diff --git a/.github/actions/setup-vortex/action.yml b/.github/actions/setup-vortex/action.yml index b27d3c1305..bd5d05da13 100644 --- a/.github/actions/setup-vortex/action.yml +++ b/.github/actions/setup-vortex/action.yml @@ -75,7 +75,7 @@ runs: - name: Install test-harness Python deps shell: bash - run: pip install --quiet pytest pyyaml + run: pip install --quiet pytest pyyaml "pyslang>=11,<13" # Re-export the SST / gem5 / openmpi tool env. ci/{sst,toolchain}_install.sh # emit these to $GITHUB_ENV, but that is per-job — the setup job that built diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a16733cf0e..5744db4ed4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -232,6 +232,13 @@ jobs: mpi) sudo bash ./ci/install_dependencies.sh --mpi ;; gem5) sudo bash ./ci/install_dependencies.sh --gem5 ;; esac + # A check's cell gathers cases from every category, so per-case tools are + # keyed on the cell's needs, not its name. + - name: Per-case tools + if: contains(matrix.needs, 'foldsim') + run: | + (cd build${{ matrix.xlen }} && bash ci/toolchain_install.sh --foldsim) + pip install --quiet pyslang==12.0.0 - name: Run cell run: | cd build${{ matrix.xlen }} diff --git a/.github/workflows/fpga_gate.yml b/.github/workflows/fpga_gate.yml index ab154187a9..232b517d14 100644 --- a/.github/workflows/fpga_gate.yml +++ b/.github/workflows/fpga_gate.yml @@ -53,6 +53,11 @@ jobs: echo "state=$HOME/.cache/vortex/fpga_gate.$(echo '${{ github.repository }}' | tr / _).sha" >> "$GITHUB_OUTPUT" echo "gating $SHA" + # The synthesis flows fold asynchronous RAM reads into block RAM with + # pyslang (hw/scripts/ram_fold.py); a no-op once installed. + - name: Install pyslang + run: python3 -m pip install --quiet --user "pyslang>=11,<13" + # The command is the one the catalog declares (ci/testcases/fpga_gate.yaml's # single `run:` case), so what CI runs and what the catalog says cannot # drift apart. Dispatch inputs only narrow it, and reach the shell through diff --git a/.github/workflows/logic_gate.yml b/.github/workflows/logic_gate.yml new file mode 100644 index 0000000000..a9d5a6e4eb --- /dev/null +++ b/.github/workflows/logic_gate.yml @@ -0,0 +1,197 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"). +# +# logic_gate — logic-level regression gate on hosted runners. +# Runs Beacon over the DUT catalog (ci/testcases/logic_gate.yaml), through +# foldsim-estimate-timing, and asserts the depth of every RTL unit against the +# golden results in ci/baselines/synthesis/beacon/, so an RTL change that makes +# a unit deeper is named on the change that does it, not the next night. +# +# Why its own workflow rather than a cell in ci.yml: it needs foldsim, which the +# base toolchain does not install (ci/toolchain_install.sh --foldsim), and it +# reads the synthesis gates' DUT flow rather than a pytest build tree. Unlike +# the synthesis gates it is cheap -- seconds to minutes per DUT, no licence -- +# so it runs on every pull request and push that touches the RTL, in one job. +# +# See docs/designs/continuous_integration.md §3.5. + +name: Timing Gate + +on: + pull_request: + paths: + - 'hw/rtl/**' + - 'hw/unittest/**' + - 'hw/scripts/**' + - 'hw/syn/xilinx/dut/**' + - 'ci/synth_gate.py' + - 'ci/logic_gate.py' + - 'ci/testcases/logic_gate.yaml' + - 'ci/baselines/synthesis/beacon/**' + push: + branches: [master] + paths: + - 'hw/rtl/**' + - 'hw/unittest/**' + - 'hw/scripts/**' + - 'hw/syn/xilinx/dut/**' + - 'ci/synth_gate.py' + - 'ci/logic_gate.py' + - 'ci/testcases/logic_gate.yaml' + - 'ci/baselines/synthesis/beacon/**' + workflow_dispatch: + inputs: + builds: + description: "build ids/groups, space separated (blank = all)" + default: "" + gpu: + description: "run the full-GPU builds (group gpu) on the self-hosted machine instead" + type: boolean + default: false + +concurrency: + group: logic-gate-${{ github.ref }} + cancel-in-progress: true + +env: + CCACHE_DISABLE: 1 + +jobs: + beacon: + if: ${{ !inputs.gpu }} + runs-on: ubuntu-22.04 + timeout-minutes: 90 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + # Restores the prebuilt toolchain (CIRCT among it) and exports TOOLDIR; + # prepare=true installs it on a cache miss. + - name: Setup Vortex + uses: ./.github/actions/setup-vortex + with: + prepare: true + + # foldsim is not in the base toolchain. Its prebuilt is a small fetch. + - name: Install foldsim + run: | + set -euo pipefail + mkdir -p tc_foldsim && cd tc_foldsim + ../configure --tooldir=$TOOLDIR + ci/toolchain_install.sh --foldsim + + # The command is the one the catalog declares (the `gate` case of + # ci/testcases/logic_gate.yaml), so what CI runs and what the catalog + # says cannot drift apart. + - name: Gate + id: run + env: + IN_BUILDS: ${{ github.event.inputs.builds }} + run: | + set -uo pipefail + CMD=$(python3 -c "import yaml; print(next(t['run'] for t in yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'] if t['id'] == 'gate'))") + # Dispatch inputs reach the shell through an env var, never + # interpolated into the script. + ARGS=() + for b in ${IN_BUILDS:-}; do ARGS+=(-b "$b"); done + rc=0 + $CMD "${ARGS[@]}" --timeout 3600 --report logic_gate.json || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Summarize + if: always() + run: | + set -uo pipefail + python3 ci/synth_report.py --annotate error logic_gate.json \ + > logic_gate_report.md || true + cat logic_gate_report.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logic-gate + path: | + logic_gate.json + logic_gate_report.md + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/build.log + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/beacon_* + build32_logic_gate/hw/syn/xilinx/dut/logic_gate_*/beacon/report.txt + if-no-files-found: warn + + - name: Verdict + run: | + case "${{ steps.run.outputs.rc }}" in + 0) echo "logic_gate passed" ;; + 1) echo "::error::logic_gate FAILED — a unit got deeper than its baseline"; exit 1 ;; + *) echo "::error::logic_gate build error (see logs)"; exit 1 ;; + esac + + # The full-GPU builds: ~30 min and ~95 GB each, run one at a time, which no + # hosted runner has the memory for. Manual dispatch only (gpu: true), on the + # self-hosted machine the FPGA gate uses. + gpu: + if: ${{ inputs.gpu }} + runs-on: [self-hosted, linux, vivado] + timeout-minutes: 480 + steps: + - uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Vortex + uses: ./.github/actions/setup-vortex + with: + prepare: true + + - name: Install foldsim + run: | + set -euo pipefail + mkdir -p tc_foldsim && cd tc_foldsim + ../configure --tooldir=$TOOLDIR + ci/toolchain_install.sh --foldsim + + # The catalog's `gpu` case; a dispatch's builds narrow it. + - name: Gate + id: run + env: + IN_BUILDS: ${{ github.event.inputs.builds }} + run: | + set -uo pipefail + CMD=$(python3 -c "import yaml; print(next(t['run'] for t in yaml.safe_load(open('ci/testcases/logic_gate.yaml'))['tests'] if t['id'] == 'gpu'))") + ARGS=() + for b in ${IN_BUILDS:-}; do ARGS+=(-b "$b"); done + rc=0 + $CMD "${ARGS[@]}" --report logic_gate_gpu.json || rc=$? + echo "rc=$rc" >> "$GITHUB_OUTPUT" + exit 0 + + - name: Summarize + if: always() + run: | + set -uo pipefail + python3 ci/synth_report.py --annotate error logic_gate_gpu.json \ + > logic_gate_gpu_report.md || true + cat logic_gate_gpu_report.md >> "$GITHUB_STEP_SUMMARY" + + - uses: actions/upload-artifact@v4 + if: always() + with: + name: logic-gate-gpu + path: | + logic_gate_gpu.json + logic_gate_gpu_report.md + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/build.log + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/beacon_* + build*_logic_gate/hw/syn/xilinx/dut/logic_gate_gpu*/beacon/report.txt + if-no-files-found: warn + + - name: Verdict + run: | + case "${{ steps.run.outputs.rc }}" in + 0) echo "logic_gate gpu passed" ;; + 1) echo "::error::logic_gate gpu FAILED — a unit got deeper than its baseline"; exit 1 ;; + *) echo "::error::logic_gate gpu build error (see logs)"; exit 1 ;; + esac diff --git a/AGENTS.md b/AGENTS.md index 1324ebf732..995d39cf47 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -8,7 +8,7 @@ This is the canonical entry point for **both human contributors and AI coding ag ### Setup & build - [docs/install_vortex.md](docs/install_vortex.md) — initial install + build setup -- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL from source +- [docs/building_toolchain.md](docs/building_toolchain.md) — building Verilator, RISC-V GNU, LLVM, compiler-rt, musl, POCL, CIRCT, Foldsim from source - [docs/environment_setup.md](docs/environment_setup.md) — environment variables and toolchain layout - [docs/fpga_setup.md](docs/fpga_setup.md) — FPGA target setup @@ -34,8 +34,7 @@ This is the canonical entry point for **both human contributors and AI coding ag - [CONTRIBUTING.md](CONTRIBUTING.md) — public fork/PR contribution flow - [docs/bug_fixes.md](docs/bug_fixes.md) — bug-fix discipline (root-cause vs patch) - [docs/continuous_integration.md](docs/continuous_integration.md) — CI pipeline -- [docs/proposals/](docs/proposals/) — design and migration proposals (drafts and in-progress) -- [docs/designs/](docs/designs/) — accepted designs (post-proposal, post-implementation) +- [docs/designs/](docs/designs/) — accepted designs (implemented) --- @@ -87,7 +86,8 @@ See [docs/testing.md](docs/testing.md) and [docs/debugging.md](docs/debugging.md - **RTL coverage path is `xrt`, not `rtlsim`.** When discussing or planning RTL verification, `xrt` is the canonical path — `rtlsim` bypasses the AFU surface. `rtlsim` remains useful for fast iteration on processor RTL; `xrt` is what proves the full integration. - **`ci/regression.sh` is the canonical source of tested configurations.** Use it to discover supported parameter combinations before inventing ad hoc ones. - **Perf-regression baselines (`ci/baselines/perf/*.json`) are golden data — never hand-edit them, and never "fix" a red perf gate by bumping the number.** They are regenerated only by `pytest ci -m perf_gate --update-baselines` (a human-run, reviewed step), and CI must never pass that flag. A `perf_gate` failure means real cycles moved: root-cause it, or — if the change is intended — regenerate the baseline so the diff shows the perf delta for review. Same rule as image goldens and `known_issue:`. -- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved. Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Synthesis baselines (`ci/baselines/synthesis/{xilinx,yosys,beacon}/*.json`) are golden data under the same rule** — never hand-edited, regenerated only by a human running `ci/fpga_gate.py --update-baseline` / `ci/asic_gate.py --update-baseline` / `ci/logic_gate.py --update-baseline`, never by CI. A red `fpga_gate`/`asic_gate` means real Fmax or area moved; a red `logic_gate` means a unit got deeper or the design's area grew past 5 %, and its reasons name the unit or the resource (`beacon_area.txt` has the area by unit and each memory's placement). Note the two gates measure the *same* `hw/unittest/*/VX_*_top.sv` wrappers through different flows, so a divergence between them is a finding, not noise — keep `hw/syn/{xilinx,yosys}/dut/catalog.mk` in step when you add or rename a DUT. See [docs/designs/continuous_integration.md §3.5](docs/designs/continuous_integration.md). +- **Beacon first for any RTL change that can move timing; synthesis only to confirm.** A change to a pipeline, arbiter, queue, interface or datapath can add logic levels that only the nightly `fpga_gate` would find, hours later. Beacon finds them in minutes, so iterate on it, not on Vivado or Yosys: run `ci/logic_gate.py -b ` on the parent revision once (its own build tree, e.g. `git worktree add ../ref HEAD` and `--build-dir build32_beacon_ref`), then `ci/logic_gate.py -b --against build32_beacon_ref` after each edit, and fix what `/beacon_diff.txt` reports until nothing fails (`ci/beacon.py paths` shows a deep unit's paths op by op). Run synthesis once per change, on the DUTs whose depth moved, and feed the result back with `ci/beacon.py validate`. A change to the cluster, socket, L2 or anything only the whole GPU assembles also runs the `gpu` group (`ci/logic_gate.py -b gpu -j 1`, ~30 min and ~95 GB per build). A unit that gets deeper is fixed or, if intended, re-recorded with the reason in the commit message. `beacon_diff.txt` also lists each resource of the design (LUT, FF, distributed and block RAM, DSP) and the units whose area moved, so a memory that leaves block RAM shows in minutes. Beacon does not see wires, placement or congestion; a change that moves those still needs synthesis. See [docs/designs/beacon.md §6.3](docs/designs/beacon.md). - **SimX is the RTL's timing model — keep them in lockstep.** Any change that moves RTL cycles (pipeline structure, arbitration, queue depths, cache/memory behavior) must land together with the matching SimX timing-model update, and vice-versa. The `model_parity` CI gate enforces this: a `check: model_parity` case (`ci/testcases/core.yaml` + per-extension parity categories) runs the same app/args/configs on simx and rtlsim and asserts exact retired-instruction match plus cycle agreement within the case tolerance (default 5%). Never widen a tolerance to absorb a divergence — model the behavior. When adding a hardware feature, add or extend a parity case that exercises it. - **When RTL debugging stalls, switch to the SimX-as-oracle pattern.** For numerical bugs, deep pipeline races, or any failure mode where rtlsim is "close but wrong": (1) build/extend the SimX C++ model so it mirrors the *new* RTL architecture and gets to PASS; (2) add matching trace dumps to both SimX and RTL (cycle, FU events, SRAM addresses+data, hazards) — same CSV format on both sides; (3) diff trace files — the first divergence is the bug. Don't keep guessing from output values; localize via trace diff. See [docs/debugging.md](docs/debugging.md#simx-as-oracle-for-rtl-debug). @@ -135,6 +135,12 @@ python3 ci/roofline.py --app=sgemm --driver=simx --args="-n128" --flops=4194304 - **Align with mainstream CUDA, HIP, OpenCL, and Vulkan API and driver stacks.** For any design question — driver surface, command-processor model, memory model, scheduling — pick the solution those stacks would use. This keeps Vortex's externals familiar to mainstream software and avoids one-off abstractions. +- **Vivado floating-point IP latency is an Fmax knob, and it is already at its maximum.** + - `hw/scripts/xilinx_ip_gen.tcl` sets each `floating_point` IP's `C_Latency`. More cycles means more internal pipelining, up to a per-operation maximum; Vivado rejects any value above it. + - The single-precision maxima on V80 and U55C are: FMA 16 (`Medium_Usage`; 19 with `Full_Usage`), divide 28, square root 28, multiply 8, add 11. `xil_fma`, `xil_fdiv`, `xil_fsqrt` and the TCU's `xil_fmul`/`xil_fadd` already use them, so FMA with `Full_Usage` (more DSPs) is the only headroom left. + - A change must move together: `C_Latency`, the matching `VX_CFG_*_LATENCY` in `VX_config.toml` (checked by the `STATIC_ASSERT`s in `hw/rtl/fpu/VX_f*_unit.sv`), and the SimX timing that reads it. + - This applies only to `FPU_TYPE_DSP`. The STD FPU's div/sqrt latency is fixed by its SRT stage count, and only its FMA latency is configurable. + - **SimX Cardinal Rule — modules communicate *only* through channels.** A `SimObject` may observe or mutate another module's state *only* via its bound `SimChannel` ports (`MemReq`/`MemRsp`, `result_if`, …). **Never reach across the ownership hierarchy** to touch another object directly (`core_->processor()->memsim()`, `parent()->child()->field`, a leaf unit grabbing the global `Memory`). See [docs/designs/simobject.md](docs/designs/simobject.md#the-cardinal-rule). --- @@ -154,19 +160,11 @@ See [docs/coding_guidelines_cpp.md](docs/coding_guidelines_cpp.md) and [docs/cod --- -## 7. Proposals - -Design and migration proposals belong in [`docs/proposals/`](docs/proposals/) — never in build dirs or the repo root. A proposal captures the *why* of a non-trivial change before the code lands. - -Accepted designs that outlive their proposal phase live in [`docs/designs/`](docs/designs/). - ---- - -## 8. Living Document Policy +## 7. Living Document Policy This file is versioned alongside the code and evolves with it. Both humans and agents are expected to update it: - **Add a rule** when you discover a footgun the next contributor will hit. Include the *why*. - **Update or remove a rule** that's gone stale. A stale rule is worse than no rule. - **Don't duplicate** content from the topic docs in `docs/`. AGENTS.md links; the topic doc explains. -- **Branch-specific addenda are forbidden.** Branch context goes in a `docs/proposals/_proposal.md`, not in branch-local `AGENTS.md` variants. If the rule is general, promote it here; if it's a project-tracking detail, keep it in the proposal. +- **Branch-specific addenda are forbidden.** No branch-local `AGENTS.md` variants: if a rule is general, promote it here; project-tracking details stay out of the repo docs. diff --git a/CHANGELOG.md b/CHANGELOG.md index b384f6a82b..35ab713ccb 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,12 +9,45 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, ### Added -- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `rtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow +- **Hardware profiles.** `profiles/.json` names a machine (`tinygpu`, `gtx1080`, `v100`, `a100`, `h100`, `rtx5080`) that every flow builds through `configure --profile=` or `blackbox.sh --profile=`; explicit `CONFIGS` win over the profile. + The GPU profiles are 64-bit machines with VM, atomics, MUFU, shared memory and the tensor, DXA, ray-tracing and graphics units of + the real part; a profile's `xlen` sets `configure`'s XLEN, and a build tree of the other width rejects it. See [docs/simulation.md](docs/simulation.md#hardware-profiles). +- **Per-partition fetch (`VX_CFG_FETCH_WIDTH`, default 1).** The front end splits into FETCH_WIDTH slices, each with its own warp + scheduler, fetch, decode and icache port, serving the warps with `wid % FETCH_WIDTH == f`; CTA dispatch, barriers and warp control + stay core-wide. The core takes one of three structures from the fetch width, issue width and per-unit block counts; with one block + of every unit per issue slice and FETCH_WIDTH equal to ISSUE_WIDTH, each slice is a self-contained compute slice (the NVIDIA + processing-block organization). *Why:* sgemm at 16 warps x 16 threads, ISSUE_WIDTH 4: -31.6% cycles at FETCH_WIDTH 2. +- **Hashed cache bank select (`VX_CFG_{DCACHE,L2}_BANK_HASH`).** The bank index is the raw bank bits XORed with a hash of the line + address above them (IPOLY(5) for 16 banks, IPOLY(37) for 32, an XOR fold otherwise), so power-of-two strides spread over the banks. + On by default for the L2 and for a dcache with 4 or more banks. *Why:* rtlsim sgemv 512x512 at 16 threads -32.8%, fp16 TCU SGEMM + on 2 cores + L2 at 16 threads -13.8%, WGMMA with DXA -13.5%. +- **Dcache read merge (`VX_CFG_DCACHE_READ_MERGE`, on by default).** Plain reads of the same word from the ports of one LSU block in + the same cycle are served by one bank access, and the response is multicast to every port; each port's order into the bank is + unchanged. *Why:* uniform loads (kernel arguments, SGEMM's shared operand) no longer serialize on one bank: rtlsim sgemm -n128 at + 16 threads -6.5%, -17.2% with an 8-byte dcache word. +- **Write-allocate without fetch (`VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`, on where the cache is a write-back LLC).** A store miss in a write-back cache installs + the sector with its dirty-byte mask and a partial bit instead of fetching the line; a later load or AMO merges memory data under the + unwritten bytes, and evictions write back only dirty bytes. *Why:* streaming writes no longer read every line first: 1-core vecadd + DRAM reads -33%. +- **Command-processor DMA through the shared LLC (`VX_CFG_CP_LLC_CLIENT`, on by default).** When one LLC serves every core, the + CP's transfers enter it on their own client slot, so host transfers need only an L1 flush, not a walk of the whole LLC; transfers + of at least half the LLC write around it. A launch behind DMA writes waits for an L1-only flush token sent on the slot. The runtime + sees the new `DMA_LLC_CLIENT` CP capability. *Why:* rtlsim at 2 clusters + L2 + L3: dogfood -30%, multikernel -42.7%; dogfood + -64% with an 8 MB L3. +- **Foldsim: threaded FPGA simulation of Vortex on the Alveo V80, built on CIRCT.** Every class of identical units (sockets, issue slices, ALUs, FPUs, load-store units) is time-multiplexed through one physical copy, cycle-exact against the RTL, with no Yosys or sv2v: `circt-verilog` imports Vortex, Foldsim's `foldsim-extract-models` / `foldsim-compose` cut it into threaded models joined by one token fabric (Foldsim is its own project on CIRCT, installed by `ci/toolchain_install.sh --foldsim`; [hw/syn/foldsim/](hw/syn/foldsim/)), and the V80's native flow packages it with a latency-bandwidth memory endpoint and a command-queue control block ([hw/rtl/afu/foldsim/](hw/rtl/afu/foldsim/)). `--driver=foldsim` runs it on the card (`TARGET=hw`, VRT + QDMA) or in-process in Verilator (`avedsim`, the default). Vortex's RTL is unchanged: units get plain-port boundaries and instance IDs as ports from a source-to-source pass on the flow's copy ([hw/syn/foldsim/lift_boundaries.py](hw/syn/foldsim/lift_boundaries.py)). See [docs/designs/foldsim.md](docs/designs/foldsim.md). + *Why:* the simulated GPU is no longer bounded by the card's size, and the shell around it is ~8k LUTs instead of FireSim's ~93k — s2w4i2 (2 cores, NW=NT=4, IW=2) routes in 138k LUTs vs 219k and runs sgemm n=128 in 196 ms on V80: 50× SimX on the same host, 1.7× the FireSim image, cycle counts identical to Verilator. + *Limits:* V80 only; one memory port, single-beat transactions; classes are named (`FOLDSIM_CLASSES`), not found; kernel register offsets 0x40–0xBF are unusable on the V80 shell (its PCIe MSI-X table answers them). ### Changed +- **Foldsim's shell lives in the Foldsim repository.** The control and memory endpoints, the DRAM bank timing model, the register + map, the in-process host, the endpoint's unit tests and the boundary lifting (`foldsim-lift`) moved out of `hw/rtl/afu/foldsim`, + `sim/foldsim`, `sw/common`, `hw/unittest/foldsim_mem*` and `hw/syn/foldsim`; they come with the Foldsim installation + (`FOLDSIM_SHELL`, `$(FOLDSIM_PATH)/share/foldsim/shell`). Vortex keeps the target wrapper, the image flow, the runtime driver and + the parity case. *Why:* Foldsim is a general-purpose RTL simulator; nothing in the shell is specific to Vortex. + - **RV64 portability.** Device index math, DXA/OM addresses, pack-load strides, RTU scene/payload pointers and the ray log carry full width on RV64 (`size_t`/`uintptr_t`, identical RV32 code); RTU slot reads into FP registers are NaN-boxed. - **Zero warnings restored.** Every C/C++ build (simulators, runtime, kernel library, tests, OpenCL/HIP/Vulkan apps) is `-Werror` and @@ -23,6 +56,52 @@ follows the version pins recorded in [VERSION](VERSION) (`VORTEX_VERSION`, (JEDEC HBM2/HBM3/GDDR6 organization and nanosecond timing, 32 B bursts, permuted open-page bank interleave, per-channel responses), configured from data-sheet values: `VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`. The default machine keeps its idle read latency (14/26 cycles open-row/conflict, was 13/27) and one-block-per-cycle peak. +- **Cache flush architecture.** The CP starts every core's flush with one broadcast `CACHE_FLUSH` DCR write, then reads each core's + completion, so the flushes overlap; a flush read without a preceding write no longer starts one. A cache walks its banks, sends one + flush token down every memory port and answers once all ports have; every independently buffered write path (each dcache lane, each + OM input of the ocache) carries its own token. Read-only caches (icache, tcache, rtcache, rcache) no longer forward flushes. + `DBG_FLUSH_STALL` adds random input backpressure to the flush path for stress tests. +- **L1-only flush between launches.** When a single LLC serves every memory client it is the point of coherence, so a launch + boundary cleans only the L1s: `CMD_CACHE_FLUSH` gains an L1-only flag, carried as a flush-token kind that a shared LLC answers + without a walk. Replicated LLCs and LLCs with peer clients (TEX, RTU, RASTER, OM, DXA) still walk. The runtime posts flushes + lazily and flushes fully before any device-memory transfer, and device busy stays high until every core write, uncached IO stores + included, is visible in memory. *Why:* per-launch flush cost at 4 cores + L2 drops from 2,750 to 180 cycles. +- **Targeted fences.** A memory fence sends an L1-only flush token that the shared LLC answers without a walk. A fence that orders + only device (I/O) accesses decodes as its own LSU op, and its token makes each level drain to its memory ports without walking. + *Why:* printf, which fences after each device write, runs 13.6x faster on rtlsim at 2 cores + L2. +- **LMEM DMA arbitration.** DXA writes into shared memory and the TCU's WGMMA operand reads now share the local-memory DMA port + round-robin. DXA had fixed priority and starved the TCU for about 510 cycles per tiled-scatter transfer. *Why:* rtlsim WGMMA with + DXA -10.2%, sparse WGMMA with DXA -11.6%. +- **SimX timing model follows the RTL's structure.** Buffer capacities and registered ready at every pipeline boundary, per-slice + cache arbiter trees and network depths, registered cache core-response ports, flush and fence tokens end to end, and CTA launch onto + the nearest ready core with two launch credits per core. SimX's per-core cycle counter (`mcycle`) now advances only while the core + has work, as the RTL's does. rtlsim completes DRAM requests in per-bank arrival order, as the AXI platform does. + +### Fixed + +- **Global barriers across clusters.** Each cluster ran its own global-barrier unit that saw only its own cores, while + `vx_num_cores()`, and so `vortex::gbarrier`, counts the whole device. On SimX every global barrier hung with more than one + cluster; on the RTL, 2 clusters x 1 core timed out and 2 clusters x 2 cores released each cluster on its own arrivals, with no + ordering between clusters. One device-level unit now counts every core. +- **Writes left dirty by a flush.** A cache forwarded its flush token on memory port 0 only, so writes queued on another port could + reach the next level after that level's walk and stay dirty there. +- **Dirty sectors dropped on a sector refill.** With dirty-byte tracking and several sectors per line, installing a sector into a + resident line cleared the whole line's byte mask, so a dirty sibling sector wrote back with no bytes enabled and its data was lost + (mstress at 4 cores with a write-back, sectored L2). +- **Local-memory LR/SC livelock at 16 threads.** One warp's per-lane LR burst used up a reservation's fixed hold budget of 7 refusals, + so contended LR/SC loops on shared memory stopped making progress. The budget now scales with the core's harts. +- **`VX_stream_arb` trees with a partial last slice.** When the request count was not a whole number of slices, the last slice's + select was sized and placed as if it were full: the build failed lint, and with several outputs requests went to the wrong output. +- **`-DSCOPE` builds.** The scope buses of fetch, issue, LSU, TCU, TEX, RASTER and OM were declared only under their `DBG_SCOPE_*` + taps, left unbound, or did not parse, so debug-scope builds failed to elaborate. +- **mstress checked no data.** Its host reference was always 0, so the test passed with corrupted loads; it now checks every loaded + value exactly. +- **FPGA run instructions.** The docs exported `TARGET=hw`, which `blackbox.sh` ignores, so runs fell back to xrtsim; they now pass + `--target=hw`. +- **Block RAM for asynchronous-read RAMs is a CIRCT pass, not a Vivado netlist patch; every synthesis flow reads one CIRCT-elaborated netlist.** A RAM that must be block RAM but reads asynchronously is a plain array tagged `(* ram_style = "block" *)`; CIRCT's `seq-retime-firmem-reads` makes its read synchronous, reading at the address its registers are about to take, and stops the build with the reason if it cannot. `hw/scripts/circt_elaborate.sh` (`gen_sources.sh -E`) elaborates the design for Vivado (xrt, aved, dut, sandbox), FireSim, Foldsim and Yosys alike, and writes a report of every asynchronous-read RAM. `ASYNC_BRAM_PATCH`, `VX_async_ram_patch`, `VX_placeholder`, `xilinx_async_bram_patch.tcl` and its pre-opt hooks are gone; Xilinx floating-point cores are declared to the elaboration in `hw/scripts/xilinx_ip_stubs.sv`. + *Why:* the patch rewired Vivado's synthesized netlist in Tcl, ran in Vivado flows only, failed late on large builds (`DRC INBB-3`), and produced a netlist nothing simulated. On the core at 16 warps and 8 threads (V80, 300 MHz met both ways) the CIRCT netlist implements in 90,481 logic LUTs against 92,608, with the same BRAM, LUTRAM and DSPs. + *Limits:* generate-block instances are named `g_x_0_inst`, not `g_x[0].inst`, in the netlist (`xilinx_slr_pblocks.tcl` accepts both); the modules between a converted RAM and its address registers are inlined; the synthesis baselines move with the netlist and are regenerated by hand. +- **CIRCT is the SystemVerilog frontend of every synthesis flow; sv2v and Verilator's preprocessor are gone.** `gen_sources.sh -P` preprocesses with `circt-verilog -E` (identical code to `verilator -E` over the whole RTL tree; only comments differ), and the Yosys flow elaborates with CIRCT into one Verilog-2005 file ([hw/scripts/sv_to_v.sh](hw/scripts/sv_to_v.sh)) instead of sv2v: packed arrays and structs become vectors, and the SRAM macros (`BB_MODULES`) stay instances with their parameter values, read in SystemVerilog mode beside their stubs. CIRCT installs under `$TOOLDIR/circt` (`toolchain_install.sh --circt`; `hw/syn/common.mk` exports `CIRCT_VERILOG`, `CIRCT_OPT`, `FIRTOOL`), and the `asic_gate` fingerprints its baselines on the CIRCT version. Verilator remains the simulator (and `--xml-only` scope metadata). On the cache DUT (ASAP7), SRAM area is identical and logic area is 5.6% higher than through sv2v: CIRCT folds synchronous resets into the next-value logic and lowers array reads to mux trees, both open items. ## [3.0] — 2026-10-07 diff --git a/README.md b/README.md index 39658e3c24..f110ebad2b 100644 --- a/README.md +++ b/README.md @@ -81,7 +81,7 @@ The following dependencies will be fetched prebuilt by `toolchain_install.sh`. - [SoftFloat](https://github.com/ucb-bar/berkeley-softfloat-3.git) - [Ramulator](https://github.com/CMU-SAFARI/ramulator.git) - [Yosys](https://github.com/YosysHQ/yosys) -- [Sv2v](https://github.com/zachjs/sv2v) +- [CIRCT](https://github.com/llvm/circt) (Vortex's fork) - [Mesa](https://www.mesa3d.org/) - [chipStar](https://github.com/CHIP-Star/chipStar) ### Install Vortex codebase diff --git a/VERSION b/VERSION index 60550bfcb5..be388f21fc 100644 --- a/VERSION +++ b/VERSION @@ -1,3 +1,4 @@ VORTEX_VERSION=3.0 TOOLCHAIN_REV=v3.0.1 GEM5_REV=v25.0.0.1 +FIRESIM_REV=57bb93f1e23daf46198659ddd93608f70121fee0 diff --git a/VX_config.toml b/VX_config.toml index ca184b4886..82f527aac1 100644 --- a/VX_config.toml +++ b/VX_config.toml @@ -56,9 +56,12 @@ VX_CFG_MAX_BAR_EVENTS = 32 VX_CFG_IBUF_SIZE = 4 VX_CFG_DISPATCH_QUEUE_SIZE = 4 VX_CFG_ISSUE_WIDTH = "expr: up($VX_CFG_NUM_WARPS / 16)" +# Fetch slices, each scheduling and fetching its own warps on its own icache +# port: a power of two that divides ISSUE_WIDTH. One per issue slice by +# default (the largest power of two dividing ISSUE_WIDTH). +VX_CFG_FETCH_WIDTH = "expr: $VX_CFG_ISSUE_WIDTH & -$VX_CFG_ISSUE_WIDTH" VX_CFG_SIMD_WIDTH = "expr: $VX_CFG_NUM_THREADS" VX_CFG_NUM_OPCS = "expr: up($VX_CFG_NUM_WARPS / (4 * $VX_CFG_ISSUE_WIDTH))" - VX_CFG_NUM_GPR_BANKS = 4 VX_CFG_NUM_VGPR_BANKS = 2 @@ -139,8 +142,9 @@ VX_CFG_FMA_LATENCY = "expr: 16 if $fpu_dsp_vivado else (4 if $fpu_dsp_quartus e # STD timing; DPI is a fast STD and takes the same values. fdiv_latency_std = "expr: 32 if $VX_CFG_EXT_D_ENABLE else 17" fsqrt_latency_std = "expr: $fdiv_latency_std" -fdiv_latency_dsp = "expr: 15 if $QUARTUS else (28 if $VIVADO else (32 if $VX_CFG_EXT_D_ENABLE else 17))" -fsqrt_latency_dsp = "expr: 10 if $QUARTUS else (28 if $VIVADO else (32 if $VX_CFG_EXT_D_ENABLE else 17))" +# The vendor divide and square-root IP is F32 only; with D they are the RTL cores. +fdiv_latency_dsp = "expr: 32 if $VX_CFG_EXT_D_ENABLE else (15 if $QUARTUS else (28 if $VIVADO else 17))" +fsqrt_latency_dsp = "expr: 32 if $VX_CFG_EXT_D_ENABLE else (10 if $QUARTUS else (28 if $VIVADO else 17))" VX_CFG_FDIV_LATENCY = "expr: 16 if $VX_CFG_FPU_TYPE_FPNEW else ($fdiv_latency_dsp if $VX_CFG_FPU_TYPE_DSP else $fdiv_latency_std)" VX_CFG_FSQRT_LATENCY= "expr: 16 if $VX_CFG_FPU_TYPE_FPNEW else ($fsqrt_latency_dsp if $VX_CFG_FPU_TYPE_DSP else $fsqrt_latency_std)" @@ -175,6 +179,15 @@ VX_CFG_DTLB_SIZE = 16 VX_CFG_ITLB_SIZE = 8 VX_CFG_L1_TLB_MSHR_SIZE = 4 +# Socket unit MMU: one TLB per socket for the socket-resident units' memory +# ports (TEX tcache misses, RTU rtcache misses, DXA gmem reads), kept apart +# from the dTLB so texture and BVH traffic cannot evict the cores' translations. +VX_CFG_UTLB_SIZE = 8 + +# Cluster unit MMU: one TLB per cluster for the rcache and ocache memory ports +# (raster and OM), a client of the cluster TLB like a socket. +VX_CFG_CTLB_SIZE = 8 + # Shared cluster TLB and page-table walker complex. VX_CFG_L2_TLB_SIZE = 512 VX_CFG_L2_TLB_NUM_WAYS = 4 @@ -205,6 +218,13 @@ l3_is_llc = "expr: $VX_CFG_L3_ENABLED == 1" single_core = "expr: $VX_CFG_NUM_CORES == 1 and $VX_CFG_NUM_CLUSTERS == 1" single_cluster = "expr: $VX_CFG_NUM_CLUSTERS == 1" +# The command processor's DMA becomes a client of the last-level cache, which +# is then the point of coherence for host transfers: they need no cache flush. +# Only a last-level cache that alone serves every core can take it (an L2 with +# one cluster, or an L3); VX_CFG_CP_ON_LLC is set where it took effect. +VX_CFG_CP_LLC_CLIENT = 1 +VX_CFG_CP_ON_LLC = "expr: int($VX_CFG_CP_LLC_CLIENT != 0 and (($l2_is_llc and $single_cluster) or $l3_is_llc))" + [l1cache] VX_CFG_ICACHE_SIZE = 16384 VX_CFG_ICACHE_NUM_WAYS = 4 @@ -215,6 +235,11 @@ VX_CFG_ICACHE_MRSQ_SIZE = 0 VX_CFG_ICACHE_MREQ_SIZE = 0 VX_CFG_ICACHE_CRSQ_SIZE = 0 VX_CFG_ICACHE_MEM_PORTS = 1 +# banks behind the fetch-slice request ports +VX_CFG_ICACHE_NUM_BANKS = "expr: pow(2, clog2($VX_CFG_FETCH_WIDTH))" +# Merge same-line fetches from the core's slices in the same cycle into one +# bank access; the response goes to every merged slice, each with its word. +VX_CFG_ICACHE_READ_MERGE = 1 VX_CFG_ICACHE_LATENCY = "expr: 2 + max(0, clog2($VX_CFG_ICACHE_SIZE) - clog2(16384))" VX_CFG_DCACHE_SIZE = 16384 @@ -224,7 +249,13 @@ VX_CFG_DCACHE_NUM_WAYS = 4 # stays moderate: NT 1/2/4/8/16/32/64 -> word 4/8/8/16/16/32/32, banks 1/1/2/2/4/4/8. VX_CFG_DCACHE_WORD_SIZE = "expr: min($VX_CFG_L1_LINE_SIZE, min($VX_CFG_MEM_BLOCK_SIZE, ($VX_CFG_XLEN / 8) * pow(2, (clog2($VX_CFG_NUM_LSU_LANES) + 1) / 2)))" VX_CFG_DCACHE_WRITEBACK = "expr: int($dcache_is_llc and $single_core)" -VX_CFG_DCACHE_DIRTYBYTES = 0 +# LAZY_FETCH: a store miss allocates its line without a fill and a later read +# fetches and merges under the dirty bytes (write-validate, as GPU L2s do). It +# is on wherever the cache is a write-back LLC, the only place it is legal. It +# needs the per-byte dirty masks, so it turns DIRTYBYTES on; DIRTYBYTES alone +# keeps fetch-on-write. The same pair exists for L2 and L3. +VX_CFG_DCACHE_DIRTYBYTES = "expr: $VX_CFG_DCACHE_LAZY_FETCH" +VX_CFG_DCACHE_LAZY_FETCH = "expr: int($VX_CFG_DCACHE_WRITEBACK and $dcache_is_llc)" VX_CFG_DCACHE_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_DCACHE_MSHR_SIZE = 16 # dcache is unsectored: line = sector = mem transaction granule = L1_LINE (which @@ -243,6 +274,14 @@ VX_CFG_NUM_ICACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_ICACHE_ENABLE VX_CFG_NUM_DCACHES = "expr: up($VX_CFG_SOCKET_SIZE / 4) if $VX_CFG_DCACHE_ENABLE else 0" VX_CFG_DCACHE_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_DCACHE_NUM_REQS, 16)))" +# XOR/IPOLY bank-select hash, so power-of-two strides spread over the banks. +# With two banks it can only swap a line on parity, and the extra parallel +# misses cost more in memory queuing than the spreading saves. +VX_CFG_DCACHE_BANK_HASH = "expr: int($VX_CFG_DCACHE_NUM_BANKS >= 4)" +# One bank access serves a warp's same-word reads arriving together on its +# coalescer channels (uniform loads), and the response returns to all of +# them in the same cycle. +VX_CFG_DCACHE_READ_MERGE = 1 VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if ($VX_CFG_ICACHE_ENABLE or $VX_CFG_DCACHE_ENABLE) else min($VX_CFG_DCACHE_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" @@ -250,7 +289,8 @@ VX_CFG_L1_MEM_PORTS = "expr: min($VX_CFG_DCACHE_NUM_BANKS, $VX_CFG_PLATFORM_MEMO VX_CFG_L2_SIZE = 1048576 VX_CFG_L2_NUM_WAYS = 8 VX_CFG_L2_WRITEBACK = "expr: int($l2_is_llc and $single_cluster)" -VX_CFG_L2_DIRTYBYTES = 0 +VX_CFG_L2_DIRTYBYTES = "expr: $VX_CFG_L2_LAZY_FETCH" +VX_CFG_L2_LAZY_FETCH = "expr: int($VX_CFG_L2_WRITEBACK and $l2_is_llc)" VX_CFG_L2_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L2_MSHR_SIZE = 16 VX_CFG_L2_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L2_SIZE) - clog2(1048576))" @@ -259,13 +299,17 @@ VX_CFG_L2_MREQ_SIZE = 0 VX_CFG_L2_CRSQ_SIZE = 0 VX_CFG_L2_NUM_BANKS = "expr: pow(2, clog2(min($VX_CFG_L2_NUM_REQS, 16)))" +# On at every bank count: unlike the dcache, a 2-bank L2 measured neutral +# (sgemm-mc-nt4 -0.6%, fp16-mc-nt4 +0.1%), and it gains from 4 banks up. +VX_CFG_L2_BANK_HASH = 1 VX_CFG_L2_MEM_PORTS = "expr: min($VX_CFG_L2_NUM_BANKS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS) if $VX_CFG_L2_ENABLE else min($VX_CFG_L2_NUM_REQS, $VX_CFG_PLATFORM_MEMORY_NUM_BANKS)" [l3cache] VX_CFG_L3_SIZE = 2097152 VX_CFG_L3_NUM_WAYS = 8 VX_CFG_L3_WRITEBACK = "expr: int($l3_is_llc)" -VX_CFG_L3_DIRTYBYTES = 0 +VX_CFG_L3_DIRTYBYTES = "expr: $VX_CFG_L3_LAZY_FETCH" +VX_CFG_L3_LAZY_FETCH = "expr: int($VX_CFG_L3_WRITEBACK and $l3_is_llc)" VX_CFG_L3_REPL_POLICY = "expr: $__cache_repl_fifo" VX_CFG_L3_MSHR_SIZE = 16 VX_CFG_L3_LATENCY = "expr: 4 + max(0, clog2($VX_CFG_L3_SIZE) - clog2(2097152))" diff --git a/VX_types.toml b/VX_types.toml index 80ec9c17f6..04072db399 100644 --- a/VX_types.toml +++ b/VX_types.toml @@ -19,7 +19,10 @@ VX_ISA_IMPL_ID = 0 [memmap] VX_MEM_USER_BASE_ADDR = "expr: 0x0000000000010000 if ($XLEN == 64) else 0x00010000" VX_MEM_STACK_LOG2_SIZE = 13 -VX_MEM_STACK_BASE_ADDR = "expr: 0x00000001FFFF0000 if ($XLEN == 64) else 0xFFFF0000" +# The stack window's top is 256 KB aligned, so the window lies on whole groups +# of NUM_THREADS stacks (up to 32 threads) and the stack interleave is a +# pure bit permutation. +VX_MEM_STACK_BASE_ADDR = "expr: 0x00000001FFFC0000 if ($XLEN == 64) else 0xFFFC0000" VX_MEM_LMEM_BASE_ADDR = "expr: $VX_MEM_STACK_BASE_ADDR" VX_MEM_IO_BASE_ADDR = "expr: 0x0000000000000040 if ($XLEN == 64) else 0x00000040" VX_MEM_PAGE_TABLE_BASE_ADDR = "expr: 0x00000000F0000000 if ($XLEN == 64) else 0xF0000000" @@ -76,6 +79,11 @@ VX_DCR_BASE_CACHE_FLUSH = 0x000 VX_DCR_BASE_MPM_VALUE = 0x001 VX_DCR_BASE_STATE_END = 0x002 +[cache_flush] +# VX_DCR_BASE_CACHE_FLUSH write value: flush only the levels above the +# last-level cache, which stays the point of coherence for the next kernel. +VX_CACHE_FLUSH_L1_ONLY = 0x10000 + [dcr_mmu] VX_DCR_MMU_STATE_BEGIN = 0x004 VX_DCR_MMU_SATP_LO = 0x004 diff --git a/ci/asic_gate.py b/ci/asic_gate.py index 1a5dd579a2..84b11ec4c6 100755 --- a/ci/asic_gate.py +++ b/ci/asic_gate.py @@ -13,7 +13,7 @@ from the reports/ set the flow already produces Unlike the FPGA gate this one runs on a hosted runner as a normal pytest cell -(`check: asic_gate`, tier `full`), because yosys/sv2v/OpenSTA ship in the +(`check: asic_gate`, tier `full`), because yosys/CIRCT/OpenSTA ship in the prebuilt toolchain and ASAP7 is a 59 MB fetch. Usage (from a source checkout): diff --git a/ci/baselines/perf/core.json b/ci/baselines/perf/core.json index 8467f80317..8257cb1e42 100644 --- a/ci/baselines/perf/core.json +++ b/ci/baselines/perf/core.json @@ -1,177 +1,177 @@ { "core:raycast-nt16:rtlsim": { "32": { - "cycles": 563524, - "instrs": 137486 + "cycles": 11182301, + "instrs": 2241925 }, "app": "raycast", "args": "", - "config_hash": "30868719eb39dd56", + "config_hash": "b9d9ee09370158c4", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:raycast-nt4:rtlsim": { "32": { - "cycles": 1498642, - "instrs": 259228 + "cycles": 21106989, + "instrs": 5664693 }, "app": "raycast", "args": "", - "config_hash": "bbb5e759fb64a2d2", + "config_hash": "d0bce0794d7bfdbc", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemm-mc-nt16:rtlsim": { "32": { - "cycles": 787850, + "cycles": 635244, "instrs": 623680 }, "app": "sgemm", "args": "-n128", - "config_hash": "f3cdbb1a54b60bd7", + "config_hash": "14d91b803afc0b16", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-mc-nt4:rtlsim": { "32": { - "cycles": 2239426, + "cycles": 2204469, "instrs": 2494528 }, "app": "sgemm", "args": "-n128", - "config_hash": "1f13c513ce419a18", + "config_hash": "ea174630d2a4f571", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "core:sgemm-nt16:rtlsim": { "32": { - "cycles": 1838654, + "cycles": 1661746, "instrs": 623632 }, "app": "sgemm", "args": "-n128", - "config_hash": "6c4c16bd5ba783fb", + "config_hash": "912a20e26fc4d53b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemm-nt4:rtlsim": { "32": { - "cycles": 5939731, + "cycles": 5942823, "instrs": 2494480 }, "app": "sgemm", "args": "-n128", - "config_hash": "e654782241f3196f", + "config_hash": "23fd0d2a32b65356", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemmx-nt16:rtlsim": { "32": { - "cycles": 1290130, + "cycles": 1224642, "instrs": 266896 }, "app": "sgemmx", "args": "-n128", - "config_hash": "406f5c17ca35ee9d", + "config_hash": "0dae7e6b06c6facd", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemmx-nt4:rtlsim": { "32": { - "cycles": 4219466, + "cycles": 4217077, "instrs": 1067536 }, "app": "sgemmx", "args": "-n128", - "config_hash": "5016dd4b1102f6fc", + "config_hash": "e2fe551b6ed5319a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:sgemv-nt16:rtlsim": { "32": { - "cycles": 685107, + "cycles": 381483, "instrs": 66800 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "b631a0bafc91d264", + "config_hash": "6075eb2ef46a71f7", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:sgemv-nt4:rtlsim": { "32": { - "cycles": 872926, + "cycles": 866110, "instrs": 267152 }, "app": "sgemv", "args": "-m512 -n512", - "config_hash": "14333df532bad1d6", + "config_hash": "800b8c9f7edc548a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:softmax-nt16:rtlsim": { "32": { - "cycles": 1272929, + "cycles": 1270624, "instrs": 464089 }, "app": "softmax", "args": "-n64", - "config_hash": "fdbcac76ee256139", + "config_hash": "3580b572151bfcde", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:softmax-nt4:rtlsim": { "32": { - "cycles": 4860309, + "cycles": 4860368, "instrs": 1789797 }, "app": "softmax", "args": "-n64", - "config_hash": "d1489b9fccb46500", + "config_hash": "15f1137753d108a7", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:stencil3d-nt16:rtlsim": { "32": { - "cycles": 2306960, + "cycles": 2235935, "instrs": 892944 }, "app": "stencil3d", "args": "-n32", - "config_hash": "7514f2e770fda7af", + "config_hash": "a6ddac00bf4be79b", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:stencil3d-nt4:rtlsim": { "32": { - "cycles": 4180969, + "cycles": 4180121, "instrs": 1785872 }, "app": "stencil3d", "args": "-n32", - "config_hash": "7b1283f03793abb6", + "config_hash": "63352c381701eb2a", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "core:vecadd-nt16:rtlsim": { "32": { - "cycles": 100640, + "cycles": 96002, "instrs": 29712 }, "app": "vecadd", "args": "-n16384", - "config_hash": "3c79fc19ad8549a0", + "config_hash": "d14a86b1d94d9967", "configs": "-DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "core:vecadd-nt4:rtlsim": { "32": { - "cycles": 367765, + "cycles": 368700, "instrs": 118800 }, "app": "vecadd", "args": "-n16384", - "config_hash": "75b65abb22c3753b", + "config_hash": "dcca7a6e7c88ad28", "configs": "-DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/dxa.json b/ci/baselines/perf/dxa.json index a57e2798f3..89fa130f36 100644 --- a/ci/baselines/perf/dxa.json +++ b/ci/baselines/perf/dxa.json @@ -1,23 +1,23 @@ { "dxa:perf_gate-copy-mcast:rtlsim": { "32": { - "cycles": 3381, + "cycles": 3389, "instrs": 961 }, "app": "dxa_copy_mcast", "args": "", - "config_hash": "1d98c347f39baa17", + "config_hash": "b6b95c0822edf48b", "configs": "-DVX_CFG_EXT_DXA_ENABLE", "driver": "rtlsim" }, "dxa:perf_gate-copy-nt16:rtlsim": { "32": { - "cycles": 1994, + "cycles": 1961, "instrs": 592 }, "app": "dxa_copy", "args": "-d1", - "config_hash": "f0ace578558d01a1", + "config_hash": "a6138216544c603d", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, @@ -28,95 +28,95 @@ }, "app": "dxa_copy", "args": "-d1", - "config_hash": "9a3f1b02710f7458", + "config_hash": "461ebf20c7b39b93", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 14022240, + "cycles": 14016558, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "4d3b7f000ce31563", + "config_hash": "6d3b0c78bc6bce82", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 14063502, + "cycles": 14051366, "instrs": 4962320 }, "app": "sgemm2_dxa_mcast", "args": "-n128", - "config_hash": "4dffe3629adfe705", + "config_hash": "44732317d093efc0", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt16:rtlsim": { "32": { - "cycles": 3941267, + "cycles": 3935575, "instrs": 1263632 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "6aac48586a19e5d1", + "config_hash": "ad5d3a1ba925762f", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-sgemm2-dxa-nt4:rtlsim": { "32": { - "cycles": 14352313, + "cycles": 14351025, "instrs": 5017616 }, "app": "sgemm2_dxa", "args": "-n128 -t4 -m1", - "config_hash": "15066287b6cf674a", + "config_hash": "84141dd86fe75797", "configs": "-DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt16:rtlsim": { "32": { - "cycles": 82160, + "cycles": 78350, "instrs": 32384 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "6f3f943387268cd9", + "config_hash": "cb07e8565e928aa5", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa-mcast-nt4:rtlsim": { "32": { - "cycles": 360010, + "cycles": 358377, "instrs": 234560 }, "app": "sgemm_tcu_wg_dxa_mcast", "args": "-m 128 -n 128 -k 64", - "config_hash": "99f4567a3f3d04a6", + "config_hash": "0f396d75e95e93fe", "configs": "-DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-dxa:rtlsim": { "32": { - "cycles": 36021, + "cycles": 29643, "instrs": 9160 }, "app": "sgemm_tcu_wg_dxa", "args": "-m 128 -n 128 -k 64", - "config_hash": "a45156b86aef0add", + "config_hash": "93664ea8afb18300", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" }, "dxa:perf_gate-wgmma-sp-dxa:rtlsim": { "32": { - "cycles": 59789, + "cycles": 48722, "instrs": 9328 }, "app": "sgemm_tcu_wg_sp_dxa", "args": "-m 128 -n 128 -k 128", - "config_hash": "63afe161fb9d2aac", + "config_hash": "73097ce78c7e35b3", "configs": "-DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_NUM_THREADS=32", "driver": "rtlsim" } diff --git a/ci/baselines/perf/graphics.json b/ci/baselines/perf/graphics.json index 761b756cb9..e0e39c0977 100644 --- a/ci/baselines/perf/graphics.json +++ b/ci/baselines/perf/graphics.json @@ -1,100 +1,100 @@ { "graphics:perf_gate-draw3d-mc:rtlsim": { "32": { - "cycles": 530195, + "cycles": 529951, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "71e067fd7765f075", + "config_hash": "02ff3eb8cc7c891c", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt16:rtlsim": { "32": { - "cycles": 143620, + "cycles": 141529, "instrs": 51652 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "ac94e8341d7c171e", + "config_hash": "33bc329a5c31e08d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-draw3d-nt4:rtlsim": { "32": { - "cycles": 529393, + "cycles": 529326, "instrs": 203136 }, "app": "gfx_draw3d", "args": "-w 128 -h 128 -t box.cgltrace -r box_ref_128.png", - "config_hash": "00c3aa3428bf4e86", + "config_hash": "bd063408e042748e", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-om-nt16:rtlsim": { "32": { - "cycles": 189231, + "cycles": 189991, "instrs": 68624 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "a99eb5442dfe8bd5", + "config_hash": "a283b22b5fc238d4", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-om-nt4:rtlsim": { "32": { - "cycles": 740701, + "cycles": 740707, "instrs": 274448 }, "app": "gfx_om", "args": "-w 128 -h 128 -r whitebox_128.png", - "config_hash": "b1631e4a43822c5a", + "config_hash": "5b99293235b7c2fa", "configs": "-DVX_CFG_EXT_OM_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt16:rtlsim": { "32": { - "cycles": 14785, + "cycles": 14481, "instrs": 4708 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "2c54468d2adf668e", + "config_hash": "e6d78d84f54ee283", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-raster-nt4:rtlsim": { "32": { - "cycles": 55565, + "cycles": 55551, "instrs": 18640 }, "app": "gfx_raster", "args": "-w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png", - "config_hash": "1f50896f410bc97f", + "config_hash": "15f7ab48d7b2edce", "configs": "-DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt16:rtlsim": { "32": { - "cycles": 35441, + "cycles": 35278, "instrs": 12304 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "b72e550c7f6c7aca", + "config_hash": "c808376069d5b819", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "graphics:perf_gate-tex-nt4:rtlsim": { "32": { - "cycles": 136487, + "cycles": 136502, "instrs": 49168 }, "app": "gfx_tex", "args": "-i toad.png -r toad_ref_f0.png -f 0 -g 0", - "config_hash": "15f81408ca91fc9f", + "config_hash": "9d3f7a8db252c33d", "configs": "-DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/raytracing.json b/ci/baselines/perf/raytracing.json index db73f213c5..a9a61b8666 100644 --- a/ci/baselines/perf/raytracing.json +++ b/ci/baselines/perf/raytracing.json @@ -1,34 +1,34 @@ { "raytracing:perf_gate-rt_raycast-mc:rtlsim": { "32": { - "cycles": 85045, + "cycles": 85069, "instrs": 39346 }, "app": "rt_raycast", "args": "", - "config_hash": "048a348462ce8158", + "config_hash": "75e25538007e3fd7", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt16:rtlsim": { "32": { - "cycles": 53434, + "cycles": 52944, "instrs": 11372 }, "app": "rt_raycast", "args": "", - "config_hash": "bbab935bba11ff85", + "config_hash": "4d66c19f8df23f2e", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=16", "driver": "rtlsim" }, "raytracing:perf_gate-rt_raycast-nt4:rtlsim": { "32": { - "cycles": 159655, + "cycles": 159677, "instrs": 39330 }, "app": "rt_raycast", "args": "", - "config_hash": "f2e45a7a7018b151", + "config_hash": "fb0b2247eb1c36e6", "configs": "-DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_NUM_THREADS=4", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor.json b/ci/baselines/perf/tensor.json index 85797034f5..89bfffe2f7 100644 --- a/ci/baselines/perf/tensor.json +++ b/ci/baselines/perf/tensor.json @@ -1,56 +1,56 @@ { "tensor:perf_gate-fp16-isw4:rtlsim": { "32": { - "cycles": 197465, + "cycles": 196204, "instrs": 130592 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "d538172b26e2967c", + "config_hash": "51141703c93beb72", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt16:rtlsim": { "32": { - "cycles": 42404, + "cycles": 38764, "instrs": 47296 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "02375c8f3588b56d", + "config_hash": "8afdd549e2e91df6", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=16 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16-mc-nt4:rtlsim": { "32": { - "cycles": 221592, + "cycles": 221653, "instrs": 336448 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "38834e790f4f6a3f", + "config_hash": "33b3e7a8d1fd702d", "configs": "-DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_L2_ENABLE", "driver": "rtlsim" }, "tensor:perf_gate-fp16:rtlsim": { "32": { - "cycles": 256743, + "cycles": 252792, "instrs": 130576 }, "app": "sgemm_tcu", "args": "-m 128 -n 128 -k 128", - "config_hash": "fcf4a952a5f93107", + "config_hash": "0371a3688be32c73", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32", "driver": "rtlsim" }, "tensor:perf_gate-sgemm2-fp16:rtlsim": { "32": { - "cycles": 590798, + "cycles": 594162, "instrs": 409888 }, "app": "sgemm2_tcu", "args": "-m 128 -n 128 -k 64", - "config_hash": "66b2e78e9663c54b", + "config_hash": "c410f503b55870cf", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_mx.json b/ci/baselines/perf/tensor_mx.json index a9ed6b1e98..c10ac69fc3 100644 --- a/ci/baselines/perf/tensor_mx.json +++ b/ci/baselines/perf/tensor_mx.json @@ -1,12 +1,12 @@ { "tensor_mx:perf_gate-mxfp8:rtlsim": { "32": { - "cycles": 76393, + "cycles": 76658, "instrs": 34832 }, "app": "sgemm_tcu_mx", "args": "-m 128 -n 128 -k32", - "config_hash": "eac1b740108b63ac", + "config_hash": "31c2447f43720b96", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_sp.json b/ci/baselines/perf/tensor_sp.json index d898fae929..9e03868a83 100644 --- a/ci/baselines/perf/tensor_sp.json +++ b/ci/baselines/perf/tensor_sp.json @@ -1,12 +1,12 @@ { "tensor_sp:perf_gate-sparse-fp16:rtlsim": { "32": { - "cycles": 183094, + "cycles": 183077, "instrs": 100880 }, "app": "sgemm_tcu_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "1ab2d7ff293df75a", + "config_hash": "069b267054841b9f", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE", "driver": "rtlsim" } diff --git a/ci/baselines/perf/tensor_wg.json b/ci/baselines/perf/tensor_wg.json index 1caa93120b..690531b2c5 100644 --- a/ci/baselines/perf/tensor_wg.json +++ b/ci/baselines/perf/tensor_wg.json @@ -1,34 +1,34 @@ { "tensor_wg:perf_gate-wgmma-fedp2k-rs:rtlsim": { "32": { - "cycles": 158181, + "cycles": 158241, "instrs": 108512 }, "app": "sgemm_tcu_wg", "args": "-m64 -n64 -k64", - "config_hash": "23715dbc37d808e1", + "config_hash": "5a6929889f5a9768", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-fp16-ss:rtlsim": { "32": { - "cycles": 834873, + "cycles": 838095, "instrs": 525536 }, "app": "sgemm_tcu_wg", "args": "-m 128 -n 128 -k 128", - "config_hash": "3eafd3c24fc1aba9", + "config_hash": "531700427d640d57", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" }, "tensor_wg:perf_gate-wgmma-sparse:rtlsim": { "32": { - "cycles": 400023, + "cycles": 399994, "instrs": 258976 }, "app": "sgemm_tcu_wg_sp", "args": "-m 128 -n 128 -k 128", - "config_hash": "448955a5824dd803", + "config_hash": "1bece58b69db57f0", "configs": "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1", "driver": "rtlsim" } diff --git a/ci/beacon.py b/ci/beacon.py new file mode 100755 index 0000000000..a2af644667 --- /dev/null +++ b/ci/beacon.py @@ -0,0 +1,1247 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""beacon -- run Beacon, foldsim's logic-level and area count, on a Vortex +netlist, and read its results by RTL unit or against Vivado. + + run map an elaborated CIRCT netlist and count the logic levels of + every end point (foldsim-estimate-timing) and its LUTs, + flip-flops, memories and DSPs (foldsim-estimate-area) + summarize the deep end points grouped by RTL unit, optionally beside + Vivado's failing ones + paths every deep path op by op, by RTL unit (one per pair of + registers, generate copies folded) + diff a change against its parent revision: the registers that got + deeper or shallower, by register and by unit + validate agreement per register with a Vivado end point table + retime-split the same agreement, split by whether Vivado retimed the paths + ports the depth at the ports of chosen modules' instances, by unit + buffers which elastic buffers pass a deep ready chain or a computed + output through: evidence for an OUT_BUF (OUT_REG) change + calibrate train the learned calibration (scikit-learn) on DUT builds: + Beacon runs and the Vivado end point tables of the same sources + score rank a run's registers by the calibrated near-critical + probability, with predicted slack (pure Python; advisory) + + ci/beacon.py run gen/design.mlir runs/full + ci/beacon.py run runs/full/mapped.mlirbc runs/socket \\ + top=VX_socket__lift black-boxes=VX_core__lift + ci/beacon.py summarize runs/full --min-levels 12 + ci/beacon.py paths runs/full --unit lsu_unit + ci/beacon.py diff runs/parent runs/change --min-levels 8 + ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut + ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \\ + port-modules=VX_elastic_buffer + ci/beacon.py buffers runs/eb + ci/beacon.py calibrate cal.json --build build_core/beacon ep.csv core + ci/beacon.py score build_core/beacon cal.json --top 30 + +A Vivado end point table is a CSV of `slack,levels,logic,net,start,end`, one +row per failing end point (a gate's ep.csv). The DUT flow and logic_gate run +Beacon through the Makefiles instead (make -C hw/syn/xilinx/dut -beacon, +ci/logic_gate.py); see docs/designs/beacon.md. +""" +import argparse +import collections +import csv +import json +import math +import os +import re +import resource +import subprocess +import sys +import time + +# Fold classes of the static fold: module in the design -> cell prefix in +# the folded netlist. +FOLD_CLASSES = { + 'VX_socket__lift': 'VX_socket__lift_model_copy', + 'VX_core__lift': 'VX_core__lift_model_copy', + 'VX_compute_slice__lift': 'VX_compute_slice__lift_copy', + 'foldsim_target': 'foldsim_target_model_copy', +} + +# Vivado appends replication and retiming suffixes in any order +# (_rep__0_replica_1). +VIVADO_CELL = re.compile( + r'_reg((\[\w+\])*(_replica(_\d+)?|_[fb]ret(_+\d+)?|_rep(_+\d+)?)*' + r'|_\d+(_\d+)*|_bram_\d+)$') +FOLD_SUFFIX = re.compile( + r'(_ext(_\d+)?|_fold|_read(_\d+)?|_[fb]ret(_\d+)?|_replica(_\d+)?)+$') +FOLD_SHELL = re.compile(r'^fold\d+_') +DSP_SUBCELL = re.compile(r'^DSP\w*(<\d+>)?_INST$') + + +def norm(s): + return s.replace('.', '_').replace('[', '_').replace(']', '') + + +def ep_key(cell_path, native): + """(class, key) of a Vivado end point pin, or None for fold-shell cells. + + Folded: sim//[/]/. + Native: //[/]/, being the + harness prefix. + """ + # The halves of a dual-port LUTRAM (RAM32X1D/RAM64X1D: DP, SP) sit + # below the memory cell. + cell_path = re.sub(r'/(DP|SP)(/[^/]+)$', r'\2', cell_path) + parts = cell_path.split('/') + if native: + cls = parts[0] + hier = parts[1:-1] + # Distributed RAM primitives below the memory cell, and the sub-cells + # of a DSP macro (DSP58C<0>_INST/DSP_M_DATA58_INST, + # DSP_OUTPUT_INST) below the register it absorbed. + for i, p in enumerate(hier): + if (p.startswith('Memory_reg') or re.match(r'RAM[A-H]', p) + or DSP_SUBCELL.match(p)): + hier = hier[:i] + break + else: + if parts[0] != 'sim' or len(parts) < 3: + return None + cls = parts[1] + hier = parts[2].split('2F') + if not hier or FOLD_SHELL.match(hier[-1]): + return None + leaf = VIVADO_CELL.sub('', hier[-1]) + leaf = FOLD_SUFFIX.sub('', leaf) + return cls, norm('/'.join(hier[:-1] + [leaf])) + + +def numpy(): + # Only the comparisons with Vivado need numpy; counting levels does not. + import numpy as np + return np + + +def spearman(a, b): + np = numpy() + + def rank(x): + x = np.asarray(x, dtype=float) + order = np.argsort(x, kind='mergesort') + ranks = np.empty(len(x)) + ranks[order] = np.arange(len(x)) + # Average the ranks of ties. + _, inv, counts = np.unique(x, return_inverse=True, return_counts=True) + sums = np.bincount(inv, weights=ranks) + return (sums / counts)[inv] + ra, rb = rank(a), rank(b) + if ra.std() == 0 or rb.std() == 0: + return float('nan') + return float(np.corrcoef(ra, rb)[0, 1]) + + +def endpoints(run_dir): + """The rows of a run's end point table, as dicts.""" + with open(f'{run_dir}/endpoints.csv') as f: + yield from csv.DictReader(f) + + +def load_instances(path): + inst = {} + with open(path) as f: + for row in csv.DictReader(f): + inst[int(row['id'])] = (int(row['parent']), row['instance'], + row['module']) + return inst + + +def class_paths(inst, classes): + """ctx -> (class cell prefix, path below the class root).""" + memo = {} + + def get(ctx): + if ctx in memo: + return memo[ctx] + parent, name, module = inst[ctx] + if module in classes: + res = (classes[module], '') + elif ctx == 0: + res = (None, '') + else: + cls, p = get(parent) + res = (cls, f'{p}/{name}' if p else name) + memo[ctx] = res + return res + + sys.setrecursionlimit(10000) + return {ctx: get(ctx) for ctx in inst} + + +INDEX = re.compile(r'_\d+(?=_|/|$)') + + +def unit(key): + d = key.rpartition('/')[0] + return INDEX.sub('_*', d) + + +def cmd_run(args): + """Map (unless the input is already mapped bytecode), then count levels; + from a netlist, also map for area and count it.""" + opt = os.environ.get("FOLDSIM_OPT") or "foldsim-opt" + os.makedirs(args.out, exist_ok=True) + # Map to the cells of the target the levels are counted on. + target = next((o for o in args.options if o.startswith("target=")), + "target=versal") + mapped = args.design + steps = [] + # MLIR maps modules in parallel: the full GPU maps in 53 s and 18 GB, or + # 107 s and 9 GB on one thread. + threads = ["--mlir-disable-threading"] if args.no_threads else [] + if not args.design.endswith(".mlirbc"): + mapped = os.path.join(args.out, "mapped.mlirbc") + pipeline = "foldsim-beacon-map{%s}" % target + if args.keep: + # Keep these modules' instances through inlining, so their ports + # can be reported (ports=, port-modules=). + pipeline = "foldsim-keep-modules{modules=%s},%s" % ( + ",".join(args.keep), pipeline) + steps.append(("map", [opt, args.design, "--emit-bytecode", "-o", mapped, + "--pass-pipeline=builtin.module(%s)" % pipeline] + + threads)) + files = ["output=%s/endpoints.csv" % args.out, + "instances=%s/instances.csv" % args.out, + "report=%s/report.txt" % args.out, + "paths=%s/paths.jsonl" % args.out] + if any(o.startswith("port-modules=") for o in args.options): + files.append("ports=%s/ports.csv" % args.out) + opts = " ".join(files + args.options) + steps.append(("levels", [opt, mapped, "-o", os.devnull, + "--pass-pipeline=builtin.module(" + "foldsim-estimate-timing{%s})" % opts])) + if not args.design.endswith(".mlirbc"): + # Area reads the netlist mapped again, for area. + top = [o for o in args.options if o.startswith(("top=", "black-boxes="))] + area = " ".join([target, "output=%s/area.csv" % args.out, + "memories=%s/area_memories.csv" % args.out, + "report=%s/area.txt" % args.out] + top) + steps.append(("area", [opt, args.design, "-o", os.devnull, + "--pass-pipeline=builtin.module(" + "foldsim-beacon-map{%s area}," + "foldsim-estimate-area{%s})" % (target, area)] + + threads)) + for name, cmd in steps: + start = time.time() + rc = subprocess.call(cmd) + peak = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss / 1e6 + print("%-7s %6.1f s peak %.1f GB" % (name, time.time() - start, peak)) + if rc: + return rc + return 0 + + +def cmd_validate(args): + np = numpy() + + inst = load_instances(f'{args.run_dir}/instances.csv') + classes = FOLD_CLASSES + sub = None + harness = '' + if args.native: + module, prefix = args.native.split('=') + prefix, _, sub = prefix.partition(':') + # A DUT build's table names its cells from the DUT's top down, with + # no harness prefix. + harness = '' if prefix else 'dut/' + prefix = prefix or 'dut' + classes = {module: prefix} + paths = class_paths(inst, classes) + + ours = collections.defaultdict(lambda: [0, 0]) # key -> [max levels, n] + # (class, instance path) -> {register base name: key}, for Vivado names + # that place a register in the generate scope that assigns it rather + # than the one that declares it. + by_dir = collections.defaultdict(dict) + # Vendor IPs, black boxes to Beacon: Vivado's end points inside them have + # no counterpart. + ips = set() + + def class_key(inst_id, name): + cls, p = paths[int(inst_id)] + if cls is None: + return None + # Registers of inlined modules are named /. + base = name.rsplit('.', 1)[0] + full = f'{p}/{base}' if p else base + if sub: + if not full.startswith(sub + '/'): + return None + full = full[len(sub) + 1:] + return cls, full + + for row in endpoints(args.run_dir): + if row['start_kind'] == 'ip': + ip = class_key(row['start_inst'], row['start']) + if ip: + ips.add((ip[0], norm(ip[1]))) + ck = class_key(row['end_inst'], row['end']) + if ck is None: + continue + cls, full = ck + key = (cls, norm(full)) + v = ours[key] + if not v[1]: + d, _, base = full.rpartition('/') + by_dir[(cls, norm(d))][norm(base.rsplit('.', 1)[-1])] = key + v[0] = max(v[0], int(row[args.metric])) + v[1] += 1 + + def fallback(key): + cls, k = key + d, _, leaf = k.rpartition('/') + best = None + for name, cand in by_dir.get((cls, d), {}).items(): + if (leaf == name or leaf.endswith('_' + name)) and ( + best is None or len(name) > len(best[0])): + best = (name, cand) + return best[1] if best else key + + def in_ip(key): + cls, k = key + parts = k.split('/') + return any((cls, '/'.join(parts[:i])) in ips for i in range(1, len(parts))) + + viv = collections.defaultdict(lambda: [0, float('inf'), 0]) # max lv, min slack, n + rows = unmatched_rows = shell_rows = ip_rows = 0 + unmatched = collections.Counter() + with open(args.ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + rows += 1 + key = ep_key(harness + end, args.native) + if key is None or key[0] not in classes.values(): + shell_rows += 1 + continue + if in_ip(key): + ip_rows += 1 + continue + if key not in ours and key[1].endswith('_rdata'): + # The read register of a synchronous RAM is the memory's + # read port in the design. + key = (key[0], key[1][:-len('_rdata')]) + if (key not in ours and key[1].endswith('_rdata_r') + and (key[0], key[1][:-len('_rdata_r')] + '_ram') in ours): + # So is the registered read of a synchronous LUTRAM. + key = (key[0], key[1][:-len('_rdata_r')] + '_ram') + if key not in ours: + key = fallback(key) + if key not in ours: + unmatched_rows += 1 + unmatched[key] += 1 + continue + v = viv[key] + v[0] = max(v[0], int(levels)) + v[1] = min(v[1], float(slack)) + v[2] += 1 + + keys = sorted(viv) + if not keys: + print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' + f'inside vendor IPs {ip_rows}, unmatched {unmatched_rows}, ' + f'matched 0') + for (cls, k), n in unmatched.most_common(args.top): + print(f' {n:6d} {cls}: {k}') + return + a = np.array([ours[k][0] for k in keys]) + lv = np.array([viv[k][0] for k in keys]) + sl = np.array([viv[k][1] for k in keys]) + print(f'ep rows {rows}: fold shell / channels {shell_rows}, ' + f'inside vendor IPs {ip_rows}, unmatched {unmatched_rows}, ' + f'matched {rows - shell_rows - ip_rows - unmatched_rows} ' + f'in {len(keys)} registers/memories') + print(f'Beacon keys in the classes: {len(ours)}') + print(f'Spearman(Beacon levels, Vivado levels) = {spearman(a, lv):.3f}') + print(f'Spearman(Beacon levels, -slack) = {spearman(a, -sl):.3f}') + ratio = a[lv > 0] / lv[lv > 0] + print(f'Beacon/Vivado levels: median {np.median(ratio):.2f}, ' + f'p10 {np.percentile(ratio, 10):.2f}, p90 {np.percentile(ratio, 90):.2f}') + + print('\nVivado levels -> Beacon levels (registers/memories):') + for v in range(0, int(lv.max()) + 1): + sel = a[lv == v] + if len(sel): + print(f' {v:3d}: n={len(sel):6d} Beacon median {np.median(sel):5.1f} ' + f'p10 {np.percentile(sel, 10):5.1f} p90 {np.percentile(sel, 90):5.1f}') + + if args.native: + # The table lists the end points near or past the period; a measured + # register fails when its worst slack is negative. + scr = np.array([v[0] for v in ours.values()]) + fails = np.array([k in viv and viv[k][1] < 0 for k in ours]) + print('\nflag registers at >= T levels (failing: negative slack):') + print(' T flagged precision recall') + for t in range(4, 21, 2): + flagged = scr >= t + if not flagged.any(): + break + prec = fails[flagged].mean() + rec = flagged[fails].mean() + print(f' {t:4d} {flagged.sum():8d} {prec:10.2f} {rec:7.2f}') + + if args.json: + summary = { + 'rows': rows, 'shell': shell_rows, 'vendor_ip': ip_rows, + 'unmatched': unmatched_rows, + 'registers': len(keys), 'measured': len(ours), + 'rho_levels': round(spearman(a, lv), 3), + 'rho_slack': round(spearman(a, -sl), 3), + 'ratio': round(float(np.median(ratio)), 2), + 'pairs': [[int(x), int(y), n] for (x, y), n in + sorted(collections.Counter(zip(a, lv)).items())], + } + if args.native: + summary['pr'] = [ + [t, int((scr >= t).sum()), round(float(fails[scr >= t].mean()), 3), + round(float((scr >= t)[fails].mean()), 3)] + for t in range(2, 23) if (scr >= t).any()] + with open(args.json, 'w') as f: + json.dump(summary, f) + + print(f'\nunmatched Vivado cells (top {args.top} by rows):') + for (cls, k), n in unmatched.most_common(args.top): + print(f' {n:6d} {cls}: {k}') + + order = np.argsort(-a, kind='mergesort') + print(f'\ndeepest measured registers with failing Vivado end points:') + for i in order[:args.top]: + k = keys[i] + print(f' beacon {a[i]:3d} vivado {lv[i]:3d} slack {sl[i]:7.3f} ' + f'n {viv[k][2]:5d} {k[0]}: {k[1]}') + + + +def cmd_summarize(args): + + inst = load_instances(f'{args.run_dir}/instances.csv') + classes = FOLD_CLASSES + if args.native: + module, prefix = args.native.split('=') + classes = {module: prefix} + paths = class_paths(inst, classes) + + groups = collections.defaultdict( + lambda: {'n': 0, 'max': -1, 'hist': collections.Counter(), + 'worst': None}) + for row in endpoints(args.run_dir): + levels = int(row[args.metric]) + if levels < args.min_levels: + continue + cls, p = paths[int(row['end_inst'])] + end, start = row['end'], row['start'] + full = f'{p}/{end}' if p else end + g = groups[(cls, unit(norm(full)))] + g['n'] += 1 + g['hist'][levels] += 1 + if levels > g['max']: + sp = paths[int(row['start_inst'])][1] + g['max'] = levels + g['worst'] = (f'{sp}/{start}' if sp else start, full, row['fanout']) + + viv = collections.defaultdict(lambda: [0, 0]) # n, max levels + if args.ep: + with open(args.ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + if int(levels) < args.min_levels: + continue + key = ep_key(end, args.native) + if key is None: + continue + v = viv[(key[0], unit(key[1]))] + v[0] += 1 + v[1] = max(v[1], int(levels)) + + order = sorted(groups, key=lambda k: (-groups[k]['max'], -groups[k]['n'])) + print(f'end point bits of at least {args.min_levels} levels, by unit ' + f'({sum(g["n"] for g in groups.values())} in {len(groups)} units)') + hdr = f'{"max":>4} {"mode":>4} {"bits":>8}' + if args.ep: + hdr += f' {"viv n":>7} {"viv max":>7}' + print(hdr + ' unit') + for k in order[:args.top]: + g = groups[k] + mode = g['hist'].most_common(1)[0][0] + line = f'{g["max"]:4d} {mode:4d} {g["n"]:8d}' + if args.ep: + v = viv.get(k, [0, 0]) + line += f' {v[0]:7d} {v[1]:7d}' + print(f'{line} {k[0] or ""}: {k[1]}') + s, e, fo = g['worst'] + print(f'{"":>20}from {s}\n{"":>20}to {e}') + if args.ep: + missed = sorted((k for k in viv if k not in groups), + key=lambda k: -viv[k][0]) + print(f'\nVivado units at or above {args.min_levels} levels the ' + f'Beacon ranks below it ({len(missed)}):') + for k in missed[:args.top]: + print(f'{viv[k][0]:8d} {viv[k][1]:4d} {k[0]}: {k[1]}') + + + +def cmd_paths(args): + """Every path of at least --min-levels, by unit, op by op.""" + sys.path.insert(0, os.path.join(os.path.dirname(os.path.abspath(__file__)), + '..', 'hw', 'scripts')) + import beacon_summary + count, _ = beacon_summary.write_paths(sys.stdout, args.run_dir, + args.min_levels, args.unit) + if not count: + print('(Beacon lists paths from its path-levels= on, 12 by ' + 'default)') + + +def load_ports(run_dir): + """instance -> {port: (direction, levels)} from a run's ports.csv.""" + ports = collections.defaultdict(dict) + with open(f'{run_dir}/ports.csv') as f: + for row in csv.DictReader(f): + ports[(row['instance'], row['module'])][row['port']] = ( + row['direction'], int(row['levels'])) + return ports + + +def cmd_ports(args): + """The deepest instance of each unit, port by port.""" + units = {} + for (inst, module), ports in load_ports(args.run_dir).items(): + deepest = max(levels for _, levels in ports.values()) + key = (INDEX.sub('_*', norm(inst)), module.rstrip('0123456789').rstrip('_')) + if key not in units or deepest > units[key][0]: + units[key] = (deepest, inst, ports) + for (unit, module), (deepest, inst, ports) in sorted( + units.items(), key=lambda kv: -kv[1][0])[:args.top]: + if deepest < args.min_levels: + break + print(f'{deepest:3d} {module} {unit}') + for port, (direction, levels) in ports.items(): + print(f' {direction:>3} {port:24s} {levels:3d}') + + +BIT_INDEX = re.compile(r'(\[\d+\])+$') + + +def register_depths(run_dir, metric): + """register -> (levels, deepest start) over its bits, for a run.""" + inst = load_instances(f'{run_dir}/instances.csv') + paths = class_paths(inst, FOLD_CLASSES) + regs = {} + for row in endpoints(run_dir): + levels = int(row[metric]) + cls, p = paths[int(row['end_inst'])] + end = row['end'] + full = f'{p}/{end}' if p else end + key = (cls or '', BIT_INDEX.sub('', full)) + if key not in regs or levels > regs[key][0]: + sp = paths[int(row['start_inst'])][1] + regs[key] = (levels, f'{sp}/{row["start"]}' if sp else row['start']) + return regs + + +def cmd_diff(args): + """A change's effect on depth: every register at or above --min-levels in + either run whose depth moved, worst first. Exits 1 on a regression: a + register --fail-delta or more levels deeper, one that reaches --fix-levels + from below, or one only the change has at --fix-levels. If both runs + counted area, also the design's resources and the units whose area moved; + exits 3 if only area regressed: a resource of the design more than + --area-tolerance larger. + + Against routed Vivado runs of the same DUT's revisions, a register that + got four or more levels deeper lost slack every time, while a rise of one + to three is within Vivado's own spread from run to run. The mean depth of + the deepest registers tracks the change in worst slack; the single deepest + one does not.""" + before = register_depths(args.before, args.metric) + after = register_depths(args.after, args.metric) + + def deepest(regs): + return max((v[0] for v in regs.values()), default=0) + + def watched(regs): + return sum(1 for v in regs.values() if v[0] >= args.min_levels) + + def top_mean(regs, n=50): + depths = sorted((v[0] for v in regs.values()), reverse=True)[:n] + return sum(depths) / len(depths) if depths else 0.0 + + area_before, area_after = load_area(args.before), load_area(args.after) + area_head = '' + if bool(area_before) != bool(area_after): + print('area: only the %s run counted it; not compared' + % ('reference' if area_before else 'changed')) + if area_before and area_after: + tb, ta = area_before[''], area_after[''] + area_head = '; ' + ', '.join( + f'{k} {tb[k]:g} -> {ta[k]:g}' for k in ('lut', 'bram', 'dsp')) + print(f'deepest path {deepest(before)} -> {deepest(after)} levels; ' + f'mean of the 50 deepest {top_mean(before):.1f} -> {top_mean(after):.1f}; ' + f'registers at or above {args.min_levels} levels ' + f'{watched(before)} -> {watched(after)}{area_head}') + + deeper, shallower, new, gone = [], [], [], [] + for key, (lv, start) in after.items(): + if key not in before: + if lv >= args.min_levels: + new.append((lv, key, start)) + continue + old = before[key][0] + if lv > old and lv >= args.min_levels: + deeper.append((lv, old, key, start)) + elif lv < old and old >= args.min_levels: + shallower.append((lv, old, key, start)) + for key, (lv, start) in before.items(): + if key not in after and lv >= args.min_levels: + gone.append((lv, key, start)) + + def name(key): + return f'{key[0]}: {key[1]}' if key[0] else key[1] + + def show(title, rows, moved): + if not rows: + return + print(f'\n{title} ({len(rows)})') + for row in rows[:args.top]: + if moved: + lv, old, key, start = row + print(f'{old:4d} -> {lv:<4d} {name(key)}') + else: + lv, key, start = row + print(f'{lv:12d} {name(key)}') + print(f'{"":>13}from {start}') + + deeper.sort(key=lambda r: (-r[0], r[1] - r[0])) + shallower.sort(key=lambda r: (-r[1], r[0] - r[1])) + new.sort(key=lambda r: -r[0]) + gone.sort(key=lambda r: -r[0]) + regress = [r for r in deeper if r[0] - r[1] >= args.fail_delta + or (r[0] >= args.fix_levels > r[1])] + regress_keys = {r[2] for r in regress} + show(f'REGRESSION: {args.fail_delta} or more levels deeper, or now at ' + f'{args.fix_levels}', regress, True) + show('deeper by less (watch)', [r for r in deeper if r[2] not in regress_keys], True) + show('new at or above the watch', new, False) + show('shallower', shallower, True) + show('no longer present', gone, False) + + # The same movement by unit: the deepest register of each unit. + units = collections.defaultdict(lambda: [0, 0]) + for i, regs in enumerate((before, after)): + for (cls, reg), (lv, _) in regs.items(): + u = units[(cls, unit(norm(reg)))] + u[i] = max(u[i], lv) + moved = sorted(((b, a, k) for k, (b, a) in units.items() + if a != b and max(a, b) >= args.min_levels), + key=lambda r: (-max(r[0], r[1]), r[0] - r[1])) + if moved: + print(f'\nunits whose deepest register moved ({len(moved)})') + for b, a, (cls, u) in moved[:args.top]: + print(f'{b:4d} -> {a:<4d} {cls + ": " if cls else ""}{u}') + + area_regress = False + if area_before and area_after: + area_regress = show_area(area_before, area_after, args) + + # A register the reference lacks may be a renamed one, so it fails only at + # the fix threshold, as a new unit does in logic_gate. + if regress or any(r[0] >= args.fix_levels for r in new): + return 1 + return 3 if area_regress else 0 + + +AREA_KEYS = ('lut', 'ff', 'lutram', 'bram', 'uram', 'dsp') + + +def load_area(run_dir): + """The design's resources ('') and each folded unit's own, summed over its + copies, from a run's area.csv, or None. A unit's own resources leave out + its children's, so a change shows at the unit it is in. `lut` counts + distributed RAM too, as Vivado's utilization does.""" + path = f'{run_dir}/area.csv' + if not os.path.exists(path): + return None + units = {} + with open(path) as fh: + for row in csv.DictReader(fh): + prefix = 'total_' if not row['path'] else '' + res = {k: float(row[prefix + k]) for k in AREA_KEYS} + res['lut'] += res['lutram'] + key = unit(norm(row['path']) + '/x') if row['path'] else '' + if key in units: + for k in AREA_KEYS: + units[key][k] += res[k] + else: + units[key] = res + return units if '' in units else None + + +def show_area(before, after, args): + """The design's resources before and after, and the units whose own LUTs, + memories or DSPs moved most; True if a resource of the design grew more + than --area-tolerance.""" + tb, ta = before[''], after[''] + print('\narea (design)') + regress = False + for k in AREA_KEYS: + b, a = tb[k], ta[k] + pct = (a - b) / b * 100 if b else (100.0 if a else 0.0) + flag = '' + if pct > 100 * args.area_tolerance: + flag, regress = ' REGRESSION', True + print(f' {k:7s} {b:10g} -> {a:<10g} {pct:+6.1f}%{flag}') + moved = [] + for key in set(before) | set(after): + if not key: + continue + b = before.get(key, dict.fromkeys(AREA_KEYS, 0.0)) + a = after.get(key, dict.fromkeys(AREA_KEYS, 0.0)) + if any(abs(a[k] - b[k]) >= 0.5 for k in AREA_KEYS): + moved.append((abs(a['lut'] - b['lut']) + 100 * ( + abs(a['bram'] - b['bram']) + abs(a['uram'] - b['uram']) + + abs(a['dsp'] - b['dsp'])), key, b, a)) + if moved: + moved.sort(key=lambda r: -r[0]) + print(f'\nunits whose own area moved ({len(moved)}), all copies') + for _, key, b, a in moved[:args.top]: + parts = [f'{k} {b[k]:g} -> {a[k]:g}' for k in AREA_KEYS + if abs(a[k] - b[k]) >= 0.5] + print(f' {key}: ' + ', '.join(parts)) + return regress + + +def cmd_buffers(args): + """VX_elastic_buffer instances whose ready chain or output is deep. + + ready_out is the input from downstream and ready_in the output upstream: + a buffer that passes ready combinationally drives ready_in at least as + deep as ready_out arrives, and a skid buffer (SIZE 2) or OUT_BUF cuts the + chain. An output (data_out, valid_out) above one level is computed, not + registered: OUT_REG would register it. + """ + rows = [] + for (inst, module), ports in load_ports(args.run_dir).items(): + if not module.startswith('VX_elastic_buffer'): + continue + get = lambda p: ports.get(p, ('', -1))[1] + chain = get('ready_in') + through = chain >= 1 and chain >= get('ready_out') + out = max(get('data_out'), get('valid_out')) + advice = [] + if through and chain >= args.min_levels: + advice.append(f'ready chain of {chain} levels passes through') + if out >= args.min_levels: + advice.append(f'output computed at {out} levels') + if advice: + rows.append((max(chain if through else 0, out), inst, module, + get('valid_in'), get('ready_out'), chain, out, + '; '.join(advice))) + units = {} + for r in rows: + key = INDEX.sub('_*', norm(r[1])) + if key not in units or r[0] > units[key][0][0]: + units[key] = (r, 1 + (units[key][1] if key in units else 0)) + else: + units[key] = (units[key][0], units[key][1] + 1) + print(f'elastic buffers at or above {args.min_levels} levels: {len(rows)} ' + f'in {len(units)} units') + print(f'{"lv":>3} {"n":>4} {"valid_in":>8} {"ready_out":>9} ' + f'{"ready_in":>8} {"out":>4} unit / advice') + for key, (r, n) in sorted(units.items(), key=lambda kv: -kv[1][0][0]): + print(f'{r[0]:3d} {n:4d} {r[3]:8d} {r[4]:9d} {r[5]:8d} {r[6]:4d} ' + f'{key}\n{"":32}{r[7]}') + + +RETIMED = re.compile(r'_[fb]ret') + + +def cmd_retime_split(args): + np = numpy() + run_dir, ep, native, metric = args.run_dir, args.ep, args.native, args.metric + module, prefix = native.split('=') + inst = load_instances(f'{run_dir}/instances.csv') + paths = class_paths(inst, {module: prefix}) + ours = collections.defaultdict(int) + for row in endpoints(run_dir): + cls, p = paths[int(row['end_inst'])] + base = row['end'].rsplit('.', 1)[0] + key = (cls, norm(f'{p}/{base}' if p else base)) + ours[key] = max(ours[key], int(row[metric])) + viv = collections.defaultdict(lambda: [0, False]) + with open(ep) as f: + r = csv.reader(f) + next(r) + for slack, levels, logic, net, start, end in r: + key = ep_key(end, True) + if key not in ours: + continue + v = viv[key] + v[0] = max(v[0], int(levels)) + v[1] |= bool(RETIMED.search(start) or RETIMED.search(end)) + for retimed in (False, True): + keys = [k for k, v in viv.items() if v[1] == retimed] + a = np.array([ours[k] for k in keys]) + b = np.array([viv[k][0] for k in keys]) + if len(keys) < 3: + continue + ratio = np.median(a[b > 0] / b[b > 0]) if (b > 0).any() else float('nan') + print(f'retimed={retimed!s:5}: {len(keys):5d} registers, Spearman ' + f'{spearman(a, b):.3f}, median Beacon/Vivado {ratio:.2f}, ' + f'Beacon-Vivado median {np.median(a - b):+.1f}') + + +# Learned calibration (calibrate, score): per-register features of a run, +# all from Beacon's own outputs. The p_*/n_* columns come from the deepest +# path in paths.jsonl, for registers at or above the model's path levels; +# below that they are missing, as in training. +CAL_FEATURES = ( + 'lev', 'lev_mean', 'log2_bits', 'frac_deep', 'log2_nbits_deep', + 'k_d', 'k_wd', 'k_wctl', 'k_rd', 'k_port', + 's_reg', 's_port', 's_bram', 's_ip', 's_other', + 'log2_sfo', 'log2_sfomax', 'log2_in_fo', 'log2_in_fo_deep', 'cross', 'xmax', + 'hier_depth', 'log2_lca_bits', 'lca_up', + 'p_lut', 'p_carry', 'p_cmp', 'p_lutram', 'p_bram', 'p_dsp', + 'n_carry', 'n_cmp', 'p_maxfo', 'p_sumlog_fo', 'p_nfo64', 'p_ninst', 'p_nsrc', + 'hand', +) +CAL_OPS = ('lut', 'carry', 'cmp', 'lutram', 'bram', 'dsp') +CAL_KINDS = {'d': 'd', 'wd': 'wd', 'wa': 'wctl', 'we': 'wctl', 'wm': 'wctl', + 'ra': 'rd', 're': 'rd'} +CAL_NEAR = 0.3 # ns: the near-critical set the model ranks +CAL_CENSOR = 1.0 # ns: a gate's ep.csv lists end points under 1 ns only + + +def log2p(x): + return math.log2(1 + max(0, x)) + + +def register_features(run_dir, path_levels): + """(features, by_dir, ips) of a run: register -> {feature: value}, all + end point bits under one name (a memory's ports included), keyed by its + normalized path below the run's top; the instance -> {base name: key} + map and the vendor IP instances, for matching Vivado's names.""" + inst = load_instances(f'{run_dir}/instances.csv') + memo = {} + + def ipath(ctx): + if ctx not in memo: + parent, name, _ = inst[ctx] + p = '' if ctx == 0 else ipath(parent) + memo[ctx] = '' if ctx == 0 else (f'{p}/{name}' if p else name) + return memo[ctx] + + def full(ctx, name): + p = ipath(int(ctx)) + return f'{p}/{name}' if p else name + + sys.setrecursionlimit(10000) + regs, ips = {}, set() + by_dir = collections.defaultdict(dict) + for row in endpoints(run_dir): + if row['start_kind'] == 'ip': + ips.add(norm(full(row['start_inst'], row['start'].rsplit('.', 1)[0]))) + f = full(row['end_inst'], row['end'].rsplit('.', 1)[0]) + key = norm(f) + lv, fo, sfo = int(row['levels']), int(row.get('in_fanout') or 0), int(row['fanout']) + r = regs.get(key) + if r is None: + r = regs[key] = dict(lev=-1, bits=0, sum=0, sfomax=0, fo=0, xmax=0, lvs=[]) + d, _, b = f.rpartition('/') + by_dir[norm(d)][norm(b.rsplit('.', 1)[-1])] = key + r['bits'] += 1 + r['sum'] += lv + r['lvs'].append(lv) + r['fo'] = max(r['fo'], fo) + r['xmax'] = max(r['xmax'], int(row['crossings'])) + if row['start_kind'] == 'reg': + r['sfomax'] = max(r['sfomax'], sfo) + if lv > r['lev']: + r.update(lev=lv, kind=row['kind'], skind=row['start_kind'], sfo=sfo, + fo_deep=fo, cross=int(row['crossings']), + start=norm(full(row['start_inst'], row['start']))) + + paths = {} + if os.path.exists(f'{run_dir}/paths.jsonl'): + with open(f'{run_dir}/paths.jsonl') as f: + for line in f: + p = json.loads(line) + key = norm(full(p['end_inst'], p['end'].rsplit('.', 1)[0])) + if key not in paths or p['levels'] > paths[key]['levels']: + paths[key] = p + + # Bits below each instance: the span of a register and its deepest start. + under = collections.Counter() + for k, r in regs.items(): + parts = k.split('/')[:-1] + for i in range(len(parts) + 1): + under['/'.join(parts[:i])] += r['bits'] + + feats = {} + for k, r in regs.items(): + lv = r['lev'] + a, c = k.split('/')[:-1], r['start'].split('/')[:-1] + n = 0 + while n < min(len(a), len(c)) and a[n] == c[n]: + n += 1 + deep = sum(1 for v in r['lvs'] if v >= lv - 1) + kind = CAL_KINDS.get(r['kind'], 'port') + skind = r['skind'] if r['skind'] in ('reg', 'port', 'bram', 'ip') else 'other' + x = dict(lev=lv, lev_mean=r['sum'] / r['bits'], log2_bits=log2p(r['bits']), + frac_deep=deep / r['bits'], log2_nbits_deep=log2p(deep), + log2_sfo=log2p(r['sfo']), log2_sfomax=log2p(r['sfomax']), + log2_in_fo=log2p(r['fo']), log2_in_fo_deep=log2p(r['fo_deep']), + cross=r['cross'], xmax=r['xmax'], hier_depth=len(a), + log2_lca_bits=log2p(under['/'.join(a[:n])]), lca_up=len(a) - n) + for v in ('d', 'wd', 'wctl', 'rd', 'port'): + x['k_' + v] = float(kind == v) + for v in ('reg', 'port', 'bram', 'ip', 'other'): + x['s_' + v] = float(skind == v) + p = paths.get(k) if lv >= path_levels else None + if p is None: + x.update((f, float('nan')) for f in CAL_FEATURES if f.startswith(('p_', 'n_'))) + x['hand'] = lv + else: + ops = p['ops'] # [levels, kind, fanout, instance, name, source], end first + cost, count = collections.Counter(), collections.Counter() + for i in range(len(ops) - 1): + op = ops[i][1] if ops[i][1] in CAL_OPS else 'lut' + cost[op] += ops[i][0] - ops[i + 1][0] + count[op] += 1 + fos = [o[2] for o in ops[1:-1]] + x.update((f'p_{op}', cost[op]) for op in CAL_OPS) + x.update(n_carry=count['carry'], n_cmp=count['cmp'], + p_maxfo=log2p(max(fos, default=0)), + p_sumlog_fo=sum(log2p(v) for v in fos), + p_nfo64=sum(1 for v in fos if v >= 64), + p_ninst=len({o[3] for o in ops}), + p_nsrc=len({o[5].split(':')[0] for o in ops})) + # Against Vivado's path delay, a carry level costs about 0.42 of + # a LUT level and a compare 0.15. + x['hand'] = lv - 0.58 * cost['carry'] - 0.85 * cost['cmp'] + feats[k] = x + return feats, by_dir, ips + + +def vivado_slacks(ep, regs, by_dir, ips): + """register -> worst slack of a DUT build's end point table (cells named + from the DUT's top down), matched to a run's registers as validate does.""" + def in_ip(k): + parts = k.split('/') + return any('/'.join(parts[:i]) in ips for i in range(1, len(parts))) + + def fallback(k): + d, _, leaf = k.rpartition('/') + best = None + for name, cand in by_dir.get(d, {}).items(): + if (leaf == name or leaf.endswith('_' + name)) and ( + best is None or len(name) > len(best[0])): + best = (name, cand) + return best[1] if best else k + + slack = {} + with open(ep) as f: + r = csv.reader(f) + next(r) + for s, levels, logic, net, start, end in r: + key = ep_key('top/' + end, True) + if key is None or in_ip(key[1]): + continue + k = key[1] + if k not in regs and k.endswith('_rdata'): + k = k[:-len('_rdata')] + if k not in regs and k.endswith('_rdata_r') and k[:-len('_rdata_r')] + '_ram' in regs: + k = k[:-len('_rdata_r')] + '_ram' + if k not in regs: + k = fallback(k) + if k in regs: + slack[k] = min(slack.get(k, float('inf')), float(s)) + return slack + + +def average_precision(score, positive): + """Average precision of a ranking, tied scores taken together.""" + np = numpy() + order = np.argsort(-np.asarray(score, float), kind='mergesort') + s, y = np.asarray(score, float)[order], np.asarray(positive, bool)[order] + tp = np.cumsum(y) + last = np.r_[s[1:] != s[:-1], True] + prec, rec = tp[last] / (np.nonzero(last)[0] + 1), tp[last] / max(y.sum(), 1) + return float(np.sum(prec * np.diff(np.r_[0.0, rec]))) + + +def hgb_json(m, link): + """A fitted HistGradientBoosting model as plain trees: a node goes left + when its feature is <= t, or is missing and m is set; a leaf has f -1.""" + trees = [] + for (pred,) in m._predictors: + n = pred.nodes + leaf = n['is_leaf'].astype(bool) + trees.append(dict( + f=[-1 if lf else int(i) for i, lf in zip(n['feature_idx'], leaf)], + t=[0 if lf else float(t) for t, lf in zip(n['num_threshold'], leaf)], + m=[int(v) for v in n['missing_go_to_left']], + l=[int(v) for v in n['left']], r=[int(v) for v in n['right']], + v=[float(f'{v:.7g}') if lf else 0 for v, lf in zip(n['value'], leaf)])) + return dict(link=link, base=float(m._baseline_prediction.ravel()[0]), trees=trees) + + +def hgb_predict(model, x): + """Evaluate hgb_json trees on one feature row (a list).""" + raw = model['base'] + for t in model['trees']: + i = 0 + f = t['f'] + while f[i] >= 0: + v = x[f[i]] + i = t['l'][i] if (t['m'][i] if v != v else v <= t['t'][i]) else t['r'][i] + raw += t['v'][i] + return 1 / (1 + math.exp(-raw)) if model['link'] == 'logistic' else raw + + +def cal_models(X, slack, weight, features): + """Fit the slack regressor and the near-critical and failing classifiers: + gradient-boosted trees, never ranking a deeper register (lev, hand) safer.""" + from sklearn.ensemble import (HistGradientBoostingClassifier, + HistGradientBoostingRegressor) + kw = dict(max_iter=300, learning_rate=0.05, max_leaf_nodes=15, min_samples_leaf=50, + l2_regularization=1.0, early_stopping=False, random_state=0) + up = [1 if f in ('lev', 'hand') else 0 for f in features] + reg = HistGradientBoostingRegressor(monotonic_cst=[-v for v in up], **kw) + near = HistGradientBoostingClassifier(monotonic_cst=up, **kw) + fail = HistGradientBoostingClassifier(monotonic_cst=up, **kw) + return (reg.fit(X, slack, sample_weight=weight), + near.fit(X, slack < CAL_NEAR, sample_weight=weight), + fail.fit(X, slack < 0, sample_weight=weight)) + + +def cmd_calibrate(args): + """Train the calibration on DUT builds: each a Beacon run and the Vivado + end point table of the same sources. Writes the model as JSON.""" + np = numpy() + builds = [] + for b in args.build: + if len(b) not in (2, 3): + sys.exit('--build RUN_DIR EP_CSV [GROUP]') + run_dir, ep = b[:2] + feats, by_dir, ips = register_features(run_dir, args.path_levels) + slacks = vivado_slacks(ep, feats, by_dir, ips) + keys = sorted(feats) + X = np.array([[feats[k][f] for f in CAL_FEATURES] for k in keys], float) + y = np.array([min(slacks.get(k, CAL_CENSOR), CAL_CENSOR) for k in keys]) + measured = np.array([k in slacks for k in keys]) + builds.append(dict(run=run_dir, group=b[2] if len(b) == 3 else run_dir, + X=X, y=y, measured=measured)) + print(f'{run_dir}: {len(keys)} registers, {measured.sum()} in the table, ' + f'{(y < 0).sum()} failing, {(y < CAL_NEAR).sum()} under {CAL_NEAR} ns') + + def fit(train): + X = np.vstack([b['X'] for b in train]) + y = np.concatenate([b['y'] for b in train]) + # Every build weighs the same, whatever its register count. + w = np.concatenate([np.full(len(b['y']), len(y) / len(train) / len(b['y'])) + for b in train]) + return cal_models(X, y, w, CAL_FEATURES) + + if args.holdout: + print('\nleave one build out (builds of its group left out of training):') + print(f'{"":40} near-critical AP Spearman vs -slack') + print(f'{"build":40} {"levels":>7} {"model":>7} {"levels":>7} {"model":>7}') + for b in builds: + train = [o for o in builds if o['group'] != b['group']] + if not train: + continue + reg, near, _ = fit(train) + lev, m, y = b['X'][:, 0], b['measured'], b['y'] + ap = [average_precision(s, y < CAL_NEAR) if (y < CAL_NEAR).any() else float('nan') + for s in (lev, near.predict_proba(b['X'])[:, 1])] + rho = [spearman(s[m], -y[m]) for s in (lev, -reg.predict(b['X']))] + name = '/'.join(os.path.normpath(b['run']).split('/')[-2:]) + print(f'{name[-40:]:40} {ap[0]:7.2f} {ap[1]:7.2f} {rho[0]:7.2f} {rho[1]:7.2f}') + + import sklearn + reg, near, fail = fit(builds) + model = dict( + features=list(CAL_FEATURES), path_levels=args.path_levels, near=CAL_NEAR, + censor=CAL_CENSOR, sklearn=sklearn.__version__, + builds=[dict(run=b['run'], group=b['group'], registers=len(b['y']), + failing=int((b['y'] < 0).sum())) for b in builds], + slack=hgb_json(reg, 'identity'), p_near=hgb_json(near, 'logistic'), + p_fail=hgb_json(fail, 'logistic')) + with open(args.model, 'w') as f: + json.dump(model, f, separators=(',', ':')) + print(f'\nwrote {args.model} ({os.path.getsize(args.model) // 1024} KB, ' + f'{len(builds)} builds)') + + +def cmd_score(args): + """Rank a run's registers by the calibrated near-critical probability, + with the predicted worst slack. Advisory: the gates stay on levels.""" + with open(args.model) as f: + model = json.load(f) + feats, _, _ = register_features(args.run_dir, model['path_levels']) + names = model['features'] + if any(math.isnan(x['p_lut']) for x in feats.values() + if x['lev'] >= model['path_levels']): + print(f'warning: registers at or above {model["path_levels"]} levels lack ' + f'a path in paths.jsonl; rerun with path-levels={model["path_levels"]}') + rows = [] + for k, x in feats.items(): + v = [x[n] for n in names] + rows.append((hgb_predict(model['p_near'], v), hgb_predict(model['slack'], v), + hgb_predict(model['p_fail'], v), x['lev'], k)) + rows.sort(key=lambda r: (-r[0], r[1])) + low = sorted(r[1] for r in rows)[:50] + print(f'{len(rows)} registers; P(slack < {model["near"]} ns) >= 0.5: ' + f'{sum(r[0] >= 0.5 for r in rows)}; mean of the 50 lowest predicted ' + f'slacks {sum(low) / max(1, len(low)):+.3f} ns (model trained on ' + f'{len(model["builds"])} builds)') + print(f'{"P(near)":>7} {"slack":>6} {"P(fail)":>7} {"lv":>3} register') + for p, s, pf, lv, k in [r for r in rows if r[3] >= args.min_levels][:args.top]: + print(f'{p:7.2f} {s:+6.2f} {pf:7.2f} {lv:3d} {k}') + if args.csv: + with open(args.csv, 'w', newline='') as f: + w = csv.writer(f) + w.writerow(['register', 'levels', 'p_near', 'slack', 'p_fail']) + for p, s, pf, lv, k in rows: + w.writerow([k, lv, f'{p:.4f}', f'{s:.4f}', f'{pf:.4f}']) + + + +def main(): + ap = argparse.ArgumentParser(description=__doc__, + formatter_class=argparse.RawDescriptionHelpFormatter) + sub = ap.add_subparsers(dest="cmd", required=True) + + p = sub.add_parser("run", help="map a netlist and count its levels") + p.add_argument("design", help="elaborated netlist (.mlir), or mapped bytecode (.mlirbc)") + p.add_argument("out", help="output directory") + p.add_argument("options", nargs="*", + help="foldsim-estimate-timing options (top=, target=, black-boxes=, " + "trace=, retime-levels=, ...); the design is mapped to the " + "cells of target= (versal by default)") + p.add_argument("--no-threads", action="store_true", + help="map on one thread: half the peak memory, twice the time") + p.add_argument("--keep", action="append", default=[], metavar="MODULE", + help="keep MODULE's instances through inlining, to report " + "their ports with port-modules=MODULE (repeatable)") + p.set_defaults(fn=cmd_run) + + p = sub.add_parser("summarize", help="deep end points by RTL unit") + p.add_argument('run_dir') + p.add_argument('--min-levels', type=int, default=10) + p.add_argument('--ep', help="a Vivado end point table to group beside") + p.add_argument('--native', metavar='MODULE=PREFIX') + p.add_argument('--top', type=int, default=40) + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the run's column to group by") + p.set_defaults(fn=cmd_summarize) + + p = sub.add_parser("paths", help="deep paths op by op, by RTL unit") + p.add_argument('run_dir') + p.add_argument('--min-levels', type=int, default=12) + p.add_argument('--unit', help="only the units whose name contains this") + p.set_defaults(fn=cmd_paths) + + p = sub.add_parser("validate", help="agreement per register with Vivado") + p.add_argument('run_dir') + p.add_argument('ep') + p.add_argument('--top', type=int, default=30) + p.add_argument('--native', metavar='MODULE=PREFIX[:SUBPATH]', + help='native gate: the measured top and the harness ' + 'prefix of its cells (e.g. VX_socket__lift=dut); with ' + 'SUBPATH, only the measured end points below it, which ' + 'the harness instantiates as PREFIX; an empty PREFIX (VX_core_top=) for ' + 'a DUT build, whose cells have no harness prefix') + p.add_argument('--json', help='also write the agreement to this file') + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the run's column to compare") + p.set_defaults(fn=cmd_validate) + + p = sub.add_parser("ports", help="port depths of chosen modules, by unit") + p.add_argument('run_dir') + p.add_argument('--min-levels', type=int, default=8) + p.add_argument('--top', type=int, default=40) + p.set_defaults(fn=cmd_ports) + + p = sub.add_parser("diff", help="a change's effect on depth, register by " + "register, and on area, by unit") + p.add_argument('before', help="run directory of the reference (the parent revision)") + p.add_argument('after', help="run directory of the change") + p.add_argument('--min-levels', type=int, default=8, + help="watch registers at or above this depth in either run") + p.add_argument('--fix-levels', type=int, default=12, + help="a register that reaches this depth from below, or " + "only the change has at it, fails") + p.add_argument('--fail-delta', type=int, default=4, + help="a register this many levels deeper fails; a smaller " + "rise is listed to watch") + p.add_argument('--top', type=int, default=40) + p.add_argument('--metric', default='levels', choices=['levels', 'retimed'], + help="the runs' column to compare") + p.add_argument('--area-tolerance', type=float, default=0.05, + help="a resource of the design that grows by more than this " + "fraction fails (exit 3)") + p.set_defaults(fn=cmd_diff) + + p = sub.add_parser("buffers", help="elastic buffers that pass deep paths") + p.add_argument('run_dir') + p.add_argument('--min-levels', type=int, default=8) + p.set_defaults(fn=cmd_buffers) + + p = sub.add_parser("retime-split", help="agreement split by Vivado's retiming") + p.add_argument('run_dir') + p.add_argument('ep') + p.add_argument('native', metavar='MODULE=PREFIX') + p.add_argument('metric', nargs='?', default='levels', choices=['levels', 'retimed']) + p.set_defaults(fn=cmd_retime_split) + + p = sub.add_parser("calibrate", help="train the learned slack calibration") + p.add_argument('model', help="the model JSON to write") + p.add_argument('--build', nargs='+', action='append', required=True, + metavar='RUN_DIR EP_CSV [GROUP]', + help="a DUT build: its Beacon run and Vivado end point table; " + "builds of one GROUP (one RTL lineage) are held out together " + "(repeatable)") + p.add_argument('--path-levels', type=int, default=8, + help="path features from this depth on (the runs' path-levels)") + p.add_argument('--holdout', action='store_true', + help="also print a leave-one-build-out check against levels") + p.set_defaults(fn=cmd_calibrate) + + p = sub.add_parser("score", help="registers by calibrated near-critical risk") + p.add_argument('run_dir') + p.add_argument('model', help="a model JSON from calibrate") + p.add_argument('--min-levels', type=int, default=0) + p.add_argument('--top', type=int, default=40) + p.add_argument('--csv', help="also write every register's prediction here") + p.set_defaults(fn=cmd_score) + + args = ap.parse_args() + return args.fn(args) or 0 + + +if __name__ == '__main__': + # Exit codes 1 and 3 are regressions to logic_gate (DEEPER, AREA); a crash + # must read as a failed run, not as one. + try: + sys.exit(main()) + except Exception: + import traceback + traceback.print_exc() + sys.exit(4) diff --git a/ci/blackbox.sh b/ci/blackbox.sh index 21215167a0..99a926e502 100755 --- a/ci/blackbox.sh +++ b/ci/blackbox.sh @@ -26,8 +26,12 @@ show_help() { show_usage echo " where" - echo "--driver: gpu, simx, rtlsim, opae, xrt, aved" - echo "--target: aved driver only -- avedsim (default), sim, hw" + echo "--driver: gpu, simx, rtlsim, opae, xrt, aved, firesim, foldsim" + echo "--target: xrt: xrtsim (default), hw_emu, hw -- aved: avedsim (default), sim, hw" + echo " firesim: the targets of the card's native driver; xrtsim (default) or" + echo " avedsim need no image, the others run FPGA_BIN_DIR's (hw/syn/firesim)" + echo " foldsim: avedsim (default) simulates the kernel in Verilator, built" + echo " by the run; hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim)" echo "--app: any subfolder test under regression, graphics, mpi, opencl, or hip" echo "--perf: 0=disable, 1=core, 3=icache, 4=dcache, 5=l2cache, 6=l3cache, 7=mem," echo " 11=tcu, 12=raster, 13=tex, 14=om, 15=rtu, 16=dxa" @@ -96,7 +100,7 @@ parse_args() { set_driver_path() { case $DRIVER in gpu) DRIVER_PATH="" ;; - simx|rtlsim|opae|xrt|aved|firesim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; + simx|rtlsim|opae|xrt|aved|firesim|foldsim) DRIVER_PATH="$ROOT_DIR/sw/runtime/$DRIVER" ;; *) echo "Invalid driver: $DRIVER"; exit 1 ;; esac } diff --git a/ci/circt_install.sh.in b/ci/circt_install.sh.in new file mode 100755 index 0000000000..cfa17f83c0 --- /dev/null +++ b/ci/circt_install.sh.in @@ -0,0 +1,101 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# CIRCT install: the SystemVerilog frontend of every synthesis flow +# (circt-verilog, circt-opt, firtool) and the compiler Foldsim builds on. +# +# Builds Vortex's CIRCT fork (LLVM, MLIR and CIRCT in one build) and installs +# it into $TOOLDIR/circt: tools, libraries, headers and CMake packages, which +# ci/foldsim_install.sh builds against. This script is the *producer* of the +# circt prebuilt: build here (once per OSVERSION, in that OS's container), +# then `ci/toolchain_prebuilt.sh --circt` packages the three tools and +# `ci/toolchain_install.sh --circt` fetches them. Most users install the +# prebuilt and never run this script. +# +# Release, assertions off, libstdc++ linked statically, no zlib/zstd: the +# tools run on any machine of the OSVERSION. Needs git, cmake >= 3.28 (for +# slang, the SystemVerilog frontend), ninja, a C++20 compiler (GCC >= 11) and +# python3; installs nothing system-wide. +# +# Idempotent: re-running rebuilds only what changed. + +# exit when any command fails +set -e + +TOOLDIR=${TOOLDIR:=@TOOLDIR@} + +CIRCT_REPO=${CIRCT_REPO:=https://github.com/vortexgpgpu/circt.git} +CIRCT_REV=${CIRCT_REV:=vortex_3.x} + +CIRCT_SRC=${CIRCT_SRC:=$TOOLDIR/circt-src} +CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} +CIRCT_BUILD=$CIRCT_SRC/build +JOBS=${JOBS:=$(nproc)} + +# slang's CMakeLists sets cmake_minimum_required(3.28), which is NEWER than +# the cmake Ubuntu 22.04 ships (3.22); the failure would come only after the +# LLVM checkout, so check first. CMAKE may point at a newer cmake (e.g. pip's). +CMAKE=${CMAKE:=cmake} +cmake_ver=$($CMAKE --version | head -1 | awk '{print $3}') +cmake_maj=${cmake_ver%%.*}; cmake_rest=${cmake_ver#*.}; cmake_min=${cmake_rest%%.*} +if [ "$cmake_maj" -lt 3 ] || { [ "$cmake_maj" -eq 3 ] && [ "$cmake_min" -lt 28 ]; }; then + echo "error: CIRCT's slang frontend needs cmake >= 3.28, found $cmake_ver ($CMAKE)." + echo " set CMAKE=/path/to/newer/cmake (pip install cmake) and re-run." + exit 1 +fi + +# ---------------------------------------------------------------------- +# 1. Fetch: LLVM is a submodule pinned by the CIRCT revision. +# ---------------------------------------------------------------------- +# CIRCT_REV is a branch, taken at its tip on origin, or a tag or a commit: the +# checkout is detached either way, so a re-run never stays on a stale local +# branch. +if [ ! -d "$CIRCT_SRC/.git" ]; then + git clone "$CIRCT_REPO" "$CIRCT_SRC" +fi +git -C "$CIRCT_SRC" fetch --force --tags origin +if git -C "$CIRCT_SRC" rev-parse -q --verify "refs/remotes/origin/$CIRCT_REV^{commit}" >/dev/null; then + git -C "$CIRCT_SRC" checkout --detach "origin/$CIRCT_REV" +else + git -C "$CIRCT_SRC" checkout --detach "$CIRCT_REV^{commit}" +fi +git -C "$CIRCT_SRC" submodule update --init --depth 1 llvm + +# ---------------------------------------------------------------------- +# 2. Configure and build +# ---------------------------------------------------------------------- +$CMAKE -G Ninja -S "$CIRCT_SRC/llvm/llvm" -B "$CIRCT_BUILD" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$CIRCT_PREFIX" \ + -DLLVM_ENABLE_PROJECTS=mlir \ + -DLLVM_EXTERNAL_PROJECTS=circt \ + -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR="$CIRCT_SRC" \ + -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_ENABLE_ASSERTIONS=OFF \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ + -DLLVM_ENABLE_ZLIB=OFF \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_INSTALL_UTILS=ON \ + -DCIRCT_SLANG_FRONTEND_ENABLED=ON \ + -DCIRCT_SLANG_BUILD_FROM_SOURCE=ON +ninja -C "$CIRCT_BUILD" -j "$JOBS" circt-verilog circt-opt firtool +ninja -C "$CIRCT_BUILD" -j "$JOBS" install + +# ---------------------------------------------------------------------- +# 3. Check +# ---------------------------------------------------------------------- +"$CIRCT_PREFIX/bin/circt-verilog" --version | head -1 +echo "circt installed in $CIRCT_PREFIX" diff --git a/ci/firesim_env.sh b/ci/firesim_env.sh new file mode 100644 index 0000000000..a9cae0f7f0 --- /dev/null +++ b/ci/firesim_env.sh @@ -0,0 +1,65 @@ +# Environment for building and running Vortex through FireSim. Source it from +# the build directory, after setting up the card's Xilinx tools exactly as the +# native flow for that card needs them (Vitis and XRT for an XRT card; Vivado +# and VRT for an aved card): +# +# source ci/firesim_env.sh +# +# It adds only what FireSim needs on top of those: +# - FireSim's conda environment, through its env.sh (conda must be on PATH); +# - the host compiler and linker, which VRT and the aved simulator build +# need: conda's cannot link against the host glibc those libraries use; +# - Vivado's simulator libraries, which an aved TARGET=sim run loads. +# +# FIRESIM_PATH the FireSim fork checkout (default: config.mk's) +# FIRESIM_ENV_SH FireSim's env.sh, if its conda environment lives in another +# checkout (default: $FIRESIM_PATH/env.sh) + +_fs_fail() { echo "firesim_env: $*" >&2; return 1; } + +_fs_main() { + if [ -z "${FIRESIM_PATH:-}" ] && [ -f config.mk ]; then + FIRESIM_PATH=$(make -s -f config.mk -f - <<<'print-fs: ; @echo $(FIRESIM_PATH)' print-fs 2>/dev/null) + fi + [ -n "${FIRESIM_PATH:-}" ] || { _fs_fail "set FIRESIM_PATH, or source this from a configured build directory"; return 1; } + [ -d "$FIRESIM_PATH/sim" ] || { _fs_fail "no FireSim checkout at FIRESIM_PATH=$FIRESIM_PATH"; return 1; } + export FIRESIM_PATH + + command -v conda >/dev/null 2>&1 || { _fs_fail "conda is not on PATH (FireSim's env.sh needs it)"; return 1; } + local env_sh=${FIRESIM_ENV_SH:-$FIRESIM_PATH/env.sh} + [ -f "$env_sh" ] || { _fs_fail "FireSim's env.sh not found at $env_sh (set FIRESIM_ENV_SH)"; return 1; } + # env.sh reads variables it has not set, so it cannot run under set -u. + local had_u=0; case $- in *u*) had_u=1; set +u ;; esac + source "$env_sh" >/dev/null 2>&1 + [ $had_u = 1 ] && set -u + # env.sh may repoint FIRESIM_PATH at its own checkout; keep the one asked for. + export FIRESIM_PATH + + # The host compiler, and the host linker ahead of conda's under the plain + # name `ld` that CMake-driven builds invoke. + export CC=/usr/bin/gcc CXX=/usr/bin/g++ + local hostld=${XDG_CACHE_HOME:-$HOME/.cache}/vortex-firesim/hostld + mkdir -p "$hostld" && ln -sf /usr/bin/ld "$hostld/ld" + case ":$PATH:" in *":$hostld:"*) ;; *) export PATH="$hostld:$PATH" ;; esac + + if [ -n "${XILINX_VIVADO:-}" ]; then + case ":${LD_LIBRARY_PATH:-}:" in + *":$XILINX_VIVADO/lib/lnx64.o:"*) ;; + *) export LD_LIBRARY_PATH="${LD_LIBRARY_PATH:+$LD_LIBRARY_PATH:}$XILINX_VIVADO/lib/lnx64.o" ;; + esac + else + echo "firesim_env: XILINX_VIVADO is not set; set up the card's Xilinx tools before building" >&2 + fi + + # A stale VRT ahead of the packaged one on the library path runs every V80 + # image at the shell's boot clock instead of the frequency it was built for. + case ":${LD_LIBRARY_PATH:-}:" in + *":/opt/xilinx/slash/lib:"*) + echo "firesim_env: warning: /opt/xilinx/slash/lib is on LD_LIBRARY_PATH; make sure it is the same VRT as the installed vrtd" >&2 ;; + esac + + echo "firesim_env: FIRESIM_PATH=$FIRESIM_PATH" +} + +_fs_main +unset -f _fs_main _fs_fail diff --git a/ci/foldsim_install.sh.in b/ci/foldsim_install.sh.in new file mode 100755 index 0000000000..65ff94e84c --- /dev/null +++ b/ci/foldsim_install.sh.in @@ -0,0 +1,90 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# Foldsim install: the FAME-5 simulator compiler of hw/syn/foldsim (the +# `foldsim` driver and foldsim-opt). +# +# Foldsim is its own project and builds on CIRCT, as POCL builds on LLVM: it +# needs the CIRCT installation ci/circt_install.sh leaves in $TOOLDIR/circt +# (libraries, headers and CMake packages, not only the prebuilt's tools). This +# script is the *producer* of the Foldsim prebuilt: build here (once per +# OSVERSION, in that OS's container, after ci/circt_install.sh), then +# `ci/toolchain_prebuilt.sh --foldsim` packages $TOOLDIR/foldsim and +# `ci/toolchain_install.sh --foldsim` fetches it. Most users install the +# prebuilt and never run this script. +# +# Release, assertions off, libstdc++ linked statically. Needs git, cmake >= +# 3.20, ninja, lit (pip) and a C++17 compiler; installs nothing system-wide. +# +# Idempotent: re-running rebuilds only what changed. + +# exit when any command fails +set -e + +TOOLDIR=${TOOLDIR:=@TOOLDIR@} + +FOLDSIM_REPO=${FOLDSIM_REPO:=https://github.com/vortexgpgpu/foldsim.git} +FOLDSIM_REV=${FOLDSIM_REV:=vortex_3.x} + +CIRCT_PREFIX=${CIRCT_PREFIX:=$TOOLDIR/circt} +FOLDSIM_SRC=${FOLDSIM_SRC:=$TOOLDIR/foldsim-src} +FOLDSIM_PREFIX=${FOLDSIM_PREFIX:=$TOOLDIR/foldsim} +FOLDSIM_BUILD=$FOLDSIM_SRC/build + +if [ ! -f "$CIRCT_PREFIX/lib/cmake/circt/CIRCTConfig.cmake" ]; then + echo "error: no CIRCT installation in $CIRCT_PREFIX (CIRCTConfig.cmake)." + echo " run ci/circt_install.sh first, or set CIRCT_PREFIX." + exit 1 +fi + +# ---------------------------------------------------------------------- +# 1. Fetch +# ---------------------------------------------------------------------- +# FOLDSIM_REV is a branch, taken at its tip on origin, or a tag or a commit: the +# checkout is detached either way, so a re-run never stays on a stale local +# branch. +if [ ! -d "$FOLDSIM_SRC/.git" ]; then + git clone "$FOLDSIM_REPO" "$FOLDSIM_SRC" +fi +git -C "$FOLDSIM_SRC" fetch --force --tags origin +if git -C "$FOLDSIM_SRC" rev-parse -q --verify "refs/remotes/origin/$FOLDSIM_REV^{commit}" >/dev/null; then + git -C "$FOLDSIM_SRC" checkout --detach "origin/$FOLDSIM_REV" +else + git -C "$FOLDSIM_SRC" checkout --detach "$FOLDSIM_REV^{commit}" +fi + +# ---------------------------------------------------------------------- +# 2. Configure, build and test +# ---------------------------------------------------------------------- +# An installed LLVM has no llvm-lit; the tests take lit from PATH (pip's +# `lit`) unless LIT names one. +LIT=${LIT:-$(command -v lit || command -v llvm-lit || true)} +if [ -z "$LIT" ]; then + echo "error: no lit for Foldsim's tests: pip install lit, or set LIT." + exit 1 +fi +cmake -G Ninja -S "$FOLDSIM_SRC" -B "$FOLDSIM_BUILD" \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$FOLDSIM_PREFIX" \ + -DCIRCT_DIR="$CIRCT_PREFIX/lib/cmake/circt" \ + -DMLIR_DIR="$CIRCT_PREFIX/lib/cmake/mlir" \ + -DLLVM_DIR="$CIRCT_PREFIX/lib/cmake/llvm" \ + -DLLVM_EXTERNAL_LIT="$LIT" \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON +ninja -C "$FOLDSIM_BUILD" +ninja -C "$FOLDSIM_BUILD" check-foldsim +cmake --install "$FOLDSIM_BUILD" + +echo "Foldsim installed in $FOLDSIM_PREFIX" diff --git a/ci/gen_config.py b/ci/gen_config.py index 316c8d0a97..0d5f58ed99 100755 --- a/ci/gen_config.py +++ b/ci/gen_config.py @@ -484,7 +484,7 @@ def set_enum(name: str, val: Any) -> None: # ----------------------------- PROFILE_SECTIONS = ("core", "chip", "platform") -PROFILE_FIELDS = {"name", "extends", "description", "reference", "sources"} | set(PROFILE_SECTIONS) +PROFILE_FIELDS = {"name", "extends", "description", "xlen", "reference", "sources"} | set(PROFILE_SECTIONS) @dataclass @@ -495,6 +495,7 @@ class Profile: sections: Dict[str, Set[str]] # section -> knobs it sets reference: Dict[str, Any] sources: Any + xlen: Optional[int] # ISA width the machine needs (None: any) def _profile_path(name: str, profiles_dir: str) -> str: @@ -519,7 +520,7 @@ def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> P base = load_profile(data["extends"], profiles_dir, _seen + (name,)) else: base = Profile(name="", chain=[], values={}, sections={s: set() for s in PROFILE_SECTIONS}, - reference={}, sources=None) + reference={}, sources=None, xlen=None) values = dict(base.values) sections = {s: set(base.sections[s]) for s in PROFILE_SECTIONS} @@ -532,8 +533,11 @@ def load_profile(name: str, profiles_dir: str, _seen: Tuple[str, ...] = ()) -> P reference = dict(base.reference) reference.update(data.get("reference", {})) sources = data.get("sources", base.sources) + xlen = data.get("xlen", base.xlen) + if xlen not in (None, 32, 64): + raise ValueError(f"Profile '{name}': xlen must be 32 or 64") return Profile(name=data.get("name", name), chain=base.chain + [name], values=values, - sections=sections, reference=reference, sources=sources) + sections=sections, reference=reference, sources=sources, xlen=xlen) def validate_profile(prof: Profile, toml_defs: Dict[str, Any], enums: Dict[str, EnumSpec], @@ -590,11 +594,20 @@ def _canonical_define(tok: str, enums: Dict[str, EnumSpec]) -> Optional[str]: def merge_configs(prof: Profile, cflags: str, enums: Dict[str, EnumSpec]) -> str: explicit = shlex.split(cflags) if cflags else [] - given = {_canonical_define(t, enums) for t in explicit} + knobs = [_canonical_define(t, enums) for t in explicit] + given = set(knobs) given.discard(None) merged = [t for t in profile_defines(prof) if _canonical_define(t, enums) not in given] - # The caller's flags pass through verbatim, quoting included. - return " ".join(merged + ([cflags.strip()] if cflags.strip() else [])) + # A later flag for a knob replaces an earlier one, as on a compiler command + # line: a Makefile appends its own flags after an already merged profile. + last = {k: i for i, k in enumerate(knobs) if k is not None} + kept = [t for i, (t, k) in enumerate(zip(explicit, knobs)) if k is None or last[k] == i] + if len(kept) == len(explicit): + # Nothing replaced: the caller's flags pass through verbatim, quoting included. + caller = [cflags.strip()] if cflags.strip() else [] + else: + caller = [shlex.join(kept)] + return " ".join(merged + caller) # ----------------------------- @@ -1621,6 +1634,10 @@ def main(argv: List[str]) -> int: ap.add_argument("--profile", default="", help="hardware profile: profiles/.json next to the config, or a .json path") ap.add_argument("--profiles-dir", default=None, help="profile directory (default: /profiles)") + ap.add_argument("--xlen", type=int, default=None, + help="the build's XLEN; a profile that names another xlen is rejected") + ap.add_argument("--profile-xlen", action="store_true", + help="print the profile's xlen (empty if it names none) and exit") ap.add_argument("--merge-configs", action="store_true", help="print --cflags with the profile folded in (explicit -D flags win) and exit") args, unknown = ap.parse_known_args(argv) @@ -1641,6 +1658,13 @@ def main(argv: List[str]) -> int: prof = load_profile(args.profile, profiles_dir) validate_profile(prof, toml_defs, enums, params_set | set(builtins.keys())) + if args.profile_xlen: + sys.stdout.write(f"{prof.xlen if prof and prof.xlen else ''}\n") + return 0 + if prof is not None and prof.xlen is not None and args.xlen is not None and args.xlen != prof.xlen: + raise ValueError(f"Profile '{prof.name}' models a {prof.xlen}-bit machine but this build is XLEN={args.xlen}: " + f"use a build{prof.xlen} tree (configure --xlen={prof.xlen})") + if args.merge_configs: out = merge_configs(prof, args.cflags, enums) if prof else args.cflags sys.stdout.write(out.strip() + "\n") diff --git a/ci/logic_gate.py b/ci/logic_gate.py new file mode 100755 index 0000000000..f6f1773657 --- /dev/null +++ b/ci/logic_gate.py @@ -0,0 +1,44 @@ +#!/usr/bin/env python3 +"""logic_gate — logic-level and area regression gate (Beacon). + +Gates the depth of every RTL unit of the DUT catalog, and each DUT's area: +foldsim-estimate-timing counts the logic levels of every register-to-register +path of the CIRCT netlist the Xilinx flow elaborates, as Vivado counts them on +DEVICE's family, and foldsim-estimate-area its LUTs, flip-flops, distributed +and block RAM and DSPs, without running Vivado. A unit that gets one level +deeper than its baseline fails, by name, and a resource more than 5% larger, +in seconds -- where ci/fpga_gate.py finds the Fmax or area it costs the next +night. The +gating machinery is shared with the synthesis gates and lives in +ci/synth_gate.py; this entry point only pins the tool. + + Spec : ci/testcases/logic_gate.yaml (what to measure) + Baselines : ci/baselines/synthesis/beacon/.json (last accepted levels) + Builds : hw/syn/xilinx/dut/logic_gate_ (no Vivado) + Metrics : /beacon_summary.csv, beacon_units.json, beacon_area.txt + (written by hw/syn/xilinx/dut's `beacon` target) + +It needs CIRCT and foldsim (ci/toolchain_install.sh --circt --foldsim); the +Xilinx environment is not used. + +Usage (from a source checkout): + ci/logic_gate.py --list + ci/logic_gate.py # all builds, gate against baselines + ci/logic_gate.py -b cache -b tcu # explicit builds + ci/logic_gate.py --update-baseline # re-record (human-reviewed, never in CI) + +Beacon runs by hand through the flow's own entry point -- +`make -C hw/syn/xilinx/dut cache-beacon` -- so the gate adds no second way to +build. + +See docs/designs/continuous_integration.md 3.5. +""" + +import os +import sys + +sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) +import synth_gate # noqa: E402 + +if __name__ == "__main__": + sys.exit(synth_gate.main(default_tool="beacon")) diff --git a/ci/synth_gate.py b/ci/synth_gate.py index 820089a69d..a4237c0756 100755 --- a/ci/synth_gate.py +++ b/ci/synth_gate.py @@ -7,15 +7,17 @@ post-synthesis timing and area against the recorded numbers, so an RTL change that quietly costs timing closure or area cannot land green. -Two backends, one gate. Everything that is not "how do I drive the tool and what -does it call its numbers" is shared: +Three backends, one gate. Everything that is not "how do I drive the tool and +what does it call its numbers" is shared: --tool=xilinx Vivado synthesis + place-and-route -> ci/fpga_gate.py --tool=yosys Yosys + OpenSTA on an ASIC PDK -> ci/asic_gate.py + --tool=beacon logic levels per RTL unit, no synthesis -> ci/logic_gate.py Spec : ci/testcases/.yaml (what to build) Baselines : ci/baselines/synthesis//.json (last accepted metrics) - Metrics : /synth_summary.csv, written by the flow itself + Metrics : /synth_summary.csv (beacon_summary.csv for Beacon), + written by the flow itself Same division as every other check in the catalog: the yaml is hand-authored, commented and reviewed; the baseline is machine-written and never hand-edited. @@ -40,16 +42,20 @@ import yaml -# Gated metrics must stay within +/-5% of baseline. The regression side is the -# gate; the improvement side is the ratchet -- a win beyond it must be recorded -# so the gain is locked in and a later silent slide back to the old number still -# fails. +# Fmax is held to the build's target clock and area to its fixed budget, both +# from the catalog and both one-sided: Fmax may not fall more than 2% below the +# target, area may not grow more than 5% over the budget, and doing better never +# fails. Neither is measured against the last recorded baseline, so re-recording +# cannot ratchet the limit; raising a budget is a reviewed catalog edit. Any +# other metric gated with --gate is compared with its baseline both ways: the +# improvement side is the ratchet, so a win must be recorded to lock it in. # -# This is the GLOBAL default. It is overridden per-metric on the command line -# (--metric-threshold lut=0.10) and per-build in the catalog ("thresholds": +# These are the GLOBAL defaults. They are overridden per-metric on the command +# line (--metric-threshold lut=0.10) and per-build in the catalog ("thresholds": # {"fmax_mhz": 0.03}) -- most specific wins. Same shape as model_parity's # `tolerance` (per-case entry > category defaults > DEFAULT_PARITY_TOLERANCE). TOLERANCE = 0.05 +METRIC_TOLERANCE = {"fmax_mhz": 0.02} # Higher-is-better metrics. Everything else is lower-is-better. HIGHER_IS_BETTER = ("fmax_mhz",) @@ -88,6 +94,17 @@ class Tool: metrics = () # recorded, in report order; build_time_s is bookkeeping gated = () # asserted by default (--gate overrides) + area = "" # the gated area metric, held to the catalog's budget + budget_key = "" # the catalog field holding that budget + # The metric whose presence in a baseline entry means "recorded". + key_metric = "fmax_mhz" + # The flow's one-row metric file, in the build's work directory. + summary = "synth_summary.csv" + # Default gate tolerance, overridden by --threshold and the catalog. + tolerance = TOLERANCE + # Per-metric defaults over `tolerance`, overridden by --metric-threshold + # and the catalog. + metric_tolerance = {} integral = () # recorded as ints; every other metric is a rounded float columns = () # (header, metric, format, width) for the report table @@ -142,6 +159,16 @@ def extras(self, work): """Diagnostics recorded next to the metrics, never gated.""" return {} + def compare(self, metrics, base): + """Reasons beyond the per-metric tolerance check, as gate() words them: + a reason containing "improvement" asks for a re-record, any other + fails.""" + return [] + + def evidence(self, work): + """Files of a finished build that explain its reasons.""" + return [] + def incomplete(self, metrics): """Whether a cached result predates something this tool now records. @@ -190,7 +217,10 @@ class Xilinx(Tool): metrics = ("fmax_mhz", "wns_ns", "lut", "lutram", "ff", "bram", "uram", "dsp", "build_time_s") - gated = ("fmax_mhz", "lut") + # Fmax is held to the build's clock target (gate()'s BELOW TARGET check). + gated = ("lut",) + area = "lut" + budget_key = "lut_budget" integral = ("lut", "lutram", "ff", "bram", "uram", "dsp", "build_time_s") columns = (("fmax", "fmax_mhz", "%.0f", 9), ("lut", "lut", "%d", 9), ("ff", "ff", "%d", 9), ("lutram", "lutram", "%d", 9), @@ -309,6 +339,8 @@ class Yosys(Tool): # whatever else the design contains: #424 left the RTU's worst path in an # unchanged FMA stage yet moved it 957 -> 903 MHz. gated = ("cell_area_um2",) + area = "cell_area_um2" + budget_key = "area_budget_um2" integral = ("cell_count", "build_time_s") columns = (("fmax", "fmax_mhz", "%.0f", 9), ("area_um2", "cell_area_um2", "%.0f", 11), @@ -322,15 +354,15 @@ class Yosys(Tool): fallback_weight = 900 job_label = "make" - # sv2v is the parser: a bad define, a missing include or a hierarchy error - # dies there, before Yosys ever reads a line. run_synth.sh stamps `gen-ys` - # immediately after source generation and conversion have both succeeded, so - # that stamp is this flow's "past the typo risk" marker. + # CIRCT is the parser: a bad define, a missing include or a hierarchy error + # dies in circt_elaborate.sh, before Yosys ever reads a line. run_synth.sh stamps + # `gen-ys` immediately after source generation and conversion have both + # succeeded, so that stamp is this flow's "past the typo risk" marker. elab_done = "TIME gen-ys" elab_label = "source conversion" phases = ( ("gen_sources.sh", "sources"), - ("sv2v.sh", "sv2v"), + ("circt_elaborate.sh", "verilog"), (elab_done, "synthesis"), ("TIME yosys", "sram"), ("run_sta.tcl", "timing"), @@ -344,7 +376,8 @@ def env(self, args): "corner": os.environ.get("CORNER", "tt"), "yosys": tool_version("yosys", "-V"), "sta": tool_version("sta", "-version"), - "sv2v": tool_version("sv2v", "--version"), + "circt": tool_version("circt-verilog", "--version", + package="circt", line_prefix="CIRCT"), "xlen": args.xlen} def hash_key(self, env): @@ -359,13 +392,13 @@ def hash_key(self, env): # tool change. All three ship in one tarball pinned by TOOLCHAIN_REV, so # this makes that pin part of the baseline contract. return [env["pdk"], env["vt"], env["corner"], str(env["xlen"]), - env["yosys"], env["sta"], env["sv2v"]] + env["yosys"], env["sta"], env["circt"]] def env_warning(self, build, env): # Area and Fmax move with the Yosys/ABC version far more than Vivado's # do, so a version change is worth saying out loud on every build. for key, label in (("yosys", "Yosys"), ("sta", "OpenSTA"), - ("sv2v", "sv2v")): + ("circt", "CIRCT")): ref = build.get("baseline_env", {}).get(key) if ref and env.get(key) and ref != env[key]: # The version is in config_hash, so this build is already going @@ -427,10 +460,191 @@ def describe_baseline(self, base): base.get("cell_area_um2") or 0) -TOOLS = {t.name: t for t in (Xilinx(), Yosys())} +class Beacon(Tool): + """Beacon: logic levels per RTL unit and the design's area + (hw/syn/xilinx/dut, target `beacon`). + + foldsim-estimate-timing ranks every register-to-register path of the CIRCT + netlist the Xilinx flow elaborates by its logic levels, counted as Vivado + counts them on DEVICE's family; foldsim-estimate-area counts its LUTs, + flip-flops, distributed and block RAM, UltraRAM and DSPs, placing each + memory as Vivado infers it. It needs no Vivado and no licence and takes + seconds per DUT, so it can run on every change where the Fmax gate runs + nightly. + + Depth is deterministic -- the same RTL and tools give the same levels -- so + the gate has no tolerance: a unit that gets deeper fails, by name, until it + is fixed or a human re-records it. Area is as deterministic, but a feature + adds some by design, so a resource of the design fails past its baseline + by more than fpga_gate's LUT tolerance (metric_tolerance), and a memory + that moves between distributed and block RAM shows as both columns + moving. What it gates is depth and cells, not Fmax or placement: Beacon + does not see wires (docs/designs/continuous_integration.md 3.5). + """ + + name = "beacon" + gate = "logic_gate" + flow = "hw/syn/xilinx/dut" + + area_metrics = ("lut", "ff", "lutram", "bram", "uram", "dsp") + metrics = ("max_levels", "deep_endpoints", "watch_endpoints", + "endpoints") + area_metrics + ("build_time_s",) + gated = ("max_levels",) + area_metrics + integral = ("max_levels", "deep_endpoints", "watch_endpoints", + "endpoints", "lut", "ff", "lutram", "uram", "dsp", + "build_time_s") + key_metric = "max_levels" + summary = "beacon_summary.csv" + tolerance = 0.0 + metric_tolerance = dict.fromkeys(area_metrics, 0.05) + columns = (("max_lv", "max_levels", "%d", 7), + ("deep", "deep_endpoints", "%d", 9), + ("watch", "watch_endpoints", "%d", 9), + ("ends", "endpoints", "%d", 9), + ("lut", "lut", "%d", 9), ("ff", "ff", "%d", 9), + ("lutram", "lutram", "%d", 9), ("bram", "bram", "%g", 8), + ("uram", "uram", "%d", 6), ("dsp", "dsp", "%d", 6)) + + # Elaboration and mapping of the widest DUTs; the level count itself is + # seconds. + weights = {"top": 600, "vortex": 600, "gfx": 600, "core": 300, + "tensor": 300, "tcu": 300} + fallback_weight = 120 + job_label = "make" + + # The netlist exists once foldsim-opt starts mapping it; a typo in a define + # or an include dies in gen_sources.sh or circt_elaborate.sh before that. + elab_done = "foldsim-beacon-map" + elab_label = "elaboration" + phases = ( + ("gen_sources.sh", "sources"), + ("circt_elaborate.sh", "elaboration"), + (elab_done, "mapping"), + ("foldsim-estimate-timing", "levels"), + ("foldsim-estimate-area", "area"), + ("beacon_summary.py", "reporting"), + ) + error_re = re.compile(r"^(?:ERROR|FATAL|Error)\b|: error:|^\S+: \*\*\* ") + + UNITS = "beacon_units.json" + PATHS = "beacon_paths.txt" + AREA = "beacon_area.txt" + + def env(self, args): + return {"circt": tool_version("circt-verilog", "--version", + package="circt", line_prefix="CIRCT"), + "foldsim": tool_version("foldsim-opt", "--version", + package="foldsim", + line_prefix="foldsim"), + "device": os.environ.get("DEVICE", "xcu55c-fsvh2892-2L-e"), + "xlen": args.xlen} + + def hash_key(self, env): + # The levels are the model's: a new foldsim or CIRCT can map the same + # RTL to other levels, and another device's family counts them + # differently; either must read as STALE, not as a regression. + return [env["circt"], env["foldsim"], env["device"], str(env["xlen"])] + def env_warning(self, build, env): + for key, label in (("foldsim", "foldsim"), ("circt", "CIRCT")): + ref = build.get("baseline_env", {}).get(key) + if ref and env.get(key) and ref != env[key]: + return ("baseline was recorded on %s %s, running %s -- this " + "reads as STALE until re-recorded" % (label, ref, env[key])) + return None -def tool_version(name, flag): + def work_dir(self, dut_root, prefix, build): + return os.path.join(dut_root, "%s_%s" % (prefix, build["dut"])) + + def clean(self, dut_root, work, build, benv): + subprocess.run(["make", "-C", work, "clean"], env=benv, + stdout=subprocess.DEVNULL, stderr=subprocess.STDOUT, + check=True) + + def build_command(self, dut_root, work, build, env, args): + os.makedirs(work, exist_ok=True) + shutil.copy2(os.path.join(dut_root, "build.mk"), + os.path.join(work, "Makefile")) + return ["make", "-C", work, "beacon"], dict( + DUT=build["dut"], CONFIGS=build["configs"], + BEACON_INTERFACES=build.get("interfaces", "")) + + def extras(self, work): + try: + with open(os.path.join(work, self.UNITS)) as fh: + doc = json.load(fh) + except (OSError, ValueError): + return {} + return {"units": doc.get("units", {}), + "interfaces": doc.get("interfaces", {}), + "deepest_paths": doc.get("deepest_paths", []), + "fix_levels": doc.get("fix_levels"), + "watch_levels": doc.get("watch_levels")} + + def incomplete(self, metrics): + return metrics.get("units") is None + + def compare(self, metrics, base): + """Per unit: deeper than its baseline fails; a new unit at the fix + threshold fails; shallower, or gone from the watch list, is an + improvement to lock in. Per interface output: more bits reached + combinationally from outside its instance fails, fewer is an + improvement.""" + cur, ref = metrics.get("units"), base.get("units") + if cur is None or ref is None: + return [] + fix = metrics.get("fix_levels") or 0 + reasons = [] + for unit in sorted(set(cur) | set(ref)): + now, was = cur.get(unit), ref.get(unit) + if was is None: + if now >= fix: + reasons.append("unit %s: new at %d levels (fix threshold " + "%d) -- REGRESSION" % (unit, now, fix)) + elif now is None: + reasons.append("unit %s: %d levels -> below the watch threshold " + "-- improvement, update the baseline to lock it " + "in" % (unit, was)) + elif now != was: + reasons.append("unit %s: %d -> %d levels -- %s" + % (unit, was, now, + "REGRESSION" if now > was else + "improvement, update the baseline to lock " + "it in")) + cur, ref = metrics.get("interfaces") or {}, base.get("interfaces") or {} + for port in sorted(set(cur) | set(ref)): + now, was = cur.get(port, 0), ref.get(port, 0) + if now != was: + reasons.append("interface %s: %d -> %d bits reached " + "combinationally from outside -- %s" + % (port, was, now, + "REGRESSION, register it" if now > was else + "improvement, update the baseline to lock " + "it in")) + return reasons + + def evidence(self, work): + # Every unit at or above the watch threshold, its paths op by op: a + # failing unit is a heading there; the area by unit and each memory's + # placement. + return [p for p in (os.path.join(work, self.PATHS), + os.path.join(work, self.AREA)) + if os.path.exists(p)] + + def describe(self, m): + area = (", %d LUTs, %g BRAM, %d DSP" % (m["lut"], m["bram"], m["dsp"]) + if m.get("lut") is not None else "") + return "max %d levels, %d end points >= %s%s" % ( + m["max_levels"], m["deep_endpoints"], m.get("fix_levels"), area) + + def describe_baseline(self, base): + return "max %d levels" % base["max_levels"] + + +TOOLS = {t.name: t for t in (Xilinx(), Yosys(), Beacon())} + + +def tool_version(name, flag, package=None, line_prefix=None): """First line of ` `, or "" if it cannot be asked. Resolved out of $TOOLDIR the way hw/syn/common.mk does, with a PATH fallback: @@ -438,7 +652,11 @@ def tool_version(name, flag): copy of yosys would make the baseline's tool env a lie. """ tooldir = os.environ.get("TOOLDIR") or os.path.expanduser("~/tools") - binary = os.path.join(tooldir, name, "bin", name) + package = package or name + # _PATH overrides the toolchain copy, as in hw/syn/common.mk. + root = os.environ.get(package.upper() + "_PATH") or \ + os.path.join(tooldir, package) + binary = os.path.join(root, "bin", name) if not os.path.exists(binary): binary = name try: @@ -449,6 +667,12 @@ def tool_version(name, flag): text = out.stdout.decode(errors="replace").strip() if not text: return "" + # A tool that prints several components' versions names the one that + # matters on a line of its own (circt-verilog: "CIRCT "). + if line_prefix: + lines = [l.strip() for l in text.splitlines() + if l.strip().startswith(line_prefix)] + return lines[0] if lines else "" # Up to the build details: yosys reports its git sha AND the g++ that built # it ("Yosys 0.57+157 (git sha1 ..., g++ 11.4 ...)"). Those identify the # binary, not the tool, and this string is part of the baseline fingerprint @@ -483,7 +707,7 @@ def baseline_path(root, tool, group): return os.path.join(baseline_dir(root, tool), group + ".json") -def load_catalog(root, tool): +def load_catalog(root, tool, xlen=32): """Builds keyed by id: the SPEC from ci/testcases/.yaml, joined to its recorded metrics from ci/baselines/synthesis//.json. @@ -516,13 +740,20 @@ def load_catalog(root, tool): if bid in builds: sys.exit("ERROR: duplicate build id '%s' in %s" % (bid, spec_path(root, tool.gate))) - missing = {"dut", "clock_mhz", "configs"} - set(entry) + required = {"dut", "clock_mhz", "configs"} | ({tool.budget_key} if tool.budget_key else set()) + missing = required - set(entry) if missing: sys.exit("ERROR: %s: missing %s" % (bid, ", ".join(sorted(missing)))) group = entry.get("group", "default") build = dict(entry) build["group"] = group build["configs"] = entry["configs"] or "" + # A build may pin its own ISA width and hardware profile + # (profiles/.json, folded under its configs by config.mk); a + # profile that models a 64-bit machine needs xlen: 64. + build["xlen"] = entry.get("xlen", xlen) + build["profile"] = entry.get("profile", "") + build["opt_in"] = bool(entry.get("opt_in", False)) build["known_issue"] = entry.get("known_issue", defaults.get("known_issue", "")) build["thresholds"] = entry.get("thresholds", @@ -531,7 +762,8 @@ def load_catalog(root, tool): result = recorded.get("result") or {} # An all-null skeleton means "never recorded" -- the file ships with the # shape visible, not with numbers. - build["baseline"] = result if result.get("fmax_mhz") is not None else None + build["baseline"] = result if result.get(tool.key_metric) is not None \ + else None build["baseline_env"] = result.get("env") or {} build["config_hash"] = recorded.get("config_hash") builds[bid] = build @@ -540,9 +772,10 @@ def load_catalog(root, tool): def select(builds, selectors): """Resolve --build selectors: an exact build id wins over a group of the - same name; otherwise a group selects all its builds.""" + same name; otherwise a group selects all its builds. With no selector, every + build but the opt-in ones, which run only when named.""" if not selectors: - return [builds[b] for b in sorted(builds)] + return [builds[b] for b in sorted(builds) if not builds[b]["opt_in"]] picked = {} for sel in selectors: hits = [b for b in builds.values() if b["id"] == sel] @@ -558,11 +791,15 @@ def select(builds, selectors): def config_hash(build, env, tool): """Fingerprint the synthesis inputs; a change invalidates the stored metrics.""" key = "|".join([build["dut"], str(build["clock_mhz"]), build["configs"]] - + tool.hash_key(env)) - # Appended only when set, so builds without a strategy override keep the - # hash their baselines were recorded under. + + tool.hash_key(dict(env, xlen=build["xlen"]))) + # Appended only when set, so builds without a strategy override or a + # profile keep the hash their baselines were recorded under. if build.get("impl_strategy"): key += "|" + build["impl_strategy"] + if build.get("profile"): + key += "|profile=" + build["profile"] + if build.get("interfaces"): + key += "|interfaces=" + build["interfaces"] return hashlib.sha256(key.encode()).hexdigest()[:16] @@ -593,10 +830,12 @@ def save_baselines(root, tool, results, env): "clock_mhz": b["clock_mhz"], "configs": b["configs"], "config_hash": config_hash(b, env, tool), - "result": dict(r["metrics"], env=dict(env)), + "result": dict(r["metrics"], env=dict(env, xlen=b["xlen"])), } if b.get("impl_strategy"): entries[b["id"]]["impl_strategy"] = b["impl_strategy"] + if b.get("profile"): + entries[b["id"]]["profile"] = b["profile"] with open(path, "w") as fh: json.dump(entries, fh, indent=2, sort_keys=True) fh.write("\n") @@ -826,6 +1065,18 @@ def save_stamp(work, chash, status, stamp_file, metrics=None): fh.write("\n") +def build_work(build, build_dir, tool): + """A build's work directory. + + Unique PREFIX per build id: its own build tree and its own log, so parallel + builds (and other synthesis runs on this machine) cannot collide. It is + also what keeps the Yosys flow's cached $(BUILD_DIR)/src from serving one + DUT's sources to the next. + """ + return tool.work_dir(tool.dut_root(build_dir), tool.gate + "_" + build["id"], + build) + + def run_build(build, build_dir, env, args, tool): """Synthesize one DUT; return its metrics. Raises BuildError on failure.""" dut_root = tool.dut_root(build_dir) @@ -833,12 +1084,7 @@ def run_build(build, build_dir, env, args, tool): raise BuildError("no DUT target '%s' in %s/catalog.mk" % (build["dut"], dut_root)) - # Unique PREFIX per build id: its own build tree and its own log, so parallel - # builds (and other synthesis runs on this machine) cannot collide. It is - # also what keeps the Yosys flow's cached $(BUILD_DIR)/src from serving one - # DUT's sources to the next. - prefix = tool.gate + "_" + build["id"] - work = tool.work_dir(dut_root, prefix, build) + work = build_work(build, build_dir, tool) stamp_file = tool.gate + ".json" chash = config_hash(build, env, tool) @@ -863,6 +1109,9 @@ def run_build(build, build_dir, env, args, tool): benv = dict(os.environ) argv, extra = tool.build_command(dut_root, work, build, env, args) benv.update(extra) + # Set even when empty, so a PROFILE in the caller's environment cannot + # leak into a build that names none. + benv["PROFILE"] = build["profile"] if resume: benv["RESUME"] = "1" # make clean is a no-op; keeps project_1 + *.dcp @@ -900,10 +1149,10 @@ def run_build(build, build_dir, env, args, tool): rc, elapsed // 60, "".join("\n | " + e for e in errors), log)) - summary = os.path.join(work, "synth_summary.csv") + summary = os.path.join(work, tool.summary) if not os.path.exists(summary): - raise BuildError("no synth_summary.csv (the flow did not reach report), " - "see %s" % log) + raise BuildError("no %s (the flow did not reach report), see %s" + % (tool.summary, log)) metrics = read_summary(summary, tool) metrics["build_time_s"] = elapsed metrics.update(tool.extras(work)) @@ -942,14 +1191,17 @@ def run_all(builds, build_dir, env, args, tool): for b in order} for fut in concurrent.futures.as_completed(futures): b = futures[fut] + work = build_work(b, build_dir, tool) try: metrics = fut.result() say(b, "done", "%s (%dm)" % (tool.describe(metrics), metrics["build_time_s"] // 60)) - results.append({"build": b, "metrics": metrics, "error": None}) + results.append({"build": b, "metrics": metrics, "error": None, + "work": work}) except (BuildError, subprocess.CalledProcessError) as e: say(b, "FAIL", str(e)) - results.append({"build": b, "metrics": None, "error": str(e)}) + results.append({"build": b, "metrics": None, "error": str(e), + "work": work}) return sorted(results, key=lambda r: r["build"]["id"]) @@ -986,16 +1238,31 @@ def verdict(name, reasons): if result["error"]: return "BUILD-FAIL", [result["error"]] + reasons = [] + vs_base = [m for m in gated if m != tool.area] + + if tool.area in gated: + budget = float(build[tool.budget_key]) + cur = result["metrics"].get(tool.area) + if cur is None: + reasons.append("%s: not measured by this run" % tool.area) + else: + tol = threshold(build, tool.area, args) + excess = (cur - budget) / budget + if excess > tol: + reasons.append("%s: %g vs %g budget (%+.1f%%, tolerance %.0f%%) " + "-- OVER BUDGET" + % (tool.area, cur, budget, 100 * excess, 100 * tol)) + base = build.get("baseline") or {} - if not any(base.get(m) is not None for m in gated): + if vs_base and not any(base.get(m) is not None for m in vs_base): return verdict("NO-BASELINE", ["no recorded baseline (run --update-baseline)"]) - if build.get("config_hash") != config_hash(build, env, tool): + if vs_base and build.get("config_hash") != config_hash(build, env, tool): return verdict("STALE", ["config changed since the baseline was recorded " "(re-record with --update-baseline)"]) - reasons = [] - for m in gated: + for m in vs_base: cur, ref = result["metrics"].get(m), base.get(m) if ref is None: reasons.append("%s: missing from baseline" % m) @@ -1013,11 +1280,11 @@ def verdict(name, reasons): "improvement, update the baseline to lock it in" if better else "REGRESSION")) + reasons += tool.compare(result["metrics"], base) + # Target-frequency gate: Fmax must be within tolerance of the clock the design - # was BUILT for, independent of the baseline. A baseline recorded below target - # (a design that never met its clock) must not let a build pass just by matching - # that number -- the gate answers "does it meet the frequency it targets?", not - # only "did it get worse than last time?". + # was BUILT for, never of the last recording. One-sided: a faster build passes, + # since placement noise alone moves Fmax by more than the tolerance. target = build.get("clock_mhz") fmax = result["metrics"].get("fmax_mhz") if target and fmax is not None: @@ -1056,6 +1323,9 @@ def report_entry(r, env, tool): "reasons": r["reasons"]} if b.get("impl_strategy"): entry["impl_strategy"] = b["impl_strategy"] + entry["xlen"] = b["xlen"] + if b.get("profile"): + entry["profile"] = b["profile"] return entry @@ -1103,10 +1373,55 @@ def cell(key, fmt): + tuple(cell(c[1], c[2]) for c in tool.columns)) + " %5dm %s" % (m["build_time_s"] // 60, verdict)) + works = {r["build"]["id"]: r.get("work") for r in results} for bid, verdict, reasons in failures: print("\n%s: %s" % (bid, verdict)) for reason in reasons: print(" - %s" % reason) + if works.get(bid) and verdict not in ("BUILD-FAIL", "STALE", + "NO-BASELINE"): + for path in tool.evidence(works[bid]): + print(" see %s" % path) + return failures + + +def compare_against(results, ref_dir, tool): + """Each finished build's Beacon run against the same build in ref_dir: + end points register by register, and area by unit; return the builds a + register got deeper in, or whose area grew past the tolerance, as failing + verdicts.""" + script = os.path.join(os.path.dirname(os.path.abspath(__file__)), "beacon.py") + failures = [] + for r in results: + b, work = r["build"], r.get("work") + if not r["metrics"] or not work: + continue + ref = os.path.join(build_work(b, ref_dir, tool), "beacon") + cur = os.path.join(work, "beacon") + if not os.path.exists(os.path.join(ref, "endpoints.csv")): + print("\n%s: no reference run in %s" % (b["id"], ref)) + continue + out = os.path.join(work, "beacon_diff.txt") + with open(out, "w") as fh: + m = r["metrics"] + rc = subprocess.run([sys.executable, script, "diff", ref, cur, + "--min-levels", str(m.get("watch_levels") or 8), + "--fix-levels", str(m.get("fix_levels") or 12)], + stdout=fh, stderr=subprocess.STDOUT).returncode + with open(out) as fh: + head = fh.readline().strip() + print("\n%s against %s: %s%s\n see %s" + % (b["id"], ref_dir, head, " -- REGRESSION" if rc in (1, 3) else "", + out)) + if rc == 1: + failures.append((b["id"], "DEEPER", ["a register got deeper than in " + "the reference: %s" % out])) + elif rc == 3: + failures.append((b["id"], "AREA", ["the design's area grew past the " + "tolerance against the " + "reference: %s" % out])) + elif rc: + failures.append((b["id"], "BUILD-FAIL", ["beacon.py diff failed: %s" % out])) return failures @@ -1115,7 +1430,7 @@ def cell(key, fmt): def main(argv=None, default_tool="xilinx"): tool = TOOLS[default_tool] ap = argparse.ArgumentParser( - description="Synthesis-regression gate (Fmax/area vs golden baseline)") + description="Synthesis-regression gate (Fmax vs target clock, area vs budget)") ap.add_argument("-b", "--build", action="append", default=[], metavar="ID", help="build id or group to run (repeatable; default: all)") ap.add_argument("-j", "--jobs", type=int, default=2, metavar="N", @@ -1129,15 +1444,18 @@ def main(argv=None, default_tool="xilinx"): "ci/testcases/.yaml and the golden file tree " "ci/baselines/synthesis// (default: %s)" % default_tool) - ap.add_argument("--threshold", type=float, default=TOLERANCE, metavar="F", - help="global gate tolerance, fraction (default: %.2f). " + ap.add_argument("--threshold", type=float, default=None, metavar="F", + help="global gate tolerance, fraction (default: the " + "tool's, %.2f for synthesis, 0 for Beacon). " "Overridden per-metric by --metric-threshold and " "per-build by the catalog's \"thresholds\"." % TOLERANCE) ap.add_argument("--metric-threshold", action="append", default=[], metavar="M=F", help="global tolerance for one metric, e.g. lut=0.10 " - "(repeatable); still overridden per-build") + "(repeatable; default: %s); still overridden " + "per-build" % ", ".join("%s=%g" % kv for kv in + METRIC_TOLERANCE.items())) ap.add_argument("--gate", metavar="M,M", help="metrics to assert on (default: the tool's, %s for %s); " "all metrics are always recorded and reported" @@ -1148,8 +1466,16 @@ def main(argv=None, default_tool="xilinx"): ap.add_argument("--build-dir", metavar="DIR", help="build tree to synthesize in (default: " "/build_, configured on demand)") + ap.add_argument("--against", metavar="DIR", + help="Beacon only: the build tree of a reference run, " + "typically the parent revision's; each build's end " + "points are compared register by register, and its " + "area by unit, with the same build there " + "(ci/beacon.py diff); a register that got deeper, or " + "area grown past the tolerance, fails the run") ap.add_argument("--xlen", type=int, default=32, choices=(32, 64), - help="ISA width (default: 32)") + help="ISA width of the builds that do not pin their own " + "(default: 32)") ap.add_argument("--opt-level", type=int, default=3, metavar="N", help="Vivado optimization level, 0=fastest 3=default") ap.add_argument("--timeout", type=int, default=8 * 3600, metavar="SEC", @@ -1172,13 +1498,15 @@ def main(argv=None, default_tool="xilinx"): help="print the schedule without running the tool") args = ap.parse_args(argv) tool = TOOLS[args.tool] + if args.threshold is None: + args.threshold = tool.tolerance # Line-buffer stdout: progress is the point of the elaboration watch, and it # is worthless if it sits in a block buffer until the run ends (which is what # happens whenever this is piped -- CI, nohup, a log file). sys.stdout.reconfigure(line_buffering=True) - overrides = {} + overrides = dict(METRIC_TOLERANCE, **tool.metric_tolerance) for spec in args.metric_threshold: metric, _, value = spec.partition("=") if metric not in tool.metrics or not value: @@ -1188,7 +1516,7 @@ def main(argv=None, default_tool="xilinx"): args.metric_threshold = overrides root = source_root() - builds = load_catalog(root, tool) + builds = load_catalog(root, tool, args.xlen) if args.matrix: # Ordered longest-first so the slowest DUT is dispatched into the fan-out @@ -1196,7 +1524,7 @@ def main(argv=None, default_tool="xilinx"): # argument run_all() makes for its thread pool. print(json.dumps(sorted( [{"id": b["id"], "group": b["group"], "dut": b["dut"], - "clock_mhz": b["clock_mhz"], + "clock_mhz": b["clock_mhz"], "xlen": b["xlen"], "known_issue": bool(b.get("known_issue"))} for b in select(builds, args.build)], key=lambda b: (-weight(builds[b["id"]], tool), b["id"])))) @@ -1205,16 +1533,21 @@ def main(argv=None, default_tool="xilinx"): if args.list: for b in select(builds, args.build): base = b.get("baseline") or {} - print("%-10s [%s] dut=%-8s %4d MHz %s%s" - % (b["id"], b["group"], b["dut"], b["clock_mhz"], + print("%-10s [%s] dut=%-8s %4d MHz xlen %d%s%s %s%s" + % (b["id"], b["group"], b["dut"], b["clock_mhz"], b["xlen"], + " profile " + b["profile"] if b["profile"] else "", + " (opt-in)" if b["opt_in"] else "", "baseline: " + tool.describe_baseline(base) - if base.get("fmax_mhz") else "no baseline", + if base.get(tool.key_metric) is not None + else "no baseline", " KNOWN ISSUE: " + b["known_issue"] if b.get("known_issue") else "")) print(" %s" % b["configs"]) return 0 tool.check_env() + if args.against and tool.name != "beacon": + sys.exit("ERROR: --against compares Beacon runs; use it with --tool=beacon") if args.jobs < 1: sys.exit("ERROR: --jobs must be >= 1") @@ -1222,8 +1555,6 @@ def main(argv=None, default_tool="xilinx"): args.tool_jobs = max(MIN_TOOL_JOBS, min(MAX_TOOL_JOBS, os.cpu_count() // args.jobs)) - build_dir = args.build_dir or os.path.join( - root, "build%d_%s" % (args.xlen, tool.gate)) env = tool.env(args) selected = select(builds, args.build) @@ -1232,15 +1563,30 @@ def main(argv=None, default_tool="xilinx"): if warning: print("WARNING: %s %s" % (b["id"], warning)) - if not args.dry_run: - prepare_build_tree(root, build_dir, args.xlen) - results = run_all(selected, build_dir, env, args, tool) + # One configured build tree per ISA width: a build that pins its own xlen + # runs in that width's tree. + xlens = sorted({b["xlen"] for b in selected}) + if len(xlens) > 1 and (args.build_dir or args.against): + sys.exit("ERROR: --build-dir/--against name one build tree, and the " + "selected builds span xlen %s; select one width" + % "/".join(map(str, xlens))) + results = [] + for xlen in xlens: + build_dir = args.build_dir or os.path.join( + root, "build%d_%s" % (xlen, tool.gate)) + if not args.dry_run: + prepare_build_tree(root, build_dir, xlen) + results += run_all([b for b in selected if b["xlen"] == xlen], + build_dir, env, args, tool) if args.dry_run: return 0 + results.sort(key=lambda r: r["build"]["id"]) gated = tuple(m for m in (args.gate.split(",") if args.gate else tool.gated) if m) outcomes = report(results, env, args, gated, tool, args.update_baseline) failures = [o for o in outcomes if o[1] not in PASSING] + if args.against: + failures += compare_against(results, args.against, tool) ok = [r for r in results if r["metrics"]] if args.update_baseline and ok: diff --git a/ci/synth_report.py b/ci/synth_report.py index 03356ac238..dbfe8fa35b 100755 --- a/ci/synth_report.py +++ b/ci/synth_report.py @@ -1,12 +1,12 @@ #!/usr/bin/env python3 """synth_report — render synthesis-gate reports as one Markdown summary. -Both gates end a run with `--report` JSON: asic_gate.yml one per DUT job, -fpga_gate.yml one for the whole sweep. This joins them into a single Markdown -document for the workflow summary -- every build's metrics against its -baseline, why each build that did not pass did not, and where each design is -tight -- so a run is diagnosed from its GitHub page rather than from the machine -that built it. It exits with the worst outcome so the caller can decide whether +Every gate ends a run with `--report` JSON: asic_gate.yml one per DUT job, +fpga_gate.yml and logic_gate.yml one for the whole sweep. This joins them +into a single Markdown document for the workflow summary -- every build's +metrics against its baseline, why each build that did not pass did not, and +where each design is tight -- so a run is diagnosed from its GitHub page +rather than from the machine that built it. It exits with the worst outcome so the caller can decide whether the commit was actually gated: 0 every build passed (or is a tracked known issue) @@ -28,7 +28,7 @@ # pass are reported, not failed. PASSING = ("PASS", "RECORDED", "KNOWN-ISSUE", "XPASS") -# Every metric either tool records, shown in this order when a build reports it. +# Every metric any tool records, shown in this order when a build reports it. COLUMNS = (("Fmax (MHz)", "fmax_mhz", "%.1f"), ("WNS (ns)", "wns_ns", "%.3f"), ("TNS (ns)", "tns_ns", "%.1f"), @@ -42,7 +42,10 @@ ("LUTRAM", "lutram", "%d"), ("BRAM", "bram", "%d"), ("URAM", "uram", "%d"), - ("DSP", "dsp", "%d")) + ("DSP", "dsp", "%d"), + ("Max levels", "max_levels", "%d"), + ("End points >= 12", "deep_endpoints", "%d"), + ("End points >= 8", "watch_endpoints", "%d")) # Slack sits near zero by design, so a relative change of it says nothing. NO_DELTA = ("wns_ns", "tns_ns") diff --git a/ci/test_ram_fold.py b/ci/test_ram_fold.py new file mode 100644 index 0000000000..2d2799d48d --- /dev/null +++ b/ci/test_ram_fold.py @@ -0,0 +1,487 @@ +#!/usr/bin/env python3 +"""hw/scripts/ram_fold.py on a small design that crosses every construct the +pass handles, simulated in lockstep against the unfolded design. + +The design reads four block-RAM-sized RAMs, one per generate iteration, at an +address that comes from registers in other modules: a library pipe register +with a partial reset, reached through an interface array, a function, a +concatenated assign target and a declaration initializer, plus a hand-written +always_ff register. The four RAMs live in instances of one module, so slang +analyzes one body and the pass maps the other three onto it. + +Needs Verilator (VERILATOR or PATH), circt-verilog (CIRCT_VERILOG or PATH) and +pyslang >= 11 in RAM_FOLD_PYTHON (default: this interpreter). +""" + +import os +import pathlib +import re +import shutil +import subprocess +import sys +import tempfile +import unittest + +ROOT = pathlib.Path(__file__).resolve().parents[1] +FOLD = ROOT / "hw/scripts/ram_fold.py" +GEN = ROOT / "hw/scripts/gen_sources.sh" +LIBS = [ROOT / "hw/rtl/VX_platform.vh", ROOT / "hw/rtl/VX_scope.vh", + ROOT / "hw/rtl/libs/VX_dp_ram.sv", ROOT / "hw/rtl/libs/VX_pipe_register.sv"] + +DESIGN = r''' +`include "VX_platform.vh" + +typedef struct packed { + logic [3:0] hi; + logic [3:0] lo; +} rf_pair_t; + +interface rf_if; + logic [7:0] addr; + rf_pair_t pair; + modport master (output addr, pair); + modport slave (input addr, pair); +endinterface + +function automatic logic [7:0] rf_swz(input logic [7:0] x); + return {x[3:0], x[7:4]}; +endfunction + +module rf_src ( + input wire clk, + input wire reset, + input wire en, + input wire [7:0] din, + input wire [7:0] din2, + rf_if o +); + VX_pipe_register #( + .DATAW (16), + .RESETW (1) + ) pr ( + .clk (clk), + .reset (reset), + .enable (en), + .data_in ({din, din2}), + .data_out ({o.addr, o.pair}) + ); +endmodule + +module rf_leaf #( + parameter SIZE = 256 +) ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (SIZE), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +module rf_leaf_in ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (256), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +// instantiated once foldable and once not: neither instance may fold +module rf_mixed ( + input wire clk, + input wire reset, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + input wire [7:0] raddr, + output wire [15:0] rdata +); + VX_dp_ram #( + .DATAW (16), + .SIZE (256), + .RDW_MODE ("R") + ) ram ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (raddr), + .rdata (rdata) + ); +endmodule + +module rf_mid ( + input wire clk, + input wire reset, + rf_if i, + input wire [7:0] bias, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [15:0] rdata +); + reg [7:0] cnt_r; + always_ff @(posedge clk) begin + if (reset) begin + cnt_r <= '0; + end else if (write) begin + cnt_r <= cnt_r + 8'd1; + end + end + wire [3:0] lo_part, hi_part; + assign {hi_part, lo_part} = i.pair; + wire [7:0] mixed; + assign mixed = rf_swz(i.addr) ^ {hi_part, lo_part} ^ cnt_r; + wire [7:0] addr = mixed + bias; + rf_leaf leaf ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (addr), + .rdata (rdata) + ); +endmodule + +// three addresses the pass must refuse: another clock, a register assigned +// inside a concatenation, a register written by two blocks +module rf_odd ( + input wire clk, + input wire clk2, + input wire reset, + input wire [7:0] din, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [2:0][15:0] rdata +); + reg [7:0] slow_r, cat_r, tag_r, two_r; + always_ff @(posedge clk2) begin + slow_r <= din; + end + always_ff @(posedge clk) begin + {cat_r, tag_r} <= {din, ~din}; + end + always_ff @(posedge clk) begin + two_r[3:0] <= din[3:0]; + end + always_ff @(posedge clk) begin + two_r[7:4] <= din[7:4]; + end + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_slow ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (slow_r), + .rdata (rdata[0]) + ); + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_cat ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (cat_r), + .rdata (rdata[1]) + ); + VX_dp_ram #( + .DATAW (16), + .SIZE (256) + ) u_two ( + .clk (clk), + .reset (reset), + .read (1'b0), + .write (write), + .wren (1'b1), + .waddr (waddr), + .wdata (wdata), + .raddr (two_r), + .rdata (rdata[2]) + ); +endmodule + +module rf_top ( + input wire clk, + input wire clk2, + input wire reset, + input wire en, + input wire [7:0] din0, + input wire [7:0] din1, + input wire [7:0] din2, + input wire [7:0] bias_in, + input wire [7:0] top_addr, + input wire write, + input wire [7:0] waddr, + input wire [15:0] wdata, + output wire [9:0][15:0] rdata +); + reg [7:0] bias_r; + always_ff @(posedge clk) begin + bias_r <= bias_in; + end + rf_if ifs [4] (); + for (genvar g = 0; g < 4; ++g) begin : g_lane + rf_src src ( + .clk (clk), + .reset (reset), + .en (en), + .din (din0 + 8'(g)), + .din2 (g[0] ? din1 : din2), + .o (ifs[g]) + ); + rf_mid mid ( + .clk (clk), + .reset (reset), + .i (ifs[g]), + .bias (bias_r), + .write (write), + .waddr (waddr), + .wdata (wdata), + .rdata (rdata[g]) + ); + end + rf_mixed mixed_reg ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (bias_r), + .rdata (rdata[5]) + ); + rf_mixed mixed_in ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (top_addr), + .rdata (rdata[6]) + ); + rf_odd odd ( + .clk (clk), + .clk2 (clk2), + .reset (reset), + .din (din0), + .write (write), + .waddr (waddr), + .wdata (wdata), + .rdata (rdata[9:7]) + ); + // read straight from a top-level input: nothing to fold + rf_leaf_in direct ( + .clk (clk), + .reset (reset), + .write (write), + .waddr (waddr), + .wdata (wdata), + .raddr (top_addr), + .rdata (rdata[4]) + ); +endmodule +''' + +TB = r''' +module tb; + reg clk = 0, clk2 = 0; + reg reset = 1, en = 1, write = 0; + reg [7:0] din0 = 0, din1 = 0, din2 = 0, bias_in = 0, top_addr = 0, waddr = 0; + reg [15:0] wdata = 0; + wire [9:0][15:0] rdata; + rf_top dut (.*); + always #5 clk = ~clk; + always #7 clk2 = ~clk2; + integer f, c; + initial begin + f = $fopen("trace.txt", "w"); + // every RAM word written before anything is traced + for (c = 0; c < 256; c = c + 1) begin + @(negedge clk); + if (c > 3) reset = 0; + write = 1; waddr = c[7:0]; wdata = 16'(c * 7919); + end + for (c = 0; c < 20000; c = c + 1) begin + @(negedge clk); + $fwrite(f, "%0d %h\n", c, rdata); + en = $urandom_range(3) != 0; + write = $urandom_range(1); + din0 = 8'($urandom); din1 = 8'($urandom); din2 = 8'($urandom); + bias_in = 8'($urandom); top_addr = 8'($urandom); + waddr = 8'($urandom); wdata = 16'($urandom); + end + $fclose(f); + $finish; + end +endmodule +''' + + +def tool(env, name): + path = os.environ.get(env) or shutil.which(name) + return path if path and os.path.exists(path) else None + + +VERILATOR = tool("VERILATOR", "verilator") +CIRCT = tool("CIRCT_VERILOG", "circt-verilog") +PYTHON = os.environ.get("RAM_FOLD_PYTHON", sys.executable) + + +def have_pyslang(): + return subprocess.run([PYTHON, "-c", "import pyslang.analysis"], capture_output=True).returncode == 0 + + +@unittest.skipUnless(VERILATOR and CIRCT and have_pyslang(), + "needs verilator, circt-verilog and pyslang >= 11") +class RamFoldTest(unittest.TestCase): + + @classmethod + def setUpClass(cls): + cls.tmp = pathlib.Path(tempfile.mkdtemp(prefix="ram_fold_test_")) + rtl = cls.tmp / "rtl" + rtl.mkdir() + for f in LIBS: + shutil.copy(f, rtl) + (rtl / "rf_design.sv").write_text(DESIGN) + (cls.tmp / "tb.sv").write_text(TB) + env = dict(os.environ, CIRCT_VERILOG=CIRCT, RAM_FOLD_PYTHON=PYTHON) + # the flow's own path: gen_sources.sh preprocesses, then folds (-F) + for name, fold in (("orig", []), ("flow", ["-F"])): + r = subprocess.run(["bash", str(GEN), "-DSIMULATION", "-I" + str(rtl), "-P", *fold, + "-Trf_top", "-C" + str(cls.tmp / name)], + cwd=cls.tmp, env=env, capture_output=True, text=True) + assert r.returncode == 0, r.stdout + r.stderr + if fold: + cls.flow_out = r.stdout + # the same fold with the next-address check, for simulation + shutil.copytree(cls.tmp / "orig", cls.tmp / "chk") + r = subprocess.run([PYTHON, str(FOLD), "--top", "rf_top", "--check", "--diff", "chk.diff", + *sorted(str(p) for p in (cls.tmp / "chk").glob("*.sv"))], + cwd=cls.tmp, capture_output=True, text=True) + assert r.returncode == 0, r.stdout + r.stderr + cls.report = r.stdout + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls.tmp, ignore_errors=True) + + def simulate(self, name): + src = self.tmp / name + obj = self.tmp / ("obj_" + name) + files = [str(p) for p in sorted(src.glob("*.sv"))] + r = subprocess.run([VERILATOR, "--binary", "--timing", "-Wno-fatal", "-Wno-lint", "-Wno-style", + "--x-initial", "0", "--top-module", "tb", "-I" + str(src), "-Mdir", str(obj), + *files, str(self.tmp / "tb.sv")], + capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout[-3000:] + r.stderr[-3000:]) + run = self.tmp / ("run_" + name) + run.mkdir(exist_ok=True) + r = subprocess.run([str(obj / "Vtb"), "+verilator+seed+7"], cwd=run, capture_output=True, text=True) + return r.stdout + r.stderr, (run / "trace.txt").read_text() + + def test_report(self): + folded = [l for l in self.report.splitlines() if l.startswith("fold ")] + kept = [l for l in self.report.splitlines() if l.startswith("keep ")] + self.assertEqual(len(folded), 4, self.report) + self.assertEqual(len(kept), 6, self.report) + self.assertTrue(all(".mid.leaf.ram" in l for l in folded), self.report) + why = {l.split()[1]: l for l in kept} + self.assertIn("top-level input", why["rf_top.direct.ram"]) + self.assertIn("top-level input", why["rf_top.mixed_in.ram"]) + # the foldable instance shares its instantiation with one that is not + self.assertIn("rf_top.mixed_in.ram cannot fold", why["rf_top.mixed_reg.ram"]) + self.assertIn("not the RAM's clock", why["rf_top.odd.u_slow"]) + self.assertIn("assigned as part of", why["rf_top.odd.u_cat"]) + self.assertIn("procedural blocks", why["rf_top.odd.u_two"]) + + def test_flow_patch_is_the_pass_patch(self): + # gen_sources.sh -F folds exactly as the pass does on its own + self.assertEqual(self.flow_out.count("fold "), 4, self.flow_out) + plain = self.tmp / "plain" + shutil.copytree(self.tmp / "orig", plain) + r = subprocess.run([PYTHON, str(FOLD), "--top", "rf_top", + *sorted(str(p) for p in plain.glob("*.sv"))], capture_output=True, text=True) + self.assertEqual(r.returncode, 0, r.stdout + r.stderr) + for f in sorted((self.tmp / "flow").glob("*.sv")): + self.assertEqual(f.read_text(), (plain / f.name).read_text(), f.name) + + def test_rams_take_the_registered_address_form(self): + leaf = (self.tmp / "chk" / "rf_design.sv").read_text() + self.assertIn(".OUT_REG (1)", leaf) + self.assertIn('.RDW_MODE ("W")', leaf) + self.assertIn(".raddr (raddr__nx)", leaf) + + def test_lockstep(self): + out_o, trace_o = self.simulate("orig") + out_c, trace_c = self.simulate("chk") + self.assertNotIn("ram_fold:", out_c, out_c[-2000:]) + self.assertEqual(trace_o.count("\n"), 20000) + self.assertTrue(trace_o == trace_c, "folded design diverges from the original") + + def test_known_bad_patch_is_caught(self): + # read the counter's current value instead of its next one: a fold off by a cycle + bad = self.tmp / "bad" + if not bad.exists(): + shutil.copytree(self.tmp / "chk", bad) + d = bad / "rf_design.sv" + text, n = re.subn(r"(assign mixed__nx = .*?)cnt_r__nx", r"\1cnt_r", d.read_text()) + self.assertEqual(n, 1) + d.write_text(text) + out_b, trace_b = self.simulate("bad") + _, trace_o = self.simulate("orig") + self.assertIn("ram_fold:", out_b) + self.assertNotEqual(trace_b, trace_o) + + +if __name__ == "__main__": + unittest.main() diff --git a/ci/test_runner.py b/ci/test_runner.py index 5feae9f5d6..446d12c1bb 100644 --- a/ci/test_runner.py +++ b/ci/test_runner.py @@ -11,11 +11,12 @@ its failure is expected and does not fail CI (and an unexpected pass shows as XPASS). Use it only for triaged, documented breakage. -A `check: model_parity` case runs the same app/args/configs on simx and rtlsim -and compares the runtime's final `PERF: instrs=…, cycles=…` summary: retired -instructions must match exactly (both are deterministic ISA-level executions, -so any delta is functional divergence) and cycles must agree within the case's -`tolerance` (SimX is the timing model of the RTL, not just a functional oracle). +A `check: model_parity` case runs the same app/args/configs on its two `legs` +(simx and rtlsim by default) and compares the runtime's final `PERF: instrs=…, +cycles=…` summary: retired instructions must match exactly (both are +deterministic ISA-level executions, so any delta is functional divergence) and +cycles must agree within the case's `tolerance` (SimX is the timing model of the +RTL, not just a functional oracle; Foldsim is cycle-exact, so its tolerance is 0). A case with `parity_counters` also compares that MPM class's counters. """ @@ -92,22 +93,57 @@ def _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles): assert not fails, "{}: {} counter mismatch: {}".format(case.id, prefix, "; ".join(fails)) +def _perf_of(text, what): + perf = _PERF_RE.findall(text) + assert perf, "{}: no 'PERF: instrs=…, cycles=…' summary " \ + "(app must call vx_device_dump_perf)".format(what) + instrs, cycles = perf[-1] + return int(instrs), int(cycles) + + +def _release_parity(case, xlen): + """Run the debug case, then (xlen 32) the same run built release; both + must report identical instructions and cycles.""" + argv, env = case.run_command(xlen) + rc = tc.execute(argv, env) + assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) + if xlen != 32: + return + # a debug run sends its output to the --log file (blackbox.sh) + log = next((a.split("=", 1)[1] for a in argv if a.startswith("--log=")), "run.log") + with open(log, errors="replace") as fh: + dbg_instrs, dbg_cycles = _perf_of(fh.read(), case.id + " [debug]") + rel_argv = [a for a in argv if not a.startswith(("--debug", "--log="))] + rc, out = tc.execute_capture(rel_argv, env) + assert rc == 0, "{} [release] failed (exit {}): {}".format( + case.id, rc, " ".join(rel_argv)) + rel_instrs, rel_cycles = _perf_of(out, case.id + " [release]") + print("RELEASE-PARITY: {}: instrs debug={} release={}, cycles debug={} " + "release={}".format(case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles)) + assert (dbg_instrs, dbg_cycles) == (rel_instrs, rel_cycles), \ + "{}: the debug build changes the run (instrs {} vs {}, cycles {} vs {}) " \ + "-- debug-only state must not reach timing".format( + case.id, dbg_instrs, rel_instrs, dbg_cycles, rel_cycles) + + def _model_parity(case, xlen): - simx_instrs, simx_cycles, simx_out = _run_one(case, xlen, "simx") - rtl_instrs, rtl_cycles, rtl_out = _run_one(case, xlen, "rtlsim") - gap = abs(rtl_cycles - simx_cycles) / float(rtl_cycles) - print("PARITY: {}: instrs simx={} rtlsim={}, cycles simx={} rtlsim={}, " + model, ref = case.legs + model_instrs, model_cycles, model_out = _run_one(case, xlen, model) + ref_instrs, ref_cycles, ref_out = _run_one(case, xlen, ref) + gap = abs(ref_cycles - model_cycles) / float(ref_cycles) + print("PARITY: {}: instrs {m}={} {r}={}, cycles {m}={} {r}={}, " "gap={:.2%} (tolerance {:.0%})".format( - case.id, simx_instrs, rtl_instrs, simx_cycles, rtl_cycles, - gap, case.tolerance)) - assert simx_instrs == rtl_instrs, \ - "{}: retired-instruction mismatch (simx={}, rtlsim={}) — functional " \ - "divergence, not a timing gap".format(case.id, simx_instrs, rtl_instrs) + case.id, model_instrs, ref_instrs, model_cycles, ref_cycles, + gap, case.tolerance, m=model, r=ref)) + assert model_instrs == ref_instrs, \ + "{}: retired-instruction mismatch ({m}={}, {r}={}) — functional " \ + "divergence, not a timing gap".format(case.id, model_instrs, ref_instrs, + m=model, r=ref) assert gap <= case.tolerance, \ - "{}: cycle gap {:.2%} exceeds tolerance {:.0%} (simx={}, rtlsim={})".format( - case.id, gap, case.tolerance, simx_cycles, rtl_cycles) + "{}: cycle gap {:.2%} exceeds tolerance {:.0%} ({m}={}, {r}={})".format( + case.id, gap, case.tolerance, model_cycles, ref_cycles, m=model, r=ref) if case.parity_counters: - _compare_counters(case, simx_out, rtl_out, simx_cycles, rtl_cycles) + _compare_counters(case, model_out, ref_out, model_cycles, ref_cycles) def _perf_gate(case, xlen, update): @@ -149,6 +185,9 @@ def test_case(case, sim_build, request): if case.check == "perf_gate": _perf_gate(case, xlen, request.config.getoption("--update-baselines")) return + if case.release_parity: + _release_parity(case, xlen) + return argv, env = case.run_command(xlen) rc = tc.execute(argv, env) assert rc == 0, "{} failed (exit {}): {}".format(case.id, rc, " ".join(argv)) diff --git a/ci/testcase.py b/ci/testcase.py index c9845ca45a..09d25e362f 100644 --- a/ci/testcase.py +++ b/ci/testcase.py @@ -48,12 +48,18 @@ # nightly), and everything is what ci.yml's matrix can run as a CELL -- these # cannot be. `fpga` needs a licensed Vivado and hours of a whole machine; # `asic` needs 1-2 hours PER DUT, which is a fan-out of standalone jobs, not one -# cell. Each has its own workflow (.github/workflows/{fpga,asic}_gate.yml). -# Opting in is explicit, never implied. -OPT_IN_TIERS = {"fpga", "asic"} +# cell; `timing` needs foldsim, which the base toolchain does not install, and +# `timing_gpu` also ~95 GB per build. Each has its own workflow +# (.github/workflows/{fpga,asic,logic}_gate.yml). Opting in is explicit, never +# implied. +OPT_IN_TIERS = {"fpga", "asic", "timing", "timing_gpu"} # simx<->rtlsim cycle-parity default: both timing models must agree within 5%. DEFAULT_PARITY_TOLERANCE = 0.05 +DEFAULT_PARITY_LEGS = ["simx", "rtlsim"] +# A parity leg runs through blackbox. foldsim is a leg but not a cell driver: it +# generates and builds its simulator inside the run, so there is no sim build. +PARITY_DRIVERS = VALID_DRIVERS | {"foldsim"} # Per-case wall times (seconds, per xlen) measured by a CI run, recorded by # ci/record_runtimes.py from the run's junit reports. A cell whose recorded total @@ -61,6 +67,8 @@ # into the tests job's timeout. RUNTIMES_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "runtimes.json") SHARD_BUDGET_S = 150 * 60 +# Cells of long rtlsim runs split finer: each shard of these stays near an hour. +CELL_SHARD_BUDGET_S = {"model_parity": 60 * 60, "perf_gate": 60 * 60} # Weight of a case with no recorded time (new, or not yet run on that xlen). DEFAULT_CASE_S = 120 @@ -106,6 +114,12 @@ def __init__(self, category, entry, driver, defaults): # coverage can't creep back in per-file. if self.check in ("model_parity", "perf_gate"): self.xlens = [32] + # release_parity: a debug run (its flags carry --debug) must report + # exactly the cycles of the same run built release. A debug build widens + # the RTL UUID to 44 bits and enables assertions and traces; none of that + # may move timing. The runner adds the release leg itself, at xlen 32 + # only like the cycle gates; other xlens run the debug leg alone. + self.release_parity = bool(entry.get("release_parity", False)) self.tolerance = float(entry.get("tolerance", defaults.get("tolerance", DEFAULT_PARITY_TOLERANCE))) # model_parity can also compare an MPM class's counters, read from the @@ -113,6 +127,9 @@ def __init__(self, category, entry, driver, defaults): # must pass the class with --perf=N): `exact` counters must match, and # `tolerance` counters must agree within the case tolerance. self.parity_counters = dict(entry.get("parity_counters", {})) + # The two drivers a model_parity case runs: the model, then the RTL it + # is checked against. + self.legs = list(entry.get("legs", defaults.get("legs", DEFAULT_PARITY_LEGS))) self.authored_drivers = ("driver" in entry) or ("drivers" in entry) # make-run / script fields self.dir = entry.get("dir", "") @@ -415,19 +432,19 @@ def cell_name(case): return case.check or case.category -def shard_plan(case_ids, xlen, shards=None): +def shard_plan(case_ids, xlen, shards=None, budget=SHARD_BUDGET_S): """Split a cell's cases into shards balanced on their recorded runtimes. Longest case first onto the least-loaded shard, with ties broken on case id and shard index, so the planner and every shard derive the same split from the same ids. Unless `shards` is given, the count is the fewest that keep - the cell's recorded total within SHARD_BUDGET_S per shard. + the cell's recorded total within `budget` per shard. Returns (shards, {case id: 0-based shard}). """ runtimes = load_runtimes(xlen) weight = {i: runtimes.get(i, DEFAULT_CASE_S) for i in case_ids} if shards is None: - shards = max(1, math.ceil(sum(weight.values()) / SHARD_BUDGET_S)) + shards = max(1, math.ceil(sum(weight.values()) / budget)) load = [0.0] * shards plan = {} for i in sorted(weight, key=lambda i: (-weight[i], i)): @@ -466,7 +483,7 @@ def cmd_matrix(args): # the CHECK, not on a category that happens to share its name -- a check is a # marker across suites, and no category name may control whether it runs. # - # A cell whose recorded runtime exceeds SHARD_BUDGET_S is emitted as `shards` + # A cell whose recorded runtime exceeds its budget is emitted as `shards` # cells, each running shard `shard` of the split (shard_plan). The split is # over every case the cell's -m expression selects, not only those this # event's filters planned the cell for, because that is what the cell runs. @@ -479,21 +496,24 @@ def cmd_matrix(args): if xfilter and xlen not in xfilter: continue name = cell_name(c) - key = (name, drv, xlen) - cell = cells.setdefault(key, { - "category": name, "driver": drv, "xlen": xlen, "needs": set(), + cells.setdefault((name, drv, xlen), { + "category": name, "driver": drv, "xlen": xlen, }) - cell["needs"].update(c.needs) - members = {} + # A cell's needs are those of every case its -m expression runs, not only of + # the cases this event's filters planned it for. + members, needs = {}, {} for c in cases: if c.tier in OPT_IN_TIERS: continue for xlen in c.xlens: - members.setdefault((cell_name(c), c.marker_driver or "host", xlen), []).append(c.id) + key = (cell_name(c), c.marker_driver or "host", xlen) + members.setdefault(key, []).append(c.id) + needs.setdefault(key, set()).update(c.needs) out = [] for key, cell in cells.items(): - cell["needs"] = sorted(cell["needs"]) - shards, _ = shard_plan(members.get(key, []), key[2]) + cell["needs"] = sorted(needs.get(key, ())) + shards, _ = shard_plan(members.get(key, []), key[2], + budget=CELL_SHARD_BUDGET_S.get(key[0], SHARD_BUDGET_S)) if shards == 1: out.append(cell) continue @@ -580,6 +600,13 @@ def cmd_lint(args): errors.append("{}: script case missing 'run'".format(c.id)) if any(int(x) not in (32, 64) for x in c.xlens): errors.append("{}: xlen must be 32 and/or 64".format(c.id)) + if c.release_parity: + if c.via != "blackbox" or "--debug" not in c.flags: + errors.append("{}: release_parity needs a blackbox case run " + "with --debug".format(c.id)) + if c.check: + errors.append("{}: release_parity cannot combine with a " + "check".format(c.id)) if c.check: if c.check not in VALID_CHECK: errors.append("{}: invalid check {!r}".format(c.id, c.check)) @@ -587,9 +614,15 @@ def cmd_lint(args): errors.append("{}: check cases must be via blackbox".format(c.id)) if c.authored_drivers: errors.append("{}: check cases must not set driver/drivers " - "(the runner drives simx and rtlsim)".format(c.id)) - if not (0.0 < c.tolerance < 1.0): - errors.append("{}: tolerance must be in (0, 1)".format(c.id)) + "(the runner drives the legs itself)".format(c.id)) + if len(c.legs) != 2 or len(set(c.legs)) != 2 or \ + not set(c.legs) <= PARITY_DRIVERS: + errors.append("{}: legs must be two distinct drivers of {}".format( + c.id, sorted(PARITY_DRIVERS))) + # 0 asks for exact cycles, from a model that is cycle-exact by + # construction. + if not (0.0 <= c.tolerance < 1.0): + errors.append("{}: tolerance must be in [0, 1)".format(c.id)) if c.parity_counters: pc = c.parity_counters if c.check != "model_parity": diff --git a/ci/testcases/amo.yaml b/ci/testcases/amo.yaml index de2b638690..1e63cc43c0 100644 --- a/ci/testcases/amo.yaml +++ b/ci/testcases/amo.yaml @@ -7,7 +7,7 @@ defaults: configs: "-DVX_CFG_EXT_A_ENABLE" xlen: [32, 64] tier: smoke - touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_slice.sv, sim/simx/amo, sim/simx/mem] + touches: [hw/rtl/cache, hw/rtl/mem, hw/rtl/core/VX_decode.sv, hw/rtl/core/VX_lsu_block.sv, sim/simx/amo, sim/simx/mem] tests: # RISC-V 'A' ISA conformance (rv{XLEN}ua-p-{amo*,lrsc}); shares the A-built sim. - id: isa-conformance @@ -30,7 +30,7 @@ tests: - id: wb-dirtybytes0 app: amo drivers: [rtlsim] - configs+: "-DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4" + configs+: "-DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_LAZY_FETCH=0 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4" - id: wb-dirtybytes1 app: amo drivers: [rtlsim] diff --git a/ci/testcases/asic_gate.yaml b/ci/testcases/asic_gate.yaml index 00def8e7cc..678a863f9f 100644 --- a/ci/testcases/asic_gate.yaml +++ b/ci/testcases/asic_gate.yaml @@ -78,6 +78,10 @@ tests: # clock_mhz target clock the design is constrained to, via CLOCK_FREQ. Fmax # is derived from it and the worst slack, and the gate also asserts # the design MEETS it — so this is a commitment, not a hint. +# area_budget_um2 cell-area ceiling: the gate fails when cell area grows +# more than 5% over it. Fixed here rather than taken from the +# baseline, so re-recording cannot creep it upward; raising it +# is a reviewed edit. # configs EXTRA defines, appended to the catalog's. Usually empty: the DUT # configuration belongs in one place, and that place is catalog.mk. # group which golden file the results land in (mirrors the Xilinx groups) @@ -103,6 +107,7 @@ builds: group: core dut: cache clock_mhz: 800 + area_budget_um2: 48726 configs: "" # Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC @@ -115,8 +120,18 @@ builds: group: core dut: core clock_mhz: 400 + area_budget_um2: 84624 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 400 + area_budget_um2: 15867 + configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE" + # Multi-core system top: 2 cores with atomics and virtual memory behind the AFU # wrapper. Closes the cross-module boundaries (socket arbiters, socket MMU, # cluster mem network) that the per-unit DUTs do not. @@ -124,6 +139,7 @@ builds: group: core dut: top clock_mhz: 400 + area_budget_um2: 187230 # 11.2 GB Yosys peak at NT=NW=4 (8/8: 18.6 GB). configs: "-DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4" @@ -134,6 +150,7 @@ builds: group: dxa dut: dxa clock_mhz: 800 + area_budget_um2: 12449 configs: "" # --- mufu -------------------------------------------------------------------- @@ -144,6 +161,7 @@ builds: group: mufu dut: mufu clock_mhz: 800 + area_budget_um2: 2073 configs: "" # --- tensor ------------------------------------------------------------------ @@ -154,6 +172,7 @@ builds: group: tensor dut: tcu clock_mhz: 500 + area_budget_um2: 74407 configs: "" # Full tensor subsystem: TCU (WGMMA + MX) + dxa behind a shared L2, 2 cores per @@ -162,6 +181,7 @@ builds: group: tensor dut: tensor clock_mhz: 400 + area_budget_um2: 104438 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" # --- graphics ---------------------------------------------------------------- @@ -171,6 +191,7 @@ builds: group: graphics dut: raster clock_mhz: 800 + area_budget_um2: 10527 configs: "" # Output merger: depth/stencil test, blend, ocache port. @@ -178,6 +199,7 @@ builds: group: graphics dut: om clock_mhz: 800 + area_budget_um2: 5926 configs: "" # Texture unit: address generation, LOD, bilinear filter, tcache port. @@ -186,6 +208,7 @@ builds: group: graphics dut: tex clock_mhz: 800 + area_budget_um2: 61159 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" # Ray-tracing unit: CW-BVH4 walker, box/triangle PEs, short stack. @@ -193,6 +216,7 @@ builds: group: graphics dut: rtu clock_mhz: 400 + area_budget_um2: 98965 configs: "" # Full graphics subsystem: raster + om + tex + rtu + dxa behind a shared L2, @@ -201,6 +225,7 @@ builds: group: graphics dut: gfx clock_mhz: 400 + area_budget_um2: 242203 # 14.6 GB Yosys peak at NT=NW=4 (8/8: 19.5 GB). configs: "-DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4" @@ -212,4 +237,5 @@ builds: group: vm dut: vm clock_mhz: 400 + area_budget_um2: 146445 configs: "-DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8" diff --git a/ci/testcases/cache.yaml b/ci/testcases/cache.yaml index de3e6c289d..cd69bd7f37 100644 --- a/ci/testcases/cache.yaml +++ b/ci/testcases/cache.yaml @@ -161,7 +161,7 @@ tests: drivers: - rtlsim app: mstress - configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4 + configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_LAZY_FETCH=0 -DVX_CFG_DCACHE_DIRTYBYTES=0 -DVX_CFG_DCACHE_NUM_WAYS=4 - id: mstress-2 via: blackbox drivers: @@ -196,6 +196,18 @@ tests: clusters: 2 l2cache: true l3cache: true +# Dirty-byte write-back on a sectored L2: a sector refill keeps the other +# sectors' dirty bytes. +- id: mstress-6 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_DIRTYBYTES=1 + shape: + cores: 4 + l2cache: true - id: sgemm-20 via: blackbox drivers: @@ -252,7 +264,7 @@ tests: args: -n1 # Non-power-of-two cache sharing: 3 cores over 2 shared icaches exercises the -# SimX arbiter's ceiling-division input grouping. A floor-grouped arbiter +# arbiter's ceiling-division input grouping. A floor-grouped arbiter # (inputs/outputs) never polls the trailing core's fetch port and the run # hangs at boot; VX_stream_arb sizes its groups by ceiling division, so the # SimX model must match. @@ -260,7 +272,130 @@ tests: via: blackbox drivers: - simx + - rtlsim app: demo configs: -DVX_CFG_SOCKET_SIZE=3 -DVX_CFG_NUM_ICACHES=2 shape: cores: 3 +# Hashed bank select, on by default for the dcache and the L2. These cover the +# hash off at both levels, 16 threads, and a write-back L2, where every fill and +# write-back rebuilds the address. +- id: bank_hash-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_DCACHE_BANK_HASH=0 -DVX_CFG_L2_BANK_HASH=0 + shape: + cores: 2 + l2cache: true +- id: bank_hash-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + shape: + threads: 16 +- id: bank_hash-3 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 + shape: + cores: 4 + l2cache: true +- id: bank_hash-4 + via: blackbox + drivers: + - simx + - rtlsim + app: mstress + configs: -DVX_CFG_DCACHE_WRITEBACK=1 -DVX_CFG_DCACHE_DIRTYBYTES=1 + shape: + threads: 16 +# Lazy fetch-on-read on a write-back LLC: a store miss allocates its line +# without a fill, and a load or AMO of a partially written line fetches it and +# merges under the dirty bytes. The default 1-core dcache is such an LLC. +- id: lazy_fetch-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 +# The same on a write-back L2 shared by 4 cores behind write-through L1s. +- id: lazy_fetch-2 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + shape: + cores: 4 + l2cache: true +# A sector every store has covered reads no memory when a load refetches it. +# The app compares the memory reads of fully written lines against lines with +# a word per sector unwritten, so it runs with the memory counters. +- id: lazy_fetch-3 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 + flags: --perf=7 +- id: lazy_fetch-4 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true +# The same on a dcache with a deferred commit stage (latency 3): a queued AMO +# result drains through the core-request slot while the lazy queue holds its +# pipeline reservation, and fully written sectors still loop back data-less. +- id: lazy_fetch-5 + via: blackbox + drivers: + - simx + - rtlsim + app: lazy_fetch + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_DCACHE_LAZY_FETCH=1 -DVX_CFG_DCACHE_LATENCY=3 +# Both sectors of one sectored L2 line miss together from different warps and +# cores: each must get its own fill, never chain behind the other sector's. +- id: sector_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sector_merge + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +# A thread's load must return its own earlier store to the same word. On the +# write-through L1 under an L2, the stores of a miss chain reach the array only +# as they replay behind the fill: a load landing after the chain's read head +# has replayed, and before the store has, must link behind the store rather +# than hit the line ahead of it. The delay sweep lands the load across that +# window. +- id: raw_order-wt + via: blackbox + drivers: + - simx + - rtlsim + app: raw_order + args: -n16384 -s64 -d2 + shape: + l2cache: true diff --git a/ci/testcases/config1.yaml b/ci/testcases/config1.yaml index feab45fba1..a5d8bdcf45 100644 --- a/ci/testcases/config1.yaml +++ b/ci/testcases/config1.yaml @@ -174,18 +174,57 @@ tests: shape: cores: 4 args: -n1 +# Core and cluster counts that are not powers of two: the shared L2 serves +# three sockets, the global barrier counts three cores, and three clusters +# share the L3. +- id: async_gbarrier-npot-1 + via: blackbox + drivers: + - rtlsim + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: async_gbarrier-npot-2 + via: blackbox + drivers: + - simx + app: async_gbarrier + shape: + cores: 3 + l2cache: true +- id: vecadd-npot-1 + via: blackbox + drivers: + - rtlsim + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true +- id: vecadd-npot-2 + via: blackbox + drivers: + - simx + app: vecadd + shape: + cores: 2 + clusters: 3 + l2cache: true + l3cache: true - id: diverge-17 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: diverge-18 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-19 via: blackbox drivers: @@ -254,13 +293,13 @@ tests: drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=2 -DVX_CFG_NUM_ALU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: diverge-23 via: blackbox drivers: - rtlsim app: diverge - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_ALU_BLOCKS=4 -DVX_CFG_NUM_ALU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: diverge-24 via: blackbox drivers: @@ -284,13 +323,13 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 + configs: -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_NUM_FPU_LANES=2 -DVX_CFG_FETCH_WIDTH=2 - id: regression/vecadd-2 via: blackbox drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_FPU_BLOCKS=4 -DVX_CFG_NUM_FPU_LANES=4 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-3 via: blackbox drivers: @@ -349,7 +388,7 @@ tests: drivers: - rtlsim app: regression/vecadd - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 - id: regression/vecadd-7 via: blackbox drivers: @@ -362,3 +401,21 @@ tests: - simx app: regression/vecadd configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_NUM_LSU_BLOCKS=4 -DVX_CFG_NUM_LSU_LANES=4 +# Fetch slices reading one line in the same cycle share one icache access, and +# each slice must get its own word: sgemm's three-line loop merges almost every +# cycle. The second case puts the merged responses behind 16 banks. +- id: icache_merge-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 +- id: icache_merge-2 + via: blackbox + drivers: + - simx + - rtlsim + app: raycast + configs: -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_THREADS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_ICACHE_NUM_BANKS=16 diff --git a/ci/testcases/core.yaml b/ci/testcases/core.yaml index b09ebb949e..51e8dcb36d 100644 --- a/ci/testcases/core.yaml +++ b/ci/testcases/core.yaml @@ -131,6 +131,34 @@ tests: via: blackbox app: vecadd args: -n16384 +# Streaming stores into a lazy-fetch write-back L2: store misses allocate +# through the bank loopback instead of a memory fill. +- id: parity-vecadd-lazy-l2 + check: model_parity + via: blackbox + app: vecadd + args: -n16384 + configs: -DVX_CFG_L2_LAZY_FETCH=1 + shape: {l2cache: true} +# Lazy-fetch refetch paths on the default write-back dcache: partial-sector +# loads park and refetch, and same-set thrash evicts lines mid-refetch. +- id: parity-lazy-fetch + check: model_parity + via: blackbox + app: lazy_fetch + configs: -DVX_CFG_DCACHE_LAZY_FETCH=1 +# Fully written sectors on a lazy-fetch write-back L2 shared by 4 cores: +# partial-miss loads and parked chain heads refetch them through the loopback. +- id: parity-lazy-fetch-full + check: model_parity + via: blackbox + app: lazy_fetch + args: -f -n1 + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_SOCKET_SIZE=1 -DVX_CFG_L2_WRITEBACK=1 -DVX_CFG_L2_LAZY_FETCH=1 + flags: --perf=7 + shape: + cores: 4 + l2cache: true # FPU + LSU locality, multi-warp reuse (~1.29M cycles on the default shape). - id: parity-sgemm check: model_parity @@ -160,6 +188,118 @@ tests: args: -n128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 shape: {cores: 2, l2cache: true} +# The same shape at n64: L2 hits on a miss-heavy working set, two operand +# collectors per issue slice, and the load-response path's exact entry counts. +- id: parity-sgemm-mc64 + check: model_parity + # Tighter than the default: the shape exercises those entries and the + # collectors' output register exactly. + tolerance: 0.03 + via: blackbox + app: sgemm + args: -n64 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 + shape: {cores: 2, l2cache: true} +# SGEMM on two frontend slices feeding four issue slices: per-slice fetch on a +# two-port icache, CTAs as wide as the core, a 16-port dcache. +- id: parity-sgemm-fw2 + check: model_parity + # Tighter than the default: icache port 0, which carries the flush + # injection, holds four requests, and the shape exercises all of them. + tolerance: 0.03 + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Four frontend slices on a four-port icache: every slice but the first fetches +# through two request entries, the first through four. +- id: parity-sgemm-fw4 + check: model_parity + via: blackbox + app: sgemm + args: -n128 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 +# Device printf: per-character IO ring stores, each published behind a fence +# that drains, flushes and invalidates the data cache. +- id: parity-printf + check: model_parity + via: blackbox + app: printf +# The same fences on data caches with more banks than memory ports: every +# port's flush token must be answered on that port. +- id: parity-printf-iw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-printf-fw2 + check: model_parity + via: blackbox + app: printf + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=2 +# Same-word reads from several coalescer channels of one warp, served by one +# dcache bank access and returned to every channel. +- id: parity-uniform-ld + check: model_parity + via: blackbox + app: uniform_ld + configs: -DVX_CFG_NUM_THREADS=16 +# Warp-wide local memory: broadcast reads, bank-conflict replays, sub-word and +# same-word stores, with one LSU block and with two, where the read engine +# works the banks' second port. +- id: parity-lmem-access + check: model_parity + via: blackbox + app: lmem_access + configs: -DVX_CFG_NUM_THREADS=16 +- id: parity-lmem-access-iw2 + check: model_parity + via: blackbox + app: lmem_access + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 +# -s leaves out the LR/SC phase, whose retry count depends on timing; the +# lmem-access-amo* regression cases run it. +- id: parity-lmem-access-amo-iw2 + check: model_parity + via: blackbox + app: lmem_access + args: -n16 -s + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 +# Wide loads and stores: SimX and RTL expand and time them alike. +- id: parity-ldst-wide + check: model_parity + via: blackbox + app: ldst_wide + configs: -DVX_CFG_NUM_THREADS=16 +- id: parity-ldst-wide-lanes + check: model_parity + via: blackbox + app: ldst_wide + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 +# Matrix fragment loads and stores: SimX and RTL expand and time them alike. +- id: parity-ldst-matrix + check: model_parity + via: blackbox + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE +- id: parity-sgemm2-lmem + check: model_parity + via: blackbox + app: sgemm2 + args: -n32 -t4 + configs: -DVX_CFG_NUM_THREADS=16 +- id: parity-sgemm2-lmem-iw2 + check: model_parity + via: blackbox + app: sgemm2 + args: -n64 -t16 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_ISSUE_WIDTH=2 +- id: parity-sgemm2-lmem-wide + check: model_parity + via: blackbox + app: sgemm2 + args: -n32 -t4 + configs: -DVX_CFG_NUM_THREADS=16 -DLDST_WIDE=1 # SGEMM with local-memory tiling (kernel-managed shared memory). - id: parity-sgemmx check: model_parity @@ -184,3 +324,12 @@ tests: via: blackbox app: raycast configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 +# The raycaster on four cores sharing an L2: its image columns carry very +# different work, so where the launch places each CTA sets the critical path. +- id: parity-raycast-mc + check: model_parity + via: blackbox + app: raycast + shape: + cores: 4 + l2cache: true diff --git a/ci/testcases/debug.yaml b/ci/testcases/debug.yaml index ee8d117b88..21b4ccd8fc 100644 --- a/ci/testcases/debug.yaml +++ b/ci/testcases/debug.yaml @@ -66,13 +66,20 @@ tests: app: sgemm_tcu configs: -DVX_CFG_EXT_TCU_ENABLE flags: --debug=3 --log=run_simx.log +# The debug build (44-bit UUID, assertions, traces) must not move RTL timing: +# the runner reruns this case built release and requires identical cycles. The +# shape keeps the data cache under pressure, with two channels whose same-word +# reads merge, so a debug-only tag width reaching the merge shows in cycles. - id: sgemm_tcu-2 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_EXT_TCU_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_DCACHE_SIZE=4096 -DVX_CFG_EXT_TCU_ENABLE + shape: + l2cache: true flags: --debug=3 --log=run_rtlsim.log + release_parity: true - id: simx_mt-1 via: blackbox drivers: diff --git a/ci/testcases/dxa.yaml b/ci/testcases/dxa.yaml index 8cad41a2b3..4e84687297 100644 --- a/ci/testcases/dxa.yaml +++ b/ci/testcases/dxa.yaml @@ -216,6 +216,21 @@ tests: - rtlsim app: sgemm2_dxa configs: -DVX_CFG_EXT_DXA_ENABLE +# The tile reads through the wide loads (LDST_WIDE=1). +- id: sgemm2_dxa-wide + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DLDST_WIDE=1 +- id: sgemm2_dxa_mcast-wide + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa_mcast + configs: -DVX_CFG_EXT_DXA_ENABLE -DLDST_WIDE=1 - id: sgemm2_dxa_mcast-1 via: blackbox drivers: @@ -353,6 +368,45 @@ tests: configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 shape: cores: 4 +# Shared L2 behind multi-core sockets: the socket routes each DXA lmem write +# to the core named by its cluster-local index modulo the socket size, at a +# power-of-two shape and at sockets of three. +- id: sgemm2_dxa-l2-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-l2-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true +- id: sgemm2_dxa-npot-1 + via: blackbox + drivers: + - simx + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true +- id: sgemm2_dxa-npot-2 + via: blackbox + drivers: + - rtlsim + app: sgemm2_dxa + configs: -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_DXA_UNITS=4 -DVX_CFG_SOCKET_SIZE=3 + shape: + cores: 6 + l2cache: true - id: sgemm2_dxa-9 via: blackbox drivers: @@ -388,7 +442,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_FETCH_WIDTH=2 shape: threads: 32 warps: 8 @@ -419,7 +473,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp_dxa - configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_TCU_TYPE=DPI -DVX_CFG_FETCH_WIDTH=4 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 shape: threads: 32 @@ -565,4 +619,14 @@ tests: configs: -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 shape: - threads: [4, 16] \ No newline at end of file + threads: [4, 16] + +# The DXA-staged WGMMA accumulator stored with vx_stm (LDST_MATRIX=1). +- id: sgemm_tcu_wg_dxa-ldm + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg_dxa + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DWGMMA_NRC=16 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + shape: {threads: 32, warps: 8} + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/foldsim.yaml b/ci/testcases/foldsim.yaml new file mode 100644 index 0000000000..c1d37f73a5 --- /dev/null +++ b/ci/testcases/foldsim.yaml @@ -0,0 +1,52 @@ +# Foldsim suite: the threaded simulator hw/syn/foldsim compiles from Vortex +# (docs/designs/foldsim.md), checked against the RTL it was compiled from. +# +# check: model_parity with legs [foldsim, rtlsim] and tolerance 0. Foldsim is +# cycle-exact by construction, so retired instructions AND cycles must be +# identical. The reference is rtlsim behind the AXI4 adapter on the same +# DRAM bank model (AXI_BUS=1 AXI_MEM_MODEL=bank); the env reaches only +# rtlsim's build. Exactness holds at any workload size, so the configuration +# is the smallest: tinygpu, generated and simulated in Verilator by the +# foldsim driver's in-process target. +# +# The configuration is the toml's but for two choices of the test: +# - FPU_TYPE_STD: the toml's FPU under Vivado synthesis is the Xilinx cores, +# whose simulation models need a Vivado install. +# - PLATFORM_MEMORY_ID_WIDTH=5 for rtlsim: the AXI id width the Foldsim kernel +# drives with Vortex's tag buffer engaged, 4 + log2(memory banks), which the +# flow derives from the toml's 2 banks. At rtlsim's default of 32 there is no +# tag buffer and cycles differ. +# +# needs: foldsim -- the foldsim prebuilt and pyslang (the boundary lifting); ci.yml +# installs them in a cell whose cases need them. circt is in the lite toolchain. +category: foldsim +defaults: + xlen: + - 32 + tier: full + needs: + - foldsim + legs: + - foldsim + - rtlsim + tolerance: 0 + configs: >- + -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -DVX_CFG_NUM_WARPS=2 + -DVX_CFG_NUM_THREADS=2 -DVX_CFG_ICACHE_DISABLE -DVX_CFG_DCACHE_DISABLE + -DVX_CFG_LMEM_DISABLE -DVX_CFG_FPU_TYPE_STD -DPLATFORM_MEMORY_ID_WIDTH=5 +tests: +- id: parity-demo + check: model_parity + via: blackbox + app: demo + env: + AXI_BUS: 1 + AXI_MEM_MODEL: bank +- id: parity-sgemm + check: model_parity + via: blackbox + app: sgemm + args: -n16 + env: + AXI_BUS: 1 + AXI_MEM_MODEL: bank diff --git a/ci/testcases/fpga_gate.yaml b/ci/testcases/fpga_gate.yaml index cae9b0f0a8..65750815c6 100644 --- a/ci/testcases/fpga_gate.yaml +++ b/ci/testcases/fpga_gate.yaml @@ -48,7 +48,12 @@ tests: # builds — read by ci/fpga_gate.py, ignored by the pytest catalog loader. # # dut hw/syn/xilinx/dut/ target to synthesize -# clock_mhz target clock, baked into the constraint via CLK_FREQ_MHZ +# clock_mhz target clock, baked into the constraint via CLK_FREQ_MHZ. The +# gate fails when Fmax falls more than 2% below it. +# lut_budget LUT ceiling: the gate fails when LUT grows more than 5% over +# it. Fixed here rather than taken from the baseline, so +# re-recording cannot creep it upward; raising it is a reviewed +# edit. # group which golden file the results land in # impl_strategy Vivado implementation strategy for this build (IMPL_STRATEGY # env into project.tcl); absent = the OPT_LEVEL default. Part of @@ -71,6 +76,7 @@ builds: group: core dut: cache clock_mhz: 300 + lut_budget: 28369 configs: -DVX_CFG_EXT_A_ENABLE # Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC @@ -79,8 +85,18 @@ builds: group: core dut: core clock_mhz: 300 + lut_budget: 171712 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 300 + lut_budget: 26822 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE + # Multi-core system top: 2 cores at NT=NW=4, atomics + virtual memory (MMU). # Closes the cross-module boundaries (socket arbiters, socket MMU, cluster mem # network) that the per-unit DUTs do not; 250 MHz (routing-bound, misses 300). @@ -88,6 +104,7 @@ builds: group: core dut: top clock_mhz: 250 + lut_budget: 136762 configs: >- -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_CORES=2 @@ -101,6 +118,7 @@ builds: group: dxa dut: dxa clock_mhz: 300 + lut_budget: 10102 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- mufu -------------------------------------------------------------------- @@ -111,6 +129,7 @@ builds: group: mufu dut: mufu clock_mhz: 300 + lut_budget: 2030 configs: "-DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16" # --- tensor ------------------------------------------------------------------ @@ -121,6 +140,7 @@ builds: group: tensor dut: tcu clock_mhz: 300 + lut_budget: 117377 configs: >- -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_TCU_FP16_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_INT8_ENABLE @@ -138,6 +158,7 @@ builds: group: graphics dut: rtu clock_mhz: 300 + lut_budget: 67881 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Rasterizer: bin/block/quad walker, edge evaluators, rcache port. @@ -145,6 +166,7 @@ builds: group: graphics dut: raster clock_mhz: 300 + lut_budget: 8346 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Output merger: depth/stencil test, blend, ocache port. @@ -152,6 +174,7 @@ builds: group: graphics dut: om clock_mhz: 300 + lut_budget: 2618 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Texture unit: address generation, LOD, bilinear filter, tcache port. @@ -159,6 +182,7 @@ builds: group: graphics dut: tex clock_mhz: 300 + lut_budget: 61245 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- vm ---------------------------------------------------------------------- @@ -170,6 +194,7 @@ builds: group: vm dut: vm clock_mhz: 250 + lut_budget: 119695 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_CORES=2 # --- graphics (combined subsystem) ------------------------------------------- @@ -182,6 +207,7 @@ builds: group: graphics dut: gfx clock_mhz: 250 + lut_budget: 204871 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # --- tensor (combined subsystem) --------------------------------------------- @@ -193,9 +219,10 @@ builds: group: tensor dut: tensor clock_mhz: 250 + lut_budget: 239260 configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 # Route-dominated at this size: identical-area netlists swung 207-242 MHz - # under the default strategy (placement variance), which no 5% gate + # under the default strategy (placement variance), which no Fmax gate # tolerance survives. The worst arc is 80%+ route (L1 tag way-decode to the # data-store write enables), so the closure pick is the post-route physical # optimizer: 251.8 vs 244.2 for NetDelay_high on the same netlist. diff --git a/ci/testcases/graphics.yaml b/ci/testcases/graphics.yaml index c337b03306..252736abaf 100644 --- a/ci/testcases/graphics.yaml +++ b/ci/testcases/graphics.yaml @@ -693,4 +693,25 @@ tests: app: gfx_draw3d args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE - shape: {cores: 2, l2cache: true, threads: 4} \ No newline at end of file + shape: {cores: 2, l2cache: true, threads: 4} +# Flush ordering of OM aperture stores under random input backpressure +# (DBG_FLUSH_STALL): the last fragments' stores are often still in transit to +# the OM ingress when the kernel-boundary flush starts, so the flush must wait +# for them behind the dcache's flush tokens held at the OM steers. +- id: draw3d-flush-stress-1 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {threads: 16} +- id: draw3d-flush-stress-2 + via: blackbox + drivers: + - rtlsim + app: gfx_draw3d + args: -w 128 -h 128 -t box.cgltrace -r box_ref_128.png + configs: -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE -DDBG_FLUSH_STALL + shape: {cores: 2, l2cache: true, threads: 4} + tier: full diff --git a/ci/testcases/logic_gate.yaml b/ci/testcases/logic_gate.yaml new file mode 100644 index 0000000000..05909f5588 --- /dev/null +++ b/ci/testcases/logic_gate.yaml @@ -0,0 +1,264 @@ +# Logic-level regression gate (Beacon): the deepest end point of every RTL +# unit must not get deeper than a checked-in golden result +# (ci/baselines/synthesis/beacon/.json). See ci/logic_gate.py. +# +# Sibling of fpga_gate (fpga_gate.yaml) and asic_gate (asic_gate.yaml): same +# discipline -- golden baseline, results only ever re-recorded by a human -- +# over foldsim-estimate-timing instead of a synthesis tool. It counts logic +# levels as Vivado counts them on DEVICE's family (LUTs, CARRY8s, DSP cells, +# distributed-RAM reads) on the CIRCT netlist the Xilinx flow elaborates, so +# it gates depth, not Fmax: wires are not in it. +# +# - Per unit. A unit is an end point's instance path with generate indices +# folded (g_banks_3_bank -> g_banks_*_bank). Every unit at or above the +# watch threshold (8 levels at 300 MHz) is recorded; a unit that gets deeper +# fails, and so does a new unit at or above the fix threshold (12). +# - No tolerance. Beacon is deterministic, so any change in depth is a +# real one; a shallower unit reads IMPROVED, to be locked in by a re-record. +# - The tool versions (CIRCT, foldsim) are part of config_hash: a new model +# reads STALE, not as a regression. +# +# --------------------------------------------------------------------------- +# Two blocks, two consumers, as in the synthesis gates: +# +# tests: what pytest/CI sees -- ONE case, whose `run:` is literally the +# command CI executes. +# builds: what ci/logic_gate.py reads -- the build list. The pytest loader +# ignores this key. +# +# The builds are fpga_gate.yaml's, DUT for DUT and config for config (all but +# `top`, see below), so where both gates run a DUT they measure the same RTL, and a unit this gate flags +# can be read against the Fmax gate's critical paths. clock_mhz is the Fmax +# gate's target, recorded for that comparison; Beacon's thresholds are +# 300 MHz ones. +# +# Tier `timing` is OPT-IN (OPT_IN_TIERS in testcase.py): it needs foldsim, +# which the base toolchain does not install. .github/workflows/logic_gate.yml +# installs it and runs the gate. +# +# Run one by name: ci/logic_gate.py -b tcu +# By hand, no gate: make -C hw/syn/xilinx/dut tcu-beacon +# Re-record: ci/logic_gate.py --update-baseline (human-reviewed, never CI) +# --------------------------------------------------------------------------- +category: logic_gate +defaults: + # 32-bit only, as the synthesis gates. + xlen: + - 32 + tier: timing + via: script + # Any RTL change can move depth. + touches: + - hw/rtl/ +tests: +# Seconds to minutes per DUT: the whole catalog runs in one job. +- id: gate + run: ci/logic_gate.py -j 2 + +# The full-GPU builds (group gpu, below): ~30 min and ~95 GB of memory each, +# so one at a time, on a large self-hosted machine -- never a hosted runner. +# Its own opt-in tier keeps `--tier timing` from sweeping it in. +- id: gpu + tier: timing_gpu + run: ci/logic_gate.py -b gpu -j 1 --timeout 7200 + +# --------------------------------------------------------------------------- +# builds — read by ci/logic_gate.py, ignored by the pytest catalog loader. +# +# dut hw/syn/xilinx/dut/catalog.mk target to measure +# clock_mhz the Fmax gate's target for this DUT (recorded, not asserted) +# configs EXTRA defines, appended to the catalog's +# group which golden file the results land in +# xlen ISA width, when not the gate's default (32); the build runs in +# that width's own build tree +# profile hardware profile (profiles/.json) under the configs; +# configs win over it +# opt_in run only when selected by id or group, never by default +# interfaces modules whose outputs must be registered (space separated): +# an output bit a combinational path from outside the +# instance reaches is recorded, and one more than the +# baseline fails +# known_issue tracked expected failure: still measures and reports, does not +# fail the run +# --------------------------------------------------------------------------- +builds: + +# --- core: the base pipeline and its memory system --------------------------- + +# L1/LLC datapath with atomics: tag/data pipeline, MSHR, AMO unit. +- id: cache + group: core + dut: cache + clock_mhz: 300 + configs: -DVX_CFG_EXT_A_ENABLE + +# Wide-SIMD core: issue/operand-collect/LSU/FPU at NT=NW=16, plus the RVC +# decompressor and atomics. +- id: core + group: core + dut: core + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE + interfaces: VX_core + +# Warp-wide local memory with two LSU blocks: both pass engines, the +# dual-port banks the read engine reads through, and the atomic logic. +- id: lmem + group: core + dut: lmem + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_EXT_A_ENABLE + +# fpga_gate's `top` is not measured: CIRCT's import leaves the command +# processor's AXI-Lite response ports (VX_cp_core) as LLHD references, which +# no HW/Comb/Seq tool reads. It joins this list when the import lowers them. + +# --- dxa --------------------------------------------------------------------- + +# DXA engine: descriptor queue, address generation, multicast distribution. +- id: dxa + group: dxa + dut: dxa + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- tensor ------------------------------------------------------------------ + +# TFR tensor core with every datatype plus sparsity and WGMMA. +- id: tcu + group: tensor + dut: tcu + clock_mhz: 300 + configs: >- + -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_TCU_TYPE_TFR + -DVX_CFG_TCU_FP16_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_INT8_ENABLE + -DVX_CFG_TCU_INT4_ENABLE -DVX_CFG_TCU_TF32_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + +# --- graphics ---------------------------------------------------------------- + +# Ray-tracing unit: CW-BVH4 walker, box/triangle PEs, short stack. +- id: rtu + group: graphics + dut: rtu + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Rasterizer: bin/block/quad walker, edge evaluators, rcache port. +- id: raster + group: graphics + dut: raster + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Output merger: depth/stencil test, blend, ocache port. +- id: om + group: graphics + dut: om + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# Texture unit: address generation, LOD, bilinear filter, tcache port. +- id: tex + group: graphics + dut: tex + clock_mhz: 300 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- vm ---------------------------------------------------------------------- + +# Standalone MMU network: dTLB + iTLB, pooled L1 TLB MSHR, L2 TLB, PTW walker +# hierarchy, fed by a shared L2. The memory-translation bellwether; the rest of +# the extension config comes from hw/syn/xilinx/dut/vm/Makefile. +- id: vm + group: vm + dut: vm + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_NUM_CORES=2 + +# --- graphics (combined subsystem) ------------------------------------------- + +# Full graphics subsystem in one wrapper: raster + om + tex + rtu + dxa behind a +# shared L2, 2 cores / socket. Closes the graphics cross-unit boundaries that the +# per-unit DUTs above do not. Subsystem enables come from +# hw/syn/xilinx/dut/gfx/Makefile. +- id: gfx + group: graphics + dut: gfx + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# The same slice under VM: the unit MMUs in front of the graphics caches and the +# DXA, which neither gfx (VM off) nor vm (no units) builds. +- id: gfxvm + group: graphics + dut: gfxvm + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- tensor (combined subsystem) --------------------------------------------- + +# Full tensor subsystem in one wrapper: TCU (WGMMA + MX) + dxa behind a shared L2, +# 2 cores / socket. Enables come from hw/syn/xilinx/dut/tensor/Makefile. +- id: tensor + group: tensor + dut: tensor + clock_mhz: 250 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + +# --- gpu (opt-in): the whole processor with every feature on ------------------ +# +# The per-unit DUTs above miss what only the assembled GPU has: the +# cluster/socket/L2 crossbars at full width, the kernel launch path into every +# core, and the cross-unit boundaries with every extension on at once. Vivado +# needs 48 h and more on these; Beacon ~30 min. Four cores, not the products' +# 84-132: core count widens only the arbiters and crossbars -- the deepest path +# was the same 43 levels at 1, 2, 4 and 8 cores -- while memory grows with it +# (~95 GB at 4, 121 GB at 8). Run with `ci/logic_gate.py -b gpu -j 1`. + +# Everything the 32-bit machine has: every extension and tensor format, VM, +# DXA and DMA, the graphics pipeline, ray tracing with TLAS, a 4-wide +# frontend and issue. (D is 64-bit only.) +- id: gpu32_full + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + configs: >- + -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 + -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 + -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_FETCH_WIDTH=4 + -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE -DVX_CFG_EXT_ZACAS_ENABLE + -DVX_CFG_EXT_ZFHMIN_ENABLE -DVX_CFG_EXT_MUFU_ENABLE -DVX_CFG_VM_ENABLE + -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TF32_ENABLE -DVX_CFG_TCU_FP16_ENABLE + -DVX_CFG_TCU_FP8_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_INT8_ENABLE + -DVX_CFG_TCU_INT4_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE + -DVX_CFG_TCU_NVFP4_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_DSM_ENABLE -DVX_CFG_EXT_DXA_ENABLE -DVX_CFG_EXT_DMA_ENABLE + -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_RTU_TLAS_ENABLE + -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + -DVX_CFG_L2_ENABLE + +# RTX 5080's SM: 48 warps x 32 threads, the graphics pipeline, ray tracing, +# MX/FP4 tensor formats, a 32-bank L2 -- at XLEN 64 with D. +- id: gpu_rtx5080 + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + xlen: 64 + profile: rtx5080 + configs: -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 + +# H100's SM: 64 warps and WGMMA, which the RTX 5080 profile does not have. +- id: gpu_h100 + group: gpu + opt_in: true + dut: vortex + clock_mhz: 300 + interfaces: VX_core VX_socket VX_cluster + xlen: 64 + profile: h100 + configs: -DVX_CFG_NUM_CORES=4 -DVX_CFG_NUM_CLUSTERS=1 diff --git a/ci/testcases/mufu.yaml b/ci/testcases/mufu.yaml index c7f70873b9..42f075d0a1 100644 --- a/ci/testcases/mufu.yaml +++ b/ci/testcases/mufu.yaml @@ -8,7 +8,7 @@ defaults: - 32 - 64 tier: smoke - touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] + touches: [hw/rtl/mufu, hw/rtl/core/VX_sfu_unit.sv, hw/rtl/core/VX_sfu_block.sv, sim/simx/mufu_unit.cpp, sim/simx/sfu_unit.cpp, sw/common/mufu.cpp, sw/common/mufu_tables.h, sw/kernel/include/vx_mufu.h] tests: - id: mufu-1 via: blackbox diff --git a/ci/testcases/regression.yaml b/ci/testcases/regression.yaml index 50e45420c0..3ad1590526 100644 --- a/ci/testcases/regression.yaml +++ b/ci/testcases/regression.yaml @@ -18,6 +18,163 @@ tests: - rtlsim dir: tests/regression target: run-{driver} +# Local memory: shared-word reads (one lane to the whole warp, and lanes of +# different LSU blocks), partial masks, bank-conflict replays, sub-word loads +# and stores, same-word stores, and per-lane order against the lane's own +# stores and atomics. +- id: lmem-access-1 + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_access + shape: + threads: 16 +# Two LSU blocks: the read engine on the banks' second port. +- id: lmem-access-2 + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_access + configs: -DVX_CFG_ISSUE_WIDTH=2 + shape: + threads: 32 + warps: 8 +# With atomics: a warp's reads around its own atomic adds to the word. +- id: lmem-access-amo + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_access + configs: -DVX_CFG_EXT_A_ENABLE + shape: + threads: 16 +# Atomics with two LSU blocks: the read engine drains before an atomic enters +# the first engine. +- id: lmem-access-amo-iw2 + via: blackbox + drivers: + - simx + - rtlsim + app: lmem_access + configs: -DVX_CFG_EXT_A_ENABLE -DVX_CFG_ISSUE_WIDTH=2 + shape: + threads: 16 +# Wide loads and stores (vx_flw.d/.q, vx_fsw.d/.q) on local and global memory: +# lane patterns, register groups, word alignments and the offset limits. +- id: ldst-wide-1 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_wide + shape: + threads: 16 +# Two LSU blocks. +- id: ldst-wide-2 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_wide + configs: -DVX_CFG_ISSUE_WIDTH=2 + shape: + threads: 32 + warps: 8 +# Fewer LSU lanes than SIMD lanes: each uop's request goes out in four packets. +- id: ldst-wide-3 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_wide + configs: -DVX_CFG_NUM_LSU_LANES=4 + shape: + threads: 16 +# One SIMD lane: the dispatcher splits every warp into single-lane packets. +- id: ldst-wide-4 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_wide + configs: -DVX_CFG_SIMD_WIDTH=1 + shape: + threads: 4 +# Matrix fragment loads and stores (vx_ldm, vx_stm): every shape of the +# configuration, local and global memory, row strides that are not powers of +# two, zero/positive/negative offsets, aligned and unaligned register bases. +- id: ldst-matrix-1 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI +# Symmetric sparse geometry (4 lanes). +- id: ldst-matrix-2 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE +# WGMMA accumulator chunks (16 lanes). +- id: ldst-matrix-3 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_WGMMA_ENABLE +# Each uop split into 4-lane packets, symmetric sparse shapes included. +- id: ldst-matrix-4 + via: blackbox + drivers: + - simx + - rtlsim + app: ldst_matrix + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_NUM_LSU_LANES=4 +# The sgemm benchmarks with each thread's consecutive words read and written +# through the wide forms (LDST_WIDE=1). +- id: ldst-wide-sgemm + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm-v1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_v1 + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemmx + via: blackbox + drivers: + - simx + - rtlsim + app: sgemmx + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2 + args: -n32 -t4 + configs: -DLDST_WIDE=1 +- id: ldst-wide-sgemm2-v1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_v1 + configs: -DLDST_WIDE=1 - id: occupancy-1 via: blackbox drivers: @@ -90,6 +247,57 @@ tests: app: gbar_phase shape: cores: 2 +# Four cores in one cluster: every core must arrive under its own index. +- id: gbar-phase-3 + via: blackbox + drivers: + - xrt + app: gbar_phase + shape: + cores: 4 +# A global barrier spans clusters: the device-level unit counts every core, so +# one core per cluster still meets the count and two clusters of two cores do +# not release on a cluster's own arrivals. +- id: gbar-phase-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 1 +- id: gbar-phase-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: gbar_phase + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +- id: async-gbarrier-cl2-1 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 1 +- id: async-gbarrier-cl2-2 + via: blackbox + drivers: + - simx + - rtlsim + app: async_gbarrier + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true # SimX<->rtlsim cycle parity on the multi-core global barrier. Until this case # existed the gate could not observe the gbar path at all: every other parity # case is single-core, so USE_GBAR is 0 and the request queue is not built. @@ -105,6 +313,16 @@ tests: app: async_gbarrier shape: cores: 2 +# Two cores sharing one socket's instruction and data caches: their requests +# and responses cross each cache's input arbiter. +- id: model_parity-async-gbarrier-ss2 + check: model_parity + via: blackbox + app: async_gbarrier + configs: -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 2 + l2cache: true - id: bar-slot-phase-1 via: blackbox @@ -155,3 +373,277 @@ tests: drivers: - simx app: module_reload +# Kernel-boundary flush coherence: back-to-back launches with host rewrites, +# rotating line ownership across cores and cross-launch read-modify-writes. +# The shapes cover a write-back L1, a write-back L2 shared by 4 cores, and a +# write-back L3 shared by two clusters, where every core flushes at once. +- id: flush_coherence-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence +# The xrt leg runs the RTL command processor's flush (broadcast write, then +# per-core reads). +- id: flush_coherence-2 + via: blackbox + drivers: + - simx + - xrt + app: flush_coherence + shape: + cores: 4 + l2cache: true +- id: flush_coherence-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# A memory fence carries data from one core to another across the caches they +# do not share: the producer's token reaches the point of coherence, the +# consumer's invalidates its copies above it. Two clusters with an L2 each +# replicate the last level, so the fence must walk both L2s there. +- id: fence_mp-1 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp +- id: fence_mp-2 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + shape: + cores: 2 + l2cache: true +- id: fence_mp-3 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 1 + l2cache: true +- id: fence_mp-4 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# A release must drain: bank 0's writes are held back by the debug delay, and +# the flag, in a bank-1 line, overtakes the data in bank-0 lines unless the +# release fence (-m1) or the .rl atomic (-m2) waits for them. +- id: fence_mp-5 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m1 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true +- id: fence_mp-6 + via: blackbox + drivers: + - simx + - rtlsim + app: fence_mp + args: -n64 -m2 + configs: -DVX_CFG_EXT_A_ENABLE -DDBG_MEM_BANK0_WDELAY + shape: + cores: 2 + l2cache: true +# Kernel to kernel with no host transfer between (one command batch): the +# launch flushes are L1-only, so the last-level cache alone carries each +# round's writes to the next round, and the full flush runs before readback. +- id: flush_coherence-chain-1 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c +- id: flush_coherence-chain-2 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +- id: flush_coherence-chain-3 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true +# Replicated last-level caches (per-core dcaches without an L2, per-cluster L2s +# without an L3) are not a point of coherence, so an L1-only flush must still +# invalidate them. +- id: flush_coherence-chain-4 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + cores: 2 +- id: flush_coherence-chain-5 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c + shape: + clusters: 2 + cores: 2 + l2cache: true +# Each round loads its input with a DXA tile copy, which reaches memory beside +# the single dcache, so that dcache is not a point of coherence and must still +# be written back on an L1-only flush. +- id: flush_coherence-chain-6 + via: blackbox + drivers: + - simx + - rtlsim + app: flush_coherence + args: -n2048 -r16 -c -x + configs: -DVX_CFG_EXT_DXA_ENABLE +# The command processor's DMA as a client of the shared L2: host transfers +# are timed through the L2, and each launch waits for the DMA's fence. +- id: model_parity-dogfood-cp-llc + check: model_parity + via: blackbox + app: dogfood + args: -n16 -xbar -xgbar + shape: + cores: 4 + l2cache: true +# The same with the DMA off the L2, reaching memory on its own path. +- id: model_parity-dogfood-cp-direct + check: model_parity + via: blackbox + app: dogfood + args: -n16 -xbar -xgbar + configs: -DVX_CFG_CP_LLC_CLIENT=0 + shape: + cores: 4 + l2cache: true +# Eight L1 ports per core and the DMA's slot give the shared L2 2^k + 1 +# ports, so each bank's port arbiter is a tree whose last slice has one port: +# 17 ports on two cores, and 65 on eight, past a 64-bit port mask. +- id: model_parity-sgemm-cp-llc-17 + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 2 + l2cache: true +- id: sgemm-cp-llc-65 + via: blackbox + drivers: + - simx + app: sgemm + args: -n32 + configs: -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=8 + shape: + cores: 8 + l2cache: true +# More dcache banks and memory ports than core requests: the uncached path keeps +# each core request on its own memory port. +- id: model_parity-sgemm-dcache-16-banks + check: model_parity + via: blackbox + app: sgemm + args: -n32 + configs: -DVX_CFG_DCACHE_NUM_BANKS=16 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=16 +# The shared L2 keeps its lines across the L1-only launch flushes. +- id: model_parity-flush-chain + check: model_parity + via: blackbox + app: flush_coherence + args: -n2048 -r16 -c + configs: -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true +# Flush ordering under random input backpressure (DBG_FLUSH_STALL): each round +# ends on a store burst, so a core's last writes are often still queued on one +# memory port or dcache lane while its token on another reaches the next level. +# Without a token per path these shapes lose those writes. +- id: flush_coherence-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + cores: 4 + l2cache: true +- id: flush_coherence-stress-2 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 + configs: -DDBG_FLUSH_STALL + shape: + clusters: 2 + cores: 2 + l2cache: true + l3cache: true + tier: full +# Cores sharing a dcache: a lost lane store is only visible when it belongs to +# the core answered last, so this shape needs more rounds to expose it. +- id: flush_coherence-stress-3 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r24 -b16 + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=2 + shape: + cores: 4 + l2cache: true + tier: full +- id: flush_coherence-chain-stress-1 + via: blackbox + drivers: + - rtlsim + app: flush_coherence + args: -n2048 -r8 -b16 -c + configs: -DDBG_FLUSH_STALL -DVX_CFG_SOCKET_SIZE=1 + shape: + cores: 4 + l2cache: true diff --git a/ci/testcases/tensor.yaml b/ci/testcases/tensor.yaml index 062a8dd331..ff94ff8a3b 100644 --- a/ci/testcases/tensor.yaml +++ b/ci/testcases/tensor.yaml @@ -31,7 +31,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=bf16 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=fp32 - id: sgemm_tcu-5 via: blackbox drivers: @@ -43,7 +43,7 @@ tests: drivers: - simx app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=bf8 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=bf8 -DOTYPE=fp32 - id: sgemm_tcu-7 via: blackbox drivers: @@ -123,13 +123,13 @@ tests: drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 - id: sgemm_tcu-19 via: blackbox drivers: - rtlsim app: sgemm_tcu - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_ISSUE_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=4 -DPROFILE_ENABLE - id: sgemm_tcu-20 via: blackbox @@ -148,7 +148,7 @@ tests: drivers: - rtlsim app: sgemm2_tcu - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 -DITYPE=fp16 -DOTYPE=fp32 args: -w 4 @@ -193,6 +193,9 @@ tests: check: model_parity via: blackbox app: sgemm_tcu + # Sized like its perf_gate twin: at the default 64^3 the grid is four waves of + # CTAs, so one CTA running long moves the per-core split and the cycle count. + args: -m 128 -n 128 -k 128 configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 shape: {cores: 2, l2cache: true} @@ -226,4 +229,76 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# Fragments moved with vx_ldm/vx_stm (LDST_MATRIX=1), alone and with the K loop +# unrolled (KUNROLL), whose tile offsets the instructions take as immediates. +- id: sgemm_tcu-ldm-1 + via: blackbox + drivers: + - simx + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=bf16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-3 + via: blackbox + drivers: + - simx + app: sgemm_tcu + args: -m 64 -n 64 -k 128 + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=fp8 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-4 + via: blackbox + drivers: + - rtlsim + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu-ldm-5 + via: blackbox + drivers: + - rtlsim + app: sgemm_tcu + args: -m 32 -n 32 -k 96 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm2_tcu-ldm + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_NUM_SFU_BLOCKS=4 + -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + args: -w 4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-fp16-ldm + check: model_parity + via: blackbox + app: sgemm_tcu + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-fp16-ldm-u4 + check: model_parity + via: blackbox + app: sgemm_tcu + args: -m 64 -n 64 -k 128 + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=4 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-sgemm2-fp16-ldm + check: model_parity + via: blackbox + app: sgemm2_tcu + args: -m 128 -n 128 -k 64 + # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is + # several times the size, and a hosted CI runner cannot build it in a job. + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_mx.yaml b/ci/testcases/tensor_mx.yaml index 9a90236244..7662bacd36 100644 --- a/ci/testcases/tensor_mx.yaml +++ b/ci/testcases/tensor_mx.yaml @@ -247,6 +247,22 @@ tests: configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE -DITYPE=nvfp4 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI -DTCU_MX_TLS args: "-k32" +# An LSU narrower than the warp: TCU_LD fetches the scale factors in several passes. +- id: sgemm_tcu_mx-lsu-lanes + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k32" +- id: model_parity-mxfp8-lsu-lanes + check: model_parity + via: blackbox + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI + args: "-k32" + # SimX<->rtlsim cycle parity on the MX datapath: same config on both drivers, # cycles must agree within tolerance (see core.yaml). - id: model_parity-mxfp8 @@ -265,4 +281,23 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DVX_CFG_TCU_TYPE_DPI # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# MX fragments moved with vx_ldm/vx_stm (LDST_MATRIX=1). +- id: sgemm_tcu_mx-ldm-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp8 -DOTYPE=fp32 -DLDST_MATRIX=1 + args: -k64 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_mx-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_mx + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE -DITYPE=mxfp4 -DOTYPE=fp32 -DLDST_MATRIX=1 -DKUNROLL=2 + args: -k64 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_sp.yaml b/ci/testcases/tensor_sp.yaml index 8457299edb..1b4d69a159 100644 --- a/ci/testcases/tensor_sp.yaml +++ b/ci/testcases/tensor_sp.yaml @@ -32,6 +32,15 @@ tests: - simx app: sgemm_tcu_sp configs: -DVX_CFG_NUM_THREADS=2 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TF32_ENABLE -DITYPE=tf32 -DOTYPE=fp32 +# A long K: the fp32 tolerance grows with the accumulation chain. +- id: sgemm_tcu_sp-k256 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 + args: -k256 - id: sgemm_tcu_sp-5 via: blackbox drivers: @@ -250,6 +259,28 @@ tests: app: sgemm_tcu_sp configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_TF32_ENABLE -DITYPE=tf32 -DOTYPE=fp32 +# An LSU narrower than the warp: TCU_LD fetches the metadata in several passes, +# through a shared AGU port in the second case. +- id: sgemm_tcu_sp-lsu-lanes-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 +- id: sgemm_tcu_sp-lsu-lanes-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 +- id: model_parity-sparse-lsu-lanes + check: model_parity + via: blackbox + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_LSU_LANES=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE + # SimX<->rtlsim cycle parity on the native sparse fp16 datapath: same config on # both drivers, cycles must agree within tolerance (see core.yaml). - id: model_parity-sparse-fp16 @@ -267,4 +298,27 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# Sparse fragments moved with vx_ldm (LDST_MATRIX=1). +- id: sgemm_tcu_sp-ldm-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_TYPE_DPI -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_sp-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-sp-ldm + check: model_parity + via: blackbox + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/tensor_wg.yaml b/ci/testcases/tensor_wg.yaml index a08f6cc835..9b7787be1c 100644 --- a/ci/testcases/tensor_wg.yaml +++ b/ci/testcases/tensor_wg.yaml @@ -44,14 +44,14 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-6 via: blackbox drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-7 via: blackbox @@ -72,7 +72,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_SS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg-10 via: blackbox @@ -116,7 +116,7 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=4 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-5 via: blackbox @@ -130,8 +130,25 @@ tests: drivers: - rtlsim app: sgemm_tcu_wg_sp - configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + configs: -DVX_CFG_NUM_THREADS=32 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_FETCH_WIDTH=2 -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=8 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +# WMMA_SP in a WGMMA build: at NT 4 and 16 it keeps the symmetric layout. +- id: sgemm_tcu_sp-wgmma-1 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 +- id: sgemm_tcu_sp-wgmma-2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm_tcu_sp + configs: -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE + -DVX_CFG_TCU_SPARSE_ENABLE -DVX_CFG_TCU_INT8_ENABLE -DITYPE=int8 -DOTYPE=int32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - id: sgemm_tcu_wg_sp-7 via: blackbox drivers: @@ -232,8 +249,9 @@ tests: args: -m64 -n64 -k64 # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is # several times the size, and a hosted CI runner cannot build it in a job. + # Two frontend slices: covers the frontend crossbar on both models. configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 - -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS + -DVX_CFG_TCU_FEDP2K -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_FETCH_WIDTH=2 # Cooperative sparse WGMMA (default fp16 operands, 2:4 structured sparsity). # Not warp-width swept: the tensor-core fragment layout is defined over a fixed @@ -248,4 +266,29 @@ tests: configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_SPARSE_ENABLE -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 # Not warp-width swept: the tensor-core fragment layout is defined over a fixed - # lane count, which the configs pin. \ No newline at end of file + # lane count, which the configs pin. + +# The WGMMA accumulator stored with vx_stm in 8-register chunks (LDST_MATRIX=1). +- id: sgemm_tcu_wg-ldm-1 + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=32 -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: sgemm_tcu_wg-ldm-2 + via: blackbox + drivers: + - simx + app: sgemm_tcu_wg + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_TCU_FEDP2K -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=8 -DWGMMA_RS -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DLDST_MATRIX=1 -DKUNROLL=2 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] +- id: model_parity-wgmma-ldm + check: model_parity + via: blackbox + app: sgemm_tcu_wg + args: -m64 -n64 -k64 + # One execution block per unit: the four-block model ISSUE_WIDTH=4 implies is + # several times the size, and a hosted CI runner cannot build it in a job. + configs: -DVX_CFG_NUM_THREADS=8 -DVX_CFG_NUM_WARPS=8 -DVX_CFG_ISSUE_WIDTH=4 -DVX_CFG_EXT_TCU_ENABLE -DVX_CFG_TCU_WGMMA_ENABLE -DVX_CFG_NUM_ALU_BLOCKS=1 -DVX_CFG_NUM_LSU_BLOCKS=1 -DVX_CFG_NUM_FPU_BLOCKS=1 -DVX_CFG_TCU_TYPE_DPI -DITYPE=fp16 -DOTYPE=fp32 -DWGMMA_NRC=16 -DLDST_MATRIX=1 + touches: [hw/rtl/tcu, hw/rtl/core/VX_uop_packld.sv, hw/rtl/core/VX_decode.sv, sim/simx/tcu, sim/simx/lsu_unit.cpp, sim/simx/decode.cpp, sw/kernel/include/vx_tensor.h] diff --git a/ci/testcases/unittest.yaml b/ci/testcases/unittest.yaml index be00b76514..44055d72f5 100644 --- a/ci/testcases/unittest.yaml +++ b/ci/testcases/unittest.yaml @@ -23,14 +23,46 @@ tests: - id: hw-tcu via: script run: "make -C hw/unittest run-tcu" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] + # VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit-exact + # against each datapath's reference: native TFR against the TfrFedp reference + # model, DPI against its discrete mul/add. Then the native FEDP's + # fp8 and block-scaled (MX/NVFP4) formats against the same model. The `hw` + # lane only builds these. + - id: hw-tcu-unit + via: script + run: "make -C hw/unittest run-tcu-unit" + touches: [hw/rtl/tcu, hw/dpi, hw/unittest/tcu_unit, hw/unittest/tcu_fedp, sw/common/fedp.h] # Same native-TFR formats with the multipliers mapped onto DSP48 (USE_DSP=1) — # the FPGA-synthesis path, off by default in sim. Bit-identical to LUT by # contract, so a mismatch is a DSP-packing bug. - id: hw-tcu-dsp via: script run: "make -C hw/unittest run-tcu-dsp" - touches: [hw/rtl/tcu, hw/unittest/tcu_fedp] + touches: [hw/rtl/tcu, hw/unittest/tcu_fedp, sw/common/fedp.h] + # The command processor's DMA port into the last-level cache: a launch behind + # DMA writes must wait for them to reach the cache. No runtime path launches + # right behind a DMA write, so no system test observes this ordering. + - id: hw-cp-llc-port + via: script + run: "make -C hw/unittest/cp_llc_port run" + touches: [hw/rtl/mem/VX_cp_llc_port.sv, hw/unittest/cp_llc_port] + # Stream arbiters over more than 1.5x MAX_FANOUT requests become trees of + # slices; a last slice narrower than the arbiter's other side must keep every + # request on its own output and select. The system shapes reach only a few of + # these slice shapes. + - id: hw-stream-arb + via: script + run: "make -C hw/unittest/stream_arb run" + touches: [hw/rtl/libs/VX_stream_arb.sv, hw/unittest/stream_arb] + # The ID-routed AXI arbiter shared by the AFU and the command processor: + # write data must reach the slave in address order whichever channel a + # master offers first, and a slave that takes data before the address must + # not deadlock it. xrtsim's always-ready slave exercises neither. + - id: hw-mm-axi-arb + via: script + run: "make -C hw/unittest/mm_axi_arb run" + touches: [hw/rtl/libs/VX_mm_axi_arb.sv, hw/unittest/mm_axi_arb] # Texel blend (VX_tex_lerp, both forms the sampler uses) against the C model # the SW sampler and SimX share, and the exact Vulkan f/256 blend, over every # input. diff --git a/ci/testcases/vm.yaml b/ci/testcases/vm.yaml index 2c4d2b7c2b..89ff412191 100644 --- a/ci/testcases/vm.yaml +++ b/ci/testcases/vm.yaml @@ -235,6 +235,67 @@ tests: # above) are reported, via the shared walker. Re-enable when L1 permission-fault # reporting is added. +# Two SFU blocks: every warp writes SATP at entry, from either block. +- id: vm-sfu2 + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm + configs: -DVX_CFG_VM_ENABLE -DVX_CFG_ISSUE_WIDTH=2 -DVX_CFG_NUM_SFU_BLOCKS=2 + +# Socket unit MMU: the RTU's BVH fetches and the DXA's global reads translate, +# so a scene or a tile source at a minted VA (!= PA) is read correctly (both +# apps check their results; dxa_copy does not). +- id: vm-rtu + via: blackbox + drivers: + - simx + - rtlsim + app: rt_smoke + configs+: -DVX_CFG_EXT_RTU_ENABLE + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-dxa + via: blackbox + drivers: + - simx + - rtlsim + app: sgemm2_dxa + configs+: -DVX_CFG_EXT_DXA_ENABLE + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-tex + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_tex + configs+: -DVX_CFG_EXT_TEX_ENABLE + args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-raster + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_raster + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png + env: + VORTEX_RANDOMIZE_VA: 1 +- id: vm-om + via: blackbox + drivers: + - simx + - rtlsim + app: gfx_om + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -d -b -r whitebox_db_128.png + env: + VORTEX_RANDOMIZE_VA: 1 + # --------------------------------------------------------------------------- # check: model_parity — same VM-enabled app on both drivers (xlen 32); exact # instret + cycles within tolerance. SimX is the timing model of the RTL. @@ -249,3 +310,50 @@ tests: via: blackbox app: vecadd args: -n16384 +- id: parity-vm-rt_raycast + check: model_parity + via: blackbox + app: rt_raycast + configs+: -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_DXA_ENABLE +- id: parity-vm-dxa_copy + check: model_parity + via: blackbox + app: dxa_copy + configs+: -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_DXA_ENABLE + args: -d3 +- id: parity-vm-gfx_tex + check: model_parity + via: blackbox + app: gfx_tex + configs+: -DVX_CFG_EXT_TEX_ENABLE + args: -i toad.png -r toad_ref_f0.png -f 0 -g 0 +- id: parity-vm-gfx_raster + check: model_parity + via: blackbox + app: gfx_raster + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -t triangle.cgltrace -r triangle_ref_128.png +- id: parity-vm-gfx_om + check: model_parity + via: blackbox + app: gfx_om + configs+: -DVX_CFG_EXT_RASTER_ENABLE -DVX_CFG_EXT_OM_ENABLE + args: -w 128 -h 128 -d -b -r whitebox_db_128.png +# The RTL command processor's DMA walks the page table for every upload and +# download; only xrt runs it (simx and rtlsim use the C++ command processor). +- id: cp-dma-walk + via: blackbox + drivers: + - xrt + app: vm_test + args: -t1 -n16 -p1 + env: + VORTEX_RANDOMIZE_VA: 1 +- id: cp-dma-walk-megapage + via: blackbox + drivers: + - xrt + app: sgemm + args: -n16 + env: + VORTEX_VM_MEGAPAGE_HEAP: 1 diff --git a/ci/toolchain_install.sh.in b/ci/toolchain_install.sh.in index ac6f49e943..e2df33253c 100755 --- a/ci/toolchain_install.sh.in +++ b/ci/toolchain_install.sh.in @@ -163,12 +163,36 @@ verilator() mkdir -p $TOOLDIR && rm -rf $TOOLDIR/verilator && mv verilator $TOOLDIR rm -rf verilator verilator.tar.bz2* } -sv2v() +# CIRCT: the SystemVerilog frontend of the synthesis flows (circt-verilog, +# circt-opt, firtool), packaged in parts as verilator is. +circt() { - wget $REPOSITORY/sv2v/$OSVERSION/sv2v.tar.bz2 - tar -xvf sv2v.tar.bz2 - mkdir -p $TOOLDIR && rm -rf $TOOLDIR/sv2v && mv sv2v $TOOLDIR - rm -rf sv2v sv2v.tar.bz2 + # The count of what ci/toolchain_prebuilt.sh's `split -b 50M` produced for + # this release, as for verilator: re-check it whenever the circt prebuilt + # is refreshed. v3.0.1's bundle is 51.2 MB, one part. + case $OSVERSION in + *) parts=$(eval echo {a..a}) ;; + esac + echo $parts + rm -f circt.tar.bz2.parta* + for x in $parts + do + wget $REPOSITORY/circt/$OSVERSION/circt.tar.bz2.parta$x + done + cat circt.tar.bz2.parta* > circt.tar.bz2 + tar -xvf circt.tar.bz2 + mkdir -p $TOOLDIR && rm -rf $TOOLDIR/circt && mv circt $TOOLDIR + rm -rf circt circt.tar.bz2* +} +# Foldsim: the FAME-5 simulator compiler of hw/syn/foldsim (the Foldsim +# driver and foldsim-opt), built on CIRCT. +foldsim() +{ + rm -f foldsim.tar.bz2 + wget $REPOSITORY/foldsim/$OSVERSION/foldsim.tar.bz2 + tar -xvf foldsim.tar.bz2 + mkdir -p $TOOLDIR && rm -rf $TOOLDIR/foldsim && mv foldsim $TOOLDIR + rm -rf foldsim foldsim.tar.bz2 } yosys() @@ -218,8 +242,8 @@ gem5() rm -rf gem5 gem5.tar.bz2 } -# Every component except gem5 — the default when the script is run with no -# arguments, and the shared body of --all. +# Every component but the optional ones, Foldsim and gem5 — the default when +# the script is run with no arguments, and the shared body of --all. install_base() { pocl @@ -232,7 +256,7 @@ install_base() libc64 riscv32 riscv64 - sv2v + circt yosys sta mesa @@ -244,13 +268,13 @@ install_base() show_usage() { echo "Install Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--sv2v] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" - echo " (no arguments: install all components except gem5)" - echo " gem5 is opt-in: it is a large source build." + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo " (no arguments: install all components except Foldsim and gem5)" + echo " Foldsim and gem5 are opt-in: Foldsim serves only the Foldsim flow, and" + echo " gem5 is a large source build. --all installs them too." } -# Default with no arguments: install every component except gem5, which is -# opt-in because it is large. +# Default with no arguments: install every component except the opt-in ones. if [ "$#" -eq 0 ]; then install_base exit 0 @@ -280,7 +304,9 @@ while [ "$1" != "" ]; do ;; --libc64 ) libc64 ;; - --sv2v ) sv2v + --circt ) circt + ;; + --foldsim ) foldsim ;; --yosys ) yosys ;; @@ -289,6 +315,7 @@ while [ "$1" != "" ]; do --gem5 ) gem5 ;; --all ) install_base + foldsim gem5 ;; -h | --help ) show_usage diff --git a/ci/toolchain_prebuilt.sh.in b/ci/toolchain_prebuilt.sh.in index a720eae002..da17faefe2 100755 --- a/ci/toolchain_prebuilt.sh.in +++ b/ci/toolchain_prebuilt.sh.in @@ -124,12 +124,23 @@ verilator() rm verilator.tar.bz2 } -sv2v() +circt() { - echo "prebuilt sv2v..." - tar -C $TOOLDIR -cvjf sv2v.tar.bz2 sv2v - mkdir -p ./sv2v/$OSVERSION - mv sv2v.tar.bz2 ./sv2v/$OSVERSION + echo "prebuilt circt..." + tar -C $TOOLDIR -cvjf circt.tar.bz2 circt/bin/circt-verilog circt/bin/circt-opt circt/bin/firtool + split -b 50M circt.tar.bz2 "circt.tar.bz2.part" + mkdir -p ./circt/$OSVERSION + mv circt.tar.bz2.part* ./circt/$OSVERSION + rm circt.tar.bz2 +} + +# Foldsim's driver and foldsim-opt (ci/foldsim_install.sh), one tarball. +foldsim() +{ + echo "prebuilt foldsim..." + tar -C $TOOLDIR -cvjf foldsim.tar.bz2 foldsim/bin/foldsim foldsim/bin/foldsim-opt + mkdir -p ./foldsim/$OSVERSION + mv foldsim.tar.bz2 ./foldsim/$OSVERSION } yosys() @@ -172,7 +183,7 @@ gem5() show_usage() { echo "Setup Pre-built Vortex Toolchain" - echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--sv2v] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" + echo "Usage: $0 [--pocl] [--chipstar] [--mesa] [--verilator] [--riscv32] [--riscv64] [--llvm] [--libcrt32] [--libcrt64] [--libc32] [--libc64] [--circt] [--foldsim] [--yosys] [--sta] [--gem5] [--all] [-h|--help]" } while [ "$1" != "" ]; do @@ -199,7 +210,9 @@ while [ "$1" != "" ]; do ;; --libc64 ) libc64 ;; - --sv2v ) sv2v + --circt ) circt + ;; + --foldsim ) foldsim ;; --yosys ) yosys ;; @@ -217,7 +230,8 @@ while [ "$1" != "" ]; do libcrt64 libc32 libc64 - sv2v + circt + foldsim yosys sta mesa diff --git a/ci/v80_preflight.sh b/ci/v80_preflight.sh new file mode 100755 index 0000000000..bbd9f7f89a --- /dev/null +++ b/ci/v80_preflight.sh @@ -0,0 +1,66 @@ +#!/bin/bash +# Refuses to let a run program an Alveo V80 that is not in a state to take it. +# +# Every host reset seen with this card followed one of these: a design write to +# a card that was off the bus or unhealthy, a write while another process still +# had it open, or two writes back to back. Run before anything that programs the +# card (run-firesim TARGET=hw does); it exits non-zero, saying why, instead. +# +# VRT_DEVICE_BDF the card to check (bus:dev, e.g. 01:00); found on the bus if unset +# V80_COOLDOWN_S seconds to wait after the last design write (default 60) + +set -u +COOLDOWN=${V80_COOLDOWN_S:-60} + +fail() { echo "v80_preflight: $*" >&2; exit 3; } + +# The card: PF0 of a V80 (10ee:50b4), or the one named. +BDF=${VRT_DEVICE_BDF:-} +if [ -z "$BDF" ]; then + BDF=$(lspci -D -d 10ee:50b4 2>/dev/null | head -1 | cut -d' ' -f1) + [ -n "$BDF" ] || fail "no V80 (10ee:50b4) on the PCIe bus; if it dropped off, the host needs a cold power cycle" + BDF=${BDF%.*} +fi +case "$BDF" in *:*:*) ;; *) BDF="0000:$BDF" ;; esac + +# All three physical functions, and the port above them. +NFUNC=$(lspci -D -s "$BDF" 2>/dev/null | wc -l) +[ "$NFUNC" -eq 3 ] || fail "$BDF shows $NFUNC of its 3 functions on the bus" +PORT=$(basename "$(dirname "$(readlink -f /sys/bus/pci/devices/$BDF.0)")") +[ -e "/sys/bus/pci/devices/$PORT" ] || fail "the root port above $BDF ($PORT) is not present" + +# The board as the shell and vrtd see it. +SHORT=${BDF#0000:} +LIST=$(timeout 20 v80-smi list 2>&1) +echo "$LIST" | grep -q "Board 0000:$SHORT OK (PF0: OK) (PF1: OK) (PF2: OK) (VRTD: OK)" \ + || fail "v80-smi does not report $BDF healthy: +$LIST" + +# Nobody else holding a VRT session: a design write under a live user is what +# takes the card down. +USERS="" +for m in /proc/[0-9]*/maps; do + pid=${m#/proc/}; pid=${pid%/maps} + [ "$pid" = "$$" ] && continue + grep -q 'libvrt\.so' "$m" 2>/dev/null && USERS="$USERS $pid" +done +[ -z "$USERS" ] || fail "processes still hold VRT sessions:$(ps -o pid=,etime=,args= -p ${USERS// /,} | sed 's/^/\n /')" + +# A cool-down since the last design write. The journal is the only record of +# it; a user who cannot read vrtd's entries there cannot have the wait judged, +# which is reported rather than passed off as "no recent write". +JOURNAL=$(journalctl -t vrtd --since "-$((COOLDOWN + 5))s" -o short-unix --no-pager 2>&1) +if echo "$JOURNAL" | grep -qiE "insufficient permissions|no journal files"; then + echo "v80_preflight: cannot read the vrtd journal; not enforcing the ${COOLDOWN}s cool-down" >&2 +else + LAST=$(echo "$JOURNAL" | grep "Design write completed" | tail -1 | cut -d' ' -f1 | cut -d. -f1) + if [ -n "$LAST" ]; then + WAIT=$(( LAST + COOLDOWN - $(date +%s) )) + if [ "$WAIT" -gt 0 ]; then + echo "v80_preflight: last design write was $(( COOLDOWN - WAIT ))s ago; waiting ${WAIT}s" + sleep "$WAIT" + fi + fi +fi + +echo "v80_preflight: $BDF ready (port $PORT)" diff --git a/config.mk.in b/config.mk.in index 1dc8f4efbd..b7de36b210 100644 --- a/config.mk.in +++ b/config.mk.in @@ -42,6 +42,14 @@ RISCV_TOOLCHAIN_PATH ?= $(TOOLDIR)/riscv$(XLEN)-gnu-toolchain # and a path derived from VORTEX_HOME would make the dependency's location a # function of which tree is being built. FIRESIM_PATH ?= $(TOOLDIR)/firesim +# vortexgpgpu/firesim commit this tree is built and tested against (VERSION). +FIRESIM_REV := @FIRESIM_REV@ +# A Foldsim installation (ci/foldsim_install.sh): the compiler that builds a +# FAME-5 simulator of Vortex (hw/syn/foldsim), and the FPGA shell the simulator +# runs in, which the image, the in-process simulator (sim/foldsim) and the +# runtime (sw/runtime/foldsim) all take from it. +FOLDSIM_PATH ?= $(TOOLDIR)/foldsim +FOLDSIM_SHELL ?= $(FOLDSIM_PATH)/share/foldsim/shell RISCV_PREFIX ?= riscv$(XLEN)-unknown-elf RISCV_SYSROOT ?= $(RISCV_TOOLCHAIN_PATH)/$(RISCV_PREFIX) @@ -61,8 +69,15 @@ VRT_HOME ?= /usr/local # CONFIGS win over the profile. PROFILE ?= @PROFILE@ ifneq ($(strip $(PROFILE)),) -override CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --merge-configs --cflags='$(CONFIGS)') +PROFILE_CONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') ifneq ($(.SHELLSTATUS),0) $(error invalid hardware profile '$(PROFILE)') endif +# A Makefile that includes this one appends its own -D flags to CONFIGS, and an +# override would silently drop them; only a command-line CONFIGS needs one. +ifeq ($(origin CONFIGS),command line) +override CONFIGS := $(PROFILE_CONFIGS) +else +CONFIGS := $(PROFILE_CONFIGS) +endif endif diff --git a/configure b/configure index c5e1e21147..848dcef99e 100755 --- a/configure +++ b/configure @@ -77,7 +77,7 @@ copy_files() { continue fi mkdir -p "$dest_dir" - sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g" "$file" > "$dest_file" + sed "s|@VORTEX_HOME@|$SOURCE_DIR|g; s|@XLEN@|$XLEN|g; s|@TOOLDIR@|$TOOLDIR|g; s|@OSVERSION@|$OSVERSION|g; s|@INSTALLDIR@|$PREFIX|g; s|@BUILDDIR@|$CURRENT_DIR|g; s|@TOOLCHAIN_REV@|$TOOLCHAIN_REV|g; s|@VORTEX_VERSION@|$VORTEX_VERSION|g; s|@GEM5_REV@|$GEM5_REV|g; s|@PROFILE@|$PROFILE|g; s|@FIRESIM_REV@|$FIRESIM_REV|g" "$file" > "$dest_file" # apply permissions to bash scripts read -r firstline < "$dest_file" if [[ "$firstline" =~ ^#!.*bash ]]; then @@ -177,7 +177,7 @@ usage() { } while [[ "$#" -gt 0 ]]; do case $1 in - --xlen=*) XLEN="${1#*=}" ;; + --xlen=*) XLEN="${1#*=}"; XLEN_SET=1 ;; --tooldir=*) TOOLDIR="${1#*=}" ;; --osversion=*) OSVERSION="${1#*=}" ;; --prefix=*|--installdir=*) PREFIX="${1#*=}" ;; @@ -194,6 +194,19 @@ if [ -n "$PROFILE" ] && [ ! -f "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" & exit -1 fi +# a profile that names an xlen picks it, and an explicit --xlen must agree +if [ -n "$PROFILE" ]; then + PROFILE_XLEN=$(python3 "$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/ci/gen_config.py" \ + --config="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )/VX_config.toml" --profile="$PROFILE" --profile-xlen) || exit -1 + if [ -n "$PROFILE_XLEN" ]; then + if [ -n "$XLEN_SET" ] && [ "$XLEN" != "$PROFILE_XLEN" ]; then + echo "Error: profile '$PROFILE' models a $PROFILE_XLEN-bit machine; --xlen=$XLEN disagrees" + exit -1 + fi + XLEN=$PROFILE_XLEN + fi +fi + # check OS if [ "$OSVERSION" == "unsupported" ]; then echo "Error: Unsupported OS." @@ -208,12 +221,15 @@ SOURCE_DIR="$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) THIRD_PARTY_DIR=$SOURCE_DIR/third_party -# load version pins (VORTEX_VERSION, TOOLCHAIN_REV) +# load version pins (VORTEX_VERSION, TOOLCHAIN_REV, GEM5_REV, FIRESIM_REV) source "$SOURCE_DIR/VERSION" -# stamp file tracks config parameter changes; touched only when params change +# stamp file tracks config parameter changes; touched only when params change. +# The version pins are substituted into the same templates, so they are part of +# the signature: without them a bumped pin never reaches an existing build tree. CONFIG_STAMP="$CURRENT_DIR/.config.stamp" CONFIG_SIG="XLEN=$XLEN TOOLDIR=$TOOLDIR OSVERSION=$OSVERSION PREFIX=$PREFIX PROFILE=$PROFILE SOURCE_DIR=$SOURCE_DIR BUILDDIR=$CURRENT_DIR" +CONFIG_SIG="$CONFIG_SIG VORTEX_VERSION=$VORTEX_VERSION TOOLCHAIN_REV=$TOOLCHAIN_REV GEM5_REV=$GEM5_REV FIRESIM_REV=$FIRESIM_REV" if [ ! -f "$CONFIG_STAMP" ] || [ "$(cat "$CONFIG_STAMP" 2>/dev/null)" != "$CONFIG_SIG" ]; then echo "$CONFIG_SIG" > "$CONFIG_STAMP" fi diff --git a/docs/assets/img/amo_architecture.svg b/docs/assets/img/amo_architecture.svg index d95de9c2cf..b9d02886a8 100644 --- a/docs/assets/img/amo_architecture.svg +++ b/docs/assets/img/amo_architecture.svg @@ -44,7 +44,7 @@ aq / rl are captured and carry no behavior - VX_lsu_slice — per-lane sideband + VX_lsu_block — per-lane sideband packs amo_req_t into each lane's mem_bus_attr_t: hart_id {core, warp, thread} amo_cmp comparand (Zacas only) @@ -86,8 +86,8 @@ Commit point 1 — shared (local) memory - VX_local_mem bank - single-ported SRAM: an atomic owns its bank for 3 cycles + VX_lmem_unit bank + an atomic owns its bank for 3 cycles (first port; nothing else is issued) ① read the word ② register it, resolve the reservation ③ VX_amo_alu result written back under the request byteen VX_amo_unit: min(AMO_RS_SIZE, 4) stations, 3-bit hold credit diff --git a/docs/assets/img/beacon_compiler.svg b/docs/assets/img/beacon_compiler.svg new file mode 100644 index 0000000000..b9b8c0a057 --- /dev/null +++ b/docs/assets/img/beacon_compiler.svg @@ -0,0 +1,133 @@ + + + + + + + + + + + Beacon — the two passes + + + + foldsim-beacon-map (per module) + + + 1. hw-flatten-modules: inline modules < 300 ops + LUTs pack across buffers and arbiters, as Vivado flattens + + + 2. canonicalize, CSE, comb-balance-mux + mux chains become balanced trees + + + 3. foldsim passes + lower-equality lower-constant-division + fold-zero-width + + + 4. convert-comb-to-synth + to an and-inverter graph; add, sub, icmp, mul, + div and mod stay word-level ops + + + 5. lower-word-to-bits, timing-aware variadic + lowering, maximum-and-cover, SOP balancing + depth-optimal AIG, structurally hashed + + + 6. synth-generic-lut-mapper (6 inputs, timing) + comb.truth_table ops: one LUT6 each + + + Result: mapped.mlirbc + LUT6s, word-level arithmetic, registers (seq.firreg, + seq.compreg), memories (seq.firmem: distributed RAM + if read latency 0, block RAM otherwise), instances of + the modules not inlined, external modules (vendor IP) + CIRCT upstream passes + three foldsim passes + + + + + + + + + + + foldsim-estimate-timing (whole design, elaborated) + + + Collect end points + walk the instance tree from top=; one context + per instance. End points: register D, reset and + enable; memory write ports and registered read + address; inputs of vendor IP and black boxes; + top outputs + + + Resolve a bit + see through wires, extracts, concats, replicates + and instance ports to the op that drives it: one + node per (instance, value, bit). Start points: + registers, block-RAM reads, vendor IP outputs, + black-box outputs, top inputs + + + Evaluate: iterative depth-first, one worst predecessor per bit + levels(node) = max over inputs ( levels(input) + edge(input → node) ) + edge levels per op kind, for the device's family: + LUT 1 + add/sub bit j→i 2 + ⌊i/8⌋ − ⌊j/8⌋ (1 if the bit reads ≤ 6 inputs) + compare as an adder, to the top non-constant bit + multiply ≤ 10 variable bits: a table; else DSP cascade + divide ≤ 10 support bits: a table; else a ripple + LUTRAM read 1 + doublings past 64 words + a combinational loop's closing edge is left out and counted; time and memory are + linear in the design's bits (full GPU: 16-21 s, 2.7 GB) + + + End point levels + the input node's levels, plus a DSP's output + stage unless the end point is a register the + DSP absorbs (with its enable logic) + optional retime-levels=: one-hop balancing + + + Report + walk worst predecessors back to the start: + start, largest fanout, module crossings; + histogram; top-paths and trace= op by op; + paths=: each deep register pair, op by op + endpoints.csv, instances.csv, report.txt + + + Targets (target=): the FPGAs differ in their DSPs + versal DSP58 27×24 in 1 out 6 cascade 3 + virtexuplusHBM DSP48E2 27×18 in 1 out 5 cascade 2 + asap7 cell stages (3-input cuts), prefix adders, Dadda products + + + + + + + mapped + diff --git a/docs/assets/img/beacon_system.svg b/docs/assets/img/beacon_system.svg new file mode 100644 index 0000000000..ad70ab047a --- /dev/null +++ b/docs/assets/img/beacon_system.svg @@ -0,0 +1,136 @@ + + + + + + + + + + + + + + Beacon — where each piece lives, and what flows between them + + + + Vortex repository + + + RTL + VX_config.toml + hw/rtl/**, hw/unittest/*/VX_*_top.sv + the DUT catalog: hw/syn/xilinx/dut/catalog.mk + + + hw/syn/xilinx/dut — target beacon + gen_sources.sh (hw/rtl, toml defines) + circt_elaborate.sh -f mlir -x xilinx_ip_stubs.sv + gen_platform.py --device=$(DEVICE) --emit=family + + + hw/scripts/beacon_summary.py + beacon_summary.csv, beacon_units.json + beacon_paths.txt (deep paths by unit) + units = instance paths, generate indices folded + + + ci/logic_gate.py (synth_gate.py Beacon backend) + ci/testcases/logic_gate.yaml + ci/baselines/synthesis/beacon/*.json + .github/workflows/logic_gate.yml + PASS / FAIL per unit, zero tolerance on depth + + + + Toolchain ($(TOOLDIR), prebuilt) + + + CIRCT: circt-verilog, circt-opt + SystemVerilog → HW / Comb / Seq netlist + (the same import foldsim and the Yosys flow use) + + + foldsim-opt (foldsim, on CIRCT) + + foldsim-beacon-map + inline small modules, balance, lower to AIG, + map to 6-input LUTs; adders, compares, + multipliers and dividers stay word-level + + foldsim-estimate-timing + per-bit arrival over the elaborated instances, + levels as Vivado counts them on target= + versal (V80) | virtexuplusHBM (U55C) + optional: retime-levels=, trace=, black-boxes= + + + Outputs (per DUT, beacon/) + endpoints.csv one row per end point bit + instances.csv the elaborated instance tree + report.txt histogram, deepest paths op by op + paths.jsonl every deep path, source lines + mapped.mlirbc kept to measure other tops + + + + Who reads it + + + Pull requests touching RTL + logic_gate.yml, ~4 min for the 11 DUTs, + no Vivado, no licence: names the unit that + got deeper on the change that did it + + + RTL timing work (native-timing loop) + run Beacon on a whole GPU in ~1 min before a + 20-40 min Vivado run; fix list = units at + ≥ 12 levels, watch list ≥ 8 (300 MHz) + + + ci/beacon.py, hw/scripts/xilinx_*.tcl + run summarize validate retime-split + xilinx_cone_dump.tcl xilinx_cone_probe.tcl + xilinx_wire_probe.tcl + Beacon ↔ Vivado agreement, and the Vivado + probes for what levels cannot show + + + fpga_gate (nightly, Vivado) + same DUTs and configurations: a unit the + Beacon flags is checked against its + critical paths; Fmax stays its job + + + + + sources, defines + + netlist.mlir + + mapped.mlirbc + + + endpoints, instances + + + + + solid: data; dashed: who consumes it. Engine in foldsim (ships in the prebuilt); scripts and integration in Vortex. + diff --git a/docs/assets/img/cache_flush_sequence.svg b/docs/assets/img/cache_flush_sequence.svg index 780747378b..843f55ec28 100644 --- a/docs/assets/img/cache_flush_sequence.svg +++ b/docs/assets/img/cache_flush_sequence.svg @@ -1,4 +1,4 @@ - + @@ -22,6 +22,7 @@ .cm { font-family:"DejaVu Sans Mono", monospace; font-size:10px; fill:#1b2733; } .act { fill:#e8f0f7; stroke:#2e5e82; stroke-width:1.3; } .actg { fill:#f4faf4; stroke:#3a7d44; stroke-width:1.3; } + .acty { fill:#fdf6e3; stroke:#b58900; stroke-width:1.3; } .note { fill:#fbfcfd; stroke:#5a7184; stroke-width:1.2; } .loop { fill:none; stroke:#a03020; stroke-width:1.5; stroke-dasharray:7 4; } .loopl{ fill:#fbeeea; stroke:#a03020; stroke-width:1.3; } @@ -37,52 +38,51 @@ .lbr { fill:#a03020; font-size:11px; font-weight:bold; } - - Cache flush — DCR read to last writeback drained - One flush primitive exists: whole-cache invalidate-and-writeback. It is triggered by a DCR READ, carried as an ordinary MemReq with attr.is_flush, and it freezes the cache for its whole sweep. + + Cache flush — broadcast write to the last core's completion read + One flush primitive exists: whole-cache invalidate-and-writeback, requested by a flush token (a MemReq with attr.is_flush). Every ordered path carries its own token, and a cache answers a token only after flushing the next level. host / CP VX_dcr_data - VX_dcr_flush + VX_dcr_flush (per lane) VX_cache_init VX_cache_flush (per bank) next level - - - - - - + + + + + + - dcr_read(VX_DCR_BASE_CACHE_FLUSH, core_id) + dcr_write(VX_DCR_BASE_CACHE_FLUSH) - latch flush_pending_r - a DCR READ, not a write — the read's - response is what acknowledges the flush + every core: flush_pending_r = 1 + a write reaches all cores at once and has no + response, so all cores flush concurrently - dcr_flush_if.req = 1 (level-held) + dcr_flush_if.req = 1 (level-held, once stores drain) - - synthesize a degenerate MemReq: + a flush token per buffered dcache lane: rw=0, addr=0, data=0, byteen=0 attr = 1 << MEM_ATTR_FLUSH_OFFS - the flush rides port 0 as ordinary cache traffic + each token behind its own lane's stores - LOCK EVERY INPUT PORT + BLOCK EVERY INPUT input_enable = ~flush_req_enable - || lock_released[i] - valid=0 downstream, ready=0 upstream + while any token is visible at any input: + valid=0 downstream, ready=0 upstream STATE_WAIT1: drain the @@ -115,74 +115,89 @@ write-through: valid cleared, zero writebacks ever - writeback - - one dirty sector - per beat + writeback (on its bank's port) flush_ready gates the counter, so the walk pauses whenever the memory-request queue is near full. A write-through bank never gates on it — it emits no writebacks at all. - - STATE_WAIT2 — every bank EXCEPT bank 0 - Banks 1..N−1 wait for bank_empty. Bank 0 skips to - DONE, so its request — the one that propagates the - flush downstream — is the last to be issued. + + STATE_WAIT2 — every bank + waits for bank_empty: its write-backs must leave + the bank before the tokens go out behind them + + + flush_end — one pulse per bank + + + STATE_WAIT2 done: the tokens visible NOW are + this walk's; a later token starts the next walk + + + + STATE_FORWARD (FLUSH_FORWARD=1) + wait mem_req_empty: the crossbar has drained + + + one flush token on EVERY memory port, behind that port's write-backs - - flush_end — one pulse per bank + + same sequence one level down: one walk + for every core's tokens, then answers + each port's token - - STATE_WAIT2: accumulate until - flush_done == {NUM_BANKS{1'b1}} + + one response per port — recognized by phase (cache quiescent) - - release ONLY the originating port; its MemReq acks + + + STATE_RESPOND: each token answered locally - - dcr_flush_if.done — LEVEL-HELD until req drops + + done — all lanes answered; LEVEL-HELD - - DCR response — the flush read finally returns + + dcr_read(CACHE_FLUSH, cid), one core at a time + + answered when that core is done (at once if already) - - - - Core traffic has arbitration PRIORITY - The 2:1 arb puts core_bus_if on input 0, not the - flush. The flush waits until the LSU goes quiet — - which it does, because the host only issues a flush - after busy drops. STICKY hardens the 1-cycle gaps. - - - Why done is level-held rather than a pulse - One core's flush fans out to the dcache, the icache, and — with graphics enabled — the - socket and cluster gfx caches. Each leg finishes at a different time. Holding every - done high until req drops lets the parent simply AND them together, race-free. - A pulse would need a latch at every junction and would drop legs that finished early. - - - What the lock buys - Once locked, no core request can reach any bank pipeline. That single fact is what lets - STATE_FLUSH assume the only in-flight entries are its own walk, and it is why WAIT2's - bank_empty check need not re-test mshr_empty — nothing can refill the MSHR. - The cost: the cache is effectively OFF for the whole walk. Every other warp's loads and - stores stall. Flush is linear in cache size and is meant to be infrequent. - - - Walk length - write-through: LINES_PER_BANK - write-back: LINES_PER_BANK x NUM_WAYS - Write-through clears all ways in parallel (no way_en gate), so one pass over the lines - suffices. Write-back must visit each (way, line) separately because each dirty line - emits its own memory request. All banks walk concurrently. - - - Reset reuses the same machinery - The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid - bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. - A flush arriving during init is latched and answered immediately: init already invalidated - everything, so the init walk stands in for the flush walk. + + Why one token per memory port + Each port is a separately buffered path to the next level, and nothing orders one port's + writes against another port's token. A single token could reach the next level while another + port's last writes are still queued in front of it: that level's walk would release the token + and the writes would land after the walk, still dirty. A token behind each port's own data + is answered only once that data is in. The same rule gives one token per dcache lane. + The no-ack equivalent of waiting for store acknowledgements on every path. + + + Why done is level-held rather than a pulse + One core's flush fans out to every dcache lane, the icache, and — with graphics enabled — the + socket and cluster gfx caches. Each leg finishes at a different time. Holding every + done high until req drops lets the parent simply AND them together, race-free. + A pulse would need a latch at every junction and would drop legs that finished early. + + + What the block buys + While any token is visible, no core request can reach any bank pipeline. That is what lets + STATE_FLUSH assume the only in-flight entries are its own walk, makes every memory response + in STATE_FORWARD a token response, and makes a late token safe: it can only become visible + once all of its input's earlier requests were accepted, before the walk. + The cost: the cache is effectively OFF for the walk. Flush is linear in cache size. + + + Walk length + write-through: LINES_PER_BANK + write-back: LINES_PER_BANK x NUM_WAYS + Write-through clears all ways in parallel (no way_en gate), so one pass over the lines + suffices. Write-back must visit each (way, line) separately because each dirty line + emits its own memory request. All banks walk concurrently. + + + Reset reuses the same machinery + The FSM's reset state is STATE_INIT, not STATE_IDLE: it walks every line clearing valid + bits, so the tag SRAM needs no async reset — which is what makes it map cleanly to BRAM. + A flush arriving during init is latched and answered immediately: init already invalidated + everything, so the init walk stands in for the flush walk. diff --git a/docs/assets/img/cache_flush_tree.svg b/docs/assets/img/cache_flush_tree.svg index 8da61c02cd..aac49bd251 100644 --- a/docs/assets/img/cache_flush_tree.svg +++ b/docs/assets/img/cache_flush_tree.svg @@ -1,4 +1,4 @@ - + @@ -6,9 +6,6 @@ - - - - The flush fan-out tree — one DCR read, every cache + The flush fan-out tree — one broadcast write, every cache Solid blue = req fans DOWN. Dashed green = done ANDs back UP. Every done is level-held until req drops, which is what makes the AND race-free. - - VX_dcr_data - dcr_read(CACHE_FLUSH, cid) - holds req until done comes back + + VX_dcr_data (per core) + dcr_write(CACHE_FLUSH): all cores start + dcr_read(CACHE_FLUSH, cid): report one + holds req until done comes back - + - + VX_core — the fan-out point dcr_flush_if - - - req + + + req && store_drained - - VX_dcr_flush (dcache) - injects the synthetic MemReq onto - dcache port 0 only - VX_mem_unit.sv — i==0 && j==0 + + VX_dcr_flush per buffered lane + lane 0 of a private dcache, else every + lane. Lane dones AND together + VX_mem_unit.sv - - dcache - the lock freezes ALL its ports - - - - ORDERING CONSTRAINT - icache.req = dcr_flush_if.req - && dcache.done - The icache flush is SEQUENCED - BEHIND the dcache flush. - - - - - - - VX_dcr_flush (icache) - VX_core.sv — between fetch and iMMU - - - - icache - no dirty data — retires fast - - - Why the icache waits for the dcache - Both L1s forward their flush to the shared next level, and a cache that is - flushing locks out incoming core requests for its whole sweep. - The icache carries no dirty data, so it retires almost immediately. If it ran - concurrently, its forward would arrive at the next level while the dcache is - still evicting — and that level's lock would strand the dcache's writebacks - upstream of memory. - - - Why the icache is flushed at all - A kernel re-loaded to the same virtual address after a CACHE_FLUSH would - otherwise execute stale lines from the previous launch. The host issues - CMD_CACHE_FLUSH after every kernel, so this is the instruction-side half - of that contract. + + dcache + walks, then one token per + memory port to the next level + + + + req (concurrent with the dcache) + + + VX_dcr_flush (icache) + VX_core.sv — between fetch and iMMU + + + + icache (FLUSH_FORWARD=0) + invalidates locally; no forward + + + + Why a token per write path + A token only orders the requests ahead of it on its own path. Separate paths + into a cache can be buffered independently (a shared dcache's arbiter buffers + each lane), so a store on one lane could still be queued when another lane's + token arrives. The same rule gives one token per memory port between levels. + + + Why the icache flushes, and why only locally + A kernel re-loaded to the same virtual address would otherwise execute stale + lines from the previous launch. The icache holds no dirty data, and the dcache's + flush already walks and invalidates every shared level the icache reads through, + so the icache's flush ends at the icache. - - EXT_GFX_ANY_ENABLE only + + EXT_GFX_ANY_ENABLE only VX_socket — OR of core reqs socket_flush_req = |per_core_req - + tcache (TEX) - + rtcache (RTU) - + VX_cluster — OR of socket reqs cluster_flush_if.req - - rcache (raster) - - ocache (output merger) - - Sockets fire their reqs - sequentially; the shared - resource flushes once per - req. Later reqs see an - already-empty cache, so the - redundant sweeps are cheap, - not incorrect. - - rcache/ocache are cluster- - level, inside VX_graphics; - tcache/rtcache are socket- - level. + + rcache (raster, 1 token) + + ocache (1 token / OM input) + + tcache, rtcache and rcache + flush once for all cores; + the OR drops when the + last core finishes. + The ocache flushes once the + OM has drained, for the + cluster flush and for each + dcache token a steer holds + on its way to the L2. + tcache/rtcache: socket; + rcache/ocache: cluster. - - The completion AND tree + + The completion AND tree - VX_cluster: per_socket.done = cluster_flush_if.done - VX_socket: per_core.done = cluster_flush_if.done - & tcache.done - & rtcache.done - VX_core: dcr_flush_if.done = dcache.done - & icache.done - & cluster_flush_if.done + VX_cluster: per_socket.done = cluster_flush_if.done + VX_socket: per_core.done = cluster_flush_if.done + & tcache.done + & rtcache.done + VX_core: dcr_flush_if.done = dcache.done + & icache.done + & cluster_flush_if.done + VX_mem_unit: dcache.done = &lane_flush_done - Only when every leg reports done does VX_dcr_data return the DCR read - and drop req — which re-arms every VX_dcr_flush for the next flush. + When every leg reports done, VX_dcr_data drops req (re-arming every + VX_dcr_flush) and answers this core's flush read, at once if it came first. - - - done + + + done - - + + diff --git a/docs/assets/img/core_partitioning_structures.svg b/docs/assets/img/core_partitioning_structures.svg new file mode 100644 index 0000000000..40a1198394 --- /dev/null +++ b/docs/assets/img/core_partitioning_structures.svg @@ -0,0 +1,192 @@ + + + + + + + + + + + Core partitioning at issue width 4: the widths select the structure + + Compute slices + FW = IW = every B = 4 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + Backend slices + FW = 2 < IW = every B = 4 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + Stage-sliced + FW = IW = 4, but the SFU has B = 1 + + core-wide: CTA dispatch, barriers, retirement, + SFU/TCU shared services, icache · dcache · LMEM + + compute slice 0 + + frontend + slice + wid mod 4 = 0 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 1 + + frontend + slice + wid mod 4 = 1 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 2 + + frontend + slice + wid mod 4 = 2 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + compute slice 3 + + frontend + slice + wid mod 4 = 3 + + + backend + slice + issue + ALU · FPU + LSU · SFU + TCU + commit + + + frontend slice 0 + wid mod 2 = 0 + own icache port + + + frontend slice 1 + wid mod 2 = 1 + own icache port + + + VX_frontend_xbar: combinational routing + + + backend + slice 0 + wid mod 4 = 0 + issue + 1 block of + every unit + commit + + + + backend + slice 1 + wid mod 4 = 1 + issue + 1 block of + every unit + commit + + + + backend + slice 2 + wid mod 4 = 2 + issue + 1 block of + every unit + commit + + + + backend + slice 3 + wid mod 4 = 3 + issue + 1 block of + every unit + commit + + + frontend + slice 0 + wid mod 4 = 0 + + + frontend + slice 1 + wid mod 4 = 1 + + + frontend + slice 2 + wid mod 4 = 2 + + + frontend + slice 3 + wid mod 4 = 3 + + + VX_frontend_xbar: combinational routing + + + VX_issue + 4 issue slices: ibuffer, scoreboard, operands + + + VX_execute + ALU, FPU, LSU, TCU: 4 blocks (wid mod 4) + SFU: 1 block, takes the 4 issue slices in turn + + + VX_commit: 4 commit slices + + diff --git a/docs/assets/img/cta_dispatch_hierarchy.svg b/docs/assets/img/cta_dispatch_hierarchy.svg index 64634eab83..ca0ce66ba3 100644 --- a/docs/assets/img/cta_dispatch_hierarchy.svg +++ b/docs/assets/img/cta_dispatch_hierarchy.svg @@ -56,7 +56,7 @@ kmu_arb — Vortex.sv:186 1 -> NUM_CLUSTERS - DEST_LSB_DEVICE + DEST_DIV_DEVICE @@ -64,7 +64,7 @@ kmu_arb — VX_cluster.sv:105 2 -> NUM_SOCKETS, ARBITER "R" - DEST_LSB_CLUSTER + DEST_DIV_CLUSTER the only level with 2 inputs @@ -83,7 +83,7 @@ kmu_arb — VX_socket.sv:69 1 -> VX_CFG_SOCKET_SIZE - DEST_LSB_SOCKET + DEST_DIV_SOCKET socket 1..N diff --git a/docs/assets/img/firesim_build_flow.svg b/docs/assets/img/firesim_build_flow.svg new file mode 100644 index 0000000000..4c2b845074 --- /dev/null +++ b/docs/assets/img/firesim_build_flow.svg @@ -0,0 +1,82 @@ + + + + + + + + + + Build flow — how Vortex RTL becomes a FireSim image + Vortex stays Verilog: Golden Gate never sees inside it. Runs from hw/syn/firesim/Makefile with the native flows' tokens. + Threaded images (FAME-5) are the Foldsim flow, compiled by CIRCT: see foldsim.md. + + + + Front end — the configured tree, flattened + make -C hw/syn/firesim [PLATFORM=<xsa> | DEVICE=<part>] TARGET=hw|sim|hw_emu + gen_sources.sh -P preprocesses hw/rtl into one self-contained directory, top VX_firesim_wrap: + vivado_srcs/ + .f ← -DSYNTHESIS -DVIVADO -DNDEBUG (+ -DSIMULATION for TARGET=sim/hw_emu) + vortex_srcs/ (metasim) ← -DNDEBUG only: behavioural RAMs, soft FPU + Platform defines: VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 · PLATFORM_MEMORY_ID_WIDTH=4 + (paired with idBits) · [PERF_ENABLE] + + + FAME-1: Vortex as a Verilog BlackBox + + stage → replace-rtl — Golden Gate on the harness only + VortexTarget.scala elaborates: VortexDUT holds VX_firesim_wrap as an opaque BlackBox; + Golden Gate applies FAME-1 at its boundary, inserts bridges and token channels. + → FireSim-generated.sv (harness; Vortex NOT inside) + + + xilinx-ips — vendor floating-point cores + xil_fma, xil_fdiv, … generated for FPGA_PART; the flattened tree only instantiates them. + (On aved the RTU keeps the soft FMA/divider: the clock-enable conversion cannot enter an OOC core.) + + + Platform build — the card's own flow + vitis: v++ link · aved: slashkit. The kernel is packaged from three inputs: + VORTEX_VIVADO_SRCS = the CIRCT netlist VORTEX_XILINX_IP_DIR = the vendor cores + FireSim-generated.sv = the harness + Vivado synthesis + place-and-route at FREQUENCY (default 60 MHz), STRATEGY=TIMING; + on aved, FireSim's clock gates become clock enables (no BUFGCE in the partition). + + + Image — build32_<aved|xsa>_<hw|sim|hw_emu>/bin/ + firesim.vbin | firesim.xclbin + firesim.card + + + Metasimulator (no image) — TARGET=xrtsim / avedsim + `elaborate` runs Golden Gate alone; sim/firesim/Makefile Verilates + FireSim-generated.sv + the vortex_srcs tree (library search) into libfiresim.so, + with simif_emul as the host interface — the design itself is the simulator. + EMUL_DEBUG=1 adds --trace +define+DEBUG → +waveformfile=… writes a VCD + Same target, same tokens as the card: the step before silicon. + + + + + + + + + Run the image: FPGA_BIN_DIR=<bin> ./ci/blackbox.sh --driver=firesim --target=hw --app=… + The Makefile guards the classic failure modes: SYNTHESIS missing from the flattened tree (every RAM + lands in LUTRAM), VX_MEM_PORTS resolving to 0, a changed CONFIGS reusing a stale flatten. + diff --git a/docs/assets/img/firesim_fpga_architecture.svg b/docs/assets/img/firesim_fpga_architecture.svg new file mode 100644 index 0000000000..d6bb715603 --- /dev/null +++ b/docs/assets/img/firesim_fpga_architecture.svg @@ -0,0 +1,174 @@ + + + + + + + + + + + + + A Vortex instance on the FPGA — V80 (AVED shell) + Everything inside pblock_slash is FireSim-generated.sv, one reconfigurable module. Solid orange = memory data; solid grey = control (MMIO); dashed green = clocks. The U55C build is the same picture with the Vitis shell around it and HBM[0:7] as the memory binding. + + + + xcv80-lsva4737-2MHP-e-S + + + + AVED static region (in flash) + + + PCIe Gen5 ×8 + QDMA + BAR0 → AXI-Lite control network + (the HOST slave bridge is unused: + its reads never complete) + + + Base logic + uuid_rom · SMBus RPU · firewall · + clock/reset: programs the kernel clock + to the linked frequency — no MMCM, + FREQUENCY=60 → 60.000 MHz + + + dfx_decoupler + isolates the RM during reprogramming + + + HBM VNOC (NoC) + Routes the RM's one wide master to the + HBM stacks. The MEM segment maps + 32 GB at + 0x40_0000_0000 (HOST_MEM_SP=MEM) + covering the target's whole 32-bit map + + + HBM2e — 32 GB + Functional target memory: every byte the + GPU reads or writes lives here; FASED + only decides *when* answers return + + + + Reconfigurable partition pblock_slash — RM “slash_firesim” = FireSim-generated.sv + + + + AvedShim / SimulationMaster + Fingerprint 0x46697265 (“Fire”), transport gate at open; every FireSim MMIO register gets its own 16-byte AXI-Lite block + (the shell resolves AXI-Lite in 16 B units and LoadMem's R_DATA has read side effects), driver scales addresses ×4; + host-memory base check: the memory port must be bound at the elaborated base before any widget is touched + + + + PeekPoke bridge + The driver's hands: pokes + ctrl_dcr_req_*, ctrl_start + peeks + ctrl_busy, ctrl_dcr_rsp_* + step(n) = advance n target cycles + + + LoadMem widget + Bulk target-memory access from + the host, beside the timing model: + kernel images, buffers, readback. + Chunk unit = 32-bit words + + + Clock bridge + Generates the target clock token + stream; clockmodule counts + tcycle (target) / hcycle (host) + their ratio is FMR + + + FASED memory timing model + Latency-bandwidth pipe, per-run settings + readLatency=30 writeLatency=30 + maxReads=16 maxWrites=16 + functional half: ingress/egress queues, + reorder buffer, host-memory master + + + + FAMETop — the FAME-1 hub (SimWrapper) + Every wire between a bridge and the target is a token channel (ready/valid + payload). One target cycle fires exactly when every input + channel holds a token and every output channel has space; otherwise the target clock is gated and the design waits, cycle-accurately. + This is what host-decoupling means: the target cannot outrun the host, and the host cannot observe a half-advanced cycle. + Clock gating on the V80 is a clock-enable conversion (FireSimFabricClockGate): the partition has no BUFGCE site. + + + + Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex + Control surface: DCR request/response + start/busy (poked by PeekPoke). Memory: one AXI4 port, 512-bit data, 32-bit address, 4-bit id. + + + Single-copy logic (built once) + KMU (CTA dispatch) · DCR bus · global barrier unit · + cluster: socket↔memory arbiter · VX_mem_to_axi + (tag buffer: 16 outstanding reads, 4-bit AXI id) + Built once for the whole GPU. + + + g_sockets — N × VX_socket_top + Each socket: 1 core + I$/D$/LMEM, socket_id as a port. + One physical copy per socket. Threading the sockets + through one copy (FAME-5) is the Foldsim flow: foldsim.md. + + + + Control path: PCIe BAR0 → AXI-Lite → shim register blocks → widget MMIO. Memory data path: FASED / LoadMem host-memory master → HBM VNOC → HBM. + Host clock: free-running 60 MHz from the shell. Target clock: the host clock, enabled only on token-complete cycles — the target has no clock of its own. + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/docs/assets/img/firesim_memory_system.svg b/docs/assets/img/firesim_memory_system.svg new file mode 100644 index 0000000000..ea3e498c17 --- /dev/null +++ b/docs/assets/img/firesim_memory_system.svg @@ -0,0 +1,134 @@ + + + + + + + + + + + + + The memory system — functional data in HBM, latency from a model + Nothing on this path is a DRAM: the “GPU memory” the kernel sees is the card's HBM behind a timing model that makes it look like whatever the +mm_ settings say. Determinism follows — congestion on the real HBM stalls target time instead of changing results. + + + + Vortex (target time) + + + Cores → L1 → (L2/L3 if enabled) + the sockets' misses meet at the + cluster arbiter + + + VX_mem_to_axi + Tag buffer: UUID-wide upstream tags → + 16 outstanding reads, 4-bit AXI id + (paired: PLATFORM_MEMORY_ID_WIDTH=4 ↔ idBits=4) + + + m_axi_mem_0 — one AXI4 port + 512-bit data · 32-bit addr · INCR bursts + NUM_BANKS=1: the wrapper presents one port, + so Vortex builds one + + + + FASED bridge (FireSim-generated) + + + Timing model — latency-bandwidth pipe + Runtime registers, set per run, no rebuild: + readLatency = 30 · writeLatency = 30 (target cycles) + readMaxReqs / writeMaxReqs ≤ maxReads/Writes = 16 + Set via VORTEX_FIRESIM_ARGS plusargs: + +mm_readLatency_0=100 +mm_writeLatency_0=100 + +mm_useHardwareDefaultRuntimeSettings_0 (covers the rest) + + + Functional model + Ingress/egress queues, per-id reorder buffer; reads and writes + go to host memory when accepted, responses are held until the + timing model's release cycle. Instrumented: + totalReads/Writes(Beats) · readMaxReqs · rrespError · … + printed by the driver on every stall report and watchdog + + + LoadMem widget — the host's side door + Kernel upload, buffer copy, readback: same host-memory master, + no target time passes. Chunk unit is 32-bit words. + + + + Shell binding (per card, from its descriptor) + + + V80 (aved) — hw/syn/xilinx/platforms/v80.toml + host_mem_sp = "MEM" → HBM VNOC, 32 GB window + base = 0x40_0000_0000 (WithHostMemBase) + The whole 32-bit target map fits the window; the AFU + rebases every access. HBM0 (1 GB) would truncate it — + the stack sits just under 4 GB. HOST (QDMA slave + bridge) never completes reads: refused by the descriptor. + + + U55C (vitis) + HOST_MEM_SP = HBM[0:7] → 8 × 512 MB = 4 GiB + base = 0x0 + A narrower binding silently serves only part of the map: + kernel uploads land outside it, reads return junk. + + + One host memory channel, 64-bit + Host-side AXI: beatBytes=8 at the host clock. This bounds + *simulation speed* under memory traffic, never target + timing: a starved FASED gates the target clock and FMR + rises; the kernel's cycle count is unchanged. + + + + + + + + + + + What the model does — and does not — capture + + The default pipe is a flat-latency, fixed-parallelism abstraction. Its ceiling is easy to compute and easy to hit: + peak read bandwidth ≈ maxReads × 64 B / readLatency = 16 × 64 / 30 ≈ 34 B per target cycle — for the whole GPU + demo at 64 cores (memory-bound, IPC 0.18) and sgemm 1024³ (14.7 FLOP/cycle against ~2000 available) both sit on this ceiling. Raising it means + raising maxReads (compile time, area) or the modelled latencies (runtime), not a faster card. + + There is no DRAM structure in the default model: no banks, rows, refresh, or contention. FASED has DDR3 FIFO/FR-FCFS models with those (compile-time + choice, WithDDR3FIFOMAS), and an optional tags-only LLC model; neither is enabled for Vortex today. HBM timing does not exist in FASED at all. + + Configuration knobs, by where they live: + image build: FREQUENCY=60 STRATEGY=TIMING DEVICE=<part> HOST_MEM_SP=<tag> (hw/syn/firesim → platform build) + elaboration: NastiParameters(dataBits=512, addrBits=32, idBits=4) · BaseParams(maxReads/maxWrites=16) · platform toml [firesim] + per run: VORTEX_FIRESIM_ARGS="+mm_…_0=…" — the only knob that needs no rebuild + + Address-width warning: the wrapper takes its AXI width from VX_CFG_MEM_ADDR_WIDTH (48 at XLEN=64) while the Scala side pins addrBits=32; + a disagreement is padded or truncated with no diagnostic. XLEN=64 on FireSim needs both sides changed together. + idBits is bounded both ways: 1<<idBits wraps Scala's Int at 32 (silently wrong responses); per-id state made idBits=8 cost 751K LUTs on a U55C. 4 matches what Vortex drives. + diff --git a/docs/assets/img/firesim_system_stack.svg b/docs/assets/img/firesim_system_stack.svg new file mode 100644 index 0000000000..1710193a79 --- /dev/null +++ b/docs/assets/img/firesim_system_stack.svg @@ -0,0 +1,121 @@ + + + + + + + + + + FireSim + Vortex — software and hardware stack + One stack, two vehicles: the card (left of the red cut: PCIe) and the in-process metasimulator (right: the same design, Verilated). Every layer above the cut is identical in both. + + + + Host process (one per run) + + + Application — tests/regression/<app> + vx_dev_open / vx_mem_alloc / vx_copy_to_dev / vx_start / vx_ready_wait, linked against libvortex.so + + + Runtime HAL — sw/runtime/firesim/vortex.cpp → libvortex-firesim.so + Driver ABI + software command processor (cmd_processor.cpp); kernel upload, DCR writes, launch, flush + VRT_VBIN_PATH / XCLBIN_PATH · VORTEX_FIRESIM_ARGS → extra plusargs (+mm_…) + + + Transport — sim/firesim/firesim_sim.cpp (in libfiresim.so) + Owns the simulator thread; every bridge call is a labelled request marshalled onto it. + Adaptive step batching, DCR single-step polling, launch watchdog, stall reports + + + FireSim driver core — $(FIRESIM_PATH)/sim/midas/src/main/cc + Widget registry from FireSim-generated.const.h: + peek_poke_t · loadmem_t · clockmodule_t · FASEDMemoryTimingModel (driver half) + simif_t: MMIO read/write + bulk memory, one implementation per vehicle + + + simif_aved (VRT) + V80: vrtd programs the PDI, + MMIO over PCIe BAR + + + simif_vitis (XRT) + U55C: xclbin load, + MMIO + DMA through XRT + + + + Metasimulator (TARGET=xrtsim / avedsim) — no card, no image + + + Same application · same runtime HAL · same transport + Built with -DFIRESIM_EMUL; libfiresim.so is the Verilated model itself + + + simif_emul — the emulated host interface + MMIO and DMA become DPI calls into the Verilated top (top.sv / emul); + firesim_emul_main.cpp runs Verilator's event loop on the simulator thread. + VORTEX_FIRESIM_EMUL_ARGS → +waveformfile=… +max-cycles=… (EMUL_DEBUG=1 build) + + + Vemul — the whole FPGA payload, Verilated + FireSim-generated.sv + the flattened Vortex tree, + bit-identical to what the card synthesizes. Same tokens, same schedule: + what passes here passes on silicon, cycle for cycle. + Build: sw/runtime/firesim → sim/firesim/Makefile (EMUL_DIR per card, + and a _dbg variant); run: blackbox.sh --driver=firesim + --target=avedsim|xrtsim [EMUL_DEBUG=1] + + + + PCIe — MMIO (control) + memory writes; everything below is on the FPGA + + + + FPGA (V80 shown; U55C differs only in the shell) + + + Shell — AVED static region + AvedShim / SimulationMaster + PCIe endpoint, AXI-Lite control (16 B blocks), host clock, HBM NoC — see the FPGA architecture figure + + + FPGATop — FireSim's generated harness (FireSim-generated.sv) + Bridges: PeekPoke (ctrl_* ports) · LoadMem (bulk memory) · ClockBridge · FASED (memory timing) + Token channels host-decouple everything inside; the target advances only when stepped. + + + Target — VortexDUT ▸ VX_firesim_wrap ▸ Vortex + The design under test: DCR + start/busy control surface, one 512-bit AXI4 memory port. + + + + + + + + + + + + + + diff --git a/docs/assets/img/firesim_timing_model.svg b/docs/assets/img/firesim_timing_model.svg new file mode 100644 index 0000000000..fc74bd7b39 --- /dev/null +++ b/docs/assets/img/firesim_timing_model.svg @@ -0,0 +1,122 @@ + + + + + + + + + + How simulation timing is modelled + Three layers, strictly separated: token exchange decides *when a target cycle may happen*; timing models decide *what latency the target observes*; the driver decides *how many cycles to ask for*. Wall-clock speed never leaks into target time. + + + + 1 — Host decoupling (FAME-1): the target clock is a resource, not a fact + + + PeekPoke + tokens on demand (step) + + FASED + one token per mem beat + + Clock bridge + the cycle budget itself + + + + + + + + + + + + token channels (ready/valid queues) + + + FAMETop hub + target + Fires one target cycle when EVERY input + channel holds a token and every output + has space. Otherwise the clock enable + stays low: the target is frozen mid-state, + not observing time. ○ = the missing token + that is gating the cycle right now. + + + Why this is exact + The target cannot advance past a bridge that has not answered, and a bridge + cannot answer early: however slow the host is — PCIe hiccups, a paused driver, + Verilator instead of silicon — the target sees the identical cycle sequence. + Metasim and FPGA runs agree cycle-for-cycle for this reason. + Two clocks, never conflated: + tcycle — target cycles advanced (clockmodule; Vortex PERF: cycles) + hcycle — host cycles spent simulating them + FMR = hcycle/tcycle; target freq = host_freq / FMR + + + + 2 — Observed latency: FASED's timing model, in target cycles + + The memory the GPU sees is split in two. The functional half reads/writes real HBM through the host memory port. The timing half is a latency-bandwidth + pipe that schedules, in target time, when each response is released. If HBM is slow, target time simply stops until the data is there — the GPU never notices. + + + target time → + + + T: AR accepted + + T+30: R data released + + readLatency = 30 target cycles (runtime register, +mm_readLatency_0=…) + + ≤16 reads in flight + + Bandwidth ceiling of this model: 16 outstanding × 64 B / 30 cycles ≈ 34 B per target cycle for the whole GPU — sgemm at 64 cores runs into exactly this. + Counters (totalReads, readMaxReqs, …) are read from the model over MMIO and printed by the driver on every stall report. + + + + 3 — The driver's schedule: adaptive step batching (firesim_sim.cpp) + + The target only moves when stepped, and each poll costs a thread handoff plus an MMIO round trip. The driver grows the batch ×4 from a floor of 16 target + cycles up to a ceiling (default 100,000; VORTEX_FIRESIM_STEP_BATCH), and drops back to the floor whenever the target goes idle: + + + + 16 + 64 + 256 + 1024 + 4096 + …×4… + 100000 (ceiling) + busy sampled after each batch + + + Consequences worth knowing: + • “Target Cycles Emulated” is quantized to this schedule — Vortex's own PERF: cycles counter is the cycle authority. + • A short kernel is only *observed* to finish at the next poll; the floor of 16 exists so a launch's busy edge is never stepped over. + • DCR reads poll one cycle at a time (dcr_rsp_valid is a one-cycle pulse), bounded by VORTEX_FIRESIM_DCR_POLL_ROUNDS — slow, but exact. + diff --git a/docs/assets/img/foldsim_architecture.svg b/docs/assets/img/foldsim_architecture.svg new file mode 100644 index 0000000000..43226ed218 --- /dev/null +++ b/docs/assets/img/foldsim_architecture.svg @@ -0,0 +1,105 @@ + + + + + + + + + + Compile: one IR from SystemVerilog to the kernel (the Foldsim driver, on CIRCT) + + circt-verilog + Vortex, flat but for + the classes + + foldsim-extract-models + each level a netlist of its own + logic and the class instances + + model cleanups + share-adders, low-bits + narrowing, const-prop + + foldsim-compose + classes threaded (FAME-5), + channels in one token fabric + + firtool, Vivado + foldsim_sim.sv, packaged + and linked as on the V80 + + + + + + Run: Foldsim's shell (share/foldsim/shell) and its host + + foldsim_afu + + + foldsim_sim (generated) + every channel: two slots per producer thread, a count per reader + + rest of Vortex + 1 thread + + socket + 1 copy, N threads + + issue, ALU, LSU, FPU + 1 copy each, N x M threads + + + + token fabric: registers, or memories read at the reader's thread + a thread finishes its target cycle once each input has a token and each output was taken; + state in LUTRAM per thread, written with the clock enable of that finish + + + foldsim_ctrl + AXI4-Lite registers + command queue: DCR, + start, reset; busy, + cycle counters + + foldsim_mem + latency-bandwidth pipe + in target cycles; + requests to HBM at once, + a cycle waits for data + + + Endpoints offer and take tokens with valid and ready from their own state, so they connect to the + simulator's channels directly. The target runs free: one token per channel per target cycle. + Shown for one memory port and Vortex's DCR, start and busy (the target foldsim_target). + + + host: --driver=foldsim + the runtime's command + processor model, in software; + VRT on the V80, or the kernel + in Verilator (avedsim) + + HBM + target address + base; + host DMA by QDMA + + AXI-Lite + + AXI4 + + QDMA + diff --git a/docs/assets/img/foldsim_mem_bank.svg b/docs/assets/img/foldsim_mem_bank.svg new file mode 100644 index 0000000000..257cd41d39 --- /dev/null +++ b/docs/assets/img/foldsim_mem_bank.svg @@ -0,0 +1,73 @@ + + foldsim memory endpoint: the bank model + + + + + + + + + per target cycle (at finish) + + AR / AW + W tokens + read takes a slot (32) + AW+W form a write + + + R / B stages + R: oldest ready read + B: oldest buffered write + data: slot buffer (HBM) + + + per DRAM clock (host cycles, before the cycle finishes) + + ingress + in order, from the + next target cycle + + + read buffer + + write buffer (acked) + write drain: 26 / 7 bursts + + + reservation (one request) + bank table (32 banks): open row, ACT, column-, PRE-ready + PRE, ACT, column, column: each a first-free search, + 64 clocks a host cycle, from its lower bound + horizon check, then mark what each command forbids + + + ring calendars, 512 clocks (LUTRAM) + command bus: row, column commands + row bus (PRE, two-clock ACT) + forbidden to a read / write column: + per pseudo-channel (tCCD_S, turnarounds) + per bank group (tCCD_L, tWTR_L) + forbidden to an ACT: tRRD_S, tRRD_L + column directions (controller queues) + departures: read slot per clock + a word of 64 clocks cleared as it passes; + a reservation reaches at most 448 clocks + + + departure at clock d+1 + read ready LATENCY + 1 target cycles + after its clock's target cycle + (ready FIFO, in departure order) + + + + + + + + + + + + write acknowledged + diff --git a/docs/assets/img/lsu_architecture.svg b/docs/assets/img/lsu_architecture.svg index c173c657b1..782e4efae4 100644 --- a/docs/assets/img/lsu_architecture.svg +++ b/docs/assets/img/lsu_architecture.svg @@ -30,7 +30,7 @@ VX_lsu_unit execute stage - one VX_lsu_slice per + one VX_lsu_block per LSU block (the issue width) @@ -42,7 +42,7 @@ becomes packets: pid, sop, eop - VX_lsu_slice — request side + VX_lsu_block — request side VX_lsu_agu base + offset, or @@ -68,7 +68,7 @@ tag = header, operation, alignment, slot, fence - VX_lsu_slice — response side + VX_lsu_block — response side load formatter sub-word extract by the saved @@ -152,11 +152,11 @@ streams local path - VX_lsu_adapter, one per block → VX_local_mem - banked SRAM: lanes to different banks are served - in the same cycle - a second port serves the DXA and tensor - direct-memory clients + whole warp requests → VX_lmem_unit + a pass per cycle: each bank to its lowest lane, + same-word reads broadcast, conflicts replay + several LSU blocks: a read engine on the second + bank port; DXA and tensor DMA take whole rows per-core shared memory global path diff --git a/docs/assets/img/mlp_afu_inflight.svg b/docs/assets/img/mlp_afu_inflight.svg new file mode 100644 index 0000000000..ffc14da73f --- /dev/null +++ b/docs/assets/img/mlp_afu_inflight.svg @@ -0,0 +1,34 @@ + + Before: each request waits a full 100-cycle round trip before the next is sent. After: four requests overlap and finish in about the time of one round trip plus three cycles. + + + + + Before: one in flight + R1 · 100 cyc + R2 + R3 + R4 + ≈ 400 cyc + After: many in flight + + R1–R4 overlap: ≈ 104 cyc + time → + + diff --git a/docs/assets/img/mlp_afu_ports.svg b/docs/assets/img/mlp_afu_ports.svg new file mode 100644 index 0000000000..1a51bb3d18 --- /dev/null +++ b/docs/assets/img/mlp_afu_ports.svg @@ -0,0 +1,57 @@ + + The AFU now has one AXI port per GPU memory port, each mapped to its own HBM group, and a write's data now leaves in the same cycle as its address. + + + +Before: fixed per board +After: follows the GPU + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +1 AXI port +HBM + + + +GPU port 0 +GPU port 1 +GPU port 2 +GPU port 3 +AXI 0 +AXI 1 +AXI 2 +AXI 3 +HBM group 0 +HBM group 1 +HBM group 2 +HBM group 3 + + +Write fall-through +before +AW +W +2 cycles per write +after +AW + W +1 cycle per write + + diff --git a/docs/assets/img/mlp_arg_slots.svg b/docs/assets/img/mlp_arg_slots.svg new file mode 100644 index 0000000000..a770b2c019 --- /dev/null +++ b/docs/assets/img/mlp_arg_slots.svg @@ -0,0 +1,36 @@ + + Before, uploading a launch's arguments forced a full LLC flush; now arguments go into a slot no kernel has read since the last full flush, so the launch follows right away. + + + + +Before +upload args +full LLC flush +launch + + +After +upload args to a clean slot +launch + +no kernel has read this slot +since the last full flush + + diff --git a/docs/assets/img/mlp_cp_dma_llc.svg b/docs/assets/img/mlp_cp_dma_llc.svg new file mode 100644 index 0000000000..58c436f523 --- /dev/null +++ b/docs/assets/img/mlp_cp_dma_llc.svg @@ -0,0 +1,46 @@ + + Before: CP DMA writes DRAM directly, leaving stale lines in L2 that force a full LLC walk. After: CP DMA enters L2 on its own client slot, so only the L1s are flushed. + + + +Before: DMA beside the LLC + After: DMA as an LLC client + + host + CP DMA + core L1s + L2: stale lines + DRAM + + + writes DRAM + + needs full walk + + host + CP DMA + core L1s + L2: coherent + DRAM + + L1-only flush + client slot + + + diff --git a/docs/assets/img/mlp_fence_split.svg b/docs/assets/img/mlp_fence_split.svg new file mode 100644 index 0000000000..2027ea9bf6 --- /dev/null +++ b/docs/assets/img/mlp_fence_split.svg @@ -0,0 +1,40 @@ + + An acquire only invalidates afterward, a release only drains beforehand, and only a full fence does both. + + + + +Acquire +Release +Full +fence r,rw · amo.aq +fence rw,w · amo.rl +fence rw,rw · amo.aqrl +before +nothing +drain writes, no walk +drain writes, no walk +after +invalidate L1 +nothing +invalidate L1 +No L1 action at all where the dcache is itself the coherence point +(one socket, one dcache, no other memory client) + + diff --git a/docs/assets/img/mlp_fences.svg b/docs/assets/img/mlp_fences.svg new file mode 100644 index 0000000000..5c4b1d5056 --- /dev/null +++ b/docs/assets/img/mlp_fences.svg @@ -0,0 +1,41 @@ + + Before: a fence token walks every line in L1 and in the L2 and writes back to DRAM. After: L1 writes back to L2, and the shared L2 acknowledges the token without a walk; DRAM is not touched. + + + +Before: full flush per fence + After: L1-only token + + core: fence + L1: walk all lines + L2 (LLC): walk all lines + DRAM: write-backs + token + token + dirty + token + dirty + + core: fence + L1: write back dirty + L2 (LLC): ack, no walk + DRAM: not touched + L1-only token + dirty lines + ack + + diff --git a/docs/assets/img/mlp_fill_forward.svg b/docs/assets/img/mlp_fill_forward.svg new file mode 100644 index 0000000000..1923b1c975 --- /dev/null +++ b/docs/assets/img/mlp_fill_forward.svg @@ -0,0 +1,45 @@ + + Before, every waiting request replayed through the bank pipeline after a fill. With fill forwarding, the waiting reads take their data from the staged fill, and only the first write replays. + + + +Before: every waiter replays +After: reads forward from the fill + +fill: line L +ld L+0 (waiting) +ld L+4 (waiting) +st L+8 (waiting) +bank pipeline3 replay passes + + + +new requests wait + +fill: line L +ld L+0: done +ld L+4: done +st L+8 (waiting) + + +forward +bank pipeline1 replay, keeps order + + + diff --git a/docs/assets/img/mlp_fill_staging.svg b/docs/assets/img/mlp_fill_staging.svg new file mode 100644 index 0000000000..ac92df36e8 --- /dev/null +++ b/docs/assets/img/mlp_fill_staging.svg @@ -0,0 +1,40 @@ + + Before, a bank took a new fill every other cycle; now fills enter back to back, one per cycle. + + + + +Before +fill 1 +busy +fill 2 +busy +fill 3 +busy +fill 4 +4 fills in 7 cycles +After +fill 1 +fill 2 +fill 3 +fill 4 +4 fills in 4 cycles +cycles → + + diff --git a/docs/assets/img/mlp_flush_tokens.svg b/docs/assets/img/mlp_flush_tokens.svg new file mode 100644 index 0000000000..7a77a0177c --- /dev/null +++ b/docs/assets/img/mlp_flush_tokens.svg @@ -0,0 +1,45 @@ + + A broadcast flush overlaps every core's flush, and a flush token on every memory port makes sure no write arrives after the next level has finished its walk. + + + +Before: cores flush in turn +After: one broadcast, all overlap + +core 0 +core 1 +core 2 +core 3 + +core 0 +core 1 +core 2 +core 3 +Before: token on port 0 only +After: a token on every port +L1 (2 memory ports) +next level: line left dirty +token +write,late +L1 (2 memory ports) +next level: answers after both +token +token + + diff --git a/docs/assets/img/mlp_hashed_banks.svg b/docs/assets/img/mlp_hashed_banks.svg new file mode 100644 index 0000000000..259a3d19a4 --- /dev/null +++ b/docs/assets/img/mlp_hashed_banks.svg @@ -0,0 +1,53 @@ + + Four requests with a power-of-two stride all queue on bank 0 with plain bank select; with the hash they spread over banks 0 to 3. + + + +Before: bank = line[1:0] + After: bank = line[1:0] XOR hash(line) + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 · 4 queued + bank 1 idle + bank 2 idle + bank 3 idle + + + + + + line 0x000 + line 0x100 + line 0x200 + line 0x300 + bank 0 + bank 1 + bank 2 + bank 3 + + + + + serialized: 4 cycles + parallel: 1 cycle + + diff --git a/docs/assets/img/mlp_icache_merge.svg b/docs/assets/img/mlp_icache_merge.svg new file mode 100644 index 0000000000..da923d6298 --- /dev/null +++ b/docs/assets/img/mlp_icache_merge.svg @@ -0,0 +1,54 @@ + + Before: four fetch slices request the same icache line and are served one per cycle. After: one access returns word 0 to 3 to slices 0 to 3 in the same cycle. + + + +Before: slices take turns + After: one access, a word per slice + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + S0 + S1 + S2 + S3 + 4 cycles + + slice 0: line L + slice 1: line L + slice 2: line L + slice 3: line L + icache + + + + + + w0w1w2w3 + S0–3 + 1 cycle + + diff --git a/docs/assets/img/mlp_launch_flush.svg b/docs/assets/img/mlp_launch_flush.svg new file mode 100644 index 0000000000..5d10af1f49 --- /dev/null +++ b/docs/assets/img/mlp_launch_flush.svg @@ -0,0 +1,30 @@ + + Flush cycles per launch, drawn to scale: a full flush walks the L2; an L1-only flush skips it. + + + + +1 core, full flush422 +1 core, L1-only152 +4 cores + L2, full flush2,750 +4 cores + L2, L1-only180 + +02,750 cycles per launch + + diff --git a/docs/assets/img/mlp_lazy_fetch.svg b/docs/assets/img/mlp_lazy_fetch.svg new file mode 100644 index 0000000000..4a58bc1d2b --- /dev/null +++ b/docs/assets/img/mlp_lazy_fetch.svg @@ -0,0 +1,48 @@ + + Stores fill the sector's byte mask without a DRAM read. When a load arrives on a fully written sector, the old design reads DRAM anyway; design G loops back inside the bank with no DRAM read. + + + + + 1 · store miss + 2 · more stores + 3 · load arrives + + sector installed, partial = 1 + no DRAM read + + byte mask full + partial still = 1 + ld → partial line + + + + + + + Without the mask check + With the mask check + refetch from DRAM + merge under 0 bytes + + mask full → loop back + clear partial, no DRAM + + + diff --git a/docs/assets/img/mlp_llc_atomics.svg b/docs/assets/img/mlp_llc_atomics.svg new file mode 100644 index 0000000000..250253c402 --- /dev/null +++ b/docs/assets/img/mlp_llc_atomics.svg @@ -0,0 +1,33 @@ + + An atomic executes once at the last-level cache, and every L1 it passes through drops its copy of the line. + + + + +core 0: amoadd X +core 1: ld X +L1: pass, drop X +L1: miss on X +L2 (LLC): read-modify-write X + +AMO + +sees new X + + diff --git a/docs/assets/img/mlp_lmem_round_robin.svg b/docs/assets/img/mlp_lmem_round_robin.svg new file mode 100644 index 0000000000..6009b161f7 --- /dev/null +++ b/docs/assets/img/mlp_lmem_round_robin.svg @@ -0,0 +1,39 @@ + + Before: DXA holds the LMEM port for about 540 cycles while TCU reads wait. After: grants alternate between DXA and TCU every cycle. + + + + + Before: fixed priority + DXA writes + holds the port ~540 cycles + TCU reads + stalled ~510 cycles + go + After: round-robin + DXA writes + TCU reads + + + + + grants alternate + time → + + diff --git a/docs/assets/img/mlp_lsu_paths.svg b/docs/assets/img/mlp_lsu_paths.svg new file mode 100644 index 0000000000..641b348ff8 --- /dev/null +++ b/docs/assets/img/mlp_lsu_paths.svg @@ -0,0 +1,42 @@ + + Each LSU block now owns its path to its dcache ports, so the core no longer funnels every block through one shared memory path. + + + +Before: one shared path +After: a path per LSU block + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +core memorypath +dcache + + + +LSU block 0 +LSU block 1 +LSU block 2 +LSU block 3 +dcacheports 0–3 + +own path, own status + + diff --git a/docs/assets/img/mlp_read_merge.svg b/docs/assets/img/mlp_read_merge.svg new file mode 100644 index 0000000000..fad93e4e00 --- /dev/null +++ b/docs/assets/img/mlp_read_merge.svg @@ -0,0 +1,54 @@ + + Before: four ports load word A and the bank serves them in four cycles. After: one access at cycle 1, with the reply multicast to all four ports. + + + +Before: one access per port + After: one access, multicast + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + P0 + P1 + P2 + P3 + 4 cycles + + port 0: ld A + port 1: ld A + port 2: ld A + port 3: ld A + bank + + + + + + P0–3 + 1 cycle + dashed: one reply to every port + + diff --git a/docs/assets/img/mlp_sectoring.svg b/docs/assets/img/mlp_sectoring.svg new file mode 100644 index 0000000000..228691552e --- /dev/null +++ b/docs/assets/img/mlp_sectoring.svg @@ -0,0 +1,46 @@ + + L2 and L3 lines carry one tag over two 64-byte sectors; dcache banks scale with the thread count. + + + +L2/L3: one tag, two sectors +Dcache: banks follow the lanes + +before +tag +64 B line +tag +64 B line +2 tags per 128 B +after +tag +sector 0 · 64 B +sector 1 · 64 B +1 tag per 128 B: half the tags +fills and evictions move one 64 B sector + +4 threads → 4 banks + +8 threads → 8 banks + +16 threads → 16 banks + +banks = line footprint / word size, line size unchanged + + diff --git a/docs/assets/img/mlp_wt_order.svg b/docs/assets/img/mlp_wt_order.svg new file mode 100644 index 0000000000..1abc739ba6 --- /dev/null +++ b/docs/assets/img/mlp_wt_order.svg @@ -0,0 +1,41 @@ + + In a write-through cache, a load to a word with an older store still waiting must wait behind that store, not hit the old data. + + + +Before: the load hits stale data +After: the load waits for the store + +read miss, line L +store X (waiting) + +load X (new) +hit: returns old X + + +read miss, line L +store X (waiting) +load X: linked behind + + +returns new X + +a waiting storenow counts asa hazard + + diff --git a/docs/building_toolchain.md b/docs/building_toolchain.md index 9508651472..7f38db077a 100644 --- a/docs/building_toolchain.md +++ b/docs/building_toolchain.md @@ -23,8 +23,10 @@ The components covered here are: 9. [Mesa for Vortex](#9-mesa-for-vortex-vulkan) — Vulkan software stack (lavapipe) the Vortex Vulkan driver builds on (optional) 10. [gem5](#10-gem5) — cycle-level CPU simulator hosting the Vortex SimObject (X86 + ARM); consumed by `ci/regression.sh --gem5` 11. [SST](#11-sst) — Structural Simulation Toolkit (sst-core + sst-elements + OpenMPI); consumed by the SST-driven regression configurations -12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C; needed only to elaborate a target or build a bitstream (optional) +12. [FireSim](#12-firesim) — FPGA-accelerated host-decoupled simulation on the Alveo U55C or V80; needed only to elaborate a target or build a bitstream (optional) 13. [SLASH / Alveo V80](#13-slash--alveo-v80-the-aved-backend) — AMD Alveo V80 platform stack for the `aved` backend (optional) +14. [CIRCT](#14-circt) — the SystemVerilog frontend of every synthesis flow (`circt-verilog`, `circt-opt`, `firtool`) +15. [Foldsim](#15-foldsim) — FAME-5 simulator compiler of `hw/syn/foldsim`, built on CIRCT (optional) --- @@ -880,15 +882,17 @@ rm -rf "$SST_CORE_HOME/include" "$SST_ELEMENTS_HOME/include" \ ## 12. FireSim **Purpose**: FPGA-accelerated, host-decoupled simulation of Vortex on a -Xilinx Alveo U55C. Consumed by the `firesim` driver +Xilinx Alveo U55C (the `vitis` platform, XRT) or an AMD Alveo V80 (the `aved` +platform, SLASH/VRT). Consumed by the `firesim` driver (`./ci/blackbox.sh --driver=firesim`) and by the bitstream build under `hw/syn/firesim`. See [`designs/firesim_integration.md`](designs/firesim_integration.md) for the architecture. -**Only needed to build.** Running an existing `.xclbin` requires XRT and the -Vortex runtime and nothing from `$TOOLDIR/firesim` — the transport links -against XRT directly. Install this component if you intend to elaborate a +**Only needed to build.** Running an existing image — a `.xclbin` with XRT, +a `.vbin` with VRT — requires the card's runtime and the Vortex runtime and +nothing from `$TOOLDIR/firesim`: the transport links against XRT or VRT +directly. Install this component if you intend to elaborate a target or build a bitstream. ### Installing @@ -899,10 +903,36 @@ it, and only an FPGA flow can use it. Install it by hand: ```bash git clone --branch vortex_3.x https://github.com/vortexgpgpu/firesim.git ~/dev/firesim -cd ~/dev/firesim && ./build-setup.sh --skip-validate +cd ~/dev/firesim && ./build-setup.sh # needs `conda` on PATH (e.g. Miniforge) ln -s ~/dev/firesim $TOOLDIR/firesim ``` +The checkout must contain the fork commit pinned as `FIRESIM_REV` in `VERSION`; +`make -C hw/syn/firesim stage` checks it. + +**From the prebuilt (faster).** `vortex-toolchain-prebuilt` carries the same +tree for ubuntu/focal with its conda environment already resolved — the part +of a setup that takes time — conda-packed, plus a shallow `.git` of the +`FIRESIM_REV` commit so `check-firesim` passes and `git fetch` works: + +```bash +REPO=https://github.com/vortexgpgpu/vortex-toolchain-prebuilt/raw/v3.0.1/firesim/ubuntu/focal +for x in a b c d e f g h i j k l m n o p q r s t u v; do + wget $REPO/firesim.tar.bz2.parta$x +done +cat firesim.tar.bz2.parta* | tar -C ~/dev -xjf - && rm firesim.tar.bz2.parta* +cd ~/dev/firesim/.conda-env +tar -xf firesim-conda-env.tar && rm firesim-conda-env.tar +python3 bin/conda-unpack # rewrite the packed prefixes for this location +ln -s ~/dev/firesim $TOOLDIR/firesim +``` + +The v3.0.1 bundle is 22 parts (`partaa`..`partav`). `env.sh` locates its own +directory, so the tree works wherever it is unpacked; `conda` must still be on +`PATH` for `ci/firesim_env.sh`. The first elaboration downloads sbt's own +compiler jars into `~/.cache/coursier` and `~/.sbt`, so it needs network +access once. + `vortex_3.x` is based on upstream tag 1.21.0. Clone it wherever you keep sources; `$TOOLDIR/firesim` only has to resolve to the checkout. @@ -947,8 +977,10 @@ make -C hw/syn/firesim check-firesim host with `libgmp-dev` installed, building the transport works without activating conda at all. - **Vortex's changes to FireSim live on the branch, not as patches.** - `git diff 1.21.0..vortex_3.x` is the authoritative delta — 8 files, mostly - Vitis platform fixes and cycle-counter accounting, all upstreamable. + `git diff 1.21.0..$FIRESIM_REV` is the authoritative delta: Vitis platform + fixes, the AVED platform for the V80 (shim, fabric clock gates, kernel + clock, partitioned synthesis of large models), the host-memory base as a + config field, driver build and teardown, and cycle-counter accounting. ## 13. SLASH / Alveo V80 (the `aved` backend) @@ -1094,6 +1126,107 @@ for the backend's architecture and [`xilinx_slash_setup.md`](xilinx_slash_setup.md) for board bring-up. +--- + +## 14. CIRCT + +**Purpose**: the SystemVerilog frontend of every synthesis flow. Its +preprocessor writes the sources each flow reads (`gen_sources.sh -P`, then the +RAM fold, `-F`; see the hardware library's "Block RAM and asynchronous reads"); +Vivado reads them as they are, and Yosys, Beacon and Foldsim read the netlist +CIRCT elaborates from them (`hw/scripts/circt_elaborate.sh`). + +> **Baseline:** Vortex's CIRCT fork, branch `vortex_3.x` of +> `vortexgpgpu/circt`: upstream CIRCT with the import, lowering and +> memory fixes the Vortex flows need. It pins its own LLVM (a submodule), +> separate from `llvm-vortex`: CIRCT tracks LLVM main. + +### Build + +`ci/circt_install.sh` builds LLVM, MLIR and CIRCT in one build and installs +the whole of it (tools, libraries, headers, CMake packages) into +`$TOOLDIR/circt`: + +```bash +./ci/circt_install.sh # CIRCT_REPO, CIRCT_REV, JOBS, CMAKE override the defaults +``` + +It needs cmake >= 3.28 (slang, CIRCT's SystemVerilog frontend, requires it) +and a C++20 compiler (GCC >= 11). Ubuntu 22.04 ships cmake 3.22: install a +newer one (`pip install cmake`) and point `CMAKE` at it if it is not first on +`PATH`. + +It configures with: + +```bash +cmake -G Ninja -S $CIRCT_SRC/llvm/llvm -B $CIRCT_SRC/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TOOLDIR/circt \ + -DLLVM_ENABLE_PROJECTS=mlir -DLLVM_EXTERNAL_PROJECTS=circt \ + -DLLVM_EXTERNAL_CIRCT_SOURCE_DIR=$CIRCT_SRC -DLLVM_TARGETS_TO_BUILD=host \ + -DLLVM_ENABLE_ASSERTIONS=OFF -DLLVM_STATIC_LINK_CXX_STDLIB=ON \ + -DLLVM_ENABLE_ZLIB=OFF -DLLVM_ENABLE_ZSTD=OFF -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_INSTALL_UTILS=ON -DCIRCT_SLANG_FRONTEND_ENABLED=ON \ + -DCIRCT_SLANG_BUILD_FROM_SOURCE=ON +``` + +### Notes + +- For a prebuilt, build once per `OSVERSION` in that OS's container + (`ubuntu/focal`, `centos/7`): assertions off and libstdc++ linked + statically, so the tools run on any machine of that OS. +- The prebuilt ships only `circt-verilog`, `circt-opt` and `firtool`. Building + Foldsim needs the full installation this script leaves behind. +- To develop CIRCT itself, build it in its own checkout with assertions on + and point the flows at it: `CIRCT_PATH=` (the build directory has + the same `bin/` layout). + +--- + +## 15. Foldsim + +**Purpose**: the compiler of the FAME-5 FPGA simulators (`hw/syn/foldsim`, +[designs/foldsim.md](designs/foldsim.md)): the `foldsim` driver and +`foldsim-opt`. Foldsim is its own project and builds on CIRCT, as POCL builds +on LLVM. + +> **Baseline:** branch `vortex_3.x` of `vortexgpgpu/foldsim`. + +### Build + +Build CIRCT first (§14), then: + +```bash +./ci/foldsim_install.sh # FOLDSIM_REPO, FOLDSIM_REV, CIRCT_PREFIX override the defaults +``` + +which configures Foldsim against `$TOOLDIR/circt`, builds it, runs its lit +tests (`check-foldsim`) and installs `bin/foldsim` and `bin/foldsim-opt` into +`$TOOLDIR/foldsim`: + +```bash +cmake -G Ninja -S $FOLDSIM_SRC -B $FOLDSIM_SRC/build \ + -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=$TOOLDIR/foldsim \ + -DCIRCT_DIR=$TOOLDIR/circt/lib/cmake/circt \ + -DMLIR_DIR=$TOOLDIR/circt/lib/cmake/mlir \ + -DLLVM_DIR=$TOOLDIR/circt/lib/cmake/llvm \ + -DLLVM_EXTERNAL_LIT=$TOOLDIR/circt/bin/llvm-lit \ + -DLLVM_STATIC_LINK_CXX_STDLIB=ON +``` + +### Notes + +- `foldsim-opt` links CIRCT statically; the driver finds it next to itself + and the CIRCT tools through `--circt-bin`. `config.mk` sets + `FOLDSIM_PATH ?= $(TOOLDIR)/foldsim` and `FOLDSIM_SHELL ?= + $(FOLDSIM_PATH)/share/foldsim/shell` (the FPGA shell the simulator runs in); + `hw/syn/common.mk` sets `FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim` and + `FOLDSIM_LIFT_TOOL ?= $(FOLDSIM_PATH)/bin/foldsim-lift`. +- A CIRCT change Foldsim's passes rely on lands in CIRCT first; both + prebuilts are refreshed together. +- The prebuilt is opt-in: `./ci/toolchain_install.sh --foldsim` (or `--all`) + installs it; the default install leaves it out, since only the Foldsim + flow needs it. + --- ## Verifying an installed toolchain @@ -1115,6 +1248,8 @@ $TOOLDIR/ │ ├── bin/ etc/ lib/libOpenCL.so* share/pocl/ │ └── include/CL/*.h (REQUIRED — see §6; absence breaks tests/opencl) ├── chipstar/ (bin/hipcc, lib/libCHIP.so, include/hip/) +├── circt/bin/ (circt-verilog, circt-opt, firtool) +├── foldsim/bin/ (foldsim, foldsim-opt; optional) ├── sta/ (OpenSTA, optional) ├── mesa-vortex/ (lavapipe Vulkan ICD; share/vulkan/icd.d/, optional) ├── gem5/ (slim runtime — build/{X86,ARM}/gem5.opt + configs/) @@ -1136,6 +1271,8 @@ $LLVM_PREFIX/bin/llvm-spirv --version # if SPIRV-LLVM-Translator installed $TOOLDIR/riscv32-gnu-toolchain/bin/riscv32-unknown-elf-gcc --version $TOOLDIR/verilator/bin/verilator --version $TOOLDIR/chipstar/bin/hipcc --version # if chipStar installed +$TOOLDIR/circt/bin/circt-verilog --version +$TOOLDIR/foldsim/bin/foldsim-opt --version # if foldsim installed test -f $TOOLDIR/pocl/include/CL/opencl.h && echo "POCL host headers OK" # if gem5 installed — both ISAs are built by default @@ -1167,11 +1304,14 @@ Both accept the same per-component flags: ``` --pocl --chipstar --verilator --riscv32 --riscv64 --llvm --mesa ---libcrt32 --libcrt64 --libc32 --libc64 --sv2v --yosys --sta +--libcrt32 --libcrt64 --libc32 --libc64 --circt --foldsim --yosys --sta --gem5 (slim runtime — see §10) --all (every component above; SST is NOT included — see §11) ``` +With no flag, `ci/toolchain_install.sh` installs every component but the +opt-in `foldsim` and `gem5`; `--all` adds them. + > **Note**: `--all` covers every component in this list except SST. > SST is intentionally not packaged as a prebuilt (autoconf bakes > absolute paths into `sstsimulator.conf` and the `sst` wrapper @@ -1184,7 +1324,7 @@ Both accept the same per-component flags: Run this **after** building the components above (or after rebuilding just one — e.g. POCL). For each selected component it `tar`s the component directory out of `$TOOLDIR`, bzip2-compresses it, and — for -the large ones (`riscv32`, `riscv64`, `llvm`, `verilator`, `yosys`) — +the large ones (`riscv32`, `riscv64`, `llvm`, `verilator`, `circt`, `yosys`) — `split`s the archive into 50 MB parts to stay under GitHub's per-file limit. The bundles are written **relative to the current directory**, into `.//$OSVERSION/` (`libc*` / `libcrt*` go straight into @@ -1215,7 +1355,7 @@ split parts) from the `vortex-toolchain-prebuilt` repo at the directory: ```bash -# install the full toolchain: +# install the toolchain (all but the opt-in foldsim and gem5): ./ci/toolchain_install.sh # or refresh a single component (e.g. after a new POCL release): diff --git a/docs/coding_guidelines_verilog.md b/docs/coding_guidelines_verilog.md index 3222a6d340..790601eb11 100644 --- a/docs/coding_guidelines_verilog.md +++ b/docs/coding_guidelines_verilog.md @@ -248,6 +248,9 @@ Strive for moderate combinatorial logic depths that balance latency with synthes ## 11. Reuse the Hardware IP Library Before writing new RTL, consult the hardware IP library in [hw/rtl/libs/](../hw/rtl/libs/) — the [hardware_library.md](hardware_library.md) reference catalogs the reusable, parameterized modules it provides: elastic buffers and flow control, arbiters, mux/demux, stream fork/join/pack/dispatch, crossbars and interconnect, encoders/decoders, arithmetic (multipliers, dividers, adders, CSA trees), RAM/FIFO primitives, memory adapters, and bit-manipulation utilities. Prefer instantiating an existing library module over hand-rolling equivalent logic: the library modules carry consistent valid/ready handshake semantics, inherit the FPGA/ASIC synthesis support, and are already verified, so reuse avoids duplicating tested logic and the subtle handshake/timing bugs that re-implementation invites. If a needed primitive is genuinely missing, add it to the library rather than embedding a one-off in a block. + +**Memories go through `VX_dp_ram` / `VX_sp_ram`.** Never hand-write a RAM array. Choose the read form the design needs; a combinational read (`OUT_REG=0`) of a block-RAM-sized `VX_dp_ram` becomes block RAM through the synthesis flows' RAM fold, which computes its address one cycle early from the registers that feed it, wherever they are. Keep that address computable: drive it from registers on the RAM's clock through assigns, ports, interfaces and functions, not from a top-level input or an `always_comb` block; connect the modules on its path by name; and give a RAM whose read form is a parameter one instantiation per form, since the fold edits the instantiation every instance shares. `/ram_fold.rpt` names every RAM the fold kept and why. See [Block RAM and asynchronous reads](hardware_library.md#block-ram-and-asynchronous-reads). + ## 12. Module & Interface Declarations & Instantiations Declare and instantiate modules **and parameterized interfaces** with one diff --git a/docs/designs/atomic_memory_operations.md b/docs/designs/atomic_memory_operations.md index 065ff80f0c..500aac5ff8 100644 --- a/docs/designs/atomic_memory_operations.md +++ b/docs/designs/atomic_memory_operations.md @@ -8,7 +8,7 @@ cache bank and the local-memory bank), the reservation stations, and the SimX model. RTL: [`VX_amo_alu.sv`](../../hw/rtl/cache/VX_amo_alu.sv), [`VX_amo_unit.sv`](../../hw/rtl/cache/VX_amo_unit.sv), [`VX_cache_amo.sv`](../../hw/rtl/cache/VX_cache_amo.sv), -[`VX_local_mem.sv`](../../hw/rtl/mem/VX_local_mem.sv). SimX: +[`VX_lmem_unit.sv`](../../hw/rtl/mem/VX_lmem_unit.sv). SimX: [`sim/simx/amo/`](../../sim/simx/amo/), [`sim/simx/mem/cache.cpp`](../../sim/simx/mem/cache.cpp), [`sim/simx/mem/local_mem.cpp`](../../sim/simx/mem/local_mem.cpp). @@ -90,7 +90,7 @@ LSU as `rs3_data` with no new operand path. ## 3. The LSU sideband -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) packs one `amo_req_t` +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) packs one `amo_req_t` per lane into the lane's memory attribute ([`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)): @@ -321,7 +321,7 @@ deferring, at most, one cycle longer than strictly needed. ## 7. Commit at the local-memory bank Shared (local) memory is a per-core SRAM, not a cache, so there is no line -to fetch and no level below: [`VX_local_mem.sv`](../../hw/rtl/mem/VX_local_mem.sv) +to fetch and no level below: [`VX_lmem_unit.sv`](../../hw/rtl/mem/VX_lmem_unit.sv) commits in place. [`VX_lmem_switch.sv`](../../hw/rtl/mem/VX_lmem_switch.sv) passes the attribute through unchanged on the local path. diff --git a/docs/designs/aved_driver_architecture.md b/docs/designs/aved_driver_architecture.md index 9f8715e495..748a9ef2be 100644 --- a/docs/designs/aved_driver_architecture.md +++ b/docs/designs/aved_driver_architecture.md @@ -258,7 +258,7 @@ refused with `CTL_RESET_ERROR`, correctly, because that read really is still outstanding; it reads like a reset bug and is not one. This happened on 2026-08-27 and cost a bitstream and three reboots. The cause -was `platforms.mk` pinning `MEM_TAG` but not `HOST_TAG`, leaving the Makefile's +was the card description pinning `MEM_TAG` but not `HOST_TAG`, leaving the Makefile's `HOST_TAG ?= HOST` to win. Both are pinned now, and `HOST_TAG=HOST` is refused at Makefile parse time rather than after two and a half hours of Vivado. Before blaming a hardware symptom on RTL, diff the build's generated `config.cfg` @@ -391,15 +391,11 @@ threads. `xrt` and `aved` both guard it. ## 8. Open directions -1. **A software reset for the CP.** `q_reset_pulse` is decoded in the regfile - and discarded in `VX_cp_core`. Wiring it would remove the §3.5 workaround - and make a hung kernel recoverable without reconfiguring the partition — - today a kernel that never completes leaves the CP unusable. -2. **Unify the two sync engines** (§4.3) behind one publish/refresh interface +1. **Unify the two sync engines** (§4.3) behind one publish/refresh interface with a pluggable transport. -3. **Drop the VRT-internal reach** once VRT exposes the simulated memory +2. **Drop the VRT-internal reach** once VRT exposes the simulated memory windows and the ZMQ server through public API. -4. **Gate the build on timing closure.** The V80 flow packages a bitstream +3. **Gate the build on timing closure.** The V80 flow packages a bitstream that does not meet timing without saying so. --- @@ -425,5 +421,6 @@ loopback (`minimal -l`) remains the sim-side check of the CP DMA path. Simulated host buffers are drawn from VRT's DDR window, not its HBM window, so they cannot collide with device memory. The authoritative settings and -their rationale are in -[`platforms.mk`](../../hw/syn/xilinx/aved/platforms.mk). +their rationale are in the card descriptor +[`v80.toml`](../../hw/syn/xilinx/platforms/v80.toml), which both the `aved` and +`xrt` flows read through [`gen_platform.py`](../../hw/scripts/gen_platform.py). diff --git a/docs/designs/beacon.md b/docs/designs/beacon.md new file mode 100644 index 0000000000..29d1448b04 --- /dev/null +++ b/docs/designs/beacon.md @@ -0,0 +1,831 @@ +# Beacon — logic depth and area, before Vivado + +Beacon counts the logic levels of every register-to-register path +of a Vortex design the way Vivado counts them, and the LUTs, flip-flops, +distributed and block RAM and DSPs it maps to, in seconds and without +Vivado. It names the RTL units whose paths are too deep for the target clock, +gates their depth and the design's area on every RTL change (`logic_gate`), +and is the first step of RTL timing work: run Beacon, fix the deep units, +then confirm with one Vivado run. + +| § | contents | +|---|---| +| [1](#1-overview) | Overview and how to run it | +| [2](#2-system-architecture) | System architecture: where each piece lives | +| [3](#3-compiler-architecture) | Compiler architecture: the two passes | +| [4](#4-the-model) | The model: levels per element, devices, area (§4.5) | +| [5](#5-outputs) | Outputs | +| [6](#6-integration) | Integration: the DUT flow, `logic_gate`, timing work | +| [7](#7-validation) | Validation against Vivado; the learned calibration (§7.1); area (§7.2) | +| [8](#8-diagnostics) | Diagnostics beyond levels | +| [9](#9-invariants-and-limitations) | Invariants and limitations | + +--- + +## 1. Overview + +It answers one question per register: **how many logic levels is its worst +input path?** At 300 MHz on the V80, failing end points have a median of +7–8 levels, and between half and four fifths of the registers at 12 or more +levels fail, depending on the gate (§7). So Beacon reports: + +- **fix list:** units with end points at ≥ 12 levels; +- **watch list:** ≥ 8 levels. + +It does not predict slack, because it does not see wires (§9). + +One DUT of the synthesis catalog, from a configured build tree: + +```bash +make -C hw/syn/xilinx/dut tcu-beacon +# build_tcu/beacon_paths.txt every path at >= 8 levels, op by op, by unit +# build_tcu/beacon_units.json deepest end point per RTL unit, area per unit +# build_tcu/beacon_area.txt LUTs, FFs, memories and DSPs, by unit; each memory's placement +``` + +The whole catalog as a gate, and a baseline update (a human step): + +```bash +ci/logic_gate.py -j 2 +ci/logic_gate.py -b om --update-baseline +``` + +Any CIRCT netlist, with `ci/beacon.py` (subcommands `run`, +`summarize`, `paths`, `validate`, `retime-split`, `ports`, `buffers`, +`calibrate`, `score`; +`foldsim-opt` from the `PATH` or `FOLDSIM_OPT`): + +```bash +ci/beacon.py run design.mlir runs/full # map, then count levels and area +ci/beacon.py run runs/full/mapped.mlirbc runs/socket \ + top=VX_socket__lift black-boxes=VX_core__lift # one unit of a mapped design +ci/beacon.py summarize runs/full --min-levels 12 # deep end points by RTL unit +ci/beacon.py paths runs/full --unit lsu_unit # its paths, op by op (§5) +ci/beacon.py validate runs/socket ep.csv --native VX_socket__lift=dut +``` + +Or the passes directly: + +```bash +foldsim-opt design.mlir --emit-bytecode -o mapped.mlirbc \ + --pass-pipeline='builtin.module(foldsim-beacon-map{target=virtexuplusHBM})' +foldsim-opt mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module( + foldsim-estimate-timing{top=VX_core_top target=virtexuplusHBM + output=endpoints.csv instances=instances.csv report=report.txt + top-paths=50 trace=lsu_unit})' +foldsim-opt design.mlir -o /dev/null --pass-pipeline='builtin.module( + foldsim-beacon-map{target=virtexuplusHBM area}, + foldsim-estimate-area{top=VX_core_top target=virtexuplusHBM + output=area.csv memories=area_memories.csv report=area.txt})' +``` + +| `foldsim-estimate-timing` option | meaning | +|---|---| +| `top=` | the top module (default: the one nothing instantiates) | +| `target=` | the Beacon target (§4.2): `versal` (V80, default), `virtexuplus`, `virtexuplusHBM` (U55C), `zynquplus`, or `asap7` (ASIC, §4.3); map with `foldsim-beacon-map` for the same target | +| `black-boxes=` | modules to treat as external, as a harness stubs them | +| `output=`, `instances=`, `report=` | the files of §5 | +| `min-levels=` | write only end points at or above this many levels | +| `top-paths=` | deepest paths to print op by op (default 50) | +| `trace=` | also print the deepest path of each end point whose name contains this | +| `paths=`, `path-levels=` | every path of at least `path-levels` (12) levels, one per pair of registers, op by op (§5) | +| `retime-levels=` | also estimate the depth after Vivado's retiming to this period (§4.4) | +| `ports=`, `port-modules=` | the depth at each port of the named modules' instances (§8) | + +| `foldsim-estimate-area` option | meaning | +|---|---| +| `top=`, `target=`, `black-boxes=` | as for `foldsim-estimate-timing`; map with `foldsim-beacon-map{area}` for the same target | +| `output=` | the instance table of §5 (`area.csv`) | +| `memories=` | every memory, its shape and where it goes (`area_memories.csv`) | +| `report=` | the design's totals, the mapping's LUTs by op, the vendor IP | +| `lut-scale=` | device LUTs per mapped LUT (default: the target's, §4.5) | +| `modules=`, `state-threads=` | per module, its totals and its state's cost folded over that many threads (JSON; foldsim's fold-degree search prices copies with it) | + +## 2. System architecture + +![Beacon system](../assets/img/beacon_system.svg) + +The engine is two compiler passes of **foldsim** (`foldsim-opt`), built on +CIRCT like the rest of foldsim, and shipped in the foldsim prebuilt. Vortex +never builds it. The integration is in Vortex: + +| Piece | Where | Role | +|---|---|---| +| `foldsim-beacon-map`, `foldsim-estimate-timing`, `foldsim-estimate-area` | foldsim `lib/Transforms/Beacon.cpp`, `EstimateTiming.cpp`, `EstimateArea.cpp` | map to the target's cells; count levels; count area | +| `ci/beacon.py` | `ci/` | run Beacon on any netlist; read it by unit; compare with Vivado | +| `xilinx_cone_dump.tcl`, `xilinx_cone_probe.tcl`, `xilinx_wire_probe.tcl` | `hw/scripts` | Vivado probes for what levels cannot show (§8) | +| target `beacon` | `hw/syn/xilinx/dut/common.mk` | one DUT: sources → netlist → levels and area → summary | +| `circt_elaborate.sh -f mlir` | `hw/scripts` | the lowered HW/Comb/Seq netlist, no `firtool` | +| `gen_platform.py --emit=family` | `hw/scripts` | the family of `DEVICE`, from `hw/syn/xilinx/platforms` | +| `beacon_summary.py` | `hw/scripts` | `beacon_summary.csv`, per-unit `beacon_units.json`, `beacon_area.txt` | +| `Beacon` backend, `logic_gate.py` | `ci/synth_gate.py`, `ci/` | the gate, beside `fpga_gate` and `asic_gate` | +| `logic_gate.yml` | `.github/workflows` | runs it on pull requests that touch RTL | + +Beacon reads the same CIRCT import that foldsim and the Yosys flow +use (`circt-verilog --ir-hw`), elaborated from the sources and defines +`gen_sources.sh` writes for Vivado, with the Xilinx floating-point IPs as +black boxes (`xilinx_ip_stubs.sv`). + +## 3. Compiler architecture + +![Beacon compiler](../assets/img/beacon_compiler.svg) + +### 3.1 `foldsim-beacon-map` + +Every pass loads the same **target** from foldsim's target table +(`foldsim/Beacon.h`): its cells for the mapping, its DSPs and delays for the +level count, its LUT scale for the area (§4.5). A target is named after its FPGA family or its PDK, so every +flow passes the name it already has: the DUT flow the family +`gen_platform.py --emit=family` gives for `DEVICE`, the ASIC flow `PDK`, +`ci/beacon.py run` its `target=`. The mapping is a pipeline of CIRCT's +synthesis passes plus three of foldsim's, per module: + +1. **Inline small modules** (`hw-flatten-modules`, < 300 ops) so LUTs pack + across buffers, arbiters and switches, as Vivado's flattened synthesis + does. Larger modules stay instances; the estimator elaborates them. +2. **Canonicalize and balance** mux chains (`comb-balance-mux`). +3. **foldsim passes:** + - `foldsim-lower-equality`: an equality compare is XORs and an OR tree, + which pack into LUTs; + - `foldsim-lower-constant-division`: a division by a constant is an + extract (power of two) or a table (dividend of ≤ 10 bits), as + synthesis writes it; + - `foldsim-fold-zero-width`: zero-width values are constants, which + CIRCT's lowering otherwise rejects. +4. **Lower to an and-inverter graph** (`convert-comb-to-synth`). Adders, + subtracters, compares, multipliers, dividers and remainders stay + word-level: the estimator models them as the carry chains, DSPs and + tables Vivado builds, which a LUT mapping of their gates would not + reproduce. +5. **Depth-optimize** the graph: timing-aware variadic lowering, maximum + AND cover, SOP balancing, structural hashing. +6. **Map to the target's cells** (`synth-generic-lut-mapper`, timing + strategy): 6-input LUTs on the FPGA families, 3-input cuts on ASAP7. + +The full GPU maps in about a minute and 18–20 GB. + +With `area`, the same lowering maps the design for `foldsim-estimate-area` +instead. A depth-oriented cover duplicates logic to save levels, so its +cells are no measure of area; the area mapping is what Vivado does off the +critical paths: + +- every module stays an instance, so each one's resources stay its own + (inlining moves the count by 1–4 %), and each module maps once however + many instances it has; +- no mux balancing, no maximum AND cover and no SOP balancing; +- variadic logic lowered sharing common subsets (`reuse-subsets`), and the + cut mapper's area strategy. + +A DUT of the catalog maps for area in 1–16 s. + +### 3.2 `foldsim-estimate-timing` + +The estimator works on the elaborated design: one **context** per instance, +and one **node** per (context, value, bit). It never enumerates paths: + +1. **Collect end points** by walking the instance tree from `top`. +2. **Resolve** each end point bit through wires, extracts, concatenations, + replicates and instance ports to the op that drives it. +3. **Evaluate** with an iterative depth-first walk: a node's levels are the + maximum over its inputs of the input's levels plus the edge's (§4), and + it keeps its one worst predecessor. An input still being evaluated + closes a combinational loop; that edge is left out and counted. +4. **Report**: each end point's start point is found by walking the worst + predecessors back; the walk also yields the path's largest fanout and its + module crossings. + +Time and memory are linear in the design's bits: the full GPU runs in +16–21 s and 2.7 GB, a DUT of the catalog in seconds. + +### 3.3 `foldsim-estimate-area` + +The area estimator counts each module's own ops once (§4.5), then sums the +instance tree from `top`: a module's total is its own resources and its +instances' totals. It writes one row per instance, depth first, with the +instance's own resources and its subtree's, and one row per memory with the +placement chosen for it. It takes under a second on a DUT. + +## 4. The model + +### 4.1 Levels per element + +| Element | Levels | +|---|---| +| LUT (`comb.truth_table`) | 1 | +| Adder or subtracter, input bit j to output bit i | 2 + ⌊i/8⌋ − ⌊j/8⌋: a LUT, then the 8-bit carry blocks; 1 if the output bit reads ≤ 6 input bits (one LUT6) | +| Order compare | as an adder, up to the highest bit not constant in both operands | +| Equality compare | lowered to LUTs by the mapping | +| Multiply, ≤ 10 variable operand bits | a table of constants: 1 + ⌈(bits − 6)/2⌉ | +| Multiply, wider | a DSP cascade over the significant bits (§4.2) | +| Divide or remainder | a table over the bits its operand cones depend on, if ≤ 10; else a ripple of one subtracter per quotient bit | +| Distributed-RAM read (`seq.firmem`, latency 0) | 1, plus 1 per doubling past 64 words (MUXF7, MUXF8, LUT), from the address | +| Other ops | 1 per op, listed in the report | + +Significant bits exclude constant bits on top and copies of the bit below +(a sign extension). A table is a LUT6 for six inputs and a level of 4:1 +multiplexers per two more. + +### 4.2 Devices + +Versal and the UltraScale+ families all have LUT6s, 8-bit carry blocks +(LOOKAHEAD8, CARRY8) and 64-word distributed RAMs; they differ in their DSPs. +The target is selected by `target=`, with the family names of +`hw/syn/xilinx/platforms`: + +| `target=` | Device | DSP multiplier | fabric → registered product | product → fabric | per cascaded DSP | +|---|---|---|---|---|---| +| `versal` | V80 | DSP58, 27 × 24 | 1 | 6 | 3 | +| `virtexuplus` | U200, U250 | DSP48E2, 27 × 18 | 1 | 5 | 2 | +| `virtexuplusHBM` | U50, U55C, U280 | DSP48E2, 27 × 18 | 1 | 5 | 2 | +| `zynquplus` | Zynq UltraScale+ | DSP48E2, 27 × 18 | 1 | 5 | 2 | + +A product takes ⌈a/A⌉·⌈b/B⌉ DSPs (the better orientation). The cell counts +are Vivado's: on Versal, M_DATA, ALUMUX, ALUADD and OUTPUT out of a DSP, +and ALUMUX, ALUADD and OUTPUT through each cascaded one. A register fed by +the product is the DSP's own (MREG or PREG), and so is a LUT that reads that +register back, its clock enable or feedback; any other load pays the output +stage. The fabric-to-product value is fitted on the V80 data. A name that +is not a target is an error that lists the targets. + +### 4.3 ASAP7 + +For the ASIC flow (`hw/syn/yosys`, ASAP7 7.5-track RVT, TT corner) +Beacon counts **cell stages**, not LUT levels. `target=asap7` +maps the same balanced and-inverter graph to 3-input cuts, each one cell +(NAND, AOI, MAJ, XOR), and models the word-level operators +as an ASIC builds them: + +| Element | Stages | +|---|---| +| cell (`comb.truth_table`, 3 inputs) or unmapped and-inverter gate | 1 | +| adder, input bit j to output bit i | 2 + ⌈log₂(i − j + 1)⌉: a prefix adder | +| order compare | 1 + ⌈log₂ span⌉ | +| multiply | partial products, a Dadda tree of full adders (2 stages each), a prefix adder | +| table of k inputs (small products and dividers) | k (a multiplexer tree) | +| memory of flops (no macro) | 1 + ⌈log₂ depth⌉: a decoder and a multiplexer tree | + +SRAM macros (`VX_dp_ram_asic`, `VX_sp_ram_asic`) are black boxes, as in the +flow's synthesis: their pins end and start paths. The report converts the +deepest path to a period: + +``` +deepest path 45 stages: 58 + 167 + 45 x 24 + 6 = 1311 ps (763 MHz) + clk→Q path stages × cell setup +``` + +The cell (24 ps at fanout 3 and its own slew), clock-to-output (58 ps) and +setup (6 ps) delays come from the Liberty, by `hw/syn/yosys/liberty_stage.py`; +the path term (167 ps: the inverter after a DFFHQN's QN, buffering, slew) is +fitted on OpenSTA (§7). Re-derive them for another corner or threshold +voltage. + +### 4.4 Retiming (optional) + +All of Vortex's Vivado runs retime. With `retime-levels=N`, the `retimed` +column estimates each end point's depth after one hop of balancing to a +period of N levels. The end register moves backward into the room its +output stage leaves, and the start register moves forward into the room its +input stage leaves. Neither crosses a distributed RAM or a DSP, and only a +register Vivado can retime (synchronous reset) moves. Top ports and +black-box pins count as registers with empty neighbour stages, as a gate +harness makes them. It ranks no better than plain levels, so `levels` stays +the ranking; `retimed ≥ 12` at `retime-levels=10` is a short list of fix +candidates retiming is unlikely to absorb. + +### 4.5 Area + +The area mapping (§3.1) first takes out of the netlist what Vivado keeps +out of LUTs: + +- **Clock enables and synchronous resets** (`foldsim-register-enables`). A + select between a register's next value and the register itself is the + flip-flop's CE pin, and a select of a constant under it the R or S pin; + neither takes a LUT (Vivado's `VX_pipe_register`s with reset and enable + have none). The register becomes a `seq.compreg.ce`; its enable and reset + are one gate per register, not per bit. +- **Register bits that never leave 0** (`foldsim-constant-registers`). A + flip-flop powers up at 0, so a bit that only loads 0 or itself, and resets + to 0 if at all, is the constant 0, and the register narrows to its other + bits (`VX_cache_amo`'s write-back offset, of which only bit 5 varies, makes + its shift a select of two amounts). +- **Bits a select passes through** (`foldsim-narrow-muxes`). A procedural + write of part of a vector in a loop elaborates to a select per iteration + of the whole vector, whose arms agree on every bit outside that part; + lowered to and-inverter gates, each such bit stays a pair of gates and + every iteration's condition reaches it. `VX_cache_amo`'s read-forward + loop over the L2's 512-bit word (64 bytes of 4 write-back slots and the + settling entry) is a chain of 320 whole-word selects, every one reaching + every bit. Each select narrows to the runs where its arms differ. +- **Styled ROMs** (`foldsim-styled-roms`). A constant table on a wire with + `rom_style` `block` or `distributed` becomes a memory of its shape, placed + as below (`VX_mufu`'s table: one block RAM, as Vivado places it, instead + of 12,321 mapped LUTs of logic). + +Then, per op: + +| Element | FPGA | ASAP7 | +|---|---|---| +| LUT (`comb.truth_table`) | 1 LUT | 1 cell | +| Adder or subtracter | a LUT per bit into the carry chain | a prefix adder: w · (2 + ⌈log₂ w⌉) cells | +| Order compare | a LUT per two bits | 2 cells per bit | +| Multiply by a constant | a LUT per bit (shifts and adders) | 3 cells per bit | +| Multiply, ≤ 10 variable operand bits | a table: a LUT per output bit (at most as many as the operands' variable bits), per 64 entries past six bits | the same in cells | +| Multiply, wider | DSPs: the fewer of ⌈a/A⌉·⌈b/B⌉ in either orientation over the significant bits (§4.2) | partial products and a Dadda tree: 3·a·b cells | +| Divide or remainder (output bits: at most the dividend's) | a table over ≤ 10 variable bits; a constant divisor, 2 LUTs per bit; else a subtracter per quotient bit, a LUT per result bit | the same, but 3 cells per subtracter bit | +| Register | a flip-flop per bit | a flip-flop per bit | +| Vendor IP (`xil_*`) | its logic LUTs, flip-flops and DSPs: for `xil_fma`, `xil_fdiv` and `xil_fsqrt` the medians of their routed instances in the U55C builds (616, 743, 412 LUTs; 1,064, 1,360, 786 flip-flops) | — | +| SRAM macro (black box) | — | a macro | + +**Memories** go where Vivado infers them. A `ram_style` the RTL asks for wins +(`VX_dp_ram` writes `block` for `FORCE_BRAM` sizes and `distributed` for +`LUTRAM`; the RAM fold, `ram_fold.py`, registers the read of the RAMs it +folds, so `VX_dp_ram` writes `block` for them too). Otherwise a +read of latency 0 is distributed RAM, and a registered read of at least 32 +words and 2 Kb is block RAM. The shapes are the same on every FPGA family: + +| Placement | Cost | +|---|---| +| distributed RAM | 0.57 LUTs per bit up to 32 words (RAM32M16), 1.14 per 64 words beyond (RAM64M8), per read port, plus a multiplexer tree over the 64-word LUTs | +| block RAM | the fewest 36 Kb blocks of any shape (512 × 72 … 32K × 1) or 18 Kb halves; a copy per two ports, a read-write port one of them | +| UltraRAM (`ultra`) | 4096 × 72 each | +| `registers` | flip-flops and a read multiplexer | + +Two shapes override the inference: + +- A memory distributed RAM cannot hold is **flip-flops**: one read of + latency 0 at each of its words, at constant addresses (an associative + lookup, as the MSHR's `addr_table`, `word_table` and `sector_table` are), + or more than seven read ports besides a read-write port. Its + constant-address reads are wires (in a copy folded over threads, a + multiplexer over the threads), every other read a multiplexer per bit, + and each word a write enable on its flip-flops' CE pins. The L2 MSHR goes from 629 to 348 distributed RAM LUTs + and from 167 to 567 flip-flops per bank, against Vivado's 352 and 579. +- A block RAM writes 8- or 9-bit bytes under a mask, so a memory masked in + **lanes that are not whole bytes** (neither a multiple of 8 nor of 9 bits) + is a block RAM per lane: the L2 tag store (64 × 136 in eight 17-bit + lanes) is eight 18 Kb halves, the I-cache's (64 × 80 in four 20-bit lanes) + four, as Vivado builds them. +- A memory without a write port is a **ROM**: in distributed form, logic + (a table over the address per bit and read, nothing for one word), not + distributed RAM, which is how Vivado reports it. A memory nothing reads + costs nothing. + +On ASAP7, a memory without a macro is flip-flops, a write decoder and a +read multiplexer tree. + +**The LUT scale.** The cut mapper has none of Vivado's LUT combining or +wide multiplexers (MUXF7/F8), and maps module by module, without the logic +Vivado trims across module boundaries, so it covers a design with 2.6× the +logic LUTs Vivado leaves at the geometric mean, from 1.4× (`lmem`) to 7.5× +(`rtu`) (§7.2). The target's `lutScale` +converts the mapping's LUTs to Vivado's: 0.391 on the LUT6 families, the +geometric mean over the 14 routed U55C `fpga_gate` DUTs of Vivado's logic +LUTs, less what Beacon prices outside the mapping (vendor IP, memory +multiplexers), over Beacon's mapped LUTs (§7.2). The V80 shares that value +without a fit of its own, and `asap7` has none (1.0): its cell counts are +not yet compared with Yosys. Vendor IP and distributed RAM, and the +memories' read multiplexers, are not scaled; `lut-scale=1` gives the raw cover. + +## 5. Outputs + +| File | Contents | +|---|---| +| `endpoints.csv` | one row per end point bit: `levels,retimed,kind,start_kind,fanout,crossings,start_inst,start,end_inst,end` | +| `instances.csv` | the instance tree: `id,parent,instance,module` (the `*_inst` columns' ids) | +| `report.txt` | counts, combinational loops, ops without a model, the histogram of end points by levels, the `top-paths` deepest paths op by op, and the deepest path of each `trace=` match | +| `beacon_summary.csv` | the DUT flow's one row: max levels, end points ≥ 12, ≥ 8, all, and the area | +| `paths.jsonl` | `paths=`: one JSON line per pair of start and end registers whose deepest bit has at least `path-levels` levels: that bit's levels, the pair's bit count, start, end, and `ops`, the path from the end back to the start as `[levels, kind, fanout, instance id, name, source]` | +| `beacon_units.json` | the deepest end point of every unit at ≥ 8 levels, and the 10 deepest paths | +| `beacon_paths.txt` | the DUT flow's `paths.jsonl` at ≥ 8 levels, by unit (`ci/beacon.py paths` for any run) | +| `area.csv` | one row per instance, depth first: `depth,path,module`, its own `lut,lutram,ff,bram,uram,dsp,macros`, then its subtree's (`total_*`); `lut` is logic LUTs, scaled (§4.5) | +| `area_memories.csv` | one row per memory: its instance, name, depth, width, read, write and read-write ports, read latency, `ram_style`, placement and cost | +| `area.txt` | the design's totals, the mapping's LUTs by op before scaling, and the vendor IP | +| `beacon_area.txt` | `area.txt`, the units with ≥ 1 % of the LUTs (one copy's subtree, × copies), and every memory with its placement | + +`beacon_summary.csv` also holds the design's `lut` (distributed RAM +included, as Vivado's utilization counts it), `ff`, `lutram`, `bram`, +`uram` and `dsp`, and `beacon_units.json` an `area` map of every unit's +resources (one copy's subtree) and copy count. + +A path in `report.txt`, deepest first, one op per line: levels at the op, +its kind, the fanout of its bit, and its instance path and name: + +``` +#1 levels 20: .../mem_req_queue/pending_size/used_r.d[0] (VX_socket__lift) + 20 lut fo 0 .../comb.truth_table[0] + 19 carry fo 1 .../comb.add[0] + ... + 0 reg fo 59 .../req_pending/in_fire_r[13] +``` + +`report.txt`'s deepest paths are often one pair of registers, bit after bit. +The path list has **every** deep path once: + +- one per pair of start and end registers, its deepest bit, with the + number of bits it stands for; +- pairs whose names differ only in generate indices (`g_banks_0`, + `g_banks_1`) and bits fold into one, as units fold them, so a unit the + gate names is a heading of the list; +- units deepest first, and in each unit its paths deepest first; +- each op with the **source line** it came from. Mapped LUTs have no name + of their own, so the file and line are what locate them in the RTL. The + DUT flow elaborates `hw/rtl` itself (not preprocessed copies), so the + lines are the repository's. + +``` +== cache/g_cache_cache/g_banks_*_bank/g_amo_amo: 24 levels, 23 paths + + 24 levels, 1024 d bits in 4 copies + from cache/g_cache.cache/g_banks_0.bank/g_amo.amo/g_commit.chain_match_r[0] + to cache/g_cache.cache/g_banks_0.bank/g_amo.amo/g_commit.wbq_data.d[6] + 24 lut fo 0 VX_cache_amo.sv:211 .../g_amo.amo/comb.truth_table[0] + 22 lut fo 1 VX_cache_amo.sv:700 .../g_amo.amo/comb.truth_table[0] + ... + 10 carry fo 839 VX_cache_amo.sv:689 .../g_amo.amo/comb.sub[1] + ... + 0 reg fo 2 VX_cache_amo.sv:773 .../g_commit.chain_match_r[0] +``` + +On the full GPU, the 3,283 pairs at ≥ 12 levels fold into 169 paths in +69 units. + +## 6. Integration + +### 6.1 The DUT flow + +`hw/syn/xilinx/dut` builds every DUT of `catalog.mk` for Vivado; its +`beacon` target runs Beacon on the same DUT with the same sources, defines and +`DEVICE` (the U55C by default): + +1. `gen_sources.sh` into `beacon/sources.f`, a filelist of `hw/rtl` with + the defines (not preprocessed copies, so source lines are the + repository's); +2. `circt_elaborate.sh -f mlir` into `beacon/netlist.mlir`, the FP IPs as + black boxes, with every op's source location; +3. `foldsim-beacon-map`, then `foldsim-estimate-timing`; and + `foldsim-beacon-map{area}`, then `foldsim-estimate-area`; all with + `target=` the family `gen_platform.py --device=$(DEVICE) --emit=family` + names; +4. `beacon_summary.py`: `beacon_summary.csv`, `beacon_units.json`, + `beacon_paths.txt` and `beacon_area.txt` (§5). + +The ASIC flow has the same target, with the same sources, defines and SRAM +black boxes as its synthesis, and `target=$(PDK)`: + +```bash +make -C hw/syn/yosys/dut om TARGET=beacon +# om_VX_om_core_top/beacon/report.txt: the deepest paths and the period +``` + +### 6.2 `logic_gate` + +The third synthesis gate, beside `fpga_gate` (Vivado, nightly) and +`asic_gate` (Yosys + OpenSTA, nightly); see +[continuous_integration.md §3.5](continuous_integration.md). It runs Beacon on +`fpga_gate`'s builds (`ci/testcases/logic_gate.yaml`, all but `top`, whose +`VX_cp_core` keeps LLHD reference ports) in about four minutes, and checks +each against `ci/baselines/synthesis/beacon/`: + +- a **unit** that gets deeper fails, by name; +- a **new unit** at or above the fix threshold (12) fails; +- a unit that gets shallower, or leaves the watch list, is an improvement + to lock in by re-recording; +- end point counts are recorded, not gated (they grow with lanes and warps); +- a **resource** of the design (`lut`, `ff`, `lutram`, `bram`, `uram`, + `dsp`) more than 5 % above its baseline fails, `fpga_gate`'s LUT + tolerance (catalog `thresholds` and `--metric-threshold` override it); + 5 % below is an improvement to lock in. A memory that moves between + distributed and block RAM shows as both columns moving. + +A failing build names its `beacon_paths.txt`, where the failing unit is a +heading with its paths op by op, and its `beacon_area.txt`; CI uploads them +with the build's logs. + +Depth is deterministic, so the tolerance is zero. Area is as +deterministic, but a feature adds some by design, so it has the Fmax gate's +tolerance. The baseline fingerprint +is the CIRCT and foldsim versions, `DEVICE` and `XLEN`: a new foldsim reads +as STALE, not as a regression. Baselines are golden data, re-recorded only +by a human (`--update-baseline`), as for the other gates. + +### 6.3 RTL timing work: Beacon first + +Beacon is the inner loop of any RTL change that can move timing. A DUT takes +minutes in Beacon and hours in Vivado or Yosys, so a deeper path is found and +fixed while the change is still open, and synthesis runs once, to confirm. + +1. **Reference.** Beacon on the parent revision, once per base, in a build + tree of its own: + + ```bash + git worktree add ../ref HEAD + ../ref/ci/logic_gate.py -b core -b lmem --build-dir $PWD/build32_beacon_ref + ``` + +2. **Change.** The same builds on the working tree, each compared with the + reference register by register: + + ```bash + ci/logic_gate.py -b core -b lmem --against build32_beacon_ref + ``` + + Besides the per-unit verdict against the golden baseline, each build + writes `beacon_diff.txt` (`ci/beacon.py diff`). Its headline is the + deepest register, the mean of the 50 deepest and the count at the watch + threshold, before and after; then it lists the registers that got deeper, + new deep registers, and the ones that got shallower, then the units whose + deepest register moved. With area, the headline adds the LUTs, block RAMs + and DSPs before and after, and a section lists each resource of the + design and the units whose own area moved, all copies; a resource more + than 5 % larger fails the run (`AREA`). The run fails (`DEEPER`) on a register 4 or more + levels deeper, one that reaches the fix threshold (12) from below, or a new + one at it. These limits come from 13 routed U55C builds: a register 4 or + more levels deeper lost slack in Vivado in every case (170 of 170), while + 1 to 3 levels is within the placer's noise, so those are listed as + "deeper by less" to watch, not failed. Of the headline numbers, the top-50 + mean follows Vivado's worst slack (Spearman 0.56) and the single deepest + register does not (0.29). A new register below 12 is listed but passes, + since a rename looks the same. + +3. **Fix.** `ci/beacon.py paths /beacon --unit ` prints a deep + unit's paths op by op. Edit, and repeat step 2 until nothing got deeper, + or until each deeper path is one the change intends. + +4. **Confirm.** Run synthesis once per change, not once per edit: the + `fpga_gate` or `asic_gate` builds of the DUTs whose depth moved. Then feed + the result back. `ci/beacon.py validate /beacon ep.csv` compares the + run with Vivado's end point table, and a disagreement there is a Beacon bug + to fix in foldsim. + +Beacon counts logic levels and cells; it does not see wire delay, +placement, SLR crossings or congestion. A change that moves those (a wide +fan-out, a floorplan) still needs synthesis before it lands. Compare a fix +against a native gate (`hw/syn/xilinx/dut` or the OOC gates), not the folded +design (§9). + +## 7. Validation + +Against Vivado, per RTL register or memory: Beacon's deepest bit +against Vivado's deepest failing path and worst slack, on seven V80 datasets +with no new Vivado runs (`ci/beacon.py validate`): + +| Dataset | Vivado run | Spearman vs levels | Spearman vs slack | ≥ 12: precision / recall | +|---|---|---|---|---| +| full GPU, folded | 4.0 ns, routed, 270 k end points | 0.75 | 0.45 | — | +| socket | native gate, 4.0 ns, routed | 0.55 | 0.53 | 0.64 / 0.31 | +| D-cache | native, 4.0 ns, routed | 0.71 | 0.55 | 0.67 / 0.38 | +| top | native gate, 3.333 ns, placed | 0.53 | 0.39 | 0.19 / 0.53 | +| socket | native gate, 3.333 ns, placed | 0.50 | 0.33 | 0.52 / 0.18 | +| core | native gate, 3.333 ns, placed | 0.93 | 0.62 | 0.81 / 0.13 | +| compute slice | native gate, 3.333 ns, placed | 0.87 | 0.75 | 0.58 / 0.57 | + +Beacon's depths run about 1.25× Vivado's on the native gates. The U55C +profile was checked against `fpga_gate`'s critical paths: it agrees with +the V80 profile wherever no DSP is on the path, and the remaining +disagreement is the same on both devices. + +**ASAP7**, against OpenSTA on fresh syntheses of five DUTs (Yosys + ABC as +`hw/syn/yosys` builds them, flip-flops named after their registers, +`hw/syn/yosys/sta_endpoints.tcl` for the slack of each): + +| DUT | Registers | Spearman, stages vs arrival | Worst path: OpenSTA → Beacon | +|---|---|---|---| +| om | 93 | 0.91 | 827 → 879 ps | +| cache | 526 | 0.82 | 1450 → 1311 ps | +| dxa | 169 | 0.56 | 1018 → 1215 ps | +| raster | 134 | 0.58 | 879 → 879 ps | +| tex | 1002 | 0.93 | 840 → 807 ps | + +Per register, the fitted delay per stage is 17–25 ps, the library's 24. +The worst path is within a mean 8 % on these five, from which the path term +was fitted; on the six larger DUTs, the predicted Fmax is within a mean +13 % (−32 % to +3 %) of the asic_gate baselines, some of which predate the +current RTL. Beacon runs a DUT in 2–52 s, where Yosys and OpenSTA take +2.5–37 min. + +### 7.1 Learned calibration (advisory) + +Levels rank Vivado's near-critical registers at about half the ceiling set +by one Vivado run predicting another. `ci/beacon.py score` ranks them with a +calibration learned from routed U55C builds instead: gradient-boosted trees +on 38 per-register features, all from Beacon's own outputs (no Vivado input): + +- **depth:** levels, mean levels over the bits, the share of bits within one + level of the deepest; +- **width:** bits, and bits within one level of the deepest; +- **end and start kind:** D, RAM write data or control, RAM read; register, + port, BRAM, IP; +- **fan-out:** the deepest start's, the cone's (`in_fanout`), the largest + along the deepest path, nets of 64 or more loads on it; +- **span:** hierarchy depth, crossings, the bits under the lowest common + instance of the register and its deepest start; +- **the deepest path's op mix** (`paths.jsonl`, at 8 levels or more): levels + spent in LUT, carry, compare, LUTRAM, BRAM and DSP ops, distinct + instances and source files, and the levels with a carry level costed at + 0.42 of a LUT level and a compare at 0.15. + +The model is three tree ensembles: the worst slack (capped at 1 ns, the +`ep.csv` horizon), the probability of slack under 0.3 ns (near-critical) and +under 0 ns. A register never ranks safer for being deeper (a monotone +constraint on levels). `calibrate` trains them with scikit-learn and writes +them as plain JSON trees; `score` evaluates them in pure Python, in about +4 s for a core. The model trained on the 13 builds below ships as +[`hw/scripts/beacon_calibration_u55c.json`](../../hw/scripts/beacon_calibration_u55c.json); +it applies to the U55C at 300 MHz, the target it was trained on. + +```bash +ci/beacon.py score build_core/beacon hw/scripts/beacon_calibration_u55c.json \ + --top 30 --csv score.csv +ci/beacon.py calibrate cal.json --holdout \ + --build build_core/beacon build_core/ep.csv core \ + --build build_cache/beacon build_cache/ep.csv cache +``` + +Each `--build` is a DUT build's Beacon run (`path-levels=8`, as the DUT flow +writes it) and the end point table of the same sources. `--holdout` prints +the check below for the given builds. + +**Validation.** 13 routed U55C builds at 3.333 ns: 8 core revisions, 4 lmem, +1 cache, 19 k registers. Each build is held out and scored by a model +trained on the builds outside its group. Revisions of one RTL lineage share +most of their registers, so they are held out together: + +- core with the coretiming lineage (origin, it1, it2, lsuagu); +- core with the lmemwarp lineage (v1, v2, final, p2lsulmem); +- cache; +- lmem. + +The stricter test holds out the whole DUT (core trained on cache and lmem +only). + +| held out | near-critical AP: levels | levels + 0.35·log2(fan-out) | carry/compare cost | model, lineage out | model, DUT out | Spearman vs −slack: levels / model | +|---|---|---|---|---|---|---| +| core origin | 0.36 | 0.38 | 0.41 | 0.72 | 0.65 | 0.15 / 0.64 | +| core it1 | 0.40 | 0.42 | 0.42 | 0.75 | 0.65 | 0.18 / 0.63 | +| core it2 | 0.44 | 0.46 | 0.48 | 0.77 | 0.68 | 0.24 / 0.68 | +| core lsuagu | 0.68 | 0.70 | 0.70 | 0.80 | 0.71 | 0.33 / 0.54 | +| core lmemwarp v1 | 0.37 | 0.40 | 0.48 | 0.75 | 0.56 | 0.27 / 0.66 | +| core lmemwarp v2 | 0.44 | 0.50 | 0.54 | 0.82 | 0.61 | 0.35 / 0.73 | +| core lmemwarp | 0.37 | 0.41 | 0.52 | 0.81 | 0.59 | 0.32 / 0.64 | +| core p2lsulmem | 0.55 | 0.63 | 0.60 | 0.87 | 0.65 | 0.40 / 0.78 | +| cache | 0.47 | 0.57 | 0.52 | 0.65 | 0.65 | 0.50 / 0.56 | +| **mean** | **0.44** | **0.50** | **0.52** | **0.77** | **0.64** | **0.30 / 0.65** | + +On the three lmem builds, with 4–5 near-critical registers each, the AP goes +from 0.02–0.03 to 0.63–0.68. One Vivado run predicting another of nearly +the same RTL (it1, it2) reaches 0.80. AP for failing registers goes from +0.21 to 0.44; that set is mostly placement noise at a WNS near −0.1 ns. + +- **Width carries most of it.** Shuffling the bit counts costs 0.29 of AP. + This is partly selection: a wide register shows the worst of many bits. + Within registers of one deep bit, though, the AP still goes from 0.47 to + 0.79. +- **Span, the op mix and fan-out carry the rest** (0.03–0.08 each). The op + mix matters most on the cache (0.27). +- **The DUT-out column is the honest number for a new DUT.** The lineage-out + column is the one for a new revision of a DUT already trained on. +- **It does not track a whole design's slowdown.** The mean of the 50 lowest + predicted slacks follows the change in WNS over 10 revision pairs worse + than the top-50 mean of levels: Spearman 0.15 (lineage out) or 0.42 (DUT + out), against 0.53. The lsuagu regression, −1.3 ns, moves it by 0.02 ns. + +So the model is a ranking for where to look first, not a gate. `diff` and +`logic_gate` keep gating on levels. + +### 7.2 Area + +Against the routed `fpga_gate` builds of the same sources (U55C, Vivado +2024.2.2, 1 November 2026): + +Logic LUTs are Vivado's logic LUTs (without distributed RAM and shift +registers) against Beacon's at the fitted scale; *before* is the estimator +before the rules of §4.5 above, at its earlier scale of 0.337 and IP costs. +`top` leaves out the command processor (`cp_core`) on both sides: CIRCT +elaborates it with LLHD signals Beacon does not read. + +| DUT | LUT ratio before | LUT (Beacon / Vivado) | FF | distributed RAM LUTs | block RAM | DSP | +|---|---|---|---|---|---|---| +| cache | 1.19 | 27,313 / 28,317 (0.96) | 34,922 / 25,280 (1.38) | 2,542 / 2,722 | 32 / 32 | 0 / 0 | +| core | 0.86 | 122,879 / 165,478 (0.74) | 158,903 / 146,008 (1.09) | 6,838 / 6,220 | 129 / 137 | 97 / 96 | +| dxa | 1.05 | 8,170 / 9,666 (0.85) | 9,215 / 7,132 (1.29) | 568 / 638 | 0 / 0 | 9 / 7 | +| gfx | 1.45 | 241,768 / 210,879 (1.15) | 379,837 / 236,308 (1.61) | 11,906 / 12,834 | 191.5 / 172.5 | 1,140 / 992 | +| lmem | 0.68 | 14,766 / 26,322 (0.56) | 14,361 / 13,623 (1.05) | 726 / 740 | 16 / 16 | 0 / 0 | +| mufu | 2.92 | 1,387 / 1,966 (0.71) | 1,806 / 1,105 (1.63) | 0 / 0 | 1 / 1 | 5 / 3 | +| om | 2.37 | 4,730 / 2,298 (2.06) | 4,456 / 3,496 (1.27) | 367 / 408 | 0 / 0 | 10 / 22 | +| raster | 1.00 | 5,044 / 7,981 (0.63) | 12,069 / 8,974 (1.34) | 496 / 606 | 8 / 8 | 14 / 14 | +| rtu | 2.01 | 99,325 / 63,721 (1.56) | 146,917 / 94,885 (1.55) | 1,904 / 1,840 | 16.5 / 18.5 | 140 / 140 | +| tcu | 1.04 | 121,388 / 115,727 (1.05) | 49,056 / 42,675 (1.15) | 915 / 1,360 | 0 / 0 | 769 / 768 | +| tensor | 1.12 | 220,251 / 225,309 (0.98) | 166,688 / 138,986 (1.20) | 8,962 / 7,842 | 116.5 / 133.5 | 267 / 265 | +| tex | 1.50 | 71,506 / 55,791 (1.28) | 96,949 / 34,582 (2.80) | 3,241 / 4,208 | 0 / 0 | 896 / 528 | +| top | 1.04 | 104,728 / 125,511 (0.83) | 157,573 / 144,522 (1.09) | 8,551 / 7,836 | 76 / 76 | 50 / 48 | +| vm | 1.55 | 109,070 / 116,552 (0.94) | 151,175 / 121,587 (1.24) | 10,276 / 10,366 | 168 / 172 | 0 / 0 | + +- **LUTs**: the geometric mean error over the 14 DUTs goes from 41 % to + 31 %, the range from 0.68–2.92 to 0.56–2.06. The L2 of `vm` came to + 105,071 LUTs before (311,783 mapped) and comes to 47,555 now (121,625 + mapped), against Vivado's 54,150; each bank's `VX_cache_amo` to 14,733 + before and 3,608 now, against 5,001. The scale is one number per family, + so a DUT whose logic Vivado trims or packs more or less than the average + moves by that much; `logic_gate` compares a DUT with its own baseline, so + the scale cancels there. The largest + residues are logic Vivado removes across module boundaries, which Beacon, + mapping module by module, keeps: `rtu`'s logic outside its 73 + floating-point cores is 19,536 LUTs and 18,088 flip-flops in Vivado against 53,970 LUTs + (138,032 mapped) and 68,357 flip-flops in Beacon; `tex` keeps 2.8× + Vivado's flip-flops; `om`'s `VX_om_dcr` and `VX_om_blend` come to 3,407 + LUTs against Vivado's 741. `lmem`, `raster` and `core` come out low. +- **Flip-flops** run 5–63 % high, and 2.8× on `tex`, for the same reason: + Vivado drops the bits a parent never reads or ties to a constant (the + cache DUT's stream buffers, 20,936 flip-flops in Beacon against 12,324), and moves pipeline registers into DSPs and block RAM + output registers, none of which a module-by-module count sees. +- **Distributed RAM** is within 15 % on 10 of the 13 DUTs that have any + (`raster` 0.82, `tex` 0.77, `tcu` 0.67). +- **Block RAM** is exact on 5 of the 10 DUTs that have any; the mask-lane + rule matches Vivado on every masked tag store. The RTU scheduler's + `window_store` (64 × 512 in eight 64-bit lanes) packs in 7.5 blocks, its + lanes being whole bytes, where Vivado builds a 36 Kb block per lane, 8. + `gfx` is 19 high: Beacon places four RTU scheduler memories without a + `ram_style` (`fbuf_ram`, `ray_store`, `xfres_ram`, `trires_ram`, 23.5 + blocks) in block RAM by its size and latency rule, and Vivado does not; + `core` and `tensor` are short by 8 and 17, not yet traced. +- **DSPs** are within two on 11 DUTs; `gfx` and `tex` are 148 and 368 high + and `om` 12 low, where Vivado chooses between DSPs and LUTs for narrow + products by a heuristic of its own. + +The repository's Vivado baselines (`ci/baselines/synthesis/xilinx/`) date +from 5 October 2026 and predate later RTL changes, so they are not a +reference for this table. + +## 8. Diagnostics beyond levels + +**Port depths.** `port-modules=` names modules whose instances' ports +Beacon reports: the levels at which each input arrives, the levels behind +each output, and `through`, the bits of each output that a combinational +path from outside the instance reaches. A module that registers its +outgoing interface has `through` 0 on every output; `logic_gate` gates it +for the modules a build lists under `interfaces` (`BEACON_INTERFACES` in +the DUT flow). Small modules are inlined by the mapping, so keep them with +`run --keep`: + +```bash +ci/beacon.py run --keep VX_elastic_buffer design.mlir runs/eb \ + port-modules=VX_elastic_buffer +ci/beacon.py buffers runs/eb # the OUT_BUF advisor +ci/beacon.py ports runs/eb # every kept module, by unit +``` + +- The **buffer advisor** lists the `VX_elastic_buffer`s that pass a deep + ready chain through combinationally (`ready_in`, upstream, driven at + least as deep as `ready_out` arrives from downstream), where a skid + buffer (SIZE 2) or `OUT_BUF` would cut it, and those whose output is + computed rather than registered, where `OUT_REG` would register it. It + is evidence for an RTL review: the change also moves cycles, so it comes + with the matching SimX timing update. +- For foldsim, the ports of the class modules (`VX_socket__lift`, + `VX_core__lift`, ...) show the outputs computed from logic, and how deep, + rather than registered: in the fold they cost host cycles on the class's + links. + +**Vivado probes.** + +`hw/scripts` has three Vivado Tcl probes for questions +Beacon cannot answer, run on a checkpoint +(`vivado -mode batch -source -tclargs ...`): + +| Probe | Answers | +|---|---| +| `xilinx_cone_dump.tcl` | the worst path into a register, cell by cell, following retimed start registers back | +| `xilinx_cone_probe.tcl` | whether paths exist between two register sets, and how deep Vivado built them | +| `xilinx_wire_probe.tcl` | per path: length in tiles, bounding box and fanout beside logic and net delay | + +## 9. Invariants and limitations + +- **Depth, not slack.** On the placed 3.333 ns socket gate, wires are 83 % + of the worst paths' delay; levels explain 1 % of net delay, and the real + placement two thirds. No pre-placement wire term ranked slack better than + levels on every held-out gate, so Beacon has none. Route-dominated + failures with few levels are invisible to it: recall at ≥ 12 levels is + 0.13–0.57. The learned calibration (§7.1) ranks near-critical registers + better from width, span and fan-out, but it is trained on three U55C DUTs + and is advisory. +- **Retiming and packing.** Vivado retimes and packs across what the RTL + separates: a popcount, its adders and a compare become a LUT6 tree with a + retimed register inside. Beacon's depths are RTL stage depths, about + 1.25× Vivado's. +- **The fold.** In foldsim's folded designs, the fold turns some registers + into LUTRAM and cuts some chains, so its paths differ from the RTL's. The + native gates are the reference for native RTL. +- **Black boxes.** Vendor IPs and `black-boxes=` modules end and start + paths at their pins: their insides are not timed. Vendor IP's area is + its measured cost per instance; a `black-boxes=` module's is left out. +- **Area is a count, not a utilization report.** The mapping is a cut + mapper's cover scaled to Vivado's average (§4.5, §7.2), memories go by + Vivado's inference rules, and what Vivado packs into DSP, block RAM and + SRL cells Beacon counts as fabric. Logic that nothing observable reads is + counted; Vivado removes it. +- **Devices.** The V80 and the U55C (Vivado), and ASAP7 RVT TT (the Yosys + flow). The ASAP7 numbers are Beacon's period estimate, a sizing + guide, not a sign-off: they come within about 10–30 % of OpenSTA. +- **Toolchain.** The catalog maps only with three CIRCT Synth fixes + (zero-width mux, constant SOP cut, a failing longest-path query), which + reach CI through the foldsim prebuilt. diff --git a/docs/designs/build_configuration_system.md b/docs/designs/build_configuration_system.md index ccc4a9a3bb..79c9523bf5 100644 --- a/docs/designs/build_configuration_system.md +++ b/docs/designs/build_configuration_system.md @@ -322,14 +322,14 @@ mix of knobs and computed results. RTL reads `` `VX_CFG_* `` and `` `VX_MEM_* `` macros directly; there is no typed-`localparam` configuration package. One was built and reverted: -- **sv2v drops wildcard re-exports.** The Yosys ASIC flow converts through - sv2v, which discards `export pkg::*` symbols — the flow failed on - `L3_CACHE_SIZE` and its siblings. +- **sv2v drops wildcard re-exports.** The Yosys ASIC flow converted through + sv2v at the time, which discards `export pkg::*` symbols — the flow failed + on `L3_CACHE_SIZE` and its siblings. (The flow now elaborates with CIRCT.) - **Package compile order is tool-divergent.** Verilator is strict, Vivado auto-sorts, VCS is order-sensitive. Backtick macros are the one representation every tool handles identically. -Do not re-attempt a package without first solving sv2v re-export. +Do not re-attempt a package without first checking re-exports in every flow. ### 9.2 Superseded directions diff --git a/docs/designs/cache_subsystem.md b/docs/designs/cache_subsystem.md index 0ed3e50e98..8fc8e61c06 100644 --- a/docs/designs/cache_subsystem.md +++ b/docs/designs/cache_subsystem.md @@ -224,6 +224,22 @@ rather than being a field of its own. Two consequences are worth internalizing: pipeline carrying only `word_idx` and still tell the tag and data stages which sector to mark and write. +**Hashed bank select (`BANK_HASH`).** With the bank taken straight from the +address bits, any power-of-two stride that is a multiple of `NUM_BANKS × LINE` +lands every access in one or two banks. With `BANK_HASH=1` the bank is the raw +bank bits XORed with `cache_bank_hash` of the line address above them +([VX_gpu_pkg.sv](../../hw/rtl/VX_gpu_pkg.sv)): IPOLY for 16 and 32 banks, which +spreads every power-of-two stride evenly, and an XOR fold of the higher bits for +other bank counts. The same mask XORed again restores the raw bits, so it is +applied once on the request side (`core_req_bid`) and once where the memory +address is rebuilt from `{line, bank}`. The banks keep storing the line address +without the bank field, so tags, MSHR, the flush walk and sectoring are +unchanged. `VX_CFG_DCACHE_BANK_HASH` and `VX_CFG_L2_BANK_HASH` set it for the +dcache and the L2; SimX uses the same functions. The L2 default is on; the dcache +default is on from 4 banks, because with two banks the hash can only swap lines on +parity and the extra parallel misses cost more in memory queuing than the +spreading saves. + The memory side transacts in sectors, so the memory address carries `CS_SECTOR_SEL_BITS` below the line address (`CS_MEM_SECTOR_ADDR_WIDTH`, `CS_LINE_SECTOR_ADDR_WIDTH`). With one sector per line these collapse back to the @@ -395,7 +411,10 @@ to new requests. This removes the read-after-fill round trip from miss latency. The window closes on the **first write or AMO head**. That head and every later chain entry replay through the pipeline as usual, preserving program order: an -older store must merge before a younger same-line read responds. +older store must merge before a younger same-line read responds. Stores still +waiting in the chain keep its line pending, on a write-through cache too, where +they have already been written through: a same-word access that arrives while +they drain links behind them instead of hitting the array ahead of them. Two details make it correct: @@ -449,6 +468,35 @@ Replay is the mirror image: it gates only when **not** `WRITEBACK` and the replayed request is a write (`replay_rw`), because a write-through bank forwards every write downstream and so does need a slot for it. +### 4.9 Requests in flight and device `busy` + +Stores are ack-less, so a core goes idle while its last requests are still +on their way out, and the console rings (`VX_MEM_IO_COUT`) are read right +after a launch retires, with no flush in between. Device `busy` therefore +covers every request the cores issued until memory has completed it, the way +a GPU's outstanding-write counters do: + +- the core stays busy while a request waits at its dcache port, past the flush + injector's buffer; +- every place outside the banks where a request can wait reports it through + `req_empty`: `VX_cache_bypass` counts from its uncached inputs and the + cache's memory side to its memory ports, `VX_cache_cluster` adds its + core-side and memory-side arbiters, and the socket memory arbiter and the OM + steers do the same. Each counter is a `VX_mem_req_pending` over the + section's input and output handshakes, which holds because these sections + neither create nor drop requests; the socket, cluster and device fold them + into `busy`; +- the platform wrapper extends `busy` past the device's memory ports to the + point where a write is visible to the host's DMA: acceptance by the bank in + `rtlsim_shim`, the AXI B response in `Vortex_axi`. The OPAE AFU needs + neither, since its DMA queues behind Vortex's writes on the same in-order + Avalon path. + +Cacheable requests left in the banks are covered by the next flush instead +(§6). A request's next holding point reaches `busy` through no more registers +than the one it leaves, so the coverage handed from one counter to the next has +no gap. + --- ## 5. Parallelism and scaling @@ -511,11 +559,22 @@ The same machinery is reused for reset-time tag initialization. ![Flush fan-out tree](../assets/img/cache_flush_tree.svg) -The flush is triggered by a DCR **read** of `VX_DCR_BASE_CACHE_FLUSH` — a read, -not a write, because the read's *response* is what acknowledges completion. -[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) latches `flush_pending_r`, -holds `dcr_flush_if.req` high, and returns the DCR response only once `done` -comes back. +The command processor flushes in two phases, because the DCR response bus has +no back-pressure and two cores answering in the same cycle would lose one: + +1. one DCR **write** to `VX_DCR_BASE_CACHE_FLUSH`. Writes reach every core, so + all cores start flushing together; a write has no response; +2. a DCR **read** of the same address per core, one at a time. The read's + *response* acknowledges that core's completion, at once if it has already + finished. + +[VX_dcr_data.sv](../../hw/rtl/core/VX_dcr_data.sv) keeps three bits per core: +`flush_pending_r` (running; holds `dcr_flush_if.req` high), `flush_complete_r` +(finished, not yet reported) and `flush_read_r` (a read waiting). A write opens +a new flush and clears `flush_complete_r`, so an older completion cannot answer +it. A read with no write before it starts the flush itself, so a requester that +only reads still works. All cores' tokens reach the shared levels together, +where one walk serves them all. From `VX_core` the request fans out to six possible destinations through six `VX_dcr_flush` instances — dcache, icache, tcache, rtcache, rcache, ocache — @@ -532,18 +591,17 @@ holding every `done` high until `req` drops is what lets each parent simply AND them together race-free. A pulse would need a latch at every junction and would drop legs that finished early. -**The icache flush is sequenced behind the dcache flush:** +**The icache flush stays local** (`FLUSH_FORWARD=0`): it holds no dirty data, +and the dcache's flush already walks and invalidates every shared level the +icache reads through. So the two L1s flush concurrently, and only the dcache's +tokens reach the next level. + +The dcache flush waits for the core's stores to leave the LSU: ```systemverilog -assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; +assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; ``` -Both L1s forward their flush to the shared next level, and a cache that is -flushing locks out incoming core requests for its whole sweep. The icache carries -no dirty data and so retires almost immediately; if it ran concurrently, its -forward would arrive at the next level while the dcache is still evicting, and -that level's lock would strand the dcache's writebacks upstream of memory. - The icache is flushed at all because a kernel re-loaded to the same virtual address after a `CACHE_FLUSH` would otherwise execute stale lines from the previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — see @@ -552,8 +610,20 @@ previous launch. The host issues `CMD_CACHE_FLUSH` after every kernel launch — ### 6.2 VX_dcr_flush — injecting the request `VX_dcr_flush` synthesizes a degenerate `MemReq` (`rw=0, addr=0, data=0, -byteen=0, attr = 1 << MEM_ATTR_FLUSH_OFFS`) and merges it into the cache's port 0 -stream through a 2:1 `VX_mem_bus_arb`. +byteen=l1_only, attr = 1 << MEM_ATTR_FLUSH_OFFS`), a **flush token**, and merges it +into one cache input stream through a 2:1 `VX_mem_bus_arb`. Byteen bit 0 +marks an **L1-only** token (§6.4; a receiving cache tests for any byteen bit, +since a level with wider words sees the bit at the token's line offset); `VX_dcr_data` latches it from the START +write's `VX_CACHE_FLUSH_L1_ONLY` bit, and the dcache and icache injectors carry +it. The ocache, whose tokens reach the L2, sends an L1-only token only when +every requester of its flush round is L1-only: the cluster flush (kind latched +from the same broadcast write) and the dcache tokens held at the OM steers. +The other graphics caches do not forward and send full tokens. A memory fence +sends an L1-only token. A fence ordering only device accesses sends an **IO** +token (the `io` attribute): no level walks, each quiesces its banks, waits +until nothing is held on its uncached path either, and forwards an IO token +down every memory port, so the answer proves the earlier IO writes reached +memory. Kinds visible together combine to the weakest that serves them all. **Core traffic sits on the priority input, not the flush.** The flush waits until upstream is quiescent — which it reliably is, because the host only issues @@ -567,41 +637,108 @@ the first completion until the initiator drops `req`. Without that latch, a shar `req` across several instances would make the fast ones re-flush forever while waiting on the slow one. -Only **port 0** of the dcache is routed through `VX_dcr_flush` -([VX_mem_unit.sv:381](../../hw/rtl/core/VX_mem_unit.sv#L381), `i==0 && j==0`). The -input lock inside `VX_cache_init` is what propagates the freeze to every other -port. +**Every buffered write path gets its own token.** A token only orders the +requests ahead of it on its own path. A private dcache takes the lanes straight +from the LSU adapter, whose drain `store_drained` already waits for, so one +token on lane 0 suffices. Otherwise the lanes are buffered independently (a +shared dcache's socket arbiter, or a disabled dcache's passthrough), so each +lane gets its own `VX_dcr_flush` ([VX_mem_unit.sv](../../hw/rtl/core/VX_mem_unit.sv)) +and their level-held dones AND together. Each OM input of the ocache likewise has its own token +([VX_graphics.sv](../../hw/rtl/VX_graphics.sv)). The read-only rcache keeps a +single token. ### 6.3 End-to-end sequence ![Flush sequence](../assets/img/cache_flush_sequence.svg) -### 6.4 VX_cache_init — the input lock +### 6.4 VX_cache_init — the flush sequencer -`VX_cache_init` is a 5-state FSM (`IDLE → WAIT1 → FLUSH → WAIT2 → DONE`) sitting -between the cache's input ports and the request crossbar. +`VX_cache_init` sits between the cache's input ports and the request crossbar. +Its FSM is `IDLE → WAIT1 → FLUSH → WAIT2 → FORWARD → RESPOND`. ```systemverilog -wire input_enable = ~flush_req_enable || lock_released[i]; +wire input_enable = ~flush_req_enable; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; -assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; +assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); ``` -While a flush is in flight every input port presents `valid=0` downstream and -`ready=0` upstream, so upstream stalls. Only the port that originated the flush is -unlocked (`lock_released_n = flush_req_mask`), so its `MemReq` enters the cache and -generates the acknowledging response; the rest unlock on the return to IDLE. +While any token is visible at any input, every input presents `valid=0` +downstream and `ready=0` upstream, so upstream stalls. - `STATE_WAIT1` waits for `BANK_SEL_LATENCY × NUM_BANKS` outstanding crossbar requests to drain. When there is no crossbar latency this state is bypassed and IDLE goes straight to FLUSH. - `STATE_FLUSH` is one cycle: it pulses `flush_begin` to **all** banks at once. - `STATE_WAIT2` accumulates `flush_done |= flush_end` until every bank has - reported. - -That lock is the load-bearing piece for correctness. Once engaged, no core request -can reach any bank pipeline — which is what lets the per-bank walk assume the only -in-flight entries are its own. + reported. The tokens visible at that point are the ones this walk answers; + a token that arrives later starts the next walk. +- `STATE_FORWARD` (only with `FLUSH_FORWARD=1`) flushes the next level: once + the memory-request crossbar has drained (`mem_req_empty`), the cache sends + **one token on every memory port** and waits for every port's response. The + token for port *i* takes bank *i*'s lane into the request crossbar (bank *i* + maps to port *i*), so it leaves through the same registered outputs as the + write-backs ahead of it. +- `STATE_RESPOND` answers the tokens one per cycle. Each answer carries the + token's own tag, so upstream arbiters route it back, and takes bank 0's lane + into the response crossbar whenever bank 0 has no response of its own; the + token is consumed at its input as its answer is sent. Then the inputs reopen. + +**L1-only tokens.** Between kernels nothing outside the GPU reads memory, so +the last-level cache, the point of coherence, need not be written back. A +token with a byteen bit set asks only the levels above the LLC to flush: + +- a non-LLC level walks as usual (write-back levels write back, write-through + levels invalidate, so the next kernel sees other cores' and clusters' + updates) and forwards the token on every port **with the same kind** + (`mem_flush_l1_only`, byteen bit 0 of the forwarded token). Forwarding is what + lands a write-through level's last stores in the next level before the + token is answered; +- a last-level cache that alone serves every memory client (`SHARED_LLC`: + the L3; the L2 of a single cluster; the dcache of a single core with no + texture, ray-tracing, DXA, raster or OM unit, since those reach memory below + or beside it) goes from IDLE (or WAIT1) straight to RESPOND, with no walk and + no forward: once the token is visible, its input's earlier requests are + inside the cache, ahead of anything the next kernel sends. Any other LLC + (per-core dcaches without an L2, per-cluster L2s without an L3, a dcache + beside fixed-function or DXA ports) walks like any other level: its lines + may be stale copies of another instance's writes, or dirty data a client + beside it must read from memory. `Vortex.sv` asserts that a write-back LLC + has a single instance. + +The kind of a walk is L1-only only if every token it answers is, so a full +token is never answered by an L1-only walk. A token without the bit behaves +exactly as before. The runtime issues a full flush before any transfer to or +from device memory, because the host DMA path bypasses the LLC. + +**Why a token per memory port.** Each port is a separate, separately buffered +path to the next level, and nothing orders one port's data against another +port's token. A token sent on one port only could reach the next level while +another port's last writes are still queued in front of it; that level's walk +would release the token, and the writes would enter after the walk and stay +dirty. With one token per port, each token sits behind its own port's data in +that port's FIFO path, so the next level answers it only once that data is in. + +**Why a late token is safe.** A token becomes visible only at the head of its +input. Once any token is visible, every input is blocked, so a token that +appears during the walk had all of its input's earlier requests accepted before +the walk started. + +**Recognizing the token responses.** During `STATE_FORWARD` the cache is +quiescent: inputs blocked, MSHRs empty, no misses outstanding, and writes get no +responses. So every response on a memory port is a token response; they are +recognized by phase, with no extra tag bit. Non-cacheable requests are steered +around the cache by `VX_cache_bypass` and never reach this interface. + +**IO ordering.** `VX_cache_bypass` merges the cache's memory requests with +non-cacheable (IO) requests in a round-robin per-port arbiter, and the IO path +is unbuffered up to it. An IO request waiting there holds its own input, so the +token behind it on that input is not yet visible: a later walk answers it, and +that walk's tokens follow the IO request. A private dcache's lanes 1+ carry no +token, and `store_drained` waits for them before the core injects its token. + +**Passthrough levels** (a disabled cache) need nothing: `VX_mem_bus_arb` maps +input *i* to output *i* mod `MEM_PORTS`, so each port's stream, token included, +stays in order on one output. ### 6.5 VX_cache_flush — the per-bank walk @@ -613,7 +750,7 @@ A 6-state FSM per bank, about 14 flip-flops: | `STATE_IDLE` | wait for `flush_begin` | | `STATE_WAIT1` | wait for `mshr_empty` **and** `bank_empty` | | `STATE_FLUSH` | counter walk; drives `flush_valid`, `flush_line`, `flush_way`; increments on `flush_ready` | -| `STATE_WAIT2` | **every bank except bank 0** waits for `bank_empty` | +| `STATE_WAIT2` | wait for `bank_empty`: the write-backs must leave the bank before the tokens go out | | `STATE_DONE` | one-cycle `flush_end` pulse | **The counter width depends on the write policy:** @@ -627,18 +764,6 @@ since `do_flush = flush && (!WRITEBACK || way_en)` drops the way gate. Write-bac walks each `(way, line)` separately because each dirty line emits its own memory request. -**`STATE_WAIT2` is every bank *except* bank 0.** Bank 0 goes straight to DONE: - -```systemverilog -state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; -``` - -Bank 0 is the canonical egress that propagates the flush downstream to lower -levels, and that request must be issued **last**. The other banks therefore hold -their `flush_end` until their own writebacks have drained; since `VX_cache_init` -waits for the AND of all `flush_end`, bank 0's downstream forward cannot be -overtaken. - `flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall` gates the counter, so under write-back with a near-full egress queue the walk pauses until it drains. @@ -676,8 +801,9 @@ tag/data write ports and the existing memory-request queue. | Component | Storage | |---|---| | `VX_cache_flush` | 3 bits state, 1 bit pending, ~10 bits counter — ~14 FF/bank | -| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, `NUM_REQS`-bit `lock_released`, UUID register | -| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter | +| `VX_cache_init` | 3 bits state, `NUM_BANKS`-bit `flush_done`, 2 × `MEM_PORTS`-bit token tracking, `NUM_REQS`-bit answer tracking, UUID register | +| `VX_dcr_flush` | 2 bits (`flush_inflight_r`, `flush_done_r`) + a 2:1 arbiter, per write path | +| `VX_cache` | a request-crossbar in-flight counter (forwarding caches only) and 2:1 muxes on bank 0's response lane and on the first `MEM_PORTS` request lanes | | tag / data RAM | **no extra storage** | | mreq queue | **no extra entries** | @@ -704,18 +830,20 @@ primitive. ### 6.8 Correctness invariants -1. **Atomicity against normal traffic.** Once a flush is in flight no core request - reaches a bank pipeline — enforced by the `VX_cache_init` input lock. +1. **Atomicity against normal traffic.** While any token is visible no core + request reaches a bank pipeline — enforced by the `VX_cache_init` input block. 2. **The bank is fully quiesced before the walk starts.** `STATE_WAIT1` waits for `mshr_empty && bank_empty` — both, not just the MSHR. Otherwise a fill could install a fresh line behind the walk pointer and survive the flush. 3. **Reset-time valid bits are zero** without async reset on the tag SRAM, via the `STATE_INIT` walk. This is what lets the tag array map cleanly to BRAM. The bank's highest-priority `init_valid` masks every other source while it runs. -4. **All banks finish before the cache acks** — `VX_cache_init` waits for the AND - of `flush_end` across all banks before unlocking the originating input. -5. **Bank 0's downstream forward goes out last** — banks 1..N−1 hold `flush_end` - until `bank_empty`. +4. **All banks finish before the next level is flushed** — `VX_cache_init` + waits for the AND of `flush_end` across all banks, and every bank holds + `flush_end` until `bank_empty`. +5. **A token is answered only after the next level has each port's data** — + with `FLUSH_FORWARD`, one token per memory port goes out once the request + crossbar has drained, and every port must answer before `STATE_RESPOND`. 6. **The init walk is one-shot per reset.** `flush_pending_r` only records a `flush_begin` arriving *during* init; it never re-enters `STATE_INIT`. @@ -724,14 +852,15 @@ Edge cases: - **Flush during init** → `flush_pending_r` records it, and `STATE_DONE` fires as soon as init completes. The init walk stands in for the flush walk, which is correct because init invalidated everything anyway. -- **Multiple ports racing the flush flag** → the loop in `STATE_IDLE` counts down, - so the last write wins and it latches the lowest-indexed one's UUID; all flagged - ports unlock together at the end. +- **Tokens on several inputs** → the loop in `STATE_IDLE` counts down, so it + latches the lowest-indexed one's UUID. Every token visible at the end of the + walk is answered, one per cycle through bank 0's response lane (behind any + bank response); a token that arrives later starts the next walk. - **Flush during a fill-forward drain** is asserted impossible (`RUNTIME_ASSERT(~(flush_fire && fwd_pending))`) — `STATE_WAIT1`'s `mshr_empty` precondition already excludes it. - **`STATE_WAIT2`'s `bank_empty` does not re-check `mshr_empty`.** It does not need - to: WAIT1 established it and the input lock guarantees nothing can refill. + to: WAIT1 established it and the input block guarantees nothing can refill. ### 6.9 Why flush is not the substrate for line-granular invalidation @@ -959,8 +1088,8 @@ identically at all three levels — the dcache is not special. - **Miss latency hiding** insufficient (banks idle while misses pend) → raise `*_MSHR_SIZE` and check the LSU pending pool upstream; verify `MEM_PORTS` is not serializing the drain. -- **Bank conflicts** (hit-path stalls) → more banks via a smaller - `DCACHE_WORD_SIZE`; check address strides against the line-interleaved mapping. +- **Bank conflicts** (hit-path stalls) → check that the bank hash is on for that + level, then more banks via a smaller `DCACHE_WORD_SIZE`. - **Store bandwidth** bound → enable a shared level so `WRITEBACK` derives to 1 where legal; add `*_DIRTYBYTES` for partial-line store patterns. - **BRAM / timing** pressure → cut `*_NUM_WAYS` or `*_SIZE` before cutting banks; diff --git a/docs/designs/command_processor.md b/docs/designs/command_processor.md index 118c5dbdcb..2283d89f5a 100644 --- a/docs/designs/command_processor.md +++ b/docs/designs/command_processor.md @@ -108,7 +108,7 @@ architecturally defined; FENCE reads only `arg0[1:0]`, which is inside its |---|---|---| | 0 | `F_PROFILE` | Appends an 8 B `profile_slot` trailer; every size below grows by 8. | | 1 | `F_FENCE_PRE` | Declared in the package; not acted on by any consumer. | -| 2 | `F_MEM_PHYSICAL` | `CMD_MEM_*` device operand is a physical address — the MMU-aware DMA skips translation (§8). Emulation CP only ([`cmd_processor.h:160`](../../sim/common/cmd_processor.h#L160)). | +| 2 | `F_MEM_PHYSICAL` | `CMD_MEM_*` device operand is a physical address — the MMU-aware DMA skips translation (§8). | ### 2.2 Opcode map @@ -129,7 +129,7 @@ come from `cmd_size_bytes` | `CMD_FENCE` | 0x07 | 8 | — | yes | yes | Ordering barrier (retires as a NOP today, §10) | | `CMD_EVENT_SIGNAL` | 0x08 | 20 | EVENT | yes | yes | Write a counter slot | | `CMD_EVENT_WAIT` | 0x09 | 28 | EVENT | yes | yes | Spin until a counter satisfies a compare | -| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Per-core cache flush sweep | +| `CMD_CACHE_FLUSH` | 0x0A | 12 | DCR | yes | yes | Cache flush: broadcast start, per-core completion | | `CMD_LAUNCH_QMD` | 0x0B | 12 | KMU | **no** | yes | Atomic launch: KMU descriptor read from memory (§2.3) | | `CMD_DRAW` | 0x0C | 12 | KMU | **no** | yes | Device-orchestrated graphics draw (§8.1) | @@ -168,7 +168,13 @@ come from `cmd_size_bytes` (`WAIT_OP_EQ/GE/GT/NE`, [`VX_cp_pkg.sv:109-114`](../../hw/rtl/cp/VX_cp_pkg.sv#L109)). - **`CMD_CACHE_FLUSH`** — `arg0` = number of cores. Provides AMD `ACQUIRE_MEM`-style cache maintenance; the host fills `arg0` from - `VX_CAPS_NUM_CORES`. + `VX_CAPS_NUM_CORES`. Header flag bit 2 (`F_FLUSH_L1_ONLY`) asks for an + L1-only flush: the START write carries `VX_CACHE_FLUSH_L1_ONLY`, and the + last-level cache answers without its walk + ([cache_subsystem.md §6.4](cache_subsystem.md#64-vx_cache_init--the-flush-sequencer)). + The runtime flushes lazily: an L1-only flush ahead of the next launch, a + full flush ahead of the next device-memory transfer (see the launch + encoding in §9). --- @@ -317,10 +323,10 @@ is a single-engine CP. Three things gate raising it: 3. **The host serializes anyway.** All host queues funnel into one ring behind one mutex (§9). -The **host-side** scaling limit is the launch path, not the ring: 22 +The **host-side** scaling limit is the launch path, not the ring: 21 commands per launch, unbatched, each with its own doorbell and its own busy-poll loop (§9). `CMD_LAUNCH_QMD` collapses the 20 DCR writes to one -command; batching collapses 22 doorbells to one. Both are implemented in +command; batching collapses 21 doorbells to one. Both are implemented in the Emulation CP and the runtime; the RTL CP needs the QMD mirror. ### 4.5 The submit handshake end to end @@ -343,6 +349,7 @@ subtracts it first. | `0x000` | `CP_CTRL` | RW | bit0 = enable_global, bit1 = reset_all | | `0x004` | `CP_STATUS` | RO | bit0 = busy, bit1 = error | | `0x008` | `CP_DEV_CAPS` | RO | `[7:0]` NUM_QUEUES, `[15:8]` RING_SIZE_LOG2_MAX, `[23:16]` AXI_TID_WIDTH — plus feature bits 24-26 on the Emulation CP (§6) | +| `0x00C` | `GPU_DEV_CAPS_EXT` | RO | device-config caps extension (§6) | | `0x010` | `CP_CYCLE_LO` | RO | free-running cycle counter, low 32 | | `0x014` | `CP_CYCLE_HI` | RO | high 32 | | `0x018` / `0x01C` | `GPU_DEV_CAPS_LO/HI` | RO | device-config caps (§6) | @@ -381,10 +388,8 @@ Reads are registered (AR latches, R returns the decoded value the next cycle). Undecoded addresses return `0xDEADBEEF` with DECERR on both channels. -> **FPGA/sim divergence:** the RTL regfile has **no** `CP_SATP_LO/HI` -> registers. The Emulation CP *does* (`0x028/0x02C`, -> [`cmd_processor.cpp:72-73`](../../sim/common/cmd_processor.cpp#L72)), -> and the runtime writes them only when VM is advertised. See §8. +`CP_SATP_LO/HI` (`0x028/0x02C`) hold the DMA's page-table root; the runtime +writes them only when `VM_ENABLED` is advertised. See §8. --- @@ -428,6 +433,11 @@ The exact counts in `[63:42]` let a machine that is not a power of two falls back to the log2 fields only when they read zero, as on devices built before they existed. +**`GPU_DEV_CAPS_EXT`** (`0x00C`) carries the exact counts that no longer fit +in `GPU_DEV_CAPS`: `[7:0]` socket_size (`VX_CAPS_SOCKET_SIZE`), the rest zero. +It reads zero on devices built before it existed, so the decoder falls back to +the log2 socket_size field `[16:14]`. + `VX_CAPS_NUM_CORES` is derived, not a field: `num_clusters × cluster_size × socket_size`, from the exact counts. `CACHE_LINE_SIZE`, `GLOBAL_MEM_SIZE`, `CLOCK_RATE`, `PEAK_MEM_BW` and the VM ids are not @@ -450,11 +460,9 @@ upper byte of the CP's own caps word is free, and the Emulation CP uses it | 25 | `SUPPORTS_DRAW` | The CP decodes `CMD_DRAW` / `OP_DRAW` (§8.1) | | 26 | `SUPPORTS_QMD` | The CP decodes `CMD_LAUNCH_QMD` (§2.3) | -**The RTL CP hardwires all three to zero.** Its `CP_DEV_CAPS` read returns -`{8'd0, AXI_TID_W, RING_SIZE_LOG2_MAX, NUM_QUEUES}` -([`VX_cp_axil_regfile.sv:177-180`](../../hw/rtl/cp/VX_cp_axil_regfile.sv#L177)), -so on FPGA every launch takes the 20-DCR fallback, SATP is never -programmed, and draws stream as a ring batch. The runtime consumes the +**The RTL CP sets `VM_ENABLED` from `VX_CFG_VM_ENABLE` and hardwires +`SUPPORTS_DRAW` / `SUPPORTS_QMD` to zero**, so on FPGA every launch takes the +20-DCR fallback and draws stream as a ring batch. The runtime consumes the bits at open ([`device.cpp:274-282`](../../sw/runtime/common/device.cpp#L274)) and branches accordingly, so the graphics driver calls `vx_enqueue_draw` unconditionally on every backend. @@ -470,10 +478,18 @@ unconditionally on every backend. barrier** — the property §8.1 depends on. - **DCR proxy** ([`VX_cp_dcr_proxy.sv`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv)) — `IDLE→REQ→(WAIT_RSP)→DONE`. Writes complete at `REQ`; reads wait for - `dcr_rsp_valid`. `CMD_CACHE_FLUSH` re-enters `REQ` once per core, - sweeping a DCR *read* of `VX_DCR_BASE_CACHE_FLUSH` with the target core - index in `dcr_req_data[15:0]`, and retires only once the last core's - flush completes ([`:105-134`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv#L105)). + `dcr_rsp_valid`. `CMD_CACHE_FLUSH` first passes through `START`: one DCR + *write* of `VX_DCR_BASE_CACHE_FLUSH`, which reaches every core and starts + all their flushes at once. It then re-enters `REQ` once per core, sweeping + a DCR *read* of the same address with the target core index in + `dcr_req_data[15:0]`; each read is answered when that core's flush has + completed, and the command retires after the last one; a full flush then + holds `DONE` until Vortex `busy` drops, which covers every write until its + AXI B response, so a DMA after the flush cannot overtake the last-level + cache's write-backs on bank 0. The DCR response + bus has no back-pressure, so the reads stay one at a time; the flushes + themselves overlap and share one walk of the shared caches + ([cache_subsystem.md §6](cache_subsystem.md#6-flush-architecture)). - **DMA** ([`VX_cp_dma.sv`](../../hw/rtl/cp/VX_cp_dma.sv)) — dual-port multi-beat bursts in ≤4 KB INCR chunks (`MAX_BURST` = 64 × 64 B beats), each chunk clipped so no burst crosses a 4 KB boundary on either @@ -537,19 +553,17 @@ a divergence — with one queue the RTL's arbiters are degenerate too. The Emulation CP's MMIO map matches the RTL regfile **plus**: -- `CP_SATP_LO/HI` at `0x028/0x02C` - ([`cmd_processor.cpp:72-73`](../../sim/common/cmd_processor.cpp#L72)), -- the `CP_DEV_CAPS` feature bits 24-26 (§6), -- a software page-table walk `cp_translate` (Sv32/Sv39, megapage-aware, - [`:157-201`](../../sim/common/cmd_processor.cpp#L157)) honoring the - `MEM_FLAG_PHYSICAL` flag ([`:519-520`](../../sim/common/cmd_processor.cpp#L519)), +- the `CP_DEV_CAPS` feature bits 25-26 (§6), - byte-exact `CMD_MEM_*` copies, staged through a 64 KiB bounce buffer ([`:531-541`](../../sim/common/cmd_processor.cpp#L531)), - the two extra opcodes, `CMD_LAUNCH_QMD` (§2.3) and `CMD_DRAW` (§8.1). -This means **CP DMA is MMU-aware in simulation but not on FPGA** — a -deliberate phased rollout (runtime + emulation first, RTL walker later; -see §10). Two smaller divergences: `CP_STATUS` returns only the busy bit +Both CPs are MMU-aware: with a root in `CP_SATP`, each `CMD_MEM_*` walks +the Sv32/Sv39 table (megapage-aware) for its device operands, once per +command, since an allocation is one contiguous physical range, and +`F_MEM_PHYSICAL` skips the walk (`cp_translate` in the emulation CP; the +`S_XLATE_*` states of `VX_cp_dma`, which read the PTEs on the device port). +Two smaller divergences: `CP_STATUS` returns only the busy bit (no error bit), and unknown MMIO offsets are silently ignored on write rather than raising DECERR. @@ -622,8 +636,7 @@ is written **last**, after the per-queue enable. It then reads `CP_DEV_CAPS` once and latches the VM / DRAW / QMD feature bits ([`:274-282`](../../sw/runtime/common/device.cpp#L274)); `CP_SATP_LO/HI` is written only inside the `if (vm_enabled_)` branch -([`:305-306`](../../sw/runtime/common/device.cpp#L305)) — so on every RTL -CP today (§6) SATP is never programmed. +([`:305-306`](../../sw/runtime/common/device.cpp#L305)). **Submit path.** Each `vx_enqueue_*` pushes a `Command` lambda onto a per-queue worker FIFO ([`queue.cpp:28`](../../sw/runtime/common/queue.cpp#L28) @@ -644,13 +657,24 @@ whole run costs one doorbell and one poll. **Launch encoding.** A kernel launch emits **20** `CMD_DCR_WRITE`s ([`queue.cpp:401-427`](../../sw/runtime/common/queue.cpp#L401)) — 4 for the startup/entry PCs, 2 for the args pointer, and 14 for block/grid/cluster -dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then -`CMD_CACHE_FLUSH` with `arg0 = num_cores`, then a COUT drain +dims, lmem size, block size and warp steps — then `CMD_LAUNCH`, then a COUT drain ([`device.cpp:502-520`](../../sw/runtime/common/device.cpp#L502)). Each group is independently skippable through the legacy escape hatches (no kernel, empty args, `ndim == 0`), so the real count ranges from 20 down to -0. This path does **not** batch: all 22 lines take the non-batch -`cp_submit_cl_`, so one launch costs 22 doorbells and 22 poll loops. Only +0. The kernel-boundary `CMD_CACHE_FLUSH` (`arg0 = num_cores`) is posted lazily +by `cp_ring_append_`, under the same ring lock, from two flags: `l1_dirty_` (a +kernel ran since the last flush) puts an L1-only flush ahead of the next +launch; `l1_dirty_` or `llc_dirty_` (a kernel ran since the last full flush) +puts a full flush ahead of the next `CMD_MEM_*` that touches device memory +outside the IO aperture, because the CP DMA reaches device memory beneath +the last-level cache. A launch followed by a transfer therefore costs one +full flush, and back-to-back launches one L1-only flush each. The COUT drain +needs no flush: the console rings are uncached, and the device `busy` the +launch retires on stays high until memory has completed every write the cores +issued ([cache_subsystem.md §4.9](cache_subsystem.md#49-requests-in-flight-and-device-busy)). The drain's +own transfers stay in the IO aperture and never trigger a flush. +This path does **not** batch: all 21 lines take the non-batch +`cp_submit_cl_`, so one launch costs 21 doorbells and 21 poll loops. Only `enqueue_commands` and the `enqueue_draw` fallback batch. **Queues and serialization.** `vx_queue_create` makes an unbounded number @@ -674,7 +698,8 @@ host-side instead. They become usable once queues get independent rings. ### 9.1 FPGA backends -- **XRT** ([`hw/rtl/afu/xrt/VX_afu_wrap.sv`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv)) — +- **XRT and AVED** ([`hw/rtl/afu/common/VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv), + wrapped by `afu/xrt/vortex_afu.v` and `afu/aved/vortex_afu.v`) — AXI-Lite is split on bit 12: `0x000–0x0FFF` → a minimal `VX_afu_ctrl` ap_ctrl stub + SCOPE; `0x1000–0x1FFF` → the CP regfile. The legacy launch FSM, DCR path, and dev-caps were **removed**; CP is the sole @@ -715,50 +740,46 @@ The following gaps remain **open**. ([`cmd_processor.cpp:531-541`](../../sim/common/cmd_processor.cpp#L531)). Non-cache-line-aligned transfers can over-write on FPGA. Needs tail `wstrb` on the last beat. -2. **VM in RTL.** Add `CP_SATP_LO/HI` regfile decode + a hardware - page-table walker + TLB in `VX_cp_dma`, and route `F_MEM_PHYSICAL`, so - FPGA matches the simulator's MMU-aware DMA. Today VM works on - simx/rtlsim/gem5 and silently no-ops on FPGA. -3. **Real `CMD_FENCE` semantics.** The engine retires FENCE as a NOP +2. **Real `CMD_FENCE` semantics.** The engine retires FENCE as a NOP ([`VX_cp_engine.sv:109-112`](../../hw/rtl/cp/VX_cp_engine.sv#L109)); it should honor `FENCE_DMA_BIT` / `FENCE_GPU_BIT` ordering. -4. **`dcr_req_ready` backpressure.** `VX_cp_dcr_proxy` assumes the DCR bus +3. **`dcr_req_ready` backpressure.** `VX_cp_dcr_proxy` assumes the DCR bus accepts every request in one cycle ([`:115-122`](../../hw/rtl/cp/VX_cp_dcr_proxy.sv#L115)); `VX_cp_core` discards `dcr_req_ready` outright ([`:340`](../../hw/rtl/cp/VX_cp_core.sv#L340)) even though `VX_cp_gpu_slice` now carries a real elastic buffer that can stall. -5. **Resource re-entry under multi-queue** (§4.4, item 2). A grant landing +4. **Resource re-entry under multi-queue** (§4.4, item 2). A grant landing on a busy resource unit is silently dropped while the granted CPE still retires on the previous command's `done`. Latent at `NUM_QUEUES=1`; blocking for anything more. **Performance / feature gaps:** -6. **QMD `CMD_LAUNCH_QMD` in the RTL CP.** The atomic-launch opcode, the +5. **QMD `CMD_LAUNCH_QMD` in the RTL CP.** The atomic-launch opcode, the in-memory descriptor format, the runtime encoder, and the `SUPPORTS_QMD` gate all exist and are validated on the Emulation CP (§2.3); the RTL CP does not decode 0x0B and advertises 0. Until it does, every FPGA launch pays the 20-`CMD_DCR_WRITE` dance, and per-queue launch concurrency stays blocked on item 5. -7. **`CMD_DRAW` in the RTL CP** (§8.1) — same shape: shipped in emulation, +6. **`CMD_DRAW` in the RTL CP** (§8.1) — same shape: shipped in emulation, cap bit cleared in RTL. -8. **Multi-queue everywhere.** RTL is parameterized on `NUM_QUEUES` but +7. **Multi-queue everywhere.** RTL is parameterized on `NUM_QUEUES` but defaults to 1; the Emulation CP models only `q0_`; the runtime programs only queue 0 and funnels every host queue through one ring lock (§9). Real concurrency depends on items 5 and 6. -9. **Batch the launch path.** `enqueue_launch` submits its 22 commands - unbatched — 22 doorbells, 22 busy-polls — while `cp_batch_begin/end` +8. **Batch the launch path.** `enqueue_launch` submits its 21 commands + unbatched — 21 doorbells, 21 busy-polls — while `cp_batch_begin/end` already exists and is used by `enqueue_commands`. Cheap win independent of item 6. -10. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter +9. **EVENT_WAIT fairness + backoff.** `EVENT_WAIT` holds its arbiter grant for the whole wait ([`VX_cp_event_unit.sv:124-129`](../../hw/rtl/cp/VX_cp_event_unit.sv#L124)); it should release between polls and back off. -11. **Priority arbitration.** `VX_cp_engine` already drives `bid_priority` +10. **Priority arbitration.** `VX_cp_engine` already drives `bid_priority` from `q_state.prio`; `VX_cp_arbiter` ties it off ([`:41`](../../hw/rtl/cp/VX_cp_arbiter.sv#L41)). Wire it through. -12. **Profiling writeback.** [`VX_cp_profiling.sv`](../../hw/rtl/cp/VX_cp_profiling.sv) +11. **Profiling writeback.** [`VX_cp_profiling.sv`](../../hw/rtl/cp/VX_cp_profiling.sv) is **not instantiated by anything** — `VX_cp_axil_regfile` carries its own free-running counter for `CP_CYCLE_LO/HI` at `0x010/0x014` (the profiling module's header comment claiming `0x040/0x044` is stale). @@ -767,19 +788,19 @@ The following gaps remain **open**. To give `vx_event_get_profiling` hardware backing: instantiate the module, emit the 32 B `{queued,submit,start,end}` record, and add a `CP_CYCLE_FREQ_HZ` register. -13. **Interrupt path.** Add IP_ISR/IER/GIER in the AFU and +12. **Interrupt path.** Add IP_ISR/IER/GIER in the AFU and `xrt::ip::interrupt` so the runtime sleeps instead of busy-polling `Q_SEQNUM` (the `irq` pulse already exists, [`VX_cp_core.sv:505`](../../hw/rtl/cp/VX_cp_core.sv#L505)). -14. **Host-coherent completion mailbox / `head_addr` writeback.** The CP +13. **Host-coherent completion mailbox / `head_addr` writeback.** The CP tracks `head` internally and never DMAs it to `head_addr`; the runtime likewise ignores the cmpl slot it programs (§9). A cacheable host mailbox would let both replace the per-poll MMIO read. -15. **A CP DCR range.** `command_processor` has no DCR block of its own for +14. **A CP DCR range.** `command_processor` has no DCR block of its own for CP↔GPU coordination / multi-context KMU. The range once earmarked for it (`0x080–0x0BF`) is now occupied by `[dcr_om]` in `VX_types.toml`, so this needs a fresh allocation rather than the old reservation. -16. **Opcode-constant duplication.** The command opcodes are re-declared in +15. **Opcode-constant duplication.** The command opcodes are re-declared in `VX_cp_pkg.sv`, `cmd_processor.h`, `device.cpp` **and** `queue.cpp`, with no shared header — four copies of one ABI, and a standing drift hazard. diff --git a/docs/designs/continuous_integration.md b/docs/designs/continuous_integration.md index c959934e45..90facb0f39 100644 --- a/docs/designs/continuous_integration.md +++ b/docs/designs/continuous_integration.md @@ -133,7 +133,10 @@ A case with `known_issue:` still builds and runs. `conftest.py` marks it `xfail`, so its failure does not fail the run; if it starts passing it surfaces as `XPASS`. Reserve it for triaged, documented breakage. `needs:` does not skip either — it records what a cell must provision, and a -missing dependency is a real failure. +missing dependency is a real failure. A cell's `needs` cover every case its +`-m` expression selects, and the workflow provisions per-case tools (the +foldsim prebuilt) from them rather than from the cell's name, because a +check's cell gathers cases from every category. #### Lint rules @@ -195,9 +198,9 @@ A check is an assertion across runs rather than a clean exit. | | `model_parity` | `perf_gate` | |---|---|---| -| compares | SimX against rtlsim, same case | rtlsim against a checked-in baseline | +| compares | a model against rtlsim, same case (SimX by default; Foldsim) | rtlsim against a checked-in baseline | | asserts | instructions equal; cycles within tolerance | cycles within tolerance; instructions equal | -| tolerance | 5 %, overridable per case or per category | 2 %, one constant | +| tolerance | 5 %, overridable per case or per category (0 for Foldsim) | 2 %, one constant | | golden data | none | `ci/baselines/perf/.json` | | tier | `full` | `full` | | pinned to | the rtlsim driver, which elaborates the RTL | the rtlsim driver | @@ -227,9 +230,28 @@ its own parity case with only that extension enabled, so a regression is attributable. Workloads are sized so that steady state dominates — a tiny kernel is all boot and dispatch skew, and its gap ratio is noise. +A case's `legs:` names its two drivers, the model and then the reference; +the default is `[simx, rtlsim]`. `ci/testcases/foldsim.yaml` runs +`[foldsim, rtlsim]` at tolerance 0: Foldsim is compiled from the RTL and is +cycle-exact by construction, so any cycle difference is a defect. A tiny +configuration is enough there. Its reference is rtlsim behind the AXI4 adapter +on the same DRAM bank model (`AXI_BUS=1 AXI_MEM_MODEL=bank`). + **Never widen a tolerance to absorb a divergence.** Model the behavior, or mark the case `known_issue` while it is investigated. +#### `release_parity` + +A debug build must not change timing. A debug build widens the RTL UUID to 44 +bits and enables assertions and traces. A blackbox case run with `--debug` can +set `release_parity: true`. The runner then reruns it built release, at +xlen 32, and requires identical instructions and cycles; it reads the debug +run's counts from its `--log` file. This is a case field, not a check, so the +case stays in its category cell and costs one extra run there, with no new +cell. `debug:sgemm_tcu-2` carries it. That case is shaped to keep the data +cache under pressure, with two channels whose same-word reads merge; a +debug-only tag width reaching the read merge would show there. + ### 3.4 `perf_gate` rtlsim cycle counts are deterministic and host-independent, so the threshold @@ -254,22 +276,27 @@ so the parity and perf views of one workload are two cases. `perf_gate` catches a change that costs cycles. The synthesis gates catch one that costs **timing closure or area**: they synthesize a catalog of designs -and assert the results against golden baselines. - -| | `fpga_gate` | `asic_gate` | -|---|---|---| -| flow | Vivado synthesis, place and route | sv2v → Yosys → ABC → OpenSTA, on ASAP7 | -| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | -| gated against the baseline | `fmax_mhz`, `lut` | `cell_area_um2` (Fmax only against the target clock) | -| also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | -| comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | -| runner | self-hosted, licensed Vivado | hosted, one job per design | -| default tolerance | 5 % | 5 % | - -Neither is a pytest cell. A cell is a build tree plus a driver; a synthesis -build is neither, and both gates are hours long. They are standalone scripts +and assert the results against the target clock and area budget each build +declares, recording every run's metrics in golden baselines. `logic_gate` +catches one that makes a unit **deeper** or the design **larger**, in seconds +and without synthesis. + +| | `fpga_gate` | `asic_gate` | `logic_gate` | +|---|---|---|---| +| flow | Vivado synthesis, place and route | CIRCT → Yosys → ABC → OpenSTA, on ASAP7 | CIRCT → `foldsim-estimate-timing` (logic levels as Vivado counts them on `DEVICE`'s family) and `foldsim-estimate-area` (its cells, memories and DSPs) | +| design tree | `hw/syn/xilinx/dut` | `hw/syn/yosys/dut` | `hw/syn/xilinx/dut`, target `beacon` | +| gated | `fmax_mhz` against the target clock, `lut` against `lut_budget` | `fmax_mhz` against the target clock, `cell_area_um2` against `area_budget_um2` | `max_levels` against the baseline, the deepest end point of every RTL unit, and `lut`, `ff`, `lutram`, `bram`, `uram`, `dsp` against the baseline | +| also recorded | `wns_ns`, `lutram`, `ff`, `bram`, `uram`, `dsp`, critical paths, high-fanout nets | `wns_ns`, `tns_ns`, `seq_area_um2`, `sram_area_um2`, `cell_count`, `power_mw` | end point bits at ≥ 12 and ≥ 8 levels, the deepest paths | +| comparable when | same device, optimization level, xlen | same PDK, VT, corner, xlen | same CIRCT, foldsim, xlen | +| runner | self-hosted, licensed Vivado | hosted, one job per design | hosted, one job for the catalog; the opt-in `gpu` group self-hosted | +| default tolerance | Fmax 2 %, LUT 5 % | Fmax 2 %, cell area 5 % | depth none: Beacon is deterministic; area 5 % | + +None is a pytest cell. A cell is a build tree plus a driver; a gate build is +neither, and the synthesis gates are hours long. They are standalone scripts driven by their own workflows, and their tiers are **opt-in**: an empty tier -selection means "everything a cell can run", which excludes them. +selection means "everything a cell can run", which excludes them. The timing +gate's tier is opt-in because it needs foldsim, which the base toolchain does +not install. #### One implementation, two adapters @@ -277,24 +304,35 @@ selection means "everything a cell can run", which excludes them. tool-specific — catalog loading, the config hash, threshold resolution, `known_issue`, the resumable session, the scheduler, the gate and the report. A tool adapter supplies only what differs: metric names, how the flow is -invoked, what its log looks like, and which environment fields make two runs -comparable. `fpga_gate.py` and `asic_gate.py` are entry points that pin the -tool. +invoked, what its log looks like, which environment fields make two runs +comparable, and any assertion beyond the per-metric check (`logic_gate`'s +per-unit depths). `fpga_gate.py`, `asic_gate.py` and `logic_gate.py` are +entry points that pin the tool. #### Spec and baseline | file | written by | holds | |---|---|---| -| `ci/testcases/{fpga,asic}_gate.yaml` | a person | per build: a design, a target clock, `CONFIGS`, a group, optional `thresholds` and `known_issue` | -| `ci/baselines/synthesis/{xilinx,yosys}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | +| `ci/testcases/{fpga,asic,logic}_gate.yaml` | a person | per build: a design, a target clock, an area budget (synthesis gates), `CONFIGS`, a group, optional `thresholds`, `known_issue`, `xlen`, `profile` and `opt_in` | +| `ci/baselines/synthesis/{xilinx,yosys,beacon}/.json` | the gate | measured metrics, the config hash, the tool versions and environment | -Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`. `asic_gate` uses the +Groups are `core`, `tensor`, `graphics`, `dxa` and `vm`, plus `logic_gate`'s +opt-in `gpu`. `asic_gate` uses the same group names and gates a subset of the builds (§4.5), so where the two gates share a design, a divergence between the tools on the same module is a -finding, not noise. +finding, not noise. `logic_gate` measures exactly `fpga_gate`'s builds, so a +unit it flags can be read against the Fmax gate's critical paths; its `gpu` +group adds builds no synthesis gate can afford. + +A build may pin its own `xlen`, and runs in that width's build tree +(`build_`); `profile` folds a hardware profile +(`profiles/.json`) under its `CONFIGS`, which win over it. Both are in +the config hash. An `opt_in` build runs only when selected by id or group. The config hash ties the two files together: edit the spec, and the gate -refuses to compare against numbers recorded for the old one. Its +refuses to compare against numbers recorded for the old one. It matters only +for a metric gated against its baseline (`--gate`); the default assertions +read the target clock and the budget, which are not part of the hash. Its tool-specific half is the device and optimization level for Vivado and the PDK, VT and corner for Yosys. @@ -306,9 +344,9 @@ developer runs cannot diverge. | assertion | reason | |---|---| -| each gated metric within tolerance of its baseline | regression detection | -| an improvement beyond tolerance also fails | the gain must be recorded | -| Fmax within tolerance of the **target clock**, independent of the baseline | a baseline recorded below target must not let a build pass by matching it | +| Fmax no more than its tolerance **below the target clock** | the design meets the clock it is built for; a faster result never fails, since placement noise alone moves Fmax by more than 2 % | +| area no more than its tolerance **over the build's budget** | the budget is fixed in the catalog, so re-recording a baseline cannot creep area upward; raising it is a reviewed edit | +| any other metric gated with `--gate`: within tolerance of its baseline, both ways | an improvement beyond tolerance must be recorded | Thresholds resolve most-specific first: a build's own `thresholds`, then a global per-metric threshold, then the global default. @@ -327,21 +365,80 @@ so the flow reports the signed worst slack instead. | verdict | meaning | fails the run | |---|---|---| | `PASS` | within tolerance | no | -| `REGRESSION` | a gated metric got worse | yes | -| `IMPROVED` | a gated metric got better; record it | yes | +| `FAIL` | Fmax below target, area over budget, or a baseline-gated metric worse | yes | +| `IMPROVED` | a baseline-gated metric got better; record it | yes | | `STALE` | the spec or the tool changed since the baseline | yes | -| `FAIL` | the build produced no metrics | yes | +| `BUILD-FAIL` | the build produced no metrics | yes | | `KNOWN-ISSUE` | a tracked expected failure | no | | `XPASS` | a known issue stopped reproducing; clear the flag | no | | `RECORDED` | a baseline was written | no | +#### What `logic_gate` asserts + +Beacon ([beacon.md](beacon.md)) counts logic levels, not +slack: it does not see wires, so it gates depth, which it predicts (Spearman +0.50–0.93 against Vivado per register on the V80 gates), and leaves Fmax to +`fpga_gate`. It counts them for the family of +the DUT flow's `DEVICE` (the U55C by default), which +`gen_platform.py --emit=family` reads from `hw/syn/xilinx/platforms`; the +device is part of the baseline fingerprint, as it is for `fpga_gate`. +Beacon has profiles for the V80 and the U55C only, and fails on any other +`DEVICE`. A +**unit** is an end point's instance path with generate indices folded +(`g_banks_3_bank` → `g_banks_*_bank`). + +| assertion | reason | +|---|---| +| no unit deeper than its baseline | the change that adds a level is named, by unit | +| no new unit at the fix threshold (12 levels at 300 MHz) | a new deep cone is a regression even with no baseline to compare | +| a shallower unit, or one that drops below the watch threshold (8), reads `IMPROVED` | the gain must be recorded | +| no output of a build's `interfaces` modules reached combinationally from outside the instance in more bits than its baseline | modules register their outgoing interfaces: a path that runs through a core, socket or cluster boundary is what fails routing across SLRs, and Beacon names it by port | +| no resource of the design (`lut`, `ff`, `lutram`, `bram`, `uram`, `dsp`) more than 5 % above its baseline | area found in minutes, not in a nightly Vivado run; a memory that moves between distributed and block RAM moves two columns, and `beacon_area.txt` names each memory's placement | + +`interfaces` lists modules by name; their instances keep their ports through +Beacon's mapping (`BEACON_INTERFACES` in the DUT flow), and `ports.csv` +counts, per output port, the bits a combinational path from outside the +instance reaches. + +Depth has no tolerance: the same RTL and tools give the same levels. Area is +as deterministic, but a feature adds some by design, so it has `fpga_gate`'s +LUT tolerance; its LUTs are the area mapping's scaled to Vivado's average +([beacon.md §4.5](beacon.md)), so it gates their change, not a budget. A new CIRCT +or foldsim can map the same RTL differently, so both versions are in the config +hash and a tool change reads `STALE`. `make -C hw/syn/xilinx/dut -beacon` +runs Beacon on one DUT by hand; its `beacon_paths.txt` lists every deep path op by op, +by unit, with source lines, and a failing build names it. + +#### Full-GPU builds + +The `gpu` group runs Beacon on the whole processor (`vortex` DUT) with every +feature on: the 32-bit machine with every extension, tensor format and +graphics unit, and the `rtx5080` and `h100` profiles at XLEN 64. It catches +what the per-unit DUTs cannot: the cluster, socket and L2 crossbars at full +width, the kernel launch path into every core, and every extension at once. +Vivado needs 48 hours and more on these designs; Beacon needs about 30 minutes. + +| cores | time | peak memory | end points | +|---|---|---|---| +| 1 | 37 min | 101 GB | 7.1 M | +| 2 | 24 min | 90 GB | 13.2 M | +| 4 | 32 min | 94 GB | 25.4 M | +| 8 | 49 min | 121 GB | 50.1 M | + +Measured with the `rtx5080` features at XLEN 64 on a loaded machine. The +builds use 4 cores: the core count widens only the arbiters and crossbars, +and the deepest path was the same at every count, while memory grows with it. +The group is opt-in in the gate (`ci/logic_gate.py -b gpu -j 1`) and its own +tier, `timing_gpu`, in the catalog, so neither the default run nor +`--tier timing` sweeps it in. + #### Running a multi-hour sweep | mechanism | what it does | |---|---| | scheduling | longest build first, from the recorded build times, so the longest is in flight from the start; `-j` caps parallel builds and each build's tool threads are derived from it | | isolation | every build gets a unique `PREFIX`, hence its own tree and log. On the Yosys flow this is load-bearing: its Makefile caches generated sources and does not regenerate them when the include set changes, so two designs sharing a tree would synthesize the first one's sources | -| early-failure watch | the runner follows each log and announces the point past which a front-end error can no longer occur — the end of RTL elaboration for Vivado, the end of source generation and sv2v conversion for Yosys. A build that fails before it is reported as failed before synthesis, with its error lines quoted | +| early-failure watch | the runner follows each log and announces the point past which a front-end error can no longer occur — the end of RTL elaboration for Vivado, the end of source generation and CIRCT elaboration for Yosys. A build that fails before it is reported as failed before synthesis, with its error lines quoted | | resumable sessions | each build directory carries a stamp of config hash, status and metrics. `--resume` reuses a build finished for this config, lets an unfinished one continue from its checkpoint, and rebuilds one whose config changed | | progress | phase transitions as they land, with a heartbeat between; `-v` streams the raw log | @@ -482,6 +579,21 @@ The largest designs — `top`, `gfx`, `tensor`, `vm` — are therefore gated on `fpga_gate` only, and `core` and `tex` are gated at a reduced width. Measure a design's peak memory before adding it: an over-budget design costs a silent job kill, not a red gate. +### 4.6 `logic_gate.yml` + +On every pull request and push to master that touches the RTL, the DUT +wrappers, Beacon's scripts, the gate or its baselines; and by manual +dispatch. One hosted job runs Beacon on the whole catalog: it restores the toolchain, +installs foldsim (`ci/toolchain_install.sh --foldsim`), runs the catalog's +`run:` command with `--report`, and renders the report with +[`synth_report.py`](../../ci/synth_report.py) into the workflow summary. The +artifacts are the report and, per design, the build log, the summary and +Beacon's deepest paths. + +A manual dispatch with `gpu: true` runs the catalog's `gpu` case instead, on +the self-hosted machine: one build at a time, up to two hours each, since each +needs about 95 GB of memory. + --- ## 5. Running locally and coverage diff --git a/docs/designs/core_partitioning.md b/docs/designs/core_partitioning.md new file mode 100644 index 0000000000..5034f058c4 --- /dev/null +++ b/docs/designs/core_partitioning.md @@ -0,0 +1,184 @@ +# Core Partitioning — Design + +**Scope:** how a Vortex core divides its warps among parallel pipelines — the +fetch width, the issue width and the per-unit block counts, the three core +structures they select, the frontend crossbar that joins slices of different +widths, and the services that stay core-wide. Covers the RTL +([`VX_core.sv`](../../hw/rtl/core/VX_core.sv), +[`VX_frontend_slice.sv`](../../hw/rtl/core/VX_frontend_slice.sv), +[`VX_frontend_xbar.sv`](../../hw/rtl/core/VX_frontend_xbar.sv), +[`VX_backend_slice.sv`](../../hw/rtl/core/VX_backend_slice.sv), +[`VX_compute_slice.sv`](../../hw/rtl/core/VX_compute_slice.sv), +[`VX_issue.sv`](../../hw/rtl/core/VX_issue.sv), +[`VX_execute.sv`](../../hw/rtl/core/VX_execute.sv), +[`VX_commit.sv`](../../hw/rtl/core/VX_commit.sv), +[`VX_lane_dispatch.sv`](../../hw/rtl/core/VX_lane_dispatch.sv)) and the +configuration knobs in [`VX_config.toml`](../../VX_config.toml). + +The pipeline stages themselves are described in +[`microarchitecture.md`](microarchitecture.md); the CTA dispatcher that feeds +warps to the slices in [`cta_dispatch_architecture.md`](cta_dispatch_architecture.md); +the LSU blocks in [`lsu_pipeline_design.md`](lsu_pipeline_design.md). + +![Core partitioning](../assets/img/core_partitioning_structures.svg) + +--- + +## 1. Slicing by warp id + +A core holds `VX_CFG_NUM_WARPS` warps. Every parallel structure in it owns a +fixed subset of them, selected by the low bits of the warp id: + +| Width | Knob | Owner of warp `wid` | Unit of replication | +|---|---|---|---| +| FW | `VX_CFG_FETCH_WIDTH` | frontend slice `wid mod FW` | warp scheduler, fetch, decode, one icache port | +| IW | `VX_CFG_ISSUE_WIDTH` | issue slice `wid mod IW` | ibuffer, scoreboard, operand collector, commit | +| B | `VX_CFG_NUM_{ALU,FPU,LSU,SFU,TCU}_BLOCKS` | block `wid mod B` of that unit | one block of the execution unit | + +The widths must nest: FW and every B divide IW. FW is a power of two +(`VX_CFG_FETCH_WIDTH` defaults to the lowest set bit of IW), so `wid mod FW` +and `wid mod B` are bit slices of `wid mod IW`, and a warp never changes +slice. Each structure names its warps locally — a frontend slice's warp +`wfs` is core warp `wfs·FW + slice_id` — and instructions leave a slice +carrying the core-wide `wid`. + +Defaults: IW is `up(NUM_WARPS / 16)`, so a core of up to 16 warps has one +issue slice; FW follows IW; the ALU, FPU, LSU and TCU have one block per +issue slice (B = IW); the SFU has one block (`VX_CFG_NUM_SFU_BLOCKS = 1`). + +## 2. Three structures + +[`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv) derives the structure from +the widths; there is no switch for it. + +``` +BACKEND_SLICES = every enabled unit has B == IW (ALU, LSU, SFU, FPU if F, TCU if TCU) +COMPUTE_SLICES = BACKEND_SLICES && FW == IW +``` + +| Widths | Structure | Modules | +|---|---|---| +| every B = IW = FW | **compute slices** | IW × `VX_compute_slice`, no crossbar | +| every B = IW, FW < IW | **backend slices** | FW × `VX_frontend_slice`, `VX_frontend_xbar`, IW × `VX_backend_slice` | +| some unit has B ≠ IW | **stage-sliced** | FW × `VX_frontend_slice`, `VX_frontend_xbar`, `VX_issue` / `VX_execute` / `VX_commit` | + +All three run the same warps through the same stages with the same +timing; they differ in how many instances there are and what connects them. +With IW = FW = 1 and the default block counts, every condition holds and the +core is a single compute slice. + +### 2.1 Compute slices + +A compute slice ([`VX_compute_slice.sv`](../../hw/rtl/core/VX_compute_slice.sv)) +is a frontend slice and a backend slice joined directly: one end-to-end +pipeline for the warps `wid mod IW == i`, the counterpart of an NVIDIA SM +processing block. Frontend slice *i* serves only issue slice *i* and the +blocks of slice *i*, so there is nothing to route between slices. Only the +core scheduler, the icache, the memory system and the shared SFU/TCU services +are reached through its ports. This is the structure to floorplan by: each +slice is self-contained apart from those ports. + +Compute slices with IW > 1 have one SFU block per slice, so each slice's +warp-control requests are arbitrated against the other slices', and a slice +retires its own warps when its exit request is granted (`SLICE_RETIRE`). +With a single slice the core retires them from the registered request. + +### 2.2 Backend slices + +When FW < IW, frontend slice *f* feeds issue slices `f, f+FW, f+2·FW, …`. +The back end is still IW identical slices +([`VX_backend_slice.sv`](../../hw/rtl/core/VX_backend_slice.sv)): issue, +execute (one block of every unit) and commit for one issue slice. Between +them sits the frontend crossbar (§3). + +### 2.3 Stage-sliced + +When some unit has fewer blocks than issue slices — by default the SFU — a +block serves several issue slices, so issue and execute cannot be cut into +identical end-to-end slices. The core is cut by stage instead: +[`VX_issue`](../../hw/rtl/core/VX_issue.sv) holds IW issue slices, +[`VX_execute`](../../hw/rtl/core/VX_execute.sv) holds B blocks of each unit, +and [`VX_commit`](../../hw/rtl/core/VX_commit.sv) holds IW commit slices +([`VX_commit_slice`](../../hw/rtl/core/VX_commit_slice.sv)). Each unit's +[`VX_lane_dispatch`](../../hw/rtl/core/VX_lane_dispatch.sv) groups the IW issue +ports into IW / B batches: block *b* takes the issue slices `b, b+B, …` in +turn, which is the `wid mod B` ownership of §1. + +## 3. The frontend crossbar + +[`VX_frontend_xbar.sv`](../../hw/rtl/core/VX_frontend_xbar.sv) connects FW +frontend slices to IW issue slices and to the ALU and SFU blocks whenever the +core is not built from compute slices. It is purely combinational: it adds +no cycles, so a configuration's timing does not depend on which structure the +widths select. + +| Direction | Traffic | Routed by | +|---|---|---| +| frontend → issue | decoded instructions | issue slice *i* takes frontend slice `i mod FW` | +| issue → frontend | issue and commit events, per-warp drain status | the frontend slice that owns the warp | +| ALU blocks → frontend | branch resolution | the owning frontend slice | +| SFU blocks → frontend | warp control, CSR reads of scheduler state, RT unlocks | the owning frontend slice; a read is answered by the slice owning the read `wid` | + +## 4. Core-wide services + +Some state belongs to the whole core, not to a slice. Each is reached through +one port per slice or per block, in every structure. + +| Service | Module | Why core-wide | +|---|---|---| +| CTA launch | [`VX_cta_dispatch`](../../hw/rtl/core/VX_cta_dispatch.sv) in the core scheduler, one lane per frontend slice (`NUM_LANES = FW`) | a CTA's warps span slices | +| barriers, wspawn, warp retirement | [`VX_warp_shared_unit`](../../hw/rtl/core/VX_warp_shared_unit.sv) | a barrier counts warps of every slice | +| DXA, TEX and RTU buses | [`VX_sfu_shared`](../../hw/rtl/core/VX_sfu_shared.sv) | one accelerator of each kind per core; the SFU blocks are arbitrated onto it | +| WGMMA tile buffers | [`VX_tcu_shared`](../../hw/rtl/tcu/VX_tcu_shared.sv) | a warpgroup's B operand is common to its TCU blocks | +| icache, dcache, local memory | [`VX_mem_unit`](../../hw/rtl/core/VX_mem_unit.sv) | one memory system per core; the icache has one request port per frontend slice (`ICACHE_NUM_REQS = FW`) | + +`VX_warp_shared_unit` takes one request per cycle, round-robin. Its +requesters register their requests toward the arbiter; a sole requester is +connected directly and its request is registered inside the unit instead, so +a request takes effect at the same distance from its instruction either way. + +## 5. Configurations + +| Configuration | NW | IW | FW | B (ALU/FPU/LSU/TCU, SFU) | Structure | +|---|---|---|---|---|---| +| default | 4–16 | 1 | 1 | 1, 1 | one compute slice | +| `logic_gate` core (`NT=NW=16`) | 16 | 1 | 1 | 1, 1 | one compute slice | +| U55C image C (`ISSUE_WIDTH=4`) | 8 | 4 | 4 | 4, 1 | stage-sliced | +| `rtx5080` profile | 48 | 4 | 4 | 4, 1 | stage-sliced | +| `rtx5080` with `NUM_SFU_BLOCKS=4` | 48 | 4 | 4 | 4, 4 | 4 compute slices | +| IW = 4, `FETCH_WIDTH=1` | — | 4 | 1 | 4, 4 | backend slices | + +To get compute slices at IW > 1, give the SFU one block per issue slice +(`-DVX_CFG_NUM_SFU_BLOCKS=`). Lowering a block count below IW trades that +unit's throughput and area against arbitration, and moves the core to the +stage-sliced structure. + +## 6. Performance and area + +**Fetch width.** Each frontend slice schedules and fetches its own warps on its +own icache port, so FW sets how many instructions the core can fetch per +cycle. At FW > 1, slices often fetch from the same icache line in the same +cycle; with `VX_CFG_ICACHE_READ_MERGE` (on by default, no effect at FW = 1) +those reads share one bank access and each slice gets its own word. The +merge cut sgemm cycles by 12.1% at IW = FW = 4, and by 25.8% on one +`rtx5080` core in rtlsim. + +**Issue width and blocks.** IW sets how many instructions issue per cycle; +each issue slice has its own ibuffer, scoreboard, operand collector and +commit port. A unit with B < IW issues at most B instructions of that unit +per cycle. + +**Area.** A frontend slice replicates the warp scheduler, fetch and decode; a +backend slice, the issue logic and one block of every unit. The crossbar is +combinational routing only. Compute slices replicate the most — every unit +in every slice — and have the shortest connections, since a slice talks to +the rest of the core only through the core-wide services. The stage-sliced +structure shares the units with B < IW at the cost of the dispatch and commit +arbitration between their issue slices. + +## 7. SimX + +SimX models the timing, not the structure: its core has one pipeline +parameterized by the same widths, and `model_parity` holds it to the RTL in +every structure. A change that moves cycles in one structure moves the SimX +model with it (see [`AGENTS.md`](../../AGENTS.md)). diff --git a/docs/designs/cta_dispatch_architecture.md b/docs/designs/cta_dispatch_architecture.md index 7b8af01a77..48c6aa0a85 100644 --- a/docs/designs/cta_dispatch_architecture.md +++ b/docs/designs/cta_dispatch_architecture.md @@ -41,12 +41,12 @@ between them, so the slave-port `ready` would cross both stages combinationally. Where a level needs both, it composes two instances around a registered launch link: -| Level | File | Shape | `dest` slice | +| Level | File | Shape | `dest` digit | |---|---|---|---| -| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_LSB_DEVICE` | +| Device | [`Vortex.sv:185`](../../hw/rtl/Vortex.sv#L185) | 1 → `VX_CFG_NUM_CLUSTERS` | `KMU_DEST_DIV_DEVICE` | | Cluster merge | [`VX_cluster.sv:121`](../../hw/rtl/VX_cluster.sv#L121) | 2 → 1 | — | -| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_LSB_CLUSTER` | -| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_LSB_SOCKET` | +| Cluster distribute | [`VX_cluster.sv:133`](../../hw/rtl/VX_cluster.sv#L133) | 1 → `NUM_SOCKETS` | `KMU_DEST_DIV_CLUSTER` | +| Socket | [`VX_socket.sv:72`](../../hw/rtl/VX_socket.sv#L72) | 1 → `VX_CFG_SOCKET_SIZE` | `KMU_DEST_DIV_SOCKET` | `NUM_SOCKETS = UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE)` ([`VX_gpu_pkg.sv:149`](../../hw/rtl/VX_gpu_pkg.sv#L149)). The cluster merge is @@ -228,7 +228,7 @@ Two things are worth noticing about what is *not* here: placement hint, so the fan-out drops its first member on any ready core and the message lock keeps the rest with it. This is what spreads a grid across the machine. -- **FRAGMENT** → routed by this level's slice of `dest`. Fragment work is +- **FRAGMENT** → routed by this level's digit of `dest`. Fragment work is bin→core affine, and that affinity is what keeps same-pixel blend order correct; load-balancing it would break the blend order outright. @@ -462,10 +462,10 @@ Two preconditions make this sound, both already enforced: SimX has no bus, so its `Kmu` carries the routing itself: the destination is fixed **before** a message is sent — a locked cluster's remaining members go -only to the core that took the first, and a new message goes to the core the -round-robin already points at, stalling on a busy core rather than yielding -its turn to a neighbour -([`kmu.cpp:122-153`](../../sim/simx/kmu/kmu.cpp#L122)). +only to the core that took the first, and a new message goes to the nearest +ready core from the round-robin pointer, as the RTL fan-out picks the nearest +ready output +([`kmu.cpp:122-156`](../../sim/simx/kmu/kmu.cpp#L122)). **Why this is load-bearing rather than an optimisation.** The obvious consequence of a split cluster is that LMEM peers resolve to another CTA's @@ -733,16 +733,20 @@ is cycle-for-cycle identical to it on `vecadd` / `sgemm` / `sgemm_tcu_wg` / The KMU→core transport is modelled as per-core **launch lanes**: each lane is a registered boundary stage feeding the dispatcher's endpoint, and the KMU -additionally paces every core with explicit credits — `LAUNCH_CREDITS = 4` -per core ([`kmu.h:59`](../../sim/simx/kmu/kmu.h#L59)) — spent when a CTA is +additionally paces every core with explicit credits — `LAUNCH_CREDITS = 2` +per core ([`kmu.h:58`](../../sim/simx/kmu/kmu.h#L58)) — spent when a CTA is sent and returned by the dispatcher one message per admission -([`cta_dispatcher.cpp:142`](../../sim/simx/cta_dispatcher.cpp#L142)). The -bound is the transport's occupancy made explicit — the boundary stage's -endpoint (2) plus the dispatcher endpoint (1) plus one credit in return -flight — so a busy core never accumulates work beyond what the RTL's skids -would hold, and pacing rests on slot accounting rather than on peeking at -the consumer's state. RTL needs no counter: its registered output skids -apply the same bound as wires. +([`cta_dispatcher.cpp:175`](../../sim/simx/cta_dispatcher.cpp#L175)). The +bound is the two-entry output skid in front of each core made explicit, so a +busy core never holds more unadmitted CTAs than the RTL's skid would, and +pacing rests on slot accounting rather than on peeking at the consumer's +state. The depth matters for placement, not only for pacing: every CTA +committed to a busy core is one a core that frees a slot sooner cannot take. +RTL needs no counter: its registered output skids apply the same bound as +wires. The lanes are flat, one per core, so the model is exact while a single +level of the tree fans out; where two levels fan out (several clusters of +several cores, or several sockets of several cores) the upper level's shared +skids are not modelled. The differences are deliberate, not drift: @@ -754,7 +758,7 @@ The differences are deliberate, not drift: | Bus payload | `cluster_size` (scalar), `aligned_lmem_size` | `cluster_dim[3]`, raw `lmem_size` | | `cta_id` on the bus | absent | present (model-only, unused as identity) | | Stride rounding | in the KMU | in the dispatcher | -| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 4`) | +| Launch pacing | elastic-skid back-pressure | explicit credits (`LAUNCH_CREDITS = 2`) | | Fragment path | through `kmu_bus_if` → `VX_cta_dispatch` | injected at the scheduler by the FWD | The last row is the significant one: in RTL a fragment wave is a launch that diff --git a/docs/designs/custom_accelerator_isa_extensions.md b/docs/designs/custom_accelerator_isa_extensions.md index a4593709a3..d6c611a042 100644 --- a/docs/designs/custom_accelerator_isa_extensions.md +++ b/docs/designs/custom_accelerator_isa_extensions.md @@ -233,7 +233,67 @@ the table in the same change. | | | `7` | `vx_rt_trace`, `vx_rt_wait` | R4 | | | | `1`, `2`, `4` | free | | | custom-2 `0x5B` | `funct7` | `0x00` | MUFU special functions; `fn` in `rs2`, `fmt` in `funct3` | R | -| custom-3 `0x7B` | | | free | | +| custom-3 `0x7B` | `funct3` | `0` | `vx_ldm fd, off(rs1), shape`: a TCU matrix fragment ← memory | I | +| | | `2` | `vx_stm fs2, off(rs1), shape`: memory ← a TCU matrix fragment | S | +| | | `3` | `vx_flw.d fd, imm(rs1)`: `fd`, `fd+1` ← two words | I | +| | | `4` | `vx_flw.q fd, imm(rs1)`: `fd`..`fd+3` ← four words | I | +| | | `5` | `vx_fsw.d fs2, imm(rs1)`: two words ← `fs2`, `fs2+1` | S | +| | | `6` | `vx_fsw.q fs2, imm(rs1)`: four words ← `fs2`..`fs2+3` | S | +| | | `1`, `7` | reserved | | + +The wide loads and stores have the layout of `fld`/`flq` and `fsd`/`fsq`, with a +signed 12-bit byte offset. Each is a macro-op of 2 or 4 plain `flw`/`fsw`: uop +*k* moves word *k* at `rs1 + imm + 4k` to or from register `fd + k` (`fs2 + k`), +so the result is exactly that of the scalar sequence, with the same address +windows, exceptions and lane mask, and no atomicity across the words. Each word +needs only 4-byte alignment. The register must be a multiple of 2 (`.d`) or 4 +(`.q`) and `imm` a multiple of 8 or 16, which lets the expander place *k* in the +low bits of both without an adder. An encoding that breaks either rule is +reserved and its result undefined; the assembler rejects it. They are defined +with F and FLEN 32 (no D). + +The matrix fragment load and store move one tensor-core fragment (the +registers `vx_tensor.h`'s `load_matrix_sync`/`store_matrix_sync` fill) with one +instruction. `imm[4:0]` is the fragment **shape**, `imm[11:5]` a byte offset +`off` (a multiple of 32), and `rs1` the first of a **base group** of J integer +registers, a multiple of J, where register `rs1 + j` holds the lane's address +of the fragment's row group *j*. Each is a macro-op of one plain `flw`/`fsw` per +fragment register: uop *e* moves register `fd + e` (`fs2 + e`) to or from the +word at `x[rs1 | j(r)] + (off | c(r)·CS)`, where *r* = π⁻¹(*e*) is the fragment +element the WMMA/WGMMA expansion reads from that register (π is its +bank-conflict-free register order), `(j, c)` a bit split of *r*, and CS the byte +stride between a lane's words in a row. The register add takes the uop counter +directly; the shape only selects the row group and column. So neither the LSU +nor the address path does anything new: the +row step and the lane's own offset live in the base group, computed by +software once per tile, and the column offsets are bit selects. `off` must be +zero in the bits the column offsets use, which makes the OR an add; unrolled K +tiles then share one base group and differ only in `off`. The shapes are +`{ctx, role[1:0], trans, sparse}`; those a configuration has are + +| shape | fragment | J | per row group | registers | +|---|---|---|---|---| +| `0x00` | WMMA A, row-major | M steps | K steps, `4·tcK` B apart | 8 | +| `0x01` | WMMA sparse A, row-major | M steps | K/2 steps, `4·tcK` B apart | 4 | +| `0x06` | WMMA B, col-major | N steps per register | K steps, `4·tcK` B apart | 4 or 8 | +| `0x07` | WMMA symmetric sparse B, col-major | one N step per register | 1 word | 4 or 8 | +| `0x08` | WMMA C, row-major | M steps | N steps, `4·tcN` B apart | 8 | +| `0x18` | WGMMA C, row-major, one 8-register chunk | 2 | 4 columns, `4·tcN` B apart | 8 | + +with the sparse shapes under `VX_CFG_TCU_SPARSE_ENABLE` (sparse B only in the +symmetric layouts, NT=4 and NT=16) and the WGMMA one under +`VX_CFG_TCU_WGMMA_ENABLE`; every other shape is reserved (SimX stops on it). +In the RTL, [`VX_tcu_pkg.sv`](../../hw/rtl/tcu/VX_tcu_pkg.sv) (`tcu_ldm_uop`) +holds the shapes' geometry next to the operand permutations `VX_tcu_uops` +reads with, and `VX_uop_packld` asserts at elaboration that every uop of +every shape maps back through those permutations to its own register; SimX +derives its order from the same permutations by search. So the two cannot +drift apart. The words of a fragment register span its element type, +so one shape serves every ITYPE. A layout whose register gathers words from +several rows (a sub-word B row-major, A col-major) has no shape: software stages +such a tile transposed. A 16- or 32-register WGMMA accumulator takes 2 or 4 +`vx_stm`, one per chunk (`fs2 = f0, f8, f16, f24`), each chunk's column offset +in `off`. --- @@ -261,7 +321,7 @@ a priority encoder in the sequencer: | slot | expander | macro-ops | |---|---|---| -| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads | +| `UOP_PACKLD` | [`VX_uop_packld.sv`](../../hw/rtl/core/VX_uop_packld.sv) | packed loads, wide loads and stores, matrix fragment loads and stores | | `UOP_TCU` | [`VX_tcu_uops.sv`](../../hw/rtl/tcu/VX_tcu_uops.sv) | WMMA, WGMMA | | `UOP_GFX` | [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv) | ray-trace windows, fragment export | diff --git a/docs/designs/firesim_integration.md b/docs/designs/firesim_integration.md index a467c86948..4935db1a97 100644 --- a/docs/designs/firesim_integration.md +++ b/docs/designs/firesim_integration.md @@ -1,49 +1,114 @@ # FireSim integration — host-decoupled FPGA simulation of Vortex -This document describes how Vortex runs as a [FireSim](https://fires.im/) -target on a Xilinx Alveo U55C. It covers three things: - -1. The **execution model** — what host-decoupled simulation means for a - driver, and why the runtime *steps* the target rather than waiting on it. -2. The **integration architecture** — the Chisel target, the RTL wrapper, the - transport layer, and how a kernel launch reaches the card. -3. The **build flow** — elaboration through place-and-route, and the - invariants that make the difference between a design that fits and one - that silently does not. - -FireSim itself is an external dependency, developed in a clone of -[`vortexgpgpu/firesim`](https://github.com/vortexgpgpu/firesim) (branch -`vortex_3.x`, based on upstream 1.21.0) at `~/dev/firesim`, and consumed by -the Vortex build through `$(FIRESIM_PATH)`, which `config.mk` defaults to -`$(TOOLDIR)/firesim` — the same source/tool split POCL, Mesa and LLVM use. -Vortex's own changes to FireSim are small — 8 files, ~324 lines — and are bug -fixes and U55C platform plumbing rather than divergence; see §6. - +This document specifies how Vortex runs as a [FireSim](https://fires.im/) +target: a multi-core GPU simulated on an FPGA with cycle exactness, on the +Alveo U55C (Vitis/XRT) and the Alveo V80 (AVED/VRT). + +| § | contents | +|---|---| +| [1](#1-system-architecture) | System architecture: host stack, FPGA payload, execution model | +| [2](#2-synthesis-of-the-vortex-rtl) | Synthesis of the Vortex RTL | +| [3](#3-threaded-simulation) | Threaded simulation: Foldsim | +| [4](#4-the-simulation-timing-model) | The simulation timing model | +| [5](#5-the-memory-system) | The memory system and its configuration; shell configuration | +| [6](#6-observability) | Observability: wave traces, RTL messages and assertions, counters | +| [7](#7-the-transport-layer) | The transport layer | +| [8](#8-running) | Run targets and commands | +| [9](#9-measured-results) | Measured results | +| [10](#10-relationship-to-upstream-firesim) | Relationship to upstream FireSim | +| [11](#11-design-invariants) | Design invariants | +| [12](#12-known-limitations) | Known limitations | + +FireSim itself is an external dependency: the +[`vortexgpgpu/firesim`](https://github.com/vortexgpgpu/firesim) fork, consumed +through `$(FIRESIM_PATH)` (default `$(TOOLDIR)/firesim`) and pinned by +`FIRESIM_REV` in `VERSION` — see [§10](#10-relationship-to-upstream-firesim). For running kernels on hardware generally, see -[`fpga_setup.md`](../fpga_setup.md); for the other simulation vehicles, see +[`fpga_setup.md`](../fpga_setup.md); for the other simulation vehicles, [`simulation.md`](../simulation.md). --- -## 1. Execution model - -### 1.1 What "host-decoupled" means - -FireSim does not emulate Vortex at wall-clock speed and then let the host -observe it. It compiles the design through **Golden Gate** (MIDAS) into a -FAME-1 transformed model in which *every* target clock edge is explicitly -scheduled by the host. The FPGA holds the transformed design; the host driver -decides how many target cycles to advance and when. - -The practical consequence, and the thing that shapes the entire driver: **the -target does not run unless the host tells it to.** There is no free-running -clock to poll. A driver that issues an MMIO read and waits will wait forever, -because nothing advanced the design between the request and the read. - -This is the single most important difference from the XRT flow, where the -kernel clock free-runs and the host merely observes. - -### 1.2 Target cycles vs. host cycles +## 1. System architecture + +![FireSim + Vortex system stack](../assets/img/firesim_system_stack.svg) + +### 1.1 The layering + +Everything above the PCIe boundary except the FireSim driver core lives in +this repository — deliberately, so the FireSim checkout stays close to +upstream. From the application down: + +- **Application** (`tests/regression/`): the ordinary `vx_*` API; it + cannot tell FireSim from any other backend. +- **Runtime HAL** (`sw/runtime/firesim/vortex.cpp` → + `libvortex-firesim.so`): the driver ABI plus the software command + processor; kernel upload, DCR configuration, launch, flush. +- **Transport** (`sim/firesim/firesim_sim.cpp`, in `libfiresim.so`): owns the + simulator thread and marshals every operation onto it; stepping policy, + watchdogs, diagnostics ([§7](#7-the-transport-layer)). +- **FireSim driver core** (`$(FIRESIM_PATH)/sim/midas/src/main/cc`): the + widgets generated alongside the hardware — `peek_poke_t`, `loadmem_t`, + `clockmodule_t`, FASED's driver half — over a `simif_t` that is + `simif_aved` (VRT) on the V80, `simif_vitis` (XRT) on the U55C, and + `simif_emul` in the metasimulator. +- **FPGA payload** (`FireSim-generated.sv`): the shell shim, FireSim's + bridges, the FAME hub, and the target — Vortex behind + `hw/rtl/afu/firesim/VX_firesim_wrap.sv`. + +The **metasimulator** (`TARGET=xrtsim`/`avedsim`) is the same stack with the +bottom swapped: `libfiresim.so` *is* the Verilated FPGA payload, reached +through `simif_emul` over DPI, and needs no card, image, XRT or VRT. It runs +the same tokens on the same generated design, which is why it agrees with +silicon cycle for cycle and is the step before every bitstream. + +### 1.2 The FPGA payload + +![A Vortex instance + FireSim shell on the FPGA](../assets/img/firesim_fpga_architecture.svg) + +On the V80, the AVED static region (permanently in flash) carries the PCIe +endpoint, the AXI-Lite control network, the clock/reset logic and the HBM +NoC. The reconfigurable partition holds one module — `FireSim-generated.sv` — +containing: + +- **`AvedShim` / SimulationMaster**: the control-plane adapter. The compute + shell resolves AXI-Lite in 16-byte blocks and FireSim registers have read + side effects (LoadMem's `R_DATA` dequeues), so every register gets its own + block and the shim answers the padding words itself; the driver scales + addresses by 4. At open, the driver checks the `0x46697265` ("Fire") + fingerprint, writes and reads back a probe, and verifies the memory port is + bound at the elaborated base before any widget is touched. +- **Bridges** (the host-visible half of the simulation): PeekPoke (the + target's DCR/start/busy surface), LoadMem (bulk target memory), the clock + bridge (`tcycle`/`hcycle`), and FASED (the memory model, + [§5](#5-the-memory-system)). +- **The FAME hub** (`FAMETop`): token channels between the bridges and the + target; the mechanism of host decoupling ([§4](#4-the-simulation-timing-model)). +- **The target**: `VortexDUT` ▸ `VX_firesim_wrap` ▸ Vortex. + +Clocking on the V80 has one platform quirk worth knowing: FireSim gates the +target clock, the partition has no `BUFGCE` site, and a fabric clock gate is +not safe. `AvedConfig` therefore selects `FireSimFabricClockGate` — a plain +AND marked `gated_clock` that synthesis's gated-clock conversion moves onto +the clock-enable pin of every register, RAM and DSP it drives. `BUFGCE` +semantics, no derived clock; a pre-opt check fails the build if any kernel +clock is still driven by logic. (This is also why vendor OOC IP cannot sit +inside the kernel on aved: the conversion cannot enter it, so no unit takes +one there. The FPU is the STD one (`VX_CFG_FPU_TYPE_STD`), its FMA, divider +and square root take their soft path in the flow's copy of the sources, and +the RTU uses the soft FMA/divider — `VX_CFG_RTU_USE_DSP=0`. The metasimulator +and the image therefore run the same FPU, as the Foldsim flow's do.) + +### 1.3 Host-decoupled execution + +FireSim does not emulate Vortex at wall-clock speed and let the host observe +it. Golden Gate (MIDAS) compiles the design into a FAME-1 transformed model +in which *every* target clock edge is explicitly scheduled. The practical +consequence, and the thing that shapes the entire driver: **the target does +not run unless the host tells it to.** There is no free-running clock to +poll; a driver that issues an MMIO read and waits, waits forever. This is the +single most important difference from the XRT flow, where the kernel clock +free-runs and the host merely observes. Two clocks matter and must not be conflated: @@ -52,324 +117,482 @@ Two clocks matter and must not be conflated: | **target cycles** | cycles the Vortex design advanced | `clockmodule_t::tcycle()`, and Vortex's own `PERF: cycles=` | | **host cycles** | cycles the FPGA spent simulating them | `clockmodule_t::hcycle()` | -Their ratio is **FMR** (FPGA-to-Model Ratio) — how many host cycles it costs -to simulate one target cycle. FMR is the honest measure of simulation -efficiency; the effective target frequency is `host_freq / FMR`. - -**Do not quote FireSim's `Target Cycles Emulated` as a kernel cycle count.** -It reports cycles the *driver stepped*, not cycles the kernel took, and the -driver steps in growing batches (§3.3). A kernel is only observed to have -finished at the first poll *after* it does, so the figure is quantized to the -batch schedule and can overstate a short kernel by up to 4×. Two workloads -that finish inside the same inter-poll interval report the identical number. -Vortex's `PERF: cycles=` counter is exact and is what every measurement in -this repository uses. +Their ratio is **FMR** (FPGA-to-Model Ratio); the effective target frequency +is `host_freq / FMR`. **Do not quote FireSim's `Target Cycles Emulated` as a +kernel cycle count** — it reports cycles the *driver stepped*, quantized by +the batching schedule ([§4.3](#43-the-drivers-schedule)), and can overstate a +short kernel by up to 4×. Vortex's `PERF: cycles=` counter is exact and is +what every measurement in this repository uses. --- -## 2. Integration architecture +## 2. Synthesis of the Vortex RTL -### 2.1 Layering +![Build flow](../assets/img/firesim_build_flow.svg) -``` - tests/regression/ application - │ vx_* API - sw/runtime/firesim/vortex.cpp runtime HAL (driver ABI, XCLBIN_PATH) - │ firesim_sim class - sim/firesim/firesim_sim.cpp transport (request queue, stepping) - │ peek_poke / loadmem bridges - $(FIRESIM_PATH)/sim/… FireSim driver (simif_vitis, widgets) - │ XRT - ─────────────────────────────── PCIe ─────────────────────────────── - FireSim-generated.sv FAME-1 model (Golden Gate) - │ ctrl_* / AXI - hw/rtl/afu/firesim/VX_firesim_wrap.sv AFU wrapper - │ - Vortex the design under test +Vortex stays Verilog: Golden Gate sees it as an opaque BlackBox, transforms +only its boundary (FAME-1), and the card's platform flow synthesizes the +Verilog directly. The build is driven from `hw/syn/firesim/Makefile` with the +same tokens as the native flows: + +```bash +source ci/firesim_env.sh # after the card's Xilinx environment +# U55C (XRT): TARGET=hw | hw_emu +make -C hw/syn/firesim PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 TARGET=hw +# V80 (AVED): TARGET=hw | sim +make -C hw/syn/firesim TARGET=hw PERF=1 CONFIGS="-DVX_CFG_NUM_WARPS=16" ``` -Everything above the PCIe line except the FireSim driver lives in this -repository; that is deliberate, so the FireSim checkout stays close to -upstream. +The image lands in `__/bin` as +`firesim.vbin` or `firesim.xclbin`, beside `firesim.card`, which names the +card for `run-firesim`. `PREFIX`, `XLEN`, `CONFIGS`, `PERF=1` work as in the +other synthesis flows; extensions pull their sources in through +`hw/syn/extensions.mk`. Nothing given on this command line reaches FireSim's +own makes (`MAKEOVERRIDES` is cleared): FireSim uses `PLATFORM`, `TARGET` and +`BUILD_DIR` for its own purposes, and each sub-make is handed exactly what it +needs. -### 2.2 The Chisel target — `hw/syn/firesim/src/main/scala/VortexTarget.scala` +### 2.1 The flattened trees -`VortexDUT` instantiates Vortex as an opaque Verilog **BlackBox**. Golden Gate -cannot see inside it, which is the point: the FAME transform channelizes the -BlackBox's boundary signals and leaves its internals alone. +Golden Gate and Vivado take a file list, not include paths, so +`gen_sources.sh -P` preprocesses `hw/rtl` into one self-contained directory +with the macros already inlined, top `VX_firesim_wrap`. Two flattenings +exist because simulation and synthesis need opposite defines: -`VortexConfig` sets the memory interface the target presents: +| flattening | defines | consumed by | +|---|---|---| +| `vivado_srcs/` | `-DSYNTHESIS -DVIVADO -DNDEBUG` (+ `-DSIMULATION` for `TARGET=sim`/`hw_emu`) | platform synthesis | +| `vortex_srcs/` | `-DNDEBUG` only | the Verilator metasimulator | + +The invariants here have each cost hours when broken: + +- **`-DSYNTHESIS` is not optional.** `VX_dp_ram`/`VX_sp_ram` gate all + block-RAM inference behind it, and because the tree is *preprocessed*, the + BRAM branch is not inactive without it — it is deleted. Every memory then + lands in distributed RAM and place-and-route fails hours later at ~750K + LUTs-as-memory with nothing pointing at the cause. +- **Verilator gets a library search, not a flat list.** Handing it every + file makes it stop treating the macro-computed width parameters as + constants; only the packages (in dependency order) and the wrapper are + named, the rest resolves via `-y`. +- **A changed `CONFIGS` must invalidate the flatten.** Defines arrive on the + command line, which no file timestamp reflects; a stamp file rewritten only + when the flags differ hangs the dependency. +- **Only overrides and type selectors are passed, never the resolved + config.** Passing the whole resolved `XCONFIGS` replaces `VX_config.vh`'s + computations with report values, which sets the memory port count to zero; + both flatten rules grep the result for `VX_MEM_PORTS = 0` and fail fast. + +### 2.2 Vortex as a BlackBox + +`hw/syn/firesim/src/main/scala/VortexTarget.scala` instantiates Vortex as a +Chisel BlackBox. Golden Gate cannot see inside it, which is the point: the +FAME-1 transform channelizes the boundary signals — the AXI memory port, the +DCR/start/busy control surface — and leaves the internals alone. `image` +then runs, in order: `vivado-sources`, `xilinx-ips` (the `xil_fma`/`xil_fdiv` +floating-point cores for the card's part), `stage` (copy the Scala targets, +harnesses and the generated platform config into the fork — only what +changed, since FireSim's make judges elaboration by these timestamps), +`replace-rtl` (Golden Gate), and the platform's own build (`v++` on vitis, +slashkit on aved), handed: + +- `VORTEX_VIVADO_SRCS` — the netlist CIRCT elaborates from the flattened + tree, packaged into the kernel; +- `VORTEX_XILINX_IP_DIR` — the vendor cores, copied in fresh directories + (Tcl's `file copy` onto an existing directory nests instead of replacing, + which silently reuses the previous run's cores); +- `FREQUENCY` (default 60 MHz) and `STRATEGY` (default `TIMING`). -```scala -NastiParameters(dataBits = 512, addrBits = 32, idBits = 4) -``` +--- -`idBits` is **paired with `PLATFORM_MEMORY_ID_WIDTH` in -`hw/syn/firesim/Makefile`** and neither may be changed alone — one shapes the -IO the BlackBox is instantiated with, the other shapes the module's own ports, -and Verilog pads or truncates a disagreement with no diagnostic. - -It is also bounded from both sides, which is not obvious: - -- **Too large and it overflows.** FASED sizes its read-response egress with - `1 << idBits` in Scala `Int` arithmetic. At 32 that shift wraps to 1, which - both selects the untranslated egress branch and gives it a single queue for - every AXI id. Responses then leave carrying the requesting transaction's id - and another transaction's data. Elaboration succeeds and nothing warns — the - generated design is simply smaller and wrong. -- **Too large and it does not fit.** The response deinterleaver and the id-yank - queues carry per-id state, so model area scales with the id space. At 8 that - is 750,955 LUTs as memory against the 600,960 a U55C has. At 4 it is 51,678. - -4 is what Vortex actually drives: `VX_mem_to_axi`'s tag buffer is engaged -because `UUID_WIDTH` puts the upstream tag past any AXI id width, so reads -carry `CLOG2(TAG_BUFFER_SIZE) = 4` bits. It also matches FASED's `maxReads` -of 16. - -### 2.3 The RTL wrapper — `hw/rtl/afu/firesim/VX_firesim_wrap.sv` - -A thin AFU presenting Vortex as: - -- one AXI4 memory port (`m_axi_mem_0_*`), -- a DCR request/response pair for configuration, -- `start` / `busy` for launch and completion. - -`VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1` is set for this wrapper. Left at the -default of 2, Vortex builds two internal memory ports and `VX_mem_to_axi` -arbitrates them onto the one bank, recovering the originating port from the -*returned* AXI id (`rsp_xbar_sel_in = m_axi_rid[0 +: NUM_PORTS_IN_BITS]`). -That makes correctness depend on the memory model echoing `rid` bit-exact — an -assumption real AXI satisfies but a timing model need not. One port removes the -dependency entirely and matches what the wrapper presents. - -### 2.4 Memory model - -Target DRAM is served by **FASED**, FireSim's parameterized memory timing -model, backed by host DRAM through the LoadMem widget. The Vitis platform -binds `host_mem_0` across HBM[0:7] with a 4 GiB window — a 32-bit target spans -4 GB, with its program image at `0x80000000` and its stack just under 4 GB, so -a narrower binding puts kernel uploads outside the served window and the target -fetches memory nothing ever wrote, with no error anywhere. +## 3. Threaded simulation + +This backend simulates Vortex unthreaded: one physical copy of every module. +Threading a GPU's identical units through one physical copy each (FAME-5) is +the Foldsim flow, which compiles Vortex with CIRCT rather than Golden Gate and +has its own kernel, driver and image: see [foldsim.md](foldsim.md). --- -## 3. The transport layer — `sim/firesim/firesim_sim.cpp` +## 4. The simulation timing model -The largest single piece of the integration, and where host-decoupling shows -up as real complexity. +![Timing model](../assets/img/firesim_timing_model.svg) -### 3.1 Threading +Three layers, strictly separated: -FireSim's bridges are **not thread-safe** and must only be touched by the -thread running the simulation loop. The transport therefore owns a simulator -thread and marshals every operation onto it as a closure: +### 4.1 Token exchange decides *when a cycle may happen* -``` -caller ──push labelled_request_t──> queue ──> simulator thread ──> bridges - <──────── condition variable ──────── -``` +Every wire between a bridge and the target is a token channel. The FAME hub +fires one target cycle exactly when every input channel holds a token and +every output channel has space; otherwise the target clock stays gated and +the design is frozen mid-state, not observing time. However slow the host +is — PCIe latency, a paused driver, Verilator instead of silicon — the target +sees the identical cycle sequence. This is why metasim and FPGA agree cycle +for cycle, and why wall-clock speed never contaminates a measurement. -Each request carries a label so a trace attributes a stall to an operation -rather than showing a column of identical entries. +### 4.2 Timing models decide *what latency the target observes* -### 3.2 Memory access — the chunk-unit invariant +Latency the GPU experiences is not the card's memory latency — it is whatever +the FASED timing model schedules, in target cycles ([§5](#5-the-memory-system)). +A memory response that takes 400 host cycles to fetch from HBM still arrives, +in target time, exactly `readLatency` cycles after its request; the missing +host cycles simply never existed for the target. -`access_mem` moves data through LoadMem one host beat at a time, with -read-modify-write on unaligned edges. +### 4.3 The driver's schedule -**LoadMem's chunk is counted in 32-bit words, not bytes.** `read_mem` and -`write_mem` push exactly that many *words* through the data register. Treating -it as a byte count uploads a quarter of each beat and advances the address by a -quarter of a beat — storing two live bytes per eight and dropping every write -that does not land on a beat boundary. A kernel image uploaded that way is -fetched as a mixture of written and stale memory, and nothing reports an error. +The driver decides *how many cycles to ask for*. `is_busy()` must both catch +the launch edge (busy is a level; a batch stepped past a short kernel would +wait forever on a rise that already happened) and give long kernels +throughput (each poll costs a thread handoff plus an MMIO round trip). The +batch therefore grows ×4 from a floor of 16 target cycles to a ceiling +(default 100,000, `VORTEX_FIRESIM_STEP_BATCH`), dropping back to the floor +when the target goes idle. This schedule is what quantizes +`Target Cycles Emulated` (§1.3). DCR reads are the exception: `dcr_rsp_valid` +is a one-cycle pulse, so the poll advances one target cycle at a time, +bounded by `VORTEX_FIRESIM_DCR_POLL_ROUNDS` ([§7](#7-the-transport-layer)). -Whole-beat writes bypass the read-modify-write path. +--- -### 3.3 Advancing the target — adaptive step batching +## 5. The memory system -`is_busy()` must both catch the launch edge and give a long kernel throughput. -These pull in opposite directions: a small step never misses `busy` asserting -but costs an MMIO round trip per few cycles; a large step is efficient but can -step past a short kernel entirely. +![Memory system](../assets/img/firesim_memory_system.svg) -The driver grows the batch: floor of 16 target cycles, ×4 per poll up to a -ceiling, dropping back to the floor when the target goes idle. Cumulative poll -boundaries therefore land at 16, 80, 336, 1360, 5456, 21840, … +### 5.1 The path -This is also the origin of the `Target Cycles Emulated` quantization in §1.2 — -the counter is sampled at the poll that *observes* completion, not at the -busy→idle transition itself. +Vortex presents **one AXI4 memory port** (512-bit data, 32-bit address, +4-bit id, INCR bursts at full width). `VX_firesim_wrap` is built with +`VX_CFG_PLATFORM_MEMORY_NUM_BANKS=1` so Vortex builds exactly one internal +port: at the default of two, `VX_mem_to_axi` recovers the originating port +from the *returned* AXI id, which makes correctness depend on the memory +model echoing `rid` bit-exact — an assumption real AXI satisfies but a timing +model need not. -### 3.4 DCR reads must single-step +That port terminates in **FASED**, FireSim's memory timing model, which is +two halves: -`dcr_rsp_valid` is a one-cycle pulse passed straight out of the core, so a -batched step lands past it and the read reports unanswered on a target that -replied. `dcr_read` therefore advances one target cycle per iteration, and its -bound is expressed in cycles rather than polls. +- **Functional**: data lives in the card's real memory, reached over the + shell's host-memory port. Reads and writes are performed when accepted; + responses are held in egress queues (with a per-id reorder buffer) until + released. +- **Timing**: a latency-bandwidth pipe decides the release cycle. Runtime + registers, defaults: `readLatency=30`, `writeLatency=30` target cycles, + ≤16 reads and ≤16 writes in flight. -The cost is one MMIO round trip per target cycle — tenths of a second for a -cache flush on the card, minutes under emulation. Batched polling would need -the response **latched at the wrapper**, which is a target-side change and a -new bitstream; it is a known follow-up rather than a driver-side workaround. +The **LoadMem widget** is the host's side door onto the same functional +memory — kernel upload, buffer copies, readback — consuming no target time. -### 3.5 Diagnostics +### 5.2 Configuring the timing model per run -Vortex's own performance counters are the only view inside the core that does -not require re-elaborating the design, and they are read over the DCR path. -They are gated behind `PERF_ENABLE` (see `PERF_DEFINES` in -`hw/syn/firesim/Makefile`); **without it every counter reads zero, which is -indistinguishable from an idle core rather than an absent instrument.** The -driver says so explicitly rather than leaving it to be inferred. +FASED's registers are set by plusargs, which the runtime forwards from +`VORTEX_FIRESIM_ARGS` (card and metasim alike). Any register set explicitly +requires the rest to be covered by the hardware-defaults plusarg: + +```bash +VORTEX_FIRESIM_ARGS="+mm_readLatency_0=100 +mm_writeLatency_0=100 +mm_useHardwareDefaultRuntimeSettings_0" \ + ./ci/blackbox.sh --driver=firesim --target=avedsim --app=demo ... +``` + +Validated: a 4-core demo runs 15,541 cycles at the default 30-cycle latency +and 47,371 at 100/100 — same 9,776 instructions, `PASSED` both ways. +This is the only memory knob that needs no rebuild. + +The model's ceiling is worth computing before trusting a bandwidth-bound +number: `maxReads × 64 B / readLatency` ≈ 34 B per target cycle for the +whole GPU at the defaults. Raising +the ceiling means `maxReads` (compile-time, and the egress area scales with +it) or lower modelled latency (runtime). There is no DRAM structure — banks, +rows, refresh — in the default pipe; FASED has DDR3 FIFO/FR-FCFS models and a +tags-only LLC model as compile-time alternatives (`WithDDR3FIFOMAS`, +`WithLLCModel`), none enabled for Vortex today, and no HBM timing model at +all. + +FASED's counters (`totalReads`, `totalWrites`, beat counts, `readMaxReqs`, +response errors) are read over MMIO and printed by the driver on every stall +report and watchdog — the first place to look when a kernel starves. + +### 5.3 Configuring the shell + +Everything card-specific comes from the platform descriptor +(`hw/syn/xilinx/platforms/.toml`), whose `[firesim]` section names the +FireSim platform, the part, and the host-memory binding; `gen_platform.py` +generates from it both the make fragment and the platform config class +(`V80PlatformConfig` = `WithHostMemBase(0x40_0000_0000)` + `DefaultAvedConfig`), +so one file governs elaboration and link alike. + +| | V80 (aved) | U55C (vitis) | +|---|---|---| +| binding | `host_mem_sp = "MEM"`: the HBM VNOC, **32 GB at 0x40_0000_0000** | `HOST_MEM_SP = HBM[0:7]`: 8 × 512 MB = 4 GiB at 0 | +| why | the whole 32-bit target map in one window; HBM0 (1 GB) would truncate it — the stack sits just under 4 GB; `HOST` (QDMA slave bridge) never completes reads and is refused by the descriptor | the card has no DDR bank; a narrower binding silently serves part of the map and kernel uploads land outside it | +| host channel | one, 64-bit, at the host clock | one, 64-bit | +| image-build knobs | `FREQUENCY` (60), `STRATEGY` (`TIMING`), `DEVICE` | + `PLATFORM=` | + +The host channel's width bounds *simulation speed* under memory traffic, +never target timing: a starved FASED gates the target clock, FMR rises, and +the kernel's cycle count is unchanged. + +Two width couplings must move together or nothing warns: + +- **`idBits` (VortexTarget.scala) ↔ `PLATFORM_MEMORY_ID_WIDTH` + (hw/syn/firesim/Makefile).** One shapes the IO the BlackBox is instantiated + with, the other the module's ports; Verilog pads or truncates a + disagreement silently. The value is bounded both ways: at 32, + `1 << idBits` wraps Scala's `Int` and every AXI id shares one response + queue (silently wrong data); at 8, per-id egress state cost 750,955 LUTs + against a U55C's 600,960. 4 matches what Vortex drives + (`CLOG2(TAG_BUFFER_SIZE)`) and FASED's `maxReads` of 16. +- **`addrBits` ↔ `VX_CFG_MEM_ADDR_WIDTH`.** The Scala side pins 32 while the + wrapper follows XLEN (48 at XLEN=64): XLEN=64 on FireSim needs both changed + together or addresses truncate silently. --- -## 4. Build flow +## 6. Observability -``` -make -C hw/syn/firesim stage copy Chisel/C++ targets into $(FIRESIM_PATH) -make -C hw/syn/firesim sources Verilator source list (packages + wrapper) -make -C hw/syn/firesim vivado-sources - flatten Vortex with macros inlined (-P) -cd $(FIRESIM_PATH)/sim && make replace-rtl PLATFORM=vitis DESIGN=VortexTarget - Golden Gate elaboration -cd platforms/vitis/… && make bitstream DEVICE=… FREQUENCY=60.0 STRATEGY=TIMING - synthesis + place-and-route +### 6.1 Wave traces + +**Metasimulator (the primary path).** Build the metasim library with tracing +and pass the dump plusargs at run time: + +```bash +EMUL_DEBUG=1 \ +VORTEX_FIRESIM_EMUL_ARGS="+waveformfile=$PWD/demo.vcd +max-cycles=60000" \ + ./ci/blackbox.sh --driver=firesim --target=avedsim --cores=4 --warps=4 --threads=4 --app=demo ``` -### 4.1 Two source lists, and why - -Verilator resolves Vortex by **library search**, so `sources` names only the -packages and the wrapper — handing it the whole tree as a flat list makes it -stop treating the macro-computed width parameters as constants. Vivado has no -such search and must be handed every file, so `vivado-sources` flattens the -tree into one directory with macros already inlined. The constant-parameter -failure that rules flattening out for Verilator is an elaboration behaviour, -not a synthesis one. - -### 4.2 Synthesis invariants - -**`SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG` is not optional.** -`VX_dp_ram` and `VX_sp_ram` gate all block-RAM inference behind `` `ifdef -SYNTHESIS ``, and `gen_sources.sh -P` *preprocesses* into the flattened tree — -so without it the BRAM branch is not merely inactive, it is deleted before -Vivado sees it and every memory in the design lands in distributed RAM. The -symptom is a place-and-route failure hours later at ~750k LUTs-as-memory, with -nothing pointing at the cause. - -**The Makefile is a prerequisite of the flatten rule.** Without it, changing -any define does not invalidate the flattened tree and the next build silently -reuses the previous elaboration. Note it must be `$(firstword $(MAKEFILE_LIST))` -— `lastword` resolves to `build/config.mk`. - -**Guard the elaboration, not just the build.** `v++` runs synthesis into -place-and-route without stopping, so an over-utilized design surfaces only -after the placer has been tried. Two cheap checks catch the failure modes -above before hours are spent: that FASED elaborated a per-id response path -(`ReorderBuffer` present, multi-bit `MultiQueue` selectors), and that the -post-synthesis utilization report fits. Any such watchdog **must be -freshness-qualified** against the current run's timestamp — an unqualified -lookup reads the previous build's report and kills a healthy build. +`EMUL_DEBUG=1` (sim/firesim/Makefile) Verilates with `--trace +--trace-structs +define+DEBUG` into its own build directory (`…_dbg`) — the +model rule does not depend on flags, so debug and normal builds must not +share one. The plusargs are FireSim's emul-top contract (`top.sv`): +`+waveformfile=` opens the VCD, and `+max-cycles=` is **required** — +dumping runs from `+dump-start` (default 0) for `max-cycles` host clocks and +`$dumpoff`s at the end, so without it the window is empty. The run itself +continues to completion after the window closes. `VORTEX_FIRESIM_EMUL_ARGS` +also takes ordinary Verilator runtime args (e.g. `+verilator+rand+reset+2` +to randomize uninitialized state). + +**Size the window before running.** The dump covers every signal of the +FPGA payload: a 4-core demo writes ~23 GB for a 60,000 +host-cycle window (~380 KB per host cycle). Place `+dump-start` just before +the region of interest and keep `+max-cycles` in the low tens of thousands; +the simulation itself runs to completion after the window closes, unperturbed +(same 15,541 cycles traced and untraced). + +**Reading the waves.** The hierarchy is +`TOP.emul.FPGATop.sim.target.…` — `emul` is the DPI testbench, `FPGATop` +the FPGA payload, `sim.target` the FAME hub's interior. Orientation: + +- everything toggles on the **host clock**; one target cycle is one firing + of the hub, so anchor on the clock channel or `ctrl_busy` rather than + counting clock edges; +- the target lives under `FPGATop.sim.target…`; +- the bridges' AXI (FASED) and MMIO surfaces are at the top level — the + fastest way to see whether a hang is the target or the transport. + +**U55C `hw_emu`** uses XSIM, not Verilator: waveform capture is governed by +`sw/runtime/firesim/xrt.ini` (`debug_mode=batch` writes the `.wdb`; it is +~800 MB for a demo run and the dominant cost, so it is off by default). + +**On the card: none.** There is no ILA in these images (AutoILA is off on +the V80 — Versal has no `ila` core — and not wired on the U55C flow). On-card +visibility is the counters and driver diagnostics below; anything deeper +moves to the metasimulator with the same image configuration. + +### 6.2 `$display`, `$error`, assertions + +All of Vortex's messages and assertions live behind `SIMULATION` +(`hw/rtl/VX_platform.vh`: `STATIC_ASSERT`, `ASSERT`/`RUNTIME_ASSERT`, +`ERROR`, and the `DBG_TRACE_*` machinery). Where that define exists in this +flow: + +| vehicle | `SIMULATION`? | consequence | +|---|---|---| +| `TARGET=sim` (V80 XSIM image), `TARGET=hw_emu` (U55C) | **yes** | `$display`/`$error`/asserts fire in the platform simulator's log | +| metasim `xrtsim`/`avedsim` | no (`-DNDEBUG` only) | compiled out — the metasim flattening carries neither synthesis nor simulation defines, and adding `-DSIMULATION` there is untested (a few library modules would also want the `dpi_trace` objects the metasim does not link) | +| `TARGET=hw` | no (`SYNTHESIS`) | compiled out, as in every synthesis flow | + +Golden Gate's own synthesized printf/assert machinery (`SynthPrints`, +`SynthAsserts` — both enabled in the platform config) applies only to +printf/stop nodes authored in Chisel or present in the FIRRTL; Vortex arrives +as a BlackBox containing neither, so it contributes nothing here. In short: +**for RTL-level messages and asserts, use `TARGET=sim`/`hw_emu`, or rtlsim +outside FireSim entirely; on `hw`, instrument via counters and the driver.** + +### 6.3 Performance counters and driver diagnostics + +Vortex's performance counters are the only view inside the core that needs no +re-elaboration. They are read over the DCR path and gated behind +`PERF_ENABLE`: build the image with `PERF=1`, and note that **without it +every counter reads zero, which is indistinguishable from an idle core**. +The driver prints, on every stall report, watchdog expiry and (optionally) +completion: FASED's request/beat/error counters, the core's scheduler and +pipeline-stall counters, and the D-cache counters — enough to separate "a +stalled pipeline" from "a memory request that never returned". + +The transport's own knobs and reports: + +| env var | effect | +|---|---| +| `VORTEX_FIRESIM_TRACE=1` | timestamped log of every request the transport services | +| `VORTEX_FIRESIM_ARGS` | extra plusargs to the FireSim driver (`+mm_*`, §5.2) | +| `VORTEX_FIRESIM_EMUL_ARGS` | Verilator runtime args, metasim only (waves, rand-reset) | +| `VORTEX_FIRESIM_TIMEOUT_S` | launch watchdog, wall seconds; expiry prints the full counter report and exits 124 | +| `VORTEX_FIRESIM_STALL_CYCLES` | target-cycles-busy threshold for the wedge report | +| `VORTEX_FIRESIM_STEP_BATCH` | batching ceiling (§4.3) | +| `VORTEX_FIRESIM_DCR_POLL_ROUNDS` | DCR read poll bound, in target cycles | +| `VORTEX_FIRESIM_PREFILL`, `VORTEX_FIRESIM_DUMP_UPLOADS`, `VORTEX_FIRESIM_VERIFY_WRITES`, `VORTEX_FIRESIM_REPORT_ON_IDLE` | debugging aids: pattern-stamp a region before launch; dump every upload to files; read back every write; counter dump on healthy completion | + +### 6.4 Choosing a vehicle + +| need | vehicle | +|---|---| +| waves of the exact FPGA payload | metasim + `EMUL_DEBUG=1` | +| Vortex `$display`/asserts | `TARGET=sim` / `hw_emu` (or rtlsim) | +| speed with full fidelity | the card | +| memory-timing experiments without rebuilds | any vehicle + `VORTEX_FIRESIM_ARGS` | --- -## 5. Measured results +## 7. The transport layer + +`sim/firesim/firesim_sim.cpp` is the largest single piece of the +integration, and where host-decoupling shows up as real complexity. + +- **Threading.** FireSim's bridges are not thread-safe and are only touched + by the simulator thread; every operation is a labelled closure marshalled + through a queue (the label is what makes a trace attribute a stall to an + operation). In the metasimulator the same thread runs Verilator's event + loop, and `entry()` is reached from the emul top's initial block. +- **Memory access.** `access_mem` moves data through LoadMem one host beat + at a time with read–modify–write on unaligned edges. **LoadMem's chunk is + counted in 32-bit words, not bytes**; treating it as bytes stores two live + bytes per eight and drops every unaligned write — a kernel image uploaded + that way is fetched as a mixture of written and stale memory with no error + anywhere. Whole-beat writes bypass the RMW path. +- **Stepping** is §4.3's adaptive batching; **DCR reads** single-step + because the response valid is a one-cycle pulse. Batched DCR polling would + need the response latched in the wrapper — a target-side change and a new + bitstream, noted as a follow-up. +- **Peeks after a blocking step are non-blocking.** A blocking peek waits up + to 0.1 s for the `PRECISE_PEEKABLE` flag before returning; after a blocking + step the value is already final and the wait adds nothing but time. +- **Watchdogs.** A launch that exceeds `VORTEX_FIRESIM_TIMEOUT_S` or a + busy-level that exceeds `VORTEX_FIRESIM_STALL_CYCLES` produces the full + counter report (§6.3) and a clean abort — on this vehicle, "hung" must be + distinguishable from "slow" without a reboot. -U55C, kernel clock constrained at 60 MHz, `STRATEGY=TIMING`. Configuration: -1 cluster / 1 core / 4 warps / 4 threads, SIMD 4, L1 only, no A or C -extension, no TCU/DXA/RTU/TEX/RASTER/OM, `FPU_USE_DSP=0`. +--- + +## 8. Running -### 5.1 Utilization (kernel only) +```bash +# metasimulator (no image, no card): +./ci/blackbox.sh --driver=firesim --target=avedsim --warps=4 --threads=4 --app=demo +# the card: +FPGA_BIN_DIR=$PWD/hw/syn/firesim/build32_aved_hw/bin \ + ./ci/blackbox.sh --driver=firesim --target=hw --warps=4 --threads=4 --app=demo +``` -| resource | used | available | % | +| card | FireSim platform | run targets | image / card named by | |---|---|---|---| -| CLB LUTs | 131,578 | 1,303,680 | 10.09 | -| — as logic | 79,900 | 1,303,680 | 6.13 | -| — as memory | 51,678 | 600,960 | 8.60 | -| CLB registers | 62,004 | 2,607,360 | 2.38 | -| Block RAM tiles | 109 | 2,016 | 5.41 | -| URAM | 8 | 960 | 0.83 | -| DSP | 24 | 9,024 | 0.27 | +| `u55c` | `vitis` (XRT) | `xrtsim`, `hw_emu`, `hw` | `XCLBIN_PATH` | +| `v80` | `aved` (VRT) | `avedsim`, `sim`, `hw` | `VRT_VBIN_PATH`, `VRT_DEVICE_BDF` (discovered when unset) | + +`run-firesim` takes the card from `$(FPGA_BIN_DIR)/firesim.card`; the +metasim targets need no image and default to the card of their platform +(`FIRESIM_CARD` overrides). The architecture flags (`--cores`, `--warps`, +`--threads`, `CONFIGS=`) must match the image's — they configure the runtime +and, in the metasim, the model build. On the V80, `TARGET=hw` runs +`ci/v80_preflight.sh` first (link state, cooldown between PDI loads), and a +first `SIGINT`/`SIGTERM` stops the run cleanly (`FAILED`, exit 130/143; 124 +is the launch watchdog). A FireSim run uses the card's native driver +environment — on the V80 that means the slashkit-linked Xilinx tools and the +static shell in flash, as for the native `aved` backend. -DSP usage is near zero because this configuration builds the FPU from LUTs by -choice (`FPU_USE_DSP=0`), not for want of DSPs. +--- -### 5.2 Timing +## 9. Measured results -Kernel clock `clk_out1_firesim_clocking` closes at **WNS +5.283 ns** over -548,101 endpoints with zero failing, implying Fmax ≈ 87.8 MHz against the -60 MHz constraint. The design-level WNS of +0.003 ns belongs to the Vitis -shell's 450 MHz `hbm_aclk`, **not** to Vortex — reading the global number as -the kernel's inverts the conclusion. +### 9.1 U55C -### 5.3 Kernels +1 cluster / 1 core / 4 warps / 4 threads, L1 only, `FPU_USE_DSP=0`, kernel +clock 60 MHz. Kernel-only utilization: 131,578 LUTs (10.1%), 62,004 FFs, +109 BRAM tiles, 8 URAM, 24 DSP. The kernel clock closes at WNS +5.283 ns +(Fmax ≈ 87.8 MHz); the design-level WNS of +0.003 ns belongs to the Vitis +shell's 450 MHz `hbm_aclk`, not to Vortex. `sgemm -n1024`: +instrs=1,187,250,192, cycles=3,229,839,120, 84.6 s of kernel time (~153× +faster than rtlsim on `sgemm -n32`). Instruction counts match rtlsim +exactly; cycle counts differ −1.3%…+7.1% — the FASED timing model, not the +core: FPGA and hw_emu agree exactly. -| app | args | instrs | cycles | IPC | -|---|---|---|---|---| -| demo | -n64 | 2,444 | 6,429 | 0.380 | -| vecadd | -n1024 | 6,160 | 18,837 | 0.327 | -| sgemm | -n32 | 48,400 | 115,490 | 0.419 | -| sgemm | -n1024 | 1,187,250,192 | 3,229,839,120 | 0.368 | +### 9.2 V80 -`sgemm -n1024` completes in **84.6 s** of kernel time. Instruction counts match -rtlsim exactly; cycle counts differ by −1.3% to +7.1%, which is the FASED -memory timing model rather than the core — FPGA and hw_emu cycle counts agree -exactly. +tinygpu: 1 cluster / 1 core / 2 warps / 2 threads, no L1 caches or local +memory, STD FPU, one memory bank, kernel clock 143 MHz (WNS +0.752 ns). +Kernel-only synthesis: 103,369 LUTs (51,943 as distributed RAM), 34,376 +FFs, 5 BRAM tiles, 8 URAM, 8 DSP. -For scale: `sgemm -n32` takes 1228 ms on rtlsim and 8 ms on the card, ~153×. +| app | instrs | cycles, metasim (`avedsim`) | cycles, V80 | kernel time, V80 | +|---|---:|---:|---:|---:| +| `demo` | 1,222 | 25,624 | 25,624 | | +| `sgemm -n16` | 15,240 | 317,163 | 317,163 | 5 ms | +| `sgemm -n64` | 673,800 | | 13,937,835 | 178 ms | ---- +The metasimulator and the image agree cycle for cycle. Instruction counts +match rtlsim's. Cycles are 5.0 % above rtlsim's on the same latency-bandwidth +settings (`AXI_BUS=1`; `demo`, `sgemm -n16`), and 5.1 % above the Foldsim +image's on `sgemm -n64`, from the memory model's channel mechanics. -## 6. Relationship to upstream FireSim +Threaded images, and the same tinygpu on the Foldsim flow, are in +[foldsim.md](foldsim.md) §7. -The FireSim checkout is a **git clone, not a vendored copy**. Vortex's changes -live on branch `vortex_3.x` of `vortexgpgpu/firesim`, based on upstream tag -1.21.0, and amount to 8 files: +--- -| area | files | why | -|---|---|---| -| Vitis platform | `simif_vitis.cc`, `VitisShim.scala` | DRAM aperture across HBM[0:7] with a 4 GiB window; host-memory base address | -| driver lifetime | `entry.cc` | tear the platform down at the end of `entry()`; a global `unique_ptr` is destroyed during static destruction, after XRT's own statics, and segfaults in the device registry | -| cycle accounting | `simulation.{cc,h}` | `end_tcycle` is read absolute and never rebased, so it accumulates across runs and FMR divides a per-run quantity by a cumulative one | -| build | `cl_firesim/Makefile`, `gen_xo.tcl`, `package_kernel.tcl` | U55C build fixes; package the flattened Vortex sources into the kernel | +## 10. Relationship to upstream FireSim -All are candidates for upstreaming: only the `HOST_MEM_SP` default is -U55C-specific, and it is parameterized rather than hardcoded. The branch should -shrink over time as changes are accepted upstream. +The FireSim checkout is a clone of the `vortexgpgpu/firesim` fork (branch +`vortex_3.x`, based on upstream 1.21.0), consumed through `FIRESIM_PATH` and +pinned by `FIRESIM_REV` in `VERSION`. `make stage` refuses a checkout that +does not contain the pinned commit; being ahead of it is normal while +developing the fork. The fork's changes are platform plumbing and bug fixes +rather than divergence — the AVED platform (shim, register blocks, fabric +clock gates, host-memory base as a config field), driver lifetime and +toolchain fixes, cycle-accounting fixes, and kernel packaging — all +candidates for upstreaming. Everything Vortex-specific (targets, wrapper, +transport) lives in this repository and is staged in at +build time. --- -## 7. Design invariants - -1. **`idBits` and `PLATFORM_MEMORY_ID_WIDTH` move together.** A disagreement is - padded or truncated silently. Raising either requires re-checking - utilization, not merely avoiding the `1 << idBits` overflow. -2. **`SYNTHESIS` must reach the flattened tree.** Its absence is invisible - until place-and-route and costs hours. -3. **Vortex's `PERF:` counters are the cycle authority.** FireSim's - `Target Cycles Emulated` is quantized by the step schedule. -4. **The target only advances when stepped.** Any new driver operation that - waits on target state must step, and must know whether the signal it waits - on is a pulse or a level. -5. **All-zero performance counters mean the instrument is absent** - (`PERF_ENABLE` off), not that the core is idle. +## 11. Design invariants + +1. **The target only advances when stepped.** Any new driver operation that + waits on target state must step, and must know whether the signal it + waits on is a pulse or a level. +2. **`idBits` ↔ `PLATFORM_MEMORY_ID_WIDTH` move together**, and `addrBits` ↔ + `VX_CFG_MEM_ADDR_WIDTH`; disagreements pad or truncate silently. +3. **`SYNTHESIS` must reach the flattened tree** — its absence surfaces + hours later in place-and-route. +4. **Vortex's `PERF:` counters are the cycle authority**, never + `Target Cycles Emulated`. +5. **All-zero performance counters mean `PERF_ENABLE` is off**, not that the + core is idle. +6. **LoadMem's chunk unit is 32-bit words**, not bytes. --- -## 8. Known limitations - -- **Metasimulation is dead.** FireSim's Verilator metasim builds and starts, - then deadlocks before target cycle 0 (65 threads in `futex_wait`, 0% CPU, no - output past `entering simif.run`). It is deliberately not fixed: the card - supersedes every purpose it served. Anyone reaching for it should expect this - rather than read the silence as a slow build. -- **DCR reads are slow** by one MMIO round trip per target cycle (§3.4). -- **Regression coverage is partial.** The kernels in §5.3 plus `conv3`, - `dogfood`, `dotproduct` and `mstress` pass on the card; the suite has not - been swept exhaustively, and a large share of it requires extensions this - configuration disables. -- **The conda environment is not relocatable** as built. It records its own - prefix in every script shebang and in package metadata, so the tree must - stay where `build-setup.sh` created it, or be re-packaged with `conda-pack` - and unpacked with `conda-unpack` at the new location. See - [`../building_toolchain.md`](../building_toolchain.md) (section: FireSim). +## 12. Known limitations + +- **No RTL `$display`/asserts on `hw`** (§6.2); the metasim + compiles them out too (no DPI linked). RTL-message debugging belongs to + `TARGET=sim`/`hw_emu` or rtlsim. +- **DCR reads single-step** — one MMIO round trip per target cycle; batched + polling needs a latched response in the wrapper (target-side follow-up). +- **XLEN=64 is not wired through FireSim**: `addrBits` is pinned at 32 + (§5.3). +- **One host memory channel** bounds simulation speed under heavy memory + traffic (never target timing). +- **The default memory model is a flat latency-bandwidth pipe** — no banks, + rows, refresh or HBM timing (§5.2). +- **The conda environment is not relocatable** as built; re-package with + `conda-pack` to move it (see + [`../building_toolchain.md`](../building_toolchain.md)). diff --git a/docs/designs/foldsim.md b/docs/designs/foldsim.md new file mode 100644 index 0000000000..9f8b46a509 --- /dev/null +++ b/docs/designs/foldsim.md @@ -0,0 +1,580 @@ +# Foldsim — threaded FPGA simulation of Vortex, on CIRCT + +Foldsim runs Vortex on the Alveo V80 as a host-decoupled, cycle-exact +simulator in which every class of identical units — sockets, issue slices, +ALUs, FPUs, load-store units — is time-multiplexed through **one physical +copy** (FAME-5 model multithreading). A GPU many times the card's size fits +on it, and runs orders of magnitude faster than Verilator. + +| § | contents | +|---|---| +| [1](#1-overview) | Overview and how to run it | +| [2](#2-the-compiler) | The compiler: netlists, threading, composition | +| [3](#3-the-simulator) | The simulator: threads, channels, the token fabric; timing options; domains and relays | +| [4](#4-the-kernel) | The kernel: memory and control endpoints | +| [5](#5-the-runtime) | The runtime and its register contract | +| [6](#6-validation) | Validation | +| [7](#7-measured-results) | Measured results | +| [8](#8-invariants-and-limitations) | Invariants and limitations | + +--- + +## 1. Overview + +![Foldsim architecture](../assets/img/foldsim_architecture.svg) + +The whole flow is one IR from SystemVerilog to Verilog: `circt-verilog` +imports Vortex, Foldsim's passes cut it into models and compose the +simulator, `firtool` writes it out, and the card's native flow (IP-XACT +packaging, `slashkit link`) builds the image. (`foldsim-opt` also carries +Beacon, which counts the logic levels of a Vortex netlist before Vivado: +[beacon.md](beacon.md).) Foldsim is a project of its own +that builds on CIRCT, as POCL builds on LLVM: the `foldsim` driver and +`foldsim-opt` come from the toolchain (`$(TOOLDIR)/foldsim`), CIRCT from +`$(TOOLDIR)/circt`. Nothing else is in the flow: no Yosys, no sv2v. + +Foldsim is a general-purpose RTL simulator: everything that is not specific +to Vortex is in the Foldsim repository. That is the compiler, the boundary +lifting (`foldsim-lift`), and the FPGA shell the simulator runs in (control +and memory endpoints, the DRAM bank timing model, the register map, the +in-process host and the endpoint's unit tests), installed as +`$(FOLDSIM_PATH)/share/foldsim/shell` (`FOLDSIM_SHELL`). Vortex keeps only its +glue: + +| Vortex path | what | +|---|---| +| `hw/rtl/afu/foldsim/target/foldsim_target.sv` | Vortex with its AXI4 ports, DCR, start and busy as the shell's plain target ports | +| `hw/syn/foldsim` | which units to lift and fold, and the V80 image flow on the native aved flow | +| `sim/foldsim` | the shell's in-process host built for a Vortex configuration, with the Xilinx FP cores' models | +| `sw/runtime/foldsim` | the Vortex runtime driver over the shell's registers | +| `ci/testcases/foldsim.yaml` | the cycle-exact parity check against rtlsim | + +```bash +export FOLDSIM_CLASSES="VX_socket__lift VX_core__lift VX_compute_slice__lift" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID VX_core:drop=INSTANCE_ID VX_compute_slice:drop=INSTANCE_ID,SLICE_STR" +CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --app=sgemm # Verilator +make -C hw/syn/foldsim CONFIGS="$CONFIGS" # the image +FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin CONFIGS="$CONFIGS" \ + ./ci/blackbox.sh --driver=foldsim --target=hw --app=sgemm # the V80 +``` + +`FOLDSIM_CLASSES` names the modules threaded as classes; a class may contain +others (the socket contains the issue slices). `FOLDSIM_LIFT` gives +interface-port units a plain-port boundary first, so they can be classes +(Foldsim's `foldsim-lift`, a source-to-source pass on pyslang; the +RTL is not changed). `drop=P` removes a parameter that only names an instance (`INSTANCE_ID`), so +all instances of the class elaborate to one module; instance IDs that reach +logic are already input ports in the RTL (the socket's `socket_id`). +`port=P` turns such a parameter into an input port where a unit still has one. +The tools are found through `FOLDSIM` (the driver), `FOLDSIM_SHELL` and +`CIRCT_BIN`, by default under `config.mk`'s `FOLDSIM_PATH` and +`hw/syn/common.mk`'s `CIRCT_PATH`. + +A class `A+B` fuses two modules of the same parent (`foldsim-fuse-models`): +each `A` is paired with the one `B` it exchanges bits with, and each pair is +simulated as one model, so a path from one through the other and back takes +a thread one host cycle instead of a round trip through their channels. The +LSU slice and its scheduler are such a pair (the slice's request, the +scheduler's `ready` on it, the slice's `ready` on that): fused, tinygpu runs +at 3.00 host cycles per target cycle instead of 4.00 (V80), with the same +threads and copies and fewer LUTs. + +The flow builds the configuration's FPU, as the native Vivado flows do: of +type DSP, the Xilinx floating-point cores. Their netlists are encrypted, so +their state cannot be threaded; each core declares a pipeline contract +(`hw/scripts/xilinx_ip_stubs.sv`: latency, clock enable, data ports, no other +state), and the compiler simulates each by one unthreaded copy of the core and +a per-thread store of its results (`foldsim --fold-pipelines`, see the +compiler's `foldsim-find-pipelines`). `hw/scripts/check_ip_contracts.py` checks +each contract's latency against the core's generation, the RTL and the +configuration; the in-process simulator runs the cores' models +(`sim/foldsim/xilinx_fp_models.sv`). `-DVX_CFG_FPU_TYPE_STD` in `CONFIGS` +selects the soft FPU. A block-RAM-sized RAM that reads asynchronously gets a registered read +from the RAM fold before the target is lifted (`/ram_fold.rpt`, see the +hardware library's "Block RAM and asynchronous reads"); `gen/memories.rpt` lists +the RAMs that still read asynchronously. + +--- + +## 2. The compiler + +The `foldsim` driver drives four steps, all in one MLIR module of the HW, +Comb and Seq dialects; steps 2 to 4 run in `foldsim-opt`: + +1. **Import and flatten.** `circt-verilog` elaborates the target + (`foldsim_target`: Vortex with its AXI4 port, DCR, start and busy as plain + ports) with `--detect-memories`; everything but the class modules is + inlined. +2. **Cut netlists, innermost first** (`foldsim-extract-models{netlist}`). A + module with class instances becomes `_model` — its own logic, with + ports for what it exchanges with the instances — and `` becomes a + netlist of the same ports, instantiating `_model` beside the class + instances, wired as before. Instances of the module elsewhere stay valid, + so the socket's netlist is cut out of the top in turn. Bits of a model + output that are always the same constant stay in the parent, and an input + driven only by the parent's inputs and constants, or exactly by another + model's output, is wired to those directly. Small logic (up to 64 ops) + that drives the same input of every instance of a model from other models' + outputs is moved into the model (`absorb-logic`), so that input comes from + those models rather than through the parent's model. +3. **Clean up the models**: `hw-expand-array-writes` (a conditional write of + one array element becomes a select per element), `seq-reg-const-prop`, + `comb-share-adders` (`mux(c, a+b, a-b)` → one adder), + `comb-low-bits-narrowing` (adders whose operands have known-zero low bits), + canonicalization, and CSE after a topological sort (in a module body, CSE + cannot merge a duplicate whose users come first). These are what a + threaded model's area is sensitive to; ExportVerilog emits sign-extended + multiplies as `$signed` so Vivado maps them to single DSPs. +4. **Compose** (`foldsim-compose`). The top netlist's nested netlists are + inlined, every instance of a model becomes one thread of that model's + single threaded copy (§3), and every wire between them a channel of the + token fabric. The result is `foldsim_sim`: `hostClock`, `hostReset`, a + channel per target port (`_valid`, `_ready`, `_bits`) and a + `profile_finish` bit per class. + +The driver also writes `foldsim_sim.json` (the ports, and each class's thread +count) and `foldsim_sim.vh` (the class count, for the kernel). + +--- + +## 3. The simulator + +**Threads.** A model of N instances is one module that simulates them in turn +(`foldsim-fame5`'s threader): every register becomes an N-entry memory read at +the current thread, every memory N times deeper. A thread finishes its +target cycle when each input channel has its token and each output's token +was taken; its state is then written, on the host clock with that finish as +the enable — no gated clocks. Outputs that depend on no input fire eagerly, +the next cycle's value as soon as the thread finishes. The thread index +advances every host cycle and is a port of the module, so the fabric reads it +rather than keeping its own counter. + +**Channels.** Every output of a model, and every input of the target, is a +*producer*; every input of a model, and every output of the target, a +*consumer* that reads bits of one or more producers, for each of its threads. +Each producer thread has two token slots and a count of tokens written; each +reader of it a count of tokens read. A producer is ready while every reader +of the thread has read all but one of its tokens; a consumer's token is valid +when each producer it reads has one for it. This is a two-entry channel +with no flow-through, and with fan-out, so the target's behaviour does not +depend on the host's schedule. + +**Storage.** A producer read only by threaded consumers that read the same +bits of it for every thread keeps its slots in a memory addressed by the +reader's thread and slot (LUTRAM); any other, in registers. A target port +that is exactly one port of a model of one thread is wired to the shell +directly, since the shell's valid and ready depend on its own state alone. + +### 3.1 Timing: the options the image flow uses + +A threaded model's critical paths run from its thread index, through every +state memory's read and the target logic, into every state memory's write. +The image flow's default options (`FOLDSIM_FLAGS` in `hw/syn/foldsim`) cut +those paths without changing what the target computes; each keeps the +simulator cycle-exact (§6). + +| option | what it does | +|---|---| +| `--region-loads 2048` | splits a model's state into regions of at most 2,048 loads, each with its own copy of the thread index and of the edge, written a host cycle after the target edge | +| `--registered-state` | reads the next thread's state a host cycle ahead into registers, forwarding the write landing in between; threads then go in turn (`--schedule`) | +| `--commit-select` | a model of one thread commits its state by selecting a registered candidate, not by writing it in the edge's cycle | +| `--state-stages 2` | writes the state two host cycles after its edge in models of three threads or more, so synthesis can retime into the logic before it | +| `--pipeline-multipliers` | with two stages, a state computed by a multiplier takes its first stage at the multiplier's inputs | +| `--staged-ram-writes` | with regions, memories with a registered read are written a host cycle late too, their reads forwarded | +| `--staged-tokens` | a channel writes each token into its slot a host cycle after it is taken, forwarded to its readers until then | +| `--fold-pipelines` | simulates a fixed-latency pipeline by one unthreaded copy and a per-thread store of its results (the Xilinx FP cores, §1) | + +`--compact-staging` (staging placed at the state it stages; for large images, +s2w4i2's Verilog 34 → 16 MB) and `--channel-depth` are not in the default. + +Out-of-context results for s2w4i2 (`make -C hw/syn/foldsim ooc`, xcv80, +4.0 ns unless noted; Vivado retiming moves a run by about ±20 MHz): + +| options | FPU | MHz | LUTs | FFs | +|---|---|---:|---:|---:| +| none | soft | 186.8 | 130.1k | 26.8k | +| regions of 2,048 loads | soft | 214.3 | 136.9k | 83.6k | +| the default set but folding | soft | 250.0 | 211.8k | 174.7k | +| the default set | soft | 235.3 | 194.6k | 157.0k | +| the default set | Xilinx cores | 250.0 (262.7 at 3.33 ns) | 182.8k | 162.4k | + +The soft FPU's own pipeline stages limit it: the same options with ready +scheduling instead of threads in turn reached 250.0 MHz, the difference being +which stages retiming balanced. + +Small targets pay area for these options that they do not need; +`FOLDSIM_FLAGS=` (empty) builds the plain simulator. + +### 3.2 Domains and relays + +`--domains` emits the simulator as a tree of modules, one per class, nested as +the classes are, with every domain boundary a set of module ports; +`--relay ` gives the channels of the named models (or +`:` pairs) a relay stage, so a channel crossing an SLR costs a +host cycle of latency instead of a long wire. Both are off by default. +Measured on s2w4i2 (out of context): with the default options, domains and +relays reach the same 250 MHz at 4.0 ns; at 3.33 ns 273.6 against 262.7 MHz +with the Xilinx cores (equal slack after synthesis), but 225.7 against +238.9 MHz with the soft FPU; without the state-staging options +(regions and folding only) 231.8 against 226.2 MHz at 4.0 ns, for 1.4 % more +LUTs. Both differences are inside the retiming spread. A relay costs host +cycles where a loop runs through it (up to 22 % on img6b), so relays are +chosen per image from a Verilator run. + +--- + +## 4. The kernel + +The shell's `rtl/foldsim_afu.sv` has the native V80 kernel's ports but for +the host-memory master: `s_axi_control` and one device-memory master +`m_axi_mem_

` per memory port of the target, each linked to the `MEM` HBM +aperture (the linker gives each its own way in). The target has as many +memory ports as its configuration has platform memory banks +(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS`: 1, 2, 4 or 8; `VX_config.toml`'s 2 unless +`CONFIGS` sets it); the flow writes their number, the shift to a port's +channel, their AXI id width and `FOLDSIM_MEM_REPEAT`, which lists the ports +whose names carry their index, into `gen/foldsim_kernel.vh` (spelled out, as +Vivado's IP packager does not paste an expanded macro argument into a macro +name). It instantiates `foldsim_sim`, a +memory endpoint per port and the control endpoint, all models of the target's +surroundings that take one token per channel per target cycle +(`foldsim_token.sv`): + +- **`foldsim_mem`** is one of the target's AXI4 ports, timed as a DRAM + channel's banks (§4.1). A read goes to HBM (`base` + target address) + the cycle the target makes it, a write the cycle it has both its AW and its + W, so HBM only ever sees whole single-beat writes; a target cycle that + should present a response memory has not returned yet does not finish until + it has, and a write is acknowledged to the target only after HBM + acknowledged it. A burst sets the memory error and goes nowhere; from the + next cycle on the endpoint is ready for nothing, and HBM sees none of it. +- **`foldsim_ctrl`** is the AXI4-Lite register block and the endpoint of the + target's reset, DCR, start and busy. The target runs free: the endpoint + gives it a token every target cycle. Host commands are queued and each is + applied for one target cycle (a reset for its count). Kernel reset holds the + target in reset for 64 cycles. + +With several ports, Vortex's AXI adapter puts the index of the GPU memory +port a read came from above its tag buffer's 4 index bits, so the target's +AXI ids are 4 + log2(ports) bits wide; the flow sets +`PLATFORM_MEMORY_ID_WIDTH` and the endpoints' id width to match. + +### 4.1 The memory model + +The memory model takes its settings at a target reset: the kernel's own reset +gives it the default machine's DRAM (HBM2 at 800 MT/s behind a 400 MHz core), +and the runtime writes the configuration's before the reset it makes on open +(§5). + +**The bank model** (`foldsim_mem_bank.sv`) is cycle for cycle +`sim/common/dram_bank_model.h` (`DramBankAxi` over `DramBankModel`), the +reservation-based model of dram_sim's DRAM, configured with dram_sim's own +knobs (`VX_CFG_PLATFORM_MEMORY_{TYPE,DATA_RATE,LATENCY,QUEUE_SIZE}`, +`VX_CFG_PLATFORM_CLOCK_RATE`) and its data-sheet timings +(`sw/common/dram_timing.h`, converted to DRAM clocks with Ramulator's +rounding). The same model is a `DramSim` backend in the simulators +(`VORTEX_DRAM_MODEL=bank`) and rtlsim's AXI4 memory (`AXI_BUS=1 +AXI_MEM_MODEL=bank`). + +![the bank model](../assets/img/foldsim_mem_bank.svg) + +- *Port.* A read holds one of 32 slots from acceptance until R takes it; its + data, from HBM, waits in the slot. An AW and its W form a write. Reads and + writes reach the channel in the order they formed, from the next target + cycle; at most `QUEUE_SIZE` requests are in flight (0: the slots only). +- *Channel.* One request a request's bursts' worth of DRAM clocks enters the + controller's read or write buffer (32 bursts each); a write is acknowledged + there. The controller serves reads and drains writes in batches between + dram_sim's watermarks (26 and 7 bursts). +- *Reservation.* The request at the head of the buffer being served is + scheduled once: its PRE, ACT and column commands take the first DRAM clocks + every JEDEC constraint allows (tRCD, tRP, tRAS, tRC, tRTP, tWR per bank; + tCCD, tRRD, the read/write turnarounds per pseudo-channel and bank group; + one command per clock; HBM's two-clock ACT), before or after commands + reserved earlier. Rows stay open; a bank serves its requests in order. A + batch of writes goes after every column reserved before it, the reads after + it after the batch. +- *Completion.* A read is ready `LATENCY` + 1 target cycles after the target + cycle whose DRAM clocks its data departs in (tCL + tBL after its last + column); R and B take the oldest ready response, so reads complete out of + order — legal, since Vortex never has two reads of one id outstanding. +- *Channels.* Each memory port is a channel of its own, with its own endpoint + and engine, as dram_sim gives each memory bank its own channel. Ports + interleave the target's memory by block, so a port's channel sees the + block index without the port's bits (`PLATFORM_MEMORY_INTERLEAVE`); device + memory takes the target's address on every port. +- *Refresh* (off unless `VORTEX_DRAM_REFRESH` asks for it; dram_sim models + none). Every interval a refresh takes the DRAM clock's turn to reserve. + Per bank (HBM REFsb, GDDR6 REFpb; the banks in turn, an interval of + tREFI / banks): the bank is precharged if open, once its reserved commands + allow, then refreshed once the precharge and its last activate allow, and + may not activate for tRFCsb. All banks (an interval of tREFI): one + precharge after every open bank's commands, one REF, and no bank activates + for tRFC. Both commands take the row-command bus as a precharge does; a + refresh that would reach past the horizon waits, and the requests with it. + +In hardware the calendars are bitmaps over a ring of 512 DRAM clocks in +distributed RAM: per clock, whether it holds a command, the row bus, and per +pseudo-channel and bank group the clocks a read, a write or an ACT may not +take. Every command marks, once reserved, the clocks its constraints forbid +to later ones, so placing one is a first-free search over 64 clocks a host +cycle; a word of the ring is cleared, by a valid bit, as the DRAM clock +passes it, and a reservation reaches at most 448 clocks ahead (the model's +horizon). A refresh goes through the same search and commit as a request's +precharge; for all-bank refresh the channel keeps the latest time a +precharge and an activate are allowed over the banks opened since the last +one, so it never reads the whole bank table. A target cycle's DRAM clocks run in host cycles while the target +evaluates it — two host cycles a clock, about fifteen more for a +reservation — and the cycle finishes only once they have; the `memory +waits` counter includes the host cycles a finished cycle waits for them. tFAW is left to tRRD, +which covers it wherever four tRRD_S span a tFAW (HBM2, HBM3); the runtime +refuses a memory where it does not (GDDR6). + +Both endpoints' token ends are staged (`foldsim_token`'s `STAGED`): `have` +and `done` are registers, so an endpoint's finish starts at its own state, +not at the simulator's valid and ready. A token counts a host cycle later, +which s2w4i2's eight host cycles a target cycle hide (its counts are +unchanged). + +**Reset.** The control endpoint also gives the memory endpoint a token each +target cycle: the target's reset in that cycle. A reset command thereby +resets the target's whole memory port, not only the target. A target cycle in +reset takes no request, and finishes only once HBM has answered every +transaction it has from before the reset — reads returned, writes +acknowledged, nothing left offered to it — so the AXI4 port to HBM stays in +protocol. The cycle then drops the model's state: requests the target made +and was not answered, data not delivered to it, the lone half of a write, and the memory +error. No response from before a reset reaches the target after it; writes +HBM already took stay written, as they would behind a real memory +controller. + +--- + +## 5. The runtime + +`sw/runtime/foldsim/vortex.cpp` (`--driver=foldsim`) is a transport under the +runtime's software command-processor model: the +model's DCR writes and reads, start and busy become commands and status reads +of the control block, and its device-memory accesses DMA at the target's +addresses. `TARGET=hw` uses VRT — the kernel's registers through +`vrt::Kernel`, device memory through a QDMA queue — and `TARGET=avedsim` +the same kernel in Verilator (`sim/foldsim`), with a sparse device memory and +each register access costing the kernel cycles a PCIe access would. + +The register map is the shell's `include/foldsim_regs.h` (`rtl/foldsim_regs.vh` +for the RTL): + +| offset | register | +|---|---| +| `0x00`, `0x04` | ID `"FOLD"`, version | +| `0x08` | control: bit 0 the target runs | +| `0x0C` | status: busy (anything queued, running, or in flight), queue full, DCR response ready, memory error | +| `0x10`, `0x14`, `0x18` | command address, data, and the command: DCR write, DCR read, start, reset | +| `0x1C` | the last DCR read's response | +| `0x20`–`0x2C` | target cycles, kernel-clock cycles | +| `0x30`, `0x38` | kernel-clock cycles a target cycle waited for HBM; the number of classes | +| `0xC0`, `0xC4` | device base of the target's memory | +| `0xD8`, `0xDC` | the refresh in effect (read only); the bank model's capacities: ports, engines and masters, slots, horizon (read only) | +| `0xE0`–`0xF8` | bank model: data rate and core clock, fixed latency, requests in flight and controller queues, geometry, write watermarks, refresh interval and busy time | +| `0x100 + 8c` | per class, threads finished | +| `0x800`–`0x844` | bank model: DRAM timings, a word each (tBL … tWTR_L, read-to-write extra) | +| `0xC00 + 0x40p` | memory port p's bank model: reads, writes, row hits, misses, conflicts, summed read latency, refreshes, target cycles it could take no read (64-bit each) | + +`0x40`–`0xBF` is empty: on the V80 the PCIe block keeps its MSI-X table at +that offset of the BAR the first kernel is mapped at, and answers those +accesses itself (§8). The runtime refuses an image whose `VERSION` differs +from its own map. + +`busy` covers queued commands, a start the target has not yet answered with +busy, and transactions the memory model has in flight, at the target or in +HBM, so a host that sees the target idle has every effect of what it asked +for. + +The runtime resets the target on open, as a card's native driver resets its +device; that also clears a memory error an earlier process left, and the reset +command's address chooses, with `VORTEX_DRAM_REFRESH=bank|all`, the bank +model's refresh. Before it the runtime writes the bank model's registers from +the platform memory it was built for (`VX_CFG_PLATFORM_MEMORY_*`), every +channel alike, refusing what the image cannot hold and warning when a refresh +keeps a bank past the image's horizon; the counters print at close, per +channel and in all. The command +processor model's hooks return no status, so the runtime latches a device +error instead — a memory error, a failed register or memory access, or a wait +that timed out — and from then on every register access the runtime core +makes fails, and the launch, DCR access or copy that hit it fails with it. A +launch longer than `VORTEX_FOLDSIM_TIMEOUT_S` seconds is such an error (no +limit by default), as is a command the target does not take, a DCR read it +does not answer, or a reset it does not finish while its target cycles stop +advancing for that time (60 s when it is unset). + +--- + +## 6. Validation + +- **Token co-simulation** (`foldsim_sim` against the RTL it was compiled from, + in Verilator): random target inputs, every channel stalled at random, every + output token compared. s2w4i2: 3,000 target cycles, 51,000 tokens, 0 + differences. +- **The memory endpoint** (the shell's `test`, `ninja check-foldsim-shell` in + the Foldsim build, in Verilator): an + AXI4 master as the target, HBM with random ready and latency checking the + device-side protocol, resets at random points with HBM busy, and each kind + of burst; every target cycle's outputs compared with the C++ model's + (`DramBankAxi`), every response, id and read's data checked, none from + before a reset delivered after it. Twelve bank-model settings (HBM2 and HBM3 at the default, v100, a100 and + h100 rates; small queues; row hits, random rows, one bank to the horizon, + streams, read- and write-heavy) and five with refresh (per bank and all + banks, at the standard intervals and faster), on the endpoint alone, with + staged token ends, with speculative reads, and as one of two ports (`ch2`): 0 + differences. +- **Refresh against its interval** (`tests/unittest/dram_bank_model + refresh`): one refresh per interval, to the operation, at every memory and + pattern; all-bank refresh costs a stream at least tRFC / tREFI of its + bandwidth (9.3–12 %, against 9.0 %). +- **Applications** in the in-process simulator and on the V80, against rtlsim + behind the AXI4 adapter on the same bank model (`AXI_BUS=1 + AXI_MEM_MODEL=bank`) with the same memory settings: retired instructions and + cycles identical (§7). CI holds this for tinygpu demo and sgemm in the + in-process simulator (`ci/testcases/foldsim.yaml`, a `model_parity` check at + tolerance 0). +- **Foldsim's own tests** (`check-foldsim` in the Foldsim repository): lit + tests of every pass and of the driver, and small threaded simulators run in + Verilator against their expected tokens — one and several threads, + registered memory reads, register initial values, eager outputs, a reset of + one cycle. + +--- + +## 7. Measured results + +Configuration s2w4i2 (2 cores, 4 warps, 4 threads, issue width 2), seven +classes: one rest-of-Vortex model, the socket (2 threads) and six unit +classes (4 threads each). Every setting is the configuration's +(`VX_config.toml`): 2 memory channels on the default machine's DRAM, and the +Xilinx floating-point cores, which Vivado synthesis selects; the flow's +default options (§3.1). + +| | where | demo cycles | sgemm cycles | host / target cycle | +|---|---|---:|---:|---:| +| Foldsim, Xilinx cores | Verilator | 6,616 (`-n64`) | 404,896 | 12.04 / 8.18 | +| Foldsim, soft FPU (`-DVX_CFG_FPU_TYPE_STD`) | Verilator | 6,616 | 399,646 | 12.04 / 8.17 | +| rtlsim, soft FPU, AXI4 adapter on the bank model (`AXI_BUS=1 AXI_MEM_MODEL=bank`, `PLATFORM_MEMORY_ID_WIDTH=5`) | Verilator | 6,624 | 399,654 | | + +rtlsim's harness starts the kernel 8 cycles after reset, while each L1 bank +still invalidates one set per cycle (64 for the instruction cache), so its +first fetch waits 8 cycles the counters see; a host's launch comes thousands +of cycles later. Given that settle time, rtlsim's cycles are Foldsim's +exactly. rtlsim must take the platform's AXI id width (4 + log2 of the +channels): at its own default of 32 the AXI adapter needs no tag buffer and +times reads differently (demo 6,552). + +**On the V80** this configuration, with every setting the toml's and the +flow's default options, closes **250 MHz** in the image flow (226k LUTs, 195k +FFs and 20 DSPs in the kernel). It runs demo, sgemm and eight FP tests +(`dogfood`) at Verilator's instruction and cycle counts: demo 6,616, sgemm +404,896, and dogfood 21 1,657,630 cycles, at 8.1–8.2 host cycles per target +cycle. A one-channel variant with domains and issue/LSU relays (§3.2) also +closed 250 MHz and ran the same tests exactly. The plain flow's s2w4i2 image had closed at 149.8 MHz in +138k routed LUTs (219k for the firesim backend's image of the same +configuration). + +### 7.1 The bank model + +s2w4i2 in the in-process simulator (Verilator), against SimX on the same +configuration (one memory bank, Ramulator behind dram_sim) and rtlsim behind +the AXI4 adapter on the same model +(`AXI_MEM_MODEL=bank`). Memories: the default (HBM2, 800 MT/s, 400 MHz core) +and the a100 and h100 profiles' (`VX_CFG_PLATFORM_MEMORY_*` from +`profiles/*.json`). + +| app | memory | Foldsim, bank model | rtlsim, bank model | SimX, Ramulator | SimX, bank model | +|---|---|---:|---:|---:|---:| +| demo | default | 10,740 | 10,748 | 10,190 | 9,148 | +| sgemm | default | 399,337 | 399,345 | 401,434 | 401,421 | +| sgemm | a100 | 691,372 | | 690,935 | 690,780 | +| sgemm | h100 | 422,294 | | 425,547 | 425,541 | +| vecadd `-n16384` | default | 194,242 | | 193,267 | 192,901 | +| vecadd `-n16384` | a100 | 311,917 | | 312,529 | 312,587 | +| vecadd `-n16384` | h100 | 223,875 | | 224,921 | 224,921 | + +One memory channel (`-DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1`), soft FPU. +Instruction counts are equal everywhere. rtlsim's cycles are Foldsim's plus +the 8-cycle launch settle (§7). Against SimX on +Ramulator Foldsim is within 0.8 % on sgemm and vecadd at all three memories; +the store-dominated demo (2,048 writes to 268 reads) is 5 % slower, and +SimX's own bank backend 10 % faster: the reservation model drains a write +stream differently from Ramulator's per-clock scheduler (a +mixed-traffic residual). A fixed-latency memory model, which Foldsim had +before, gave sgemm 415,454 cycles at every profile. The bank model's DRAM +clocks cost host cycles (memory waits): 7.3–12 host cycles per target cycle +on s2w4i2. + +### 7.2 tinygpu: Foldsim and the firesim backend + +The same Vortex tree, configuration and V80 on both flows: 1 core, 2 warps, +2 threads, no L1 caches or local memory, STD FPU, one memory bank, kernel +clock 143 MHz, measured 2026-10-13 with the plain flow and the fixed-latency +memory model Foldsim had then (FASED's latency-bandwidth pipe, which rtlsim +keeps as `AxiLatencyPipe`). Every class has one instance, so Foldsim folds +nothing here: this compares the two simulators' fidelity and overheads, not +threading. + +| | Foldsim | firesim backend | +|---|---:|---:| +| kernel LUTs (synthesis) | 34,393 | 103,369 | +| of them distributed RAM | 1,822 | 51,943 | +| FFs / BRAM tiles / URAM / DSP | 30,198 / 4 / 0 / 8 | 34,376 / 5 / 8 / 8 | +| WNS at 143 MHz | +0.282 ns | +0.752 ns | +| image build (side by side, same host) | 56 min | 80 min | +| `demo` cycles, V80 (= Verilator) | 24,402 | 25,624 | +| `sgemm -n16` cycles, V80 (= Verilator) | 301,923 | 317,163 | +| `sgemm -n64` cycles, V80 | 13,264,035 | 13,937,835 | +| `sgemm -n64` kernel time, V80 | 278 ms | 178 ms | +| host cycles per target cycle, `sgemm -n64` | 3.00 ¹ | 1.84 (FMR) | + +¹ With the plain flow (`FOLDSIM_FLAGS=`) and the fixed-latency memory. The +flow's default options (§3.1) took it to 4.00, cycles unchanged: they buy Fmax +for large targets with host cycles a small one pays. With today's defaults (2 +channels, the bank model) tinygpu takes demo 16,375 and `sgemm -n16` 195,152 +cycles, equal to rtlsim's on the bank model, at 7.1–7.2 host cycles per target +cycle (Verilator). + +Instruction counts are identical on both flows, in Verilator and on the V80. +Foldsim's cycles equalled rtlsim's behind the AXI4 adapter on `AxiLatencyPipe` +with the same memory settings (`AXI_BUS=1`) exactly; the firesim backend's +are 5.0–5.1 % above, from its memory model's channel mechanics. With nothing +to fold, a Foldsim target cycle costs three host cycles (the token exchange +of each one-thread model), so the firesim backend runs tinygpu faster; +Foldsim's advantage is the GPU it fits, its area falling with every class it +folds (s2w4i2 above). + +--- + +## 8. Invariants and limitations + +1. **Valid and ready of an endpoint depend on its state alone.** The + simulator wires target ports to the shell directly; a combinational path + from a channel's ready to its valid would close a loop. +2. **Every model ticks on its first finish.** A threaded model's target clock + is enabled from the first finish after reset, so token k of every channel + is target cycle k. +3. **The memory endpoint takes single-beat transactions**, as Vortex issues + them; a burst sets the status register's memory error, stops the + endpoint, and only a reset clears it. Under the bank model, no two reads + of one id may be outstanding (Vortex's AXI adapter never has them). +4. **Memory ports**: 1, 2, 4 or 8, one per platform memory bank, each a + channel with its own engine (about 11k LUTs). A refresh that keeps its + bank longer than the horizon (per-bank refresh at HBM3's clock: tRFCsb is + 524 clocks against 448) holds the requests behind one to that bank at the + channel's ingress, which overstates refresh's cost; the runtime warns. + With refresh on, a run's cycles depend on where the refresh schedule + stands at launch, so on the host's timing before it: such runs equal + another simulator's only from a common reset (the endpoint + co-simulation), not across hosts. With refresh off they do not depend on + host timing. +5. **Classes are named, not found**: `FOLDSIM_CLASSES` lists them. +6. **No kernel register in `0x40`–`0xBF`.** The V80 shell configures the + PCIe block's MSI-X table internally, at offset `0x40` of the BAR whose + offset 0 is the first kernel; reads there return the table's entries and + writes stick in it, without reaching the kernel. diff --git a/docs/designs/fpga_afu_shell.md b/docs/designs/fpga_afu_shell.md index f866d40b7a..4556c9a93d 100644 --- a/docs/designs/fpga_afu_shell.md +++ b/docs/designs/fpga_afu_shell.md @@ -1,9 +1,12 @@ -# AFU Shell (XRT / OPAE) — Design +# AFU Shell (XRT / AVED / OPAE) — Design **Scope:** the FPGA acceleration-function-unit (AFU) shells that adapt -Vortex to a host platform — the Xilinx/XRT shell -([`hw/rtl/afu/xrt/`](../../hw/rtl/afu/xrt/)) and the Intel/OPAE shell -([`hw/rtl/afu/opae/`](../../hw/rtl/afu/opae/)). This document covers the +Vortex to a host platform. The shell proper is shared +([`hw/rtl/afu/common/`](../../hw/rtl/afu/common/)); what differs per platform +is the kernel top that wraps it — Xilinx/XRT +([`hw/rtl/afu/xrt/`](../../hw/rtl/afu/xrt/)), AMD/AVED for the Alveo V80 +([`hw/rtl/afu/aved/`](../../hw/rtl/afu/aved/)) — plus the Intel/OPAE shell +([`hw/rtl/afu/opae/`](../../hw/rtl/afu/opae/)), which is separate. This document covers the **shell structure** (platform glue, memory adaptation, control/reset/ discovery contracts, and the XRT↔OPAE asymmetries). The Command Processor integration these shells host is documented in @@ -17,17 +20,20 @@ DMA-command engine have been removed. --- -## 1. XRT shell +## 1. XRT and AVED shells + +The two share every file below except the kernel top: `afu/xrt/vortex_afu.v` +is the Vitis RTL-kernel top and `afu/aved/vortex_afu.v` the SLASH one. | File | Role | |---|---| | [`vortex_afu.v`](../../hw/rtl/afu/xrt/vortex_afu.v) | Vitis RTL-kernel top — thin wrapper instantiating `VX_afu_wrap`. | -| [`vortex_afu.vh`](../../hw/rtl/afu/xrt/vortex_afu.vh) | Defines/macros (`GEN_AXI_MEM`, `GEN_AXI_HOST`, the bit-12 window). | +| [`vortex_afu.vh`](../../hw/rtl/afu/common/vortex_afu.vh) | Defines/macros (`GEN_AXI_MEM`, `GEN_AXI_HOST`, the bit-12 window). | | [`VX_afu_wrap.sv`](../../hw/rtl/afu/common/VX_afu_wrap.sv) | The real shell (~1,200 LOC): AXI-Lite bit-12 demux, `VX_cp_core`, `m_axi_host`, the Vortex instance and the device-memory path shared with the CP. | -| [`VX_afu_ctrl.sv`](../../hw/rtl/afu/xrt/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | +| [`VX_afu_ctrl.sv`](../../hw/rtl/afu/common/VX_afu_ctrl.sv) | Slimmed AXI-Lite slave (~322 LOC): `ap_ctrl` stub at 0x00 + a SCOPE serial register pair + SCOPE watchdog. | | [`VX_afu_axil_demux.sv`](../../hw/rtl/afu/common/VX_afu_axil_demux.sv) | AXI-Lite demux splitting the control space on `addr[12]`, one outstanding transaction per direction. | -| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing. | -| [`VX_afu_axi_limit.sv`](../../hw/rtl/afu/common/VX_afu_axi_limit.sv) | Holds an AXI master to one read and one write in flight; applied to port 0 of the multi-port shell. | +| [`VX_afu_axi_drain.sv`](../../hw/rtl/afu/common/VX_afu_axi_drain.sv) | Outstanding-transaction tracker per AXI master; reports when a port owes the interconnect nothing, including a write burst under way ahead of its AW. | +| [`VX_afu_req_gate.sv`](../../hw/rtl/afu/common/VX_afu_req_gate.sv) / [`VX_afu_wr_gate.sv`](../../hw/rtl/afu/common/VX_afu_wr_gate.sv) | Withhold new reads / whole new writes while the reset sequencer quiesces, never withdrawing an offer. | | [`VX_afu_reset_seq.sv`](../../hw/rtl/afu/common/VX_afu_reset_seq.sv) | Quiesce-before-reset sequencer for the soft reset; refuses rather than resetting a master that will not drain. | - **Control.** Host AXI-Lite `addr[12]` splits the slave: `addr[12]=0` → @@ -40,18 +46,45 @@ DMA-command engine have been removed. with its address, and routing W by a register that only updates at the AW handshake sent AW and W to different slaves and deadlocked the interface. Covered by `hw/unittest/afu_axil_demux`. -- **Memory.** Two structures, selected by the number of memory ports. - *One port* (the merged platforms): `Vortex_axi` and the CP's `axi_dev` - share the port through a 2:1 `VX_mm_axi_arb`, and CP bursts reach the - platform as bursts. *Several ports*: Vortex's memory ports and `axi_dev` - (bridged by `VX_membus_from_axi`) are the inputs of one bank adapter, - `VX_mem_to_axi`, so the CP is merged **upstream** of bank selection and - reaches every bank. In both, port 0 carries one read and one write in - flight (`VX_afu_axi_limit` in the second case). CP `axi_host` drives a +- **Memory.** The AFU exposes one AXI master per GPU memory port: + `AFU_MEM_PORTS_OF(VX_MEM_PORTS)` ([`vortex_afu.vh`](../../hw/rtl/afu/common/vortex_afu.vh)), + the last-level cache's memory ports (already capped by the platform's + channel count, `PLATFORM_MEMORY_NUM_BANKS`) rounded up to a power of two. + The kernel top must spell its ports out, so every flow resolves the count + to the literal `AFU_MEM_PORTS` by elaborating a probe against its own flags + ([`afu_mem_ports.sh`](../../hw/scripts/afu_mem_ports.sh)); `VX_afu_wrap` + asserts the two agree, `package_kernel.tcl` checks the packaged kernel's + master count, and the XRT Makefile generates the `--connectivity.sp` lines + from the same value. Lines interleave across the N ports + (`PLATFORM_MEMORY_INTERLEAVE`; without it each port takes a contiguous range), and port *i* owns the *i*-th contiguous + 1/N of device memory: its AXI address is the line index divided by N + within that share. The platform maps the share to its own contiguous group + of `PLATFORM_MEMORY_CHANNELS / N` channels (`HBM[2i:2i+1]` for 16 ports on + a U55C), so ports never contend for a channel. `PLATFORM_MEMORY_OFFSET` is + applied once, at every port. + Two structures, selected by N. + *One port*: `Vortex_axi` and the CP's `axi_dev` + share the port through a 2:1 `VX_mm_axi_arb` in its ID-routed + multi-outstanding mode (`MULTI_OUT=1`: the top ID bit names the source, + Vortex gets the remaining ID bits), and CP bursts reach the platform as + bursts. Writes keep AW order on the shared W channel, and a write's data + goes with its address, so single-beat writes issue one per cycle. + *Several ports*: Vortex's memory ports and `axi_dev` (bridged by + `VX_membus_from_axi`) are the inputs of one bank adapter, `VX_mem_to_axi`, + so the CP is merged **upstream** of bank selection and reaches every bank. + In both, every port carries as many reads and writes in flight as its + masters issue; one outstanding transaction per direction would cap a port + at one line per DRAM round trip. CP `axi_host` drives a **dedicated `m_axi_host` AXI master** for the command ring + host DMA. - `PLATFORM_MEMORY_OFFSET` is applied per bank, at the port. + While the reset sequencer quiesces, `VX_afu_req_gate` holds new reads and + `VX_afu_wr_gate` holds new writes on both channels together: AXI4 lets a + master present write data before or with its address, so gating AW alone + would strand data the slave can never pair. An address or data offer + already on the bus obliges the other channel, since a slave may wait for + one before accepting the other. xrtsim aborts on a request outside the + issuing port's share of memory. - **Interrupt.** The AFU `interrupt` pin is driven from `cp_interrupt` - ([`:335`](../../hw/rtl/afu/xrt/VX_afu_wrap.sv#L335)). + ([`:335`](../../hw/rtl/afu/common/VX_afu_wrap.sv#L335)). --- diff --git a/docs/designs/graphics_hardware_stack.md b/docs/designs/graphics_hardware_stack.md index 43132045a6..669ff6568a 100644 --- a/docs/designs/graphics_hardware_stack.md +++ b/docs/designs/graphics_hardware_stack.md @@ -300,17 +300,14 @@ kernel), never by the host. --- -## 8. VM / pinned-buffer tie-in - -Under `VX_CFG_VM_ENABLE` the per-core MMU translates VA→PA for kernel LSU -traffic, but the RASTER/TEX/OM AXI masters **bypass** the MMU and use the -physical addresses written into their DCRs. `VX_MEM_PHYS` buffers are -identity-mapped and carved from a dedicated pinned slab so VA == PA. DCR -writes targeting graphics buffer-address registers are validated against -the pinned slab on the CP submit path (returning `VX_ERR_INVALID_VALUE` -for a PA outside the slab). The slab size is `VX_CFG_VM_PINNED_REGION_SIZE` -(overridable via `VORTEX_VM_PINNED_SIZE`). Tests allocate every HW-bound -buffer with `VX_MEM_PHYS` and omit it for write-only LSU buffers. The +## 8. Virtual memory + +Under `VX_CFG_VM_ENABLE` every graphics master translates: TEX through the +socket unit MMU, RASTER and OM through the cluster unit MMU in front of the +rcache and ocache (see +[`virtual_memory_subsystem.md`](virtual_memory_subsystem.md) §3.8-3.9), so the +addresses in the TEX/RASTER/OM DCRs are ordinary device virtual addresses and +a texture, tile, primitive, color or depth buffer needs no `VX_MEM_PHYS`. The VM/MMU subsystem is documented in [`virtual_memory_subsystem.md`](virtual_memory_subsystem.md). diff --git a/docs/designs/lsu_pipeline_design.md b/docs/designs/lsu_pipeline_design.md index c1d19bbeea..fe0b1f0f1d 100644 --- a/docs/designs/lsu_pipeline_design.md +++ b/docs/designs/lsu_pipeline_design.md @@ -6,7 +6,7 @@ interleave, per-lane classification and formatting, fence ordering, multi-packet tracking, the shared memory scheduler and its outstanding pool, address-class steering, coalescing, and the response path. Covers the RTL ([`VX_lsu_unit.sv`](../../hw/rtl/core/VX_lsu_unit.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv), +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv), [`VX_lsu_agu.sv`](../../hw/rtl/core/VX_lsu_agu.sv), [`VX_lsu_scheduler.sv`](../../hw/rtl/core/VX_lsu_scheduler.sv), [`VX_mem_scheduler.sv`](../../hw/rtl/libs/VX_mem_scheduler.sv), @@ -87,12 +87,12 @@ Load-bearing properties: - **The XOR with the group index is what spreads warps across sets.** A group spans a power of two larger than a cache way; without the skew, the same frame slot of every warp would share a set. -- **It costs two narrow operations, not three wide ones.** Written naively - the map subtracts the window base, permutes, and adds the base back. The - permutation moves bits only inside the group field and the group index - keeps its position, so the subtract and add cancel on every bit above the - field; the window base has trailing zeros, so the field itself needs - arithmetic only on the bits above them. +- **It is a bit permutation plus one narrow XOR, with no arithmetic.** The + window top `VX_MEM_STACK_BASE_ADDR` is a multiple of the group size + (`NUM_THREADS` stacks), so groups start on absolute group boundaries and + the map needs no subtract or add of the window base: it rewires the + thread and word fields and XORs the word field with the low bits of the + group index. The 256 KB-aligned base covers up to 32 threads. - **Window membership is a narrow compare.** Both window bounds are asserted to be multiples of the stack size, so the test uses only the bits above it. @@ -104,7 +104,7 @@ that is provably equivalent. ## 3. The slice — request side -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv). +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv). ### 3.1 Classification @@ -148,7 +148,9 @@ A fence is a **total barrier on its slice**. The slice sets `fence_lock` when a fence's last packet is accepted and clears it when that fence's response returns. While locked, the slice accepts nothing. -The fence itself travels to memory as a request with `is_flush`, behind every +The fence itself travels to memory as a request with `is_flush` (an IO token, +the `io` attribute set, for a fence ordering only device accesses; otherwise an +L1-only token, byte enables set), behind every request the slice issued before it, so its response proves they have drained. Only the last packet of a multi-packet fence is sent; the earlier packets complete immediately. @@ -208,7 +210,7 @@ library block also used by the caches' other clients. | client | source | present | |---|---|---| -| 0 | the block's `VX_lsu_slice` | always | +| 0 | the block's `VX_lsu_block` | always | | 1 | the tensor unit's warp-level address generator | on block 0, when the tensor metadata path is enabled | Arbitration is round-robin, and the granted client's index is prepended to @@ -303,14 +305,29 @@ The attribute passes through untouched on both paths. ### 5.2 Local path -Each LSU block has its **own** adapter into -[`VX_local_mem`](../../hw/rtl/mem/VX_local_mem.sv). A shared adapter would -close a loop through its pack and unpack buffers and deadlock with more than -one block. - -Local memory is a banked SRAM, so lanes addressing different banks are served -in the same cycle. Its second port is shared by the direct-memory clients — -the DXA writes and the tensor unit's reads. +Each LSU block hands its local requests whole to +[`VX_lmem_unit`](../../hw/rtl/mem/VX_lmem_unit.sv), the warp-wide local +memory. A block arbiter takes one request at a time into a pass engine +([`VX_lmem_engine`](../../hw/rtl/mem/VX_lmem_engine.sv)), which compares the +lanes' words once and then serves the request in bank-conflict-free passes: +every bank goes to its lowest pending lane, every pending lane reading that +lane's word rides along (a broadcast), and the remaining lanes replay. A +request retires before the next one starts, so every lane's accesses reach the +banks in program order. A request without bank conflicts takes the banks for a +single cycle, so one such request issues every cycle; the response +carries every lane the pass served. + +With several LSU blocks, a second engine for plain reads only works the banks' +second port, so two warps' reads go through in a cycle. Stores and atomics use +the first engine alone and wait for the read engine to drain; the read engine +takes no read while a store or atomic sits in the first engine, while one +waited at an input the cycle before, or in a cycle the first engine may be +taking one. Each engine's block is chosen a cycle ahead, so the ready a block +sees comes from registers. + +The direct-memory clients (the DXA writes and the tensor unit's reads) take +whole bank rows on the first port: a DMA request seen in one cycle stops the +passes and gets the banks the next. ### 5.3 Global path @@ -318,7 +335,7 @@ the DXA writes and the tensor unit's reads. |---|---| | `VX_mem_coalescer` | merges word-sized lanes onto data-cache-word channels; lanes addressing the same cache word become one request | | `VX_lsu_adapter` | turns the lane bundle into one memory bus per channel; registers the request boundary | -| `VX_dcr_flush` | injects the host's cache-flush request, on channel 0 of block 0 | +| `VX_dcr_flush` | injects the host's cache-flush token on lane 0 of a private dcache, and on every lane otherwise (a shared dcache's socket arbiter, or a disabled dcache's passthrough, buffers each lane on its own) | The coalescer exists when there is more than one lane and the data-cache word is wider than an LSU word; otherwise the path is a pass-through that @@ -391,8 +408,9 @@ pool does. | `VX_CFG_LSU_QUEUE_IN_SIZE` | §4.3 | staging depth | | `VX_CFG_LSU_PENDING_SIZE` | §4.3 | outstanding-pool depth | -`NUM_THREADS` must be a power of two for the stack interleave, and a stack -access must fit a word (`FLEN ≤ XLEN`). +`NUM_THREADS` must be a power of two for the stack interleave, the stack +window top must be a multiple of `NUM_THREADS` stacks, and a stack access +must fit a word (`FLEN ≤ XLEN`). --- diff --git a/docs/designs/microarchitecture.md b/docs/designs/microarchitecture.md index 3a9913d82d..44eb6c4313 100644 --- a/docs/designs/microarchitecture.md +++ b/docs/designs/microarchitecture.md @@ -75,6 +75,18 @@ Vortex has a 6-stage pipeline: - **Commit** - Write result back to the register file and update the Scoreboard. +### Core partitioning + +The core splits its warps among parallel pipelines by warp id: +`VX_CFG_FETCH_WIDTH` frontend slices (`wid mod FW`), `VX_CFG_ISSUE_WIDTH` issue +slices (`wid mod IW`), and B blocks per execution unit (`wid mod B`). The +widths select the structure: IW compute slices when every width is equal, FW +frontend slices behind a combinational frontend crossbar otherwise, and a +stage-sliced issue/execute/commit when some unit has fewer blocks than issue +slices. CTA launch, barriers, warp retirement, the shared SFU/TCU services and +the memory system stay core-wide. See +[core_partitioning.md](core_partitioning.md). + ### Vortex clustering architecture - Sockets - Grouping multiple cores sharing L1 cache diff --git a/docs/designs/mlp_optimizations.md b/docs/designs/mlp_optimizations.md new file mode 100644 index 0000000000..c71469b390 --- /dev/null +++ b/docs/designs/mlp_optimizations.md @@ -0,0 +1,378 @@ +# Memory-Level Parallelism Optimizations + +A GPU hides memory latency by keeping many memory requests in flight at once. This is memory-level parallelism (MLP). It runs from the lanes of one warp, through the caches and the command processor's DMA, out to the DRAM channels. This document describes the features that raise Vortex's MLP: +- how the cache hierarchy is organized; +- how misses are handled; +- how same-address requests are merged; +- how coherence and ordering are kept cheap; +- how the DMA engines share the memory system; +- how the FPGA shell reaches HBM. + +Each section says what problem the feature removes, how it works, and what it gains. + +## Reading the numbers + +- Gains are changes in total kernel cycles, so −10% means 10% faster. Each gain is measured against the same configuration with the feature off. +- **rtlsim** is the Verilator model of the RTL. **SimX** is the cycle-level C++ model. **xrtsim** is the RTL plus the FPGA shell. +- The GPU shapes (`a100`, `h100`, `rtx5080`, …) are hardware profiles. A profile sets cores, warps, threads, cache sizes, and the DRAM type and timing (HBM2, HBM3, GDDR6) from data-sheet values. See [simulation.md](../simulation.md#hardware-profiles). + +## Summary + +| Feature | Level | What it removes | Best measured gain | Default | +|---|---|---|---|---| +| [Sectored L2/L3](#sectored-l2-and-l3-caches) | L2/L3 | Half of the LLC tags | Tag area halved | On | +| [Word-size dcache banking](#dcache-banks-follow-the-thread-count) | L1D | Fixed bank count as threads grow | Bank count scales with threads | On | +| [Hashed bank select](#hashed-bank-select) | L1D, L2 | Power-of-two strides on one bank | sgemv −32.8% | On (L2; dcache ≥ 4 banks) | +| [Fill forwarding](#fill-forwarding) | All caches | Replaying waiters after a fill | sgemv −5.0% | On | +| [Back-to-back fills](#back-to-back-fills) | All caches | A bubble after each fill | rtx5080 stream −6.2% | On | +| [Write-allocate without fetch](#write-allocate-without-fetch-lazy-fetch) | Write-back caches | Reading a line only to overwrite it | DRAM reads −33% | On where the cache is a write-back LLC | +| [Full-sector refetch skip](#full-sector-refetch-skip) | Write-back caches | Refetching a fully written sector | rtx5080 sgemm −43.5% with lazy L2 | On with lazy fetch | +| [Dcache read merge](#dcache-read-merge) | L1D | Same-word loads serialized on one bank | sgemm −17.2% | On | +| [Icache same-line merge](#icache-same-line-merge) | L1I | Fetch slices serialized on one line | rtx5080 sgemm −25.8% | On | +| [Broadcast flush, per-port tokens](#broadcast-flush-and-per-port-flush-tokens) | Whole hierarchy | Serial per-core flushes; lost writes | Flush 2.76M → 16.5k cycles per launch | On | +| [L1-only launch flush](#l1-only-flush-between-launches) | Whole hierarchy | Walking the LLC at every launch | Flush 2,750 → 180 cycles per launch | On | +| [Argument upload slots](#launch-argument-slots) | Runtime + LLC | A full flush after each argument upload | multikernel −7.5% | On | +| [Targeted fences](#targeted-fences) | L1D, LLC | A full flush per fence | printf 13.6× faster | On | +| [Acquire/release fences](#acquire-and-release-fences) | L1D | Doing both halves of a fence | Ordering-correct atomics at minimal cost | On | +| [Atomics at the LLC](#atomics-at-the-last-level-cache) | L1D, LLC | Atomics racing across private caches | Correct atomics across the hierarchy | With the A extension | +| [Write-through ordering](#write-through-ordering) | L1D | A load returning data older than its own store | Correctness | On | +| [CP DMA through the LLC](#command-processor-dma-through-the-llc) | CP, LLC | An LLC walk for every host transfer | multikernel −42.7% | Off (on in GPU profiles) | +| [LMEM DMA round-robin](#local-memory-dma-round-robin) | Local memory | DXA starving tensor-core reads | WGMMA with DXA −10.2% | On | +| [Per-block LSU memory paths](#per-block-lsu-memory-paths) | Core | One shared core memory path | Scales with LSU blocks | On | +| [Many AXI transactions in flight](#many-axi-transactions-in-flight) | FPGA shell | One outstanding request per port | 4-core sgemv 1.30× | On | +| [Ports follow the GPU; write fall-through](#memory-ports-follow-the-gpu-and-write-fall-through) | FPGA shell | A fixed port count; a cycle per write | Write throughput 2× | On | + +## Configuration knobs + +Some of these features can be set with a `VX_CFG_*` knob (see [Build Configuration System](build_configuration_system.md)). The rest are always on, or are selected automatically from the cache layout. + +| Feature | Knob | Default | +|---|---|---| +| Hashed bank select | `VX_CFG_DCACHE_BANK_HASH`, `VX_CFG_L2_BANK_HASH` | dcache: on with 4 or more banks; L2: on. The L3 has no bank hash. | +| Write-allocate without fetch | `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH` | On where the cache is a write-back LLC (the dcache with no L2, the L2 with no L3, or the L3), off elsewhere. It also enables the dirty-byte masks (`*_DIRTYBYTES`). | +| Full-sector refetch skip | Part of lazy fetch | Follows `*_LAZY_FETCH` | +| Dcache read merge | `VX_CFG_DCACHE_READ_MERGE` | 1 (on) | +| Icache same-line merge | `VX_CFG_ICACHE_READ_MERGE` | 1 (on). No effect with one fetch slice. | +| CP DMA through the LLC | `VX_CFG_CP_LLC_CLIENT` | 1 (on). Takes effect only when one LLC serves every core. | +| Atomics at the LLC | `VX_CFG_EXT_A_ENABLE` | Off | +| Sectored L2/L3 | `VX_CFG_{L2,L3}_SECTOR_SIZE` (a size) | One bus granule. Setting it to the line size removes sectoring. | +| Dcache word-size banking | `VX_CFG_DCACHE_WORD_SIZE` (a size) | Derived from the lane count and XLEN | + +**Always on:** fill forwarding, back-to-back fills, broadcast flush and per-port flush tokens, launch argument slots, targeted fences, acquire/release fences, write-through ordering, local-memory DMA round-robin, per-block LSU memory paths, many AXI transactions in flight, and write fall-through. + +**Selected automatically:** +- The L1-only launch flush and the fence shortcuts apply whenever a single LLC is the point of coherence. +- Write-allocate without fetch, the full-sector refetch skip and the dirty-byte masks turn on in whichever cache is a write-back LLC. +- The FPGA shell's memory port count follows the GPU's memory ports. + +--- + +## Cache organization + +### Sectored L2 and L3 caches + +![L2/L3 sectoring and dcache banking](../assets/img/mlp_sectoring.svg) + +**Problem.** An LLC line as small as the memory bus granule needs one tag per 64 bytes. At GPU LLC sizes (megabytes), that is a large tag array. + +**How it works.** L2 and L3 lines are twice the bus granule. One tag covers the line, and the line is split into two sectors of one bus granule each, with a valid bit per sector. Fills and evictions move one sector at a time, so memory traffic stays at bus granularity, while the tag array is half the size. Sector size is set with `VX_CFG_{L2,L3}_SECTOR_SIZE`. + +**Gain.** The LLC tag area is halved at the same capacity and the same memory traffic. + +### Dcache banks follow the thread count + +**Problem.** A dcache with a fixed bank count serializes a warp's lanes once the thread count outgrows it. + +**How it works.** The coalescer's word size sets the dcache banking: banks = line footprint / word size, and there is one bank per request lane. Wider warps get more banks without a smaller line. See the right half of the figure above. + +**Gain.** Dcache parallelism tracks `NUM_THREADS`, so a 16- or 32-thread warp is not throttled by a 4-bank cache. + +### Hashed bank select + +![Hashed bank select](../assets/img/mlp_hashed_banks.svg) + +**Problem.** A cache picks its bank from the lowest line-address bits. GPU kernels walk arrays with power-of-two strides: matrix rows, tiles, per-thread blocks. Every request in such a stride has the same low bits, so all of them queue on one bank while the others sit idle. + +**How it works.** The bank index is the low line-address bits XORed with a hash of the higher bits. The hash is an IPOLY polynomial: IPOLY(5) for 16 banks, IPOLY(37) for 32, and an XOR fold for other counts. Strided requests spread over all banks. Each line still has exactly one bank, so the rest of the cache is unchanged. It is controlled by `VX_CFG_{DCACHE,L2}_BANK_HASH`: on for the L2, and on for a dcache with 4 or more banks. Below 4 banks, the extra queuing outweighs the spreading. + +**Gain (rtlsim):** +- sgemv 512×512 at 16 threads: −32.8% +- fp16 tensor sgemm on 2 cores + L2: −13.8% +- WGMMA with DXA: −13.5% +- multi-core sgemm: −8.2% +- SimX, rtx5080, N=256: −34% + +--- + +## Miss handling + +### Fill forwarding + +![Fill forwarding](../assets/img/mlp_fill_forward.svg) + +**Problem.** When a miss returns, every request waiting on that line (the MSHR chain) used to replay through the bank pipeline one at a time. Each replay is a full pipeline pass, and new requests are blocked while the chain drains. + +**How it works.** The reads at the head of the chain take their data straight from the staged fill sector, which is fill-buffer forwarding, standard in GPU L1s. The first write or atomic in the chain, and everything after it, still replays, so same-line accesses keep program order. The input arbiter stays open to new requests while the chain completes. Always on. + +**Gain (rtlsim):** sgemm −2.7%, sgemv −5.0%, multi-core sgemm −2.2%. + +### Back-to-back fills + +![Back-to-back fills](../assets/img/mlp_fill_staging.svg) + +**Problem.** A bank could not take a fill in the cycle right after another fill. Under heavy miss traffic, fills arrived in bursts and the bank accepted only one every other cycle. + +**How it works.** Fills follow each other into the bank pipeline back to back. Each staged fill keeps its own sector until it installs, so consecutive fills don't conflict. + +**Gain:** +- rtx5080 profile, 4 MiB stream: −6.2% +- the stock configuration (shallow L2): within ±0.5% +- the cache also closed timing faster: Fmax up by 3% + +### Write-allocate without fetch (lazy fetch) + +**Problem.** A write-back cache that misses on a store normally reads the whole line from memory, then overwrites part or all of it. Streaming writes therefore cost a read for every line they write. + +**How it works.** A store miss installs the sector at once, with a per-byte dirty mask and a *partial* bit, without reading memory. A later load or atomic on a partial sector refetches it and merges memory data only under the bytes that were never written. An eviction writes back only the dirty bytes. A refill of one sector keeps the dirty bytes of the line's other sectors. Controlled by `VX_CFG_{DCACHE,L2,L3}_LAZY_FETCH`. + +**Gain:** 1-core vecadd DRAM reads −33% and memory requests −25%. Cycles are unchanged because that kernel is latency-bound. On where the cache is a write-back LLC, the same write-validate policy NVIDIA's L2 uses: the DRAM traffic it removes counts even where cycles are flat. + +### Full-sector refetch skip + +![Full-sector refetch skip](../assets/img/mlp_lazy_fetch.svg) + +**Problem.** With lazy fetch, once stores have written every byte of a sector, the partial bit still triggered a refetch on the next load. The cache read memory only to merge it under zero unwritten bytes. On L2-heavy kernels, these useless reads made lazy fetch slower than leaving it off. + +**How it works.** When the refetch commits, the bank checks the byte mask. If the mask is full, the refetch loops back through the bank as a fill with no data, clears the partial bit, and reads nothing from memory. If the loopback queue has no room, the refetch falls back to memory as before. + +**Gain:** +- DRAM reads: 145 → 17 at the dcache and 539 → 27 at the L2, on a fully written stream. +- SimX, rtx5080 profile, 4 cores, sgemm -n192 with L2 lazy fetch: 285,006 → 161,121 cycles (−43.5%). Lazy fetch is now slightly faster than having it off. + +--- + +## Request merging + +### Dcache read merge + +![Dcache read merge](../assets/img/mlp_read_merge.svg) + +**Problem.** Uniform loads make every lane of a warp read the same word in the same cycle. Kernel arguments and the shared operand of a matrix multiply are typical. Each LSU port sent its own request to the same bank, and the bank served them one per cycle. + +**How it works.** Plain reads of the same word from the ports of one LSU block in the same cycle are served by one bank access, and the reply goes to every port that asked. Each port's order into the bank is unchanged, so loads stay ordered against stores. The merge compares request tags in the same form on every port, so it doesn't depend on how wide the tag is. Controlled by `VX_CFG_DCACHE_READ_MERGE`. + +**Gain (rtlsim sgemm -n128):** +- 16 threads with an 8-byte dcache word: −17.2% +- 2 banks: −9.8% +- 32 threads: −7.9% +- 16 threads: −6.5% +- 4 cores + L2: −6.5% + +At 16 threads, dcache reads fall 25% and bank stalls 55%. + +### Icache same-line merge + +![Icache same-line merge](../assets/img/mlp_icache_merge.svg) + +**Problem.** With several fetch slices, the front end fetches up to one instruction per slice per cycle. A dense kernel's inner loop often spans only a few cache lines (sgemm: 3). So the slices keep requesting the same line and take turns at its bank, losing 1 to 2 of 4 fetch slots every cycle even with a 100% hit rate. A warp has only one fetch in flight, so each cycle of waiting lowers its fetch rate directly. + +**How it works.** Reads of the same icache line from different fetch slices in the same cycle become one bank access. The bank returns each slice the word it asked for. This differs from the dcache merge: the slices want different words of one line, not the same word. Controlled by `VX_CFG_ICACHE_READ_MERGE`. It has no effect with one fetch slice. + +**Gain (rtlsim / SimX):** + +| Shape | sgemm | vecadd | conv3 | raycast | +|---|---|---|---|---| +| Issue and fetch width 4 | −12.1% / −12.4% | | | −4.7% / −4.7% | +| rtx5080 core | −25.8% / −20.2% | −15.7% / −20.1% | −3.4% / −4.6% | | + +With the merge, 16 icache banks are no longer slower than 4. + +--- + +## Coherence and ordering + +### Broadcast flush and per-port flush tokens + +![Broadcast flush and per-port tokens](../assets/img/mlp_flush_tokens.svg) + +**Problem:** +- The command processor flushed the cores one after another, so a launch boundary cost the sum of every core's flush. +- A cache forwarded its flush token on memory port 0 only. A write still queued on another port could reach the next level after that level had finished its walk, and stay dirty there. + +**How it works:** +- One broadcast `CACHE_FLUSH` write starts every core's flush at once. The command processor then collects the completions, so the flushes overlap. +- A cache walks its banks and sends a flush token down **every** memory port. It answers only once all ports have answered. +- Every independently buffered write path carries its own token. +- Read-only caches (icache, texture, ray-tracing caches) no longer forward flushes at all. + +**Gain:** flush cost per launch dropped from 2.76M to 16.5k cycles on the measured shape. It also fixed the lost-write case above. + +### L1-only flush between launches + +![L1-only launch flush](../assets/img/mlp_launch_flush.svg) + +**Problem.** Every kernel launch flushed the whole hierarchy, including a walk of the LLC. Back-to-back launches paid for it each time. + +**How it works.** When a single LLC serves every memory client, it is the point of coherence: launches only need the L1s cleaned. The launch flush becomes an L1-only token, which the shared LLC answers without walking. LLCs that are replicated per core or per cluster, and LLCs with peer clients beside them (texture, ray tracing, raster, output merger, DXA), still walk. A dcache counts as the LLC only when it is the single dcache of a single socket with no other memory client. The runtime posts flushes lazily and flushes fully before any device-memory transfer that needs it. The device reports busy until every core write, including uncached I/O stores, is visible in memory. + +**Gain:** flush cycles per launch 422 → 152 at 1 core and 2,750 → 180 at 4 cores + L2. Chained launches at 4 cores + L2: −11%. + +### Launch argument slots + +![Launch argument slots](../assets/img/mlp_arg_slots.svg) + +**Problem.** Each launch uploads its arguments and launch descriptors to device memory. That upload made the LLC dirty, so the next launch needed a full flush. Workloads that launch many small kernels behaved as if the L1-only flush didn't exist. + +**How it works.** Arguments and descriptors go into a pool of slots. A slot is reused only if no kernel has read it since the last full flush, so no cache can hold a stale copy and the upload needs no flush. This is the same dirty tracking AMD's driver does for its argument buffers. + +**Gain:** multikernel at 4 cores + L2: −7.5% on rtlsim, −8.5% on SimX. A workload that reads results back after every launch still needs the full flush and is unchanged. + +### Targeted fences + +![Targeted fences](../assets/img/mlp_fences.svg) + +**Problem.** A memory fence flushed the whole hierarchy: every level walked all of its lines and wrote dirty data to DRAM. `printf` fences after every device write, so printf-heavy kernels spent most of their time walking caches. + +**How it works.** +- A memory fence sends an L1-only token. The L1s write their dirty lines back to the shared LLC, and the LLC answers without walking. DRAM is not touched. +- A fence that orders only device (I/O) accesses becomes its own LSU operation. Every level drains its in-flight writes to its memory ports, with no walk anywhere. + +**Gain (rtlsim):** +- printf at 2 cores + L2: 4,006,822 → 295,165 cycles (13.6×) +- printf at the default configuration: −42.9% +- fence test at 2 cores + L2: −55% +- async global barrier: −13.3% + +### Acquire and release fences + +![Acquire and release fences](../assets/img/mlp_fence_split.svg) + +**Problem.** A fence did both halves of a full barrier, even when the program asked for only one. Atomics marked acquire or release (`.aq`/`.rl`) were not ordered at all. + +**How it works.** The fence's predecessor and successor sets choose the action: +- **Acquire** (`fence r,rw`, `amo.aq`): invalidates clean L1 lines after the operation. Nothing written back. +- **Release** (`fence rw,w`, `amo.rl`): drains pending writes before the operation, without a walk. Nothing invalidated. +- **Full** (`fence rw,rw`, `amo.aqrl`): does both. + +Atomics with `.aq`/`.rl` get these steps around them. All of it is skipped when the dcache is itself the point of coherence. + +**Gain:** correct acquire/release ordering for atomics, at the minimum cost the memory model allows. Fences, printf, global barriers and sgemm are unchanged; their kernels use full fences. A test that makes every atomic sequentially consistent pays for the ordering it now gets. + +### Atomics at the last-level cache + +![Atomics at the LLC](../assets/img/mlp_llc_atomics.svg) + +**How it works.** RISC-V `A`-extension atomics run once, as a read-modify-write at the last-level cache. Each private L1 they pass through drops its copy of the line, so the next load of that word from any core misses and sees the new value. `LR`/`SC` use a reservation table. This keeps atomics correct across L1, L2 and L3 without a coherence protocol between the L1s. Controlled by `VX_CFG_EXT_A_ENABLE`. + +### Write-through ordering + +![Write-through ordering](../assets/img/mlp_wt_order.svg) + +**Problem.** In a write-through L1 (every configuration with more than one core, or with an L2 or L3), a store can be waiting in the MSHR behind an earlier read miss to the same line. A load of the same word that arrived then could hit the cache and return the value from before the store. + +**How it works.** A waiting store counts as a hazard. A later access to the same word links behind it in the MSHR chain and sees the stored value. The same rule keeps a newer write from overtaking an older one. + +**Gain:** correctness. The ordering test fails without the rule and passes with it. + +--- + +## DMA + +### Command-processor DMA through the LLC + +![CP DMA through the LLC](../assets/img/mlp_cp_dma_llc.svg) + +**Problem.** The command processor copied host data straight into DRAM, beside the L2. The L2 could hold stale copies of those lines, so every host transfer needed a full flush and invalidate of the LLC. Launches with uploads between them paid this every time. + +**How it works:** +- When one LLC serves every core, the command processor's DMA becomes a client of it on its own slot. A page-table walker attaches the same way. +- Host data flows through the LLC and keeps it coherent, so a transfer needs only an L1-only flush. +- Transfers of at least half the LLC write around it, so they don't evict the working set. +- A launch issued after DMA writes waits for an L1-only flush token sent on the DMA's slot. +- The runtime detects the feature through the `DMA_LLC_CLIENT` capability. +- Controlled by `VX_CFG_CP_LLC_CLIENT`: on by default. + +**Gain (rtlsim):** + +| Shape | dogfood | multikernel | sgemm -n32 | +|---|---|---|---| +| 2 clusters + L2 + L3 | −30.0% | −42.7% | −14.1% | +| 4 cores + L2 | −9.1% | −23.4% | | +| 8 MB L3 | −64% | | | + +**Virtual memory.** Both command processors translate DMA addresses before they enter the LLC, so a client of the LLC sees physical addresses with virtual memory on. + +### Local-memory DMA round-robin + +![LMEM DMA round-robin](../assets/img/mlp_lmem_round_robin.svg) + +**Problem.** Two clients share the local-memory DMA port: the DXA engine, which writes tiles into shared memory, and the tensor core, which reads WGMMA operands from it. The DXA had fixed priority. One tiled-scatter transfer held the port for about 540 cycles, and a tensor-core batch caught behind it stalled for about 510. That stall delayed the warpgroup barrier, and the barrier delayed the next DXA request. + +**How it works.** The port grants round-robin, so a tensor-core read waits at most one DXA write. It costs one bit of state, and a single-client configuration is unchanged. + +**Gain (rtlsim):** WGMMA with DXA −10.2%, sparse WGMMA with DXA −11.6%. + +--- + +## Load/store unit + +### Per-block LSU memory paths + +![Per-block LSU memory paths](../assets/img/mlp_lsu_paths.svg) + +**Problem.** With several LSU blocks per core (one per issue slice), all of them went through one shared core memory path. Both bandwidth and floorplanning were limited by that single funnel. + +**How it works.** Each LSU block owns its path to its own dcache ports, along with its own status signals. The tensor-core blocks also get their own metadata ports into their slice's LSU. This is the processing-block organization of NVIDIA SMs: each partition has its own load/store path. + +**Gain:** cycle-identical at today's shapes. It is the structure that lets memory bandwidth scale with the number of LSU blocks, and it shortens the wires in the core. + +--- + +## FPGA memory interface + +### Many AXI transactions in flight + +![Many AXI transactions in flight](../assets/img/mlp_afu_inflight.svg) + +**Problem.** The FPGA shell limited each AXI memory port to very few outstanding transactions. By Little's law, a port's throughput is the number of requests in flight divided by latency. One request in flight at 100 cycles of latency delivers one line per 100 cycles, however much bandwidth the HBM has. + +**How it works.** The limit is gone. Requests are arbitrated with their AXI IDs, and responses are routed back by ID, so many transactions overlap their latency. A grant is held while a request is being offered, which keeps the AXI rule that a valid request stays stable. + +**Gain (xrtsim):** + +| Case | Speedup | +|---|---| +| One port, 100-cycle latency | 1.25–1.27× | +| 4-core sgemv | 1.30× | +| Several ports | 1.06–1.08× | +| Default timing | 1.00–1.13× | + +### Memory ports follow the GPU, and write fall-through + +![Memory ports and write fall-through](../assets/img/mlp_afu_ports.svg) + +**Problem.** +- The number of AXI memory ports was fixed per board, not set by the GPU configuration. A larger GPU could not use more HBM channels, and some boards left channels unmapped. +- A write's data waited a cycle behind its address. + +**How it works.** +- The shell gets one AXI port per GPU memory port, rounded up to a power of two. +- Each port owns a contiguous part of memory, mapped to its own HBM channel group, and the board connectivity is generated from the port count. +- A write's data now leaves in the same cycle as its address. Data stays in address order through a small queue. +- Reset waits for any write already in progress, so it never splits a burst. + +**Gain:** +- Single-beat writes: twice the throughput (1,000 writes in 1,000 cycles instead of 2,000). +- Small GPUs (1–4 cores) are not port-bound and see within ±1.2%. The port scaling is what lets larger GPUs use the HBM. + +--- + +## How the gains were measured + +- **DRAM model.** All simulators share one DRAM model: + - independent channels, refresh, and open-page bank interleave; + - JEDEC-style HBM2, HBM3 and GDDR6 timing from data-sheet values; + - per-channel response queues. + + The A100 profile's idle latency (216/257 core cycles open-row/conflict) matches GPGPU-Sim's SM80 model (about 215/255). +- **Model agreement.** SimX models every feature here with the same structure as the RTL, and CI checks SimX against rtlsim on each feature's shapes. So gains measured on SimX at full GPU scale reflect the hardware. Each feature has a test that fails when it is broken or turned off. diff --git a/docs/designs/multicache_amo_coherence.md b/docs/designs/multicache_amo_coherence.md index e6ad6901ea..9525593bc1 100644 --- a/docs/designs/multicache_amo_coherence.md +++ b/docs/designs/multicache_amo_coherence.md @@ -240,19 +240,25 @@ updates it, core 0's copy is stale. The LLC never reaches up. The weak model resolves this the way a GPU does: the **consumer** discards its own copies at an acquire point and refetches. -There is no dedicated acquire-invalidate. What exists today: - -| model | what a `fence` does to the issuing core's data cache | -|---|---| -| RTL | the LSU sends the fence as a request with `is_flush` set, which starts the **whole-cache flush**; on a write-through cache the walk clears every valid bit | -| SimX | the fence is an ordering barrier in the LSU only; a flush of a write-through cache is a no-op and no line is invalidated | - -So an RTL `fence` happens to give the consumer-side invalidate, at the cost -of a full flush walk that locks the cache's inputs for its duration, and -SimX does not model it. **The two models disagree, and no regression case -exercises fence-managed cross-core visibility** — `atomic_critical`, which -runs plain loads and stores inside a lock, is skipped on multi-core -configurations for exactly this reason. +There is no dedicated acquire-invalidate. In the RTL the LSU sends a memory +`fence` (any R or W in its predecessor or successor set) to the data cache as +an **L1-only** flush token (`is_flush` with byte enables set). Each cache above +the point of coherence holds its inputs, drains, walks every set (every way of +a write-back cache), writing back dirty lines and clearing every valid bit, and +forwards the token. The shared LLC, the point of coherence, answers it without +a walk; a replicated LLC still walks. The fence holds its LSU block until the +answer. + +So a memory `fence` gives the consumer-side invalidate above the point of +coherence, at the cost of a flush walk of each private level. `fence_mp` +passes data between cores this way on shared and replicated last levels. +`atomic_critical`, which runs plain loads and stores inside a lock, is still +skipped on multi-core configurations. + +A fence ordering only device accesses (`fence ow, ow`, no R or W) sends an +**IO** token (`is_flush` with `io`): no level walks; each quiesces and waits +until its uncached path is empty, then forwards, so the answer proves the +earlier IO writes reached memory. Programs that share data across cores through plain memory are outside what is validated. Share through atomics, or through the single shared level. diff --git a/docs/designs/output_merger_architecture.md b/docs/designs/output_merger_architecture.md index 5c4d7adecf..a8aeacb10d 100644 --- a/docs/designs/output_merger_architecture.md +++ b/docs/designs/output_merger_architecture.md @@ -8,7 +8,7 @@ and deadlock arguments, and the parallelism/scaling model. Covers the RTL ([`hw/rtl/om/`](../../hw/rtl/om/)), the SIMT-side plumbing ([`VX_decode.sv`](../../hw/rtl/core/VX_decode.sv), [`VX_gfx_uops.sv`](../../hw/rtl/core/VX_gfx_uops.sv), -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv)), the cluster +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv)), the cluster integration ([`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)), the SimX model ([`sim/simx/om/`](../../sim/simx/om/)), and the software contract ([`sw/kernel/include/vx_graphics.h`](../../sw/kernel/include/vx_graphics.h), @@ -170,7 +170,7 @@ byte-enables splitting the write (`byteen[2:0]` = depth writemask, ### 4.1 LSU attribute tagging -[`VX_lsu_slice.sv`](../../hw/rtl/core/VX_lsu_slice.sv) range-checks every +[`VX_lsu_block.sv`](../../hw/rtl/core/VX_lsu_block.sv) range-checks every lane address against the aperture and sets **two** attribute bits: `is_addr_om` (the steer's select) and `is_addr_io` (alongside it). The IO bit is not a hack — *uncached, bypassed, posted* is exactly the behaviour @@ -183,9 +183,21 @@ bit arrives at the cluster untouched. One steer per socket→L2 trunk input (`L2_SOCKET_REQS = NUM_SOCKETS × L1_MEM_PORTS` of them, in [`VX_graphics.sv`](../../hw/rtl/VX_graphics.sv)). The steer demuxes on the single `is_addr_om` bit: aperture writes go to the -OM leg (registered), everything else passes **combinationally** to L2 (the -socket already registers its outgoing bus, and a skid here would sit on the -critical L1→L2 path). +OM leg, everything else to L2; each leg presents a registered master +interface (`OUT_BUF`). + +**Flush ordering: the steer is a fork.** A flush token from the socket (a +dcache's kernel-boundary flush or a fence) bound for L2 is held at the steer +until the OM side has been flushed. When the token reaches the steer, every +aperture store its trunk carried before it is already on the OM leg. A +controller in `VX_graphics` serves the requests present when a round starts +(the held tokens and the cluster flush), waits until no complete fragment is +left in the OM (steer OM legs, ingress record stages and OM cores idle), +flushes the ocache, and then releases those tokens to L2. A half-received +two-word record is not waited for: its depth beat can be queued behind a held +token, and the store it belongs to is not one the token orders. So a flush covers the OM writes of the aperture stores +issued before it, whatever `busy` reports while those stores are in transit +between the dcache and the ingress (a span no busy signal counts). **Placement is the deadlock argument.** The join is the last point where a request has left the socket but has not touched anything L2-owned. A full OM diff --git a/docs/designs/simx_simulator_architecture.md b/docs/designs/simx_simulator_architecture.md index 9da381bc39..a37cfe2425 100644 --- a/docs/designs/simx_simulator_architecture.md +++ b/docs/designs/simx_simulator_architecture.md @@ -105,7 +105,9 @@ atomics, threads, locks, or deferred-call plumbing — enforced by **Functional units** - `alu_unit`, `fpu_unit` — own private `execute()` (ISA semantics). -- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads; pulls load +- `lsu_unit` — loads/stores/AMO; `LsuUopGen` for packed loads, wide loads/stores and + matrix fragment loads/stores (`vx_ldm`/`vx_stm`, whose geometry and register order + `TcuUopGen::ldm_uop` derives with the WMMA expansion's permutations); pulls load data from `MemRsp::data`. - `sfu_unit` — special-function dispatcher routing to sub-units: WCTL, CSR, TEX, RASTER, DXA, OM. diff --git a/docs/designs/tensor_core_wgmma_engine.md b/docs/designs/tensor_core_wgmma_engine.md index d6eefb36f1..dbd58a28e6 100644 --- a/docs/designs/tensor_core_wgmma_engine.md +++ b/docs/designs/tensor_core_wgmma_engine.md @@ -67,6 +67,11 @@ Flags of an MMA instruction: | `rs2[2:1]` | `cd_nregs` — accumulator registers: 0 = 8, 1 = 16, 2 = 32 | | `rs2[3]` | `a_from_smem` — A comes from shared memory; B always does for WGMMA | +`fmt_d` is fp32 for float and MX inputs and int32 for integer inputs; the +tensor core has no other output format. The kernel API rejects any other output +type at compile time (`static_assert` in `vx_tensor.h`). An instruction that +encodes one anyway trips an assertion in the RTL (`VX_tcu_core`) and in SimX. + | `op_type` | value | operation | |---|---|---| | `INST_TCU_WMMA` | 0 | warp MMA | @@ -92,6 +97,16 @@ micro-op arguments do not carry a sparse bit. An MX format has bit 3 of the id set, which is how decode recognizes that the instruction depends on scale metadata. +Scales follow the OCP MX and NVFP4 definitions: + +- **mxfp8, mxbf8, mxfp4** take an E8M0 scale per 32 elements, worth + 2^(e-127). A product of two scales spans 2^-254 to 2^254. 0xFF is NaN. +- **nvfp4** takes a UE4M3 scale per 16 elements: unsigned, so bit 7 is + padding, and 0x7F is NaN (PTX `.ue4m3`). A zero exponent field is read as a + normal; subnormal scales are an open question. +- **Out-of-range products.** A NaN scale makes its block NaN. A product scaled + below the datapath's exponent range only reaches the sticky bit. + ### 2.2 Metadata dependencies Metadata is not an architectural register, so the scoreboard tracks it with @@ -192,7 +207,8 @@ is muxed by `op_type`: `VX_tcu_wgmma` is a third party that only **observes** the dispatch path. It builds the tile-buffer request from each WGMMA micro-op and returns operand data and a ready signal, but never drives the handshake itself; the core -does. +does. `VX_tcu_agu` and `VX_tcu_wgmma` serve every block, so they live in +the core-level `VX_tcu_shared` and reach each block through per-block ports. --- @@ -363,7 +379,10 @@ simulation assertion catches a forgotten mask. [`VX_tcu_agu.sv`](../../hw/rtl/tcu/VX_tcu_agu.sv) issues **one warp-wide fetch per `TCU_LD`**: lane `T` loads word `T` of the metadata tile at the base address in `rs1`. The kernel pre-advances the base per slot, so a -multi-slot load is several instructions. +multi-slot load is several instructions. An LSU narrower than the warp +takes the fetch in `NUM_THREADS / NUM_LSU_LANES` passes of consecutive +words, as the LSU splits a warp into packets; each request carries its pass +in its tag, so the responses can return in any order. The generator is **client 1 of the core's LSU scheduler** on block 0. It owns no memory port; it reuses the LSU's staging, outstanding pool and cache @@ -397,9 +416,10 @@ MMA (§2.2). ## 8. FEDP backends -`VX_CFG_TCU_TYPE` selects one implementation of the dot product. All compute -the same function; each asserts the configured latency against its own -stage structure. +`VX_CFG_TCU_TYPE` selects one implementation of the dot product. Each asserts +the configured latency against its own stage structure. They agree up to +rounding, not bit for bit: `TFR` rounds once over a fixed window, the others +round each multiply and add. | backend | implementation | latency | |---|---|---| @@ -411,8 +431,13 @@ stage structure. `TFR` is the universal default. It is the only backend that does not pay a full floating-point rounding per tree level: it aligns every product to one -exponent and adds them as integers, rounding once. On an FPGA its -multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. +exponent and adds them as integers, rounding once. The alignment keeps a +25-bit window below the largest product's exponent field, with a sticky bit +for what falls out of it. A row of subnormals, or products that nearly cancel, +can therefore round a few ULP away from the exact sum. +[`sw/common/fedp.h`](../../sw/common/fedp.h) (`TfrFedp`) models it bit for +bit. On an FPGA its multipliers map onto DSP blocks through +`VX_CFG_TCU_USE_DSP`. --- @@ -431,6 +456,13 @@ multipliers map onto DSP blocks through `VX_CFG_TCU_USE_DSP`. Operand loads go through simulation channels; the tensor path never reads memory directly. +Numerics follow the configured backend for fp32 results from float and +block-scaled (MX, NVFP4) sources. Under `TFR`, each dot product is the +`TfrFedp` model, the same one the RTL is checked against bit for bit in +`hw-tcu-unit`. Otherwise it is a discrete fp32 multiply-add: in the DPI +backend's order for unscaled formats, and per element for block-scaled ones. +C and D are fp32 (int32 for integer inputs) in both models; see §2. + Perf: MPM class `TCU` — `TCU_TBUF_STALLS` (0xB03, cycles a WGMMA micro-op waited on tile-buffer data), `TCU_TBUF_CACHE_HITS` (0xB04, B rows reused), `TCU_LMEM_READS` (0xB05). diff --git a/docs/designs/virtual_memory_subsystem.md b/docs/designs/virtual_memory_subsystem.md index 8b6a34a6e4..2b2cc7abef 100644 --- a/docs/designs/virtual_memory_subsystem.md +++ b/docs/designs/virtual_memory_subsystem.md @@ -160,9 +160,13 @@ kernel image once the loader knows its extents. Buffers allocated with (`VX_CFG_VM_PINNED_REGION_SIZE`), so a kernel can hold PA-stable pointers while paged and identity translations contend in the same TLBs. -The graphics fixed-function masters (RASTER, TEX, OM) and the RTU issue -physical addresses and never pass through an MMU; see -[`graphics_hardware_stack.md`](graphics_hardware_stack.md). +The socket-resident units translate too: the TEX unit's tcache misses, the +RTU's rtcache misses and the DXA's global reads go through the socket **unit +MMU** (§3.8), so a texture base, a BVH root, an acceleration-structure pointer +or a tensor descriptor is a virtual address like any kernel pointer. The +cluster-resident raster and OM units translate through the cluster **unit +MMU** (§3.9), so their tile, primitive, color and depth buffers are ordinary +buffers too; see [`graphics_hardware_stack.md`](graphics_hardware_stack.md). --- @@ -212,7 +216,7 @@ mutually exclusive by priority: | `cat_bypass` | `valid ∧ req_bypass` | fields forwarded unchanged | | `cat_hit` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ perm_ok` | `PA = {ppn_spliced, addr[11:0]}` forwarded; MRU bump | | `cat_park` | `valid ∧ ¬bypass ∧ (mshr_match ∨ ¬cam_hit)` | lowest parking lane parks; others hold and retry | -| `cat_pfault` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ ¬perm_ok` | accepted and dropped; fault sideband pulses | +| `cat_pfault` | `valid ∧ ¬bypass ∧ ¬mshr_match ∧ cam_hit ∧ ¬perm_ok` | accepted, never sent to memory; a data-side load or atomic is answered with zeroed data (§3.5); fault sideband pulses | `mshr_match` outranks `cam_hit` on purpose: a request to a page whose walk is still landing must queue behind that page's parked requests, or a same-lane @@ -267,7 +271,10 @@ or atomic still owes the pipeline a response, so `VX_mmu` injects a zeroed response carrying the parked tag ahead of the cache reply for that lane (`kill_needs_rsp = (EXEC_SIDE == 0) ∧ (¬rw ∨ amo)`). A plain store and every instruction fetch retire silently — a fabricated instruction word would be -decoded as real. +decoded as real. A permission fault caught on +a hit is answered the same way: the load or atomic waits in a one-entry +per-lane slot (the lane holds while it is full) and its zeroed response goes +out behind any kill on that lane. Ordering follows from the structure. The stage is hit-under-miss, so only same-page order is guaranteed: accesses to one VPN share an entry and drain in @@ -278,6 +285,20 @@ joiners: the next same-VPN request re-walks rather than inheriting the kill. Allocation is refused during a flush, because every entry clears at that edge and an accepted request would vanish with neither replay nor kill. +A cache flush token bypasses translation, so on its own it would overtake its +lane's parked requests. Each lane counts the requests it has in the miss +station (up on a park, down on the replay or kill that drains one), and a +flush token on a lane holds while that count is non-zero: it reaches the cache +behind every earlier request of its lane, which is the order a +`FLUSH_FORWARD` cache relies on to forward the token behind that port's +writes. Only a data-side MMU with writing clients counts (the dMMU and the +cluster unit MMU): a read or fetch overtaken by a token is still answered +correctly. The token of one lane does not wait on another lane's parks; the +core covers that case for the dcache: under VM its flush request also waits +until its own dMMU lanes are idle (`lane_idle`, which counts the parks), as it +waits for its stores, and is then latched until the flush request drops, since +the token's own trip through the MMU makes a lane busy again. + ### 3.6 Flush and drain `flush_clear = flush_if.req ∧ tlb_empty`: the CAM and the miss station clear @@ -295,6 +316,99 @@ request, one lane per core, `ITLB_SIZE` (8) entries. Fetch reaches it through icache stream; that request carries the FLUSH attribute and bypasses translation. +### 3.8 The socket unit MMU + +The socket-resident units reach memory through the socket memory arbiter, not +through the dcache, so the dMMU never sees them. Under VM a third `VX_mmu` +instance per socket, `ummu` +([`VX_socket.sv`](../../hw/rtl/VX_socket.sv)), sits **on their memory side**: +between the tcache's and rtcache's miss ports and the DXA's gmem port on one +side and the socket arbiter's `TCACHE_MEM_ARB_IDX` / `RTCACHE_MEM_ARB_IDX` / +`DXA_MEM_ARB_IDX` inputs on the other, one lane per port (`UMMU_TCACHE_IDX`, +`UMMU_RTCACHE_IDX`, `UMMU_DXA_IDX` in +[`VX_gpu_pkg.sv`](../../hw/rtl/VX_gpu_pkg.sv)), each lane an L1 line wide with +the socket arbiter's tag. It is the same module as the dMMU, with five +differences: + +- **Its own TLB**, `VX_CFG_UTLB_SIZE` (8) entries, so texture, BVH and tensor streams + cannot evict the cores' translations, and its misses ride a third socket + `tlb_bus` into `tlb_socket_arb` (3 inputs), on to the cluster TLB and the + walker like any L1 miss. +- **A registered reply** (`RSP_OUT_BUF = 2`). Each reply leaves through an + MMU-owned stage, so the select that injects kill and permission-fault + answers stays off the units' response logic. The requests arrive from + flops their clients own: the unit caches register their memory requests + (`MEM_OUT_BUF`), and the DXA registers its gmem output under VM. The TLB + lookup therefore starts at a client flop and its accept ends at the + client's registered ready, never deeper inside the unit. A unit access pays + the reply stage plus the MMU's own output stage, under VM only. +- **A registered park** (`PARK_REG = 1`). The lane that wins the park + arbitration is first registered in a one-entry stage and allocates in the + miss station the next cycle, deduplicating against the staged VPN itself + (`ALLOC_PROBE`) since the lane's probe is a cycle old by then. This keeps + the lane arbitration off the miss station's update cone. A later request + on the staged page holds until the stage drains, so it cannot overtake the + staged one. The stage takes a park only while empty, so the lanes park at + most every other cycle, and a park (new or joining) pays one more cycle. An + entry that filled and freed while the park was staged still matches: it is + re-armed as filled and replays from its kept translation (`retired_r`), + so the staged park neither walks the page again nor installs it twice. +- **`WRITE_ENABLE = 0`.** Its clients only read, so a parked request keeps + `{rw, addr, attr, tag}` and the replay restores zero data and all byte + enables; the miss-station payload is a line's address, not a line. +- **The unit caches are virtually indexed and tagged**, since translation + happens after them. They are invalidated whenever translation changes: the + socket snoop's `SATP_HI` flush (§8.2) also flushes the unit MMU's TLB and, + from a latch of its own, injects a flush token into the tcache and the + rtcache through their `VX_dcr_flush` legs (OR-ed with the core-driven + graphics flush), held until both caches report done. The two legs are + independent: a cache flush waits for its outstanding misses, which may need + a unit-TLB park, and a TLB under flush refuses parks, so holding the TLB + flush for a cache's walk could deadlock. Both caches are read-only, so the + flush only invalidates. + A single address space has no synonyms, so a VIVT read-only cache flushed + on every `satp` change is exact. + +Without VM, or with none of TEX, RTU and DXA (`SOCKET_UNIT_MMU_ENABLE` undefined), +none of this exists: the unit ports wire straight into the arbiter as before. +A unit's DCR base address is a virtual address, which on Sv32 spans the +whole 32-bit space, so under VM the TEX base field (`TEX_ADDR_BITS`, a +cache-block index) is 26 bits; without VM, 25 bits cover the physical range. + +The unit MMU's `empty` holds the socket `busy`, but not the per-core +`mmu_drained` barrier gate: that gate orders a core's own loads and stores, +none of which pass through the unit MMU. + +### 3.9 The cluster unit MMU + +Raster and OM live in the cluster (`VX_graphics`) and reach the L2 directly +through the rcache and the ocache, past every socket. Under VM a fourth +`VX_mmu`, `cmmu` ([`VX_cluster.sv`](../../hw/rtl/VX_cluster.sv)), sits between +those caches' memory ports and their L2 client slots (`L2_GFX_RASTER_IDX`, +`L2_GFX_OM_IDX`), one lane each (`CMMU_RCACHE_IDX`, `CMMU_OCACHE_IDX`), with +the same registered reply (`RSP_OUT_BUF = 2`) and registered park +(`PARK_REG = 1`) as the socket unit MMU; the rcache and ocache register their +memory requests (`MEM_OUT_BUF`). Its TLB is +`VX_CFG_CTLB_SIZE` (8) entries, and its misses enter the cluster TLB arbiter +as one more client after the sockets (`CLUSTER_NUM_TLB_PORTS`), its slot id +zero-extended to the socket buses' width. `WRITE_ENABLE` follows OM, since the +ocache writes (write-through) through it, so its park stage holds a full +line. `vm_active` comes from the device's +`satp` mode, fanned to the clusters from [`Vortex.sv`](../../hw/rtl/Vortex.sv). + +The rcache and ocache are virtually tagged, so the device's `SATP_HI` flush +(`mmu_flush_req`, §8.1) also reaches `VX_graphics` as an address-space-change +requester: the rcache invalidates through its `VX_dcr_flush` leg, and the OM +flush sequencer takes it as an L1-only round, draining the OM's fragments and +the ocache's writes into the L2 before invalidating. Only then does the +cluster flush the `cmmu` TLB those writes went through; the cluster's +`mmu_flush_done` waits on the L2 TLB and on `cmmu`. `cmmu.empty` joins the +cluster `busy`. With both units, the raster's early-Z depth reads go through +the ocache and so through `cmmu` too; its enable (`OM_EARLYZ_SAFE`) and depth +function reset off, so a draw that never programs the OM depth state reads no +depth buffer. As for TEX, the raster and OM buffer-base fields +(`RASTER_ADDR_BITS`, `OM_ADDR_BITS`) widen to 26 bits on Sv32 under VM. + --- ## 4. The miss/fill fabric @@ -317,11 +431,11 @@ elastic slice (`OUT_BUF = 3`). The id therefore grows deterministically ``` L1_TLB_ID_WIDTH = clog2(L1_TLB_MSHR_SIZE) the L1 miss-station slot -TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + sel(2) + {dMMU, iMMU} -TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + sel(NUM_SOCKETS) + socket +TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + sel(SOCKET_NUM_MMUS) + {dMMU, iMMU[, unit MMU]} +TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + sel(CLUSTER_NUM_TLB_PORTS) + socket [or cluster unit MMU] ``` -The socket arbiter merges the dMMU and iMMU ports onto `cluster_tlb_bus_if`, +The socket arbiter merges the dMMU, iMMU and (when present) unit MMU ports onto `cluster_tlb_bus_if`, the cluster arbiter merges every socket onto the cluster TLB's client port. A fill routes back to exactly one L1 slot with no table lookups anywhere. @@ -516,7 +630,8 @@ held until the next DCR access retires it, muxed ahead of the cluster reply. socket's DCR chain, forwarding it to the cores unchanged while snooping the satp writes. It derives `vm_active` from the mode field (`SATP_LO[31]` on Sv32, `SATP_HI[31:28] ≠ 0` on Sv39) and, on the `SATP_HI` -write, requests a flush of both L1 MMUs, held until both report done. The +write, requests a flush of every L1 MMU, held until all report done; with a +unit MMU the tcache and rtcache invalidation of §3.8 runs as a separate leg. The flush is **self-timed**: satp only changes between kernels, once the TLBs are already drained, so no completion handshake crosses the socket boundary. This is what keeps `satp`, flush, and fault ports off the socket interface — only @@ -548,7 +663,8 @@ DCRs. Then: maps of §2.4. - **`vx_mem_alloc`** takes a PA from the global pool, mints a VA (identity by default; `VORTEX_RANDOMIZE_VA=1` picks a random page-aligned - base seeded by `VORTEX_VA_SEED`, keeping multi-page buffers VA-contiguous) + base seeded by `VORTEX_VA_SEED`, keeping multi-page buffers VA-contiguous + and outside the OM aperture, which the LSU decodes by address) and installs one leaf PTE per page. Re-mapping a VA already covered by a leaf is idempotent when the translation matches (identity superpages installed at init are legitimately re-covered) and a conflict otherwise. @@ -563,14 +679,15 @@ DCRs. Then: ### 8.5 The command processor -The emulation CP ([`cmd_processor.cpp`](../../sim/common/cmd_processor.cpp)) -is MMU-aware: `cp_translate()` performs the same Sv32/Sv39 walk as +Both command processors are MMU-aware: the emulation CP +([`cmd_processor.cpp`](../../sim/common/cmd_processor.cpp), `cp_translate()`) +and the RTL DMA ([`VX_cp_dma.sv`](../../hw/rtl/cp/VX_cp_dma.sv), the +`S_XLATE_*` states) perform the same Sv32/Sv39 walk as `VMManager::page_table_walk` against device memory for every `CMD_MEM_*` VA -operand, skipped by `F_MEM_PHYSICAL`; its root is `CP_SATP_LO/HI` (regfile -0x028/0x02C), programmed at open. The RTL CP has **no MMU**: `CP_SATP` reads -zero, `DEV_CAPS.VM_ENABLED` is 0, and the runtime never mints VAs on FPGA. -VM is therefore a simulation-only feature today -([`command_processor.md`](command_processor.md) §8, §10). +operand, skipped by `F_MEM_PHYSICAL`. Their root is `CP_SATP_LO/HI` (regfile +0x028/0x02C), programmed at open, and both advertise `DEV_CAPS.VM_ENABLED` +in a VM build, so VM runs the same on simulators and FPGA +([`command_processor.md`](command_processor.md) §8). --- @@ -583,10 +700,16 @@ SimObjects bound through channels: `ReqIn/ReqOut` per port, a `Tlb` CAM (`mmu_tlb.cpp`, MRU replacement, superpage splice), an `L1_TLB_MSHR_SIZE`-entry miss station with two parked requests per entry, per-port replay queues that drain ahead of new input, - per-port kill-response queues, and the `TlbMissOut/TlbFillIn` link. Fills - install, then re-check each parked request's own permission and either - replay it translated or kill it. Responses pass through with no added - latency: the stage's cost is charged on the request path. + per-port kill-response queues, a one-entry permission-fault slot per port + (a port stalls its next fault while it is full), and the + `TlbMissOut/TlbFillIn` link. Fills install, then re-check each parked + request's own permission and either replay it translated or kill it. With + writing data-side clients a flush token holds while its port has requests + in the miss station. With a registered reply (the unit MMU) each response + leaves a cycle late; otherwise responses pass through. The DXA's gmem + output carries its own request register under VM, as in the RTL. + The dcache flush needs no MMU wait: SimX flushes the caches from the host + after the run has drained, when every MMU is already empty. - **`L2Tlb`** ([`tlb_l2.cpp`](../../sim/simx/mem/tlb_l2.cpp)) — one request per tick accepted round-robin over clients into a pipe that retires after `VX_CFG_L2_TLB_LATENCY` ticks; megapage `Tlb` probed first, then the @@ -599,7 +722,12 @@ SimObjects bound through channels: walker index in the tag, and the first-fault latch. Wiring ([`cluster.cpp`](../../sim/simx/cluster.cpp)): every core's two `Mmu` -instances are clients of the cluster `L2Tlb`; the `Ptw` shares L2-cache port 0 +instances are clients of the cluster `L2Tlb`, followed by each socket's unit +`Mmu` ([`socket.cpp`](../../sim/simx/socket.cpp)) and the cluster's unit `Mmu` +in front of the rcache / ocache, placed as in the RTL +between the tcache / rtcache / DXA gmem ports and the socket arbiter; `set_satp` +flushes its TLB and invalidates the unit caches (`Cache::invalidate`; the +ocache writes through and holds no write between launches); the `Ptw` shares L2-cache port 0 with socket 0 through a **priority** arbiter that favours PTE fetches — every translated access behind a walk is blocked until it completes, and the walkers issue far too little traffic to starve the demand stream (the reverse @@ -669,6 +797,8 @@ untranslated. | `VX_CFG_VM_ENABLE` | false | build the MMU, TLB fabric, cluster TLB and walker at all | | `VX_CFG_DTLB_SIZE` | 16 | socket dMMU CAM entries | | `VX_CFG_ITLB_SIZE` | 8 | socket iMMU CAM entries | +| `VX_CFG_UTLB_SIZE` | 8 | socket unit MMU CAM entries (TEX, RTU, DXA) | +| `VX_CFG_CTLB_SIZE` | 8 | cluster unit MMU CAM entries (RASTER, OM) | | `VX_CFG_L1_TLB_MSHR_SIZE` | 4 | L1 miss-station entries (distinct outstanding VPNs); 2 tokens each | | `VX_CFG_L2_TLB_SIZE` / `NUM_WAYS` | 512 / 4 | cluster TLB set array | | `VX_CFG_L2_TLB_MEGA_SIZE` | 8 | cluster TLB superpage CAM | @@ -704,6 +834,14 @@ sizes must be powers of two ≥ 2 (static asserts). | `tlb-concurrent`, `mc-l2-amo`, `mc-l3` | 2–4 cores, L2 / L3 | shared cluster TLB and walker across cores and cache levels | | `vm-fault` | `vm_fault -t0` | unmapped page → device error and first-fault report | | `parity-vm-sgemm/vecadd` | `check: model_parity` | exact instret + cycles within tolerance, simx vs rtlsim | +| `vm-rtu` | `rt_smoke`, `EXT_RTU`, randomized VA | rtcache misses through the socket unit MMU | +| `vm-dxa` | `sgemm2_dxa`, `EXT_DXA`, randomized VA | DXA gmem reads through the socket unit MMU | +| `parity-vm-rt_raycast`, `parity-vm-dxa_copy` | `check: model_parity`, RTU + DXA | unit-MMU timing, simx vs rtlsim | +| `vm-tex` | `gfx_tex`, `EXT_TEX`, randomized VA | tcache misses through the socket unit MMU | +| `parity-vm-gfx_tex` | `check: model_parity`, TEX | unit-MMU texture timing, simx vs rtlsim | +| `vm-raster` | `gfx_raster`, `EXT_RASTER` + `EXT_OM`, randomized VA | rcache misses through the cluster unit MMU | +| `vm-om` | `gfx_om -d -b`, `EXT_RASTER` + `EXT_OM`, randomized VA | ocache reads and write-throughs through the cluster unit MMU | +| `parity-vm-gfx_raster`, `parity-vm-gfx_om` | `check: model_parity`, RASTER + OM | cluster-unit-MMU timing, simx vs rtlsim | `vm_fault -t1` (a store to a read-only page) is not in CI: it needs the L1 permission-fault *report*, which the RTL does not surface (§9, §13). @@ -725,6 +863,8 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | area | state | note | |---|:--:|---| | Socket L1 translation stage, lane-parallel, hit-under-miss | ✅ | RTL + SimX | +| Socket unit MMU for TEX, RTU and DXA, VIVT tcache/rtcache invalidated on `satp` | ✅ | RTL + SimX | +| Cluster unit MMU for RASTER and OM, rcache invalidated / ocache drained and invalidated on `satp` | ✅ | RTL + SimX | | VPN-dedup miss stations with per-entry requester queues, ordered drain | ✅ | L1 and cluster | | Narrow miss/fill fabric with id-prepending arbiters | ✅ | socket + cluster levels | | Cluster TLB: pipelined set-associative array + megapage CAM | ✅ | RTL + SimX | @@ -741,6 +881,18 @@ Grades: ✅ done · ⚠️ partial · ❌ pending. | Chip-wide L3 TLB, TLB inclusion policy, a `VX_dma` block with its own TLB | ❌ | not started | | ASIDs, A/D write-back, PMP, Sv48/Sv57 | ❌ | deliberate: single global address space; runtime pre-sets A/D | +**Known limitation.** The `satp` commit's invalidation of the tcache, the +rtcache and the rcache shares each cache's flush leg with the core-driven +graphics flush: the two requests are OR-ed onto one `VX_dcr_flush`, so +whichever is raised second completes on the walk the first one started — a +core flush raised while the `satp` request is high completes on the `satp` +flush's done, and the reverse. Both directions are benign for these read-only +caches: the walk blocks the cache's inputs and invalidates every line, so a +walk that completes after a request was raised leaves no line the request +could still see. The runtime also commits `satp` once, before the first +launch, with no core flush in progress. A writing cache, or a commit that must +complete strictly after a given core flush, would need its own requester. + **Deliberate simplifications** (recorded so they are not mistaken for bugs): there is one global address space and no ASID — `satp` is programmed once per address space, never per dispatch, and the ASID field is ignored diff --git a/docs/fpga_setup.md b/docs/fpga_setup.md index 0419cfd18d..7701c11383 100644 --- a/docs/fpga_setup.md +++ b/docs/fpga_setup.md @@ -148,20 +148,22 @@ sudo bash -c 'source /opt/xilinx/xrt/setup.sh && \ The [blackbox.sh](./simulation.md) script within the build directory can be used to run a test with Vortex’s xrt driver using the following command: -`FPGA_BIN_DIR= TARGET=hw|hw_emu PLATFORM= ./ci/blackbox.sh --driver=xrt --app=` +`FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw|hw_emu --app=` + +The target is a `blackbox.sh` option, not an environment variable: `blackbox.sh` resets `TARGET`, so an exported `TARGET=hw` is ignored and the run falls back to the `xrtsim` simulator. For example: -```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin TARGET=hw PLATFORM=xilinx_u280_gen3x16_xdma_1_202211_1 ./ci/blackbox.sh --driver=xrt --app=demo``` +```FPGA_BIN_DIR= hw/syn/xilinx/xrt/build_4c_u280_xilinx_u280_gen3x16_xdma_1_202211_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo``` Run sgemm on a U55C `hw` bitstream (host memory enabled per the section above), `-n` sets the matrix size: ```bash -FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) TARGET=hw \ - ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n128" +FPGA_BIN_DIR=$(realpath hw/syn/xilinx/xrt//bin) \ + ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n128" ``` -The same command with `TARGET=hw_emu` against a `hw_emu` build runs it in emulation +The same command with `--target=hw_emu` against a `hw_emu` build runs it in emulation (use a small `-n`, e.g. `-n16`, since emulation is slow). Synthesis for Intel (Altera) Boards diff --git a/docs/hardware_library.md b/docs/hardware_library.md index 0ba4a8bdb5..1b81ed2428 100644 --- a/docs/hardware_library.md +++ b/docs/hardware_library.md @@ -2,7 +2,7 @@ ## Overview -The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 76 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. +The Vortex hardware IP library (`hw/rtl/libs/`) provides a comprehensive collection of 77 reusable, parameterized SystemVerilog IP modules used throughout the Vortex GPGPU processor. These modules cover arbitration, buffering, arithmetic, memory, interconnect, and debug infrastructure, forming the foundational building blocks of the Vortex microarchitecture. All modules are parameterized for data width, depth, and implementation strategy, with support for both FPGA (Xilinx/Intel) and ASIC synthesis flows. @@ -27,7 +27,6 @@ All modules are parameterized for data width, depth, and implementation strategy 12. [Synchronization and Timing](#12-synchronization-and-timing) 13. [Allocators and Counters](#13-allocators-and-counters) 14. [Debug and Tracing](#14-debug-and-tracing) -15. [Utility and Placeholder](#15-utility-and-placeholder) --- @@ -1221,16 +1220,15 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn | `OUT_REG` | 0 | Output register enable | | `LUTRAM` | 0 | Force LUTRAM implementation | | `RDW_MODE` | "W" | Read-during-write mode: "W" (write-first), "R" (read-first) | -| `RADDR_REG` | 0 | Read address registered hint | | `INIT_ENABLE` | 0 | Enable initialization | | `INIT_FILE` | "" | Initialization memory file | **Features:** - Separate read and write ports with independent addresses - Bit-level write enable (`WRENW` granularity) -- Write-first or read-first semantics for simultaneous read/write to same address +- Three read forms: registered address, registered data, combinational (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) - FPGA: infers LUTRAM or BRAM based on `LUTRAM` parameter -- ASIC: uses register-based implementation +- ASIC: a registered read of block-RAM size maps to the SRAM macro (`VX_dp_ram_asic`); the registered-address form forwards the lanes written at the captured address around it. Other RAMs are flops - Optional output register for timing improvement - File-based initialization for preloaded content @@ -1255,11 +1253,36 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn **Features:** - Single address shared between read and write - Three read-during-write modes including "N" (no-change) which holds previous read data -- `ASYNC_BRAM_PATCH` support for asynchronous read BRAM workarounds +- Read forms and block-RAM rules as in VX_dp_ram (see [Block RAM and asynchronous reads](#block-ram-and-asynchronous-reads)) - Same FPGA/ASIC implementation paths as VX_dp_ram --- +### VX_1rw1r_ram + +**File:** `VX_1rw1r_ram.sv` + +**Purpose:** RAM with one read/write port and one read-only port, both with a registered read. + +| Parameter | Default | Description | +|-----------|---------|-------------| +| `DATAW` | - | Data width in bits | +| `SIZE` | - | Number of entries | +| `WRENW` | 1 | Write enable granularity | +| `LUTRAM` | 0 | Force LUTRAM implementation | +| `RESET_RAM` | 0 | Clear the contents on reset (simulation) | +| `INIT_ENABLE` | 0 | Enable initialization | +| `INIT_FILE` | "" | Initialization memory file | + +**Features:** +- Port A reads or writes (read-first); port B reads, each with its own address +- A port B read of the word port A writes in the same cycle is undefined; callers must not issue one +- FPGA: the two ports of a true dual-port block RAM, or distributed RAM with `LUTRAM` +- ASIC: two `VX_dp_ram` copies that both take every write, one per read port, since the macro libraries have no read/write + read macro +- Used by the warp-wide local memory's second, read-only pass engine + +--- + ### VX_rom **File:** `VX_rom.sv` @@ -1279,32 +1302,67 @@ Parameterized memory primitives with support for FPGA (LUTRAM/BRAM) and ASIC syn --- -### VX_async_ram_patch - -**File:** `VX_async_ram_patch.sv` -**Purpose:** Asynchronous RAM wrapper with patching support for synthesis tools. - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `DATAW` | 1 | Data width | -| `SIZE` | 1 | Number of entries | -| `WRENW` | 1 | Write enable granularity | -| `DUAL_PORT` | 0 | Enable dual-port operation | -| `FORCE_BRAM` | 0 | Force block RAM inference | -| `RADDR_REG` | 0 | Read address registered | -| `RADDR_RESET` | 0 | Read address has reset | -| `WRITE_FIRST` | 0 | Write-first semantics | -| `INIT_ENABLE` | 0 | Enable initialization | -| `INIT_FILE` | "" | Initialization file | -| `INIT_VALUE` | 0 | Default init value | - -**Features:** -- Wraps synchronous RAM with asynchronous read path emulation -- Uses placeholder modules for tool-specific register/memory patching -- Supports dual-port and single-port configurations -- Handles vendor-specific synthesis quirks (Xilinx BRAM async read workaround) - ---- +### Block RAM and asynchronous reads + +Block RAMs and SRAM macros read only synchronously. `VX_dp_ram` and +`VX_sp_ram` have three read forms: + +| Form | Parameters | `rdata` | Implementation | +|------|------------|---------|----------------| +| Registered address | `OUT_REG=1`, `RDW_MODE="W"` | `ram[raddr captured on read]`; sees a write on the same edge | block RAM (with a collision bypass) or distributed RAM | +| Registered data | `OUT_REG=1`, `RDW_MODE="R"` (`"N"` on `VX_sp_ram` also holds it across a write) | `ram[raddr]` captured on read; does not see a write on the same edge | block RAM or distributed RAM | +| Combinational | `OUT_REG=0` | `ram[raddr]` | distributed RAM or flops, unless the RAM fold turns it into the registered-address form | + +The form fixes the behaviour, identically in simulation and synthesis; size +and `LUTRAM` only pick the implementation (`FORCE_BRAM`), locally, inside the +RAM. + +**The RAM fold.** Write a read in whatever form the design needs; a +combinational read of a `VX_dp_ram` is fine at any size. Every synthesis flow +but Altera OPAE (which does not preprocess) runs `hw/scripts/ram_fold.py` on its preprocessed sources (`gen_sources.sh -P +-F`) before any tool reads them. For each block-RAM-sized `VX_dp_ram` that reads +combinationally and does not ask for `LUTRAM`, it computes the read address one +cycle early and gives the RAM the registered-address form, reading on every +cycle (`read` tied high), which reads exactly what the combinational read did: + +- The pass elaborates the design with slang (CIRCT's front end) and follows the + read address back to the registers that feed it, through assigns, ports, + interfaces, struct fields, functions and generate scopes, in any module. +- Each signal on that path gets a twin, `__nx`, carrying its next value: a + copy of the signal's own driver with its operands renamed. A register's twin + is a combinational copy of its clocked block that keeps only the assignments + to that register; their right-hand sides still read current values. Ports + and interface signals on the path get twin ports and members. +- The edits only add code, except the RAM's parameters and address + connection. The patched sources are elaborated again, and the patch is + dropped if it adds an error. + +`/ram_fold.rpt` lists every candidate RAM, folded or kept with the reason; +`/ram_fold.diff` is the patch. A RAM is kept, and stays distributed RAM, when: + +- its address depends on a top-level input, or on a signal driven by an + `always_comb` block or by a clocked block with blocking assignments; +- an instance on the path is an element of an instance array, or connects its + ports by position; +- another RAM made by the same instantiation statement cannot fold, or + registers its read (`OUT_REG` from a parameter, as in `VX_fifo_queue`): the patch + edits the statement, which all its instances share; +- a register on the path is written by more than one block, inside a + concatenation, or on a clock other than the RAM's; +- the address reaches a signal through a module body slang analyzed only as + another instance and the signal lies outside it, or a signal on the path has + no driver. + +`ram_fold.py --check` adds a simulation check that each folded RAM's next +address is its address a cycle later; `ci/test_ram_fold.py` runs a folded design +in lockstep against the original, with a known-bad patch the check must catch. + +The fold needs a python3 with pyslang 11 or 12 (`pip install "pyslang>=11,<13"`), named by +`RAM_FOLD_PYTHON` (default `python3`, `hw/syn/common.mk`). Vivado reads the +folded preprocessed sources, so its own RAM, FSM and shift-register inference +sees the RTL as written. The CIRCT elaboration of the same sources feeds Yosys, +Beacon and Foldsim; its `.memories.rpt` lists the RAMs that still read +combinationally. ### VX_fifo_queue @@ -1798,36 +1856,12 @@ Infrastructure for runtime debug observation and serial trace extraction. --- -## 15. Utility and Placeholder - -### VX_placeholder - -**File:** `VX_placeholder.sv` -**Purpose:** Empty black-box module for hierarchical design placeholder. - -| Parameter | Default | Description | -|-----------|---------|-------------| -| `I` | 1 | Input width | -| `O` | 1 | Output width | - -| Port | Direction | Width | Description | -|------|-----------|-------|-------------| -| `data_in` | input | I | Input (unconnected) | -| `data_out` | output | O | Output (unconnected) | - -**Features:** -- Marked with `BLACKBOX_CELL` synthesis directive -- Used as a structural placeholder for tool-specific cell replacement -- Zero logic -- purely a synthesis anchor point - ---- - ## Module Summary Table | # | Module | Category | Area | Latency | |---|--------|----------|------|---------| -| 1 | VX_allocator | Allocator | Small | 0 cycles | -| 2 | VX_async_ram_patch | Memory | Varies | 0-1 cycles | +| 1 | VX_1rw1r_ram | Memory | Varies | 1 cycle | +| 2 | VX_allocator | Allocator | Small | 0 cycles | | 3 | VX_avs_adapter | Memory Adapter | Large | Multi-cycle | | 4 | VX_axi_adapter | Memory Adapter | Large | Multi-cycle | | 5 | VX_axi_write_ack | Memory Adapter | Small | 0-1 cycles | @@ -1871,34 +1905,33 @@ Infrastructure for runtime debug observation and serial trace extraction. | 43 | VX_pending_size | Counter | Small | 0-1 cycles | | 44 | VX_pipe_buffer | Buffer | Varies | N cycles | | 45 | VX_pipe_register | Buffer | Varies | N cycles | -| 46 | VX_placeholder | Utility | Zero | 0 cycles | -| 47 | VX_popcount | Arithmetic | Small | O(log N) | -| 48 | VX_priority_arbiter | Arbiter | Small | 0 cycles | -| 49 | VX_priority_encoder | Encoder | Small | O(log N) | -| 50 | VX_reduce_tree | Arithmetic | Small | O(log N) | -| 51 | VX_reset_relay | Sync | Small | 1 cycle | -| 52 | VX_rom | Memory | Varies | 0-1 cycles | -| 53 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | -| 54 | VX_scan | Encoder | Medium | O(log N) | -| 55 | VX_scope_switch | Debug | Small | 0 cycles | -| 56 | VX_scope_tap | Debug | Large | Multi-cycle | -| 57 | VX_serial_div | Arithmetic | Small | Multi-cycle | -| 58 | VX_serial_mul | Arithmetic | Small | Multi-cycle | -| 59 | VX_shift_register | Buffer | Varies | N cycles | -| 60 | VX_skid_buffer | Buffer | Small | 0-1 cycles | -| 61 | VX_sp_ram | Memory | Varies | 0-1 cycles | -| 62 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | -| 63 | VX_stream_buffer | Buffer | Small | 0-1 cycles | -| 64 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | -| 65 | VX_stream_fork | Stream Util | Small | 0 cycles | -| 66 | VX_stream_join | Stream Util | Small | 0 cycles | -| 67 | VX_stream_omega | Interconnect | Large | O(log N) | -| 68 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | -| 69 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | -| 70 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | -| 71 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | -| 72 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | -| 73 | VX_ticket_lock | Allocator | Small | 0-1 cycles | -| 74 | VX_toggle_buffer | Buffer | Small | 1 cycle | -| 75 | VX_transpose | Bit Manip | Zero | 0 cycles | -| 76 | VX_wallace_mul | Arithmetic | Large | O(log N) | +| 46 | VX_popcount | Arithmetic | Small | O(log N) | +| 47 | VX_priority_arbiter | Arbiter | Small | 0 cycles | +| 48 | VX_priority_encoder | Encoder | Small | O(log N) | +| 49 | VX_reduce_tree | Arithmetic | Small | O(log N) | +| 50 | VX_reset_relay | Sync | Small | 1 cycle | +| 51 | VX_rom | Memory | Varies | 0-1 cycles | +| 52 | VX_rr_arbiter | Arbiter | Medium | 0 cycles | +| 53 | VX_scan | Encoder | Medium | O(log N) | +| 54 | VX_scope_switch | Debug | Small | 0 cycles | +| 55 | VX_scope_tap | Debug | Large | Multi-cycle | +| 56 | VX_serial_div | Arithmetic | Small | Multi-cycle | +| 57 | VX_serial_mul | Arithmetic | Small | Multi-cycle | +| 58 | VX_shift_register | Buffer | Varies | N cycles | +| 59 | VX_skid_buffer | Buffer | Small | 0-1 cycles | +| 60 | VX_sp_ram | Memory | Varies | 0-1 cycles | +| 61 | VX_stream_arb | Interconnect | Medium | 0-1 cycles | +| 62 | VX_stream_buffer | Buffer | Small | 0-1 cycles | +| 63 | VX_stream_dispatch | Stream Util | Medium | 0-1 cycles | +| 64 | VX_stream_fork | Stream Util | Small | 0 cycles | +| 65 | VX_stream_join | Stream Util | Small | 0 cycles | +| 66 | VX_stream_omega | Interconnect | Large | O(log N) | +| 67 | VX_stream_pack | Stream Util | Medium | 0-1 cycles | +| 68 | VX_stream_switch | Interconnect | Medium | 0-1 cycles | +| 69 | VX_stream_unpack | Stream Util | Medium | Multi-cycle | +| 70 | VX_stream_xbar | Interconnect | Large | 0-1 cycles | +| 71 | VX_stream_xpoint | Interconnect | Medium | 0-1 cycles | +| 72 | VX_ticket_lock | Allocator | Small | 0-1 cycles | +| 73 | VX_toggle_buffer | Buffer | Small | 1 cycle | +| 74 | VX_transpose | Bit Manip | Zero | 0 cycles | +| 75 | VX_wallace_mul | Arithmetic | Large | O(log N) | diff --git a/docs/index.md b/docs/index.md index 82631b4f45..9e4b858360 100644 --- a/docs/index.md +++ b/docs/index.md @@ -4,7 +4,7 @@ - [Environment Setup](environment_setup.md): system prerequisites and toolchain setup - [Installing Vortex](install_vortex.md): building and installing the Vortex stack -- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, and gem5 +- [Building the Toolchain from Source](building_toolchain.md): maintainer-facing build recipes for Verilator, RISC-V GNU, LLVM (with X86 + lld + SPIR-V), compiler-rt, musl, POCL, Mesa, chipStar, gem5, CIRCT, and Foldsim - [FPGA Setup](fpga_setup.md): running Vortex on Altera and Xilinx FPGA cards - [SLASH / Alveo V80 Setup](xilinx_slash_setup.md): building the SLASH platform stack, the prerequisite for the `aved` backend @@ -47,6 +47,7 @@ Detailed architectural specifications for Vortex subsystems live under - [Custom Accelerator ISA Extensions](designs/custom_accelerator_isa_extensions.md) - **Memory system** - [Cache Subsystem](designs/cache_subsystem.md) + - [Memory-Level Parallelism Optimizations](designs/mlp_optimizations.md) - [Atomic Memory Operations](designs/atomic_memory_operations.md) - [Multi-Cache AMO Coherence](designs/multicache_amo_coherence.md) - [Memory Fabric Attributes](designs/memory_fabric_attributes.md) @@ -74,4 +75,5 @@ Detailed architectural specifications for Vortex subsystems live under - [SimX Simulator Architecture](designs/simx_simulator_architecture.md) - [gem5 Integration](designs/vortex_gem5_integration.md) - [FireSim Integration](designs/firesim_integration.md) + - [Foldsim: threaded FPGA simulation on CIRCT](designs/foldsim.md) - [Continuous Integration Design](designs/continuous_integration.md) diff --git a/docs/simulation.md b/docs/simulation.md index 8fbc55f94d..aef657e962 100644 --- a/docs/simulation.md +++ b/docs/simulation.md @@ -28,12 +28,51 @@ running orders of magnitude faster than Verilator. For scale, `sgemm -n32` takes ~1228 ms on rtlsim and ~8 ms on an Alveo U55C, and `sgemm -n1024` — over a billion instructions — completes in ~85 s. +FireSim is driven exactly like the card's native flow; only the driver name +differs. It takes the same targets as the card's native driver: + +| card | targets | image built by | +|---|---|---| +| XRT (e.g. U55C) | `xrtsim` (default), `hw_emu`, `hw` | `make -C hw/syn/firesim PLATFORM= TARGET=hw_emu\|hw` | +| aved (V80) | `avedsim` (default), `sim`, `hw` | `make -C hw/syn/firesim TARGET=sim\|hw` | + +`xrtsim` and `avedsim` run the FireSim-transformed design in Verilator inside +the runtime and need no image or Vivado. The image builds take `PREFIX`, +`XLEN`, `CONFIGS` and `PERF=1` as `hw/syn/xilinx/{xrt,aved}` do, and leave the +image in `__/bin` together with `firesim.card`, which +tells the run which card it is for. From the build directory: + ```bash -export TARGET=hw -export FPGA_BIN_DIR= -./ci/blackbox.sh --driver=firesim --app=sgemm --args="-n1024" +source ci/firesim_env.sh # after the card's Xilinx tools; see the script + +# no image needed +./ci/blackbox.sh --driver=firesim --target=avedsim --app=demo + +# build an image, then run on it +CONFIGS="-DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4" make -C hw/syn/firesim TARGET=hw PERF=1 +FPGA_BIN_DIR=$PWD/hw/syn/firesim/build32_aved_hw/bin \ + ./ci/blackbox.sh --driver=firesim --target=hw --warps=4 --threads=4 --app=demo ``` +On the V80, `TARGET=hw` runs `ci/v80_preflight.sh` first: it refuses to +program a card that is off the bus, unhealthy or still in use, and waits out a +cool-down since the last programming. A first `Ctrl-C` (or `SIGTERM`) stops a +FireSim run cleanly — the launch in flight is reported and completed, and the +application tears down normally — and a second one kills it. A stopped run +never reads as a pass: FireSim reports it `FAILED` and the process exits 130 +(143 for `SIGTERM`, 124 when `VORTEX_FIRESIM_TIMEOUT_S` gave up). + +Debugging knobs, all read from the environment by the runtime: + +| variable | effect | +|---|---| +| `VORTEX_FIRESIM_TIMEOUT_S` | give up on a launch after this many seconds, with a report | +| `VORTEX_FIRESIM_STALL_CYCLES` | report (once) after this many busy target cycles; default 500000 | +| `VORTEX_FIRESIM_REPORT_ON_IDLE` | also report the counters when a launch completes | +| `VORTEX_FIRESIM_TRACE` | time-stamp every request to the simulator thread | +| `VORTEX_FIRESIM_EMUL_ARGS` | extra Verilator arguments for `xrtsim`/`avedsim`, e.g. `+verilator+rand+reset+2` | +| `FIRESIM_MM_REORDER`, `FIRESIM_MM_LATENCY` | `xrtsim`/`avedsim` host memory: reorder (1), also interleave (2) and delay reads | + The architecture, build flow and design invariants are documented in [designs/firesim_integration.md](designs/firesim_integration.md); the toolchain component is [section 12 of the toolchain @@ -44,6 +83,48 @@ One caveat worth knowing when reading its output: FireSim's kernel took, and is quantized by the driver's polling schedule. Use Vortex's own `PERF: cycles=` line, which is exact. +### Threaded FPGA Simulation (Foldsim) + +Foldsim compiles Vortex into a simulator for the V80 with CIRCT: each class of +identical units (sockets, issue slices, ALUs, ...) is threaded through one +physical copy, so a GPU far larger than the card fits on it, still cycle-exact +against the RTL. Its kernel carries a small control block and a +latency-bandwidth memory model; the target runs free, and the runtime talks to +it like the native V80 driver does (AXI4-Lite registers, QDMA for memory). + +| target | what runs | built by | +|---|---|---| +| `avedsim` (default) | the same kernel in Verilator, in-process | the run itself: the Foldsim compiler, then a Verilator build of the kernel, for the app's `CONFIGS` | +| `hw` | the image on the V80 | `make -C hw/syn/foldsim` | + +```bash +# the classes to thread, and the units to give plain-port boundaries first +export FOLDSIM_CLASSES="VX_socket__lift VX_core__lift VX_compute_slice__lift" +export FOLDSIM_LIFT="VX_socket:drop=INSTANCE_ID VX_core:drop=INSTANCE_ID VX_compute_slice:drop=INSTANCE_ID,SLICE_STR" +CONFIGS="-DVX_CFG_NUM_CORES=2 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_THREADS=4 -DVX_CFG_ISSUE_WIDTH=2" + +# in Verilator, no image needed +CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --app=sgemm + +# build the image, then run on the V80 +make -C hw/syn/foldsim CONFIGS="$CONFIGS" +FPGA_BIN_DIR=$PWD/hw/syn/foldsim/build32_foldsim_hw/bin \ + CONFIGS="$CONFIGS" ./ci/blackbox.sh --driver=foldsim --target=hw --app=sgemm +``` + +The flow needs CIRCT (`circt-verilog`, `circt-opt`, `firtool`) and Foldsim +(the `foldsim` driver and `foldsim-opt`) from the toolchain +(`./ci/toolchain_install.sh --circt --foldsim`; `CIRCT_BIN=

` and +`FOLDSIM=` override them), and pyslang for the boundary lifting +(`LIFT_PYTHON`). The memory is timed as the configuration's DRAM +(`VX_CFG_PLATFORM_MEMORY_*`), a channel per memory bank, as `dram_sim` times +it; `VORTEX_DRAM_REFRESH=bank|all` adds refresh. +`VORTEX_FOLDSIM_TIMEOUT_S=` fails a launch that runs longer; a command, +DCR read or reset the target leaves unanswered fails too once the target has +advanced no cycle for that long, or for 60 s when it is unset. At exit the runtime reports the target and kernel-clock cycles the simulator ran +and how busy each threaded class was. The design is +[designs/foldsim.md](designs/foldsim.md). + ### How to Test (using `blackbox.sh`) Running tests under specific drivers (rtlsim,simx,fpga) is done using the script named `blackbox.sh` located in the `ci` folder. Running command `./ci/blackbox.sh --help` from the Vortex root directory will display the following command line arguments for `blackbox.sh`: @@ -54,7 +135,7 @@ Running tests under specific drivers (rtlsim,simx,fpga) is done using the script - *Threads* - used to specify the number of threads (smallest unit of computation) within a configuration. - *L2cache* - used to enable the shared l2cache among the Vortex cores. - *L3cache* - used to enable the shared l3cache among the Vortex clusters. -- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim). +- *Driver* - used to specify which driver to run the Vortex simulation (either rtlsim, opae, xrt, simx, firesim, foldsim). - *Debug* - used to enable debug mode for the Vortex simulation. - *Perf* - used to enable the detailed performance counters within the Vortex simulation. - *App* - used to specify which test/benchmark to run in the Vortex simulation. The main choices are vecadd, sgemm, basic, demo, and dogfood. Other tests/benchmarks are located in the `/benchmarks/opencl` folder though not all of them work wit the current version of Vortex. @@ -150,11 +231,25 @@ machine from it: SimX, rtlsim, the FPGA and ASIC flows, and the kernels. | Profile | Machine | |---|---| | `tinygpu` | 1 core, 2 warps x 2 threads, shared memory | -| `rtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 2 MB L2, GDDR5X as 16 GDDR6 channels | -| `v100` | Tesla V100 (Volta): 80 cores, 4 MB L2, 32 HBM2 channels at 1752 MT/s | -| `a100` | A100 (Ampere): 108 cores, 32 MB L2, HBM2 at the card's 1555 GB/s | -| `h100` | H100 SXM5 (Hopper): 132 cores, 32 MB L2, HBM3 at the card's 3.35 TB/s | -| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 MB L2, GDDR7 bandwidth on the GDDR6 model | +| `gtx1080` | GeForce GTX 1080 (Pascal): 20 cores x 64 warps x 32 threads, 96 KB shared memory (64 KB modelled), 2 MB L2, GDDR5X as 16 GDDR6 channels; no tensor cores | +| `gtx1060` | GeForce GTX 1060 6 GB (Pascal), on `gtx1080`: 10 cores, 1 MB L2, 192 GB/s as 8 GDDR6 channels | +| `rtx3060` | GeForce RTX 3060 12 GB (Ampere GA106): 28 cores x 48 warps, 64 KB shared memory, 2 MB L2, GDDR6 at 360 GB/s; FP16/TF32/INT8/INT4 tensor cores with sparsity; no RTU (it needs full-warp ALU lanes, GA106 has 16 INT32 per block) | +| `rx6600` | Radeon RX 6600 (RDNA 2): 28 CUs x 40 waves x 32 lanes, 64 KB LDS, 2 MB L2, 32 MB Infinity Cache as the L3, GDDR6 at 224 GB/s; ray accelerators as RTU | +| `rx9070xt` | Radeon RX 9070 XT (RDNA 4): 64 CUs x 32 waves x 32 lanes, 64 KB LDS, 8 MB L2, 64 MB Infinity Cache as the L3, GDDR6 at 640 GB/s; FP16/FP8/INT8/INT4 matrix units with sparsity, ray accelerators as RTU | +| `v100` | Tesla V100 (Volta): 80 cores x 64 warps, 64 KB shared memory, 4 MB L2, 32 HBM2 channels at 1752 MT/s; FP16 tensor cores | +| `a100` | A100 (Ampere): 108 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM2 at 1555 GB/s; FP16/TF32/INT8/INT4 tensor cores with 2:4 sparsity | +| `h100` | H100 SXM5 (Hopper): 132 cores x 64 warps, 128 KB shared memory, 32 MB L2, HBM3 at 3.35 TB/s; FP16/TF32/FP8/INT8 tensor cores with sparsity and wgmma, TMA as DXA | +| `rtx5080` | GeForce RTX 5080 (Blackwell): 84 cores x 48 warps, 64 KB shared memory, 64 MB L2, GDDR7 bandwidth on the GDDR6 model; FP16/TF32/FP8/FP4/MX/INT8 tensor cores with sparsity, TMA as DXA, RT cores as RTU | + +Every GPU profile also turns on virtual memory, the A extension with Zacas, +Zfhmin, MUFU at the part's special-function throughput, the part's INT32, +FP32 and load/store lanes per processing block (`NUM_ALU_LANES`, +`NUM_FPU_LANES`, `NUM_LSU_LANES`; a warp instruction takes `NUM_THREADS` / +lanes cycles, as on the part), and shared memory at the part's 128 B per clock +(16 banks: a Vortex bank is XLEN wide). The real parts are 64-bit machines, so these profiles set `"xlen": 64`: +`configure --profile=` picks XLEN from the profile (an explicit, +conflicting `--xlen` is an error), and a `build32` tree rejects them. A profile +without an `xlen` field, like `tinygpu`, builds at either width. Select one per build with `../configure --profile=`, or per run: diff --git a/docs/synthesis_analysis.md b/docs/synthesis_analysis.md index b2bb361da5..1cbd13a011 100644 --- a/docs/synthesis_analysis.md +++ b/docs/synthesis_analysis.md @@ -2,6 +2,8 @@ This guide explains how to run synthesis, timing analysis, area analysis, and power analysis for Vortex across all supported back-ends: Xilinx (Vivado), Altera (Quartus), Yosys (open-source), and Synopsys Design Compiler. +**Timing work starts in Beacon, not here.** For an RTL change that can move timing, iterate on [Beacon](designs/beacon.md) (§6.3): it counts the logic levels of every register in minutes, compares them with the parent revision's (`ci/logic_gate.py -b --against `), and names the registers a change made deeper. Synthesize once the change passes there, to confirm Fmax and to measure what Beacon does not see: wires, placement, congestion, area and power. + --- ## Table of Contents @@ -62,6 +64,23 @@ NUM_CORES=16 # equivalent to -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -DVX NUM_CORES=32 # equivalent to -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -DVX_CFG_L2_ENABLE ``` +### Block RAM for asynchronous reads (the RAM fold) + +Every flow reads preprocessed copies of the sources (`gen_sources.sh -P`) that +the RAM fold has patched first (`-F`, [hw/scripts/ram_fold.py](../hw/scripts/ram_fold.py)): +a block-RAM-sized `VX_dp_ram` that reads combinationally gets its read address one +cycle early and a registered read, so it maps to block RAM or an SRAM macro. The +build's source folder holds `ram_fold.rpt` (each candidate RAM, folded or kept +with the reason) and `ram_fold.diff` (the patch). The fold needs a python3 with +pyslang 11 or 12, named by `RAM_FOLD_PYTHON` (default `python3`): + +```bash +pip install "pyslang>=11,<13" +RAM_FOLD_PYTHON=/path/to/python3 make core # when python3 has no pyslang +``` + +See [Block RAM and asynchronous reads](hardware_library.md#block-ram-and-asynchronous-reads). + ### Using PREFIX for Isolated Builds Use `PREFIX=` to keep builds separate. Each flow creates a build directory derived from `PREFIX` so that multiple configurations can coexist without overwriting each other: @@ -434,7 +453,7 @@ Output reports are written to `build/hw/syn/yosys/tfr_fedp_VX_tcu_fedp_tfr/repor Build directory: `_/` (e.g., `test_Vortex/`). -The flow uses `sv2v` to convert SystemVerilog sources to Verilog before feeding them to Yosys. +The flow elaborates the SystemVerilog sources with CIRCT ([hw/scripts/circt_elaborate.sh](../hw/scripts/circt_elaborate.sh) `-f v2005`) into one Verilog-2005 file for Yosys: packed arrays and structs become plain vectors, the SRAM macros (`BB_MODULES`) stay instances with their parameter values, bound to their stubs, and a RAM that asks for block RAM reads synchronously, so that it maps to an SRAM macro. #### Per-DUT synthesis @@ -451,7 +470,7 @@ make -C build/hw/syn/yosys/dut om CLOCK_FREQ=500 # override the target clock #### Synthesis-regression gate (`asic_gate`) -`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails on a Fmax or cell-area move beyond ±5%. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). +`ci/asic_gate.py` runs that catalog against checked-in goldens in `ci/baselines/synthesis/yosys/` and fails when Fmax falls more than 2% below the build's target clock or cell area grows more than 5% over its `area_budget_um2`; both limits live in the catalog. It is the ASIC sibling of `ci/fpga_gate.py` and shares its implementation (`ci/synth_gate.py`). ```bash ci/asic_gate.py --list # builds and their recorded baselines diff --git a/docs/xilinx_slash_setup.md b/docs/xilinx_slash_setup.md index 03d02918a8..ebbcca6102 100644 --- a/docs/xilinx_slash_setup.md +++ b/docs/xilinx_slash_setup.md @@ -97,12 +97,15 @@ Artifacts land in `deb/`. ```bash cd deb -sudo apt install --no-install-recommends \ - $(ls -1 *.deb | grep -v '^ami_' | sed 's|^|./|') +sudo apt install --no-install-recommends $(ls -1 *.deb | sed 's|^|./|') ``` -`ami` is built but deliberately left out: nothing in the `slash` dependency -chain needs it, and it is the component that heartbeats the AMC over GCQ. +**Install `ami` too.** Nothing in the `slash` dependency chain pulls it in, but +the card is not fully usable without it: `ami.ko` binds PF0, and the AMC behind +PF0 is how the static shell is written to flash (§5) and how the board is +recovered. Skipping it leaves PF0 unbound at every boot and the board reporting +`NOT READY`. Its package registers the module with DKMS, so — like `slash` — it +rebuilds on kernel updates and loads automatically. If a previous **source** install exists, remove its leftovers first or they win: files under `/etc/systemd/system/` override the packaged units, and @@ -146,8 +149,18 @@ systemctl is-active vrtd v80-smi list ``` -`v80-smi list` reports **PF0 NOT READY** when `ami` is absent. That is a -readiness *report*, not a requirement: PF1, PF2 and VRTD are what matter. +All three functions should bind: PF0 to `ami`, PF1 to `slash_qdma`, PF2 to +`slash_ctl`. + +```bash +lsmod | grep -E '^ami|^slash' +for f in 0 1 2; do basename "$(readlink /sys/bus/pci/devices/0000:01:00.$f/driver)"; done +``` + +`v80-smi list` reporting **PF0 NOT READY** means `ami` is missing. Running +kernels does work in that state — PF1, PF2 and VRTD carry the data and control +paths — but flashing the shell and recovering the board do not, so treat it as +an incomplete install rather than a cosmetic warning. --- @@ -290,6 +303,69 @@ from the static shell in flash. On a board that has never had it programmed, PF0-only is expected — complete §5 first. `slash.ko` binds *only* PF1/PF2, so until they appear the driver has nothing to attach to. +**The whole host resets when the card is touched.** The machine reboots with +no panic and no log; the next boot prints a fatal ACPI **BERT** record naming a +processor cache error (`Check Information 0x602001f`), often on a different +core each time. It is not a CPU fault. + +``` +x86/amd: Previous system reset reason [0x80000800]: a software sync flood event occurred +GHES: APEI firmware first mode is enabled by APEI bit +``` + +The chain is: a driver reads the card's BAR while the link is down -> that read +is an unsupported request or completion timeout, an uncorrectable PCIe error -> +firmware owns error handling (APEI firmware-first) and answers with a **sync +flood**, resetting the host before Linux can log anything. Firmware then +records it in the only vocabulary BERT gives it, which is why it masquerades as +a CPU defect. + +The reader is usually AMI's logging thread: `dump_amc_log()` reads every 500 ms +and, before the fix, did so unconditionally -- so it covered every bus reset, +every reconfiguration and every wedged card. Symptoms line up with a card event +0-80 s earlier, and the giveaway in the log is: + +``` +ami 0000:01:00.0: ERROR : Logging thread error - invalid PCI data read! +eqdma_cpm5_hw_error_process: Global Err Reg(0x248) = 0xffffffff +``` + +Three things fix it, and all three are worth doing: + +1. **Patched drivers.** `dump_amc_log()` must return when the service is halted + or `pci_device_is_present()` is false, and `ami` must implement + `reset_prepare`/`reset_done` so a reset quiesces its threads. The reset path + must use `pci_reset_bus()`, not a bare `pci_bridge_secondary_bus_reset()`, + or those callbacks are never delivered. +2. **BIOS: `NBIO SyncFlood Generation` = Disabled** (Advanced -> AMD CBS -> + NBIO Common Options), with `Enable AER Cap` and EDPC/DPC enabled. This stops + *any* unanswered access from resetting the host, and is the only way to see + the real PCIe error -- with firmware-first the evidence is destroyed by the + reset. +3. **Both flash partitions written (§5).** `Shell: unknown` makes the daemon + reset the board on every image load: one boot logged 115 of them. Each is a + window for the failure above. + +**No V80 on the bus at all, and no root port.** `lspci -d 10ee:` prints +nothing and `ls /sys/bus/pci/devices/0000:00:01.1` does not exist. AMD firmware +only instantiates a root port whose link is up at POST, so a card that was +wedged — mid-program, or by a host crash — disappears entirely and the bus +numbers shift down. **No rescan can bring it back**: there is no parent bridge +to scan behind, and `v80-smi --jtag` cannot help either, because it resolves the +device through VRTD, which only knows cards that are on PCIe. + +Recover over JTAG, which needs no root and reads no flash: + +```bash +bash hw/syn/xilinx/aved/tools/jtag_load_shell.sh # loads the _nofpt PDI +sudo reboot # POST now sees a trained link +``` + +Check `0000:00:01.1` before attempting anything rescan-based. Unload `slash` +and `ami` first if they are loaded — a link transition under a live driver is +what turns this into a host reset. A card whose **both** flash partitions hold +a good shell (§5) boots on its own and does not need this. + **BARs unassigned / `enable=0` / config space reads `0xff`.** The Versal can finish booting from OSPI *after* the host has enumerated PCIe, leaving the kernel's view frozen from a failed enumeration. Symptom in `dmesg`: diff --git a/hw/rtl/VX_cluster.sv b/hw/rtl/VX_cluster.sv index 86fc979410..caf860a54c 100644 --- a/hw/rtl/VX_cluster.sv +++ b/hw/rtl/VX_cluster.sv @@ -15,7 +15,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CLUSTER_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -24,6 +23,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -37,6 +38,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // KMU bus VX_kmu_bus_if.slave kmu_bus_if[1], + // Global barrier bus to the device-level barrier unit + VX_gbar_bus_if.master gbar_bus_if, + `ifdef VX_CFG_EXT_RASTER_ENABLE // Delegated draw launch (device KMU → raster engines) VX_raster_launch_if.slave raster_launch_if[1], @@ -50,16 +54,29 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if.slave dev_ptw_mem_if, input wire mmu_flush_req, output wire mmu_flush_done, +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Translation active vs BARE (satp.MODE), for the cluster unit MMU. + input wire mmu_vm_active, +`endif `endif + // Command-processor DMA, entering as an L2 client when this cluster's L2 + // is the LLC (CP_ON_L2); tied off otherwise. + VX_mem_bus_if.slave cp_mem_if, + // Status output wire busy ); `ifdef SCOPE localparam scope_socket = 0; +`ifdef EXT_GFX_CLUSTER_ENABLE + localparam scope_gfx = NUM_SOCKETS; + `SCOPE_IO_SWITCH (NUM_SOCKETS + 1); +`else `SCOPE_IO_SWITCH (NUM_SOCKETS); `endif +`endif `ifdef PERF_ENABLE cache_perf_t l2_perf; @@ -132,7 +149,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_dist ( .clk (clk), @@ -144,7 +161,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (NUM_SOCKETS), - .DEST_LSB (KMU_DEST_LSB_CLUSTER), + .DEST_DIV (KMU_DEST_DIV_CLUSTER), .OUT_BUF ((NUM_SOCKETS > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), @@ -155,7 +172,6 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif VX_gbar_bus_if per_socket_gbar_bus_if[NUM_SOCKETS](); - VX_gbar_bus_if gbar_bus_if(); VX_gbar_arb #( .NUM_REQS (NUM_SOCKETS), @@ -168,19 +184,11 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (gbar_bus_if) ); - VX_gbar_unit #( - .INSTANCE_ID (`SFORMATF(("gbar%0d", CLUSTER_ID))) - ) gbar_unit ( - .clk (clk), - .reset (reset), - .gbar_bus_if (gbar_bus_if) - ); - // L2 input buses (socket ports + cluster-resident gfx cache ports) VX_mem_bus_if #( .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH) - ) per_socket_mem_bus_if[L2_NUM_REQS](); + ) per_socket_mem_bus_if[L2_CACHE_REQS](); // Socket memory output buses (socket arb output tag width) VX_mem_bus_if #( @@ -206,6 +214,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L2_MEM_TAG_WIDTH) ) l2_mem_bus_if[L2_MEM_PORTS](); + wire l2_req_empty; VX_cache_wrap #( .INSTANCE_ID (`SFORMATF(("%s-l2cache", INSTANCE_ID))), @@ -215,7 +224,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_BANKS (L2_NUM_BANKS), .NUM_WAYS (`VX_CFG_L2_NUM_WAYS), .WORD_SIZE (L2_WORD_SIZE), - .NUM_REQS (L2_NUM_REQS), + .NUM_REQS (L2_CACHE_REQS), .MEM_PORTS (L2_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L2_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L2_MSHR_SIZE), @@ -226,12 +235,16 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L2), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L2_ENABLED), .IS_LLC (L2_IS_LLC), + .SHARED_LLC (L2_SHARED_LLC), + .BANK_HASH (`VX_CFG_L2_BANK_HASH), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l2cache ( .clk (clk), @@ -240,7 +253,8 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (l2_perf), `endif .core_bus_if (per_socket_mem_bus_if), - .mem_bus_if (l2_mem_bus_if) + .mem_bus_if (l2_mem_bus_if), + .req_empty (l2_req_empty) ); // Cluster DCR distribution — declared before its consumers (sockets/gfx). @@ -286,14 +300,20 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], l2_mem_bus_if[i]); end + if (CP_ON_L2 != 0) begin : g_cp_l2_client + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_CP_IDX], cp_mem_if); + end else begin : g_no_cp_l2_client + `UNUSED_VX_MEM_BUS_IF (cp_mem_if) + end + /////////////////////////////////////////////////////////////////////////// wire [NUM_SOCKETS-1:0] per_socket_busy; `ifdef EXT_GFX_ANY_ENABLE // OR all socket flush reqs together; broadcast .done to every socket. - // Sockets fire their reqs sequentially; the shared resource flushes - // once per req (subsequent reqs see an already-empty cache). + // The cores start their flushes together, so the read-only graphics caches + // flush once for all of them; the OR drops when the last core has finished. VX_dcr_flush_if cluster_flush_if(); VX_dcr_flush_if per_socket_cluster_flush_if [NUM_SOCKETS](); wire [NUM_SOCKETS-1:0] per_socket_cluster_flush_req; @@ -308,14 +328,15 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // --------------------------------------------------------------------- // Shared cluster TLB + page-table walker // --------------------------------------------------------------------- + // Every socket's tlb_bus, then the cluster unit MMU's (when present). VX_tlb_bus_if #(.ID_WIDTH (TLB_SOCKET_ID_WIDTH)) - per_socket_tlb_bus_if [NUM_SOCKETS] (); + per_socket_tlb_bus_if [CLUSTER_NUM_TLB_PORTS] (); VX_tlb_bus_if #(.ID_WIDTH (TLB_CLUSTER_ID_WIDTH)) l2_client_if (); VX_tlb_bus_if #(.ID_WIDTH (L2_TLB_SLOT_WIDTH)) l2_ptw_if (); VX_tlb_bus_arb #( - .NUM_INPUTS (NUM_SOCKETS), + .NUM_INPUTS (CLUSTER_NUM_TLB_PORTS), .ID_WIDTH_IN (TLB_SOCKET_ID_WIDTH), .OUT_BUF (3) ) tlb_cluster_arb ( @@ -358,9 +379,21 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // TLB flush off the SATP DCR write, and the device walker reports its // own done leg at the top. assign l2_flush_if.req = mmu_flush_req; +`ifdef CLUSTER_UNIT_MMU_ENABLE + // The rcache and ocache are virtually tagged: the address-space change + // drains the ocache's writes and invalidates both, then flushes the + // cluster unit MMU's TLB that those writes went through. + VX_dcr_flush_if gfx_mmu_flush_if(); + VX_tlb_flush_if cmmu_flush_if(); + wire cmmu_empty; + assign gfx_mmu_flush_if.req = mmu_flush_req; + assign cmmu_flush_if.req = mmu_flush_req && gfx_mmu_flush_if.done; + assign mmu_flush_done = l2_flush_if.done && cmmu_flush_if.done; + wire mmu_busy = ~(l2_empty && cmmu_empty); +`else assign mmu_flush_done = l2_flush_if.done; - wire mmu_busy = ~l2_empty; +`endif `else wire mmu_busy = 1'b0; `endif @@ -368,13 +401,13 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar socket_id = 0; socket_id < NUM_SOCKETS; ++socket_id) begin : g_sockets VX_socket #( - .SOCKET_ID ((CLUSTER_ID * NUM_SOCKETS) + socket_id), .INSTANCE_ID (`SFORMATF(("%s-socket%0d", INSTANCE_ID, socket_id))) ) socket ( `SCOPE_IO_BIND (scope_socket+socket_id) .clk (clk), .reset (reset), + .socket_id (CORE_ID_WIDTH'(cluster_id) * CORE_ID_WIDTH'(NUM_SOCKETS) + CORE_ID_WIDTH'(socket_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -421,10 +454,12 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; assign per_socket_dcr_bus_if[DCR_GFX_IDX].rsp_data = gfx_dcr_bus_if.rsp_data; VX_graphics #( - .CLUSTER_ID (CLUSTER_ID) + .INSTANCE_ID (INSTANCE_ID) ) graphics ( + `SCOPE_IO_BIND (scope_gfx) .clk (clk), .reset (reset), + .cluster_id (cluster_id), `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_perf (gfx_raster_perf), @@ -447,9 +482,95 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif .dcr_bus_if (gfx_dcr_bus_if), .cluster_flush_if (cluster_flush_if), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_flush_if (gfx_mmu_flush_if), + `endif .busy (gfx_busy) ); +`ifdef CLUSTER_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Cluster unit MMU — translates the rcache and ocache memory ports on + // their way to the L2, so those caches are virtually indexed and tagged. + // Its misses go straight to the cluster TLB as one more client. + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_mem_bus_if[CMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_phys_bus_if[CMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_RASTER_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_RCACHE_IDX], rcache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_RASTER_IDX], gfx_phys_bus_if[CMMU_RCACHE_IDX]); +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_OCACHE_IDX], ocache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_OM_IDX], gfx_phys_bus_if[CMMU_OCACHE_IDX]); +`endif + + // Its miss bus enters the cluster TLB arbiter after the sockets' buses, + // its slot id zero-extended to their width. + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) cmmu_tlb_bus_if(); + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_valid = cmmu_tlb_bus_if.req_valid; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.id = `UP(TLB_SOCKET_ID_WIDTH)'(cmmu_tlb_bus_if.req_data.id); + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.access = cmmu_tlb_bus_if.req_data.access; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.amo = cmmu_tlb_bus_if.req_data.amo; + assign per_socket_tlb_bus_if[NUM_SOCKETS].req_data.vpn = cmmu_tlb_bus_if.req_data.vpn; + assign cmmu_tlb_bus_if.req_ready = per_socket_tlb_bus_if[NUM_SOCKETS].req_ready; + assign cmmu_tlb_bus_if.rsp_valid = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_valid; + assign cmmu_tlb_bus_if.rsp_data.id = `UP(L1_TLB_ID_WIDTH)'(per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.id); + assign cmmu_tlb_bus_if.rsp_data.fault = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.fault; + assign cmmu_tlb_bus_if.rsp_data.level = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.level; + assign cmmu_tlb_bus_if.rsp_data.ppn = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.ppn; + assign cmmu_tlb_bus_if.rsp_data.flags = per_socket_tlb_bus_if[NUM_SOCKETS].rsp_data.flags; + assign per_socket_tlb_bus_if[NUM_SOCKETS].rsp_ready = cmmu_tlb_bus_if.rsp_ready; + + VX_mmu_fault_if cmmu_fault_if(); + // Faults are reported by the walker, as for the socket MMUs. + `UNUSED_VAR (cmmu_fault_if.valid) + `UNUSED_VAR (cmmu_fault_if.va) + `UNUSED_VAR (cmmu_fault_if.access) + `UNUSED_VAR (cmmu_fault_if.amo) + +`ifdef PERF_ENABLE + mmu_perf_t cmmu_perf; + `UNUSED_VAR (cmmu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("cmmu"), + .NUM_REQS (CMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_CTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (`VX_CFG_EXT_OM_ENABLED), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) cmmu ( + .clk (clk), + .reset (reset), + .vm_active (mmu_vm_active), + `ifdef PERF_ENABLE + .mmu_perf (cmmu_perf), + `endif + .core_bus_if (gfx_mem_bus_if), + .mem_bus_if (gfx_phys_bus_if), + .tlb_bus_if (cmmu_tlb_bus_if), + .flush_if (cmmu_flush_if), + .fault_if (cmmu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (cmmu_empty) + ); +`else `ifdef VX_CFG_EXT_RASTER_ENABLE `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_RASTER_IDX], rcache_l2_bus_if); `endif @@ -457,6 +578,7 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_OM_ENABLE `ASSIGN_VX_MEM_BUS_IF (per_socket_mem_bus_if[L2_GFX_OM_IDX], ocache_l2_bus_if); `endif +`endif // CLUSTER_UNIT_MMU_ENABLE `elsif EXT_GFX_ANY_ENABLE // No cluster-resident graphics caches: the socket-local flush legs @@ -470,10 +592,15 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; // combinationally -- the beat presented at the cluster input and any beat resident // in a per-socket output skid -- so a launch in transit is never invisible on the // presented cycle. Child (per_socket_busy) stays registered. + // A socket's busy has no path from its inputs: the launch and DCR beats this + // level presents to the sockets join the registered aggregation here. wire [NUM_SOCKETS-1:0] per_socket_kmu_valid; + wire [NUM_SOCKETS-1:0] per_socket_dcr_valid; for (genvar i = 0; i < NUM_SOCKETS; ++i) begin : g_kmu_link_valid assign per_socket_kmu_valid[i] = per_socket_kmu_bus_if[i].valid; + assign per_socket_dcr_valid[i] = per_socket_dcr_bus_if[i].req_valid; end + wire socket_issued_busy = (| per_socket_kmu_valid) | (| per_socket_dcr_valid); `ifdef VX_CFG_EXT_RASTER_ENABLE // The merge->distribute link is a registered cluster-internal launch bus; a beat // resident in it is covered here just like the input and per-socket links. @@ -484,9 +611,9 @@ module VX_cluster import VX_gpu_pkg::*, VX_tlb_pkg::*; wire busy_r; `ifdef EXT_GFX_ANY_ENABLE - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | gfx_busy | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | socket_issued_busy | gfx_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `else - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | mmu_busy, 1'b1, 1, (NUM_SOCKETS > 1)); + `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_socket_busy) | socket_issued_busy | mmu_busy | ~l2_req_empty, 1'b1, 1, (NUM_SOCKETS > 1)); `endif assign busy = busy_r | dcr_bus_if.req_valid | launch_link_busy; diff --git a/hw/rtl/VX_define.vh b/hw/rtl/VX_define.vh index 7607b2bdf6..d3ff4a3849 100644 --- a/hw/rtl/VX_define.vh +++ b/hw/rtl/VX_define.vh @@ -83,6 +83,25 @@ `define EXT_GFX_CLUSTER_ENABLE `endif +// Under VM, the cluster unit MMU translates the rcache and ocache memory ports. +`ifdef VX_CFG_VM_ENABLE +`ifdef EXT_GFX_CLUSTER_ENABLE + `define CLUSTER_UNIT_MMU_ENABLE +`endif +`endif + +// Under VM, the socket unit MMU translates the memory ports of the +// socket-resident units: the tcache and rtcache misses and the DXA gmem reads. +`ifdef VX_CFG_VM_ENABLE +`ifdef VX_CFG_EXT_TEX_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`elsif VX_CFG_EXT_RTU_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`elsif VX_CFG_EXT_DXA_ENABLE + `define SOCKET_UNIT_MMU_ENABLE +`endif +`endif + // Early-Z occlusion cull requires BOTH the rasterizer (produces the covered-quad // waves + depth plane) and the OM (owns the ocache the depth read is coherent // with). It is illegal without them — reading committed depth needs the ocache. diff --git a/hw/rtl/VX_gpu_pkg.sv b/hw/rtl/VX_gpu_pkg.sv index 88a1b908a4..a9f7ab8c1e 100644 --- a/hw/rtl/VX_gpu_pkg.sv +++ b/hw/rtl/VX_gpu_pkg.sv @@ -30,6 +30,11 @@ package VX_gpu_pkg; localparam NT_WIDTH = `UP(NT_BITS); localparam NB_WIDTH = `UP(NB_BITS); + // core index across all clusters + localparam CORE_ID_BITS = `CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES); + localparam CORE_ID_WIDTH = `UP(CORE_ID_BITS); + localparam CLUSTER_ID_WIDTH = `LOG2UP(`VX_CFG_NUM_CLUSTERS); + localparam NUM_CTA_MAX = `VX_CFG_NUM_WARPS; localparam NCTA_BITS = `CLOG2(NUM_CTA_MAX); localparam NCTA_WIDTH = `UP(NCTA_BITS); @@ -83,7 +88,7 @@ package VX_gpu_pkg; localparam BAR_ADDR_W = `UP(BAR_ADDR_BITS); localparam BAR_ID_SHIFT = 8; - localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, NC_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); + localparam BAR_SIZE_W = `MAX(`MAX(NW_WIDTH, CORE_ID_WIDTH), `CLOG2(`VX_CFG_MAX_BAR_EVENTS)); localparam UOP_PACKLD = 0; localparam UOP_TCU = UOP_PACKLD + 1; @@ -93,7 +98,9 @@ package VX_gpu_pkg; // owning one: the two are combinational rewrites of ibuf_in with mutually // exclusive selects (EX_LSU + export_mask vs EX_SFU + INST_SFU_RTUW). localparam UOP_GFX = UOP_TCU + `VX_CFG_EXT_TCU_ENABLED; - localparam UOP_MAX = UOP_GFX + `EXT_GFX_ANY_ENABLED; + // atomics with .aq/.rl expand into fences around the atomic + localparam UOP_AMO = UOP_GFX + `EXT_GFX_ANY_ENABLED; + localparam UOP_MAX = UOP_AMO + `VX_CFG_EXT_A_ENABLED; localparam UOP_CTR_W = 8; localparam CTA_TID_WIDTH = `UP(NW_BITS + NT_BITS); @@ -150,8 +157,9 @@ package VX_gpu_pkg; // table keys on hart_id = make_hart_id(cid, wid, tid). // MINU/MAXU collapse into MIN/MAX + amo_unsigned. - // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. - localparam HART_ID_BITS = NC_BITS + NW_BITS + NT_BITS; + // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid; cid is + // device-wide, so harts of different clusters stay distinct at a shared LLC. + localparam HART_ID_BITS = CORE_ID_BITS + NW_BITS + NT_BITS; localparam HART_ID_WIDTH = `UP(HART_ID_BITS); // ─────────────────────────────────────────────────────────────────────── @@ -204,8 +212,11 @@ package VX_gpu_pkg; // is_flush at LSB so dcr_flush can drive it deterministically. // is_addr_om is appended ABOVE amo, deliberately: the amo field is cast by // offset (VX_cache_bank), so inserting a bit below it would silently shift - // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. + // MEM_ATTR_AMO_OFFS and reinterpret the AMO sideband. is_noalloc sits above + // amo for the same reason: a write that misses a write-around cache goes to + // memory instead of allocating. typedef struct packed { + logic is_noalloc; // MEM_ATTR_NOALLOC_OFFS amo_req_t amo; // MEM_ATTR_AMO_OFFS = 4 logic is_addr_om; // MEM_ATTR_OM_OFFS = 3 logic is_addr_local; // MEM_ATTR_LOCAL_OFFS = 2 @@ -218,12 +229,41 @@ package VX_gpu_pkg; localparam MEM_ATTR_LOCAL_OFFS = 2; localparam MEM_ATTR_OM_OFFS = 3; localparam MEM_ATTR_AMO_OFFS = 4; + localparam MEM_ATTR_NOALLOC_OFFS = MEM_ATTR_AMO_OFFS + AMO_REQ_BITS; // Total width of the mem-bus attr field. Use this as the parameter // default for VX_mem_bus_if's ATTR_WIDTH parameter and as the // wire/reg sizing for any per-cache plumbing. localparam MEM_ATTR_WIDTH = $bits(mem_bus_attr_t); + // Cache bank-select hash: XORed into the raw bank bits, a function of the + // line address above them, so XORing it again restores the raw bits. IPOLY + // for 16 and 32 banks (Rau, ISCA 1991: conflict-free for power-of-two + // strides); an XOR fold of the bits above the bank field otherwise. + function automatic logic [7:0] cache_bank_hash(input logic [31:0] hi, input integer bits); + logic [7:0] f; + f = '0; + if (bits == 4) begin + f[0] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[6] ^ hi[4] ^ hi[3] ^ hi[0]; + f[1] = hi[12] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[1] ^ hi[0]; + f[2] = hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[2] ^ hi[1]; + f[3] = hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[5] ^ hi[3] ^ hi[2]; + end else if (bits == 5) begin + f[0] = hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[6] ^ hi[5] ^ hi[3] ^ hi[0]; + f[1] = hi[14] ^ hi[13] ^ hi[12] ^ hi[11] ^ hi[10] ^ hi[7] ^ hi[6] ^ hi[4] ^ hi[1]; + f[2] = hi[14] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[6] ^ hi[3] ^ hi[2] ^ hi[0]; + f[3] = hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[7] ^ hi[4] ^ hi[3] ^ hi[1]; + f[4] = hi[12] ^ hi[11] ^ hi[10] ^ hi[9] ^ hi[8] ^ hi[5] ^ hi[4] ^ hi[2]; + end else if (bits > 0 && bits <= 8) begin + for (integer i = 0; i < 15; ++i) begin + if ((i % bits) == 0) begin + f = f ^ (8'(hi >> i) & 8'((1 << bits) - 1)); + end + end + end + return f; + endfunction + localparam VX_DCR_ADDR_WIDTH = `VX_DCR_ADDR_BITS; localparam VX_DCR_DATA_WIDTH = `VX_DCR_DATA_BITS; @@ -455,6 +495,7 @@ package VX_gpu_pkg; localparam INST_LSU_SH = 4'b1001; localparam INST_LSU_SW = 4'b1010; localparam INST_LSU_SD = 4'b1011; // new for RV64I SD + localparam INST_LSU_FENCE_IO = 4'b1110; // a fence ordering only device (IO) accesses localparam INST_LSU_FENCE = 4'b1111; localparam INST_LSU_BITS = 4; @@ -471,7 +512,11 @@ package VX_gpu_pkg; endfunction function automatic logic inst_lsu_is_fence(input logic [INST_LSU_BITS-1:0] op); - return (op == INST_LSU_FENCE); + return (op[3:1] == 3'b111); + endfunction + + function automatic logic inst_lsu_is_fence_io(input logic [INST_LSU_BITS-1:0] op); + return (op == INST_LSU_FENCE_IO); endfunction /////////////////////////////////////////////////////////////////////////// @@ -575,7 +620,7 @@ package VX_gpu_pkg; localparam DISPATCH_QSIZE = `VX_CFG_DISPATCH_QUEUE_SIZE; - localparam PER_OPC_WARPS = PER_ISSUE_WARPS / `VX_CFG_NUM_OPCS; + localparam PER_OPC_WARPS = `CDIV(PER_ISSUE_WARPS, `VX_CFG_NUM_OPCS); localparam PER_OPC_NW_BITS = `CLOG2(PER_OPC_WARPS); localparam PER_OPC_NW_W = `UP(PER_OPC_NW_BITS); @@ -612,6 +657,111 @@ package VX_gpu_pkg; end endfunction + // Frontend slice f schedules and fetches the warps with wid mod FETCH_WIDTH == f, + // naming them by their local id wfs = wid / FETCH_WIDTH. + localparam FETCH_FSW_BITS = `CLOG2(`VX_CFG_FETCH_WIDTH); + localparam FETCH_FSW_W = `UP(FETCH_FSW_BITS); + localparam PER_FETCH_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_FETCH_WIDTH; + localparam FETCH_WFS_BITS = `CLOG2(PER_FETCH_WARPS); + localparam FETCH_WFS_W = `UP(FETCH_WFS_BITS); + + function automatic logic [NW_WIDTH-1:0] wfs_to_wid( + input logic [FETCH_WFS_W-1:0] wfs, + input logic [FETCH_FSW_W-1:0] fsw + ); + if (FETCH_WFS_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(fsw); + end else if (FETCH_FSW_BITS == 0) begin + wfs_to_wid = NW_WIDTH'(wfs); + end else begin + wfs_to_wid = NW_WIDTH'({wfs, fsw}); + end + endfunction + + function automatic logic [FETCH_FSW_W-1:0] wid_to_fsw( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_FSW_BITS != 0) begin + wid_to_fsw = wid[FETCH_FSW_W-1:0]; + end else begin + wid_to_fsw = 0; + end + endfunction + + // FU block ports per frontend slice: a unit with B blocks serves warps with + // wid mod B == b, so B >= FETCH_WIDTH gives each frontend slice B / FETCH_WIDTH + // blocks and B < FETCH_WIDTH shares each block among FETCH_WIDTH / B slices. + localparam FETCH_ALU_PORTS = (`VX_CFG_NUM_ALU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_ALU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + localparam FETCH_SFU_PORTS = (`VX_CFG_NUM_SFU_BLOCKS > `VX_CFG_FETCH_WIDTH) ? (`VX_CFG_NUM_SFU_BLOCKS / `VX_CFG_FETCH_WIDTH) : 1; + + function automatic logic [FETCH_WFS_W-1:0] wid_to_wfs( + input logic [NW_WIDTH-1:0] wid + ); + if (FETCH_WFS_BITS != 0) begin + wid_to_wfs = FETCH_WFS_W'(wid >> FETCH_FSW_BITS); + end else begin + wid_to_wfs = 0; + end + endfunction + + // A unit with B blocks serves, in block b, exactly the warps with wid mod B == b + // (lane dispatch maps issue slot {batch, block} and isw = wid mod ISSUE_WIDTH), + // so a block's warp state is indexed by the remaining high wid bits. + localparam SFU_BLOCK_BITS = `CLOG2(`VX_CFG_NUM_SFU_BLOCKS); + localparam SFU_BLOCK_W = `UP(SFU_BLOCK_BITS); + localparam SFU_BLOCK_WARPS = `VX_CFG_NUM_WARPS / `VX_CFG_NUM_SFU_BLOCKS; + localparam SFU_BLOCK_NW_BITS = `CLOG2(SFU_BLOCK_WARPS); + localparam SFU_BLOCK_NW_W = `UP(SFU_BLOCK_NW_BITS); + + function automatic logic [SFU_BLOCK_W-1:0] wid_to_sfu_block( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_BITS != 0) begin + wid_to_sfu_block = wid[SFU_BLOCK_W-1:0]; + end else begin + wid_to_sfu_block = 0; + end + endfunction + + function automatic logic [SFU_BLOCK_NW_W-1:0] wid_to_sfu_local( + input logic [NW_WIDTH-1:0] wid + ); + if (SFU_BLOCK_NW_BITS != 0) begin + wid_to_sfu_local = SFU_BLOCK_NW_W'(wid >> SFU_BLOCK_BITS); + end else begin + wid_to_sfu_local = 0; + end + endfunction + + function automatic logic [NW_WIDTH-1:0] sfu_local_to_wid( + input logic [SFU_BLOCK_NW_W-1:0] lid, + input logic [SFU_BLOCK_W-1:0] block + ); + if (SFU_BLOCK_NW_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(block); + end else if (SFU_BLOCK_BITS == 0) begin + sfu_local_to_wid = NW_WIDTH'(lid); + end else begin + sfu_local_to_wid = NW_WIDTH'({lid, block}); + end + endfunction + + // The core structure follows from the widths. When every execution unit has + // one block per issue slice, the back end is ISSUE_WIDTH backend slices + // (VX_backend_slice: issue, execute, commit); otherwise it is stage-sliced. + // When the fetch width matches too, each backend slice and its frontend slice + // form one compute slice (VX_compute_slice), with no frontend crossbar. + localparam BACKEND_SLICES = (`VX_CFG_NUM_ALU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_LSU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (`VX_CFG_NUM_SFU_BLOCKS == `VX_CFG_ISSUE_WIDTH) + && (!`VX_CFG_EXT_F_ENABLED || (`VX_CFG_NUM_FPU_BLOCKS == `VX_CFG_ISSUE_WIDTH)) + && (!`VX_CFG_EXT_TCU_ENABLED || (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH)); + localparam COMPUTE_SLICES = BACKEND_SLICES && (`VX_CFG_FETCH_WIDTH == `VX_CFG_ISSUE_WIDTH); + // A compute slice whose warp-control requests are arbitrated with other + // slices' retires its own warps when its exit request is granted. With a + // sole requester, the core retires them from its registered request. + localparam SLICE_RETIRE = COMPUTE_SLICES && (`VX_CFG_NUM_SFU_BLOCKS > 1); + /////////////////////////////// TENSOR UNIT /////////////////////////////// `ifdef VX_CFG_EXT_TCU_ENABLE @@ -665,10 +815,26 @@ package VX_gpu_pkg; logic [BAR_SIZE_W-1:0] size_m1; } barrier_t; + // Core-wide warp-control requests (barrier, wspawn, warp exit), one per + // SFU block, served by VX_warp_shared_unit. + localparam WSH_BAR = 2'd0; + localparam WSH_WSPAWN = 2'd1; + localparam WSH_EXIT = 2'd2; + + typedef struct packed { + logic [1:0] kind; + logic has_rsp; // the requester waits for its completion + logic [NW_WIDTH-1:0] wid; + barrier_t bar; + logic [BAR_ADDR_W-1:0] bar_addr; + wspawn_t wspawn; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; // spawning warp's, inherited by the spawned warps + } warp_shared_req_t; + typedef struct packed { logic [NB_WIDTH-1:0] id; - logic [NC_WIDTH-1:0] size_m1; - logic [NC_WIDTH-1:0] core_id; + logic [CORE_ID_WIDTH-1:0] size_m1; + logic [CORE_ID_WIDTH-1:0] core_id; } gbar_req_t; typedef struct packed { @@ -800,16 +966,17 @@ package VX_gpu_pkg; // Placement hint: the global index of the core a fragment launch must land // on. Fragment work is bin->core affine (that is what makes same-pixel blend // order safe), so a fragment message cannot be load-balanced like a compute - // CTA. Each fan-out level consumes its own slice of this index, LSB first: - // core-in-socket, then socket-in-cluster, then cluster (see KMU_DEST_LSB_*). + // CTA. Each fan-out level picks its output as (dest / DIV) % NUM_OUTPUTS, + // where DIV is the number of cores below each of its outputs: core-in-socket, + // then socket-in-cluster, then cluster (see KMU_DEST_DIV_*). // Device-wide, so the hint can name any core: VX_CFG_NUM_CORES is per-cluster. // `UP` so a single-core build still has a 1-bit (unused) hint rather than a // zero-width vector. localparam KMU_DEST_W = `UP(`CLOG2(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_CORES)); - localparam KMU_DEST_LSB_SOCKET = 0; - localparam KMU_DEST_LSB_CLUSTER = `CLOG2(`VX_CFG_SOCKET_SIZE); - localparam KMU_DEST_LSB_DEVICE = KMU_DEST_LSB_CLUSTER + `CLOG2(NUM_SOCKETS); + localparam KMU_DEST_DIV_SOCKET = 1; + localparam KMU_DEST_DIV_CLUSTER = `VX_CFG_SOCKET_SIZE; + localparam KMU_DEST_DIV_DEVICE = KMU_DEST_DIV_CLUSTER * NUM_SOCKETS; typedef struct packed { logic [NCTA_WIDTH-1:0] cta_id; @@ -872,13 +1039,82 @@ package VX_gpu_pkg; cta_lane_t [`VX_CFG_NUM_THREADS-1:0] lane_launch; } cta_warp_t; + // CTA dispatcher state broadcast to the CTA-CSR stores that serve CSR reads: + // the per-CTA context, shared by every launch lane. + typedef struct packed { + logic ctx_write; + logic [NCTA_WIDTH-1:0] ctx_waddr; + cta_ctx_t ctx_wdata; + } cta_bcast_t; + + // One launch lane's per-warp state: the warp's rank/lane record and CTA id. + typedef struct packed { + logic warp_write; + logic [NW_WIDTH-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [NW_WIDTH-1:0] id_wid; + logic [NCTA_WIDTH-1:0] id_cta; + } cta_lane_bcast_t; + + // Scheduler traffic between an FU block and the frontend slice owning the warp, + // addressed by core-wide wid. + + // ALU branch resolution + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic taken; + logic [PC_BITS-1:0] dest; + logic is_trap; + logic is_mret; + logic [3:0] trap_cause; + } sched_branch_t; + + // SFU warp control + typedef struct packed { + logic tmc_valid; + logic split_valid; + logic sjoin_valid; + logic wsync_valid; + logic [NW_WIDTH-1:0] wid; + tmc_t tmc; + split_t split; + join_t sjoin; + } sched_wctl_t; + + // SFU CSR writes to per-warp state + typedef struct packed { + logic valid; + logic [NW_WIDTH-1:0] wid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] data; + logic [NUM_TRAP_CSRS-1:0] trap_valid; + logic [`VX_CFG_XLEN-1:0] trap_data; + } sched_csr_wr_t; + + // SFU CSR reads of per-warp state, selected by the read wid and CTA id + typedef struct packed { + logic [`VX_CFG_NUM_THREADS-1:0] thread_mask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; + logic [`VX_CFG_XLEN-1:0] mstatus; + logic [`VX_CFG_XLEN-1:0] mtvec; + logic [`VX_CFG_XLEN-1:0] mepc; + logic [`VX_CFG_XLEN-1:0] mcause; + logic [`VX_CFG_XLEN-1:0] mtval; + logic [NW_WIDTH-1:0] cta_rank; + cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; + } sched_csr_rd_t; + //////////////////////// instruction arguments //////////////////////////// // tcu_args_t at 27 bits = 1+1+1+2+5+5+4+4+4. Sparse variants are distinct opcodes. localparam INST_ARGS_BITS = 3 + ALU_TYPE_BITS + 20 + 2; + // imm_hi: imm20 is bits [31:12] of the immediate (LUI, AUIPC). + // imm_x2: imm20 counts halfwords (JAL, conditional branches). typedef struct packed { - logic [1:0] __padding; + logic imm_hi; + logic imm_x2; logic use_PC; logic use_imm; logic is_w; @@ -889,7 +1125,8 @@ package VX_gpu_pkg; // Branch instructions (JAL/JALR/B/SYS) execute on the ALU pipeline. typedef struct packed { - logic [1:0] __padding; + logic imm_hi; + logic imm_x2; logic use_PC; logic use_imm; logic is_rvc; @@ -908,8 +1145,11 @@ package VX_gpu_pkg; // lsu_args_t carries AMO sideband when EXT_A_ENABLE. // amo_valid / amo_op are inspected only when amo_valid==1; for plain - // loads/stores they're zero. aq/rl are decoded but unused (sequentially - // consistent). amo_unsigned distinguishes the U-variants of MIN/MAX. + // loads/stores they're zero. aq/rl order the AMO with the hart's other + // accesses (fence uops around it); on a memory fence they say what it + // orders: aq = later reads see others' writes (acquire: invalidate), + // rl = earlier writes are visible (release: drain). amo_unsigned + // distinguishes the U-variants of MIN/MAX. // export_mask marks a vx_om_export (OM fragment export) and says which // words of the aperture record the shader is writing: // bit 0 = colour (record + 0), bit 1 = depth (record + 4) @@ -1415,7 +1655,7 @@ package VX_gpu_pkg; // Width of the tag carried over VX_lsu_sched_if: lsu_header + op_type // + per-lane align + packet allocator addr + fence flag (matches the - // local TAG_WIDTH built inside VX_lsu_slice). + // local TAG_WIDTH built inside VX_lsu_block). localparam LSU_CLIENT_TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (`VX_CFG_NUM_LSU_LANES * `CLOG2(LSU_WORD_SIZE)) + `LOG2UP(`VX_CFG_LSU_PENDING_SIZE) + 1; @@ -1476,8 +1716,12 @@ package VX_gpu_pkg; // Block size in bytes localparam ICACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; - // Core request tag Id bits - localparam ICACHE_TAG_ID_BITS = NW_WIDTH; + // Request ports per core: one per frontend slice, each fetching its own warps + localparam ICACHE_NUM_REQS = `VX_CFG_FETCH_WIDTH; + localparam ICACHE_NUM_BANKS = `VX_CFG_ICACHE_NUM_BANKS; + + // Core request tag Id bits: the warp id local to the requesting frontend slice + localparam ICACHE_TAG_ID_BITS = FETCH_WFS_W; // Core request tag bits as VX_fetch writes them (UUID + warp id). localparam ICACHE_FETCH_TAG_WIDTH = (UUID_WIDTH + ICACHE_TAG_ID_BITS); @@ -1498,9 +1742,9 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_ICACHE_ENABLE - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, 1, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_MEM_TAG_WIDTH(`VX_CFG_ICACHE_MSHR_SIZE, ICACHE_NUM_BANKS, 1, `VX_CFG_NUM_ICACHES, UUID_WIDTH); `else - localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(1, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); + localparam ICACHE_MEM_TAG_WIDTH = `CACHE_CLUSTER_BYPASS_MEM_TAG_WIDTH(ICACHE_NUM_REQS, 1, ICACHE_LINE_SIZE, ICACHE_WORD_SIZE, ICACHE_TAG_WIDTH, `VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_ICACHES); `endif ////////////////////////// Dcache Parameters ////////////////////////////// @@ -1533,7 +1777,7 @@ package VX_gpu_pkg; localparam DCACHE_CORE_TAG_WIDTH = (UUID_WIDTH + DCACHE_TAG_ID_BITS); // Core request tag bits on dcache_bus_if port of VX_core (+1 for flush-arb sel bit) - // Pre-MMU width (what VX_mem_unit produces, including the flush sel bit). + // Pre-MMU width (an LSU block's D-cache port, including the flush sel bit). // When VM_ENABLE is on, the dMMU widens this by DCACHE_TLB_SOURCE_BITS // for its internal bypass/TLB/PTW arbiter, so the cache cluster sees // the wider DCACHE_TAG_WIDTH below. @@ -1557,13 +1801,24 @@ package VX_gpu_pkg; // L1 dcache is the LLC iff neither L2 nor L3 is enabled. localparam DCACHE_IS_LLC = !`VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // It is the point of coherence between kernels only as the one dcache, + // shared by every core of the one socket, with no other memory client + // beside it (the texture, ray-tracing and DXA ports join below it, the + // raster and OM caches beside it). + localparam DCACHE_SHARED_LLC = DCACHE_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_NUM_CORES == `VX_CFG_SOCKET_SIZE) && (`VX_CFG_NUM_DCACHES == 1) + && !`VX_CFG_EXT_TEX_ENABLED && !`VX_CFG_EXT_RTU_ENABLED + && !`VX_CFG_EXT_RASTER_ENABLED && !`VX_CFG_EXT_OM_ENABLED + && !`VX_CFG_EXT_DXA_ENABLED; ////////////////////////// Tex / Tcache Parameters //////////////////////// `ifdef VX_CFG_EXT_TEX_ENABLE // tex unit: per-request queue + per-request tag localparam TEX_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(`VX_CFG_TEX_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's texture bus + localparam TEX_CORE_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket TEX units - localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); + localparam TEX_REQ_ARB1_TAG_WIDTH = (TEX_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_TEX_CORES)); localparam TCACHE_WORD_SIZE = 4; localparam TCACHE_ADDR_WIDTH = (`VX_CFG_MEM_ADDR_WIDTH - `CLOG2(TCACHE_WORD_SIZE)); @@ -1581,9 +1836,8 @@ package VX_gpu_pkg; localparam TCACHE_TAG_ID_BITS = (`CLOG2(`VX_CFG_TEX_MEM_QUEUE_SIZE) + TCACHE_BATCH_SEL_BITS); // Core-side (tex_core's cache_bus_if) tag width. localparam TCACHE_TAG_WIDTH = (UUID_WIDTH + TCACHE_TAG_ID_BITS); - // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush so a - // host-issued CACHE_FLUSH DCR-read can inject a synthetic flush request - // onto port 0 without aliasing real requests. + // Cache-side tag width = core-side + 1 bit reserved by VX_dcr_flush, so + // a flush token can be injected without aliasing real requests. localparam TCACHE_BUS_TAG_WIDTH = (TCACHE_TAG_WIDTH + 1); localparam TCACHE_MEM_DATA_WIDTH = (TCACHE_LINE_SIZE * 8); localparam TCACHE_MEM_PORTS = 1; @@ -1686,8 +1940,10 @@ package VX_gpu_pkg; `ifdef VX_CFG_EXT_RTU_ENABLE localparam RTU_REQ_QUEUE_SIZE = 4; localparam RTU_REQ_TAG_WIDTH = (UUID_WIDTH + `CLOG2(RTU_REQ_QUEUE_SIZE)); + // core-level arb: SFU blocks -> the core's RTU bus + localparam RTU_CORE_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_SFU_BLOCKS, 1)); // Single socket-level arb: socket cores -> per-socket RTU units - localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_REQ_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); + localparam RTU_REQ_ARB1_TAG_WIDTH = (RTU_CORE_TAG_WIDTH + `ARB_SEL_BITS(`VX_CFG_SOCKET_SIZE, `VX_CFG_NUM_RTU_CORES)); // The RTU fetches a whole CW-BVH node (one 64 B cache line) per request, // so its cache is line-granular: word size == line size, one port/core. @@ -1695,7 +1951,8 @@ package VX_gpu_pkg; localparam RTCACHE_ADDR_WIDTH = (`VX_CFG_MEM_ADDR_WIDTH - `CLOG2(RTCACHE_WORD_SIZE)); localparam RTCACHE_LINE_SIZE = `VX_CFG_L1_LINE_SIZE; localparam RTCACHE_NUM_REQS = 1; - localparam RTCACHE_TAG_ID_BITS = `CLOG2(`VX_CFG_RTCACHE_MSHR_SIZE); + // The core-side tag carries the RTU context index. + localparam RTCACHE_TAG_ID_BITS = `CLOG2(`VX_CFG_RTU_NUM_CTX); localparam RTCACHE_TAG_WIDTH = (UUID_WIDTH + RTCACHE_TAG_ID_BITS); localparam RTCACHE_BUS_TAG_WIDTH = (RTCACHE_TAG_WIDTH + 1); localparam RTCACHE_MEM_DATA_WIDTH= (RTCACHE_LINE_SIZE * 8); @@ -1717,6 +1974,12 @@ package VX_gpu_pkg; localparam DXA_MEM_ARB_IDX = RTCACHE_MEM_ARB_IDX + `VX_CFG_EXT_RTU_ENABLED; localparam SOCKET_MEM_ARB_REQS = 2 + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; + // Socket unit MMU ports (SOCKET_UNIT_MMU_ENABLE), in socket arb order. + localparam UMMU_TCACHE_IDX = 0; + localparam UMMU_RTCACHE_IDX = UMMU_TCACHE_IDX + `VX_CFG_EXT_TEX_ENABLED; + localparam UMMU_DXA_IDX = UMMU_RTCACHE_IDX + `VX_CFG_EXT_RTU_ENABLED; + localparam UMMU_NUM_REQS = `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_DXA_ENABLED; + // Widest tag among the socket arb inputs (DXA adapts to the given budget) `ifdef VX_CFG_EXT_TEX_ENABLE localparam SOCKET_MEM_TAG_W1 = `MAX(L1_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH); @@ -1746,6 +2009,11 @@ package VX_gpu_pkg; localparam L2_GFX_RASTER_IDX = L2_SOCKET_REQS; localparam L2_GFX_OM_IDX = L2_GFX_RASTER_IDX + `VX_CFG_EXT_RASTER_ENABLED; + // Cluster unit MMU ports (CLUSTER_UNIT_MMU_ENABLE), in L2 client order. + localparam CMMU_RCACHE_IDX = 0; + localparam CMMU_OCACHE_IDX = CMMU_RCACHE_IDX + `VX_CFG_EXT_RASTER_ENABLED; + localparam CMMU_NUM_REQS = `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_OM_ENABLED; + // With one cluster, an L2 and no L3, that L2 is the LLC: the device // walker's PTE fetches ride it through a dedicated client slot, so a // leaf-PTE miss is a cache lookup rather than a DRAM round trip. In @@ -1758,6 +2026,19 @@ package VX_gpu_pkg; localparam L2_PTW_IDX = L2_SOCKET_REQS + L2_GFX_REQS; localparam L2_NUM_REQS = L2_SOCKET_REQS + L2_GFX_REQS + L2_PTW_REQS; + + // The command processor's DMA enters the shared LLC on one more client + // slot (VX_CFG_CP_ON_LLC): the cluster L2 when it is the LLC of a single + // cluster, the L3 otherwise. The slot does not size the banks, so + // L2_NUM_REQS / L3_NUM_REQS exclude it and *_CACHE_REQS count it. + localparam CP_ON_L2 = ((`VX_CFG_CP_ON_LLC != 0) + && (`VX_CFG_NUM_CLUSTERS == 1) + && (`VX_CFG_L2_ENABLED != 0) + && (`VX_CFG_L3_ENABLED == 0)) ? 1 : 0; + localparam CP_ON_L3 = ((`VX_CFG_CP_ON_LLC != 0) && (`VX_CFG_L3_ENABLED != 0)) ? 1 : 0; + localparam L2_CP_IDX = L2_NUM_REQS; + localparam L2_CACHE_REQS = L2_NUM_REQS + CP_ON_L2; + localparam L2_NUM_BANKS = `VX_CFG_L2_NUM_BANKS; localparam L2_MEM_PORTS = `VX_CFG_L2_MEM_PORTS; @@ -1767,10 +2048,21 @@ package VX_gpu_pkg; // TLB miss/fill fabric id widths. The id starts as the L1 miss-station slot // and each arb level prepends its grant index so fills route back exactly. localparam L1_TLB_ID_WIDTH = `CLOG2(`VX_CFG_L1_TLB_MSHR_SIZE); - // A socket exposes two L1 MMUs (dtlb + itlb), regardless of SOCKET_SIZE, each - // driving one bus into the socket TLB arbiter. - localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(2, 1); - localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(NUM_SOCKETS, 1); + // A socket exposes two L1 MMUs (dtlb + itlb), regardless of SOCKET_SIZE, plus + // the unit MMU when present, each driving one bus into the socket TLB arbiter. +`ifdef SOCKET_UNIT_MMU_ENABLE + localparam SOCKET_NUM_MMUS = 3; +`else + localparam SOCKET_NUM_MMUS = 2; +`endif + localparam TLB_SOCKET_ID_WIDTH = L1_TLB_ID_WIDTH + `ARB_SEL_BITS(SOCKET_NUM_MMUS, 1); + // The cluster TLB's clients: every socket, plus the cluster unit MMU. +`ifdef CLUSTER_UNIT_MMU_ENABLE + localparam CLUSTER_NUM_TLB_PORTS = NUM_SOCKETS + 1; +`else + localparam CLUSTER_NUM_TLB_PORTS = NUM_SOCKETS; +`endif + localparam TLB_CLUSTER_ID_WIDTH = TLB_SOCKET_ID_WIDTH + `ARB_SEL_BITS(CLUSTER_NUM_TLB_PORTS, 1); localparam L2_TLB_SLOT_WIDTH = `CLOG2(`VX_CFG_L2_TLB_MSHR_SIZE); // Device-level walker: cluster L2-TLB miss buses arb into one walker. localparam TLB_DEV_ID_WIDTH = L2_TLB_SLOT_WIDTH + `ARB_SEL_BITS(`VX_CFG_NUM_CLUSTERS, 1); @@ -1780,13 +2072,18 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L2_ENABLE - localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L2_MSHR_SIZE, L2_NUM_BANKS, L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH, UUID_WIDTH); `else - localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_NUM_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); + localparam L2_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L2_CACHE_REQS, L2_MEM_PORTS, L2_SECTOR_SIZE, L2_WORD_SIZE, L2_TAG_WIDTH); `endif // L2 is the LLC iff L2 is enabled and L3 is not. localparam L2_IS_LLC = `VX_CFG_L2_ENABLED && !`VX_CFG_L3_ENABLED; + // Every memory client of a cluster reaches memory through its L2. + localparam L2_SHARED_LLC = L2_IS_LLC && (`VX_CFG_NUM_CLUSTERS == 1); + // An acquire must also invalidate the cache below the dcache unless that + // one is the point of coherence (an L3 always is; memory always is). + localparam DCACHE_ACQ_FORWARD = `VX_CFG_L2_ENABLED && !L2_SHARED_LLC; /////////////////////////////// L3 Parameters ///////////////////////////// @@ -1803,6 +2100,8 @@ package VX_gpu_pkg; localparam L3_PTW_IDX = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS; localparam L3_NUM_REQS = `VX_CFG_NUM_CLUSTERS * L2_MEM_PORTS + ((`VX_CFG_VM_ENABLED != 0) && (PTW_ON_L2 == 0) ? 1 : 0); + localparam L3_CP_IDX = L3_NUM_REQS; + localparam L3_CACHE_REQS = L3_NUM_REQS + CP_ON_L3; localparam L3_NUM_BANKS = `VX_CFG_L3_NUM_BANKS; localparam L3_MEM_PORTS = `VX_CFG_L3_MEM_PORTS; @@ -1814,12 +2113,20 @@ package VX_gpu_pkg; // Memory request tag bits `ifdef VX_CFG_L3_ENABLE - localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_NC_MEM_TAG_WIDTH(`VX_CFG_L3_MSHR_SIZE, L3_NUM_BANKS, L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH, UUID_WIDTH); `else - localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_NUM_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); + localparam L3_MEM_TAG_WIDTH = `CACHE_BYPASS_TAG_WIDTH(L3_CACHE_REQS, L3_MEM_PORTS, L3_SECTOR_SIZE, L3_WORD_SIZE, L3_TAG_WIDTH); `endif // L3 is the LLC whenever it is enabled. localparam L3_IS_LLC = `VX_CFG_L3_ENABLED; + localparam L3_SHARED_LLC = L3_IS_LLC; + + // CP DMA port into the LLC client slot: one LLC word per request. The + // slot tag carries the port's tag plus a fence marker. + localparam CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? L2_WORD_SIZE : L3_WORD_SIZE; + localparam CP_MEM_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(CP_MEM_DATA_SIZE); + localparam CP_SLOT_TAG_WIDTH = (CP_ON_L2 != 0) ? L2_TAG_WIDTH : L3_TAG_WIDTH; + localparam CP_MEM_TAG_WIDTH = CP_SLOT_TAG_WIDTH - UUID_WIDTH - 1; /////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/VX_graphics.sv b/hw/rtl/VX_graphics.sv index 64510dba1b..257db1f23e 100644 --- a/hw/rtl/VX_graphics.sv +++ b/hw/rtl/VX_graphics.sv @@ -21,11 +21,15 @@ module VX_graphics import VX_gpu_pkg::* , VX_om_pkg::* `endif ; #( - parameter CLUSTER_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( + `SCOPE_IO_DECL + input wire clk, input wire reset, + input wire [CLUSTER_ID_WIDTH-1:0] cluster_id, + `ifdef PERF_ENABLE `ifdef VX_CFG_EXT_RASTER_ENABLE output raster_perf_t raster_perf, @@ -61,11 +65,37 @@ module VX_graphics import VX_gpu_pkg::* // Cluster-level flush trigger. VX_dcr_flush_if.slave cluster_flush_if, +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Address-space change (SATP commit): the rcache and ocache are virtually + // tagged, so the ocache drains its writes and both invalidate. + VX_dcr_flush_if.slave mmu_flush_if, +`endif + // Producer busy — the raster engine signals frame drain out-of-band (it has // no in-band `done`); the cluster ORs this into the device busy aggregation. output wire busy ); - `UNUSED_PARAM (CLUSTER_ID) + +`ifdef SCOPE +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam scope_raster_cnt = `VX_CFG_NUM_RASTER_CORES; +`else + localparam scope_raster_cnt = 0; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam scope_om_cnt = `VX_CFG_NUM_OM_CORES; +`else + localparam scope_om_cnt = 0; +`endif + `SCOPE_IO_SWITCH (scope_raster_cnt + scope_om_cnt); +`endif + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_EXT_RASTER_ENABLE + `UNUSED_VAR (cluster_id) +`endif + + wire rcache_req_empty; + wire ocache_req_empty; // Fan one DCR slave input out to one master per consumer unit. Each // unit's internal case-statement filters by DCR address range. The @@ -134,8 +164,7 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_RASTER_CORES; ++i) begin : g_raster_core VX_raster_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-raster%0d", CLUSTER_ID, i))), - .INSTANCE_IDX (CLUSTER_ID * `VX_CFG_NUM_RASTER_CORES + i), + .INSTANCE_ID (`SFORMATF(("%s-raster%0d", INSTANCE_ID, i))), .NUM_INSTANCES (`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES), .NUM_SLICES (`VX_CFG_RASTER_NUM_SLICES), // The front end's top-level walk unit is a coarse bin: bin_x/bin_y @@ -149,8 +178,11 @@ module VX_graphics import VX_gpu_pkg::* .QUAD_FIFO_DEPTH (`VX_CFG_RASTER_QUAD_FIFO_DEPTH), .OUTPUT_QUADS (`VX_CFG_NUM_THREADS) ) raster_core ( + `SCOPE_IO_BIND (i) .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(cluster_id) * `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(`VX_CFG_NUM_RASTER_CORES) + + `LOG2UP(`VX_CFG_NUM_CLUSTERS * `VX_CFG_NUM_RASTER_CORES)'(i)), `ifdef PERF_ENABLE .perf_raster_if (per_core_raster_perf_if[i]), `endif @@ -223,7 +255,11 @@ module VX_graphics import VX_gpu_pkg::* ) rcache_flushable_bus_if [`VX_CFG_NUM_RASTER_CORES * RCACHE_NUM_REQS] (); VX_dcr_flush_if rcache_flush_if(); +`ifdef CLUSTER_UNIT_MMU_ENABLE + assign rcache_flush_if.req = cluster_flush_if.req || mmu_flush_if.req; +`else assign rcache_flush_if.req = cluster_flush_if.req; +`endif VX_dcr_flush #( .WORD_SIZE (RCACHE_WORD_SIZE), @@ -233,6 +269,7 @@ module VX_graphics import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rcache_flush_if), + .l1_only (1'b0), .core_bus_if (rcache_bus_if[0]), .cache_bus_if (rcache_flushable_bus_if[0]) ); @@ -243,7 +280,7 @@ module VX_graphics import VX_gpu_pkg::* end VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-rcache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-rcache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_RCACHES), .NUM_INPUTS (`VX_CFG_NUM_RASTER_CORES), .TAG_SEL_IDX (0), @@ -263,6 +300,8 @@ module VX_graphics import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rcache ( @@ -272,7 +311,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (rcache_perf), `endif .core_bus_if (rcache_flushable_bus_if), - .mem_bus_if (rcache_mem_bus_tmp_if) + .mem_bus_if (rcache_mem_bus_tmp_if), + .req_empty (rcache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (rcache_mem_bus_if, rcache_mem_bus_tmp_if[0], @@ -297,6 +337,7 @@ module VX_graphics import VX_gpu_pkg::* om_dcrs_t om_core_dcrs [`VX_CFG_NUM_OM_CORES]; wire [L2_SOCKET_REQS-1:0] om_ingress_busy; + wire [L2_SOCKET_REQS-1:0] om_ingress_pending; // The DCRs are broadcast, so every OM core decodes identical values; the // ingresses read the aperture fields from core 0. om_dcrs_t om_dcrs; @@ -321,35 +362,61 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (L2_TAG_WIDTH) ) om_aperture_bus_if [L2_SOCKET_REQS](); + // Flush tokens are held at the steers until the OM side is flushed (below). + wire [L2_SOCKET_REQS-1:0] steer_flush_req; + wire [L2_SOCKET_REQS-1:0] steer_flush_l1_only; + reg [L2_SOCKET_REQS-1:0] steer_flush_ack_r; + wire [L2_SOCKET_REQS-1:0] steer_om_pending; + wire [L2_SOCKET_REQS-1:0] steer_req_empty; + for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_steer VX_om_steer #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-steer%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-steer%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .OUT_BUF (3) ) om_steer ( - .clk (clk), - .reset (reset), - .bus_in_if (socket_mem_bus_if[i]), - .l2_out_if (l2_out_bus_if[i]), - .om_out_if (om_aperture_bus_if[i]) + .clk (clk), + .reset (reset), + .bus_in_if (socket_mem_bus_if[i]), + .l2_out_if (l2_out_bus_if[i]), + .om_out_if (om_aperture_bus_if[i]), + .flush_req (steer_flush_req[i]), + .flush_l1_only (steer_flush_l1_only[i]), + .flush_ack (steer_flush_ack_r[i]), + .om_pending (steer_om_pending[i]) + ); + wire [1:0] steer_out_fire = { + om_aperture_bus_if[i].req_valid && om_aperture_bus_if[i].req_ready, + l2_out_bus_if[i].req_valid && l2_out_bus_if[i].req_ready + }; + VX_mem_req_pending #( + .NUM_INPUTS (1), + .NUM_OUTPUTS (2) + ) steer_pending ( + .clk (clk), + .reset (reset), + .in_fire (socket_mem_bus_if[i].req_valid && socket_mem_bus_if[i].req_ready), + .out_fire (steer_out_fire), + .empty (steer_req_empty[i]) ); end for (genvar i = 0; i < L2_SOCKET_REQS; ++i) begin : g_om_ingress VX_om_ingress #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om-ingress%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om-ingress%0d", INSTANCE_ID, i))), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (L2_TAG_WIDTH), .NUM_LANES (OM_CORE_LANES), .OUT_BUF (3) ) om_ingress ( - .clk (clk), - .reset (reset), - .dcrs (om_dcrs), - .mem_bus_if (om_aperture_bus_if[i]), - .om_bus_if (om_arb_in_if[i]), - .busy (om_ingress_busy[i]) + .clk (clk), + .reset (reset), + .dcrs (om_dcrs), + .mem_bus_if (om_aperture_bus_if[i]), + .om_bus_if (om_arb_in_if[i]), + .busy (om_ingress_busy[i]), + .frag_pending (om_ingress_pending[i]) ); end @@ -374,9 +441,10 @@ module VX_graphics import VX_gpu_pkg::* for (genvar i = 0; i < `VX_CFG_NUM_OM_CORES; ++i) begin : g_om_core VX_om_core #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-om%0d", CLUSTER_ID, i))), + .INSTANCE_ID (`SFORMATF(("%s-om%0d", INSTANCE_ID, i))), .NUM_LANES (OM_CORE_LANES) ) om_core ( + `SCOPE_IO_BIND (scope_raster_cnt + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE @@ -424,37 +492,173 @@ module VX_graphics import VX_gpu_pkg::* .TAG_WIDTH (OCACHE_BUS_TAG_WIDTH) ) ocache_flushable_bus_if [OCACHE_NUM_INPUTS * OCACHE_NUM_REQS] (); + // OM-side flush. Each round serves the requests present when it starts: the + // steers holding a flush token (every aperture store their trunk carried + // before it is past the steer) and the cluster flush. It waits until no + // complete fragment is left in the OM, so the ocache's flush tokens queue + // behind its writes, then flushes the ocache and releases those requesters. + // A half-received two-word record is not waited for: its depth beat can sit + // behind a held token, and it belongs to a store the token does not order. + localparam OMF_IDLE = 0; + localparam OMF_DRAIN = 1; + localparam OMF_FLUSH = 2; + localparam OMF_REARM = 3; + VX_dcr_flush_if ocache_flush_if(); - assign ocache_flush_if.req = cluster_flush_if.req; + + reg [1:0] omf_state_r; + reg [L2_SOCKET_REQS-1:0] omf_steers_r; // steers this round answers + reg omf_cluster_r; // this round answers the cluster flush + reg omf_cluster_done_r; + reg omf_l1_only_r; // every requester of this round is L1-only + wire omf_cluster_req = cluster_flush_if.req && ~omf_cluster_done_r; + // The address-space change is an L1-only requester: the ocache drains its + // writes and invalidates, and the L2 (physically tagged) is left alone. +`ifdef CLUSTER_UNIT_MMU_ENABLE + reg omf_mmu_r; + reg omf_mmu_done_r; + wire omf_mmu_req = mmu_flush_if.req && ~omf_mmu_done_r; +`else + wire omf_mmu_req = 1'b0; +`endif + + // The cluster flush takes its kind from the broadcast write that starts it, + // which every core receives too. The cores raise the cluster flush request + // only after taking that write, so the latch is current by then. + reg cluster_flush_l1_only_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_l1_only_r <= 1'b0; + end else if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH)) begin + cluster_flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end + end + wire omf_drained = ~(| om_busy_w) && ~(| om_ingress_pending) && ~(| steer_om_pending); + +`ifndef SYNTHESIS + // Each cluster flush round must find the latch written for it, strictly + // before the round takes the request. + wire cluster_flush_kind_wr = dcr_bus_if.req_valid && dcr_bus_if.req_data.rw + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire omf_cluster_take = (omf_state_r == OMF_IDLE) && omf_cluster_req; + reg cluster_flush_kind_new_r; + always @(posedge clk) begin + if (reset) begin + cluster_flush_kind_new_r <= 1'b0; + end else if (cluster_flush_kind_wr) begin + cluster_flush_kind_new_r <= 1'b1; + end else if (omf_cluster_take) begin + cluster_flush_kind_new_r <= 1'b0; + end + end + `RUNTIME_ASSERT (~omf_cluster_take || (cluster_flush_kind_new_r && ~cluster_flush_kind_wr), + ("%t: cluster flush taken before its kind was latched", $time)) +`endif + + always @(posedge clk) begin + if (reset) begin + omf_state_r <= OMF_IDLE; + omf_steers_r <= '0; + omf_cluster_r <= 1'b0; + omf_cluster_done_r <= 1'b0; + omf_l1_only_r <= 1'b0; + steer_flush_ack_r <= '0; + end else begin + steer_flush_ack_r <= '0; + if (~cluster_flush_if.req) begin + omf_cluster_done_r <= 1'b0; + end + case (omf_state_r) + OMF_IDLE: begin + if ((| steer_flush_req) || omf_cluster_req || omf_mmu_req) begin + omf_steers_r <= steer_flush_req; + omf_cluster_r <= omf_cluster_req; + omf_l1_only_r <= (& (~steer_flush_req | steer_flush_l1_only)) + && (~omf_cluster_req || cluster_flush_l1_only_r); + omf_state_r <= OMF_DRAIN; + end + end + OMF_DRAIN: begin + if (omf_drained) begin + omf_state_r <= OMF_FLUSH; + end + end + OMF_FLUSH: begin + if (ocache_flush_if.done) begin + steer_flush_ack_r <= omf_steers_r; + if (omf_cluster_r) begin + omf_cluster_done_r <= 1'b1; + end + omf_state_r <= OMF_REARM; + end + end + default: begin // OMF_REARM: the injectors re-arm once req drops + if (~ocache_flush_if.done) begin + omf_state_r <= OMF_IDLE; + end + end + endcase + end + end + assign ocache_flush_if.req = (omf_state_r == OMF_FLUSH); + +`ifdef CLUSTER_UNIT_MMU_ENABLE + // The address-space change joins a round like the cluster flush does. + always @(posedge clk) begin + if (reset) begin + omf_mmu_r <= 1'b0; + omf_mmu_done_r <= 1'b0; + end else begin + if (~mmu_flush_if.req) begin + omf_mmu_done_r <= 1'b0; + end + if ((omf_state_r == OMF_IDLE) && ((| steer_flush_req) || omf_cluster_req || omf_mmu_req)) begin + omf_mmu_r <= omf_mmu_req; + end + if ((omf_state_r == OMF_FLUSH) && ocache_flush_if.done && omf_mmu_r) begin + omf_mmu_done_r <= 1'b1; + end + end + end +`endif // OM-core flush chain carries the OM tag + 1 flush bit. When early-Z widens // the shared bus tag (a wider early-Z requester), the flushed OM bus is - // zero-extended up to OCACHE_BUS_TAG_WIDTH below. + // zero-extended up to OCACHE_BUS_TAG_WIDTH below. Each OM input is a + // separately ordered write path into the ocache, so each carries its own + // flush token behind its writes. + localparam OCACHE_OM_INPUTS = `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; localparam OCACHE_OM_FLUSH_TAG_WIDTH = OCACHE_TAG_WIDTH + 1; VX_mem_bus_if #( .DATA_SIZE (OCACHE_WORD_SIZE), .TAG_WIDTH (OCACHE_OM_FLUSH_TAG_WIDTH) - ) ocache_om_flush_bus_if [1] (); + ) ocache_om_flush_bus_if [OCACHE_OM_INPUTS] (); - VX_dcr_flush #( - .WORD_SIZE (OCACHE_WORD_SIZE), - .TAG_WIDTH (OCACHE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF - ) ocache_dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (ocache_flush_if), - .core_bus_if (ocache_bus_if[0]), - .cache_bus_if (ocache_om_flush_bus_if[0]) - ); + VX_dcr_flush_if ocache_om_flush_if [OCACHE_OM_INPUTS] (); + wire [OCACHE_OM_INPUTS-1:0] ocache_om_flush_done; + + for (genvar i = 0; i < OCACHE_OM_INPUTS; ++i) begin : g_ocache_om_flush + assign ocache_om_flush_if[i].req = ocache_flush_if.req; + assign ocache_om_flush_done[i] = ocache_om_flush_if[i].done; - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[0], ocache_om_flush_bus_if[0], - OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); + VX_dcr_flush #( + .WORD_SIZE (OCACHE_WORD_SIZE), + .TAG_WIDTH (OCACHE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register cache-request master boundary; rsp registered by cache CORE_OUT_BUF + ) ocache_dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (ocache_om_flush_if[i]), + .l1_only (omf_l1_only_r), + .core_bus_if (ocache_bus_if[i]), + .cache_bus_if (ocache_om_flush_bus_if[i]) + ); - for (genvar i = 1; i < `VX_CFG_NUM_OM_CORES * OCACHE_NUM_REQS; ++i) begin : g_ocache_passthru - `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_bus_if[i], - OCACHE_BUS_TAG_WIDTH, OCACHE_TAG_WIDTH, 0); + `ASSIGN_VX_MEM_BUS_IF_EX (ocache_flushable_bus_if[i], ocache_om_flush_bus_if[i], + OCACHE_BUS_TAG_WIDTH, OCACHE_OM_FLUSH_TAG_WIDTH, 0); end + assign ocache_flush_if.done = (& ocache_om_flush_done); `ifdef VX_CFG_RASTER_EARLYZ_ENABLE // Attach the early-Z depth readers as the trailing ocache inputs. Each reader @@ -467,7 +671,7 @@ module VX_graphics import VX_gpu_pkg::* `endif VX_cache_cluster #( - .INSTANCE_ID (`SFORMATF(("cluster%0d-ocache", CLUSTER_ID))), + .INSTANCE_ID (`SFORMATF(("%s-ocache", INSTANCE_ID))), .NUM_UNITS (`VX_CFG_NUM_OCACHES), .NUM_INPUTS (OCACHE_NUM_INPUTS), .TAG_SEL_IDX (0), @@ -496,7 +700,8 @@ module VX_graphics import VX_gpu_pkg::* .cache_perf (ocache_perf), `endif .core_bus_if (ocache_flushable_bus_if), - .mem_bus_if (ocache_mem_bus_tmp_if) + .mem_bus_if (ocache_mem_bus_tmp_if), + .req_empty (ocache_req_empty) ); `ASSIGN_VX_MEM_BUS_IF_EX (ocache_mem_bus_if, ocache_mem_bus_tmp_if[0], @@ -516,12 +721,20 @@ module VX_graphics import VX_gpu_pkg::* assign rcache_flush_done = 1'b1; `endif `ifdef VX_CFG_EXT_OM_ENABLE - assign ocache_flush_done = ocache_flush_if.done; + assign ocache_flush_done = omf_cluster_done_r; `else assign ocache_flush_done = 1'b1; `endif assign cluster_flush_if.done = rcache_flush_done & ocache_flush_done; +`ifdef CLUSTER_UNIT_MMU_ENABLE +`ifdef VX_CFG_EXT_OM_ENABLE + assign mmu_flush_if.done = rcache_flush_done & omf_mmu_done_r; +`else + assign mmu_flush_if.done = rcache_flush_done; +`endif +`endif + // Producer busy = a raster engine still draining a frame (out-of-band // drain) or an OM core with fragments in flight (vx_om4 is fire-and-forget, // so nothing else holds the device busy until the ROP commits). @@ -534,9 +747,16 @@ module VX_graphics import VX_gpu_pkg::* `endif `ifdef VX_CFG_EXT_OM_ENABLE wire om_busy_any = (| om_busy_w) || (| om_ingress_busy); + // requests held in a steer's buffers on the way to L2 or the OM + wire om_req_busy = ~(& steer_req_empty); `else wire om_busy_any = 1'b0; + wire om_req_busy = 1'b0; + assign ocache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RASTER_ENABLE + assign rcache_req_empty = 1'b1; `endif - assign busy = raster_busy_any || om_busy_any; + assign busy = raster_busy_any || om_busy_any || om_req_busy || ~rcache_req_empty || ~ocache_req_empty; endmodule diff --git a/hw/rtl/VX_platform.vh b/hw/rtl/VX_platform.vh index 85fae1d635..5b76c4fab3 100644 --- a/hw/rtl/VX_platform.vh +++ b/hw/rtl/VX_platform.vh @@ -192,9 +192,6 @@ `define PRESERVE_NET (* keep = "true" *) `define BLACKBOX_CELL (* black_box *) `define STRING -`ifndef SIMULATION - `define ASYNC_BRAM_PATCH -`endif `elsif ASIC `define MAX_FANOUT 8 `define LATENCY_IMUL 3 diff --git a/hw/rtl/VX_socket.sv b/hw/rtl/VX_socket.sv index af9b3258aa..b5bea180cc 100644 --- a/hw/rtl/VX_socket.sv +++ b/hw/rtl/VX_socket.sv @@ -18,7 +18,6 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; import VX_dxa_pkg::*; `endif #( - parameter SOCKET_ID = 0, parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +26,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] socket_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -64,7 +65,12 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef SCOPE localparam scope_core = 0; +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam scope_tex = `VX_CFG_SOCKET_SIZE; + `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE + `VX_CFG_NUM_TEX_CORES); +`else `SCOPE_IO_SWITCH (`VX_CFG_SOCKET_SIZE); +`endif `endif VX_kmu_bus_if per_core_kmu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -72,7 +78,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_SOCKET_SIZE), - .DEST_LSB (KMU_DEST_LSB_SOCKET), + .DEST_DIV (KMU_DEST_DIV_SOCKET), .OUT_BUF ((`VX_CFG_SOCKET_SIZE > 1) ? 3 : 0) ) kmu_arb ( .clk (clk), @@ -135,17 +141,20 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE](); + ) per_core_icache_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE](); + ) icache_phys_bus_if[`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS](); VX_mem_bus_if #( .DATA_SIZE (ICACHE_LINE_SIZE), .TAG_WIDTH (ICACHE_MEM_TAG_WIDTH) ) icache_mem_bus_if[1](); + wire icache_req_empty; + wire tcache_req_empty; + wire rtcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-icache", INSTANCE_ID))), @@ -154,10 +163,10 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .TAG_SEL_IDX (0), .CACHE_SIZE (`VX_CFG_ICACHE_SIZE), .LINE_SIZE (ICACHE_LINE_SIZE), - .NUM_BANKS (1), + .NUM_BANKS (ICACHE_NUM_BANKS), .NUM_WAYS (`VX_CFG_ICACHE_NUM_WAYS), .WORD_SIZE (ICACHE_WORD_SIZE), - .NUM_REQS (1), + .NUM_REQS (ICACHE_NUM_REQS), .MEM_PORTS (1), .CRSQ_SIZE (`VX_CFG_ICACHE_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_ICACHE_MSHR_SIZE), @@ -168,6 +177,11 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), + // a core's fetch slices read different words of the same lines + .READ_MERGE (`VX_CFG_ICACHE_READ_MERGE ? ICACHE_NUM_REQS : 0), + .READ_MERGE_LINE (1), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( @@ -177,7 +191,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (icache_phys_bus_if), - .mem_bus_if (icache_mem_bus_if) + .mem_bus_if (icache_mem_bus_if), + .req_empty (icache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -198,6 +213,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .DATA_SIZE (DCACHE_SECTOR_SIZE), .TAG_WIDTH (DCACHE_MEM_TAG_WIDTH) ) dcache_mem_bus_if[L1_MEM_PORTS](); + wire dcache_req_empty; VX_cache_cluster #( .INSTANCE_ID (`SFORMATF(("%s-dcache", INSTANCE_ID))), @@ -221,11 +237,16 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .IS_LLC (DCACHE_IS_LLC), + .SHARED_LLC (DCACHE_SHARED_LLC), + .ACQ_FORWARD (DCACHE_ACQ_FORWARD), + .BANK_HASH (`VX_CFG_DCACHE_BANK_HASH), + .READ_MERGE (`VX_CFG_DCACHE_READ_MERGE ? DCACHE_CHANNELS : 0), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) dcache ( `ifdef PERF_ENABLE @@ -234,7 +255,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .core_bus_if (dcache_phys_bus_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + .req_empty (dcache_req_empty) ); /////////////////////////////////////////////////////////////////////////// @@ -254,20 +276,29 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_tlb_flush_if immu_flush_if(); VX_mmu_fault_if dmmu_fault_if(); VX_mmu_fault_if immu_fault_if(); +`ifdef SOCKET_UNIT_MMU_ENABLE + VX_tlb_flush_if ummu_flush_if(); + VX_dcr_flush_if ucache_flush_if(); +`endif VX_mmu_snoop mmu_snoop ( - .clk (clk), - .reset (reset), - .dcr_bus_in_if (dcr_bus_if), - .dcr_bus_out_if (socket_dcr_bus_if), - .vm_active (vm_active), - .dmmu_flush_if (dmmu_flush_if), - .immu_flush_if (immu_flush_if) + .clk (clk), + .reset (reset), + .dcr_bus_in_if (dcr_bus_if), + .dcr_bus_out_if (socket_dcr_bus_if), + .vm_active (vm_active), + .dmmu_flush_if (dmmu_flush_if), + `ifdef SOCKET_UNIT_MMU_ENABLE + .ummu_flush_if (ummu_flush_if), + .ucache_flush_if (ucache_flush_if), + `endif + .immu_flush_if (immu_flush_if) ); - // dtlb + itlb each drive one tlb_bus into the socket arbiter below. + // dtlb + itlb (+ the unit MMU) each drive one tlb_bus into the socket + // arbiter below. VX_tlb_bus_if #(.ID_WIDTH (L1_TLB_ID_WIDTH)) - socket_tlb_bus_if [2] (); + socket_tlb_bus_if [SOCKET_NUM_MMUS] (); wire dmmu_empty, immu_empty; @@ -294,12 +325,13 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .tlb_bus_if (socket_tlb_bus_if[0]), .flush_if (dmmu_flush_if), .fault_if (dmmu_fault_if), + .lane_idle (dmmu_lane_idle), .empty (dmmu_empty) ); VX_mmu #( .INSTANCE_ID ("immu"), - .NUM_REQS (`VX_CFG_SOCKET_SIZE), + .NUM_REQS (`VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS), .TLB_SIZE (`VX_CFG_ITLB_SIZE), .EXEC_SIDE (1), .DATA_SIZE (ICACHE_WORD_SIZE), @@ -316,6 +348,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .tlb_bus_if (socket_tlb_bus_if[1]), .flush_if (immu_flush_if), .fault_if (immu_fault_if), + `UNUSED_PIN (lane_idle), .empty (immu_empty) ); @@ -332,8 +365,18 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; // Socket MMU drain status, broadcast to every core so its busy/barrier logic // waits for in-flight translations. Shared MMU: a core waits for the whole - // socket to drain (conservative, always correct). + // socket to drain (conservative, always correct). The unit MMU carries no + // core load or store, so it holds only the socket busy, not the barriers. wire socket_mmu_drained = dmmu_empty && immu_empty; + // Per dMMU lane: none of that lane's requests is in the MMU; each core's + // dcache flush waits only on its own lanes. + wire [`VX_CFG_SOCKET_SIZE * DCACHE_NUM_REQS-1:0] dmmu_lane_idle; +`ifdef SOCKET_UNIT_MMU_ENABLE + wire ummu_empty; + wire unit_mmu_drained = ummu_empty; +`else + wire unit_mmu_drained = 1'b1; +`endif `ifdef PERF_ENABLE // Surfacing these through sysmem_perf needs a dedicated mmu slot; until that @@ -342,9 +385,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_VAR (immu_perf) `endif - // Merge the two MMU tlb buses onto the one cluster-facing bus. + // Merge the MMU tlb buses onto the one cluster-facing bus. VX_tlb_bus_arb #( - .NUM_INPUTS (2), + .NUM_INPUTS (SOCKET_NUM_MMUS), .ID_WIDTH_IN (L1_TLB_ID_WIDTH), .OUT_BUF (3) ) tlb_socket_arb ( @@ -359,7 +402,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * DCACHE_NUM_REQS; ++i) begin : g_dcache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (dcache_phys_bus_if[i], per_core_dcache_bus_if[i]); end - for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_icache_phys_passthru + for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE * ICACHE_NUM_REQS; ++i) begin : g_icache_phys_passthru `ASSIGN_VX_MEM_BUS_IF (icache_phys_bus_if[i], per_core_icache_bus_if[i]); end `endif @@ -400,7 +443,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_EXT_TEX_ENABLE VX_tex_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) ) per_core_tex_bus_if[`VX_CFG_SOCKET_SIZE](); VX_tex_bus_if #( @@ -412,7 +455,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_TEX_CORES), - .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH), .ARBITER ("R"), .OUT_BUF_REQ ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0), // register request fan-in .OUT_BUF_RSP ((`VX_CFG_SOCKET_SIZE != `VX_CFG_NUM_TEX_CORES) ? 3 : 0) // register response distribution (skid registers the core-facing rsp and its backward ready) @@ -438,6 +481,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .TAG_WIDTH (TEX_REQ_ARB1_TAG_WIDTH) ) tex_core ( + `SCOPE_IO_BIND (scope_tex + i) .clk (clk), .reset (reset), `ifdef PERF_ENABLE @@ -493,6 +537,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -523,6 +568,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( @@ -532,7 +579,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (socket_tcache_perf), `endif .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + .req_empty (tcache_req_empty) ); `endif // VX_CFG_EXT_TEX_ENABLE @@ -549,7 +597,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; VX_rtu_bus_if #( .NUM_LANES (`VX_CFG_NUM_SFU_LANES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .SRC_WIDTH (1) // a core does not know its own index; the arbiter fills it in ) per_core_rtu_bus_if[`VX_CFG_SOCKET_SIZE](); @@ -563,7 +611,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .NUM_INPUTS (`VX_CFG_SOCKET_SIZE), .NUM_LANES (`VX_CFG_NUM_SFU_LANES), .NUM_OUTPUTS (`VX_CFG_NUM_RTU_CORES), - .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), .ARBITER ("R"), // The arm may be buffered: it does not mean "the RTU has taken your ray" // (the RTU stages a ray per {src, wid}), so registering it cannot let a @@ -642,6 +690,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -672,6 +721,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + // read-only: no dirty data, and the dcache's flush walks the shared levels + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( @@ -681,7 +732,8 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .cache_perf (socket_rtcache_perf), `endif .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + .req_empty (rtcache_req_empty) ); `endif // VX_CFG_EXT_RTU_ENABLE @@ -730,9 +782,9 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `UNUSED_PIN (busy) ); - // Route DXA lmem requests to per-core buses using core_local_id from tag. - // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]} - // core_local_id = core_id[CORE_LOCAL_BITS-1:0] + // Route DXA lmem requests to per-core buses by the core named in the tag. + // Tag value layout: {core_id[NC_BITS-1:0], engine_value[0]}, core_id being + // the cluster-local index, so the core within this socket is core_id % SOCKET_SIZE. localparam DXA_LMEM_CORE_LOCAL_BITS = `CLOG2(`VX_CFG_SOCKET_SIZE); VX_mem_bus_if #( .DATA_SIZE (DXA_LMEM_WORD_SIZE), @@ -743,7 +795,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire [`UP(DXA_LMEM_CORE_LOCAL_BITS)-1:0] dxa_lmem_core_sel; if (`VX_CFG_SOCKET_SIZE > 1) begin : g_dxa_lmem_sel - assign dxa_lmem_core_sel = dxa_lmem_bus_if[0].req_data.tag.value[1 +: DXA_LMEM_CORE_LOCAL_BITS]; + assign dxa_lmem_core_sel = DXA_LMEM_CORE_LOCAL_BITS'(dxa_lmem_bus_if[0].req_data.tag.value[1 +: NC_BITS] % `VX_CFG_SOCKET_SIZE); end else begin : g_dxa_lmem_sel assign dxa_lmem_core_sel = '0; end @@ -766,8 +818,86 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; ); `endif // VX_CFG_EXT_DXA_ENABLE +`ifdef SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Socket unit MMU — translates the socket-resident units' memory ports + // (tcache and rtcache misses, DXA gmem reads) between them and the socket + // memory arbiter, so the unit caches are virtually indexed and tagged. Its + // own TLB keeps texture, BVH and tensor streams from evicting the dTLB's + // entries, and its misses ride the third socket tlb_bus. + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_mem_bus_if[UMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_phys_bus_if[UMMU_NUM_REQS](); + + +`ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_TCACHE_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (unit_mem_bus_if[UMMU_DXA_IDX], dxa_gmem_bus_if[0]); +`endif + + VX_mmu_fault_if ummu_fault_if(); + +`ifdef PERF_ENABLE + mmu_perf_t ummu_perf; + `UNUSED_VAR (ummu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("ummu"), + .NUM_REQS (UMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_UTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (0), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) ummu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (ummu_perf), + `endif + .core_bus_if (unit_mem_bus_if), + .mem_bus_if (unit_phys_bus_if), + .tlb_bus_if (socket_tlb_bus_if[2]), + .flush_if (ummu_flush_if), + .fault_if (ummu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (ummu_empty) + ); + + // Faults are reported by the cluster walker, as for the dMMU/iMMU. + `UNUSED_VAR (ummu_fault_if.valid) + `UNUSED_VAR (ummu_fault_if.va) + `UNUSED_VAR (ummu_fault_if.access) + `UNUSED_VAR (ummu_fault_if.amo) + +`ifndef EXT_GFX_ANY_ENABLE + // no unit cache to invalidate + assign ucache_flush_if.done = 1'b1; + `UNUSED_VAR (ucache_flush_if.req) +`endif +`endif // SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + wire mem_arb_empty; + // L2-facing memory ports. Port 0 arbitrates icache and dcache with the // socket-resident units' memory ports (tcache, rtcache, DXA gmem) as // peers; priority order keeps icache first and DXA bulk traffic last so @@ -787,6 +917,17 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[ICACHE_MEM_ARB_IDX], icache_mem_bus_if[0], SOCKET_MEM_TAG_WIDTH, ICACHE_MEM_TAG_WIDTH, UUID_WIDTH); `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[DCACHE_MEM_ARB_IDX], dcache_mem_bus_if[0], SOCKET_MEM_TAG_WIDTH, DCACHE_MEM_TAG_WIDTH, UUID_WIDTH); + `ifdef SOCKET_UNIT_MMU_ENABLE + `ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[TCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_TCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); + `endif + `else `ifdef VX_CFG_EXT_TEX_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif @@ -795,6 +936,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `endif `ifdef VX_CFG_EXT_DXA_ENABLE `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], dxa_gmem_bus_if[0]); + `endif `endif VX_mem_bus_arb #( @@ -813,6 +955,21 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .bus_out_if (socket_arb_out_if) ); + wire [SOCKET_MEM_ARB_REQS-1:0] arb_in_fire; + for (genvar j = 0; j < SOCKET_MEM_ARB_REQS; ++j) begin : g_arb_in_fire + assign arb_in_fire[j] = socket_arb_in_if[j].req_valid && socket_arb_in_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (SOCKET_MEM_ARB_REQS), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (arb_in_fire), + .out_fire (socket_arb_out_if[0].req_valid && socket_arb_out_if[0].req_ready), + .empty (mem_arb_empty) + ); + `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[0], socket_arb_out_if[0]); end else begin : g_i VX_mem_bus_if #( @@ -828,6 +985,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; /////////////////////////////////////////////////////////////////////////// wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_busy; + wire [`VX_CFG_SOCKET_SIZE-1:0][`VX_CFG_NUM_LSU_BLOCKS-1:0] per_core_lsu_busy; `ifdef EXT_GFX_ANY_ENABLE // Core flush requests fan out to the socket-local gfx caches (tcache, // rtcache) and up to the cluster (rcache, ocache); each leg's done is @@ -839,14 +997,26 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; wire tcache_flush_done_w; wire rtcache_flush_done_w; +`ifdef SOCKET_UNIT_MMU_ENABLE + // The unit caches are virtually tagged: the SATP commit invalidates them too. + wire unit_cache_flush_req = ucache_flush_if.req; + assign ucache_flush_if.done = tcache_flush_done_w && rtcache_flush_done_w; +`else + wire unit_cache_flush_req = 1'b0; +`endif +`ifndef VX_CFG_EXT_TEX_ENABLE +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR (unit_cache_flush_req) // no socket-resident unit cache +`endif +`endif `ifdef VX_CFG_EXT_TEX_ENABLE - assign tcache_flush_if.req = socket_flush_req; + assign tcache_flush_if.req = socket_flush_req || unit_cache_flush_req; assign tcache_flush_done_w = tcache_flush_if.done; `else assign tcache_flush_done_w = 1'b1; `endif `ifdef VX_CFG_EXT_RTU_ENABLE - assign rtcache_flush_if.req = socket_flush_req; + assign rtcache_flush_if.req = socket_flush_req || unit_cache_flush_req; assign rtcache_flush_done_w = rtcache_flush_if.done; `else assign rtcache_flush_done_w = 1'b1; @@ -873,13 +1043,13 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; );*/ VX_core #( - .CORE_ID ((SOCKET_ID * `VX_CFG_SOCKET_SIZE) + core_id), .INSTANCE_ID (`SFORMATF(("%s-core%0d", INSTANCE_ID, core_id))) ) core ( `SCOPE_IO_BIND (scope_core + core_id) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(socket_id * `VX_CFG_SOCKET_SIZE) + CORE_ID_WIDTH'(core_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf_tmp), @@ -889,7 +1059,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .dcache_bus_if (per_core_dcache_bus_if[core_id * DCACHE_NUM_REQS +: DCACHE_NUM_REQS]), - .icache_bus_if (per_core_icache_bus_if[core_id]), + .icache_bus_if (per_core_icache_bus_if[core_id * ICACHE_NUM_REQS +: ICACHE_NUM_REQS]), `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (per_core_dxa_req_bus_if[core_id]), @@ -912,6 +1082,7 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; `ifdef VX_CFG_VM_ENABLE .mmu_drained (socket_mmu_drained), + .dcache_mmu_drained (& dmmu_lane_idle[core_id * DCACHE_NUM_REQS +: DCACHE_NUM_REQS]), `endif `ifdef EXT_GFX_ANY_ENABLE @@ -922,20 +1093,43 @@ module VX_socket import VX_gpu_pkg::*, VX_tlb_pkg::*; .gbar_bus_if (per_core_gbar_bus_if[core_id]), + .lsu_busy (per_core_lsu_busy[core_id]), .busy (per_core_busy[core_id]) ); end - // Launch liveness: fold this level's launch-link `valid` at both ends into busy - // combinationally -- the beat presented at the socket input and any beat resident - // in a per-core output skid -- so an in-transit launch stays visible on the - // presented cycle. The child (per_core_busy) aggregation stays registered. + // Launch liveness: a request is covered by its issuer, so busy has no path from + // the socket's inputs. The cluster covers the launch and DCR beats it presents + // here; this socket covers the beats resident in its per-core output skids and + // the cycle a core accepts one, which the core leaves out of its busy. With a + // single core, the skid is a wire from the input and the cluster covers it. + localparam BUSY_REG = (`VX_CFG_SOCKET_SIZE > 1); wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_kmu_valid; + wire [`VX_CFG_SOCKET_SIZE-1:0] per_core_kmu_fire; for (genvar i = 0; i < `VX_CFG_SOCKET_SIZE; ++i) begin : g_kmu_link_valid assign per_core_kmu_valid[i] = per_core_kmu_bus_if[i].valid; + assign per_core_kmu_fire[i] = per_core_kmu_bus_if[i].valid && per_core_kmu_bus_if[i].ready; end +`ifdef VX_CFG_VM_ENABLE + wire mmu_busy = ~(socket_mmu_drained && unit_mmu_drained); +`else + wire mmu_busy = 1'b0; +`endif + // Stores are ack-less, so the cores go idle with their last requests still + // on the way out; the socket stays busy until none is held in its caches' + // uncached paths or arbiters. +`ifndef VX_CFG_EXT_TEX_ENABLE + assign tcache_req_empty = 1'b1; +`endif +`ifndef VX_CFG_EXT_RTU_ENABLE + assign rtcache_req_empty = 1'b1; +`endif + wire mem_req_empty = icache_req_empty && dcache_req_empty && tcache_req_empty + && rtcache_req_empty && mem_arb_empty; + wire issued_busy = BUSY_REG ? (dcr_bus_if.req_valid | (| per_core_kmu_fire)) : 1'b0; + wire skid_busy = BUSY_REG ? (| per_core_kmu_valid) : 1'b0; wire busy_r; - `BUFFER_EX(busy_r, dcr_bus_if.req_valid | (|per_core_busy), 1'b1, 1, (`VX_CFG_SOCKET_SIZE > 1)); - assign busy = busy_r | dcr_bus_if.req_valid | kmu_bus_if[0].valid | (|per_core_kmu_valid); + `BUFFER_EX(busy_r, (|per_core_busy) | (|per_core_lsu_busy) | ~mem_req_empty | mmu_busy | issued_busy, 1'b1, 1, BUSY_REG); + assign busy = busy_r | skid_busy; endmodule diff --git a/hw/rtl/VX_trace_pkg.sv b/hw/rtl/VX_trace_pkg.sv index 8c3f8933c1..0842b8b26d 100644 --- a/hw/rtl/VX_trace_pkg.sv +++ b/hw/rtl/VX_trace_pkg.sv @@ -44,7 +44,8 @@ package VX_trace_pkg; task trace_ex_op(input int level, input [EX_BITS-1:0] ex_type, input [INST_OP_BITS-1:0] op_type, - input op_args_t op_args + input op_args_t op_args, + input [BYTESEL_BITS-1:0] bytesel = BYTESEL_DEFAULT ); case (ex_type) EX_ALU: begin @@ -176,9 +177,19 @@ package VX_trace_pkg; 2'd1: begin `TRACE(level, ("PACKLB.F")) end - default: begin + 2'd2: begin `TRACE(level, ("PACKLH.F")) end + default: begin + // the bytesel offset marks a quad (1) or a matrix fragment (2, 3) + if (bytesel[1]) begin + `TRACE(level, (op_args.lsu.is_store ? "VX_STM" : "VX_LDM")) + end else if (op_args.lsu.is_store) begin + `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FSW.Q" : "VX_FSW.D")) + end else begin + `TRACE(level, ((bytesel != BYTESEL_DEFAULT) ? "VX_FLW.Q" : "VX_FLW.D")) + end + end endcase end else if (op_args.lsu.is_float) begin case (INST_LSU_BITS'(op_type)) @@ -202,6 +213,7 @@ package VX_trace_pkg; INST_LSU_SW: `TRACE(level, ("SW")) INST_LSU_SD: `TRACE(level, ("SD")) INST_LSU_FENCE:`TRACE(level,("FENCE")) + INST_LSU_FENCE_IO:`TRACE(level,("FENCE.IO")) default: `TRACE(level, ("?")) endcase end diff --git a/hw/rtl/Vortex.sv b/hw/rtl/Vortex.sv index 2b4b49df89..8dd18a2b12 100644 --- a/hw/rtl/Vortex.sv +++ b/hw/rtl/Vortex.sv @@ -45,14 +45,33 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache (VX_CFG_CP_ON_LLC) + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy ); - // Check clustering configuration - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CLUSTERS), ("NUM_CLUSTERS must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_CORES), ("NUM_CORES must be a power of 2")); - `STATIC_ASSERT(`IS_POW2(`VX_CFG_SOCKET_SIZE), ("SOCKET_SIZE must be a power of 2")); + `STATIC_ASSERT((`VX_CFG_NUM_CORES % `VX_CFG_SOCKET_SIZE) == 0, ("NUM_CORES must be a multiple of SOCKET_SIZE")); + + // A write-back LLC must be the only instance: replicated ones never make + // their dirty lines visible to each other. + `STATIC_ASSERT(!(DCACHE_IS_LLC && `VX_CFG_DCACHE_WRITEBACK) || ((`VX_CFG_NUM_CORES == 1) && (`VX_CFG_NUM_CLUSTERS == 1)), + ("a write-back dcache that is the last-level cache must be the only dcache")); + `STATIC_ASSERT(!(L2_IS_LLC && `VX_CFG_L2_WRITEBACK) || (`VX_CFG_NUM_CLUSTERS == 1), + ("a write-back L2 that is the last-level cache must be the only L2")); // Every cache strictly above the LLC must be write-through. // A WB intermediate could absorb a hart-B store without the LLC @@ -75,6 +94,9 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign dcr_rsp_valid = dcr_bus_if.rsp_valid; assign dcr_rsp_data = dcr_bus_if.rsp_data; + wire kmu_start; + wire cp_mem_busy; + // Kernel Management Unit VX_kmu_bus_if kmu_bus_in[1](); wire kmu_busy; @@ -86,7 +108,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( ) kmu ( .clk (clk), .reset (reset), - .start (start), + .start (kmu_start), .busy (kmu_busy), .dcr_req_valid (dcr_req_valid), .dcr_req_rw (dcr_req_rw), @@ -117,12 +139,14 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_mem_bus_if #( .DATA_SIZE (L2_SECTOR_SIZE), .TAG_WIDTH (L3_TAG_WIDTH) - ) per_cluster_mem_bus_if[L3_NUM_REQS](); + ) per_cluster_mem_bus_if[L3_CACHE_REQS](); VX_mem_bus_if #( .DATA_SIZE (L3_SECTOR_SIZE), .TAG_WIDTH (L3_MEM_TAG_WIDTH) ) mem_bus_if[L3_MEM_PORTS](); + // no request held in the L3 bypass path short of the memory ports + wire l3_req_empty; VX_cache_wrap #( .INSTANCE_ID ("l3cache"), @@ -132,7 +156,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .NUM_BANKS (L3_NUM_BANKS), .NUM_WAYS (`VX_CFG_L3_NUM_WAYS), .WORD_SIZE (L3_WORD_SIZE), - .NUM_REQS (L3_NUM_REQS), + .NUM_REQS (L3_CACHE_REQS), .MEM_PORTS (L3_MEM_PORTS), .CRSQ_SIZE (`VX_CFG_L3_CRSQ_SIZE), .MSHR_SIZE (`VX_CFG_L3_MSHR_SIZE), @@ -143,12 +167,15 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L3_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L3_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L3_LAZY_FETCH), + .WRITE_AROUND (CP_ON_L3), .REPL_POLICY (`VX_CFG_L3_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), .NC_ENABLE (1), .PASSTHRU (!`VX_CFG_L3_ENABLED), .IS_LLC (L3_IS_LLC), + .SHARED_LLC (L3_SHARED_LLC), .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) ) l3cache ( .clk (clk), @@ -159,7 +186,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `endif .core_bus_if (per_cluster_mem_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + .req_empty (l3_req_empty) ); for (genvar i = 0; i < L3_MEM_PORTS; ++i) begin : g_mem_bus_if @@ -178,6 +206,68 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign mem_rsp_ready[i] = mem_bus_if[i].rsp_ready; end + // Command-processor DMA: one leg per cluster so every instance owns its + // port's drivers; only cluster 0's leg carries traffic, and only under + // CP_ON_L2. Under CP_ON_L3 the DMA is the L3's last client. + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) per_cluster_cp_mem_if [`VX_CFG_NUM_CLUSTERS] (); + + if ((CP_ON_L2 != 0) || (CP_ON_L3 != 0)) begin : g_cp_mem + VX_mem_bus_if #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .TAG_WIDTH (CP_SLOT_TAG_WIDTH) + ) cp_mem_if (); + + VX_cp_llc_port #( + .DATA_SIZE (CP_MEM_DATA_SIZE), + .SLOT_TAG_WIDTH (CP_SLOT_TAG_WIDTH), + .TAG_WIDTH (CP_MEM_TAG_WIDTH), + .ADDR_WIDTH (CP_MEM_ADDR_WIDTH) + ) cp_llc_port ( + .clk (clk), + .reset (reset), + .req_valid (cp_mem_req_valid), + .req_rw (cp_mem_req_rw), + .req_noalloc (cp_mem_req_noalloc), + .req_addr (cp_mem_req_addr), + .req_data (cp_mem_req_data), + .req_byteen (cp_mem_req_byteen), + .req_tag (cp_mem_req_tag), + .req_ready (cp_mem_req_ready), + .rsp_valid (cp_mem_rsp_valid), + .rsp_data (cp_mem_rsp_data), + .rsp_tag (cp_mem_rsp_tag), + .rsp_ready (cp_mem_rsp_ready), + .start_in (start), + .start_out (kmu_start), + .busy (cp_mem_busy), + .mem_bus_if (cp_mem_if) + ); + + if (CP_ON_L2 != 0) begin : g_l2 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0], cp_mem_if); + end else begin : g_l3 + `ASSIGN_VX_MEM_BUS_IF (per_cluster_mem_bus_if[L3_CP_IDX], cp_mem_if); + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[0]) + end + for (genvar c = 1; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end else begin : g_no_cp_mem + assign cp_mem_req_ready = 1'b0; + assign cp_mem_rsp_valid = 1'b0; + assign cp_mem_rsp_data = '0; + assign cp_mem_rsp_tag = '0; + `UNUSED_VAR ({cp_mem_req_valid, cp_mem_req_rw, cp_mem_req_noalloc, cp_mem_req_addr, cp_mem_req_data, cp_mem_req_byteen, cp_mem_req_tag, cp_mem_rsp_ready}) + assign kmu_start = start; + assign cp_mem_busy = 1'b0; + for (genvar c = 0; c < `VX_CFG_NUM_CLUSTERS; ++c) begin : g_cp_mem_legs + `INIT_VX_MEM_BUS_IF (per_cluster_cp_mem_if[c]) + end + end + wire [`VX_CFG_NUM_CLUSTERS-1:0] per_cluster_busy; VX_kmu_bus_if per_cluster_kmu_bus_if[`VX_CFG_NUM_CLUSTERS](); @@ -185,7 +275,7 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( VX_kmu_bus_arb #( .NUM_INPUTS (1), .NUM_OUTPUTS (`VX_CFG_NUM_CLUSTERS), - .DEST_LSB (KMU_DEST_LSB_DEVICE), + .DEST_DIV (KMU_DEST_DIV_DEVICE), .OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) // register per-cluster kmu fan-out (SLR-crossing skid) ) kmu_arb ( .clk (clk), @@ -213,6 +303,14 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( `ifdef VX_CFG_VM_ENABLE wire [`VX_CFG_XLEN-1:0] mmu_satp; wire mmu_flush_req; +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Translation active vs BARE (satp.MODE), for the cluster unit MMUs. +`ifdef VX_VM_ADDR_MODE_SV32 + wire mmu_vm_active = mmu_satp[31]; +`else + wire mmu_vm_active = (| mmu_satp[63:60]); +`endif +`endif wire [`VX_CFG_NUM_CLUSTERS-1:0] cl_mmu_flush_done; // Per-cluster L2-TLB miss export buses (used by the device-level walker). @@ -319,17 +417,41 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .bus_out_if (per_cluster_dcr_bus_if) ); + // A global barrier's participants can span clusters, so its arrivals are + // counted once for the whole device. + VX_gbar_bus_if per_cluster_gbar_bus_if[`VX_CFG_NUM_CLUSTERS](); + VX_gbar_bus_if gbar_bus_if(); + + VX_gbar_arb #( + .NUM_REQS (`VX_CFG_NUM_CLUSTERS), + .REQ_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0), + .RSP_OUT_BUF ((`VX_CFG_NUM_CLUSTERS > 1) ? 3 : 0) + ) gbar_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (per_cluster_gbar_bus_if), + .bus_out_if (gbar_bus_if) + ); + + VX_gbar_unit #( + .INSTANCE_ID ("gbar") + ) gbar_unit ( + .clk (clk), + .reset (reset), + .gbar_bus_if (gbar_bus_if) + ); + // Generate all clusters for (genvar cluster_id = 0; cluster_id < `VX_CFG_NUM_CLUSTERS; ++cluster_id) begin : g_clusters VX_cluster #( - .CLUSTER_ID (cluster_id), .INSTANCE_ID (`SFORMATF(("cluster%0d", cluster_id))) ) cluster ( `SCOPE_IO_BIND (scope_cluster + cluster_id) .clk (clk), .reset (reset), + .cluster_id (CLUSTER_ID_WIDTH'(cluster_id)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), @@ -341,6 +463,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .kmu_bus_if (per_cluster_kmu_bus_if[cluster_id +: 1]), + .gbar_bus_if (per_cluster_gbar_bus_if[cluster_id]), + `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_launch_if (per_cluster_raster_launch_if[cluster_id +: 1]), `endif @@ -350,8 +474,13 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( .dev_ptw_mem_if (per_cluster_ptw_mem_if[cluster_id]), .mmu_flush_req (mmu_flush_req), .mmu_flush_done (cl_mmu_flush_done[cluster_id]), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_vm_active (mmu_vm_active), + `endif `endif + .cp_mem_if (per_cluster_cp_mem_if[cluster_id]), + .busy (per_cluster_busy[cluster_id]) ); end @@ -364,8 +493,8 @@ module Vortex import VX_gpu_pkg::*, VX_trace_pkg::*, VX_tlb_pkg::*; ( assign per_cluster_kmu_valid[c] = per_cluster_kmu_bus_if[c].valid; end wire busy_r; - `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); - assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid); + `BUFFER_EX(busy_r, kmu_busy | dcr_bus_if.req_valid | (|per_cluster_busy) | dev_mmu_busy | ~l3_req_empty, 1'b1, 1, (`VX_CFG_NUM_CLUSTERS > 1)); + assign busy = busy_r | kmu_busy | dcr_bus_if.req_valid | (|per_cluster_kmu_valid) | cp_mem_busy; `ifdef PERF_ENABLE diff --git a/hw/rtl/Vortex_axi.sv b/hw/rtl/Vortex_axi.sv index 4566016c42..109f2b4898 100644 --- a/hw/rtl/Vortex_axi.sv +++ b/hw/rtl/Vortex_axi.sv @@ -107,6 +107,8 @@ module Vortex_axi import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] mem_rsp_tag [VX_MEM_PORTS]; wire mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -136,8 +138,21 @@ module Vortex_axi import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -269,4 +284,29 @@ module Vortex_axi import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_rresp) ); + // AXI orders a write against another master's access only once its B + // response is back, so the device stays busy until every write is acked. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [AXI_NUM_BANKS-1:0] axi_b_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < AXI_NUM_BANKS; ++i) begin : g_axi_b_fire + assign axi_b_fire[i] = m_axi_bvalid[i] && m_axi_bready[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (AXI_NUM_BANKS), + .SIZE (32 * AXI_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (axi_b_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/hw/rtl/afu/aved/vortex_afu.v b/hw/rtl/afu/aved/vortex_afu.v index 5335b376bb..54f5a5f288 100644 --- a/hw/rtl/afu/aved/vortex_afu.v +++ b/hw/rtl/afu/aved/vortex_afu.v @@ -23,22 +23,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -69,6 +61,9 @@ module vortex_afu #( output wire interrupt ); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CONTROL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CONTROL_DATA_WIDTH), @@ -79,11 +74,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (~ap_rst_n), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_control_awvalid), .s_axi_ctrl_awready (s_axi_control_awready), diff --git a/hw/rtl/afu/common/VX_afu_axi_drain.sv b/hw/rtl/afu/common/VX_afu_axi_drain.sv index 95e4c05459..b023f7fd0e 100644 --- a/hw/rtl/afu/common/VX_afu_axi_drain.sv +++ b/hw/rtl/afu/common/VX_afu_axi_drain.sv @@ -27,6 +27,10 @@ // directions, so it stays right when AXI4 allows write data to be presented // before its address (which would drive an up/down counter negative). // +// A write burst whose first beats have gone out is in flight too, even when +// its AW has not: the counters only see last beats, so the burst in progress +// is tracked separately. +// // COUNT_WIDTH must exceed the port's maximum outstanding transactions; the // comparison is exact only while fewer than 2**COUNT_WIDTH are in flight. // ============================================================================ @@ -39,6 +43,7 @@ module VX_afu_axi_drain #( // Handshakes observed on the shell side of the request gate. input wire aw_fire, // awvalid && awready + input wire w_fire, // wvalid && wready input wire w_fire_last, // wvalid && wready && wlast input wire b_fire, // bvalid && bready input wire ar_fire, // arvalid && arready @@ -60,15 +65,20 @@ module VX_afu_axi_drain #( reg [COUNT_WIDTH-1:0] b_count; reg [COUNT_WIDTH-1:0] ar_count; reg [COUNT_WIDTH-1:0] r_count; + reg w_in_burst; always @(posedge clk) begin if (reset) begin - aw_count <= '0; - w_count <= '0; - b_count <= '0; - ar_count <= '0; - r_count <= '0; + aw_count <= '0; + w_count <= '0; + b_count <= '0; + ar_count <= '0; + r_count <= '0; + w_in_burst <= 1'b0; end else begin + if (w_fire) begin + w_in_burst <= ~w_fire_last; + end if (aw_fire) begin aw_count <= aw_count + 1'b1; end @@ -94,7 +104,8 @@ module VX_afu_axi_drain #( assign dbg_ar_count = ar_count; assign dbg_r_count = r_count; - assign idle = (aw_count == w_count) + assign idle = ~w_in_burst + && (aw_count == w_count) && (aw_count == b_count) && (ar_count == r_count); diff --git a/hw/rtl/afu/common/VX_afu_axi_limit.sv b/hw/rtl/afu/common/VX_afu_axi_limit.sv deleted file mode 100644 index 37bc3bf736..0000000000 --- a/hw/rtl/afu/common/VX_afu_axi_limit.sv +++ /dev/null @@ -1,99 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -// ============================================================================ -// VX_afu_axi_limit — holds an AXI master to one read and one write in flight. -// -// A new AR is withheld until the previous read's last beat, and a new AW until -// the previous write's B. The limits only ever tighten on a handshake, so an -// offer already made to the slave is never withdrawn. -// -// AW and W of one write may fire in either order. W is let through whenever -// its AW has already gone, and AW whenever its W has, whatever the limit says -// by then: withholding half of an accepted write would hang it. -// ============================================================================ - -`TRACING_OFF -module VX_afu_axi_limit ( - input wire clk, - input wire reset, - - input wire in_awvalid, - output wire in_awready, - input wire in_wvalid, - output wire in_wready, - input wire in_wlast, - input wire in_arvalid, - output wire in_arready, - - output wire out_awvalid, - input wire out_awready, - output wire out_wvalid, - input wire out_wready, - output wire out_arvalid, - input wire out_arready, - - input wire b_fire, - input wire r_fire_last -); - reg rd_busy; - reg wr_busy; - reg w_owed; // AW sent, its W still to come - reg aw_owed; // W sent, its AW still to come - - wire aw_allow = aw_owed || ~wr_busy; - wire w_allow = w_owed || (~wr_busy && ~aw_owed); - wire ar_allow = ~rd_busy; - - assign out_awvalid = in_awvalid && aw_allow; - assign in_awready = out_awready && aw_allow; - assign out_wvalid = in_wvalid && w_allow; - assign in_wready = out_wready && w_allow; - assign out_arvalid = in_arvalid && ar_allow; - assign in_arready = out_arready && ar_allow; - - wire aw_fire = out_awvalid && out_awready; - wire w_fire_last = out_wvalid && out_wready && in_wlast; - wire ar_fire = out_arvalid && out_arready; - - always @(posedge clk) begin - if (reset) begin - rd_busy <= 1'b0; - wr_busy <= 1'b0; - w_owed <= 1'b0; - aw_owed <= 1'b0; - end else begin - if (ar_fire) begin - rd_busy <= 1'b1; - end else if (r_fire_last) begin - rd_busy <= 1'b0; - end - if (aw_fire) begin - wr_busy <= 1'b1; - end else if (b_fire) begin - wr_busy <= 1'b0; - end - if (aw_fire && ~w_fire_last) begin - aw_owed <= 1'b0; - w_owed <= ~aw_owed; - end else if (w_fire_last && ~aw_fire) begin - w_owed <= 1'b0; - aw_owed <= ~w_owed; - end - end - end - -endmodule -`TRACING_ON diff --git a/hw/rtl/afu/common/VX_afu_wr_gate.sv b/hw/rtl/afu/common/VX_afu_wr_gate.sv new file mode 100644 index 0000000000..821f0526f8 --- /dev/null +++ b/hw/rtl/afu/common/VX_afu_wr_gate.sv @@ -0,0 +1,98 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "vortex_afu.vh" + +// ============================================================================ +// VX_afu_wr_gate — withhold new AXI writes without splitting one. +// +// The write-channel counterpart of VX_afu_req_gate. Gating AW alone is not +// enough: AXI4 lets a master present a write's data before or with its +// address, so a master that does would push W beats of a write whose AW the +// gate then withholds. The slave is left holding data no address will ever +// claim, the drain counters never balance, and the reset sequencer refuses. +// +// A write is the k-th AW paired with the k-th W burst. While stopped, the gate +// lets through only what completes a write already begun on the shell side: +// the AW owed to a W burst that has started or is being offered, the W burst +// owed to an AW that has been accepted or is being offered, the rest of a +// burst in progress, and any offer already presented (VALID is never +// withdrawn, AXI4 §A3.2.1). A slave may hold AWREADY until it sees the data, +// or WREADY until it sees the address, so an offer alone obliges the other +// channel. Everything else waits, so the shell sees whole writes only and the +// drain reaches zero. +// +// LEAD_WIDTH must exceed the number of AWs a master may run ahead of its data. +// ============================================================================ + +module VX_afu_wr_gate #( + parameter LEAD_WIDTH = 16 +) ( + input wire clk, + input wire reset, + + input wire stop_req, // withhold new writes + + input wire in_awvalid, // from the AFU-internal master + output wire in_awready, + output wire out_awvalid, // to the shell + input wire out_awready, + + input wire in_wvalid, + input wire in_wlast, + output wire in_wready, + output wire out_wvalid, + input wire out_wready +); + reg blocked; + reg aw_offered; + reg w_offered; + reg w_in_burst; + reg [LEAD_WIDTH-1:0] aw_lead; // AWs accepted minus W bursts started + + wire aw_fire = out_awvalid && out_awready; + wire w_fire = out_wvalid && out_wready; + wire w_start = w_fire && ~w_in_burst; + + always @(posedge clk) begin + if (reset) begin + blocked <= 1'b0; + aw_offered <= 1'b0; + w_offered <= 1'b0; + w_in_burst <= 1'b0; + aw_lead <= '0; + end else begin + blocked <= stop_req; + aw_offered <= out_awvalid && ~out_awready; + w_offered <= out_wvalid && ~out_wready; + if (w_fire) begin + w_in_burst <= ~in_wlast; + end + aw_lead <= aw_lead + LEAD_WIDTH'(aw_fire) - LEAD_WIDTH'(w_start); + end + end + + wire lead_neg = aw_lead[LEAD_WIDTH-1]; + wire lead_zero = (aw_lead == '0); + wire aw_owed = lead_neg || (lead_zero && w_offered && ~w_in_burst); + wire w_owed = (~lead_neg && ~lead_zero) || (lead_zero && aw_offered); + + wire aw_pass = ~blocked || aw_offered || aw_owed; + wire w_pass = ~blocked || w_offered || w_in_burst || w_owed; + + assign out_awvalid = in_awvalid && aw_pass; + assign in_awready = out_awready && aw_pass; + assign out_wvalid = in_wvalid && w_pass; + assign in_wready = out_wready && w_pass; + +endmodule diff --git a/hw/rtl/afu/common/VX_afu_wrap.sv b/hw/rtl/afu/common/VX_afu_wrap.sv index 5f21e4842d..4a38fbff28 100644 --- a/hw/rtl/afu/common/VX_afu_wrap.sv +++ b/hw/rtl/afu/common/VX_afu_wrap.sv @@ -28,7 +28,8 @@ // register at host-offset 0x000. // // Data plane: -// * Memory banks 0..N-1 ride the platform AXI4 master ports. +// * One platform AXI4 master per GPU memory port (AFU_MEM_PORTS_OF); +// port i owns the i-th contiguous 1/N of device memory. // * The CP's device master shares them with Vortex: at the AXI port when // there is one, upstream of bank selection when there are several. // @@ -41,22 +42,43 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS_OF(VX_MEM_PORTS) ) ( // System signals input wire clk, input wire reset, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 memory masters, port i in slice i of each bus + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_awready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_awlen_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_wlast_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_bready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_bresp_bus, + + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arvalid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_arready_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS*8-1:0] m_axi_mem_arlen_bus, + + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rvalid_bus, + output wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rready_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus, + input wire [C_M_AXI_MEM_NUM_BANKS-1:0] m_axi_mem_rlast_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus, + input wire [C_M_AXI_MEM_NUM_BANKS*2-1:0] m_axi_mem_rresp_bus, + // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, // AXI4-Lite slave interface @@ -86,42 +108,64 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); localparam M_AXI_MEM_ADDR_WIDTH = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH; - wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; - - wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; - wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; - wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; - - // convert memory interface to array -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_TO_ARRAY, MP_SEMI); -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_TO_ARRAY, MP_SEMI); -`endif + // The platform top declares one AXI master per GPU memory port; a count + // that disagrees with the GPU would leave ports idle or requests unrouted. + `STATIC_ASSERT(C_M_AXI_MEM_NUM_BANKS == `AFU_MEM_PORTS_OF(VX_MEM_PORTS), + ("AXI memory port count %0d does not match the GPU's %0d", C_M_AXI_MEM_NUM_BANKS, `AFU_MEM_PORTS_OF(VX_MEM_PORTS))) + + wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_mem_bus + assign m_axi_mem_awvalid_bus[i] = m_axi_mem_awvalid_a[i]; + assign m_axi_mem_awready_a[i] = m_axi_mem_awready_bus[i]; + assign m_axi_mem_awaddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_awaddr_a[i]; + assign m_axi_mem_awid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_awid_a[i]; + assign m_axi_mem_awlen_bus[i * 8 +: 8] = m_axi_mem_awlen_a[i]; + assign m_axi_mem_wvalid_bus[i] = m_axi_mem_wvalid_a[i]; + assign m_axi_mem_wready_a[i] = m_axi_mem_wready_bus[i]; + assign m_axi_mem_wdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_wdata_a[i]; + assign m_axi_mem_wstrb_bus[i * (C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)] = m_axi_mem_wstrb_a[i]; + assign m_axi_mem_wlast_bus[i] = m_axi_mem_wlast_a[i]; + assign m_axi_mem_bvalid_a[i] = m_axi_mem_bvalid_bus[i]; + assign m_axi_mem_bready_bus[i] = m_axi_mem_bready_a[i]; + assign m_axi_mem_bid_a[i] = m_axi_mem_bid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_bresp_a[i] = m_axi_mem_bresp_bus[i * 2 +: 2]; + assign m_axi_mem_arvalid_bus[i] = m_axi_mem_arvalid_a[i]; + assign m_axi_mem_arready_a[i] = m_axi_mem_arready_bus[i]; + assign m_axi_mem_araddr_bus[i * C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH] = m_axi_mem_araddr_a[i]; + assign m_axi_mem_arid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_arid_a[i]; + assign m_axi_mem_arlen_bus[i * 8 +: 8] = m_axi_mem_arlen_a[i]; + assign m_axi_mem_rvalid_a[i] = m_axi_mem_rvalid_bus[i]; + assign m_axi_mem_rready_bus[i] = m_axi_mem_rready_a[i]; + assign m_axi_mem_rdata_a[i] = m_axi_mem_rdata_bus[i * C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; + assign m_axi_mem_rlast_a[i] = m_axi_mem_rlast_bus[i]; + assign m_axi_mem_rid_a[i] = m_axi_mem_rid_bus[i * C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; + assign m_axi_mem_rresp_a[i] = m_axi_mem_rresp_bus[i * 2 +: 2]; + end reg [`VX_CFG_RESET_DELAY-1:0] vx_reset_shift_r; wire vx_reset; @@ -338,7 +382,14 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire cp_interrupt; - VX_cp_core cp_core ( + // With several memory ports the CP's device master enters the + // last-level cache's client slot when the configuration has one; with a + // single port it keeps sharing that port below the caches. + localparam CP_DMA_LLC = ((C_M_AXI_MEM_NUM_BANKS > 1) && ((CP_ON_L2 != 0) || (CP_ON_L3 != 0))) ? 1 : 0; + + VX_cp_core #( + .DMA_LLC_CLIENT (CP_DMA_LLC) + ) cp_core ( .clk (clk), .reset (subsys_reset), .axil_s (cp_axil), @@ -355,26 +406,28 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // XRT pins m_axi_host to HOST[0]; host addresses pass straight through // (no PLATFORM_MEMORY_OFFSET — that offset is device-memory specific). // Gated like the bank ports: no new host-side request leaves the AFU while - // the reset sequencer is draining. The gate is a VX_afu_req_gate rather - // than a plain AND so an offer already made to the shell is never - // withdrawn -- see the header of that module. - VX_afu_req_gate host_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (cp_axi_host.awvalid), - .in_ready (cp_axi_host.awready), - .out_valid (m_axi_host_awvalid), - .out_ready (m_axi_host_awready) + // the reset sequencer is draining. The gates never withdraw an offer + // already made to the shell and never split a write -- see their headers. + VX_afu_wr_gate host_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (cp_axi_host.awvalid), + .in_awready (cp_axi_host.awready), + .out_awvalid (m_axi_host_awvalid), + .out_awready (m_axi_host_awready), + .in_wvalid (cp_axi_host.wvalid), + .in_wlast (cp_axi_host.wlast), + .in_wready (cp_axi_host.wready), + .out_wvalid (m_axi_host_wvalid), + .out_wready (m_axi_host_wready) ); assign m_axi_host_awaddr = cp_axi_host.awaddr; assign m_axi_host_awid = {{(C_M_AXI_MEM_ID_WIDTH-`VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_host.awid}; assign m_axi_host_awlen = cp_axi_host.awlen; - assign m_axi_host_wvalid = cp_axi_host.wvalid; assign m_axi_host_wdata = cp_axi_host.wdata; assign m_axi_host_wstrb = cp_axi_host.wstrb; assign m_axi_host_wlast = cp_axi_host.wlast; - assign cp_axi_host.wready = m_axi_host_wready; assign cp_axi_host.bvalid = m_axi_host_bvalid; assign cp_axi_host.bid = m_axi_host_bid[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_host.bresp = m_axi_host_bresp; @@ -435,18 +488,39 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_u [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_u [C_M_AXI_MEM_NUM_BANKS]; - // Per-bank XRT BO base offset. Each m_axi_mem_ port targets a different - // xrt::bo (one per DDR/HBM channel) that XRT places at a different virtual - // base address. PLATFORM_MEMORY_OFFSET applies the same synthesis-time - // offset to every bank. - wire [C_M_AXI_MEM_ADDR_WIDTH-1:0] platform_memory_offsets [C_M_AXI_MEM_NUM_BANKS]; - for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_pmo - assign platform_memory_offsets[i] = C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); - end + // Port i owns the i-th contiguous 1/N of the device address space, which + // the platform maps to that port's own group of memory channels, so no + // port reaches across another's. Lines interleave across the ports, so + // within the group a line sits at its index divided by N: drop the port + // field from the line address and put the port number on top. + // Without interleaving the bank adapter already sends each port only its + // own share. PLATFORM_MEMORY_OFFSET then rebases every port by the same + // aperture. + `STATIC_ASSERT(`IS_POW2(C_M_AXI_MEM_NUM_BANKS), + ("AXI memory port count %0d is not a power of two", C_M_AXI_MEM_NUM_BANKS)) + localparam MEM_PORT_BITS = `CLOG2(C_M_AXI_MEM_NUM_BANKS); + localparam MEM_LINE_BITS = `CLOG2(C_M_AXI_MEM_DATA_WIDTH / 8); + localparam MEM_GROUP_BITS = M_AXI_MEM_ADDR_WIDTH - MEM_PORT_BITS - MEM_LINE_BITS; + + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_g [C_M_AXI_MEM_NUM_BANKS]; + wire [M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_g [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_addressing - assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_u[i]) + platform_memory_offsets[i]; - assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_u[i]) + platform_memory_offsets[i]; + if (C_M_AXI_MEM_NUM_BANKS > 1 && `VX_CFG_PLATFORM_MEMORY_INTERLEAVE != 0) begin : g_group + assign m_axi_mem_awaddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_awaddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_awaddr_u[i][MEM_LINE_BITS-1:0]}; + assign m_axi_mem_araddr_g[i] = {MEM_PORT_BITS'(i), + m_axi_mem_araddr_u[i][MEM_LINE_BITS + MEM_PORT_BITS +: MEM_GROUP_BITS], + m_axi_mem_araddr_u[i][MEM_LINE_BITS-1:0]}; + `UNUSED_VAR (m_axi_mem_awaddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + `UNUSED_VAR (m_axi_mem_araddr_u[i][MEM_LINE_BITS +: MEM_PORT_BITS]) + end else begin : g_linear + assign m_axi_mem_awaddr_g[i] = m_axi_mem_awaddr_u[i]; + assign m_axi_mem_araddr_g[i] = m_axi_mem_araddr_u[i]; + end + assign m_axi_mem_awaddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_awaddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); + assign m_axi_mem_araddr_a[i] = C_M_AXI_MEM_ADDR_WIDTH'(m_axi_mem_araddr_g[i]) + C_M_AXI_MEM_ADDR_WIDTH'(`PLATFORM_MEMORY_OFFSET); end // ======================================================================== @@ -464,19 +538,27 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // ======================================================================== `SCOPE_IO_SWITCH (2); - // Address requests ahead of the request gate. + // Address and write-data requests ahead of the request gates. wire pre_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire pre_wready_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire pre_arready_a [C_M_AXI_MEM_NUM_BANKS]; if (C_M_AXI_MEM_NUM_BANKS == 1) begin : g_single_port + // The arbiter carries the source index in the top ID bit, so each + // master gets the rest of the platform ID. + localparam ARB_SRC_BITS = 1; + localparam SRC_ID_WIDTH = C_M_AXI_MEM_ID_WIDTH - ARB_SRC_BITS; + `STATIC_ASSERT(`VX_CP_AXI_TID_WIDTH <= SRC_ID_WIDTH, ("CP device-master ID does not fit the platform ID")) + // ---- Vortex's AXI port, ahead of the arbiter ---- wire vx_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_awready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_awid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_awlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; @@ -487,26 +569,26 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire vx_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_bready_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_bid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_bresp_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_arready_a [C_M_AXI_MEM_NUM_BANKS]; wire [M_AXI_MEM_ADDR_WIDTH-1:0] vx_araddr_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_arid_a [C_M_AXI_MEM_NUM_BANKS]; wire [7:0] vx_arlen_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rready_a [C_M_AXI_MEM_NUM_BANKS]; wire [C_M_AXI_MEM_DATA_WIDTH-1:0] vx_rdata_a [C_M_AXI_MEM_NUM_BANKS]; wire vx_rlast_a [C_M_AXI_MEM_NUM_BANKS]; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [SRC_ID_WIDTH-1:0] vx_rid_a [C_M_AXI_MEM_NUM_BANKS]; wire [1:0] vx_rresp_a [C_M_AXI_MEM_NUM_BANKS]; Vortex_axi #( .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .AXI_ADDR_WIDTH (M_AXI_MEM_ADDR_WIDTH), - .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_TID_WIDTH (SRC_ID_WIDTH), .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) ) vortex_axi ( `SCOPE_IO_BIND (1) @@ -570,12 +652,12 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( ); // ---- 2:1 arbiter merges Vortex + CP axi_dev ---- - // Pad CP's narrower ID into the platform ID width so the arbiter sees - // identical signal widths from both sources. - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.awid}; - wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = - {{(C_M_AXI_MEM_ID_WIDTH - `VX_CP_AXI_TID_WIDTH){1'b0}}, cp_axi_dev.arid}; + // Zero-extend both sources' IDs to the platform width; the arbiter + // overwrites the top bit with the source index. + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_awid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_awid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_arid_padded = C_M_AXI_MEM_ID_WIDTH'(vx_arid_a[0]); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_awid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.awid); + wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_arid_padded = C_M_AXI_MEM_ID_WIDTH'(cp_axi_dev.arid); // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at @@ -592,7 +674,9 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // Packed 2-master AXI arbiter: index 0 = Vortex bank-0 (priority via // ARBITER="P"), index 1 = CP device master. Input channels are packed // {cp, vx}; the arbiter's slave-side outputs land in local packed wires - // and are split back to the two masters below. + // and are split back to the two masters below. Responses are routed by + // the source bit of their ID, so both masters keep as many transactions + // in flight as they issue. localparam BANK0_STRB_W = C_M_AXI_MEM_DATA_WIDTH/8; wire [1:0] b0_awready; @@ -613,6 +697,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ID_WIDTH (C_M_AXI_MEM_ID_WIDTH), .ARBITER ("P"), // index 0 (Vortex bank-0) > index 1 (CP) + .MULTI_OUT (1), .STRB_WIDTH (BANK0_STRB_W) ) bank0_arb ( .clk (clk), @@ -621,7 +706,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_awvalid ({cp_axi_dev.awvalid, vx_awvalid_a[0]}), .s_awready (b0_awready), .s_awaddr ({cp_awaddr_dev, vx_awaddr_a[0]}), - .s_awid ({cp_awid_padded, vx_awid_a[0]}), + .s_awid ({cp_awid_padded, vx_awid_padded}), .s_awlen ({cp_axi_dev.awlen, vx_awlen_a[0]}), .s_wvalid ({cp_axi_dev.wvalid, vx_wvalid_a[0]}), @@ -638,7 +723,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .s_arvalid ({cp_axi_dev.arvalid, vx_arvalid_a[0]}), .s_arready (b0_arready), .s_araddr ({cp_araddr_dev, vx_araddr_a[0]}), - .s_arid ({cp_arid_padded, vx_arid_a[0]}), + .s_arid ({cp_arid_padded, vx_arid_padded}), .s_arlen ({cp_axi_dev.arlen, vx_arlen_a[0]}), .s_rvalid (b0_rvalid), @@ -651,7 +736,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_awvalid (pre_awvalid_a[0]), .m_awready (pre_awready_a[0]), .m_awaddr (m_axi_mem_awaddr_u[0]), .m_awid (m_axi_mem_awid_a[0]), .m_awlen (m_axi_mem_awlen_a[0]), - .m_wvalid (m_axi_mem_wvalid_a[0]), .m_wready (m_axi_mem_wready_a[0]), + .m_wvalid (pre_wvalid_a[0]), .m_wready (pre_wready_a[0]), .m_wdata (m_axi_mem_wdata_a[0]), .m_wstrb (m_axi_mem_wstrb_a[0]), .m_wlast (m_axi_mem_wlast_a[0]), .m_bvalid (m_axi_mem_bvalid_a[0]), .m_bready (m_axi_mem_bready_a[0]), @@ -668,6 +753,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // index 0 = Vortex bank-0, index 1 = CP device master. // Declared before their first assignment below (the implicit-net footgun: // a pre-declaration use makes Vivado mint 1-bit nets and keep both). + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_bid_full; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] vx_rid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_bid_full; wire [C_M_AXI_MEM_ID_WIDTH-1:0] cp_axi_dev_rid_full; assign vx_awready_a[0] = b0_awready[0]; @@ -676,7 +763,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.wready = b0_wready[1]; assign vx_bvalid_a[0] = b0_bvalid[0]; assign cp_axi_dev.bvalid = b0_bvalid[1]; - assign vx_bid_a[0] = b0_bid[0]; + assign vx_bid_full = b0_bid[0]; assign cp_axi_dev_bid_full = b0_bid[1]; assign vx_bresp_a[0] = b0_bresp[0]; assign cp_axi_dev.bresp = b0_bresp[1]; @@ -688,14 +775,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( assign cp_axi_dev.rdata = b0_rdata[1]; assign vx_rlast_a[0] = b0_rlast[0]; assign cp_axi_dev.rlast = b0_rlast[1]; - assign vx_rid_a[0] = b0_rid[0]; + assign vx_rid_full = b0_rid[0]; assign cp_axi_dev_rid_full = b0_rid[1]; assign vx_rresp_a[0] = b0_rresp[0]; assign cp_axi_dev.rresp = b0_rresp[1]; - // Truncate the arbiter's wider ID back to CP's narrower native ID width. + // Truncate the arbiter's platform-width IDs back to each master's own. + assign vx_bid_a[0] = vx_bid_full[SRC_ID_WIDTH-1:0]; + assign vx_rid_a[0] = vx_rid_full[SRC_ID_WIDTH-1:0]; assign cp_axi_dev.bid = cp_axi_dev_bid_full[`VX_CP_AXI_TID_WIDTH-1:0]; assign cp_axi_dev.rid = cp_axi_dev_rid_full[`VX_CP_AXI_TID_WIDTH-1:0]; + `UNUSED_VAR (vx_bid_full) + `UNUSED_VAR (vx_rid_full) `UNUSED_VAR (cp_axi_dev_bid_full) `UNUSED_VAR (cp_axi_dev_rid_full) @@ -714,7 +805,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( localparam VX_MEM_TAG_A_WIDTH = VX_MEM_TAG_WIDTH + `MAX(SUB_LDATAW, 0); localparam VX_MEM_ADDR_A_WIDTH = VX_MEM_ADDR_WIDTH - SUB_LDATAW; - localparam FAB_NUM_PORTS = VX_MEM_PORTS + 1; + localparam FAB_NUM_PORTS = VX_MEM_PORTS + (CP_DMA_LLC ? 0 : 1); localparam FAB_CP_PORT = VX_MEM_PORTS; localparam FAB_TAG_WIDTH = `MAX(VX_MEM_TAG_A_WIDTH, `VX_CP_AXI_TID_WIDTH); localparam FAB_DATA_SIZE = C_M_AXI_MEM_DATA_WIDTH / 8; @@ -733,6 +824,18 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire cp_llc_req_valid; + wire cp_llc_req_rw; + wire [CP_MEM_ADDR_WIDTH-1:0] cp_llc_req_addr; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_req_data; + wire [CP_MEM_DATA_SIZE-1:0] cp_llc_req_byteen; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_req_tag; + wire cp_llc_req_ready; + wire cp_llc_rsp_valid; + wire [CP_MEM_DATA_SIZE*8-1:0] cp_llc_rsp_data; + wire [CP_MEM_TAG_WIDTH-1:0] cp_llc_rsp_tag; + wire cp_llc_rsp_ready; + Vortex vortex ( `SCOPE_IO_BIND (1) @@ -760,6 +863,19 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_llc_req_valid), + .cp_mem_req_rw (cp_llc_req_rw), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr (cp_llc_req_addr), + .cp_mem_req_data (cp_llc_req_data), + .cp_mem_req_byteen (cp_llc_req_byteen), + .cp_mem_req_tag (cp_llc_req_tag), + .cp_mem_req_ready (cp_llc_req_ready), + .cp_mem_rsp_valid (cp_llc_rsp_valid), + .cp_mem_rsp_data (cp_llc_rsp_data), + .cp_mem_rsp_tag (cp_llc_rsp_tag), + .cp_mem_rsp_ready (cp_llc_rsp_ready), + .start (vx_start), .busy (vx_busy) ); @@ -777,6 +893,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( wire [FAB_TAG_WIDTH-1:0] fab_rsp_tag [FAB_NUM_PORTS]; wire fab_rsp_ready [FAB_NUM_PORTS]; + // The GPU and the bank adapter can sit in different SLRs: both + // directions leave this adapter fully registered, data included. for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_mem_adapter VX_mem_data_adapter #( .SRC_DATA_WIDTH (VX_MEM_DATA_WIDTH), @@ -785,8 +903,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .DST_ADDR_WIDTH (VX_MEM_ADDR_A_WIDTH), .SRC_TAG_WIDTH (VX_MEM_TAG_WIDTH), .DST_TAG_WIDTH (FAB_TAG_WIDTH), - .REQ_OUT_BUF (0), - .RSP_OUT_BUF (0) + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (3) ) mem_data_adapter ( .clk (clk), .reset (vx_reset), @@ -822,11 +940,20 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( // The CP's device addresses come from the same host-side allocator as the // pointers handed to the cores -- Device::global_mem_, based at // VX_MEM_USER_BASE_ADDR -- so they are offset-relative already, exactly - // like Vortex's own. They enter the fabric unchanged and - // PLATFORM_MEMORY_OFFSET is applied once, at the bank port, for both - // masters. - wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; - wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + // like Vortex's own. They enter the fabric (or the last-level cache) + // unchanged and PLATFORM_MEMORY_OFFSET is applied once, at the bank + // port, for both masters. + wire cp_mem_req_valid; + wire cp_mem_req_rw; + wire [CP_LINE_ADDRW-1:0] cp_mem_req_addr; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_req_data; + wire [FAB_DATA_SIZE-1:0] cp_mem_req_byteen; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_req_tag; + wire cp_mem_req_ready; + wire cp_mem_rsp_valid; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] cp_mem_rsp_data; + wire [`VX_CP_AXI_TID_WIDTH-1:0] cp_mem_rsp_tag; + wire cp_mem_rsp_ready; VX_membus_from_axi #( .ADDR_W (64), @@ -836,23 +963,69 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (subsys_reset), .axi_s (cp_axi_dev), - .mem_req_valid (fab_req_valid[FAB_CP_PORT]), - .mem_req_rw (fab_req_rw[FAB_CP_PORT]), + .mem_req_valid (cp_mem_req_valid), + .mem_req_rw (cp_mem_req_rw), .mem_req_addr (cp_mem_req_addr), - .mem_req_data (fab_req_data[FAB_CP_PORT]), - .mem_req_byteen (fab_req_byteen[FAB_CP_PORT]), + .mem_req_data (cp_mem_req_data), + .mem_req_byteen (cp_mem_req_byteen), .mem_req_tag (cp_mem_req_tag), - .mem_req_ready (fab_req_ready[FAB_CP_PORT]), - .mem_rsp_valid (fab_rsp_valid[FAB_CP_PORT]), - .mem_rsp_data (fab_rsp_data[FAB_CP_PORT]), - .mem_rsp_tag (fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]), - .mem_rsp_ready (fab_rsp_ready[FAB_CP_PORT]) + .mem_req_ready (cp_mem_req_ready), + .mem_rsp_valid (cp_mem_rsp_valid), + .mem_rsp_data (cp_mem_rsp_data), + .mem_rsp_tag (cp_mem_rsp_tag), + .mem_rsp_ready (cp_mem_rsp_ready) ); - assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; - assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); - `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) - `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + if (CP_DMA_LLC != 0) begin : g_cp_llc + `STATIC_ASSERT(CP_MEM_DATA_SIZE == FAB_DATA_SIZE, ("CP DMA beat must match the last-level cache word")) + `STATIC_ASSERT(CP_MEM_TAG_WIDTH >= `VX_CP_AXI_TID_WIDTH, ("CP DMA id does not fit the last-level cache slot tag")) + // The bridge's request valid is combinational; register the + // request before it enters the last-level cache's slot. + VX_elastic_buffer #( + .DATAW (1 + CP_MEM_ADDR_WIDTH + CP_MEM_DATA_SIZE * 9 + CP_MEM_TAG_WIDTH), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) + ) cp_llc_req_buf ( + .clk (clk), + .reset (subsys_reset), + .valid_in (cp_mem_req_valid), + .ready_in (cp_mem_req_ready), + .data_in ({cp_mem_req_rw, cp_mem_req_addr[CP_MEM_ADDR_WIDTH-1:0], cp_mem_req_data, + cp_mem_req_byteen, CP_MEM_TAG_WIDTH'(cp_mem_req_tag)}), + .data_out ({cp_llc_req_rw, cp_llc_req_addr, cp_llc_req_data, cp_llc_req_byteen, cp_llc_req_tag}), + .valid_out (cp_llc_req_valid), + .ready_out (cp_llc_req_ready) + ); + assign cp_mem_rsp_valid = cp_llc_rsp_valid; + assign cp_mem_rsp_data = cp_llc_rsp_data; + assign cp_mem_rsp_tag = cp_llc_rsp_tag[`VX_CP_AXI_TID_WIDTH-1:0]; + assign cp_llc_rsp_ready = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:CP_MEM_ADDR_WIDTH]) + `UNUSED_VAR (cp_llc_rsp_tag) + end else begin : g_cp_fabric + assign fab_req_valid[FAB_CP_PORT] = cp_mem_req_valid; + assign fab_req_rw[FAB_CP_PORT] = cp_mem_req_rw; + assign fab_req_addr[FAB_CP_PORT] = cp_mem_req_addr[VX_MEM_ADDR_A_WIDTH-1:0]; + assign fab_req_data[FAB_CP_PORT] = cp_mem_req_data; + assign fab_req_byteen[FAB_CP_PORT] = cp_mem_req_byteen; + assign fab_req_tag[FAB_CP_PORT] = FAB_TAG_WIDTH'(cp_mem_req_tag); + assign cp_mem_req_ready = fab_req_ready[FAB_CP_PORT]; + assign cp_mem_rsp_valid = fab_rsp_valid[FAB_CP_PORT]; + assign cp_mem_rsp_data = fab_rsp_data[FAB_CP_PORT]; + assign cp_mem_rsp_tag = fab_rsp_tag[FAB_CP_PORT][`VX_CP_AXI_TID_WIDTH-1:0]; + assign fab_rsp_ready[FAB_CP_PORT] = cp_mem_rsp_ready; + `UNUSED_VAR (cp_mem_req_addr[CP_LINE_ADDRW-1:VX_MEM_ADDR_A_WIDTH]) + `UNUSED_VAR (fab_rsp_tag[FAB_CP_PORT]) + + assign cp_llc_req_valid = 1'b0; + assign cp_llc_req_rw = 1'b0; + assign cp_llc_req_addr = '0; + assign cp_llc_req_data = '0; + assign cp_llc_req_byteen = '0; + assign cp_llc_req_tag = '0; + assign cp_llc_rsp_ready = 1'b1; + `UNUSED_VAR ({cp_llc_req_ready, cp_llc_rsp_valid, cp_llc_rsp_data, cp_llc_rsp_tag}) + end // The optional AXI4 sideband signals (size/burst) are unused by the // bridge's reduced view — pin them sink-side so lint stays clean. @@ -862,13 +1035,6 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_VAR (cp_axi_dev.arburst) - wire ad_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_awready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_arready_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; - wire ad_wready_a [C_M_AXI_MEM_NUM_BANKS]; - VX_mem_to_axi #( .DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), .ADDR_WIDTH_IN (VX_MEM_ADDR_A_WIDTH), @@ -897,8 +1063,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .mem_rsp_tag (fab_rsp_tag), .mem_rsp_ready (fab_rsp_ready), - .m_axi_awvalid (ad_awvalid_a), - .m_axi_awready (ad_awready_a), + .m_axi_awvalid (pre_awvalid_a), + .m_axi_awready (pre_awready_a), .m_axi_awaddr (m_axi_mem_awaddr_u), .m_axi_awid (m_axi_mem_awid_a), .m_axi_awlen (m_axi_mem_awlen_a), @@ -910,8 +1076,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( `UNUSED_PIN (m_axi_awqos), `UNUSED_PIN (m_axi_awregion), - .m_axi_wvalid (ad_wvalid_a), - .m_axi_wready (ad_wready_a), + .m_axi_wvalid (pre_wvalid_a), + .m_axi_wready (pre_wready_a), .m_axi_wdata (m_axi_mem_wdata_a), .m_axi_wstrb (m_axi_mem_wstrb_a), .m_axi_wlast (m_axi_mem_wlast_a), @@ -921,8 +1087,8 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_bid (m_axi_mem_bid_a), .m_axi_bresp (m_axi_mem_bresp_a), - .m_axi_arvalid (ad_arvalid_a), - .m_axi_arready (ad_arready_a), + .m_axi_arvalid (pre_arvalid_a), + .m_axi_arready (pre_arready_a), .m_axi_araddr (m_axi_mem_araddr_u), .m_axi_arid (m_axi_mem_arid_a), .m_axi_arlen (m_axi_mem_arlen_a), @@ -942,62 +1108,33 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .m_axi_rresp (m_axi_mem_rresp_a) ); - // Bank 0 carries one read and one write at a time, the other banks - // as many as the adapter tracks. Reads in flight cost cycles on a - // saturated channel and save them across several, so moving this - // limit changes cycle counts in both directions. - VX_afu_axi_limit bank0_limit ( - .clk (clk), - .reset (subsys_reset), - .in_awvalid (ad_awvalid_a[0]), - .in_awready (ad_awready_a[0]), - .in_wvalid (ad_wvalid_a[0]), - .in_wready (ad_wready_a[0]), - .in_wlast (m_axi_mem_wlast_a[0]), - .in_arvalid (ad_arvalid_a[0]), - .in_arready (ad_arready_a[0]), - .out_awvalid (pre_awvalid_a[0]), - .out_awready (pre_awready_a[0]), - .out_wvalid (m_axi_mem_wvalid_a[0]), - .out_wready (m_axi_mem_wready_a[0]), - .out_arvalid (pre_arvalid_a[0]), - .out_arready (pre_arready_a[0]), - .b_fire (m_axi_mem_bvalid_a[0] && m_axi_mem_bready_a[0]), - .r_fire_last (m_axi_mem_rvalid_a[0] && m_axi_mem_rready_a[0] - && m_axi_mem_rlast_a[0]) - ); - - for (genvar i = 1; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_bank_direct - assign pre_awvalid_a[i] = ad_awvalid_a[i]; - assign ad_awready_a[i] = pre_awready_a[i]; - assign m_axi_mem_wvalid_a[i] = ad_wvalid_a[i]; - assign ad_wready_a[i] = m_axi_mem_wready_a[i]; - assign pre_arvalid_a[i] = ad_arvalid_a[i]; - assign ad_arready_a[i] = pre_arready_a[i]; - end - end // ======================================================================== // Request gate + drain tracking. // // Everything upstream drives pre_* instead of the bank port directly. The - // gate below withholds new AW/AR while the reset sequencer is quiescing, - // so the outstanding counts can reach zero in bounded time. W, B and R are - // never gated: a burst whose address the interconnect has already accepted - // must be allowed to finish. + // gates below withhold new reads and writes while the reset sequencer is + // quiescing, so the outstanding counts can reach zero in bounded time. + // A write already begun on either channel completes on both, and B and R + // are never gated: a transaction the interconnect has seen must finish. // ======================================================================== wire mem_idle_a [C_M_AXI_MEM_NUM_BANKS]; for (genvar i = 0; i < C_M_AXI_MEM_NUM_BANKS; ++i) begin : g_axi_gate - VX_afu_req_gate mem_aw_gate ( - .clk (clk), - .reset (reset), - .stop_req (rst_stop_req), - .in_valid (pre_awvalid_a[i]), - .in_ready (pre_awready_a[i]), - .out_valid (m_axi_mem_awvalid_a[i]), - .out_ready (m_axi_mem_awready_a[i]) + VX_afu_wr_gate mem_wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (rst_stop_req), + .in_awvalid (pre_awvalid_a[i]), + .in_awready (pre_awready_a[i]), + .out_awvalid (m_axi_mem_awvalid_a[i]), + .out_awready (m_axi_mem_awready_a[i]), + .in_wvalid (pre_wvalid_a[i]), + .in_wlast (m_axi_mem_wlast_a[i]), + .in_wready (pre_wready_a[i]), + .out_wvalid (m_axi_mem_wvalid_a[i]), + .out_wready (m_axi_mem_wready_a[i]) ); VX_afu_req_gate mem_ar_gate ( @@ -1010,10 +1147,15 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .out_ready (m_axi_mem_arready_a[i]) ); - VX_afu_axi_drain mem_drain ( + // A memory port carries every miss the last-level caches hold in + // flight, so its counters are sized well past any MSHR total. + VX_afu_axi_drain #( + .COUNT_WIDTH (16) + ) mem_drain ( .clk (clk), .reset (reset), .aw_fire (m_axi_mem_awvalid_a[i] && m_axi_mem_awready_a[i]), + .w_fire (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i]), .w_fire_last (m_axi_mem_wvalid_a[i] && m_axi_mem_wready_a[i] && m_axi_mem_wlast_a[i]), .b_fire (m_axi_mem_bvalid_a[i] && m_axi_mem_bready_a[i]), @@ -1035,6 +1177,7 @@ module VX_afu_wrap import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .aw_fire (m_axi_host_awvalid && m_axi_host_awready), + .w_fire (m_axi_host_wvalid && m_axi_host_wready), .w_fire_last (m_axi_host_wvalid && m_axi_host_wready && m_axi_host_wlast), .b_fire (m_axi_host_bvalid && m_axi_host_bready), .ar_fire (m_axi_host_arvalid && m_axi_host_arready), diff --git a/hw/rtl/afu/common/vortex_afu.vh b/hw/rtl/afu/common/vortex_afu.vh index 9544345186..dd17de49aa 100644 --- a/hw/rtl/afu/common/vortex_afu.vh +++ b/hw/rtl/afu/common/vortex_afu.vh @@ -81,7 +81,7 @@ .m_axi_mem_``i``_bid(m_axi_mem_``i``_bid) // Host-memory AXI4 master port (CP command ring + host side of DMA). -// XRT pins this kernel port to HOST[0] (see hw/syn/xilinx/xrt/platforms.mk). +// XRT pins this kernel port to the card descriptor's host tag (hw/syn/xilinx/platforms). `define GEN_AXI_HOST \ output wire m_axi_host_awvalid, \ input wire m_axi_host_awready, \ @@ -163,6 +163,97 @@ assign m_axi_mem_bresp_a[i] = m_axi_mem_``i``_bresp; \ assign m_axi_mem_bid_a[i] = m_axi_mem_``i``_bid +`define AXI_MEM_TO_BUS(i) \ + assign m_axi_mem_``i``_awvalid = m_axi_mem_awvalid_bus[i]; \ + assign m_axi_mem_awready_bus[i] = m_axi_mem_``i``_awready; \ + assign m_axi_mem_``i``_awaddr = m_axi_mem_awaddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_awid = m_axi_mem_awid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_awlen = m_axi_mem_awlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_``i``_wvalid = m_axi_mem_wvalid_bus[i]; \ + assign m_axi_mem_wready_bus[i] = m_axi_mem_``i``_wready; \ + assign m_axi_mem_``i``_wdata = m_axi_mem_wdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH]; \ + assign m_axi_mem_``i``_wstrb = m_axi_mem_wstrb_bus[(i)*(C_M_AXI_MEM_DATA_WIDTH/8) +: (C_M_AXI_MEM_DATA_WIDTH/8)]; \ + assign m_axi_mem_``i``_wlast = m_axi_mem_wlast_bus[i]; \ + assign m_axi_mem_``i``_arvalid = m_axi_mem_arvalid_bus[i]; \ + assign m_axi_mem_arready_bus[i] = m_axi_mem_``i``_arready; \ + assign m_axi_mem_``i``_araddr = m_axi_mem_araddr_bus[(i)*C_M_AXI_MEM_ADDR_WIDTH +: C_M_AXI_MEM_ADDR_WIDTH]; \ + assign m_axi_mem_``i``_arid = m_axi_mem_arid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH]; \ + assign m_axi_mem_``i``_arlen = m_axi_mem_arlen_bus[(i)*8 +: 8]; \ + assign m_axi_mem_rvalid_bus[i] = m_axi_mem_``i``_rvalid; \ + assign m_axi_mem_``i``_rready = m_axi_mem_rready_bus[i]; \ + assign m_axi_mem_rdata_bus[(i)*C_M_AXI_MEM_DATA_WIDTH +: C_M_AXI_MEM_DATA_WIDTH] = m_axi_mem_``i``_rdata; \ + assign m_axi_mem_rlast_bus[i] = m_axi_mem_``i``_rlast; \ + assign m_axi_mem_rid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_rid; \ + assign m_axi_mem_rresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_rresp; \ + assign m_axi_mem_bvalid_bus[i] = m_axi_mem_``i``_bvalid; \ + assign m_axi_mem_``i``_bready = m_axi_mem_bready_bus[i]; \ + assign m_axi_mem_bresp_bus[(i)*2 +: 2] = m_axi_mem_``i``_bresp; \ + assign m_axi_mem_bid_bus[(i)*C_M_AXI_MEM_ID_WIDTH +: C_M_AXI_MEM_ID_WIDTH] = m_axi_mem_``i``_bid + +// AXI memory masters the AFU exposes: one per GPU memory port, rounded up to +// a power of two so that address bits select the port, and at most one per +// platform memory channel. A platform top must spell its ports out, so the +// build resolves this to the literal AFU_MEM_PORTS +// (hw/scripts/afu_mem_ports.sh) and VX_afu_wrap checks that the two agree. +`define AFU_MEM_PORTS_OF(n) `MIN(1 << `CLOG2(n), `VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + +// Flat per-field buses connecting a platform top's m_axi_mem_ ports to +// VX_afu_wrap, port i in slice i. They are plain Verilog, so the platform +// tops stay Verilog, which the Vitis and AVED kernel packagers require. +`define AXI_MEM_BUSES(n) \ + wire [(n)-1:0] m_axi_mem_awvalid_bus; \ + wire [(n)-1:0] m_axi_mem_awready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_bus; \ + wire [(n)*8-1:0] m_axi_mem_awlen_bus; \ + wire [(n)-1:0] m_axi_mem_wvalid_bus; \ + wire [(n)-1:0] m_axi_mem_wready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_bus; \ + wire [(n)*(C_M_AXI_MEM_DATA_WIDTH/8)-1:0] m_axi_mem_wstrb_bus; \ + wire [(n)-1:0] m_axi_mem_wlast_bus; \ + wire [(n)-1:0] m_axi_mem_arvalid_bus; \ + wire [(n)-1:0] m_axi_mem_arready_bus; \ + wire [(n)*C_M_AXI_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_bus; \ + wire [(n)*8-1:0] m_axi_mem_arlen_bus; \ + wire [(n)-1:0] m_axi_mem_rvalid_bus; \ + wire [(n)-1:0] m_axi_mem_rready_bus; \ + wire [(n)*C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_bus; \ + wire [(n)-1:0] m_axi_mem_rlast_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_bus; \ + wire [(n)*2-1:0] m_axi_mem_rresp_bus; \ + wire [(n)-1:0] m_axi_mem_bvalid_bus; \ + wire [(n)-1:0] m_axi_mem_bready_bus; \ + wire [(n)*2-1:0] m_axi_mem_bresp_bus; \ + wire [(n)*C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_bus; + +`define AXI_MEM_BUS_ARGS \ + .m_axi_mem_awvalid_bus(m_axi_mem_awvalid_bus), \ + .m_axi_mem_awready_bus(m_axi_mem_awready_bus), \ + .m_axi_mem_awaddr_bus(m_axi_mem_awaddr_bus), \ + .m_axi_mem_awid_bus(m_axi_mem_awid_bus), \ + .m_axi_mem_awlen_bus(m_axi_mem_awlen_bus), \ + .m_axi_mem_wvalid_bus(m_axi_mem_wvalid_bus), \ + .m_axi_mem_wready_bus(m_axi_mem_wready_bus), \ + .m_axi_mem_wdata_bus(m_axi_mem_wdata_bus), \ + .m_axi_mem_wstrb_bus(m_axi_mem_wstrb_bus), \ + .m_axi_mem_wlast_bus(m_axi_mem_wlast_bus), \ + .m_axi_mem_arvalid_bus(m_axi_mem_arvalid_bus), \ + .m_axi_mem_arready_bus(m_axi_mem_arready_bus), \ + .m_axi_mem_araddr_bus(m_axi_mem_araddr_bus), \ + .m_axi_mem_arid_bus(m_axi_mem_arid_bus), \ + .m_axi_mem_arlen_bus(m_axi_mem_arlen_bus), \ + .m_axi_mem_rvalid_bus(m_axi_mem_rvalid_bus), \ + .m_axi_mem_rready_bus(m_axi_mem_rready_bus), \ + .m_axi_mem_rdata_bus(m_axi_mem_rdata_bus), \ + .m_axi_mem_rlast_bus(m_axi_mem_rlast_bus), \ + .m_axi_mem_rid_bus(m_axi_mem_rid_bus), \ + .m_axi_mem_rresp_bus(m_axi_mem_rresp_bus), \ + .m_axi_mem_bvalid_bus(m_axi_mem_bvalid_bus), \ + .m_axi_mem_bready_bus(m_axi_mem_bready_bus), \ + .m_axi_mem_bresp_bus(m_axi_mem_bresp_bus), \ + .m_axi_mem_bid_bus(m_axi_mem_bid_bus) + `include "VX_define.vh" `endif // VORTEX_AFU_VH diff --git a/hw/rtl/afu/foldsim/target/foldsim_target.sv b/hw/rtl/afu/foldsim/target/foldsim_target.sv new file mode 100644 index 0000000000..271d97ad3b --- /dev/null +++ b/hw/rtl/afu/foldsim/target/foldsim_target.sv @@ -0,0 +1,152 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "vortex_afu.vh" + +// The foldsim target: Vortex_axi with its port surface flattened to packed, +// scalar ports, the boundary foldsim channelizes. +// +// Vortex_axi declares its AXI channels as SystemVerilog unpacked arrays indexed +// by bank; each of its VX_CFG_PLATFORM_MEMORY_NUM_BANKS banks is broken out into +// discrete m_axi_mem__* signals. The +// port shape and the array conversion are the same ones the XRT AFU uses, taken +// from its macros so the two cannot drift apart. +// +// The control surface (DCR request/response, start, busy) is already scalar and +// passes straight through, so the control endpoint drives it directly rather +// than through an AXI4-Lite control block. +module foldsim_target import VX_gpu_pkg::*; #( + parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, + parameter C_M_AXI_MEM_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, + parameter C_M_AXI_MEM_ADDR_WIDTH = 64, + parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS +) ( + input wire clk, + input wire reset, + + `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + + // DCR write request + input wire dcr_req_valid, + input wire dcr_req_rw, + input wire [VX_DCR_ADDR_WIDTH-1:0] dcr_req_addr, + input wire [VX_DCR_DATA_WIDTH-1:0] dcr_req_data, + + // DCR read response + output wire dcr_rsp_valid, + output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + + // ctrl/status + input wire start, + output wire busy +); + localparam VX_MEM_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH; + + wire m_axi_mem_awvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_awready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [VX_MEM_ADDR_WIDTH-1:0] m_axi_mem_awaddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_awid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_awlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_wvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_wdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH/8-1:0] m_axi_mem_wstrb_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_wlast_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_bvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_bready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_bid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_bresp_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_arvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_arready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [VX_MEM_ADDR_WIDTH-1:0] m_axi_mem_araddr_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_arid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [7:0] m_axi_mem_arlen_a [C_M_AXI_MEM_NUM_BANKS]; + + wire m_axi_mem_rvalid_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rready_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_DATA_WIDTH-1:0] m_axi_mem_rdata_a [C_M_AXI_MEM_NUM_BANKS]; + wire m_axi_mem_rlast_a [C_M_AXI_MEM_NUM_BANKS]; + wire [C_M_AXI_MEM_ID_WIDTH-1:0] m_axi_mem_rid_a [C_M_AXI_MEM_NUM_BANKS]; + wire [1:0] m_axi_mem_rresp_a [C_M_AXI_MEM_NUM_BANKS]; + + `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_TO_ARRAY, MP_SEMI); + + Vortex_axi #( + .AXI_DATA_WIDTH (C_M_AXI_MEM_DATA_WIDTH), + .AXI_ADDR_WIDTH (VX_MEM_ADDR_WIDTH), + .AXI_TID_WIDTH (C_M_AXI_MEM_ID_WIDTH), + .AXI_NUM_BANKS (C_M_AXI_MEM_NUM_BANKS) + ) vortex_axi ( + .clk (clk), + .reset (reset), + + .m_axi_awvalid (m_axi_mem_awvalid_a), + .m_axi_awready (m_axi_mem_awready_a), + .m_axi_awaddr (m_axi_mem_awaddr_a), + .m_axi_awid (m_axi_mem_awid_a), + .m_axi_awlen (m_axi_mem_awlen_a), + .m_axi_awsize (), + .m_axi_awburst (), + .m_axi_awlock (), + .m_axi_awcache (), + .m_axi_awprot (), + .m_axi_awqos (), + .m_axi_awregion (), + + .m_axi_wvalid (m_axi_mem_wvalid_a), + .m_axi_wready (m_axi_mem_wready_a), + .m_axi_wdata (m_axi_mem_wdata_a), + .m_axi_wstrb (m_axi_mem_wstrb_a), + .m_axi_wlast (m_axi_mem_wlast_a), + + .m_axi_bvalid (m_axi_mem_bvalid_a), + .m_axi_bready (m_axi_mem_bready_a), + .m_axi_bid (m_axi_mem_bid_a), + .m_axi_bresp (m_axi_mem_bresp_a), + + .m_axi_arvalid (m_axi_mem_arvalid_a), + .m_axi_arready (m_axi_mem_arready_a), + .m_axi_araddr (m_axi_mem_araddr_a), + .m_axi_arid (m_axi_mem_arid_a), + .m_axi_arlen (m_axi_mem_arlen_a), + .m_axi_arsize (), + .m_axi_arburst (), + .m_axi_arlock (), + .m_axi_arcache (), + .m_axi_arprot (), + .m_axi_arqos (), + .m_axi_arregion (), + + .m_axi_rvalid (m_axi_mem_rvalid_a), + .m_axi_rready (m_axi_mem_rready_a), + .m_axi_rdata (m_axi_mem_rdata_a), + .m_axi_rlast (m_axi_mem_rlast_a), + .m_axi_rid (m_axi_mem_rid_a), + .m_axi_rresp (m_axi_mem_rresp_a), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (start), + .busy (busy) + ); + +endmodule diff --git a/hw/rtl/afu/opae/vortex_afu.sv b/hw/rtl/afu/opae/vortex_afu.sv index 61ad02a8e4..5e7ca4cecb 100644 --- a/hw/rtl/afu/opae/vortex_afu.sv +++ b/hw/rtl/afu/opae/vortex_afu.sv @@ -740,6 +740,19 @@ module vortex_afu import ccip_if_pkg::*; import local_mem_cfg_pkg::*; import VX_ .dcr_rsp_valid (vx_dcr_rsp_valid), .dcr_rsp_data (vx_dcr_rsp_data), + .cp_mem_req_valid (1'b0), + .cp_mem_req_rw (1'b0), + .cp_mem_req_noalloc (1'b0), + .cp_mem_req_addr ('0), + .cp_mem_req_data ('0), + .cp_mem_req_byteen ('0), + .cp_mem_req_tag ('0), + `UNUSED_PIN (cp_mem_req_ready), + `UNUSED_PIN (cp_mem_rsp_valid), + `UNUSED_PIN (cp_mem_rsp_data), + `UNUSED_PIN (cp_mem_rsp_tag), + .cp_mem_rsp_ready (1'b1), + .start (vx_start), .busy (vx_busy) ); diff --git a/hw/rtl/afu/xrt/vortex_afu.v b/hw/rtl/afu/xrt/vortex_afu.v index 6e3bd65a4d..05863d8791 100644 --- a/hw/rtl/afu/xrt/vortex_afu.v +++ b/hw/rtl/afu/xrt/vortex_afu.v @@ -21,22 +21,14 @@ module vortex_afu #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - parameter C_M_AXI_MEM_NUM_BANKS = 1 -`else - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS -`endif + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, - // AXI4 master interface -`ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, GEN_AXI_MEM, MP_COMMA), -`else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), -`endif + // AXI4 master interface, one per GPU memory port + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -75,6 +67,9 @@ module vortex_afu #( reset_r <= ~ap_rst_n; end + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -85,11 +80,7 @@ module vortex_afu #( ) afu_wrap ( .clk (ap_clk), .reset (reset_r), - `ifdef PLATFORM_MERGED_MEMORY_INTERFACE - `MP_REPEAT (1, AXI_MEM_ARGS, MP_COMMA), - `else - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), - `endif + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, .s_axi_ctrl_awvalid (s_axi_ctrl_awvalid), .s_axi_ctrl_awready (s_axi_ctrl_awready), diff --git a/hw/rtl/cache/VX_cache.sv b/hw/rtl/cache/VX_cache.sv index 7982a79cd4..def3addae7 100644 --- a/hw/rtl/cache/VX_cache.sv +++ b/hw/rtl/cache/VX_cache.sv @@ -40,6 +40,10 @@ module VX_cache import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -48,6 +52,15 @@ module VX_cache import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, parameter IS_LLC = 0, // last-level cache: banks own AMO commit + RVA reservation tracking + parameter SHARED_LLC = 0, // the LLC serves every memory client alone (point of coherence) + parameter ACQ_FORWARD = 0, // an acquire flush also invalidates the next level + + parameter FLUSH_FORWARD = 1, // a flush also flushes the next level + + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) + + parameter READ_MERGE = 0, // merge same-word reads within groups of this many adjacent ports (0: off) + parameter READ_MERGE_LINE = 0, // merge any reads of the same line instead, each port keeping its own tag and word parameter AMO_ENABLE = 0 // synthesize atomic-op logic in the banks ) ( @@ -57,12 +70,17 @@ module VX_cache import VX_gpu_pkg::*; #( input wire clk, input wire reset, VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + // nothing held between this level's inputs and its memory side, on either path + input wire nc_req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter: number of banks must be power of 2")) `STATIC_ASSERT(WRITE_ENABLE || !WRITEBACK, ("invalid parameter: writeback requires write enable")) `STATIC_ASSERT(WRITEBACK || !DIRTY_BYTES, ("invalid parameter: dirty bytes require writeback")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || WRITEBACK, ("invalid parameter: write-around requires a write-back cache")) `STATIC_ASSERT(NUM_BANKS >= MEM_PORTS, ("invalid parameter: number of banks must be greater or equal to number of memory ports")) + `STATIC_ASSERT(!BANK_HASH || NUM_BANKS <= 256, ("invalid parameter: bank hash supports up to 256 banks")) localparam REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS); localparam WORD_SEL_WIDTH = `UP(`CS_WORD_SEL_BITS); localparam MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE); @@ -73,17 +91,46 @@ module VX_cache import VX_gpu_pkg::*; #( localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); localparam LINE_ADDR_WIDTH = (`CS_WORD_ADDR_WIDTH - BANK_SEL_BITS - WORD_SEL_BITS); - localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + `UP(MEM_ATTR_WIDTH); + // Read merging carries the merged ports' mask (one bit per port of the + // leader's group) below the request tag through the bank. + localparam MERGE_GROUP = (READ_MERGE != 0) ? `MIN(READ_MERGE, NUM_REQS) : 1; + localparam DO_READ_MERGE = (MERGE_GROUP > 1); + // Line merging also carries every group port's {tag, word select}, one + // slot per port above the mask, and the bank answers with one word per slot. + localparam MERGE_LINE = DO_READ_MERGE && (READ_MERGE_LINE != 0); + localparam SLOT_WIDTH = TAG_WIDTH + WORD_SEL_WIDTH; + localparam SLOTS_WIDTH = MERGE_LINE ? (MERGE_GROUP * SLOT_WIDTH) : 0; + localparam RSP_WORDS = MERGE_LINE ? MERGE_GROUP : 1; + localparam BANK_TAG_WIDTH = TAG_WIDTH + SLOTS_WIDTH + (DO_READ_MERGE ? MERGE_GROUP : 0); + localparam CORE_REQ_DATAW = LINE_ADDR_WIDTH + 1 + WORD_SEL_WIDTH + WORD_SIZE + WORD_WIDTH + BANK_TAG_WIDTH + `UP(MEM_ATTR_WIDTH); localparam CORE_RSP_DATAW = WORD_WIDTH + TAG_WIDTH; localparam BANK_MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH; - localparam MEM_REQ_DATAW = (`CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); + localparam MEM_REQ_DATAW = (`UP(BANK_SEL_BITS) + `CS_LINE_SECTOR_ADDR_WIDTH + 1 + SECTOR_SIZE + `CS_SECTOR_WIDTH + BANK_MEM_TAG_WIDTH + `UP(MEM_ATTR_WIDTH)); localparam MEM_RSP_DATAW = `CS_SECTOR_WIDTH + MEM_TAG_WIDTH; localparam MEM_PORTS_SEL_BITS = `CLOG2(MEM_PORTS); localparam MEM_PORTS_SEL_WIDTH = `UP(MEM_PORTS_SEL_BITS); localparam MEM_ARB_SEL_BITS = `CLOG2(`CDIV(NUM_BANKS, MEM_PORTS)); localparam REQ_XBAR_BUF = (NUM_REQS > 2) ? 2 : 0; + // A bank's request arbiter splits a fan-in wider than 1.5 x MAX_FANOUT + // into registered two-entry slices, where requests from different inputs + // wait apart: entries per bank over all slice levels (0 without slices). + function automatic integer req_xbar_slice_slots(integer n); + integer m, slots; + m = n; + slots = 0; + while (m > (`MAX_FANOUT + `MAX_FANOUT / 2)) begin + m = `CDIV(m, `MAX_FANOUT); + slots = slots + 2 * m; + end + return slots; + endfunction + localparam REQ_XBAR_SLICE_SLOTS = req_xbar_slice_slots(NUM_REQS); + // requests the crossbar can hold when slices let them pass each other; + // the flush sequencer drains them before it walks or answers a token + localparam REQ_XBAR_SIZE = (REQ_XBAR_SLICE_SLOTS != 0) ? (NUM_BANKS * (REQ_XBAR_SLICE_SLOTS + `TO_OUT_BUF_SIZE(REQ_XBAR_BUF))) : 0; localparam CORE_RSP_BUF_ENABLE = (NUM_BANKS != 1) || (NUM_REQS != 1); localparam MEM_REQ_BUF_ENABLE = (NUM_BANKS != 1); + `STATIC_ASSERT(!DO_READ_MERGE || (`IS_POW2(MERGE_GROUP) && `IS_DIVISBLE(NUM_REQS, MERGE_GROUP)), ("invalid parameter: read merge group")) `ifdef PERF_ENABLE wire [NUM_BANKS-1:0] perf_read_miss_per_bank; wire [NUM_BANKS-1:0] perf_write_miss_per_bank; @@ -95,23 +142,57 @@ module VX_cache import VX_gpu_pkg::*; #( .TAG_WIDTH (TAG_WIDTH) ) core_bus2_if[NUM_REQS](); wire [NUM_BANKS-1:0] per_bank_flush_begin; + wire flush_skip; wire [`UP(UUID_WIDTH)-1:0] flush_uuid; wire [NUM_BANKS-1:0] per_bank_flush_end; + wire [NUM_REQS-1:0] core_req_xbar_fire; wire [NUM_BANKS-1:0] per_bank_core_req_fire; + wire mem_req_empty; + wire [MEM_PORTS-1:0] mem_flush_valid; + wire mem_flush_l1_only; + wire mem_flush_io; + wire mem_flush_noalloc; + wire [MEM_PORTS-1:0] mem_flush_ready; + wire mem_flush_rsp_phase; + wire [MEM_PORTS-1:0] mem_flush_rsp_valid; + wire flush_rsp_valid; + wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx; + wire [TAG_WIDTH-1:0] flush_rsp_tag; + wire flush_rsp_ready; VX_cache_init #( - .NUM_REQS (NUM_REQS), - .NUM_BANKS (NUM_BANKS), - .TAG_WIDTH (TAG_WIDTH), - .BANK_SEL_LATENCY (`TO_OUT_BUF_REG(REQ_XBAR_BUF)) // request xbar latency + .NUM_REQS (NUM_REQS), + .NUM_BANKS (NUM_BANKS), + .MEM_PORTS (MEM_PORTS), + .FLUSH_FORWARD (FLUSH_FORWARD), + .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), + .TAG_WIDTH (TAG_WIDTH), + .XBAR_SIZE (REQ_XBAR_SIZE), + .REQ_SEL_WIDTH (REQ_SEL_WIDTH) ) cache_init ( - .clk (clk), - .reset (reset), - .core_bus_in_if (core_bus_if), - .core_bus_out_if (core_bus2_if), - .bank_req_fire (per_bank_core_req_fire), - .flush_begin (per_bank_flush_begin), - .flush_uuid (flush_uuid), - .flush_end (per_bank_flush_end) + .clk (clk), + .reset (reset), + .core_bus_in_if (core_bus_if), + .core_bus_out_if (core_bus2_if), + .xbar_req_fire (core_req_xbar_fire), + .bank_req_fire (per_bank_core_req_fire), + .flush_begin (per_bank_flush_begin), + .flush_skip (flush_skip), + .flush_uuid (flush_uuid), + .flush_end (per_bank_flush_end), + .mem_req_empty (mem_req_empty), + .nc_req_empty (nc_req_empty), + .mem_flush_valid (mem_flush_valid), + .mem_flush_l1_only (mem_flush_l1_only), + .mem_flush_io (mem_flush_io), + .mem_flush_noalloc (mem_flush_noalloc), + .mem_flush_ready (mem_flush_ready), + .mem_flush_rsp_phase (mem_flush_rsp_phase), + .mem_flush_rsp_valid (mem_flush_rsp_valid), + .flush_rsp_valid (flush_rsp_valid), + .flush_rsp_idx (flush_rsp_idx), + .flush_rsp_tag (flush_rsp_tag), + .flush_rsp_ready (flush_rsp_ready) ); VX_mem_bus_if #( .DATA_SIZE (SECTOR_SIZE), @@ -161,6 +242,15 @@ module VX_cache import VX_gpu_pkg::*; #( assign mem_rsp_queue_sel[i] = 0; end end + // While the flush tokens are out the cache is quiescent (inputs blocked, + // MSHRs empty), so every memory response is a token's. + wire [MEM_PORTS-1:0] mem_rsp_xbar_valid; + wire [MEM_PORTS-1:0] mem_rsp_xbar_ready; + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_flush_rsp + assign mem_flush_rsp_valid[i] = mem_flush_rsp_phase && mem_rsp_queue_valid[i]; + assign mem_rsp_xbar_valid[i] = mem_rsp_queue_valid[i] && ~mem_flush_rsp_phase; + assign mem_rsp_queue_ready[i] = mem_flush_rsp_phase || mem_rsp_xbar_ready[i]; + end wire [NUM_BANKS-1:0] per_bank_mem_rsp_valid; wire [NUM_BANKS-1:0][MEM_RSP_DATAW-MEM_ARB_SEL_BITS-1:0] per_bank_mem_rsp_pdata; wire [NUM_BANKS-1:0] per_bank_mem_rsp_ready; @@ -175,15 +265,16 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_rsp_xbar ( .clk (clk), .reset (reset), - .valid_in (mem_rsp_queue_valid), + .valid_in (mem_rsp_xbar_valid), .data_in (mem_rsp_queue_data_s), .sel_in (mem_rsp_queue_sel), - .ready_in (mem_rsp_queue_ready), + .ready_in (mem_rsp_xbar_ready), .valid_out (per_bank_mem_rsp_valid), .data_out (per_bank_mem_rsp_pdata), `UNUSED_PIN (sel_out), .ready_out (per_bank_mem_rsp_ready), - `UNUSED_PIN (collisions) + `UNUSED_PIN (collisions), + `UNUSED_PIN (empty) ); wire [NUM_BANKS-1:0][`CS_SECTOR_WIDTH-1:0] per_bank_mem_rsp_data; wire [NUM_BANKS-1:0][BANK_MEM_TAG_WIDTH-1:0] per_bank_mem_rsp_tag; @@ -199,14 +290,15 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_BANKS-1:0][WORD_SEL_WIDTH-1:0] per_bank_core_req_wsel; wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_core_req_byteen; wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_req_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_req_tag; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_req_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_req_idx; wire [NUM_BANKS-1:0][`UP(MEM_ATTR_WIDTH)-1:0] per_bank_core_req_attr; wire [NUM_BANKS-1:0] per_bank_core_req_ready; wire [NUM_BANKS-1:0] per_bank_core_rsp_valid; - wire [NUM_BANKS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_core_rsp_tag; + wire [NUM_BANKS-1:0][RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] per_bank_core_rsp_data; + wire [NUM_BANKS-1:0][BANK_TAG_WIDTH-1:0] per_bank_core_rsp_tag; wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_core_rsp_idx; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] per_bank_core_rsp_targets; wire [NUM_BANKS-1:0] per_bank_core_rsp_ready; wire [NUM_BANKS-1:0] per_bank_mem_req_valid; wire [NUM_BANKS-1:0][`CS_LINE_SECTOR_ADDR_WIDTH-1:0] per_bank_mem_req_addr; @@ -251,11 +343,83 @@ module VX_cache import VX_gpu_pkg::*; #( end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_bid if (NUM_BANKS > 1) begin : g_multibanks - assign core_req_bid[i] = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + wire [BANK_SEL_BITS-1:0] raw_bid = core_req_addr[i][WORD_SEL_BITS +: BANK_SEL_BITS]; + if (BANK_HASH != 0) begin : g_hash + assign core_req_bid[i] = raw_bid ^ BANK_SEL_BITS'(cache_bank_hash(32'(core_req_line_addr[i]), BANK_SEL_BITS)); + end else begin : g_raw + assign core_req_bid[i] = raw_bid; + end end else begin : g_singlebank assign core_req_bid[i] = '0; end end + // Read merging: plain reads of the same word with the same tag (or, under + // line merging, of the same line) from ports of one group in the same cycle + // are served by one bank access. The lowest such port leads; the others + // stay out of the request crossbar and are accepted in the cycle their + // leader is, so every port keeps its own request order into the bank. + wire [NUM_REQS-1:0] core_req_follower; + wire [NUM_REQS-1:0] core_req_xbar_ready; + wire [NUM_REQS-1:0][BANK_TAG_WIDTH-1:0] core_req_bank_tag; + if (DO_READ_MERGE) begin : g_read_merge + // a plain read has no flush, I/O, local, OM or atomic attribute + localparam [`UP(MEM_ATTR_WIDTH)-1:0] SPECIAL_ATTRS = `UP(MEM_ATTR_WIDTH)'((1 << MEM_ATTR_FLUSH_OFFS) + | (1 << MEM_ATTR_IO_OFFS) + | (1 << MEM_ATTR_LOCAL_OFFS) + | (1 << MEM_ATTR_OM_OFFS) + | (1 << MEM_ATTR_AMO_OFFS)); + wire [NUM_REQS-1:0] plain_read; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_plain_read + assign plain_read[i] = core_req_valid[i] && ~core_req_rw[i] && ((core_req_attr[i] & SPECIAL_ATTRS) == '0); + end + // same_word[i][j]: port i reads what a lower port j of its group reads + wire [NUM_REQS-1:0][NUM_REQS-1:0] same_word; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_same_i + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_same_j + if (j < i && (j / MERGE_GROUP) == (i / MERGE_GROUP)) begin : g_cmp + if (MERGE_LINE) begin : g_line + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_line_addr[i] == core_req_line_addr[j]) + && (core_req_bid[i] == core_req_bid[j]); + end else begin : g_word + assign same_word[i][j] = plain_read[i] && plain_read[j] + && (core_req_addr[i] == core_req_addr[j]) + && (core_req_tag[i] == core_req_tag[j]); + end + end else begin : g_none + assign same_word[i][j] = 1'b0; + end + end + end + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_merge + localparam GROUP_BASE = (i / MERGE_GROUP) * MERGE_GROUP; + wire [MERGE_GROUP-1:0] merged_mask; + wire [NUM_REQS-1:0] leader_fire; + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_merged_mask + assign merged_mask[k] = same_word[GROUP_BASE + k][i]; + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_leader_fire + assign leader_fire[j] = same_word[i][j] && ~core_req_follower[j] && core_req_xbar_ready[j]; + end + wire [`UP(SLOTS_WIDTH)-1:0] slots; + if (MERGE_LINE) begin : g_slots + for (genvar k = 0; k < MERGE_GROUP; ++k) begin : g_slot + assign slots[k * SLOT_WIDTH +: SLOT_WIDTH] = {core_req_tag[GROUP_BASE + k], core_req_wsel[GROUP_BASE + k]}; + end + assign core_req_bank_tag[i] = {core_req_tag[i], slots, merged_mask}; + end else begin : g_no_slots + assign slots = '0; + `UNUSED_VAR (slots) + assign core_req_bank_tag[i] = {core_req_tag[i], merged_mask}; + end + assign core_req_follower[i] = (| same_word[i]); + assign core_req_ready[i] = core_req_follower[i] ? (| leader_fire) : core_req_xbar_ready[i]; + end + end else begin : g_no_read_merge + assign core_req_follower = '0; + assign core_req_bank_tag = core_req_tag; + assign core_req_ready = core_req_xbar_ready; + end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_data_in assign core_req_data_in[i] = { core_req_line_addr[i], @@ -263,11 +427,13 @@ module VX_cache import VX_gpu_pkg::*; #( core_req_wsel[i], core_req_byteen[i], core_req_data[i], - core_req_tag[i], + core_req_bank_tag[i], core_req_attr[i] }; end - assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_mem_req_ready; + // a merged read enters the crossbar once, as its leader + assign core_req_xbar_fire = core_req_valid & ~core_req_follower & core_req_xbar_ready; + assign per_bank_core_req_fire = per_bank_core_req_valid & per_bank_core_req_ready; `ifdef PERF_ENABLE wire [PERF_CTR_BITS-1:0] perf_collisions; `endif @@ -286,10 +452,10 @@ module VX_cache import VX_gpu_pkg::*; #( `else `UNUSED_PIN(collisions), `endif - .valid_in (core_req_valid), + .valid_in (core_req_valid & ~core_req_follower), .data_in (core_req_data_in), .sel_in (core_req_bid), - .ready_in (core_req_ready), + .ready_in (core_req_xbar_ready), .valid_out (per_bank_core_req_valid), .data_out (core_req_data_out), .sel_out (per_bank_core_req_idx), @@ -309,6 +475,7 @@ module VX_cache import VX_gpu_pkg::*; #( for (genvar bank_id = 0; bank_id < NUM_BANKS; ++bank_id) begin : g_banks VX_cache_bank #( .BANK_ID (bank_id), + .BANK_HASH (BANK_HASH), .INSTANCE_ID (`SFORMATF(("%s-bank%0d", INSTANCE_ID, bank_id))), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -320,13 +487,19 @@ module VX_cache import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), .MRSQ_SIZE (MRSQ_SIZE), .MREQ_SIZE (MREQ_SIZE), .LATENCY (LATENCY), - .TAG_WIDTH (TAG_WIDTH), + .MERGE_GROUP (DO_READ_MERGE ? MERGE_GROUP : 0), + .TAG_WIDTH (BANK_TAG_WIDTH), + .RSP_WORDS (RSP_WORDS), + .RSP_WSEL_OFFS (DO_READ_MERGE ? MERGE_GROUP : 0), + .RSP_WSEL_STRIDE (SLOT_WIDTH), .CORE_OUT_BUF (CORE_RSP_BUF_ENABLE ? 3 : 0), .MEM_OUT_BUF (MEM_REQ_BUF_ENABLE ? 3 : 0), .IS_LLC (IS_LLC), @@ -354,6 +527,7 @@ module VX_cache import VX_gpu_pkg::*; #( .core_rsp_data (per_bank_core_rsp_data[bank_id]), .core_rsp_tag (per_bank_core_rsp_tag[bank_id]), .core_rsp_idx (per_bank_core_rsp_idx[bank_id]), + .core_rsp_targets (per_bank_core_rsp_targets[bank_id]), .core_rsp_ready (per_bank_core_rsp_ready[bank_id]), .mem_req_valid (per_bank_mem_req_valid[bank_id]), .mem_req_addr (per_bank_mem_req_addr[bank_id]), @@ -368,6 +542,7 @@ module VX_cache import VX_gpu_pkg::*; #( .mem_rsp_tag (per_bank_mem_rsp_tag[bank_id]), .mem_rsp_ready (per_bank_mem_rsp_ready[bank_id]), .flush_begin (per_bank_flush_begin[bank_id]), + .flush_skip (flush_skip), .flush_uuid (flush_uuid), .flush_end (per_bank_flush_end[bank_id]) ); @@ -377,27 +552,121 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_rsp_ready; wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_bank_core_rsp_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_core_rsp_pdata - assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i], per_bank_core_rsp_tag[i]}; + assign per_bank_core_rsp_pdata[i] = {per_bank_core_rsp_data[i][0], per_bank_core_rsp_tag[i][BANK_TAG_WIDTH-1 -: TAG_WIDTH]}; + end + // A flush token's answer takes bank 0's lane into the response crossbar + // when the bank has no response of its own, so it leaves through the same + // registered outputs as every other response. + wire [NUM_BANKS-1:0] core_rsp_xbar_valid; + wire [NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] core_rsp_xbar_pdata; + wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] core_rsp_xbar_idx; + wire [NUM_BANKS-1:0] core_rsp_xbar_ready; + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_core_rsp_xbar_in + if (i == 0) begin : g_flush_rsp + wire flush_rsp_sel = flush_rsp_valid && ~per_bank_core_rsp_valid[0]; + assign core_rsp_xbar_valid[0] = per_bank_core_rsp_valid[0] || flush_rsp_valid; + assign core_rsp_xbar_pdata[0] = flush_rsp_sel ? {`CS_WORD_WIDTH'(0), flush_rsp_tag} : per_bank_core_rsp_pdata[0]; + assign core_rsp_xbar_idx[0] = flush_rsp_sel ? flush_rsp_idx : per_bank_core_rsp_idx[0]; + assign per_bank_core_rsp_ready[0] = core_rsp_xbar_ready[0]; + assign flush_rsp_ready = core_rsp_xbar_ready[0] && ~per_bank_core_rsp_valid[0]; + end else begin : g_bank_rsp + assign core_rsp_xbar_valid[i] = per_bank_core_rsp_valid[i]; + assign core_rsp_xbar_pdata[i] = per_bank_core_rsp_pdata[i]; + assign core_rsp_xbar_idx[i] = per_bank_core_rsp_idx[i]; + assign per_bank_core_rsp_ready[i] = core_rsp_xbar_ready[i]; + end + end + if (DO_READ_MERGE) begin : g_core_rsp_mcast + // A merged read's response goes to its leader's port and to every + // merged port: each output takes it when its arbiter grants it, and the + // bank's response is released once all its targets have taken it. + // the targets carry the port index; only line merging reads it, for the leader + `UNUSED_VAR (core_rsp_xbar_idx) + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_pending; + wire [NUM_BANKS-1:0][NUM_REQS-1:0] lane_taken; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_valid; + wire [NUM_REQS-1:0][NUM_BANKS-1:0] per_output_ready; + wire [NUM_REQS-1:0][NUM_BANKS-1:0][CORE_RSP_DATAW-1:0] per_output_pdata; + // Under line merging each port takes its own slot's word and tag; the + // leader keeps the lane's tag, which a flush answer also uses. + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_output_pdata + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lane + if (MERGE_LINE) begin : g_slot + localparam K = j % MERGE_GROUP; + wire [TAG_WIDTH-1:0] lane_tag = core_rsp_xbar_pdata[i][TAG_WIDTH-1:0]; + wire [TAG_WIDTH-1:0] slot_tag = per_bank_core_rsp_tag[i][MERGE_GROUP + K * SLOT_WIDTH + WORD_SEL_WIDTH +: TAG_WIDTH]; + wire is_leader = (core_rsp_xbar_idx[i] == REQ_SEL_WIDTH'(j)); + assign per_output_pdata[j][i] = {per_bank_core_rsp_data[i][K], (is_leader ? lane_tag : slot_tag)}; + end else begin : g_word + assign per_output_pdata[j][i] = core_rsp_xbar_pdata[i]; + end + end + end + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_lanes + wire [NUM_REQS-1:0] targets; + if (i == 0) begin : g_flush_targets + assign targets = per_bank_core_rsp_valid[0] ? per_bank_core_rsp_targets[0] + : (NUM_REQS'(1) << flush_rsp_idx); + end else begin : g_bank_targets + assign targets = per_bank_core_rsp_targets[i]; + end + reg [NUM_REQS-1:0] sent_r; + assign lane_pending[i] = targets & ~sent_r & {NUM_REQS{core_rsp_xbar_valid[i]}}; + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_taken + assign per_output_valid[j][i] = lane_pending[i][j]; + assign lane_taken[i][j] = lane_pending[i][j] && per_output_ready[j][i]; + end + assign core_rsp_xbar_ready[i] = core_rsp_xbar_valid[i] && ((lane_pending[i] & ~lane_taken[i]) == '0); + always @(posedge clk) begin + if (reset) begin + sent_r <= '0; + end else if (core_rsp_xbar_valid[i]) begin + sent_r <= core_rsp_xbar_ready[i] ? '0 : (sent_r | lane_taken[i]); + end + end + end + for (genvar j = 0; j < NUM_REQS; ++j) begin : g_outputs + VX_stream_arb #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (1), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_arb ( + .clk (clk), + .reset (reset), + .valid_in (per_output_valid[j]), + .data_in (per_output_pdata[j]), + .ready_in (per_output_ready[j]), + .valid_out (core_rsp_valid[j]), + .data_out (core_rsp_pdata[j]), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready[j]) + ); + end + end else begin : g_core_rsp_xbar + `UNUSED_VAR (per_bank_core_rsp_targets) + VX_stream_omega #( + .NUM_INPUTS (NUM_BANKS), + .NUM_OUTPUTS (NUM_REQS), + .DATAW (CORE_RSP_DATAW), + .ARBITER ("R"), + .OUT_BUF (CORE_OUT_BUF) + ) core_rsp_xbar ( + .clk (clk), + .reset (reset), + .valid_in (core_rsp_xbar_valid), + .data_in (core_rsp_xbar_pdata), + .sel_in (core_rsp_xbar_idx), + .ready_in (core_rsp_xbar_ready), + .valid_out (core_rsp_valid), + .data_out (core_rsp_pdata), + `UNUSED_PIN (sel_out), + .ready_out (core_rsp_ready), + `UNUSED_PIN (collisions), + `UNUSED_PIN (empty) + ); end - VX_stream_omega #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (CORE_RSP_DATAW), - .ARBITER ("R"), - .OUT_BUF (CORE_OUT_BUF) - ) core_rsp_xbar ( - .clk (clk), - .reset (reset), - .valid_in (per_bank_core_rsp_valid), - .data_in (per_bank_core_rsp_pdata), - .sel_in (per_bank_core_rsp_idx), - .ready_in (per_bank_core_rsp_ready), - .valid_out (core_rsp_valid), - .data_out (core_rsp_pdata), - `UNUSED_PIN (sel_out), - .ready_out (core_rsp_ready), - `UNUSED_PIN (collisions) - ); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_rsp assign core_bus2_if[i].rsp_valid = core_rsp_valid[i]; assign {core_bus2_if[i].rsp_data.data, core_bus2_if[i].rsp_data.tag} = core_rsp_pdata[i]; @@ -405,7 +674,16 @@ module VX_cache import VX_gpu_pkg::*; #( end wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] per_bank_mem_req_pdata; for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_mem_req_pdata + // the raw bank bits of the memory address, formed while the bank id is a constant + wire [`UP(BANK_SEL_BITS)-1:0] raw_bid; + if (NUM_BANKS > 1 && BANK_HASH != 0) begin : g_hash + assign raw_bid = BANK_SEL_BITS'(i) ^ BANK_SEL_BITS'(cache_bank_hash( + 32'(per_bank_mem_req_addr[i][`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]), BANK_SEL_BITS)); + end else begin : g_raw + assign raw_bid = `UP(BANK_SEL_BITS)'(i); + end assign per_bank_mem_req_pdata[i] = { + raw_bid, per_bank_mem_req_addr[i], per_bank_mem_req_rw[i], per_bank_mem_req_byteen[i], @@ -426,6 +704,66 @@ module VX_cache import VX_gpu_pkg::*; #( wire [MEM_PORTS-1:0] mem_req_tmp_valid; wire [MEM_PORTS-1:0][`UP(`LOG2UP(NUM_BANKS))-1:0] mem_req_tmp_idx; wire [MEM_PORTS-1:0] mem_req_tmp_ready; + // The flush token for port i is a read of the port's first line with the + // flush attribute; byteen bit 0 carries the L1-only kind and the io + // attribute the IO kind. It takes bank + // i's lane into the request crossbar (bank i maps to port i), so the + // crossbar forms its address and it leaves through the same registered + // outputs as the write-backs ahead of it. + wire [NUM_BANKS-1:0] mem_req_xbar_valid; + wire [NUM_BANKS-1:0][MEM_REQ_DATAW-1:0] mem_req_xbar_pdata; + wire [NUM_BANKS-1:0] mem_req_xbar_ready; + + // Debug: a write-through cache holds each write of bank 0 for a fixed + // delay before it leaves, so requests through the other banks overtake it + // unless something orders them. + wire [NUM_BANKS-1:0] dbg_mem_req_hold; +`ifdef DBG_MEM_BANK0_WDELAY + if (WRITEBACK == 0) begin : g_dbg_wdelay + localparam DBG_WDELAY = 2048; + reg [`CLOG2(DBG_WDELAY+1)-1:0] dbg_wdelay_cnt; + wire dbg_wait = per_bank_mem_req_valid[0] && per_bank_mem_req_rw[0]; + always @(posedge clk) begin + if (reset || ~dbg_wait || (mem_req_xbar_valid[0] && mem_req_xbar_ready[0])) begin + dbg_wdelay_cnt <= '0; + end else if (dbg_wdelay_cnt != DBG_WDELAY) begin + dbg_wdelay_cnt <= dbg_wdelay_cnt + 1; + end + end + assign dbg_mem_req_hold = NUM_BANKS'(dbg_wait && (dbg_wdelay_cnt != DBG_WDELAY)); + end else begin : g_no_dbg_wdelay + assign dbg_mem_req_hold = '0; + end +`else + assign dbg_mem_req_hold = '0; +`endif + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_mem_req_xbar_in + if (i < MEM_PORTS) begin : g_flush_req + // line 0 hashes to 0, so the raw bank bits are the lane's bank id + wire [MEM_REQ_DATAW-1:0] mem_flush_pdata = { + `UP(BANK_SEL_BITS)'(i), + `CS_LINE_SECTOR_ADDR_WIDTH'(0), + 1'b0, + SECTOR_SIZE'(mem_flush_l1_only), + `CS_SECTOR_WIDTH'(0), + BANK_MEM_TAG_WIDTH'(0), + (`UP(MEM_ATTR_WIDTH)'(1) << MEM_ATTR_FLUSH_OFFS) | (`UP(MEM_ATTR_WIDTH)'(mem_flush_io) << MEM_ATTR_IO_OFFS) + | (`UP(MEM_ATTR_WIDTH)'(mem_flush_noalloc) << MEM_ATTR_NOALLOC_OFFS) + }; + assign mem_req_xbar_valid[i] = (per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]) || mem_flush_valid[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_valid[i] ? per_bank_mem_req_pdata[i] : mem_flush_pdata; + assign mem_flush_ready[i] = mem_req_xbar_ready[i] && ~per_bank_mem_req_valid[i]; + `RUNTIME_ASSERT (~(mem_flush_valid[i] && per_bank_mem_req_valid[i]), ("%t: %s flush token collides with a bank %0d request", $time, INSTANCE_ID, i)) + end else begin : g_bank_req + assign mem_req_xbar_valid[i] = per_bank_mem_req_valid[i] && ~dbg_mem_req_hold[i]; + assign mem_req_xbar_pdata[i] = per_bank_mem_req_pdata[i]; + end + assign per_bank_mem_req_ready[i] = mem_req_xbar_ready[i] && ~dbg_mem_req_hold[i]; + end + // Write-backs from different banks are not ordered with each other inside + // the crossbar, so the tokens wait until it has drained. + wire mem_req_xbar_empty; + assign mem_req_empty = (FLUSH_FORWARD != 0) ? mem_req_xbar_empty : 1'b1; VX_stream_omega #( .NUM_INPUTS (NUM_BANKS), .NUM_OUTPUTS (MEM_PORTS), @@ -435,17 +773,19 @@ module VX_cache import VX_gpu_pkg::*; #( ) mem_req_xbar ( .clk (clk), .reset (reset), - .valid_in (per_bank_mem_req_valid), - .data_in (per_bank_mem_req_pdata), + .valid_in (mem_req_xbar_valid), + .data_in (mem_req_xbar_pdata), .sel_in (per_bank_mem_req_bid), - .ready_in (per_bank_mem_req_ready), + .ready_in (mem_req_xbar_ready), .valid_out (mem_req_tmp_valid), .data_out (mem_req_pdata), .sel_out (mem_req_tmp_idx), .ready_out (mem_req_tmp_ready), - `UNUSED_PIN (collisions) + `UNUSED_PIN (collisions), + .empty (mem_req_xbar_empty) ); for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_tmp_if + wire [`UP(BANK_SEL_BITS)-1:0] mem_req_raw_bid_w; wire mem_req_rw_w; wire [`CS_LINE_SECTOR_ADDR_WIDTH-1:0] mem_req_addr_w; wire [SECTOR_SIZE-1:0] mem_req_byteen_w; @@ -453,6 +793,7 @@ module VX_cache import VX_gpu_pkg::*; #( wire [BANK_MEM_TAG_WIDTH-1:0] mem_req_tag_w; wire [`UP(MEM_ATTR_WIDTH)-1:0] mem_req_attr_w; assign { + mem_req_raw_bid_w, mem_req_addr_w, mem_req_rw_w, mem_req_byteen_w, @@ -476,17 +817,16 @@ module VX_cache import VX_gpu_pkg::*; #( if (NUM_BANKS != MEM_PORTS) begin : g_arb_sel wire [MEM_ARB_SEL_BITS-1:0] mem_req_arb_sel; assign mem_req_arb_sel = mem_req_tmp_idx[i][`LOG2UP(NUM_BANKS)-1 -: MEM_ARB_SEL_BITS]; - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_tmp_idx[i]}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = {mem_req_tag_w, mem_req_arb_sel}; end else begin : g_no_arb_sel `UNUSED_VAR (mem_req_tmp_idx) - assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, MEM_PORTS_SEL_WIDTH'(i)}) << `CS_SECTOR_SEL_BITS) - | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); assign mem_bus_tmp_if[i].req_data.tag = MEM_TAG_WIDTH'(mem_req_tag_w); end + assign mem_bus_tmp_if[i].req_data.addr = (`CS_MEM_SECTOR_ADDR_WIDTH'({mem_req_line_w, mem_req_raw_bid_w}) << `CS_SECTOR_SEL_BITS) + | `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_sec_w); end else begin : g_singlebank `UNUSED_VAR (mem_req_tmp_idx) + `UNUSED_VAR (mem_req_raw_bid_w) `UNUSED_VAR (mem_req_line_w) `UNUSED_VAR (mem_req_sec_w) assign mem_bus_tmp_if[i].req_data.addr = `CS_MEM_SECTOR_ADDR_WIDTH'(mem_req_addr_w); @@ -506,7 +846,8 @@ module VX_cache import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] perf_core_writes_per_req; wire [NUM_REQS-1:0] perf_crsp_stall_per_req; wire [MEM_PORTS-1:0] perf_mem_stall_per_port; - `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw); + // a merged read counts once, as the bank access that serves it + `BUFFER(perf_core_reads_per_req, core_req_valid & core_req_ready & ~core_req_rw & ~core_req_follower); `BUFFER(perf_core_writes_per_req, core_req_valid & core_req_ready & core_req_rw); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_perf_crsp_stall_per_req assign perf_crsp_stall_per_req[i] = core_bus_if[i].rsp_valid && ~core_bus_if[i].rsp_ready; diff --git a/hw/rtl/cache/VX_cache_amo.sv b/hw/rtl/cache/VX_cache_amo.sv index 63a857d330..34c119f42f 100644 --- a/hw/rtl/cache/VX_cache_amo.sv +++ b/hw/rtl/cache/VX_cache_amo.sv @@ -282,13 +282,15 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // Lane select and intra-lane bit offset, split from the encoded byte // offset. The AMO's byteen always sits inside one lane (natural // alignment), so the lane of its lowest set byte is the lane of all of - // them (asserted below). + // them (asserted below). A naturally aligned 4- or 8-byte operand starts + // on a 32-bit boundary, so the offset keeps only its word bit: the + // operand and result shifts are a select, not a funnel. reg [LANE_IDXW-1:0] lane_st1; reg [LANE_OFF_BITS-1:0] bit_off_st1; always @(posedge clk) begin if (~pipe_stall) begin lane_st1 <= (NUM_LANES > 1) ? LANE_IDXW'(byte_off_n >> `CLOG2(LANE_SIZE)) : '0; - bit_off_st1 <= LANE_OFF_BITS'({byte_off_n[`UP(`CLOG2(LANE_SIZE))-1:0], 3'b0}); + bit_off_st1 <= LANE_OFF_BITS'(((32'(byte_off_n) % LANE_SIZE) >> 2) << 5); end end // This AMO's byteen slice within its lane. @@ -322,17 +324,13 @@ module VX_cache_amo import VX_gpu_pkg::*; #( wire [WBQ_SIZE-1:0][NUM_LANES-1:0] wbq_lane_hit; wire [NUM_LANES-1:0] post_lane_hit; - if ((PIPE_EX != 0) && (NUM_LANES > 1)) begin : g_hit_reg - // A deferred-commit bank has the next commit-stage key a cycle - // early. Where the word spans several lanes the forward merge is - // wide enough for the {line, word} compare to set the response - // path, so the lane hits are registered and the forward network - // and the response start from flops: each is taken between the - // queue's next contents and the request entering the commit stage - // next cycle, with the slot's occupancy folded in. The queue moves - // during a stall while the commit request holds, so both keys are - // compared and the stall selects. A single-lane word keeps the - // live compare, which is cheaper there. + // The lane hits are registered so the forward network and the + // response start from flops: each is taken between the queue's next + // contents and the request entering the commit stage next cycle (S0 + // itself when the commit is not deferred), with the slot's occupancy + // folded in. The queue moves during a stall while the commit request + // holds, so both keys are compared and the stall selects. + if (1) begin : g_hit_reg wire [WBQ_SIZE-1:0] wbq_key_n; for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq_key_n assign wbq_key_n[i] = (wbq_addr[i] == res_addr_n) && (wbq_wsel[i] == word_idx_n); @@ -386,17 +384,6 @@ module VX_cache_amo import VX_gpu_pkg::*; #( end assign wbq_lane_hit = wbq_lane_hit_r; assign post_lane_hit = post_lane_hit_r; - end else begin : g_hit_live - for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane - assign wbq_lane_hit[i][l] = (WBQ_CNTW'(i) < wbq_count) && wbq_key_st1[i] - && (wbq_lane[i] == LANE_IDXW'(l)); - end - end - for (genvar l = 0; l < NUM_LANES; ++l) begin : g_post_lane - assign post_lane_hit[l] = post_wb_valid && post_key_st1 && (post_wb_lane == LANE_IDXW'(l)); - end - `UNUSED_VAR (word_idx_n) end wire [WBQ_SIZE-1:0] wbq_word_hit; @@ -585,13 +572,54 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // independent of the drain: the highest match is the target either way // unless it is the draining head, so wb_fire, which arrives late with // the bank's stall, only picks between the two precomputed outcomes. + // The {line, word, lane} keys against the compute stage are taken a + // cycle early, when it loads from the commit stage, against each slot + // and the settling entry as they stand after this cycle's update, so + // the target select and the merge start from flops. A result pushes + // only the cycle after its load, the one cycle these keys are used. + wire [WBQ_SIZE-1:0] coal_eq_same, coal_eq_up; + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_eq + assign coal_eq_same[i] = (wbq_addr[i] == addr_st1) && (wbq_wsel[i] == word_idx_st1) + && (wbq_lane[i] == lane_st1); + if (i < WBQ_SIZE-1) begin : g_up + assign coal_eq_up[i] = (wbq_addr[i+1] == addr_st1) && (wbq_wsel[i+1] == word_idx_st1) + && (wbq_lane[i+1] == lane_st1); + end else begin : g_last + assign coal_eq_up[i] = coal_eq_same[i]; + end + end + wire coal_eq_cmp = (cmp_addr == addr_st1) && (cmp_wsel == word_idx_st1) && (cmp_lane == lane_st1); + wire coal_eq_post = (post_wb_addr == addr_st1) && (post_wb_wsel == word_idx_st1) + && (post_wb_lane == lane_st1); + reg [WBQ_SIZE-1:0] coal_key_r; + reg post_coal_key_r; + always @(posedge clk) begin + for (integer i = 0; i < WBQ_SIZE; ++i) begin + if (wb_push && (wb_slot == WBQ_IDXW'(i))) begin + coal_key_r[i] <= coal_eq_cmp; + end else if (wb_fire) begin + coal_key_r[i] <= coal_eq_up[i]; + end else begin + coal_key_r[i] <= coal_eq_same[i]; + end + end + post_coal_key_r <= wb_fire ? coal_eq_same[0] : coal_eq_post; + end + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_coal_check + `RUNTIME_ASSERT (~wb_push || (coal_key_r[i] == ((wbq_addr[i] == cmp_addr) && (wbq_wsel[i] == cmp_wsel) + && (wbq_lane[i] == cmp_lane))), + ("%t: AMO coalesce key out of sync (slot=%0d)", $time, i)) + end + `RUNTIME_ASSERT (~wb_push || (post_coal_key_r == ((post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) + && (post_wb_lane == cmp_lane))), + ("%t: AMO settling-entry coalesce key out of sync", $time)) + reg [WBQ_SIZE-1:0] wb_coal_hit; reg [WBQ_IDXW-1:0] wb_coal_idx; // pre-shift index of the coalesce target always @(*) begin wb_coal_idx = '0; for (integer i = 0; i < WBQ_SIZE; ++i) begin - wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && (wbq_addr[i] == cmp_addr) - && (wbq_wsel[i] == cmp_wsel) && (wbq_lane[i] == cmp_lane); + wb_coal_hit[i] = (WBQ_CNTW'(i) < wbq_count) && coal_key_r[i]; if (wb_coal_hit[i]) begin wb_coal_idx = WBQ_IDXW'(i); end @@ -613,10 +641,68 @@ module VX_cache_amo import VX_gpu_pkg::*; #( // than it: drop those bytes from the settling entry, so the forward // never has two writers for one byte. On a drain the settling entry // is reloaded from the head, which a same-key push never coalesces into. - wire post_push_post = wb_push && (post_wb_addr == cmp_addr) && (post_wb_wsel == cmp_wsel) - && (post_wb_lane == cmp_lane); - wire post_push_head = wb_push && (wbq_addr[0] == cmp_addr) && (wbq_wsel[0] == cmp_wsel) - && (wbq_lane[0] == cmp_lane); + wire post_push_post = wb_push && post_coal_key_r; + wire post_push_head = wb_push && coal_key_r[0]; + + // Writeback queue: a drain (wb_fire) shifts every entry toward the head; + // a completed compute (wb_push) enqueues at the tail or byte-merges into + // its word's entry, written after the shift so it wins when both target + // the same slot. Both handshakes follow the bank's stall, the latest + // signals here, so each slot's next entry is formed for either drain + // outcome from early terms (target slot, coalesce, merged bytes) and + // the drain, then the push, select last. + wire [WBQ_SIZE-1:0][LINE_ADDR_BITS-1:0] wbq_addr_n; + wire [WBQ_SIZE-1:0][WORD_SEL_WIDTH-1:0] wbq_wsel_n; + wire [WBQ_SIZE-1:0][LANE_IDXW-1:0] wbq_lane_n; + wire [WBQ_SIZE-1:0][LANE_SIZE-1:0] wbq_byteen_n; + wire [WBQ_SIZE-1:0][LANE_WIDTH-1:0] wbq_data_n; + wire [WBQ_SIZE-1:0][TAG_WIDTH-1:0] wbq_tag_n; + wire [WBQ_SIZE-1:0][REQ_SEL_WIDTH-1:0] wbq_idx_n; + wire [WBQ_SIZE-1:0][ATTR_WIDTH-1:0] wbq_attr_n; + wire [LANE_SIZE-1:0] pbyteen_d = cmp_byteen | (wb_coalesce_drain ? coal_src_byteen : {LANE_SIZE{1'b0}}); + wire [LANE_SIZE-1:0] pbyteen_h = cmp_byteen | (wb_coalesce_hold ? coal_src_byteen : {LANE_SIZE{1'b0}}); + for (genvar i = 0; i < WBQ_SIZE; ++i) begin : g_wbq_next + // a drain moves slot i+1 into slot i; the last slot keeps its entry + localparam J = (i < WBQ_SIZE-1) ? (i + 1) : i; + wire [LANE_SIZE-1:0] byteen_d = (i < WBQ_SIZE-1) + ? (wbq_byteen[J] & ~(wbq_clr_hit[J] ? clr_byteen[J] : {LANE_SIZE{1'b0}})) + : wbq_byteen[i]; + wire [LANE_SIZE-1:0] byteen_h = wbq_byteen[i] & ~(wbq_clr_hit[i] ? clr_byteen[i] : {LANE_SIZE{1'b0}}); + wire [LANE_WIDTH-1:0] data_d = wbq_data[J]; + wire [LANE_WIDTH-1:0] data_h = wbq_data[i]; + wire [LANE_WIDTH-1:0] pdata_d, pdata_h; + for (genvar b = 0; b < LANE_SIZE; ++b) begin : g_bytes + assign pdata_d[b*8 +: 8] = cmp_byteen[b] ? wb_data_w[b*8 +: 8] + : (wb_coalesce_drain ? coal_src_data[b*8 +: 8] : data_d[b*8 +: 8]); + assign pdata_h[b*8 +: 8] = cmp_byteen[b] ? wb_data_w[b*8 +: 8] + : (wb_coalesce_hold ? coal_src_data[b*8 +: 8] : data_h[b*8 +: 8]); + end + wire push_d = wb_push && (wb_slot_drain == WBQ_IDXW'(i)); + wire push_h = wb_push && (wb_slot_hold == WBQ_IDXW'(i)); + assign wbq_addr_n[i] = wb_fire ? (push_d ? cmp_addr : wbq_addr[J]) : (push_h ? cmp_addr : wbq_addr[i]); + assign wbq_wsel_n[i] = wb_fire ? (push_d ? cmp_wsel : wbq_wsel[J]) : (push_h ? cmp_wsel : wbq_wsel[i]); + assign wbq_lane_n[i] = wb_fire ? (push_d ? cmp_lane : wbq_lane[J]) : (push_h ? cmp_lane : wbq_lane[i]); + assign wbq_byteen_n[i] = wb_fire ? (push_d ? pbyteen_d : byteen_d) : (push_h ? pbyteen_h : byteen_h); + assign wbq_data_n[i] = wb_fire ? (push_d ? pdata_d : data_d) : (push_h ? pdata_h : data_h); + assign wbq_tag_n[i] = wb_fire ? (push_d ? cmp_tag : wbq_tag[J]) : (push_h ? cmp_tag : wbq_tag[i]); + assign wbq_idx_n[i] = wb_fire ? (push_d ? cmp_idx : wbq_idx[J]) : (push_h ? cmp_idx : wbq_idx[i]); + assign wbq_attr_n[i] = wb_fire ? (push_d ? cmp_attr : wbq_attr[J]) : (push_h ? cmp_attr : wbq_attr[i]); + end + + always @(posedge clk) begin + if (~reset) begin + for (integer i = 0; i < WBQ_SIZE; ++i) begin + wbq_addr[i] <= wbq_addr_n[i]; + wbq_wsel[i] <= wbq_wsel_n[i]; + wbq_lane[i] <= wbq_lane_n[i]; + wbq_byteen[i] <= wbq_byteen_n[i]; + wbq_data[i] <= wbq_data_n[i]; + wbq_tag[i] <= wbq_tag_n[i]; + wbq_idx[i] <= wbq_idx_n[i]; + wbq_attr[i] <= wbq_attr_n[i]; + end + end + end always @(posedge clk) begin if (reset) begin @@ -663,45 +749,6 @@ module VX_cache_amo import VX_gpu_pkg::*; #( cmp_valid <= 1'b0; end - // Writeback queue: a drain (wb_fire) shifts every entry toward - // the head; a completed compute (wb_push) enqueues at the tail or - // byte-merges into its word's entry. The push is written after - // the shift so it wins when both target the same slot. - if (wb_fire) begin - for (integer i = 0; i < WBQ_SIZE-1; ++i) begin - wbq_addr[i] <= wbq_addr[i+1]; - wbq_wsel[i] <= wbq_wsel[i+1]; - wbq_lane[i] <= wbq_lane[i+1]; - wbq_byteen[i] <= wbq_byteen[i+1] & ~(wbq_clr_hit[i+1] ? clr_byteen[i+1] : {LANE_SIZE{1'b0}}); - wbq_data[i] <= wbq_data[i+1]; - wbq_tag[i] <= wbq_tag[i+1]; - wbq_idx[i] <= wbq_idx[i+1]; - wbq_attr[i] <= wbq_attr[i+1]; - end - end else begin - for (integer i = 0; i < WBQ_SIZE; ++i) begin - if (wbq_clr_hit[i]) begin - wbq_byteen[i] <= wbq_byteen[i] & ~clr_byteen[i]; - end - end - end - if (wb_push) begin - wbq_addr[wb_slot] <= cmp_addr; - wbq_wsel[wb_slot] <= cmp_wsel; - wbq_lane[wb_slot] <= cmp_lane; - wbq_tag[wb_slot] <= cmp_tag; - wbq_idx[wb_slot] <= cmp_idx; - wbq_attr[wb_slot] <= cmp_attr; - // byte-merge: this AMO's bytes take the new result; a coalesce - // keeps the target entry's other bytes; a fresh entry zero-fills. - wbq_byteen[wb_slot] <= cmp_byteen | (wb_coalesce ? coal_src_byteen : {LANE_SIZE{1'b0}}); - for (integer b = 0; b < LANE_SIZE; ++b) begin - if (cmp_byteen[b]) - wbq_data[wb_slot][b*8 +: 8] <= wb_data_w[b*8 +: 8]; - else if (wb_coalesce) - wbq_data[wb_slot][b*8 +: 8] <= coal_src_data[b*8 +: 8]; - end - end // Count grows only on a new (non-coalescing) enqueue; a coalesce // updates in place. Pop removes the head. wbq_count <= wbq_count_n; @@ -820,6 +867,11 @@ module VX_cache_amo import VX_gpu_pkg::*; #( `RUNTIME_ASSERT (~(amo_st1.amo_valid && valid_st1 && is_creq_st1) || ((byteen_st1 & ~(WORD_SIZE'({LANE_SIZE{1'b1}}) << (lane_st1 * LANE_SIZE))) == '0), ("%t: AMO byteen crosses its lane (addr=0x%0h)", $time, addr_st1)) + // The word-granular offset rests on natural alignment as well. + `RUNTIME_ASSERT (~(amo_st1.amo_valid && valid_st1 && is_creq_st1) + || (&lane_byteen_st1) || (lane_byteen_st1 == LANE_SIZE'(4'hF)) + || ((LANE_SIZE == 8) && (lane_byteen_st1 == LANE_SIZE'(8'hF0))), + ("%t: AMO operand is not a naturally aligned word or doubleword (addr=0x%0h)", $time, addr_st1)) // The narrowed arithmetic width rests on the ISA: on a 32-bit hart // only the Zacas pair carries a 64-bit operand. if (`MIN(`VX_CFG_XLEN, AMO_OLD_BITS) < AMO_OLD_BITS) begin : g_arith_width_guard diff --git a/hw/rtl/cache/VX_cache_bank.sv b/hw/rtl/cache/VX_cache_bank.sv index 28576fcd39..5bef208111 100644 --- a/hw/rtl/cache/VX_cache_bank.sv +++ b/hw/rtl/cache/VX_cache_bank.sv @@ -16,6 +16,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter NUM_REQS = 1, parameter CACHE_SIZE = 1024, // cache size in bytes parameter LINE_SIZE = 16, // line size in bytes @@ -30,6 +31,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter WRITE_ENABLE = 1, parameter WRITEBACK = 0, parameter DIRTY_BYTES = 0, + parameter LAZY_FETCH = 0, // write miss allocates without a fill; a load to a partial sector refetches + parameter WRITE_AROUND = 0, // a write miss flagged no-allocate is sent to memory instead parameter REPL_POLICY = `CS_REPL_FIFO, parameter TAG_WIDTH = UUID_WIDTH + 1, parameter CORE_OUT_BUF = 0, @@ -37,6 +40,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( parameter IS_LLC = 0, // last-level cache: AMOs commit locally here parameter AMO_ENABLE = 0, // synthesize atomic-op logic parameter LATENCY = 2, // pipeline depth; >2 defers the data array to break the tag->data path + // Words per core response. Above 1, response word k is the line's word + // whose select rides in the request tag at RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE. + parameter RSP_WORDS = 1, + parameter RSP_WSEL_OFFS = 0, + parameter RSP_WSEL_STRIDE = 0, + parameter MERGE_GROUP = 0, // read-merge group: the tag's low bits mark the merged ports (0: none) parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE), parameter MEM_TAG_WIDTH = UUID_WIDTH + MSHR_ADDR_WIDTH, parameter REQ_SEL_WIDTH = `UP(`CS_REQ_SEL_BITS), @@ -66,9 +75,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Core response output wire core_rsp_valid, - output wire [`CS_WORD_WIDTH-1:0] core_rsp_data, + output wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] core_rsp_data, output wire [TAG_WIDTH-1:0] core_rsp_tag, output wire [REQ_SEL_WIDTH-1:0] core_rsp_idx, + output wire [NUM_REQS-1:0] core_rsp_targets, // the ports the response goes to input wire core_rsp_ready, // Memory request (sector-granular; = line when 1 sector/line) @@ -89,6 +99,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Flush input wire flush_begin, + input wire flush_skip, input wire [`UP(UUID_WIDTH)-1:0] flush_uuid, output wire flush_end ); @@ -101,6 +112,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // core-rsp queue depth (pow2): registered-skid minimum (2) plus CRSQ_SIZE extra slots localparam CRSQ_QUEUE_SIZE = 1 << `CLOG2(2 + CRSQ_SIZE); `STATIC_ASSERT(LATENCY >= 2, ("invalid parameter: cache bank LATENCY must be >= 2")) + `STATIC_ASSERT(!LAZY_FETCH || (WRITEBACK && DIRTY_BYTES && IS_LLC), ("invalid parameter: lazy fetch requires a write-back LLC with dirty bytes")) + `STATIC_ASSERT(!WRITE_AROUND || (WRITEBACK && WRITE_ENABLE), ("invalid parameter: write-around requires a write-back cache")) + `STATIC_ASSERT(RSP_WORDS == 1 || (!WRITE_ENABLE && !AMO_ENABLE), ("invalid parameter: multi-word responses are read-only")) `UNUSED_PARAM (MRSQ_SIZE) // ======================================================================== @@ -150,6 +164,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( lookup_t lk; } commit_t; + // Lazy-fetch request flavors ride a side pipeline beside the request; it + // exists only when the feature is enabled. + typedef struct packed { + logic is_lazy; // data-less fill of a write-miss root + logic is_refetch; // fill whose root refetches a partial sector + logic is_retry; // refetch re-issued after its sector was evicted + logic is_park; // replay parked on a partial sector: issues its refetch + logic is_loop; // fill taken from the loopback queue + } lazy_req_t; + data_t sel_req, st0, dat_in, stD; // request + fill line: sel -> S0 -> stD commit_t cmt_in, st1, stC; // request + lookup delta: S0 -> S1 -> stC lookup_t lk_st0; // S0 lookup results @@ -158,7 +182,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // Shared signals // ------------------------------------------------------------------------ wire crsp_queue_stall, mshr_alm_full; - wire mshr_allocate_st0, fill_inflight; + wire mshr_allocate_st0; (* max_fanout = 64 *) wire pipe_stall; // Authoritative "no entry held", straight off the MSHR's valid mask. The // pending-size counter below is a proxy: it is fed by separate increment and @@ -172,6 +196,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`CS_LINE_ADDR_WIDTH-1:0] mem_rsp_addr; wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mem_rsp_sector; // sector this fill installs wire [MSHR_ADDR_WIDTH-1:0] mshr_alloc_id, mshr_previd; + wire mshr_fill_refetch, mshr_fill_retry, mshr_dequeue_reads; + wire lazyq_empty, lazyq_alm_full, lazyq_full; wire mshr_pending_raw; wire mshr_pending_wr_raw; wire mshr_pending_ovl_raw; @@ -215,8 +241,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire fwd_head, fwd_fire, fwd_pending; wire [`CS_WORD_WIDTH-1:0] fwd_word; - // Fill buffer: the sector staged at fill accept owns all in-flight fill - // data; it feeds the data-array fill port and the forward-response word. + // Fill buffer: the sector staged at fill accept; it feeds the forward- + // response word and enters the fill-data delay line to the array write. reg [`CS_SECTOR_WIDTH-1:0] fbuf_data_r; reg [`CS_LINE_ADDR_WIDTH-1:0] fbuf_addr_r; @@ -242,7 +268,6 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire do_read_stc = stC.req.valid && stC.req.is_creq && ~stC.req.rw; wire do_write_stc = stC.req.valid && stC.req.is_creq && stC.req.rw; - wire do_init_std = stD.req.valid && stD.req.is_init; wire do_fill_std = stD.req.valid && stD.req.is_fill; wire do_flush_std = stD.req.valid && stD.req.is_flush; wire do_read_std = stD.req.valid && stD.req.is_creq && ~stD.req.rw; @@ -283,10 +308,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign pipe_inflight = st0.req.valid || st1.req.valid || (| commit_valid); end // Queued AMO results are still writes to land in the array. - wire no_pending_req = ~pipe_inflight && mreq_queue_empty && ~amo_commit_busy; + wire no_pending_req = ~pipe_inflight && mreq_queue_empty && lazyq_empty && ~amo_commit_busy; VX_cache_flush #( - .BANK_ID (BANK_ID), .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), .NUM_BANKS (NUM_BANKS), @@ -296,6 +320,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .flush_begin (flush_begin), + .flush_skip (flush_skip), .flush_end (flush_end), .flush_init (init_valid), .flush_valid (flush_valid), @@ -351,11 +376,41 @@ module VX_cache_bank import VX_gpu_pkg::*; #( && (replay_rw || st0.req.rw) && (replay_wsel == st0.req.word_idx); + // Lazy fetch: a write-miss root's fill loops back (lazyq) as a data-less + // fill below memory fills. A load/AMO head of a partial chain parks and + // refetches. A refetch merges into its resident sector one stage late, at + // stC; one whose sector was evicted installs nothing and retries as a + // plain fill (nothing can dirty an absent sector, so its data is current). + // A refetch whose sector is already fully written has nothing to read: it + // loops back as a data-less refetch instead of going to memory. + wire lazyq_valid = ~lazyq_empty; + wire [MSHR_ADDR_WIDTH-1:0] lazyq_id; + wire lazy_sel = ~mem_rsp_valid && lazyq_valid; + wire chain_partial, chain_retry; + // Lazy-fetch request flavors, folded to constants when the feature is off. + lazy_req_t lz_sel, lz_st0, lz_st1, lz_stc; + wire lazy_st0 = lz_st0.is_lazy; + wire refetch_st0 = lz_st0.is_refetch; + wire retry_st0 = lz_st0.is_retry; + wire park_st0 = lz_st0.is_park; + wire park_stc = lz_stc.is_park; + wire refetch_stc = lz_stc.is_refetch; + wire retry_stc = lz_stc.is_retry; + `UNUSED_VAR ({lz_st1, lz_stc.is_lazy}) + wire park_sel = (LAZY_FETCH != 0) && chain_partial && mshr_dequeue_reads; + wire refetch_hold = st0.req.valid && st0.req.is_fill && refetch_st0; + // The chain's head waits out a refetch at S0, and a head that would park + // waits for memory-request room; nothing else is selected meanwhile. Built + // from registers only and applied at the MSHR output, off the dequeue cone. + wire replay_block = refetch_hold || (park_sel && mreq_queue_alm_full); + // A fill starts a replay chain, and a chain carries at most one AMO (no // request enters behind a pending AMO), so a fill waits until the AMO // writeback queue has room for it; the queue drains meanwhile. wire replay_mux = replay_valid && ~fwd_head && ~replay_link_hold; - wire fill_mux = mem_rsp_valid && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full; + wire refetch_inflight; + wire fill_mux = (mem_rsp_valid || lazyq_valid) && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full + && ~refetch_inflight && ~replay_block; wire replay_grant = ~init_valid; wire replay_enable = replay_grant && replay_mux; @@ -373,41 +428,36 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire creq_enable = creq_grant && (amo_creq_path || amo_wb_path); assign replay_ready = replay_grant && ~fwd_head && ~replay_link_hold && ~(!WRITEBACK && replay_rw && mreq_queue_alm_full) && ~pipe_stall; - assign mem_rsp_ready = fill_grant && ~fwd_pending && ~fill_inflight && ~replay_link_hold && ~amo_wbq_full && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; + assign mem_rsp_ready = fill_grant && ~fwd_pending && ~replay_link_hold && ~amo_wbq_full && ~refetch_inflight + && ~replay_block && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; assign flush_ready = flush_grant && ~(WRITEBACK && mreq_queue_alm_full) && ~pipe_stall; - assign core_req_ready = creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall + // a core request may become a write-miss root, which pushes to lazyq. + wire creq_alm_full = mreq_queue_alm_full || lazyq_alm_full || replay_block; + // A queued AMO result drains through the core-request slot, while a full + // AMO writeback queue holds off the fills that drain the lazy queue. So + // the writeback is not held to the lazy queue's pipeline reservation when + // the pipe is empty: then only the writeback itself can push, and a free + // slot is room enough. + wire lazyq_wb_block = lazyq_alm_full && (pipe_inflight || lazyq_full); + wire amo_wb_alm_full = mreq_queue_alm_full || lazyq_wb_block || replay_block; + assign core_req_ready = creq_grant && ~creq_alm_full && ~mshr_alm_full && ~pipe_stall && ~amo_commit_busy && ~req_input_defer; wire init_fire = init_valid; wire replay_fire = replay_valid && replay_ready; wire mem_rsp_fire = mem_rsp_valid && mem_rsp_ready; + wire lazy_fire = lazy_sel && mem_rsp_ready; + wire fill_fire = mem_rsp_fire || lazy_fire; + wire park_fire = replay_fire && park_sel; wire flush_fire = flush_valid && flush_ready; // amo_wb_path already excludes the cycle a fresh AMO commits at S1, so the // writeback never races the chain update. - wire amo_wb_fire = amo_wb_path && creq_grant && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; - wire core_req_fire = (amo_creq_path || amo_wb_path) && creq_grant - && ~mreq_queue_alm_full && ~mshr_alm_full && ~pipe_stall; + wire amo_wb_fire = amo_wb_path && creq_grant && ~amo_wb_alm_full && ~mshr_alm_full && ~pipe_stall; + wire core_req_fire = ((amo_creq_path && ~creq_alm_full) || (amo_wb_path && ~amo_wb_alm_full)) + && creq_grant && ~mshr_alm_full && ~pipe_stall; wire [MSHR_ADDR_WIDTH-1:0] mem_rsp_id = mem_rsp_tag[MSHR_ADDR_WIDTH-1:0]; - - // A fill's sector lives only in the staging buffer until its data-array - // write at stD, so with deferred stages (PIPE_EX>0) a back-to-back fill - // must be held off while one is in flight. At PIPE_EX=0 the array write - // samples the buffer on the same edge a new fill re-stages it (old value - // read), so no interlock is needed. - if (PIPE_EX > 0) begin : g_fill_inflight - reg [PIPE_EX-1:0] fill_busy; - always @(posedge clk) begin - if (reset) begin - fill_busy <= '0; - end else if (~pipe_stall) begin - fill_busy <= PIPE_EX'({fill_busy, (mem_rsp_fire && ~is_passthru_fill_sel)}); - end - end - assign fill_inflight = (| fill_busy); - end else begin : g_no_fill_inflight - assign fill_inflight = 1'b0; - end + wire [MSHR_ADDR_WIDTH-1:0] fill_id = lazy_sel ? lazyq_id : mem_rsp_id; // Fill word_idx: place the installed sector in the high (sector) bits of the // in-line word offset; low (word-in-sector) bits are don't-care for a fill. @@ -441,8 +491,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign flush_tag = '0; end - // Word payload mux (store / AMO-writeback data). Fill data does not ride - // the pipeline: the staged fill buffer feeds the data array directly. + // Word payload mux (store / AMO-writeback data). Fill data takes its own + // delay line from the staged fill buffer to the data array. wire [`CS_WORD_WIDTH-1:0] wdata_sel; if (WRITE_ENABLE) begin : g_wdata_sel assign wdata_sel = replay_mux ? replay_data @@ -457,9 +507,20 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // pending wb (chained AMO replays from MSHR after a fill) and must not be // cleared by amo_wb_pending; the synthetic writeback carries amo.valid=0 so // it never re-commits at S1. + // The core-request slot's address: a pending LLC AMO writeback or the core + // request. The MSHR takes its allocate match against it a cycle early. + wire [`CS_LINE_ADDR_WIDTH-1:0] creq_addr_sel = amo_wb_pending ? amo_wb_addr : core_req_addr; + wire [WORD_SEL_WIDTH-1:0] creq_wsel_sel = amo_wb_pending ? amo_wb_word_idx : core_req_wsel; + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] creq_sector_sel; + if (`CS_SECTOR_SEL_BITS != 0) begin : g_creq_sector + assign creq_sector_sel = creq_wsel_sel[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; + end else begin : g_creq_sector_0 + assign creq_sector_sel = '0; + end + always @(*) begin sel_req = '0; - sel_req.req.valid = init_fire || replay_fire || mem_rsp_fire || flush_fire || core_req_fire; + sel_req.req.valid = init_fire || replay_fire || fill_fire || flush_fire || core_req_fire; sel_req.req.is_init = init_valid; sel_req.req.is_fill = fill_enable; sel_req.req.is_flush = flush_enable; @@ -470,17 +531,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( sel_req.req.attr = amo_wb_pending ? amo_wb_attr : (core_req_valid ? core_req_attr : '0); sel_req.req.way_idx = flush_way; sel_req.req.addr = (init_valid | flush_valid) ? `CS_LINE_ADDR_WIDTH'(flush_sel) - : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr - : (amo_wb_pending ? amo_wb_addr : core_req_addr))); + : (replay_mux ? replay_addr : (fill_mux ? mem_rsp_addr : creq_addr_sel)); sel_req.req.byteen = replay_mux ? replay_byteen : (amo_wb_pending ? amo_wb_byteen : core_req_byteen); // a fill carries the installed sector in its word_idx high bits so the // tag/data stages mark/write the right sector (0 when 1 sector/line). sel_req.req.word_idx = replay_mux ? replay_wsel : (fill_mux ? fill_word_idx - : (amo_wb_pending ? amo_wb_word_idx : core_req_wsel)); + : creq_wsel_sel); sel_req.req.req_idx = replay_mux ? replay_idx : (amo_wb_pending ? amo_wb_idx : core_req_idx); sel_req.req.tag = (init_valid | flush_valid) ? (flush_valid ? flush_tag : '0) - : (replay_mux ? replay_tag : (fill_mux ? mem_rsp_tag_s + : (replay_mux ? replay_tag : (fill_mux ? (lazy_sel ? '0 : mem_rsp_tag_s) : (amo_wb_pending ? amo_wb_tag : core_req_tag))); sel_req.req.mshr_id = replay_id; sel_req.req.amo = replay_mux ? replay_amo : (amo_wb_pending ? amo_req_t'('0) @@ -489,6 +549,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // tag_matches is computed at S0; left 0 here (overridden at the data bubble). end + assign lz_sel.is_lazy = fill_enable && lazy_sel && ~mshr_fill_refetch; + assign lz_sel.is_refetch = fill_enable && mshr_fill_refetch; + assign lz_sel.is_retry = fill_enable && ~lazy_sel && mshr_fill_retry; + assign lz_sel.is_park = replay_enable && park_sel; + assign lz_sel.is_loop = fill_enable && lazy_sel; + // UUID extraction (debug + MSHR ordering): per stage, from the carried tag. wire [`UP(UUID_WIDTH)-1:0] req_uuid_sel, req_uuid_st0, req_uuid_st1, req_uuid_stc; if (UUID_WIDTH != 0) begin : g_req_uuid @@ -520,7 +586,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0] tag_matches_st0; wire [NUM_WAYS-1:0] line_present_st0; wire [`CS_WAY_SEL_WIDTH-1:0] hit_idx_st0; - wire evict_dirty_st0; + wire evict_dirty_st0, evict_valid_st0; + `UNUSED_VAR (evict_valid_st0) wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask_st0; wire [`CS_TAG_SEL_BITS-1:0] evict_tag_st0; @@ -539,6 +606,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); wire [`CS_WAY_SEL_WIDTH-1:0] fill_way_st0 = line_present_any_st0 ? present_way_st0 : victim_way; wire [`CS_WAY_SEL_WIDTH-1:0] evict_way_st0 = st0.req.is_fill ? fill_way_st0 : st0.req.way_idx; + // A refetch (first attempt) uses no victim: it merges into the way that + // holds its sector, carried apart from evict_way so that the tag compare + // stays off the array write enables. + wire refetch_first_st0 = do_fill_st0 && refetch_st0 && ~retry_st0; + wire [`CS_WAY_SEL_WIDTH-1:0] merge_way_st0 = refetch_first_st0 ? hit_idx_st0 : evict_way_st0; // Fill way as one-hot, straight from the per-way presence vector: routing // it through the binary encode -> evict_way -> per-way compare round trip @@ -564,12 +636,27 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .lookup_hit (st1.lk.is_hit), .lookup_line (st1.req.addr[`CS_LINE_SEL_BITS-1:0]), .lookup_way (st1.req.way_idx), - .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + // a refetch merges into its resident sector and uses no victim + .repl_valid (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), .repl_line (line_idx_st0), .repl_line_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .repl_way (victim_way) ); + wire creq_hit_order_hazard_st0; + + // Loads and AMOs read the sector, so a partial one is a miss for them. The + // array enables use the raw tag match (a partial sector is still resident); + // only the commit decisions, registered at S1, see the partial check. + wire needs_data_st0 = st0.req.is_creq && ~st0.req.rw; + wire partial_match_st0; + wire partial_miss_st0 = (LAZY_FETCH != 0) && needs_data_st0 && partial_match_st0; + // A refetch finds its partial sector resident (it merges) unless the + // sector was evicted meanwhile: its data may then predate the eviction's + // writeback, so it installs nothing and its chain retries. + wire fill_resident_st0 = (| tag_matches_st0); + wire fill_evicted_st0 = refetch_first_st0 && ~fill_resident_st0; + VX_cache_tags #( .CACHE_SIZE (CACHE_SIZE), .LINE_SIZE (LINE_SIZE), @@ -578,6 +665,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .WORD_SIZE (WORD_SIZE), .SECTOR_SIZE(SECTOR_SIZE), .WRITEBACK (WRITEBACK), + .LAZY_FETCH (LAZY_FETCH), .AMO_ENABLE ((AMO_ENABLE != 0) && (IS_LLC == 0)) ) cache_tags ( .clk (clk), @@ -585,23 +673,34 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .stall (pipe_stall), .init (do_init_st0), .flush (do_flush_st0 && ~pipe_stall), - .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~pipe_stall), + .fill (do_fill_st0 && ~st0.req.is_passthru_fill && ~refetch_first_st0 && ~pipe_stall), + .refetch (refetch_first_st0 && ~pipe_stall), .read (do_read_st0 && ~pipe_stall), .write (do_write_st0 && ~pipe_stall), // non-LLC AMO forwards downstream and invalidates its own copy so the - // issuer's later plain load refetches the new value. - .invalidate (is_amo_fwd_st0 && lk_st0.is_hit && ~pipe_stall), + // issuer's later plain load refetches the new value. Only the hit's + // qualifiers are passed (a forwarded AMO has no order hazard): the + // array ANDs in each way's own tag match, which keeps the OR over the + // ways off the valid store's write enables. + .invalidate (is_amo_fwd_st0 && ~partial_miss_st0 && ~pipe_stall), + .fill_nodata (lazy_st0), .line_idx (line_idx_st0), .line_idx_n (sel_req.req.addr[`CS_LINE_SEL_BITS-1:0]), .line_tag (line_tag_st0), .sector_idx (sector_idx_st0), .evict_way (evict_way_st0), + .flush_way (st0.req.way_idx), + // a writeback leaves only from a fresh fill's victim or a flushed way + .wb_way (st0.req.is_fill ? victim_way : st0.req.way_idx), .fill_way_oh (fill_way_oh_st0), + .victim_way_oh (victim_way_oh), .tag_matches (tag_matches_st0), .line_present (line_present_st0), .evict_dirty (evict_dirty_st0), + .evict_valid (evict_valid_st0), .evict_dirty_mask (evict_dirty_mask_st0), - .evict_tag (evict_tag_st0) + .evict_tag (evict_tag_st0), + .partial_match (partial_match_st0) ); VX_onehot_encoder #( @@ -624,23 +723,24 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // disjoint words — the common case in streaming, where the whole warp's // beats land on one sector — keep the fast release path, as does // read-after-read on the same word. - wire creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay - && mshr_pending_raw - && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) - && ~is_amo_fwd_st0 - && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); + // (the probes carry their own pending test; see VX_cache_mshr) + assign creq_hit_order_hazard_st0 = st0.req.is_creq && ~st0.req.is_replay + && (mshr_pending_wr_raw || (st0.req.rw && mshr_pending_ovl_raw)) + && ~is_amo_fwd_st0 + && ~((AMO_ENABLE != 0) && st0.req.amo.amo_valid); // S0 lookup delta (single combinational driver). The AMO requester is forced // non-pending so it never coalesces onto a prior same-line entry. always @(*) begin lk_st0 = '0; - lk_st0.is_hit = (| tag_matches_st0) && ~creq_hit_order_hazard_st0; + lk_st0.is_hit = (| tag_matches_st0) && ~partial_miss_st0 && ~creq_hit_order_hazard_st0; lk_st0.is_dirty = evict_dirty_st0; lk_st0.is_refill = st0.req.is_fill && line_present_any_st0; lk_st0.evict_dirty_mask = evict_dirty_mask_st0; lk_st0.evict_tag = evict_tag_st0; lk_st0.mshr_previd = mshr_previd; - lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0; + // a parked replay is its own chain's root: it issues the refetch. + lk_st0.mshr_pending = mshr_pending_raw && ~is_amo_fwd_st0 && ~park_st0; end // ======================================================================== @@ -670,7 +770,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // plus the lookup delta. The word payload is dropped here. always @(*) begin cmt_in.req = st0.req; - cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : evict_way_st0; + cmt_in.req.way_idx = st0.req.is_creq ? hit_idx_st0 : merge_way_st0; cmt_in.req.mshr_id = st0.req.is_replay ? st0.req.mshr_id : mshr_alloc_id; cmt_in.lk = lk_st0; end @@ -698,24 +798,99 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .data_out (st1) ); + // stP is the stage that feeds stC: the dirty-byte mask is read there, so + // stC gets it from a register. + commit_t stP; VX_pipe_register #( .DATAW ($bits(commit_t)), .RESETW (1), - .DEPTH (PIPE_EX) - ) reg_cmt ( + .DEPTH ((PIPE_EX != 0) ? (PIPE_EX - 1) : 0) + ) reg_cmt_p ( .clk (clk), .reset (reset), .enable (~pipe_stall), .data_in (st1), + .data_out (stP) + ); + + VX_pipe_register #( + .DATAW ($bits(commit_t)), + .RESETW (1), + .DEPTH ((PIPE_EX != 0) ? 1 : 0) + ) reg_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (stP), .data_out (stC) ); + // A refetch whose sector is still resident merges at stC; a data-less or + // refetch fill writes no data at stD. + wire fill_merge_st0 = refetch_first_st0 && fill_resident_st0; + wire fill_nodata_st0 = lazy_st0 || refetch_first_st0; + wire fill_merge_stc, fill_nodata_std, loop_std, partial_miss_stc; + if (LAZY_FETCH != 0) begin : g_lazy_pipe + wire fill_merge_st1, partial_miss_st1; + VX_pipe_register #( + .DATAW ($bits(lazy_req_t)), + .RESETW ($bits(lazy_req_t)) + ) reg_lz_s0 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (lz_sel), + .data_out (lz_st0) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2) + ) reg_lz_s1 ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st0, fill_merge_st0, partial_miss_st0}), + .data_out ({lz_st1, fill_merge_st1, partial_miss_st1}) + ); + VX_pipe_register #( + .DATAW ($bits(lazy_req_t) + 2), + .RESETW ($bits(lazy_req_t) + 2), + .DEPTH (PIPE_EX) + ) reg_lz_cmt ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st1, fill_merge_st1, partial_miss_st1}), + .data_out ({lz_stc, fill_merge_stc, partial_miss_stc}) + ); + VX_pipe_register #( + .DATAW (2), + .RESETW (2), + .DEPTH (PIPE_EX) + ) reg_lz_dat ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in ({lz_st0.is_loop, fill_nodata_st0}), + .data_out ({loop_std, fill_nodata_std}) + ); + end else begin : g_no_lazy_pipe + assign lz_st0 = '0; + assign lz_st1 = '0; + assign lz_stc = '0; + assign fill_merge_stc = 1'b0; + assign fill_nodata_std = 1'b0; + assign loop_std = 1'b0; + assign partial_miss_stc = 1'b0; + `UNUSED_VAR ({lz_sel, fill_merge_st0, fill_nodata_st0, lz_st0.is_loop, lz_stc.is_loop}) + end + // a passthru-AMO replay carries its result word instead of an installed // line, so it counts as a hit at the commit stage. The AMO commit signals // (is_amo_replay_st1) resolve at stC, so the replay-must-hit check is taken // there; with PIPE_EX>0 an st1-staged check would sample them a stage early. wire eff_hit_stc = stC.lk.is_hit || is_amo_replay_st1; - `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~eff_hit_stc), ("missed mshr replay")) + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_replay && ~park_stc && ~eff_hit_stc), ("missed mshr replay")) // ======================================================================== // Data array (driven at stD; outputs land at stC) @@ -732,10 +907,62 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign sector_idx_std = '0; end - // The staged fill sector, replicated across the line so each sector's - // slices see their words; the data array writes only the installed sector - // (per-slice gate). One copy == the full line when 1 sector/line. - wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fbuf_data_r}}; + // A fill's sector is staged when the fill enters S0 and then rides beside + // it to its data-array write at stD, so the next fill may restage the + // buffer right behind it. Each stage has one source and holds on the + // pipeline enable; without a reset, deep stages map onto shift registers. + wire [`CS_SECTOR_WIDTH-1:0] fill_data_std; + VX_pipe_register #( + .DATAW (`CS_SECTOR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_data ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_data_r), + .data_out (fill_data_std) + ); +`ifndef SYNTHESIS + wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr_std; + VX_pipe_register #( + .DATAW (`CS_LINE_ADDR_WIDTH), + .DEPTH (PIPE_EX) + ) reg_fill_addr ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (fbuf_addr_r), + .data_out (fill_addr_std) + ); + `RUNTIME_ASSERT (~(do_fill_std && ~stD.req.is_passthru_fill && ~loop_std) || (stD.req.addr == fill_addr_std), ("%t: %s fill sector misaligned with its array write: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(stD.req.addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fill_addr_std, BANK_ID))) +`endif + + // The fill sector, replicated across the line so each sector's slices see + // their words; the data array writes only the installed sector (per-slice + // gate). One copy == the full line when 1 sector/line. + wire [`CS_LINE_WIDTH-1:0] fbuf_line = {`CS_SECTORS_PER_LINE{fill_data_std}}; + + // sector of the request at commit (the missed sector of a fill request). + wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; + // a refetch merges into its resident sector at stC (see refetch_hold) + wire do_merge_stc = stC.req.valid && stC.req.is_fill && fill_merge_stc; + // Dirty-byte mask updates land at stC in pipeline order, once per request: + // a fill held at stC by its writeback beats still reads the evicted mask. + wire refetch_first_stc = stC.req.is_fill && refetch_stc && ~retry_stc; + wire mask_set_stc = do_write_stc && stC.lk.is_hit; + wire mask_fill_stc = stC.req.valid && stC.req.is_fill && ~stC.req.is_passthru_fill && ~refetch_first_stc; + wire mask_clr_line_stc = (mask_fill_stc && ~stC.lk.is_refill) || (stC.req.valid && stC.req.is_flush); + wire mask_clr_sec_stc = mask_fill_stc && stC.lk.is_refill; + + // The data slices a read returns: its word's, or under multi-word + // responses the whole line, since a merged read may take any word (and + // decoding the slots' selects here would sit on the array's enable path). + wire [`CS_WORDS_PER_LINE-1:0] read_words_std; + if (RSP_WORDS > 1) begin : g_read_words + assign read_words_std = '1; + end else begin : g_read_word + assign read_words_std = `CS_WORDS_PER_LINE'(1) << stD.req.word_idx; + end VX_cache_data #( .CACHE_SIZE (CACHE_SIZE), @@ -746,12 +973,27 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .SECTOR_SIZE (SECTOR_SIZE), .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), - .DIRTY_BYTES (DIRTY_BYTES) + .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), + .MASK_SYNC (PIPE_EX != 0) ) cache_data ( .clk (clk), .reset (reset), - .init (do_init_std), .fill (do_fill_std && ~stD.req.is_passthru_fill && ~pipe_stall), + .fill_nodata (fill_nodata_std), + .mask_write ((mask_set_stc || mask_clr_line_stc || mask_clr_sec_stc) && ~pipe_stall), + .mask_set (mask_set_stc), + .mask_clr_line (mask_clr_line_stc), + .mask_clr_sector (mask_clr_sec_stc), + .mask_way (stC.req.way_idx), + .mask_line (stC.req.addr[`CS_LINE_SEL_BITS-1:0]), + .mask_word (stC.req.word_idx), + .mask_byteen (stC.req.byteen), + .mask_sector (sector_idx_stc), + .mask_read (~pipe_stall), + .mask_rway (stP.req.way_idx), + .mask_rline (stP.req.addr[`CS_LINE_SEL_BITS-1:0]), + .merge_write (do_merge_stc && ~pipe_stall), .flush (do_flush_std && ~pipe_stall), .read (do_read_std && ~pipe_stall), .write (do_write_std && ~pipe_stall), @@ -761,18 +1003,18 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .fill_data (fbuf_line), .write_word (stD.wdata), .word_idx (stD.req.word_idx), + .read_words (read_words_std), .sector_idx (sector_idx_std), - .way_idx_r (stC.req.way_idx), .write_byteen (stD.req.byteen), .read_data (read_data_stc), .evict_byteen (evict_byteen_stc) ); // stD write word delayed to stC for its consumers there (the AMO RMW - // operand and the write-through memory payload); elided when no stC - // consumer exists (write-through banks, or writeback banks with LLC AMO). + // operand and the write-through or write-around memory payload); elided + // when no stC consumer exists. wire [`CS_WORD_WIDTH-1:0] word_stc; - if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0))) begin : g_word_stc + if ((WRITE_ENABLE != 0) && (!WRITEBACK || (AMO_ENABLE != 0 && IS_LLC != 0) || (WRITE_AROUND != 0))) begin : g_word_stc VX_pipe_register #( .DATAW (`CS_WORD_WIDTH) ) reg_word_stc ( @@ -806,7 +1048,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // release the entry on a hit. wire mshr_release_st1; if (WRITEBACK) begin : g_mshr_release - assign mshr_release_st1 = st1.lk.is_hit && ~is_amo_fwd_rel; + // a write-around miss keeps nothing: it leaves for memory at stC. + wire write_around_st1 = (WRITE_AROUND != 0) && st1.req.rw && st1.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~st1.lk.is_hit && ~st1.lk.mshr_pending; + assign mshr_release_st1 = (st1.lk.is_hit || write_around_st1) && ~is_amo_fwd_rel; end else begin : g_mshr_release_ro // keep missed writes in MSHR if a pending entry exists for the line, so a // pending fill arriving without the write content replays them locally. @@ -815,7 +1060,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mshr_release_fire = mshr_finalize_st1 && mshr_release_st1 && ~pipe_stall; wire [1:0] mshr_dequeue; - `POP_COUNT(mshr_dequeue, {(replay_fire || fwd_fire), mshr_release_fire}); + `POP_COUNT(mshr_dequeue, {((replay_fire && ~park_fire) || fwd_fire), mshr_release_fire}); VX_pending_size #( .SIZE (MSHR_SIZE), @@ -835,14 +1080,15 @@ module VX_cache_bank import VX_gpu_pkg::*; #( VX_cache_mshr #( .INSTANCE_ID (`SFORMATF(("%s-mshr", INSTANCE_ID))), .BANK_ID (BANK_ID), + .BANK_HASH (BANK_HASH), .LINE_SIZE (LINE_SIZE), .SECTOR_SIZE (SECTOR_SIZE), .NUM_BANKS (NUM_BANKS), .MSHR_SIZE (MSHR_SIZE), - .WRITEBACK (WRITEBACK), .AMO_ENABLE (AMO_ENABLE != 0), .AMO_PASSTHRU ((AMO_ENABLE != 0) && (IS_LLC == 0)), .WORD_SEL_WIDTH (WORD_SEL_WIDTH), + .LAZY_FETCH (LAZY_FETCH), .DATA_WIDTH (WORD_SEL_WIDTH + WORD_SIZE + `CS_WORD_WIDTH + TAG_WIDTH + REQ_SEL_WIDTH + AMO_REQ_BITS) ) cache_mshr ( .clk (clk), @@ -850,10 +1096,12 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .deq_req_uuid (req_uuid_sel), .alc_req_uuid (req_uuid_st0), .fin_req_uuid (req_uuid_st1), - .fill_valid (mem_rsp_fire), - .fill_id (mem_rsp_id), + .fill_valid (fill_fire), + .fill_id (fill_id), .fill_addr (mem_rsp_addr), .fill_sector (mem_rsp_sector), + .fill_refetch (mshr_fill_refetch), + .fill_retry (mshr_fill_retry), .probe_addr (core_req_addr), .probe_pending_ld (mshr_probe_pending_ld), .probe_pending_amo (mshr_probe_pending_amo), @@ -864,17 +1112,26 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .dequeue_valid (replay_valid), .dequeue_addr (replay_addr), .dequeue_rw (replay_rw), + .dequeue_reads (mshr_dequeue_reads), .dequeue_data ({replay_wsel, replay_byteen, replay_data, replay_tag, replay_idx, replay_amo}), .dequeue_id (replay_id), .dequeue_ready (replay_ready || fwd_fire), + .dequeue_fwd (fwd_fire), + .dequeue_park (park_sel), + .dequeue_park_retry (chain_retry), + .dequeue_block (replay_block), .allocate_valid (mshr_allocate_st0 && ~pipe_stall), .allocate_addr (st0.req.addr), .allocate_sector (sector_idx_st0), + .allocate_addr_n (creq_addr_sel), + .allocate_sector_n (creq_sector_sel), + .allocate_adv (~pipe_stall), .allocate_word_idx (st0.req.word_idx), .allocate_rw (st0.req.rw), // Only non-LLC AMOs must not coalesce; at the LLC same-line AMOs coalesce // and serialize their commits on the single filled line. .allocate_is_amo (AMO_ENABLE ? st0.req.amo.amo_valid : 1'b0), + .allocate_refetch (needs_data_st0 && partial_match_st0), .allocate_data ({st0.req.word_idx, st0.req.byteen, write_word_st0, st0.req.tag, st0.req.req_idx, st0.req.amo}), .allocate_id (mshr_alloc_id), .allocate_pending (mshr_pending_raw), @@ -954,7 +1211,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .is_creq_st0 (st0.req.is_creq), // Atomics are exempt from the hit-order hazard, so their S0 hit is // the raw tag match; the MSHR probe stays off the commit_busy cone. - .is_hit_st0 ((| tag_matches_st0)), + .is_hit_st0 ((| tag_matches_st0) && ~partial_miss_st0), .is_replay_st0 (st0.req.is_replay), // Commit ports are fed from stC (the deferred data-output stage), so // the AMO RMW operands and the read word align at PIPE_EX>0. At @@ -1069,7 +1326,10 @@ module VX_cache_bank import VX_gpu_pkg::*; #( // ======================================================================== reg fwd_active_r; - wire fwd_stage = mem_rsp_fire && ~is_passthru_fill_sel; + // A refetch fill stages its sector for the array write but cannot answer + // the chain: the staged copy lacks the dirty bytes the array merges in. + wire fbuf_stage = mem_rsp_fire && ~is_passthru_fill_sel; + wire fwd_stage = fbuf_stage && ~mshr_fill_refetch; wire fwd_close = replay_valid && fwd_active_r && (replay_rw || replay_amo.amo_valid); @@ -1093,11 +1353,11 @@ module VX_cache_bank import VX_gpu_pkg::*; #( end else begin if (fwd_stage) begin fwd_active_r <= 1'b1; - end else if (fwd_close) begin + end else if (fwd_close || ((LAZY_FETCH != 0) && fill_fire)) begin fwd_active_r <= 1'b0; end end - if (fwd_stage) begin + if (fbuf_stage) begin fbuf_data_r <= mem_rsp_data; fbuf_addr_r <= mem_rsp_addr; end @@ -1110,11 +1370,48 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign fwd_word = fbuf_data_r[`CS_WORD_WIDTH-1:0]; end + // A multi-word response takes each word its tag selects, from the line at + // commit or from the staged sector on a forward. + wire [RSP_WORDS-1:0][`CS_WORD_WIDTH-1:0] crsp_words, fwd_words; + if (RSP_WORDS > 1) begin : g_rsp_words + for (genvar k = 0; k < RSP_WORDS; ++k) begin : g_word + wire [WORD_SEL_WIDTH-1:0] wsel_stc = stC.req.tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + wire [WORD_SEL_WIDTH-1:0] wsel_fwd = replay_tag[RSP_WSEL_OFFS + k * RSP_WSEL_STRIDE +: WORD_SEL_WIDTH]; + assign crsp_words[k] = read_data_stc[wsel_stc]; + if (`CS_WORDS_PER_SECTOR > 1) begin : g_fwd + wire [`CLOG2(`CS_WORDS_PER_SECTOR)-1:0] fwd_k = wsel_fwd[`CLOG2(`CS_WORDS_PER_SECTOR)-1:0]; + assign fwd_words[k] = fbuf_data_r[fwd_k * `CS_WORD_WIDTH +: `CS_WORD_WIDTH]; + end else begin : g_fwd_1 + assign fwd_words[k] = fbuf_data_r[`CS_WORD_WIDTH-1:0]; + `UNUSED_VAR (wsel_fwd) + end + end + `UNUSED_VAR ({crsp_queue_data, fwd_word}) + end else begin : g_rsp_word + assign crsp_words = crsp_queue_data; + assign fwd_words = fwd_word; + end + `RUNTIME_ASSERT (~fwd_fire || (replay_addr == fbuf_addr_r), ("%t: %s fill-forward address mismatch: addr=0x%0h, staged=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(replay_addr, BANK_ID), `CS_BANK_TO_FULL_ADDR(fbuf_addr_r, BANK_ID))) `RUNTIME_ASSERT (~(flush_fire && fwd_pending), ("%t: %s flush during fill-forward drain", $time, INSTANCE_ID)) + // The ports a response goes to, decoded as it enters the queue so that + // the response arbiters start from registers: its own port and, under + // read merging, the merged ports of its group the tag's low bits mark. + wire [TAG_WIDTH-1:0] crsp_in_tag = crsp_queue_valid ? stC.req.tag : replay_tag; + wire [REQ_SEL_WIDTH-1:0] crsp_in_idx = crsp_queue_valid ? stC.req.req_idx : replay_idx; + wire [NUM_REQS-1:0] crsp_in_targets; + if (MERGE_GROUP > 1) begin : g_merge_targets + localparam GROUP_BITS = `CLOG2(MERGE_GROUP); + wire [REQ_SEL_WIDTH-1:0] group_base = (crsp_in_idx >> GROUP_BITS) << GROUP_BITS; + assign crsp_in_targets = (NUM_REQS'(1) << crsp_in_idx) + | (NUM_REQS'(crsp_in_tag[MERGE_GROUP-1:0]) << group_base); + end else begin : g_port_target + assign crsp_in_targets = NUM_REQS'(1) << crsp_in_idx; + end + VX_elastic_buffer #( - .DATAW (TAG_WIDTH + `CS_WORD_WIDTH + REQ_SEL_WIDTH), + .DATAW (NUM_REQS + TAG_WIDTH + RSP_WORDS * `CS_WORD_WIDTH + REQ_SEL_WIDTH), .SIZE (CRSQ_QUEUE_SIZE), .OUT_REG (`TO_OUT_BUF_REG(CORE_OUT_BUF)) ) core_rsp_queue ( @@ -1122,9 +1419,8 @@ module VX_cache_bank import VX_gpu_pkg::*; #( .reset (reset), .valid_in (crsp_queue_valid || fwd_head), .ready_in (crsp_queue_ready), - .data_in (crsp_queue_valid ? {stC.req.tag, crsp_queue_data, stC.req.req_idx} - : {replay_tag, fwd_word, replay_idx}), - .data_out ({core_rsp_tag, core_rsp_data, core_rsp_idx}), + .data_in ({crsp_in_targets, crsp_in_tag, (crsp_queue_valid ? crsp_words : fwd_words), crsp_in_idx}), + .data_out ({core_rsp_targets, core_rsp_tag, core_rsp_data, core_rsp_idx}), .valid_out (core_rsp_valid), .ready_out (core_rsp_ready) ); @@ -1148,13 +1444,26 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire mreq_queue_rw; wire is_fill_or_flush_stc = stC.req.is_fill || (stC.req.is_flush && WRITEBACK); + // a no-allocate write miss with no fill pending on its line is sent to + // memory as is: nothing is allocated or fetched. + wire write_around_stc = (WRITE_AROUND != 0) && do_write_stc && stC.req.attr[MEM_ATTR_NOALLOC_OFFS] + && ~stC.lk.is_hit && ~stC.lk.mshr_pending; + // a write-miss root under lazy fetch loops its fill back instead. + wire lazy_alloc_stc = (LAZY_FETCH != 0) && do_write_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~write_around_stc; + // the mask of the request's sector, as of every older store + wire [SECTOR_SIZE-1:0] sector_byteen_stc = evict_byteen_stc[sector_idx_stc * SECTOR_SIZE +: SECTOR_SIZE]; + // a refetch root (a partial-miss read or a parked head) whose sector every + // store already covers loops back instead of reading memory, while the + // loopback has the room a new request is admitted with: a parked head is + // not admitted against it, and memory always has room for the refetch. + wire refetch_loop_stc = (LAZY_FETCH != 0) && do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && partial_miss_stc && (& sector_byteen_stc) && ~lazyq_alm_full; wire do_fill_or_flush_stc = stC.req.valid && is_fill_or_flush_stc; // a sector refill keeps the resident line (no eviction), so never writes back. - wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill; + wire do_writeback_stc = do_fill_or_flush_stc && stC.lk.is_dirty && ~stC.lk.is_refill && ~refetch_first_stc; wire [`CS_LINE_ADDR_WIDTH-1:0] evict_addr_stc = {stC.lk.evict_tag, stC.req.addr[`CS_LINE_SEL_BITS-1:0]}; - // sector of the request at commit (the missed sector of a fill request). - wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx_stc; if (`CS_SECTOR_SEL_BITS != 0) begin : g_sector_idx_stc assign sector_idx_stc = stC.req.word_idx[`CS_WORD_SEL_BITS-1 -: `CS_SECTOR_SEL_BITS]; end else begin : g_sector_idx_stc0 @@ -1224,18 +1533,60 @@ module VX_cache_bank import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_wb if (DIRTY_BYTES) begin : g_dirty_bytes wire has_dirty_bytes = (| evict_byteen_stc); - `RUNTIME_ASSERT (~do_fill_or_flush_stc || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + // a refetch evicts nothing: the mask it reads is its resident way's + wire check_evict_bytes = do_fill_or_flush_stc && ~refetch_first_stc; + `ifndef SYNTHESIS + // An invalid way's mask is undefined: the reset walk leaves the + // mask alone and every install clears its line's mask, so only a + // valid victim's bytes must agree with its dirty flag. + wire evict_valid_stc; + VX_pipe_register #( + .DATAW (1), + .DEPTH (1 + PIPE_EX) + ) reg_evict_valid ( + .clk (clk), + .reset (reset), + .enable (~pipe_stall), + .data_in (evict_valid_st0), + .data_out (evict_valid_stc) + ); + `RUNTIME_ASSERT (~(check_evict_bytes && evict_valid_stc) || (stC.lk.is_dirty == has_dirty_bytes), ("missmatch dirty bytes: dirty_line=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.is_dirty, has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `endif + // per-sector: a dirty sector must carry dirty bytes, or its writeback beat is dropped. + wire [SEC-1:0] sec_has_dirty_bytes; + for (genvar s = 0; s < SEC; ++s) begin : g_sec_has_dirty_bytes + assign sec_has_dirty_bytes[s] = (| evict_byteen_stc[s*SECTOR_SIZE +: SECTOR_SIZE]); + end + `RUNTIME_ASSERT (~check_evict_bytes || ((stC.lk.evict_dirty_mask & ~sec_has_dirty_bytes) == '0), ("dirty sector without dirty bytes: dirty_mask=%b, dirty_bytes=%b, addr=0x%0h", stC.lk.evict_dirty_mask, sec_has_dirty_bytes, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) end // fill request on a read/write miss (one sector); multi-beat writeback // on a dirty eviction. The two are mutually exclusive (a writeback is a // fill/flush commit; a fill request is a creq commit). - wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending && ~pipe_stall; - assign mreq_queue_push = fill_req_push || wb_beat_accept; + wire fill_req_push = (do_read_stc || do_write_stc) && ~stC.lk.is_hit && ~stC.lk.mshr_pending + && ~lazy_alloc_stc && ~refetch_loop_stc && ~write_around_stc && ~pipe_stall; + wire wa_req_push = write_around_stc && ~pipe_stall; + assign mreq_queue_push = fill_req_push || wa_req_push || wb_beat_accept; assign mreq_queue_addr = is_wb_beat ? wb_mreq_addr : rd_mreq_addr; - assign mreq_queue_rw = is_wb_beat; - assign mreq_queue_data = wb_data_sec; // read fill request: data unused - assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({stC.req.byteen, stC.req.is_replay}) + assign mreq_queue_rw = is_wb_beat || write_around_stc; + if (WRITE_AROUND != 0) begin : g_wa_data + wire [LINE_SIZE-1:0] full_byteen; + VX_demux #( + .DATAW (WORD_SIZE), + .N (`CS_WORDS_PER_LINE) + ) byteen_demux ( + .sel_in (stC.req.word_idx), + .data_in (stC.req.byteen), + .data_out (full_byteen) + ); + wire [SECTOR_SIZE-1:0] sec_byteen = full_byteen[sector_idx_stc*SECTOR_SIZE +: SECTOR_SIZE]; + assign mreq_queue_data = is_wb_beat ? wb_data_sec : {`CS_WORDS_PER_SECTOR{word_stc}}; + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : (write_around_stc ? sec_byteen : {SECTOR_SIZE{1'b1}}); + end else begin : g_no_wa_data + assign mreq_queue_data = wb_data_sec; // read fill request: data unused + assign mreq_queue_byteen = is_wb_beat ? wb_byteen_sec : {SECTOR_SIZE{1'b1}}; + `UNUSED_VAR (stC.req.byteen) + end + `UNUSED_VAR (stC.req.is_replay) end else begin : g_wt // word byte-enable demuxed over the line, then sliced to the word's sector. wire [LINE_SIZE-1:0] full_byteen; @@ -1257,7 +1608,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = stC.req.rw; assign mreq_queue_data = {`CS_WORDS_PER_SECTOR{word_stc}}; assign mreq_queue_byteen = (stC.req.rw || is_amo_fwd_st1) ? sec_byteen : {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, evict_addr_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, write_around_stc}) end end else begin : g_mreq_queue_ro assign mreq_queue_push = (do_read_stc && ~stC.lk.is_hit && ~stC.lk.mshr_pending) && ~pipe_stall; @@ -1265,7 +1616,7 @@ module VX_cache_bank import VX_gpu_pkg::*; #( assign mreq_queue_rw = 0; assign mreq_queue_data = '0; assign mreq_queue_byteen = {SECTOR_SIZE{1'b1}}; - `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc}) + `UNUSED_VAR ({is_wb_beat, wb_beat_accept, wb_mreq_addr, wb_data_sec, wb_byteen_sec, wb_sector, do_writeback_stc, evict_addr_stc, evict_byteen_stc, stC.lk.evict_tag, stC.lk.is_dirty, stC.lk.evict_dirty_mask, stC.req.byteen, stC.req.word_idx, stC.req.is_replay, do_write_stc, write_around_stc}) end if (UUID_WIDTH != 0) begin : g_mreq_queue_tag_uuid @@ -1296,12 +1647,99 @@ module VX_cache_bank import VX_gpu_pkg::*; #( ); assign mem_req_valid = ~mreq_queue_empty; + // Lazy-fetch loopback: write-miss roots awaiting their data-less fill, and + // refetch roots whose sector needs no data. + // Sized like the memory request queue's reservation: a core request is + // admitted only while every request already in the pipe still fits. + if (LAZY_FETCH != 0) begin : g_lazyq + // Room for twice the pipe, so a few queued entries leave the + // reservation (and the refetch loopback, which needs it) open. + localparam LAZYQ_SIZE = 1 << `CLOG2(2 * PIPELINE_STAGES); + VX_fifo_queue #( + .DATAW (MSHR_ADDR_WIDTH), + .DEPTH (LAZYQ_SIZE), + .ALM_FULL (LAZYQ_SIZE - PIPELINE_STAGES) + ) lazy_queue ( + .clk (clk), + .reset (reset), + .push ((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall), + .pop (lazy_fire), + .data_in (stC.req.mshr_id), + .data_out (lazyq_id), + .empty (lazyq_empty), + .alm_full (lazyq_alm_full), + .full (lazyq_full), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (size) + ); + + // The partial state of the draining chain's sector: set by a data-less + // fill, cleared by a data fill, and corrected at S0 when a refetch + // finds its sector evicted (the replay waits out that cycle). + reg chain_partial_r, chain_retry_r; + always @(posedge clk) begin + if (reset) begin + chain_partial_r <= 1'b0; + chain_retry_r <= 1'b0; + end else if (fill_fire) begin + chain_partial_r <= lazy_fire && ~mshr_fill_refetch; + chain_retry_r <= 1'b0; + end else if (refetch_hold && ~pipe_stall) begin + chain_partial_r <= fill_evicted_st0; + chain_retry_r <= fill_evicted_st0; + end + end + assign chain_partial = chain_partial_r; + assign chain_retry = chain_retry_r; + + // A refetch's fill sector must stay staged until its stC merge. + reg refetch_inflight_r; + always @(posedge clk) begin + if (reset) begin + refetch_inflight_r <= 1'b0; + end else if (mem_rsp_fire && mshr_fill_refetch) begin + refetch_inflight_r <= 1'b1; + end else if (stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop && ~pipe_stall) begin + refetch_inflight_r <= 1'b0; + end + end + assign refetch_inflight = refetch_inflight_r; + // A memory refetch keeps its sector staged until its stC merge. + `RUNTIME_ASSERT (~(stC.req.valid && stC.req.is_fill && refetch_stc && ~lz_stc.is_loop) || refetch_inflight_r, + ("%t: %s memory refetch reached its merge unstaged: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + + `RUNTIME_ASSERT (~((lazy_alloc_stc || refetch_loop_stc) && ~pipe_stall && lazyq_full && ~lazy_fire), + ("%t: %s lazy queue overflow", $time, INSTANCE_ID)) + // A looped-back refetch carries no data: its merge must find every byte written. + `RUNTIME_ASSERT (~(do_merge_stc && lz_stc.is_loop) || (& sector_byteen_stc), + ("%t: %s looped-back refetch of an incompletely written sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(addr_stc, BANK_ID))) + `RUNTIME_ASSERT (~(park_fire && fwd_fire), ("%t: %s park and forward in one cycle", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~replay_valid || (mshr_dequeue_reads == (~replay_rw || replay_amo.amo_valid)), + ("%t: %s chain head read flag disagrees with its payload", $time, INSTANCE_ID)) + // Nothing is selected behind a refetch, so its stC merge has the array + // to itself and the next lookup reads the cleared partial bit. + `RUNTIME_ASSERT (~(refetch_hold && sel_req.req.valid), ("%t: %s request selected behind a refetch", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(st0.req.valid && park_st0 && (| tag_matches_st0) && ~partial_miss_st0), ("%t: %s parked replay hit", $time, INSTANCE_ID)) + // Only a refetch can find its sector resident. + `RUNTIME_ASSERT (~(do_fill_st0 && ~refetch_first_st0 && ~st0.req.is_passthru_fill && fill_resident_st0), + ("%t: %s fill into a resident sector: addr=0x%0h", $time, INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID))) + end else begin : g_no_lazyq + assign lazyq_id = '0; + assign lazyq_empty = 1'b1; + assign lazyq_alm_full = 1'b0; + assign lazyq_full = 1'b0; + assign chain_partial = 1'b0; + assign chain_retry = 1'b0; + assign refetch_inflight = 1'b0; + `UNUSED_VAR ({lazy_alloc_stc, refetch_loop_stc, sector_byteen_stc, lazy_fire, refetch_hold, fill_resident_st0, fill_evicted_st0, partial_match_st0}) + end + `UNUSED_VAR (do_lookup_st0) /////////////////////////////////////////////////////////////////////////////// `ifdef PERF_ENABLE - assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit; + assign perf_read_miss = do_read_st1 && ~st1.lk.is_hit && ~lz_st1.is_park; assign perf_write_miss = do_write_st1 && ~st1.lk.is_hit; assign perf_evictions = do_writeback_stc; assign perf_mshr_stall = mshr_alm_full; @@ -1317,7 +1755,9 @@ module VX_cache_bank import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] core_req_full_addr = `CS_BANK_TO_FULL_ADDR(core_req_addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st0 = `CS_BANK_TO_FULL_ADDR(st0.req.addr, BANK_ID); wire [`VX_CFG_XLEN-1:0] full_addr_st1 = `CS_BANK_TO_FULL_ADDR(st1.req.addr, BANK_ID); - wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_addr, BANK_ID); + wire [`CS_LINE_ADDR_WIDTH-1:0] mreq_queue_line = mreq_queue_addr[`CS_LINE_SECTOR_ADDR_WIDTH-1 -: `CS_LINE_ADDR_WIDTH]; + wire [`VX_CFG_XLEN-1:0] mreq_queue_full_addr = `CS_BANK_TO_FULL_ADDR(mreq_queue_line, BANK_ID) + | ((`VX_CFG_XLEN'(mreq_queue_addr) & `VX_CFG_XLEN'((1 << `CS_SECTOR_SEL_BITS) - 1)) << `CLOG2(SECTOR_SIZE)); always @(posedge clk) begin if (input_stall || pipe_stall) begin @@ -1325,8 +1765,16 @@ module VX_cache_bank import VX_gpu_pkg::*; #( crsp_queue_stall, mreq_queue_alm_full, mshr_alm_full)) end if (mem_rsp_fire) begin - `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, data=0x%h (#%0d)\n", $time, INSTANCE_ID, - mem_rsp_full_addr, mem_rsp_id, mem_rsp_data, req_uuid_sel)) + `TRACE(2, ("%t: %s fill-rsp: addr=0x%0h, mshr_id=%0d, refetch=%b, retry=%b, data=0x%h (#%0d)\n", $time, INSTANCE_ID, + mem_rsp_full_addr, mem_rsp_id, mshr_fill_refetch, mshr_fill_retry, mem_rsp_data, req_uuid_sel)) + end + if (lazy_fire) begin + `TRACE(2, ("%t: %s lazy-fill: addr=0x%0h, mshr_id=%0d, refetch=%b\n", $time, INSTANCE_ID, + mem_rsp_full_addr, lazyq_id, mshr_fill_refetch)) + end + if (park_fire) begin + `TRACE(2, ("%t: %s mshr-park: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, + replay_full_addr, replay_tag, replay_idx, req_uuid_sel)) end if (replay_fire) begin `TRACE(2, ("%t: %s mshr-pop: addr=0x%0h, tag=0x%0h, req_idx=%0d (#%0d)\n", $time, INSTANCE_ID, diff --git a/hw/rtl/cache/VX_cache_bypass.sv b/hw/rtl/cache/VX_cache_bypass.sv index 931cf0ebab..f3e6d36ac4 100644 --- a/hw/rtl/cache/VX_cache_bypass.sv +++ b/hw/rtl/cache/VX_cache_bypass.sv @@ -39,7 +39,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], VX_mem_bus_if.master core_bus_out_if [NUM_REQS], VX_mem_bus_if.slave mem_bus_in_if [MEM_PORTS], - VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_out_if [MEM_PORTS], + + // no uncached or outgoing request held short of the memory side + output wire req_empty ); localparam DIRECT_PASSTHRU = !CACHE_ENABLE && (`CS_WORD_SEL_BITS == 0) && (NUM_REQS == MEM_PORTS); localparam CORE_DATA_WIDTH = WORD_SIZE * 8; @@ -50,6 +53,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( localparam MEM_TAG_NC1_WIDTH = UUID_WIDTH + MEM_TAG_ID_WIDTH; localparam MEM_TAG_NC2_WIDTH = MEM_TAG_NC1_WIDTH + WSEL_BITS; localparam MEM_TAG_OUT_WIDTH = CACHE_ENABLE ? `MAX(MEM_TAG_IN_WIDTH, MEM_TAG_NC2_WIDTH) : MEM_TAG_NC2_WIDTH; + // Beside a cache, the memory-side arbiter grants on valid, so spreading + // uncached requests over more memory ports by their ready would close a + // valid-ready loop: with fewer core ports, core port i keeps memory port i. + localparam NC_PORTS = CACHE_ENABLE ? `MIN(NUM_REQS, MEM_PORTS) : MEM_PORTS; `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % `VX_CFG_MEM_BLOCK_SIZE), ("invalid parameter")) VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -58,7 +65,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( wire [NUM_REQS-1:0] core_req_nc_sel; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_req_is_nc if (CACHE_ENABLE) begin : g_cache - assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + // a flush token always reaches the cache, whatever its io kind bit + assign core_req_nc_sel[i] = ~core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS] + | core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end else begin : g_no_cache assign core_req_nc_sel[i] = 1'b0; end @@ -103,10 +112,10 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (MEM_TAG_NC1_WIDTH) - ) core_bus_nc_arb_if[MEM_PORTS](); + ) core_bus_nc_arb_if[NC_PORTS](); VX_mem_bus_arb #( .NUM_INPUTS (NUM_REQS), - .NUM_OUTPUTS(MEM_PORTS), + .NUM_OUTPUTS(NC_PORTS), .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (CORE_TAG_WIDTH), .TAG_SEL_IDX(TAG_SEL_IDX), @@ -123,7 +132,7 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_NC2_WIDTH) ) mem_bus_out_nc_if[MEM_PORTS](); - for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc + for (genvar i = 0; i < NC_PORTS; ++i) begin : g_mem_bus_out_nc wire core_req_nc_arb_rw; wire [WORD_SIZE-1:0] core_req_nc_arb_byteen; wire [CORE_ADDR_WIDTH-1:0] core_req_nc_arb_addr; @@ -198,6 +207,9 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( }; assign mem_bus_out_nc_if[i].rsp_ready = core_bus_nc_arb_if[i].rsp_ready; end + for (genvar i = NC_PORTS; i < MEM_PORTS; ++i) begin : g_mem_bus_out_nc_idle + `INIT_VX_MEM_BUS_IF (mem_bus_out_nc_if[i]) + end VX_mem_bus_if #( .DATA_SIZE (LINE_SIZE), .TAG_WIDTH (MEM_TAG_OUT_WIDTH) @@ -224,4 +236,33 @@ module VX_cache_bypass import VX_gpu_pkg::*; #( .bus_in_if (mem_bus_out_src_if), .bus_out_if (mem_bus_out_if) ); + + if (!DIRECT_PASSTHRU) begin : g_req_pending + localparam NUM_SRCS = NUM_REQS + (CACHE_ENABLE ? MEM_PORTS : 0); + wire [NUM_SRCS-1:0] src_fire; + wire [MEM_PORTS-1:0] dst_fire; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_nc_fire + assign src_fire[i] = core_bus_in_nc_if[i].req_valid && core_bus_in_nc_if[i].req_ready; + end + if (CACHE_ENABLE) begin : g_cache_fire + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_in_fire + assign src_fire[NUM_REQS + i] = mem_bus_in_if[i].req_valid && mem_bus_in_if[i].req_ready; + end + end + for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_dst_fire + assign dst_fire[i] = mem_bus_out_if[i].req_valid && mem_bus_out_if[i].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_SRCS), + .NUM_OUTPUTS (MEM_PORTS) + ) req_pending ( + .clk (clk), + .reset (reset), + .in_fire (src_fire), + .out_fire (dst_fire), + .empty (req_empty) + ); + end else begin : g_no_req_pending + assign req_empty = 1'b1; + end endmodule diff --git a/hw/rtl/cache/VX_cache_cluster.sv b/hw/rtl/cache/VX_cache_cluster.sv index 886f898d62..8352630768 100644 --- a/hw/rtl/cache/VX_cache_cluster.sv +++ b/hw/rtl/cache/VX_cache_cluster.sv @@ -44,6 +44,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -55,6 +57,12 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Indicates this cache cluster is the LLC (handles AMO completion). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic memory operations. ) ( @@ -65,7 +73,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_INPUTS * NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the arbiters or bypass paths + output wire req_empty ); localparam NUM_CACHES = `UP(NUM_UNITS); localparam PASSTHRU = (NUM_UNITS == 0); @@ -87,6 +98,10 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (ARB_TAG_WIDTH) ) arb_core_bus_if[NUM_CACHES * NUM_REQS](); + wire [NUM_REQS-1:0] core_arb_empty; + wire [NUM_CACHES-1:0] cache_req_empty; + wire [MEM_PORTS-1:0] mem_arb_empty; + assign req_empty = (& core_arb_empty) && (& cache_req_empty) && (& mem_arb_empty); for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_arb VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), @@ -114,6 +129,28 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (core_bus_tmp_if), .bus_out_if (arb_core_bus_tmp_if) ); + if (NUM_INPUTS != NUM_CACHES) begin : g_core_arb_pending + wire [NUM_INPUTS-1:0] in_fire; + wire [NUM_CACHES-1:0] out_fire; + for (genvar j = 0; j < NUM_INPUTS; ++j) begin : g_in_fire + assign in_fire[j] = core_bus_tmp_if[j].req_valid && core_bus_tmp_if[j].req_ready; + end + for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_out_fire + assign out_fire[k] = arb_core_bus_tmp_if[k].req_valid && arb_core_bus_tmp_if[k].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_CACHES) + ) core_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (out_fire), + .empty (core_arb_empty[i]) + ); + end else begin : g_no_core_arb_pending + assign core_arb_empty[i] = 1'b1; + end for (genvar k = 0; k < NUM_CACHES; ++k) begin : g_arb_core_bus_if `ASSIGN_VX_MEM_BUS_IF (arb_core_bus_if[k * NUM_REQS + i], arb_core_bus_tmp_if[k]); end @@ -132,6 +169,7 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), @@ -145,6 +183,12 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .NC_ENABLE (NC_ENABLE), .PASSTHRU (PASSTHRU), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), + .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache_wrap ( `ifdef PERF_ENABLE @@ -153,7 +197,8 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (arb_core_bus_if[i * NUM_REQS +: NUM_REQS]), - .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]) + .mem_bus_if (cache_mem_bus_if[i * MEM_PORTS +: MEM_PORTS]), + .req_empty (cache_req_empty[i]) ); end for (genvar i = 0; i < MEM_PORTS; ++i) begin : g_mem_bus_if @@ -183,6 +228,24 @@ module VX_cache_cluster import VX_gpu_pkg::*; #( .bus_in_if (arb_core_bus_tmp_if), .bus_out_if (mem_bus_tmp_if) ); + if (NUM_CACHES > 1) begin : g_mem_arb_pending + wire [NUM_CACHES-1:0] in_fire; + for (genvar j = 0; j < NUM_CACHES; ++j) begin : g_in_fire + assign in_fire[j] = arb_core_bus_tmp_if[j].req_valid && arb_core_bus_tmp_if[j].req_ready; + end + VX_mem_req_pending #( + .NUM_INPUTS (NUM_CACHES), + .NUM_OUTPUTS (1) + ) mem_arb_pending ( + .clk (clk), + .reset (reset), + .in_fire (in_fire), + .out_fire (mem_bus_tmp_if[0].req_valid && mem_bus_tmp_if[0].req_ready), + .empty (mem_arb_empty[i]) + ); + end else begin : g_no_mem_arb_pending + assign mem_arb_empty[i] = 1'b1; + end if (WRITE_ENABLE) begin : g_we `ASSIGN_VX_MEM_BUS_IF (mem_bus_if[i], mem_bus_tmp_if[0]); end else begin : g_ro diff --git a/hw/rtl/cache/VX_cache_data.sv b/hw/rtl/cache/VX_cache_data.sv index abc3472ba3..71a96d1292 100644 --- a/hw/rtl/cache/VX_cache_data.sv +++ b/hw/rtl/cache/VX_cache_data.sv @@ -39,13 +39,34 @@ module VX_cache_data import VX_gpu_pkg::*; #( parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill granule); = LINE_SIZE => 1 sector parameter WRITE_ENABLE = 1, // Enable cache writeable parameter WRITEBACK = 0, // Enable cache writeback - parameter DIRTY_BYTES = 0 // Enable dirty bytes on writeback + parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Fills may be data-less or merge under the dirty bytes + parameter MASK_SYNC = 0 // Dirty-byte mask read one stage ahead of the commit stage ) ( input wire clk, input wire reset, // inputs - input wire init, input wire fill, + input wire fill_nodata, // data-less fill: allocate only, no array write + // dirty-byte mask access at the commit stage (stC), one stage after the + // array access: a store sets its bytes, a fill/flush reads and clears + input wire mask_write, + input wire mask_set, + input wire mask_clr_line, + input wire mask_clr_sector, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_way, + input wire [`CS_LINE_SEL_BITS-1:0] mask_line, + input wire [`UP(`CS_WORD_SEL_BITS)-1:0] mask_word, + input wire [WORD_SIZE-1:0] mask_byteen, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] mask_sector, + // MASK_SYNC: the {way, line} that enters the commit stage next, and its + // advance enable + input wire mask_read, + input wire [`CS_WAY_SEL_WIDTH-1:0] mask_rway, + input wire [`CS_LINE_SEL_BITS-1:0] mask_rline, + // merge of a refetched sector into its resident copy at the commit stage + // (the stage behind it is empty): only the clean bytes + input wire merge_write, input wire flush, input wire read, input wire write, @@ -56,15 +77,13 @@ module VX_cache_data import VX_gpu_pkg::*; #( input wire [`CS_WORD_WIDTH-1:0] write_word, input wire [WORD_SIZE-1:0] write_byteen, input wire [`UP(`CS_WORD_SEL_BITS)-1:0] word_idx, + input wire [`CS_WORDS_PER_LINE-1:0] read_words, // slices a read returns (word_idx's, plus a merged read's) input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // sector being filled - input wire [`CS_WAY_SEL_WIDTH-1:0] way_idx_r, // outputs output wire [`CS_LINE_WIDTH-1:0] read_data, output wire [LINE_SIZE-1:0] evict_byteen ); - // The main data array resolves the way at read-issue and folds it into the - // array address, so the S1 output-way register is only used by the (narrow, - // off-path) dirty-byte mask below. + `STATIC_ASSERT(!LAZY_FETCH || (DIRTY_BYTES && WRITEBACK), ("invalid parameter: lazy fetch requires write-back dirty bytes")) localparam WAY_SEL_BITS = `CS_WAY_SEL_BITS; localparam DATA_RAM_DEPTH = `CS_LINES_PER_BANK * NUM_WAYS; @@ -83,6 +102,7 @@ module VX_cache_data import VX_gpu_pkg::*; #( `UNUSED_PIN (valid_out) ); + wire fill_write = fill && ~((LAZY_FETCH != 0) && fill_nodata); wire is_evict = fill || flush; wire hit_any = (| tag_matches); wire [`CS_WAY_SEL_WIDTH-1:0] way_sel = is_evict ? evict_way : hit_way; @@ -90,59 +110,112 @@ module VX_cache_data import VX_gpu_pkg::*; #( // {way, line} -> physical address wire [DATA_ADDR_WIDTH-1:0] data_addr; if (NUM_WAYS > 1) begin : g_way_addr - assign data_addr = DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); + assign data_addr = merge_write ? DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}) + : DATA_ADDR_WIDTH'({way_sel[WAY_SEL_BITS-1:0], line_idx}); end else begin : g_no_way_addr - `UNUSED_VAR (way_sel) - assign data_addr = line_idx; + `UNUSED_VAR ({way_sel, mask_way}) + assign data_addr = merge_write ? mask_line : line_idx; end - // Per-byte dirty mask (writeback only). Unlike the main data array this is - // kept as one line-indexed array per way (evict way selected at S1): it is - // narrow and off the load path, and -- crucially -- it must be cleared for - // every way during the line-only init walk, which the way-folded data - // layout cannot do in a single pass. + // Per-byte dirty mask (writeback only), one mask word per {way, line}. It + // is updated only at the commit stage, in pipeline order, so each access + // sees every older update. A line's mask is cleared when it is + // installed, which leaves nothing for the reset walk to clear. if (DIRTY_BYTES != 0) begin : g_dirty_bytes - wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] write_mask; - for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_write_mask - wire word_en = (`CS_WORDS_PER_LINE == 1) || (word_idx == i); - assign write_mask[i] = write_byteen & {WORD_SIZE{word_en}}; + wire [`CS_WORDS_PER_LINE-1:0][WORD_SIZE-1:0] set_word; + for (genvar i = 0; i < `CS_WORDS_PER_LINE; ++i) begin : g_set_word + wire word_en = (`CS_WORDS_PER_LINE == 1) || (mask_word == i); + assign set_word[i] = mask_byteen & {WORD_SIZE{mask_set && word_en}}; end + // A sector refill clears only the installed sector: the other resident + // sectors may hold dirty bytes not yet written back. + wire [`CS_SECTORS_PER_LINE-1:0][SECTOR_SIZE-1:0] clr_sector; + for (genvar i = 0; i < `CS_SECTORS_PER_LINE; ++i) begin : g_clr_sector + wire sec_en = (`CS_SECTORS_PER_LINE == 1) || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(i)); + assign clr_sector[i] = {SECTOR_SIZE{mask_clr_line || (mask_clr_sector && sec_en)}}; + end + wire [LINE_SIZE-1:0] set_mask = set_word; + wire [LINE_SIZE-1:0] clr_mask = clr_sector; + + wire [DATA_ADDR_WIDTH-1:0] mask_addr, mask_raddr; + if (NUM_WAYS > 1) begin : g_mask_way_addr + assign mask_addr = DATA_ADDR_WIDTH'({mask_way[WAY_SEL_BITS-1:0], mask_line}); + assign mask_raddr = DATA_ADDR_WIDTH'({mask_rway[WAY_SEL_BITS-1:0], mask_rline}); + end else begin : g_mask_line_addr + `UNUSED_VAR (mask_rway) + assign mask_addr = mask_line; + assign mask_raddr = mask_rline; + end + + wire [LINE_SIZE-1:0] mask_rdata; + wire [LINE_SIZE-1:0] mask_wdata = (mask_rdata & ~clr_mask) | set_mask; + + if (MASK_SYNC != 0) begin : g_mask_sync + // Read at the stage ahead, so the commit stage takes the mask from + // a register rather than through a deep distributed-RAM read mux + // (and a large mask fits block RAM). A write by the request now + // committing to the address read behind it is forwarded. The + // pipeline holds as a whole, so nothing writes while the read is + // held. + wire [LINE_SIZE-1:0] mask_ram_rdata; + VX_dp_ram #( + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (1), + .RDW_MODE ("R") + ) byteen_store ( + .clk (clk), + .reset (reset), + .read (mask_read), + .write (mask_write), + .wren (1'b1), + .waddr (mask_addr), + .wdata (mask_wdata), + .raddr (mask_raddr), + .rdata (mask_ram_rdata) + ); + + reg mask_fwd_r; + reg [LINE_SIZE-1:0] mask_fwd_data_r; + reg [DATA_ADDR_WIDTH-1:0] mask_raddr_r; + always @(posedge clk) begin + if (mask_read) begin + mask_fwd_r <= mask_write && (mask_addr == mask_raddr); + mask_fwd_data_r <= mask_wdata; + mask_raddr_r <= mask_raddr; + end + end + assign mask_rdata = mask_fwd_r ? mask_fwd_data_r : mask_ram_rdata; - wire [NUM_WAYS-1:0][LINE_SIZE-1:0] byteen_rdata; - for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_byteen_store - wire [LINE_SIZE-1:0] byteen_wdata = {LINE_SIZE{write}}; // only asserted on writes - wire [LINE_SIZE-1:0] byteen_wren = {LINE_SIZE{init || fill || flush}} | write_mask; - wire byteen_write = ((fill || flush) && ((NUM_WAYS == 1) || (evict_way == i))) - || (write && tag_matches[i]) - || init; - wire byteen_read = fill || flush; - - // The dirty mask has 1-bit (per-byte) write granularity, which - // block RAM cannot do -- inferring BRAM shatters it into dozens of - // tiny RAMB18 per way. It is small, so map it to distributed RAM. + `RUNTIME_ASSERT (~mask_write || mask_read, ("%t: dirty-byte mask written while its read is held", $time)) + `RUNTIME_ASSERT (~mask_write || (mask_addr == mask_raddr_r), ("%t: dirty-byte mask read for another line: 0x%0h, 0x%0h", $time, mask_addr, mask_raddr_r)) + end else begin : g_mask_async + `UNUSED_VAR ({mask_read, mask_raddr}) + // A whole-word read-modify-write shares one write enable across the + // mask, so it maps onto wide distributed RAM rather than one cell per + // byte (or a shattered block RAM). VX_sp_ram #( - .DATAW (LINE_SIZE), - .WRENW (LINE_SIZE), - .SIZE (`CS_LINES_PER_BANK), - .OUT_REG (1), - .LUTRAM (1), - .RDW_MODE ("R") + .DATAW (LINE_SIZE), + .SIZE (DATA_RAM_DEPTH), + .OUT_REG (0), + .LUTRAM (1), + .RDW_MODE ("W") ) byteen_store ( .clk (clk), .reset (reset), - .read (byteen_read), - .write (byteen_write), - .wren (byteen_wren), - .addr (line_idx), - .wdata (byteen_wdata), - .rdata (byteen_rdata[i]) + .read (1'b1), + .write (mask_write), + .wren (1'b1), + .addr (mask_addr), + .wdata (mask_wdata), + .rdata (mask_rdata) ); end - assign evict_byteen = byteen_rdata[way_idx_r]; + assign evict_byteen = mask_rdata; end else begin : g_no_dirty_bytes - `UNUSED_VAR (init) - `UNUSED_VAR (flush) - `UNUSED_VAR (way_idx_r) + `UNUSED_VAR ({mask_write, mask_set, mask_clr_line, mask_clr_sector}) + `UNUSED_VAR ({mask_line, mask_word, mask_byteen, mask_sector}) + `UNUSED_VAR ({mask_read, mask_rway, mask_rline}) assign evict_byteen = '1; // update whole line end @@ -160,8 +233,8 @@ module VX_cache_data import VX_gpu_pkg::*; #( wire fill_sec_en = (`CS_SECTORS_PER_LINE == 1) || (sector_idx == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); - // load reads the selected slice; writeback/flush reads all slices. - wire slice_read = (read && word_en) || ((fill || flush) && WRITEBACK); + // a load reads its words' slices; writeback/flush reads all slices. + wire slice_read = (read && read_words[s]) || ((fill || flush) && WRITEBACK); wire slice_write; wire [WRENW-1:0] slice_wren; @@ -169,14 +242,20 @@ module VX_cache_data import VX_gpu_pkg::*; #( if (WRITE_ENABLE) begin : g_wren // fill writes the fetched sector's slices; a store writes only the hit slice. - assign slice_write = (fill && fill_sec_en) || (write && hit_any && word_en); - assign slice_wren = fill ? {WORD_SIZE{1'b1}} : write_byteen; - assign slice_wdata = fill ? fill_data[s] : write_word; + wire merge_sec_en = (`CS_SECTORS_PER_LINE == 1) + || (mask_sector == `UP(`CS_SECTOR_SEL_BITS)'(SLICE_SECTOR)); + assign slice_write = (fill_write && fill_sec_en) || (write && hit_any && word_en) + || (merge_write && merge_sec_en); + assign slice_wren = merge_write ? ~evict_byteen[s * WORD_SIZE +: WORD_SIZE] + : (fill ? {WORD_SIZE{1'b1}} : write_byteen); + assign slice_wdata = (fill || merge_write) ? fill_data[s] : write_word; end else begin : g_no_wren `UNUSED_VAR (write) `UNUSED_VAR (write_word) `UNUSED_VAR (write_byteen) `UNUSED_VAR (hit_any) + `UNUSED_VAR ({fill_write, merge_write}) + `UNUSED_VAR (word_en) assign slice_write = fill && fill_sec_en; assign slice_wren = 1'b1; assign slice_wdata = fill_data[s]; diff --git a/hw/rtl/cache/VX_cache_define.vh b/hw/rtl/cache/VX_cache_define.vh index 3f904816fc..c3ccbb49b0 100644 --- a/hw/rtl/cache/VX_cache_define.vh +++ b/hw/rtl/cache/VX_cache_define.vh @@ -74,7 +74,8 @@ /////////////////////////////////////////////////////////////////////////////// -`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(b << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} +// b is the bank id; with BANK_HASH the raw bank bits are b XOR the line's hash +`define CS_BANK_TO_FULL_ADDR(x, b) {x, (`VX_CFG_XLEN-$bits(x))'(((b) ^ (BANK_HASH ? int'(cache_bank_hash(32'(x), `CS_BANK_SEL_BITS)) : 0)) << (`VX_CFG_XLEN-$bits(x)-`CS_BANK_SEL_BITS))} `define CS_MEM_TO_FULL_ADDR(x) {x, (`VX_CFG_XLEN-$bits(x))'(0)} /////////////////////////////////////////////////////////////////////////////// diff --git a/hw/rtl/cache/VX_cache_flush.sv b/hw/rtl/cache/VX_cache_flush.sv index 79e5aad266..d8227664e6 100644 --- a/hw/rtl/cache/VX_cache_flush.sv +++ b/hw/rtl/cache/VX_cache_flush.sv @@ -14,7 +14,6 @@ `include "VX_cache_define.vh" module VX_cache_flush import VX_gpu_pkg::*; #( - parameter BANK_ID = 0, parameter CACHE_SIZE = 1024, // Size of cache in bytes parameter LINE_SIZE = 64, // Size of line inside a bank in bytes parameter NUM_BANKS = 1, // Number of banks @@ -24,6 +23,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( input wire clk, input wire reset, input wire flush_begin, + input wire flush_skip, // the flush only quiesces the bank: an IO token output wire flush_end, output wire flush_init, output wire flush_valid, @@ -49,6 +49,7 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // latch flush_begin that arrives while init is in progress reg flush_pending_r, flush_pending_n; + reg skip_r; always @(*) begin state_n = state; @@ -75,18 +76,17 @@ module VX_cache_flush import VX_gpu_pkg::*; #( // Wait for the bank to fully quiesce before evicting: // both MSHR must drain, and the bank pipeline as well to ensure no inflight misses. if (mshr_empty && bank_empty) begin - state_n = STATE_FLUSH; + state_n = skip_r ? STATE_WAIT2 : STATE_FLUSH; end end STATE_FLUSH: begin if (counter == ((2 ** CTR_WIDTH)-1) && flush_ready) begin - state_n = (BANK_ID == 0) ? STATE_DONE : STATE_WAIT2; + state_n = STATE_WAIT2; end end STATE_WAIT2: begin - // ensure the bank is empty before notifying the cache flush unit, - // because the flush request to lower caches only goes through bank0 - // and it is important that request gets send out last. + // the write-backs must leave the bank before the cache sends + // its flush tokens behind them if (bank_empty) begin state_n = STATE_DONE; end @@ -106,6 +106,9 @@ module VX_cache_flush import VX_gpu_pkg::*; #( end else begin state <= state_n; flush_pending_r <= flush_pending_n; + if (flush_begin) begin + skip_r <= flush_skip; + end if (state != STATE_IDLE) begin if ((state == STATE_INIT) || ((state == STATE_FLUSH) && flush_ready)) begin diff --git a/hw/rtl/cache/VX_cache_init.sv b/hw/rtl/cache/VX_cache_init.sv index a27ba56dc6..b2c19d4c80 100644 --- a/hw/rtl/cache/VX_cache_init.sv +++ b/hw/rtl/cache/VX_cache_init.sv @@ -13,134 +13,317 @@ `include "VX_cache_define.vh" +// Flush sequencer. Flush tokens (requests with the flush attribute) at any +// input start one walk: every input is blocked, in-flight requests drain and +// every bank writes back and invalidates its lines. With FLUSH_FORWARD, the +// cache then sends one token down each memory port, behind that port's +// write-backs, and waits for every port to answer, so the next level has +// each port's data before it releases that port's token. Finally each token +// visible at the end of the walk is answered locally, one at a time through +// the cache's own response path, and the inputs reopen. +// A token that arrives during the walk is visible only after all of its +// input's earlier requests were accepted, so the walk covered them. +// A request crossbar that can hold requests apart from their input order +// (XBAR_SIZE != 0) drains into the banks first, while the inputs are held, +// so no earlier request of a token's input is passed by a later one. +// An L1-only token (any byteen bit set: a next level with wider words sees +// the bit at the token's line offset) is forwarded with its kind. A +// last-level cache that alone serves every memory client is the point of +// coherence for it: once the token is visible, its input's earlier requests +// are inside the cache, ahead of anything the next kernel sends, so it answers +// the token without a walk. Any other last-level cache still walks, since a +// replicated instance or a client reaching memory beside it may hold or need +// lines it has not written back. +// An IO token (the io attribute: a fence ordering only device accesses) walks +// nowhere. Once nothing is held between the cache's inputs and its memory +// ports, on the cacheable and the non-cacheable path alike, it goes down every +// memory port, so its answer proves the earlier IO writes reached memory. +// The L1-only kind splits by what the fence orders. A release token (noalloc +// attribute) owes no invalidation: the banks only quiesce and the token goes +// down to the point of coherence behind the earlier writes. An acquire token +// (io attribute with byte enables set) owes no drain: the cache walks and +// answers, forwarding only to a cache below that is not the point of +// coherence (ACQ_FORWARD). +// Kinds visible together combine to the weakest one that serves them all. module VX_cache_init import VX_gpu_pkg::*; #( - parameter NUM_REQS = 4, // Number of Word requests per cycle - parameter NUM_BANKS = 1, // Number of banks - parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size - parameter BANK_SEL_LATENCY = 1 // Bank select latency + parameter NUM_REQS = 4, // Number of Word requests per cycle + parameter NUM_BANKS = 1, // Number of banks + parameter MEM_PORTS = 1, // Number of memory ports + parameter FLUSH_FORWARD = 1, // Forward the flush to the next level + parameter SHARED_LLC = 0, // The LLC serving every memory client alone: answers L1-only tokens without a walk + parameter ACQ_FORWARD = 0, // An acquire token must also invalidate the next level + parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size + parameter XBAR_SIZE = 0, // Requests the request crossbar can hold apart from input order + parameter REQ_SEL_WIDTH = `UP(`CLOG2(NUM_REQS)) ) ( - input wire clk, - input wire reset, - VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], - VX_mem_bus_if.master core_bus_out_if [NUM_REQS], - input wire [NUM_BANKS-1:0] bank_req_fire, - output wire [NUM_BANKS-1:0] flush_begin, + input wire clk, + input wire reset, + VX_mem_bus_if.slave core_bus_in_if [NUM_REQS], + VX_mem_bus_if.master core_bus_out_if [NUM_REQS], + input wire [NUM_REQS-1:0] xbar_req_fire, + input wire [NUM_BANKS-1:0] bank_req_fire, + output wire [NUM_BANKS-1:0] flush_begin, + output wire flush_skip, // the banks only quiesce: no walk is owed output wire [`UP(UUID_WIDTH)-1:0] flush_uuid, - input wire [NUM_BANKS-1:0] flush_end + input wire [NUM_BANKS-1:0] flush_end, + // flush tokens to the next level + input wire mem_req_empty, // no write-back left between the banks and the ports + input wire nc_req_empty, // no request held between the inputs and the memory side + output wire [MEM_PORTS-1:0] mem_flush_valid, + output wire mem_flush_l1_only, + output wire mem_flush_io, + output wire mem_flush_noalloc, + input wire [MEM_PORTS-1:0] mem_flush_ready, + output wire mem_flush_rsp_phase, + input wire [MEM_PORTS-1:0] mem_flush_rsp_valid, + // answers to the requesters' tokens + output wire flush_rsp_valid, + output wire [REQ_SEL_WIDTH-1:0] flush_rsp_idx, + output wire [TAG_WIDTH-1:0] flush_rsp_tag, + input wire flush_rsp_ready ); - `UNUSED_PARAM (TAG_WIDTH) - - localparam STATE_IDLE = 0; - localparam STATE_WAIT1 = 1; - localparam STATE_FLUSH = 2; - localparam STATE_WAIT2 = 3; - localparam STATE_DONE = 4; + localparam STATE_IDLE = 0; + localparam STATE_WAIT1 = 1; + localparam STATE_FLUSH = 2; + localparam STATE_WAIT2 = 3; + localparam STATE_FORWARD = 4; + localparam STATE_RESPOND = 5; reg [2:0] state, state_n; - // track in-flight core requests + // track requests inside the request crossbar wire no_inflight_reqs; - if (BANK_SEL_LATENCY != 0) begin : g_bank_sel_latency + if (XBAR_SIZE != 0) begin : g_xbar_inflight - localparam NUM_REQS_W = `CLOG2(NUM_REQS+1); - localparam NUM_BANKS_W = `CLOG2(NUM_BANKS+1); + localparam REQS_W = `CLOG2(NUM_REQS+1); + localparam INCRW = `CLOG2(`MIN(NUM_REQS, XBAR_SIZE)+1); + localparam BANKS_W = `CLOG2(NUM_BANKS+1); - wire [NUM_REQS-1:0] core_bus_out_fire; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_fire - assign core_bus_out_fire[i] = core_bus_out_if[i].req_valid && core_bus_out_if[i].req_ready; - end + // The fires and their counts are registered ahead of the counter, so + // the crossbar's arbitration, the popcounts and the counter's add are + // three cycles apart. A request is in flight from the cycle after it + // fires: until the counter holds it, the registered fire or count + // does. Decrements arrive as late, so the counter only overstates. + reg [NUM_REQS-1:0] xbar_req_fire_r; + reg [NUM_BANKS-1:0] bank_req_fire_r; + reg [REQS_W-1:0] xbar_req_cnt_r; + reg [BANKS_W-1:0] bank_req_cnt_r; + wire [REQS_W-1:0] xbar_req_cnt; + wire [BANKS_W-1:0] bank_req_cnt; - wire [NUM_REQS_W-1:0] core_bus_out_cnt; - wire [NUM_BANKS_W-1:0] bank_req_cnt; + `POP_COUNT(xbar_req_cnt, xbar_req_fire_r); + `POP_COUNT(bank_req_cnt, bank_req_fire_r); + + always @(posedge clk) begin + if (reset) begin + xbar_req_fire_r <= '0; + bank_req_fire_r <= '0; + xbar_req_cnt_r <= '0; + bank_req_cnt_r <= '0; + end else begin + xbar_req_fire_r <= xbar_req_fire; + bank_req_fire_r <= bank_req_fire; + xbar_req_cnt_r <= xbar_req_cnt; + bank_req_cnt_r <= bank_req_cnt; + end + end - `POP_COUNT(core_bus_out_cnt, core_bus_out_fire); - `POP_COUNT(bank_req_cnt, bank_req_fire); - `UNUSED_VAR (core_bus_out_cnt) + wire xbar_empty; VX_pending_size #( - .SIZE (BANK_SEL_LATENCY * NUM_BANKS), - .INCRW (NUM_BANKS_W), - .DECRW (NUM_BANKS_W) + .SIZE (XBAR_SIZE), + .INCRW (INCRW), + .DECRW (BANKS_W) ) pending_size ( .clk (clk), .reset (reset), - .incr (NUM_BANKS_W'(core_bus_out_cnt)), - .decr (bank_req_cnt), - .empty (no_inflight_reqs), + .incr (INCRW'(xbar_req_cnt_r)), + .decr (bank_req_cnt_r), + .empty (xbar_empty), `UNUSED_PIN (alm_empty), `UNUSED_PIN (full), `UNUSED_PIN (alm_full), `UNUSED_PIN (size) ); - end else begin : g_no_bank_sel_latency + assign no_inflight_reqs = xbar_empty && ~(| xbar_req_fire_r) && (xbar_req_cnt_r == '0); + + end else begin : g_no_xbar_inflight assign no_inflight_reqs = 0; - `UNUSED_VAR (bank_req_fire) + `UNUSED_VAR ({xbar_req_fire, bank_req_fire}) end - reg [NUM_BANKS-1:0] flush_done, flush_done_n; - wire [NUM_REQS-1:0] flush_req_mask; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_req_mask assign flush_req_mask[i] = core_bus_in_if[i].req_valid && core_bus_in_if[i].req_data.attr[MEM_ATTR_FLUSH_OFFS]; end wire flush_req_enable = (| flush_req_mask); - reg [NUM_REQS-1:0] lock_released, lock_released_n; + wire [NUM_REQS-1:0] flush_l1_mask, flush_io_mask, flush_acq_mask, flush_rel_mask; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_flush_kind_mask + wire tok_io = core_bus_in_if[i].req_data.attr[MEM_ATTR_IO_OFFS]; + wire tok_rel = core_bus_in_if[i].req_data.attr[MEM_ATTR_NOALLOC_OFFS]; + wire tok_l1 = (| core_bus_in_if[i].req_data.byteen); + assign flush_io_mask[i] = flush_req_mask[i] && tok_io && ~tok_l1; + assign flush_acq_mask[i] = flush_req_mask[i] && tok_io && tok_l1; + assign flush_rel_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && tok_rel; + assign flush_l1_mask[i] = flush_req_mask[i] && ~tok_io && tok_l1 && ~tok_rel; + end + wire any_full = (| (flush_req_mask & ~flush_l1_mask & ~flush_io_mask & ~flush_acq_mask & ~flush_rel_mask)); + wire any_l1 = (| flush_l1_mask); + wire any_io = (| flush_io_mask); + wire any_acq = (| flush_acq_mask); + wire any_rel = (| flush_rel_mask); + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + wire flush_walk = any_full || ((any_l1 || any_acq) && (SHARED_LLC == 0)); + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + wire flush_drain = any_io || (any_rel && (SHARED_LLC == 0)); + + reg [NUM_BANKS-1:0] flush_done, flush_done_n; + reg fwd_go, fwd_go_n; // the crossbar drained; only tokens enter it now + reg [MEM_PORTS-1:0] fwd_sent, fwd_sent_n; + reg [MEM_PORTS-1:0] fwd_done, fwd_done_n; + reg [NUM_REQS-1:0] rsp_pending, rsp_pending_n; // token still to be answered reg [`UP(UUID_WIDTH)-1:0] flush_uuid_r, flush_uuid_n; + reg l1_only_r, l1_only_n; + reg io_r, io_n; + reg noalloc_r, noalloc_n; + reg skip_r, skip_n; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req - // block core requests if a flush is active, - // unless the lock for this specific request has been released - wire input_enable = ~flush_req_enable || lock_released[i]; + // answer the lowest pending input; its token stays at the input until then + wire [NUM_REQS-1:0] rsp_sel_onehot; + wire [REQ_SEL_WIDTH-1:0] rsp_sel_idx; + wire rsp_sel_valid; + VX_priority_encoder #( + .N (NUM_REQS) + ) rsp_sel ( + .data_in (rsp_pending), + .onehot_out (rsp_sel_onehot), + .index_out (rsp_sel_idx), + .valid_out (rsp_sel_valid) + ); + assign flush_rsp_valid = (state == STATE_RESPOND) && rsp_sel_valid; + assign flush_rsp_idx = rsp_sel_idx; + wire flush_rsp_fire = flush_rsp_valid && flush_rsp_ready; + wire [NUM_REQS-1:0][TAG_WIDTH-1:0] core_bus_in_tag; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_tag + assign core_bus_in_tag[i] = core_bus_in_if[i].req_data.tag; + end + VX_onehot_mux #( + .DATAW (TAG_WIDTH), + .N (NUM_REQS) + ) rsp_tag_mux ( + .data_in (core_bus_in_tag), + .sel_in (rsp_sel_onehot), + .data_out (flush_rsp_tag) + ); + + wire [NUM_REQS-1:0] dbg_stall; +`ifdef DBG_FLUSH_STALL + // Random input backpressure, a new mask every 128 cycles (each input + // stalled half the time), to widen ordering windows under test. + reg [31:0] dbg_lfsr_r; + reg [6:0] dbg_cnt_r; + reg [NUM_REQS-1:0] dbg_stall_r; + always @(posedge clk) begin + if (reset) begin + dbg_lfsr_r <= 32'hACE1_2468; + dbg_cnt_r <= '0; + dbg_stall_r <= '0; + end else begin + dbg_lfsr_r <= {dbg_lfsr_r[30:0], dbg_lfsr_r[31] ^ dbg_lfsr_r[21] ^ dbg_lfsr_r[1] ^ dbg_lfsr_r[0]}; + dbg_cnt_r <= dbg_cnt_r + 7'd1; + if (dbg_cnt_r == '0) begin + dbg_stall_r <= (NUM_REQS > 1) ? NUM_REQS'(dbg_lfsr_r) : '0; + end + end + end + assign dbg_stall = dbg_stall_r; +`else + assign dbg_stall = '0; +`endif + + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_req + // Normal requests stay blocked while any token is visible. A token is + // consumed here when its answer is sent and never reaches the banks. + wire take_token = flush_rsp_fire && rsp_sel_onehot[i]; + wire input_enable = ~flush_req_enable && ~dbg_stall[i]; assign core_bus_out_if[i].req_valid = core_bus_in_if[i].req_valid && input_enable; assign core_bus_out_if[i].req_data = core_bus_in_if[i].req_data; - assign core_bus_in_if[i].req_ready = core_bus_out_if[i].req_ready && input_enable; + assign core_bus_in_if[i].req_ready = take_token || (core_bus_out_if[i].req_ready && input_enable); end for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_rsp - assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; - assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; + assign core_bus_in_if[i].rsp_valid = core_bus_out_if[i].rsp_valid; + assign core_bus_in_if[i].rsp_data = core_bus_out_if[i].rsp_data; assign core_bus_out_if[i].rsp_ready = core_bus_in_if[i].rsp_ready; end - reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_out_uuid; - wire [NUM_REQS-1:0] core_bus_out_ready; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_uuid + reg [NUM_REQS-1:0][`UP(UUID_WIDTH)-1:0] core_bus_in_uuid; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_in_uuid if (UUID_WIDTH != 0) begin : g_uuid - assign core_bus_out_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; + assign core_bus_in_uuid[i] = core_bus_in_if[i].req_data.tag.uuid; end else begin : g_no_uuid - assign core_bus_out_uuid[i] = 0; + assign core_bus_in_uuid[i] = 0; end end - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_out_ready - assign core_bus_out_ready[i] = core_bus_out_if[i].req_ready; - end + // an IO token also waits for the non-cacheable path, which carries the IO + // writes it orders + wire fwd_ready = mem_req_empty && (~io_r || l1_only_r || nc_req_empty); + + // The kind forwarded after a walk: full if a full token or an IO one asks + // for more than the point of coherence, L1-only if any token owes a drain + // to it, acquire if only acquire tokens were visible. + wire walk_fwd_acq = any_acq && ~any_full && ~any_io && ~any_l1 && ~any_rel; + wire walk_fwd_l1 = (any_l1 || any_rel) && ~any_full && ~any_io; always @(*) begin - state_n = state; - flush_done_n = flush_done; - lock_released_n = lock_released; - flush_uuid_n = flush_uuid_r; + state_n = state; + flush_done_n = flush_done; + fwd_go_n = fwd_go; + fwd_sent_n = fwd_sent; + fwd_done_n = fwd_done; + rsp_pending_n = rsp_pending & ~(rsp_sel_onehot & {NUM_REQS{flush_rsp_fire}}); + flush_uuid_n = flush_uuid_r; + l1_only_n = l1_only_r; + io_n = io_r; + noalloc_n = noalloc_r; + skip_n = skip_r; case (state) // STATE_IDLE: default: begin if (flush_req_enable) begin - state_n = (BANK_SEL_LATENCY != 0) ? STATE_WAIT1 : STATE_FLUSH; + if (XBAR_SIZE != 0) begin + state_n = STATE_WAIT1; + end else if (~flush_walk && ~flush_drain) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + skip_n = ~flush_walk; + state_n = STATE_FLUSH; + end for (integer i = NUM_REQS-1; i >= 0; --i) begin if (flush_req_mask[i]) begin - flush_uuid_n = core_bus_out_uuid[i]; + flush_uuid_n = core_bus_in_uuid[i]; end end end end STATE_WAIT1: begin if (no_inflight_reqs) begin - state_n = STATE_FLUSH; + if (~flush_walk && ~flush_drain) begin + rsp_pending_n = flush_req_mask; + state_n = STATE_RESPOND; + end else begin + skip_n = ~flush_walk; + state_n = STATE_FLUSH; + end end end STATE_FLUSH: begin @@ -151,18 +334,34 @@ module VX_cache_init import VX_gpu_pkg::*; #( // wait for all banks to finish flushing flush_done_n = flush_done | flush_end; if (flush_done_n == {NUM_BANKS{1'b1}}) begin - state_n = STATE_DONE; flush_done_n = '0; - // only release current flush requests - // and keep normal requests locked - lock_released_n = flush_req_mask; + // answer the tokens visible now; later ones start a new walk. + // A quiesce without a walk serves only the tokens owing none, + // and forwards IO if it serves an IO token, else release. + rsp_pending_n = skip_r ? (flush_io_mask | flush_rel_mask + | (SHARED_LLC ? (flush_l1_mask | flush_acq_mask) : '0)) + : flush_req_mask; + l1_only_n = skip_r ? ~any_io : (walk_fwd_l1 || walk_fwd_acq); + io_n = skip_r ? any_io : walk_fwd_acq; + noalloc_n = skip_r && ~any_io; + // an acquire walk owes the next level nothing unless it too + // may hold stale lines + state_n = (FLUSH_FORWARD && (skip_r || ~walk_fwd_acq || ACQ_FORWARD)) ? STATE_FORWARD : STATE_RESPOND; end end - STATE_DONE: begin - // wait until released flush requests are issued - // when returning to IDLE state other requests will unlock - lock_released_n = lock_released & ~core_bus_out_ready; - if (lock_released_n == 0) begin + STATE_FORWARD: begin + fwd_go_n = fwd_go | fwd_ready; + fwd_sent_n = fwd_sent | (mem_flush_valid & mem_flush_ready); + fwd_done_n = fwd_done | mem_flush_rsp_valid; + if (fwd_done_n == {MEM_PORTS{1'b1}}) begin + fwd_go_n = 1'b0; + fwd_sent_n = '0; + fwd_done_n = '0; + state_n = STATE_RESPOND; + end + end + STATE_RESPOND: begin + if (rsp_pending_n == '0) begin state_n = STATE_IDLE; end end @@ -171,18 +370,37 @@ module VX_cache_init import VX_gpu_pkg::*; #( always @(posedge clk) begin if (reset) begin - state <= STATE_IDLE; - flush_done <= '0; - lock_released <= '0; + state <= STATE_IDLE; + flush_done <= '0; + fwd_go <= 1'b0; + fwd_sent <= '0; + fwd_done <= '0; + rsp_pending <= '0; end else begin - state <= state_n; - flush_done <= flush_done_n; - lock_released <= lock_released_n; + state <= state_n; + flush_done <= flush_done_n; + fwd_go <= fwd_go_n; + fwd_sent <= fwd_sent_n; + fwd_done <= fwd_done_n; + rsp_pending <= rsp_pending_n; end flush_uuid_r <= flush_uuid_n; + l1_only_r <= l1_only_n; + io_r <= io_n; + noalloc_r <= noalloc_n; + skip_r <= skip_n; end assign flush_begin = {NUM_BANKS{state == STATE_FLUSH}}; + assign flush_skip = skip_r; assign flush_uuid = flush_uuid_r; + // The drain counter counts the tokens too, so the first one sent must not + // take the others' valid away. + assign mem_flush_valid = {MEM_PORTS{(state == STATE_FORWARD) && (fwd_go || fwd_ready)}} & ~fwd_sent; + assign mem_flush_rsp_phase = (state == STATE_FORWARD); + assign mem_flush_l1_only = l1_only_r; + assign mem_flush_io = io_r; + assign mem_flush_noalloc = noalloc_r; + endmodule diff --git a/hw/rtl/cache/VX_cache_mshr.sv b/hw/rtl/cache/VX_cache_mshr.sv index 22605fd148..2d1d8e02dd 100644 --- a/hw/rtl/cache/VX_cache_mshr.sv +++ b/hw/rtl/cache/VX_cache_mshr.sv @@ -45,15 +45,16 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID= "", parameter BANK_ID = 0, + parameter BANK_HASH = 0, // hashed bank select (cache_bank_hash) parameter LINE_SIZE = 16, // Size of line inside a bank in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (coalescing/fill granule); = LINE_SIZE => 1 sector parameter NUM_BANKS = 1, // Number of banks parameter MSHR_SIZE = 4, // Miss Reserv Queue Knob parameter DATA_WIDTH = 1, // MSHR parameters - parameter WRITEBACK = 0, // Enable cache writeback parameter AMO_ENABLE = 0, // Per-entry AMO tracking (probe outputs) parameter AMO_PASSTHRU = 0, // Non-LLC passthrough: AMO entries never coalesce parameter WORD_SEL_WIDTH = 1, // word-in-line select width (order-hazard granularity) + parameter LAZY_FETCH = 0, // Chains may park and refetch a partial sector parameter MSHR_ADDR_WIDTH = `LOG2UP(MSHR_SIZE) ) ( @@ -72,6 +73,10 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire [`CS_LINE_ADDR_WIDTH-1:0] fill_addr, // sector of the entry that initiated this fill (which sector to install) output wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] fill_sector, + // the entry that initiated this fill refetches a resident partial sector + output wire fill_refetch, + // ... after a refetch found it evicted: the sector it re-reads stayed clean + output wire fill_retry, // probe: pending requests for `probe_addr`'s line, split by type. // probe_pending_ld : a non-AMO (line-filling) request is pending — @@ -95,17 +100,31 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire dequeue_valid, output wire [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr, output wire dequeue_rw, + output wire dequeue_reads, // the head reads its sector (load or AMO) output wire [DATA_WIDTH-1:0] dequeue_data, output wire [MSHR_ADDR_WIDTH-1:0] dequeue_id, input wire dequeue_ready, + // the dequeue this cycle is the bank's fill-forward port, not a replay + input wire dequeue_fwd, + // park the chain at its head instead of consuming it: the entry stays + // allocated (it becomes the root of a refetch) and the walk stops. + input wire dequeue_park, + input wire dequeue_park_retry, + // withhold the head this cycle (registered-only inputs) + input wire dequeue_block, // allocate input wire allocate_valid, input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector, + // the core request entering S0 next cycle, and the S0 advance enable + input wire [`CS_LINE_ADDR_WIDTH-1:0] allocate_addr_n, + input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] allocate_sector_n, + input wire allocate_adv, input wire [WORD_SEL_WIDTH-1:0] allocate_word_idx, input wire allocate_rw, input wire allocate_is_amo, // AMO: never coalesce + input wire allocate_refetch, // reads a resident partial sector input wire [DATA_WIDTH-1:0] allocate_data, output wire [MSHR_ADDR_WIDTH-1:0] allocate_id, output wire allocate_pending, @@ -128,12 +147,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( output wire empty ); `UNUSED_PARAM (BANK_ID) + `UNUSED_PARAM (BANK_HASH) - reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; - // Per-entry sector. Coalescing matches on {line, sector} so same-line - // different-sector misses get independent fills (each replay then hits its - // own filled sector). Zero-width-equivalent (1 bit, all 0) when 1 sector/line. - reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; // Per-entry word-in-line select, kept outside the payload RAM so the // allocate probe can compare it: the hit-demotion order hazard is // word-granular (a chained entry constrains only requests that touch @@ -152,6 +167,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // amo_mask is the per-entry AMO flag, zero when AMO_ENABLE=0 so the // mask folds out of addr_matches and the probe logic disappears. wire [MSHR_SIZE-1:0] amo_mask; + wire [MSHR_SIZE-1:0] amo_entries; reg allocate_rdy, allocate_rdy_n; reg [MSHR_ADDR_WIDTH-1:0] allocate_id_r, allocate_id_n; @@ -163,6 +179,13 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( wire allocate_fire = allocate_valid && allocate_ready; wire dequeue_fire = dequeue_valid && dequeue_ready; + wire dequeue_park_fire = (LAZY_FETCH != 0) && dequeue_fire && dequeue_park; + wire dequeue_free = dequeue_fire && ~dequeue_park_fire; + + // Entry holds the allocate request's {line, sector}. + // Coalescing matches on {line, sector} so same-line different-sector misses + // get independent fills (each replay then hits its own filled sector). + wire [MSHR_SIZE-1:0] entry_match; wire [MSHR_SIZE-1:0] addr_matches; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_addr_matches @@ -175,9 +198,8 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // cycle: a released head never fills, so a request that linked behind // it would wait forever. Exclude it too — the requester re-looks-up // and hits the line the releasing entry just installed. - assign addr_matches[i] = valid_table[i] && (addr_table[i] == allocate_addr) - && (sector_table[i] == allocate_sector) && ~amo_mask[i] - && ~(dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + assign addr_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] + && ~(dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i))) && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); end @@ -191,7 +213,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // the next id never re-selects the slot being allocated this cycle. wire [MSHR_SIZE-1:0] dequeue_free_oh, finalize_free_oh, allocate_set_oh; for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_free_delta - assign dequeue_free_oh[i] = dequeue_fire && (dequeue_id == MSHR_ADDR_WIDTH'(i)); + assign dequeue_free_oh[i] = dequeue_free && (dequeue_id == MSHR_ADDR_WIDTH'(i)); assign finalize_free_oh[i] = finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i)); assign allocate_set_oh[i] = allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i)); end @@ -227,7 +249,9 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( dequeue_id_n = fill_id; end - if (dequeue_fire) begin + if (dequeue_park_fire) begin + dequeue_val_n = 0; + end else if (dequeue_fire) begin valid_table_n[dequeue_id] = 0; if (next_table[dequeue_id]) begin dequeue_id_n = next_index[dequeue_id]; @@ -268,8 +292,6 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( end if (allocate_fire) begin - addr_table[allocate_id] <= allocate_addr; - sector_table[allocate_id] <= allocate_sector; word_table[allocate_id] <= allocate_word_idx; write_table[allocate_id] <= allocate_rw; end @@ -283,6 +305,40 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( next_table <= next_table_n; end + reg [`CS_LINE_ADDR_WIDTH-1:0] addr_table [0:MSHR_SIZE-1]; + reg [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_table [0:MSHR_SIZE-1]; + + always @(posedge clk) begin + if (allocate_fire) begin + addr_table[allocate_id] <= allocate_addr; + sector_table[allocate_id] <= allocate_sector; + end + end + + // The allocate match is taken a cycle early, against the core request + // entering S0 and the table as it stands after this cycle's allocate, + // so S0 starts it from flops: it gates the data array's write enable + // through the order-hazard probes. A request of another kind entering + // S0 does not allocate, so its match is never used. Nothing allocates + // while S0 holds. + wire alloc_self_match = (allocate_addr == allocate_addr_n) && (allocate_sector == allocate_sector_n); + reg [MSHR_SIZE-1:0] entry_match_r; + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_entry_match + wire match_n = (allocate_fire && (allocate_id == MSHR_ADDR_WIDTH'(i))) ? alloc_self_match + : ((addr_table[i] == allocate_addr_n) && (sector_table[i] == allocate_sector_n)); + always @(posedge clk) begin + if (allocate_adv) begin + entry_match_r[i] <= match_n; + end + end + assign entry_match[i] = entry_match_r[i]; + `RUNTIME_ASSERT (~allocate_valid || (entry_match_r[i] == ((addr_table[i] == allocate_addr) && (sector_table[i] == allocate_sector))), + ("%t: *** %s allocate match out of sync: id=%0d", $time, INSTANCE_ID, i)) + end + + assign fill_addr = addr_table[fill_id]; + assign fill_sector = sector_table[fill_id]; + `RUNTIME_ASSERT(~(allocate_fire && valid_table[allocate_id_r]), ("*** %s inuse allocation: addr=0x%0h, id=%0d (#%0d)", INSTANCE_ID, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID), allocate_id_r, alc_req_uuid)) @@ -301,22 +357,9 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( dequeue_id, next_table[dequeue_id])) `ifdef SIMULATION - // Every entry the dequeue pointer lands on must belong to the line whose - // fill opened the chain, whichever way the pointer got there (fill, - // next_index walk, or same-cycle late-joiner splice). Checked here rather - // than at the bank's forward port because only the MSHR can see all three. - reg [`CS_LINE_ADDR_WIDTH-1:0] chain_addr_r; - always @(posedge clk) begin - if (fill_valid) begin - chain_addr_r <= addr_table[fill_id]; - end - end - `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != chain_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, - dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(chain_addr_r, BANK_ID))) - // The slot the dequeue pointer names must not be handed to another line - // while the pointer still rests on it: `dequeue_addr` is combinational on - // `addr_table[dequeue_id_r]`, so the rewrite retargets an in-flight replay. + // while the pointer still rests on it: a combinational `dequeue_addr` on + // the entry at `dequeue_id_r` would retarget an in-flight replay. `RUNTIME_ASSERT(~(allocate_fire && dequeue_val && (allocate_id == dequeue_id)), ("*** %s allocate over live dequeue pointer: id=%0d, addr=0x%0h", INSTANCE_ID, allocate_id, `CS_BANK_TO_FULL_ADDR(allocate_addr, BANK_ID))) `endif @@ -338,8 +381,33 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( .rdata (dequeue_data) ); - assign fill_addr = addr_table[fill_id]; - assign fill_sector = sector_table[fill_id]; + if (LAZY_FETCH != 0) begin : g_refetch + reg [MSHR_SIZE-1:0] refetch_table, retry_table; + always @(posedge clk) begin + if (allocate_fire) begin + refetch_table[allocate_id] <= allocate_refetch; + retry_table[allocate_id] <= 1'b0; + end + if (dequeue_park_fire) begin + refetch_table[dequeue_id] <= 1'b1; + retry_table[dequeue_id] <= dequeue_park_retry; + end + end + assign fill_refetch = refetch_table[fill_id]; + assign fill_retry = retry_table[fill_id]; + // registered with the pointer so the park decision stays off the + // payload read + reg dequeue_reads_r; + always @(posedge clk) begin + dequeue_reads_r <= ~write_table[dequeue_id_n] || amo_entries[dequeue_id_n]; + end + assign dequeue_reads = dequeue_reads_r; + end else begin : g_no_refetch + assign fill_refetch = 1'b0; + assign fill_retry = 1'b0; + assign dequeue_reads = 1'b0; + `UNUSED_VAR ({allocate_refetch, dequeue_park, dequeue_park_retry, dequeue_park_fire, amo_entries}) + end if (AMO_ENABLE != 0) begin : g_amo reg [MSHR_SIZE-1:0] amo_table; @@ -373,6 +441,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( // in arrival order like any request; the probe outputs alone provide // the same-line ordering guard. assign amo_mask = (AMO_PASSTHRU != 0) ? amo_table : '0; + assign amo_entries = amo_table; // At the LLC no request links behind a pending AMO: the bank defers // every request to a line while an AMO to it waits here. `RUNTIME_ASSERT (~((AMO_PASSTHRU == 0) && finalize_valid && ~finalize_is_release && finalize_is_pending @@ -412,6 +481,7 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign persist_amo = finalize_valid && ~finalize_is_release && amo_table[finalize_id]; end else begin : g_no_amo assign amo_mask = '0; + assign amo_entries = '0; assign probe_pending_ld = 1'b0; assign probe_pending_amo = 1'b0; assign {probe_any_ld, probe_any_amo, persist_ld, persist_amo} = '0; @@ -423,12 +493,12 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( assign allocate_id = allocate_id_r; assign allocate_previd = prev_idx; - if (WRITEBACK) begin : g_pending_wb - assign allocate_pending = |addr_matches; - end else begin : g_pending_wt - // exclude write requests if writethrough - assign allocate_pending = |(addr_matches & ~write_table); - end + // Any undrained entry of the line is pending, writes included. On a + // write-through cache a write entry only lives behind a read root, so once + // the root has replayed the writes still waiting are a chain mid-drain: a + // same-word access must link behind them, not overtake them through the + // array. + assign allocate_pending = |addr_matches; // Word-granular order-hazard probes. A chained entry only constrains a // requester that touches the SAME word: the entry's replay reads or @@ -438,39 +508,56 @@ module VX_cache_mshr import VX_gpu_pkg::*; #( for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_word_matches assign word_matches[i] = (word_table[i] == allocate_word_idx); end + // These probes gate the data array's write enable at S0, so they leave the + // replay dequeue out of their exclusion: a replay that could change them (a + // same-word entry, with a write on either side, against an allocating core + // request) is held that cycle by the bank's link hold, so the exclusion + // never alters them, and dropping it takes the S0 address compare behind + // that hold out of the write-enable cone. The forward port is not held, so + // its dequeue stays excluded, as does a release. + wire [MSHR_SIZE-1:0] hazard_matches; + for (genvar i = 0; i < MSHR_SIZE; ++i) begin : g_hazard_matches + assign hazard_matches[i] = valid_table[i] && entry_match[i] && ~amo_mask[i] + && ~(dequeue_fwd && ~((LAZY_FETCH != 0) && dequeue_park) && (dequeue_id == MSHR_ADDR_WIDTH'(i))) + && ~(finalize_valid && finalize_is_release && (finalize_id == MSHR_ADDR_WIDTH'(i))); + end + wire hazard_pending = |hazard_matches; // The probed word's chain holds a WRITE that has not reached the data array // yet. A same-word hit completing ahead of it would read pre-store data (or // a write-hit would be overwritten by the older store's replay), inverting // arrival order — the bank demotes such a hit to a chained miss. - assign allocate_pending_wr = |(addr_matches & write_table & word_matches); + assign allocate_pending_wr = hazard_pending && |(hazard_matches & write_table & word_matches); // Any undrained same-word entry: an incoming WRITE hit must not land ahead // of an older chained read of the same word. - assign allocate_pending_ovl = |(addr_matches & word_matches); - - assign dequeue_valid = dequeue_val; - // Sampled with dequeue_id_n, alongside the payload in mshr_store, rather - // than read live at dequeue_id_r a cycle later. The entry's slot can be - // freed and reallocated while the pointer still names it -- the free set - // offers a slot released by a dequeue or a finalize in the same cycle -- and - // a live read then retargets an in-flight replay onto the new occupant's - // line. The bank's fill-forward path answers that replay from the staged - // sector of the line it was actually issued for, so the requester silently - // receives another line's data; the guard for it is a RUNTIME_ASSERT, which - // synthesis compiles out. - // - // Taking the address from the same snapshot as the payload also removes an - // inconsistency that was there regardless: the payload was a snapshot and - // the address was not, so the two could describe different entries. - // - // Withholding the slot from the free set instead deadlocks: the pointer can - // park on a blocked replay, and if that slot is the last free one the core - // stalls and the replay never drains. + assign allocate_pending_ovl = hazard_pending && |(hazard_matches & word_matches); + // the order hazard the bank derives from the two probes is the one the full + // exclusion gives + `RUNTIME_ASSERT (~allocate_valid + || ((allocate_pending_wr || (allocate_rw && allocate_pending_ovl)) + == (allocate_pending && (|(addr_matches & write_table & word_matches) + || (allocate_rw && |(addr_matches & word_matches))))), + ("%t: *** %s order-hazard probe diverged from the full exclusion", $time, INSTANCE_ID)) + + assign dequeue_valid = dequeue_val && ~((LAZY_FETCH != 0) && dequeue_block); + // Coalescing keys on the line, so every entry a chain walks (the fill's + // root, its next_index successors, a late joiner spliced in) holds the + // fill's line: the address is captured once, at the fill, and the late + // dequeue pointer stays off it. Being a register, it also cannot follow a + // slot that is freed and reallocated while the pointer still names it, + // which would retarget an in-flight replay onto the new occupant's line. + // Withholding that slot from the free set instead deadlocks: the pointer + // can park on a blocked replay, and if that slot is the last free one the + // core stalls and the replay never drains. reg [`CS_LINE_ADDR_WIDTH-1:0] dequeue_addr_r; reg dequeue_rw_r; always @(posedge clk) begin - dequeue_addr_r <= addr_table[dequeue_id_n]; - dequeue_rw_r <= write_table[dequeue_id_n]; + if (fill_valid) begin + dequeue_addr_r <= fill_addr; + end + dequeue_rw_r <= write_table[dequeue_id_n]; end + `RUNTIME_ASSERT(~(dequeue_val && ~fill_valid && (addr_table[dequeue_id] != dequeue_addr_r)), ("*** %s chain entry on foreign line: dequeue_id=%0d, entry=0x%0h, chain=0x%0h", INSTANCE_ID, + dequeue_id, `CS_BANK_TO_FULL_ADDR(addr_table[dequeue_id], BANK_ID), `CS_BANK_TO_FULL_ADDR(dequeue_addr_r, BANK_ID))) assign dequeue_addr = dequeue_addr_r; assign dequeue_rw = dequeue_rw_r; diff --git a/hw/rtl/cache/VX_cache_tags.sv b/hw/rtl/cache/VX_cache_tags.sv index 8f208a219a..27a7e2f6ab 100644 --- a/hw/rtl/cache/VX_cache_tags.sv +++ b/hw/rtl/cache/VX_cache_tags.sv @@ -35,6 +35,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( parameter WORD_SIZE = 1, // Size of a word in bytes parameter SECTOR_SIZE = LINE_SIZE, // Size of a sector in bytes (fill/eviction granule); = LINE_SIZE => 1 sector parameter WRITEBACK = 0, // Enable cache writeback + parameter LAZY_FETCH = 0, // Track per-sector partial (allocated-by-store) state parameter AMO_ENABLE = 0 // Enable the AMO-passthrough line invalidate (non-LLC banks only) ) ( input wire clk, @@ -47,16 +48,28 @@ module VX_cache_tags import VX_gpu_pkg::*; #( input wire fill, input wire read, input wire write, - input wire invalidate, // clear valid on the hit way + input wire invalidate, // clear valid on the hit way (qualifies the way's own match) + // the fill installs no data: the sector becomes partial (cleared otherwise) + input wire fill_nodata, + // a refetch completes its resident partial sector: clear its partial bit + input wire refetch, input wire [`CS_LINE_SEL_BITS-1:0] line_idx, input wire [`CS_LINE_SEL_BITS-1:0] line_idx_n, input wire [`CS_TAG_SEL_BITS-1:0] line_tag, input wire [`UP(`CS_SECTOR_SEL_BITS)-1:0] sector_idx, // requested sector within the line input wire [`CS_WAY_SEL_WIDTH-1:0] evict_way, + // the way a flush walks: the request's registered way, never a fill's + // compare-derived target, so the flush's clears stay off the tag read + input wire [`CS_WAY_SEL_WIDTH-1:0] flush_way, + // the way a writeback evicts, for evict_tag: a fill's replacement victim + // (a refill evicts nothing) or a flush's way -- never the tag compare + input wire [`CS_WAY_SEL_WIDTH-1:0] wb_way, // fill target way as one-hot, taken from line_present/victim directly: // the binary evict_way round trip (encode, mux, per-way compare) put the // tag-read cone on this array's own write-enable pins. input wire [NUM_WAYS-1:0] fill_way_oh, + // the replacement victim as one-hot: the tag store's write enable + input wire [NUM_WAYS-1:0] victim_way_oh, // outputs output wire [NUM_WAYS-1:0] tag_matches, @@ -70,7 +83,11 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // per-sector dirty vector of the evict way (drives the multi-beat per-sector // writeback); a single bit == evict_dirty when 1 sector/line. output wire [`CS_SECTORS_PER_LINE-1:0] evict_dirty_mask, - output wire [`CS_TAG_SEL_BITS-1:0] evict_tag + output wire [`CS_TAG_SEL_BITS-1:0] evict_tag, + // the evict way holds a valid sector + output wire evict_valid, + // the requested sector is resident but partial (a load/AMO to it refetches) + output wire partial_match ); // tag store: tag only (valid + dirty decoupled into side LUTRAMs) localparam SEC = `CS_SECTORS_PER_LINE; @@ -82,14 +99,19 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] read_tag; wire [NUM_WAYS-1:0][SEC-1:0] read_valid; wire [NUM_WAYS-1:0][SEC-1:0] read_dirty; + wire [NUM_WAYS-1:0][SEC-1:0] read_partial; + wire [NUM_WAYS-1:0] partial_hit; + + assign evict_valid = (| read_valid[evict_way]); if (WRITEBACK) begin : g_evict_tag_wb assign evict_dirty = (| read_dirty[evict_way]); // dirty if any sector dirty assign evict_dirty_mask = read_dirty[evict_way]; - assign evict_tag = read_tag[evict_way]; + assign evict_tag = read_tag[wb_way]; end else begin : g_evict_tag_wt `UNUSED_VAR (read_dirty) `UNUSED_VAR (read_tag) + `UNUSED_VAR (wb_way) assign evict_dirty = 1'b0; assign evict_dirty_mask = '0; assign evict_tag = '0; @@ -97,9 +119,8 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // Per-way decoded write strobes and payloads. At most one operation type // fires per cycle (input arbitration in the bank). The tag store holds the - // tag only and is written (with the registered line_tag) only on a fill, so - // its write never depends on the tag compare. - wire [NUM_WAYS-1:0] line_write; + // tag only and is written only on a fill, so its write never depends on + // the tag compare. wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] line_wdata; wire [NUM_WAYS-1:0][`CS_TAG_SEL_BITS-1:0] tag_rdata; @@ -117,8 +138,18 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire [NUM_WAYS-1:0][SEC-1:0] dirty_wdata; wire [NUM_WAYS-1:0][SEC-1:0] dirty_rdata; + // Per-sector partial store (lazy fetch only): a fill writes its sector's + // bit, set by a data-less fill and cleared by a data fill. The bit is only + // meaningful while the sector is valid, so no other operation touches it. + wire [NUM_WAYS-1:0][SEC-1:0] partial_wren; + wire [NUM_WAYS-1:0][SEC-1:0] partial_wdata; + wire [NUM_WAYS-1:0][SEC-1:0] partial_rdata; + + // a sector refill rewrites the line's own tags; a line fill writes the + // victim way's new tag + wire fill_refill_any; for (genvar i = 0; i < NUM_WAYS; ++i) begin : g_way_decode - wire way_en = (NUM_WAYS == 1) || (evict_way == i); + wire way_en = (NUM_WAYS == 1) || (flush_way == i); wire do_init = init; // init all ways // The fill way select comes in one-hot (line_present or the registered // victim decode); the binary way_en compare stays only for flush, whose @@ -128,11 +159,12 @@ module VX_cache_tags import VX_gpu_pkg::*; #( wire do_write = WRITEBACK && write && tag_matches[i]; // only write on tag hit // Tag store: the tag changes only on a fill (a refill rewrites the - // identical tag, harmless), so its write-enable does not depend on the - // compare and its write-data is the registered line_tag. init/flush/ - // invalidate only touch the valid vector (below), never the tag. - assign line_write[i] = do_fill; - assign line_wdata[i] = line_tag; + // identical tag, harmless). init/flush/invalidate only touch the valid + // vector (below), never the tag. The fill way follows the tag compare + // (a refill targets the resident way), so a fill rewrites every way -- + // the filled one with line_tag, the others with their current tag -- + // and the compare reaches the array's data, not its write enables. + assign line_wdata[i] = fill_refill_any ? read_tag[i] : line_tag; // Read-First BRAM: a fill committed on the previous cycle isn't yet in // the readout. The bypass is keyed on the SET and the FILLED WAY: that @@ -163,8 +195,12 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // hit: tag match AND the requested sector is valid. read_valid folds in // a same-set fill from the previous cycle (see the bypass above), so - // this compare is exact even on the fill's write edge. - wire raw_hit = read_valid[i][sector_idx] && tag_eq; + // this compare is exact even on the fill's write edge. Whether a partial + // sector serves the lookup is decided outside, off the array enables. + wire sector_valid = read_valid[i][sector_idx] && tag_eq; + wire sector_partial = (LAZY_FETCH != 0) && read_partial[i][sector_idx]; + wire raw_hit = sector_valid; + assign partial_hit[i] = sector_valid && sector_partial; // AMO passthrough invalidate: clear the requested sector's valid. wire do_inval = (AMO_ENABLE != 0) && invalidate && raw_hit; @@ -179,12 +215,21 @@ module VX_cache_tags import VX_gpu_pkg::*; #( // read-modify-write): set the filled sector; clear the whole line on // init/flush, the other sectors on a fresh victim install, the requested // sector on invalidate. - wire [SEC-1:0] vset_oh = do_fill ? sec_oh : {SEC{1'b0}}; - wire [SEC-1:0] vclr_oh = ((do_init || do_flush) ? {SEC{1'b1}} : {SEC{1'b0}}) - | ((do_fill && ~fill_refill) ? ~sec_oh : {SEC{1'b0}}) - | (do_inval ? sec_oh : {SEC{1'b0}}); - assign valid_wren[i] = vset_oh | vclr_oh; - assign valid_wdata[i] = vset_oh; + // The fill way follows the tag compare (a refill targets the resident + // way), so a fill rewrites every way's valid bits -- the filled way + // with its update, the others with their current bits -- and the + // compare reaches the store's data, not its write enables. The current + // bits are read_valid, which already carries last cycle's fill, with + // last cycle's other same-set clears applied here, off the hit path. + wire [SEC-1:0] vclr_op = ((do_init || do_flush) ? {SEC{1'b1}} : {SEC{1'b0}}) + | (do_inval ? sec_oh : {SEC{1'b0}}); + wire same_set_v = (line_idx == line_idx_n); + wire [SEC-1:0] rdw_vclr; + `BUFFER_EX(rdw_vclr, (same_set_v ? vclr_op : {SEC{1'b0}}), ~stall, SEC, 1); + wire [SEC-1:0] cur_valid = read_valid[i] & ~rdw_vclr; + wire [SEC-1:0] fill_valid = do_fill ? (fill_refill ? (cur_valid | sec_oh) : sec_oh) : cur_valid; + assign valid_wren[i] = (fill ? {SEC{1'b1}} : {SEC{1'b0}}) | vclr_op; + assign valid_wdata[i] = fill ? fill_valid : {SEC{1'b0}}; // Tag from the tag BRAM, valid from the decoupled LUTRAM. Both share the // same look-ahead read / read-first access pattern, so the way_filled @@ -199,12 +244,23 @@ module VX_cache_tags import VX_gpu_pkg::*; #( if (WRITEBACK) begin : g_dirty // set the written sector on a write hit; clear sectors on // fill/inval; clear all on init/flush or a fresh (evicting) fill. + // The hit way of a write and the target way of a fill follow the + // tag compare, so those operations rewrite every way -- the target + // with its update, the others with their current bits (read_dirty + // is exact: it forwards last cycle's update) -- and the compare + // reaches the store's data, not its write enables. + wire write_op = WRITEBACK && write; wire [SEC-1:0] dset_oh = do_write ? sec_oh : {SEC{1'b0}}; wire dclr_all = do_init || do_flush || (do_fill && ~fill_refill); wire [SEC-1:0] dclr_oh = (do_inval ? sec_oh : {SEC{1'b0}}) | ((do_fill && fill_refill) ? sec_oh : {SEC{1'b0}}); - assign dirty_wren[i] = dset_oh | dclr_oh | {SEC{dclr_all}}; - assign dirty_wdata[i] = dset_oh; // 1 only where setting; 0 on any clear + wire [SEC-1:0] fill_keep = do_fill ? (fill_refill ? (read_dirty[i] & ~sec_oh) : {SEC{1'b0}}) + : read_dirty[i]; + assign dirty_wren[i] = (write_op ? sec_oh : {SEC{1'b0}}) + | {SEC{fill || do_init || do_flush}} + | (do_inval ? sec_oh : {SEC{1'b0}}); + assign dirty_wdata[i] = fill ? fill_keep + : ((write_op ? read_dirty[i] : {SEC{1'b0}}) | dset_oh); // 0 on any clear // Read-first forwarding for the dirty LUTRAM (same-set, 1 cycle, held // across stall): a dirty update to this set last cycle isn't in the @@ -224,11 +280,29 @@ module VX_cache_tags import VX_gpu_pkg::*; #( assign read_dirty[i] = {SEC{1'b0}}; end + if (LAZY_FETCH != 0) begin : g_partial + // Same access pattern as the valid store, so the way_filled bypass + // corrects a same-set fill from the previous cycle the same way. A + // refetch's clear needs none: nothing is looked up right behind it. + wire rdw_nodata; + `BUFFER_EX(rdw_nodata, fill_nodata, ~stall, 1, 1); + wire do_refetch = refetch && sector_valid; + assign partial_wren[i] = (do_fill || do_refetch) ? sec_oh : {SEC{1'b0}}; + assign partial_wdata[i] = {SEC{do_fill && fill_nodata}}; + assign read_partial[i] = way_filled ? ((partial_rdata[i] & ~rdw_sec_oh) | (rdw_sec_oh & {SEC{rdw_nodata}})) + : partial_rdata[i]; + end else begin : g_no_partial + assign partial_wren[i] = '0; + assign partial_wdata[i] = '0; + assign read_partial[i] = {SEC{1'b0}}; + end + assign tag_matches[i] = raw_hit; // line resident in this way: tag matches and at least one sector valid // (a same-cycle-prior fill is folded into tag_eq/read_valid). assign line_present[i] = tag_eq && (| read_valid[i]); end + assign fill_refill_any = (SEC > 1) && (| line_present); // Single tag array: one BRAM word holds all ways' tags; per-way write-enable // updates a single way on a fill. Read at line_idx_n (one cycle ahead), @@ -243,18 +317,23 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - // fill alone: only fills write the tag store, and the fill way select - // (one-hot) always names exactly one way, so |line_write == fill -- - // an identity synthesis cannot prove, leaving the way-select cone on - // the array's global enable. + // a fill writes the victim way's tag (see line_wdata) .write (fill), - .wren (line_write), + .wren (victim_way_oh), .waddr (line_idx), .raddr (line_idx_n), .wdata (line_wdata), .rdata (tag_rdata) ); + // The side stores' array enables are the operations that may write them, + // never the OR of their per-bit enables: those carry the tag compare, which + // would otherwise fan into every LUTRAM write enable. A bit still changes + // only where its own enable is set. + wire valid_write = init || flush || fill || ((AMO_ENABLE != 0) && invalidate); + wire dirty_write = init || flush || fill || write || ((AMO_ENABLE != 0) && invalidate); + wire partial_write = fill || refetch; + // Decoupled per-sector valid store: mirrors the tag BRAM's access pattern // (look-ahead read, read-first) so pipeline alignment is identical, with // per-bit write-enable so a sector refill sets its bit without reading back. @@ -269,7 +348,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - .write (| valid_wren), + .write (valid_write), .wren (valid_wren), .waddr (line_idx), .raddr (line_idx_n), @@ -292,7 +371,7 @@ module VX_cache_tags import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .read (~stall), - .write (| dirty_wren), + .write (dirty_write), .wren (dirty_wren), .waddr (line_idx), .raddr (line_idx_n), @@ -301,7 +380,33 @@ module VX_cache_tags import VX_gpu_pkg::*; #( ); end else begin : g_no_dirty_store assign dirty_rdata = '0; - `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata}) + `UNUSED_VAR ({dirty_wren, dirty_wdata, dirty_rdata, dirty_write}) + end + + if (LAZY_FETCH != 0) begin : g_partial_store + VX_dp_ram #( + .DATAW (NUM_WAYS * SEC), + .WRENW (NUM_WAYS * SEC), + .SIZE (`CS_LINES_PER_BANK), + .OUT_REG (1), + .LUTRAM (1), + .RDW_MODE ("R") + ) partial_store ( + .clk (clk), + .reset (reset), + .read (~stall), + .write (partial_write), + .wren (partial_wren), + .waddr (line_idx), + .raddr (line_idx_n), + .wdata (partial_wdata), + .rdata (partial_rdata) + ); + assign partial_match = (| partial_hit); + end else begin : g_no_partial_store + assign partial_rdata = '0; + assign partial_match = 1'b0; + `UNUSED_VAR ({partial_wren, partial_wdata, partial_rdata, partial_hit, read_partial, fill_nodata, refetch, partial_write}) end endmodule diff --git a/hw/rtl/cache/VX_cache_wrap.sv b/hw/rtl/cache/VX_cache_wrap.sv index caff8c34d3..6d705b20d6 100644 --- a/hw/rtl/cache/VX_cache_wrap.sv +++ b/hw/rtl/cache/VX_cache_wrap.sv @@ -42,6 +42,10 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter DIRTY_BYTES = 0, // Enable dirty bytes on writeback + parameter LAZY_FETCH = 0, // Write misses allocate without a fill (write-validate) + + parameter WRITE_AROUND = 0, // Write misses flagged no-allocate go to memory (write-back only) + parameter REPL_POLICY = `CS_REPL_FIFO, // Replacement policy parameter TAG_WIDTH = UUID_WIDTH + 1, // core request tag size @@ -55,6 +59,12 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( parameter MEM_OUT_BUF = 3, // Memory request output buffer parameter IS_LLC = 0, // Set when this cache is the LLC (enables AMO completion at this level). + parameter SHARED_LLC = 0, // The LLC serves every memory client alone (point of coherence). + parameter ACQ_FORWARD = 0, // An acquire flush also invalidates the next level. + parameter FLUSH_FORWARD = 1, // A flush also flushes the next level. + parameter BANK_HASH = 0, // Hashed bank select (cache_bank_hash). + parameter READ_MERGE = 0, // Merge same-word reads within groups of this many adjacent ports (0: off). + parameter READ_MERGE_LINE = 0, // Merge any reads of the same line instead, each port keeping its own tag and word. parameter AMO_ENABLE = 0 // Enable atomic operations in cache banks. ) ( @@ -66,7 +76,10 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( output cache_perf_t cache_perf, `endif VX_mem_bus_if.slave core_bus_if [NUM_REQS], - VX_mem_bus_if.master mem_bus_if [MEM_PORTS] + VX_mem_bus_if.master mem_bus_if [MEM_PORTS], + + // no uncached or outgoing request waiting in the bypass path + output wire req_empty ); `STATIC_ASSERT(NUM_BANKS == (1 << `CLOG2(NUM_BANKS)), ("invalid parameter")) localparam CACHE_MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH); @@ -106,9 +119,11 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .core_bus_in_if (core_bus_if), .core_bus_out_if(core_bus_cache_if), .mem_bus_in_if (mem_bus_cache_if), - .mem_bus_out_if (mem_bus_tmp_if) + .mem_bus_out_if (mem_bus_tmp_if), + .req_empty (req_empty) ); end else begin : g_no_bypass + assign req_empty = 1'b1; for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if `ASSIGN_VX_MEM_BUS_IF (core_bus_cache_if[i], core_bus_if[i]); end @@ -137,6 +152,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), + .WRITE_AROUND (WRITE_AROUND), .REPL_POLICY (REPL_POLICY), .CRSQ_SIZE (CRSQ_SIZE), .MSHR_SIZE (MSHR_SIZE), @@ -147,6 +164,12 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .CORE_OUT_BUF (BYPASS_ENABLE ? 1 : CORE_OUT_BUF), .MEM_OUT_BUF (BYPASS_ENABLE ? 1 : MEM_OUT_BUF), .IS_LLC (IS_LLC), + .SHARED_LLC (SHARED_LLC), + .ACQ_FORWARD (ACQ_FORWARD), + .FLUSH_FORWARD (FLUSH_FORWARD), + .BANK_HASH (BANK_HASH), + .READ_MERGE (READ_MERGE), + .READ_MERGE_LINE (READ_MERGE_LINE), .AMO_ENABLE (AMO_ENABLE) ) cache ( .clk (clk), @@ -155,7 +178,8 @@ module VX_cache_wrap import VX_gpu_pkg::*; #( .cache_perf (cache_perf), `endif .core_bus_if (core_bus_cache_if), - .mem_bus_if (mem_bus_cache_if) + .mem_bus_if (mem_bus_cache_if), + .nc_req_empty (req_empty) ); end else begin : g_passthru for (genvar i = 0; i < NUM_REQS; ++i) begin : g_core_bus_cache_if diff --git a/hw/rtl/core/VX_alu_block.sv b/hw/rtl/core/VX_alu_block.sv new file mode 100644 index 0000000000..b07a14a074 --- /dev/null +++ b/hw/rtl/core/VX_alu_block.sv @@ -0,0 +1,93 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One ALU block: the integer and mul/div datapaths behind a PE switch. +module VX_alu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_branch_ctl_if.master branch_ctl_if +); + localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_INT = 0; + localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; + + VX_execute_if #( + .data_t (alu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (alu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [`UP(PE_SEL_BITS)-1:0] pe_select; + always @(*) begin + pe_select = PE_IDX_INT; + if (`VX_CFG_EXT_M_ENABLED && (execute_if.data.op_args.alu.xtype == ALU_TYPE_MULDIV)) begin + pe_select = PE_IDX_MDV; + end + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_alu_int #( + .INSTANCE_ID (`SFORMATF(("%s-int%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) alu_int ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_INT]), + .branch_ctl_if (branch_ctl_if), + .result_if (pe_result_if[PE_IDX_INT]) + ); + +`ifdef VX_CFG_EXT_M_ENABLE + VX_alu_muldiv #( + .INSTANCE_ID (`SFORMATF(("%s-muldiv%s", INSTANCE_ID, BLOCK_STR))), + .NUM_LANES (NUM_LANES) + ) muldiv_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MDV]), + .result_if (pe_result_if[PE_IDX_MDV]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_alu_int.sv b/hw/rtl/core/VX_alu_int.sv index 3a680055bf..998a12d00b 100644 --- a/hw/rtl/core/VX_alu_int.sv +++ b/hw/rtl/core/VX_alu_int.sv @@ -15,12 +15,13 @@ module VX_alu_int import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_IDX = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, + input wire [`LOG2UP(`VX_CFG_NUM_ALU_BLOCKS)-1:0] block_idx, + // Inputs VX_execute_if.slave execute_if, @@ -47,7 +48,7 @@ module VX_alu_int import VX_gpu_pkg::*; #( reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] vote_result; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] shfl_result; - reg [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_result_r; `ifdef VX_CFG_XLEN_64 @@ -64,7 +65,6 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [INST_ALU_BITS-1:0] alu_op = INST_ALU_BITS'(execute_if.data.op_type); wire [INST_BR_BITS-1:0] br_op = INST_BR_BITS'(execute_if.data.op_type); wire is_br_op = (execute_if.data.op_args.alu.xtype == ALU_TYPE_BRANCH); - wire is_alu_op = (execute_if.data.op_args.alu.xtype == ALU_TYPE_ARITH); wire is_sub_op = inst_alu_is_sub(alu_op); wire is_signed = inst_alu_signed(alu_op); wire [1:0] op_class = is_br_op ? inst_br_class(alu_op) : inst_alu_class(alu_op); @@ -74,16 +74,15 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [1:0] wg_src_offset = execute_if.data.op_args.alu.imm20[1:0]; // group-relative source index - wire is_br_jal_op = is_br_op && (br_op <= INST_BR_JAL); - wire is_lui_op = is_alu_op && (alu_op == INST_ALU_LUI || alu_op == INST_ALU_AUIPC); - wire [31:0] lui_imm32 = {execute_if.data.op_args.alu.imm20, 12'd0}; wire [20:0] br_imm21 = {execute_if.data.op_args.alu.imm20, 1'b0}; wire [`VX_CFG_XLEN-1:0] alu_imm = `SEXT(`VX_CFG_XLEN, execute_if.data.op_args.alu.imm20); wire [`VX_CFG_XLEN-1:0] lui_imm = `SEXT(`VX_CFG_XLEN, lui_imm32); wire [`VX_CFG_XLEN-1:0] br_imm = `SEXT(`VX_CFG_XLEN, br_imm21); - wire [`VX_CFG_XLEN-1:0] add_imm = is_lui_op ? lui_imm : (is_br_jal_op ? br_imm : alu_imm); + // The decoder pre-selects the immediate form, so only flops steer this mux. + wire [`VX_CFG_XLEN-1:0] add_imm = execute_if.data.op_args.alu.imm_hi ? lui_imm + : (execute_if.data.op_args.alu.imm_x2 ? br_imm : alu_imm); wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] add_in1_PC = execute_if.data.op_args.alu.use_PC ? {NUM_LANES{to_fullPC(execute_if.data.header.PC)}} : alu_in1; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] add_in2_imm = execute_if.data.op_args.alu.use_imm ? {NUM_LANES{add_imm}} : alu_in2; @@ -165,14 +164,24 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] wgather_result; if (NUM_LANES > 1) begin : g_wgather wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] alu_in3 = execute_if.data.rs3_data; + // The fallback lane is shared by all lanes, so its operands are read + // once; each lane's own source is then a group-local select. + wire [`VX_CFG_XLEN-1:0] last_in1 = alu_in1[last_tid]; + wire [`VX_CFG_XLEN-1:0] last_in2 = alu_in2[last_tid]; + wire [`VX_CFG_XLEN-1:0] last_in3 = alu_in3[last_tid]; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_i - wire [LANE_BITS-1:0] nominal = (LANE_BITS'(i) & ~LANE_BITS'(3)) | LANE_BITS'(wg_src_offset); - wire [LANE_BITS-1:0] src_lane = execute_if.data.header.tmask[nominal] ? nominal : last_tid; - wire [1:0] offset = 2'(i) - wg_src_offset; // (i - src) mod 4 - assign wgather_result[i] = (offset == 2'd1) ? alu_in1[src_lane] - : (offset == 2'd2) ? alu_in2[src_lane] - : (offset == 2'd3) ? alu_in3[src_lane] - : `VX_CFG_XLEN'(0); // offset 0 = source lane (write-suppressed) + wire [LANE_BITS-1:0] nominal = (LANE_BITS'(i) & ~LANE_BITS'(3)) | LANE_BITS'(wg_src_offset); + wire [1:0] offset = 2'(i) - wg_src_offset; // (i - src) mod 4 + // offset 0 = source lane (write-suppressed) + wire [`VX_CFG_XLEN-1:0] nominal_val = (offset == 2'd1) ? alu_in1[nominal] + : (offset == 2'd2) ? alu_in2[nominal] + : (offset == 2'd3) ? alu_in3[nominal] + : `VX_CFG_XLEN'(0); + wire [`VX_CFG_XLEN-1:0] last_val = (offset == 2'd1) ? last_in1 + : (offset == 2'd2) ? last_in2 + : (offset == 2'd3) ? last_in3 + : `VX_CFG_XLEN'(0); + assign wgather_result[i] = execute_if.data.header.tmask[nominal] ? nominal_val : last_val; end end else begin : g_wgather_0 assign wgather_result[0] = alu_in1[0]; @@ -194,62 +203,70 @@ module VX_alu_int import VX_gpu_pkg::*; #( wire [LANE_BITS-1:0] lane_bfly = LANE_BITS'(i) ^ bval; wire [LANE_BITS-1:0] lane_idx = minLane | (bval & ~mask); + // The source lane is read before its bound check resolves, which + // only picks between that read and the lane's own value. reg [LANE_BITS-1:0] lane; + reg in_bounds; always @(*) begin - lane = LANE_BITS'(i); case (alu_op[1:0]) INST_SHFL_UP: begin - if ($signed(lane_up) >= $signed({1'b0, minLane})) begin - lane = lane_up[LANE_BITS-1:0]; - end + lane = lane_up[LANE_BITS-1:0]; + in_bounds = ($signed(lane_up) >= $signed({1'b0, minLane})); end INST_SHFL_DOWN: begin - if (lane_down <= {1'b0, maxLane}) begin - lane = lane_down[LANE_BITS-1:0]; - end + lane = lane_down[LANE_BITS-1:0]; + in_bounds = (lane_down <= {1'b0, maxLane}); end INST_SHFL_BFLY: begin - if (lane_bfly <= maxLane) begin - lane = lane_bfly; - end + lane = lane_bfly; + in_bounds = (lane_bfly <= maxLane); end - INST_SHFL_IDX: begin - if (lane_idx <= maxLane) begin - lane = lane_idx; - end + default: begin // INST_SHFL_IDX + lane = lane_idx; + in_bounds = (lane_idx <= maxLane); end endcase end - assign shfl_result[i] = execute_if.data.header.tmask[lane] ? alu_in1[lane] : alu_in1[i]; + wire take_lane = in_bounds && execute_if.data.header.tmask[lane]; + assign shfl_result[i] = take_lane ? alu_in1[lane] : alu_in1[i]; end end else begin : g_shfl_0 assign shfl_result[0] = alu_in1[0]; end + // The adder settles last, so its select is decoded from the op alone and + // it enters the result mux at the final level. + wire is_add_result = (execute_if.data.op_args.alu.xtype != ALU_TYPE_OTHER) + && ({is_alu_w, op_class} == 3'b000); + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_alu_result - wire [`VX_CFG_XLEN-1:0] slt_br_result = `VX_CFG_XLEN'({is_br_op && ~(| sub_result[i][`VX_CFG_XLEN-1:0]), sub_result[i][`VX_CFG_XLEN]}); + // the difference's low XLEN bits are zero exactly when the operands match + wire is_equal = (alu_in1[i] == sub_in2_imm[i]); + wire [`VX_CFG_XLEN-1:0] slt_br_result = `VX_CFG_XLEN'({is_br_op && is_equal, sub_result[i][`VX_CFG_XLEN]}); wire [`VX_CFG_XLEN-1:0] sub_slt_br_result = (is_sub_op && ~is_br_op) ? sub_result[i][`VX_CFG_XLEN-1:0] : slt_br_result; + reg [`VX_CFG_XLEN-1:0] other_result; always @(*) begin if (execute_if.data.op_args.alu.xtype == ALU_TYPE_OTHER) begin case (alu_op[3:2]) - 2'b00: alu_result[i] = vote_result[i]; - 2'b01: alu_result[i] = shfl_result[i]; - 2'b10: alu_result[i] = wgather_result[i]; - default: alu_result[i] = vote_result[i]; + 2'b00: other_result = vote_result[i]; + 2'b01: other_result = shfl_result[i]; + 2'b10: other_result = wgather_result[i]; + default: other_result = vote_result[i]; endcase end else begin case ({is_alu_w, op_class}) - 3'b000: alu_result[i] = add_result[i]; // ADD, LUI, AUIPC - 3'b001: alu_result[i] = sub_slt_br_result; // SUB, SLTU, SLTI, BR* - 3'b010: alu_result[i] = shr_zic_result[i]; // SRL, SRA, SRLI, SRAI, CZERO* - 3'b011: alu_result[i] = msc_result[i]; // AND, OR, XOR, SLL, SLLI - 3'b100: alu_result[i] = add_result_w[i]; // ADDIW, ADDW - 3'b101: alu_result[i] = sub_result_w[i]; // SUBW - 3'b110: alu_result[i] = shr_result_w[i]; // SRLW, SRAW, SRLIW, SRAIW - 3'b111: alu_result[i] = msc_result_w[i]; // SLLW + 3'b001: other_result = sub_slt_br_result; // SUB, SLTU, SLTI, BR* + 3'b010: other_result = shr_zic_result[i]; // SRL, SRA, SRLI, SRAI, CZERO* + 3'b011: other_result = msc_result[i]; // AND, OR, XOR, SLL, SLLI + 3'b100: other_result = add_result_w[i]; // ADDIW, ADDW + 3'b101: other_result = sub_result_w[i]; // SUBW + 3'b110: other_result = shr_result_w[i]; // SRLW, SRAW, SRLIW, SRAIW + 3'b111: other_result = msc_result_w[i]; // SLLW + default: other_result = 'x; // ADD, LUI, AUIPC: is_add_result endcase end end + assign alu_result[i] = is_add_result ? add_result[i] : other_result; end // branch @@ -339,7 +356,10 @@ module VX_alu_int import VX_gpu_pkg::*; #( : is_br_static ? from_fullPC(br_result) : cbr_dest_r; wire [NW_WIDTH-1:0] br_wid; - `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, BLOCK_IDX, `VX_CFG_NUM_ALU_BLOCKS) + `ASSIGN_BLOCKED_WID (br_wid, result_if.data.header.wid, block_idx, `VX_CFG_NUM_ALU_BLOCKS) + if (`VX_CFG_NUM_ALU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end VX_pipe_register #( .DATAW (1 + NW_WIDTH + 1 + PC_BITS + 1 + 1 + 4), diff --git a/hw/rtl/core/VX_alu_unit.sv b/hw/rtl/core/VX_alu_unit.sv index 3128187c54..e91603251c 100644 --- a/hw/rtl/core/VX_alu_unit.sv +++ b/hw/rtl/core/VX_alu_unit.sv @@ -27,14 +27,9 @@ module VX_alu_unit import VX_gpu_pkg::*; #( VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS] ); - `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_ALU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_ALU_LANES; localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); - localparam PE_COUNT = 1 + `VX_CFG_EXT_M_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_INT = 0; - localparam PE_IDX_MDV = PE_IDX_INT + `VX_CFG_EXT_M_ENABLED; VX_execute_if #( .data_t (alu_execute_t) @@ -51,66 +46,25 @@ module VX_alu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - - VX_execute_if #( - .data_t (alu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (alu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [`UP(PE_SEL_BITS)-1:0] pe_select; - always @(*) begin - pe_select = PE_IDX_INT; - if (`VX_CFG_EXT_M_ENABLED && (per_block_execute_if[block_idx].data.op_args.alu.xtype == ALU_TYPE_MULDIV)) - pe_select = PE_IDX_MDV; - end - - VX_pe_switch #( - .PE_COUNT (PE_COUNT), + VX_alu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))), .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF (0), - .RSP_OUT_BUF (PARTIAL_BW ? 1 : 3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[block_idx]), - .result_out_if (per_block_result_if[block_idx]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_alu_int #( - .INSTANCE_ID (`SFORMATF(("%s-int%0d", INSTANCE_ID, block_idx))), - .BLOCK_IDX (block_idx), - .NUM_LANES (NUM_LANES) - ) alu_int ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_INT]), - .branch_ctl_if (branch_ctl_if[block_idx]), - .result_if (pe_result_if[PE_IDX_INT]) - ); - - `ifdef VX_CFG_EXT_M_ENABLE - VX_alu_muldiv #( - .INSTANCE_ID (`SFORMATF(("%s-muldiv%0d", INSTANCE_ID, block_idx))), - .NUM_LANES (NUM_LANES) - ) muldiv_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MDV]), - .result_if (pe_result_if[PE_IDX_MDV]) + .PARTIAL_BW (PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .branch_ctl_if (branch_ctl_if[block_idx]) ); - `endif end VX_lane_gather #( diff --git a/hw/rtl/core/VX_backend_slice.sv b/hw/rtl/core/VX_backend_slice.sv new file mode 100644 index 0000000000..42c9630e8b --- /dev/null +++ b/hw/rtl/core/VX_backend_slice.sv @@ -0,0 +1,194 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One issue slice carried through issue, execute and commit, serving the warps +// with wid mod ISSUE_WIDTH == slice_id with one block of every execution unit. +// Every slice is the same module: its identity arrives on slice_id/core_id, and +// only the core-wide resources (frontend slices, warp-control resources, memory +// system, SFU buses, WGMMA tile buffers and the metadata AGU) are reached through +// its ports. +module VX_backend_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output issue_perf_t issue_perf, +`endif + + // decoded instructions of the slice's warps + VX_decode_if.slave decode_if, + + // issued and retired warps (registered), indexed by wid / ISSUE_WIDTH + output wire warp_issued, + output wire [ISSUE_WIS_W-1:0] warp_issued_wis, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // scheduler traffic of the slice's ALU and SFU blocks + VX_branch_ctl_if.master branch_ctl_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // LSU +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, + output wire lsu_busy, +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if, +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif + + // core-wide warp control and CTA contexts + VX_warp_shared_if.master warp_shared_if, + input cta_bcast_t cta_bcast +); + `STATIC_ASSERT (BACKEND_SLICES, ("invalid parameter")) + + `SCOPE_IO_SWITCH (2); + + VX_dispatch_if dispatch_if [NUM_EX_UNITS](); + VX_commit_if commit_if [NUM_EX_UNITS](); + VX_writeback_if writeback_if(); + + // issue + + wire issue_fire; + wire [ISSUE_WIS_W-1:0] issue_wis; + + VX_issue_slice #( + .INSTANCE_ID (`SFORMATF(("%s-issue%s", INSTANCE_ID, SLICE_STR))) + ) issue_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .warp_issued (issue_fire), + .warp_issued_wis (issue_wis) + ); + + `BUFFER(warp_issued, issue_fire); + `BUFFER(warp_issued_wis, issue_wis); + + // execute + + VX_execute_slice #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), + .SLICE_STR (SLICE_STR) + ) execute_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .dispatch_if (dispatch_if), + .commit_if (commit_if), + .branch_ctl_if (branch_ctl_if), + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if), + `endif + .dcache_bus_if (dcache_bus_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + .cta_bcast (cta_bcast) + ); + + // commit + + VX_commit_slice #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (commit_if), + .writeback_if (writeback_if), + .committed_wis (committed_wis) + ); + +endmodule diff --git a/hw/rtl/core/VX_bar_unit.sv b/hw/rtl/core/VX_bar_unit.sv index f3f7313e92..f6239012e3 100644 --- a/hw/rtl/core/VX_bar_unit.sv +++ b/hw/rtl/core/VX_bar_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_bar_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // local barrier interface input wire req_valid, input wire [NW_WIDTH-1:0] req_wid, @@ -36,16 +37,17 @@ module VX_bar_unit import VX_gpu_pkg::*; #( output wire [`VX_CFG_NUM_WARPS-1:0] unlock_mask // warps to unlock ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) // warp mask + warp count + event count localparam EVENT_WIDTH = `CLOG2(`VX_CFG_MAX_BAR_EVENTS + 1); - localparam BAR_STATEW = `VX_CFG_NUM_WARPS + NW_WIDTH + EVENT_WIDTH; - localparam GBAR_REQW = NB_WIDTH + NC_WIDTH; - localparam USE_GBAR = (`VX_CFG_NUM_CORES > 1); + // the count holds warps for a local barrier, cores for a global one + localparam COUNT_W = `MAX(NW_WIDTH, CORE_ID_WIDTH); + localparam BAR_STATEW = `VX_CFG_NUM_WARPS + COUNT_W + EVENT_WIDTH; + localparam GBAR_REQW = NB_WIDTH + CORE_ID_WIDTH; + localparam USE_GBAR = ((`VX_CFG_NUM_CORES * `VX_CFG_NUM_CLUSTERS) > 1); logic [`VX_CFG_NUM_WARPS-1:0] mask_r, mask_n; - logic [NW_WIDTH-1:0] count_r, count_n; + logic [COUNT_W-1:0] count_r, count_n; logic [EVENT_WIDTH-1:0] events_r, events_n; logic phase_r, phase_n; @@ -59,14 +61,14 @@ module VX_bar_unit import VX_gpu_pkg::*; #( logic gbar_enqueue; logic [NB_WIDTH-1:0] gbar_enqueue_id; - logic [NC_WIDTH-1:0] gbar_enqueue_size_m1; + logic [CORE_ID_WIDTH-1:0] gbar_enqueue_size_m1; wire gbar_rsp_ready = ~req_valid; wire gbar_rsp_fire = gbar_bus_if.rsp_valid && gbar_rsp_ready; wire gbar_rsp_apply = USE_GBAR && gbar_rsp_fire && gbar_pending_r[gbar_bus_if.rsp_data.id]; wire [`VX_CFG_NUM_WARPS-1:0] wait_mask = ((`VX_CFG_NUM_WARPS)'(1) << req_wid) | mask_r; - wire [NW_WIDTH-1:0] next_count = count_r + NW_WIDTH'(1); + wire [COUNT_W-1:0] next_count = count_r + COUNT_W'(1); wire next_phase = ~phase_r; always @(*) begin @@ -104,7 +106,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( end end else if (req_data.is_arrive) begin // barrier arrival - if (count_r == NW_WIDTH'(req_data.size_m1)) begin + if (count_r == COUNT_W'(NW_WIDTH'(req_data.size_m1))) begin count_n = '0; if (events_r == 0) begin mask_n = '0; @@ -148,11 +150,11 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(count_r); // was saved in barrier_arrive + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(count_r); // was saved in barrier_arrive end end else if (req_data.is_arrive) begin // barrier arrival - count_n = NW_WIDTH'(req_data.size_m1); // store participating number of cores + count_n = COUNT_W'(req_data.size_m1); // store participating number of cores if (req_data.is_sync) begin gbar_waiters_n[req_data.id][req_wid] = 1; end @@ -160,7 +162,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( mask_n = '0; gbar_enqueue = 1; // notify global barrier gbar_enqueue_id = req_data.id; - gbar_enqueue_size_m1 = NC_WIDTH'(req_data.size_m1); + gbar_enqueue_size_m1 = CORE_ID_WIDTH'(req_data.size_m1); end else begin // Add arriving warp to arrival mask mask_n = wait_mask; @@ -203,11 +205,15 @@ module VX_bar_unit import VX_gpu_pkg::*; #( wire store_phase_wdata = gbar_rsp_apply ? ~gbar_pending_phase_r[gbar_bus_if.rsp_data.id] : phase_n; wire store_phase_write = req_valid || gbar_rsp_apply; + // The state is read, updated and written back every cycle: in distributed + // RAM the loop starts from a fabric read rather than a block RAM's + // clock-to-out. VX_dp_ram #( .DATAW (BAR_STATEW), .SIZE (1 << BAR_ADDR_BITS), .RDW_MODE ("W"), - .OUT_REG (1) + .OUT_REG (1), + .LUTRAM (1) ) barrier_state_store ( .clk (clk), .reset (reset), @@ -223,8 +229,7 @@ module VX_bar_unit import VX_gpu_pkg::*; #( VX_dp_ram #( .DATAW (1), .SIZE (1 << BAR_ADDR_BITS), - .RDW_MODE ("W"), - .RADDR_REG(1) + .RDW_MODE ("W") ) barrier_phase_store ( .clk (clk), .reset (reset), @@ -243,20 +248,27 @@ module VX_bar_unit import VX_gpu_pkg::*; #( wire store_phase_rdata_v = store_valids[store_raddr] ? store_phase_rdata : '0; + // The state read's valid bit is registered with its address (forwarding a + // same-cycle write), so the state's consumers start from a flop rather than + // a store_valids lookup. + reg store_state_valid; + always @(posedge clk) begin if (reset) begin store_valids <= '0; + store_state_valid <= 0; phase_r <= '0; end else begin if (store_state_write) begin store_valids[store_waddr] <= 1'b1; end + store_state_valid <= store_valids[store_raddr] || (store_state_write && (store_waddr == store_raddr)); phase_r <= is_phase_rdw_hazard ? store_phase_wdata : store_phase_rdata_v; end store_waddr <= store_raddr; end - assign {mask_r, count_r, events_r} = store_valids[store_waddr] ? store_state_rdata : '0; + assign {mask_r, count_r, events_r} = store_state_valid ? store_state_rdata : '0; wire phase_async = is_phase_rdw_hazard ? store_phase_wdata : store_phase_rdata_v; @@ -329,12 +341,13 @@ module VX_bar_unit import VX_gpu_pkg::*; #( assign gbar_bus_if.req_valid = gbar_req_valid_r; assign {gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1} = req_queue_data; - assign gbar_bus_if.req_data.core_id = NC_WIDTH'(CORE_ID % `VX_CFG_NUM_CORES); + assign gbar_bus_if.req_data.core_id = core_id; assign gbar_bus_if.rsp_ready = gbar_rsp_ready; `RUNTIME_ASSERT(~gbar_enqueue || ~gbar_pending_r[gbar_enqueue_id], ("%s duplicate global barrier generation: id=%0d", INSTANCE_ID, gbar_enqueue_id)) `RUNTIME_ASSERT(~gbar_enqueue || (store_waddr == BAR_ADDR_W'(gbar_enqueue_id)), ("%s invalid global barrier slot: id=%0d, slot=%0d", INSTANCE_ID, gbar_enqueue_id, store_waddr)) end else begin : g_nogbar + `UNUSED_VAR (core_id) assign gbar_bus_if.req_valid = 0; assign gbar_bus_if.req_data = 'x; diff --git a/hw/rtl/core/VX_commit.sv b/hw/rtl/core/VX_commit.sv index 44a5437129..4cc0927c18 100644 --- a/hw/rtl/core/VX_commit.sv +++ b/hw/rtl/core/VX_commit.sv @@ -26,125 +26,28 @@ module VX_commit import VX_gpu_pkg::*; #( VX_writeback_if.master writeback_if [`VX_CFG_ISSUE_WIDTH], VX_commit_sched_if.master commit_sched_if ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam OUT_DATAW = $bits(commit_t); - - // commit arbitration - - VX_commit_if commit_arb_if[`VX_CFG_ISSUE_WIDTH](); - wire [`VX_CFG_ISSUE_WIDTH-1:0] committed_warps; - wire [`VX_CFG_ISSUE_WIDTH-1:0][PER_ISSUE_WARPS-1:0] commit_eop_wis; - - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_commit_arbs - - wire [NUM_EX_UNITS-1:0] valid_in; - wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; - wire [NUM_EX_UNITS-1:0] ready_in; - - // Decoded ahead of the output register so each warp's release reads - // its own flop. - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in - wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop) - << wid_to_wis(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid); - assign valid_in[j] = commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid; - assign data_in[j] = {eop_wis, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data}; - assign commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready = ready_in[j]; - end - - VX_stream_arb #( - .NUM_INPUTS (NUM_EX_UNITS), - .DATAW (PER_ISSUE_WARPS + OUT_DATAW), - .ARBITER ("P"), - .OUT_BUF (1) - ) commit_arb ( - .clk (clk), - .reset (reset), - .valid_in (valid_in), - .ready_in (ready_in), - .data_in (data_in), - .data_out ({commit_eop_wis[i], commit_arb_if[i].data}), - .valid_out (commit_arb_if[i].valid), - .ready_out (commit_arb_if[i].ready), - `UNUSED_PIN (sel_out) - ); - - wire commit_arb_fire = commit_arb_if[i].valid && commit_arb_if[i].ready; - assign committed_warps[i] = commit_arb_fire && commit_arb_if[i].data.eop; - end - - // notify scheduler: build per-warp committed mask from per-slot signals - wire [`VX_CFG_ISSUE_WIDTH-1:0][NW_WIDTH-1:0] committed_slot_wid; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_committed_wid - assign committed_slot_wid[i] = commit_arb_if[i].data.wid; - end - - logic [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask; - wire [`VX_CFG_NUM_WARPS-1:0] committed_warp_mask_r; - always_comb begin - committed_warp_mask = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin - if (committed_warps[i]) begin - committed_warp_mask[committed_slot_wid[i]] = 1'b1; - end + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + VX_commit_if slice_commit_if[NUM_EX_UNITS](); + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_commit_if + `ASSIGN_VX_IF(slice_commit_if[j], commit_if[j * `VX_CFG_ISSUE_WIDTH + i]); end - end - `BUFFER(committed_warp_mask_r, committed_warp_mask); - assign commit_sched_if.committed_warps = committed_warp_mask_r; - - // Writeback - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback - wire [XLENB_W-1:0] bytesel_size = commit_arb_if[i].data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; - wire [XLENB_W-1:0] bytesel_off = commit_arb_if[i].data.bytesel[0 +: XLENB_W]; - wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; - wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + wire [PER_ISSUE_WARPS-1:0] committed_wis; - wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : - (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : - (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : - (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : - (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : - (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : - (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); - wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; - - for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel - assign writeback_data[lane] = commit_arb_if[i].data.data[lane] << (8 * bytesel_off); - assign writeback_byteen[lane] = commit_arb_if[i].data.tmask[lane] ? base_byteen : '0; - end - - assign writeback_if[i].valid = commit_arb_if[i].valid; - assign writeback_if[i].data.uuid = commit_arb_if[i].data.uuid; - assign writeback_if[i].data.wis = wid_to_wis(commit_arb_if[i].data.wid); - assign writeback_if[i].data.eop_wis = commit_eop_wis[i]; - assign writeback_if[i].data.cta_id = commit_arb_if[i].data.cta_id; - assign writeback_if[i].data.sid = commit_arb_if[i].data.sid; - assign writeback_if[i].data.PC = commit_arb_if[i].data.PC; - assign writeback_if[i].data.tmask= commit_arb_if[i].data.tmask; - assign writeback_if[i].data.wb = commit_arb_if[i].data.wb; - assign writeback_if[i].data.wr_xregs = commit_arb_if[i].data.wr_xregs; - assign writeback_if[i].data.rd = commit_arb_if[i].data.rd; - assign writeback_if[i].data.byteen = writeback_byteen; - assign writeback_if[i].data.data = writeback_data; - assign writeback_if[i].data.sop = commit_arb_if[i].data.sop; - assign writeback_if[i].data.eop = commit_arb_if[i].data.eop; - assign commit_arb_if[i].ready = 1; - end + VX_commit_slice #( + .INSTANCE_ID (INSTANCE_ID) + ) commit_slice ( + .clk (clk), + .reset (reset), + .commit_if (slice_commit_if), + .writeback_if (writeback_if[i]), + .committed_wis (committed_wis) + ); -`ifdef DBG_TRACE_PIPELINE - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_trace - for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_j - always @(posedge clk) begin - if (commit_if[j * `VX_CFG_ISSUE_WIDTH + i].valid && commit_if[j * `VX_CFG_ISSUE_WIDTH + i].ready) begin - `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wid, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.cta_id, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sid, to_fullPC(commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.PC))) - VX_trace_pkg::trace_ex_type(1, j); - `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.tmask, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wb, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.wr_xregs, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.rd, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.sop, commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.eop)) - `TRACE_ARRAY1D(1, "0x%0h", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.data, `VX_CFG_SIMD_WIDTH) - `TRACE(1, (" (#%0d)\n", commit_if[j * `VX_CFG_ISSUE_WIDTH + i].data.uuid)) - end - end + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign commit_sched_if.committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; end end -`endif endmodule diff --git a/hw/rtl/core/VX_commit_slice.sv b/hw/rtl/core/VX_commit_slice.sv new file mode 100644 index 0000000000..b2a1133d24 --- /dev/null +++ b/hw/rtl/core/VX_commit_slice.sv @@ -0,0 +1,124 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Commit for one issue slice: arbitrates its execution units' results onto the +// slice's writeback port and reports the warps whose instruction retired. +module VX_commit_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + // inputs + VX_commit_if.slave commit_if [NUM_EX_UNITS], + + // outputs + VX_writeback_if.master writeback_if, + output wire [PER_ISSUE_WARPS-1:0] committed_wis // registered +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam OUT_DATAW = $bits(commit_t); + + VX_commit_if commit_arb_if(); + wire [PER_ISSUE_WARPS-1:0] commit_eop_wis; + + wire [NUM_EX_UNITS-1:0] valid_in; + wire [NUM_EX_UNITS-1:0][PER_ISSUE_WARPS+OUT_DATAW-1:0] data_in; + wire [NUM_EX_UNITS-1:0] ready_in; + + // Decoded ahead of the output register so each warp's release reads + // its own flop. + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_data_in + wire [PER_ISSUE_WARPS-1:0] eop_wis = PER_ISSUE_WARPS'(commit_if[j].data.eop) + << wid_to_wis(commit_if[j].data.wid); + assign valid_in[j] = commit_if[j].valid; + assign data_in[j] = {eop_wis, commit_if[j].data}; + assign commit_if[j].ready = ready_in[j]; + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_EX_UNITS), + .DATAW (PER_ISSUE_WARPS + OUT_DATAW), + .ARBITER ("P"), + .OUT_BUF (1) + ) commit_arb ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .ready_in (ready_in), + .data_in (data_in), + .data_out ({commit_eop_wis, commit_arb_if.data}), + .valid_out (commit_arb_if.valid), + .ready_out (commit_arb_if.ready), + `UNUSED_PIN (sel_out) + ); + + // notify scheduler + wire [PER_ISSUE_WARPS-1:0] committed_wis_n = commit_arb_if.valid ? commit_eop_wis : '0; + `BUFFER(committed_wis, committed_wis_n); + + // Writeback + + wire [XLENB_W-1:0] bytesel_size = commit_arb_if.data.bytesel[BYTESEL_BITS-1 -: XLENB_W]; + wire [XLENB_W-1:0] bytesel_off = commit_arb_if.data.bytesel[0 +: XLENB_W]; + wire [`VX_CFG_SIMD_WIDTH-1:0][`VX_CFG_XLEN-1:0] writeback_data; + wire [`VX_CFG_SIMD_WIDTH-1:0][XLENB-1:0] writeback_byteen; + + wire [XLENB-1:0] size_mask = (bytesel_size == XLENB_W'(7)) ? XLENB'(255) : + (bytesel_size == XLENB_W'(6)) ? XLENB'(127) : + (bytesel_size == XLENB_W'(5)) ? XLENB'(63) : + (bytesel_size == XLENB_W'(4)) ? XLENB'(31) : + (bytesel_size == XLENB_W'(3)) ? XLENB'(15) : + (bytesel_size == XLENB_W'(2)) ? XLENB'(7) : + (bytesel_size == XLENB_W'(1)) ? XLENB'(3) : XLENB'(1); + wire [XLENB-1:0] base_byteen = size_mask << bytesel_off; + + for (genvar lane = 0; lane < `VX_CFG_SIMD_WIDTH; ++lane) begin : g_bytesel + assign writeback_data[lane] = commit_arb_if.data.data[lane] << (8 * bytesel_off); + assign writeback_byteen[lane] = commit_arb_if.data.tmask[lane] ? base_byteen : '0; + end + + assign writeback_if.valid = commit_arb_if.valid; + assign writeback_if.data.uuid = commit_arb_if.data.uuid; + assign writeback_if.data.wis = wid_to_wis(commit_arb_if.data.wid); + assign writeback_if.data.eop_wis = commit_eop_wis; + assign writeback_if.data.cta_id = commit_arb_if.data.cta_id; + assign writeback_if.data.sid = commit_arb_if.data.sid; + assign writeback_if.data.PC = commit_arb_if.data.PC; + assign writeback_if.data.tmask= commit_arb_if.data.tmask; + assign writeback_if.data.wb = commit_arb_if.data.wb; + assign writeback_if.data.wr_xregs = commit_arb_if.data.wr_xregs; + assign writeback_if.data.rd = commit_arb_if.data.rd; + assign writeback_if.data.byteen = writeback_byteen; + assign writeback_if.data.data = writeback_data; + assign writeback_if.data.sop = commit_arb_if.data.sop; + assign writeback_if.data.eop = commit_arb_if.data.eop; + assign commit_arb_if.ready = 1; + +`ifdef DBG_TRACE_PIPELINE + for (genvar j = 0; j < NUM_EX_UNITS; ++j) begin : g_trace + always @(posedge clk) begin + if (commit_if[j].valid && commit_if[j].ready) begin + `TRACE(1, ("%t: %s commit: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, commit_if[j].data.wid, commit_if[j].data.cta_id, commit_if[j].data.sid, to_fullPC(commit_if[j].data.PC))) + VX_trace_pkg::trace_ex_type(1, j); + `TRACE(1, (", tmask=%b, wb=%b, wr_xregs=%b, rd=%0d, sop=%b, eop=%b, data=", commit_if[j].data.tmask, commit_if[j].data.wb, commit_if[j].data.wr_xregs, commit_if[j].data.rd, commit_if[j].data.sop, commit_if[j].data.eop)) + `TRACE_ARRAY1D(1, "0x%0h", commit_if[j].data.data, `VX_CFG_SIMD_WIDTH) + `TRACE(1, (" (#%0d)\n", commit_if[j].data.uuid)) + end + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_compute_slice.sv b/hw/rtl/core/VX_compute_slice.sv new file mode 100644 index 0000000000..4d8c0b6619 --- /dev/null +++ b/hw/rtl/core/VX_compute_slice.sv @@ -0,0 +1,332 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// A frontend slice and its backend slice: the warps with wid mod ISSUE_WIDTH == +// slice_id carried through fetch, decode, issue, execute and commit. Used when +// the fetch, issue and every execution-unit width are equal, so frontend slice i +// serves only issue slice i and the blocks of slice i, and the two meet +// directly. Only the core scheduler, the icache, the memory system and the +// shared SFU/TCU services are reached through the ports. +module VX_compute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, + output issue_perf_t issue_perf, +`endif + + // core scheduler and icache + VX_slice_sched_if.slave slice_sched_if, + VX_mem_bus_if.master icache_bus_if, + output wire [PER_ISSUE_WARPS-1:0] committed_wis, + + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] all_lsu_sched_empty, + input wire mmu_empty, + input cta_bcast_t cta_bcast, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_XLEN-1:0] satp, + output wire satp_wr_valid, + output wire [`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + + // LSU +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, + output wire lsu_busy, +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if, +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif + + // core-wide warp control + VX_warp_shared_if.master warp_shared_if +); + `STATIC_ASSERT (COMPUTE_SLICES, ("invalid parameter")) + + localparam SW = PER_FETCH_WARPS; + + `SCOPE_IO_SWITCH (2); + + VX_decode_if decode_if(); + VX_branch_ctl_if branch_ctl_if(); + VX_warp_ctl_if warp_ctl_if(); + VX_sched_csr_if sched_csr_if(); +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if(); +`endif + + wire warp_issued; + wire [ISSUE_WIS_W-1:0] warp_issued_wis; + + // slice-local view of the frontend slice's ports: block i serves only frontend slice + // i, and a warp's fetch-local and issue-local ids are the same + sched_branch_t [0:0] branch; + sched_wctl_t [0:0] wctl; + wire [0:0][NW_WIDTH-1:0] dvstack_wid; + wire [0:0][DV_STACK_SIZEW-1:0] dvstack_ptr; + wire [0:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch; + wire [0:0][NW_WIDTH-1:0] csr_rd_wid; + sched_csr_rd_t [0:0] csr_rd; + sched_csr_wr_t [0:0] csr_wr; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [0:0] rtu_unlock_valid; + wire [0:0][NW_WIDTH-1:0] rtu_unlock_wid; +`endif + wire [SW-1:0] issued; + wire [SW-1:0] pending_alm_empty; + + // The core-wide accounting retires an exiting warp when the shared unit grants + // its exit request. With several requesters, the slice sees that grant on its + // own request (registered toward the unit) and retires the warp in the same + // cycle, so the retirement does not cross the core's arbiter; with a sole + // requester, the core's retirement is already registered. + VX_slice_sched_if fs_sched_if(); + + assign fs_sched_if.cta_fire = slice_sched_if.cta_fire; + assign fs_sched_if.cta_lid = slice_sched_if.cta_lid; + assign fs_sched_if.cta_PC = slice_sched_if.cta_PC; + assign fs_sched_if.cta_tmask = slice_sched_if.cta_tmask; + assign fs_sched_if.cta_param = slice_sched_if.cta_param; + assign fs_sched_if.cta_init = slice_sched_if.cta_init; + assign fs_sched_if.warp_write = slice_sched_if.warp_write; + assign fs_sched_if.warp_waddr = slice_sched_if.warp_waddr; + assign fs_sched_if.warp_wdata = slice_sched_if.warp_wdata; + assign fs_sched_if.id_write = slice_sched_if.id_write; + assign fs_sched_if.id_waddr = slice_sched_if.id_waddr; + assign fs_sched_if.id_wdata = slice_sched_if.id_wdata; + assign fs_sched_if.wspawn_apply = slice_sched_if.wspawn_apply; + assign fs_sched_if.wspawn_wmask = slice_sched_if.wspawn_wmask; + assign fs_sched_if.wspawn_pc = slice_sched_if.wspawn_pc; + assign fs_sched_if.wspawn_src = slice_sched_if.wspawn_src; + assign fs_sched_if.wspawn_src_lid = slice_sched_if.wspawn_src_lid; + assign fs_sched_if.wspawn_mscratch = slice_sched_if.wspawn_mscratch; + assign fs_sched_if.bar_unlock_valid = slice_sched_if.bar_unlock_valid; + assign fs_sched_if.bar_unlock_mask = slice_sched_if.bar_unlock_mask; + assign fs_sched_if.slice_ibuf_full = slice_sched_if.slice_ibuf_full; +`ifdef EXT_SCHED_TIMEOUT_DUMP + assign fs_sched_if.timeout_dump = slice_sched_if.timeout_dump; +`endif + + if (SLICE_RETIRE) begin : g_retire + assign fs_sched_if.retire_valid = warp_shared_if.req_valid && warp_shared_if.req_ready + && (warp_shared_if.req_data.kind == WSH_EXIT); + assign fs_sched_if.retire_lid = wid_to_wfs(warp_shared_if.req_data.wid); + `UNUSED_VAR ({slice_sched_if.retire_valid, slice_sched_if.retire_lid}) + end else begin : g_retire_core + assign fs_sched_if.retire_valid = slice_sched_if.retire_valid; + assign fs_sched_if.retire_lid = slice_sched_if.retire_lid; + end + + assign slice_sched_if.active = fs_sched_if.active; + assign slice_sched_if.stalled = fs_sched_if.stalled; + assign slice_sched_if.ibuf_full = fs_sched_if.ibuf_full; + assign slice_sched_if.pending_empty = fs_sched_if.pending_empty; + assign slice_sched_if.branch_valid = fs_sched_if.branch_valid; + assign slice_sched_if.divergence = fs_sched_if.divergence; + assign slice_sched_if.sched_valid_room = fs_sched_if.sched_valid_room; + assign slice_sched_if.sched_valid_full = fs_sched_if.sched_valid_full; + assign slice_sched_if.sched_fire = fs_sched_if.sched_fire; + assign slice_sched_if.sched_tmask = fs_sched_if.sched_tmask; + + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((`VX_CFG_FETCH_WIDTH > 1) ? SLICE_STR : "") + ) frontend_slice ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(slice_id)), + `ifdef PERF_ENABLE + .fetch_perf (fetch_perf), + `endif + .slice_sched_if (fs_sched_if), + .icache_bus_if (icache_bus_if), + .decode_if (decode_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .issued (issued), + .committed (committed_wis), + .pending_alm_empty (pending_alm_empty) + ); + + for (genvar l = 0; l < SW; ++l) begin : g_issued + assign issued[l] = warp_issued && (warp_issued_wis == ISSUE_WIS_W'(l)); + end + + assign branch[0].valid = branch_ctl_if.valid; + assign branch[0].wid = branch_ctl_if.wid; + assign branch[0].taken = branch_ctl_if.taken; + assign branch[0].dest = branch_ctl_if.dest; + assign branch[0].is_trap = branch_ctl_if.is_trap; + assign branch[0].is_mret = branch_ctl_if.is_mret; + assign branch[0].trap_cause = branch_ctl_if.trap_cause; + + assign wctl[0].tmc_valid = warp_ctl_if.tmc_valid; + assign wctl[0].split_valid = warp_ctl_if.split_valid; + assign wctl[0].sjoin_valid = warp_ctl_if.sjoin_valid; + assign wctl[0].wsync_valid = warp_ctl_if.wsync_valid; + assign wctl[0].wid = warp_ctl_if.wid; + assign wctl[0].tmc = warp_ctl_if.tmc; + assign wctl[0].split = warp_ctl_if.split; + assign wctl[0].sjoin = warp_ctl_if.sjoin; + assign dvstack_wid[0] = warp_ctl_if.dvstack_wid; + assign warp_ctl_if.dvstack_ptr = dvstack_ptr[0]; + assign warp_ctl_if.mscratch = dvstack_mscratch[0]; + assign warp_ctl_if.warp_pending_alm_empty = pending_alm_empty; + assign warp_ctl_if.lsu_sched_empty = all_lsu_sched_empty; + assign warp_ctl_if.mmu_empty = mmu_empty; + + assign csr_rd_wid[0] = sched_csr_if.csr_rd_wid; + assign csr_wr[0].valid = sched_csr_if.csr_wr_valid; + assign csr_wr[0].wid = sched_csr_if.csr_wr_wid; + assign csr_wr[0].data = sched_csr_if.csr_wr_data; + assign csr_wr[0].trap_valid = sched_csr_if.trap_csr_wr_valid; + assign csr_wr[0].trap_data = sched_csr_if.trap_csr_wr_data; + + assign sched_csr_if.cycles = cycles; + assign sched_csr_if.instret = instret; + assign sched_csr_if.active_warps = active_warps; + assign sched_csr_if.thread_mask = csr_rd[0].thread_mask; + assign sched_csr_if.mscratch = csr_rd[0].mscratch; + assign sched_csr_if.csr_mstatus = csr_rd[0].mstatus; + assign sched_csr_if.csr_mtvec = csr_rd[0].mtvec; + assign sched_csr_if.csr_mepc = csr_rd[0].mepc; + assign sched_csr_if.csr_mcause = csr_rd[0].mcause; + assign sched_csr_if.csr_mtval = csr_rd[0].mtval; + assign sched_csr_if.cta_rank = csr_rd[0].cta_rank; + assign sched_csr_if.cta_lane = csr_rd[0].cta_lane; +`ifdef VX_CFG_VM_ENABLE + assign sched_csr_if.csr_satp = satp; + assign satp_wr_valid = sched_csr_if.satp_wr_valid; + assign satp_wr_data = sched_csr_if.satp_wr_data; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_unlock_valid[0] = sched_unlock_if.valid; + assign rtu_unlock_wid[0] = sched_unlock_if.wid; +`endif + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (SLICE_STR) + ) backend_slice ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + `ifdef PERF_ENABLE + .issue_perf (issue_perf), + `endif + .decode_if (decode_if), + .warp_issued (warp_issued), + .warp_issued_wis (warp_issued_wis), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), + `endif + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if), + `endif + .dcache_bus_if (dcache_bus_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + .warp_shared_if (warp_shared_if), + .cta_bcast (cta_bcast) + ); + +endmodule diff --git a/hw/rtl/core/VX_core.sv b/hw/rtl/core/VX_core.sv index 0de66a70f5..26891662aa 100644 --- a/hw/rtl/core/VX_core.sv +++ b/hw/rtl/core/VX_core.sv @@ -17,8 +17,11 @@ `include "VX_fpu_define.vh" `endif -module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( - parameter CORE_ID = 0, +module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL @@ -27,6 +30,8 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE input sysmem_perf_t sysmem_perf, `endif @@ -35,14 +40,14 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], - VX_mem_bus_if.master icache_bus_if, + VX_mem_bus_if.master icache_bus_if [ICACHE_NUM_REQS], `ifdef VX_CFG_VM_ENABLE - // Address translation is relocated to the socket (one MMU per L1 cache); the - // core emits virtual addresses on its cache buses. The socket returns its - // MMU drain state so the core's busy/barrier logic still waits for in-flight - // translations to retire before completion/suspend. + // Translation happens at the socket (one MMU per L1 cache): the core emits + // virtual addresses, and busy/barrier logic waits on the socket MMU draining. input wire mmu_drained, + // This core's own dMMU lanes hold none of its requests. + input wire dcache_mmu_drained, `endif `ifdef VX_CFG_EXT_DXA_ENABLE @@ -54,12 +59,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_tex_bus_if.master tex_bus_if, `endif -`ifdef VX_CFG_EXT_OM_ENABLE -`endif - -`ifdef VX_CFG_EXT_RASTER_ENABLE -`endif - `ifdef VX_CFG_EXT_RTU_ENABLE VX_rtu_bus_if.master rtu_bus_if, `endif @@ -74,68 +73,43 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // Global barrier VX_gbar_bus_if.master gbar_bus_if, - // Status + // Status: the core's own busy, and each LSU block's memory path busy + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, output wire busy ); - VX_schedule_if schedule_if(); - VX_fetch_if fetch_if(); - VX_decode_if decode_if(); - VX_sched_csr_if sched_csr_if(); - VX_decode_sched_if decode_sched_if(); - VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); - VX_commit_sched_if commit_sched_if(); - VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); - VX_warp_ctl_if warp_ctl_if(); -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if sched_unlock_if(); // RTU TRACE wstall release -> scheduler -`endif - - VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); - VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + VX_warp_shared_if warp_shared_if[`VX_CFG_NUM_SFU_BLOCKS](); `ifdef VX_CFG_EXT_DXA_ENABLE VX_txbar_bus_if dxa_txbar_bus_if(); `endif - // cta_table_if removed: cluster-contiguous LMEM placement lets the - // DXA multicast writer compute receiver addresses as - // `issuer_base + r × smem_stride`, eliminating the per-slot lookup table. - + // the LSU blocks' memory paths + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty; +`ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // VX_lsu_scheduler instantiated per-LSU-block; all blocks have NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // LSU client interfaces flow from execute as client 0. - VX_lsu_sched_if lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS](); - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; - -`ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); `endif - VX_mem_bus_if #( .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_TAG_WIDTH_BASE) - ) mmu_dcache_if[DCACHE_NUM_REQS](); - - // VX_fetch -> VX_dcr_flush -> mmu_icache_if -> (i)MMU -> icache_bus_if. - // The flush wrapper consumes the dcr-triggered cache-flush request and - // injects a synthetic flush request into the icache stream, mirroring - // the dcache path in VX_mem_unit. - VX_mem_bus_if #( - .DATA_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) - ) fetch_icache_if[1](); + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); +`ifdef PERF_ENABLE + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire; + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses; +`endif + // frontend-slice icache ports, tagged with the flush arbiter's select bit (the + // LSB); port 0 also carries the injected flush VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_TAG_WIDTH_BASE) - ) mmu_icache_if[1](); + .TAG_WIDTH (ICACHE_TAG_WIDTH) + ) fetch_icache_if[ICACHE_NUM_REQS](); `ifdef VX_CFG_TCU_WGMMA_ENABLE localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); @@ -147,7 +121,6 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( ) tcu_lmem_if(); `endif - `ifdef PERF_ENABLE lmem_perf_t lmem_perf; coalescer_perf_t coalescer_perf; @@ -166,37 +139,56 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_dcr_csr_if dcr_csr_if(); - // Single DCR-flush trigger from VX_dcr_data, fanned out below to BOTH - // the dcache (via VX_mem_unit) and the icache (via the wrapper inserted - // between VX_fetch and the iMMU). Without invalidating the icache too, - // a kernel re-loaded to the same VMA after a CACHE_FLUSH would execute - // stale lines from the previous launch. + // One DCR flush request invalidates both L1s: a kernel re-loaded at the same + // address must not execute stale icache lines. VX_dcr_flush_if dcr_flush_if(); VX_dcr_flush_if dcr_flush_dcache_if(); VX_dcr_flush_if dcr_flush_icache_if(); + wire dcr_flush_l1_only; - // High when the in-core global-store path is empty (no store still in - // flight toward the dcache). Gates `busy` so the flush waits for the tail. - wire mem_unit_empty; + // the in-core global-store path is empty + wire mem_unit_empty = (& lsu_store_empty); + wire mem_unit_busy; - // Hold the dcache flush request until this core's stores have all reached L1. - // Stores are ack-less and lag warp-exit by many cycles, and the flush arbiter - // sits at the adapter output with no upstream visibility, so without this gate a - // flush could inject ahead of a store still in the LSU/coalescer and lose it. +`ifdef VX_CFG_VM_ENABLE +`ifdef PERF_ENABLE + // the MMU perf counters are kept at the socket + assign pipeline_perf.mmu = '0; +`endif + // Translation drains at the socket MMU; the socket returns its empty status. + wire mmu_empty = mmu_drained; +`else + wire mmu_empty = 1'b1; +`endif + + + // Stores are ack-less and lag warp exit, so the dcache flush waits until every + // store of this core has reached L1. wire store_drained = (&lsu_sched_empty) && mem_unit_empty; +`ifdef VX_CFG_VM_ENABLE + // Under VM a store may still be parked on a TLB miss in this core's dMMU + // lanes, which a flush token on another lane would overtake: the flush also + // waits for those lanes. The request is level-held and the token's own trip + // through the MMU clears the lanes' idle, so the start is latched until the + // flush request drops. + reg dcache_flush_go_r; + always @(posedge clk) begin + if (reset || ~dcr_flush_if.req) begin + dcache_flush_go_r <= 1'b0; + end else if (store_drained && dcache_mmu_drained) begin + dcache_flush_go_r <= 1'b1; + end + end + assign dcr_flush_dcache_if.req = dcr_flush_if.req + && ((store_drained && dcache_mmu_drained) || dcache_flush_go_r); +`else assign dcr_flush_dcache_if.req = dcr_flush_if.req && store_drained; - // Both L1s forward their flush to the shared next level, and a cache that - // is flushing locks out incoming core requests for its whole sweep. The - // icache carries no dirty data and so retires almost immediately; gate it - // behind the dcache to keep that forward from arriving while the dcache is - // still evicting, which would strand the evictions upstream of memory. - assign dcr_flush_icache_if.req = dcr_flush_if.req && dcr_flush_dcache_if.done; - // Each VX_dcr_flush holds .done level-high until its req drops, so a - // straight AND of the two dones reports the combined completion to - // VX_dcr_data — which then drops req, re-arming both for the next flush. - // When gfx extensions are enabled, also gate on the cluster-shared - // gfx-cache flush done so that VX_dcr_data doesn't retire the flush DCR - // before tcache/rcache/ocache have actually invalidated. +`endif + // Only the dcache flush walks the shared levels (the icache does not + // forward its flush), so the icache flushes alongside it. + assign dcr_flush_icache_if.req = dcr_flush_if.req; + // Each flush holds done until its req drops, so the dones AND into the combined + // completion (with the cluster graphics caches when present). `ifdef EXT_GFX_ANY_ENABLE assign cluster_flush_if.req = dcr_flush_if.req; assign dcr_flush_if.done = dcr_flush_dcache_if.done & dcr_flush_icache_if.done & cluster_flush_if.done; @@ -205,285 +197,666 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `endif VX_dcr_data #( - .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dcr_data", INSTANCE_ID))) ) dcr_data ( .clk (clk), .reset (reset), + .core_id (core_id), .dcr_bus_if (dcr_bus_if), .dcr_csr_if (dcr_csr_if), - .dcr_flush_if(dcr_flush_if) + .dcr_flush_if(dcr_flush_if), + .flush_l1_only(dcr_flush_l1_only) ); - `SCOPE_IO_SWITCH (3); - - wire sched_busy; - VX_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) scheduler ( - .clk (clk), - .reset (reset), + wire [PERF_CTR_BITS-1:0] sched_cycles; + wire [PERF_CTR_BITS-1:0] sched_instret; + VX_csr_mpm #( + .INSTANCE_ID (`SFORMATF(("%s-mpm", INSTANCE_ID))) + ) csr_mpm ( `ifdef PERF_ENABLE - .sched_perf (pipeline_perf.sched), + .sysmem_perf (sysmem_perf_tmp), + .pipeline_perf (pipeline_perf), `endif - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if), - `ifdef VX_CFG_EXT_RTU_ENABLE - .sched_unlock_if (sched_unlock_if), - `endif - - .decode_sched_if(decode_sched_if), - .issue_sched_if (issue_sched_if), - .commit_sched_if(commit_sched_if), - - .kmu_bus_if (kmu_bus_if), - - .schedule_if (schedule_if), - .sched_csr_if (sched_csr_if), - .gbar_bus_if (gbar_bus_if), - - .busy (sched_busy) + .cycles (sched_cycles), + .instret (sched_instret), + .dcr_csr_if (dcr_csr_if) ); - VX_fetch #( - .INSTANCE_ID (`SFORMATF(("%s-fetch", INSTANCE_ID))) - ) fetch ( - `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .fetch_perf (pipeline_perf.fetch), - `endif - .icache_bus_if (fetch_icache_if[0]), - .schedule_if (schedule_if), - .fetch_if (fetch_if) - ); + `SCOPE_IO_SWITCH (3); + + // Port 0 injects the cache flush into its request stream; the cache flushes + // every bank. The merged request leaves the core registered. The response + // goes to the frontend slice as the cache returns it, and the slice drops the + // flush's own (select bit set), so no logic lies between the cache's response + // register and the slice. The injector observes the response for its done. + VX_mem_bus_if #( + .DATA_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) + ) flush_core_if(); + + assign flush_core_if.req_valid = fetch_icache_if[0].req_valid; + assign flush_core_if.req_data.rw = fetch_icache_if[0].req_data.rw; + assign flush_core_if.req_data.addr = fetch_icache_if[0].req_data.addr; + assign flush_core_if.req_data.data = fetch_icache_if[0].req_data.data; + assign flush_core_if.req_data.byteen = fetch_icache_if[0].req_data.byteen; + assign flush_core_if.req_data.attr = fetch_icache_if[0].req_data.attr; + assign flush_core_if.req_data.tag = fetch_icache_if[0].req_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign fetch_icache_if[0].req_ready = flush_core_if.req_ready; + `UNUSED_VAR (fetch_icache_if[0].req_data.tag[0]) + + assign fetch_icache_if[0].rsp_valid = icache_bus_if[0].rsp_valid; + assign fetch_icache_if[0].rsp_data = icache_bus_if[0].rsp_data; + assign flush_core_if.rsp_ready = fetch_icache_if[0].rsp_ready; + `UNUSED_VAR (flush_core_if.rsp_valid) + `UNUSED_VAR (flush_core_if.rsp_data) - // Inject CACHE_FLUSH onto the icache stream so a host-side CMD_CACHE_FLUSH - // (issued after every kernel launch) invalidates instruction-cache lines - // belonging to the previous kernel image. VX_dcr_flush #( - .WORD_SIZE (ICACHE_WORD_SIZE), - .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core icache-request boundary; rsp already registered by L1 CORE_OUT_BUF + .WORD_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH), + .REQ_OUT_BUF (3) ) icache_dcr_flush ( .clk (clk), .reset (reset), .dcr_flush_if (dcr_flush_icache_if), - .core_bus_if (fetch_icache_if[0]), - .cache_bus_if (mmu_icache_if[0]) + .l1_only (dcr_flush_l1_only), + .core_bus_if (flush_core_if), + .cache_bus_if (icache_bus_if[0]) ); - VX_decode #( - .INSTANCE_ID (`SFORMATF(("%s-decode", INSTANCE_ID))) - ) decode ( - .clk (clk), - .reset (reset), - .fetch_if (fetch_if), - .decode_if (decode_if), - .decode_sched_if(decode_sched_if) - ); + // The other ports carry their frontend slice's registered requests, whose + // select bit is clear; a response's is cleared as a constant, so the slice's + // flush check folds away. + for (genvar i = 1; i < ICACHE_NUM_REQS; ++i) begin : g_icache_ports + assign icache_bus_if[i].req_valid = fetch_icache_if[i].req_valid; + assign icache_bus_if[i].req_data = fetch_icache_if[i].req_data; + assign fetch_icache_if[i].req_ready = icache_bus_if[i].req_ready; + + assign fetch_icache_if[i].rsp_valid = icache_bus_if[i].rsp_valid; + assign fetch_icache_if[i].rsp_data.data = icache_bus_if[i].rsp_data.data; + assign fetch_icache_if[i].rsp_data.tag = {icache_bus_if[i].rsp_data.tag[ICACHE_TAG_WIDTH-1:1], 1'b0}; + assign icache_bus_if[i].rsp_ready = fetch_icache_if[i].rsp_ready; + `UNUSED_VAR (icache_bus_if[i].rsp_data.tag[0]) + end - VX_issue #( - .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) - ) issue ( - `SCOPE_IO_BIND (1) + // scheduling: the core scheduler and its per-frontend-slice interfaces - .clk (clk), - .reset (reset), + localparam FW = `VX_CFG_FETCH_WIDTH; - `ifdef PERF_ENABLE - .issue_perf (pipeline_perf.issue), - `endif + VX_slice_sched_if slice_sched_if[FW](); - .decode_if (decode_if), - .writeback_if (writeback_if), - .dispatch_if (dispatch_if), - .issue_sched_if (issue_sched_if) - ); + wire [`VX_CFG_NUM_WARPS-1:0] committed_warps; + wire [`VX_CFG_NUM_WARPS-1:0] sched_active_warps; +`ifdef VX_CFG_VM_ENABLE + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] sched_satp; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid; + wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data; +`endif - VX_execute #( - .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) execute ( - `SCOPE_IO_BIND (2) + cta_bcast_t cta_bcast; + VX_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler", INSTANCE_ID))) + ) scheduler ( .clk (clk), .reset (reset), - + .core_id (core_id), `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf_tmp), - .pipeline_perf (pipeline_perf), - `ifdef VX_CFG_EXT_TCU_ENABLE - .tcu_perf (tcu_perf), + .sched_perf (pipeline_perf.sched), `endif + .slice_if (slice_sched_if), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (dxa_txbar_bus_if), `endif - - // execute exposes LSU client interfaces; lsu_scheduler sits between - // execute and lsu_mem_if (which connects to mem_unit). - .lsu_client_if (lsu_client_if), - - `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .committed_warps(committed_warps), + .cta_bcast (cta_bcast), + .kmu_bus_if (kmu_bus_if), + .gbar_bus_if (gbar_bus_if), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), `endif + .busy (sched_busy) + ); - .dispatch_if (dispatch_if), - .commit_if (commit_if), +`ifdef PERF_ENABLE + fetch_perf_t per_slice_fetch_perf [FW]; + `PERF_COUNTER_ADD (pipeline_perf.fetch, per_slice_fetch_perf, stalls, PERF_CTR_BITS, FW, (FW > 2)) +`endif - .sched_csr_if (sched_csr_if), + // core-wide services, reached through one port per SFU or TCU block - .dcr_csr_if (dcr_csr_if), +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if block_dxa_req_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH) + ) block_tex_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH) + ) block_rtu_bus_if[`VX_CFG_NUM_SFU_BLOCKS](); +`endif +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), - `endif + VX_sfu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-sfu-shared", INSTANCE_ID))) + ) sfu_shared ( `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), + .block_dxa_req_bus_if (block_dxa_req_bus_if), + .dxa_req_bus_if (dxa_req_bus_if), `endif `ifdef VX_CFG_EXT_TEX_ENABLE - .tex_bus_if (tex_bus_if), - `endif - `ifdef VX_CFG_EXT_OM_ENABLE + .block_tex_bus_if (block_tex_bus_if), + .tex_bus_if (tex_bus_if), `endif `ifdef VX_CFG_EXT_RTU_ENABLE - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if), + .block_rtu_bus_if (block_rtu_bus_if), + .rtu_bus_if (rtu_bus_if), `endif - - .warp_ctl_if (warp_ctl_if), - .branch_ctl_if (branch_ctl_if) + .clk (clk), + .reset (reset) ); - VX_commit #( - .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) - ) commit ( - .clk (clk), - .reset (reset), +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu-shared", INSTANCE_ID))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), + `endif + .clk (clk), + .reset (reset) + ); +`endif - .commit_if (commit_if), + // fetch, issue, execute and commit, structured by COMPUTE_SLICES / BACKEND_SLICES - .writeback_if (writeback_if), + if (COMPUTE_SLICES) begin : g_compute_slices - .commit_sched_if(commit_sched_if) - ); + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end + `endif - // Per-block VX_lsu_scheduler instances: all parameterized NUM_CLIENTS=2. - // Block 0 wires client 1 to the warp-level TCU AGU; other blocks tie it off. - // Symmetric NUM_CLIENTS keeps module generation uniform — tied-off clients - // cost only a few muxes inside the round-robin arbiter. -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - for (genvar block_idx = 0; block_idx < `VX_CFG_NUM_LSU_BLOCKS; ++block_idx) begin : g_lsu_scheduler - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); - - // Client 0 — LSU (per-block). Forward the per-block lsu_client_if - // master onto the scheduler's slave port via per-signal assigns - // (interface arrays can't be wired with a single assign). - assign sched_client_if[0].req_valid = lsu_client_if[block_idx].req_valid; - assign sched_client_if[0].req_data = lsu_client_if[block_idx].req_data; - assign lsu_client_if[block_idx].req_ready = sched_client_if[0].req_ready; - assign lsu_client_if[block_idx].rsp_valid = sched_client_if[0].rsp_valid; - assign lsu_client_if[block_idx].rsp_data = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_client_if[block_idx].rsp_ready; - - `ifdef TCU_META_ENABLE - // Client 1 — TCU AGU on block 0; tied off on other blocks. - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (0) + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_compute_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) compute_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[i]), + .issue_perf (per_slice_issue_perf[i]), + `endif + .slice_sched_if (slice_sched_if[i]), + .icache_bus_if (fetch_icache_if[i]), + .committed_wis (committed_wis), + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .all_lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), + .cta_bcast (cta_bcast), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp[i]), + .satp_wr_valid (satp_wr_valid[i]), + .satp_wr_data (satp_wr_data[i]), + `endif + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if[i]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + .warp_shared_if (warp_shared_if[i]) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end end + + end else begin : g_crossbar + + VX_decode_if fs_decode_if[FW](); + VX_decode_if is_decode_if[`VX_CFG_ISSUE_WIDTH](); + VX_branch_ctl_if branch_ctl_if[`VX_CFG_NUM_ALU_BLOCKS](); + VX_warp_ctl_if warp_ctl_if[`VX_CFG_NUM_SFU_BLOCKS](); + VX_sched_csr_if sched_csr_if[`VX_CFG_NUM_SFU_BLOCKS](); + `ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if sched_unlock_if[`VX_CFG_NUM_SFU_BLOCKS](); + `endif + wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid; + wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis; + + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + sched_branch_t [FW-1:0][NPA-1:0] fs_branch; + sched_wctl_t [FW-1:0][NPS-1:0] fs_wctl; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_dvstack_wid; + wire [FW-1:0][NPS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr; + wire [FW-1:0][NPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid; + sched_csr_rd_t [FW-1:0][NPS-1:0] fs_csr_rd; + sched_csr_wr_t [FW-1:0][NPS-1:0] fs_csr_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire [FW-1:0][NPS-1:0] fs_rtu_unlock_valid; + wire [FW-1:0][NPS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid; `endif + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_issued; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_committed; + wire [FW-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty; + + VX_frontend_xbar frontend_xbar ( + .fs_decode_if (fs_decode_if), + .is_decode_if (is_decode_if), + .issue_valid (issue_valid), + .issue_wis (issue_wis), + .committed_warps (committed_warps), + .fs_issued (fs_issued), + .fs_committed (fs_committed), + .fs_pending_alm_empty (fs_pending_alm_empty), + .branch_ctl_if (branch_ctl_if), + .warp_ctl_if (warp_ctl_if), + .sched_csr_if (sched_csr_if), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if), + `endif + .cycles (sched_cycles), + .instret (sched_instret), + .active_warps (sched_active_warps), + .lsu_sched_empty (lsu_sched_empty), + .mmu_empty (mmu_empty), + `ifdef VX_CFG_VM_ENABLE + .satp (sched_satp), + .satp_wr_valid (satp_wr_valid), + .satp_wr_data (satp_wr_data), + `endif + .fs_branch (fs_branch), + .fs_wctl (fs_wctl), + .fs_dvstack_wid (fs_dvstack_wid), + .fs_dvstack_ptr (fs_dvstack_ptr), + .fs_dvstack_mscratch (fs_dvstack_mscratch), + .fs_csr_rd_wid (fs_csr_rd_wid), + .fs_csr_rd (fs_csr_rd), + `ifdef VX_CFG_EXT_RTU_ENABLE + .fs_rtu_unlock_valid (fs_rtu_unlock_valid), + .fs_rtu_unlock_wid (fs_rtu_unlock_wid), + `endif + .fs_csr_wr (fs_csr_wr) + ); - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d", INSTANCE_ID, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (lsu_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) + `ifdef SCOPE + wire [FW-1:0] frontend_scope_bus_in, frontend_scope_bus_out; + VX_scope_switch #( + .N (FW) + ) frontend_scope_switch ( + .clk (clk), + .reset (scope_reset_w[0]), + .req_in (scope_bus_in_w[0]), + .rsp_out (scope_bus_out_w[0]), + .req_out (frontend_scope_bus_in), + .rsp_in (frontend_scope_bus_out) ); + `endif + + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices + VX_frontend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR ((FW > 1) ? `SFORMATF(("%0d", f)) : "") + ) frontend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[0]), + .scope_bus_in (frontend_scope_bus_in[f]), + .scope_bus_out (frontend_scope_bus_out[f]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (FETCH_FSW_W'(f)), + `ifdef PERF_ENABLE + .fetch_perf (per_slice_fetch_perf[f]), + `endif + .slice_sched_if (slice_sched_if[f]), + .icache_bus_if (fetch_icache_if[f]), + .decode_if (fs_decode_if[f]), + .branch (fs_branch[f]), + .wctl (fs_wctl[f]), + .dvstack_wid (fs_dvstack_wid[f]), + .dvstack_ptr (fs_dvstack_ptr[f]), + .dvstack_mscratch (fs_dvstack_mscratch[f]), + .csr_rd_wid (fs_csr_rd_wid[f]), + .csr_rd (fs_csr_rd[f]), + .csr_wr (fs_csr_wr[f]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (fs_rtu_unlock_valid[f]), + .rtu_unlock_wid (fs_rtu_unlock_wid[f]), + `endif + .issued (fs_issued[f]), + .committed (fs_committed[f]), + .pending_alm_empty (fs_pending_alm_empty[f]) + ); + end + + if (BACKEND_SLICES) begin : g_backend_slices + + `ifdef PERF_ENABLE + issue_perf_t per_slice_issue_perf [`VX_CFG_ISSUE_WIDTH]; + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, ibf_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, scb_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, opd_stalls, PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_issue_perf_units_uses + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_stalls[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + `PERF_COUNTER_ADD (pipeline_perf.issue, per_slice_issue_perf, dispatch_instrs[i], PERF_CTR_BITS, `VX_CFG_ISSUE_WIDTH, (`VX_CFG_ISSUE_WIDTH > 2)) + end + `endif + + `ifdef SCOPE + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_scope_bus_in, slice_scope_bus_out; + VX_scope_switch #( + .N (`VX_CFG_ISSUE_WIDTH) + ) slice_scope_switch ( + .clk (clk), + .reset (scope_reset_w[1]), + .req_in (scope_bus_in_w[1]), + .rsp_out (scope_bus_out_w[1]), + .req_out (slice_scope_bus_in), + .rsp_in (slice_scope_bus_out) + ); + `endif + `SCOPE_IO_UNUSED (2) + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_slices + wire [PER_ISSUE_WARPS-1:0] committed_wis; + + VX_backend_slice #( + .INSTANCE_ID (INSTANCE_ID), + .SLICE_STR (`SFORMATF(("%0d", i))) + ) backend_slice ( + `ifdef SCOPE + .scope_reset (scope_reset_w[1]), + .scope_bus_in (slice_scope_bus_in[i]), + .scope_bus_out (slice_scope_bus_out[i]), + `endif + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (ISSUE_ISW_W'(i)), + `ifdef PERF_ENABLE + .issue_perf (per_slice_issue_perf[i]), + `endif + .decode_if (is_decode_if[i]), + .warp_issued (issue_valid[i]), + .warp_issued_wis (issue_wis[i]), + .committed_wis (committed_wis), + .branch_ctl_if (branch_ctl_if[i]), + .warp_ctl_if (warp_ctl_if[i]), + .sched_csr_if (sched_csr_if[i]), + `ifdef VX_CFG_EXT_RTU_ENABLE + .sched_unlock_if (sched_unlock_if[i]), + `endif + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire[i]), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire[i]), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire[i]), + .lsu_coalescer_misses (lsu_coalescer_misses[i]), + `endif + .lsu_sched_empty (lsu_sched_empty[i]), + .lsu_store_empty (lsu_store_empty[i]), + .lsu_busy (lsu_busy[i]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if[i]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]), + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if[i]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if[i]), + `endif + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid[i]), + .tcu_exec_ready (tcu_exec_ready[i]), + .tcu_exec_data (tcu_exec_data[i]), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data[i]), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data[i]), + .tcu_tbuf_ready (tcu_tbuf_ready[i]), + `endif + .warp_shared_if (warp_shared_if[i]), + .cta_bcast (cta_bcast) + ); + + // wid = {wis, isw}: slice i owns every ISSUE_WIDTH-th warp starting at i + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_committed + assign committed_warps[w * `VX_CFG_ISSUE_WIDTH + i] = committed_wis[w]; + end + end + + end else begin : g_stage_sliced + + VX_issue_sched_if issue_sched_if[`VX_CFG_ISSUE_WIDTH](); + VX_commit_sched_if commit_sched_if(); + + VX_dispatch_if dispatch_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_commit_if commit_if[NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH](); + VX_writeback_if writeback_if[`VX_CFG_ISSUE_WIDTH](); + + VX_issue #( + .INSTANCE_ID (`SFORMATF(("%s-issue", INSTANCE_ID))) + ) issue ( + `SCOPE_IO_BIND (1) + + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .issue_perf (pipeline_perf.issue), + `endif + + .decode_if (is_decode_if), + .writeback_if (writeback_if), + .dispatch_if (dispatch_if), + .issue_sched_if (issue_sched_if) + ); + + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_issue_sched + assign issue_valid[i] = issue_sched_if[i].valid; + assign issue_wis[i] = issue_sched_if[i].wis; + end + + VX_execute #( + .INSTANCE_ID (`SFORMATF(("%s-execute", INSTANCE_ID))) + ) execute ( + `SCOPE_IO_BIND (2) + + .clk (clk), + .reset (reset), + .core_id (core_id), + + `ifdef PERF_ENABLE + .lsu_perf_rd_req_fire (lsu_perf_rd_req_fire), + .lsu_perf_wr_req_fire (lsu_perf_wr_req_fire), + .lsu_perf_rsp_fire (lsu_perf_rsp_fire), + .lsu_coalescer_misses (lsu_coalescer_misses), + `endif + .lsu_sched_empty (lsu_sched_empty), + .lsu_store_empty (lsu_store_empty), + .lsu_busy (lsu_busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if), + `endif + .dcache_bus_if (lsu_dcache_bus_if), + + .dispatch_if (dispatch_if), + .commit_if (commit_if), + + .sched_csr_if (sched_csr_if), + + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tcu_exec_valid (tcu_exec_valid), + .tcu_exec_ready (tcu_exec_ready), + .tcu_exec_data (tcu_exec_data), + .tcu_tbuf_rs1_data (tcu_tbuf_rs1_data), + .tcu_tbuf_rs2_data (tcu_tbuf_rs2_data), + .tcu_tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (block_dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (block_tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (block_rtu_bus_if), + .sched_unlock_if (sched_unlock_if), + `endif + + .warp_shared_if (warp_shared_if), + + .warp_ctl_if (warp_ctl_if), + .branch_ctl_if (branch_ctl_if), + .cta_bcast (cta_bcast) + ); + + VX_commit #( + .INSTANCE_ID (`SFORMATF(("%s-commit", INSTANCE_ID))) + ) commit ( + .clk (clk), + .reset (reset), + + .commit_if (commit_if), + + .writeback_if (writeback_if), + + .commit_sched_if(commit_sched_if) + ); + + assign committed_warps = commit_sched_if.committed_warps; + + end end VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef PERF_ENABLE - .lmem_perf (lmem_perf), - .coalescer_perf(coalescer_perf), + .lmem_perf (lmem_perf), `endif `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .tcu_lmem_if (tcu_lmem_if), `endif `ifdef VX_CFG_EXT_DXA_ENABLE - .dxa_lmem_bus_if(dxa_lmem_bus_if), + .dxa_lmem_bus_if (dxa_lmem_bus_if), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), - .dcr_flush_if (dcr_flush_dcache_if), - .dcache_bus_if (mmu_dcache_if) + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), + .dcr_flush_if (dcr_flush_dcache_if), + .flush_l1_only (dcr_flush_l1_only), + .dcache_bus_if (dcache_bus_if), + .busy (mem_unit_busy) ); - // Address translation is relocated to the socket (one MMU per L1 cache). - // The core's dcache/icache buses carry the addresses the mem_unit and fetch - // produce straight through — virtual under VM, physical otherwise. Same - // widths on both sides (BASE == full for both L1s). - for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru - `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[i], mmu_dcache_if[i]); - end - `ASSIGN_VX_MEM_BUS_IF (icache_bus_if, mmu_icache_if[0]); - -`ifdef VX_CFG_VM_ENABLE -`ifdef PERF_ENABLE - // The relocated MMU's perf counters live at the socket and are not yet routed - // through sysmem_perf, so the in-core pipeline view reads zero. - assign pipeline_perf.mmu = '0; -`endif - // Translation drains at the socket MMU; the socket returns its empty status. - wire mmu_empty = mmu_drained; - // The socket MMU takes its satp from the shared DCR root, so the per-core CSR - // satp is not a translation source. - `UNUSED_VAR (sched_csr_if.csr_satp) -`else - wire mmu_empty = 1'b1; -`endif - // Fragment work drains at the producer (VX_raster_core.busy), not here. // DCR reads (cache flush, MPM readback) order on their own DCR response, so // busy tracks only kernel execution: warps resident or stores not yet at L1. - // ~mmu_empty also keeps the core busy while a translation or parked store is - // still in flight, so kernel completion cannot race un-drained accesses. - assign busy = sched_busy || ~(&lsu_sched_empty) || ~mem_unit_empty || ~mmu_empty; - - // BAR (vx_barrier / vx_barrier_arrive) drains LSU + MMU before suspending or registering arrival. - assign warp_ctl_if.lsu_sched_drained = (&lsu_sched_empty) && mmu_empty; + // A store still in an LSU block's memory path shows on lsu_busy, which the + // parent ORs in; one past a flush injector's buffer shows on mem_unit_busy. + // busy depends on no input: the socket covers the launches it sends and its + // MMU's translations. + assign busy = sched_busy || mem_unit_busy; `ifdef PERF_ENABLE @@ -491,7 +864,9 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_wr_req_per_cycle; wire [`CLOG2(LSU_NUM_REQS+1)-1:0] perf_dcache_rsp_per_cycle; - wire [1:0] perf_icache_pending_read_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_req_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)-1:0] perf_icache_rsp_per_cycle; + wire [`CLOG2(ICACHE_NUM_REQS+1)+1-1:0] perf_icache_pending_read_cycle; wire [`CLOG2(LSU_NUM_REQS+1)+1-1:0] perf_dcache_pending_read_cycle; reg [PERF_CTR_BITS-1:0] perf_icache_pending_reads; @@ -501,20 +876,33 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_loads; reg [PERF_CTR_BITS-1:0] perf_stores; - wire perf_icache_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; - wire perf_icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; + wire [ICACHE_NUM_REQS-1:0] perf_icache_req_fire; + wire [ICACHE_NUM_REQS-1:0] perf_icache_rsp_fire; + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_perf_icache + assign perf_icache_req_fire[i] = icache_bus_if[i].req_valid && icache_bus_if[i].req_ready; + assign perf_icache_rsp_fire[i] = icache_bus_if[i].rsp_valid && icache_bus_if[i].rsp_ready; + end + `POP_COUNT(perf_icache_req_per_cycle, perf_icache_req_fire); + `POP_COUNT(perf_icache_rsp_per_cycle, perf_icache_rsp_fire); wire [LSU_NUM_REQS-1:0] perf_dcache_rd_req_fire, perf_dcache_rd_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_wr_req_fire, perf_dcache_wr_req_fire_r; wire [LSU_NUM_REQS-1:0] perf_dcache_rsp_fire; - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_perf_dcache - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_j - assign perf_dcache_rd_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && ~lsu_mem_if[i].req_data.rw; - assign perf_dcache_wr_req_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].req_valid && lsu_mem_if[i].req_data.mask[j] && lsu_mem_if[i].req_ready && lsu_mem_if[i].req_data.rw; - assign perf_dcache_rsp_fire[i * `VX_CFG_NUM_LSU_LANES + j] = lsu_mem_if[i].rsp_valid && lsu_mem_if[i].rsp_data.mask[j] && lsu_mem_if[i].rsp_ready; - end - end + assign perf_dcache_rd_req_fire = lsu_perf_rd_req_fire; + assign perf_dcache_wr_req_fire = lsu_perf_wr_req_fire; + assign perf_dcache_rsp_fire = lsu_perf_rsp_fire; + + wire [PERF_CTR_BITS-1:0] coalescer_misses; + VX_reduce_tree #( + .IN_W (PERF_CTR_BITS), + .N (`VX_CFG_NUM_LSU_BLOCKS), + .OP ("+") + ) coalescer_reduce ( + .data_in (lsu_coalescer_misses), + .data_out (coalescer_misses) + ); + `BUFFER(coalescer_perf.misses, coalescer_misses); `BUFFER(perf_dcache_rd_req_fire_r, perf_dcache_rd_req_fire); `BUFFER(perf_dcache_wr_req_fire_r, perf_dcache_wr_req_fire); @@ -523,7 +911,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `POP_COUNT(perf_dcache_wr_req_per_cycle, perf_dcache_wr_req_fire_r); `POP_COUNT(perf_dcache_rsp_per_cycle, perf_dcache_rsp_fire); - assign perf_icache_pending_read_cycle = perf_icache_req_fire - perf_icache_rsp_fire; + assign perf_icache_pending_read_cycle = perf_icache_req_per_cycle - perf_icache_rsp_per_cycle; assign perf_dcache_pending_read_cycle = perf_dcache_rd_req_per_cycle - perf_dcache_rsp_per_cycle; always @(posedge clk) begin @@ -547,7 +935,7 @@ module VX_core import VX_gpu_pkg::*, VX_tlb_pkg::*; #( perf_icache_lat <= '0; perf_dcache_lat <= '0; end else begin - perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_fire); + perf_ifetches <= perf_ifetches + PERF_CTR_BITS'(perf_icache_req_per_cycle); perf_loads <= perf_loads + PERF_CTR_BITS'(perf_dcache_rd_req_per_cycle); perf_stores <= perf_stores + PERF_CTR_BITS'(perf_dcache_wr_req_per_cycle); perf_icache_lat <= perf_icache_lat + perf_icache_pending_reads; diff --git a/hw/rtl/core/VX_csr_cta.sv b/hw/rtl/core/VX_csr_cta.sv new file mode 100644 index 0000000000..8ceb33a7c4 --- /dev/null +++ b/hw/rtl/core/VX_csr_cta.sv @@ -0,0 +1,75 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Per-warp CTA state for one group of warps, addressed by group-local warp id: +// the rank, lane record and CTA id of the group's own warps. Written by the +// dispatcher's broadcast; the parent presents only writes that belong to this +// group. The CTA contexts every warp may read are held core-wide. +module VX_csr_cta import VX_gpu_pkg::*; #( + parameter NUM_WARPS = 1, + parameter LW_WIDTH = `LOG2UP(NUM_WARPS) +) ( + input wire clk, + input wire reset, + + // this group's warps + input wire warp_write, + input wire [LW_WIDTH-1:0] warp_waddr, + input cta_warp_t warp_wdata, + input wire id_write, + input wire [LW_WIDTH-1:0] id_waddr, + input wire [NCTA_WIDTH-1:0] id_wdata, + + // CSR read: rdata is valid one cycle after the address, which the CSR unit + // holds stable for that cycle + input wire [LW_WIDTH-1:0] rd_lid, + output wire [NW_WIDTH-1:0] rd_cta_rank, + output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] rd_lane, + + // CTA id of a warp, combinational + input wire [LW_WIDTH-1:0] sched_lid, + output wire [NCTA_WIDTH-1:0] sched_cta_id +); + cta_warp_t warp_rdata; + + VX_dp_ram #( + .DATAW ($bits(cta_warp_t)), + .SIZE (NUM_WARPS), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_warp_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (warp_write), + .wren (1'b1), + .waddr (warp_waddr), + .wdata (warp_wdata), + .raddr (rd_lid), + .rdata (warp_rdata) + ); + + assign rd_cta_rank = warp_rdata.cta_rank; + assign rd_lane = warp_rdata.lane_launch; + + reg [NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_r; + always @(posedge clk) begin + if (id_write) begin + cta_id_r[id_waddr] <= id_wdata; + end + end + assign sched_cta_id = cta_id_r[sched_lid]; + +endmodule diff --git a/hw/rtl/core/VX_csr_data.sv b/hw/rtl/core/VX_csr_data.sv index a47d6a86be..0214f50e99 100644 --- a/hw/rtl/core/VX_csr_data.sv +++ b/hw/rtl/core/VX_csr_data.sv @@ -17,38 +17,31 @@ `include "VX_fpu_define.vh" `endif -`ifdef VX_CFG_XLEN_64 - `define CSR_READ_64(addr, dst, src) \ - addr : dst = `VX_CFG_XLEN'(src) -`else - `define CSR_READ_64(addr, dst, src) \ - addr : dst = src[31:0]; \ - addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) -`endif - module VX_csr_data import VX_gpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE import VX_fpu_pkg::*; `endif #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + // FP CSR of the requesting warp: read live, written at fire + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, // {fcsr, frm, fflags} + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, + // context of the request's CTA, read by req_cta_id + input cta_ctx_t cta_ctx, + // The request is held for at least one cycle before it fires; that cycle // decodes it, and the fire cycle only merges the values that can still move. input wire req_fire, @@ -58,18 +51,13 @@ import VX_fpu_pkg::*; input wire [`VX_CSR_ADDR_BITS-1:0] req_addr, input wire [INST_SFU_BITS-1:0] req_op, input wire [`VX_CFG_XLEN-1:0] req_src, - output wire [`VX_CFG_XLEN-1:0] read_data, - - // Host performance-counter reads - input wire [7:0] dcr_mpm_class, - input wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr, - output wire [`VX_CFG_XLEN-1:0] dcr_data + output wire [`VX_CFG_XLEN-1:0] read_data ); `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) `UNUSED_VAR (req_uuid) - wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = sched_csr_if.cta_csrs.param; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] __cta_param = cta_ctx.param; `UNUSED_VAR (__cta_param) // Values that can change between the decode cycle and the fire cycle are @@ -91,7 +79,8 @@ import VX_fpu_pkg::*; localparam LIVE_LMEM_ADDR = 20; localparam LIVE_CLUSTER = 21; localparam LIVE_ENTRY = 22; - localparam NUM_LIVE = 23; + localparam LIVE_SATP = 23; + localparam NUM_LIVE = 24; // Writable CSRs localparam WR_MSCRATCH = 0; @@ -114,15 +103,10 @@ import VX_fpu_pkg::*; endcase endfunction -`ifdef VX_CFG_VM_ENABLE - reg [`VX_CFG_XLEN-1:0] satp; -`endif - // Decode //////////////////////////////////////////////////////////////// // Scheduler CSRs read interface assign sched_csr_if.csr_rd_wid = req_wid; - assign sched_csr_if.csr_rd_cta_id = req_cta_id; reg [`VX_CFG_XLEN-1:0] stable_w; reg [NUM_LIVE-1:0] live_sel_w; @@ -144,7 +128,7 @@ import VX_fpu_pkg::*; `endif `VX_CSR_MSCRATCH : begin stable_w = `VX_CFG_XLEN'(sched_csr_if.mscratch); wr_sel_w[WR_MSCRATCH] = 1; end - `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_id); + `VX_CSR_CTA_ID : stable_w = `VX_CFG_XLEN'(req_cta_id); `VX_CSR_CTA_RANK : live_sel_w[LIVE_CTA_RANK] = 1; `VX_CSR_CTA_SIZE : live_sel_w[LIVE_CTA_SIZE] = 1; `VX_CSR_CTA_BLOCK_ID_X : live_sel_w[LIVE_BLOCK_ID + 0] = 1; @@ -161,7 +145,7 @@ import VX_fpu_pkg::*; `VX_CSR_CTA_ENTRY : live_sel_w[LIVE_ENTRY] = 1; `VX_CSR_WARP_ID : stable_w = `VX_CFG_XLEN'(req_wid); - `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(CORE_ID); + `VX_CSR_CORE_ID : stable_w = `VX_CFG_XLEN'(core_id); `VX_CSR_ACTIVE_THREADS: live_sel_w[LIVE_THREADS] = 1; `VX_CSR_ACTIVE_WARPS: live_sel_w[LIVE_WARPS] = 1; `VX_CSR_NUM_THREADS: stable_w = `VX_CFG_XLEN'(`VX_CFG_NUM_THREADS); @@ -178,7 +162,7 @@ import VX_fpu_pkg::*; `endif `ifdef VX_CFG_VM_ENABLE - `VX_CSR_SATP : begin stable_w = satp; wr_sel_w[WR_SATP] = 1; end + `VX_CSR_SATP : begin live_sel_w[LIVE_SATP] = 1; wr_sel_w[WR_SATP] = 1; end `endif // Machine-mode trap CSRs (stored in the scheduler). @@ -228,8 +212,7 @@ import VX_fpu_pkg::*; // Live values /////////////////////////////////////////////////////////// `ifdef VX_CFG_EXT_F_ENABLE - reg [`VX_CFG_NUM_WARPS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr, fcsr_n; - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr[req_wid]; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] req_fcsr = fcsr_rd_data; `endif wire [NUM_LIVE-1:0][`VX_CFG_XLEN-1:0] live_src; @@ -245,8 +228,13 @@ import VX_fpu_pkg::*; assign live_src[LIVE_INSTRET] = sched_csr_if.instret[31:0]; assign live_src[LIVE_INSTRET_H] = 32'(sched_csr_if.instret[PERF_CTR_BITS-1:32]); `endif - assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_masks[req_wid]); - assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(sched_csr_if.active_warps); + assign live_src[LIVE_THREADS] = `VX_CFG_XLEN'(sched_csr_if.thread_mask); + // the CSR is XLEN wide: it holds warps 0..XLEN-1 + wire [`VX_CFG_NUM_WARPS-1:0] active_warps = sched_csr_if.active_warps; + assign live_src[LIVE_WARPS] = `VX_CFG_XLEN'(active_warps); + if (`VX_CFG_NUM_WARPS > `VX_CFG_XLEN) begin : g_active_warps_hi + `UNUSED_VAR (active_warps[`VX_CFG_NUM_WARPS-1:`VX_CFG_XLEN]) + end `ifdef VX_CFG_EXT_F_ENABLE assign live_src[LIVE_FFLAGS] = `VX_CFG_XLEN'(req_fcsr[`FP_FLAGS_BITS-1:0]); assign live_src[LIVE_FRM] = `VX_CFG_XLEN'(req_fcsr[INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]); @@ -258,16 +246,21 @@ import VX_fpu_pkg::*; `endif // The CTA context RAMs are addressed by the held request and return in the // fire cycle. - assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_rank); - assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cta_size); + assign live_src[LIVE_CTA_RANK] = `VX_CFG_XLEN'(sched_csr_if.cta_rank); + assign live_src[LIVE_CTA_SIZE] = `VX_CFG_XLEN'(cta_ctx.cta_size); for (genvar i = 0; i < 3; ++i) begin : g_cta_dims - assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_idx[i]); - assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.block_dim[i]); - assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.grid_dim[i]); + assign live_src[LIVE_BLOCK_ID + i] = `VX_CFG_XLEN'(cta_ctx.block_idx[i]); + assign live_src[LIVE_BLOCK_DIM + i] = `VX_CFG_XLEN'(cta_ctx.block_dim[i]); + assign live_src[LIVE_GRID_DIM + i] = `VX_CFG_XLEN'(cta_ctx.grid_dim[i]); end - assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.lmem_addr); - assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(sched_csr_if.cta_csrs.cluster_size); - assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(sched_csr_if.cta_csrs.entry)); + assign live_src[LIVE_LMEM_ADDR] = `VX_CFG_XLEN'(cta_ctx.lmem_addr); + assign live_src[LIVE_CLUSTER] = `VX_CFG_XLEN'(cta_ctx.cluster_size); + assign live_src[LIVE_ENTRY] = `VX_CFG_XLEN'(to_fullPC(cta_ctx.entry)); +`ifdef VX_CFG_VM_ENABLE + assign live_src[LIVE_SATP] = sched_csr_if.csr_satp; +`else + assign live_src[LIVE_SATP] = '0; +`endif reg [`VX_CFG_XLEN-1:0] live_data; always @(*) begin @@ -294,66 +287,17 @@ import VX_fpu_pkg::*; `ifdef VX_CFG_EXT_F_ENABLE // The FP flags keep moving under in-flight FP instructions, so their // read-modify-write stays in the fire cycle. - wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data = - (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); - - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_enable; - wire [`VX_CFG_NUM_FPU_BLOCKS-1:0][NW_WIDTH-1:0] fpu_write_wid; - fflags_t [`VX_CFG_NUM_FPU_BLOCKS-1:0] fpu_write_fflags; - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_write - assign fpu_write_enable[i] = fpu_csr_if[i].write_enable; - assign fpu_write_wid[i] = fpu_csr_if[i].write_wid; - assign fpu_write_fflags[i] = fpu_csr_if[i].write_fflags; - end - - always @(*) begin - fcsr_n = fcsr; - for (integer i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin - if (fpu_write_enable[i]) begin - fcsr_n[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] = fcsr[fpu_write_wid[i]][`FP_FLAGS_BITS-1:0] - | fpu_write_fflags[i]; - end - end - if (write_sel[WR_FFLAGS]) begin - fcsr_n[req_wid][`FP_FLAGS_BITS-1:0] = fcsr_wr_data[`FP_FLAGS_BITS-1:0]; - end - if (write_sel[WR_FRM]) begin - fcsr_n[req_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS] = fcsr_wr_data[INST_FRM_BITS-1:0]; - end - if (write_sel[WR_FCSR]) begin - fcsr_n[req_wid] = fcsr_wr_data; - end - end - - for (genvar i = 0; i < `VX_CFG_NUM_FPU_BLOCKS; ++i) begin : g_fpu_csr_read_frm - assign fpu_csr_if[i].read_frm = fcsr[fpu_csr_if[i].read_wid][INST_FRM_BITS+`FP_FLAGS_BITS-1:`FP_FLAGS_BITS]; - end - - always @(posedge clk) begin - if (reset) begin - fcsr <= '0; - end else begin - fcsr <= fcsr_n; - end - end + assign fcsr_wr_data = (INST_FRM_BITS+`FP_FLAGS_BITS)'(csr_rmw(req_op, live_data, req_src)); + assign fcsr_wr_sel = {write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}; `else `UNUSED_VAR ({write_sel[WR_FCSR], write_sel[WR_FRM], write_sel[WR_FFLAGS]}) `endif `ifdef VX_CFG_VM_ENABLE - // Per-core SATP CSR. Initialized to 0 (BARE mode); kernel writes - // it from vx_start.S after the runtime has installed the page table. - // Surfaced on sched_csr_if.satp so VX_core can pick it up directly - // off the shared interface instead of routing through SFU/execute. - always @(posedge clk) begin - if (reset) begin - satp <= '0; - end else if (write_sel[WR_SATP]) begin - satp <= wr_data_r; - end - end - assign sched_csr_if.csr_satp = satp; + // The per-core SATP CSR lives in the scheduler, shared by every SFU block, + // so its read-modify-write stays in the fire cycle. + assign sched_csr_if.satp_wr_valid = write_sel[WR_SATP]; + assign sched_csr_if.satp_wr_data = csr_rmw(req_op, live_data, req_src); `else `UNUSED_VAR (write_sel[WR_SATP]) `endif @@ -386,225 +330,4 @@ import VX_fpu_pkg::*; end end - // Host performance-counter reads //////////////////////////////////////// - - reg [`VX_CFG_XLEN-1:0] dcr_data_w; - - always @(*) begin - dcr_data_w = '0; - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, sched_csr_if.cycles); - `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, sched_csr_if.instret); - default: begin - `ifdef PERF_ENABLE - if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) - || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin - case (dcr_mpm_class) - `VX_DCR_MPM_CLASS_CORE: begin - case (dcr_addr) - // PERF: pipeline - `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); - `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); - `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); - `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); - `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); - `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); - `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); - `ifdef VX_CFG_EXT_F_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); - `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); - `endif - // PERF: branches - `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); - `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); - // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) - `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); - `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); - `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); - `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); - `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); - default:; - endcase - end - `VX_DCR_MPM_CLASS_ICACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_DCACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L2CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_L3CACHE: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); - default:; - endcase - end - `VX_DCR_MPM_CLASS_MEM: begin - case (dcr_addr) - // PERF: off-chip memory - `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); - `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); - `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); - // PERF: lmem - `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); - `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); - `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); - // PERF: coalescer - `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); - `ifdef VX_CFG_VM_ENABLE - // PERF: VM/MMU (icache + dcache MMU summed) - `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); - `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); - `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); - `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); - `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); - `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); - `endif - default:; - endcase - end - `ifdef VX_CFG_EXT_DXA_ENABLE - `VX_DCR_MPM_CLASS_DXA: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); - `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); - `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); - `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TCU_ENABLE - `VX_DCR_MPM_CLASS_TCU: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); - `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - `VX_DCR_MPM_CLASS_TEX: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `VX_DCR_MPM_CLASS_RASTER: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `VX_DCR_MPM_CLASS_OM: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); - `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); - `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); - `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); - `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); - default:; - endcase - end - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - `VX_DCR_MPM_CLASS_RTU: begin - case (dcr_addr) - `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); - `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); - `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); - `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); - `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); - `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); - `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); - `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); - default:; - endcase - end - `endif - default:; - endcase - end - `endif - end - endcase - end - - assign dcr_data = dcr_data_w; -`ifndef PERF_ENABLE - `UNUSED_VAR (dcr_mpm_class) -`endif - -`ifdef PERF_ENABLE - `UNUSED_VAR (sysmem_perf.icache); - `UNUSED_VAR (sysmem_perf.lmem); -`endif - endmodule diff --git a/hw/rtl/core/VX_csr_fpu.sv b/hw/rtl/core/VX_csr_fpu.sv new file mode 100644 index 0000000000..1aefbf8642 --- /dev/null +++ b/hw/rtl/core/VX_csr_fpu.sv @@ -0,0 +1,108 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE + +`include "VX_fpu_define.vh" + +// Per-warp FP CSRs (frm + fflags) for one residue class of warps modulo WID_STRIDE, +// stored by wid / WID_STRIDE; every request presented belongs to that class. +// CSR port c serves the warps with (wid / WID_STRIDE) mod NUM_CSR_PORTS == c and +// FPU port f those with (wid / WID_STRIDE) mod NUM_FPU_PORTS == f, so each warp's +// state is written by exactly one port of each kind. +module VX_csr_fpu import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter WID_STRIDE = 1, + parameter NUM_CSR_PORTS = 1, + parameter NUM_FPU_PORTS = 1 +) ( + input wire clk, + input wire reset, + + input wire [NUM_CSR_PORTS-1:0][NW_WIDTH-1:0] csr_wid, + output wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_rd_data, + input wire [NUM_CSR_PORTS-1:0][2:0] csr_wr_sel, // {fcsr, frm, fflags} + input wire [NUM_CSR_PORTS-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] csr_wr_data, + + VX_fpu_csr_if.slave fpu_csr_if [NUM_FPU_PORTS] +); + localparam FCSR_W = INST_FRM_BITS + `FP_FLAGS_BITS; + localparam NUM_ENTRIES = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + localparam ENTRY_W = `LOG2UP(NUM_ENTRIES); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) + + function automatic logic [ENTRY_W-1:0] entry_of(input logic [NW_WIDTH-1:0] wid); + entry_of = ENTRY_W'(wid >> STRIDE_BITS); + endfunction + + wire [NUM_FPU_PORTS-1:0] fpu_wr_enable; + wire [NUM_FPU_PORTS-1:0][NW_WIDTH-1:0] fpu_wr_wid; + fflags_t [NUM_FPU_PORTS-1:0] fpu_wr_fflags; + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_ports + assign fpu_wr_enable[f] = fpu_csr_if[f].write_enable; + assign fpu_wr_wid[f] = fpu_csr_if[f].write_wid; + assign fpu_wr_fflags[f] = fpu_csr_if[f].write_fflags; + end + + wire [NUM_ENTRIES-1:0][FCSR_W-1:0] fcsr; + + for (genvar l = 0; l < NUM_ENTRIES; ++l) begin : g_entries + localparam C = l % NUM_CSR_PORTS; + localparam F = l % NUM_FPU_PORTS; + localparam [ENTRY_W-1:0] ENTRY = ENTRY_W'(l); + + reg [FCSR_W-1:0] fcsr_r, fcsr_n; + + always @(*) begin + fcsr_n = fcsr_r; + if (fpu_wr_enable[F] && (entry_of(fpu_wr_wid[F]) == ENTRY)) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = fcsr_r[`FP_FLAGS_BITS-1:0] | fpu_wr_fflags[F]; + end + if (entry_of(csr_wid[C]) == ENTRY) begin + if (csr_wr_sel[C][0]) begin + fcsr_n[`FP_FLAGS_BITS-1:0] = csr_wr_data[C][`FP_FLAGS_BITS-1:0]; + end + if (csr_wr_sel[C][1]) begin + fcsr_n[FCSR_W-1:`FP_FLAGS_BITS] = csr_wr_data[C][INST_FRM_BITS-1:0]; + end + if (csr_wr_sel[C][2]) begin + fcsr_n = csr_wr_data[C]; + end + end + end + + always @(posedge clk) begin + if (reset) begin + fcsr_r <= '0; + end else begin + fcsr_r <= fcsr_n; + end + end + + assign fcsr[l] = fcsr_r; + end + + for (genvar c = 0; c < NUM_CSR_PORTS; ++c) begin : g_csr_rd + assign csr_rd_data[c] = fcsr[entry_of(csr_wid[c])]; + end + + for (genvar f = 0; f < NUM_FPU_PORTS; ++f) begin : g_fpu_rd + assign fpu_csr_if[f].read_frm = fcsr[entry_of(fpu_csr_if[f].read_wid)][FCSR_W-1:`FP_FLAGS_BITS]; + end + +endmodule + +`endif diff --git a/hw/rtl/core/VX_csr_mpm.sv b/hw/rtl/core/VX_csr_mpm.sv new file mode 100644 index 0000000000..4acdd07c1c --- /dev/null +++ b/hw/rtl/core/VX_csr_mpm.sv @@ -0,0 +1,272 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_XLEN_64 + `define CSR_READ_64(addr, dst, src) \ + addr : dst = `VX_CFG_XLEN'(src) +`else + `define CSR_READ_64(addr, dst, src) \ + addr : dst = src[31:0]; \ + addr+12'h80 : dst = 32'(src[$bits(src)-1:32]) +`endif + +// Host-side machine performance-monitor reads (DCR path). The counters are not +// reachable from kernel CSR instructions, so one instance serves the whole core. +module VX_csr_mpm import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef PERF_ENABLE + input sysmem_perf_t sysmem_perf, + input pipeline_perf_t pipeline_perf, +`endif + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + + VX_dcr_csr_if.slave dcr_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + wire [7:0] dcr_mpm_class = dcr_csr_if.mpm_class; + wire [`VX_CSR_ADDR_BITS-1:0] dcr_addr = dcr_csr_if.addr; + wire [`VX_CFG_XLEN-1:0] dcr_data; + + // Host counter reads decode their own address and never wait on a request. + assign dcr_csr_if.ready = 1'b1; + assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_data); + `UNUSED_VAR (dcr_data) + `UNUSED_VAR (dcr_csr_if.valid) + + // Host performance-counter reads //////////////////////////////////////// + + reg [`VX_CFG_XLEN-1:0] dcr_data_w; + + always @(*) begin + dcr_data_w = '0; + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MCYCLE, dcr_data_w, cycles); + `CSR_READ_64(`VX_CSR_MINSTRET, dcr_data_w, instret); + default: begin + `ifdef PERF_ENABLE + if ((dcr_addr >= `VX_CSR_MPM_USER && dcr_addr < (`VX_CSR_MPM_USER + 32)) + || (dcr_addr >= `VX_CSR_MPM_USER_H && dcr_addr < (`VX_CSR_MPM_USER_H + 32))) begin + case (dcr_mpm_class) + `VX_DCR_MPM_CLASS_CORE: begin + case (dcr_addr) + // PERF: pipeline + `CSR_READ_64(`VX_CSR_MPM_SCHED_IDLE, dcr_data_w, pipeline_perf.sched.idles); + `CSR_READ_64(`VX_CSR_MPM_ACTIVE_WARPS, dcr_data_w, pipeline_perf.sched.active_warps); + `CSR_READ_64(`VX_CSR_MPM_STALLED_WARPS, dcr_data_w, pipeline_perf.sched.stalled_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_WARPS, dcr_data_w, pipeline_perf.sched.issued_warps); + `CSR_READ_64(`VX_CSR_MPM_ISSUED_THREADS, dcr_data_w, pipeline_perf.sched.issued_threads); + `CSR_READ_64(`VX_CSR_MPM_STALL_FETCH, dcr_data_w, pipeline_perf.fetch.stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_IBUF, dcr_data_w, pipeline_perf.issue.ibf_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_SCRB, dcr_data_w, pipeline_perf.issue.scb_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_OPDS, dcr_data_w, pipeline_perf.issue.opd_stalls); + `CSR_READ_64(`VX_CSR_MPM_STALL_ALU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_ALU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_ALU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_LSU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_LSU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_LSU]); + `CSR_READ_64(`VX_CSR_MPM_STALL_SFU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_SFU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_SFU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_SFU]); + `ifdef VX_CFG_EXT_F_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_FPU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_FPU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_FPU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_FPU]); + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `CSR_READ_64(`VX_CSR_MPM_STALL_TCU, dcr_data_w, pipeline_perf.issue.dispatch_stalls[EX_TCU]); + `CSR_READ_64(`VX_CSR_MPM_INSTR_TCU, dcr_data_w, pipeline_perf.issue.dispatch_instrs[EX_TCU]); + `endif + // PERF: branches + `CSR_READ_64(`VX_CSR_MPM_BRANCHES, dcr_data_w, pipeline_perf.sched.branches); + `CSR_READ_64(`VX_CSR_MPM_DIVERGENCE, dcr_data_w, pipeline_perf.sched.divergence); + // PERF: memory (core-issued requests; DRAM traffic is in the MEM class) + `CSR_READ_64(`VX_CSR_MPM_IFETCHES, dcr_data_w, pipeline_perf.ifetches); + `CSR_READ_64(`VX_CSR_MPM_LOADS, dcr_data_w, pipeline_perf.loads); + `CSR_READ_64(`VX_CSR_MPM_STORES, dcr_data_w, pipeline_perf.stores); + `CSR_READ_64(`VX_CSR_MPM_IFETCH_LT, dcr_data_w, pipeline_perf.ifetch_latency); + `CSR_READ_64(`VX_CSR_MPM_LOAD_LT, dcr_data_w, pipeline_perf.load_latency); + default:; + endcase + end + `VX_DCR_MPM_CLASS_ICACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_ICACHE_READS, dcr_data_w, sysmem_perf.icache.reads); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MISS_R, dcr_data_w, sysmem_perf.icache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_ICACHE_MSHR_ST, dcr_data_w, sysmem_perf.icache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_DCACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DCACHE_READS, dcr_data_w, sysmem_perf.dcache.reads); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_WRITES, dcr_data_w, sysmem_perf.dcache.writes); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_R, dcr_data_w, sysmem_perf.dcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MISS_W, dcr_data_w, sysmem_perf.dcache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_EVICTS, dcr_data_w, sysmem_perf.dcache.evictions); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_BANK_ST, dcr_data_w, sysmem_perf.dcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_DCACHE_MSHR_ST, dcr_data_w, sysmem_perf.dcache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L2CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_READS, dcr_data_w, sysmem_perf.l2cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_WRITES, dcr_data_w, sysmem_perf.l2cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_R, dcr_data_w, sysmem_perf.l2cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MISS_W, dcr_data_w, sysmem_perf.l2cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_EVICTS, dcr_data_w, sysmem_perf.l2cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_BANK_ST, dcr_data_w, sysmem_perf.l2cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L2CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l2cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_L3CACHE: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_READS, dcr_data_w, sysmem_perf.l3cache.reads); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_WRITES, dcr_data_w, sysmem_perf.l3cache.writes); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_R, dcr_data_w, sysmem_perf.l3cache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MISS_W, dcr_data_w, sysmem_perf.l3cache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_EVICTS, dcr_data_w, sysmem_perf.l3cache.evictions); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_BANK_ST, dcr_data_w, sysmem_perf.l3cache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_L3CACHE_MSHR_ST, dcr_data_w, sysmem_perf.l3cache.mshr_stalls); + default:; + endcase + end + `VX_DCR_MPM_CLASS_MEM: begin + case (dcr_addr) + // PERF: off-chip memory + `CSR_READ_64(`VX_CSR_MPM_MEM_READS, dcr_data_w, sysmem_perf.mem.reads); + `CSR_READ_64(`VX_CSR_MPM_MEM_WRITES, dcr_data_w, sysmem_perf.mem.writes); + `CSR_READ_64(`VX_CSR_MPM_MEM_LT, dcr_data_w, sysmem_perf.mem.latency); + // PERF: lmem + `CSR_READ_64(`VX_CSR_MPM_LMEM_READS, dcr_data_w, sysmem_perf.lmem.reads); + `CSR_READ_64(`VX_CSR_MPM_LMEM_WRITES, dcr_data_w, sysmem_perf.lmem.writes); + `CSR_READ_64(`VX_CSR_MPM_LMEM_BANK_ST, dcr_data_w, sysmem_perf.lmem.bank_stalls); + // PERF: coalescer + `CSR_READ_64(`VX_CSR_MPM_COALESCER_MISS, dcr_data_w, sysmem_perf.coalescer.misses); + `ifdef VX_CFG_VM_ENABLE + // PERF: VM/MMU (icache + dcache MMU summed) + `CSR_READ_64(`VX_CSR_MPM_TLB_READS, dcr_data_w, pipeline_perf.mmu.tlb_reads); + `CSR_READ_64(`VX_CSR_MPM_TLB_HITS, dcr_data_w, pipeline_perf.mmu.tlb_hits); + `CSR_READ_64(`VX_CSR_MPM_TLB_MISSES, dcr_data_w, pipeline_perf.mmu.tlb_misses); + `CSR_READ_64(`VX_CSR_MPM_TLB_EVICTS, dcr_data_w, pipeline_perf.mmu.tlb_evictions); + `CSR_READ_64(`VX_CSR_MPM_PTW_WALKS, dcr_data_w, pipeline_perf.mmu.ptw_walks); + `CSR_READ_64(`VX_CSR_MPM_PTW_LATENCY, dcr_data_w, pipeline_perf.mmu.ptw_latency); + `endif + default:; + endcase + end + `ifdef VX_CFG_EXT_DXA_ENABLE + `VX_DCR_MPM_CLASS_DXA: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_DXA_TRANSFERS, dcr_data_w, sysmem_perf.dxa.transfers); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_READS, dcr_data_w, sysmem_perf.dxa.gmem_reads); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_DEDUP, dcr_data_w, sysmem_perf.dxa.gmem_dedup); + `CSR_READ_64(`VX_CSR_MPM_DXA_LMEM_WRITES,dcr_data_w, sysmem_perf.dxa.lmem_writes); + `CSR_READ_64(`VX_CSR_MPM_DXA_GMEM_LT, dcr_data_w, sysmem_perf.dxa.gmem_latency); + `CSR_READ_64(`VX_CSR_MPM_DXA_NOSLOT_STALLS, dcr_data_w, sysmem_perf.dxa.noslot_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TCU_ENABLE + `VX_DCR_MPM_CLASS_TCU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_STALLS, dcr_data_w, pipeline_perf.tcu.tbuf_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCU_TBUF_CACHE_HITS, dcr_data_w, pipeline_perf.tcu.tbuf_cache_hits); + `CSR_READ_64(`VX_CSR_MPM_TCU_LMEM_READS, dcr_data_w, pipeline_perf.tcu.lmem_reads); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + `VX_DCR_MPM_CLASS_TEX: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_TEX_READS, dcr_data_w, sysmem_perf.tex.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_TEX_LAT, dcr_data_w, sysmem_perf.tex.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_TEX_ST, dcr_data_w, sysmem_perf.tex.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_READS, dcr_data_w, sysmem_perf.tcache.reads); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MISS_R, dcr_data_w, sysmem_perf.tcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_BANK_ST, dcr_data_w, sysmem_perf.tcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_TCACHE_MSHR_ST, dcr_data_w, sysmem_perf.tcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + `VX_DCR_MPM_CLASS_RASTER: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RASTER_READS, dcr_data_w, sysmem_perf.raster.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_RASTER_LAT, dcr_data_w, sysmem_perf.raster.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_RASTER_ST, dcr_data_w, sysmem_perf.raster.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_READS, dcr_data_w, sysmem_perf.rcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MISS_R, dcr_data_w, sysmem_perf.rcache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_BANK_ST, dcr_data_w, sysmem_perf.rcache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_RCACHE_MSHR_ST, dcr_data_w, sysmem_perf.rcache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + `VX_DCR_MPM_CLASS_OM: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_OM_READS, dcr_data_w, sysmem_perf.om.mem_reads); + `CSR_READ_64(`VX_CSR_MPM_OM_WRITES, dcr_data_w, sysmem_perf.om.mem_writes); + `CSR_READ_64(`VX_CSR_MPM_OM_LAT, dcr_data_w, sysmem_perf.om.mem_latency); + `CSR_READ_64(`VX_CSR_MPM_OM_ST, dcr_data_w, sysmem_perf.om.stall_cycles); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_READS, dcr_data_w, sysmem_perf.ocache.reads); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_WRITES, dcr_data_w, sysmem_perf.ocache.writes); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_R, dcr_data_w, sysmem_perf.ocache.read_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MISS_W, dcr_data_w, sysmem_perf.ocache.write_misses); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_BANK_ST, dcr_data_w, sysmem_perf.ocache.bank_stalls); + `CSR_READ_64(`VX_CSR_MPM_OCACHE_MSHR_ST, dcr_data_w, sysmem_perf.ocache.mshr_stalls); + default:; + endcase + end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `VX_DCR_MPM_CLASS_RTU: begin + case (dcr_addr) + `CSR_READ_64(`VX_CSR_MPM_RTU_RAYS, dcr_data_w, sysmem_perf.rtu.rays); + `CSR_READ_64(`VX_CSR_MPM_RTU_HITS, dcr_data_w, sysmem_perf.rtu.hits); + `CSR_READ_64(`VX_CSR_MPM_RTU_BOX_TESTS, dcr_data_w, sysmem_perf.rtu.box_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_TRI_TESTS, dcr_data_w, sysmem_perf.rtu.tri_tests); + `CSR_READ_64(`VX_CSR_MPM_RTU_CALLBACKS, dcr_data_w, sysmem_perf.rtu.callbacks); + `CSR_READ_64(`VX_CSR_MPM_RTU_BUSY, dcr_data_w, sysmem_perf.rtu.busy); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_READS, dcr_data_w, sysmem_perf.rtcache.reads); + `CSR_READ_64(`VX_CSR_MPM_RTCACHE_MISS_R, dcr_data_w, sysmem_perf.rtcache.read_misses); + default:; + endcase + end + `endif + default:; + endcase + end + `endif + end + endcase + end + + assign dcr_data = dcr_data_w; +`ifndef PERF_ENABLE + `UNUSED_VAR (dcr_mpm_class) +`endif + +`ifdef PERF_ENABLE + `UNUSED_VAR (sysmem_perf.icache); + `UNUSED_VAR (sysmem_perf.lmem); +`endif + +endmodule diff --git a/hw/rtl/core/VX_csr_unit.sv b/hw/rtl/core/VX_csr_unit.sv index ae0b7038b2..139967ec63 100644 --- a/hw/rtl/core/VX_csr_unit.sv +++ b/hw/rtl/core/VX_csr_unit.sv @@ -13,25 +13,28 @@ `include "VX_define.vh" +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + module VX_csr_unit import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, `ifdef VX_CFG_EXT_F_ENABLE - VX_fpu_csr_if.slave fpu_csr_if [`VX_CFG_NUM_FPU_BLOCKS], + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, `endif VX_sched_csr_if.slave sched_csr_if, - VX_dcr_csr_if.slave dcr_csr_if, + input cta_ctx_t cta_ctx, VX_execute_if.slave execute_if, VX_result_if.master result_if ); @@ -74,47 +77,39 @@ module VX_csr_unit import VX_gpu_pkg::*; #( wire [`VX_CFG_XLEN-1:0] rs1_first; if (NUM_LANES > 1) begin : g_rs1_first - wire [`LOG2UP(NUM_LANES)-1:0] first_lane; - VX_priority_encoder #( + // the data's own select tree, not an encoded lane index driving a wide mux + VX_find_first #( .N (NUM_LANES), + .DATAW (`VX_CFG_XLEN), .REVERSE (0) ) first_lane_sel ( - .data_in (execute_if.data.header.tmask), - .index_out (first_lane), - `UNUSED_PIN (onehot_out), + .valid_in (execute_if.data.header.tmask), + .data_in (rs1_data), + .data_out (rs1_first), `UNUSED_PIN (valid_out) ); - assign rs1_first = rs1_data[first_lane]; end else begin : g_rs1_first_0 assign rs1_first = rs1_data[0]; end wire [`VX_CFG_XLEN-1:0] csr_req_src = execute_if.data.op_args.csr.use_imm ? `VX_CFG_XLEN'(csr_imm) : rs1_first; - wire [`VX_CFG_XLEN-1:0] csr_scalar_data, dcr_read_data; - - // Host counter reads decode their own address and never wait on a request. - assign dcr_csr_if.ready = 1'b1; - assign dcr_csr_if.value = VX_DCR_DATA_WIDTH'(dcr_read_data); - `UNUSED_VAR (dcr_read_data) - `UNUSED_VAR (dcr_csr_if.valid) + wire [`VX_CFG_XLEN-1:0] csr_scalar_data; VX_csr_data #( - .INSTANCE_ID (INSTANCE_ID), - .CORE_ID (CORE_ID) + .INSTANCE_ID (INSTANCE_ID) ) csr_data ( .clk (clk), .reset (reset), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), `endif .req_fire (csr_req_fire), @@ -124,19 +119,20 @@ module VX_csr_unit import VX_gpu_pkg::*; #( .req_addr (csr_addr), .req_op (execute_if.data.op_type), .req_src (csr_req_src), - .read_data (csr_scalar_data), - - .dcr_mpm_class (dcr_csr_if.mpm_class), - .dcr_addr (dcr_csr_if.addr), - .dcr_data (dcr_read_data) + .read_data (csr_scalar_data) ); +`ifdef VX_CFG_EXT_F_ENABLE + assign fcsr_wid = execute_if.data.header.wid; +`endif + // Per-lane CSRs // Thread ids are the lane index on top of a per-request base. wire [`VX_CFG_XLEN-1:0] wtid_base = (PID_BITS != 0) ? `VX_CFG_XLEN'(execute_if.data.header.pid * NUM_LANES) : '0; - wire [`VX_CFG_XLEN-1:0] gtid_base = (`VX_CFG_XLEN'(CORE_ID) << (NW_BITS + NT_BITS)) - + (`VX_CFG_XLEN'(execute_if.data.header.wid) << NT_BITS) + // mhartid is dense, (core * NUM_WARPS + wid) * NUM_THREADS + tid, for any NUM_WARPS + wire [`VX_CFG_XLEN-1:0] gtid_base = ((`VX_CFG_XLEN'(core_id) * `VX_CFG_XLEN'(`VX_CFG_NUM_WARPS) + + `VX_CFG_XLEN'(execute_if.data.header.wid)) << NT_BITS) + wtid_base; wire is_wtid_w = (csr_addr == `VX_CSR_THREAD_ID); diff --git a/hw/rtl/core/VX_cta_dispatch.sv b/hw/rtl/core/VX_cta_dispatch.sv index 4991f97899..7c1aa9d71d 100644 --- a/hw/rtl/core/VX_cta_dispatch.sv +++ b/hw/rtl/core/VX_cta_dispatch.sv @@ -13,9 +13,12 @@ `include "VX_define.vh" -// Dispatches warps for incoming CTAs from the KMU. +// Dispatches warps for incoming CTAs from the KMU. Warps launch on NUM_LANES +// lanes in parallel: lane l owns the warps with wid % NUM_LANES == l, and each +// lane launches one of its free warps per cycle until the CTA is fully launched. module VX_cta_dispatch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1 ) ( input wire clk, input wire reset, @@ -28,31 +31,27 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( input wire warp_done, input wire [NW_WIDTH-1:0] warp_done_wid, - // to scheduler (one warp per cycle) - output wire cta_fire, - output wire [NW_WIDTH-1:0] cta_wid, + // to scheduler (one warp per lane per cycle) + output wire [NUM_LANES-1:0] cta_fire, + output wire [NUM_LANES-1:0][NW_WIDTH-1:0] cta_wid, output wire [PC_BITS-1:0] cta_PC, - output wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask, + output wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask, output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param, - output wire cta_init, + output wire [NUM_LANES-1:0] cta_init, - // CTA-CSR read-back - input wire [NW_WIDTH-1:0] csr_rd_wid, - input wire [NCTA_WIDTH-1:0] csr_rd_cta_id, - output cta_csrs_t cta_rd_csrs, - // The per-lane launch record (see cta_lane_t): a compute warp's thread index and a - // fragment warp's stamp overlay the same bits, and the CSR unit reads the view the - // CSR being accessed calls for. - output cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane, - - // CTA id of the scheduled warp - input wire [NW_WIDTH-1:0] schedule_wid, - output wire [NCTA_WIDTH-1:0] schedule_cta_id, + // CTA state for the CTA-CSR stores (see VX_csr_cta): the CTA context, + // and each lane's warp records + output cta_bcast_t cta_bcast, + output cta_lane_bcast_t [NUM_LANES-1:0] cta_lane_bcast, output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) + `STATIC_ASSERT ((`VX_CFG_NUM_WARPS % NUM_LANES) == 0, ("invalid parameter")) localparam NUM_CTA_SLOTS= `VX_CFG_NUM_WARPS; + localparam LANE_WARPS = `VX_CFG_NUM_WARPS / NUM_LANES; + localparam LANE_WIDTH = `LOG2UP(NUM_LANES); + localparam LCNT_WIDTH = `CLOG2(NUM_LANES+1); localparam CS_BITS = NW_WIDTH; // UP(NW_BITS): at least 1 to avoid zero-width when NUM_WARPS=1 localparam LMEM_SIZE = (1 << `VX_CFG_LMEM_LOG_SIZE); @@ -93,7 +92,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Fixed-stride LMEM partition: resident CTA in slot i gets LMEM base // i * stride, where stride = aligned_lmem_size (uniform within a kernel). - // cur_lmem_base_r is latched at accept and held stable through DISPATCH. + // The product is formed from the slot and stride latched at accept, a cycle + // later; its first reader, the context write of the first launched warp, + // comes at least two cycles after accept. + reg [`VX_CFG_LMEM_LOG_SIZE:0] cur_stride_r; reg [`VX_CFG_LMEM_LOG_SIZE-1:0] cur_lmem_base_r; // Reverse lookup: warp-ID → CTA slot index. A flop array indexed by wid, @@ -114,7 +116,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // so a slot relaunched with lanes it has not run before must run it again. reg [7:0] cur_ctx_id_r; reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] warp_init_lanes_r; - reg warp_skip_init_r; // ------------------------------------------------------------------------- // FSM + per-dispatch registers @@ -126,33 +127,56 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( reg [PC_BITS-1:0] entry_r; reg [2:0][31:0] block_idx_r; reg [2:0][CTA_TID_WIDTH:0] block_dim_r; + reg [1:0][CTA_TID_WIDTH-1:0] block_last_r; reg [2:0][31:0] grid_dim_r; reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] param_r; reg [CTA_TID_WIDTH:0] block_size_r; reg [2:0][CTA_TID_WIDTH-1:0] warp_step_r; reg [NW_WIDTH:0] cluster_size_r; - reg warp_fire_r; - reg [NW_WIDTH-1:0] warp_id_r; - reg [`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; reg [NW_WIDTH-1:0] cta_rank_r; - reg [2:0][CTA_TID_WIDTH-1:0] thread_idx_r; reg [CS_BITS-1:0] cur_slot_r; + // Warps launched last cycle, one per lane; `order` is the warp's position + // among them, so its CTA rank is cta_rank_r + order. + reg [NUM_LANES-1:0] warp_fire_r; + reg [NUM_LANES-1:0][NW_WIDTH-1:0] warp_id_r; + reg [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] warp_tmask_r; + reg [NUM_LANES-1:0][LANE_WIDTH-1:0] warp_order_r; + reg [NUM_LANES-1:0] warp_skip_init_r; + reg [LCNT_WIDTH-1:0] nfired_r; + wire fire_any_r = (| warp_fire_r); + + // Thread-index window: thread_idx_r[k] is the base thread index of the warp + // ranked cta_rank_r + k. Launching n warps shifts the window by n; the entries + // shifted in advance by group_step_r (NUM_LANES warps), so no two advances + // chain within a cycle. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_r; + reg [2:0][CTA_TID_WIDTH-1:0] group_step_r; + // ------------------------------------------------------------------------- - // Free-warp selection + // Free-warp selection: one candidate per lane // ------------------------------------------------------------------------- reg [`VX_CFG_NUM_WARPS-1:0] dispatched_warps; - wire [NW_WIDTH-1:0] warp_id_n; - wire warp_ready; - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS), - .REVERSE (0) - ) priority_enc ( - .data_in (~(active_warps | dispatched_warps)), - `UNUSED_PIN(onehot_out), - .index_out (warp_id_n), - .valid_out (warp_ready) - ); + wire [`VX_CFG_NUM_WARPS-1:0] free_warps = ~(active_warps | dispatched_warps); + wire [NUM_LANES-1:0] lane_ready; + wire [NUM_LANES-1:0][NW_WIDTH-1:0] lane_wid; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_select + wire [LANE_WARPS-1:0] lane_free; + for (genvar j = 0; j < LANE_WARPS; ++j) begin : g_free + assign lane_free[j] = free_warps[j * NUM_LANES + l]; + end + wire [`LOG2UP(LANE_WARPS)-1:0] lane_idx; + VX_priority_encoder #( + .N (LANE_WARPS), + .REVERSE (0) + ) priority_enc ( + .data_in (lane_free), + `UNUSED_PIN(onehot_out), + .index_out (lane_idx), + .valid_out (lane_ready[l]) + ); + assign lane_wid[l] = NW_WIDTH'(lane_idx) * NW_WIDTH'(NUM_LANES) + NW_WIDTH'(l); + end wire kmu_bus_if_fire = kmu_bus_if.valid && kmu_bus_if.ready; @@ -191,7 +215,6 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // pipeline; `is_frag_warp`/`frag_stamps_sel` pick the lane-record view being // written. All benign (0) in a build with no rasterizer. wire frag_inflight; - wire is_frag_warp; wire [`VX_CFG_NUM_THREADS-1:0][FRAG_LANE_BITS-1:0] frag_stamps_sel; `ifdef VX_CFG_EXT_RASTER_ENABLE @@ -213,54 +236,190 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ------------------------------------------------------------------------- // Power-of-two NUM_THREADS arithmetic — all combinational, zero adders // ------------------------------------------------------------------------- - wire [NW_WIDTH:0] cta_num_warps; wire [NW_WIDTH:0] kmu_num_warps; wire [CTA_TID_WIDTH:0] block_size_next; - wire [CTA_TID_WIDTH:0] block_size_cur; wire [`VX_CFG_NUM_THREADS-1:0] partial_tmask; - - // block_size_r retires the previously latched warp's threads a cycle late, so - // the count left for the warp being latched now excludes them. - assign block_size_cur = warp_fire_r ? block_size_next : block_size_r; + wire part_warp_r; if (NT_BITS > 0) begin : g_nt_nonzero // Ceiling division block_size / NUM_THREADS: upper bits + OR of lower bits. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|block_size_r[NT_BITS-1:0]); // From KMU data at accept time (used to initialise table + cta_size output) assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size[CTA_TID_WIDTH:NT_BITS]) + (NW_WIDTH+1)'(|eff_block_size[NT_BITS-1:0]); - // Shared block_size decrement: low NT_BITS bits unchanged; upper bits decrement by 1. - assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - 1'b1, block_size_r[NT_BITS-1:0]}; - // Partial-warp mask: (1 << count) - 1 where count = block_size_cur[NT_BITS-1:0] - assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_cur[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); + // Shared block_size decrement: low NT_BITS bits unchanged; upper bits drop by + // the warps launched. + assign block_size_next = {block_size_r[CTA_TID_WIDTH:NT_BITS] - (CTA_TID_WIDTH+1-NT_BITS)'(nfired_r), block_size_r[NT_BITS-1:0]}; + // Partial-warp mask: (1 << count) - 1 where count = block_size_r[NT_BITS-1:0], + // which launches never change. + assign partial_tmask = (`VX_CFG_NUM_THREADS'(1) << block_size_r[NT_BITS-1:0]) - `VX_CFG_NUM_THREADS'(1); + assign part_warp_r = (| block_size_r[NT_BITS-1:0]); end else begin : g_nt_zero // NT_BITS=0: NUM_THREADS=1, each warp has exactly 1 thread, no partial warps. - assign cta_num_warps = (NW_WIDTH+1)'(block_size_r); assign kmu_num_warps = (NW_WIDTH+1)'(eff_block_size); - assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - 1'b1); + assign block_size_next = (CTA_TID_WIDTH+1)'(block_size_r - (CTA_TID_WIDTH+1)'(nfired_r)); assign partial_tmask = `VX_CFG_NUM_THREADS'(0); + assign part_warp_r = 1'b0; end - // Full-warp test: upper bits non-zero (no comparator) - wire is_full_warp = |block_size_cur[CTA_TID_WIDTH:NT_BITS]; - // The lanes this launch activates: full warp = all ones, partial = (1< FULL_W'(j)); + end else begin : g_full_never + assign full_r_gt[j] = 1'b0; + end + if (j == 0) begin : g_first + assign rem_r_gt[j] = full_r_gt[j] | part_warp_r; + end else begin : g_rest + assign rem_r_gt[j] = full_r_gt[j] | (part_warp_r & full_r_gt[j-1]); + end + end + wire [LCNT_WIDTH-1:0] nfired_cur = fire_any_r ? nfired_r : '0; + wire [NUM_LANES-1:0] full_gt = NUM_LANES'(full_r_gt >> nfired_cur); + wire [NUM_LANES-1:0] rem_gt = NUM_LANES'(rem_r_gt >> nfired_cur); + + // Launch selection: lanes with a free warp, taken in round-robin order from + // lane_rr_r, up to the warps left. A lane's order is its position among the + // selected lanes: the ready lanes ahead of it in the rotation, so a ready lane + // is selected while fewer than the warps left are ahead of it. + `STATIC_ASSERT (`IS_POW2(NUM_LANES), ("invalid parameter")) + reg [LANE_WIDTH-1:0] lane_rr_r; + wire launch_en; + wire [NUM_LANES-1:0] lane_sel; + wire [NUM_LANES-1:0][LANE_WIDTH-1:0] lane_order; + wire [LCNT_WIDTH-1:0] lane_cnt; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_sel + wire [NUM_LANES-1:0] ahead; + for (genvar m = 0; m < NUM_LANES; ++m) begin : g_ahead + assign ahead[m] = (LANE_WIDTH'(m) - lane_rr_r) < (LANE_WIDTH'(l) - lane_rr_r); + end + wire [NUM_LANES-1:0] ready_ahead = lane_ready & ahead; + wire [LCNT_WIDTH-1:0] order; + VX_popcount #( + .N (NUM_LANES) + ) order_count ( + .data_in (ready_ahead), + .data_out (order) + ); + assign lane_order[l] = LANE_WIDTH'(order); + assign lane_sel[l] = launch_en && lane_ready[l] && 1'(rem_gt >> order); + end + VX_popcount #( + .N (NUM_LANES) + ) lane_count ( + .data_in (lane_sel), + .data_out (lane_cnt) + ); - // Does slot w still owe the startup for any lane this launch activates? Resolved - // per slot, off the warp_id_n path, so the encoder still feeds only a 1-bit mux. - wire [`VX_CFG_NUM_WARPS-1:0] slot_needs_init; + // A warp is full unless it is the CTA's last and the block size is not a + // multiple of the warp width. + wire [NUM_LANES-1:0][`VX_CFG_NUM_THREADS-1:0] lane_tmask; + wire [NUM_LANES-1:0] lane_full; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_tmask + assign lane_full[l] = 1'(full_gt >> lane_order[l]); + assign lane_tmask[l] = lane_full[l] ? {`VX_CFG_NUM_THREADS{1'b1}} : partial_tmask; + end + + // Does slot w still owe the startup for any lane a full launch activates? A + // partial last warp re-runs the check on its own mask. Both answers are + // looked up per lane, and the lane's full/partial decision -- the deepest + // input -- only selects between them. + wire [`VX_CFG_NUM_WARPS-1:0] slot_owes_full, slot_owes_part; for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_needs_init - assign slot_needs_init[w] = |(launch_tmask & ~warp_init_lanes_r[w]); + assign slot_owes_full[w] = ~(&warp_init_lanes_r[w]); + assign slot_owes_part[w] = |(partial_tmask & ~warp_init_lanes_r[w]); + end + wire [NUM_LANES-1:0] lane_needs_init; + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lane_needs_init + assign lane_needs_init[l] = lane_full[l] ? slot_owes_full[lane_wid[l]] : slot_owes_part[lane_wid[l]]; end - // Last-warp test: ceiling(remaining/NUM_THREADS) == 1. - // Covers (upper==1, lower==0) full-last and (upper==0, lower!=0) partial-only cases. - wire is_last_warp = (cta_num_warps == (NW_WIDTH+1)'(1)); - // 3D thread-index carry-propagation (combinational on registered state) - wire [CTA_TID_WIDTH:0] next_x = {1'b0, thread_idx_r[0]} + {1'b0, warp_step_r[0]}; - wire wrap_x = (next_x >= block_dim_r[0]); - wire [CTA_TID_WIDTH:0] next_y = ({1'b0, thread_idx_r[1]} + {1'b0, warp_step_r[1]}) + (CTA_TID_WIDTH+1)'(wrap_x); - wire wrap_y = (next_y >= block_dim_r[1]); + // One thread-index advance by `step`, with a single wrap per dimension: every + // step is normalized below the block dimensions. Y and Z are formed for both + // carries in parallel, so the lower dimension's wrap only selects. + function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_advance ( + input logic [2:0][CTA_TID_WIDTH-1:0] base, + input logic [2:0][CTA_TID_WIDTH-1:0] step, + input logic [CTA_TID_WIDTH:0] bdim_x, + input logic [CTA_TID_WIDTH:0] bdim_y + ); + logic [CTA_TID_WIDTH:0] nx, ny0, ny1; + logic wx, wy0, wy1; + logic [CTA_TID_WIDTH-1:0] y0, y1, z0, z1; + nx = {1'b0, base[0]} + {1'b0, step[0]}; + wx = (nx >= bdim_x); + ny0 = {1'b0, base[1]} + {1'b0, step[1]}; + ny1 = {1'b0, base[1]} + {1'b0, step[1]} + (CTA_TID_WIDTH+1)'(1); + wy0 = (ny0 >= bdim_y); + wy1 = (ny1 >= bdim_y); + y0 = wy0 ? CTA_TID_WIDTH'(ny0 - bdim_y) : CTA_TID_WIDTH'(ny0); + y1 = wy1 ? CTA_TID_WIDTH'(ny1 - bdim_y) : CTA_TID_WIDTH'(ny1); + z0 = base[2] + step[2]; + z1 = base[2] + step[2] + CTA_TID_WIDTH'(1); + tid_advance[0] = wx ? CTA_TID_WIDTH'(nx - bdim_x) : CTA_TID_WIDTH'(nx); + tid_advance[1] = wx ? y1 : y0; + tid_advance[2] = (wx ? wy1 : wy0) ? z1 : z0; + endfunction + + // The window after the warps latched last cycle launch: entries shift down by + // nfired_r, and the ones shifted in are the entries they replace advanced by + // one group step. + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_grp; + wire [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_adv; + for (genvar k = 0; k < NUM_LANES; ++k) begin : g_tid_window + assign thread_idx_grp[k] = tid_advance(thread_idx_r[k], group_step_r, block_dim_r[0], block_dim_r[1]); + reg [2:0][CTA_TID_WIDTH-1:0] shifted; + always @(*) begin + shifted = thread_idx_r[k]; + for (integer n = 1; n <= NUM_LANES; ++n) begin + if (int'(nfired_r) == n) begin + shifted = ((k + n) < NUM_LANES) ? thread_idx_r[(k + n) % NUM_LANES] + : thread_idx_grp[(k + n) % NUM_LANES]; + end + end + end + assign thread_idx_adv[k] = shifted; + end + + // Kernel constants of the window: the base thread indices of ranks 0..NUM_LANES-1 + // and the NUM_LANES-warp step, built by stepping warp_step_r once per cycle + // whenever a launch's block shape differs from the last one's. + reg [NUM_LANES-1:0][2:0][CTA_TID_WIDTH-1:0] thread_idx_init_r; + reg [2:0][CTA_TID_WIDTH-1:0] shape_step_r; + reg [2:0][CTA_TID_WIDTH:0] shape_dim_r; + reg [LCNT_WIDTH-1:0] init_cnt_r; + reg [2:0][CTA_TID_WIDTH-1:0] init_acc_r; + wire init_busy = (init_cnt_r != '0); + wire [2:0][CTA_TID_WIDTH-1:0] init_next = tid_advance(init_acc_r, warp_step_r, block_dim_r[0], block_dim_r[1]); + wire shape_new = (NUM_LANES > 1) && !kmu_is_frag + && ((kmu_req.args.compute.warp_step != shape_step_r) + || (kmu_req.args.compute.block_dim != shape_dim_r)); + + // The window: a launch header reloads it (cleared for a new shape), the + // window build writes one entry per cycle, and launches shift it. No warp + // launches while the window is built, so the three writes are exclusive and + // the shift, the deepest input, takes the last select. + `RUNTIME_ASSERT(reset || ~(init_busy && fire_any_r), ("%t: %s window shifted while being built", $time, INSTANCE_ID)) + always @(posedge clk) begin + if (~reset) begin + for (integer k = 0; k < NUM_LANES; ++k) begin + if ((state == DISPATCH) && fire_any_r) begin + thread_idx_r[k] <= thread_idx_adv[k]; + end else if ((state == IDLE) && kmu_bus_if_fire) begin + thread_idx_r[k] <= shape_new ? '0 : thread_idx_init_r[k]; + end else if ((state == DISPATCH) && init_busy && (int'(init_cnt_r) == k)) begin + thread_idx_r[k] <= init_next; + end + end + end + end // ------------------------------------------------------------------------- // Retirement decode — pipeline: @@ -297,13 +456,13 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Within one kernel every CTA has the same aligned LMEM footprint, so LMEM // is partitioned into equal slots of pitch `stride = aligned_lmem_size`. // Resident CTA in slot i owns LMEM bytes [i*stride, (i+1)*stride). The - // occupancy bound `usable_slots_r = min(NUM_WARPS, floor(LMEM/stride))` is a + // occupancy bound `usable = min(NUM_WARPS, floor(LMEM/stride))` is a // kernel constant and is registered so the per-CTA admission/ready path sees // only a free-slot test, never the divide/encode. // // Cluster co-residency: the first CTA of a cluster (is_first_of_cluster) // reserves K = cluster_size CONSECUTIVE usable slots, pre-wrapping the slot - // window to 0 if it would overrun usable_slots_r. Members 2..K then take the + // window to 0 if it would overrun the usable range. Members 2..K then take the // following slots, so member r lands at issuer_base + r*stride — exactly what // DXA multicast resolves. Slots free immediately (out of order) on retire. @@ -315,34 +474,23 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( wire [LMEM_LOG:0] stride = kmu_req.aligned_lmem_size; wire is_first_of_cluster = eff_is_first; - reg [NW_WIDTH:0] usable_slots_r; - // Cluster member count K, capped at the slot count (a cluster larger than - // co-residency degenerates to a clamp). - wire [NW_WIDTH:0] cluster_k_raw = eff_cluster_size; - wire [NW_WIDTH:0] cluster_k = (cluster_k_raw > usable_slots_r) ? usable_slots_r - : (cluster_k_raw == 0) ? (NW_WIDTH+1)'(1) : cluster_k_raw; - - // Occupancy bound, registered (off the ready path). usable = largest m in - // [1, NUM_WARPS] with m*stride <= LMEM_SIZE; all slots when stride == 0. - // The product m*stride is a constant-times-variable (shift/add), so the - // NUM_WARPS-wide comparator tree has no divider. + // Occupancy bound as a thermometer, registered (off the ready path): + // usable_therm_r[m-1] is set iff m slots fit, i.e. m*stride <= LMEM_SIZE, + // tested as stride <= floor(LMEM_SIZE/m), a constant per m, so there is + // neither a multiplier nor a divider. All slots fit when stride == 0, and + // slot 0 always does. localparam PROD_W = LMEM_LOG + NW_WIDTH + 2; + reg [NUM_CTA_SLOTS-1:0] usable_therm_r; always_ff @(posedge clk) begin if (reset) begin - usable_slots_r <= (NW_WIDTH+1)'(NUM_CTA_SLOTS); + usable_therm_r <= '1; end else begin - if (stride == 0) begin - usable_slots_r <= (NW_WIDTH+1)'(NUM_CTA_SLOTS); - end else begin + for (integer m = 1; m <= NUM_CTA_SLOTS; m = m + 1) begin // Granting one slot to a CTA that exceeds capacity keeps the // launch making forward progress, at the cost of letting it // write past its slot. - usable_slots_r <= (NW_WIDTH+1)'(1); - for (integer m = 1; m <= NUM_CTA_SLOTS; m = m + 1) begin - if (PROD_W'(m) * PROD_W'(stride) <= PROD_W'(LMEM_SIZE)) begin - usable_slots_r <= (NW_WIDTH+1)'(m); - end - end + usable_therm_r[m-1] <= (m == 1) || (stride == 0) + || (PROD_W'(stride) <= PROD_W'(LMEM_SIZE / m)); end end end @@ -352,35 +500,47 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( `RUNTIME_ASSERT(~kmu_bus_if.valid || (stride == 0) || (PROD_W'(stride) <= PROD_W'(LMEM_SIZE)), ("%t: %s CTA shared-memory footprint %0d exceeds the %0d-byte capacity; the CTA overruns its slot", $time, INSTANCE_ID, stride, LMEM_SIZE)) - // Normalize the round-robin pointer to the usable range (covers a kernel - // transition that shrank usable_slots_r), then apply cluster pre-wrap. - // Compares are full (NW_WIDTH+2)-bit to admit usable_slots_r == NUM_CTA_SLOTS. - wire [NW_WIDTH+1:0] tail_ext = {2'b0, tail_r}; - wire [NW_WIDTH+1:0] usable_ext = {1'b0, usable_slots_r}; - wire [CS_BITS-1:0] base_tail = (tail_ext >= usable_ext) ? CS_BITS'(0) : tail_r; - wire cluster_prewrap = is_first_of_cluster - && (({2'b0, base_tail} + {1'b0, cluster_k}) > usable_ext); - wire [CS_BITS-1:0] base_slot = cluster_prewrap ? CS_BITS'(0) : base_tail; - - // Round-robin advance: next slot wraps to 0 at usable_slots_r. - wire [NW_WIDTH+1:0] next_tail_raw = {2'b0, base_slot} + (NW_WIDTH+2)'(1); - wire [CS_BITS-1:0] next_tail = (next_tail_raw >= usable_ext) - ? CS_BITS'(0) : next_tail_raw[CS_BITS-1:0]; - - // Slot LMEM base = base_slot * stride (small multiply, latched at accept; - // off the ready path). - wire [LMEM_LOG-1:0] base_lmem = (LMEM_LOG)'((LMEM_LOG+NW_WIDTH+1)'(base_slot) * (LMEM_LOG+NW_WIDTH+1)'(stride)); + // The slot base is either the round-robin tail or slot 0: the tail is + // normalized to 0 when a kernel transition shrank the usable range below it, + // and a cluster window that would overrun the usable range pre-wraps to 0. + // Every test is a bit of the registered maps shifted by the tail: + // tail_room[j] is set iff slot tail+j is usable. + wire [NUM_CTA_SLOTS:0] usable_room = {1'b0, usable_therm_r}; + wire [NUM_CTA_SLOTS:0] tail_room = usable_room >> tail_r; + wire [NUM_CTA_SLOTS-1:0] tail_busy = slot_valid_r >> tail_r; + if (NUM_CTA_SLOTS > 1) begin : g_room_top + `UNUSED_VAR (tail_room[NUM_CTA_SLOTS]) + end + + // Window of K = max(cluster_size, 1) slots. The fit test may take K unclamped: + // a K above the usable count never fits past slot 0, and at tail 0 both bases + // agree. The window mask is clamped to the usable count, as a cluster larger + // than co-residency degenerates to a clamp. + wire [NUM_CTA_SLOTS-1:0] cluster_therm; + for (genvar j = 0; j < NUM_CTA_SLOTS; ++j) begin : g_cluster_therm + assign cluster_therm[j] = (j == 0) || (eff_cluster_size > (NW_WIDTH+1)'(j)); + end + wire [NUM_CTA_SLOTS-1:0] window_ones = cluster_therm & usable_therm_r; + wire window_fits_tail = ((cluster_therm & ~tail_room[NUM_CTA_SLOTS-1:0]) == '0); + wire window_free_tail = ((tail_busy & window_ones) == '0); + wire window_free_zero = ((slot_valid_r & window_ones) == '0); + + wire tail_usable = tail_room[0]; + wire base_zero = is_first_of_cluster ? ~window_fits_tail : ~tail_usable; + wire [CS_BITS-1:0] base_slot = base_zero ? CS_BITS'(0) : tail_r; + + // Round-robin advance: next slot wraps to 0 at the usable count. + wire [CS_BITS-1:0] tail_inc = tail_r + CS_BITS'(1); + wire [CS_BITS-1:0] next_tail = base_zero ? CS_BITS'(usable_room[1]) + : (tail_room[1] ? tail_inc : CS_BITS'(0)); // Admission: a standalone CTA / following cluster member needs its single // slot free; a first-of-cluster needs all K window slots free. The window - // [base_slot, base_slot+K) never overruns NUM_CTA_SLOTS (pre-wrap ensures - // base_slot+K <= usable_slots_r <= NUM_CTA_SLOTS), so the mask is built at - // NUM_CTA_SLOTS width directly. K == NUM_CTA_SLOTS wraps to all-ones, which - // is the intended full mask. - wire [NUM_CTA_SLOTS-1:0] window_ones = (NUM_CTA_SLOTS'(1) << cluster_k) - NUM_CTA_SLOTS'(1); - wire [NUM_CTA_SLOTS-1:0] cluster_window = window_ones << base_slot; - wire cluster_window_free = ((slot_valid_r & cluster_window) == '0); - wire admit_ok = is_first_of_cluster ? cluster_window_free : ~slot_valid_r[base_slot]; + // [base_slot, base_slot+K) never overruns the usable range, so the mask is + // built at NUM_CTA_SLOTS width directly. + wire cluster_window_free = window_fits_tail ? window_free_tail : window_free_zero; + wire single_slot_free = tail_usable ? ~tail_busy[0] : ~slot_valid_r[0]; + wire admit_ok = is_first_of_cluster ? cluster_window_free : single_slot_free; // A fragment launch's stamps land in the warp record TID_STAGES cycles after the // warp fires (the thread-index ripple), but frag_stamps_r is a single bank. So a // new launch must not overwrite it while an earlier fragment warp is still in that @@ -411,14 +571,23 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin state <= IDLE; - warp_fire_r <= 0; + warp_fire_r <= '0; warp_id_r <= '0; warp_tmask_r <= '0; + warp_order_r <= '0; + nfired_r <= '0; + lane_rr_r <= '0; + init_cnt_r <= '0; + shape_step_r <= '0; + shape_dim_r <= '0; + thread_idx_init_r <= '0; + group_step_r <= '0; + init_acc_r <= '0; cur_ctx_id_r <= '0; warp_init_lanes_r <= '0; - warp_skip_init_r <= 0; + warp_skip_init_r <= '0; tail_r <= '0; - cur_lmem_base_r <= '0; + cur_stride_r <= '0; slot_valid_r <= '0; dispatched_warps<= '0; warp_done_r <= 0; @@ -447,8 +616,10 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // ---- wid → cta-slot map: latch on warp dispatch ---------------- // (Used internally by the retirement path to recover the slot // index of a finishing warp; no longer exposed externally.) - if ((state == DISPATCH) && warp_ready) begin - cta_slot_per_warp_r[warp_id_n] <= cur_slot_r; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (lane_sel[l]) begin + cta_slot_per_warp_r[lane_wid[l]] <= cur_slot_r; + end end // ---- Warp retirement ------------------------------------------- @@ -481,22 +652,32 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( entry_r <= kmu_req.entry; block_idx_r <= kmu_block_idx; block_dim_r <= kmu_req.args.compute.block_dim; + for (integer d = 0; d < 2; ++d) begin + block_last_r[d] <= CTA_TID_WIDTH'(kmu_req.args.compute.block_dim[d] - (CTA_TID_WIDTH+1)'(1)); + end grid_dim_r <= kmu_grid_dim; param_r <= kmu_req.param; block_size_r <= eff_block_size; warp_step_r <= kmu_req.args.compute.warp_step; cluster_size_r <= eff_cluster_size; cta_rank_r <= '0; - thread_idx_r <= '0; + group_step_r <= (NUM_LANES == 1) ? kmu_req.args.compute.warp_step : group_step_r; + // A new block shape rebuilds the window constants before launching. + if (shape_new) begin + shape_step_r <= kmu_req.args.compute.warp_step; + shape_dim_r <= kmu_req.args.compute.block_dim; + init_cnt_r <= LCNT_WIDTH'(1); + init_acc_r <= '0; + end // Fixed-stride placement: this CTA lands in slot base_slot // (the round-robin tail, pre-wrapped for a cluster window) // at LMEM base base_slot * stride. The round-robin pointer - // advances to next_tail (wraps at usable_slots_r). Slots + // advances to next_tail (wraps at the usable count). Slots // free out of order on retirement, so a cluster's window is // checked free up front (admit_ok) rather than reclaimed // in order. - cur_lmem_base_r <= base_lmem; + cur_stride_r <= stride; slot_valid_r[base_slot] <= 1'b1; tail_r <= (`VX_CFG_NUM_WARPS > 1) ? next_tail : CS_BITS'(0); cur_slot_r <= base_slot; @@ -506,30 +687,46 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end DISPATCH: begin - if (warp_ready) begin - warp_fire_r <= 1; - warp_id_r <= warp_id_n; - dispatched_warps[warp_id_n] <= 1'b1; - warp_tmask_r <= launch_tmask; - warp_skip_init_r <= ~slot_needs_init[warp_id_n]; - end else begin - warp_fire_r <= 0; + for (integer l = 0; l < NUM_LANES; ++l) begin + warp_fire_r[l] <= lane_sel[l]; + if (lane_sel[l]) begin + warp_id_r[l] <= lane_wid[l]; + warp_tmask_r[l] <= lane_tmask[l]; + warp_order_r[l] <= lane_order[l]; + warp_skip_init_r[l] <= ~lane_needs_init[l]; + dispatched_warps[lane_wid[l]] <= 1'b1; + end end + nfired_r <= lane_cnt; + lane_rr_r <= (NUM_LANES > 1) ? (lane_rr_r + LANE_WIDTH'(lane_cnt)) : '0; - if (warp_fire_r) begin - cta_rank_r <= cta_rank_r + NW_WIDTH'(1); - // Single shared adder result used for both decrement and last-warp test + if (fire_any_r) begin + cta_rank_r <= cta_rank_r + NW_WIDTH'(nfired_r); block_size_r <= block_size_next; + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + warp_init_lanes_r[warp_id_r[l]] <= warp_init_lanes_r[warp_id_r[l]] | warp_tmask_r[l]; + end + end + if (~rem_gt[0]) begin + state <= IDLE; + end + end - warp_init_lanes_r[warp_id_r] <= warp_init_lanes_r[warp_id_r] | warp_tmask_r; - thread_idx_r[0] <= wrap_x ? CTA_TID_WIDTH'(next_x - block_dim_r[0]) : CTA_TID_WIDTH'(next_x); - thread_idx_r[1] <= wrap_y ? CTA_TID_WIDTH'(next_y - block_dim_r[1]) : CTA_TID_WIDTH'(next_y); - thread_idx_r[2] <= thread_idx_r[2] + warp_step_r[2] + CTA_TID_WIDTH'(wrap_y); - - if (is_last_warp) begin - warp_fire_r <= 0; - state <= IDLE; + // window constants: step warp_step_r once per cycle + if (init_busy) begin + if (int'(init_cnt_r) < NUM_LANES) begin + for (integer k = 0; k < NUM_LANES; ++k) begin + if (int'(init_cnt_r) == k) begin + thread_idx_init_r[k] <= init_next; + end + end + init_cnt_r <= init_cnt_r + LCNT_WIDTH'(1); + end else begin + group_step_r <= init_next; + init_cnt_r <= '0; end + init_acc_r <= init_next; end end default:; @@ -537,6 +734,14 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( end end + always_ff @(posedge clk) begin + if (reset) begin + cur_lmem_base_r <= '0; + end else begin + cur_lmem_base_r <= (LMEM_LOG)'((LMEM_LOG+NW_WIDTH+1)'(cur_slot_r) * (LMEM_LOG+NW_WIDTH+1)'(cur_stride_r)); + end + end + // ------------------------------------------------------------------------- // Outputs — all driven combinationally from registered state // ------------------------------------------------------------------------- @@ -544,6 +749,8 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // and cta_param. cta_csrs_t cta_csrs; + assign launch_en = (state == DISPATCH) && !init_busy; + assign cta_fire = warp_fire_r; assign cta_wid = warp_id_r; assign cta_PC = warp_PC; @@ -572,215 +779,186 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r); assign cta_csrs.cluster_size = 32'(cluster_size_r); - // Busy from the accept cycle (kmu_bus_if_fire), not just while in DISPATCH: - // the accept->DISPATCH transition is registered, so gating on state alone - // leaves the accept cycle un-busy. With SOCKET_SIZE>1 the socket busy - // aggregation is registered, so on the final CTA kmu_busy drops before the - // buffered per-core busy rises, opening a 1-cycle device-busy gap the host's - // edge-sensitive idle-wait latches as premature completion (cores>1 launch - // failure). Covering the accept cycle closes the gap. - assign busy = (state == DISPATCH) || kmu_bus_if_fire; + // The accept cycle (kmu_bus_if_fire) is covered by the issuer of the launch, + // the socket, so busy has no path from this unit's inputs. Without it, a + // registered busy aggregation would leave a 1-cycle device-busy gap on the + // final CTA, which the host's edge-sensitive idle-wait latches as premature + // completion. + assign busy = (state == DISPATCH); // ------------------------------------------------------------------------- // CTA context storage + per-thread coordinate expansion // ------------------------------------------------------------------------- - // Per-CTA context (cta_ctx_ram) and per-warp residue (cta_warp_ram), written - // at launch and read back via csr_rd_wid / csr_rd_cta_id so CTA CSRs resolve - // in one cycle with no divider. BRAM-backed to avoid per-warp full-record flops. + // Per-CTA context and per-warp residue, broadcast at launch to the CTA-CSR + // stores, which resolve CTA CSRs in one cycle with no divider. - // Per-CTA context. + // Per-CTA context write port, shared by every lane. wire cta_ctx_write; wire [NCTA_WIDTH-1:0] cta_ctx_waddr; cta_ctx_t cta_ctx_wdata; - wire [NCTA_WIDTH-1:0] cta_ctx_raddr; - cta_ctx_t cta_ctx_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_ctx_t)), - .SIZE (NUM_CTA_MAX), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_ctx_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_ctx_write), - .wren (1'b1), - .waddr (cta_ctx_waddr), - .wdata (cta_ctx_wdata), - .raddr (cta_ctx_raddr), - .rdata (cta_ctx_rdata) - ); - - // Per-warp residue. - wire cta_warp_write; - wire [NW_WIDTH-1:0] cta_warp_waddr; - cta_warp_t cta_warp_wdata; - wire [NW_WIDTH-1:0] cta_warp_raddr; - cta_warp_t cta_warp_rdata; - - VX_dp_ram #( - .DATAW ($bits(cta_warp_t)), - .SIZE (`VX_CFG_NUM_WARPS), - .RDW_MODE ("R"), - .OUT_REG (1) - ) cta_warp_ram ( - .clk (clk), - .reset (reset), - .read (1'b1), - .write (cta_warp_write), - .wren (1'b1), - .waddr (cta_warp_waddr), - .wdata (cta_warp_wdata), - .raddr (cta_warp_raddr), - .rdata (cta_warp_rdata) - ); // Per-lane CTA thread coordinates, expanded divide-free from the warp base // (lane 0 = base; each later lane steps +1 along X with a single wrap into Y // then Z) and stored in cta_warp_ram so CTA_THREAD_ID reads cost one cycle // with no divider. The expansion is a serial ripple, so it is pipelined - // TID_STEP lanes/cycle to meet timing. CTA dispatch is infrequent and + // TID_STEP lanes/cycle to meet timing; each launch lane has its own pipeline. // cta_warp_ram is read many cycles after a warp launches (fetch/decode/issue // latency >> TID_STAGES), so the added write latency is hidden. - wire [2:0][CTA_TID_WIDTH-1:0] cta_base_tid = thread_idx_r; - - localparam TID_STEP = 2; + // one step per stage once the ripple across many threads is the long path + localparam TID_STEP = (`VX_CFG_NUM_THREADS > 8) ? 1 : 2; localparam TID_STAGES = (`VX_CFG_NUM_THREADS - 1 + TID_STEP - 1) / TID_STEP; function automatic logic [2:0][CTA_TID_WIDTH-1:0] tid_next ( input logic [2:0][CTA_TID_WIDTH-1:0] prev, - input logic [CTA_TID_WIDTH:0] bd_x, - input logic [CTA_TID_WIDTH:0] bd_y + input logic [CTA_TID_WIDTH-1:0] last_x, + input logic [CTA_TID_WIDTH-1:0] last_y ); - logic [CTA_TID_WIDTH:0] nx, ny; + // prev is a reduced coordinate (prev[0] <= last_x, prev[1] <= last_y: + // the warp bases come from tid_advance), so a +1 step wraps exactly + // when it leaves the last index, and wraps to 0. The wraps are equality + // tests on prev itself, beside the increments rather than after them. logic wx, wy; - nx = {1'b0, prev[0]} + (CTA_TID_WIDTH+1)'(1); - wx = (nx >= bd_x); - ny = {1'b0, prev[1]} + (CTA_TID_WIDTH+1)'(wx); - wy = wx && (ny >= bd_y); - tid_next[0] = wx ? CTA_TID_WIDTH'(nx - bd_x) : CTA_TID_WIDTH'(nx); - tid_next[1] = wy ? CTA_TID_WIDTH'(ny - bd_y) : CTA_TID_WIDTH'(ny); - tid_next[2] = prev[2] + CTA_TID_WIDTH'(wy); + wx = (prev[0] == last_x); + wy = wx && (prev[1] == last_y); + tid_next[0] = wx ? '0 : (prev[0] + CTA_TID_WIDTH'(1)); + tid_next[1] = wy ? '0 : (wx ? (prev[1] + CTA_TID_WIDTH'(1)) : prev[1]); + tid_next[2] = wy ? (prev[2] + CTA_TID_WIDTH'(1)) : prev[2]; endfunction typedef logic [`VX_CFG_NUM_THREADS-1:0][2:0][CTA_TID_WIDTH-1:0] cta_tid_arr_t; - cta_tid_arr_t tidp_tid [TID_STAGES+1]; - wire [TID_STAGES:0] tidp_valid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; - wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; - // block_dim spans 1..NUM_WARPS*NUM_THREADS inclusive, one bit more than a - // thread index -- truncating it to CTA_TID_WIDTH turns a full-width dimension - // into 0 and makes every lane wrap. - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdx; - wire [TID_STAGES:0][CTA_TID_WIDTH:0] tidp_bdy; - - // stage 0: combinational inputs; seed lane 0 with the warp base. - cta_tid_arr_t tidp_tid0; - always @(*) begin - tidp_tid0 = '0; - tidp_tid0[0] = cta_base_tid; - end - assign tidp_tid[0] = tidp_tid0; - assign tidp_valid[0] = cta_fire; - assign tidp_wid[0] = cta_wid; - assign tidp_rank[0] = cta_csrs.cta_rank; - assign tidp_bdx[0] = cta_csrs.block_dim[0]; - assign tidp_bdy[0] = cta_csrs.block_dim[1]; - - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe - cta_tid_arr_t nxt; + wire [NUM_LANES-1:0] lane_frag_inflight; + + for (genvar l = 0; l < NUM_LANES; ++l) begin : g_lanes + cta_tid_arr_t tidp_tid [TID_STAGES+1]; + wire [TID_STAGES:0] tidp_valid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_wid; + wire [TID_STAGES:0][NW_WIDTH-1:0] tidp_rank; + // The last index of each dimension, block_dim - 1: block_dim spans + // 1..NUM_WARPS*NUM_THREADS, so its last index is a thread index. + wire [TID_STAGES:0][CTA_TID_WIDTH-1:0] tidp_last_x; + wire [TID_STAGES:0][CTA_TID_WIDTH-1:0] tidp_last_y; + + // stage 0: combinational inputs; seed lane 0 with the warp base. + cta_tid_arr_t tidp_tid0; always @(*) begin - nxt = tidp_tid[s-1]; - for (integer k = 0; k < TID_STEP; k = k + 1) begin - if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin - nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_bdx[s-1], tidp_bdy[s-1]); + tidp_tid0 = '0; + tidp_tid0[0] = thread_idx_r[warp_order_r[l]]; + end + assign tidp_tid[0] = tidp_tid0; + assign tidp_valid[0] = warp_fire_r[l]; + assign tidp_wid[0] = warp_id_r[l]; + assign tidp_rank[0] = cta_csrs.cta_rank + NW_WIDTH'(warp_order_r[l]); + assign tidp_last_x[0] = block_last_r[0]; + assign tidp_last_y[0] = block_last_r[1]; + + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_tid_pipe + cta_tid_arr_t nxt; + always @(*) begin + nxt = tidp_tid[s-1]; + for (integer k = 0; k < TID_STEP; k = k + 1) begin + if (((s-1)*TID_STEP + k + 1) < `VX_CFG_NUM_THREADS) begin + nxt[(s-1)*TID_STEP + k + 1] = tid_next(nxt[(s-1)*TID_STEP + k], tidp_last_x[s-1], tidp_last_y[s-1]); + end end end - end - reg v_r; - reg [NW_WIDTH-1:0] wid_r, rank_r; - reg [CTA_TID_WIDTH:0] bdx_r, bdy_r; - cta_tid_arr_t tid_r; - always @(posedge clk) begin - if (reset) begin - v_r <= 1'b0; - end else begin - v_r <= tidp_valid[s-1]; + reg v_r; + reg [NW_WIDTH-1:0] wid_r, rank_r; + reg [CTA_TID_WIDTH-1:0] last_x_r, last_y_r; + cta_tid_arr_t tid_r; + always @(posedge clk) begin + if (reset) begin + v_r <= 1'b0; + end else begin + v_r <= tidp_valid[s-1]; + end + tid_r <= nxt; + wid_r <= tidp_wid[s-1]; + rank_r <= tidp_rank[s-1]; + last_x_r <= tidp_last_x[s-1]; + last_y_r <= tidp_last_y[s-1]; end - tid_r <= nxt; - wid_r <= tidp_wid[s-1]; - rank_r <= tidp_rank[s-1]; - bdx_r <= tidp_bdx[s-1]; - bdy_r <= tidp_bdy[s-1]; + assign tidp_tid[s] = tid_r; + assign tidp_valid[s] = v_r; + assign tidp_wid[s] = wid_r; + assign tidp_rank[s] = rank_r; + assign tidp_last_x[s] = last_x_r; + assign tidp_last_y[s] = last_y_r; end - assign tidp_tid[s] = tid_r; - assign tidp_valid[s] = v_r; - assign tidp_wid[s] = wid_r; - assign tidp_rank[s] = rank_r; - assign tidp_bdx[s] = bdx_r; - assign tidp_bdy[s] = bdy_r; - end -`ifdef VX_CFG_EXT_RASTER_ENABLE - // Which in-flight warps came from a fragment launch. One bit per stage rather - // than the stamps themselves: the write side needs to know WHICH view of the - // lane record to land, and the header side needs to know whether frag_stamps_r - // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 - // combinational, the rest registered -- so the bit and the warp it describes - // stay in step. - wire [TID_STAGES:0] tidp_frag; - assign tidp_frag[0] = cta_fire && is_frag_r; - for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe - reg f_r; - always @(posedge clk) begin - if (reset) begin - f_r <= 1'b0; - end else begin - f_r <= tidp_frag[s-1]; + wire is_frag_warp; + `ifdef VX_CFG_EXT_RASTER_ENABLE + // Which in-flight warps came from a fragment launch. One bit per stage rather + // than the stamps themselves: the write side needs to know WHICH view of the + // lane record to land, and the header side needs to know whether frag_stamps_r + // is still owed to someone. Shaped exactly like tidp_valid -- stage 0 + // combinational, the rest registered -- so the bit and the warp it describes + // stay in step. + wire [TID_STAGES:0] tidp_frag; + assign tidp_frag[0] = warp_fire_r[l] && is_frag_r; + for (genvar s = 1; s <= TID_STAGES; ++s) begin : g_frag_pipe + reg f_r; + always @(posedge clk) begin + if (reset) begin + f_r <= 1'b0; + end else begin + f_r <= tidp_frag[s-1]; + end end + assign tidp_frag[s] = f_r; + end + assign lane_frag_inflight[l] = |tidp_frag; + assign is_frag_warp = tidp_frag[TID_STAGES]; + `else + assign lane_frag_inflight[l] = 1'b0; + assign is_frag_warp = 1'b0; + // its only consumer (the lane-launch view mux) is RASTER-only + `UNUSED_VAR (is_frag_warp) + `endif + + // last_x/last_y are consumed only by stages still rippling; the final + // stage's forwarded copies are unused. + `UNUSED_VAR (tidp_last_x[TID_STAGES]) + `UNUSED_VAR (tidp_last_y[TID_STAGES]) + + cta_warp_t cta_warp_wdata; + assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; + + // The lane record is an overlay: a fragment launch lands its stamp slice, and a + // compute launch lands the expanded thread index. Which of the two applies is a + // property of the WARP being written, not of whatever launch the header stage + // happens to be holding now, so it rides the pipeline with it. + for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch + cta_lane_t lane_compute; + assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); + `ifdef VX_CFG_EXT_RASTER_ENABLE + cta_lane_t lane_fragment; + assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); + assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; + `else + assign cta_warp_wdata.lane_launch[i] = lane_compute; + `endif end - assign tidp_frag[s] = f_r; + + assign cta_lane_bcast[l].warp_write = tidp_valid[TID_STAGES]; + assign cta_lane_bcast[l].warp_waddr = tidp_wid[TID_STAGES]; + assign cta_lane_bcast[l].warp_wdata = cta_warp_wdata; + assign cta_lane_bcast[l].id_write = warp_fire_r[l]; + assign cta_lane_bcast[l].id_wid = warp_id_r[l]; + assign cta_lane_bcast[l].id_cta = cta_csrs.cta_id; end - assign frag_inflight = |tidp_frag; - assign is_frag_warp = tidp_frag[TID_STAGES]; + +`ifdef VX_CFG_EXT_RASTER_ENABLE + assign frag_inflight = |lane_frag_inflight; assign frag_stamps_sel = frag_stamps_r; `else assign frag_inflight = 1'b0; - assign is_frag_warp = 1'b0; assign frag_stamps_sel = '0; - // their only consumer (the lane-launch view mux) is RASTER-only - `UNUSED_VAR (is_frag_warp) + `UNUSED_VAR (lane_frag_inflight) `UNUSED_VAR (frag_stamps_sel) `endif - // block_dim is consumed only by stages still rippling; the final stage's - // forwarded copy is unused. - `UNUSED_VAR (tidp_bdx[TID_STAGES]) - `UNUSED_VAR (tidp_bdy[TID_STAGES]) - - assign cta_warp_write = tidp_valid[TID_STAGES]; - assign cta_warp_waddr = tidp_wid[TID_STAGES]; - assign cta_warp_wdata.cta_rank = tidp_rank[TID_STAGES]; - - // The lane record is an overlay: a fragment launch lands its stamp slice, and a - // compute launch lands the expanded thread index. Which of the two applies is a - // property of the WARP being written, not of whatever launch the header stage - // happens to be holding now, so it rides the pipeline with it. - for (genvar i = 0; i < `VX_CFG_NUM_THREADS; ++i) begin : g_lane_launch - cta_lane_t lane_compute; - assign lane_compute = LANE_LAUNCH_BITS'(tidp_tid[TID_STAGES][i]); - `ifdef VX_CFG_EXT_RASTER_ENABLE - cta_lane_t lane_fragment; - assign lane_fragment = LANE_LAUNCH_BITS'(frag_stamps_sel[i]); - assign cta_warp_wdata.lane_launch[i] = is_frag_warp ? lane_fragment : lane_compute; - `else - assign cta_warp_wdata.lane_launch[i] = lane_compute; - `endif - end - - assign cta_ctx_write = cta_fire; + assign cta_ctx_write = fire_any_r; assign cta_ctx_waddr = cta_csrs.cta_id; assign cta_ctx_wdata.cta_size = cta_csrs.cta_size; assign cta_ctx_wdata.block_idx = cta_csrs.block_idx; @@ -791,32 +969,9 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( assign cta_ctx_wdata.cluster_size = cta_csrs.cluster_size; assign cta_ctx_wdata.entry = cta_csrs.entry; - // rdata is valid one cycle after raddr; csr_unit holds its read address - // stable for that cycle. - assign cta_warp_raddr = csr_rd_wid; - assign cta_ctx_raddr = csr_rd_cta_id; - - assign cta_rd_csrs.cta_id = csr_rd_cta_id; - assign cta_rd_csrs.cta_rank = cta_warp_rdata.cta_rank; - - assign cta_rd_lane = cta_warp_rdata.lane_launch; - assign cta_rd_csrs.cta_size = cta_ctx_rdata.cta_size; - assign cta_rd_csrs.block_idx = cta_ctx_rdata.block_idx; - assign cta_rd_csrs.block_dim = cta_ctx_rdata.block_dim; - assign cta_rd_csrs.grid_dim = cta_ctx_rdata.grid_dim; - assign cta_rd_csrs.param = cta_ctx_rdata.param; - assign cta_rd_csrs.lmem_addr = cta_ctx_rdata.lmem_addr; - assign cta_rd_csrs.cluster_size = cta_ctx_rdata.cluster_size; - assign cta_rd_csrs.entry = cta_ctx_rdata.entry; - - // Per-warp -> CTA-id map, latched on launch and indexed by schedule_wid. - reg [`VX_CFG_NUM_WARPS-1:0][NCTA_WIDTH-1:0] cta_id_per_warp_r; - always @(posedge clk) begin - if (cta_fire) begin - cta_id_per_warp_r[cta_wid] <= cta_csrs.cta_id; - end - end - assign schedule_cta_id = cta_id_per_warp_r[schedule_wid]; + assign cta_bcast.ctx_write = cta_ctx_write; + assign cta_bcast.ctx_waddr = cta_ctx_waddr; + assign cta_bcast.ctx_wdata = cta_ctx_wdata; `ifdef DBG_TRACE_PIPELINE // Pipeline warp_done_wid alongside the retirement chain for trace logging. @@ -838,15 +993,17 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( `TRACE(1, ("%t: %s kmu-accept: cta_id=%0d, PC=0x%0h, param=0x%0h, stride=%0d, num_warps=%0d, usable_slots=%0d\n", $time, INSTANCE_ID, base_slot, to_fullPC(kmu_req.PC), kmu_req.param, - stride, kmu_num_warps, usable_slots_r)) + stride, kmu_num_warps, $countones(usable_therm_r))) end - // Warp dispatched to scheduler - if (warp_fire_r) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", - $time, INSTANCE_ID, warp_id_r, cur_slot_r, to_fullPC(warp_PC), - warp_tmask_r, param_r, - (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), - ~warp_skip_init_r)) + // Warps dispatched to the schedulers + for (integer l = 0; l < NUM_LANES; ++l) begin + if (warp_fire_r[l]) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, rank=%0d, PC=0x%0h, tmask=%b, param=0x%0h, lmem_addr=0x%0h, init=%b\n", + $time, INSTANCE_ID, warp_id_r[l], cur_slot_r, cta_rank_r + NW_WIDTH'(warp_order_r[l]), to_fullPC(warp_PC), + warp_tmask_r[l], param_r, + (`VX_CFG_MEM_ADDR_WIDTH'(`VX_MEM_LMEM_BASE_ADDR) | `VX_CFG_MEM_ADDR_WIDTH'(cur_lmem_base_r)), + ~warp_skip_init_r[l])) + end end // Warp retirement / CTA done if (warp_done_r_dly && slot_valid_r[done_slot_r_dly]) begin @@ -857,7 +1014,7 @@ module VX_cta_dispatch import VX_gpu_pkg::*; #( // Admission gate status when KMU presents a CTA but is stalled if (kmu_bus_if.valid && !kmu_bus_if.ready && state == IDLE) begin `TRACE(4, ("%t: %s stall: admit_ok=%b, base_slot=%0d, stride=%0d, usable_slots=%0d\n", - $time, INSTANCE_ID, admit_ok, base_slot, stride, usable_slots_r)) + $time, INSTANCE_ID, admit_ok, base_slot, stride, $countones(usable_therm_r))) end end `endif diff --git a/hw/rtl/core/VX_dcr_data.sv b/hw/rtl/core/VX_dcr_data.sv index 9c10d06f9b..c835d176a5 100644 --- a/hw/rtl/core/VX_dcr_data.sv +++ b/hw/rtl/core/VX_dcr_data.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dcr_data import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dcr_bus_if.slave dcr_bus_if, @@ -27,7 +28,8 @@ module VX_dcr_data import VX_gpu_pkg::*; #( VX_dcr_csr_if.master dcr_csr_if, // DCR-triggered cache flush - VX_dcr_flush_if.master dcr_flush_if + VX_dcr_flush_if.master dcr_flush_if, + output wire flush_l1_only ); `UNUSED_SPARAM (INSTANCE_ID) @@ -49,7 +51,7 @@ module VX_dcr_data import VX_gpu_pkg::*; #( wire is_mpm_read = dcr_bus_if.req_valid && ~dcr_bus_if.req_data.rw && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_MPM_VALUE) - && (mpm_target_cid == 16'(CORE_ID)) + && (mpm_target_cid == 16'(core_id)) && ~dcr_csr_pending_r; always @(posedge clk) begin @@ -74,27 +76,45 @@ module VX_dcr_data import VX_gpu_pkg::*; #( // Cache flush via DCR interface ///////////////////////////////////////// - reg flush_pending_r; - - wire is_flush_read = dcr_bus_if.req_valid - && ~dcr_bus_if.req_data.rw - && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH) - && (mpm_target_cid == 16'(CORE_ID)) - && ~flush_pending_r; + // A flush starts on a DCR write to CACHE_FLUSH, which reaches every core, + // and this core's flush read is answered once its flush has completed. The + // DCR response bus has no back-pressure, so the command processor starts + // all cores with one write and then reads them one at a time: the flushes + // overlap and the responses never collide. + reg flush_pending_r; // flush running + reg flush_complete_r; // flush finished, not yet reported + reg flush_read_r; // flush read waiting for completion + reg flush_l1_only_r; // the last-level cache keeps its lines + + wire is_flush_req = dcr_bus_if.req_valid + && (dcr_bus_if.req_data.addr == `VX_DCR_BASE_CACHE_FLUSH); + wire is_flush_write = is_flush_req && dcr_bus_if.req_data.rw; + wire is_flush_read = is_flush_req && ~dcr_bus_if.req_data.rw + && (mpm_target_cid == 16'(core_id)); + + wire flush_done = flush_pending_r && dcr_flush_if.done; + wire flush_start = ~flush_pending_r && is_flush_write; + wire flush_rsp = flush_read_r && (flush_complete_r || flush_done); always @(posedge clk) begin if (reset) begin - flush_pending_r <= 1'b0; - end else if (is_flush_read) begin - flush_pending_r <= 1'b1; - end else if (dcr_flush_if.done) begin - flush_pending_r <= 1'b0; + flush_pending_r <= 1'b0; + flush_complete_r <= 1'b0; + flush_read_r <= 1'b0; + flush_l1_only_r <= 1'b0; + end else begin + flush_pending_r <= flush_start || (flush_pending_r && ~flush_done); + // a write opens a new flush, so an older completion cannot answer it + flush_complete_r <= ~is_flush_write && ~flush_rsp && (flush_complete_r || flush_done); + flush_read_r <= ~flush_rsp && (flush_read_r || is_flush_read); + if (flush_start) begin + flush_l1_only_r <= (dcr_bus_if.req_data.data & VX_DCR_DATA_WIDTH'(`VX_CACHE_FLUSH_L1_ONLY)) != 0; + end end end assign dcr_flush_if.req = flush_pending_r; - - wire flush_done = flush_pending_r && dcr_flush_if.done; + assign flush_l1_only = flush_l1_only_r; // Return CSR data or flush-done on DCR response bus // Use intermediate registers — Vivado (Synth 8-453) forbids procedural @@ -107,10 +127,10 @@ module VX_dcr_data import VX_gpu_pkg::*; #( rsp_valid_r <= 1'b0; rsp_data_r <= '0; end else begin - rsp_valid_r <= dcr_csr_if_fire || flush_done; + rsp_valid_r <= dcr_csr_if_fire || flush_rsp; if (dcr_csr_if_fire) begin rsp_data_r <= dcr_csr_if.value; - end else if (flush_done) begin + end else if (flush_rsp) begin rsp_data_r <= '0; end end @@ -119,6 +139,12 @@ module VX_dcr_data import VX_gpu_pkg::*; #( assign dcr_bus_if.rsp_valid = rsp_valid_r; assign dcr_bus_if.rsp_data = rsp_data_r; + // the requester issues one read at a time, so the two answers never meet + `RUNTIME_ASSERT (~(dcr_csr_if_fire && flush_rsp), ("%t: %s CSR read and flush answered in the same cycle", $time, INSTANCE_ID)) + // a new flush starts only after every core reported the previous one + `RUNTIME_ASSERT (~(is_flush_write && flush_pending_r), ("%t: %s flush started while one is running", $time, INSTANCE_ID)) + `RUNTIME_ASSERT (~(is_flush_read && ~flush_pending_r && ~flush_complete_r), ("%t: %s flush read with no flush started", $time, INSTANCE_ID)) + `ifdef DBG_TRACE_PIPELINE always @(posedge clk) begin if (dcr_bus_if.req_valid && dcr_bus_if.req_data.rw) begin diff --git a/hw/rtl/core/VX_dcr_flush.sv b/hw/rtl/core/VX_dcr_flush.sv index 7617f23b46..8bf7ec4883 100644 --- a/hw/rtl/core/VX_dcr_flush.sv +++ b/hw/rtl/core/VX_dcr_flush.sv @@ -33,6 +33,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( input wire reset, VX_dcr_flush_if.slave dcr_flush_if, + input wire l1_only, // the last-level cache need not walk VX_mem_bus_if.slave core_bus_if, // TAG_WIDTH VX_mem_bus_if.master cache_bus_if // TAG_WIDTH+1 ); @@ -72,12 +73,13 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( rw: 1'b0, addr: '0, data: '0, - byteen: '0, + byteen: WORD_SIZE'(l1_only), attr: MEM_ATTR_WIDTH'(1 << MEM_ATTR_FLUSH_OFFS), tag: '0 }; - // The flush request only consumes MEM_ATTR_FLUSH_OFFS downstream; the - // remaining req_data fields are tied off and intentionally unused. + // The flush request only consumes MEM_ATTR_FLUSH_OFFS and byteen bit 0 + // (the L1-only kind) downstream; the remaining req_data fields are tied + // off and intentionally unused. `UNUSED_VAR (flush_bus_if.req_data.rw) `UNUSED_VAR (flush_bus_if.req_data.addr) `UNUSED_VAR (flush_bus_if.req_data.data) @@ -87,7 +89,7 @@ module VX_dcr_flush import VX_gpu_pkg::*; #( assign dcr_flush_if.done = flush_done_r; assign flush_bus_if.rsp_ready = 1'b1; - // 2-to-1 arb: input 0 = flush, input 1 = real dcache port 0 + // 2-to-1 arb: input 0 = cache traffic, input 1 = the flush token VX_mem_bus_if #( .DATA_SIZE (WORD_SIZE), .TAG_WIDTH (TAG_WIDTH) diff --git a/hw/rtl/core/VX_decode.sv b/hw/rtl/core/VX_decode.sv index 12a59ca372..7e2e746664 100644 --- a/hw/rtl/core/VX_decode.sv +++ b/hw/rtl/core/VX_decode.sv @@ -231,6 +231,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = `SEXT(20, i_imm); `USED_IREG (rd); `USED_IREG (rs1); @@ -240,6 +242,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 0; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; `USED_IREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); @@ -273,6 +277,8 @@ module VX_decode import op_args.alu.is_w = 1; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = `SEXT(20, iw_imm); `USED_IREG (rd); `USED_IREG (rs1); @@ -282,6 +288,8 @@ module VX_decode import op_args.alu.is_w = 1; op_args.alu.use_PC = 0; op_args.alu.use_imm = 0; + op_args.alu.imm_hi = 0; + op_args.alu.imm_x2 = 0; `USED_IREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); @@ -308,6 +316,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 0; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 1; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = ui_imm; `USED_IREG (rd); end @@ -318,6 +328,8 @@ module VX_decode import op_args.alu.is_w = 0; op_args.alu.use_PC = 1; op_args.alu.use_imm = 1; + op_args.alu.imm_hi = 1; + op_args.alu.imm_x2 = 0; op_args.alu.imm20 = ui_imm; `USED_IREG (rd); end @@ -327,6 +339,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 1; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 1; op_args.br.imm20 = jal_imm; op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -338,6 +352,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 0; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 0; op_args.br.imm20 = `SEXT(20, u_12); op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -350,6 +366,8 @@ module VX_decode import op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_PC = 1; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 1; op_args.br.imm20 = `SEXT(20, b_imm); op_args.br.is_rvc = decode_is_rvc; is_wstall = 1; @@ -358,11 +376,21 @@ module VX_decode import end INST_FENCE: begin ex_type = EX_LSU; - op_type = INST_LSU_FENCE; + // neither the predecessor nor the successor set holds a memory + // read or write (pred = instr[27:24], succ = instr[23:20], IORW) + op_type = ((funct3 == 3'b000) && (instr[25:24] == 2'b00) && (instr[21:20] == 2'b00)) + ? INST_LSU_FENCE_IO : INST_LSU_FENCE; op_args.lsu.is_store = 0; op_args.lsu.is_float = 0; op_args.lsu.pack = 0; op_args.lsu.offset = 0; + // A later read must see other harts' data (acquire) when the + // successor set reads; earlier writes must be visible (release) + // when the predecessor set writes. A fence with neither + // releases; one with a device access in its predecessor set, + // or fence.i, does both. + op_args.lsu.amo_aq = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[21]; + op_args.lsu.amo_rl = (funct3 != 3'b000) || (instr[27:26] != 2'b00) || instr[24] || ~instr[21]; end `ifdef VX_CFG_EXT_A_ENABLE INST_AMO: begin @@ -417,6 +445,8 @@ module VX_decode import op_type = INST_OP_BITS'(s_type); op_args.br.xtype = ALU_TYPE_BRANCH; op_args.br.use_imm= 1; + op_args.br.imm_hi = 0; + op_args.br.imm_x2 = 0; op_args.br.use_PC = 1; op_args.br.imm20 = 20'd4; op_args.br.is_rvc = decode_is_rvc; @@ -458,6 +488,72 @@ module VX_decode import `USED_IREG (rs2); `endif end + `ifdef VX_CFG_EXT_F_ENABLE + INST_EXT4: begin + case (funct3) + `ifdef VX_CFG_EXT_TCU_ENABLE + // vx_ldm/vx_stm: a TCU matrix fragment, expanded into one flw/fsw + // per register. imm[4:0] is the fragment shape, replaced by its + // expander code; a bytesel offset of 2 (4 registers) or 3 (8) + // marks the access as a fragment. + 3'h0: begin + if (VX_tcu_pkg::tcu_ldm_valid(u_12[4:0])) begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_LW); + op_args.lsu.is_store = 0; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = {u_12[11:5], VX_tcu_pkg::tcu_ldm_code(u_12[4:0])}; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({1'b1, VX_tcu_pkg::tcu_ldm_is8(u_12[4:0])})}; + `USED_IREG (rs1); + `USED_FREG (rd); + end + end + 3'h2: begin + if (VX_tcu_pkg::tcu_ldm_valid(rd)) begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_SW); + op_args.lsu.is_store = 1; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = {s_imm[11:5], VX_tcu_pkg::tcu_ldm_code(rd)}; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'({1'b1, VX_tcu_pkg::tcu_ldm_is8(rd)})}; + `USED_IREG (rs1); + `USED_FREG (rs2); + end + end + `endif + `ifndef VX_CFG_FLEN_64 + // vx_flw.d/.q and vx_fsw.d/.q: 2 or 4 words of an aligned FP register + // group, expanded into flw/fsw uops; the bytesel offset carries the + // group size. + 3'h3, 3'h4: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_LW); + op_args.lsu.is_store = 0; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = u_12; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(!funct3[0])}; + `USED_IREG (rs1); + `USED_FREG (rd); + end + 3'h5, 3'h6: begin + ex_type = EX_LSU; + op_type = INST_OP_BITS'(INST_LSU_SW); + op_args.lsu.is_store = 1; + op_args.lsu.is_float = 1; + op_args.lsu.pack = 2'b11; + op_args.lsu.offset = s_imm; + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(!funct3[0])}; + `USED_IREG (rs1); + `USED_FREG (rs2); + end + `endif + default:; + endcase + end + `endif `ifdef VX_CFG_EXT_F_ENABLE INST_FMADD, // 7'b1000011 INST_FMSUB, // 7'b1000111 @@ -767,6 +863,9 @@ module VX_decode import op_args.lsu.is_float = 1; op_args.lsu.pack = 2'b01; op_args.lsu.offset = '0; + // the bytesel offset of a pack/wide/fragment access + // carries its uop count: 0 → 2, 1 or 2 → 4, 3 → 8 + bytesel = {XLENB_W'(XLENB-1), XLENB_W'(1)}; `USED_FREG (rd); `USED_IREG (rs1); `USED_IREG (rs2); @@ -982,7 +1081,7 @@ module VX_decode import `TRACE(1, ("%t: %s decode: wid=%0d, cta_id=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, decode_if.data.wid, decode_if.data.cta_id, to_fullPC(decode_if.data.PC))) VX_trace_pkg::trace_ex_type(1, decode_if.data.ex_type); `TRACE(1, (", op=")) - VX_trace_pkg::trace_ex_op(1, decode_if.data.ex_type, decode_if.data.op_type, decode_if.data.op_args); + VX_trace_pkg::trace_ex_op(1, decode_if.data.ex_type, decode_if.data.op_type, decode_if.data.op_args, decode_if.data.bytesel); `TRACE(1, (", tmask=%b, wb=%b, rd_xregs=%b, wr_xregs=%b, used_rs=%b, rd=", decode_if.data.tmask, decode_if.data.wb, decode_if.data.rd_xregs, decode_if.data.wr_xregs, decode_if.data.used_rs)) VX_trace_pkg::trace_reg_idx(1, decode_if.data.rd); `TRACE(1, (", rs1=")) diff --git a/hw/rtl/core/VX_decompressor.sv b/hw/rtl/core/VX_decompressor.sv index de3aea297e..a1bf786487 100644 --- a/hw/rtl/core/VX_decompressor.sv +++ b/hw/rtl/core/VX_decompressor.sv @@ -66,7 +66,9 @@ // state[]/buf_pc[]/BRAM/follow_req/rsp_ready and sched_buffered_match all // gated on s1_can_accept. module VX_decompressor import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS) ) ( input wire clk, input wire reset, @@ -76,7 +78,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] sched_PC, input wire [`VX_CFG_NUM_THREADS-1:0] sched_tmask, input wire [NCTA_WIDTH-1:0] sched_cta_id, - input wire [NW_WIDTH-1:0] sched_wid, + input wire [WID_W-1:0] sched_wid, output wire sched_buffered_match, // Ungated buffered-hit: high whenever the scheduled PC is already in the // decompressor buffer, regardless of stage-0 stall. Used by VX_fetch to @@ -89,19 +91,19 @@ module VX_decompressor import VX_gpu_pkg::*; #( input wire [PC_BITS-1:0] rsp_PC, input wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask, input wire [NCTA_WIDTH-1:0] rsp_cta_id, - input wire [NW_WIDTH-1:0] rsp_wid, + input wire [WID_W-1:0] rsp_wid, input wire [UUID_WIDTH-1:0] rsp_uuid, output logic rsp_ready, // One-outstanding-per-warp tracking (from VX_fetch): high while a warp has // an icache request in flight. Used to gate the persistent follow arbiter. - input wire [`VX_CFG_NUM_WARPS-1:0] inflight, + input wire [NUM_WARPS-1:0] inflight, // Follow-up icache request (cross-word 32-bit). output logic follow_req_valid, output logic [PC_BITS-1:0] follow_req_PC, output logic [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask, - output logic [NW_WIDTH-1:0] follow_req_wid, + output logic [WID_W-1:0] follow_req_wid, output logic [UUID_WIDTH-1:0] follow_req_uuid, // Decoded fetch interface. @@ -151,7 +153,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( typedef struct packed { logic [31:0] payload32; logic [PC_BITS-1:0] PC; - logic [NW_WIDTH-1:0] wid; + logic [WID_W-1:0] wid; logic [UUID_WIDTH-1:0] uuid; logic [`VX_CFG_NUM_THREADS-1:0] tmask; logic [NCTA_WIDTH-1:0] cta_id; @@ -338,25 +340,25 @@ module VX_decompressor import VX_gpu_pkg::*; #( // State storage // ------------------------------------------------------------------ - buf_state_e state [`VX_CFG_NUM_WARPS]; - buf_state_e state_n [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc [`VX_CFG_NUM_WARPS]; - logic [PC_BITS-1:0] buf_pc_n [`VX_CFG_NUM_WARPS]; + buf_state_e state [NUM_WARPS]; + buf_state_e state_n [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc [NUM_WARPS]; + logic [PC_BITS-1:0] buf_pc_n [NUM_WARPS]; // Follow metadata captured on BUF_32HI entry (the persistent follow arbiter // re-issues the request from registered state, so it needs its own copy). - logic [UUID_WIDTH-1:0] hi_uuid [`VX_CFG_NUM_WARPS]; - logic [UUID_WIDTH-1:0] hi_uuid_n [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [`VX_CFG_NUM_WARPS]; - logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [`VX_CFG_NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid [NUM_WARPS]; + logic [UUID_WIDTH-1:0] hi_uuid_n [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask [NUM_WARPS]; + logic [`VX_CFG_NUM_THREADS-1:0] hi_tmask_n [NUM_WARPS]; // High half of the cross-word 32-bit, latched from the follow response and // held (BUF_32RDY) until the scheduler-gated emit combines it with the // BRAM-stashed low half. - logic [15:0] hi_hw [`VX_CFG_NUM_WARPS]; - logic [15:0] hi_hw_n [`VX_CFG_NUM_WARPS]; + logic [15:0] hi_hw [NUM_WARPS]; + logic [15:0] hi_hw_n [NUM_WARPS]; // drain[w]: the BUF_32HI straddle on warp w was squashed by a redirect // while its follow was in flight; discard the follow response (no emit). - logic [`VX_CFG_NUM_WARPS-1:0] drain, drain_n; + logic [NUM_WARPS-1:0] drain, drain_n; // spec[w]: the BUF_32HI straddle is speculative — it trails an RVC emitted // from the same word (have_D_emit_lo), which may be a taken branch, so the // warp is NOT yet committed to the straddle PC. Speculative straddles defer @@ -365,18 +367,22 @@ module VX_decompressor import VX_gpu_pkg::*; #( // (have_D_xword: the warp is already at the straddle PC and was acked by its // own fetch, so the scheduler will not re-present it) emit on the follow // response (have_B_resp). - logic [`VX_CFG_NUM_WARPS-1:0] spec, spec_n; + logic [NUM_WARPS-1:0] spec, spec_n; logic buf_we; - logic [NW_WIDTH-1:0] buf_waddr; + logic [WID_W-1:0] buf_waddr; buf_data_t buf_wdata; - logic [NW_WIDTH-1:0] buf_raddr; - buf_data_t buf_rdata; // registered (OUT_REG=1) + buf_data_t buf_rdata; logic buf_read; // gates rdata_r update during stall + // One read port per requester, so neither read address waits on the + // stage-0 case decode; stage 1 picks the port its kind was read from. + buf_data_t sched_rdata; // registered (OUT_REG=1) + buf_data_t rsp_rdata; // registered (OUT_REG=1) + VX_dp_ram #( .DATAW ($bits(buf_data_t)), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .OUT_REG (1), .LUTRAM (1), .RDW_MODE("R") @@ -388,8 +394,26 @@ module VX_decompressor import VX_gpu_pkg::*; #( .wren (1'b1), .waddr (buf_waddr), .wdata (buf_wdata), - .raddr (buf_raddr), - .rdata (buf_rdata) + .raddr (sched_wid), + .rdata (sched_rdata) + ); + + VX_dp_ram #( + .DATAW ($bits(buf_data_t)), + .SIZE (NUM_WARPS), + .OUT_REG (1), + .LUTRAM (1), + .RDW_MODE("R") + ) rsp_buffer_ram ( + .clk (clk), + .reset (reset), + .read (buf_read), + .write (buf_we), + .wren (1'b1), + .waddr (buf_waddr), + .wdata (buf_wdata), + .raddr (rsp_wid), + .rdata (rsp_rdata) ); // ------------------------------------------------------------------ @@ -498,11 +522,9 @@ module VX_decompressor import VX_gpu_pkg::*; #( && (state[rsp_wid] == BUF_EMPTY) && !pc_low && !rsp_high_c; - // BRAM raddr (cycle-N read latches data for cycle-N+1 stage 1 use). Both - // scheduler-gated emits (have_C, have_B_emit) read at sched_wid. - assign buf_raddr = (have_C || have_B_emit) ? sched_wid : rsp_wid; - // Only update raddr_r when stage 0 actually advances; keeps the - // registered rdata_r aligned with the registered s1_ctx. + // BRAM read (cycle-N read latches data for cycle-N+1 stage 1 use). Only + // update rdata_r when stage 0 actually advances; keeps the registered + // rdata_r aligned with the registered s1_ctx. assign buf_read = st0_advance; // ------------------------------------------------------------------ @@ -530,7 +552,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( buf_we = 1'b0; buf_waddr = rsp_wid; buf_wdata = '0; - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state_n[w] = state[w]; buf_pc_n[w] = buf_pc[w]; hi_uuid_n[w] = hi_uuid[w]; @@ -552,8 +574,8 @@ module VX_decompressor import VX_gpu_pkg::*; #( // BUF_RVC : no outstanding fetch -> clear directly. // (buf_pc == sched_PC is normal straddler continuation after an // RVC-then-straddler word — must NOT drop.) - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (sched_valid && (sched_wid == w[NW_WIDTH-1:0]) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (sched_valid && (sched_wid == WID_W'(w)) && (buf_pc[w] != sched_PC)) begin if (state[w] == BUF_32HI) begin drain_n[w] = 1'b1; @@ -684,14 +706,14 @@ module VX_decompressor import VX_gpu_pkg::*; #( // alias. Lowest-wid priority (reverse scan). // ------------------------------------------------------------------ logic follow_sel_valid; - logic [NW_WIDTH-1:0] follow_sel_wid; + logic [WID_W-1:0] follow_sel_wid; always_comb begin follow_sel_valid = 1'b0; follow_sel_wid = '0; - for (int w = `VX_CFG_NUM_WARPS-1; w >= 0; --w) begin - if ((state[w] == BUF_32HI) && ~inflight[w[NW_WIDTH-1:0]]) begin + for (int w = NUM_WARPS-1; w >= 0; --w) begin + if ((state[w] == BUF_32HI) && ~inflight[w]) begin follow_sel_valid = 1'b1; - follow_sel_wid = w[NW_WIDTH-1:0]; + follow_sel_wid = WID_W'(w); end end end @@ -706,6 +728,10 @@ module VX_decompressor import VX_gpu_pkg::*; #( // Stage 1 (combinational): build fsm_data from registered ctx + BRAM // ------------------------------------------------------------------ + // Both scheduler-gated emits (S1_C, S1_B) read at sched_wid; the + // response-gated straddle (S1_BR) reads at rsp_wid. + assign buf_rdata = (s1_kind == S1_BR) ? rsp_rdata : sched_rdata; + // Both RVC and straddle sources are registered 32-bit payloads pre-formed in // stage 0; this stage only selects, no decompress16. wire [31:0] instr_rvc = (s1_kind == S1_C) ? buf_rdata.payload // buffered RVC @@ -716,7 +742,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fsm_data = '0; fsm_data.PC = s1_ctx.PC; - fsm_data.wid = s1_ctx.wid; + fsm_data.wid = NW_WIDTH'(s1_ctx.wid); // cta_id: response-gated straddle (S1_BR) carries the BRAM-stashed CTA; // all other kinds captured the CTA into ctx (sched or rsp) in stage 0. fsm_data.cta_id = (s1_kind == S1_BR) ? buf_rdata.cta_id : s1_ctx.cta_id; @@ -761,7 +787,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_comb begin fast_data = '0; fast_data.uuid = rsp_uuid; - fast_data.wid = rsp_wid; + fast_data.wid = NW_WIDTH'(rsp_wid); fast_data.tmask = rsp_tmask; fast_data.cta_id = rsp_cta_id; fast_data.PC = rsp_PC; @@ -782,7 +808,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( always_ff @(posedge clk) begin if (reset) begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= BUF_EMPTY; buf_pc[w] <= '0; hi_uuid[w] <= '0; @@ -794,7 +820,7 @@ module VX_decompressor import VX_gpu_pkg::*; #( s1_kind <= S1_NONE; s1_ctx <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin + for (int w = 0; w < NUM_WARPS; ++w) begin state[w] <= state_n[w]; buf_pc[w] <= buf_pc_n[w]; hi_uuid[w] <= hi_uuid_n[w]; diff --git a/hw/rtl/core/VX_dispatcher.sv b/hw/rtl/core/VX_dispatcher.sv index d0c3a81d62..9c17114d77 100644 --- a/hw/rtl/core/VX_dispatcher.sv +++ b/hw/rtl/core/VX_dispatcher.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dispatcher import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_stalls, output wire [NUM_EX_UNITS-1:0][PERF_CTR_BITS-1:0] perf_instrs, @@ -32,7 +33,9 @@ module VX_dispatcher import VX_gpu_pkg::*; #( VX_dispatch_if.master dispatch_if [NUM_EX_UNITS] ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif localparam OUT_DATAW = $bits(dispatch_t); @@ -45,7 +48,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( end // All execution units pass operand data straight through; LSU address - // generation (pack-load included) lives in VX_lsu_slice. + // generation (pack-load included) lives in VX_lsu_block. for (genvar i = 0; i < NUM_EX_UNITS; ++i) begin : g_buffers VX_elastic_buffer #( .DATAW (OUT_DATAW), @@ -107,7 +110,7 @@ module VX_dispatcher import VX_gpu_pkg::*; #( for (genvar ex = 0; ex < NUM_EX_UNITS; ++ex) begin : g_dispatch_trace always @(posedge clk) begin if (dispatch_if[ex].valid && dispatch_if[ex].ready) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, ISSUE_ID), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, sid=%0d, PC=0x%0h, ex=", $time, INSTANCE_ID, wis_to_wid(dispatch_if[ex].data.wis, slice_id), dispatch_if[ex].data.cta_id, dispatch_if[ex].data.sid, to_fullPC(dispatch_if[ex].data.PC))) VX_trace_pkg::trace_ex_type(1, ex); `TRACE(1, (", op=")) VX_trace_pkg::trace_ex_op(1, ex, dispatch_if[ex].data.op_type, dispatch_if[ex].data.op_args); diff --git a/hw/rtl/core/VX_execute.sv b/hw/rtl/core/VX_execute.sv index 4f0222e066..7a7b394cdd 100644 --- a/hw/rtl/core/VX_execute.sv +++ b/hw/rtl/core/VX_execute.sv @@ -13,33 +13,34 @@ `include "VX_define.vh" -module VX_execute import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 +module VX_execute import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + + // LSU memory paths `ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`ifdef VX_CFG_EXT_TCU_ENABLE - output tcu_perf_t tcu_perf, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] lsu_coalescer_misses, `endif + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_busy, +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif - - // Per-block LSU client connections to VX_lsu_scheduler (at VX_core). - // The dcache port (lsu_mem_if) lives at VX_core; execute exposes - // only the LSU client interfaces. - VX_lsu_sched_if.master lsu_client_if [`VX_CFG_NUM_LSU_BLOCKS], - -`ifdef TCU_META_ENABLE - // TCU AGU memory client (single warp-level AGU shared across blocks). - // VX_core wires this to client 1 of block 0's lsu_scheduler. - VX_lsu_sched_if.master tcu_mem_if, -`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], // dispatch interface VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], @@ -47,13 +48,13 @@ module VX_execute import VX_gpu_pkg::*; #( // commit interface VX_commit_if.master commit_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], + // per-block ports of the core-wide SFU and TCU services `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_OM_ENABLE @@ -64,22 +65,25 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - // scheduler interfaces - VX_sched_csr_if.slave sched_csr_if, - VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_warp_ctl_if.master warp_ctl_if, - `ifdef VX_CFG_TCU_WGMMA_ENABLE - // TCU tile-buffer local-memory read port - VX_mem_bus_if.master tcu_lmem_if, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_ready, + output tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready, `endif - // DCR-CSR interface - VX_dcr_csr_if dcr_csr_if + // scheduler interfaces + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_branch_ctl_if.master branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + input cta_bcast_t cta_bcast ); `ifdef VX_CFG_EXT_F_ENABLE @@ -96,18 +100,41 @@ module VX_execute import VX_gpu_pkg::*; #( .branch_ctl_if (branch_ctl_if) ); - `SCOPE_IO_SWITCH (1); +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); + +`ifdef TCU_META_ENABLE + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if lsu_tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); +`endif VX_lsu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-lsu", INSTANCE_ID))) ) lsu_unit ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (core_id), .dispatch_if (dispatch_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_LSU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .per_block_client_if (lsu_client_if) + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + .busy (lsu_busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if), + `endif + .dcache_bus_if (dcache_bus_if) ); `ifdef VX_CFG_EXT_F_ENABLE @@ -126,32 +153,31 @@ module VX_execute import VX_gpu_pkg::*; #( VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))) ) tcu_unit ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if), `endif - .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), - .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) + .dispatch_if (dispatch_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), + .commit_if (commit_if[EX_TCU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]) ); `endif VX_sfu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("%s-sfu", INSTANCE_ID))) ) sfu_unit ( .clk (clk), .reset (reset), - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif + .core_id (core_id), .dispatch_if (dispatch_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), .commit_if (commit_if[EX_SFU * `VX_CFG_ISSUE_WIDTH +: `VX_CFG_ISSUE_WIDTH]), `ifdef VX_CFG_EXT_F_ENABLE @@ -159,22 +185,18 @@ module VX_execute import VX_gpu_pkg::*; #( `endif `ifdef VX_CFG_EXT_DXA_ENABLE .dxa_req_bus_if (dxa_req_bus_if), - .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif `ifdef VX_CFG_EXT_TEX_ENABLE .tex_bus_if (tex_bus_if), `endif - `ifdef VX_CFG_EXT_OM_ENABLE - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - `endif `ifdef VX_CFG_EXT_RTU_ENABLE .rtu_bus_if (rtu_bus_if), .sched_unlock_if (sched_unlock_if), `endif .sched_csr_if (sched_csr_if), + .warp_shared_if (warp_shared_if), .warp_ctl_if (warp_ctl_if), - .dcr_csr_if (dcr_csr_if) + .cta_bcast (cta_bcast) ); endmodule diff --git a/hw/rtl/core/VX_execute_slice.sv b/hw/rtl/core/VX_execute_slice.sv new file mode 100644 index 0000000000..b7e6d65892 --- /dev/null +++ b/hw/rtl/core/VX_execute_slice.sv @@ -0,0 +1,414 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// The execution units of one issue slice: one ALU, LSU, SFU, FPU and TCU block +// each, and the FP CSRs of the slice's warps. +module VX_execute_slice import VX_gpu_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +import VX_fpu_pkg::*; +`endif +`ifdef VX_CFG_EXT_TCU_ENABLE +import VX_tcu_pkg::*; +`endif +#( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [ISSUE_ISW_W-1:0] slice_id, + + VX_dispatch_if.slave dispatch_if [NUM_EX_UNITS], + VX_commit_if.master commit_if [NUM_EX_UNITS], + + // ALU + VX_branch_ctl_if.master branch_ctl_if, + + // LSU +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] lsu_perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] lsu_coalescer_misses, +`endif + output wire lsu_sched_empty, + output wire lsu_store_empty, + output wire lsu_busy, +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if, +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS], + + // SFU +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + // TCU +`ifdef VX_CFG_TCU_WGMMA_ENABLE + output wire tcu_exec_valid, + output wire tcu_exec_ready, + output tcu_execute_t tcu_exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data, + input wire tcu_tbuf_ready, +`endif + + // SFU warp control and CSRs + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if, + VX_sched_csr_if.slave sched_csr_if, + input cta_bcast_t cta_bcast +); + localparam ALU_PARTIAL_BW = (`VX_CFG_NUM_ALU_LANES != `VX_CFG_SIMD_WIDTH); +`ifdef VX_CFG_EXT_F_ENABLE + localparam FPU_PARTIAL_BW = (`VX_CFG_NUM_FPU_LANES != `VX_CFG_SIMD_WIDTH); +`endif + +`ifdef SCOPE + localparam SCOPE_COUNT = 1 + `VX_CFG_EXT_TCU_ENABLED; +`endif + `SCOPE_IO_SWITCH (SCOPE_COUNT); + + // ALU + + VX_execute_if #( + .data_t (alu_execute_t) + ) alu_execute_if[1](); + + VX_result_if #( + .data_t (alu_result_t) + ) alu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_ALU +: 1]), + .execute_if (alu_execute_if) + ); + + VX_alu_block #( + .INSTANCE_ID (`SFORMATF(("%s-alu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .PARTIAL_BW (ALU_PARTIAL_BW) + ) alu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (alu_execute_if[0]), + .result_if (alu_result_if[0]), + .branch_ctl_if (branch_ctl_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_ALU_LANES), + .OUT_BUF (ALU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) alu_gather ( + .clk (clk), + .reset (reset), + .result_if (alu_result_if), + .commit_if (commit_if[EX_ALU +: 1]) + ); + + // LSU + + VX_execute_if #( + .data_t (lsu_execute_t) + ) lsu_execute_if[1](); + + VX_result_if #( + .data_t (lsu_result_t) + ) lsu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_LSU +: 1]), + .execute_if (lsu_execute_if) + ); + +`ifdef TCU_META_ENABLE + // the TCU block's metadata loads go through this slice's LSU scheduler + VX_lsu_sched_if lsu_tcu_mem_if(); +`endif + + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%s", INSTANCE_ID, SLICE_STR))) + ) lsu_block ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (lsu_execute_if[0]), + .result_if (lsu_result_if[0]), + `ifdef PERF_ENABLE + .perf_rd_req_fire (lsu_perf_rd_req_fire), + .perf_wr_req_fire (lsu_perf_wr_req_fire), + .perf_rsp_fire (lsu_perf_rsp_fire), + .coalescer_misses (lsu_coalescer_misses), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .sched_empty (lsu_sched_empty), + .store_empty (lsu_store_empty), + .busy (lsu_busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if), + `endif + .dcache_bus_if (dcache_bus_if) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) lsu_gather ( + .clk (clk), + .reset (reset), + .result_if (lsu_result_if), + .commit_if (commit_if[EX_LSU +: 1]) + ); + + // SFU + + VX_execute_if #( + .data_t (sfu_execute_t) + ) sfu_execute_if[1](); + + VX_result_if #( + .data_t (sfu_result_t) + ) sfu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_SFU +: 1]), + .execute_if (sfu_execute_if) + ); + +`ifdef VX_CFG_EXT_F_ENABLE + wire [NW_WIDTH-1:0] fcsr_wid; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [2:0] fcsr_wr_sel; + wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif + + VX_sfu_block #( + .INSTANCE_ID ((`VX_CFG_NUM_SFU_BLOCKS > 1) ? `SFORMATF(("%s-sfu%s", INSTANCE_ID, SLICE_STR)) : `SFORMATF(("%s-sfu", INSTANCE_ID))) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(slice_id)), + .execute_if (sfu_execute_if[0]), + .result_if (sfu_result_if[0]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if(sched_unlock_if), + `endif + .sched_csr_if (sched_csr_if), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .cta_bcast (cta_bcast) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) sfu_gather ( + .clk (clk), + .reset (reset), + .result_if (sfu_result_if), + .commit_if (commit_if[EX_SFU +: 1]) + ); + + // FPU + +`ifdef VX_CFG_EXT_F_ENABLE + VX_fpu_csr_if fpu_csr_if[1](); + + VX_execute_if #( + .data_t (fpu_execute_t) + ) fpu_execute_if[1](); + + VX_result_if #( + .data_t (fpu_result_t) + ) fpu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_FPU +: 1]), + .execute_if (fpu_execute_if) + ); + + VX_fpu_block #( + .INSTANCE_ID (`SFORMATF(("%s-fpu", INSTANCE_ID))), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .PARTIAL_BW (FPU_PARTIAL_BW) + ) fpu_block ( + .clk (clk), + .reset (reset), + .block_idx (slice_id), + .execute_if (fpu_execute_if[0]), + .result_if (fpu_result_if[0]), + .fpu_csr_if (fpu_csr_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_FPU_LANES), + .OUT_BUF (FPU_PARTIAL_BW ? 3 : 0), + .NUM_PORTS (1) + ) fpu_gather ( + .clk (clk), + .reset (reset), + .result_if (fpu_result_if), + .commit_if (commit_if[EX_FPU +: 1]) + ); + + VX_csr_fpu #( + .WID_STRIDE (`VX_CFG_ISSUE_WIDTH), + .NUM_CSR_PORTS (1), + .NUM_FPU_PORTS (1) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) + ); +`endif + + // TCU + +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_execute_if #( + .data_t (tcu_execute_t) + ) tcu_execute_if[1](); + + VX_result_if #( + .data_t (tcu_result_t) + ) tcu_result_if[1](); + + VX_lane_dispatch #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_dispatch ( + .clk (clk), + .reset (reset), + .port_base (slice_id), + .dispatch_if(dispatch_if[EX_TCU +: 1]), + .execute_if (tcu_execute_if) + ); + + VX_tcu_block #( + .INSTANCE_ID (`SFORMATF(("%s-tcu", INSTANCE_ID))), + .BLOCK_STR (SLICE_STR) + ) tcu_block ( + `SCOPE_IO_BIND (1) + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .execute_if (tcu_execute_if[0]), + .result_if (tcu_result_if[0]) + ); + + VX_lane_gather #( + .BLOCK_SIZE (1), + .NUM_LANES (`VX_CFG_NUM_TCU_LANES), + .OUT_BUF (3), + .NUM_PORTS (1) + ) tcu_gather ( + .clk (clk), + .reset (reset), + .result_if (tcu_result_if), + .commit_if (commit_if[EX_TCU +: 1]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_fetch.sv b/hw/rtl/core/VX_fetch.sv index 72cc0b6754..43442d9bb0 100644 --- a/hw/rtl/core/VX_fetch.sv +++ b/hw/rtl/core/VX_fetch.sv @@ -13,8 +13,11 @@ `include "VX_define.vh" +// NUM_WARPS is the number of warps this unit fetches for; their ids arrive in +// the low bits of schedule_if.data.wid and leave the same way on fetch_if. module VX_fetch import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( `SCOPE_IO_DECL @@ -37,6 +40,14 @@ module VX_fetch import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (reset) + localparam WID_W = `LOG2UP(NUM_WARPS); + `STATIC_ASSERT (WID_W == ICACHE_TAG_ID_BITS, ("invalid parameter")) + + wire [WID_W-1:0] sched_wid = schedule_if.data.wid[WID_W-1:0]; + if (NW_WIDTH > WID_W) begin : g_sched_wid_unused + `UNUSED_VAR (schedule_if.data.wid[NW_WIDTH-1:WID_W]) + end + wire icache_req_valid; wire icache_req_ready; wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr; @@ -44,25 +55,25 @@ module VX_fetch import VX_gpu_pkg::*; #( // tag bits VX_mem_bus_arb adds downstream live in ICACHE_TAG_WIDTH but // are not driven by this stage. wire [ICACHE_FETCH_TAG_WIDTH-1:0] icache_req_tag; - wire [NW_WIDTH-1:0] icache_req_wid; + wire [WID_W-1:0] icache_req_wid; wire [UUID_WIDTH-1:0] icache_req_uuid; wire [UUID_WIDTH-1:0] rsp_uuid; wire [PC_BITS-1:0] rsp_PC; wire [`VX_CFG_NUM_THREADS-1:0] rsp_tmask; - wire [NW_WIDTH-1:0] req_tag; - wire [NW_WIDTH-1:0] rsp_tag; + wire [WID_W-1:0] req_tag; + wire [WID_W-1:0] rsp_tag; wire [NCTA_WIDTH-1:0] rsp_cta_id; wire icache_req_fire = icache_req_valid && icache_req_ready; - assign req_tag = schedule_if.data.wid; + assign req_tag = sched_wid; assign {rsp_uuid, rsp_tag} = icache_bus_if.rsp_data.tag; VX_dp_ram #( .DATAW (PC_BITS + `VX_CFG_NUM_THREADS + NCTA_WIDTH), - .SIZE (`VX_CFG_NUM_WARPS), + .SIZE (NUM_WARPS), .RDW_MODE ("R"), .LUTRAM (1) ) tag_store ( @@ -87,12 +98,12 @@ module VX_fetch import VX_gpu_pkg::*; #( wire [PC_BITS-1:0] icache_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] icache_req_tmask; - wire [NW_WIDTH-1:0] rsp_wid; + wire [WID_W-1:0] rsp_wid; wire follow_req_valid; wire [PC_BITS-1:0] follow_req_PC; wire [`VX_CFG_NUM_THREADS-1:0] follow_req_tmask; - wire [NW_WIDTH-1:0] follow_req_wid; + wire [WID_W-1:0] follow_req_wid; wire [UUID_WIDTH-1:0] follow_req_uuid; wire sched_buffered_match; @@ -102,23 +113,23 @@ module VX_fetch import VX_gpu_pkg::*; #( // One-outstanding-per-warp (Invariant B): a warp with an icache request in // flight must not issue another, else its two responses alias the single // tag_store slot. Set on request fire, cleared on response fire. - reg [`VX_CFG_NUM_WARPS-1:0] inflight; + reg [NUM_WARPS-1:0] inflight; wire icache_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin inflight <= '0; end else begin - for (int w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin - if (icache_req_fire && (icache_req_wid == w[NW_WIDTH-1:0])) + for (int w = 0; w < NUM_WARPS; ++w) begin + if (icache_req_fire && (icache_req_wid == WID_W'(w))) inflight[w] <= 1'b1; - else if (icache_rsp_fire && (rsp_wid == w[NW_WIDTH-1:0])) + else if (icache_rsp_fire && (rsp_wid == WID_W'(w))) inflight[w] <= 1'b0; end end end `UNUSED_VAR (icache_req_tmask) - // ibuffer occupancy is already gated by VX_scheduler (schedule_warps + // ibuffer occupancy is already gated by VX_warp_sched (schedule_warps // masks out warps with full ibufs), so schedule_if.valid implies space. // Suppress with the UNGATED sched_buffered (not the st0-gated match): a // word already in the decompressor buffer must never be re-fetched, even @@ -135,7 +146,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = follow_req_valid || sched_req_valid; assign icache_req_PC = follow_req_valid ? follow_req_PC : schedule_if.data.PC; assign icache_req_tmask = follow_req_valid ? follow_req_tmask : schedule_if.data.tmask; - assign icache_req_wid = follow_req_valid ? follow_req_wid : schedule_if.data.wid; + assign icache_req_wid = follow_req_valid ? follow_req_wid : sched_wid; assign icache_req_uuid = follow_req_valid ? follow_req_uuid : schedule_if.data.uuid; // NOTE: bind the full PC to a net before part-selecting — Vivado synthesis // rejects a bit-select applied directly to a function-call result. @@ -156,7 +167,8 @@ module VX_fetch import VX_gpu_pkg::*; #( || sched_buffered_match; VX_decompressor #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .NUM_WARPS (NUM_WARPS) ) decompressor ( .clk (clk), .reset (reset), @@ -164,7 +176,7 @@ module VX_fetch import VX_gpu_pkg::*; #( .sched_PC (schedule_if.data.PC), .sched_tmask (schedule_if.data.tmask), .sched_cta_id (schedule_if.data.cta_id), - .sched_wid (schedule_if.data.wid), + .sched_wid (sched_wid), .sched_buffered_match (sched_buffered_match), .sched_buffered (sched_buffered), .inflight (inflight), @@ -191,7 +203,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign icache_req_valid = schedule_if.valid; assign icache_req_addr = schedule_if.data.PC[2-(`VX_CFG_XLEN-PC_BITS) +: ICACHE_ADDR_WIDTH]; - assign icache_req_wid = schedule_if.data.wid; + assign icache_req_wid = sched_wid; assign icache_req_uuid = schedule_if.data.uuid; assign icache_req_tag = {icache_req_uuid, icache_req_wid}; assign schedule_if.ibuf_pop = fetch_if.ibuf_pop; @@ -199,7 +211,7 @@ module VX_fetch import VX_gpu_pkg::*; #( assign fetch_if.valid = icache_bus_if.rsp_valid; assign fetch_if.data.tmask = rsp_tmask; - assign fetch_if.data.wid = rsp_tag; + assign fetch_if.data.wid = NW_WIDTH'(rsp_tag); assign fetch_if.data.cta_id = rsp_cta_id; assign fetch_if.data.PC = rsp_PC; assign fetch_if.data.instr = icache_bus_if.rsp_data.data; @@ -248,8 +260,8 @@ module VX_fetch import VX_gpu_pkg::*; #( `endif `ifdef SCOPE -`ifdef DBG_SCOPE_FETCH `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_FETCH wire schedule_fire = schedule_if.valid && schedule_if.ready; wire icache_bus_req_fire = icache_bus_if.req_valid && icache_bus_if.req_ready; wire icache_bus_rsp_fire = icache_bus_if.rsp_valid && icache_bus_if.rsp_ready; diff --git a/hw/rtl/core/VX_frontend_slice.sv b/hw/rtl/core/VX_frontend_slice.sv new file mode 100644 index 0000000000..4c66ef0adb --- /dev/null +++ b/hw/rtl/core/VX_frontend_slice.sv @@ -0,0 +1,171 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One frontend slice: warp scheduling, fetch and decode of the warps with +// wid mod FETCH_WIDTH == slice_id, on the slice's own icache port. The front end +// names the slice's warps by wfs; decoded instructions leave with their +// core-wide wid, and ibuffer pops return indexed by wfs. +module VX_frontend_slice import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING SLICE_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + +`ifdef PERF_ENABLE + output fetch_perf_t fetch_perf, +`endif + + // core scheduler + VX_slice_sched_if.slave slice_sched_if, + + // instruction fetch, tagged with the core's flush select bit (the LSB) + VX_mem_bus_if.master icache_bus_if, + + // decoded instructions + VX_decode_if.master decode_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty +); + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + + `SCOPE_IO_SWITCH (1); + + VX_schedule_if schedule_if(); + VX_fetch_if fetch_if(); + VX_decode_if fe_decode_if(); + VX_decode_sched_if decode_sched_if(); + + // A response with the select bit set answers the core's cache flush on this + // port, and the core takes it: it is dropped here, on the cache's registered + // response, so the core passes the response through without logic. + VX_mem_bus_if #( + .DATA_SIZE (ICACHE_WORD_SIZE), + .TAG_WIDTH (ICACHE_FETCH_TAG_WIDTH) + ) fetch_bus_if(); + + assign icache_bus_if.req_valid = fetch_bus_if.req_valid; + assign icache_bus_if.req_data.rw = fetch_bus_if.req_data.rw; + assign icache_bus_if.req_data.addr = fetch_bus_if.req_data.addr; + assign icache_bus_if.req_data.data = fetch_bus_if.req_data.data; + assign icache_bus_if.req_data.byteen = fetch_bus_if.req_data.byteen; + assign icache_bus_if.req_data.attr = fetch_bus_if.req_data.attr; + assign icache_bus_if.req_data.tag = {fetch_bus_if.req_data.tag, 1'b0}; + assign fetch_bus_if.req_ready = icache_bus_if.req_ready; + + assign fetch_bus_if.rsp_valid = icache_bus_if.rsp_valid && ~icache_bus_if.rsp_data.tag[0]; + assign fetch_bus_if.rsp_data.data = icache_bus_if.rsp_data.data; + assign fetch_bus_if.rsp_data.tag = icache_bus_if.rsp_data.tag[ICACHE_TAG_WIDTH-1:1]; + assign icache_bus_if.rsp_ready = fetch_bus_if.rsp_ready; + + VX_warp_sched #( + .INSTANCE_ID (`SFORMATF(("%s-scheduler%s", INSTANCE_ID, SLICE_STR))) + ) warp_sched ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .slice_id (slice_id), + .sched_if (slice_sched_if), + .branch (branch), + .wctl (wctl), + .dvstack_wid (dvstack_wid), + .dvstack_ptr (dvstack_ptr), + .dvstack_mscratch (dvstack_mscratch), + .csr_rd_wid (csr_rd_wid), + .csr_rd (csr_rd), + .csr_wr (csr_wr), + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_unlock_valid (rtu_unlock_valid), + .rtu_unlock_wid (rtu_unlock_wid), + `endif + .decode_sched_if (decode_sched_if), + .issued (issued), + .committed (committed), + .ibuf_pop (decode_if.ibuf_pop[SW-1:0]), + .pending_alm_empty(pending_alm_empty), + .schedule_if (schedule_if) + ); + + VX_fetch #( + .INSTANCE_ID (`SFORMATF(("%s-fetch%s", INSTANCE_ID, SLICE_STR))), + .NUM_WARPS (SW) + ) fetch ( + `SCOPE_IO_BIND (0) + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .fetch_perf (fetch_perf), + `endif + .icache_bus_if (fetch_bus_if), + .schedule_if (schedule_if), + .fetch_if (fetch_if) + ); + + VX_decode #( + .INSTANCE_ID (`SFORMATF(("%s-decode%s", INSTANCE_ID, SLICE_STR))) + ) decode ( + .clk (clk), + .reset (reset), + .fetch_if (fetch_if), + .decode_if (fe_decode_if), + .decode_sched_if(decode_sched_if) + ); + + // decoded instructions take their core-wide wid + decode_t decode_data; + always @(*) begin + decode_data = fe_decode_if.data; + decode_data.wid = wfs_to_wid(fe_decode_if.data.wid[LW-1:0], slice_id); + end + if (NW_WIDTH > LW) begin : g_fe_wid_unused + `UNUSED_VAR (fe_decode_if.data.wid[NW_WIDTH-1:LW]) + end + + assign decode_if.valid = fe_decode_if.valid; + assign decode_if.data = decode_data; + assign fe_decode_if.ready = decode_if.ready; + + assign fe_decode_if.ibuf_pop[SW-1:0] = decode_if.ibuf_pop[SW-1:0]; + if (SW < `VX_CFG_NUM_WARPS) begin : g_ibuf_pop_pad + assign fe_decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW] = '0; + `UNUSED_VAR (decode_if.ibuf_pop[`VX_CFG_NUM_WARPS-1:SW]) + end + +endmodule diff --git a/hw/rtl/core/VX_frontend_xbar.sv b/hw/rtl/core/VX_frontend_xbar.sv new file mode 100644 index 0000000000..0426b3255c --- /dev/null +++ b/hw/rtl/core/VX_frontend_xbar.sv @@ -0,0 +1,221 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Connects the frontend slices (frontend slice f owns wid mod FETCH_WIDTH == f) to the +// issue slices (wid mod ISSUE_WIDTH) and the ALU/SFU blocks (wid mod B). Issue +// slice i takes its instructions from frontend slice i mod FETCH_WIDTH. A block +// serving several frontend slices sends each its own warps' events and answers its +// reads from the slice owning the read wid. Purely combinational. +module VX_frontend_xbar import VX_gpu_pkg::*; ( + // instruction path + VX_decode_if.slave fs_decode_if [`VX_CFG_FETCH_WIDTH], + VX_decode_if.master is_decode_if [`VX_CFG_ISSUE_WIDTH], + + // issue and commit events, and per-warp drain status + input wire [`VX_CFG_ISSUE_WIDTH-1:0] issue_valid, + input wire [`VX_CFG_ISSUE_WIDTH-1:0][ISSUE_WIS_W-1:0] issue_wis, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_issued, + output wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_committed, + input wire [`VX_CFG_FETCH_WIDTH-1:0][PER_FETCH_WARPS-1:0] fs_pending_alm_empty, + + // FU blocks + VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], + VX_warp_ctl_if.slave warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_sched_csr_if.master sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_sched_unlock_if.slave sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], +`endif + + // core scheduler state + input wire [PERF_CTR_BITS-1:0] cycles, + input wire [PERF_CTR_BITS-1:0] instret, + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty, + input wire mmu_empty, +`ifdef VX_CFG_VM_ENABLE + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + + // frontend slices + output sched_branch_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_ALU_PORTS-1:0] fs_branch, + output sched_wctl_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_wctl, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_dvstack_wid, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] fs_dvstack_ptr, + input wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] fs_dvstack_mscratch, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_csr_rd_wid, + input sched_csr_rd_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_rd, +`ifdef VX_CFG_EXT_RTU_ENABLE + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_rtu_unlock_valid, + output wire [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] fs_rtu_unlock_wid, +`endif + output sched_csr_wr_t [`VX_CFG_FETCH_WIDTH-1:0][FETCH_SFU_PORTS-1:0] fs_csr_wr +); + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam IW = `VX_CFG_ISSUE_WIDTH; + localparam NW = `VX_CFG_NUM_WARPS; + localparam SW = PER_FETCH_WARPS; + localparam BA = `VX_CFG_NUM_ALU_BLOCKS; + localparam BS = `VX_CFG_NUM_SFU_BLOCKS; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + + `STATIC_ASSERT (`IS_POW2(FW) && (FW <= IW) && ((IW % FW) == 0), ("invalid parameter")) + + // instruction path + + wire [IW-1:0] is_ready; + for (genvar i = 0; i < IW; ++i) begin : g_issue_slices + localparam F = i % FW; + assign is_decode_if[i].valid = fs_decode_if[F].valid && (wid_to_isw(fs_decode_if[F].data.wid) == ISSUE_ISW_W'(i)); + assign is_decode_if[i].data = fs_decode_if[F].data; + assign is_ready[i] = is_decode_if[i].ready; + if (PER_ISSUE_WARPS < NW) begin : g_ibuf_pop_unused + `UNUSED_VAR (is_decode_if[i].ibuf_pop[NW-1:PER_ISSUE_WARPS]) + end + end + + for (genvar f = 0; f < FW; ++f) begin : g_frontend_slices + assign fs_decode_if[f].ready = is_ready[wid_to_isw(fs_decode_if[f].data.wid)]; + for (genvar l = 0; l < SW; ++l) begin : g_ibuf_pop + localparam W = l * FW + f; + assign fs_decode_if[f].ibuf_pop[l] = is_decode_if[W % IW].ibuf_pop[W / IW]; + end + if (SW < NW) begin : g_ibuf_pop_pad + assign fs_decode_if[f].ibuf_pop[NW-1:SW] = '0; + end + end + + // issue and commit events, and per-warp drain status + + wire [NW-1:0] issued_warps; + wire [NW-1:0] pending_alm_empty; + for (genvar w = 0; w < NW; ++w) begin : g_warps + assign issued_warps[w] = issue_valid[w % IW] && (issue_wis[w % IW] == ISSUE_WIS_W'(w / IW)); + assign fs_issued[w % FW][w / FW] = issued_warps[w]; + assign fs_committed[w % FW][w / FW] = committed_warps[w]; + assign pending_alm_empty[w] = fs_pending_alm_empty[w % FW][w / FW]; + end + + // ALU blocks: frontend slice f port q is block q * FW + f, or block f mod BA when shared + + for (genvar f = 0; f < FW; ++f) begin : g_alu_slices + for (genvar q = 0; q < NPA; ++q) begin : g_ports + localparam B = (BA >= FW) ? (q * FW + f) : (f % BA); + wire own; + if (BA >= FW) begin : g_own + assign own = 1'b1; + end else begin : g_shared + assign own = (wid_to_fsw(branch_ctl_if[B].wid) == FETCH_FSW_W'(f)); + end + assign fs_branch[f][q].valid = branch_ctl_if[B].valid && own; + assign fs_branch[f][q].wid = branch_ctl_if[B].wid; + assign fs_branch[f][q].taken = branch_ctl_if[B].taken; + assign fs_branch[f][q].dest = branch_ctl_if[B].dest; + assign fs_branch[f][q].is_trap = branch_ctl_if[B].is_trap; + assign fs_branch[f][q].is_mret = branch_ctl_if[B].is_mret; + assign fs_branch[f][q].trap_cause = branch_ctl_if[B].trap_cause; + end + end + + // SFU blocks: same port mapping; reads are answered by the slice owning the read wid + + for (genvar f = 0; f < FW; ++f) begin : g_sfu_slices + for (genvar q = 0; q < NPS; ++q) begin : g_ports + localparam B = (BS >= FW) ? (q * FW + f) : (f % BS); + wire own_ctl, own_wr; + `ifdef VX_CFG_EXT_RTU_ENABLE + wire own_unlock; + `endif + if (BS >= FW) begin : g_own + assign own_ctl = 1'b1; + assign own_wr = 1'b1; + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = 1'b1; + `endif + end else begin : g_shared + assign own_ctl = (wid_to_fsw(warp_ctl_if[B].wid) == FETCH_FSW_W'(f)); + assign own_wr = (wid_to_fsw(sched_csr_if[B].csr_wr_wid) == FETCH_FSW_W'(f)); + `ifdef VX_CFG_EXT_RTU_ENABLE + assign own_unlock = (wid_to_fsw(sched_unlock_if[B].wid) == FETCH_FSW_W'(f)); + `endif + end + + assign fs_wctl[f][q].tmc_valid = warp_ctl_if[B].tmc_valid && own_ctl; + assign fs_wctl[f][q].split_valid = warp_ctl_if[B].split_valid && own_ctl; + assign fs_wctl[f][q].sjoin_valid = warp_ctl_if[B].sjoin_valid && own_ctl; + assign fs_wctl[f][q].wsync_valid = warp_ctl_if[B].wsync_valid && own_ctl; + assign fs_wctl[f][q].wid = warp_ctl_if[B].wid; + assign fs_wctl[f][q].tmc = warp_ctl_if[B].tmc; + assign fs_wctl[f][q].split = warp_ctl_if[B].split; + assign fs_wctl[f][q].sjoin = warp_ctl_if[B].sjoin; + assign fs_dvstack_wid[f][q] = warp_ctl_if[B].dvstack_wid; + + assign fs_csr_rd_wid[f][q] = sched_csr_if[B].csr_rd_wid; + assign fs_csr_wr[f][q].valid = sched_csr_if[B].csr_wr_valid && own_wr; + assign fs_csr_wr[f][q].wid = sched_csr_if[B].csr_wr_wid; + assign fs_csr_wr[f][q].data = sched_csr_if[B].csr_wr_data; + assign fs_csr_wr[f][q].trap_valid = sched_csr_if[B].trap_csr_wr_valid & {NUM_TRAP_CSRS{own_wr}}; + assign fs_csr_wr[f][q].trap_data = sched_csr_if[B].trap_csr_wr_data; + + `ifdef VX_CFG_EXT_RTU_ENABLE + assign fs_rtu_unlock_valid[f][q] = sched_unlock_if[B].valid && own_unlock; + assign fs_rtu_unlock_wid[f][q] = sched_unlock_if[B].wid; + `endif + end + end + + for (genvar b = 0; b < BS; ++b) begin : g_sfu_blocks + sched_csr_rd_t csr_rd; + if (BS >= FW) begin : g_own + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[b % FW][b / FW]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[b % FW][b / FW]; + assign csr_rd = fs_csr_rd[b % FW][b / FW]; + end else begin : g_shared + wire [FETCH_FSW_W-1:0] stack_fsw = wid_to_fsw(warp_ctl_if[b].dvstack_wid); + wire [FETCH_FSW_W-1:0] csr_fsw = wid_to_fsw(sched_csr_if[b].csr_rd_wid); + assign warp_ctl_if[b].dvstack_ptr = fs_dvstack_ptr[stack_fsw][0]; + assign warp_ctl_if[b].mscratch = fs_dvstack_mscratch[stack_fsw][0]; + assign csr_rd = fs_csr_rd[csr_fsw][0]; + end + + for (genvar k = 0; k < SFU_BLOCK_WARPS; ++k) begin : g_pending + assign warp_ctl_if[b].warp_pending_alm_empty[k] = pending_alm_empty[k * BS + b]; + end + assign warp_ctl_if[b].lsu_sched_empty = lsu_sched_empty; + assign warp_ctl_if[b].mmu_empty = mmu_empty; + + assign sched_csr_if[b].cycles = cycles; + assign sched_csr_if[b].instret = instret; + assign sched_csr_if[b].active_warps = active_warps; + assign sched_csr_if[b].thread_mask = csr_rd.thread_mask; + assign sched_csr_if[b].mscratch = csr_rd.mscratch; + assign sched_csr_if[b].csr_mstatus = csr_rd.mstatus; + assign sched_csr_if[b].csr_mtvec = csr_rd.mtvec; + assign sched_csr_if[b].csr_mepc = csr_rd.mepc; + assign sched_csr_if[b].csr_mcause = csr_rd.mcause; + assign sched_csr_if[b].csr_mtval = csr_rd.mtval; + assign sched_csr_if[b].cta_rank = csr_rd.cta_rank; + assign sched_csr_if[b].cta_lane = csr_rd.cta_lane; + `ifdef VX_CFG_VM_ENABLE + assign sched_csr_if[b].csr_satp = satp[b]; + assign satp_wr_valid[b] = sched_csr_if[b].satp_wr_valid; + assign satp_wr_data[b] = sched_csr_if[b].satp_wr_data; + `endif + end + +endmodule diff --git a/hw/rtl/core/VX_ibuffer.sv b/hw/rtl/core/VX_ibuffer.sv index 377075c7f1..6cfd72f7bd 100644 --- a/hw/rtl/core/VX_ibuffer.sv +++ b/hw/rtl/core/VX_ibuffer.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_ibuffer import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_ibuffer import VX_gpu_pkg::*; #( // outputs VX_ibuffer_if.master ibuffer_if [PER_ISSUE_WARPS] ); - `UNUSED_PARAM (ISSUE_ID) localparam OUT_DATAW = $bits(ibuffer_t); diff --git a/hw/rtl/core/VX_ipdom_stack.sv b/hw/rtl/core/VX_ipdom_stack.sv index a702d61d09..813af395bd 100644 --- a/hw/rtl/core/VX_ipdom_stack.sv +++ b/hw/rtl/core/VX_ipdom_stack.sv @@ -14,31 +14,33 @@ `include "VX_define.vh" module VX_ipdom_stack import VX_gpu_pkg::*; #( - parameter WIDTH = 1, - parameter DEPTH = 1, - parameter ADDRW = `LOG2UP(DEPTH) + parameter WIDTH = 1, + parameter DEPTH = 1, + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter ADDRW = `LOG2UP(DEPTH) ) ( input wire clk, input wire reset, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input wire [WIDTH-1:0] d_val, input wire [ADDRW-1:0] rd_ptr, input wire push, input wire pop, output wire [WIDTH-1:0] q_val, output wire q_idx, - output wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, + output wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr, output wire empty, output wire full ); localparam BRAM_DATAW = 1 + WIDTH; - localparam BRAM_SIZE = DEPTH * `VX_CFG_NUM_WARPS; + localparam BRAM_SIZE = DEPTH * NUM_WARPS; localparam BRAW_ADDRW = `LOG2UP(BRAM_SIZE); - wire [`VX_CFG_NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; - wire [`VX_CFG_NUM_WARPS-1:0] empty_w, full_w; + wire [NUM_WARPS-1:0][ADDRW-1:0] wr_ptr_w; + wire [NUM_WARPS-1:0] empty_w, full_w; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; i++) begin : g_addressing + for (genvar i = 0; i < NUM_WARPS; i++) begin : g_addressing reg [ADDRW-1:0] wr_ptr_r; reg empty_r, full_r; @@ -75,14 +77,16 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( wire [BRAW_ADDRW-1:0] raddr, waddr; - if (DEPTH > 1 && `VX_CFG_NUM_WARPS > 1) begin : g_DW - assign waddr = push ? {wr_ptr_w[wid], wid} : {rd_ptr, wid}; - assign raddr = {rd_ptr, wid}; + if (DEPTH > 1 && NUM_WARPS > 1) begin : g_DW + // entry = ptr * NUM_WARPS + wid keeps the store dense when NUM_WARPS is not a power of two + wire [ADDRW-1:0] waddr_ptr = push ? wr_ptr_w[wid] : rd_ptr; + assign waddr = BRAW_ADDRW'(waddr_ptr * NUM_WARPS + wid); + assign raddr = BRAW_ADDRW'(rd_ptr * NUM_WARPS + wid); end else if (DEPTH > 1) begin : g_D `UNUSED_VAR (wid) assign waddr = push ? wr_ptr_w : rd_ptr; assign raddr = rd_ptr; - end else if (`VX_CFG_NUM_WARPS > 1) begin : g_W + end else if (NUM_WARPS > 1) begin : g_W `UNUSED_VAR (rd_ptr) `UNUSED_VAR (wr_ptr_w) assign waddr = wid; @@ -98,8 +102,7 @@ module VX_ipdom_stack import VX_gpu_pkg::*; #( VX_dp_ram #( .DATAW (BRAM_DATAW), .SIZE (BRAM_SIZE), - .RDW_MODE ("R"), - .RADDR_REG(1) + .RDW_MODE ("R") ) ipdom_store ( .clk (clk), .reset (reset), diff --git a/hw/rtl/core/VX_issue.sv b/hw/rtl/core/VX_issue.sv index ddc745b577..03bff6cd98 100644 --- a/hw/rtl/core/VX_issue.sv +++ b/hw/rtl/core/VX_issue.sv @@ -25,7 +25,7 @@ module VX_issue import VX_gpu_pkg::*; #( output issue_perf_t issue_perf, `endif - VX_decode_if.slave decode_if, + VX_decode_if.slave decode_if [`VX_CFG_ISSUE_WIDTH], VX_writeback_if.slave writeback_if [`VX_CFG_ISSUE_WIDTH], VX_dispatch_if.master dispatch_if [NUM_EX_UNITS * `VX_CFG_ISSUE_WIDTH], VX_issue_sched_if.master issue_sched_if [`VX_CFG_ISSUE_WIDTH] @@ -43,11 +43,6 @@ module VX_issue import VX_gpu_pkg::*; #( end `endif - wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_if.data.wid); - - wire [`VX_CFG_ISSUE_WIDTH-1:0] decode_ready_in; - assign decode_if.ready = decode_ready_in[decode_isw]; - `SCOPE_IO_SWITCH (`VX_CFG_ISSUE_WIDTH); wire [`VX_CFG_ISSUE_WIDTH-1:0] issued_warps; @@ -56,27 +51,18 @@ module VX_issue import VX_gpu_pkg::*; #( for (genvar issue_id = 0; issue_id < `VX_CFG_ISSUE_WIDTH; ++issue_id) begin : g_slices VX_dispatch_if per_issue_dispatch_if[NUM_EX_UNITS](); - VX_decode_if slice_decode_if(); - - assign slice_decode_if.valid = decode_if.valid && (decode_isw == issue_id); - assign slice_decode_if.data = decode_if.data; - assign decode_ready_in[issue_id] = slice_decode_if.ready; - - for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_ibuf_pop - assign decode_if.ibuf_pop[wis_to_wid(ISSUE_WIS_W'(w), ISSUE_ISW_W'(issue_id))] = slice_decode_if.ibuf_pop[w]; - end VX_issue_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))), - .ISSUE_ID (issue_id) + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, issue_id))) ) issue_slice ( `SCOPE_IO_BIND(issue_id) .clk (clk), .reset (reset), + .slice_id (ISSUE_ISW_W'(issue_id)), `ifdef PERF_ENABLE .issue_perf (per_issue_perf[issue_id]), `endif - .decode_if (slice_decode_if), + .decode_if (decode_if[issue_id]), .writeback_if (writeback_if[issue_id]), .dispatch_if (per_issue_dispatch_if), .warp_issued (issued_warps[issue_id]), diff --git a/hw/rtl/core/VX_issue_slice.sv b/hw/rtl/core/VX_issue_slice.sv index b827b1e0ee..2ad54391fd 100644 --- a/hw/rtl/core/VX_issue_slice.sv +++ b/hw/rtl/core/VX_issue_slice.sv @@ -14,14 +14,15 @@ `include "VX_define.vh" module VX_issue_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] slice_id, + `ifdef PERF_ENABLE output issue_perf_t issue_perf, `endif @@ -39,8 +40,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( wire [NUM_EX_UNITS-1:0] fu_release; VX_ibuffer #( - .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-ibuffer", INSTANCE_ID))) ) ibuffer ( .clk (clk), .reset (reset), @@ -52,8 +52,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_scoreboard #( - .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-scoreboard", INSTANCE_ID))) ) scoreboard ( .clk (clk), .reset (reset), @@ -67,8 +66,7 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_operands #( - .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-operands", INSTANCE_ID))) ) operands ( .clk (clk), .reset (reset), @@ -81,11 +79,11 @@ module VX_issue_slice import VX_gpu_pkg::*; #( ); VX_dispatcher #( - .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))), - .ISSUE_ID (ISSUE_ID) + .INSTANCE_ID (`SFORMATF(("%s-dispatcher", INSTANCE_ID))) ) dispatcher ( .clk (clk), .reset (reset), + .slice_id (slice_id), `ifdef PERF_ENABLE .perf_stalls (issue_perf.dispatch_stalls), .perf_instrs (issue_perf.dispatch_instrs), @@ -101,8 +99,8 @@ module VX_issue_slice import VX_gpu_pkg::*; #( assign warp_issued_wis = scoreboard_if.data.wis; `ifdef SCOPE -`ifdef DBG_SCOPE_ISSUE `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_ISSUE wire decode_fire = decode_if.valid && decode_if.ready; wire operands_fire = operands_if.valid && operands_if.ready; wire reset_negedge; diff --git a/hw/rtl/core/VX_kmu_bus_arb.sv b/hw/rtl/core/VX_kmu_bus_arb.sv index 8ce6d4dcbb..55a87eb547 100644 --- a/hw/rtl/core/VX_kmu_bus_arb.sv +++ b/hw/rtl/core/VX_kmu_bus_arb.sv @@ -27,16 +27,16 @@ // lock would never release and would starve every other master forever. // // Fan-out picks the output by kind: -// FRAGMENT — routed by `dest`, this level's slice (DEST_LSB). Fragment work is -// bin->core affine; load-balancing it would break same-pixel blend -// order. +// FRAGMENT — routed by `dest`, this level's digit (dest / DEST_DIV) % NUM_OUTPUTS. +// Fragment work is bin->core affine; load-balancing it would break +// same-pixel blend order. // COMPUTE — round-robin over the ready outputs, which is what the plain // stream arbiter did before and what keeps CTAs spread across cores. module VX_kmu_bus_arb import VX_gpu_pkg::*; #( parameter NUM_INPUTS = 1, parameter NUM_OUTPUTS = 1, - parameter DEST_LSB = 0, // bit offset of this level's slice of `dest` + parameter DEST_DIV = 1, // cores below each output: this level's digit of `dest` parameter OUT_BUF = 0, parameter `STRING ARBITER = "R" ) ( @@ -167,7 +167,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( reg [SEL_O-1:0] lock_sel; reg [SEL_O-1:0] rr_ptr; - wire [SEL_O-1:0] dest_sel = SEL_O'(m_data[PW_DEST +: KMU_DEST_W] >> DEST_LSB); + wire [SEL_O-1:0] dest_sel = SEL_O'((32'(m_data[PW_DEST +: KMU_DEST_W]) / DEST_DIV) % NUM_OUTPUTS); // Round-robin over the ready outputs, starting at rr_ptr. Descending // scan so the nearest ready output (smallest step from rr_ptr) is the @@ -213,6 +213,7 @@ module VX_kmu_bus_arb import VX_gpu_pkg::*; #( end else begin : g_fanout_1 + `UNUSED_PARAM (DEST_DIV) assign out_valid[0] = m_valid; assign out_data[0] = m_data; assign m_ready = out_ready[0]; diff --git a/hw/rtl/core/VX_lane_dispatch.sv b/hw/rtl/core/VX_lane_dispatch.sv index 664df578b5..3b40279230 100644 --- a/hw/rtl/core/VX_lane_dispatch.sv +++ b/hw/rtl/core/VX_lane_dispatch.sv @@ -17,19 +17,25 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, parameter OUT_BUF = 0, - parameter MAX_FANOUT = `MAX_FANOUT + parameter MAX_FANOUT = `MAX_FANOUT, + // dispatch ports served, starting at issue slice port_base + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, + input wire [ISSUE_ISW_W-1:0] port_base, + // inputs - VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], + VX_dispatch_if.slave dispatch_if [NUM_PORTS], // outputs VX_execute_if.master execute_if [BLOCK_SIZE] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) + // a batched dispatch derives the issue slot from the batch index alone + `STATIC_ASSERT ((NUM_PORTS == `VX_CFG_ISSUE_WIDTH) || (BLOCK_SIZE == NUM_PORTS), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); localparam IN_DATAW = $bits(dispatch_t); @@ -41,9 +47,9 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam GPID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); localparam GPID_WIDTH = `UP(GPID_BITS); - localparam BATCH_COUNT = `VX_CFG_ISSUE_WIDTH / BLOCK_SIZE; + localparam BATCH_COUNT = NUM_PORTS / BLOCK_SIZE; localparam BATCH_COUNT_W= `LOG2UP(BATCH_COUNT); - localparam ISSUE_W = `LOG2UP(`VX_CFG_ISSUE_WIDTH); + localparam ISSUE_W = `LOG2UP(NUM_PORTS); localparam FANOUT_ENABLE= (`VX_CFG_SIMD_WIDTH > (MAX_FANOUT + MAX_FANOUT /2)); localparam DATA_IN_TMASK_OFF = IN_DATAW - (UUID_WIDTH + ISSUE_WIS_W + NCTA_WIDTH + SIMD_IDX_W + `VX_CFG_SIMD_WIDTH); @@ -54,11 +60,11 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( logic [NUM_LANES-1:0] tmask; } packet_t; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_valid; - wire [`VX_CFG_ISSUE_WIDTH-1:0][IN_DATAW-1:0] dispatch_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] dispatch_ready; + wire [NUM_PORTS-1:0] dispatch_valid; + wire [NUM_PORTS-1:0][IN_DATAW-1:0] dispatch_data; + wire [NUM_PORTS-1:0] dispatch_ready; - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_dispatch_data + for (genvar i = 0; i < NUM_PORTS; ++i) begin : g_dispatch_data assign dispatch_valid[i] = dispatch_if[i].valid; assign dispatch_data[i] = dispatch_if[i].data; assign dispatch_if[i].ready = dispatch_ready[i]; @@ -111,6 +117,10 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( `UNUSED_VAR (batch_done) end + if (BATCH_COUNT != 1) begin : g_no_port_base + `UNUSED_VAR (port_base) + end + wire [BLOCK_SIZE-1:0][ISSUE_W-1:0] issue_indices; for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_issue_indices assign issue_indices[block_idx] = ISSUE_W'(batch_idx * BLOCK_SIZE) + ISSUE_W'(block_idx); @@ -201,7 +211,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( assign isw = batch_idx; end end else begin : g_isw - assign isw = block_idx; + assign isw = port_base + ISSUE_ISW_W'(block_idx); end wire [NW_WIDTH-1:0] block_wid = wis_to_wid(dispatch_wis, isw); @@ -239,7 +249,7 @@ module VX_lane_dispatch import VX_gpu_pkg::*; #( end // release the dispatch interface when all packets are sent - reg [`VX_CFG_ISSUE_WIDTH-1:0] ready_in; + reg [NUM_PORTS-1:0] ready_in; always @(*) begin ready_in = 0; for (integer block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin diff --git a/hw/rtl/core/VX_lane_gather.sv b/hw/rtl/core/VX_lane_gather.sv index 0803f31bc8..905049c5e5 100644 --- a/hw/rtl/core/VX_lane_gather.sv +++ b/hw/rtl/core/VX_lane_gather.sv @@ -16,7 +16,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( parameter BLOCK_SIZE = 1, parameter NUM_LANES = 1, - parameter OUT_BUF = 0 + parameter OUT_BUF = 0, + // commit ports served + parameter NUM_PORTS = `VX_CFG_ISSUE_WIDTH ) ( input wire clk, input wire reset, @@ -25,9 +27,9 @@ module VX_lane_gather import VX_gpu_pkg::*; #( VX_result_if.slave result_if [BLOCK_SIZE], // outputs - VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH] + VX_commit_if.master commit_if [NUM_PORTS] ); - `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_ISSUE_WIDTH, BLOCK_SIZE), ("invalid parameter")) + `STATIC_ASSERT (`IS_DIVISBLE(NUM_PORTS, BLOCK_SIZE), ("invalid parameter")) `STATIC_ASSERT (`IS_DIVISBLE(`VX_CFG_SIMD_WIDTH, NUM_LANES), ("invalid parameter")) `DECL_EXECUTE_T (pe, NUM_LANES); @@ -38,34 +40,38 @@ module VX_lane_gather import VX_gpu_pkg::*; #( localparam LPID_WIDTH = `UP(LPID_BITS); localparam DATAW = $bits(pe_result_t); localparam DATA_WIS_OFF = DATAW - (UUID_WIDTH + NW_WIDTH); + localparam PORT_W = `LOG2UP(NUM_PORTS); + + // a partial gather recovers the issue slice from the warp id + `STATIC_ASSERT ((BLOCK_SIZE == NUM_PORTS) || (NUM_PORTS == `VX_CFG_ISSUE_WIDTH), ("invalid parameter")) wire [BLOCK_SIZE-1:0] result_in_valid; wire [BLOCK_SIZE-1:0][DATAW-1:0] result_in_data; wire [BLOCK_SIZE-1:0] result_in_ready; - wire [BLOCK_SIZE-1:0][ISSUE_ISW_W-1:0] result_in_isw; + wire [BLOCK_SIZE-1:0][PORT_W-1:0] result_in_isw; for (genvar i = 0; i < BLOCK_SIZE; ++i) begin : g_commit_in assign result_in_valid[i] = result_if[i].valid; assign result_in_data[i] = result_if[i].data; assign result_if[i].ready = result_in_ready[i]; - if (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) begin : g_result_in_isw_partial + if (BLOCK_SIZE != NUM_PORTS) begin : g_result_in_isw_partial if (BLOCK_SIZE != 1) begin : g_block - assign result_in_isw[i] = {result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}; + assign result_in_isw[i] = PORT_W'({result_in_data[i][DATA_WIS_OFF+BLOCK_SIZE_W +: (ISSUE_ISW_W-BLOCK_SIZE_W)], BLOCK_SIZE_W'(i)}); end else begin : g_no_block - assign result_in_isw[i] = result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]; + assign result_in_isw[i] = PORT_W'(result_in_data[i][DATA_WIS_OFF +: ISSUE_ISW_W]); end end else begin : g_result_in_isw_full - assign result_in_isw[i] = BLOCK_SIZE_W'(i); + assign result_in_isw[i] = PORT_W'(i); end end - reg [`VX_CFG_ISSUE_WIDTH-1:0] result_out_valid; - reg [`VX_CFG_ISSUE_WIDTH-1:0][DATAW-1:0] result_out_data; - wire [`VX_CFG_ISSUE_WIDTH-1:0] result_out_ready; + reg [NUM_PORTS-1:0] result_out_valid; + reg [NUM_PORTS-1:0][DATAW-1:0] result_out_data; + wire [NUM_PORTS-1:0] result_out_ready; always @(*) begin result_out_valid = '0; - for (integer i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin + for (integer i = 0; i < NUM_PORTS; ++i) begin result_out_data[i] = 'x; end for (integer i = 0; i < BLOCK_SIZE; ++i) begin @@ -78,7 +84,7 @@ module VX_lane_gather import VX_gpu_pkg::*; #( assign result_in_ready[i] = result_out_ready[result_in_isw[i]]; end - for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin: g_out_bufs + for (genvar i = 0; i < NUM_PORTS; ++i) begin: g_out_bufs VX_result_if #( .data_t (pe_result_t) ) result_tmp_if(); diff --git a/hw/rtl/core/VX_lsu_agu.sv b/hw/rtl/core/VX_lsu_agu.sv index a858877ee9..c921308993 100644 --- a/hw/rtl/core/VX_lsu_agu.sv +++ b/hw/rtl/core/VX_lsu_agu.sv @@ -14,7 +14,7 @@ `include "VX_define.vh" // VX_lsu_agu: per-lane LSU address generator. Owns all LSU address forms -// so VX_lsu_slice contains no address arithmetic: +// so VX_lsu_block contains no address arithmetic: // - plain load/store/fence: addr = base + sext(offset) // - pack-load uop: addr = base + uop_idx * stride // (uop_idx = offset[1:0], stride = rs2; a 2-bit shift-and-add @@ -24,15 +24,17 @@ // stack window are then interleaved by word across threads. // // This module sits between the LSU lane dispatch registers and the memory -// scheduler's request queue, so every full-width operation here is reduced to -// the narrowest width that is provably equivalent; see g_stack_interleave. +// scheduler's request queue, so it holds no arithmetic beyond the one adder. module VX_lsu_agu import VX_gpu_pkg::*; ( input wire [`VX_CFG_XLEN-1:0] base, // rs1 input wire [`VX_CFG_XLEN-1:0] stride, // rs2 (pack stride) input wire [11:0] offset, // immediate offset; [1:0] = pack uop_idx input wire [1:0] pack, // 0 = plain, non-zero = pack-load - output wire [`VX_CFG_XLEN-1:0] addr + output wire [`VX_CFG_XLEN-1:0] addr, + // the address before the stack interleave, which maps the stack window + // onto itself: any window disjoint from it classifies both alike + output wire [`VX_CFG_XLEN-1:0] lin_addr ); wire is_pack = (pack != 2'b00); wire [1:0] uop_idx = offset[1:0]; @@ -54,7 +56,7 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( .sum (csa_sum), .carry (csa_carry) ); - wire [`VX_CFG_XLEN-1:0] lin_addr = csa_sum[`VX_CFG_XLEN-1:0] + csa_carry[`VX_CFG_XLEN-1:0]; + assign lin_addr = csa_sum[`VX_CFG_XLEN-1:0] + csa_carry[`VX_CFG_XLEN-1:0]; `UNUSED_VAR ({csa_sum[`VX_CFG_XLEN+1:`VX_CFG_XLEN], csa_carry[`VX_CFG_XLEN+1:`VX_CFG_XLEN]}) `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE @@ -66,100 +68,41 @@ module VX_lsu_agu import VX_gpu_pkg::*; ( // depends on the address alone, so any thread may dereference another // thread's stack pointer. // - // Written naively the map costs three dependent full-width operations after - // lin_addr: subtract the window base, permute, add the base back. It does - // not need them. The permutation moves bits only WITHIN the group field -- - // the group index occupies the same position on both sides -- so subtracting - // the base and adding it back cancel on the high bits, and the window base - // has ALIGN_BITS trailing zeros, so the low field needs arithmetic only on - // the CORR_BITS above them. What survives of two XLEN-wide operations is one - // CORR_BITS-wide subtract, one CORR_BITS-wide add, and the +-1 the two - // disagree by. + // The window is aligned to a whole group, so the group index is the + // address's own bits above the group and the map is a bit permutation: + // no arithmetic follows the adder but the window compare. if (`VX_CFG_NUM_THREADS > 1) begin : g_stack_interleave localparam WORD_BITS = `CLOG2(`VX_CFG_XLEN / 8); localparam THREAD_BITS = `CLOG2(`VX_CFG_NUM_THREADS); localparam STACK_BITS = `VX_MEM_STACK_LOG2_SIZE; localparam SLOT_BITS = STACK_BITS - WORD_BITS; localparam GROUP_BITS = STACK_BITS + THREAD_BITS; - localparam HI_BITS = `VX_CFG_XLEN - GROUP_BITS; - // Trailing zeros of the window base. - localparam [`VX_CFG_XLEN-1:0] BOT_LSB = STACK_WINDOW_BOTTOM & -STACK_WINDOW_BOTTOM; - localparam ALIGN_BITS = (BOT_LSB == '0) ? `VX_CFG_XLEN : `CLOG2(BOT_LSB); - localparam CORR_BITS = (GROUP_BITS > ALIGN_BITS) ? (GROUP_BITS - ALIGN_BITS) : 0; localparam WIN_BITS = `VX_CFG_XLEN - STACK_BITS; `STATIC_ASSERT(`IS_POW2(`VX_CFG_NUM_THREADS), ("invalid parameter: NUM_THREADS=%0d", `VX_CFG_NUM_THREADS)) `STATIC_ASSERT(`VX_CFG_FLEN <= `VX_CFG_XLEN, ("invalid parameter: stack accesses must fit a word")) `STATIC_ASSERT(STACK_WINDOW_SPAN <= STACK_WINDOW_TOP, ("invalid parameter: stack window underflows")) - // The window test compares only the bits above the stack size, so a - // memory-map edit that breaks either bound's alignment must fail here. - `STATIC_ASSERT((STACK_WINDOW_BOTTOM & ((`VX_CFG_XLEN'(1) << STACK_BITS) - 1)) == '0, - ("stack window base is not a multiple of the stack size")) - `STATIC_ASSERT((STACK_WINDOW_TOP & ((`VX_CFG_XLEN'(1) << STACK_BITS) - 1)) == '0, - ("stack window top is not a multiple of the stack size")) + `STATIC_ASSERT(GROUP_BITS + SLOT_BITS <= `VX_CFG_XLEN, ("invalid parameter: NUM_THREADS=%0d", `VX_CFG_NUM_THREADS)) + // The span is a whole number of groups, so a group-aligned top makes + // the bottom group-aligned too. + `STATIC_ASSERT((STACK_WINDOW_TOP & ((`VX_CFG_XLEN'(1) << GROUP_BITS) - 1)) == '0, + ("stack window top is not aligned to a group of NUM_THREADS stacks")) - localparam [GROUP_BITS-1:0] BOT_LO = STACK_WINDOW_BOTTOM[GROUP_BITS-1:0]; - localparam [HI_BITS-1:0] BOT_HI = STACK_WINDOW_BOTTOM[`VX_CFG_XLEN-1:GROUP_BITS]; - - // Both bounds are multiples of the stack size, so membership is decided - // by the bits above it -- a WIN_BITS compare, not a full-width one. wire [WIN_BITS-1:0] lin_win = lin_addr[`VX_CFG_XLEN-1:STACK_BITS]; wire in_stack = (lin_win >= WIN_BITS'(STACK_WINDOW_BOTTOM >> STACK_BITS)) && (lin_win < WIN_BITS'(STACK_WINDOW_TOP >> STACK_BITS)); - wire [GROUP_BITS-1:0] lin_lo = lin_addr[GROUP_BITS-1:0]; - wire [HI_BITS-1:0] lin_hi = lin_addr[`VX_CFG_XLEN-1:GROUP_BITS]; - - // stack_off = lin_addr - STACK_WINDOW_BOTTOM, group field only. - wire [GROUP_BITS-1:0] off_lo; - wire off_borrow; - if (CORR_BITS == 0) begin : g_off_aligned - `UNUSED_PARAM (BOT_LO) - assign off_lo = lin_lo; - assign off_borrow = 1'b0; - end else begin : g_off_corr - wire [CORR_BITS:0] diff = {1'b0, lin_lo[GROUP_BITS-1:ALIGN_BITS]} - - {1'b0, BOT_LO[GROUP_BITS-1:ALIGN_BITS]}; - assign off_lo = {diff[CORR_BITS-1:0], lin_lo[ALIGN_BITS-1:0]}; - assign off_borrow = diff[CORR_BITS]; - end - - // Only SLOT_BITS of the group index reach the XOR, and truncation - // commutes with the subtract, so it is computed at that width. - wire [SLOT_BITS-1:0] stack_group; - if (SLOT_BITS <= HI_BITS) begin : g_group_narrow - assign stack_group = lin_hi[SLOT_BITS-1:0] - BOT_HI[SLOT_BITS-1:0] - - SLOT_BITS'(off_borrow); - end else begin : g_group_wide - assign stack_group = SLOT_BITS'(lin_hi - BOT_HI - HI_BITS'(off_borrow)); - end + wire [SLOT_BITS-1:0] stack_slot = lin_addr[STACK_BITS-1:WORD_BITS] + ^ lin_addr[GROUP_BITS +: SLOT_BITS]; - wire [SLOT_BITS-1:0] stack_slot = off_lo[STACK_BITS-1:WORD_BITS] ^ stack_group; - - wire [GROUP_BITS-1:0] swz_lo = { + wire [`VX_CFG_XLEN-1:0] stack_addr = { + lin_addr[`VX_CFG_XLEN-1:GROUP_BITS], stack_slot, - off_lo[GROUP_BITS-1:STACK_BITS], - off_lo[WORD_BITS-1:0] + lin_addr[GROUP_BITS-1:STACK_BITS], + lin_addr[WORD_BITS-1:0] }; - // addr = STACK_WINDOW_BOTTOM + swz_off, group field only. - wire [GROUP_BITS-1:0] addr_lo; - wire addr_carry; - if (CORR_BITS == 0) begin : g_out_aligned - assign addr_lo = swz_lo; - assign addr_carry = 1'b0; - end else begin : g_out_corr - wire [CORR_BITS:0] sum = {1'b0, swz_lo[GROUP_BITS-1:ALIGN_BITS]} - + {1'b0, BOT_LO[GROUP_BITS-1:ALIGN_BITS]}; - assign addr_lo = {sum[CORR_BITS-1:0], swz_lo[ALIGN_BITS-1:0]}; - assign addr_carry = sum[CORR_BITS]; - end - - // The window base cancels on the high bits; only the disagreement - // between the two group-field corrections survives. - wire [HI_BITS-1:0] addr_hi = lin_hi - HI_BITS'(off_borrow) + HI_BITS'(addr_carry); - - assign addr = in_stack ? {addr_hi, addr_lo} : lin_addr; + assign addr = in_stack ? stack_addr : lin_addr; end else begin : g_stack_linear assign addr = lin_addr; end diff --git a/hw/rtl/core/VX_lsu_slice.sv b/hw/rtl/core/VX_lsu_block.sv similarity index 84% rename from hw/rtl/core/VX_lsu_slice.sv rename to hw/rtl/core/VX_lsu_block.sv index 5a54698dd6..43f764107e 100644 --- a/hw/rtl/core/VX_lsu_slice.sv +++ b/hw/rtl/core/VX_lsu_block.sv @@ -13,24 +13,41 @@ `include "VX_define.vh" -module VX_lsu_slice import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 +module VX_lsu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_execute_if.slave execute_if, // Outputs VX_result_if.master result_if, - // The slice exposes the memory-client side as a VX_lsu_sched_if master; - // the parent wires it into VX_lsu_scheduler. - VX_lsu_sched_if.master client_if +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + output wire sched_empty, + output wire store_empty, + output wire busy, + +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if, +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] ); localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); @@ -47,8 +64,9 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( localparam [MEM_ADDRW-1:0] LMEM_BLK_BASE = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >> MEM_ASHIFT); `STATIC_ASSERT (MEM_ADDRW > LMEM_BLK_LSB, ("LMEM window wider than the block address space")) `endif - `UNUSED_PARAM (CORE_ID) - `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_EXT_A_ENABLE + `UNUSED_VAR (core_id) +`endif // tag_width = header + op_type + align + pkt_addr + fence localparam TAG_WIDTH = $bits(lsu_header_t) + INST_LSU_BITS + (NUM_LANES * REQ_ASHIFT) + LSUQ_SIZEW + 1; @@ -76,18 +94,45 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( // VX_lsu_agu; this slice contains no address arithmetic). wire req_is_fence, rsp_is_fence; + wire req_fence_io = inst_lsu_is_fence_io(execute_if.data.op_type); + // a memory fence that orders only later reads (acquire) or only earlier + // writes (release) + wire req_fence_acq = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_rl; + wire req_fence_rel = ~req_fence_io && ~execute_if.data.op_args.lsu.amo_aq; wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] full_addr; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] lin_addr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_full_addr VX_lsu_agu lsu_agu ( - .base (execute_if.data.rs1_data[i]), - .stride (execute_if.data.rs2_data[i]), - .offset (execute_if.data.op_args.lsu.offset), - .pack (execute_if.data.op_args.lsu.pack), - .addr (full_addr[i]) + .base (execute_if.data.rs1_data[i]), + .stride (execute_if.data.rs2_data[i]), + .offset (execute_if.data.op_args.lsu.offset), + .pack (execute_if.data.op_args.lsu.pack), + .addr (full_addr[i]), + .lin_addr (lin_addr[i]) ); end + // The address windows are classified on the address before the stack + // interleave, which keeps the interleave off the request's attribute + // path. The interleave maps the stack window onto itself, so this holds + // for every window outside it. +`ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) <= STACK_WINDOW_BOTTOM) + || (`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >= STACK_WINDOW_TOP), + ("the I/O window overlaps the stack window")) +`ifdef VX_CFG_LMEM_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) >= STACK_WINDOW_TOP) + || ((`VX_CFG_XLEN'(`VX_MEM_LMEM_BASE_ADDR) + (`VX_CFG_XLEN'(1) << `VX_CFG_LMEM_LOG_SIZE)) <= STACK_WINDOW_BOTTOM), + ("the local memory window overlaps the stack window")) +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `STATIC_ASSERT((`VX_CFG_XLEN'(`VX_MEM_OM_END_ADDR) <= STACK_WINDOW_BOTTOM) + || (`VX_CFG_XLEN'(`VX_MEM_OM_BASE_ADDR) >= STACK_WINDOW_TOP), + ("the OM aperture overlaps the stack window")) +`endif +`endif + // address type + AMO classification — per-lane mem_bus_attr_t // carried through the scheduler's USER channel and out via the // lsu_mem_if's per-lane req_data.user field. @@ -95,11 +140,13 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( mem_bus_attr_t [NUM_LANES-1:0] mem_req_attr_struct; wire [NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr; for (genvar i = 0; i < NUM_LANES; ++i) begin : g_mem_req_attr - wire [MEM_ADDRW-1:0] block_addr = full_addr[i][MEM_ASHIFT +: MEM_ADDRW]; + wire [MEM_ADDRW-1:0] block_addr = lin_addr[i][MEM_ASHIFT +: MEM_ADDRW]; // is I/O address wire [MEM_ADDRW-1:0] io_addr_start = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> MEM_ASHIFT); wire [MEM_ADDRW-1:0] io_addr_end = MEM_ADDRW'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> MEM_ASHIFT); assign mem_req_attr_struct[i].is_flush = req_is_fence; + // a release fence's token: no invalidation owed + assign mem_req_attr_struct[i].is_noalloc = req_is_fence && req_fence_rel; `ifdef VX_CFG_EXT_OM_ENABLE // OM fragment-export aperture. // @@ -118,8 +165,11 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( wire lane_is_om = 1'b0; assign mem_req_attr_struct[i].is_addr_om = 1'b0; `endif - assign mem_req_attr_struct[i].is_addr_io = ((block_addr >= io_addr_start) && (block_addr < io_addr_end)) - || lane_is_om; + // a fence's token carries its kind: io for a fence ordering only device + // accesses, and for an acquire fence's (byte enables set: no drain owed) + assign mem_req_attr_struct[i].is_addr_io = req_is_fence ? (req_fence_io || req_fence_acq) + : (((block_addr >= io_addr_start) && (block_addr < io_addr_end)) + || lane_is_om); `ifdef VX_CFG_LMEM_ENABLE // is local memory address assign mem_req_attr_struct[i].is_addr_local = @@ -141,7 +191,7 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( `endif // make_hart_id(cid, wid, tid) — packed concatenation, low bits = tid. assign lane_amo.hart_id = HART_ID_WIDTH'( - (HART_ID_WIDTH'(CORE_ID) << (NW_BITS + NT_BITS)) + (HART_ID_WIDTH'(core_id) << (NW_BITS + NT_BITS)) | (HART_ID_WIDTH'(execute_if.data.header.wid) << NT_BITS) | HART_ID_WIDTH'(i) ); @@ -261,7 +311,9 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( default : mem_req_byteen_w = {LSU_WORD_SIZE{1'b1}}; endcase end - assign mem_req_byteen[i] = mem_req_byteen_w; + // A fence is a flush token and carries no data. A memory fence is an + // L1-only token (byte enables set): it stops at the shared LLC. + assign mem_req_byteen[i] = req_is_fence ? (req_fence_io ? '0 : '1) : mem_req_byteen_w; end // memory misalignment not supported! @@ -375,6 +427,32 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( req_is_fence }; + VX_lsu_sched_if client_if(); + + VX_lsu_mem_path #( + .INSTANCE_ID (INSTANCE_ID) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire), + .perf_wr_req_fire (perf_wr_req_fire), + .perf_rsp_fire (perf_rsp_fire), + .coalescer_misses (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + .sched_empty (sched_empty), + .store_empty (store_empty), + .busy (busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if), + `endif + .dcache_bus_if (dcache_bus_if) + ); + // Pack per-lane mem_req signals into the client_if request payload; // unpack the response payload into per-lane mem_rsp signals. assign client_if.req_valid = mem_req_valid; @@ -546,8 +624,8 @@ module VX_lsu_slice import VX_gpu_pkg::*; #( `ifdef SCOPE -`ifdef DBG_SCOPE_LSU `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_LSU wire reset_negedge; `NEG_EDGE (reset_negedge, reset); `SCOPE_TAP_EX (0, 3, 4, 2, ( diff --git a/hw/rtl/core/VX_lsu_mem_path.sv b/hw/rtl/core/VX_lsu_mem_path.sv new file mode 100644 index 0000000000..68b6ce98a9 --- /dev/null +++ b/hw/rtl/core/VX_lsu_mem_path.sv @@ -0,0 +1,269 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The memory path of one LSU block: its memory scheduler, the global/local +// switch, the coalescer and the D-cache adapter. Local memory takes the +// switch's local requests whole. +// The D-cache port carries one extra tag LSB: requests leave with it clear, and +// a response with it set is the core's flush token, not this block's. +module VX_lsu_mem_path import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [PERF_CTR_BITS-1:0] coalescer_misses, +`endif + + VX_lsu_sched_if.slave client_if, +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if, +`endif + + // the scheduler holds no request; the global-store path holds no store + output wire sched_empty, + output wire store_empty, + output wire busy, + +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if, +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_CHANNELS] +); + `UNUSED_SPARAM (INSTANCE_ID) + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_mem_if(); + + // scheduler: client 0 is the block's lanes, client 1 the TCU metadata AGU + +`ifdef TCU_META_ENABLE + localparam NUM_CLIENTS = 2; +`else + localparam NUM_CLIENTS = 1; +`endif + + VX_lsu_sched_if sched_client_if [NUM_CLIENTS](); + + assign sched_client_if[0].req_valid = client_if.req_valid; + assign sched_client_if[0].req_data = client_if.req_data; + assign client_if.req_ready = sched_client_if[0].req_ready; + assign client_if.rsp_valid = sched_client_if[0].rsp_valid; + assign client_if.rsp_data = sched_client_if[0].rsp_data; + assign sched_client_if[0].rsp_ready = client_if.rsp_ready; + +`ifdef TCU_META_ENABLE + assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; + assign sched_client_if[1].req_data = tcu_mem_if.req_data; + assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; + assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; + assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; + assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; +`endif + + VX_lsu_scheduler #( + .INSTANCE_ID (`SFORMATF(("%s-lsusched", INSTANCE_ID))), + .NUM_CLIENTS (NUM_CLIENTS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .CORE_QUEUE_SIZE (`VX_CFG_LSU_QUEUE_IN_SIZE), + .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) + ) lsu_scheduler ( + .clk (clk), + .reset (reset), + .client_if (sched_client_if), + .empty (sched_empty), + .lsu_mem_if (lsu_mem_if) + ); + +`ifdef PERF_ENABLE + // The block's own client only: the TCU metadata loads are the TCU's, not + // the program's loads. + for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_perf + assign perf_rd_req_fire[j] = client_if.req_valid && client_if.req_data.mask[j] && client_if.req_ready && ~client_if.req_data.rw; + assign perf_wr_req_fire[j] = client_if.req_valid && client_if.req_data.mask[j] && client_if.req_ready && client_if.req_data.rw; + assign perf_rsp_fire[j] = client_if.rsp_valid && client_if.rsp_data.mask[j] && client_if.rsp_ready; + end +`endif + + // Two terms keep a store visible until it reaches the D-cache port: the + // coalescer (or the adapter input without one), and the adapter's request + // buffer. + wire path_empty; + wire adapter_empty; + assign store_empty = path_empty && adapter_empty; + assign busy = ~sched_empty || ~store_empty; + + VX_lsu_mem_if #( + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .DATA_SIZE (LSU_WORD_SIZE), + .TAG_WIDTH (LSU_TAG_WIDTH) + ) lsu_dcache_if(); + +`ifdef VX_CFG_LMEM_ENABLE + + VX_lmem_switch #( + .GLOBAL_OUT_BUF(1), + .LOCAL_OUT_BUF(1), + .RSP_OUT_BUF (1), + .ARBITER ("P") + ) lmem_switch ( + .clk (clk), + .reset (reset), + .lsu_in_if (lsu_mem_if), + .global_out_if(lsu_dcache_if), + .local_out_if (lmem_if) + ); + +`else + + `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if, lsu_mem_if); + +`endif + + VX_lsu_mem_if #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_coalesced_if(); + + if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing + + VX_mem_coalescer #( + .INSTANCE_ID (`SFORMATF(("%s-coalescer", INSTANCE_ID))), + .NUM_REQS (`VX_CFG_NUM_LSU_LANES), + .DATA_IN_SIZE (LSU_WORD_SIZE), + .DATA_OUT_SIZE (DCACHE_WORD_SIZE), + .ADDR_WIDTH (LSU_ADDR_WIDTH), + .USER_WIDTH (MEM_ATTR_WIDTH), + .TAG_WIDTH (LSU_TAG_WIDTH), + .UUID_WIDTH (UUID_WIDTH), + .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), + .PERF_CTR_BITS (PERF_CTR_BITS) + ) mem_coalescer ( + .clk (clk), + .reset (reset), + + `ifdef PERF_ENABLE + .misses (coalescer_misses), + `else + `UNUSED_PIN (misses), + `endif + + .empty (path_empty), + + // Input request + .in_req_valid (lsu_dcache_if.req_valid), + .in_req_mask (lsu_dcache_if.req_data.mask), + .in_req_rw (lsu_dcache_if.req_data.rw), + .in_req_byteen (lsu_dcache_if.req_data.byteen), + .in_req_addr (lsu_dcache_if.req_data.addr), + .in_req_user (lsu_dcache_if.req_data.user), + .in_req_no_merge(lsu_dcache_if.req_data.user[0][MEM_ATTR_AMO_OFFS]), + .in_req_data (lsu_dcache_if.req_data.data), + .in_req_tag (lsu_dcache_if.req_data.tag), + .in_req_ready (lsu_dcache_if.req_ready), + + // Input response + .in_rsp_valid (lsu_dcache_if.rsp_valid), + .in_rsp_mask (lsu_dcache_if.rsp_data.mask), + .in_rsp_data (lsu_dcache_if.rsp_data.data), + .in_rsp_tag (lsu_dcache_if.rsp_data.tag), + .in_rsp_ready (lsu_dcache_if.rsp_ready), + + // Output request + .out_req_valid (dcache_coalesced_if.req_valid), + .out_req_mask (dcache_coalesced_if.req_data.mask), + .out_req_rw (dcache_coalesced_if.req_data.rw), + .out_req_byteen (dcache_coalesced_if.req_data.byteen), + .out_req_addr (dcache_coalesced_if.req_data.addr), + .out_req_user (dcache_coalesced_if.req_data.user), + .out_req_data (dcache_coalesced_if.req_data.data), + .out_req_tag (dcache_coalesced_if.req_data.tag), + .out_req_ready (dcache_coalesced_if.req_ready), + + // Output response + .out_rsp_valid (dcache_coalesced_if.rsp_valid), + .out_rsp_mask (dcache_coalesced_if.rsp_data.mask), + .out_rsp_data (dcache_coalesced_if.rsp_data.data), + .out_rsp_tag (dcache_coalesced_if.rsp_data.tag), + .out_rsp_ready (dcache_coalesced_if.rsp_ready) + ); + + end else begin : g_no_coalescing + + // The passthrough narrows the tag from LSU_TAG_WIDTH to + // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the + // mem-scheduler slot id or responses alias. + `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) + + `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if, lsu_dcache_if); + `ifdef PERF_ENABLE + assign coalescer_misses = '0; + `endif + // Without a coalescer, a store still upstream of the adapter is one + // presented at its input. + assign path_empty = ~dcache_coalesced_if.req_valid; + + end + + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) dcache_adapter_if[DCACHE_CHANNELS](); + + VX_lsu_adapter #( + .NUM_LANES (DCACHE_CHANNELS), + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), + .ARBITER ("P"), + .REQ_OUT_BUF (3), + .RSP_OUT_BUF (0) + ) dcache_adapter ( + .clk (clk), + .reset (reset), + .lsu_mem_if (dcache_coalesced_if), + .mem_bus_if (dcache_adapter_if) + ); + + wire [DCACHE_CHANNELS-1:0] adapter_req_valid; + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + assign adapter_req_valid[j] = dcache_adapter_if[j].req_valid; + + assign dcache_bus_if[j].req_valid = dcache_adapter_if[j].req_valid; + assign dcache_bus_if[j].req_data.rw = dcache_adapter_if[j].req_data.rw; + assign dcache_bus_if[j].req_data.addr = dcache_adapter_if[j].req_data.addr; + assign dcache_bus_if[j].req_data.data = dcache_adapter_if[j].req_data.data; + assign dcache_bus_if[j].req_data.byteen = dcache_adapter_if[j].req_data.byteen; + assign dcache_bus_if[j].req_data.attr = dcache_adapter_if[j].req_data.attr; + assign dcache_bus_if[j].req_data.tag = {dcache_adapter_if[j].req_data.tag, 1'b0}; + assign dcache_adapter_if[j].req_ready = dcache_bus_if[j].req_ready; + + assign dcache_adapter_if[j].rsp_valid = dcache_bus_if[j].rsp_valid && ~dcache_bus_if[j].rsp_data.tag[0]; + assign dcache_adapter_if[j].rsp_data.data = dcache_bus_if[j].rsp_data.data; + assign dcache_adapter_if[j].rsp_data.tag = dcache_bus_if[j].rsp_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign dcache_bus_if[j].rsp_ready = dcache_adapter_if[j].rsp_ready; + end + assign adapter_empty = ~(| adapter_req_valid); + +endmodule diff --git a/hw/rtl/core/VX_lsu_scheduler.sv b/hw/rtl/core/VX_lsu_scheduler.sv index 626e43400e..f093e8cd7e 100644 --- a/hw/rtl/core/VX_lsu_scheduler.sv +++ b/hw/rtl/core/VX_lsu_scheduler.sv @@ -60,6 +60,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( wire [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] sched_req_data; wire [SCHED_TAG_WIDTH-1:0] sched_req_tag; wire sched_req_ready; + wire sched_req_ready_rd, sched_req_ready_wr; wire sched_rsp_valid; wire [NUM_LANES-1:0] sched_rsp_mask; @@ -100,6 +101,7 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign sched_req_data = cli_req_data[0].data; assign sched_req_tag = cli_req_data[0].tag; assign cli_req_ready[0] = sched_req_ready; + `UNUSED_VAR ({sched_req_ready_rd, sched_req_ready_wr}) assign cli_rsp_valid[0] = sched_rsp_valid; assign cli_rsp_data[0].mask = sched_rsp_mask; @@ -154,10 +156,26 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( assign sched_req_data = cli_req_data[grant_idx].data; assign sched_req_tag = {grant_idx, cli_req_data[grant_idx].tag}; - // Drive per-client ready: only the granted client sees ready high. + // Per-client ready: a client is granted unless a client ahead of it in + // the rotation is requesting, and it takes the scheduler's ready for its + // own rw. This equals the granted ready whenever the client requests -- + // the only time a ready is consumed -- without the client's own valid, + // the grant and the request mux in front of the requester's enables. for (genvar i = 0; i < NUM_CLIENTS; ++i) begin : g_creq_rdy - assign cli_req_ready[i] = sched_req_ready && (grant_idx == CLIENT_ID_BITS'(i)); + reg blocked; + always @(*) begin + blocked = 1'b0; + for (integer k = 0; k < NUM_CLIENTS; ++k) begin + if ((k != i) && cli_req_valid[k] + && (((k - int'(rr_ptr_r) + NUM_CLIENTS) % NUM_CLIENTS) < ((i - int'(rr_ptr_r) + NUM_CLIENTS) % NUM_CLIENTS))) begin + blocked = 1'b1; + end + end + end + assign cli_req_ready[i] = ~blocked && (cli_req_data[i].rw ? sched_req_ready_wr : sched_req_ready_rd); end + `RUNTIME_ASSERT (~any_valid || (cli_req_ready[grant_idx] == sched_req_ready), + ("%t: %s client ready out of sync with the granted ready", $time, INSTANCE_ID)) // Demux response: extract client_id from extended tag. wire [CLIENT_ID_BITS-1:0] rsp_cid = sched_rsp_tag[SCHED_TAG_WIDTH-1 -: CLIENT_ID_BITS]; @@ -231,6 +249,8 @@ module VX_lsu_scheduler import VX_gpu_pkg::*; #( .core_req_data (sched_req_data), .core_req_tag (sched_req_tag), .core_req_ready (sched_req_ready), + .core_req_ready_rd (sched_req_ready_rd), + .core_req_ready_wr (sched_req_ready_wr), .req_queue_empty(sched_req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/core/VX_lsu_unit.sv b/hw/rtl/core/VX_lsu_unit.sv index 67e5d22eea..408b3aec8e 100644 --- a/hw/rtl/core/VX_lsu_unit.sv +++ b/hw/rtl/core/VX_lsu_unit.sv @@ -14,22 +14,40 @@ `include "VX_define.vh" module VX_lsu_unit import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( `SCOPE_IO_DECL input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - // Per-block client connection to VX_lsu_scheduler (at VX_core). - VX_lsu_sched_if.master per_block_client_if [`VX_CFG_NUM_LSU_BLOCKS] +`ifdef PERF_ENABLE + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rd_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_wr_req_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] perf_rsp_fire, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] coalescer_misses, +`endif + +`ifdef TCU_META_ENABLE + VX_lsu_sched_if.slave tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] sched_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] store_empty, + output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] busy, + +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.master lmem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] ); localparam BLOCK_SIZE = `VX_CFG_NUM_LSU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_LSU_LANES; @@ -47,6 +65,7 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -56,16 +75,39 @@ module VX_lsu_unit import VX_gpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_lsu_slice #( - .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))), - .CORE_ID (CORE_ID) - ) lsu_slice( + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) block_dcache_bus_if[DCACHE_CHANNELS](); + for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if + `ASSIGN_VX_MEM_BUS_IF (dcache_bus_if[block_idx * DCACHE_CHANNELS + j], block_dcache_bus_if[j]); + end + + VX_lsu_block #( + .INSTANCE_ID (`SFORMATF(("%s%0d", INSTANCE_ID, block_idx))) + ) lsu_block( `SCOPE_IO_BIND (block_idx) .clk (clk), .reset (reset), + .core_id (core_id), .execute_if (per_block_execute_if[block_idx]), .result_if (per_block_result_if[block_idx]), - .client_if (per_block_client_if[block_idx]) + `ifdef PERF_ENABLE + .perf_rd_req_fire (perf_rd_req_fire[block_idx]), + .perf_wr_req_fire (perf_wr_req_fire[block_idx]), + .perf_rsp_fire (perf_rsp_fire[block_idx]), + .coalescer_misses (coalescer_misses[block_idx]), + `endif + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if[block_idx]), + `endif + .sched_empty (sched_empty[block_idx]), + .store_empty (store_empty[block_idx]), + .busy (busy[block_idx]), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lmem_if[block_idx]), + `endif + .dcache_bus_if (block_dcache_bus_if) ); end diff --git a/hw/rtl/core/VX_mem_unit.sv b/hw/rtl/core/VX_mem_unit.sv index 28e7b51377..6c4f5d09b2 100644 --- a/hw/rtl/core/VX_mem_unit.sv +++ b/hw/rtl/core/VX_mem_unit.sv @@ -21,7 +21,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE output lmem_perf_t lmem_perf, - output coalescer_perf_t coalescer_perf, `endif `ifdef VX_CFG_EXT_DXA_ENABLE @@ -34,27 +33,22 @@ module VX_mem_unit import VX_gpu_pkg::*; #( VX_mem_bus_if.slave tcu_lmem_if, `endif - output wire empty, + // the LSU blocks' memory paths +`ifdef VX_CFG_LMEM_ENABLE + VX_lsu_mem_if.slave lmem_if [`VX_CFG_NUM_LSU_BLOCKS], +`endif + VX_mem_bus_if.slave lsu_dcache_bus_if [DCACHE_NUM_REQS], - VX_lsu_mem_if.slave lsu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], VX_dcr_flush_if.slave dcr_flush_if, - VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS] -); - // Per-block empty: high when the global-store path holds no in-flight - // write. Folded into core `busy` so the end-of-kernel cache flush is - // ordered strictly behind the last store. Two terms: the coalescer/passthru - // producer, and the dcache adapter's request buffer (which carries state - // when REQ_OUT_BUF > 0 and would otherwise hide a still-settling store). - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_path_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_adapter_empty; - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] per_block_empty; - assign per_block_empty = per_block_path_empty & per_block_adapter_empty; - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_dcache_if[`VX_CFG_NUM_LSU_BLOCKS](); + input wire flush_l1_only, + VX_mem_bus_if.master dcache_bus_if [DCACHE_NUM_REQS], + // a flush lane holds a request at the D-cache port + output wire busy +); +`ifndef VX_CFG_LMEM_ENABLE + `UNUSED_SPARAM (INSTANCE_ID) +`endif `ifdef VX_CFG_TCU_WGMMA_ENABLE `STATIC_ASSERT(`VX_CFG_LMEM_ENABLED, ("TCU_WGMMA_ENABLE requires LMEM_ENABLE")) // The WGMMA shared-memory descriptor carries the LMEM byte offset in a 16-bit @@ -70,63 +64,6 @@ module VX_mem_unit import VX_gpu_pkg::*; #( localparam LMEM_ADDR_WIDTH = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE); - VX_lsu_mem_if #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_switches - VX_lmem_switch #( - .GLOBAL_OUT_BUF(1), - .LOCAL_OUT_BUF(1), - .RSP_OUT_BUF (1), - .ARBITER ("P") - ) lmem_switch ( - .clk (clk), - .reset (reset), - .lsu_in_if (lsu_mem_if[i]), - .global_out_if(lsu_dcache_if[i]), - .local_out_if (lsu_lmem_if[i]) - ); - end - - // Per-block local memory adapters to avoid deadlock when NUM_LSU_BLOCKS > 1. - // Each block gets dedicated ports into local memory, eliminating the circular - // dependency that arises from sharing a single adapter's unpack/pack buffers. - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_adapt_if[LSU_NUM_REQS](); - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lmem_adapters - - VX_mem_bus_if #( - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH) - ) lmem_block_if[`VX_CFG_NUM_LSU_LANES](); - - VX_lsu_adapter #( - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .DATA_SIZE (LSU_WORD_SIZE), - .TAG_WIDTH (LSU_TAG_WIDTH), - .TAG_SEL_BITS (LSU_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), - .RSP_OUT_BUF (0) - ) lmem_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (lsu_lmem_if[i]), - .mem_bus_if (lmem_block_if) - ); - - for (genvar j = 0; j < `VX_CFG_NUM_LSU_LANES; ++j) begin : g_lmem_adapt_if - `ASSIGN_VX_MEM_BUS_IF (lmem_adapt_if[i * `VX_CFG_NUM_LSU_LANES + j], lmem_block_if[j]); - end - end - // DMA arbiter: merge DXA writes and/or TCU reads onto single DMA port. VX_mem_bus_if #( @@ -165,6 +102,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( `ASSIGN_VX_MEM_BUS_IF_EX (dma_arb_in_if[LMEM_DMA_TCU_IDX], tcu_lmem_if, LMEM_DMA_IN_TAG_W, TCU_LMEM_TAG_W, UUID_WIDTH); `endif + // Round-robin, so a DXA write stream cannot starve the TCU's operand reads. VX_mem_bus_arb #( .NUM_INPUTS (LMEM_DMA_INPUTS), .NUM_OUTPUTS (1), @@ -172,7 +110,7 @@ module VX_mem_unit import VX_gpu_pkg::*; #( .TAG_WIDTH (LMEM_DMA_IN_TAG_W), .ATTR_WIDTH (LMEM_DMA_ATTR_W), .ADDR_WIDTH (LMEM_DMA_ADDR_WIDTH), - .ARBITER ("P") + .ARBITER ("R") ) lmem_dma_arb ( .clk (clk), .reset (reset), @@ -226,201 +164,114 @@ module VX_mem_unit import VX_gpu_pkg::*; #( end - VX_local_mem #( - .INSTANCE_ID (`SFORMATF(("%s-lmem", INSTANCE_ID))), - .SIZE (1 << `VX_CFG_LMEM_LOG_SIZE), - .NUM_REQS (LSU_NUM_REQS), - .NUM_BANKS (`VX_CFG_LMEM_NUM_BANKS), - .WORD_SIZE (LSU_WORD_SIZE), - .ADDR_WIDTH (LMEM_ADDR_WIDTH), - .TAG_WIDTH (LSU_TAG_WIDTH), - .DMA_ENABLE (LMEM_DMA_EN), + VX_lmem_unit #( + .INSTANCE_ID (`SFORMATF(("%s-lmem", INSTANCE_ID))), + .SIZE (1 << `VX_CFG_LMEM_LOG_SIZE), + .NUM_BLOCKS (`VX_CFG_NUM_LSU_BLOCKS), + .NUM_LANES (`VX_CFG_NUM_LSU_LANES), + .NUM_BANKS (`VX_CFG_LMEM_NUM_BANKS), + .WORD_SIZE (LSU_WORD_SIZE), + .ADDR_WIDTH (LMEM_ADDR_WIDTH), + .TAG_WIDTH (LSU_TAG_WIDTH), + .DMA_ENABLE (LMEM_DMA_EN), .DMA_TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .OUT_BUF (3) - ) local_mem ( - .clk (clk), - .reset (reset), + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + ) lmem_unit ( + .clk (clk), + .reset (reset), `ifdef PERF_ENABLE - .lmem_perf (lmem_perf), + .lmem_perf (lmem_perf), `endif - .dma_bus_if (lmem_dma_if), - .lsu_bus_if (lmem_adapt_if) + .lsu_mem_if (lmem_if), + .dma_bus_if (lmem_dma_if) ); `else - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_dcache_if - `ASSIGN_VX_MEM_BUS_IF (lsu_dcache_if[i], lsu_mem_if[i]); - end - `ifdef PERF_ENABLE assign lmem_perf = '0; `endif `endif - VX_lsu_mem_if #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_coalesced_if[`VX_CFG_NUM_LSU_BLOCKS](); - -`ifdef PERF_ENABLE - wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][PERF_CTR_BITS-1:0] per_block_coalescer_misses; - wire [PERF_CTR_BITS-1:0] coalescer_misses; - VX_reduce_tree #( - .IN_W (PERF_CTR_BITS), - .N (`VX_CFG_NUM_LSU_BLOCKS), - .OP ("+") - ) coalescer_reduce ( - .data_in (per_block_coalescer_misses), - .data_out (coalescer_misses) - ); - `BUFFER(coalescer_perf.misses, coalescer_misses); -`endif - - if ((`VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) begin : g_coalescing - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_coalescers - - VX_mem_coalescer #( - .INSTANCE_ID (`SFORMATF(("%s-coalescer%0d", INSTANCE_ID, i))), - .NUM_REQS (`VX_CFG_NUM_LSU_LANES), - .DATA_IN_SIZE (LSU_WORD_SIZE), - .DATA_OUT_SIZE (DCACHE_WORD_SIZE), - .ADDR_WIDTH (LSU_ADDR_WIDTH), - .USER_WIDTH (MEM_ATTR_WIDTH), - .TAG_WIDTH (LSU_TAG_WIDTH), - .UUID_WIDTH (UUID_WIDTH), - .QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PERF_CTR_BITS (PERF_CTR_BITS) - ) mem_coalescer ( - .clk (clk), - .reset (reset), - - `ifdef PERF_ENABLE - .misses (per_block_coalescer_misses[i]), - `else - `UNUSED_PIN (misses), - `endif - - .empty (per_block_path_empty[i]), - - // Input request - .in_req_valid (lsu_dcache_if[i].req_valid), - .in_req_mask (lsu_dcache_if[i].req_data.mask), - .in_req_rw (lsu_dcache_if[i].req_data.rw), - .in_req_byteen (lsu_dcache_if[i].req_data.byteen), - .in_req_addr (lsu_dcache_if[i].req_data.addr), - .in_req_user (lsu_dcache_if[i].req_data.user), - .in_req_no_merge(lsu_dcache_if[i].req_data.user[0][MEM_ATTR_AMO_OFFS]), - .in_req_data (lsu_dcache_if[i].req_data.data), - .in_req_tag (lsu_dcache_if[i].req_data.tag), - .in_req_ready (lsu_dcache_if[i].req_ready), - - // Input response - .in_rsp_valid (lsu_dcache_if[i].rsp_valid), - .in_rsp_mask (lsu_dcache_if[i].rsp_data.mask), - .in_rsp_data (lsu_dcache_if[i].rsp_data.data), - .in_rsp_tag (lsu_dcache_if[i].rsp_data.tag), - .in_rsp_ready (lsu_dcache_if[i].rsp_ready), - - // Output request - .out_req_valid (dcache_coalesced_if[i].req_valid), - .out_req_mask (dcache_coalesced_if[i].req_data.mask), - .out_req_rw (dcache_coalesced_if[i].req_data.rw), - .out_req_byteen (dcache_coalesced_if[i].req_data.byteen), - .out_req_addr (dcache_coalesced_if[i].req_data.addr), - .out_req_user (dcache_coalesced_if[i].req_data.user), - .out_req_data (dcache_coalesced_if[i].req_data.data), - .out_req_tag (dcache_coalesced_if[i].req_data.tag), - .out_req_ready (dcache_coalesced_if[i].req_ready), - - // Output response - .out_rsp_valid (dcache_coalesced_if[i].rsp_valid), - .out_rsp_mask (dcache_coalesced_if[i].rsp_data.mask), - .out_rsp_data (dcache_coalesced_if[i].rsp_data.data), - .out_rsp_tag (dcache_coalesced_if[i].rsp_data.tag), - .out_rsp_ready (dcache_coalesced_if[i].rsp_ready) - ); - end - - end else begin : g_no_coalescing - - // The passthrough narrows the tag from LSU_TAG_WIDTH to - // DCACHE_CORE_TAG_WIDTH: the dcache tag-id space must cover the - // mem-scheduler slot id or responses alias. - `STATIC_ASSERT (DCACHE_CORE_TAG_WIDTH >= LSU_TAG_WIDTH, ("dcache tag-id space cannot hold the LSU outstanding slot id")) - - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_coalesced_if - `ASSIGN_VX_MEM_BUS_IF (dcache_coalesced_if[i], lsu_dcache_if[i]); - `ifdef PERF_ENABLE - assign per_block_coalescer_misses[i] = '0; - `endif - // No coalescer: a store still in flight upstream of the adapter is - // one being presented to the adapter input. The adapter's own buffer - // is accounted for separately by per_block_adapter_empty. - assign per_block_path_empty[i] = ~dcache_coalesced_if[i].req_valid; - end - + // A private dcache takes the lanes straight from the adapter, whose drain + // store_drained already waits for, so one token suffices. Otherwise (a + // shared dcache, or none) the lanes are buffered independently before the + // next flush point, so a store on one lane can still be queued when another + // lane's token arrives: every lane then carries its own token behind its + // stores, and the flush is done when every lane's is. + localparam DCACHE_PRIVATE = (`VX_CFG_SOCKET_SIZE == `VX_CFG_NUM_DCACHES); + localparam FLUSH_LANES = DCACHE_PRIVATE ? 1 : DCACHE_NUM_REQS; + VX_dcr_flush_if lane_flush_if [FLUSH_LANES] (); + wire [FLUSH_LANES-1:0] lane_flush_done; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_lane_flush + assign lane_flush_if[k].req = dcr_flush_if.req; + assign lane_flush_done[k] = lane_flush_if[k].done; end + assign dcr_flush_if.done = (& lane_flush_done); - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_dcache_adapters - - VX_mem_bus_if #( - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) - ) dcache_bus_tmp_if[DCACHE_CHANNELS](); - - VX_lsu_adapter #( - .NUM_LANES (DCACHE_CHANNELS), - .DATA_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .TAG_SEL_BITS (DCACHE_CORE_TAG_WIDTH - UUID_WIDTH), - .ARBITER ("P"), - .REQ_OUT_BUF (3), // register dcache-request master boundary on all channels (passthru ports 1+) - .RSP_OUT_BUF (0) - ) dcache_adapter ( - .clk (clk), - .reset (reset), - .lsu_mem_if (dcache_coalesced_if[i]), - .mem_bus_if (dcache_bus_tmp_if) - ); - - // A store buffered in the adapter's REQ_OUT_BUF has left dcache_coalesced_if - // but not yet reached the flush-injection point; keep the block non-empty - // until every channel's request has drained past the adapter output. - wire [DCACHE_CHANNELS-1:0] adapter_req_valid; - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_adapter_req_valid - assign adapter_req_valid[j] = dcache_bus_tmp_if[j].req_valid; - end - assign per_block_adapter_empty[i] = ~(| adapter_req_valid); - - for (genvar j = 0; j < DCACHE_CHANNELS; ++j) begin : g_dcache_bus_if - if (i == 0 && j == 0) begin : g_flush_port - // Port 0: route through VX_dcr_flush to inject flush requests - VX_dcr_flush #( - .WORD_SIZE (DCACHE_WORD_SIZE), - .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), - .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF - ) dcr_flush ( - .clk (clk), - .reset (reset), - .dcr_flush_if (dcr_flush_if), - .core_bus_if (dcache_bus_tmp_if[0]), - .cache_bus_if (dcache_bus_if[0]) - ); - end else begin : g_passthru_port - // Ports 1+: pass through; tag is zero-extended from - // DCACHE_CORE_TAG_WIDTH to DCACHE_TAG_WIDTH on request, - // and MSB-stripped on response. - `ASSIGN_VX_MEM_BUS_IF_EX (dcache_bus_if[i * DCACHE_CHANNELS + j], dcache_bus_tmp_if[j], DCACHE_TAG_WIDTH_BASE, DCACHE_CORE_TAG_WIDTH, 0); - end + wire [FLUSH_LANES-1:0] flush_req_valid; + for (genvar k = 0; k < FLUSH_LANES; ++k) begin : g_flush_req_valid + assign flush_req_valid[k] = dcache_bus_if[k].req_valid; + end + assign busy = (| flush_req_valid); + + // A block's D-cache request carries a clear tag LSB, the flush arbiter's + // select bit; the block takes the cache response directly and drops the + // flush token's. + for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_bus_if + if (i < FLUSH_LANES) begin : g_flush_lane + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH) + ) flush_core_if(); + + assign flush_core_if.req_valid = lsu_dcache_bus_if[i].req_valid; + assign flush_core_if.req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign flush_core_if.req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign flush_core_if.req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign flush_core_if.req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign flush_core_if.req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign flush_core_if.req_data.tag = lsu_dcache_bus_if[i].req_data.tag[DCACHE_TAG_WIDTH-1:1]; + assign lsu_dcache_bus_if[i].req_ready = flush_core_if.req_ready; + `UNUSED_VAR (lsu_dcache_bus_if[i].req_data.tag[0]) + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data = dcache_bus_if[i].rsp_data; + assign flush_core_if.rsp_ready = lsu_dcache_bus_if[i].rsp_ready; + `UNUSED_VAR (flush_core_if.rsp_valid) + `UNUSED_VAR (flush_core_if.rsp_data) + + VX_dcr_flush #( + .WORD_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_CORE_TAG_WIDTH), + .REQ_OUT_BUF (3) // register core dcache-request boundary; rsp already registered by L1 CORE_OUT_BUF + ) dcr_flush ( + .clk (clk), + .reset (reset), + .dcr_flush_if (lane_flush_if[i]), + .l1_only (flush_l1_only), + .core_bus_if (flush_core_if), + .cache_bus_if (dcache_bus_if[i]) + ); + end else begin : g_passthru_lane + // The tag passes unchanged: its clear LSB stands where the flush lane's + // select bit does, so every lane encodes one request's tag alike and + // the cache's read merge compares like with like. + assign dcache_bus_if[i].req_valid = lsu_dcache_bus_if[i].req_valid; + assign dcache_bus_if[i].req_data.rw = lsu_dcache_bus_if[i].req_data.rw; + assign dcache_bus_if[i].req_data.addr = lsu_dcache_bus_if[i].req_data.addr; + assign dcache_bus_if[i].req_data.data = lsu_dcache_bus_if[i].req_data.data; + assign dcache_bus_if[i].req_data.byteen = lsu_dcache_bus_if[i].req_data.byteen; + assign dcache_bus_if[i].req_data.attr = lsu_dcache_bus_if[i].req_data.attr; + assign dcache_bus_if[i].req_data.tag = lsu_dcache_bus_if[i].req_data.tag; + assign lsu_dcache_bus_if[i].req_ready = dcache_bus_if[i].req_ready; + + assign lsu_dcache_bus_if[i].rsp_valid = dcache_bus_if[i].rsp_valid; + assign lsu_dcache_bus_if[i].rsp_data.data = dcache_bus_if[i].rsp_data.data; + assign lsu_dcache_bus_if[i].rsp_data.tag = dcache_bus_if[i].rsp_data.tag; + assign dcache_bus_if[i].rsp_ready = lsu_dcache_bus_if[i].rsp_ready; end - end - assign empty = (& per_block_empty); - endmodule diff --git a/hw/rtl/core/VX_opc_unit.sv b/hw/rtl/core/VX_opc_unit.sv index 203501046a..75b3b4c708 100644 --- a/hw/rtl/core/VX_opc_unit.sv +++ b/hw/rtl/core/VX_opc_unit.sv @@ -236,14 +236,21 @@ module VX_opc_unit import VX_gpu_pkg::*; #( end end + // warps are dealt round-robin across the collectors: slot = wis / NUM_OPCS + wire [PER_OPC_NW_W-1:0] wr_slot = PER_OPC_NW_W'(writeback_if.data.wis / `VX_CFG_NUM_OPCS); + wire [PER_OPC_NW_W-1:0] rd_slot = PER_OPC_NW_W'(pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: ISSUE_WIS_W] / `VX_CFG_NUM_OPCS); + if (PER_OPC_NW_BITS == 0) begin : g_slot_unused + `UNUSED_VAR ({wr_slot, rd_slot}) + end + wire [BANK_ADDR_WIDTH-1:0] gpr_wr_addr; if (SIMD_COUNT != 1) begin : g_gpr_wr_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(tmp, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) assign gpr_wr_addr = {writeback_if.data.sid, tmp}; end else begin : g_gpr_wr_addr - `CONCAT(gpr_wr_addr, writeback_if.data.wis[ISSUE_WIS_W-1 -: PER_OPC_NW_W], + `CONCAT(gpr_wr_addr, wr_slot, writeback_if.data.rd[NUM_REGS_BITS-1 -: REG_REM_BITS], PER_OPC_NW_BITS, REG_REM_BITS) end @@ -271,13 +278,13 @@ module VX_opc_unit import VX_gpu_pkg::*; #( wire [BANK_ADDR_WIDTH-1:0] gpr_rd_addr; if (SIMD_COUNT != 1) begin : g_gpr_rd_addr_sid wire [PER_OPC_NW_BITS + REG_REM_BITS-1:0] tmp; - `CONCAT(tmp, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(tmp, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) // pipe_mdata packs {uuid, wis, cta_id, simd_pid, ...}; skip uuid+wis+cta_id // to extract simd_pid, which must match writeback_if.data.sid on the write path. assign gpr_rd_addr = {pipe_mdata_st1[META_DATAW-UUID_WIDTH-ISSUE_WIS_W-NCTA_WIDTH-1 -: SIMD_IDX_W], tmp}; end else begin : g_gpr_rd_addr - `CONCAT(gpr_rd_addr, pipe_mdata_st1[META_DATAW-UUID_WIDTH-1 -: PER_OPC_NW_W], + `CONCAT(gpr_rd_addr, rd_slot, gpr_rd_reg_st1[b], PER_OPC_NW_BITS, REG_REM_BITS) end diff --git a/hw/rtl/core/VX_operands.sv b/hw/rtl/core/VX_operands.sv index 56e1ba7723..9a2d417616 100644 --- a/hw/rtl/core/VX_operands.sv +++ b/hw/rtl/core/VX_operands.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_operands import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -28,7 +27,6 @@ module VX_operands import VX_gpu_pkg::*; #( VX_scoreboard_if.slave scoreboard_if, VX_operands_if.master operands_if ); - `UNUSED_SPARAM (ISSUE_ID) localparam OUT_DATAW = $bits(operands_t); @@ -44,8 +42,8 @@ module VX_operands import VX_gpu_pkg::*; #( wire [NUM_OPCS_W-1:0] sb_opc, wb_opc; if (`VX_CFG_NUM_OPCS != 1) begin : g_wis_opc - assign sb_opc = scoreboard_if.data.wis[NUM_OPCS_W-1:0]; - assign wb_opc = writeback_if.data.wis[NUM_OPCS_W-1:0]; + assign sb_opc = NUM_OPCS_W'(scoreboard_if.data.wis % `VX_CFG_NUM_OPCS); + assign wb_opc = NUM_OPCS_W'(writeback_if.data.wis % `VX_CFG_NUM_OPCS); end else begin : g_wis_opc assign sb_opc = 0; assign wb_opc = 0; diff --git a/hw/rtl/core/VX_scheduler.sv b/hw/rtl/core/VX_scheduler.sv index 66e02120de..c16017a9d9 100644 --- a/hw/rtl/core/VX_scheduler.sv +++ b/hw/rtl/core/VX_scheduler.sv @@ -13,559 +13,254 @@ `include "VX_define.vh" +// The core-level part of scheduling: CTA dispatch and the core-wide warp-control +// resources. Each frontend slice schedules and fetches its own warps (VX_warp_sched, +// VX_fetch); core-wide events are handed to it with slice-local warp ids. module VX_scheduler import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + `ifdef PERF_ENABLE output sched_perf_t sched_perf, `endif - // inputs - VX_warp_ctl_if.slave warp_ctl_if, -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_sched_unlock_if.slave sched_unlock_if, // RTU TRACE wstall release + VX_slice_sched_if.master slice_if [`VX_CFG_FETCH_WIDTH], + VX_warp_shared_if.slave warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, `endif - VX_branch_ctl_if.slave branch_ctl_if [`VX_CFG_NUM_ALU_BLOCKS], - VX_decode_sched_if.slave decode_sched_if, - VX_issue_sched_if.slave issue_sched_if [`VX_CFG_ISSUE_WIDTH], - VX_commit_sched_if.slave commit_sched_if, + input wire [`VX_CFG_NUM_WARPS-1:0] committed_warps, + + // CTA context writes, to every SFU block's copy + output cta_bcast_t cta_bcast, - // KMU bus VX_kmu_bus_if.slave kmu_bus_if, - // outputs - VX_schedule_if.master schedule_if, - VX_sched_csr_if.master sched_csr_if, VX_gbar_bus_if.master gbar_bus_if, - // status + // per-core CSR state + output wire [PERF_CTR_BITS-1:0] cycles, + output wire [PERF_CTR_BITS-1:0] instret, + output wire [`VX_CFG_NUM_WARPS-1:0] active_warps, +`ifdef VX_CFG_VM_ENABLE + output wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0] satp_wr_valid, + input wire [`VX_CFG_NUM_SFU_BLOCKS-1:0][`VX_CFG_XLEN-1:0] satp_wr_data, +`endif + output wire busy ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - reg [`VX_CFG_NUM_WARPS-1:0] active_warps, active_warps_n; // updated when a warp is activated or disabled - reg [`VX_CFG_NUM_WARPS-1:0] stalled_warps, stalled_warps_n; // set when branch/gpgpu instructions are issued + localparam FW = `VX_CFG_FETCH_WIDTH; + localparam SW = PER_FETCH_WARPS; - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks, thread_masks_n; - reg [`VX_CFG_NUM_WARPS-1:0][PC_BITS-1:0] warp_pcs, warp_pcs_n; - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; - - // Per-warp machine-mode trap CSRs. csrw writes arrive on - // sched_csr_if.trap_csr_wr_*; ECALL/EBREAK hardware-write mepc/mcause/ - // mtval; MRET restores the warp PC from mepc. - reg [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; + // slice status in wid order (wid = {wfs, fsw}) + wire [`VX_CFG_NUM_WARPS-1:0] stalled_warps; + wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_empty; + for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_interleave + assign active_warps[w] = slice_if[w % FW].active[w / FW]; + assign stalled_warps[w] = slice_if[w % FW].stalled[w / FW]; + assign pending_warp_empty[w] = slice_if[w % FW].pending_empty[w / FW]; + end - wire [NW_WIDTH-1:0] schedule_wid; - wire [`VX_CFG_NUM_THREADS-1:0] schedule_tmask; - wire [PC_BITS-1:0] schedule_pc; - wire schedule_valid; - wire schedule_ready; + wire [FW-1:0] slice_ibuf_full; + for (genvar p = 0; p < FW; ++p) begin : g_slice_ibuf_full + assign slice_ibuf_full[p] = slice_if[p].ibuf_full; + end - // CTA dispatcher - wire cta_fire; - wire [NW_WIDTH-1:0] cta_wid; + // CTA dispatcher: lane p launches frontend slice p's warps + wire [FW-1:0] cta_fire; + wire [FW-1:0][NW_WIDTH-1:0] cta_wid; wire [PC_BITS-1:0] cta_PC; - wire [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] cta_tmask; wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + wire [FW-1:0] cta_init; + cta_lane_bcast_t [FW-1:0] cta_lane_bcast; wire cta_dispatcher_busy; - wire cta_init; - - // CTA-CSR read-back from the dispatcher (it owns the per-CTA/per-warp tables). - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; - - // Warp retirement: TMC with tmask==0 permanently deactivates the warp - wire cta_warp_done = warp_ctl_if.tmc_valid && (warp_ctl_if.tmc.tmask == 0); + wire cta_warp_done; + wire [NW_WIDTH-1:0] cta_warp_done_wid; VX_cta_dispatch #( - .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))) + .INSTANCE_ID (`SFORMATF(("%s-cta_dispatch", INSTANCE_ID))), + .NUM_LANES (FW) ) cta_dispatcher ( - .clk (clk), - .reset (reset), - .kmu_bus_if (kmu_bus_if), - .active_warps(active_warps), - .warp_done (cta_warp_done), - .warp_done_wid(warp_ctl_if.wid), - .cta_fire (cta_fire), - .cta_wid (cta_wid), - .cta_PC (cta_PC), - .cta_tmask (cta_tmask), - .cta_param (cta_param), - .cta_init (cta_init), - .csr_rd_wid (sched_csr_if.csr_rd_wid), - .csr_rd_cta_id(sched_csr_if.csr_rd_cta_id), - .cta_rd_csrs(cta_rd_csrs), - .cta_rd_lane(cta_rd_lane), - .schedule_wid(schedule_wid), - .schedule_cta_id(schedule_cta_id), - .busy (cta_dispatcher_busy) + .clk (clk), + .reset (reset), + .kmu_bus_if (kmu_bus_if), + .active_warps (active_warps), + .warp_done (cta_warp_done), + .warp_done_wid(cta_warp_done_wid), + .cta_fire (cta_fire), + .cta_wid (cta_wid), + .cta_PC (cta_PC), + .cta_tmask (cta_tmask), + .cta_param (cta_param), + .cta_init (cta_init), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), + .busy (cta_dispatcher_busy) ); - assign sched_csr_if.cta_csrs = cta_rd_csrs; - assign sched_csr_if.cta_lane = cta_rd_lane; - - assign sched_csr_if.mscratch = mscratch_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mstatus = mstatus_r[sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtvec = mtvec_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mepc = mepc_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mcause = mcause_r [sched_csr_if.csr_rd_wid]; - assign sched_csr_if.csr_mtval = mtval_r [sched_csr_if.csr_rd_wid]; - - // split/join - wire join_valid; - wire join_is_dvg; - wire join_is_else; - wire [NW_WIDTH-1:0] join_wid; - wire [`VX_CFG_NUM_THREADS-1:0] join_tmask; - wire [PC_BITS-1:0] join_pc; - - reg [PERF_CTR_BITS-1:0] cycles; - - wire schedule_fire = schedule_valid && schedule_ready; - wire schedule_if_fire = schedule_if.valid && schedule_if.ready; -`ifdef VX_CFG_EXT_C_ENABLE - // PC advance is driven by decompress_finished under EXT_C; - `UNUSED_VAR (schedule_if_fire) -`endif - - // branch - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_valid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][NW_WIDTH-1:0] branch_wid; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_taken; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][PC_BITS-1:0] branch_dest; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_trap; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0] branch_is_mret; - wire [`VX_CFG_NUM_ALU_BLOCKS-1:0][3:0] branch_trap_cause; - for (genvar i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin : g_branch_init - assign branch_valid[i] = branch_ctl_if[i].valid; - assign branch_wid[i] = branch_ctl_if[i].wid; - assign branch_taken[i] = branch_ctl_if[i].taken; - assign branch_dest[i] = branch_ctl_if[i].dest; - assign branch_is_trap[i] = branch_ctl_if[i].is_trap; - assign branch_is_mret[i] = branch_ctl_if[i].is_mret; - assign branch_trap_cause[i] = branch_ctl_if[i].trap_cause; - end - - // barriers - wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + // core-wide warp-control resources wire bar_unlock_valid; + wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask; + wire wspawn_apply; + wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; + wire [PC_BITS-1:0] wspawn_pc; + wire [NW_WIDTH-1:0] wspawn_wid; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + VX_warp_shared_unit #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_REQS (`VX_CFG_NUM_SFU_BLOCKS) + ) warp_shared_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .warp_shared_if (warp_shared_if), + `ifdef VX_CFG_EXT_DXA_ENABLE + .txbar_bus_if (txbar_bus_if), + `endif + .active_warps (active_warps), + .bar_unlock_valid(bar_unlock_valid), + .bar_unlock_mask (bar_unlock_mask), + .gbar_bus_if (gbar_bus_if), + .wspawn_apply (wspawn_apply), + .wspawn_wmask (wspawn_wmask), + .wspawn_pc (wspawn_pc), + .wspawn_wid (wspawn_wid), + .wspawn_mscratch (wspawn_mscratch), + .warp_done (cta_warp_done), + .warp_done_wid (cta_warp_done_wid), + `UNUSED_PIN (wspawn_pending) + ); - // wspawn - wspawn_t wspawn; - reg wspawn_valid; - reg [NW_WIDTH-1:0] wspawn_wid; - reg is_single_warp; - - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; - `POP_COUNT(active_warps_cnt, active_warps); - - always @(*) begin - active_warps_n = active_warps; - stalled_warps_n = stalled_warps; - thread_masks_n = thread_masks; - warp_pcs_n = warp_pcs; - - // dispatch warps - if (cta_fire) begin - active_warps_n[cta_wid] = 1; - // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the - // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that - // reloads the entry pointer and kargs before re-calling. - warp_pcs_n[cta_wid] = cta_init ? cta_PC : (warp_pcs[cta_wid] - from_fullPC(`VX_CFG_XLEN'(20))); - thread_masks_n[cta_wid] = cta_tmask; - end - - // decode unlock - if (decode_sched_if.valid && decode_sched_if.unlock) begin - stalled_warps_n[decode_sched_if.wid] = 0; - end - - // wspawn handling - if (wspawn_valid && is_single_warp) begin - active_warps_n |= wspawn.wmask; - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - thread_masks_n[i][0] = 1; - warp_pcs_n[i] = wspawn.pc; - end - end - stalled_warps_n[wspawn_wid] = 0; // unlock warp - end - - // TMC handling - if (warp_ctl_if.tmc_valid) begin - active_warps_n[warp_ctl_if.wid] = (warp_ctl_if.tmc.tmask != 0); - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.tmc.tmask; - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // split handling - if (warp_ctl_if.split_valid) begin - if (warp_ctl_if.split.is_dvg) begin - thread_masks_n[warp_ctl_if.wid] = warp_ctl_if.split.then_tmask; - end - stalled_warps_n[warp_ctl_if.wid] = 0; // unlock warp - end - - // join handling - if (join_valid) begin - if (join_is_dvg) begin - if (join_is_else) begin - warp_pcs_n[join_wid] = join_pc; - end - thread_masks_n[join_wid] = join_tmask; - end - stalled_warps_n[join_wid] = 0; // unlock warp - end - - // barrier unlock handling - if (bar_unlock_valid) begin - stalled_warps_n &= ~bar_unlock_mask; - end - - // wsync unlock: warp pipeline drained - if (warp_ctl_if.wsync_valid) begin - stalled_warps_n[warp_ctl_if.wid] = 0; - end - - // Branch handling - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i]) begin - if (branch_is_trap[i]) begin - // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). - warp_pcs_n[branch_wid[i]] = from_fullPC(mtvec_r[branch_wid[i]] & ~`VX_CFG_XLEN'(3)); - end else if (branch_is_mret[i]) begin - // MRET/SRET/URET: restore the saved PC from mepc. ECALL/EBREAK - // are the only traps and they do not narrow the tmask, so there - // is nothing to restore beyond the PC. - warp_pcs_n[branch_wid[i]] = from_fullPC(mepc_r[branch_wid[i]]); - end else if (branch_taken[i]) begin - warp_pcs_n[branch_wid[i]] = branch_dest[i]; - end - stalled_warps_n[branch_wid[i]] = 0; // unlock warp - end - end - - // stall the warp until decode stage - if (schedule_fire) begin - stalled_warps_n[schedule_wid] = 1; - end + // counters - // advance PC. - `ifdef VX_CFG_EXT_C_ENABLE - // With RVC, the decompressor may emit a 2-byte instruction. Advance - // from the committed warp PC rather than the redirect-muxed next-PC: - // every redirect source (branch/trap/mret, split/join, wspawn) stalls - // its warp from schedule until it resolves, so no redirect writes a - // warp's PC on the same cycle that warp decode-advances. Reading the - // registered PC keeps the +2/+4 adder off the branch/trap redirect - // cone, matching the non-RVC path which advances the pipeline-carried - // PC instead of the combinational next-PC. - if (decode_sched_if.valid) begin - warp_pcs_n[decode_sched_if.wid] = - warp_pcs[decode_sched_if.wid] - + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); - end - `else - if (schedule_if_fire) begin - warp_pcs_n[schedule_if.data.wid] = schedule_if.data.PC + from_fullPC(`VX_CFG_XLEN'(4)); - end - `endif + // a CTA counts from its accept cycle, which busy leaves to the issuer + wire kmu_fire = kmu_bus_if.valid && kmu_bus_if.ready; - `ifdef VX_CFG_EXT_RTU_ENABLE - // A wstall'd TRACE retires (its traversal's first response landed and the - // arm op wrote back the handle): resume the warp so it proceeds to WAIT, - // which returns the response status (terminal or candidate). No trap, no - // redirect — the candidate is serviced inline by the warp's loop. - if (sched_unlock_if.valid) begin - stalled_warps_n[sched_unlock_if.wid] = 1'b0; + // An issue slice commits at most one warp per cycle (warp w belongs to slice + // w % ISSUE_WIDTH), so the commits are counted per slice, not per warp. + wire [`VX_CFG_ISSUE_WIDTH-1:0] committed_slices; + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_committed_slices + wire [PER_ISSUE_WARPS-1:0] slice_warps; + for (genvar w = 0; w < PER_ISSUE_WARPS; ++w) begin : g_warps + assign slice_warps[w] = committed_warps[w * `VX_CFG_ISSUE_WIDTH + i]; end - `endif + assign committed_slices[i] = (| slice_warps); end + reg [PERF_CTR_BITS-1:0] cycles_r; + reg [PERF_CTR_BITS-1:0] instret_r; + wire [`CLOG2(`VX_CFG_ISSUE_WIDTH+1)-1:0] committed_cnt; + `POP_COUNT(committed_cnt, committed_slices); always @(posedge clk) begin if (reset) begin - stalled_warps <= '0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - cycles <= '0; - wspawn_valid <= 0; - warp_pcs <= '0; - active_warps <= '0; - thread_masks <= '0; - is_single_warp <= 0; - mscratch_r <= '0; - mstatus_r <= '0; - mtvec_r <= '0; - mepc_r <= '0; - mcause_r <= '0; - mtval_r <= '0; + cycles_r <= '0; + instret_r <= '0; end else begin - active_warps <= active_warps_n; - stalled_warps <= stalled_warps_n; - thread_masks <= thread_masks_n; - warp_pcs <= warp_pcs_n; - is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); - - // wspawn handling - if (warp_ctl_if.wspawn_valid) begin - wspawn_valid <= 1; - wspawn.wmask <= warp_ctl_if.wspawn.wmask; - wspawn.pc <= warp_ctl_if.wspawn.pc; - wspawn_wid <= warp_ctl_if.wid; - end - if (wspawn_valid && is_single_warp) begin - wspawn_valid <= 0; - // copy mscratch from spawning warp to all newly spawned warps - for (integer i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin - if (wspawn.wmask[i] && (NW_WIDTH'(i) != wspawn_wid)) begin - mscratch_r[i] <= mscratch_r[wspawn_wid]; - end - end - end - - // CTA dispatch: latch this warp's mscratch (param). The per-CTA / - // per-warp tables and the wid->cta_id map live in VX_cta_dispatch. - if (cta_fire) begin - mscratch_r[cta_wid] <= cta_param; - end - - // MSCRATCH write-back from CSR unit (CSR instruction) - if (sched_csr_if.csr_wr_valid) begin - mscratch_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.csr_wr_data; - end - - // Trap CSR write-back from CSR unit (csrw mstatus/mtvec/mepc/...) - if (sched_csr_if.trap_csr_wr_valid[0]) begin - mstatus_r[sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[1]) begin - mtvec_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[2]) begin - mepc_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[3]) begin - mcause_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - if (sched_csr_if.trap_csr_wr_valid[4]) begin - mtval_r [sched_csr_if.csr_wr_wid] <= sched_csr_if.trap_csr_wr_data; - end - - // Hardware trap entry (ECALL/EBREAK): snapshot the faulting PC - // into mepc and the cause into mcause. Ordered after the - // software write so a hardware trap wins a same-cycle conflict. - for (integer i = 0; i < `VX_CFG_NUM_ALU_BLOCKS; ++i) begin - if (branch_valid[i] && branch_is_trap[i]) begin - mepc_r [branch_wid[i]] <= to_fullPC(branch_dest[i]); - mcause_r[branch_wid[i]] <= `VX_CFG_XLEN'(branch_trap_cause[i]); - mtval_r [branch_wid[i]] <= '0; - end - end - - if (busy) begin - cycles <= cycles + 1; + if (busy || kmu_fire) begin + cycles_r <= cycles_r + 1; end + instret_r <= instret_r + PERF_CTR_BITS'(committed_cnt); end end - - // Barrier unit - - VX_bar_unit #( - .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) bar_unit ( - .clk (clk), - .reset (reset), - .req_valid (warp_ctl_if.bar_valid), - .req_wid (warp_ctl_if.wid), - .req_data (warp_ctl_if.bar), - .read_addr (warp_ctl_if.bar_addr), - .read_phase (warp_ctl_if.bar_phase), - .active_warps(active_warps), - .gbar_bus_if(gbar_bus_if), - .unlock_valid(bar_unlock_valid), - .unlock_mask(bar_unlock_mask) - ); - - // split/join handling - - VX_split_join #( - .INSTANCE_ID (`SFORMATF(("%s-splitjoin", INSTANCE_ID))), - .OUT_REG (1) - ) split_join ( - .clk (clk), - .reset (reset), - .split_valid(warp_ctl_if.split_valid), - .sjoin_valid(warp_ctl_if.sjoin_valid), - .wid (warp_ctl_if.wid), - .split (warp_ctl_if.split), - .sjoin (warp_ctl_if.sjoin), - .join_valid (join_valid), - .join_is_dvg(join_is_dvg), - .join_is_else(join_is_else), - .join_wid (join_wid), - .join_tmask (join_tmask), - .join_pc (join_pc), - .stack_wid (warp_ctl_if.dvstack_wid), - .stack_ptr (warp_ctl_if.dvstack_ptr) - ); - - // schedule the next ready warp - - wire [`VX_CFG_NUM_WARPS-1:0] ready_warps = active_warps & ~stalled_warps; - - // Per-warp ibuffer occupancy counter (registered full[i] keeps arbitration - // off the critical path; full_n feeds an externally registered aggregate - // so all_full is valid the same cycle as full[i]). - localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); - - wire [`VX_CFG_NUM_WARPS-1:0] schedule_onehot; - logic [`VX_CFG_NUM_WARPS-1:0] ibuf_full, ibuf_full_n; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_ibuf_cnt - logic [IBUF_CW-1:0] size_r, size_n; - wire incr = schedule_fire && schedule_onehot[i]; - wire decr = schedule_if.ibuf_pop[i]; - assign size_n = size_r + IBUF_CW'(incr) - IBUF_CW'(decr); - assign ibuf_full_n[i] = (size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); - always @(posedge clk) begin - if (reset) begin - size_r <= '0; - ibuf_full[i] <= 1'b0; - end else begin - size_r <= size_n; - ibuf_full[i] <= ibuf_full_n[i]; - end - end + assign cycles = cycles_r; + assign instret = instret_r; + +`ifdef VX_CFG_VM_ENABLE + // Per-core SATP CSR, written by any SFU block's CSR unit. Accesses in one + // cycle take effect in block order: each block reads the value the lower + // blocks leave. + reg [`VX_CFG_XLEN-1:0] satp_r; + wire [`VX_CFG_NUM_SFU_BLOCKS:0][`VX_CFG_XLEN-1:0] satp_chain; + assign satp_chain[0] = satp_r; + for (genvar b = 0; b < `VX_CFG_NUM_SFU_BLOCKS; ++b) begin : g_satp + assign satp[b] = satp_chain[b]; + assign satp_chain[b+1] = satp_wr_valid[b] ? satp_wr_data[b] : satp_chain[b]; end - - wire [`VX_CFG_NUM_WARPS-1:0] preferred_warps = ready_warps & ~ibuf_full; -`ifndef L1_ENABLE - // without L1, we should ensure the icache never stalls, - // because it could deadlock dcache response since they share the same bus. - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = preferred_warps; -`else - reg all_ibuf_full; always @(posedge clk) begin - if (reset) all_ibuf_full <= 1'b0; - else all_ibuf_full <= (& ibuf_full_n); - end - wire [`VX_CFG_NUM_WARPS-1:0] schedule_warps = all_ibuf_full ? ready_warps : preferred_warps; -`endif - - VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) - ) wid_select ( - .data_in (schedule_warps), - .index_out (schedule_wid), - .valid_out (schedule_valid), - .onehot_out(schedule_onehot) - ); - - wire [`VX_CFG_NUM_WARPS-1:0][(`VX_CFG_NUM_THREADS + PC_BITS)-1:0] schedule_data; - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_schedule_data - assign schedule_data[i] = {thread_masks[i], warp_pcs[i]}; + if (reset) begin + satp_r <= '0; + end else begin + satp_r <= satp_chain[`VX_CFG_NUM_SFU_BLOCKS]; + end end - - assign {schedule_tmask, schedule_pc} = { - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-1:(`VX_CFG_NUM_THREADS + PC_BITS)-4], - schedule_data[schedule_wid][(`VX_CFG_NUM_THREADS + PC_BITS)-5:0] - }; - - wire [UUID_WIDTH-1:0] instr_uuid; -`ifdef UUID_ENABLE - VX_uuid_gen #( - .CORE_ID (CORE_ID) - ) uuid_gen ( - .clk (clk), - .reset (reset), - .incr (schedule_fire), - .wid (schedule_wid), - .uuid (instr_uuid) - ); -`else - assign instr_uuid = '0; `endif - // schedule_cta_id is produced by VX_cta_dispatch from its wid->cta_id map. - - VX_elastic_buffer #( - .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), - .SIZE (2), // need to buffer out ready_in - .OUT_REG (1) // should be registered for BRAM acces in fetch unit - ) out_buf ( - .clk (clk), - .reset (reset), - .valid_in (schedule_valid), - .ready_in (schedule_ready), - .data_in ({schedule_tmask, schedule_pc, schedule_wid, schedule_cta_id, instr_uuid}), - .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), - .valid_out (schedule_if.valid), - .ready_out (schedule_if.ready) - ); - - // Track committed instructions - - reg [PERF_CTR_BITS-1:0] instret; - - wire [`VX_CFG_NUM_WARPS-1:0] committed_warps_v = commit_sched_if.committed_warps; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] committed_warps_cnt_v; - `POP_COUNT(committed_warps_cnt_v, committed_warps_v); + // per-slice view of the core-wide events + + wire [FETCH_FSW_W-1:0] wspawn_fsw = wid_to_fsw(wspawn_wid); + + for (genvar p = 0; p < FW; ++p) begin : g_slices + localparam [FETCH_FSW_W-1:0] FSW = FETCH_FSW_W'(p); + + assign slice_if[p].cta_fire = cta_fire[p]; + assign slice_if[p].cta_lid = wid_to_wfs(cta_wid[p]); + assign slice_if[p].cta_PC = cta_PC; + assign slice_if[p].cta_tmask = cta_tmask[p]; + assign slice_if[p].cta_param = cta_param; + assign slice_if[p].cta_init = cta_init[p]; + + assign slice_if[p].warp_write = cta_lane_bcast[p].warp_write; + assign slice_if[p].warp_waddr = wid_to_wfs(cta_lane_bcast[p].warp_waddr); + assign slice_if[p].warp_wdata = cta_lane_bcast[p].warp_wdata; + assign slice_if[p].id_write = cta_lane_bcast[p].id_write; + assign slice_if[p].id_waddr = wid_to_wfs(cta_lane_bcast[p].id_wid); + assign slice_if[p].id_wdata = cta_lane_bcast[p].id_cta; + `UNUSED_VAR ({wid_to_fsw(cta_wid[p]), wid_to_fsw(cta_lane_bcast[p].warp_waddr), wid_to_fsw(cta_lane_bcast[p].id_wid)}) + + assign slice_if[p].wspawn_apply = wspawn_apply; + assign slice_if[p].wspawn_pc = wspawn_pc; + assign slice_if[p].wspawn_src = (wspawn_fsw == FSW); + assign slice_if[p].wspawn_src_lid = wid_to_wfs(wspawn_wid); + assign slice_if[p].wspawn_mscratch = wspawn_mscratch; + + assign slice_if[p].bar_unlock_valid = bar_unlock_valid; + + if (SLICE_RETIRE) begin : g_retire_in_slice + assign slice_if[p].retire_valid = 1'b0; + assign slice_if[p].retire_lid = '0; + end else begin : g_retire + assign slice_if[p].retire_valid = cta_warp_done && (wid_to_fsw(cta_warp_done_wid) == FSW); + assign slice_if[p].retire_lid = wid_to_wfs(cta_warp_done_wid); + end - always @(posedge clk) begin - if (reset) begin - instret <= '0; - end else begin - instret <= instret + PERF_CTR_BITS'(committed_warps_cnt_v); + for (genvar l = 0; l < SW; ++l) begin : g_lw + assign slice_if[p].wspawn_wmask[l] = wspawn_wmask[l * FW + p]; + assign slice_if[p].bar_unlock_mask[l] = bar_unlock_mask[l * FW + p]; end - end - // Track pending instructions per warp + assign slice_if[p].slice_ibuf_full = slice_ibuf_full; + end - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_empty; - wire [`VX_CFG_NUM_WARPS-1:0] pending_warp_alm_empty; - - for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_pending_warps - localparam logic [ISSUE_ISW_W-1:0] isw = wid_to_isw(i); - localparam logic [ISSUE_WIS_W-1:0] wis = wid_to_wis(i); - - VX_pending_size #( - .SIZE (256), - .ALM_EMPTY (1) - ) per_warp_ctr ( - .clk (clk), - .reset (reset), - .incr (issue_sched_if[isw].valid && (issue_sched_if[isw].wis == ISSUE_WIS_W'(wis))), - .decr (commit_sched_if.committed_warps[i]), - .empty (pending_warp_empty[i]), - .alm_empty (pending_warp_alm_empty[i]), - `UNUSED_PIN (full), - `UNUSED_PIN (alm_full), - `UNUSED_PIN (size) - ); + // the slices schedule from the core-wide view: every ibuffer of the core is full + wire all_ibuf_full = (& slice_ibuf_full); + wire [FW-1:0] sched_valid, sched_fire; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] sched_tmask; + for (genvar p = 0; p < FW; ++p) begin : g_sched_status + assign sched_valid[p] = all_ibuf_full ? slice_if[p].sched_valid_full : slice_if[p].sched_valid_room; + assign sched_fire[p] = slice_if[p].sched_fire; + assign sched_tmask[p] = slice_if[p].sched_tmask; end +`ifndef PERF_ENABLE + `UNUSED_VAR ({sched_valid, sched_fire, sched_tmask}) +`endif + // active_warps is the slices' registered state, so a CTA launch shows there one cycle after + // cta_fire; counting the launch itself keeps busy from dropping between the dispatcher going + // idle and the launched warps becoming visible (a CTA launched in a single cycle). wire busy_buf; - `BUFFER_EX(busy_buf, (active_warps_n != 0 || ~(&pending_warp_empty)), 1'b1, 1, 1); + `BUFFER_EX(busy_buf, (active_warps != 0 || (| cta_fire) || ~(&pending_warp_empty)), 1'b1, 1, 1); assign busy = busy_buf || cta_dispatcher_busy; - assign warp_ctl_if.warp_pending_alm_empty = pending_warp_alm_empty; - - // export CSRs - assign sched_csr_if.cycles = cycles; - assign sched_csr_if.instret = instret; - assign sched_csr_if.active_warps = active_warps; - assign sched_csr_if.thread_masks = thread_masks; - - // timeout handling + // timeout handling reg [31:0] timeout_ctr; reg timeout_enable; always @(posedge clk) begin @@ -573,7 +268,7 @@ module VX_scheduler import VX_gpu_pkg::*; #( timeout_ctr <= '0; timeout_enable <= 0; end else begin - if (decode_sched_if.valid && decode_sched_if.unlock) begin + if (| cta_fire) begin timeout_enable <= 1; end if (timeout_enable && active_warps !=0 && active_warps == stalled_warps) begin @@ -590,15 +285,15 @@ module VX_scheduler import VX_gpu_pkg::*; #( localparam SCHED_STALL_TIMEOUT = STALL_TIMEOUT; `endif `ifdef EXT_SCHED_TIMEOUT_DUMP + wire timeout_dump = !reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1)); always @(posedge clk) begin - if (!reset && (timeout_ctr == (SCHED_STALL_TIMEOUT - 1))) begin + if (timeout_dump) begin $display("*** %s scheduler-timeout dump: active=%b stalled=%b", INSTANCE_ID, active_warps, stalled_warps); - for (integer wi = 0; wi < `VX_CFG_NUM_WARPS; ++wi) begin - $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", - wi, stalled_warps[wi], to_fullPC(warp_pcs[wi]), thread_masks[wi]); - end end end + for (genvar p = 0; p < FW; ++p) begin : g_timeout_dump + assign slice_if[p].timeout_dump = timeout_dump; + end `endif `RUNTIME_ASSERT(timeout_ctr < SCHED_STALL_TIMEOUT, ("*** %s timeout: active_warps=%b, stalled_warps=%b", INSTANCE_ID, active_warps, stalled_warps)) @@ -611,17 +306,34 @@ module VX_scheduler import VX_gpu_pkg::*; #( reg [PERF_CTR_BITS-1:0] perf_branches; reg [PERF_CTR_BITS-1:0] perf_divergence; - wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] stalled_warps_cnt; - wire [`CLOG2(`VX_CFG_NUM_ALU_BLOCKS+1)-1:0] branches_cnt; - wire [`CLOG2(`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; - wire schedule_idle = ~schedule_valid; - wire has_divergence = warp_ctl_if.split_valid && warp_ctl_if.split.is_dvg; - wire [`VX_CFG_NUM_THREADS-1:0] issued_threads = {`VX_CFG_NUM_THREADS{schedule_if_fire}} & schedule_if.data.tmask; + wire [FW-1:0][NPA-1:0] slice_branch; + wire [FW-1:0][NPS-1:0] slice_divergence; + for (genvar p = 0; p < FW; ++p) begin : g_perf_slices + assign slice_branch[p] = slice_if[p].branch_valid; + assign slice_divergence[p] = slice_if[p].divergence; + end + + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt, stalled_warps_cnt; + wire [`CLOG2(FW*NPA+1)-1:0] branches_cnt; + wire [`CLOG2(FW*NPS+1)-1:0] divergence_cnt; + wire [`CLOG2(FW+1)-1:0] issued_warps_cnt; + wire [`CLOG2(FW*`VX_CFG_NUM_THREADS+1)-1:0] issued_threads_cnt; + wire [FW-1:0][`VX_CFG_NUM_THREADS-1:0] issued_threads; + for (genvar p = 0; p < FW; ++p) begin : g_issued_threads + assign issued_threads[p] = {`VX_CFG_NUM_THREADS{sched_fire[p]}} & sched_tmask[p]; + end + `POP_COUNT(active_warps_cnt, active_warps); `POP_COUNT(stalled_warps_cnt, stalled_warps); `POP_COUNT(issued_threads_cnt, issued_threads); - `POP_COUNT(branches_cnt, branch_valid); + `POP_COUNT(issued_warps_cnt, sched_fire); + `POP_COUNT(branches_cnt, slice_branch); + `POP_COUNT(divergence_cnt, slice_divergence); + + wire sched_idle = ~(| sched_valid); always @(posedge clk) begin if (reset) begin @@ -633,13 +345,13 @@ module VX_scheduler import VX_gpu_pkg::*; #( perf_branches <= '0; perf_divergence <= '0; end else begin - perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(schedule_idle); + perf_sched_idles <= perf_sched_idles + PERF_CTR_BITS'(sched_idle); perf_active_warps <= perf_active_warps + PERF_CTR_BITS'(active_warps_cnt); perf_stalled_warps <= perf_stalled_warps + PERF_CTR_BITS'(stalled_warps_cnt); - perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(schedule_if_fire); + perf_issued_warps <= perf_issued_warps + PERF_CTR_BITS'(issued_warps_cnt); perf_issued_threads<= perf_issued_threads + PERF_CTR_BITS'(issued_threads_cnt); perf_branches <= perf_branches + PERF_CTR_BITS'(branches_cnt); - perf_divergence <= perf_divergence + PERF_CTR_BITS'(has_divergence); + perf_divergence <= perf_divergence + PERF_CTR_BITS'(divergence_cnt); end end @@ -650,25 +362,9 @@ module VX_scheduler import VX_gpu_pkg::*; #( assign sched_perf.issued_threads= perf_issued_threads; assign sched_perf.branches = perf_branches; assign sched_perf.divergence = perf_divergence; -`endif - -`ifdef DBG_TRACE_PIPELINE - for (genvar w = 0; w < `VX_CFG_NUM_WARPS; ++w) begin : g_trace_warp_status - always @(posedge clk) begin - if (active_warps_n[w] != active_warps[w] - || (active_warps[w] && (stalled_warps_n[w] != stalled_warps[w] - || thread_masks_n[w] != thread_masks[w]))) begin - `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", - $time, INSTANCE_ID, w, active_warps_n[w], stalled_warps_n[w], thread_masks_n[w] - )) - end - end - end - - always @(posedge clk) begin - if (schedule_fire) begin - `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, schedule_wid, schedule_cta_id, to_fullPC(schedule_pc), schedule_tmask, instr_uuid)) - end +`else + for (genvar p = 0; p < FW; ++p) begin : g_perf_unused + `UNUSED_VAR ({slice_if[p].branch_valid, slice_if[p].divergence}) end `endif diff --git a/hw/rtl/core/VX_scoreboard.sv b/hw/rtl/core/VX_scoreboard.sv index 54f9f1e6e8..9c1349c249 100644 --- a/hw/rtl/core/VX_scoreboard.sv +++ b/hw/rtl/core/VX_scoreboard.sv @@ -14,8 +14,7 @@ `include "VX_define.vh" module VX_scoreboard import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter ISSUE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, @@ -30,7 +29,6 @@ module VX_scoreboard import VX_gpu_pkg::*; #( VX_scoreboard_if.master scoreboard_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (ISSUE_ID) `UNUSED_VAR ({writeback_if.data.sop, writeback_if.data.eop, writeback_if.data.wis}) localparam NUM_OPDS = NUM_SRC_OPDS + 1; @@ -126,8 +124,8 @@ module VX_scoreboard import VX_gpu_pkg::*; #( end end - // Writeback release feeds wb_inuse_regs, which the busy check reads; the - // staging reserve is added on top to form the next state, inuse_regs_n. + // Writeback release feeds wb_inuse_regs; the staging reserve is added on + // top to form the next state, inuse_regs_n. reg [NUM_REGS-1:0] wb_inuse_regs; reg [NUM_XREGS-1:0] wb_inuse_xregs; always @(*) begin @@ -164,12 +162,23 @@ module VX_scoreboard import VX_gpu_pkg::*; #( // fire: staging hands off, its rd is reserved, the ibuffer instr moves in // Either way the flop gets the value a post-grant evaluation would. // - // Either outcome checks one of only two instrs, so each is reduced - // against the released in-use set once and the outcomes select bits. + // Either outcome checks one of only two instrs, so each is checked + // once and the outcomes select bits. wire [NUM_REGS-1:0] ibf_regs_mask = ibf_opd_mask[0] | ibf_opd_mask[1] | ibf_opd_mask[2] | ibf_opd_mask[3]; wire [NUM_REGS-1:0] stg_regs_mask = stg_opd_mask[0] | stg_opd_mask[1] | stg_opd_mask[2] | stg_opd_mask[3]; - wire ibf_busy = | (wb_inuse_regs & ibf_regs_mask); - wire stg_busy = | (wb_inuse_regs & stg_regs_mask); + // Each operand reads its in-use bit by index and compares its index with + // this cycle's release, which is shallower than reducing the operand + // masks against the whole released set. + wire wb_release = writeback_fire && writeback_if.data.wb; + wire [NUM_OPDS-1:0] ibf_opd_busy, stg_opd_busy; + for (genvar i = 0; i < NUM_OPDS; ++i) begin : g_opd_busy + assign ibf_opd_busy[i] = ibf_used_rs[i] && inuse_regs[ibf_opds[i]] + && ~(wb_release && (writeback_if.data.rd == ibf_opds[i])); + assign stg_opd_busy[i] = stg_used_rs[i] && inuse_regs[stg_opds[i]] + && ~(wb_release && (writeback_if.data.rd == stg_opds[i])); + end + wire ibf_busy = | ibf_opd_busy; + wire stg_busy = | stg_opd_busy; wire ibf_xbusy = | (wb_inuse_xregs & ibf_xregs_mask); wire stg_xbusy = | (wb_inuse_xregs & stg_xregs_mask); @@ -216,6 +225,7 @@ module VX_scoreboard import VX_gpu_pkg::*; #( `UNUSED_VAR (xregs_busy) `else `UNUSED_VAR (ibuffer_fire) + `UNUSED_VAR ({ibf_regs_mask, stg_regs_mask}) `endif always @(posedge clk) begin diff --git a/hw/rtl/core/VX_sfu_block.sv b/hw/rtl/core/VX_sfu_block.sv new file mode 100644 index 0000000000..aa15c2bb81 --- /dev/null +++ b/hw/rtl/core/VX_sfu_block.sv @@ -0,0 +1,276 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" +`endif + +// One SFU block: the PEs serving the warps with wid mod NUM_SFU_BLOCKS == block_idx. +module VX_sfu_block import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [SFU_BLOCK_W-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + +`ifdef VX_CFG_EXT_F_ENABLE + output wire [NW_WIDTH-1:0] fcsr_wid, + input wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data, + output wire [2:0] fcsr_wr_sel, + output wire [INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data, +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.master rtu_bus_if, + VX_sched_unlock_if.master sched_unlock_if, +`endif + + VX_sched_csr_if.slave sched_csr_if, + VX_warp_shared_if.master warp_shared_if, + VX_warp_ctl_if.master warp_ctl_if, + + // CTA dispatch's context writes + input cta_bcast_t cta_bcast +); + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR (block_idx) +`endif + localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_MUFU_ENABLED; + localparam PE_SEL_BITS = `CLOG2(PE_COUNT); + localparam PE_IDX_WCTL = 0; + localparam PE_IDX_CSRS = 1; +`ifdef VX_CFG_EXT_DXA_ENABLE + localparam PE_IDX_DXA = 2; +`endif +`ifdef VX_CFG_EXT_TEX_ENABLE + localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; +`endif +`ifdef VX_CFG_EXT_RASTER_ENABLE + localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; +`endif +`ifdef VX_CFG_EXT_MUFU_ENABLE + localparam PE_IDX_MUFU = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; +`endif + + VX_execute_if #( + .data_t (sfu_execute_t) + ) pe_execute_if[PE_COUNT](); + + VX_result_if#( + .data_t (sfu_result_t) + ) pe_result_if[PE_COUNT](); + + reg [PE_SEL_BITS-1:0] pe_select; + always @(*) begin + pe_select = PE_SEL_BITS'(PE_IDX_WCTL); + if (inst_sfu_is_csr(execute_if.data.op_type)) begin + pe_select = PE_SEL_BITS'(PE_IDX_CSRS); + end + `ifdef VX_CFG_EXT_DXA_ENABLE + if (execute_if.data.op_type == INST_SFU_DXA) begin + pe_select = PE_SEL_BITS'(PE_IDX_DXA); + end + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + if (execute_if.data.op_type == INST_SFU_TEX) begin + pe_select = PE_SEL_BITS'(PE_IDX_TEX); + end + `endif + `ifdef VX_CFG_EXT_OM_ENABLE + if (execute_if.data.op_type == INST_SFU_OM) begin + pe_select = PE_SEL_BITS'(PE_IDX_OM); + end + `endif + `ifdef VX_CFG_EXT_RASTER_ENABLE + if (execute_if.data.op_type == INST_SFU_RASTER) begin + pe_select = PE_SEL_BITS'(PE_IDX_RASTER); + end + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + if (execute_if.data.op_type == INST_SFU_RTUW) begin + pe_select = PE_SEL_BITS'(PE_IDX_RTUW); + end + `endif + `ifdef VX_CFG_EXT_MUFU_ENABLE + if (execute_if.data.op_type == INST_SFU_MUFU) begin + pe_select = PE_SEL_BITS'(PE_IDX_MUFU); + end + `endif + end + + VX_pe_switch #( + .PE_COUNT (PE_COUNT), + .NUM_LANES (NUM_LANES), + .ARBITER ("R"), + .REQ_OUT_BUF(0), + .RSP_OUT_BUF(3) + ) pe_switch ( + .clk (clk), + .reset (reset), + .pe_sel (pe_select), + .execute_in_if (execute_if), + .result_out_if (result_if), + .execute_out_if (pe_execute_if), + .result_in_if (pe_result_if) + ); + + VX_wctl_unit #( + .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) wctl_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_WCTL]), + .warp_shared_if (warp_shared_if), + .warp_ctl_if (warp_ctl_if), + .result_if (pe_result_if[PE_IDX_WCTL]) + ); + + // The block's copy of the CTA contexts, read at the CSR request's CTA and + // valid a cycle later. + cta_ctx_t cta_ctx; + + VX_dp_ram #( + .DATAW ($bits(cta_ctx_t)), + .SIZE (NUM_CTA_MAX), + .RDW_MODE ("R"), + .OUT_REG (1) + ) cta_ctx_ram ( + .clk (clk), + .reset (reset), + .read (1'b1), + .write (cta_bcast.ctx_write), + .wren (1'b1), + .waddr (cta_bcast.ctx_waddr), + .wdata (cta_bcast.ctx_wdata), + .raddr (pe_execute_if[PE_IDX_CSRS].data.header.cta_id), + .rdata (cta_ctx) + ); + + VX_csr_unit #( + .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) csr_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_CSRS]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid), + .fcsr_rd_data (fcsr_rd_data), + .fcsr_wr_sel (fcsr_wr_sel), + .fcsr_wr_data (fcsr_wr_data), + `endif + .sched_csr_if (sched_csr_if), + .cta_ctx (cta_ctx), + .result_if (pe_result_if[PE_IDX_CSRS]) + ); + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_unit #( + .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))) + ) dxa_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .execute_if (pe_execute_if[PE_IDX_DXA]), + .result_if (pe_result_if[PE_IDX_DXA]), + .dxa_req_bus_if (dxa_req_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_unit #( + .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) tex_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_TEX]), + .result_if (pe_result_if[PE_IDX_TEX]), + .tex_bus_if (tex_bus_if) + ); +`endif + +`ifdef VX_CFG_EXT_OM_ENABLE + // A fragment leaves the shader as a store to the OM aperture through the LSU, + // so the SFU services no OM op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_OM].ready = 1'b1; + assign pe_result_if[PE_IDX_OM].valid = 1'b0; + assign pe_result_if[PE_IDX_OM].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) +`endif + +`ifdef VX_CFG_EXT_RASTER_ENABLE + // The fragment payload reaches the warp at launch, so the SFU services no + // raster op; the PE slot is kept for index stability. + assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; + assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; + assign pe_result_if[PE_IDX_RASTER].data = '0; + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) + `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) + `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), + .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .NUM_LANES (NUM_LANES) + ) rtu_unit ( + .clk (clk), + .reset (reset), + .block_idx (block_idx), + .execute_if (pe_execute_if[PE_IDX_RTUW]), + .result_if (pe_result_if[PE_IDX_RTUW]), + .rtu_bus_if (rtu_bus_if), + .sched_unlock_if (sched_unlock_if) + ); +`endif + +`ifdef VX_CFG_EXT_MUFU_ENABLE + VX_mufu_unit #( + .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), + .NUM_LANES (NUM_LANES) + ) mufu_unit ( + .clk (clk), + .reset (reset), + .execute_if (pe_execute_if[PE_IDX_MUFU]), + .result_if (pe_result_if[PE_IDX_MUFU]) + ); +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_shared.sv b/hw/rtl/core/VX_sfu_shared.sv new file mode 100644 index 0000000000..2d8cfb9c64 --- /dev/null +++ b/hw/rtl/core/VX_sfu_shared.sv @@ -0,0 +1,156 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Merges the SFU blocks' requests onto the core's single DXA, TEX and RTU buses. +module VX_sfu_shared import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if.slave block_dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_dxa_req_bus_if.master dxa_req_bus_if, +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if.slave block_tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_tex_bus_if.master tex_bus_if, +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + VX_rtu_bus_if.slave block_rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_rtu_bus_if.master rtu_bus_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam NB = `VX_CFG_NUM_SFU_BLOCKS; + // the merged request leaves the core: register it when it is a real merge + localparam REQ_OUT_BUF = (NB > 1) ? 3 : 0; + +`ifndef VX_CFG_EXT_DXA_ENABLE +`ifndef VX_CFG_EXT_TEX_ENABLE +`ifndef VX_CFG_EXT_RTU_ENABLE + `UNUSED_VAR ({clk, reset}) + `UNUSED_PARAM (REQ_OUT_BUF) +`endif +`endif +`endif + +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_dxa_req_bus_if dxa_arb_out_if[1](); + + VX_dxa_req_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .OUT_BUF (REQ_OUT_BUF), + .ARBITER ("R") + ) dxa_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_dxa_req_bus_if), + .bus_out_if (dxa_arb_out_if) + ); + + assign dxa_req_bus_if.req_valid = dxa_arb_out_if[0].req_valid; + assign dxa_req_bus_if.req_data = dxa_arb_out_if[0].req_data; + assign dxa_arb_out_if[0].req_ready = dxa_req_bus_if.req_ready; +`endif + +`ifdef VX_CFG_EXT_TEX_ENABLE + VX_tex_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_CORE_TAG_WIDTH) + ) tex_arb_out_if[1](); + + VX_tex_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (TEX_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_RSP (0) + ) tex_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_tex_bus_if), + .bus_out_if (tex_arb_out_if) + ); + + assign tex_bus_if.req_valid = tex_arb_out_if[0].req_valid; + assign tex_bus_if.req_data = tex_arb_out_if[0].req_data; + assign tex_arb_out_if[0].req_ready = tex_bus_if.req_ready; + assign tex_arb_out_if[0].rsp_valid = tex_bus_if.rsp_valid; + assign tex_arb_out_if[0].rsp_data = tex_bus_if.rsp_data; + assign tex_bus_if.rsp_ready = tex_arb_out_if[0].rsp_ready; +`endif + +`ifdef VX_CFG_EXT_RTU_ENABLE + localparam RTU_SEL_W = `UP(`ARB_SEL_BITS(NB, 1)); + + VX_rtu_bus_if #( + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_CORE_TAG_WIDTH), + .SRC_WIDTH (RTU_SEL_W) + ) rtu_arb_out_if[1](); + + VX_rtu_bus_arb #( + .NUM_INPUTS (NB), + .NUM_OUTPUTS (1), + .NUM_LANES (`VX_CFG_NUM_SFU_LANES), + .TAG_WIDTH (RTU_REQ_TAG_WIDTH), + .ARBITER ("R"), + .OUT_BUF_ARM (REQ_OUT_BUF), + .OUT_BUF_REQ (REQ_OUT_BUF), + .OUT_BUF_WIN (0) + ) rtu_arb ( + .clk (clk), + .reset (reset), + .bus_in_if (block_rtu_bus_if), + .bus_out_if (rtu_arb_out_if) + ); + + // A warp lives in exactly one SFU block, so {wid} alone names a beat's owner + // on the core's bus; the socket re-derives the core index. + `UNUSED_VAR (rtu_arb_out_if[0].arm_data.src) + `UNUSED_VAR (rtu_arb_out_if[0].req_data.src) + + assign rtu_bus_if.arm_valid = rtu_arb_out_if[0].arm_valid; + assign rtu_bus_if.arm_data.src = 1'b0; + assign rtu_bus_if.arm_data.wid = rtu_arb_out_if[0].arm_data.wid; + assign rtu_bus_if.arm_data.mask = rtu_arb_out_if[0].arm_data.mask; + assign rtu_bus_if.arm_data.scene_base = rtu_arb_out_if[0].arm_data.scene_base; + assign rtu_bus_if.arm_data.flags = rtu_arb_out_if[0].arm_data.flags; + assign rtu_bus_if.arm_data.cull_mask = rtu_arb_out_if[0].arm_data.cull_mask; + assign rtu_bus_if.arm_data.payload_ptr = rtu_arb_out_if[0].arm_data.payload_ptr; + assign rtu_bus_if.arm_data.tag = rtu_arb_out_if[0].arm_data.tag; + assign rtu_arb_out_if[0].arm_ready = rtu_bus_if.arm_ready; + + assign rtu_bus_if.req_valid = rtu_arb_out_if[0].req_valid; + assign rtu_bus_if.req_data.kind = rtu_arb_out_if[0].req_data.kind; + assign rtu_bus_if.req_data.src = 1'b0; + assign rtu_bus_if.req_data.wid = rtu_arb_out_if[0].req_data.wid; + assign rtu_bus_if.req_data.data = rtu_arb_out_if[0].req_data.data; + assign rtu_bus_if.req_data.cb_action = rtu_arb_out_if[0].req_data.cb_action; + assign rtu_arb_out_if[0].req_ready = rtu_bus_if.req_ready; + + assign rtu_arb_out_if[0].win_valid = rtu_bus_if.win_valid; + assign rtu_arb_out_if[0].win_data = rtu_bus_if.win_data; + assign rtu_bus_if.win_ready = rtu_arb_out_if[0].win_ready; +`endif + +endmodule diff --git a/hw/rtl/core/VX_sfu_unit.sv b/hw/rtl/core/VX_sfu_unit.sv index d7a200d11a..59ba4c8dc3 100644 --- a/hw/rtl/core/VX_sfu_unit.sv +++ b/hw/rtl/core/VX_sfu_unit.sv @@ -13,21 +13,17 @@ `include "VX_define.vh" -module VX_sfu_unit import VX_gpu_pkg::*; -`ifdef VX_CFG_EXT_RASTER_ENABLE -import VX_raster_pkg::*; +`ifdef VX_CFG_EXT_F_ENABLE +`include "VX_fpu_define.vh" `endif -#( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + +module VX_sfu_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, -`ifdef PERF_ENABLE - input sysmem_perf_t sysmem_perf, - input pipeline_perf_t pipeline_perf, -`endif + input wire [CORE_ID_WIDTH-1:0] core_id, // Inputs VX_dispatch_if.slave dispatch_if [`VX_CFG_ISSUE_WIDTH], @@ -37,59 +33,29 @@ import VX_raster_pkg::*; `endif `ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_req_bus_if.master dxa_req_bus_if, - VX_txbar_bus_if.slave dxa_txbar_bus_if, + VX_dxa_req_bus_if.master dxa_req_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif `ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_bus_if.master tex_bus_if, -`endif - -`ifdef VX_CFG_EXT_OM_ENABLE + VX_tex_bus_if.master tex_bus_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - // RASTER has no SFU port at all now: the fragment shader is launched by the - // raster engine (push, not pull) and its stamp arrives in the launch, so - // nothing about a fragment passes through this unit. - `ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_bus_if.master rtu_bus_if, - VX_sched_unlock_if.master sched_unlock_if, + VX_rtu_bus_if.master rtu_bus_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_sched_unlock_if.master sched_unlock_if [`VX_CFG_NUM_SFU_BLOCKS], `endif - VX_sched_csr_if.slave sched_csr_if, - - VX_dcr_csr_if dcr_csr_if, + VX_sched_csr_if.slave sched_csr_if [`VX_CFG_NUM_SFU_BLOCKS], // Outputs VX_commit_if.master commit_if [`VX_CFG_ISSUE_WIDTH], - VX_warp_ctl_if.master warp_ctl_if + VX_warp_shared_if.master warp_shared_if [`VX_CFG_NUM_SFU_BLOCKS], + VX_warp_ctl_if.master warp_ctl_if [`VX_CFG_NUM_SFU_BLOCKS], + + input cta_bcast_t cta_bcast ); - `UNUSED_SPARAM (INSTANCE_ID) - localparam BLOCK_SIZE = 1; - localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; - localparam PE_COUNT = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED + `VX_CFG_EXT_MUFU_ENABLED; - localparam PE_SEL_BITS = `CLOG2(PE_COUNT); - localparam PE_IDX_WCTL = 0; - localparam PE_IDX_CSRS = 1; -`ifdef VX_CFG_EXT_DXA_ENABLE - localparam PE_IDX_DXA = 2; -`endif -`ifdef VX_CFG_EXT_TEX_ENABLE - localparam PE_IDX_TEX = 2 + `VX_CFG_EXT_DXA_ENABLED; -`endif -`ifdef VX_CFG_EXT_OM_ENABLE - localparam PE_IDX_OM = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED; -`endif -`ifdef VX_CFG_EXT_RASTER_ENABLE - localparam PE_IDX_RASTER = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED; -`endif -`ifdef VX_CFG_EXT_RTU_ENABLE - localparam PE_IDX_RTUW = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED; -`endif -`ifdef VX_CFG_EXT_MUFU_ENABLE - localparam PE_IDX_MUFU = 2 + `VX_CFG_EXT_DXA_ENABLED + `VX_CFG_EXT_TEX_ENABLED + `VX_CFG_EXT_OM_ENABLED + `VX_CFG_EXT_RASTER_ENABLED + `VX_CFG_EXT_RTU_ENABLED; -`endif + localparam BLOCK_SIZE = `VX_CFG_NUM_SFU_BLOCKS; + localparam NUM_LANES = `VX_CFG_NUM_SFU_LANES; VX_execute_if #( .data_t (sfu_execute_t) @@ -106,208 +72,63 @@ import VX_raster_pkg::*; ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); - VX_execute_if #( - .data_t (sfu_execute_t) - ) pe_execute_if[PE_COUNT](); - - VX_result_if#( - .data_t (sfu_result_t) - ) pe_result_if[PE_COUNT](); - - reg [PE_SEL_BITS-1:0] pe_select; - always @(*) begin - pe_select = PE_SEL_BITS'(PE_IDX_WCTL); - if (inst_sfu_is_csr(per_block_execute_if[0].data.op_type)) begin - pe_select = PE_SEL_BITS'(PE_IDX_CSRS); - end - `ifdef VX_CFG_EXT_DXA_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_DXA) begin - pe_select = PE_SEL_BITS'(PE_IDX_DXA); - end - `endif - `ifdef VX_CFG_EXT_TEX_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_TEX) begin - pe_select = PE_SEL_BITS'(PE_IDX_TEX); - end - `endif - `ifdef VX_CFG_EXT_OM_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_OM) begin - pe_select = PE_SEL_BITS'(PE_IDX_OM); - end - `endif - `ifdef VX_CFG_EXT_RASTER_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RASTER) begin - pe_select = PE_SEL_BITS'(PE_IDX_RASTER); - end - `endif - `ifdef VX_CFG_EXT_RTU_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_RTUW) begin - pe_select = PE_SEL_BITS'(PE_IDX_RTUW); - end - `endif - `ifdef VX_CFG_EXT_MUFU_ENABLE - if (per_block_execute_if[0].data.op_type == INST_SFU_MUFU) begin - pe_select = PE_SEL_BITS'(PE_IDX_MUFU); - end - `endif +`ifdef VX_CFG_EXT_F_ENABLE + wire [BLOCK_SIZE-1:0][NW_WIDTH-1:0] fcsr_wid; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_rd_data; + wire [BLOCK_SIZE-1:0][2:0] fcsr_wr_sel; + wire [BLOCK_SIZE-1:0][INST_FRM_BITS+`FP_FLAGS_BITS-1:0] fcsr_wr_data; +`endif + + for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks + VX_sfu_block #( + .INSTANCE_ID ((BLOCK_SIZE > 1) ? `SFORMATF(("%s%0d", INSTANCE_ID, block_idx)) : INSTANCE_ID) + ) sfu_block ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .block_idx (SFU_BLOCK_W'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + `ifdef VX_CFG_EXT_F_ENABLE + .fcsr_wid (fcsr_wid[block_idx]), + .fcsr_rd_data (fcsr_rd_data[block_idx]), + .fcsr_wr_sel (fcsr_wr_sel[block_idx]), + .fcsr_wr_data (fcsr_wr_data[block_idx]), + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + .dxa_req_bus_if (dxa_req_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_TEX_ENABLE + .tex_bus_if (tex_bus_if[block_idx]), + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + .rtu_bus_if (rtu_bus_if[block_idx]), + .sched_unlock_if(sched_unlock_if[block_idx]), + `endif + .sched_csr_if (sched_csr_if[block_idx]), + .warp_shared_if (warp_shared_if[block_idx]), + .warp_ctl_if (warp_ctl_if[block_idx]), + .cta_bcast (cta_bcast) + ); end - VX_pe_switch #( - .PE_COUNT (PE_COUNT), - .NUM_LANES (NUM_LANES), - .ARBITER ("R"), - .REQ_OUT_BUF(0), - .RSP_OUT_BUF(3) - ) pe_switch ( - .clk (clk), - .reset (reset), - .pe_sel (pe_select), - .execute_in_if (per_block_execute_if[0]), - .result_out_if (per_block_result_if[0]), - .execute_out_if (pe_execute_if), - .result_in_if (pe_result_if) - ); - - VX_txbar_bus_if txbar_bus_if(); - - VX_wctl_unit #( - .INSTANCE_ID (`SFORMATF(("%s-wctl", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) wctl_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_WCTL]), - .warp_ctl_if(warp_ctl_if), - .txbar_bus_if(txbar_bus_if), - .result_if (pe_result_if[PE_IDX_WCTL]) - ); - - VX_csr_unit #( - .INSTANCE_ID (`SFORMATF(("%s-csr", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) csr_unit ( - .clk (clk), - .reset (reset), - - .execute_if (pe_execute_if[PE_IDX_CSRS]), - - `ifdef PERF_ENABLE - .sysmem_perf (sysmem_perf), - .pipeline_perf (pipeline_perf), - `endif - - `ifdef VX_CFG_EXT_F_ENABLE - .fpu_csr_if (fpu_csr_if), - `endif - - .sched_csr_if (sched_csr_if), - .result_if (pe_result_if[PE_IDX_CSRS]), - .dcr_csr_if (dcr_csr_if) - ); - -`ifdef VX_CFG_EXT_DXA_ENABLE - VX_dxa_unit #( - .INSTANCE_ID (`SFORMATF(("%s-dxa", INSTANCE_ID))), - .CORE_ID (CORE_ID) - ) dxa_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_DXA]), - .result_if (pe_result_if[PE_IDX_DXA]), - .dxa_req_bus_if (dxa_req_bus_if) - ); - - // The only txbar producer is the SMEM-completion path through - // dxa_txbar_bus_if (DXA release); no arbitration is needed. - assign txbar_bus_if.valid = dxa_txbar_bus_if.valid; - assign txbar_bus_if.data = dxa_txbar_bus_if.data; - assign dxa_txbar_bus_if.ready = txbar_bus_if.ready; -`else - assign txbar_bus_if.valid = 1'b0; - assign txbar_bus_if.data = 'x; - `UNUSED_VAR (txbar_bus_if.ready) -`endif - -// The hit window has no FF consumers left. TEX was the last one — it spilled -// its 2x2 quad's eight (u,v) operands into window slots because no RISC-V encoding -// holds eight inputs, and that cost the window two full RAM mirrors. It now takes -// u/v/lod in registers (vx_tex, R4-type) and its shader computes the mip LOD with -// vx_tex_quad_lod(), so the window is down to a single mirror and its only tenant -// is the RTU. - -`ifdef VX_CFG_EXT_TEX_ENABLE - VX_tex_unit #( - .INSTANCE_ID (`SFORMATF(("%s-tex", INSTANCE_ID))), - .CORE_ID (CORE_ID), - .NUM_LANES (NUM_LANES) - ) tex_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_TEX]), - .result_if (pe_result_if[PE_IDX_TEX]), - .tex_bus_if (tex_bus_if) - ); -`endif - -`ifdef VX_CFG_EXT_OM_ENABLE - // OM export (v2): a fragment leaves the shader as a STORE to the OM aperture - // (vx_om_export -> VX_gfx_uops -> the LSU), so the SFU services no OM op. The - // PE slot is retained for index stability and tied off, as RASTER's is. - assign pe_execute_if[PE_IDX_OM].ready = 1'b1; - assign pe_result_if[PE_IDX_OM].valid = 1'b0; - assign pe_result_if[PE_IDX_OM].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_OM].data) -`endif - -`ifdef VX_CFG_EXT_RASTER_ENABLE - // RASTER push (v2): the fragment payload reaches the warp at launch via the - // core-level distributor's window seed (rast_win_wr_*), so the SFU services - // no raster op. The PE slot is retained for index stability and tied off — - // no kernel issues INST_SFU_RASTER under the push model. - assign pe_execute_if[PE_IDX_RASTER].ready = 1'b1; - assign pe_result_if[PE_IDX_RASTER].valid = 1'b0; - assign pe_result_if[PE_IDX_RASTER].data = '0; - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].valid) - `UNUSED_VAR (pe_execute_if[PE_IDX_RASTER].data) - `UNUSED_VAR (pe_result_if[PE_IDX_RASTER].ready) -`endif - -`ifdef VX_CFG_EXT_RTU_ENABLE - VX_rtu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-rtuw", INSTANCE_ID))), - .CORE_ID (CORE_ID), - `ifdef VX_CFG_EXT_RTU_ENABLE - .RTU_TAG_WIDTH (RTU_REQ_TAG_WIDTH), - `endif - .NUM_LANES (NUM_LANES) - ) rtu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_RTUW]), - .result_if (pe_result_if[PE_IDX_RTUW]) - `ifdef VX_CFG_EXT_RTU_ENABLE - , - .rtu_bus_if (rtu_bus_if), - .sched_unlock_if (sched_unlock_if) - `endif - ); -`endif - -`ifdef VX_CFG_EXT_MUFU_ENABLE - VX_mufu_unit #( - .INSTANCE_ID (`SFORMATF(("%s-mufu", INSTANCE_ID))), - .NUM_LANES (NUM_LANES) - ) mufu_unit ( - .clk (clk), - .reset (reset), - .execute_if (pe_execute_if[PE_IDX_MUFU]), - .result_if (pe_result_if[PE_IDX_MUFU]) +`ifdef VX_CFG_EXT_F_ENABLE + VX_csr_fpu #( + .NUM_CSR_PORTS (BLOCK_SIZE), + .NUM_FPU_PORTS (`VX_CFG_NUM_FPU_BLOCKS) + ) csr_fpu ( + .clk (clk), + .reset (reset), + .csr_wid (fcsr_wid), + .csr_rd_data (fcsr_rd_data), + .csr_wr_sel (fcsr_wr_sel), + .csr_wr_data (fcsr_wr_data), + .fpu_csr_if (fpu_csr_if) ); `endif diff --git a/hw/rtl/core/VX_split_join.sv b/hw/rtl/core/VX_split_join.sv index 7da6d4df6d..36fe514539 100644 --- a/hw/rtl/core/VX_split_join.sv +++ b/hw/rtl/core/VX_split_join.sv @@ -15,20 +15,22 @@ module VX_split_join import VX_gpu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter OUT_REG = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS, + parameter WID_W = `LOG2UP(NUM_WARPS), + parameter OUT_REG = 0 ) ( input wire clk, input wire reset, input wire split_valid, input wire sjoin_valid, - input wire [NW_WIDTH-1:0] wid, + input wire [WID_W-1:0] wid, input split_t split, input join_t sjoin, - input wire [NW_WIDTH-1:0] stack_wid, + input wire [WID_W-1:0] stack_wid, output wire join_valid, output wire join_is_dvg, output wire join_is_else, - output wire [NW_WIDTH-1:0] join_wid, + output wire [WID_W-1:0] join_wid, output wire [`VX_CFG_NUM_THREADS-1:0] join_tmask, output wire [PC_BITS-1:0] join_pc, output wire [DV_STACK_SIZEW-1:0] stack_ptr @@ -36,7 +38,7 @@ module VX_split_join import VX_gpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) if (NT_BITS != 0) begin : g_enable - wire [`VX_CFG_NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; + wire [NUM_WARPS-1:0][DV_STACK_SIZEW-1:0] ipdom_wr_ptr; wire [`VX_CFG_NUM_THREADS-1:0] orig_tmask; wire [PC_BITS-1:0] next_pc; wire ipdom_idx; @@ -46,8 +48,10 @@ module VX_split_join import VX_gpu_pkg::*; #( wire sjoin_is_dvg = (sjoin.stack_ptr != ipdom_wr_ptr[wid]); VX_ipdom_stack #( - .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), - .DEPTH (DV_STACK_SIZE) + .WIDTH (`VX_CFG_NUM_THREADS + PC_BITS), + .DEPTH (DV_STACK_SIZE), + .NUM_WARPS (NUM_WARPS), + .WID_W (WID_W) ) ipdom_stack ( .clk (clk), .reset (reset), @@ -66,7 +70,7 @@ module VX_split_join import VX_gpu_pkg::*; #( wire [`VX_CFG_NUM_THREADS-1:0] join_tmask_n = ipdom_idx ? orig_tmask : (~sjoin.tmask & orig_tmask); VX_pipe_register #( - .DATAW (1 + NW_WIDTH + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), + .DATAW (1 + WID_W + 1 + 1 + `VX_CFG_NUM_THREADS + PC_BITS), .RESETW (1), .DEPTH (OUT_REG) ) pipe_reg ( diff --git a/hw/rtl/core/VX_uop_amo.sv b/hw/rtl/core/VX_uop_amo.sv new file mode 100644 index 0000000000..c0e54dd216 --- /dev/null +++ b/hw/rtl/core/VX_uop_amo.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// VX_uop_amo: orders an atomic with the hart's other accesses. +// +// An AMO, LR or SC with .rl is preceded by a release fence (earlier writes +// become visible first) and one with .aq is followed by an acquire fence +// (later reads see what the atomic observed): +// [release fence] atomic [acquire fence] +// The fences are plain LSU fence ops whose aq/rl arguments carry their kind; +// they read and write no register. + +module VX_uop_amo import VX_gpu_pkg::*; ( + input clk, + input reset, + + input ibuffer_t ibuf_in, + output ibuffer_t ibuf_out, + + input wire start, + input wire advance, + input wire [UOP_CTR_W-1:0] uop_idx, + output wire [UOP_CTR_W-1:0] uop_count +); + `UNUSED_VAR ({clk, reset, start, advance}) + + wire aq = ibuf_in.op_args.lsu.amo_aq; + wire rl = ibuf_in.op_args.lsu.amo_rl; + assign uop_count = UOP_CTR_W'(1) + UOP_CTR_W'(aq) + UOP_CTR_W'(rl); + + wire [UOP_CTR_W-1:0] amo_idx = UOP_CTR_W'(rl); + wire is_rel = rl && (uop_idx == '0); + wire is_acq = aq && (uop_idx == amo_idx + UOP_CTR_W'(1)); + + ibuffer_t ibuf_r; + always_comb begin + ibuf_r = ibuf_in; + if (is_rel || is_acq) begin + ibuf_r.op_type = INST_OP_BITS'(INST_LSU_FENCE); + ibuf_r.op_args = '0; + ibuf_r.op_args.lsu.amo_aq = is_acq; + ibuf_r.op_args.lsu.amo_rl = is_rel; + ibuf_r.wb = 1'b0; + ibuf_r.rd_xregs = '0; + ibuf_r.wr_xregs = '0; + ibuf_r.used_rs = '0; + end + end + assign ibuf_out = ibuf_r; + +endmodule diff --git a/hw/rtl/core/VX_uop_packld.sv b/hw/rtl/core/VX_uop_packld.sv index f33483f579..f745fce632 100644 --- a/hw/rtl/core/VX_uop_packld.sv +++ b/hw/rtl/core/VX_uop_packld.sv @@ -13,20 +13,32 @@ `include "VX_define.vh" -// VX_uop_packld: Micro-op sequencer for pack-load instructions. +// VX_uop_packld: Micro-op sequencer for pack-load and wide load/store instructions. // // Expands one vx_packlb_f / vx_packlh_f instruction into N sequential LSU uops: // pack==1 (PACKLB) → 4 uops of INST_LSU_LBU (unsigned byte loads) // pack==2 (PACKLH) → 2 uops of INST_LSU_LHU (unsigned halfword loads) +// pack==3 (vx_flw.d/.q, vx_fsw.d/.q) → 2 or 4 plain flw/fsw uops +// pack==3 with bytesel offset 2/3 (vx_ldm, vx_stm) → 4/8 flw/fsw, one per +// register of a TCU matrix fragment // -// Each uop encodes its index in op_args.lsu.offset[1:0]. -// The dispatcher reads pack (non-zero) to identify pack-load uops and computes: +// A pack-load uop encodes its index in op_args.lsu.offset[1:0]; the LSU AGU +// reads pack (non-zero) to identify it and computes // eff_rs1[lane] = rs1[lane] + rs2[lane] * uop_idx (shift-and-add) -// The pack field is cleared in op_args forwarded to the LSU slice so it sees a -// plain LBU or LHU. The NaN-boxing of the FLW result is handled automatically -// because is_float==1 and rd is a float register (triggers nan-box in lsu_slice). +// A wide uop carries word k in offset[3:2] and register fd+k / fs2+k, and +// leaves with pack cleared, so the LSU sees a plain flw/fsw. The NaN-boxing of +// a loaded word is handled automatically because is_float==1 and rd is a float +// register (triggers nan-box in lsu_block). +// A fragment uop e moves register fd/fs2 + e: it reads its base from +// rs1 | grp(e) and ORs its column into the immediate (whose low 5 bits decode +// set to the shape's code), for the element the TCU expansion reads from that +// register. -module VX_uop_packld import VX_gpu_pkg::*; ( +module VX_uop_packld import +`ifdef VX_CFG_EXT_TCU_ENABLE + VX_tcu_pkg::*, +`endif + VX_gpu_pkg::*; ( input clk, input reset, @@ -41,18 +53,76 @@ module VX_uop_packld import VX_gpu_pkg::*; ( `UNUSED_VAR ({clk, reset, start, advance, uop_idx}) // pack==1 → PACKLB: 4 byte-load uops; - // pack==2 → PACKLH: 2 halfword-load uops + // pack==2 → PACKLH: 2 halfword-load uops; + // pack==3 → bytesel offset 0/1: a wide pair/quad, 2 or 4 uops; + // bytesel offset 2/3: a 4/8-register fragment + wire is_packlb = (ibuf_in.op_args.lsu.pack == 2'b01); wire is_packlh = (ibuf_in.op_args.lsu.pack == 2'b10); - assign uop_count = is_packlh ? UOP_CTR_W'(2) : UOP_CTR_W'(4); + wire is_wide = (ibuf_in.op_args.lsu.pack == 2'b11); +`ifdef VX_CFG_EXT_TCU_ENABLE + wire is_matrix = is_wide && ibuf_in.bytesel[1]; + tcu_ldm_uop_t ldm_uop; + assign ldm_uop = tcu_ldm_uop(ibuf_in.op_args.lsu.offset[2:0], uop_idx[2:0]); + + // uop e of every valid shape reads or writes the register its element + // occupies in the WMMA/WGMMA expansion + for (genvar s = 0; s < 6; ++s) begin : g_ldm_check + localparam bit VALID = (s == TCU_LDM_IDX_A_SP) ? `VX_CFG_TCU_SPARSE_ENABLED + : (s == TCU_LDM_IDX_B_SP) ? (`VX_CFG_TCU_SPARSE_ENABLED && SYM_SPARSE) + : (s == TCU_LDM_IDX_WG_C) ? `VX_CFG_TCU_WGMMA_ENABLED : 1; + if (VALID) begin : g_valid + for (genvar e = 0; e < tcu_ldm_count(3'(s)); ++e) begin : g_uops + `STATIC_ASSERT (tcu_ldm_reg(3'(s), tcu_ldm_uop(3'(s), 3'(e))) == e, + ("vx_ldm shape %0d uop %0d does not invert the TCU register permutation", s, e)) + end + end + end +`endif + // Decode sets the bytesel offset to the uop count: 0 → 2 (PACKLH, a wide + // pair), 1 → 4 (PACKLB, a wide quad), 2 → 4 and 3 → 8 (fragments) +`ifdef VX_CFG_EXT_TCU_ENABLE + wire cnt8 = ibuf_in.bytesel[1] && ibuf_in.bytesel[0]; + wire cnt4 = ibuf_in.bytesel[1] ^ ibuf_in.bytesel[0]; +`else + wire cnt8 = 1'b0; + wire cnt4 = ibuf_in.bytesel[0]; +`endif + assign uop_count = UOP_CTR_W'({cnt8, cnt4, ~(cnt8 || cnt4), 1'b0}); wire [1:0] idx2 = uop_idx[1:0]; - wire [XLENB_W-1:0] byte_size = is_packlh ? '1 : '0; - wire [XLENB_W-1:0] byte_off = is_packlh ? XLENB_W'({idx2[0], 1'b0}) : XLENB_W'(idx2); + wire [XLENB_W-1:0] byte_size = ibuf_in.op_args.lsu.pack[1] ? '1 : '0; + wire [XLENB_W-1:0] byte_off = is_packlb ? XLENB_W'(idx2) : (is_packlh ? XLENB_W'({idx2[0], 1'b0}) : '0); + + // Decode leaves a packed load's offset zero, and a wide access's offset and + // register have their low group bits clear, so each uop ORs its index in: + // into offset[1:0] for a packed load; into offset[3:2] (word k) and the + // register (fd+k, fs2+k) for a wide access, which becomes a plain flw/fsw. + // A fragment uop replaces offset[4:0] (the shape code) with its column and + // adds its register offset, through the same adder the wide uops use. + wire [1:0] wide_idx = is_wide ? idx2 : 2'b00; + wire [1:0] pack_idx = is_wide ? 2'b00 : idx2; + +`ifdef VX_CFG_EXT_TCU_ENABLE + wire [11:0] off_base = is_matrix ? {ibuf_in.op_args.lsu.offset[11:5], 5'd0} : ibuf_in.op_args.lsu.offset; + wire [11:0] off_add = is_matrix ? 12'(ldm_uop.col) : {8'd0, wide_idx, pack_idx}; + wire [4:0] reg_add = is_matrix ? 5'(uop_idx[2:0]) : 5'(wide_idx); + wire [1:0] grp = is_matrix ? ldm_uop.grp : 2'b00; +`endif ibuffer_t ibuf_r; always_comb begin ibuf_r = ibuf_in; - ibuf_r.op_args.lsu.offset = {10'd0, idx2}; + ibuf_r.op_args.lsu.pack = is_wide ? 2'b00 : ibuf_in.op_args.lsu.pack; + `ifdef VX_CFG_EXT_TCU_ENABLE + ibuf_r.op_args.lsu.offset = off_base | off_add; + ibuf_r.rs1 = ibuf_in.rs1 | NUM_REGS_BITS'(grp); + ibuf_r.rd = {ibuf_in.rd[NUM_REGS_BITS-1:5], ibuf_in.rd[4:0] + reg_add}; + ibuf_r.rs2 = {ibuf_in.rs2[NUM_REGS_BITS-1:5], ibuf_in.rs2[4:0] + reg_add}; + `else + ibuf_r.op_args.lsu.offset = ibuf_in.op_args.lsu.offset | {8'd0, wide_idx, pack_idx}; + ibuf_r.rd = ibuf_in.rd | NUM_REGS_BITS'(wide_idx); + ibuf_r.rs2 = ibuf_in.rs2 | NUM_REGS_BITS'(wide_idx); + `endif ibuf_r.bytesel = {byte_size, byte_off}; end assign ibuf_out = ibuf_r; diff --git a/hw/rtl/core/VX_uop_sequencer.sv b/hw/rtl/core/VX_uop_sequencer.sv index 268450164c..2b888a8cce 100644 --- a/hw/rtl/core/VX_uop_sequencer.sv +++ b/hw/rtl/core/VX_uop_sequencer.sv @@ -193,6 +193,28 @@ module VX_uop_sequencer import ); `endif +`ifdef VX_CFG_EXT_A_ENABLE + // ------------------------------------------------------------------ + // Atomic ordering uop expander: fences around an AMO/LR/SC with .aq/.rl + // ------------------------------------------------------------------ + // A dcache that is itself the point of coherence orders the atomic with + // the hart's other accesses already: no fence is owed. + assign uop_in_valid[UOP_AMO] = !DCACHE_SHARED_LLC + && (uop_in_data.ex_type == EX_LSU) + && uop_in_data.op_args.lsu.amo_valid + && (uop_in_data.op_args.lsu.amo_aq || uop_in_data.op_args.lsu.amo_rl); + VX_uop_amo uop_amo ( + .clk (clk), + .reset (reset), + .ibuf_in (uop_in_data), + .start (uop_in_start[UOP_AMO]), + .advance (uop_in_next[UOP_AMO]), + .uop_idx (uop_ctr), + .ibuf_out (uop_out_data[UOP_AMO]), + .uop_count (uop_out_count[UOP_AMO]) + ); +`endif + wire uop_hold = is_uop_input && ~uop_active; assign output_if.valid = uop_active || (input_if.valid && ~uop_hold); diff --git a/hw/rtl/core/VX_uuid_gen.sv b/hw/rtl/core/VX_uuid_gen.sv index 61ec538e35..5e2da82927 100644 --- a/hw/rtl/core/VX_uuid_gen.sv +++ b/hw/rtl/core/VX_uuid_gen.sv @@ -13,31 +13,45 @@ `include "VX_define.vh" +// Per-warp instruction ids. NUM_WARPS is the number of warps counted here: all +// of the core's, or the PER_FETCH_WARPS of one frontend slice (wid = {wfs, slice}). module VX_uuid_gen import VX_gpu_pkg::*; #( - parameter CORE_ID = 0 + parameter NUM_WARPS = `VX_CFG_NUM_WARPS ) ( input wire clk, input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, input wire incr, input wire [NW_WIDTH-1:0] wid, output wire [UUID_WIDTH-1:0] uuid ); localparam GNW_WIDTH = UUID_WIDTH - 32; - reg [31:0] uuid_cntrs [0:`VX_CFG_NUM_WARPS-1]; - reg [`VX_CFG_NUM_WARPS-1:0] has_uuid_cntrs; + localparam LW = `LOG2UP(NUM_WARPS); + + wire [LW-1:0] lid; + if (NUM_WARPS == `VX_CFG_NUM_WARPS) begin : g_core_warps + assign lid = LW'(wid); + end else begin : g_slice_warps + `STATIC_ASSERT (NUM_WARPS == PER_FETCH_WARPS, ("invalid parameter")) + assign lid = wid_to_wfs(wid); + end + + reg [31:0] uuid_cntrs [0:NUM_WARPS-1]; + reg [NUM_WARPS-1:0] has_uuid_cntrs; always @(posedge clk) begin if (reset) begin has_uuid_cntrs <= '0; end else if (incr) begin - has_uuid_cntrs[wid] <= 1; + has_uuid_cntrs[lid] <= 1; end if (incr) begin - uuid_cntrs[wid] <= has_uuid_cntrs[wid] ? (uuid_cntrs[wid] + 1) : 1; + uuid_cntrs[lid] <= has_uuid_cntrs[lid] ? (uuid_cntrs[lid] + 1) : 1; end end - wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(CORE_ID) << NW_BITS) + GNW_WIDTH'(wid); - assign uuid = {g_wid, (has_uuid_cntrs[wid] ? uuid_cntrs[wid] : 0)}; + wire [GNW_WIDTH-1:0] g_wid = (GNW_WIDTH'(core_id) << NW_BITS) + GNW_WIDTH'(wid); + assign uuid = {g_wid, (has_uuid_cntrs[lid] ? uuid_cntrs[lid] : 0)}; endmodule diff --git a/hw/rtl/core/VX_warp_sched.sv b/hw/rtl/core/VX_warp_sched.sv new file mode 100644 index 0000000000..fca7e336b1 --- /dev/null +++ b/hw/rtl/core/VX_warp_sched.sv @@ -0,0 +1,518 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The warp scheduler of one frontend slice: the state of the slice's warps +// (wid = {wfs, slice_id}), its warp selection and a registered fetch request to +// the slice's fetch unit, which names warps by wfs. FU block port q carries the +// traffic of the local warps with wfs mod PORTS == q, addressed by core-wide wid; +// core-wide events arrive through sched_if already localized. +module VX_warp_sched import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "" +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + input wire [FETCH_FSW_W-1:0] slice_id, + + VX_slice_sched_if.slave sched_if, + + // ALU ports + input sched_branch_t [FETCH_ALU_PORTS-1:0] branch, + + // SFU ports + input sched_wctl_t [FETCH_SFU_PORTS-1:0] wctl, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] dvstack_wid, + output wire [FETCH_SFU_PORTS-1:0][DV_STACK_SIZEW-1:0] dvstack_ptr, + output wire [FETCH_SFU_PORTS-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] dvstack_mscratch, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] csr_rd_wid, + output wire sched_csr_rd_t [FETCH_SFU_PORTS-1:0] csr_rd, + input sched_csr_wr_t [FETCH_SFU_PORTS-1:0] csr_wr, +`ifdef VX_CFG_EXT_RTU_ENABLE + input wire [FETCH_SFU_PORTS-1:0] rtu_unlock_valid, + input wire [FETCH_SFU_PORTS-1:0][NW_WIDTH-1:0] rtu_unlock_wid, +`endif + + // per-warp pipeline events, indexed by wfs + VX_decode_sched_if.slave decode_sched_if, + input wire [PER_FETCH_WARPS-1:0] issued, + input wire [PER_FETCH_WARPS-1:0] committed, + input wire [PER_FETCH_WARPS-1:0] ibuf_pop, + output wire [PER_FETCH_WARPS-1:0] pending_alm_empty, + + // fetch request + VX_schedule_if.master schedule_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + localparam NPA = FETCH_ALU_PORTS; + localparam NPS = FETCH_SFU_PORTS; + localparam NPS_BITS = `CLOG2(NPS); + localparam PSW = SW / NPS; // warps per SFU port + localparam KW = `LOG2UP(PSW); + localparam IBUF_CW = $clog2(`VX_CFG_IBUF_SIZE + 1); + + `UNUSED_VAR (schedule_if.ibuf_pop) + + // wfs -> SFU port-local index + function automatic logic [KW-1:0] wfs_to_k(input logic [LW-1:0] wfs); + wfs_to_k = KW'(wfs >> NPS_BITS); + endfunction + + wire [LW-1:0] dec_lid = decode_sched_if.wid[LW-1:0]; + if (NW_WIDTH > LW) begin : g_dec_wid_unused + `UNUSED_VAR (decode_sched_if.wid[NW_WIDTH-1:LW]) + end + + wire [NPA-1:0][LW-1:0] br_lid; + for (genvar q = 0; q < NPA; ++q) begin : g_br_lid + assign br_lid[q] = wid_to_wfs(branch[q].wid); + `UNUSED_VAR (wid_to_fsw(branch[q].wid)) + end + + wire [NPS-1:0][LW-1:0] ctl_lid, csr_rd_lid, csr_wr_lid; +`ifdef VX_CFG_EXT_RTU_ENABLE + wire [NPS-1:0][LW-1:0] rtu_lid; +`endif + + // split/join, one stack per SFU port over the port's warps + wire [NPS-1:0] join_valid; + wire [NPS-1:0] join_is_dvg; + wire [NPS-1:0] join_is_else; + wire [NPS-1:0][KW-1:0] join_k; + wire [NPS-1:0][`VX_CFG_NUM_THREADS-1:0] join_tmask; + wire [NPS-1:0][PC_BITS-1:0] join_pc; + + for (genvar q = 0; q < NPS; ++q) begin : g_sfu_ports + assign ctl_lid[q] = wid_to_wfs(wctl[q].wid); + assign csr_rd_lid[q] = wid_to_wfs(csr_rd_wid[q]); + assign csr_wr_lid[q] = wid_to_wfs(csr_wr[q].wid); + `UNUSED_VAR ({wid_to_fsw(wctl[q].wid), wid_to_fsw(csr_rd_wid[q]), wid_to_fsw(csr_wr[q].wid), wid_to_fsw(dvstack_wid[q])}) + `ifdef VX_CFG_EXT_RTU_ENABLE + assign rtu_lid[q] = wid_to_wfs(rtu_unlock_wid[q]); + `UNUSED_VAR (wid_to_fsw(rtu_unlock_wid[q])) + `endif + + VX_split_join #( + .INSTANCE_ID ((NPS > 1) ? `SFORMATF(("%s-splitjoin%0d", INSTANCE_ID, q)) : `SFORMATF(("%s-splitjoin", INSTANCE_ID))), + .NUM_WARPS (PSW), + .WID_W (KW), + .OUT_REG (1) + ) split_join ( + .clk (clk), + .reset (reset), + .split_valid (wctl[q].split_valid), + .sjoin_valid (wctl[q].sjoin_valid), + .wid (wfs_to_k(ctl_lid[q])), + .split (wctl[q].split), + .sjoin (wctl[q].sjoin), + .join_valid (join_valid[q]), + .join_is_dvg (join_is_dvg[q]), + .join_is_else(join_is_else[q]), + .join_wid (join_k[q]), + .join_tmask (join_tmask[q]), + .join_pc (join_pc[q]), + .stack_wid (wfs_to_k(wid_to_wfs(dvstack_wid[q]))), + .stack_ptr (dvstack_ptr[q]) + ); + end + + wire [SW-1:0] active_warps, stalled_warps; + wire [SW-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [SW-1:0][PC_BITS-1:0] warp_pcs; + wire [SW-1:0] ibuf_full, ibuf_full_n; + wire [SW-1:0] pending_empty; + wire [SW-1:0] cand_onehot; + wire cand_valid; + wire sched_fire; + + reg [SW-1:0][`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch_r; + reg [SW-1:0][`VX_CFG_XLEN-1:0] mstatus_r, mtvec_r, mepc_r, mcause_r, mtval_r; + + for (genvar l = 0; l < SW; ++l) begin : g_warps + localparam [LW-1:0] LID = LW'(l); + localparam QA = l % NPA; + localparam QS = l % NPS; + localparam [KW-1:0] K = KW'(l / NPS); + + reg active_r, active_n; + reg stalled_r, stalled_n; + reg [`VX_CFG_NUM_THREADS-1:0] tmask_r, tmask_n; + reg [PC_BITS-1:0] pc_r, pc_n; + + wire ctl_hit = (ctl_lid[QS] == LID); + wire br_hit = branch[QA].valid && (br_lid[QA] == LID); + wire join_hit = join_valid[QS] && (join_k[QS] == K); + + always @(*) begin + active_n = active_r; + stalled_n = stalled_r; + tmask_n = tmask_r; + pc_n = pc_r; + + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + active_n = 1; + // Reusing a warp for the next CTA skips the one-time prologue and rewinds to the + // kernel's per-CTA dispatch window: a fixed 20-byte (5-instruction) sequence that + // reloads the entry pointer and kargs before re-calling. + pc_n = sched_if.cta_init ? sched_if.cta_PC : (pc_r - from_fullPC(`VX_CFG_XLEN'(20))); + tmask_n = sched_if.cta_tmask; + end + + if (decode_sched_if.valid && decode_sched_if.unlock && (dec_lid == LID)) begin + stalled_n = 0; + end + + if (sched_if.wspawn_apply) begin + if (sched_if.wspawn_wmask[l]) begin + active_n = 1; + if (!(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + tmask_n[0] = 1; + pc_n = sched_if.wspawn_pc; + end + end + if (sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID)) begin + stalled_n = 0; + end + end + + if (wctl[QS].tmc_valid && ctl_hit) begin + active_n = (wctl[QS].tmc.tmask != 0); + tmask_n = wctl[QS].tmc.tmask; + stalled_n = 0; + end + + if (wctl[QS].split_valid && ctl_hit) begin + if (wctl[QS].split.is_dvg) begin + tmask_n = wctl[QS].split.then_tmask; + end + stalled_n = 0; + end + + if (join_hit) begin + if (join_is_dvg[QS]) begin + if (join_is_else[QS]) begin + pc_n = join_pc[QS]; + end + tmask_n = join_tmask[QS]; + end + stalled_n = 0; + end + + if (sched_if.bar_unlock_valid && sched_if.bar_unlock_mask[l]) begin + stalled_n = 0; + end + + // an exiting warp is retired as its CTA accounting takes it, before + // the slot can be relaunched + if (sched_if.retire_valid && (sched_if.retire_lid == LID)) begin + active_n = 0; + tmask_n = '0; + stalled_n = 0; + end + + // wsync unlock: warp pipeline drained + if (wctl[QS].wsync_valid && ctl_hit) begin + stalled_n = 0; + end + + if (br_hit) begin + if (branch[QA].is_trap) begin + // ECALL/EBREAK: redirect to trap vector (mtvec[1:0] = MODE field; mask off to get base address). + pc_n = from_fullPC(mtvec_r[l] & ~`VX_CFG_XLEN'(3)); + end else if (branch[QA].is_mret) begin + // MRET/SRET/URET: restore the saved PC from mepc. + pc_n = from_fullPC(mepc_r[l]); + end else if (branch[QA].taken) begin + pc_n = branch[QA].dest; + end + stalled_n = 0; + end + + // stall the warp until decode stage + if (sched_fire && cand_onehot[l]) begin + stalled_n = 1; + `ifndef VX_CFG_EXT_C_ENABLE + pc_n = pc_r + from_fullPC(`VX_CFG_XLEN'(4)); + `endif + end + + `ifdef VX_CFG_EXT_C_ENABLE + // Every redirect source stalls its warp from schedule until it resolves, so + // no redirect writes a warp's PC on the cycle that warp decode-advances. + if (decode_sched_if.valid && (dec_lid == LID)) begin + pc_n = pc_r + from_fullPC(decode_sched_if.is_rvc ? `VX_CFG_XLEN'(2) : `VX_CFG_XLEN'(4)); + end + `endif + + `ifdef VX_CFG_EXT_RTU_ENABLE + // A wstall'd TRACE retires: resume the warp so it proceeds to WAIT. + if (rtu_unlock_valid[QS] && (rtu_lid[QS] == LID)) begin + stalled_n = 1'b0; + end + `endif + end + + always @(posedge clk) begin + if (reset) begin + active_r <= 0; + stalled_r <= 0; + tmask_r <= '0; + pc_r <= '0; + mscratch_r[l] <= '0; + mstatus_r[l] <= '0; + mtvec_r[l] <= '0; + mepc_r[l] <= '0; + mcause_r[l] <= '0; + mtval_r[l] <= '0; + end else begin + active_r <= active_n; + stalled_r <= stalled_n; + tmask_r <= tmask_n; + pc_r <= pc_n; + + // spawned warps inherit the spawning warp's mscratch + if (sched_if.wspawn_apply && sched_if.wspawn_wmask[l] + && !(sched_if.wspawn_src && (sched_if.wspawn_src_lid == LID))) begin + mscratch_r[l] <= sched_if.wspawn_mscratch; + end + // CTA dispatch: latch this warp's mscratch (param) + if (sched_if.cta_fire && (sched_if.cta_lid == LID)) begin + mscratch_r[l] <= sched_if.cta_param; + end + if (csr_wr[QS].valid && (csr_wr_lid[QS] == LID)) begin + mscratch_r[l] <= csr_wr[QS].data; + end + + if (csr_wr_lid[QS] == LID) begin + if (csr_wr[QS].trap_valid[0]) begin + mstatus_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[1]) begin + mtvec_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[2]) begin + mepc_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[3]) begin + mcause_r[l] <= csr_wr[QS].trap_data; + end + if (csr_wr[QS].trap_valid[4]) begin + mtval_r[l] <= csr_wr[QS].trap_data; + end + end + + // Hardware trap entry (ECALL/EBREAK) wins a same-cycle software write. + if (br_hit && branch[QA].is_trap) begin + mepc_r[l] <= to_fullPC(branch[QA].dest); + mcause_r[l] <= `VX_CFG_XLEN'(branch[QA].trap_cause); + mtval_r[l] <= '0; + end + end + end + + assign active_warps[l] = active_r; + assign stalled_warps[l] = stalled_r; + assign thread_masks[l] = tmask_r; + assign warp_pcs[l] = pc_r; + + // ibuffer occupancy: registered full keeps arbitration off the critical path + logic [IBUF_CW-1:0] ibuf_size_r, ibuf_size_n; + logic ibuf_full_r; + wire ibuf_incr = sched_fire && cand_onehot[l]; + assign ibuf_size_n = ibuf_size_r + IBUF_CW'(ibuf_incr) - IBUF_CW'(ibuf_pop[l]); + always @(posedge clk) begin + if (reset) begin + ibuf_size_r <= '0; + ibuf_full_r <= 1'b0; + end else begin + ibuf_size_r <= ibuf_size_n; + ibuf_full_r <= ibuf_full_n[l]; + end + end + assign ibuf_full_n[l] = (ibuf_size_n == IBUF_CW'(`VX_CFG_IBUF_SIZE)); + assign ibuf_full[l] = ibuf_full_r; + + VX_pending_size #( + .SIZE (256), + .ALM_EMPTY (1) + ) pending_ctr ( + .clk (clk), + .reset (reset), + .incr (issued[l]), + .decr (committed[l]), + .empty (pending_empty[l]), + .alm_empty (pending_alm_empty[l]), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + + `ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (active_n != active_r + || (active_r && (stalled_n != stalled_r || tmask_n != tmask_r))) begin + `TRACE(1, ("%t: %s warp-state: wid=%0d, active=%b, stalled=%b, tmask=%b\n", + $time, INSTANCE_ID, wfs_to_wid(LID, slice_id), active_n, stalled_n, tmask_n)) + end + end + `endif + end + + // the slice's lowest ready warp + wire [SW-1:0] ready_warps = active_warps & ~stalled_warps; + wire [SW-1:0] preferred_warps = ready_warps & ~ibuf_full; +`ifndef L1_ENABLE + // without L1, the icache must never stall: it shares the bus with dcache responses + wire [SW-1:0] full_warps = preferred_warps; + wire [SW-1:0] schedule_warps = preferred_warps; + `UNUSED_VAR (sched_if.slice_ibuf_full) +`else + // once every ibuffer of the core is full, any ready warp may fetch + wire [SW-1:0] full_warps = ready_warps; + wire [SW-1:0] schedule_warps = (& sched_if.slice_ibuf_full) ? full_warps : preferred_warps; +`endif + + wire [`UP(FETCH_WFS_BITS)-1:0] cand_index; + VX_priority_encoder #( + .N (SW) + ) wid_select ( + .data_in (schedule_warps), + .index_out (cand_index), + .valid_out (cand_valid), + .onehot_out(cand_onehot) + ); + wire [LW-1:0] cand_lid = LW'(cand_index); + + // CTA CSRs and CTA ids, one store per SFU port over the port's warps + wire [NPS-1:0][NCTA_WIDTH-1:0] port_cta_id; + + for (genvar q = 0; q < NPS; ++q) begin : g_csr_cta + wire [LW-1:0] warp_waddr = sched_if.warp_waddr; + wire [LW-1:0] id_waddr = sched_if.id_waddr; + + VX_csr_cta #( + .NUM_WARPS (PSW) + ) csr_cta ( + .clk (clk), + .reset (reset), + .warp_write (sched_if.warp_write && ((32'(warp_waddr) % NPS) == q)), + .warp_waddr (wfs_to_k(warp_waddr)), + .warp_wdata (sched_if.warp_wdata), + .id_write (sched_if.id_write && ((32'(id_waddr) % NPS) == q)), + .id_waddr (wfs_to_k(id_waddr)), + .id_wdata (sched_if.id_wdata), + .rd_lid (wfs_to_k(csr_rd_lid[q])), + .rd_cta_rank (csr_rd[q].cta_rank), + .rd_lane (csr_rd[q].cta_lane), + .sched_lid (wfs_to_k(cand_lid)), + .sched_cta_id (port_cta_id[q]) + ); + + assign csr_rd[q].thread_mask = thread_masks[csr_rd_lid[q]]; + assign csr_rd[q].mscratch = mscratch_r[csr_rd_lid[q]]; + assign csr_rd[q].mstatus = mstatus_r[csr_rd_lid[q]]; + assign csr_rd[q].mtvec = mtvec_r[csr_rd_lid[q]]; + assign csr_rd[q].mepc = mepc_r[csr_rd_lid[q]]; + assign csr_rd[q].mcause = mcause_r[csr_rd_lid[q]]; + assign csr_rd[q].mtval = mtval_r[csr_rd_lid[q]]; + + assign dvstack_mscratch[q] = mscratch_r[wid_to_wfs(dvstack_wid[q])]; + end + + wire [NCTA_WIDTH-1:0] cand_cta_id = port_cta_id[32'(cand_lid) % NPS]; + + // fetch request, registered at the slice boundary + wire sched_ready; + assign sched_fire = cand_valid && sched_ready; + + wire [UUID_WIDTH-1:0] sched_uuid; +`ifdef UUID_ENABLE + VX_uuid_gen #( + .NUM_WARPS (SW) + ) uuid_gen ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .incr (sched_fire), + .wid (wfs_to_wid(cand_lid, slice_id)), + .uuid (sched_uuid) + ); +`else + assign sched_uuid = '0; + `UNUSED_VAR (core_id) +`ifndef DBG_TRACE_PIPELINE + `UNUSED_VAR (slice_id) +`endif +`endif + + VX_elastic_buffer #( + .DATAW (`VX_CFG_NUM_THREADS + PC_BITS + NW_WIDTH + NCTA_WIDTH + UUID_WIDTH), + .SIZE (2), + .OUT_REG (1) + ) sched_buf ( + .clk (clk), + .reset (reset), + .valid_in (cand_valid), + .ready_in (sched_ready), + .data_in ({thread_masks[cand_lid], warp_pcs[cand_lid], NW_WIDTH'(cand_lid), cand_cta_id, sched_uuid}), + .data_out ({schedule_if.data.tmask, schedule_if.data.PC, schedule_if.data.wid, schedule_if.data.cta_id, schedule_if.data.uuid}), + .valid_out (schedule_if.valid), + .ready_out (schedule_if.ready) + ); + + wire schedule_if_fire = schedule_if.valid && schedule_if.ready; + + // status + assign sched_if.active = active_warps; + assign sched_if.stalled = stalled_warps; + assign sched_if.ibuf_full = (& ibuf_full); + assign sched_if.pending_empty = pending_empty; + + // performance events + for (genvar q = 0; q < NPA; ++q) begin : g_branch_valid + assign sched_if.branch_valid[q] = branch[q].valid; + end + for (genvar q = 0; q < NPS; ++q) begin : g_divergence + assign sched_if.divergence[q] = wctl[q].split_valid && wctl[q].split.is_dvg; + end + assign sched_if.sched_valid_room = (| preferred_warps); + assign sched_if.sched_valid_full = (| full_warps); + assign sched_if.sched_fire = schedule_if_fire; + assign sched_if.sched_tmask = schedule_if.data.tmask; + +`ifdef EXT_SCHED_TIMEOUT_DUMP + always @(posedge clk) begin + if (sched_if.timeout_dump) begin + for (integer l = 0; l < SW; ++l) begin + $display(" wid=%0d stalled=%0d pc=0x%0h tmask=%b", + wfs_to_wid(LW'(l), slice_id), stalled_warps[l], to_fullPC(warp_pcs[l]), thread_masks[l]); + end + end + end +`endif + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (sched_fire) begin + `TRACE(1, ("%t: %s dispatch: wid=%0d, cta_id=%0d, PC=0x%0h, tmask=%b (#%0d)\n", $time, INSTANCE_ID, + wfs_to_wid(cand_lid, slice_id), cand_cta_id, to_fullPC(warp_pcs[cand_lid]), thread_masks[cand_lid], sched_uuid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_warp_shared_unit.sv b/hw/rtl/core/VX_warp_shared_unit.sv new file mode 100644 index 0000000000..de5ed3db5f --- /dev/null +++ b/hw/rtl/core/VX_warp_shared_unit.sv @@ -0,0 +1,268 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// The core-wide warp-control resources: barrier unit, wspawn and warp retirement. +// One request per cycle across the requesters, round-robin; a request that +// asks for it is completed back to its requester. Transaction-barrier +// completions use the barrier unit when no request does. +// +// Several requesters register their requests toward the arbiter, which acts on +// the grant and completes it registered. A sole requester is connected +// directly: it is completed in the cycle it asks, and its request is registered +// here instead, so every request takes effect at the same distance from its +// instruction in both cases. +module VX_warp_shared_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_REQS = 1 +) ( + input wire clk, + input wire reset, + + input wire [CORE_ID_WIDTH-1:0] core_id, + + VX_warp_shared_if.slave warp_shared_if [NUM_REQS], +`ifdef VX_CFG_EXT_DXA_ENABLE + VX_txbar_bus_if.slave txbar_bus_if, +`endif + input wire [`VX_CFG_NUM_WARPS-1:0] active_warps, + + // barrier release + output wire bar_unlock_valid, + output wire [`VX_CFG_NUM_WARPS-1:0] bar_unlock_mask, + VX_gbar_bus_if.master gbar_bus_if, + + // wspawn, applied once a single warp remains + output wire wspawn_apply, + output wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask, + output wire [PC_BITS-1:0] wspawn_pc, + output wire [NW_WIDTH-1:0] wspawn_wid, + output wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch, + + // warp retirement + output wire warp_done, + output wire [NW_WIDTH-1:0] warp_done_wid, + + output wire wspawn_pending +); + localparam REQ_DATAW = $bits(warp_shared_req_t); + localparam SEL_W = `LOG2UP(NUM_REQS); + localparam DIRECT = (NUM_REQS == 1); + + wire [NUM_REQS-1:0] req_valid_in; + wire [NUM_REQS-1:0][REQ_DATAW-1:0] req_data_in; + wire [NUM_REQS-1:0] req_ready_in; + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_in + assign req_valid_in[i] = warp_shared_if[i].req_valid; + assign req_data_in[i] = warp_shared_if[i].req_data; + assign warp_shared_if[i].req_ready = req_ready_in[i]; + end + + wire req_valid; + warp_shared_req_t req; + wire [SEL_W-1:0] req_sel; + + VX_stream_arb #( + .NUM_INPUTS (NUM_REQS), + .NUM_OUTPUTS (1), + .DATAW (REQ_DATAW), + .ARBITER ("R"), + .OUT_BUF (0) + ) req_arb ( + .clk (clk), + .reset (reset), + .valid_in (req_valid_in), + .ready_in (req_ready_in), + .data_in (req_data_in), + .data_out (req), + .valid_out (req_valid), + .ready_out (1'b1), + .sel_out (req_sel) + ); + + wire is_bar = req_valid && (req.kind == WSH_BAR); + wire is_wspawn = req_valid && (req.kind == WSH_WSPAWN); + wire is_exit = req_valid && (req.kind == WSH_EXIT); + + // wspawn and retirement act on the registered request + localparam ACT_DATAW = NW_WIDTH + $bits(wspawn_t) + `VX_CFG_MEM_ADDR_WIDTH; + wire act_wspawn, act_exit; + wire [NW_WIDTH-1:0] act_wid; + wspawn_t act_wspawn_data; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] act_mscratch; + if (DIRECT) begin : g_act_reg + VX_pipe_register #( + .DATAW (2 + ACT_DATAW), + .RESETW (2) + ) act_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}), + .data_out ({act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch}) + ); + end else begin : g_act + assign {act_wspawn, act_exit, act_wid, act_wspawn_data, act_mscratch} + = {is_wspawn, is_exit, req.wid, req.wspawn, req.mscratch}; + end + + // barrier unit + + wire txbar_req_valid; + barrier_t txbar_req; + wire [BAR_ADDR_W-1:0] bar_read_addr; +`ifdef VX_CFG_EXT_DXA_ENABLE + assign txbar_bus_if.ready = ~is_bar; + wire txbar_fire = txbar_bus_if.valid && txbar_bus_if.ready; + assign bar_read_addr = is_bar ? req.bar_addr : txbar_bus_if.data.addr; + // a completion releases one pending transaction event + barrier_t txbar_req_w; + assign txbar_req_w.id = '0; + assign txbar_req_w.is_event = 1'b1; + assign txbar_req_w.is_sync = 1'b0; + assign txbar_req_w.is_global = 1'b0; + assign txbar_req_w.is_arrive = 1'b0; + assign txbar_req_w.phase = ~txbar_bus_if.data.is_done; + assign txbar_req_w.size_m1 = '0; + VX_pipe_register #( + .DATAW (1 + $bits(barrier_t)), + .RESETW (1) + ) txbar_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({txbar_fire, txbar_req_w}), + .data_out ({txbar_req_valid, txbar_req}) + ); +`else + assign bar_read_addr = req.bar_addr; + assign txbar_req_valid = 1'b0; + assign txbar_req = '0; +`endif + + // the barrier request applies the cycle after its phase is read, as the + // read must see the barrier before this arrival + reg bar_valid_r; + barrier_t bar_r; + reg [NW_WIDTH-1:0] bar_wid_r; + always @(posedge clk) begin + if (reset) begin + bar_valid_r <= 1'b0; + end else begin + bar_valid_r <= is_bar; + end + bar_r <= req.bar; + bar_wid_r <= req.wid; + end + + wire bar_read_phase; + + VX_bar_unit #( + .INSTANCE_ID (`SFORMATF(("%s-barrier", INSTANCE_ID))) + ) bar_unit ( + .clk (clk), + .reset (reset), + .core_id (core_id), + .req_valid (bar_valid_r || txbar_req_valid), + .req_wid (bar_wid_r), + .req_data (bar_valid_r ? bar_r : txbar_req), + .read_addr (bar_read_addr), + .read_phase (bar_read_phase), + .active_warps (active_warps), + .gbar_bus_if (gbar_bus_if), + .unlock_valid (bar_unlock_valid), + .unlock_mask (bar_unlock_mask) + ); + + // wspawn + + reg wspawn_valid_r; + wspawn_t wspawn_r; + reg [NW_WIDTH-1:0] wspawn_wid_r; + reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch_r; + reg is_single_warp; + + wire [`CLOG2(`VX_CFG_NUM_WARPS+1)-1:0] active_warps_cnt; + `POP_COUNT(active_warps_cnt, active_warps); + + assign wspawn_apply = wspawn_valid_r && is_single_warp; + + always @(posedge clk) begin + if (reset) begin + wspawn_valid_r <= 1'b0; + is_single_warp <= 1'b0; + end else begin + is_single_warp <= (active_warps_cnt == $bits(active_warps_cnt)'(1)); + if (act_wspawn) begin + wspawn_valid_r <= 1'b1; + end + if (wspawn_apply) begin + wspawn_valid_r <= 1'b0; + end + end + if (act_wspawn) begin + wspawn_r <= act_wspawn_data; + wspawn_wid_r <= act_wid; + wspawn_mscratch_r <= act_mscratch; + end + end + + assign wspawn_wmask = wspawn_r.wmask; + assign wspawn_pc = wspawn_r.pc; + assign wspawn_wid = wspawn_wid_r; + assign wspawn_mscratch = wspawn_mscratch_r; + assign wspawn_pending = wspawn_valid_r; + + // warp retirement + + assign warp_done = act_exit; + assign warp_done_wid = act_wid; + + // completion + + wire [NUM_REQS-1:0] rsp_valid = NUM_REQS'(req_valid && req.has_rsp) << req_sel; + wire [NUM_REQS-1:0] rsp_valid_out; + wire rsp_phase_out; + if (DIRECT) begin : g_rsp_direct + assign rsp_valid_out = rsp_valid; + assign rsp_phase_out = bar_read_phase; + end else begin : g_rsp_reg + reg [NUM_REQS-1:0] rsp_valid_r; + reg rsp_phase_r; + always @(posedge clk) begin + if (reset) begin + rsp_valid_r <= '0; + end else begin + rsp_valid_r <= rsp_valid; + end + rsp_phase_r <= bar_read_phase; + end + assign rsp_valid_out = rsp_valid_r; + assign rsp_phase_out = rsp_phase_r; + end + + for (genvar i = 0; i < NUM_REQS; ++i) begin : g_rsp + assign warp_shared_if[i].rsp_valid = rsp_valid_out[i]; + assign warp_shared_if[i].rsp_phase = rsp_phase_out; + end + +`ifdef DBG_TRACE_PIPELINE + always @(posedge clk) begin + if (req_valid) begin + `TRACE(1, ("%t: %s request: sel=%0d, kind=%0d, wid=%0d\n", $time, INSTANCE_ID, req_sel, req.kind, req.wid)) + end + end +`endif + +endmodule diff --git a/hw/rtl/core/VX_wctl_unit.sv b/hw/rtl/core/VX_wctl_unit.sv index f2a3857ab5..7b137d586a 100644 --- a/hw/rtl/core/VX_wctl_unit.sv +++ b/hw/rtl/core/VX_wctl_unit.sv @@ -22,16 +22,15 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // Inputs VX_execute_if.slave execute_if, - VX_txbar_bus_if.slave txbar_bus_if, // Outputs VX_warp_ctl_if.master warp_ctl_if, + VX_warp_shared_if.master warp_shared_if, VX_result_if.master result_if ); `UNUSED_SPARAM (INSTANCE_ID) - localparam LANE_BITS = `CLOG2(NUM_LANES); localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam WCTL_WIDTH = $bits(tmc_t) + $bits(wspawn_t) + $bits(split_t) + $bits(join_t) + $bits(barrier_t); + localparam WCTL_WIDTH = $bits(tmc_t) + $bits(split_t) + $bits(join_t); `UNUSED_VAR (execute_if.data.rs3_data) @@ -50,25 +49,31 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire is_wsync = (execute_if.data.op_type == INST_SFU_WSYNC); wire wctl_valid; - wire wspawn_valid, tmc_valid, split_valid, sjoin_valid, bar_valid, wsync_valid; - - wire [`UP(LANE_BITS)-1:0] last_tid; - if (LANE_BITS != 0) begin : g_last_tid - VX_priority_encoder #( - .N (NUM_LANES), - .REVERSE (1) - ) last_tid_select ( - .data_in (execute_if.data.header.tmask), - .index_out (last_tid), - `UNUSED_PIN (onehot_out), - `UNUSED_PIN (valid_out) - ); - end else begin : g_no_tid - assign last_tid = 0; + wire tmc_valid, split_valid, sjoin_valid, wsync_valid; + + // The last active lane's operands, and whether their thread masks are + // empty (reduced per lane, ahead of the select), are picked by the mask's + // own select tree rather than by an encoded lane index driving a wide mux. + wire [NUM_LANES-1:0][2*`VX_CFG_XLEN+1:0] lane_scalars; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lane_scalars + assign lane_scalars[i] = {(| execute_if.data.rs1_data[i][`VX_CFG_NUM_THREADS-1:0]), + (| execute_if.data.rs2_data[i][`VX_CFG_NUM_THREADS-1:0]), + execute_if.data.rs1_data[i], + execute_if.data.rs2_data[i]}; end - wire [`VX_CFG_XLEN-1:0] rs1_data = execute_if.data.rs1_data[last_tid]; - wire [`VX_CFG_XLEN-1:0] rs2_data = execute_if.data.rs2_data[last_tid]; + wire rs1_tmask_nz, rs2_tmask_nz; + wire [`VX_CFG_XLEN-1:0] rs1_data, rs2_data; + VX_find_first #( + .N (NUM_LANES), + .DATAW (2*`VX_CFG_XLEN+2), + .REVERSE (1) + ) last_lane_sel ( + .valid_in (execute_if.data.header.tmask), + .data_in (lane_scalars), + .data_out ({rs1_tmask_nz, rs2_tmask_nz, rs1_data, rs2_data}), + `UNUSED_PIN (valid_out) + ); `UNUSED_VAR (rs1_data) wire not_pred = execute_if.data.op_args.wctl.is_cond_neg; @@ -82,9 +87,10 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( logic [`VX_CFG_NUM_THREADS-1:0] else_tmask; if (PID_BITS != 0) begin : g_pid - reg [`VX_CFG_NUM_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; + reg [SFU_BLOCK_WARPS-1:0][2*`VX_CFG_NUM_THREADS-1:0] tmask_table; - wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[execute_if.data.header.wid]; + wire [SFU_BLOCK_NW_W-1:0] tmask_lid = wid_to_sfu_local(execute_if.data.header.wid); + wire [2*`VX_CFG_NUM_THREADS-1:0] tmask_r = tmask_table[tmask_lid]; always @(*) begin {else_tmask, then_tmask} = execute_if.data.header.sop ? '0 : tmask_r; @@ -94,7 +100,7 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( always @(posedge clk) begin if (execute_if.valid) begin - tmask_table[execute_if.data.header.wid] <= {else_tmask, then_tmask}; + tmask_table[tmask_lid] <= {else_tmask, then_tmask}; end end end else begin : g_no_pid @@ -108,7 +114,6 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( // tmc / pred wire [`VX_CFG_NUM_THREADS-1:0] pred_mask = has_then ? then_tmask : rs2_data[`VX_CFG_NUM_THREADS-1:0]; - assign tmc_valid = wctl_valid && (is_tmc || is_pred); assign tmc.tmask = is_pred ? pred_mask : rs1_data[`VX_CFG_NUM_THREADS-1:0]; // split @@ -137,62 +142,153 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( wire [BAR_ADDR_W-1:0] wctl_bar_addr; `CONCAT(wctl_bar_addr, rs1_data[NW_BITS-1:0], rs1_data[BAR_ID_SHIFT +: NB_BITS], NW_BITS, NB_BITS) - wire wctl_bar_enable = wctl_valid && is_bar; - // expect_tx variant of barrier_arrive: rs2[31]=1 flags pre-registration // of pending transaction events. Routed through the existing event path // (is_event + phase=1) so VX_bar_unit's increment branch handles it. - wire is_tx_expect = wctl_bar_enable + wire is_tx_expect = is_bar && execute_if.data.op_args.wctl.is_bar_arrive && rs2_data[31]; - assign bar_valid = wctl_bar_enable || txbar_bus_if.valid; assign bar.id = rs1_data[BAR_ID_SHIFT +: NB_BITS]; - assign bar.is_event = (txbar_bus_if.valid && ~wctl_bar_enable) || is_tx_expect; + assign bar.is_event = is_tx_expect; assign bar.is_sync = execute_if.data.op_args.wctl.is_sync_bar; - assign bar.is_global= wctl_bar_enable && rs1_data[31]; + assign bar.is_global= rs1_data[31]; // expect_tx is not a normal arrival (it does not advance arrive_count) assign bar.is_arrive= (execute_if.data.op_args.wctl.is_bar_arrive && ~rs2_data[31]) || execute_if.data.op_args.wctl.is_sync_bar; - // expect_tx → phase=1 (attach/increment); txbar release → phase=0 - assign bar.phase = is_tx_expect ? 1'b1 : (wctl_bar_enable ? rs2_data[0] : ~txbar_bus_if.data.is_done); + // expect_tx → phase=1 (attach/increment) + assign bar.phase = is_tx_expect ? 1'b1 : rs2_data[0]; // For expect_tx: size_m1 carries (count - 1); for normal arrive: (num_warps - 1). // bar_unit will + 1 when consuming on the event path so the actual increment is `count`. assign bar.size_m1 = rs2_data[BAR_SIZE_W-1:0] - BAR_SIZE_W'(1); - assign txbar_bus_if.ready = ~wctl_bar_enable; - // wspawn wire [`VX_CFG_NUM_WARPS-1:0] wspawn_wmask; for (genvar i = 0; i < `VX_CFG_NUM_WARPS; ++i) begin : g_wspawn_wmask assign wspawn_wmask[i] = (i < rs1_data[NW_BITS:0]) && (i != execute_if.data.header.wid); end - assign wspawn_valid = wctl_valid && is_wspawn; assign wspawn.wmask = wspawn_wmask; assign wspawn.pc = from_fullPC(rs2_data); // Scheduler response setup assign warp_ctl_if.dvstack_wid = execute_if.data.header.wid; - assign warp_ctl_if.bar_addr = wctl_bar_enable ? wctl_bar_addr : txbar_bus_if.data.addr; - - // Send scheduler request // WSYNC: hold execute until this warp's pipeline is drained (alm_empty=1 // means only WSYNC itself is pending — all prior instructions committed). wire wsync_drain = execute_if.valid && is_wsync - && !warp_ctl_if.warp_pending_alm_empty[execute_if.data.header.wid]; + && !warp_ctl_if.warp_pending_alm_empty[wid_to_sfu_local(execute_if.data.header.wid)]; // BAR (vx_barrier or vx_barrier_arrive) drains LSU before continuing. - wire bar_drain = execute_if.valid && is_bar && !warp_ctl_if.lsu_sched_drained; + wire lsu_sched_drained = (& warp_ctl_if.lsu_sched_empty) && warp_ctl_if.mmu_empty; + wire bar_drain = execute_if.valid && is_bar && !lsu_sched_drained; + + // Barriers, wspawn and warp retirement are core-wide requests to the shared + // unit. Only a barrier arrive writes a result, the barrier phase, so it alone + // completes once the unit has served it; every other request is posted, + // completing as it is accepted, and the warp resumes through its unlock path. + wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; + // tmc.tmask == 0; a predicate mask with active then-lanes is never empty + wire is_exit = (is_tmc && ~rs1_tmask_nz) || (is_pred && ~has_then && ~rs2_tmask_nz); + wire is_shared = execute_if.data.header.eop && (is_bar || is_wspawn || is_exit); + wire sh_posted = is_shared && ~bar_rsp_valid; + + // Ready is decided from the opcode: whether a tmc/pred exits depends on its + // operand mask, so every tmc/pred waits for room for a posted request. + wire may_share = execute_if.data.header.eop && (is_bar || is_wspawn || is_tmc || is_pred); + wire may_post = may_share && ~bar_rsp_valid; + + // an exit is retired by the shared unit, which deactivates the warp itself + assign tmc_valid = wctl_valid && (is_tmc || is_pred) && !is_shared; + + localparam SH_IDLE = 2'd0, SH_WAIT = 2'd1, SH_DONE = 2'd2; + reg [1:0] sh_state; + reg sh_phase; + + warp_shared_req_t sh_req; + assign sh_req.kind = is_bar ? WSH_BAR : (is_wspawn ? WSH_WSPAWN : WSH_EXIT); + assign sh_req.has_rsp = ~sh_posted; + assign sh_req.wid = execute_if.data.header.wid; + assign sh_req.bar = bar; + assign sh_req.bar_addr = wctl_bar_addr; + assign sh_req.wspawn = wspawn; + assign sh_req.mscratch = warp_ctl_if.mscratch; + + // A sole requester reaches the shared unit directly and is answered in the + // cycle it asks; with several, requests are registered toward the unit's + // arbiter and answered two cycles later. + localparam SH_DIRECT = (`VX_CFG_NUM_SFU_BLOCKS == 1); + + wire rsp_buf_ready; + // a posted or directly answered request is sent only in the cycle its instruction completes + wire sh_req_valid = execute_if.valid && is_shared && (sh_state == SH_IDLE) && !bar_drain + && ((!sh_posted && !SH_DIRECT) || rsp_buf_ready); + wire sh_req_ready; + + if (SH_DIRECT) begin : g_sh_direct + assign warp_shared_if.req_valid = sh_req_valid; + assign warp_shared_if.req_data = sh_req; + assign sh_req_ready = warp_shared_if.req_ready; + end else begin : g_sh_buf + VX_elastic_buffer #( + .DATAW ($bits(warp_shared_req_t)), + .SIZE (2), + .OUT_REG (1) + ) sh_req_buf ( + .clk (clk), + .reset (reset), + .valid_in (sh_req_valid), + .ready_in (sh_req_ready), + .data_in (sh_req), + .data_out (warp_shared_if.req_data), + .valid_out (warp_shared_if.req_valid), + .ready_out (warp_shared_if.req_ready) + ); + end + + wire sh_req_fire = sh_req_valid && sh_req_ready; + wire sh_can_req = (sh_state == SH_IDLE) && !bar_drain && sh_req_ready; + // a direct request is answered in the cycle it is sent, so it never waits + wire sh_ok = ~may_share || (sh_state == SH_DONE) + || (!SH_DIRECT && (sh_state == SH_WAIT) && warp_shared_if.rsp_valid) + || (sh_can_req && (may_post || SH_DIRECT)); + wire exec_ok = rsp_buf_ready && !wsync_drain && sh_ok && (may_share || !bar_drain); + assign execute_if.ready = exec_ok; wire execute_fire = execute_if.valid && execute_if.ready; assign wctl_valid = execute_fire && execute_if.data.header.eop; + always @(posedge clk) begin + if (reset) begin + sh_state <= SH_IDLE; + end else begin + case (sh_state) + SH_IDLE: begin + if (sh_req_fire && !sh_posted && !execute_fire) begin + sh_state <= warp_shared_if.rsp_valid ? SH_DONE : SH_WAIT; + end + end + SH_WAIT: begin + if (warp_shared_if.rsp_valid) begin + sh_state <= execute_fire ? SH_IDLE : SH_DONE; + end + end + default: begin + if (execute_fire) begin + sh_state <= SH_IDLE; + end + end + endcase + end + if (warp_shared_if.rsp_valid) begin + sh_phase <= warp_shared_if.rsp_phase; + end + end + assign wsync_valid = wctl_valid && is_wsync; VX_pipe_register #( - .DATAW (6 + NW_WIDTH + WCTL_WIDTH), + .DATAW (4 + NW_WIDTH + WCTL_WIDTH), .RESETW (1) ) wctl_reg ( .clk (clk), @@ -200,56 +296,46 @@ module VX_wctl_unit import VX_gpu_pkg::*; #( .enable (1'b1), .data_in ({ tmc_valid, - wspawn_valid, split_valid, sjoin_valid, - bar_valid, wsync_valid, execute_if.data.header.wid, tmc, - wspawn, split, - sjoin, - bar + sjoin }), .data_out ({ warp_ctl_if.tmc_valid, - warp_ctl_if.wspawn_valid, warp_ctl_if.split_valid, warp_ctl_if.sjoin_valid, - warp_ctl_if.bar_valid, warp_ctl_if.wsync_valid, warp_ctl_if.wid, warp_ctl_if.tmc, - warp_ctl_if.wspawn, warp_ctl_if.split, - warp_ctl_if.sjoin, - warp_ctl_if.bar + warp_ctl_if.sjoin }) ); // Send result + wire rsp_phase = warp_shared_if.rsp_valid ? warp_shared_if.rsp_phase : sh_phase; wire [DV_STACK_SIZEW-1:0] dvstack_ptr_r; - wire bar_rsp_valid = is_bar && execute_if.data.op_args.wctl.is_bar_arrive; wire bar_rsp_valid_r; wire bar_phase_r; - wire rsp_buf_ready; VX_elastic_buffer #( .DATAW ($bits(sfu_header_t) + DV_STACK_SIZEW + 1 + 1), .SIZE (2) ) rsp_buf ( .clk (clk), .reset (reset), - .valid_in (execute_if.valid && !wsync_drain && !bar_drain), + .valid_in (execute_if.valid && exec_ok), .ready_in (rsp_buf_ready), - .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, warp_ctl_if.bar_phase, bar_rsp_valid}), - .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), + .data_in ({execute_if.data.header, warp_ctl_if.dvstack_ptr, rsp_phase, bar_rsp_valid}), + .data_out ({result_if.data.header, dvstack_ptr_r, bar_phase_r, bar_rsp_valid_r}), .valid_out (result_if.valid), .ready_out (result_if.ready) ); - assign execute_if.ready = rsp_buf_ready && !wsync_drain && !bar_drain; // Result data for (genvar i = 0; i < NUM_LANES; ++i) begin : g_result_if diff --git a/hw/rtl/cp/VX_cp_axil_regfile.sv b/hw/rtl/cp/VX_cp_axil_regfile.sv index 806a996da9..e874dd36c8 100644 --- a/hw/rtl/cp/VX_cp_axil_regfile.sv +++ b/hw/rtl/cp/VX_cp_axil_regfile.sv @@ -15,7 +15,9 @@ // 0x000 CP_CTRL RW bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS RO bit0=busy, bit1=error // 0x008 CP_DEV_CAPS RO [7:0]NUM_QUEUES | [15:8]RING_SIZE_LOG2_MAX -// [23:16]AXI_TID_WIDTH +// [23:16]AXI_TID_WIDTH | [24]VM_ENABLED +// [28]DMA_LLC_CLIENT +// 0x00C GPU_DEV_CAPS_EXT RO device-config caps extension: [7:0]SOCKET_SIZE // 0x010 CP_CYCLE_LO RO free-running cycle counter low 32 bits // 0x014 CP_CYCLE_HI RO high 32 bits // 0x018 GPU_DEV_CAPS_LO RO device-config caps, low 32 bits @@ -51,7 +53,9 @@ module VX_cp_axil_regfile parameter int ADDR_W = 16, // Static device-caps fields (set at synthesis time from VX_cp_pkg). parameter int RING_SIZE_LOG2_MAX = VX_CP_RING_SIZE_LOG2_C, - parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C + parameter int AXI_TID_W = VX_CP_AXI_TID_WIDTH_C, + // The device-side DMA reaches memory through the last-level cache. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -75,6 +79,9 @@ module VX_cp_axil_regfile // Programmed state out to every CPE. output cpe_state_t q_state [NUM_QUEUES], + // CP_SATP: page-table root of the DMA's device operands. + output wire [63:0] satp, + // One-cycle reset pulse per queue when the host writes Q_CONTROL.reset. output logic q_reset_pulse [NUM_QUEUES], @@ -114,6 +121,8 @@ module VX_cp_axil_regfile // ---- Global registers ---- logic [31:0] r_cp_ctrl; + logic [63:0] r_satp; + assign satp = r_satp; logic [63:0] r_cycle_count; always_ff @(posedge clk) begin @@ -128,7 +137,8 @@ module VX_cp_axil_regfile localparam int GPU_BANK_ADDR_W = `VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - `CLOG2(`VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256, + `STATIC_ASSERT(`VX_CFG_NUM_WARPS < 128 && `VX_CFG_NUM_CLUSTERS < 128 && GPU_CLUSTER_SIZE < 256 + && `VX_CFG_SOCKET_SIZE < 256, ("GPU_DEV_CAPS exact-count fields overflow")) wire [63:0] gpu_dev_caps = { @@ -147,6 +157,11 @@ module VX_cp_axil_regfile 8'(`VX_ISA_IMPL_ID) }; + wire [31:0] gpu_dev_caps_ext = { + 24'd0, + 8'(`VX_CFG_SOCKET_SIZE) + }; + wire [63:0] gpu_isa_caps = { 32'(`VX_CFG_MISA_EXT), 2'(`CLOG2(`VX_CFG_XLEN)-4), @@ -204,25 +219,22 @@ module VX_cp_axil_regfile case (addr[5:0]) 6'h00: return r_cp_ctrl; 6'h04: return {30'd0, cp_error, cp_busy}; - 6'h08: return {8'd0, + 6'h08: return {3'd0, + 1'(DMA_LLC_CLIENT), + 3'd0, + 1'(`VX_CFG_VM_ENABLED), 8'(AXI_TID_W), 8'(RING_SIZE_LOG2_MAX), 8'(NUM_QUEUES)}; + 6'h0C: return gpu_dev_caps_ext; 6'h10: return r_cycle_count[31:0]; 6'h14: return r_cycle_count[63:32]; 6'h18: return gpu_dev_caps[31:0]; 6'h1C: return gpu_dev_caps[63:32]; 6'h20: return gpu_isa_caps[31:0]; 6'h24: return gpu_isa_caps[63:32]; - // Block padding -- see the 16-byte block note on is_decoded(). Read - // as zero; they exist so their enclosing block is fully populated. - // 0x28/0x2C are CP_SATP_LO/HI in the software ABI (sw/runtime/common/ - // device.cpp), i.e. the CP DMA MMU page-table root. This CP has no - // MMU -- there is no satp anywhere in hw/rtl/cp -- so it correctly - // reports VM_ENABLED=0 in CP_DEV_CAPS and the runtime never writes - // them. Reading zero is the honest answer; implement them for real if - // an MMU is added. - 6'h0C, 6'h28, 6'h2C: return 32'd0; + 6'h28: return r_satp[31:0]; + 6'h2C: return r_satp[63:32]; // AFU host-port drain counters (debug, read-only). 6'h30: return dbg_host_w_counts; 6'h34: return dbg_host_r_counts; @@ -347,6 +359,7 @@ module VX_cp_axil_regfile automatic logic [5:0] off; if (reset) begin r_cp_ctrl <= '0; + r_satp <= '0; for (int i = 0; i < NUM_QUEUES; ++i) begin r_ring_base[i] <= '0; r_head_addr[i] <= '0; @@ -378,6 +391,10 @@ module VX_cp_axil_regfile if (wr_data_buf[1]) begin for (int i = 0; i < NUM_QUEUES; ++i) q_reset_pulse[i] <= 1'b1; end + end else if (is_global(wr_addr_buf, 8'h28)) begin + r_satp[31:0] <= wr_data_buf; + end else if (is_global(wr_addr_buf, 8'h2C)) begin + r_satp[63:32] <= wr_data_buf; end else if (decode_queue(wr_addr_buf, qid, off)) begin case (off) 6'h00: r_ring_base[qid][31:0] <= wr_data_buf; diff --git a/hw/rtl/cp/VX_cp_core.sv b/hw/rtl/cp/VX_cp_core.sv index 9ff16e575a..28995c62fe 100644 --- a/hw/rtl/cp/VX_cp_core.sv +++ b/hw/rtl/cp/VX_cp_core.sv @@ -81,7 +81,9 @@ module VX_cp_core parameter int ADDR_W = 64, parameter int DATA_W = 512, parameter int ID_W = VX_CP_AXI_TID_WIDTH_C, - parameter int AXIL_AW = 16 + parameter int AXIL_AW = 16, + // axi_dev enters the last-level cache's client slot instead of memory. + parameter int DMA_LLC_CLIENT = 0 )( input wire clk, input wire reset, @@ -147,9 +149,12 @@ module VX_cp_core logic fetch_idle [NUM_QUEUES]; logic engine_idle [NUM_QUEUES]; + wire [63:0] cp_satp; + VX_cp_axil_regfile #( - .NUM_QUEUES (NUM_QUEUES), - .ADDR_W (AXIL_AW) + .NUM_QUEUES (NUM_QUEUES), + .ADDR_W (AXIL_AW), + .DMA_LLC_CLIENT (DMA_LLC_CLIENT) ) u_regfile ( .clk (clk), .reset (reset), @@ -161,6 +166,7 @@ module VX_cp_core .q_error (q_error_to_reg), .last_dcr_rsp (dcr_last_rsp_data), .q_state (q_state), + .satp (cp_satp), .q_reset_pulse (q_reset_pulse), .q_clear_ack (q_clear), .dbg_host_w_counts (dbg_host_w_counts), @@ -358,7 +364,8 @@ module VX_cp_core .dcr_req_addr (gpu_if_int.dcr_req_addr), .dcr_req_data (gpu_if_int.dcr_req_data), .dcr_rsp_valid (gpu_if_int.dcr_rsp_valid), - .dcr_rsp_data (gpu_if_int.dcr_rsp_data) + .dcr_rsp_data (gpu_if_int.dcr_rsp_data), + .gpu_busy (gpu_if_int.busy) ); `UNUSED_VAR (gpu_if_int.dcr_req_ready) @@ -374,6 +381,7 @@ module VX_cp_core .grant (any_dma_grant), .cmd (granted_dma_cmd), .done (dma_done), + .satp (cp_satp), .axi_host (dma_host_axi), .axi_dev (dma_dev_axi) ); diff --git a/hw/rtl/cp/VX_cp_dcr_proxy.sv b/hw/rtl/cp/VX_cp_dcr_proxy.sv index 94428c07ed..3ae9a3d583 100644 --- a/hw/rtl/cp/VX_cp_dcr_proxy.sv +++ b/hw/rtl/cp/VX_cp_dcr_proxy.sv @@ -15,13 +15,19 @@ // when valid) → DONE → IDLE. // // For CMD_CACHE_FLUSH (cmd.arg0 = number of cores): -// A cache flush is a per-core DCR-read to VX_DCR_BASE_CACHE_FLUSH whose -// response is the per-core flush-complete signal (see VX_dcr_data). One -// CMD_CACHE_FLUSH sweeps that read across every core — REQ/WAIT_RSP per -// core — and retires only once the last core's flush completes. This is -// the ACQUIRE_MEM model: a single command in the ring, executed by -// the CP across all cores, that the host posts after CMD_LAUNCH so it -// sees coherent results. The host fills cmd.arg0 from VX_CAPS_NUM_CORES. +// IDLE → START (DCR write to VX_DCR_BASE_CACHE_FLUSH, which every core +// receives and starts flushing on; F_FLUSH_L1_ONLY sets +// VX_CACHE_FLUSH_L1_ONLY in its value) → then REQ/WAIT_RSP once per core: a +// DCR read of VX_DCR_BASE_CACHE_FLUSH answered when that core's flush has +// completed (see VX_dcr_data). The cores flush concurrently, so their dcache +// flushes reach the shared caches together and share one walk; the reads +// are serial because the DCR response bus cannot carry two at once. The +// command retires once the last core reports and, for a full flush, once +// Vortex busy drops: busy covers every write until memory acknowledges it, +// so a DMA after the flush cannot overtake the write-backs. This is the ACQUIRE_MEM +// model: a single command in the ring, executed by the CP across all cores, +// that the host posts after CMD_LAUNCH so it sees coherent results. The +// host fills cmd.arg0 from VX_CAPS_NUM_CORES. // // The most-recent read response is published on `last_rsp_data` and is // also exposed on the AXI-Lite regfile so the host can poll it after @@ -36,7 +42,7 @@ module VX_cp_dcr_proxy input wire grant, // verilator lint_off UNUSED - // Only cmd.hdr.opcode, cmd.arg0, and cmd.arg1 are read here. arg2 and + // Only cmd.hdr.opcode/flags, cmd.arg0, and cmd.arg1 are read here. arg2 and // profile_slot pass through untouched on the way to the engine; the // top-level instantiation hands us the full struct. input cmd_t cmd, @@ -54,11 +60,14 @@ module VX_cp_dcr_proxy output logic [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output logic [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + + input wire gpu_busy ); - typedef enum logic [1:0] { + typedef enum logic [2:0] { S_IDLE, + S_START, // CMD_CACHE_FLUSH: broadcast write starting every core S_REQ, // hold dcr_req_valid until consumed (single cycle here) S_WAIT_RSP, // read commands only S_DONE @@ -71,6 +80,7 @@ module VX_cp_dcr_proxy state_e state; logic pending_is_read; logic pending_is_flush; + logic pending_l1_only; // cmd fields are only valid for one cycle (grant pulse); capture on IDLE → REQ. logic [`VX_DCR_ADDR_BITS-1:0] pending_addr; logic [`VX_DCR_DATA_BITS-1:0] pending_data; @@ -84,11 +94,17 @@ module VX_cp_dcr_proxy wire [`VX_DCR_DATA_BITS-1:0] cmd_data = cmd.arg1[`VX_DCR_DATA_BITS-1:0]; wire [CIDW-1:0] cmd_ncores = cmd.arg0[CIDW-1:0]; + // The last-level cache forwards a full token behind its write-backs, so they + // are already counted in busy when the last core reports. An L1-only flush + // writes nothing back past the last-level cache. + wire flush_drain = pending_is_flush && !pending_l1_only && gpu_busy; + always_ff @(posedge clk) begin if (reset) begin state <= S_IDLE; pending_is_read <= 1'b0; pending_is_flush <= 1'b0; + pending_l1_only <= 1'b0; pending_addr <= '0; pending_data <= '0; rsp_data_r <= '0; @@ -100,18 +116,22 @@ module VX_cp_dcr_proxy if (grant) begin pending_is_read <= is_read; pending_is_flush <= is_flush; + pending_l1_only <= cmd.hdr.flags[F_FLUSH_L1_ONLY]; pending_addr <= cmd_addr; pending_data <= cmd_data; if (is_flush) begin flush_total <= cmd_ncores; flush_cid <= '0; // a 0-core flush (degenerate) retires immediately - state <= (cmd_ncores == '0) ? S_DONE : S_REQ; + state <= (cmd_ncores == '0) ? S_DONE : S_START; end else begin state <= S_REQ; end end end + S_START: begin + state <= S_REQ; + end S_REQ: begin // The Vortex DCR bus consumes the request in a single cycle // (req_valid handshakes combinationally; no req_ready backpressure). @@ -133,7 +153,9 @@ module VX_cp_dcr_proxy end end S_DONE: begin - state <= S_IDLE; + if (!flush_drain) begin + state <= S_IDLE; + end end default: state <= S_IDLE; endcase @@ -141,19 +163,20 @@ module VX_cp_dcr_proxy end always_comb begin - dcr_req_valid = (state == S_REQ); + dcr_req_valid = (state == S_START) || (state == S_REQ); if (pending_is_flush) begin - // a cache flush is a per-core DCR read to VX_DCR_BASE_CACHE_FLUSH; - // the target core index rides in the low 16 bits of dcr_req_data. - dcr_req_rw = 1'b0; + // the START write reaches every core; each per-core read carries the + // target core index in the low 16 bits of dcr_req_data. + dcr_req_rw = (state == S_START); dcr_req_addr = `VX_DCR_ADDR_BITS'(`VX_DCR_BASE_CACHE_FLUSH); - dcr_req_data = `VX_DCR_DATA_BITS'(flush_cid); + dcr_req_data = (state == S_START) ? (pending_l1_only ? `VX_DCR_DATA_BITS'(`VX_CACHE_FLUSH_L1_ONLY) : '0) + : `VX_DCR_DATA_BITS'(flush_cid); end else begin dcr_req_rw = !pending_is_read; dcr_req_addr = pending_addr; dcr_req_data = pending_data; end - done = (state == S_DONE); + done = (state == S_DONE) && !flush_drain; last_rsp_data = rsp_data_r; end diff --git a/hw/rtl/cp/VX_cp_dma.sv b/hw/rtl/cp/VX_cp_dma.sv index 0357b9427a..de454a37f4 100644 --- a/hw/rtl/cp/VX_cp_dma.sv +++ b/hw/rtl/cp/VX_cp_dma.sv @@ -27,7 +27,11 @@ // not pipelined. // // FSM: -// S_IDLE : grant -> latch op/dst/src/size -> S_SETUP +// S_IDLE : grant -> latch op/dst/src/size -> S_XLATE_AR +// (no device operand to translate) -> S_SETUP +// S_XLATE_AR/S_XLATE_R : page-table walk of each device operand, one PTE +// read per level on the device port; the leaf rewrites the +// operand with its physical address -> S_SETUP // S_SETUP : size 0 -> S_DONE; else size the chunk -> S_REQ_AR // S_REQ_AR : drive AR on the read port; arready -> S_READ // S_READ : capture rdata beats into buf_r; last beat -> S_REQ_AW @@ -56,6 +60,12 @@ // host polls, and the completion line that follows it) fire. // Device-direction writes don't need this: nothing reads them through a // foreign port on the strength of Q_SEQNUM alone. +// +// VIRTUAL MEMORY. With a page-table root in CP_SATP, a device operand is a +// virtual address, as on a GPU copy engine. Every allocation maps to one +// contiguous physical range, so translating the operand's base covers the +// whole transfer. Commands flagged F_MEM_PHYSICAL (the page tables +// themselves) skip the walk. // ============================================================================ module VX_cp_dma @@ -71,6 +81,9 @@ module VX_cp_dma input cmd_t cmd, output logic done, + // CP_SATP: page-table root for the device operands (0 or bare: physical). + input wire [63:0] satp, + // Host-memory AXI master (command-ring side / upload source / download dst). VX_mem_axi_if.master axi_host, // Device-memory AXI master. @@ -82,10 +95,20 @@ module VX_cp_dma localparam int BCNT_W = 7; // chunk length 1..64 typedef enum logic [3:0] { - S_IDLE, S_SETUP, S_REQ_AR, S_READ, S_REQ_AW, S_WRITE, S_WAIT_B, - S_FLUSH_AR, S_FLUSH_R, S_DONE + S_IDLE, S_XLATE_AR, S_XLATE_R, S_SETUP, S_REQ_AR, S_READ, S_REQ_AW, + S_WRITE, S_WAIT_B, S_FLUSH_AR, S_FLUSH_R, S_DONE } state_e; + // Page-table geometry: Sv32 at XLEN=32, Sv39 at XLEN=64. + localparam int PAGE_LOG2 = `VX_VM_PAGE_LOG2_SIZE; + localparam int PT_LEVELS = `VX_VM_PT_LEVEL; + localparam int LEVEL_BITS = `CLOG2(`VX_VM_PT_SIZE / `VX_VM_PTE_SIZE); + localparam int PTE_BITS = `VX_VM_PTE_SIZE * 8; + localparam int PTE_SHIFT = `CLOG2(`VX_VM_PTE_SIZE); + localparam int PTE_SEL_W = 6 - PTE_SHIFT; + localparam int PPN_W = (PTE_BITS == 32) ? 22 : 44; + localparam int LVL_W = `UP(`CLOG2(PT_LEVELS)); + state_e state; logic [7:0] op_r; // latched opcode (host/dev routing) logic [63:0] dst_r, src_r; @@ -124,6 +147,50 @@ module VX_cp_dma wire rd_from_host = (cp_opcode_e'(op_r) == CMD_MEM_WRITE); // upload: read host wire wr_to_host = (cp_opcode_e'(op_r) == CMD_MEM_READ); // download: write host + // ---- Page-table walk of the device operands ---- +`ifdef VX_VM_ADDR_MODE_SV32 + wire satp_on = satp[31]; + wire [PPN_W-1:0] satp_ppn = satp[PPN_W-1:0]; + `UNUSED_VAR (satp[63:32]) + `UNUSED_VAR (satp[30:PPN_W]) +`else + wire satp_on = (satp[63:60] != '0); + wire [PPN_W-1:0] satp_ppn = satp[PPN_W-1:0]; + `UNUSED_VAR (satp[59:PPN_W]) +`endif + + wire cmd_is_mem = (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_WRITE) + || (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_READ) + || (cp_opcode_e'(cmd.hdr.opcode) == CMD_MEM_COPY); + wire cmd_xlate = (`VX_CFG_VM_ENABLED != 0) && satp_on && cmd_is_mem + && ~cmd.hdr.flags[F_MEM_PHYSICAL]; + wire cmd_xl_dst = cmd_xlate && (cp_opcode_e'(cmd.hdr.opcode) != CMD_MEM_READ); + wire cmd_xl_src = cmd_xlate && (cp_opcode_e'(cmd.hdr.opcode) != CMD_MEM_WRITE); + + logic xl_dst_r, xl_src_r; // operands still to translate (dst first) + logic [LVL_W-1:0] xl_level_r; + logic [PPN_W-1:0] xl_ppn_r; // table base at the current level + + wire [63:0] xl_va = xl_dst_r ? dst_r : src_r; + wire [LEVEL_BITS-1:0] xl_vpn = xl_va[PAGE_LOG2 + xl_level_r * LEVEL_BITS +: LEVEL_BITS]; + wire [63:0] pte_addr = (64'(xl_ppn_r) << PAGE_LOG2) | (64'(xl_vpn) << PTE_SHIFT); + wire [PTE_SEL_W-1:0] pte_sel = pte_addr[5:PTE_SHIFT]; + wire [PTE_BITS-1:0] pte = axi_dev.rdata[pte_sel * PTE_BITS +: PTE_BITS]; + wire [PPN_W-1:0] pte_ppn = pte[10 +: PPN_W]; + wire pte_v = pte[0]; + wire pte_leaf = pte[1] | pte[2] | pte[3]; // R | W | X + // A leaf above level 0 is a superpage: the VA supplies its low bits. + wire [63:0] off_mask = (64'(1) << (PAGE_LOG2 + xl_level_r * LEVEL_BITS)) - 64'(1); + wire [63:0] leaf_pa = ((64'(pte_ppn) << PAGE_LOG2) & ~off_mask) | (xl_va & off_mask); + // An unmapped or leafless walk leaves the address as it is, as the + // functional model does. + wire xl_last = ~pte_v || pte_leaf || (xl_level_r == '0); + wire [63:0] xl_result = (pte_v && pte_leaf) ? leaf_pa : xl_va; + if (PTE_BITS > 10 + PPN_W) begin : g_pte_unused + `UNUSED_VAR (pte[PTE_BITS-1:10+PPN_W]) + end + `UNUSED_VAR (pte[9:4]) + // Last beat of the current chunk. wire last_beat = (BCNT_W'({1'b0, beat_idx}) == (chunk_beats - BCNT_W'(1))); @@ -137,16 +204,49 @@ module VX_cp_dma rem_beats <= '0; chunk_beats <= '0; beat_idx <= '0; + xl_dst_r <= 1'b0; + xl_src_r <= 1'b0; + xl_level_r <= '0; + xl_ppn_r <= '0; end else begin case (state) S_IDLE: begin if (grant) begin - op_r <= cmd.hdr.opcode; - dst_r <= cmd.arg0; - src_r <= cmd.arg1; + op_r <= cmd.hdr.opcode; + dst_r <= cmd.arg0; + src_r <= cmd.arg1; // Round the byte count up to a whole cache line. - rem_beats <= (cmd.arg2 + 64'd63) >> 6; - state <= S_SETUP; + rem_beats <= (cmd.arg2 + 64'd63) >> 6; + xl_dst_r <= cmd_xl_dst; + xl_src_r <= cmd_xl_src; + xl_level_r <= LVL_W'(PT_LEVELS - 1); + xl_ppn_r <= satp_ppn; + state <= (cmd_xl_dst || cmd_xl_src) ? S_XLATE_AR : S_SETUP; + end + end + S_XLATE_AR: begin + if (axi_dev.arvalid && axi_dev.arready) begin + state <= S_XLATE_R; + end + end + S_XLATE_R: begin + if (axi_dev.rvalid && axi_dev.rready) begin + if (xl_last) begin + if (xl_dst_r) begin + dst_r <= xl_result; + xl_dst_r <= 1'b0; + end else begin + src_r <= xl_result; + xl_src_r <= 1'b0; + end + xl_level_r <= LVL_W'(PT_LEVELS - 1); + xl_ppn_r <= satp_ppn; + state <= (xl_dst_r && xl_src_r) ? S_XLATE_AR : S_SETUP; + end else begin + xl_level_r <= xl_level_r - LVL_W'(1); + xl_ppn_r <= pte_ppn; + state <= S_XLATE_AR; + end end end S_SETUP: begin @@ -249,7 +349,7 @@ module VX_cp_dma // Replicating the decode lets each copy enable a nearby group. Logically // identical to inlining these expressions; affects implementation only. (* MAX_FANOUT = 32 *) wire host_rready = (rd_rready & rd_from_host) || (state == S_FLUSH_R); - (* MAX_FANOUT = 32 *) wire dev_rready = rd_rready & ~rd_from_host; + (* MAX_FANOUT = 32 *) wire dev_rready = (rd_rready & ~rd_from_host) || (state == S_XLATE_R); // ---- Drive both AXI masters; only the routed port asserts valid ---- always_comb begin @@ -275,10 +375,10 @@ module VX_cp_dma axi_host.bready = wr_bready & wr_to_host; // ----- axi_dev ----- - axi_dev.arvalid = rd_arvalid & ~rd_from_host; - axi_dev.araddr = src_r; + axi_dev.arvalid = (rd_arvalid & ~rd_from_host) || (state == S_XLATE_AR); + axi_dev.araddr = (state == S_XLATE_AR) ? {pte_addr[63:6], 6'd0} : src_r; axi_dev.arid = TID_PREFIX; - axi_dev.arlen = burst_len; + axi_dev.arlen = (state == S_XLATE_AR) ? 8'd0 : burst_len; axi_dev.arsize = 3'd6; axi_dev.arburst = 2'b01; axi_dev.rready = dev_rready; @@ -300,6 +400,7 @@ module VX_cp_dma // Sanity / unused. `UNUSED_VAR (cmd.hdr.flags) + `UNUSED_VAR (pte_addr[5:0]) `UNUSED_VAR (cmd.hdr.reserved) `UNUSED_VAR (cmd.profile_slot) `UNUSED_VAR (axi_host.bid) diff --git a/hw/rtl/cp/VX_cp_pkg.sv b/hw/rtl/cp/VX_cp_pkg.sv index a7f1d1ae70..64ee445bf6 100644 --- a/hw/rtl/cp/VX_cp_pkg.sv +++ b/hw/rtl/cp/VX_cp_pkg.sv @@ -80,6 +80,10 @@ package VX_cp_pkg; localparam int F_PROFILE = 0; localparam int F_FENCE_PRE = 1; + // CMD_CACHE_FLUSH: flush only the levels above the last-level cache + localparam int F_FLUSH_L1_ONLY = 2; + // CMD_MEM_*: the device operands are physical addresses (no page walk). + localparam int F_MEM_PHYSICAL = 2; typedef struct packed { logic [15:0] reserved; diff --git a/hw/rtl/dxa/VX_dxa_addr_gen.sv b/hw/rtl/dxa/VX_dxa_addr_gen.sv index df1d34bbc0..f7067aee95 100644 --- a/hw/rtl/dxa/VX_dxa_addr_gen.sv +++ b/hw/rtl/dxa/VX_dxa_addr_gen.sv @@ -74,7 +74,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // ---- Registered state ---- reg active_r; reg [`VX_CFG_MEM_ADDR_WIDTH-1:0] gmem_cursor_r; // Current row's GMEM byte base - reg [31:0] row_len_r; // row_len_bytes (constant) + reg [CL_OFF_BITS-1:0] row_len_off_r; // row_len_bytes within a CL (constant) + reg [31:0] row_len_ceil_r; // row_len_bytes + LINE - 1 // Inner-loop counters/flags (advance every beat). reg [GMEM_ADDR_WIDTH-1:0] cl_addr_r; // Current CL address (GMEM) @@ -84,7 +85,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Outer-dim odometer. reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_count_r; // Current index per dim - reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_last_r; // Last index per dim (tile-1) + reg [DXA_MAX_OUTER_DIMS-1:0][31:0] dim_penult_r; // Index before the last (tile-2) + reg [DXA_MAX_OUTER_DIMS-1:0] dim_single_r; // tile == 1: index 0 is the last + reg [DXA_MAX_OUTER_DIMS-1:0] dim_at_last_r; // dim_count_r == tile-1 reg [DXA_MAX_OUTER_DIMS-1:0][31:0] delta_r; // Wrap delta per dim reg [DXA_MAX_OUTER_DIMS-1:0][31:0] oob_limit_r; // OOB limit per dim @@ -150,14 +153,12 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( || (dim_count_r[3] >= oob_limit_r[3]); // ---- Last-row detection: all outer dims at their last index ---- - // An odometer index reaches its bound by equality, not magnitude: with - // dim_count in [0, tile-1], (dim_count+1 >= tile) ⟺ (dim_count == tile-1). - // Equality is ~2 LUT levels vs a CARRY8 chain, which keeps the once-per-row - // step_delta / num_lines wrap path short. - wire dim0_last = (dim_count_r[0] == dim_last_r[0]); - wire dim1_last = (dim_count_r[1] == dim_last_r[1]); - wire dim2_last = (dim_count_r[2] == dim_last_r[2]); - wire dim3_last = (dim_count_r[3] == dim_last_r[3]); + // Registered alongside the odometer, so the once-per-row step_delta / + // num_lines wrap path starts at a flop instead of a 32-bit compare. + wire dim0_last = dim_at_last_r[0]; + wire dim1_last = dim_at_last_r[1]; + wire dim2_last = dim_at_last_r[2]; + wire dim3_last = dim_at_last_r[3]; wire is_last_outer = dim0_last && dim1_last && dim2_last && dim3_last; // ---- Output ---- @@ -204,7 +205,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] next_cursor = gmem_cursor_r + step_delta; wire [31:0] start_span = 32'(setup_params.initial_gmem_base[CL_OFF_BITS-1:0]) + setup_params.row_len_bytes; - wire [31:0] wrap_span = 32'(next_cursor[CL_OFF_BITS-1:0]) + row_len_r; + wire [CL_OFF_BITS-1:0] wrap_end_off = next_cursor[CL_OFF_BITS-1:0] + row_len_off_r; + wire [31:0] wrap_lines = (32'(next_cursor[CL_OFF_BITS-1:0]) + row_len_ceil_r) >> CL_OFF_BITS; function automatic logic [31:0] calc_num_lines(input logic [31:0] span); calc_num_lines = (span + GMEM_LINE_SIZE - 1) >> CL_OFF_BITS; @@ -245,8 +247,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( end else if (start) begin active_r <= 1'b1; gmem_cursor_r <= setup_params.initial_gmem_base; - row_len_r <= setup_params.row_len_bytes; - cl_addr_r <= setup_params.initial_gmem_base[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; + row_len_off_r <= setup_params.row_len_bytes[CL_OFF_BITS-1:0]; + row_len_ceil_r <= setup_params.row_len_bytes + GMEM_LINE_SIZE - 1; + cl_addr_r <= setup_params.initial_gmem_base[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; lines_left_r <= calc_num_lines(start_span); is_first_r <= 1'b1; last_vend_r <= calc_last_vend(start_span[CL_OFF_BITS-1:0]); @@ -267,8 +270,10 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( n_base_r <= '0; tiled_base_r <= setup_params.initial_smem_base; for (int d = 0; d < DXA_MAX_OUTER_DIMS; d++) begin - dim_count_r[d] <= '0; - dim_last_r[d] <= setup_params.dim_tiles[d] - 32'd1; + dim_count_r[d] <= '0; + dim_penult_r[d] <= setup_params.dim_tiles[d] - 32'd2; + dim_single_r[d] <= (setup_params.dim_tiles[d] == 32'd1); + dim_at_last_r[d] <= (setup_params.dim_tiles[d] == 32'd1); delta_r[d] <= setup_params.delta[d]; oob_limit_r[d] <= setup_params.oob_limit[d]; end @@ -293,9 +298,9 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // next beat is the new row's CL0. gmem_cursor_r <= next_cursor; cl_addr_r <= next_cursor[`VX_CFG_MEM_ADDR_WIDTH-1:CL_OFF_BITS]; - lines_left_r <= calc_num_lines(wrap_span); + lines_left_r <= wrap_lines; is_first_r <= 1'b1; - last_vend_r <= calc_last_vend(wrap_span[CL_OFF_BITS-1:0]); + last_vend_r <= calc_last_vend(wrap_end_off); // K-major: SMEM jumps back to the next row base (initial + // (i1+1) * esize). Maintained via km_row_base_r which we bump // by esize at each row wrap. Row-major path keeps its linear @@ -308,18 +313,25 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Tiled scatter: each new row restarts at N=0 (tile-relative). n_base_r <= '0; if (dim0_steps) begin - dim_count_r[0] <= dim_count_r[0] + 1; + dim_count_r[0] <= dim_count_r[0] + 1; + dim_at_last_r[0] <= (dim_count_r[0] == dim_penult_r[0]); end else begin - dim_count_r[0] <= '0; + dim_count_r[0] <= '0; + dim_at_last_r[0] <= dim_single_r[0]; if (dim1_steps) begin - dim_count_r[1] <= dim_count_r[1] + 1; + dim_count_r[1] <= dim_count_r[1] + 1; + dim_at_last_r[1] <= (dim_count_r[1] == dim_penult_r[1]); end else begin - dim_count_r[1] <= '0; + dim_count_r[1] <= '0; + dim_at_last_r[1] <= dim_single_r[1]; if (dim2_steps) begin - dim_count_r[2] <= dim_count_r[2] + 1; + dim_count_r[2] <= dim_count_r[2] + 1; + dim_at_last_r[2] <= (dim_count_r[2] == dim_penult_r[2]); end else begin - dim_count_r[2] <= '0; - dim_count_r[3] <= dim_count_r[3] + 1; + dim_count_r[2] <= '0; + dim_at_last_r[2] <= dim_single_r[2]; + dim_count_r[3] <= dim_count_r[3] + 1; + dim_at_last_r[3] <= (dim_count_r[3] == dim_penult_r[3]); end end end @@ -336,6 +348,8 @@ module VX_dxa_addr_gen import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // km_step is LMEM-bounded; only the low DXA_SMEM_ADDR_W bits feed smem_byte_addr_r. `UNUSED_VAR (km_step[31:DXA_SMEM_ADDR_W]) + // the outermost dim only steps (all-last ends the transfer), never wraps to 0 + `UNUSED_VAR (dim_single_r[DXA_MAX_OUTER_DIMS-1]) `ifdef DBG_TRACE_DXA always @(posedge clk) begin diff --git a/hw/rtl/dxa/VX_dxa_core.sv b/hw/rtl/dxa/VX_dxa_core.sv index af7d5c3021..e0a1330af7 100644 --- a/hw/rtl/dxa/VX_dxa_core.sv +++ b/hw/rtl/dxa/VX_dxa_core.sv @@ -180,13 +180,20 @@ module VX_dxa_core import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // ================================================================ // Output arbitration // ================================================================ +`ifdef SOCKET_UNIT_MMU_ENABLE + localparam GMEM_OUT_REG = 1; +`else + localparam GMEM_OUT_REG = 0; +`endif VX_mem_bus_arb #( .NUM_INPUTS (`VX_CFG_NUM_DXA_CORES), .NUM_OUTPUTS (GMEM_OUT_PORTS), .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), .TAG_WIDTH (WORKER_GMEM_TAG_WIDTH), .ARBITER ("R"), - .REQ_OUT_BUF ((`VX_CFG_NUM_DXA_CORES > 1) ? 3 : 0) + // Registered whenever its consumer needs a flop to start from: the + // arbiter's merge, or the unit MMU's TLB lookup under VM. + .REQ_OUT_BUF (((`VX_CFG_NUM_DXA_CORES > 1) || GMEM_OUT_REG) ? 3 : 0) ) gmem_arb ( .clk (clk), .reset (reset), diff --git a/hw/rtl/dxa/VX_dxa_pkg.sv b/hw/rtl/dxa/VX_dxa_pkg.sv index 5712b499de..f87414b545 100644 --- a/hw/rtl/dxa/VX_dxa_pkg.sv +++ b/hw/rtl/dxa/VX_dxa_pkg.sv @@ -24,8 +24,8 @@ package VX_dxa_pkg; localparam DXA_LMEM_WORD_SIZE = `VX_CFG_LMEM_NUM_BANKS * (`VX_CFG_XLEN / 8); localparam DXA_LMEM_ADDR_W = LMEM_DMA_ADDR_WIDTH; - // LMEM byte-address width: an LMEM-relative byte offset, same as - // VX_local_mem's ADDR_WIDTH = `CLOG2(SIZE) = `VX_CFG_LMEM_LOG_SIZE. + // LMEM byte-address width: an LMEM-relative byte offset into the + // (1 << `VX_CFG_LMEM_LOG_SIZE)-byte local memory. localparam DXA_SMEM_ADDR_W = `VX_CFG_LMEM_LOG_SIZE; localparam DXA_DESC_SLOT_BITS = `CLOG2(`VX_DCR_DXA_DESC_COUNT); diff --git a/hw/rtl/dxa/VX_dxa_smem_wr.sv b/hw/rtl/dxa/VX_dxa_smem_wr.sv index 05dff2b23a..d535ecfe7c 100644 --- a/hw/rtl/dxa/VX_dxa_smem_wr.sv +++ b/hw/rtl/dxa/VX_dxa_smem_wr.sv @@ -167,6 +167,9 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( reg [DXA_SMEM_ADDR_W-1:0] tiled_step_q; reg [DXA_SMEM_ADDR_W-1:0] tiled_wrap_q; reg [4:0] calc_sh2_q; + reg [15:0] intra_khi_mask_q, intra_klo_mask_q; + reg [4:0] intra_khi_sh_q, intra_n_sh_q; + reg [3:0] esize_q; wire is_flat_w = (dest_mode_q == DXA_DEST_FLAT); wire [15:0] tcn_mask_w = (16'd1 << lg_tcN_q) - 16'd1; wire [4:0] step_sh_w = is_flat_w ? (5'(lg_ratio_q) + 5'(esize)) @@ -183,6 +186,16 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( calc_sh2_q <= is_flat_w ? (5'(lg_tcN_q) + 5'(lg_tcN_q) + 5'd1 + 5'(lg_ratio_q) + 5'(esize)) : (5'(lg_tcN_q) + 5'(lg_bkK_q) + 5'(esize)); + // intra-block term of the per-CL dest calc (stage 1 below), as disjoint + // k/n bit fields shifted into place: + // FLAT: k[hi] << (lg_tcN+esize) | n_in << (lg_ratio+esize) | k[lo] << esize + // hi = ((2 << lg_tcN) - 1) << lg_ratio, lo = (1 << lg_ratio) - 1 + // BM: n_in << (lg_bkK+esize) | k[lo] << esize, lo = (1 << lg_bkK) - 1 + intra_khi_mask_q <= is_flat_w ? (((16'd2 << lg_tcN_q) - 16'd1) << lg_ratio_q) : 16'd0; + intra_khi_sh_q <= 5'(lg_tcN_q) + 5'(esize); + intra_n_sh_q <= step_sh_w; + intra_klo_mask_q <= (16'd1 << (is_flat_w ? lg_ratio_q : lg_bkK_q)) - 16'd1; + esize_q <= esize; end // ════════════════════════════════════════════════════════════════════ @@ -426,23 +439,18 @@ module VX_dxa_smem_wr import VX_gpu_pkg::*, VX_dxa_pkg::*; #( wire [15:0] calc_k = calc_pend ? pend_k_r : defer_k_r; wire [15:0] calc_n = calc_pend ? pend_n_r : defer_n_r; - wire [15:0] ratio_mask_w = (16'd1 << lg_ratio_q) - 16'd1; - wire [15:0] ktck_mask_w = (16'd2 << lg_tcN_q) - 16'd1; - wire [15:0] kw_mask_w = (16'd1 << lg_bkK_q) - 16'd1; wire [15:0] calc_n_blk = calc_n >> lg_tcN_q; wire [15:0] calc_n_in = calc_n & tcn_mask_q; wire [15:0] f_k_word = calc_k >> lg_ratio_q; - wire [15:0] f_elem = calc_k & ratio_mask_w; wire [15:0] f_k_blk = f_k_word >> (lg_tcN_q + 4'd1); - wire [15:0] f_kw_in = f_k_word & ktck_mask_w; wire [15:0] b_k_blk = calc_k >> lg_bkK_q; - wire [15:0] b_r_in = calc_k & kw_mask_w; wire [15:0] calc_k_blk = is_flat_w ? f_k_blk : b_k_blk; wire [31:0] calc_blk_w = (32'(calc_k_blk) << lg_nsteps_q) + 32'(calc_n_blk); - wire [31:0] calc_intra_w = is_flat_w - ? (((((32'(f_kw_in) << lg_tcN_q) + 32'(calc_n_in)) << lg_ratio_q) + 32'(f_elem)) << esize) - : (((32'(calc_n_in) << lg_bkK_q) + 32'(b_r_in)) << esize); + // the masked fields are disjoint, so the nested shift-adds are one OR + wire [31:0] calc_intra_w = (32'(calc_k & intra_khi_mask_q) << intra_khi_sh_q) + | (32'(calc_n_in) << intra_n_sh_q) + | (32'(calc_k & intra_klo_mask_q) << esize_q); reg [31:0] calc_blk_r, calc_intra_r; reg calc2_valid_r, calc2_pend_r; diff --git a/hw/rtl/dxa/VX_dxa_unit.sv b/hw/rtl/dxa/VX_dxa_unit.sv index ed9136fd95..61f5a72d2e 100644 --- a/hw/rtl/dxa/VX_dxa_unit.sv +++ b/hw/rtl/dxa/VX_dxa_unit.sv @@ -14,12 +14,13 @@ `include "VX_define.vh" module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0 + parameter `STRING INSTANCE_ID = "" ) ( input wire clk, input wire reset, + input wire [CORE_ID_WIDTH-1:0] core_id, + VX_execute_if.slave execute_if, VX_result_if.master result_if, VX_dxa_req_bus_if.master dxa_req_bus_if @@ -27,6 +28,8 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_VAR (execute_if.data.rs3_data) + `STATIC_ASSERT((`VX_CFG_NUM_ALU_LANES >= 4), ("invalid parameter: NUM_ALU_LANES=%0d: the DXA descriptor is packed by WGATHER, which gathers within groups of 4 lanes, so the ALU must hold a whole group in one packet", `VX_CFG_NUM_ALU_LANES)) + // Wgather-based layout (lane index = thread_id & 3): // Lane 0: rs1=smem_addr, rs2=coord2 // Lane 1: rs1=meta, rs2=coord3 @@ -58,7 +61,8 @@ module VX_dxa_unit import VX_gpu_pkg::*, VX_dxa_pkg::*; #( // Build dxa_req payload dxa_req_data_t dxa_req_data_in; - assign dxa_req_data_in.core_id = NC_WIDTH'(CORE_ID); + // cluster-local core index + assign dxa_req_data_in.core_id = NC_WIDTH'(core_id % `VX_CFG_NUM_CORES); assign dxa_req_data_in.uuid = execute_if.data.header.uuid; assign dxa_req_data_in.wid = execute_if.data.header.wid; assign dxa_req_data_in.smem_addr = lmem_rel_byte_addr[DXA_SMEM_ADDR_W-1:0]; diff --git a/hw/rtl/dxa/VX_dxa_worker.sv b/hw/rtl/dxa/VX_dxa_worker.sv index 8f2a9500ea..fd8f0d5e25 100644 --- a/hw/rtl/dxa/VX_dxa_worker.sv +++ b/hw/rtl/dxa/VX_dxa_worker.sv @@ -45,6 +45,8 @@ module VX_dxa_worker import VX_gpu_pkg::*, VX_dxa_pkg::*; #( localparam MAX_OUTSTANDING = `VX_CFG_DXA_MAX_INFLIGHT; localparam TAG_W = `CLOG2(MAX_OUTSTANDING); localparam SEQ_W = `CLOG2(MAX_OUTSTANDING + 1); + // the outstanding-request queue and its tags need a power of two of at least 2 + `STATIC_ASSERT (`IS_POW2(MAX_OUTSTANDING) && (MAX_OUTSTANDING >= 2), ("invalid parameter")) // ════════════════════════════════════════════════════════════════════ // Inter-module wires diff --git a/hw/rtl/fpu/VX_fpu_block.sv b/hw/rtl/fpu/VX_fpu_block.sv new file mode 100644 index 0000000000..eb97c427e6 --- /dev/null +++ b/hw/rtl/fpu/VX_fpu_block.sv @@ -0,0 +1,261 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_fpu_define.vh" + +// One FPU block: the FP datapath with its tag store and fflags accumulation. +module VX_fpu_block import VX_gpu_pkg::*, VX_fpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 1, + parameter PARTIAL_BW = 0 +) ( + input wire clk, + input wire reset, + + input wire [`LOG2UP(`VX_CFG_NUM_FPU_BLOCKS)-1:0] block_idx, + + VX_execute_if.slave execute_if, + VX_result_if.master result_if, + VX_fpu_csr_if.master fpu_csr_if +); + `UNUSED_SPARAM (INSTANCE_ID) + localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); + localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); + + wire fpu_req_valid, fpu_req_ready; + wire fpu_rsp_valid, fpu_rsp_ready; + wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; + fflags_t fpu_rsp_fflags; + wire fpu_rsp_has_fflags; + + fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; + + // Carry the decoded wb through the tag store: FPU compares may target + // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the + // scoreboard, so forcing wb=1 on readback would trip the invalid + // writeback assertion. + always_comb begin + fpu_hdr_store = execute_if.data.header; + end + + always_comb begin + fpu_hdr_wb = fpu_hdr; + end + + wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; + wire mdata_full; + + wire [INST_FMT_BITS-1:0] fpu_fmt = execute_if.data.op_args.fpu.fmt; + wire [INST_FRM_BITS-1:0] fpu_frm = execute_if.data.op_args.fpu.frm; + + wire execute_fire = execute_if.valid && execute_if.ready; + wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; + + VX_index_buffer #( + .DATAW ($bits(fpu_header_t)), + .SIZE (`VX_CFG_FPU_QUEUE_SIZE) + ) tag_store ( + .clk (clk), + .reset (reset), + .acquire_en (execute_fire), + .write_addr (fpu_req_tag), + .write_data (fpu_hdr_store), + .read_data (fpu_hdr), + .read_addr (fpu_rsp_tag), + .release_en (fpu_rsp_fire), + .full (mdata_full), + `UNUSED_PIN (empty) + ); + + // resolve dynamic FRM from CSR + wire [INST_FRM_BITS-1:0] fpu_req_frm; + `ASSIGN_BLOCKED_WID (fpu_csr_if.read_wid, execute_if.data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + if (`VX_CFG_NUM_FPU_BLOCKS == 1) begin : g_block_idx_unused + `UNUSED_VAR (block_idx) + end + assign fpu_req_frm = (execute_if.data.op_type != INST_FPU_MISC + && fpu_frm == INST_FRM_DYN) ? fpu_csr_if.read_frm : fpu_frm; + + // submit FPU request + + assign fpu_req_valid = execute_if.valid && ~mdata_full; + assign execute_if.ready = fpu_req_ready && ~mdata_full; + +`ifdef VX_CFG_FPU_TYPE_DPI + + VX_fpu_dpi #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dpi ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_FPNEW + + VX_fpu_fpnew #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_fpnew ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_DSP + + VX_fpu_dsp #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_dsp ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`elsif VX_CFG_FPU_TYPE_STD + + VX_fpu_std #( + .NUM_LANES (NUM_LANES), + .TAG_WIDTH (TAG_WIDTH), + .OUT_BUF (PARTIAL_BW ? 1 : 3) + ) fpu_std ( + .clk (clk), + .reset (reset), + + .valid_in (fpu_req_valid), + .mask_in (execute_if.data.header.tmask), + .op_type (execute_if.data.op_type), + .fmt (fpu_fmt), + .frm (fpu_req_frm), + .dataa (execute_if.data.rs1_data), + .datab (execute_if.data.rs2_data), + .datac (execute_if.data.rs3_data), + .tag_in (fpu_req_tag), + .ready_in (fpu_req_ready), + + .valid_out (fpu_rsp_valid), + .result (fpu_rsp_result), + .has_fflags (fpu_rsp_has_fflags), + .fflags (fpu_rsp_fflags), + .tag_out (fpu_rsp_tag), + .ready_out (fpu_rsp_ready) + ); + +`endif + + // handle CSR update + fflags_t fpu_rsp_fflags_q; + + if (PID_BITS != 0) begin : g_fflags_pid + fflags_t fpu_rsp_fflags_r; + always @(posedge clk) begin + if (reset) begin + fpu_rsp_fflags_r <= '0; + end else if (fpu_rsp_fire) begin + fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); + end + end + assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; + end else begin : g_fflags_no_pid + assign fpu_rsp_fflags_q = fpu_rsp_fflags; + end + + VX_fpu_csr_if fpu_csr_tmp_if(); + assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; + `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) + assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; + + VX_pipe_register #( + .DATAW (1 + NW_WIDTH + $bits(fflags_t)), + .RESETW (1) + ) fpu_csr_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), + .data_out ({fpu_csr_if.write_enable, fpu_csr_if.write_wid, fpu_csr_if.write_fflags}) + ); + + // send response + + VX_elastic_buffer #( + .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), + .SIZE (0) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (fpu_rsp_valid), + .ready_in (fpu_rsp_ready), + .data_in ({fpu_hdr_wb, fpu_rsp_result}), + .data_out (result_if.data), + .valid_out (result_if.valid), + .ready_out (result_if.ready) + ); + +endmodule diff --git a/hw/rtl/fpu/VX_fpu_unit.sv b/hw/rtl/fpu/VX_fpu_unit.sv index 9e2386d54f..b64bf5825e 100644 --- a/hw/rtl/fpu/VX_fpu_unit.sv +++ b/hw/rtl/fpu/VX_fpu_unit.sv @@ -29,8 +29,6 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( `UNUSED_SPARAM (INSTANCE_ID) localparam BLOCK_SIZE = `VX_CFG_NUM_FPU_BLOCKS; localparam NUM_LANES = `VX_CFG_NUM_FPU_LANES; - localparam PID_BITS = `CLOG2(`VX_CFG_NUM_THREADS / NUM_LANES); - localparam TAG_WIDTH = `LOG2UP(`VX_CFG_FPU_QUEUE_SIZE); localparam PARTIAL_BW = (BLOCK_SIZE != `VX_CFG_ISSUE_WIDTH) || (NUM_LANES != `VX_CFG_SIMD_WIDTH); VX_execute_if #( @@ -44,6 +42,7 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -53,227 +52,17 @@ module VX_fpu_unit import VX_gpu_pkg::*, VX_fpu_pkg::*; #( ) per_block_result_if[BLOCK_SIZE](); for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - wire fpu_req_valid, fpu_req_ready; - wire fpu_rsp_valid, fpu_rsp_ready; - wire [NUM_LANES-1:0][`VX_CFG_XLEN-1:0] fpu_rsp_result; - fflags_t fpu_rsp_fflags; - wire fpu_rsp_has_fflags; - - fpu_header_t fpu_hdr, fpu_hdr_wb, fpu_hdr_store; - - // Carry the decoded wb through the tag store: FPU compares may target - // integer x0 (e.g. feq.s x0, ...), which decode does not reserve in the - // scoreboard, so forcing wb=1 on readback would trip the invalid - // writeback assertion. - always_comb begin - fpu_hdr_store = per_block_execute_if[block_idx].data.header; - end - - always_comb begin - fpu_hdr_wb = fpu_hdr; - end - - wire [TAG_WIDTH-1:0] fpu_req_tag, fpu_rsp_tag; - wire mdata_full; - - wire [INST_FMT_BITS-1:0] fpu_fmt = per_block_execute_if[block_idx].data.op_args.fpu.fmt; - wire [INST_FRM_BITS-1:0] fpu_frm = per_block_execute_if[block_idx].data.op_args.fpu.frm; - - wire execute_fire = per_block_execute_if[block_idx].valid && per_block_execute_if[block_idx].ready; - wire fpu_rsp_fire = fpu_rsp_valid && fpu_rsp_ready; - - VX_index_buffer #( - .DATAW ($bits(fpu_header_t)), - .SIZE (`VX_CFG_FPU_QUEUE_SIZE) - ) tag_store ( - .clk (clk), - .reset (reset), - .acquire_en (execute_fire), - .write_addr (fpu_req_tag), - .write_data (fpu_hdr_store), - .read_data (fpu_hdr), - .read_addr (fpu_rsp_tag), - .release_en (fpu_rsp_fire), - .full (mdata_full), - `UNUSED_PIN (empty) - ); - - // resolve dynamic FRM from CSR - wire [INST_FRM_BITS-1:0] fpu_req_frm; - `ASSIGN_BLOCKED_WID (fpu_csr_if[block_idx].read_wid, per_block_execute_if[block_idx].data.header.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_req_frm = (per_block_execute_if[block_idx].data.op_type != INST_FPU_MISC - && fpu_frm == INST_FRM_DYN) ? fpu_csr_if[block_idx].read_frm : fpu_frm; - - // submit FPU request - - assign fpu_req_valid = per_block_execute_if[block_idx].valid && ~mdata_full; - assign per_block_execute_if[block_idx].ready = fpu_req_ready && ~mdata_full; - - `ifdef VX_CFG_FPU_TYPE_DPI - - VX_fpu_dpi #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dpi ( + VX_fpu_block #( + .INSTANCE_ID (INSTANCE_ID), + .NUM_LANES (NUM_LANES), + .PARTIAL_BW (PARTIAL_BW) + ) fpu_block ( .clk (clk), .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_FPNEW - - VX_fpu_fpnew #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_fpnew ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_DSP - - VX_fpu_dsp #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_dsp ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `elsif VX_CFG_FPU_TYPE_STD - - VX_fpu_std #( - .NUM_LANES (NUM_LANES), - .TAG_WIDTH (TAG_WIDTH), - .OUT_BUF (PARTIAL_BW ? 1 : 3) - ) fpu_std ( - .clk (clk), - .reset (reset), - - .valid_in (fpu_req_valid), - .mask_in (per_block_execute_if[block_idx].data.header.tmask), - .op_type (per_block_execute_if[block_idx].data.op_type), - .fmt (fpu_fmt), - .frm (fpu_req_frm), - .dataa (per_block_execute_if[block_idx].data.rs1_data), - .datab (per_block_execute_if[block_idx].data.rs2_data), - .datac (per_block_execute_if[block_idx].data.rs3_data), - .tag_in (fpu_req_tag), - .ready_in (fpu_req_ready), - - .valid_out (fpu_rsp_valid), - .result (fpu_rsp_result), - .has_fflags (fpu_rsp_has_fflags), - .fflags (fpu_rsp_fflags), - .tag_out (fpu_rsp_tag), - .ready_out (fpu_rsp_ready) - ); - - `endif - - // handle CSR update - fflags_t fpu_rsp_fflags_q; - - if (PID_BITS != 0) begin : g_fflags_pid - fflags_t fpu_rsp_fflags_r; - always @(posedge clk) begin - if (reset) begin - fpu_rsp_fflags_r <= '0; - end else if (fpu_rsp_fire) begin - fpu_rsp_fflags_r <= fpu_hdr.eop ? '0 : (fpu_rsp_fflags_r | fpu_rsp_fflags); - end - end - assign fpu_rsp_fflags_q = fpu_rsp_fflags_r | fpu_rsp_fflags; - end else begin : g_fflags_no_pid - assign fpu_rsp_fflags_q = fpu_rsp_fflags; - end - - VX_fpu_csr_if fpu_csr_tmp_if(); - assign fpu_csr_tmp_if.write_enable = fpu_rsp_fire && fpu_hdr.eop && fpu_rsp_has_fflags; - `ASSIGN_BLOCKED_WID (fpu_csr_tmp_if.write_wid, fpu_hdr.wid, block_idx, `VX_CFG_NUM_FPU_BLOCKS) - assign fpu_csr_tmp_if.write_fflags = fpu_rsp_fflags_q; - - VX_pipe_register #( - .DATAW (1 + NW_WIDTH + $bits(fflags_t)), - .RESETW (1) - ) fpu_csr_reg ( - .clk (clk), - .reset (reset), - .enable (1'b1), - .data_in ({fpu_csr_tmp_if.write_enable, fpu_csr_tmp_if.write_wid, fpu_csr_tmp_if.write_fflags}), - .data_out ({fpu_csr_if[block_idx].write_enable, fpu_csr_if[block_idx].write_wid, fpu_csr_if[block_idx].write_fflags}) - ); - - // send response - - VX_elastic_buffer #( - .DATAW ($bits(fpu_header_t) + (NUM_LANES * `VX_CFG_XLEN)), - .SIZE (0) - ) rsp_buf ( - .clk (clk), - .reset (reset), - .valid_in (fpu_rsp_valid), - .ready_in (fpu_rsp_ready), - .data_in ({fpu_hdr_wb, fpu_rsp_result}), - .data_out (per_block_result_if[block_idx].data), - .valid_out (per_block_result_if[block_idx].valid), - .ready_out (per_block_result_if[block_idx].ready) + .block_idx (`LOG2UP(BLOCK_SIZE)'(block_idx)), + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]), + .fpu_csr_if (fpu_csr_if[block_idx]) ); end diff --git a/hw/rtl/interfaces/VX_kmu_bus_if.sv b/hw/rtl/interfaces/VX_kmu_bus_if.sv index a1f1936898..4351a4976c 100644 --- a/hw/rtl/interfaces/VX_kmu_bus_if.sv +++ b/hw/rtl/interfaces/VX_kmu_bus_if.sv @@ -37,7 +37,7 @@ // `dest` is the global core index a FRAGMENT launch must land on. Fragment work // is bin->core affine — that affinity is what keeps same-pixel blend order // correct — so a fragment message cannot be load-balanced the way a compute CTA -// can. Each fan-out level consumes its own slice of `dest` (KMU_DEST_LSB_*); +// can. Each fan-out level consumes its own digit of `dest` (KMU_DEST_DIV_*); // compute launches ignore it and go to any ready output. interface VX_kmu_bus_if import VX_gpu_pkg::*; (); diff --git a/hw/rtl/interfaces/VX_sched_csr_if.sv b/hw/rtl/interfaces/VX_sched_csr_if.sv index 5ac508cbfc..cc88a4d0e2 100644 --- a/hw/rtl/interfaces/VX_sched_csr_if.sv +++ b/hw/rtl/interfaces/VX_sched_csr_if.sv @@ -18,21 +18,21 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); wire [PERF_CTR_BITS-1:0] cycles; wire [PERF_CTR_BITS-1:0] instret; wire [`VX_CFG_NUM_WARPS-1:0] active_warps; - wire [`VX_CFG_NUM_WARPS-1:0][`VX_CFG_NUM_THREADS-1:0] thread_masks; + wire [`VX_CFG_NUM_THREADS-1:0] thread_mask; // selected by csr_rd_wid - // Read port: slave sends wid + cta_id, master returns selected mscratch - // and cta_csrs. csr_rd_cta_id must carry the executing instruction's - // cta_id so the scheduler can index its per-CTA table directly. + // Read port: slave sends wid, master returns the selected warp's mscratch + // and CTA rank. logic [NW_WIDTH-1:0] csr_rd_wid; - logic [NCTA_WIDTH-1:0] csr_rd_cta_id; logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; - cta_csrs_t cta_csrs; + logic [NW_WIDTH-1:0] cta_rank; // Per-lane launch record; the CSR unit reads the view (thread index or fragment // stamp) the accessed CSR needs. cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_lane; `ifdef VX_CFG_VM_ENABLE logic [`VX_CFG_XLEN-1:0] csr_satp; + logic satp_wr_valid; + logic [`VX_CFG_XLEN-1:0] satp_wr_data; `endif // Write port: slave notifies scheduler of MSCRATCH CSR writes @@ -57,9 +57,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output cycles, output instret, output active_warps, - output thread_masks, + output thread_mask, output mscratch, - output cta_csrs, + output cta_rank, output cta_lane, output csr_mstatus, output csr_mtvec, @@ -67,10 +67,11 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); output csr_mcause, output csr_mtval, `ifdef VX_CFG_VM_ENABLE - input csr_satp, + output csr_satp, + input satp_wr_valid, + input satp_wr_data, `endif input csr_rd_wid, - input csr_rd_cta_id, input csr_wr_valid, input csr_wr_wid, input csr_wr_data, @@ -82,9 +83,9 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input cycles, input instret, input active_warps, - input thread_masks, + input thread_mask, input mscratch, - input cta_csrs, + input cta_rank, input cta_lane, input csr_mstatus, input csr_mtvec, @@ -92,10 +93,11 @@ interface VX_sched_csr_if import VX_gpu_pkg::*; (); input csr_mcause, input csr_mtval, `ifdef VX_CFG_VM_ENABLE - output csr_satp, + input csr_satp, + output satp_wr_valid, + output satp_wr_data, `endif output csr_rd_wid, - output csr_rd_cta_id, output csr_wr_valid, output csr_wr_wid, output csr_wr_data, diff --git a/hw/rtl/interfaces/VX_slice_sched_if.sv b/hw/rtl/interfaces/VX_slice_sched_if.sv new file mode 100644 index 0000000000..95d8c79d6d --- /dev/null +++ b/hw/rtl/interfaces/VX_slice_sched_if.sv @@ -0,0 +1,156 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Core scheduler state exchanged with one frontend slice's warp scheduler. +// Core-wide events arrive already addressed to the slice: warp ids are +// slice-local (wid = {wfs, fsw}) and per-warp masks hold only its warps. +interface VX_slice_sched_if import VX_gpu_pkg::*; (); + + localparam SW = PER_FETCH_WARPS; + localparam LW = FETCH_WFS_W; + localparam FW = `VX_CFG_FETCH_WIDTH; + + // CTA launch + logic cta_fire; + logic [LW-1:0] cta_lid; + logic [PC_BITS-1:0] cta_PC; + logic [`VX_CFG_NUM_THREADS-1:0] cta_tmask; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] cta_param; + logic cta_init; + + // CTA-CSR store writes + logic warp_write; + logic [LW-1:0] warp_waddr; + cta_warp_t warp_wdata; + logic id_write; + logic [LW-1:0] id_waddr; + logic [NCTA_WIDTH-1:0] id_wdata; + + // wspawn + logic wspawn_apply; + logic [SW-1:0] wspawn_wmask; + logic [PC_BITS-1:0] wspawn_pc; + logic wspawn_src; // the spawning warp is in this slice + logic [LW-1:0] wspawn_src_lid; + logic [`VX_CFG_MEM_ADDR_WIDTH-1:0] wspawn_mscratch; + + // barrier release + logic bar_unlock_valid; + logic [SW-1:0] bar_unlock_mask; + + // warp retirement, as the core-wide CTA accounting takes it + logic retire_valid; + logic [LW-1:0] retire_lid; + + // every slice's ibuf_full; the slice ANDs them into its core-wide view + logic [FW-1:0] slice_ibuf_full; +`ifdef EXT_SCHED_TIMEOUT_DUMP + // the core-wide stall timeout is about to fire + logic timeout_dump; +`endif + + // slice status and performance events + logic [SW-1:0] active; + logic [SW-1:0] stalled; + logic [SW-1:0] pending_empty; + // every warp of the slice has a full ibuffer + logic ibuf_full; + logic [FETCH_ALU_PORTS-1:0] branch_valid; + logic [FETCH_SFU_PORTS-1:0] divergence; + // a warp is schedulable, with the core's ibuffers not all full and all full + logic sched_valid_room; + logic sched_valid_full; + logic sched_fire; + logic [`VX_CFG_NUM_THREADS-1:0] sched_tmask; + + modport master ( + output cta_fire, + output cta_lid, + output cta_PC, + output cta_tmask, + output cta_param, + output cta_init, + output warp_write, + output warp_waddr, + output warp_wdata, + output id_write, + output id_waddr, + output id_wdata, + output wspawn_apply, + output wspawn_wmask, + output wspawn_pc, + output wspawn_src, + output wspawn_src_lid, + output wspawn_mscratch, + output bar_unlock_valid, + output bar_unlock_mask, + output retire_valid, + output retire_lid, + output slice_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + output timeout_dump, +`endif + input active, + input stalled, + input ibuf_full, + input pending_empty, + input branch_valid, + input divergence, + input sched_valid_room, + input sched_valid_full, + input sched_fire, + input sched_tmask + ); + + modport slave ( + input cta_fire, + input cta_lid, + input cta_PC, + input cta_tmask, + input cta_param, + input cta_init, + input warp_write, + input warp_waddr, + input warp_wdata, + input id_write, + input id_waddr, + input id_wdata, + input wspawn_apply, + input wspawn_wmask, + input wspawn_pc, + input wspawn_src, + input wspawn_src_lid, + input wspawn_mscratch, + input bar_unlock_valid, + input bar_unlock_mask, + input retire_valid, + input retire_lid, + input slice_ibuf_full, +`ifdef EXT_SCHED_TIMEOUT_DUMP + input timeout_dump, +`endif + output active, + output stalled, + output ibuf_full, + output pending_empty, + output branch_valid, + output divergence, + output sched_valid_room, + output sched_valid_full, + output sched_fire, + output sched_tmask + ); + +endinterface diff --git a/hw/rtl/interfaces/VX_warp_ctl_if.sv b/hw/rtl/interfaces/VX_warp_ctl_if.sv index 9da68980c9..26d648a659 100644 --- a/hw/rtl/interfaces/VX_warp_ctl_if.sv +++ b/hw/rtl/interfaces/VX_warp_ctl_if.sv @@ -15,89 +15,64 @@ interface VX_warp_ctl_if import VX_gpu_pkg::*; (); - wire wspawn_valid; wire tmc_valid; wire split_valid; wire sjoin_valid; - wire bar_valid; wire wsync_valid; - wire [NW_WIDTH-1:0] wid; - tmc_t tmc; - wspawn_t wspawn; split_t split; join_t sjoin; - barrier_t bar; - // Per-warp pipeline drain status (from scheduler) - wire [`VX_CFG_NUM_WARPS-1:0] warp_pending_alm_empty; + // Pipeline drain status of the scheduler's warps, by SFU-block-local warp id + wire [SFU_BLOCK_WARPS-1:0] warp_pending_alm_empty; - // LSU memory-scheduler drained signal (AND-reduced across all LSU blocks). - // Used by VX_wctl_unit to gate BAR (both vx_barrier and vx_barrier_arrive) - // so the barrier acts as a SMEM/GMEM fence — pending LSU memory ops finish - // before the warp suspends or the barrier arrival is registered. This is - // the CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) contract. - wire lsu_sched_drained; + // Every LSU block's memory scheduler is empty, and the translation path + // drained. VX_wctl_unit ANDs them to gate BAR (both vx_barrier and + // vx_barrier_arrive) so the barrier acts as a SMEM/GMEM fence — pending + // LSU memory ops finish before the warp suspends or the barrier arrival is + // registered. This is the CUDA __syncthreads / OpenCL + // barrier(CLK_LOCAL_MEM_FENCE) contract. + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_sched_empty; + wire mmu_empty; - // Return IPDOM stack address + // IPDOM stack address and mscratch of the warp at dvstack_wid wire [NW_WIDTH-1:0] dvstack_wid; wire [DV_STACK_SIZEW-1:0] dvstack_ptr; - - // Return barrier phase - wire [BAR_ADDR_W-1:0] bar_addr; - wire bar_phase; + wire [`VX_CFG_MEM_ADDR_WIDTH-1:0] mscratch; modport master ( - output wspawn_valid, output tmc_valid, output split_valid, output sjoin_valid, - output bar_valid, output wsync_valid, - output wid, - - output wspawn, output tmc, output split, output sjoin, - output bar, - - output bar_addr, - input bar_phase, - input warp_pending_alm_empty, - input lsu_sched_drained, - + input lsu_sched_empty, + input mmu_empty, output dvstack_wid, - input dvstack_ptr + input dvstack_ptr, + input mscratch ); modport slave ( - input wspawn_valid, - input tmc_valid, - input split_valid, - input sjoin_valid, - input bar_valid, - input wsync_valid, - - input wid, - - input wspawn, - input tmc, - input split, - input sjoin, - input bar, - - input bar_addr, - output bar_phase, - + input tmc_valid, + input split_valid, + input sjoin_valid, + input wsync_valid, + input wid, + input tmc, + input split, + input sjoin, output warp_pending_alm_empty, - output lsu_sched_drained, - - input dvstack_wid, - output dvstack_ptr + output lsu_sched_empty, + output mmu_empty, + input dvstack_wid, + output dvstack_ptr, + output mscratch ); endinterface diff --git a/hw/rtl/interfaces/VX_warp_shared_if.sv b/hw/rtl/interfaces/VX_warp_shared_if.sv new file mode 100644 index 0000000000..86d1f13786 --- /dev/null +++ b/hw/rtl/interfaces/VX_warp_shared_if.sv @@ -0,0 +1,44 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// A core-wide warp-control request and its completion. A requester waits on at +// most one request with has_rsp set; rsp_valid completes it and carries the +// barrier phase. Other requests are posted and get no completion. +interface VX_warp_shared_if import VX_gpu_pkg::*; (); + + logic req_valid; + warp_shared_req_t req_data; + logic req_ready; + + logic rsp_valid; + logic rsp_phase; + + modport master ( + output req_valid, + output req_data, + input req_ready, + input rsp_valid, + input rsp_phase + ); + + modport slave ( + input req_valid, + input req_data, + output req_ready, + output rsp_valid, + output rsp_phase + ); + +endinterface diff --git a/hw/rtl/libs/VX_1rw1r_ram.sv b/hw/rtl/libs/VX_1rw1r_ram.sv new file mode 100644 index 0000000000..fef0aabad1 --- /dev/null +++ b/hw/rtl/libs/VX_1rw1r_ram.sv @@ -0,0 +1,319 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +`ifndef ASIC + `define RAM_INITIALIZATION \ + if (INIT_ENABLE != 0) begin : g_init \ + if (INIT_FILE != "") begin : g_file \ + initial $readmemh(INIT_FILE, ram); \ + end else begin : g_value \ + initial begin \ + for (integer i = 0; i < SIZE; ++i) begin : g_i \ + ram[i] = INIT_VALUE; \ + end \ + end \ + end \ + end +`else + `define RAM_INITIALIZATION +`endif + +`ifdef SIMULATION + `define RAM_RESET_BLOCK if (RESET_RAM && reset) begin \ + for (integer i = 0; i < SIZE; ++i) begin \ + ram[i] <= DATAW'(INIT_VALUE); \ + end \ + end else +`else + `define RAM_RESET_BLOCK +`endif + +`define RAM_WRITE_ALL `RAM_RESET_BLOCK \ + if (write_a) begin \ + ram[addr_a] <= wdata_a; \ + end + +`ifdef QUARTUS + `define RAM_ARRAY_WREN reg [WRENW-1:0][WSELW-1:0] ram [0:SIZE-1]; + `define RAM_WRITE_WREN `RAM_RESET_BLOCK \ + if (write_a) begin \ + for (integer i = 0; i < WRENW; ++i) begin \ + if (wren_a[i]) begin \ + ram[addr_a][i] <= wdata_a[i * WSELW +: WSELW]; \ + end \ + end \ + end +`else + `define RAM_ARRAY_WREN reg [DATAW-1:0] ram [0:SIZE-1]; + `define RAM_WRITE_WREN `RAM_RESET_BLOCK \ + if (write_a) begin \ + for (integer i = 0; i < WRENW; ++i) begin \ + if (wren_a[i]) begin \ + ram[addr_a][i * WSELW +: WSELW] <= wdata_a[i * WSELW +: WSELW]; \ + end \ + end \ + end +`endif + +// A RAM with one read/write port (A) and one read-only port (B), both with a +// registered read: the two ports of a true dual-port block RAM. Port A reads +// before it writes. A port B read of the word port A writes in the same cycle +// returns an undefined value on block RAM; the caller must not issue one. +// +// The ASIC libraries have no read/write + read macro, so on ASIC the RAM is two +// VX_dp_ram copies, one per read port, that both take every write: each maps +// onto the VX_dp_ram_asic macro the synthesis flows and their SRAM cost model +// already know. +`TRACING_OFF +module VX_1rw1r_ram #( + parameter DATAW = 1, + parameter SIZE = 1, + parameter WRENW = 1, + parameter LUTRAM = 0, + parameter RESET_RAM = 0, + parameter INIT_ENABLE = 0, + parameter INIT_FILE = "", + parameter [DATAW-1:0] INIT_VALUE = 0, + parameter ADDRW = `LOG2UP(SIZE) +) ( + input wire clk, + input wire reset, + + input wire read_a, + input wire write_a, + input wire [WRENW-1:0] wren_a, + input wire [ADDRW-1:0] addr_a, + input wire [DATAW-1:0] wdata_a, + output wire [DATAW-1:0] rdata_a, + + input wire read_b, + input wire [ADDRW-1:0] addr_b, + output wire [DATAW-1:0] rdata_b +); + localparam WSELW = DATAW / WRENW; + + `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) + +`ifdef SYNTHESIS +`ifdef ASIC + `UNUSED_PARAM (LUTRAM) + VX_dp_ram #( + .DATAW (DATAW), + .SIZE (SIZE), + .WRENW (WRENW), + .OUT_REG (1), + .RDW_MODE ("R"), + .RESET_RAM (RESET_RAM), + .INIT_ENABLE (INIT_ENABLE), + .INIT_FILE (INIT_FILE), + .INIT_VALUE (INIT_VALUE) + ) ram_a ( + .clk (clk), + .reset (reset), + .read (read_a), + .write (write_a), + .wren (wren_a), + .waddr (addr_a), + .wdata (wdata_a), + .raddr (addr_a), + .rdata (rdata_a) + ); + VX_dp_ram #( + .DATAW (DATAW), + .SIZE (SIZE), + .WRENW (WRENW), + .OUT_REG (1), + .RDW_MODE ("R"), + .RESET_RAM (RESET_RAM), + .INIT_ENABLE (INIT_ENABLE), + .INIT_FILE (INIT_FILE), + .INIT_VALUE (INIT_VALUE) + ) ram_b ( + .clk (clk), + .reset (reset), + .read (read_b), + .write (write_a), + .wren (wren_a), + .waddr (addr_a), + .wdata (wdata_a), + .raddr (addr_b), + .rdata (rdata_b) + ); +`else + localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); + if (FORCE_BRAM) begin : g_bram + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end else if (LUTRAM) begin : g_lutram + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `USE_FAST_BRAM `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end else begin : g_auto + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_WREN + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + `RAM_WRITE_ALL + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + end + always @(posedge clk) begin + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; + end + end +`endif +`else + // simulation + `UNUSED_PARAM (LUTRAM) + reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + + if (WRENW != 1) begin : g_wren + reg [DATAW-1:0] wdata_n; + always @(*) begin + wdata_n = ram[addr_a]; + for (integer i = 0; i < WRENW; ++i) begin + if (wren_a[i]) begin + wdata_n[i * WSELW +: WSELW] = wdata_a[i * WSELW +: WSELW]; + end + end + end + always @(posedge clk) begin + `RAM_RESET_BLOCK + if (write_a) begin + ram[addr_a] <= wdata_n; + end + end + end else begin : g_no_wren + `UNUSED_VAR (wren_a) + always @(posedge clk) begin + `RAM_WRITE_ALL + end + end + + reg [DATAW-1:0] rdata_a_r; + reg [DATAW-1:0] rdata_b_r; + always @(posedge clk) begin + if (read_a) begin + rdata_a_r <= ram[addr_a]; + end + if (read_b) begin + rdata_b_r <= ram[addr_b]; + end + end + assign rdata_a = rdata_a_r; + assign rdata_b = rdata_b_r; +`endif + +endmodule +`TRACING_ON diff --git a/hw/rtl/libs/VX_async_ram_patch.sv b/hw/rtl/libs/VX_async_ram_patch.sv deleted file mode 100644 index dd4d2b42d5..0000000000 --- a/hw/rtl/libs/VX_async_ram_patch.sv +++ /dev/null @@ -1,277 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -`define RAM_INITIALIZATION \ - if (INIT_ENABLE != 0) begin : g_init \ - if (INIT_FILE != "") begin : g_file \ - initial $readmemh(INIT_FILE, ram); \ - end else begin : g_value \ - initial begin \ - for (integer i = 0; i < SIZE; ++i) begin : g_i \ - ram[i] = INIT_VALUE; \ - end \ - end \ - end \ - end - -`define SYNC_RAM_WF_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [ADDRW-1:0] raddr_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - if (__re) begin \ - raddr_r <= __ra; \ - end \ - end \ - assign __d = ram[raddr_r] - -`define SYNC_RAM_WF_WREN_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [ADDRW-1:0] raddr_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - if (__re) begin \ - raddr_r <= __ra; \ - end \ - end \ - assign __d = ram[raddr_r] - -`define SYNC_RAM_RF_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [DATAW-1:0] rdata_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - if (__re) begin \ - rdata_r <= ram[__ra]; \ - end \ - end \ - assign __d = rdata_r - -`define SYNC_RAM_RF_WREN_BLOCK(__d, __re, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - reg [DATAW-1:0] rdata_r; \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - if (__re) begin \ - rdata_r <= ram[__ra]; \ - end \ - end \ - assign __d = rdata_r - -`define ASYNC_RAM_BLOCK(__d, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - always @(posedge clk) begin \ - if (__we) begin \ - ram[__wa] <= wdata; \ - end \ - end \ - assign __d = ram[__ra] - -`define ASYNC_RAM_BLOCK_WREN(__d, __we, __ra, __wa) \ - `RAM_ATTRIBUTES reg [DATAW-1:0] ram [0:SIZE-1]; \ - `RAM_INITIALIZATION \ - always @(posedge clk) begin \ - if (__we) begin \ - for (integer i = 0; i < WRENW; ++i) begin \ - if (wren[i]) begin \ - ram[__wa][i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; \ - end \ - end \ - end \ - end \ - assign __d = ram[__ra] - -`TRACING_OFF -module VX_async_ram_patch #( - parameter DATAW = 1, - parameter SIZE = 1, - parameter WRENW = 1, - parameter DUAL_PORT = 0, - parameter FORCE_BRAM = 0, - parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset - parameter WRITE_FIRST = 0, - parameter INIT_ENABLE = 0, - parameter INIT_FILE = "", - parameter [DATAW-1:0] INIT_VALUE = 0, - parameter ADDRW = `LOG2UP(SIZE) -) ( - input wire clk, - input wire reset, - input wire read, - input wire write, - input wire [WRENW-1:0] wren, - input wire [ADDRW-1:0] waddr, - input wire [DATAW-1:0] wdata, - input wire [ADDRW-1:0] raddr, - output wire [DATAW-1:0] rdata -); - localparam WSELW = DATAW / WRENW; - - `UNUSED_VAR (reset) - - (* keep = "true" *) wire [ADDRW-1:0] raddr_w, raddr_s; - (* keep = "true" *) wire read_s; - assign raddr_w = raddr; - - wire raddr_reset_w; - if (RADDR_RESET) begin : g_raddr_reset - (* keep = "true" *) wire raddr_reset; - assign raddr_reset = 0; - assign raddr_reset_w = raddr_reset; - end else begin : g_no_raddr_reset - assign raddr_reset_w = 0; - end - - VX_placeholder #( - .I (ADDRW + 1), - .O (ADDRW + 1) - ) placeholder1 ( - .in ({raddr_w, raddr_reset_w}), - .out ({raddr_s, read_s}) - ); - - wire [DATAW-1:0] rdata_s; - - if (1) begin : g_sync_ram - if (WRENW != 1) begin : g_wren - if (FORCE_BRAM) begin : g_bram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_WF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_RF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_lutram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES - `SYNC_RAM_WF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES - `SYNC_RAM_RF_WREN_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end - end else begin : g_no_wren - if (FORCE_BRAM) begin : g_bram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_WF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `USE_BLOCK_BRAM - `SYNC_RAM_RF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_lutram - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES - `SYNC_RAM_WF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES - `SYNC_RAM_RF_BLOCK(rdata_s, read_s, write, raddr_s, waddr); - `undef RAM_ATTRIBUTES - end - end - end - end - - if (RADDR_REG) begin : g_raddr_reg - assign rdata = rdata_s; - end else begin : g_async_ram - (* keep = "true" *) wire is_raddr_reg; - VX_placeholder #( - .O (1) - ) placeholder2 ( - .in (1'b0), - .out (is_raddr_reg) - ); - wire [DATAW-1:0] rdata_a; - if (DUAL_PORT) begin : g_dp - if (WRENW != 1) begin : g_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_no_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, raddr, waddr); - `undef RAM_ATTRIBUTES - end - end - end else begin : g_sp - if (WRENW != 1) begin : g_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK_WREN(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end - end else begin : g_no_wren - if (WRITE_FIRST) begin : g_write_first - `define RAM_ATTRIBUTES `RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end else begin : g_read_first - `define RAM_ATTRIBUTES `NO_RW_RAM_CHECK - `ASYNC_RAM_BLOCK(rdata_a, write, waddr, waddr); - `undef RAM_ATTRIBUTES - end - end - end - assign rdata = is_raddr_reg ? rdata_s : rdata_a; - end - -endmodule -`TRACING_ON diff --git a/hw/rtl/libs/VX_dp_ram.sv b/hw/rtl/libs/VX_dp_ram.sv index ab9cafde29..93893811da 100644 --- a/hw/rtl/libs/VX_dp_ram.sv +++ b/hw/rtl/libs/VX_dp_ram.sv @@ -67,6 +67,15 @@ end `endif +// Read forms. The form fixes the behaviour; size and LUTRAM alone pick block or +// distributed RAM. +// OUT_REG=1, RDW_MODE "W": registered address. raddr is the address's next +// value; rdata = ram[raddr captured on read], seeing a write on the same edge. +// OUT_REG=1, RDW_MODE "R": registered data. rdata = ram[raddr] captured on +// read, not seeing a write on the same edge. +// OUT_REG=0: combinational, rdata = ram[raddr]. Never block RAM by itself: the +// synthesis flows' RAM fold (hw/scripts/ram_fold.py) turns a block-RAM-sized +// one into the registered-address form when its address comes from registers. `TRACING_OFF module VX_dp_ram #( parameter DATAW = 1, @@ -75,9 +84,6 @@ module VX_dp_ram #( parameter OUT_REG = 0, parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first - parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset - parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, parameter INIT_FILE = "", @@ -96,36 +102,59 @@ module VX_dp_ram #( ); localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) - `UNUSED_PARAM (RADDR_REG) - `UNUSED_PARAM (RADDR_RESET) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W"), ("invalid parameter")) - `UNUSED_PARAM (RDW_ASSERT) `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form, so forcing BRAM pulls in the - // async-RAM patch. That only pays off for a read-first RAM; a write-first one - // adds a read/write collision bypass that is larger than plain distributed RAM. - localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC - if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic + // The macro reads synchronously, so a combinational read stays in logic. + if (FORCE_BRAM && OUT_REG != 0) begin : g_asic + `UNUSED_VAR (reset) + wire [WRENW-1:0] wr_lanes = (WRENW != 1) ? wren : {WRENW{1'b1}}; + wire [DATAW-1:0] macro_rdata; VX_dp_ram_asic #( .DATAW (DATAW), .SIZE (SIZE), .WRENW (WRENW) ) dp_ram_asic ( .clk (clk), - .reset (reset), + .reset (1'b0), .read (read), .write (write), - .wren (wren), + .wren (wr_lanes), .waddr (waddr), .wdata (wdata), .raddr (raddr), - .rdata (rdata) + .rdata (macro_rdata) ); + if (RDW_MODE == "W") begin : g_write_first + // The macro reads before it writes and then holds its output, while + // this form reads the captured address continuously: the lanes written + // at that address since the read are forwarded around the macro. + reg [ADDRW-1:0] raddr_r; + reg [WRENW-1:0] fwd_en_r; + reg [DATAW-1:0] fwd_data_r; + wire [ADDRW-1:0] fwd_addr = read ? raddr : raddr_r; + wire [WRENW-1:0] fwd_lanes = {WRENW{write && waddr == fwd_addr}} & wr_lanes; + always @(posedge clk) begin + if (read) begin + raddr_r <= raddr; + end + fwd_en_r <= (read ? '0 : fwd_en_r) | fwd_lanes; + for (integer i = 0; i < WRENW; ++i) begin + if (fwd_lanes[i]) begin + fwd_data_r[i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; + end + end + end + for (genvar i = 0; i < WRENW; ++i) begin : g_lanes + assign rdata[i * WSELW +: WSELW] = fwd_en_r[i] ? fwd_data_r[i * WSELW +: WSELW] : macro_rdata[i * WSELW +: WSELW]; + end + end else begin : g_read_first + assign rdata = macro_rdata; + end end else begin : g_no_asic `else if (1) begin : g_no_asic @@ -287,99 +316,37 @@ module VX_dp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (FORCE_BRAM_ASYNC) begin : g_bram - `ifdef ASYNC_BRAM_PATCH - VX_async_ram_patch #( - .DATAW (DATAW), - .SIZE (SIZE), - .WRENW (WRENW), - .DUAL_PORT (1), - .FORCE_BRAM (FORCE_BRAM), - .RADDR_REG (RADDR_REG), - .RADDR_RESET(RADDR_RESET), - .WRITE_FIRST(RDW_MODE == "W"), - .INIT_ENABLE(INIT_ENABLE), - .INIT_FILE (INIT_FILE), - .INIT_VALUE (INIT_VALUE) - ) async_ram_patch ( - .clk (clk), - .reset (reset), - .read (read), - .write (write), - .wren (wren), - .waddr (waddr), - .wdata (wdata), - .raddr (raddr), - .rdata (rdata) - ); - `else - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + if (RDW_MODE == "W") begin : g_write_first + if (WRENW != 1) begin : g_wren + `RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + assign rdata = ram[raddr]; + end else begin : g_no_wren + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[raddr]; end - `endif - end else begin : g_auto - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + end else begin : g_read_first + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[raddr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[raddr]; + assign rdata = ram[raddr]; + end else begin : g_no_wren + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[raddr]; end end end @@ -432,30 +399,7 @@ module VX_dp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (RDW_MODE == "W") begin : g_write_first - assign rdata = ram[raddr]; - end else begin : g_read_first - reg [DATAW-1:0] prev_data; - reg [ADDRW-1:0] prev_waddr; - reg prev_write; - - always @(posedge clk) begin - if (reset) begin - prev_write <= 0; - prev_data <= '0; - prev_waddr <= '0; - end else begin - prev_write <= write; - prev_data <= ram[waddr]; - prev_waddr <= waddr; - end - end - - assign rdata = (prev_write && (prev_waddr == raddr)) ? prev_data : ram[raddr]; - if (RDW_ASSERT) begin : g_rw_asert - `RUNTIME_ASSERT(~read || (rdata == ram[raddr]), ("read after write hazard")) - end - end + assign rdata = ram[raddr]; end `endif diff --git a/hw/rtl/libs/VX_fifo_queue.sv b/hw/rtl/libs/VX_fifo_queue.sv index 624ad5786b..145989ee3c 100644 --- a/hw/rtl/libs/VX_fifo_queue.sv +++ b/hw/rtl/libs/VX_fifo_queue.sv @@ -92,22 +92,30 @@ module VX_fifo_queue #( end end - // OUT_REG=0: asynchronous read of rd_ptr_r (combinational head). + // OUT_REG=0: the head is ram[rd_ptr_r]. Read combinationally, except at + // block-RAM size, where the RAM registers rd_ptr_r's next value itself + // (registered-address form) and so reads the same word on every cycle. // OUT_REG=1: synchronous BRAM. rd_ptr_r already leads the head by one, so // present the look-ahead address (rd_ptr_n, the next read pointer): the // registered read then returns ram[rd_ptr_r] on the same cycle the async // read would have, making the output bit-identical while letting Vivado - // infer native BRAM (no async patch). - wire [ADDRW-1:0] rd_addr = (OUT_REG != 0) ? rd_ptr_n : rd_ptr_r; + // infer native BRAM. + localparam RAM_OUT_REG = (OUT_REG != 0) || (LUTRAM == 0 && `FORCE_BRAM(DEPTH, DATAW)); + wire [ADDRW-1:0] rd_addr; + if (OUT_REG != 0) begin : g_rd_lookahead + assign rd_addr = rd_ptr_n; + end else if (RAM_OUT_REG) begin : g_rd_next + assign rd_addr = reset ? '0 : rd_ptr_n; + end else begin : g_rd_head + assign rd_addr = rd_ptr_r; + end VX_dp_ram #( .DATAW (DATAW), .SIZE (DEPTH), - .OUT_REG (OUT_REG), + .OUT_REG (RAM_OUT_REG), .LUTRAM (LUTRAM), - .RDW_MODE ("W"), - .RADDR_REG (OUT_REG == 0), - .RADDR_RESET (OUT_REG == 0) + .RDW_MODE ("W") ) dp_ram ( .clk (clk), .reset (reset), diff --git a/hw/rtl/libs/VX_mem_scheduler.sv b/hw/rtl/libs/VX_mem_scheduler.sv index 96f80e5ae4..5ac1bca106 100644 --- a/hw/rtl/libs/VX_mem_scheduler.sv +++ b/hw/rtl/libs/VX_mem_scheduler.sv @@ -69,6 +69,10 @@ module VX_mem_scheduler #( input wire [CORE_REQS-1:0][WORD_WIDTH-1:0] core_req_data, input wire [TAG_WIDTH-1:0] core_req_tag, output wire core_req_ready, + // core_req_ready for a read and for a write request: a requester whose + // rw is known ahead of a shared request mux can take its own + output wire core_req_ready_rd, + output wire core_req_ready_wr, // Core request queue output wire req_queue_empty, @@ -223,6 +227,8 @@ module VX_mem_scheduler #( // can accept another request? assign core_req_ready = reqq_ready_in && ibuf_ready; + assign core_req_ready_rd = reqq_ready_in && ~ibuf_full; + assign core_req_ready_wr = reqq_ready_in; // request queue status. `coalescer_empty` (1 when no coalescer) keeps a // store still buffered in the coalescer from prematurely signalling empty. diff --git a/hw/rtl/libs/VX_mem_to_axi.sv b/hw/rtl/libs/VX_mem_to_axi.sv index 8c928152f4..a58e74c79d 100644 --- a/hw/rtl/libs/VX_mem_to_axi.sv +++ b/hw/rtl/libs/VX_mem_to_axi.sv @@ -19,6 +19,11 @@ // (interface-free) so it can sit directly at a platform pin boundary; the // optional AXI sideband (lock/cache/prot/qos/region) is driven to constants. // The inverse of VX_membus_from_axi. +// +// The M masters share one ADDR_WIDTH_OUT byte address space, and each request +// leaves on its bank's master with its own address unchanged. INTERLEAVE +// selects the bank from the low line-address bits; otherwise from the top +// bits, so bank i serves the i-th contiguous 1/M of the space. // ============================================================================ `TRACING_OFF @@ -108,7 +113,7 @@ module VX_mem_to_axi #( localparam LOG2_DATA_SIZE = `CLOG2(DATA_SIZE); localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS_OUT); localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); - localparam DST_ADDR_WDITH = (ADDR_WIDTH_OUT - LOG2_DATA_SIZE) + BANK_SEL_BITS; // convert byte-addressable output addresss to block-addressable input space + localparam DST_ADDR_WDITH = ADDR_WIDTH_OUT - LOG2_DATA_SIZE; // line address width of the output space localparam BANK_ADDR_WIDTH = DST_ADDR_WDITH - BANK_SEL_BITS; localparam NUM_PORTS_IN_BITS = `CLOG2(NUM_PORTS_IN); localparam NUM_PORTS_IN_WIDTH = `UP(NUM_PORTS_IN_BITS); @@ -116,14 +121,12 @@ module VX_mem_to_axi #( localparam NEEDED_TAG_WIDTH = TAG_WIDTH_IN + NUM_PORTS_IN_BITS; localparam READ_TAG_WIDTH = (NEEDED_TAG_WIDTH > TAG_WIDTH_OUT) ? TAG_BUFFER_ADDRW : TAG_WIDTH_IN; localparam READ_FULL_TAG_WIDTH = READ_TAG_WIDTH + NUM_PORTS_IN_BITS; - localparam WRITE_TAG_WIDTH = `MIN(TAG_WIDTH_IN, TAG_WIDTH_OUT); - localparam DST_TAG_WIDTH = `MAX(READ_FULL_TAG_WIDTH, WRITE_TAG_WIDTH); - localparam XBAR_TAG_WIDTH = `MAX(READ_TAG_WIDTH, WRITE_TAG_WIDTH); - localparam REQ_XBAR_DATAW = 1 + BANK_ADDR_WIDTH + DATA_SIZE + DATA_WIDTH + XBAR_TAG_WIDTH; + // Writes carry no tag: every write leaves with AXI id 0 (the write tracker). + localparam REQ_XBAR_DATAW = 1 + BANK_ADDR_WIDTH + DATA_SIZE + DATA_WIDTH + READ_TAG_WIDTH; localparam RSP_XBAR_DATAW = DATA_WIDTH + READ_TAG_WIDTH; `STATIC_ASSERT ((DST_ADDR_WDITH >= ADDR_WIDTH_IN), ("invalid address width: current=%0d, expected=%0d", DST_ADDR_WDITH, ADDR_WIDTH_IN)) - `STATIC_ASSERT ((TAG_WIDTH_OUT >= DST_TAG_WIDTH), ("invalid output tag width: current=%0d, expected=%0d", TAG_WIDTH_OUT, DST_TAG_WIDTH)) + `STATIC_ASSERT ((TAG_WIDTH_OUT >= READ_FULL_TAG_WIDTH), ("invalid output tag width: current=%0d, expected=%0d", TAG_WIDTH_OUT, READ_FULL_TAG_WIDTH)) `STATIC_ASSERT ((WRITE_TRACK_DEPTH >= 2) && `IS_POW2(WRITE_TRACK_DEPTH), ("WRITE_TRACK_DEPTH must be a power of 2 >= 2: current=%0d", WRITE_TRACK_DEPTH)) // Bank selection @@ -197,7 +200,7 @@ module VX_mem_to_axi #( for (genvar i = 0; i < NUM_PORTS_IN; ++i) begin : g_req_xbar_data_in wire tag_ready = mem_req_rw[i] || mem_rd_req_tag_ready[i]; - wire [XBAR_TAG_WIDTH-1:0] tag_value = mem_req_rw[i] ? XBAR_TAG_WIDTH'(mem_req_tag[i]) : XBAR_TAG_WIDTH'(mem_rd_req_tag[i]); + wire [READ_TAG_WIDTH-1:0] tag_value = mem_req_rw[i] ? '0 : mem_rd_req_tag[i]; assign req_xbar_valid_in[i] = mem_req_valid[i] && tag_ready; assign req_xbar_data_in[i] = {mem_req_rw[i], req_bank_addr[i], mem_req_byteen[i], mem_req_data[i], tag_value}; assign mem_req_ready[i] = req_xbar_ready_in[i] && tag_ready; @@ -227,7 +230,7 @@ module VX_mem_to_axi #( wire xbar_rw_out; wire [BANK_ADDR_WIDTH-1:0] xbar_addr_out; - wire [XBAR_TAG_WIDTH-1:0] xbar_tag_out; + wire [READ_TAG_WIDTH-1:0] xbar_tag_out; wire [DATA_WIDTH-1:0] xbar_data_out; wire [DATA_SIZE-1:0] xbar_byteen_out; @@ -347,7 +350,7 @@ module VX_mem_to_axi #( assign xbar_tag_r_out = READ_FULL_TAG_WIDTH'({xbar_tag_out, req_xbar_sel_out[i]}); end else begin : g_no_input_sel `UNUSED_VAR (req_xbar_sel_out) - assign xbar_tag_r_out = READ_TAG_WIDTH'(xbar_tag_out); + assign xbar_tag_r_out = xbar_tag_out; end assign m_axi_arvalid[i] = req_xbar_valid_out[i] && ~xbar_rw_out && ~rd_hazard; @@ -407,14 +410,36 @@ module VX_mem_to_axi #( in_burst_r <= ~m_axi_rlast[i]; end end - assign rsp_xbar_valid_in[i] = m_axi_rvalid[i] && ~in_burst_r; - assign rsp_xbar_data_in[i] = {m_axi_rdata[i], m_axi_rid[i][NUM_PORTS_IN_BITS +: READ_TAG_WIDTH]}; + + // The response crossbar's ready depends on every bank's rvalid and + // rid, so a skid buffer keeps it off the AXI R handshake: rready + // comes from registers only. + wire rsp_buf_ready_in; + wire [DATA_WIDTH-1:0] rsp_buf_data; + wire [READ_FULL_TAG_WIDTH-1:0] rsp_buf_id; + + VX_elastic_buffer #( + .DATAW (DATA_WIDTH + READ_FULL_TAG_WIDTH), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (m_axi_rvalid[i] && ~in_burst_r), + .ready_in (rsp_buf_ready_in), + .data_in ({m_axi_rdata[i], m_axi_rid[i][READ_FULL_TAG_WIDTH-1:0]}), + .data_out ({rsp_buf_data, rsp_buf_id}), + .valid_out (rsp_xbar_valid_in[i]), + .ready_out (rsp_xbar_ready_in[i]) + ); + + assign rsp_xbar_data_in[i] = {rsp_buf_data, rsp_buf_id[NUM_PORTS_IN_BITS +: READ_TAG_WIDTH]}; if (NUM_PORTS_IN > 1) begin : g_input_sel - assign rsp_xbar_sel_in[i] = m_axi_rid[i][0 +: NUM_PORTS_IN_BITS]; + assign rsp_xbar_sel_in[i] = rsp_buf_id[0 +: NUM_PORTS_IN_BITS]; end else begin : g_no_input_sel assign rsp_xbar_sel_in[i] = 0; end - assign m_axi_rready[i] = in_burst_r ? 1'b1 : rsp_xbar_ready_in[i]; + assign m_axi_rready[i] = in_burst_r || rsp_buf_ready_in; `RUNTIME_ASSERT(~(m_axi_rvalid[i] && m_axi_rlast[i] == 0), ("*** AXI response error")) `RUNTIME_ASSERT(~(m_axi_rvalid[i] && m_axi_rresp[i] != 0), ("*** AXI response error")) end diff --git a/hw/rtl/libs/VX_mm_axi_arb.sv b/hw/rtl/libs/VX_mm_axi_arb.sv index 0c9536d093..a45d96375d 100644 --- a/hw/rtl/libs/VX_mm_axi_arb.sv +++ b/hw/rtl/libs/VX_mm_axi_arb.sv @@ -41,8 +41,9 @@ // until WLAST. B/R route back to the owner. IDs pass through untouched. // * MULTI_OUT == 1: ID-routed, multi-outstanding fan-in. The top // LOG2UP(NUM_INPUTS) bits of the AXI ID carry the source index (masters -// must leave them free); reads are fully concurrent, write bursts serialize -// on the shared W channel (AW gated until WLAST) while B stays concurrent. +// must leave them free); reads are fully concurrent, writes share the W +// channel in AW order (a write's W may go with its AW, and later AWs may +// run ahead of earlier data) while B stays concurrent. // Requires ID_WIDTH > LOG2UP(NUM_INPUTS) and NUM_OUTPUTS == 1. // // OUT_REG (0/1): registered AXI stage (VX_mm_axi_slice) on each slave port. @@ -175,6 +176,20 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg aw_held; + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_held ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + if (reset) begin + aw_held <= 1'b0; + end else begin + aw_held <= c_awvalid && ~c_awready; + end + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -182,7 +197,7 @@ module VX_mm_axi_arb #( ) aw_arb ( .clk (clk), .reset (reset), - .requests (s_awvalid), + .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), @@ -245,6 +260,20 @@ module VX_mm_axi_arb #( assign ar_pick = '0; assign ar_any = s_arvalid[0]; end else begin : g_ar_arb + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), @@ -252,7 +281,7 @@ module VX_mm_axi_arb #( ) ar_arb ( .clk (clk), .reset (reset), - .requests (s_arvalid), + .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), @@ -296,10 +325,10 @@ module VX_mm_axi_arb #( // ===== Multi-outstanding, ID-routed (source index in ID high bits) ===== // The top SEL_WIDTH bits of the AXI ID carry the source index; masters - // must leave them free. Reads are fully concurrent; write bursts serialize - // on the shared W channel (AW gated until the prior burst's WLAST) while B - // responses stay concurrent. Sticky arbitration holds the granted addr/id - // stable until the handshake completes (AXI stability requirement). + // must leave them free. Reads are fully concurrent; write data shares + // the W channel in AW order while B responses stay concurrent. An offer + // the slave has not yet taken holds the grant, so its addr/id stay + // stable until the handshake. localparam LOW_ID = ID_WIDTH - SEL_WIDTH; // source's own low ID bits `STATIC_ASSERT((ID_WIDTH > SEL_WIDTH), ("VX_mm_axi_arb MULTI_OUT: ID_WIDTH must exceed LOG2UP(NUM_INPUTS) to carry the source index")) @@ -312,12 +341,26 @@ module VX_mm_axi_arb #( assign ar_any = s_arvalid[0]; end else begin : g_ar_arb wire ar_fire = c_arvalid && c_arready; + // Once offered, the AR must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg ar_held; + reg [SEL_WIDTH-1:0] ar_held_idx; + wire [NUM_INPUTS-1:0] ar_reqs = ar_held ? (s_arvalid & (NUM_INPUTS'(1) << ar_held_idx)) + : s_arvalid; + always @(posedge clk) begin + if (reset) begin + ar_held <= 1'b0; + end else begin + ar_held <= c_arvalid && ~c_arready; + end + ar_held_idx <= ar_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), - .STICKY (1) // hold grant stable until AR fires (AXI stability) + .STICKY (1) ) ar_arb ( - .clk (clk), .reset (reset), .requests (s_arvalid), + .clk (clk), .reset (reset), .requests (ar_reqs), .grant_index (ar_pick), `UNUSED_PIN (grant_onehot), .grant_valid (ar_any), .grant_ready (ar_fire) ); @@ -340,9 +383,19 @@ module VX_mm_axi_arb #( end assign c_rready = s_rready[r_src]; - // ---- Write: sticky RR grant gated on W-drain, tag awid, demux B by bid ---- - reg w_active; - reg [SEL_WIDTH-1:0] w_route; + // ---- Write: sticky RR grant, tag awid, W in AW order, demux B by bid ---- + // An AW is committed the first cycle it is offered; the held grant then + // keeps it on the bus until the slave takes it, so AWs reach the slave + // in commit order. W bursts follow that order through a small queue of + // committed sources, and the burst of the AW being committed this cycle + // bypasses an empty queue: a single-beat write moves AW and W in the + // same cycle, and W never waits for AWREADY (a slave may accept the + // data first). A full queue holds new AWs back. + localparam WQ_SIZE = 4; + localparam WQ_ADDRW = `LOG2UP(WQ_SIZE); + localparam WQ_CNTW = WQ_ADDRW + 1; + + reg aw_pend; // offered last cycle, not yet taken wire [SEL_WIDTH-1:0] aw_pick; wire aw_any; wire aw_fire = c_awvalid && c_awready; @@ -350,43 +403,72 @@ module VX_mm_axi_arb #( assign aw_pick = '0; assign aw_any = s_awvalid[0]; end else begin : g_aw_arb + // Once offered, the AW must stay unchanged until the slave takes it + // (AXI stability): only the offering master may compete meanwhile. + reg [SEL_WIDTH-1:0] aw_held_idx; + wire [NUM_INPUTS-1:0] aw_reqs = aw_pend ? (s_awvalid & (NUM_INPUTS'(1) << aw_held_idx)) + : s_awvalid; + always @(posedge clk) begin + aw_held_idx <= aw_pick; + end VX_generic_arbiter #( .NUM_REQS (NUM_INPUTS), .TYPE (ARBITER), .STICKY (1) ) aw_arb ( - .clk (clk), .reset (reset), .requests (s_awvalid), + .clk (clk), .reset (reset), .requests (aw_reqs), .grant_index (aw_pick), `UNUSED_PIN (grant_onehot), .grant_valid (aw_any), .grant_ready (aw_fire) ); end - // Gate a new AW until the previous write's W burst drains. - assign c_awvalid = aw_any && !w_active; + + reg [WQ_SIZE-1:0][SEL_WIDTH-1:0] wq_src; + reg [WQ_ADDRW-1:0] wq_rd_ptr, wq_wr_ptr; + reg [WQ_CNTW-1:0] wq_count; + wire wq_empty = (wq_count == '0); + wire wq_full = (wq_count == WQ_CNTW'(WQ_SIZE)); + + assign c_awvalid = aw_any && (aw_pend || ~wq_full); assign c_awaddr = s_awaddr[aw_pick]; assign c_awlen = s_awlen [aw_pick]; assign c_awid = {aw_pick, s_awid[aw_pick][LOW_ID-1:0]}; - assign s_awready = {NUM_INPUTS{!w_active && c_awready}} + assign s_awready = {NUM_INPUTS{(aw_pend || ~wq_full) && c_awready}} & ({{(NUM_INPUTS-1){1'b0}}, aw_any} << aw_pick); + wire aw_commit = c_awvalid && ~aw_pend; + + wire w_route_valid = ~wq_empty || aw_commit; + wire [SEL_WIDTH-1:0] w_route = wq_empty ? aw_pick : wq_src[wq_rd_ptr]; + wire w_last_fire = c_wvalid && c_wready && c_wlast; + wire wq_push = aw_commit && ~(wq_empty && w_last_fire); + wire wq_pop = ~wq_empty && w_last_fire; + always @(posedge clk) begin if (reset) begin - w_active <= 1'b0; - w_route <= '0; + aw_pend <= 1'b0; + wq_rd_ptr <= '0; + wq_wr_ptr <= '0; + wq_count <= '0; end else begin - if (aw_fire) begin - w_active <= 1'b1; - w_route <= aw_pick; - end else if (w_active && c_wvalid && c_wready && c_wlast) begin - w_active <= 1'b0; + aw_pend <= c_awvalid && ~c_awready; + if (wq_push) begin + wq_wr_ptr <= wq_wr_ptr + WQ_ADDRW'(1); + end + if (wq_pop) begin + wq_rd_ptr <= wq_rd_ptr + WQ_ADDRW'(1); end + wq_count <= wq_count + WQ_CNTW'(wq_push) - WQ_CNTW'(wq_pop); + end + if (wq_push) begin + wq_src[wq_wr_ptr] <= aw_pick; end end - assign c_wvalid = w_active && s_wvalid[w_route]; + assign c_wvalid = w_route_valid && s_wvalid[w_route]; assign c_wdata = s_wdata[w_route]; assign c_wstrb = s_wstrb[w_route]; assign c_wlast = s_wlast[w_route]; - assign s_wready = {NUM_INPUTS{w_active && c_wready}} + assign s_wready = {NUM_INPUTS{w_route_valid && c_wready}} & ({{(NUM_INPUTS-1){1'b0}}, 1'b1} << w_route); wire [SEL_WIDTH-1:0] b_src = c_bid[ID_WIDTH-1 -: SEL_WIDTH]; diff --git a/hw/rtl/libs/VX_pending_size.sv b/hw/rtl/libs/VX_pending_size.sv index b94889e6e1..a9a1958544 100644 --- a/hw/rtl/libs/VX_pending_size.sv +++ b/hw/rtl/libs/VX_pending_size.sv @@ -64,7 +64,72 @@ module VX_pending_size #( reg empty_r, alm_empty_r; reg full_r, alm_full_r; - if (INCRW != 1 || DECRW != 1) begin : g_wide_step + localparam MAX_INCR = (1 << INCRW) - 1; + localparam MAX_DECR = (1 << DECRW) - 1; + localparam NUM_STEPS = MAX_INCR + MAX_DECR + 1; + + if ((INCRW != 1 || DECRW != 1) && (NUM_STEPS <= 8)) begin : g_select_step + + // incr/decr typically arrive late, so every next size and its flags + // are computed from size_r alone and they only select one. + localparam DELTAW = `MIN(SIZEW, `MAX(INCRW, DECRW)+1); + + logic [SIZEW-1:0] size_r; + wire [NUM_STEPS-1:0][SIZEW+3:0] step_data; + + for (genvar s = 0; s < NUM_STEPS; ++s) begin : g_steps + wire [SIZEW-1:0] step_size = size_r + SIZEW'(s - MAX_DECR); + assign step_data[s] = {step_size, + (step_size == SIZEW'(0)), + (step_size == SIZEW'(SIZE)), + (step_size <= SIZEW'(ALM_EMPTY)), + (step_size >= SIZEW'(ALM_FULL))}; + end + + // each (incr, decr) pair is decoded directly, so no adder sits on them + wire [NUM_STEPS-1:0] step_hot; + for (genvar s = 0; s < NUM_STEPS; ++s) begin : g_hot + wire [MAX_INCR:0][MAX_DECR:0] pair_hit; + for (genvar i = 0; i <= MAX_INCR; ++i) begin : g_i + for (genvar j = 0; j <= MAX_DECR; ++j) begin : g_j + assign pair_hit[i][j] = (i - j + MAX_DECR == s) && (incr == INCRW'(i)) && (decr == DECRW'(j)); + end + end + assign step_hot[s] = (| pair_hit); + end + + wire [SIZEW-1:0] size_n; + wire empty_n, full_n, alm_empty_n, alm_full_n; + VX_onehot_mux #( + .DATAW (SIZEW + 4), + .N (NUM_STEPS) + ) step_mux ( + .data_in (step_data), + .sel_in (step_hot), + .data_out ({size_n, empty_n, full_n, alm_empty_n, alm_full_n}) + ); + + always @(posedge clk) begin + if (reset) begin + empty_r <= 1; + full_r <= 0; + alm_empty_r <= 1; + alm_full_r <= 0; + size_r <= '0; + end else begin + `ASSERT((DELTAW'(incr) <= DELTAW'(decr)) || (size_n >= size_r), ("runtime error: counter overflow")); + `ASSERT((DELTAW'(incr) >= DELTAW'(decr)) || (size_n <= size_r), ("runtime error: counter underflow")); + empty_r <= empty_n; + full_r <= full_n; + alm_empty_r <= alm_empty_n; + alm_full_r <= alm_full_n; + size_r <= size_n; + end + end + + assign size = size_r; + + end else if (INCRW != 1 || DECRW != 1) begin : g_wide_step localparam DELTAW = `MIN(SIZEW, `MAX(INCRW, DECRW)+1); diff --git a/hw/rtl/libs/VX_placeholder.sv b/hw/rtl/libs/VX_placeholder.sv deleted file mode 100644 index 7642bf353a..0000000000 --- a/hw/rtl/libs/VX_placeholder.sv +++ /dev/null @@ -1,26 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_platform.vh" - -`TRACING_OFF -`BLACKBOX_CELL module VX_placeholder #( - parameter I = 0, - parameter O = 0 -) ( - input wire [`UP(I)-1:0] in, - output wire [`UP(O)-1:0] out -); - -endmodule -`TRACING_ON diff --git a/hw/rtl/libs/VX_sp_ram.sv b/hw/rtl/libs/VX_sp_ram.sv index 0d3694a636..13e00641fc 100644 --- a/hw/rtl/libs/VX_sp_ram.sv +++ b/hw/rtl/libs/VX_sp_ram.sv @@ -67,6 +67,9 @@ end `endif +// Read forms, as in VX_dp_ram. OUT_REG=1 with RDW_MODE "W" registers the +// address, with "R" registers the data, with "N" registers the data and holds it +// across a write. OUT_REG=0 is combinational and never block RAM. `TRACING_OFF module VX_sp_ram #( parameter DATAW = 1, @@ -75,9 +78,6 @@ module VX_sp_ram #( parameter OUT_REG = 0, parameter LUTRAM = 0, parameter `STRING RDW_MODE = "W", // W: write-first, R: read-first, N: no-change - parameter RADDR_REG = 0, // read address registered hint - parameter RADDR_RESET = 0, // read address has reset - parameter RDW_ASSERT = 0, parameter RESET_RAM = 0, parameter INIT_ENABLE = 0, parameter INIT_FILE = "", @@ -95,35 +95,56 @@ module VX_sp_ram #( ); localparam WSELW = DATAW / WRENW; `UNUSED_PARAM (LUTRAM) - `UNUSED_PARAM (RADDR_REG) - `UNUSED_PARAM (RADDR_RESET) `STATIC_ASSERT(!(WRENW * WSELW != DATAW), ("invalid parameter")) `STATIC_ASSERT((RDW_MODE == "R" || RDW_MODE == "W" || RDW_MODE == "N"), ("invalid parameter")) - `UNUSED_PARAM (RDW_ASSERT) `ifdef SYNTHESIS localparam FORCE_BRAM = !LUTRAM && `FORCE_BRAM(SIZE, DATAW); - // Async reads have no native block-RAM form, so forcing BRAM pulls in the - // async-RAM patch. That only pays off for a read-first RAM; a write-first one - // adds a read/write collision bypass that is larger than plain distributed RAM. - localparam FORCE_BRAM_ASYNC = FORCE_BRAM && (OUT_REG == 0) && (RDW_MODE == "R"); `ifdef ASIC - if (FORCE_BRAM && (OUT_REG != 0 || RADDR_REG != 0)) begin : g_asic + // The macro reads synchronously, so a combinational read stays in logic. + if (FORCE_BRAM && OUT_REG != 0) begin : g_asic + `UNUSED_VAR (reset) + wire [WRENW-1:0] wr_lanes = (WRENW != 1) ? wren : {WRENW{1'b1}}; + wire [DATAW-1:0] macro_rdata; VX_sp_ram_asic #( .DATAW (DATAW), .SIZE (SIZE), .WRENW (WRENW) ) sp_ram_asic ( .clk (clk), - .reset (reset), - .read (read), + .reset (1'b0), + .read ((RDW_MODE == "N") ? (read && ~write) : read), .write (write), - .wren (wren), + .wren (wr_lanes), .addr (addr), .wdata (wdata), - .rdata (rdata) + .rdata (macro_rdata) ); + if (RDW_MODE == "W") begin : g_write_first + // As in VX_dp_ram: the lanes written at the captured address since + // the read are forwarded around the read-first macro. + reg [ADDRW-1:0] addr_r; + reg [WRENW-1:0] fwd_en_r; + reg [DATAW-1:0] fwd_data_r; + wire [WRENW-1:0] fwd_lanes = {WRENW{write && (read || addr == addr_r)}} & wr_lanes; + always @(posedge clk) begin + if (read) begin + addr_r <= addr; + end + fwd_en_r <= (read ? '0 : fwd_en_r) | fwd_lanes; + for (integer i = 0; i < WRENW; ++i) begin + if (fwd_lanes[i]) begin + fwd_data_r[i * WSELW +: WSELW] <= wdata[i * WSELW +: WSELW]; + end + end + end + for (genvar i = 0; i < WRENW; ++i) begin : g_lanes + assign rdata[i * WSELW +: WSELW] = fwd_en_r[i] ? fwd_data_r[i * WSELW +: WSELW] : macro_rdata[i * WSELW +: WSELW]; + end + end else begin : g_read_first + assign rdata = macro_rdata; + end end else begin : g_no_asic `else if (1) begin : g_no_asic @@ -143,20 +164,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -213,7 +230,7 @@ module VX_sp_ram #( // columns whose long routes dominate the path delay. if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren - `USE_FAST_BRAM `RAM_ARRAY_WREN + `RW_RAM_CHECK `USE_FAST_BRAM `RAM_ARRAY_WREN `RAM_INITIALIZATION reg [ADDRW-1:0] addr_r; always @(posedge clk) begin @@ -224,20 +241,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK `USE_FAST_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -291,7 +304,7 @@ module VX_sp_ram #( end else begin : g_auto if (RDW_MODE == "W") begin : g_write_first if (WRENW != 1) begin : g_wren - `RAM_ARRAY_WREN + `RW_RAM_CHECK `RAM_ARRAY_WREN `RAM_INITIALIZATION reg [ADDRW-1:0] addr_r; always @(posedge clk) begin @@ -302,20 +315,16 @@ module VX_sp_ram #( end assign rdata = ram[addr_r]; end else begin : g_no_wren - reg [DATAW-1:0] ram [0:SIZE-1]; + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; `RAM_INITIALIZATION - reg [DATAW-1:0] rdata_r; + reg [ADDRW-1:0] addr_r; always @(posedge clk) begin `RAM_WRITE_ALL if (read) begin - if (write) begin - rdata_r <= wdata; - end else begin - rdata_r <= ram[addr]; - end + addr_r <= addr; end end - assign rdata = rdata_r; + assign rdata = ram[addr_r]; end end else if (RDW_MODE == "R") begin : g_read_first if (WRENW != 1) begin : g_wren @@ -369,99 +378,37 @@ module VX_sp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (FORCE_BRAM_ASYNC) begin : g_bram - `ifdef ASYNC_BRAM_PATCH - VX_async_ram_patch #( - .DATAW (DATAW), - .SIZE (SIZE), - .WRENW (WRENW), - .DUAL_PORT (0), - .FORCE_BRAM (FORCE_BRAM), - .RADDR_REG (RADDR_REG), - .RADDR_RESET(RADDR_RESET), - .WRITE_FIRST(RDW_MODE == "W"), - .INIT_ENABLE(INIT_ENABLE), - .INIT_FILE (INIT_FILE), - .INIT_VALUE (INIT_VALUE) - ) async_ram_patch ( - .clk (clk), - .reset (reset), - .read (read), - .write (write), - .wren (wren), - .waddr (addr), - .wdata (wdata), - .raddr (addr), - .rdata (rdata) - ); - `else - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + if (RDW_MODE == "W") begin : g_write_first + if (WRENW != 1) begin : g_wren + `RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK `USE_BLOCK_BRAM reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + assign rdata = ram[addr]; + end else begin : g_no_wren + `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[addr]; end - `endif - end else begin : g_auto - if (RDW_MODE == "W") begin : g_write_first - if (WRENW != 1) begin : g_wren - `RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + end else begin : g_read_first + if (WRENW != 1) begin : g_wren + `NO_RW_RAM_CHECK `RAM_ARRAY_WREN + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_WREN end - end else begin : g_read_first - if (WRENW != 1) begin : g_wren - `NO_RW_RAM_CHECK `RAM_ARRAY_WREN - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_WREN - end - assign rdata = ram[addr]; - end else begin : g_no_wren - `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; - `RAM_INITIALIZATION - always @(posedge clk) begin - `RAM_WRITE_ALL - end - assign rdata = ram[addr]; + assign rdata = ram[addr]; + end else begin : g_no_wren + `NO_RW_RAM_CHECK reg [DATAW-1:0] ram [0:SIZE-1]; + `RAM_INITIALIZATION + always @(posedge clk) begin + `RAM_WRITE_ALL end + assign rdata = ram[addr]; end end end @@ -522,28 +469,7 @@ module VX_sp_ram #( end end else begin : g_async `UNUSED_VAR (read) - if (RDW_MODE == "W") begin : g_write_first - assign rdata = ram[addr]; - end else begin : g_read_first - reg [DATAW-1:0] prev_data; - reg [ADDRW-1:0] prev_waddr; - reg prev_write; - always @(posedge clk) begin - if (reset) begin - prev_write <= 0; - prev_data <= '0; - prev_waddr <= '0; - end else begin - prev_write <= write; - prev_data <= ram[addr]; - prev_waddr <= addr; - end - end - assign rdata = (prev_write && (prev_waddr == addr)) ? prev_data : ram[addr]; - if (RDW_ASSERT) begin : g_rw_asert - `RUNTIME_ASSERT(~read || (rdata == ram[addr]), ("read after write hazard")) - end - end + assign rdata = ram[addr]; end `endif diff --git a/hw/rtl/libs/VX_stream_arb.sv b/hw/rtl/libs/VX_stream_arb.sv index 9ef71a6cfd..64e4eba814 100644 --- a/hw/rtl/libs/VX_stream_arb.sv +++ b/hw/rtl/libs/VX_stream_arb.sv @@ -50,24 +50,29 @@ module VX_stream_arb #( localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); localparam DATAW2 = DATAW + LOG_NUM_REQS2; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_OUTPUTS; + // a last slice with fewer inputs than outputs feeds only the outputs of its inputs + localparam LAST_OUTPUTS = `MIN(NUM_INPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_OUTPUTS); + localparam JOIN_INPUTS = (NUM_SLICES - 1) * NUM_OUTPUTS + LAST_OUTPUTS; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0][DATAW2-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_OUTPUTS-1:0] ready_tmp; + wire [JOIN_INPUTS-1:0] valid_tmp; + wire [JOIN_INPUTS-1:0][DATAW2-1:0] data_tmp; + wire [JOIN_INPUTS-1:0] ready_tmp; for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_OUTPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_INPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_OUTPUTS = `MIN(SLICE_SIZE, NUM_OUTPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_OUTPUTS))); - wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; - wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS2-1:0] sel_tmp_u; + wire [SLICE_OUTPUTS-1:0][DATAW-1:0] data_tmp_u; + wire [SLICE_OUTPUTS-1:0][SLICE_SEL_W-1:0] sel_tmp_u; VX_stream_arb #( .NUM_INPUTS (SLICE_SIZE), - .NUM_OUTPUTS (NUM_OUTPUTS), + .NUM_OUTPUTS (SLICE_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -79,14 +84,14 @@ module VX_stream_arb #( .valid_in (valid_in[SLICE_END-1: SLICE_BEGIN]), .data_in (data_in[SLICE_END-1: SLICE_BEGIN]), .ready_in (ready_in[SLICE_END-1: SLICE_BEGIN]), - .valid_out (valid_tmp[s]), + .valid_out (valid_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .data_out (data_tmp_u), - .ready_out (ready_tmp[s]), + .ready_out (ready_tmp[s * NUM_OUTPUTS +: SLICE_OUTPUTS]), .sel_out (sel_tmp_u) ); - for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_tmp - assign data_tmp[s][o] = {data_tmp_u[o], sel_tmp_u[o]}; + for (genvar o = 0; o < SLICE_OUTPUTS; ++o) begin : g_data_tmp + assign data_tmp[s * NUM_OUTPUTS + o] = {data_tmp_u[o], LOG_NUM_REQS2'(sel_tmp_u[o])}; end end @@ -94,7 +99,7 @@ module VX_stream_arb #( wire [NUM_OUTPUTS-1:0][LOG_NUM_REQS3-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_SLICES * NUM_OUTPUTS), + .NUM_INPUTS (JOIN_INPUTS), .NUM_OUTPUTS (NUM_OUTPUTS), .DATAW (DATAW2), .ARBITER (ARBITER), @@ -210,15 +215,19 @@ module VX_stream_arb #( localparam NUM_SLICES = `CDIV(NUM_REQS, MAX_FANOUT); localparam LOG_NUM_REQS2 = `CLOG2(MAX_FANOUT); localparam LOG_NUM_REQS3 = `CLOG2(NUM_SLICES); - - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] valid_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0][DATAW-1:0] data_tmp; - wire [NUM_SLICES-1:0][NUM_INPUTS-1:0] ready_tmp; + localparam SLICE_STRIDE = MAX_FANOUT * NUM_INPUTS; + // a last slice with fewer outputs than inputs takes only the inputs of its outputs + localparam LAST_INPUTS = `MIN(NUM_OUTPUTS - (NUM_SLICES - 1) * SLICE_STRIDE, NUM_INPUTS); + localparam FORK_OUTPUTS = (NUM_SLICES - 1) * NUM_INPUTS + LAST_INPUTS; + + wire [FORK_OUTPUTS-1:0] valid_tmp; + wire [FORK_OUTPUTS-1:0][DATAW-1:0] data_tmp; + wire [FORK_OUTPUTS-1:0] ready_tmp; wire [NUM_INPUTS-1:0][LOG_NUM_REQS3-1:0] sel_tmp; VX_stream_arb #( .NUM_INPUTS (NUM_INPUTS), - .NUM_OUTPUTS (NUM_SLICES * NUM_INPUTS), + .NUM_OUTPUTS (FORK_OUTPUTS), .DATAW (DATAW), .ARBITER (ARBITER), .STICKY (STICKY), @@ -240,15 +249,16 @@ module VX_stream_arb #( for (genvar s = 0; s < NUM_SLICES; ++s) begin : g_slice_arbs - localparam SLICE_STRIDE= MAX_FANOUT * NUM_INPUTS; localparam SLICE_BEGIN = s * SLICE_STRIDE; localparam SLICE_END = `MIN(SLICE_BEGIN + SLICE_STRIDE, NUM_OUTPUTS); localparam SLICE_SIZE = SLICE_END - SLICE_BEGIN; + localparam SLICE_INPUTS= `MIN(SLICE_SIZE, NUM_INPUTS); + localparam SLICE_SEL_W = `UP(`CLOG2(`CDIV(SLICE_SIZE, SLICE_INPUTS))); - wire [NUM_INPUTS-1:0][LOG_NUM_REQS2-1:0] sel_out_u; + wire [SLICE_INPUTS-1:0][SLICE_SEL_W-1:0] sel_out_u; VX_stream_arb #( - .NUM_INPUTS (NUM_INPUTS), + .NUM_INPUTS (SLICE_INPUTS), .NUM_OUTPUTS (SLICE_SIZE), .DATAW (DATAW), .ARBITER (ARBITER), @@ -258,14 +268,23 @@ module VX_stream_arb #( ) fanout_slice_arb ( .clk (clk), .reset (reset), - .valid_in (valid_tmp[s]), - .ready_in (ready_tmp[s]), - .data_in (data_tmp[s]), + .valid_in (valid_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .ready_in (ready_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), + .data_in (data_tmp[s * NUM_INPUTS +: SLICE_INPUTS]), .data_out (data_out[SLICE_END-1: SLICE_BEGIN]), .valid_out (valid_out[SLICE_END-1: SLICE_BEGIN]), .ready_out (ready_out[SLICE_END-1: SLICE_BEGIN]), - .sel_out (sel_out_w[s]) + .sel_out (sel_out_u) ); + + // inputs past a narrow last slice are never forked to it + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out_w + if (i < SLICE_INPUTS) begin : g_sel + assign sel_out_w[s][i] = LOG_NUM_REQS2'(sel_out_u[i]); + end else begin : g_pad + assign sel_out_w[s][i] = '0; + end + end end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_out @@ -284,7 +303,11 @@ module VX_stream_arb #( wire [NUM_INPUTS-1:0] requests; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_i localparam o = r * NUM_INPUTS + i; - assign requests[i] = ready_out[o]; + if (o < NUM_OUTPUTS) begin : g_req_ready + assign requests[i] = ready_out[o]; + end else begin : g_req_pad + assign requests[i] = 1'b0; + end end assign arb_requests[r] = (| requests); end @@ -318,7 +341,11 @@ module VX_stream_arb #( wire [NUM_REQS-1:0] ready_out_s; for (genvar r = 0; r < NUM_REQS; ++r) begin : g_r localparam o = r * NUM_INPUTS + i; - assign ready_out_s[r] = ready_out_w[o]; + if (o < NUM_OUTPUTS) begin : g_ready + assign ready_out_s[r] = ready_out_w[o]; + end else begin : g_padding + assign ready_out_s[r] = 1'b0; + end end assign ready_in[i] = (NUM_INPUTS == 1) ? arb_valid : (| (ready_out_s & arb_onehot)); end diff --git a/hw/rtl/libs/VX_stream_buffer.sv b/hw/rtl/libs/VX_stream_buffer.sv index 9f0f08df4c..eb561140b6 100644 --- a/hw/rtl/libs/VX_stream_buffer.sv +++ b/hw/rtl/libs/VX_stream_buffer.sv @@ -46,7 +46,6 @@ module VX_stream_buffer #( reg [DATAW-1:0] data_out_r, buffer_r; reg valid_out_r, valid_in_r; - wire fire_in = valid_in && ready_in; wire flow_out = ready_out || ~valid_out; always @(posedge clk) begin @@ -67,8 +66,11 @@ module VX_stream_buffer #( if (OUT_REG != 0) begin : g_out_reg + // The skid register is read only once the input side closes, which + // takes an accepted input; while open it may follow the input bus, + // so its wide enable does not wait on valid_in. always @(posedge clk) begin - if (fire_in) begin + if (valid_in_r) begin buffer_r <= data_in; end end @@ -83,6 +85,8 @@ module VX_stream_buffer #( end else begin : g_no_out_reg + wire fire_in = valid_in && ready_in; + always @(posedge clk) begin if (fire_in) begin data_out_r <= data_in; diff --git a/hw/rtl/libs/VX_stream_omega.sv b/hw/rtl/libs/VX_stream_omega.sv index 1723a21e4b..ca4b0cdedb 100644 --- a/hw/rtl/libs/VX_stream_omega.sv +++ b/hw/rtl/libs/VX_stream_omega.sv @@ -39,10 +39,20 @@ module VX_stream_omega #( output wire [NUM_OUTPUTS-1:0][IN_WIDTH-1:0] sel_out, input wire [NUM_OUTPUTS-1:0] ready_out, - output wire [PERF_CTR_BITS-1:0] collisions + output wire [PERF_CTR_BITS-1:0] collisions, + output wire empty ); `STATIC_ASSERT (`IS_POW2(RADIX), ("inavlid parameters")) + // With one or two entries per switch output and no arbiter tree, an + // output buffer holds an item exactly when its registered valid is set, + // so the network is empty when no switch output is valid. + localparam BUF_KIND = OUT_BUF & 7; + localparam VALID_HOLDS = (BUF_KIND >= 1 && BUF_KIND <= 3) + && (MAX_FANOUT == 0 || RADIX <= (MAX_FANOUT + MAX_FANOUT / 2)); + + wire buf_any_valid; + // If network size smaller than radix, simply use a crossbar. if (NUM_INPUTS <= RADIX && NUM_OUTPUTS <= RADIX) begin : g_fallback VX_stream_xbar #( @@ -66,6 +76,7 @@ module VX_stream_omega #( .ready_out, .collisions ); + assign buf_any_valid = (| valid_out); end else begin : g_omega localparam RADIX_LG = `LOG2UP(RADIX); localparam N_INPUTS_M = `MAX(NUM_INPUTS, NUM_OUTPUTS); @@ -207,6 +218,40 @@ module VX_stream_omega #( end assign collisions = collisions_r; + assign buf_any_valid = (| switch_valid_out); + end + + if (BUF_KIND == 0) begin : g_empty_passthru + `UNUSED_VAR (buf_any_valid) + assign empty = 1'b1; + end else if (VALID_HOLDS) begin : g_empty_valid + assign empty = ~buf_any_valid; + end else begin : g_empty_count + // Deeper output buffers hide items behind an invalid output, so the + // network counts what it takes and gives. + localparam IN_W = `CLOG2(NUM_INPUTS+1); + localparam OUT_W = `CLOG2(NUM_OUTPUTS+1); + localparam CAPACITY = 4 * `MAX(NUM_INPUTS, NUM_OUTPUTS) * (`CLOG2(`MAX(NUM_INPUTS, NUM_OUTPUTS)) + 1); + wire [IN_W-1:0] in_cnt; + wire [OUT_W-1:0] out_cnt; + `POP_COUNT(in_cnt, valid_in & ready_in); + `POP_COUNT(out_cnt, valid_out & ready_out); + `UNUSED_VAR (buf_any_valid) + VX_pending_size #( + .SIZE (CAPACITY), + .INCRW (IN_W), + .DECRW (OUT_W) + ) pending ( + .clk (clk), + .reset (reset), + .incr (in_cnt), + .decr (out_cnt), + .empty (empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); end endmodule diff --git a/hw/rtl/libs/VX_stream_xbar.sv b/hw/rtl/libs/VX_stream_xbar.sv index e7b18362c2..c8002a0146 100644 --- a/hw/rtl/libs/VX_stream_xbar.sv +++ b/hw/rtl/libs/VX_stream_xbar.sv @@ -63,8 +63,11 @@ module VX_stream_xbar #( .data_out (per_output_ready_in_w) ); + // An output arbiter may assert ready for an input that does not + // target it (a single-input fan-in slice passes its buffer's + // ready through), so take the ready of the selected output only. for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_ready_in - assign ready_in[i] = | per_output_ready_in_w[i]; + assign ready_in[i] = per_output_ready_in_w[i][sel_in[i]]; end for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_sel_in_demux diff --git a/hw/rtl/mem/VX_cp_llc_port.sv b/hw/rtl/mem/VX_cp_llc_port.sv new file mode 100644 index 0000000000..ac764277f8 --- /dev/null +++ b/hw/rtl/mem/VX_cp_llc_port.sv @@ -0,0 +1,157 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Command-processor DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; +// a write flagged no-allocate writes around it on a miss. +// A launch that follows DMA writes is held until those writes are inside the +// cache banks: the port sends an L1-only flush token behind them, which the +// last-level cache answers once its input crossbar has drained, and only then +// passes the start pulse on. Reads need no such fence, since the DMA waits for +// their data. +module VX_cp_llc_port import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 2, + parameter TAG_WIDTH = SLOT_TAG_WIDTH - UUID_WIDTH - 1, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + VX_mem_bus_if.master mem_bus_if +); + `STATIC_ASSERT(TAG_WIDTH >= 1, ("invalid parameter: TAG_WIDTH=%0d", TAG_WIDTH)) + `STATIC_ASSERT(0 == (`VX_MEM_IO_BASE_ADDR % DATA_SIZE), ("invalid parameter")) + + localparam ASHIFT = `CLOG2(DATA_SIZE); + localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_SIZE * 8 + DATA_SIZE + `UP(MEM_ATTR_WIDTH) + SLOT_TAG_WIDTH; + + localparam STATE_IDLE = 0; + localparam STATE_FENCE = 1; + localparam STATE_WAIT = 2; + + reg [1:0] state; + reg wr_pending; + + wire fence_rsp = mem_bus_if.rsp_valid && mem_bus_if.rsp_data.tag.value[0]; + + wire [ADDR_WIDTH-1:0] io_addr_start = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_BASE_ADDR) >> ASHIFT); + wire [ADDR_WIDTH-1:0] io_addr_end = ADDR_WIDTH'(`VX_CFG_XLEN'(`VX_MEM_IO_END_ADDR) >> ASHIFT); + + mem_bus_attr_t req_attr; + always @(*) begin + req_attr = '0; + req_attr.is_addr_io = (req_addr >= io_addr_start) && (req_addr < io_addr_end); + req_attr.is_noalloc = req_rw && req_noalloc; + end + + wire [`UP(MEM_ATTR_WIDTH)-1:0] fence_attr = `UP(MEM_ATTR_WIDTH)'(1 << MEM_ATTR_FLUSH_OFFS); + + wire [SLOT_TAG_WIDTH-1:0] req_slot_tag = SLOT_TAG_WIDTH'({req_tag, 1'b0}); + wire [SLOT_TAG_WIDTH-1:0] fence_slot_tag = SLOT_TAG_WIDTH'(1); + + wire fence_req = (state == STATE_FENCE); + + wire buf_valid_in = fence_req || ((state == STATE_IDLE) && req_valid); + wire buf_ready_in; + wire [REQ_DATAW-1:0] buf_data_in = fence_req ? + {1'b0, ADDR_WIDTH'(0), (DATA_SIZE*8)'(0), DATA_SIZE'(1), fence_attr, fence_slot_tag} : + {req_rw, req_addr, req_data, req_byteen, `UP(MEM_ATTR_WIDTH)'(req_attr), req_slot_tag}; + + assign req_ready = (state == STATE_IDLE) && buf_ready_in; + + wire req_fire = req_valid && req_ready; + wire fence_fire = fence_req && buf_ready_in; + + // Registered ready: the slot's ready is the last-level cache's crossbar + // grant, which must not reach this buffer's enable combinationally. + VX_elastic_buffer #( + .DATAW (REQ_DATAW), + .SIZE (`TO_OUT_BUF_SIZE(3)), + .OUT_REG (`TO_OUT_BUF_REG(3)) + ) req_buf ( + .clk (clk), + .reset (reset), + .valid_in (buf_valid_in), + .ready_in (buf_ready_in), + .data_in (buf_data_in), + .data_out ({mem_bus_if.req_data.rw, + mem_bus_if.req_data.addr, + mem_bus_if.req_data.data, + mem_bus_if.req_data.byteen, + mem_bus_if.req_data.attr, + mem_bus_if.req_data.tag}), + .ready_out (mem_bus_if.req_ready), + .valid_out (mem_bus_if.req_valid) + ); + + always @(posedge clk) begin + if (reset) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end else begin + case (state) + STATE_FENCE: begin + if (fence_fire) begin + state <= STATE_WAIT; + end + end + STATE_WAIT: begin + if (fence_rsp) begin + state <= STATE_IDLE; + wr_pending <= 1'b0; + end + end + default: begin + if (start_in && wr_pending) begin + state <= STATE_FENCE; + end else if (req_fire && req_rw) begin + wr_pending <= 1'b1; + end + end + endcase + end + end + + assign start_out = ((state == STATE_IDLE) && start_in && ~wr_pending) + || ((state == STATE_WAIT) && fence_rsp); + assign busy = (state != STATE_IDLE); + + assign rsp_valid = mem_bus_if.rsp_valid && ~fence_rsp; + assign rsp_data = mem_bus_if.rsp_data.data; + assign rsp_tag = mem_bus_if.rsp_data.tag.value[1 +: TAG_WIDTH]; + assign mem_bus_if.rsp_ready = fence_rsp || rsp_ready; + + `RUNTIME_ASSERT (~(start_in && (state != STATE_IDLE)), ("%t: launch while a DMA fence is pending", $time)) + +endmodule diff --git a/hw/rtl/mem/VX_gbar_unit.sv b/hw/rtl/mem/VX_gbar_unit.sv index 9af6345ef9..072105d63a 100644 --- a/hw/rtl/mem/VX_gbar_unit.sv +++ b/hw/rtl/mem/VX_gbar_unit.sv @@ -23,36 +23,35 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( ); `UNUSED_SPARAM (INSTANCE_ID) - reg [`VX_CFG_NUM_BARRIERS-1:0][`VX_CFG_NUM_CORES-1:0] barrier_masks; - wire [`CLOG2(`VX_CFG_NUM_CORES+1)-1:0] active_barrier_count; + // A core sends one request per barrier generation, so counting requests + // tracks the participants without a per-core mask over the whole device. + reg [`VX_CFG_NUM_BARRIERS-1:0][CORE_ID_WIDTH-1:0] arrive_counts; wire [NB_WIDTH-1:0] req_id = gbar_bus_if.req_data.id; - wire [`VX_CFG_NUM_CORES-1:0] curr_barrier_mask = barrier_masks[req_id]; - - `POP_COUNT(active_barrier_count, curr_barrier_mask); - `UNUSED_VAR (active_barrier_count) + wire [CORE_ID_WIDTH-1:0] curr_arrive_count = arrive_counts[req_id]; + `UNUSED_VAR (gbar_bus_if.req_data.core_id) reg rsp_valid; reg [NB_WIDTH-1:0] rsp_bar_id; - wire req_file = gbar_bus_if.req_valid && gbar_bus_if.req_ready; + wire req_fire = gbar_bus_if.req_valid && gbar_bus_if.req_ready; wire rsp_fire = gbar_bus_if.rsp_valid && gbar_bus_if.rsp_ready; wire rsp_stall = gbar_bus_if.rsp_valid && ~gbar_bus_if.rsp_ready; always @(posedge clk) begin if (reset) begin - barrier_masks <= '0; + arrive_counts <= '0; rsp_valid <= 0; end else begin if (rsp_fire) begin rsp_valid <= 0; end - if (req_file) begin - if (active_barrier_count[NC_WIDTH-1:0] == gbar_bus_if.req_data.size_m1) begin - barrier_masks[req_id] <= '0; + if (req_fire) begin + if (curr_arrive_count == gbar_bus_if.req_data.size_m1) begin + arrive_counts[req_id] <= '0; rsp_valid <= 1; rsp_bar_id <= req_id; end else begin - barrier_masks[req_id][gbar_bus_if.req_data.core_id] <= 1; + arrive_counts[req_id] <= curr_arrive_count + CORE_ID_WIDTH'(1); end end end @@ -67,7 +66,7 @@ module VX_gbar_unit import VX_gpu_pkg::*; #( `ifdef DBG_TRACE_GBAR always @(posedge clk) begin - if (req_file) begin + if (req_fire) begin `TRACE(2, ("%t: %s arrive: bar_id=%0d, size=%0d, core_id=%0d\n", $time, INSTANCE_ID, gbar_bus_if.req_data.id, gbar_bus_if.req_data.size_m1, gbar_bus_if.req_data.core_id)) end diff --git a/hw/rtl/mem/VX_lmem_engine.sv b/hw/rtl/mem/VX_lmem_engine.sv new file mode 100644 index 0000000000..ef083fc8e0 --- /dev/null +++ b/hw/rtl/mem/VX_lmem_engine.sv @@ -0,0 +1,442 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One pass engine of the warp-wide local memory. It takes a whole warp request +// and gives it to the banks in passes: in a pass every bank goes to the lowest +// pending lane that maps to it, and a plain read also serves every pending lane +// reading that lane's word. Lanes left over replay in the next passes, and the +// request retires before the next one starts. +// +// S0 the engine's input queue of two requests; the head's lanes' words are +// compared pairwise +// S1 the request, its comparison matrices, its pending lanes and the next +// pass's bank winners; a pass moves the winners' rows and payloads into +// the bank registers +// then the banks access their SRAM, two cycles after the pass each served +// lane takes its bank's word into the response register, and from there the +// response enters the response queue, or passes it when it is empty, on its +// way to its block's response register. +// +// The pair matrices are computed once per request, and each pass's bank +// winners and whether it is the last are registered ahead of it, so nothing +// combinational sits in front of the bank registers but the winners' row and +// payload mux. The first pass's winners come from S0; a replay finds the next +// winners in the cycle after a pass, so a conflicting request replays at one +// pass per two cycles. The last pass frees S1 in its own cycle. +module VX_lmem_engine import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter NUM_LANES = 4, + parameter NUM_BANKS = 4, + parameter WORD_SIZE = 4, + // word address width + parameter ADDR_WIDTH = 12, + parameter TAG_WIDTH = 8, + // per-lane write payload carried to the banks (data, byte enables, atomic sideband) + parameter PAYLOAD_WIDTH = 1, + // request attributes common to every lane + parameter ATTR_WIDTH = 1, + parameter NUM_BLOCKS = 1, + parameter RSP_QUEUE_SIZE = 8, + + parameter WORD_WIDTH = WORD_SIZE * 8, + parameter BLOCK_WIDTH = `UP(`CLOG2(NUM_BLOCKS)), + parameter DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1), + parameter BANK_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS) +) ( + input wire clk, + input wire reset, + + // request + input wire req_valid, + input wire [BLOCK_WIDTH-1:0] req_block, + // reads and atomics respond, stores do not + input wire req_rw, + // a plain read: its lanes of one word share a bank access + input wire req_read, + input wire req_amo, + input wire [ATTR_WIDTH-1:0] req_attr, + input wire [NUM_LANES-1:0] req_mask, + input wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, + input wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] req_payload, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + // the engine holds a request; the one it holds is a store or an atomic + output wire busy, + output wire holds_write, + + // the banks are free for a pass this cycle + input wire pass_en, + // an atomic pass went out + output wire pass_amo, + + // bank requests, registered + output wire [NUM_BANKS-1:0] bank_valid, + output wire bank_rw, + output wire bank_amo, + output wire [ATTR_WIDTH-1:0] bank_attr, + output wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] bank_addr, + output wire [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] bank_payload, + + // the banks' read data, two cycles after the pass + input wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] bank_rdata, + + // the response queue's head, taken into a block's response register; + // rsp_done counts this engine's responses the blocks took this cycle + output wire rsp_valid, + output wire [BLOCK_WIDTH-1:0] rsp_block, + output wire [NUM_LANES-1:0] rsp_mask, + output wire [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_take, + input wire [DONE_WIDTH-1:0] rsp_done, + + // perf: lanes served by this cycle's pass, whether they store, and + // whether lanes were left over + output wire [NUM_LANES-1:0] perf_served, + output wire perf_store, + output wire perf_replay +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); + localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); + localparam CREDIT_WIDTH = `CLOG2(RSP_QUEUE_SIZE + 1); + localparam RSP_DATAW = BLOCK_WIDTH + NUM_LANES + NUM_LANES * WORD_WIDTH + TAG_WIDTH; + + `STATIC_ASSERT(ADDR_WIDTH > BANK_SEL_BITS, ("invalid parameter")) + + // S0: the engine's input queue. Holding two requests, it takes one in any + // cycle that starts with at most one, so its ready is a register and the + // head stays put while S1 is busy. + + localparam S0_DATAW = BLOCK_WIDTH + 3 + ATTR_WIDTH + NUM_LANES + + NUM_LANES * (ADDR_WIDTH + PAYLOAD_WIDTH) + TAG_WIDTH; + + wire s0_valid; + wire [BLOCK_WIDTH-1:0] s0_block; + wire s0_rw; + wire s0_read; + wire s0_amo; + wire [ATTR_WIDTH-1:0] s0_attr; + wire [NUM_LANES-1:0] s0_mask; + wire [NUM_LANES-1:0][ADDR_WIDTH-1:0] s0_addr; + wire [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s0_payload; + wire [TAG_WIDTH-1:0] s0_tag; + wire s1_free; + + VX_elastic_buffer #( + .DATAW (S0_DATAW), + .SIZE (2), + .OUT_REG (1) + ) s0_buf ( + .clk (clk), + .reset (reset), + .valid_in (req_valid), + .ready_in (req_ready), + .data_in ({req_block, req_rw, req_read, req_amo, req_attr, req_mask, req_addr, req_payload, req_tag}), + .data_out ({s0_block, s0_rw, s0_read, s0_amo, s0_attr, s0_mask, s0_addr, s0_payload, s0_tag}), + .valid_out (s0_valid), + .ready_out (s1_free) + ); + + // pairwise comparison: same_bank[i][j] and same_word[i][j] for lower lanes j < i + + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_same_bank; + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_same_word; + wire [NUM_LANES-1:0][NUM_LANES-1:0] s0_conflict; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_cmp_i + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_cmp_j + if (j < i) begin : g_lower + if (BANK_SEL_BITS != 0) begin : g_banks + assign s0_same_bank[i][j] = (s0_addr[i][BANK_SEL_BITS-1:0] == s0_addr[j][BANK_SEL_BITS-1:0]); + end else begin : g_one_bank + assign s0_same_bank[i][j] = 1'b1; + end + assign s0_same_word[i][j] = (s0_addr[i] == s0_addr[j]); + // two lanes that need different passes + assign s0_conflict[i][j] = s0_mask[i] && s0_mask[j] && s0_same_bank[i][j] + && ~(s0_read && s0_same_word[i][j]); + end else begin : g_none + assign s0_same_bank[i][j] = 1'b0; + assign s0_same_word[i][j] = 1'b0; + assign s0_conflict[i][j] = 1'b0; + end + end + end + wire s0_one_pass = ~(| s0_conflict); + + // the first pass's bank winners: each bank's lowest lane + wire [NUM_LANES-1:0] s0_first; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_s0_first + assign s0_first[i] = s0_mask[i] && ~(| (s0_mask & s0_same_bank[i])); + end + + // S1 + + reg s1_valid; + // the bank winners of the next pass are in s1_winners + reg s1_ready; + // the next pass serves every pending lane + reg s1_last; + reg [NUM_LANES-1:0] s1_winners; + reg [BLOCK_WIDTH-1:0] s1_block; + reg s1_rw; + reg s1_read; + reg s1_amo; + reg [ATTR_WIDTH-1:0] s1_attr; + reg [NUM_LANES-1:0] s1_pending; + reg [NUM_LANES-1:0][ADDR_WIDTH-1:0] s1_addr; + reg [NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] s1_payload; + reg [TAG_WIDTH-1:0] s1_tag; + reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_bank; + reg [NUM_LANES-1:0][NUM_LANES-1:0] s1_same_word; + + // A pass serves the registered bank winners and the pending read lanes of + // their words. After a pass that leaves lanes pending, the next cycle finds + // the next winners among them and whether the pass after serves them all: + // a request replays at one pass per two cycles, and its last pass frees S1. + + wire [NUM_LANES-1:0] served; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_served + assign served[i] = s1_winners[i] || (s1_read && s1_pending[i] && (| (s1_winners & s1_same_word[i]))); + end + + wire [NUM_LANES-1:0] winners_n; + wire [NUM_LANES-1:0][NUM_LANES-1:0] pending_conflict; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_winners_n + assign winners_n[i] = s1_pending[i] && ~(| (s1_pending & s1_same_bank[i])); + for (genvar j = 0; j < NUM_LANES; ++j) begin : g_conflict + assign pending_conflict[i][j] = s1_pending[i] && s1_pending[j] && s1_same_bank[i][j] + && ~(s1_read && s1_same_word[i][j]); + end + end + wire last_n = ~(| pending_conflict); + + reg [CREDIT_WIDTH-1:0] credits; + wire has_rsp = ~s1_rw; + wire pass_fire = s1_valid && s1_ready && pass_en && (~has_rsp || (credits != '0)); + wire s1_release = pass_fire && s1_last; + assign s1_free = ~s1_valid || s1_release; + wire s1_load = s0_valid && s1_free; + + always @(posedge clk) begin + if (reset) begin + s1_valid <= 1'b0; + end else if (s1_load) begin + s1_valid <= 1'b1; + end else if (s1_release) begin + s1_valid <= 1'b0; + end + if (s1_load) begin + s1_ready <= 1'b1; + s1_last <= s0_one_pass; + s1_winners <= s0_first; + s1_pending <= s0_mask; + s1_block <= s0_block; + s1_rw <= s0_rw; + s1_read <= s0_read; + s1_amo <= s0_amo; + s1_attr <= s0_attr; + s1_addr <= s0_addr; + s1_payload <= s0_payload; + s1_tag <= s0_tag; + s1_same_bank <= s0_same_bank; + s1_same_word <= s0_same_word; + end else if (pass_fire) begin + s1_ready <= 1'b0; + s1_pending <= s1_pending & ~served; + end else if (~s1_ready) begin + s1_ready <= 1'b1; + s1_last <= last_n; + s1_winners <= winners_n; + end + end + + // stores and atomics held between S0's two entries and S1 + reg [1:0] num_writes; + wire write_in = req_valid && req_ready && ~req_read; + wire write_out = s1_release && ~s1_read; + always @(posedge clk) begin + if (reset) begin + num_writes <= '0; + end else begin + num_writes <= num_writes + 2'(write_in) - 2'(write_out); + end + end + + assign busy = s0_valid || s1_valid; + assign holds_write = (num_writes != '0); + assign pass_amo = pass_fire && s1_amo; + + // bank registers: each bank takes its first lane's row and payload + + wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] s1_bank; + wire [NUM_LANES-1:0][BANK_ADDR_WIDTH+PAYLOAD_WIDTH-1:0] s1_bank_data; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lane_addr + if (BANK_SEL_BITS != 0) begin : g_banks + assign s1_bank[i] = s1_addr[i][BANK_SEL_BITS-1:0]; + end else begin : g_one_bank + assign s1_bank[i] = '0; + end + assign s1_bank_data[i] = {s1_addr[i][BANK_SEL_BITS +: BANK_ADDR_WIDTH], s1_payload[i]}; + end + + reg [NUM_BANKS-1:0] bank_valid_r; + reg bank_rw_r; + reg bank_amo_r; + reg [ATTR_WIDTH-1:0] bank_attr_r; + reg [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] bank_addr_r; + reg [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] bank_payload_r; + + for (genvar b = 0; b < NUM_BANKS; ++b) begin : g_banks + wire [NUM_LANES-1:0] sel; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_sel + assign sel[i] = s1_winners[i] && (s1_bank[i] == BANK_SEL_WIDTH'(b)); + end + + wire [BANK_ADDR_WIDTH-1:0] sel_addr; + wire [PAYLOAD_WIDTH-1:0] sel_payload; + + VX_onehot_mux #( + .DATAW (BANK_ADDR_WIDTH + PAYLOAD_WIDTH), + .N (NUM_LANES) + ) bank_mux ( + .data_in (s1_bank_data), + .sel_in (sel), + .data_out ({sel_addr, sel_payload}) + ); + + always @(posedge clk) begin + if (reset) begin + bank_valid_r[b] <= 1'b0; + end else begin + bank_valid_r[b] <= pass_fire && (| sel); + end + bank_addr_r[b] <= sel_addr; + bank_payload_r[b] <= sel_payload; + end + end + + always @(posedge clk) begin + bank_rw_r <= s1_rw; + bank_amo_r <= s1_amo; + bank_attr_r <= s1_attr; + end + + assign bank_valid = bank_valid_r; + assign bank_rw = bank_rw_r; + assign bank_amo = bank_amo_r; + assign bank_attr = bank_attr_r; + assign bank_addr = bank_addr_r; + assign bank_payload = bank_payload_r; + + // response: the pass's lanes and banks follow it to the SRAM output + + wire m2_valid; + wire [BLOCK_WIDTH-1:0] m2_block; + wire [NUM_LANES-1:0] m2_mask; + wire [NUM_LANES-1:0][BANK_SEL_WIDTH-1:0] m2_bank; + wire [TAG_WIDTH-1:0] m2_tag; + + VX_pipe_register #( + .DATAW (1 + BLOCK_WIDTH + NUM_LANES + NUM_LANES * BANK_SEL_WIDTH + TAG_WIDTH), + .RESETW (1), + .DEPTH (2) + ) rsp_meta_pipe ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({pass_fire && has_rsp, s1_block, served, s1_bank, s1_tag}), + .data_out ({m2_valid, m2_block, m2_mask, m2_bank, m2_tag}) + ); + + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m2_data; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_rsp_data + assign m2_data[i] = bank_rdata[m2_bank[i]]; + end + + // The lane mux reads the SRAM outputs; its result is registered before it + // travels to the response queue. + wire m3_valid; + wire [BLOCK_WIDTH-1:0] m3_block; + wire [NUM_LANES-1:0] m3_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] m3_data; + wire [TAG_WIDTH-1:0] m3_tag; + + VX_pipe_register #( + .DATAW (RSP_DATAW + 1), + .RESETW (1) + ) rsp_data_reg ( + .clk (clk), + .reset (reset), + .enable (1'b1), + .data_in ({m2_valid, m2_block, m2_mask, m2_data, m2_tag}), + .data_out ({m3_valid, m3_block, m3_mask, m3_data, m3_tag}) + ); + + // The queue has a slot for every pass that responds and is in flight + // toward its block, so it never fills. A response that finds the queue + // empty passes straight on to its block's register. + wire rsp_empty; + wire [RSP_DATAW-1:0] rsp_head; + wire rsp_push = m3_valid && ~(rsp_empty && rsp_take); + wire rsp_pop = ~rsp_empty && rsp_take; + + VX_fifo_queue #( + .DATAW (RSP_DATAW), + .DEPTH (RSP_QUEUE_SIZE), + .OUT_REG (0), + .LUTRAM (1) + ) rsp_queue ( + .clk (clk), + .reset (reset), + .push (rsp_push), + .pop (rsp_pop), + .data_in ({m3_block, m3_mask, m3_data, m3_tag}), + .data_out (rsp_head), + .empty (rsp_empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + `UNUSED_PIN (size) + ); + assign rsp_valid = ~rsp_empty || m3_valid; + assign {rsp_block, rsp_mask, rsp_data, rsp_tag} = rsp_empty ? {m3_block, m3_mask, m3_data, m3_tag} : rsp_head; + + // a response's slot is free again once its block has taken it on + wire credit_take = pass_fire && has_rsp; + always @(posedge clk) begin + if (reset) begin + credits <= CREDIT_WIDTH'(RSP_QUEUE_SIZE); + end else begin + credits <= credits - CREDIT_WIDTH'(credit_take) + CREDIT_WIDTH'(rsp_done); + end + end + + assign perf_served = pass_fire ? served : '0; + assign perf_store = s1_rw; + assign perf_replay = pass_fire && ~s1_last; + +`ifdef DBG_TRACE_MEM + always @(posedge clk) begin + if (pass_fire) begin + `TRACE(2, ("%t: %s pass: rw=%b, amo=%b, pending=%b, served=%b, tag=0x%0h\n", + $time, INSTANCE_ID, s1_rw, s1_amo, s1_pending, served, s1_tag)) + end + end +`endif + +endmodule diff --git a/hw/rtl/mem/VX_lmem_unit.sv b/hw/rtl/mem/VX_lmem_unit.sv new file mode 100644 index 0000000000..a48400fb16 --- /dev/null +++ b/hw/rtl/mem/VX_lmem_unit.sv @@ -0,0 +1,961 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Warp-wide local memory. It takes whole warp requests from the LSU blocks and +// serves each in bank-conflict-free passes (VX_lmem_engine): same-word reads +// are broadcast to every lane reading the word, conflicting lanes replay, and +// requests retire in order, so every lane's accesses reach the banks in program +// order. +// +// With several LSU blocks a second engine, for plain reads only, works the +// banks' second port, so two warps' reads go through each cycle. Stores and +// atomics use the first engine alone: one enters it only once the read engine +// has drained, and the read engine takes no read while a store or atomic sits in +// the first engine, while one waited at an input the cycle before, or in a cycle +// the first engine may be taking one. A write therefore never meets a read of +// the other port in a bank, and keeps its order against every read. +// +// The DMA port (DXA writes, TCU reads) takes whole bank rows. A DMA request seen +// in one cycle gets the banks the next: no pass is issued in a cycle a DMA +// request is seen, and it is accepted in the cycle that follows. An atomic owns +// its bank for three cycles (read, capture, write back), so no pass or DMA access +// is issued in the two cycles after an atomic pass. +module VX_lmem_unit import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + + // Size in bytes + parameter SIZE = (1024*16), + // Number of LSU blocks + parameter NUM_BLOCKS = 1, + // Lanes of a warp request + parameter NUM_LANES = 4, + // Number of banks + parameter NUM_BANKS = 4, + // Size of a word in bytes + parameter WORD_SIZE = `VX_CFG_XLEN/8, + // Word address width + parameter ADDR_WIDTH = `CLOG2(SIZE / WORD_SIZE), + // Request tag size + parameter TAG_WIDTH = 16, + + // Enable DMA port + parameter DMA_ENABLE = 0, + parameter DMA_TAG_WIDTH = 1, + + // Synthesize atomic-op + LR/SC logic in the banks + parameter AMO_ENABLE = 0 +) ( + input wire clk, + input wire reset, + +`ifdef PERF_ENABLE + output lmem_perf_t lmem_perf, +`endif + + // LSU blocks' warp requests + VX_lsu_mem_if.slave lsu_mem_if [NUM_BLOCKS], + + // DMA read/write port + VX_mem_bus_if.slave dma_bus_if +); + `UNUSED_SPARAM (INSTANCE_ID) + + localparam WORD_WIDTH = WORD_SIZE * 8; + localparam NUM_WORDS = SIZE / WORD_SIZE; + localparam WORDS_PER_BANK = NUM_WORDS / NUM_BANKS; + localparam BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK); + localparam BLOCK_BITS = `CLOG2(NUM_BLOCKS); + localparam BLOCK_WIDTH = `UP(BLOCK_BITS); + localparam DUAL = (NUM_BLOCKS > 1); + // A pass's response reaches its block's response register four cycles + // after the pass at the earliest, and its slot is free for a pass the cycle + // after the block takes it: eight slots keep one pass per cycle going. + localparam RSP_QUEUE_SIZE = 8; + localparam DONE_WIDTH = `CLOG2(NUM_BLOCKS + 1); + localparam AMO_OP_WIDTH = $bits(amo_op_e); + // Compare-and-swap adds a comparand: the request's data word already + // carries the swap value, so the third operand needs its own room. +`ifdef VX_CFG_EXT_ZACAS_ENABLE + localparam AMO_CMP_WIDTH = `VX_CFG_XLEN; +`else + localparam AMO_CMP_WIDTH = 0; +`endif + // per lane: the hart id that holds a reservation, and the comparand + localparam LANE_SB_WIDTH = AMO_ENABLE ? (HART_ID_WIDTH + AMO_CMP_WIDTH) : 0; + localparam PAYLOAD_WIDTH = WORD_WIDTH + WORD_SIZE + LANE_SB_WIDTH; + // common to all lanes: the atomic op and its signedness + localparam ATTR_WIDTH = AMO_OP_WIDTH + 1; + + `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) + `STATIC_ASSERT(`IS_POW2(NUM_BANKS), ("invalid parameter")) + + // the blocks' requests + + wire [NUM_BLOCKS-1:0] in_valid; + wire [NUM_BLOCKS-1:0] in_rw; + wire [NUM_BLOCKS-1:0] in_amo; + wire [NUM_BLOCKS-1:0] in_read; + wire [NUM_BLOCKS-1:0][ATTR_WIDTH-1:0] in_attr; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] in_mask; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] in_addr; + wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][PAYLOAD_WIDTH-1:0] in_payload; + wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] in_tag; + wire [NUM_BLOCKS-1:0] in_ready; + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_in + // the atomic op is the same on every lane of a request + mem_bus_attr_t attr0; + assign attr0 = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[0]); + assign in_valid[k] = lsu_mem_if[k].req_valid; + assign in_rw[k] = lsu_mem_if[k].req_data.rw; + assign in_amo[k] = AMO_ENABLE ? attr0.amo.amo_valid : 1'b0; + assign in_read[k] = ~in_rw[k] && ~in_amo[k]; + assign in_attr[k] = AMO_ENABLE ? {attr0.amo.amo_op, attr0.amo.amo_unsigned} : '0; + assign in_mask[k] = lsu_mem_if[k].req_data.mask; + assign in_tag[k] = lsu_mem_if[k].req_data.tag; + `UNUSED_VAR (attr0) + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_lanes + assign in_addr[k][i] = lsu_mem_if[k].req_data.addr[i][ADDR_WIDTH-1:0]; + if (AMO_ENABLE != 0) begin : g_amo + mem_bus_attr_t lane_attr; + assign lane_attr = mem_bus_attr_t'(lsu_mem_if[k].req_data.user[i]); + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign in_payload[k][i] = {lane_attr.amo.hart_id, lane_attr.amo.amo_cmp, + lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `else + assign in_payload[k][i] = {lane_attr.amo.hart_id, + lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `endif + `UNUSED_VAR (lane_attr) + end else begin : g_no_amo + assign in_payload[k][i] = {lsu_mem_if[k].req_data.byteen[i], lsu_mem_if[k].req_data.data[i]}; + `UNUSED_VAR (lsu_mem_if[k].req_data.user[i]) + end + `UNUSED_VAR (lsu_mem_if[k].req_data.addr[i]) + end + assign lsu_mem_if[k].req_ready = in_ready[k]; + end + + // the engines + + wire pass_en; + wire a_pass_amo; + + wire a_req_valid; + wire a_req_ready; + wire a_busy; + wire a_holds_write; + wire [BLOCK_WIDTH-1:0] a_req_block; + wire b_req_valid; + wire b_req_ready; + wire [BLOCK_WIDTH-1:0] b_req_block; + + wire [NUM_BANKS-1:0] a_bank_valid; + wire a_bank_rw; + wire a_bank_amo; + wire [ATTR_WIDTH-1:0] a_bank_attr; + wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] a_bank_addr; + wire [NUM_BANKS-1:0][PAYLOAD_WIDTH-1:0] a_bank_payload; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] a_bank_rdata; + + wire a_rsp_valid; + wire a_rsp_take; + wire [DONE_WIDTH-1:0] a_rsp_done; + wire [BLOCK_WIDTH-1:0] a_rsp_block; + wire [NUM_LANES-1:0] a_rsp_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] a_rsp_data; + wire [TAG_WIDTH-1:0] a_rsp_tag; + + wire [NUM_LANES-1:0] a_perf_served; + wire a_perf_store; + wire a_perf_replay; + + wire b_busy; + wire [NUM_BANKS-1:0] b_bank_valid; + wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] b_bank_addr; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] b_bank_rdata; + + wire b_rsp_valid; + wire b_rsp_take; + wire [DONE_WIDTH-1:0] b_rsp_done; + wire [BLOCK_WIDTH-1:0] b_rsp_block; + wire [NUM_LANES-1:0] b_rsp_mask; + wire [NUM_LANES-1:0][WORD_WIDTH-1:0] b_rsp_data; + wire [TAG_WIDTH-1:0] b_rsp_tag; + + wire [NUM_LANES-1:0] b_perf_served; + wire b_perf_replay; + + // Block arbitration. The ready each block sees comes from registers only: + // each engine's block is chosen a cycle ahead, from the requests waiting + // then, and the engine takes it if it is still there. A block that was + // just served may present a new request of either kind, so a choice of it + // counts as a store: the first engine takes a store only while the read + // engine is idle, and the read engine takes nothing in that cycle. + + wire [NUM_BLOCKS-1:0] in_fire = in_valid & in_ready; + + if (DUAL) begin : g_dual_arb + + reg a_sel_valid_r; + reg [BLOCK_WIDTH-1:0] a_sel_r; + reg a_sel_read_r; + reg b_sel_valid_r; + reg [BLOCK_WIDTH-1:0] b_sel_r; + reg write_waiting_r; + + wire a_take = a_sel_valid_r && a_req_ready && (a_sel_read_r || ~b_busy); + wire b_take = b_sel_valid_r && b_req_ready && ~a_holds_write && ~write_waiting_r + && ~(a_take && ~a_sel_read_r); + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_ready + assign in_ready[k] = (a_take && (a_sel_r == BLOCK_WIDTH'(k))) + || (b_take && (b_sel_r == BLOCK_WIDTH'(k))); + end + + assign a_req_valid = a_take && in_valid[a_sel_r]; + assign a_req_block = a_sel_r; + assign b_req_valid = b_take && in_valid[b_sel_r]; + assign b_req_block = b_sel_r; + + // the next cycle's choices, from the requests waiting now + wire [NUM_BLOCKS-1:0] known_read = in_valid & in_read & ~in_fire; + wire b_busy_n = b_busy || (b_req_valid && b_req_ready); + wire [NUM_BLOCKS-1:0] a_eligible = known_read | (in_valid & {NUM_BLOCKS{~b_busy_n}}); + + wire a_next_valid; + wire [BLOCK_WIDTH-1:0] a_next; + wire [NUM_BLOCKS-1:0] a_next_onehot; + + VX_generic_arbiter #( + .NUM_REQS (NUM_BLOCKS), + .TYPE ("R") + ) a_arbiter ( + .clk (clk), + .reset (reset), + .requests (a_eligible), + .grant_index (a_next), + .grant_onehot (a_next_onehot), + .grant_valid (a_next_valid), + .grant_ready (1'b1) + ); + + wire [NUM_BLOCKS-1:0] b_eligible = known_read & ~a_next_onehot; + + wire b_next_valid; + wire [BLOCK_WIDTH-1:0] b_next; + wire [NUM_BLOCKS-1:0] b_next_onehot; + + VX_generic_arbiter #( + .NUM_REQS (NUM_BLOCKS), + .TYPE ("R") + ) b_arbiter ( + .clk (clk), + .reset (reset), + .requests (b_eligible), + .grant_index (b_next), + .grant_onehot (b_next_onehot), + .grant_valid (b_next_valid), + .grant_ready (1'b1) + ); + `UNUSED_VAR (b_next_onehot) + + always @(posedge clk) begin + if (reset) begin + a_sel_valid_r <= 1'b0; + b_sel_valid_r <= 1'b0; + write_waiting_r <= 1'b0; + end else begin + a_sel_valid_r <= a_next_valid; + b_sel_valid_r <= b_next_valid; + write_waiting_r <= (| (in_valid & ~in_read & ~in_fire)); + end + a_sel_r <= a_next; + a_sel_read_r <= known_read[a_next]; + b_sel_r <= b_next; + end + + end else begin : g_single_arb + + // one block: its request goes straight to the first engine + assign a_req_valid = in_valid[0]; + assign a_req_block = '0; + assign in_ready[0] = a_req_ready; + assign b_req_valid = 1'b0; + assign b_req_block = '0; + `UNUSED_VAR (in_fire) + `UNUSED_VAR (b_req_ready) + `UNUSED_VAR (b_busy) + + end + + VX_lmem_engine #( + .INSTANCE_ID (`SFORMATF(("%s-engine0", INSTANCE_ID))), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .PAYLOAD_WIDTH (PAYLOAD_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH), + .NUM_BLOCKS (NUM_BLOCKS), + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + ) engine_a ( + .clk (clk), + .reset (reset), + .req_valid (a_req_valid), + .req_block (a_req_block), + .req_rw (in_rw[a_req_block]), + .req_read (in_read[a_req_block]), + .req_amo (in_amo[a_req_block]), + .req_attr (in_attr[a_req_block]), + .req_mask (in_mask[a_req_block]), + .req_addr (in_addr[a_req_block]), + .req_payload (in_payload[a_req_block]), + .req_tag (in_tag[a_req_block]), + .req_ready (a_req_ready), + .busy (a_busy), + .holds_write (a_holds_write), + .pass_en (pass_en), + .pass_amo (a_pass_amo), + .bank_valid (a_bank_valid), + .bank_rw (a_bank_rw), + .bank_amo (a_bank_amo), + .bank_attr (a_bank_attr), + .bank_addr (a_bank_addr), + .bank_payload (a_bank_payload), + .bank_rdata (a_bank_rdata), + .rsp_valid (a_rsp_valid), + .rsp_block (a_rsp_block), + .rsp_mask (a_rsp_mask), + .rsp_data (a_rsp_data), + .rsp_tag (a_rsp_tag), + .rsp_take (a_rsp_take), + .rsp_done (a_rsp_done), + .perf_served (a_perf_served), + .perf_store (a_perf_store), + .perf_replay (a_perf_replay) + ); + `UNUSED_VAR (a_busy) + + if (DUAL) begin : g_read_engine + + wire [NUM_LANES-1:0][0:0] b_req_payload = '0; + wire b_bank_rw; + wire b_bank_amo; + wire b_holds_write; + wire b_pass_amo; + wire b_perf_store; + wire [ATTR_WIDTH-1:0] b_bank_attr; + wire [NUM_BANKS-1:0][0:0] b_bank_payload; + + VX_lmem_engine #( + .INSTANCE_ID (`SFORMATF(("%s-engine1", INSTANCE_ID))), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .PAYLOAD_WIDTH (1), + .ATTR_WIDTH (ATTR_WIDTH), + .NUM_BLOCKS (NUM_BLOCKS), + .RSP_QUEUE_SIZE (RSP_QUEUE_SIZE) + ) engine_b ( + .clk (clk), + .reset (reset), + .req_valid (b_req_valid), + .req_block (b_req_block), + .req_rw (1'b0), + .req_read (1'b1), + .req_amo (1'b0), + .req_attr ('0), + .req_mask (in_mask[b_req_block]), + .req_addr (in_addr[b_req_block]), + .req_payload (b_req_payload), + .req_tag (in_tag[b_req_block]), + .req_ready (b_req_ready), + .busy (b_busy), + .holds_write (b_holds_write), + .pass_en (pass_en), + .pass_amo (b_pass_amo), + .bank_valid (b_bank_valid), + .bank_rw (b_bank_rw), + .bank_amo (b_bank_amo), + .bank_attr (b_bank_attr), + .bank_addr (b_bank_addr), + .bank_payload (b_bank_payload), + .bank_rdata (b_bank_rdata), + .rsp_valid (b_rsp_valid), + .rsp_block (b_rsp_block), + .rsp_mask (b_rsp_mask), + .rsp_data (b_rsp_data), + .rsp_tag (b_rsp_tag), + .rsp_take (b_rsp_take), + .rsp_done (b_rsp_done), + .perf_served (b_perf_served), + .perf_store (b_perf_store), + .perf_replay (b_perf_replay) + ); + `UNUSED_VAR ({b_bank_rw, b_bank_amo, b_holds_write, b_pass_amo, b_bank_attr, b_bank_payload, b_perf_store}) + + end else begin : g_no_read_engine + + assign b_busy = 1'b0; + assign b_bank_valid = '0; + assign b_bank_addr = '0; + assign b_rsp_valid = 1'b0; + assign b_rsp_block = '0; + assign b_rsp_mask = '0; + assign b_rsp_data = '0; + assign b_rsp_tag = '0; + assign b_perf_served = '0; + assign b_perf_replay = 1'b0; + `UNUSED_VAR (b_bank_rdata) + `UNUSED_VAR (b_rsp_take) + `UNUSED_VAR (b_rsp_done) + `UNUSED_VAR (a_holds_write) + `UNUSED_VAR (b_req_valid) + `UNUSED_VAR (b_req_block) + assign b_req_ready = 1'b0; + + end + + // Responses. Each block has a response register, which takes the first + // engine's queue head ahead of the read engine's when it is empty or is + // being emptied. A response's queue slot is free again once its block has + // taken it on. + + wire [NUM_BLOCKS-1:0] a_load; + wire [NUM_BLOCKS-1:0] b_load; + wire [NUM_BLOCKS-1:0] a_done; + wire [NUM_BLOCKS-1:0] b_done; + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_rsp + reg rsp_valid_r; + reg rsp_from_b_r; + reg [NUM_LANES-1:0] rsp_mask_r; + reg [NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data_r; + reg [TAG_WIDTH-1:0] rsp_tag_r; + + wire rsp_fire = rsp_valid_r && lsu_mem_if[k].rsp_ready; + wire rsp_free = ~rsp_valid_r || lsu_mem_if[k].rsp_ready; + wire a_hit = a_rsp_valid && (a_rsp_block == BLOCK_WIDTH'(k)); + wire b_hit = b_rsp_valid && (b_rsp_block == BLOCK_WIDTH'(k)); + + assign a_load[k] = rsp_free && a_hit; + assign b_load[k] = rsp_free && ~a_hit && b_hit; + assign a_done[k] = rsp_fire && ~rsp_from_b_r; + assign b_done[k] = rsp_fire && rsp_from_b_r; + + always @(posedge clk) begin + if (reset) begin + rsp_valid_r <= 1'b0; + end else if (rsp_free) begin + rsp_valid_r <= a_hit || b_hit; + end + if (rsp_free) begin + rsp_from_b_r <= ~a_hit; + rsp_mask_r <= a_hit ? a_rsp_mask : b_rsp_mask; + rsp_data_r <= a_hit ? a_rsp_data : b_rsp_data; + rsp_tag_r <= a_hit ? a_rsp_tag : b_rsp_tag; + end + end + + assign lsu_mem_if[k].rsp_valid = rsp_valid_r; + assign lsu_mem_if[k].rsp_data.mask = rsp_mask_r; + assign lsu_mem_if[k].rsp_data.data = rsp_data_r; + assign lsu_mem_if[k].rsp_data.tag = rsp_tag_r; + end + + `POP_COUNT(a_rsp_done, a_done); + `POP_COUNT(b_rsp_done, b_done); + assign a_rsp_take = (| a_load); + assign b_rsp_take = (| b_load); + + // DMA and atomic slots + + reg [1:0] amo_stall_r; + wire amo_stall = (amo_stall_r != 2'd0); + always @(posedge clk) begin + if (reset) begin + amo_stall_r <= 2'd0; + end else if (a_pass_amo) begin + amo_stall_r <= 2'd2; + end else if (amo_stall) begin + amo_stall_r <= amo_stall_r - 2'd1; + end + end + + wire dma_valid = DMA_ENABLE && dma_bus_if.req_valid; + reg dma_slot_r; + always @(posedge clk) begin + if (reset) begin + dma_slot_r <= 1'b0; + end else begin + dma_slot_r <= dma_valid && ~amo_stall; + end + end + assign pass_en = ~dma_valid && ~amo_stall; + + // DMA port handshake + // rw=0 reads : accepted when the response pipe-buffer has space. + // rw=1 writes : no response issued. + + wire dma_rsp_buf_ready; + wire dma_fire = dma_slot_r && dma_valid && (dma_bus_if.req_data.rw || dma_rsp_buf_ready); + wire dma_rd_fire = dma_fire && ~dma_bus_if.req_data.rw; + wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_rdata_a; + + if (DMA_ENABLE) begin : g_dma_enable + `UNUSED_VAR (dma_bus_if.req_data.attr) + + assign dma_bus_if.req_ready = dma_slot_r && (dma_bus_if.req_data.rw || dma_rsp_buf_ready); + + // the tag follows the SRAM read to its output register + VX_pipe_buffer #( + .DATAW (DMA_TAG_WIDTH) + ) dma_rsp_buf ( + .clk (clk), + .reset (reset), + .valid_in (dma_rd_fire), + .ready_in (dma_rsp_buf_ready), + .data_in (dma_bus_if.req_data.tag), + .valid_out (dma_bus_if.rsp_valid), + .data_out (dma_bus_if.rsp_data.tag), + .ready_out (dma_bus_if.rsp_ready) + ); + + // The response buffer holds only the tag; the data is the live SRAM + // output, which the next pass's reads overwrite. While a response is + // back-pressured, serve a copy of the row latched on its first cycle. + reg [NUM_BANKS-1:0][WORD_WIDTH-1:0] dma_rsp_hold_data_r; + reg dma_rsp_hold_valid_r; + wire dma_rsp_consumed = dma_bus_if.rsp_valid && dma_bus_if.rsp_ready; + always @(posedge clk) begin + if (reset) begin + dma_rsp_hold_valid_r <= 1'b0; + end else if (dma_rsp_consumed) begin + dma_rsp_hold_valid_r <= 1'b0; + end else if (dma_bus_if.rsp_valid && ~dma_rsp_hold_valid_r) begin + dma_rsp_hold_data_r <= per_bank_rdata_a; + dma_rsp_hold_valid_r <= 1'b1; + end + end + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_dma_rsp_data + assign dma_bus_if.rsp_data.data[i*WORD_WIDTH +: WORD_WIDTH] = + dma_rsp_hold_valid_r ? dma_rsp_hold_data_r[i] : per_bank_rdata_a[i]; + end + + end else begin : g_no_dma + assign dma_rsp_buf_ready = 1'b0; + assign dma_bus_if.req_ready = 1'b0; + assign dma_bus_if.rsp_valid = 1'b0; + assign dma_bus_if.rsp_data = '0; + `UNUSED_VAR (dma_bus_if.req_valid) + `UNUSED_VAR (dma_bus_if.req_data) + `UNUSED_VAR (dma_bus_if.rsp_ready) + end + + // banks + + for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_data_store + + wire dma_wr_b = dma_fire && dma_bus_if.req_data.rw + && (| dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE]); + + wire lsu_active = a_bank_valid[i]; + + wire [WORD_WIDTH-1:0] req_data; + wire [WORD_SIZE-1:0] req_byteen; + wire [HART_ID_WIDTH-1:0] req_hart_id; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + wire [AMO_CMP_WIDTH-1:0] req_amo_cmp; + `endif + if (AMO_ENABLE != 0) begin : g_payload_amo + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign {req_hart_id, req_amo_cmp, req_byteen, req_data} = a_bank_payload[i]; + `else + assign {req_hart_id, req_byteen, req_data} = a_bank_payload[i]; + `endif + end else begin : g_payload + assign {req_byteen, req_data} = a_bank_payload[i]; + assign req_hart_id = '0; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + assign req_amo_cmp = '0; + `endif + end + + // Bank-facing AMO interface, tied off when the A extension is absent so + // none of the atomic hardware is synthesized. + wire amo_wb_store; // owed atomic write-back + wire [BANK_ADDR_WIDTH-1:0] amo_wb_addr; + wire [WORD_SIZE-1:0] amo_wb_byteen; + wire [WORD_WIDTH-1:0] amo_wb_data; + wire sc_resolving; // SC outcome replaces the response + wire sc_fail; + wire sc_lane_hi; // outcome lane of a resolving .W SC + + if (AMO_ENABLE != 0) begin : g_amo + + // Atomics. An atomic carries rw=0, so the read returning the old + // value is the ordinary read path. What the bank owes is the write + // of the computed value. + // + // The old word is registered before the ALU sees it. Feeding the ALU + // straight from the SRAM output puts the BRAM clock-to-out, the adder + // and the BRAM setup in a single period; the register splits that + // into BRAM-out-to-flop and flop-through-adder-to-BRAM-in. + // + // An atomic owns its bank for three cycles: read, capture, write back. + reg amo_rd_valid_r; + reg amo_wb_valid_r; + reg [WORD_WIDTH-1:0] amo_old_r; + reg [BANK_ADDR_WIDTH-1:0] amo_wb_addr_r; + reg [WORD_SIZE-1:0] amo_wb_byteen_r; + reg [AMO_OP_WIDTH-1:0] amo_wb_op_r; + reg amo_wb_unsigned_r; + reg [WORD_WIDTH-1:0] amo_wb_rhs_r; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + reg [AMO_CMP_WIDTH-1:0] amo_wb_cmp_r; + `endif + + wire [AMO_OP_WIDTH-1:0] req_amo_op = a_bank_attr[1 +: AMO_OP_WIDTH]; + wire req_amo_unsigned = a_bank_attr[0]; + + wire amo_accept = lsu_active && a_bank_amo; + + // Reservation commit stage. Every accepted request that can touch a + // reservation -- an atomic, or a plain store -- reaches it one cycle + // after acceptance, so a single address port serves both. The + // look-ahead read is driven from the acceptance cycle, which is what + // makes the registered reservation entry land here. + wire is_lr_in = a_bank_amo && (amo_op_e'(req_amo_op) == AMO_OP_LR); + wire is_sc_in = a_bank_amo && (amo_op_e'(req_amo_op) == AMO_OP_SC); + + reg res_valid_r; + reg res_is_lr_r; + reg res_is_sc_r; + reg res_is_amo_r; + reg res_is_store_r; + reg [BANK_ADDR_WIDTH-1:0] res_addr_r; + reg [HART_ID_WIDTH-1:0] res_hart_id_r; + + always @(posedge clk) begin + if (reset) begin + res_valid_r <= 1'b0; + end else begin + res_valid_r <= lsu_active; + end + if (lsu_active) begin + res_is_lr_r <= is_lr_in; + res_is_sc_r <= is_sc_in; + res_is_amo_r <= a_bank_amo; + res_is_store_r <= a_bank_rw; + res_addr_r <= a_bank_addr[i]; + res_hart_id_r <= req_hart_id; + end + end + + // A store-conditional fails when its reservation is gone. The + // outcome is known at the commit stage, one cycle before the + // write-back, so it can both suppress the write and be returned as + // this request's response. + wire res_check; + assign sc_fail = res_is_sc_r && ~res_check; + assign sc_resolving = res_valid_r && res_is_sc_r; + + // The SC outcome must land in the addressed lane: the LSU extracts + // the subword by address, so a .W outcome left at bit 0 would read + // as 0 ("success") for every upper-lane SC, including a failed one + // whose store was suppressed. + if (WORD_WIDTH == 64) begin : g_sc_lane + reg res_lane_hi_r; + always @(posedge clk) begin + if (lsu_active) begin + res_lane_hi_r <= ~req_byteen[0]; + end + end + assign sc_lane_hi = res_lane_hi_r; + end else begin : g_sc_lane0 + assign sc_lane_hi = 1'b0; + end + + // Everything that ends up writing the word: a plain store, a + // read-modify-write atomic (which carries rw=0 and so is not a store + // by the request flag), and a store-conditional that found its + // reservation. + wire res_commits_store = res_is_store_r + || (res_is_amo_r && ~res_is_lr_r && ~res_is_sc_r) + || (res_is_sc_r && res_check); + + reg amo_wb_is_lr_r; + reg amo_wb_sc_fail_r; + always @(posedge clk) begin + if (amo_rd_valid_r) begin + amo_wb_is_lr_r <= res_is_lr_r; + amo_wb_sc_fail_r <= sc_fail; + end + end + + // A load-reserved leaves the word alone, and a failed + // store-conditional must not store; every other atomic writes its + // computed value back. + assign amo_wb_store = amo_wb_valid_r && ~amo_wb_is_lr_r && ~amo_wb_sc_fail_r; + + always @(posedge clk) begin + if (reset) begin + amo_rd_valid_r <= 1'b0; + amo_wb_valid_r <= 1'b0; + end else begin + amo_rd_valid_r <= amo_accept; + amo_wb_valid_r <= amo_rd_valid_r; + end + if (amo_rd_valid_r) begin + amo_old_r <= per_bank_rdata_a[i]; + end + if (amo_accept) begin + amo_wb_addr_r <= a_bank_addr[i]; + amo_wb_byteen_r <= req_byteen; + amo_wb_op_r <= req_amo_op; + amo_wb_unsigned_r <= req_amo_unsigned; + amo_wb_rhs_r <= req_data; + `ifdef VX_CFG_EXT_ZACAS_ENABLE + amo_wb_cmp_r <= req_amo_cmp; + `endif + end + end + + wire [63:0] amo_new_word; + wire [63:0] amo_ret_word; + + // Lane-align the RMW operands. The bank word is XLEN wide, so on a + // 64-bit hart a .W atomic addresses one of two 32-bit lanes; the + // ALU computes right-aligned. Store data arrives lane-positioned + // (same packing as a plain store), so old and rhs shift down by the + // lane offset and the result shifts back up under the request + // byteen. The comparand arrives right-aligned from rs3 and needs no + // shift. The addressed lane is recovered from the byteen: an atomic + // is naturally aligned, so a .W to the upper lane never covers byte + // 0 and a .D covers the full word. + wire [63:0] amo_old_lane; + wire [63:0] amo_rhs_lane; + wire [1:0] amo_width; + if (WORD_WIDTH == 64) begin : g_amo_lane + wire lane_hi = ~amo_wb_byteen_r[0]; + assign amo_width = (&amo_wb_byteen_r) ? 2'd3 : 2'd2; + assign amo_old_lane = lane_hi ? {32'h0, amo_old_r[63:32]} : amo_old_r; + assign amo_rhs_lane = lane_hi ? {32'h0, amo_wb_rhs_r[63:32]} : amo_wb_rhs_r; + assign amo_wb_data = lane_hi ? {amo_new_word[31:0], 32'h0} : amo_new_word; + end else begin : g_amo_word + assign amo_width = 2'd2; + assign amo_old_lane = 64'(amo_old_r); + assign amo_rhs_lane = 64'(amo_wb_rhs_r); + assign amo_wb_data = amo_new_word[WORD_WIDTH-1:0]; + end + // VX_CFG_AMO_RS_SIZE sizes ONE shared LLC bank's contention; deployed + // per lmem bank it multiplies by NUM_LMEM_BANKS x NUM_CORES. After + // the holder-credit protocol a hot word maps to exactly one station, + // so depth only reduces false eviction between distinct reserved + // words aliasing in one bank -- and a spurious SC failure is + // architecturally legal. Clamp the depth. + // A credit is spent per refused LR, not per cycle: a warp's LR + // reaches a bank one lane per pass, so between a holder's LR and its + // SC every other hart of the core can be refused, and a refused hart + // can retry once more. The budget covers two refusals per hart; a + // smaller one lets one warp's own LR burst displace its holder and a + // contended retry loop stops making progress. + VX_amo_unit #( + .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), + .HOLD_CREDIT_BITS (`CLOG2(`VX_CFG_NUM_WARPS * `VX_CFG_NUM_THREADS) + 1), + .LINE_ADDR_BITS (BANK_ADDR_WIDTH), + .DATA_WIDTH (WORD_WIDTH) + ) amo_unit ( + .clk (clk), + .reset (reset), + .pipe_stall (1'b0), + .compute_op (amo_op_e'(amo_wb_op_r)), + .compute_unsigned (amo_wb_unsigned_r), + .compute_width (amo_width), + .compute_old (amo_old_lane), + .compute_rhs (amo_rhs_lane), + `ifdef VX_CFG_EXT_ZACAS_ENABLE + .compute_cmp (64'(amo_wb_cmp_r)), + `else + .compute_cmp (64'b0), + `endif + .compute_new_word (amo_new_word), + .compute_ret_word (amo_ret_word), + // A load-reserved claims the word; a store-conditional gives up + // its own reservation whether it succeeds or fails; any committing + // store breaks whoever else holds one. + .res_reserve (res_valid_r && res_is_lr_r), + .res_clear (res_valid_r && res_is_sc_r), + .res_invalidate (res_valid_r && res_commits_store), + .res_hart_id (res_hart_id_r), + .res_line_addr (res_addr_r), + .res_line_addr_n (a_bank_addr[i]), + .res_check (res_check) + ); + `UNUSED_VAR (amo_ret_word) + if (WORD_WIDTH < 64) begin : g_amo_hi_unused + `UNUSED_VAR (amo_new_word[63:WORD_WIDTH]) + end + + assign amo_wb_addr = amo_wb_addr_r; + assign amo_wb_byteen = amo_wb_byteen_r; + + end else begin : g_no_amo + assign amo_wb_store = 1'b0; + assign amo_wb_addr = '0; + assign amo_wb_byteen = '0; + assign amo_wb_data = '0; + assign sc_resolving = 1'b0; + assign sc_fail = 1'b0; + assign sc_lane_hi = 1'b0; + `UNUSED_VAR (req_hart_id) + `ifdef VX_CFG_EXT_ZACAS_ENABLE + `UNUSED_VAR (req_amo_cmp) + `endif + end + + // Port A address / write-data / write-enable mux: DMA, an owed atomic + // write-back, or the first engine's pass. They never coincide. + wire [BANK_ADDR_WIDTH-1:0] bank_sram_addr; + wire [WORD_WIDTH-1:0] bank_sram_wdata; + wire [WORD_SIZE-1:0] bank_sram_wren; + + assign bank_sram_addr = dma_fire ? BANK_ADDR_WIDTH'(dma_bus_if.req_data.addr) + : amo_wb_store ? amo_wb_addr + : a_bank_addr[i]; + assign bank_sram_wdata = dma_fire ? dma_bus_if.req_data.data[i*WORD_WIDTH +: WORD_WIDTH] + : amo_wb_store ? amo_wb_data + : req_data; + assign bank_sram_wren = dma_fire ? dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE] + : amo_wb_store ? amo_wb_byteen + : req_byteen; + + wire sram_read = dma_rd_fire || (lsu_active && ~a_bank_rw); + wire sram_write = dma_wr_b || amo_wb_store || (lsu_active && a_bank_rw); + + if (DUAL) begin : g_dual_port + VX_1rw1r_ram #( + .DATAW (WORD_WIDTH), + .SIZE (WORDS_PER_BANK), + .WRENW (WORD_SIZE) + ) lmem_store ( + .clk (clk), + .reset (reset), + .read_a (sram_read), + .write_a (sram_write), + .wren_a (bank_sram_wren), + .addr_a (bank_sram_addr), + .wdata_a (bank_sram_wdata), + .rdata_a (per_bank_rdata_a[i]), + .read_b (b_bank_valid[i]), + .addr_b (b_bank_addr[i]), + .rdata_b (b_bank_rdata[i]) + ); + end else begin : g_single_port + VX_sp_ram #( + .DATAW (WORD_WIDTH), + .SIZE (WORDS_PER_BANK), + .WRENW (WORD_SIZE), + .OUT_REG (1), + .RDW_MODE ("R") + ) lmem_store ( + .clk (clk), + .reset (reset), + .read (sram_read), + .write (sram_write), + .wren (bank_sram_wren), + .addr (bank_sram_addr), + .wdata (bank_sram_wdata), + .rdata (per_bank_rdata_a[i]) + ); + assign b_bank_rdata[i] = '0; + `UNUSED_VAR (b_bank_valid[i]) + `UNUSED_VAR (b_bank_addr[i]) + end + + // A store-conditional returns its outcome, not the word it read: zero + // for success, one for failure, positioned in the addressed lane for + // the LSU's subword extraction. The outcome is resolved at the commit + // stage, the cycle the response takes the SRAM output. + assign a_bank_rdata[i] = sc_resolving + ? (sc_lane_hi ? (WORD_WIDTH'(sc_fail) << (WORD_WIDTH/2)) + : WORD_WIDTH'(sc_fail)) + : per_bank_rdata_a[i]; + end + + if (AMO_ENABLE == 0) begin : g_no_amo_attr + `UNUSED_VAR ({a_bank_amo, a_bank_attr}) + end + +`ifdef PERF_ENABLE + wire [`CLOG2(NUM_LANES+1)-1:0] perf_a_served; + wire [`CLOG2(NUM_LANES+1)-1:0] perf_b_served; + `POP_COUNT(perf_a_served, a_perf_served); + `POP_COUNT(perf_b_served, b_perf_served); + + wire [NUM_BLOCKS-1:0] perf_crsp_stall_per_blk; + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_perf_crsp + assign perf_crsp_stall_per_blk[k] = lsu_mem_if[k].rsp_valid && ~lsu_mem_if[k].rsp_ready; + end + wire [`CLOG2(NUM_BLOCKS+1)-1:0] perf_crsp_stalls_per_cycle; + `POP_COUNT(perf_crsp_stalls_per_cycle, perf_crsp_stall_per_blk); + + reg [PERF_CTR_BITS-1:0] perf_reads; + reg [PERF_CTR_BITS-1:0] perf_writes; + reg [PERF_CTR_BITS-1:0] perf_bank_stalls; + reg [PERF_CTR_BITS-1:0] perf_crsp_stalls; + + always @(posedge clk) begin + if (reset) begin + perf_reads <= '0; + perf_writes <= '0; + perf_bank_stalls <= '0; + perf_crsp_stalls <= '0; + end else begin + perf_reads <= perf_reads + PERF_CTR_BITS'(a_perf_store ? 0 : perf_a_served) + PERF_CTR_BITS'(perf_b_served); + perf_writes <= perf_writes + PERF_CTR_BITS'(a_perf_store ? perf_a_served : 0); + perf_bank_stalls <= perf_bank_stalls + PERF_CTR_BITS'(a_perf_replay) + PERF_CTR_BITS'(b_perf_replay); + perf_crsp_stalls <= perf_crsp_stalls + PERF_CTR_BITS'(perf_crsp_stalls_per_cycle); + end + end + + assign lmem_perf.reads = perf_reads; + assign lmem_perf.writes = perf_writes; + assign lmem_perf.bank_stalls = perf_bank_stalls; + assign lmem_perf.crsp_stalls = perf_crsp_stalls; +`else + `UNUSED_VAR ({a_perf_served, a_perf_store, a_perf_replay, b_perf_served, b_perf_replay}) +`endif + +`ifdef DBG_TRACE_MEM + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_trace + always @(posedge clk) begin + if (lsu_mem_if[k].req_valid && lsu_mem_if[k].req_ready) begin + `TRACE(2, ("%t: %s req[%0d]: rw=%b, mask=%b, tag=0x%0h\n", + $time, INSTANCE_ID, k, lsu_mem_if[k].req_data.rw, lsu_mem_if[k].req_data.mask, lsu_mem_if[k].req_data.tag)) + end + if (lsu_mem_if[k].rsp_valid && lsu_mem_if[k].rsp_ready) begin + `TRACE(2, ("%t: %s rsp[%0d]: mask=%b, tag=0x%0h\n", + $time, INSTANCE_ID, k, lsu_mem_if[k].rsp_data.mask, lsu_mem_if[k].rsp_data.tag)) + end + end + end +`endif + +endmodule diff --git a/hw/rtl/mem/VX_local_mem.sv b/hw/rtl/mem/VX_local_mem.sv deleted file mode 100644 index 6d780960e5..0000000000 --- a/hw/rtl/mem/VX_local_mem.sv +++ /dev/null @@ -1,795 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_define.vh" - -module VX_local_mem import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - - // Size of cache in bytes - parameter SIZE = (1024*16*8), - - // Number of Word requests per cycle - parameter NUM_REQS = 4, - // Number of banks - parameter NUM_BANKS = 4, - - // Address width - parameter ADDR_WIDTH = `CLOG2(SIZE), - // Size of a word in bytes - parameter WORD_SIZE = `VX_CFG_XLEN/8, - - // Request tag size - parameter TAG_WIDTH = 16, - - // Enable DMA port - parameter DMA_ENABLE = 0, - parameter DMA_TAG_WIDTH = 1, - - // Synthesize atomic-op + LR/SC logic in the banks - parameter AMO_ENABLE = 0, - - // Response buffer - parameter OUT_BUF = 0 - ) ( - input wire clk, - input wire reset, - - // PERF -`ifdef PERF_ENABLE - output lmem_perf_t lmem_perf, -`endif - - // LSU read/write port - VX_mem_bus_if.slave lsu_bus_if [NUM_REQS], - - // DMA read/write port - VX_mem_bus_if.slave dma_bus_if -); - `UNUSED_SPARAM (INSTANCE_ID) - - localparam REQ_SEL_BITS = `CLOG2(NUM_REQS); - localparam REQ_SEL_WIDTH = `UP(REQ_SEL_BITS); - localparam WORD_WIDTH = WORD_SIZE * 8; - localparam NUM_WORDS = SIZE / WORD_SIZE; - localparam WORDS_PER_BANK = NUM_WORDS / NUM_BANKS; - localparam BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK); - localparam BANK_SEL_BITS = `CLOG2(NUM_BANKS); - localparam BANK_SEL_WIDTH = `UP(BANK_SEL_BITS); - // AMO sideband carried to the banks: valid + op + unsigned + hart_id. The - // operand width comes from the request's byteen; hart_id identifies the - // reserver so a store-conditional can tell its own reservation from - // another hart's. - // Compare-and-swap adds a comparand: the request's data word already - // carries the swap value, so the third operand needs its own room. -`ifdef VX_CFG_EXT_ZACAS_ENABLE - localparam AMO_CMP_WIDTH = `VX_CFG_XLEN; -`else - localparam AMO_CMP_WIDTH = 0; -`endif - localparam AMO_SB_WIDTH = 1 + $bits(amo_op_e) + 1 + HART_ID_WIDTH + AMO_CMP_WIDTH; - localparam REQ_DATAW = 1 + BANK_ADDR_WIDTH + WORD_SIZE + WORD_WIDTH + TAG_WIDTH + AMO_SB_WIDTH; - localparam RSP_DATAW = WORD_WIDTH + TAG_WIDTH; - - `STATIC_ASSERT(ADDR_WIDTH == (BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS)), ("invalid parameter")) - - // bank selection - - wire [NUM_REQS-1:0][BANK_SEL_WIDTH-1:0] req_bank_idx; - if (NUM_BANKS > 1) begin : g_req_bank_idx - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_idxs - assign req_bank_idx[i] = lsu_bus_if[i].req_data.addr[0 +: BANK_SEL_BITS]; - end - end else begin : g_req_bank_idx_0 - assign req_bank_idx = 0; - end - - // bank addressing - - wire [NUM_REQS-1:0][BANK_ADDR_WIDTH-1:0] req_bank_addr; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_bank_addr - assign req_bank_addr[i] = lsu_bus_if[i].req_data.addr[BANK_SEL_BITS +: BANK_ADDR_WIDTH]; - end - - // AMO sideband per request, from the shared mem-bus attr. - wire [NUM_REQS-1:0] req_amo_valid; - wire [NUM_REQS-1:0][$bits(amo_op_e)-1:0] req_amo_op; - wire [NUM_REQS-1:0] req_amo_unsigned; - wire [NUM_REQS-1:0][HART_ID_WIDTH-1:0] req_amo_hart_id; -`ifdef VX_CFG_EXT_ZACAS_ENABLE - wire [NUM_REQS-1:0][AMO_CMP_WIDTH-1:0] req_amo_cmp; -`endif - // Gated by AMO_ENABLE: the widths stay constant and the zeros constant-fold - // through the crossbar payload (the VX_cache_bank pattern). - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_amo - mem_bus_attr_t lane_attr; - assign lane_attr = mem_bus_attr_t'(lsu_bus_if[i].req_data.attr); - assign req_amo_valid[i] = AMO_ENABLE ? lane_attr.amo.amo_valid : 1'b0; - assign req_amo_op[i] = AMO_ENABLE ? lane_attr.amo.amo_op : '0; - assign req_amo_unsigned[i] = AMO_ENABLE ? lane_attr.amo.amo_unsigned : 1'b0; - assign req_amo_hart_id[i] = AMO_ENABLE ? lane_attr.amo.hart_id : '0; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - assign req_amo_cmp[i] = AMO_ENABLE ? lane_attr.amo.amo_cmp : '0; - `endif - `UNUSED_VAR (lane_attr) - end - - // bank requests dispatch - - wire [NUM_BANKS-1:0] per_bank_req_valid; - wire [NUM_BANKS-1:0] per_bank_req_rw; - wire [NUM_BANKS-1:0][BANK_ADDR_WIDTH-1:0] per_bank_req_addr; - wire [NUM_BANKS-1:0][WORD_SIZE-1:0] per_bank_req_byteen; - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_req_data; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_req_tag; - wire [NUM_BANKS-1:0] per_bank_req_amo_valid; - wire [NUM_BANKS-1:0][$bits(amo_op_e)-1:0] per_bank_req_amo_op; - wire [NUM_BANKS-1:0] per_bank_req_amo_unsigned; - wire [NUM_BANKS-1:0][HART_ID_WIDTH-1:0] per_bank_req_amo_hart_id; -`ifdef VX_CFG_EXT_ZACAS_ENABLE - wire [NUM_BANKS-1:0][AMO_CMP_WIDTH-1:0] per_bank_req_amo_cmp; -`endif - wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_req_idx; - wire [NUM_BANKS-1:0] per_bank_req_ready; - - wire [NUM_BANKS-1:0][REQ_DATAW-1:0] per_bank_req_data_aos; - - wire [NUM_REQS-1:0] req_valid_in; - wire [NUM_REQS-1:0][REQ_DATAW-1:0] req_data_in; - wire [NUM_REQS-1:0] req_ready_in; - -`ifdef PERF_ENABLE - wire [PERF_CTR_BITS-1:0] perf_collisions; -`endif - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_data_in - assign req_valid_in[i] = lsu_bus_if[i].req_valid; - assign req_data_in[i] = { - req_amo_valid[i], - req_amo_op[i], - req_amo_unsigned[i], - req_amo_hart_id[i], - `ifdef VX_CFG_EXT_ZACAS_ENABLE - req_amo_cmp[i], - `endif - lsu_bus_if[i].req_data.rw, - req_bank_addr[i], - lsu_bus_if[i].req_data.data, - lsu_bus_if[i].req_data.byteen, - lsu_bus_if[i].req_data.tag - }; - assign lsu_bus_if[i].req_ready = req_ready_in[i]; - end - - VX_stream_xbar #( - .NUM_INPUTS (NUM_REQS), - .NUM_OUTPUTS (NUM_BANKS), - .DATAW (REQ_DATAW), - .PERF_CTR_BITS (PERF_CTR_BITS), - .ARBITER ("P"), - .OUT_BUF (3) // output should be registered for the data_store addressing - ) req_xbar ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .collisions (perf_collisions), - `else - `UNUSED_PIN (collisions), - `endif - .valid_in (req_valid_in), - .data_in (req_data_in), - .sel_in (req_bank_idx), - .ready_in (req_ready_in), - .valid_out (per_bank_req_valid), - .data_out (per_bank_req_data_aos), - .sel_out (per_bank_req_idx), - .ready_out (per_bank_req_ready) - ); - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_req_data_soa - assign { - per_bank_req_amo_valid[i], - per_bank_req_amo_op[i], - per_bank_req_amo_unsigned[i], - per_bank_req_amo_hart_id[i], - `ifdef VX_CFG_EXT_ZACAS_ENABLE - per_bank_req_amo_cmp[i], - `endif - per_bank_req_rw[i], - per_bank_req_addr[i], - per_bank_req_data[i], - per_bank_req_byteen[i], - per_bank_req_tag[i] - } = per_bank_req_data_aos[i]; - end - - // banks access (declared here so g_dma_enable can reference per_bank_rsp_data) - - wire [NUM_BANKS-1:0] per_bank_rsp_valid; - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] per_bank_rsp_data; - // Back-pressure-safe copy of the LSU response data (see g_data_store). - wire [NUM_BANKS-1:0][WORD_WIDTH-1:0] bank_lsu_rsp_data; - wire [NUM_BANKS-1:0][REQ_SEL_WIDTH-1:0] per_bank_rsp_idx; - wire [NUM_BANKS-1:0][TAG_WIDTH-1:0] per_bank_rsp_tag; - wire [NUM_BANKS-1:0] per_bank_rsp_ready; - - // DMA port handshake - // rw=0 reads : accepted when the response pipe-buffer has space. - // rw=1 writes : always accepted; no response issued. - // DMA has priority over LSU at every bank SRAM. - - wire dma_rsp_buf_ready; // driven by pipe-buffer or tied 0 when disabled - - if (DMA_ENABLE) begin : g_dma_enable - `UNUSED_VAR (dma_bus_if.req_data.attr) - - assign dma_bus_if.req_ready = dma_bus_if.req_data.rw || dma_rsp_buf_ready; - - wire dma_rd_fire = dma_bus_if.req_valid && ~dma_bus_if.req_data.rw && dma_rsp_buf_ready; - - // Delay tag by 1 cycle to align with SRAM OUT_REG latency - VX_pipe_buffer #( - .DATAW (DMA_TAG_WIDTH) - ) dma_rsp_buf ( - .clk (clk), - .reset (reset), - .valid_in (dma_rd_fire), - .ready_in (dma_rsp_buf_ready), - .data_in (dma_bus_if.req_data.tag), - .valid_out (dma_bus_if.rsp_valid), - .data_out (dma_bus_if.rsp_data.tag), - .ready_out (dma_bus_if.rsp_ready) - ); - - // Pack all bank SRAM outputs into the read response. - // - // Same back-pressure hazard as the LSU side: dma_rsp_buf buffers only - // the tag/valid; the data (all banks) is the live SRAM OUT_REG. While a - // DMA read response is back-pressured, an interleaving LSU read - // (~dma_active) re-drives a bank's OUT_REG and corrupts the held DMA - // line. Latch the whole line on the first response-valid cycle (rdata - // is still valid then) and serve the latched copy while stalled. One - // DMA response spans all banks, so this is a single full-line hold. - reg [NUM_BANKS-1:0][WORD_WIDTH-1:0] dma_rsp_hold_data_r; - reg dma_rsp_hold_valid_r; - wire dma_rsp_consumed = dma_bus_if.rsp_valid && dma_bus_if.rsp_ready; - always @(posedge clk) begin - if (reset) begin - dma_rsp_hold_valid_r <= 1'b0; - end else if (dma_rsp_consumed) begin - dma_rsp_hold_valid_r <= 1'b0; - end else if (dma_bus_if.rsp_valid && ~dma_rsp_hold_valid_r) begin - dma_rsp_hold_data_r <= per_bank_rsp_data; - dma_rsp_hold_valid_r <= 1'b1; - end - end - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_dma_rsp_data - assign dma_bus_if.rsp_data.data[i*WORD_WIDTH +: WORD_WIDTH] = - dma_rsp_hold_valid_r ? dma_rsp_hold_data_r[i] : per_bank_rsp_data[i]; - end - - end else begin : g_no_dma - assign dma_rsp_buf_ready = 1'b0; - assign dma_bus_if.req_ready = 1'b0; - assign dma_bus_if.rsp_valid = 1'b0; - assign dma_bus_if.rsp_data = '0; - `UNUSED_VAR (dma_bus_if.req_valid) - `UNUSED_VAR (dma_bus_if.req_data) - `UNUSED_VAR (dma_bus_if.rsp_ready) - end - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_data_store - wire bank_rsp_valid, bank_rsp_ready; - - // DMA active signals (priority over LSU) - wire dma_wr_b = DMA_ENABLE - && dma_bus_if.req_valid - && dma_bus_if.req_data.rw - && (|dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE]); - - wire dma_rd_b = DMA_ENABLE - && dma_bus_if.req_valid - && ~dma_bus_if.req_data.rw - && dma_rsp_buf_ready; - - wire dma_active = dma_wr_b | dma_rd_b; - - wire lsu_active = per_bank_req_valid[i] && per_bank_req_ready[i]; - - // Bank-facing AMO interface, tied off when the A extension is absent so - // none of the atomic hardware is synthesized (the dcache counterpart is - // AMO_ENABLE in VX_cache_bank). - wire amo_busy; // an atomic owns the bank port - wire amo_wb_store; // owed atomic write-back - wire [BANK_ADDR_WIDTH-1:0] amo_wb_addr; - wire [WORD_SIZE-1:0] amo_wb_byteen; - wire [WORD_WIDTH-1:0] amo_wb_data; - wire sc_resolving; // SC outcome replaces the response - wire sc_fail; - wire sc_lane_hi; // outcome lane of a resolving .W SC - - if (AMO_ENABLE != 0) begin : g_amo - - // Atomics. The LSU sends an AMO with rw=0, so the read returning the old - // value is the ordinary read path and needs no change. What the bank owes - // is the write of the computed value. - // - // The old word is registered before the ALU sees it. Feeding the ALU - // straight from the SRAM output puts the BRAM clock-to-out, the adder and - // the BRAM setup in a single period; the register splits that into - // BRAM-out-to-flop and flop-through-adder-to-BRAM-in, which is what keeps - // the atomic adder off the critical path. - // - // The port is single-ported, so an atomic owns its bank for three cycles: - // read, capture, write back. - reg amo_rd_valid_r; - reg amo_wb_valid_r; - reg [WORD_WIDTH-1:0] amo_old_r; - reg [BANK_ADDR_WIDTH-1:0] amo_wb_addr_r; - reg [WORD_SIZE-1:0] amo_wb_byteen_r; - reg [$bits(amo_op_e)-1:0] amo_wb_op_r; - reg amo_wb_unsigned_r; - reg [WORD_WIDTH-1:0] amo_wb_rhs_r; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - reg [AMO_CMP_WIDTH-1:0] amo_wb_cmp_r; - `endif - - wire amo_accept = lsu_active && per_bank_req_amo_valid[i]; - - // Reservation commit stage. Every accepted request that can touch a - // reservation -- an atomic, or a plain store -- reaches it one cycle - // after acceptance, so a single address port serves both. The look-ahead - // read is driven from the acceptance cycle, which is what makes the - // registered reservation entry land here. - wire is_lr_in = per_bank_req_amo_valid[i] - && (amo_op_e'(per_bank_req_amo_op[i]) == AMO_OP_LR); - wire is_sc_in = per_bank_req_amo_valid[i] - && (amo_op_e'(per_bank_req_amo_op[i]) == AMO_OP_SC); - - reg res_valid_r; - reg res_is_lr_r; - reg res_is_sc_r; - reg res_is_amo_r; - reg res_is_store_r; - reg [BANK_ADDR_WIDTH-1:0] res_addr_r; - reg [HART_ID_WIDTH-1:0] res_hart_id_r; - - always @(posedge clk) begin - if (reset) begin - res_valid_r <= 1'b0; - end else begin - res_valid_r <= lsu_active; - end - if (lsu_active) begin - res_is_lr_r <= is_lr_in; - res_is_sc_r <= is_sc_in; - res_is_amo_r <= per_bank_req_amo_valid[i]; - res_is_store_r <= per_bank_req_rw[i]; - res_addr_r <= per_bank_req_addr[i]; - res_hart_id_r <= per_bank_req_amo_hart_id[i]; - end - end - - // A store-conditional fails when its reservation is gone. The outcome is - // known at the commit stage, one cycle before the write-back, so it can - // both suppress the write and be returned as this request's response. - wire res_check; - assign sc_fail = res_is_sc_r && ~res_check; - assign sc_resolving = res_valid_r && res_is_sc_r; - - // The SC outcome must land in the addressed lane: the LSU extracts - // the subword by address, so a .W outcome left at bit 0 would read - // as 0 ("success") for every upper-lane SC, including a failed one - // whose store was suppressed. - if (WORD_WIDTH == 64) begin : g_sc_lane - reg res_lane_hi_r; - always @(posedge clk) begin - if (lsu_active) begin - res_lane_hi_r <= ~per_bank_req_byteen[i][0]; - end - end - assign sc_lane_hi = res_lane_hi_r; - end else begin : g_sc_lane0 - assign sc_lane_hi = 1'b0; - end - - // Everything that ends up writing the word: a plain store, a - // read-modify-write atomic (which carries rw=0 and so is not a store by - // the request flag), and a store-conditional that found its reservation. - wire res_commits_store = res_is_store_r - || (res_is_amo_r && ~res_is_lr_r && ~res_is_sc_r) - || (res_is_sc_r && res_check); - - reg amo_wb_is_lr_r; - reg amo_wb_sc_fail_r; - always @(posedge clk) begin - if (amo_rd_valid_r) begin - amo_wb_is_lr_r <= res_is_lr_r; - amo_wb_sc_fail_r <= sc_fail; - end - end - - // A load-reserved leaves the word alone, and a failed store-conditional - // must not store; every other atomic writes its computed value back. - assign amo_wb_store = amo_wb_valid_r && ~amo_wb_is_lr_r && ~amo_wb_sc_fail_r; - assign amo_busy = amo_rd_valid_r || amo_wb_valid_r; - - always @(posedge clk) begin - if (reset) begin - amo_rd_valid_r <= 1'b0; - amo_wb_valid_r <= 1'b0; - end else begin - amo_rd_valid_r <= amo_accept; - amo_wb_valid_r <= amo_rd_valid_r; - end - if (amo_rd_valid_r) begin - amo_old_r <= per_bank_rsp_data[i]; - end - if (amo_accept) begin - amo_wb_addr_r <= per_bank_req_addr[i]; - amo_wb_byteen_r <= per_bank_req_byteen[i]; - amo_wb_op_r <= per_bank_req_amo_op[i]; - amo_wb_unsigned_r <= per_bank_req_amo_unsigned[i]; - amo_wb_rhs_r <= per_bank_req_data[i]; - `ifdef VX_CFG_EXT_ZACAS_ENABLE - amo_wb_cmp_r <= per_bank_req_amo_cmp[i]; - `endif - end - end - - // old_word is this cycle's SRAM output -- the same value the response - // carries back, which is why ret_word is not used here. - wire [63:0] amo_new_word, amo_ret_word; - - // Lane-align the RMW operands. The bank word is XLEN wide, so on a - // 64-bit hart a .W atomic addresses one of two 32-bit lanes; the - // ALU computes right-aligned. Store data arrives lane-positioned - // (same packing as a plain store), so old and rhs shift down by - // the lane offset and the result shifts back up under the request - // byteen. The comparand arrives right-aligned from rs3 and needs - // no shift. The addressed lane is recovered from the byteen: an - // atomic is naturally aligned, so a .W to the upper lane never - // covers byte 0 and a .D covers the full word. - wire [63:0] amo_old_lane, amo_rhs_lane; - wire [1:0] amo_width; - if (WORD_WIDTH == 64) begin : g_amo_lane - wire lane_hi = ~amo_wb_byteen_r[0]; - assign amo_width = (&amo_wb_byteen_r) ? 2'd3 : 2'd2; - assign amo_old_lane = lane_hi ? {32'h0, amo_old_r[63:32]} : amo_old_r; - assign amo_rhs_lane = lane_hi ? {32'h0, amo_wb_rhs_r[63:32]} : amo_wb_rhs_r; - assign amo_wb_data = lane_hi ? {amo_new_word[31:0], 32'h0} : amo_new_word; - end else begin : g_amo_word - assign amo_width = 2'd2; - assign amo_old_lane = 64'(amo_old_r); - assign amo_rhs_lane = 64'(amo_wb_rhs_r); - assign amo_wb_data = amo_new_word[WORD_WIDTH-1:0]; - end - // VX_CFG_AMO_RS_SIZE sizes ONE shared LLC bank's contention; deployed - // per lmem bank it multiplies by NUM_LMEM_BANKS x NUM_CORES. After the - // holder-credit protocol a hot word maps to exactly one station, so - // depth only reduces false eviction between distinct reserved words - // aliasing in one bank -- and a spurious SC failure is architecturally - // legal. Clamp the depth and shorten the credit budget to the local - // round trip instead of inheriting the L1-miss sizing. - VX_amo_unit #( - .NUM_RES_ENTRIES (`MIN(`VX_CFG_AMO_RS_SIZE, 4)), - .HOLD_CREDIT_BITS (3), - .LINE_ADDR_BITS (BANK_ADDR_WIDTH), - .DATA_WIDTH (WORD_WIDTH) - ) amo_unit ( - .clk (clk), - .reset (reset), - .pipe_stall (1'b0), - .compute_op (amo_op_e'(amo_wb_op_r)), - .compute_unsigned (amo_wb_unsigned_r), - .compute_width (amo_width), - .compute_old (amo_old_lane), - .compute_rhs (amo_rhs_lane), - `ifdef VX_CFG_EXT_ZACAS_ENABLE - .compute_cmp (64'(amo_wb_cmp_r)), - `else - .compute_cmp (64'b0), - `endif - .compute_new_word (amo_new_word), - .compute_ret_word (amo_ret_word), - // A load-reserved claims the word; a store-conditional gives up its - // own reservation whether it succeeds or fails; any committing store - // breaks whoever else holds one. - .res_reserve (res_valid_r && res_is_lr_r), - .res_clear (res_valid_r && res_is_sc_r), - .res_invalidate (res_valid_r && res_commits_store), - .res_hart_id (res_hart_id_r), - .res_line_addr (res_addr_r), - .res_line_addr_n (per_bank_req_addr[i]), - .res_check (res_check) - ); - `UNUSED_VAR (amo_ret_word) - if (WORD_WIDTH < 64) begin : g_amo_hi_unused - `UNUSED_VAR (amo_new_word[63:WORD_WIDTH]) - end - - assign amo_wb_addr = amo_wb_addr_r; - assign amo_wb_byteen = amo_wb_byteen_r; - - end else begin : g_no_amo - assign amo_busy = 1'b0; - assign amo_wb_store = 1'b0; - assign amo_wb_addr = '0; - assign amo_wb_byteen = '0; - assign amo_wb_data = '0; - assign sc_resolving = 1'b0; - assign sc_fail = 1'b0; - assign sc_lane_hi = 1'b0; - `UNUSED_VAR (per_bank_req_amo_valid[i]) - `UNUSED_VAR (per_bank_req_amo_op[i]) - `UNUSED_VAR (per_bank_req_amo_unsigned[i]) - `UNUSED_VAR (per_bank_req_amo_hart_id[i]) - `ifdef VX_CFG_EXT_ZACAS_ENABLE - `UNUSED_VAR (per_bank_req_amo_cmp[i]) - `endif - end - - // SRAM address / write-data / write-enable mux: DMA first, then an owed - // atomic write-back, then the incoming request - wire [BANK_ADDR_WIDTH-1:0] bank_sram_addr; - wire [WORD_WIDTH-1:0] bank_sram_wdata; - wire [WORD_SIZE-1:0] bank_sram_wren; - - assign bank_sram_addr = dma_active ? BANK_ADDR_WIDTH'(dma_bus_if.req_data.addr) - : amo_wb_store ? amo_wb_addr - : per_bank_req_addr[i]; - assign bank_sram_wdata = dma_wr_b ? dma_bus_if.req_data.data[i*WORD_WIDTH +: WORD_WIDTH] - : amo_wb_store ? amo_wb_data - : per_bank_req_data[i]; - assign bank_sram_wren = dma_wr_b ? dma_bus_if.req_data.byteen[i*WORD_SIZE +: WORD_SIZE] - : amo_wb_store ? amo_wb_byteen - : per_bank_req_byteen[i]; - - VX_sp_ram #( - .DATAW (WORD_WIDTH), - .SIZE (WORDS_PER_BANK), - .WRENW (WORD_SIZE), - .OUT_REG (1), - .RDW_MODE ("R") - ) lmem_store ( - .clk (clk), - .reset (reset), - .read (dma_rd_b || (lsu_active && ~per_bank_req_rw[i])), - .write (dma_wr_b || amo_wb_store || (lsu_active && per_bank_req_rw[i])), - .wren (bank_sram_wren), - .addr (bank_sram_addr), - .wdata (bank_sram_wdata), - .rdata (per_bank_rsp_data[i]) - ); - - // Read-during-write hazard: stalls LSU reads to an address written last cycle - // (SRAM OUT_REG + RDW_MODE="R" returns stale data on same-cycle read-after-write). - // DMA reads bypass this check. - - reg [BANK_ADDR_WIDTH-1:0] last_wr_addr; - reg last_wr_valid; - always @(posedge clk) begin - if (reset) begin - last_wr_valid <= 0; - end else begin - last_wr_valid <= dma_wr_b || amo_wb_store || (lsu_active && per_bank_req_rw[i]); - end - last_wr_addr <= bank_sram_addr; - end - wire is_rdw_hazard = last_wr_valid && ~per_bank_req_rw[i] && (per_bank_req_addr[i] == last_wr_addr); - - // LSU response valid / request ready — blocked by DMA and RDW hazards - - assign bank_rsp_valid = per_bank_req_valid[i] - && ~dma_active - && ~amo_busy - && ~per_bank_req_rw[i] - && ~is_rdw_hazard; - - assign per_bank_req_ready[i] = ~dma_active - && ~amo_busy - && (bank_rsp_ready || per_bank_req_rw[i]) - && ~is_rdw_hazard; - - // Delay tag/idx to align with SRAM 1-cycle output latency - VX_pipe_buffer #( - .DATAW (REQ_SEL_WIDTH + TAG_WIDTH) - ) bram_buf ( - .clk (clk), - .reset (reset), - .valid_in (bank_rsp_valid), - .ready_in (bank_rsp_ready), - .data_in ({per_bank_req_idx[i], per_bank_req_tag[i]}), - .data_out ({per_bank_rsp_idx[i], per_bank_rsp_tag[i]}), - .valid_out (per_bank_rsp_valid[i]), - .ready_out (per_bank_rsp_ready[i]) - ); - - // A store-conditional returns its outcome, not the word it read: zero - // for success, one for failure, positioned in the addressed lane for - // the LSU's subword extraction. The outcome is resolved at the commit - // stage, which is the same cycle the response carries the SRAM output, - // so it substitutes here rather than needing a response of its own. - wire [WORD_WIDTH-1:0] bank_rsp_word = sc_resolving - ? (sc_lane_hi ? (WORD_WIDTH'(sc_fail) << (WORD_WIDTH/2)) - : WORD_WIDTH'(sc_fail)) - : per_bank_rsp_data[i]; - - // Back-pressure-safe LSU response data. The bank SRAM OUT_REG - // (per_bank_rsp_data) is read live by the response xbar, but the - // response valid/tag is elastic-buffered, so a back-pressured LSU - // response can sit valid for >1 cycle. During that wait an interleaving - // DMA read on this bank re-drives the shared OUT_REG, corrupting the - // held LSU response (WGMMA+DXA issues concurrent LSU A-reads and DMA - // B-reads to the same banks — the only workload that does). Latch the - // data on the first response-valid cycle (rdata is still valid then) - // and serve the latched copy while the response is stalled. The latched - // copy is the substituted word, so a stalled SC keeps its outcome. - reg [WORD_WIDTH-1:0] lsu_rsp_hold_data_r; - reg lsu_rsp_hold_valid_r; - always @(posedge clk) begin - if (reset) begin - lsu_rsp_hold_valid_r <= 1'b0; - end else if (per_bank_rsp_valid[i] && per_bank_rsp_ready[i]) begin - lsu_rsp_hold_valid_r <= 1'b0; // response consumed - end else if (per_bank_rsp_valid[i] && ~lsu_rsp_hold_valid_r) begin - lsu_rsp_hold_data_r <= bank_rsp_word; // capture before DMA can clobber - lsu_rsp_hold_valid_r <= 1'b1; - end - end - assign bank_lsu_rsp_data[i] = lsu_rsp_hold_valid_r ? lsu_rsp_hold_data_r - : bank_rsp_word; - - end - - // bank responses gather - - wire [NUM_BANKS-1:0][RSP_DATAW-1:0] per_bank_rsp_data_aos; - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_rsp_data_aos - assign per_bank_rsp_data_aos[i] = {bank_lsu_rsp_data[i], per_bank_rsp_tag[i]}; - end - - wire [NUM_REQS-1:0] rsp_valid_out; - wire [NUM_REQS-1:0][RSP_DATAW-1:0] rsp_data_out; - wire [NUM_REQS-1:0] rsp_ready_out; - - VX_stream_xbar #( - .NUM_INPUTS (NUM_BANKS), - .NUM_OUTPUTS (NUM_REQS), - .DATAW (RSP_DATAW), - .ARBITER ("P"), // this priority arbiter has negligeable impact on performance - .OUT_BUF (OUT_BUF) - ) rsp_xbar ( - .clk (clk), - .reset (reset), - `UNUSED_PIN (collisions), - .sel_in (per_bank_rsp_idx), - .valid_in (per_bank_rsp_valid), - .data_in (per_bank_rsp_data_aos), - .ready_in (per_bank_rsp_ready), - .valid_out (rsp_valid_out), - .data_out (rsp_data_out), - .ready_out (rsp_ready_out), - `UNUSED_PIN (sel_out) - ); - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_lsu_bus_if - assign lsu_bus_if[i].rsp_valid = rsp_valid_out[i]; - assign lsu_bus_if[i].rsp_data = rsp_data_out[i]; - assign rsp_ready_out[i] = lsu_bus_if[i].rsp_ready; - end - -`ifdef PERF_ENABLE - // per cycle: reads, writes - wire [`CLOG2(NUM_REQS+1)-1:0] perf_reads_per_cycle; - wire [`CLOG2(NUM_REQS+1)-1:0] perf_writes_per_cycle; - wire [`CLOG2(NUM_REQS+1)-1:0] perf_crsp_stall_per_cycle; - - wire [NUM_REQS-1:0] req_rw; - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_rw - assign req_rw[i] = lsu_bus_if[i].req_data.rw; - end - - wire [NUM_REQS-1:0] perf_reads_per_req, perf_writes_per_req; - wire [NUM_REQS-1:0] perf_crsp_stall_per_req = rsp_valid_out & ~rsp_ready_out; - - `BUFFER(perf_reads_per_req, req_valid_in & req_ready_in & ~req_rw); - `BUFFER(perf_writes_per_req, req_valid_in & req_ready_in & req_rw); - - `POP_COUNT(perf_reads_per_cycle, perf_reads_per_req); - `POP_COUNT(perf_writes_per_cycle, perf_writes_per_req); - `POP_COUNT(perf_crsp_stall_per_cycle, perf_crsp_stall_per_req); - - reg [PERF_CTR_BITS-1:0] perf_reads; - reg [PERF_CTR_BITS-1:0] perf_writes; - reg [PERF_CTR_BITS-1:0] perf_crsp_stalls; - - always @(posedge clk) begin - if (reset) begin - perf_reads <= '0; - perf_writes <= '0; - perf_crsp_stalls <= '0; - end else begin - perf_reads <= perf_reads + PERF_CTR_BITS'(perf_reads_per_cycle); - perf_writes <= perf_writes + PERF_CTR_BITS'(perf_writes_per_cycle); - perf_crsp_stalls <= perf_crsp_stalls + PERF_CTR_BITS'(perf_crsp_stall_per_cycle); - end - end - - assign lmem_perf.reads = perf_reads; - assign lmem_perf.writes = perf_writes; - assign lmem_perf.bank_stalls = perf_collisions; - assign lmem_perf.crsp_stalls = perf_crsp_stalls; - -`endif - -`ifdef DBG_TRACE_MEM - - wire [NUM_BANKS-1:0][TAG_WIDTH-UUID_WIDTH-1:0] per_bank_req_tag_value; - wire [NUM_BANKS-1:0][UUID_WIDTH-1:0] per_bank_req_uuid; - - wire [NUM_BANKS-1:0][TAG_WIDTH-UUID_WIDTH-1:0] per_bank_rsp_tag_value; - wire [NUM_BANKS-1:0][UUID_WIDTH-1:0] per_bank_rsp_uuid; - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_per_bank_req_uuid - assign per_bank_req_tag_value[i] = per_bank_req_tag[i][TAG_WIDTH-UUID_WIDTH-1:0]; - assign per_bank_rsp_tag_value[i] = per_bank_rsp_tag[i][TAG_WIDTH-UUID_WIDTH-1:0]; - if (UUID_WIDTH != 0) begin : g_uuid - assign per_bank_req_uuid[i] = per_bank_req_tag[i][TAG_WIDTH-1 -: UUID_WIDTH]; - assign per_bank_rsp_uuid[i] = per_bank_rsp_tag[i][TAG_WIDTH-1 -: UUID_WIDTH]; - end else begin : g_no_uuid - assign per_bank_req_uuid[i] = 0; - assign per_bank_rsp_uuid[i] = 0; - end - end - - for (genvar i = 0; i < NUM_REQS; ++i) begin : g_req_trace - always @(posedge clk) begin - if (lsu_bus_if[i].req_valid && lsu_bus_if[i].req_ready) begin - if (lsu_bus_if[i].req_data.rw) begin - `TRACE(2, ("%t: %s core-wr-req[%0d]: addr=0x%0h, byteen=0x%h, data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].req_data.addr, lsu_bus_if[i].req_data.byteen, lsu_bus_if[i].req_data.data, lsu_bus_if[i].req_data.tag.value, lsu_bus_if[i].req_data.tag.uuid)) - end else begin - `TRACE(2, ("%t: %s core-rd-req[%0d]: addr=0x%0h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].req_data.addr, lsu_bus_if[i].req_data.tag.value, lsu_bus_if[i].req_data.tag.uuid)) - end - end - if (lsu_bus_if[i].rsp_valid && lsu_bus_if[i].rsp_ready) begin - `TRACE(2, ("%t: %s core-rd-rsp[%0d]: data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, lsu_bus_if[i].rsp_data.data, lsu_bus_if[i].rsp_data.tag.value, lsu_bus_if[i].rsp_data.tag.uuid)) - end - end - end - - for (genvar i = 0; i < NUM_BANKS; ++i) begin : g_bank_trace - always @(posedge clk) begin - if (per_bank_req_valid[i] && per_bank_req_ready[i]) begin - if (per_bank_req_rw[i]) begin - `TRACE(2, ("%t: %s bank-wr-req[%0d]: addr=0x%0h, byteen=0x%h, data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_req_addr[i], per_bank_req_byteen[i], per_bank_req_data[i], per_bank_req_tag_value[i], per_bank_req_uuid[i])) - end else begin - `TRACE(2, ("%t: %s bank-rd-req[%0d]: addr=0x%0h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_req_addr[i], per_bank_req_tag_value[i], per_bank_req_uuid[i])) - end - end - if (per_bank_rsp_valid[i] && per_bank_rsp_ready[i]) begin - `TRACE(2, ("%t: %s bank-rd-rsp[%0d]: data=0x%h, tag=0x%0h (#%0d)\n", - $time, INSTANCE_ID, i, per_bank_rsp_data[i], per_bank_rsp_tag_value[i], per_bank_rsp_uuid[i])) - end - end - end - -`endif - -endmodule diff --git a/hw/rtl/mem/VX_lsu_mem_arb.sv b/hw/rtl/mem/VX_lsu_mem_arb.sv index 29aa903045..90df8a233a 100644 --- a/hw/rtl/mem/VX_lsu_mem_arb.sv +++ b/hw/rtl/mem/VX_lsu_mem_arb.sv @@ -37,6 +37,8 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( ); localparam DATA_WIDTH = (8 * DATA_SIZE); localparam LOG_NUM_REQS = `ARB_SEL_BITS(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); localparam REQ_DATAW = 1 + NUM_LANES * (1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + `UP(USER_WIDTH)) + TAG_WIDTH; localparam RSP_DATAW = NUM_LANES * (1 + DATA_WIDTH) + TAG_WIDTH; @@ -45,7 +47,7 @@ module VX_lsu_mem_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; wire [NUM_OUTPUTS-1:0] req_ready_out; - wire [NUM_OUTPUTS-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; diff --git a/hw/rtl/mem/VX_mem_bus_arb.sv b/hw/rtl/mem/VX_mem_bus_arb.sv index 3b9fc8198f..70f16613d7 100644 --- a/hw/rtl/mem/VX_mem_bus_arb.sv +++ b/hw/rtl/mem/VX_mem_bus_arb.sv @@ -37,6 +37,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( localparam REQ_DATAW = 1 + ADDR_WIDTH + DATA_WIDTH + DATA_SIZE + ATTR_WIDTH + TAG_WIDTH; localparam RSP_DATAW = DATA_WIDTH + TAG_WIDTH; localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(`CDIV(`MAX(NUM_INPUTS, NUM_OUTPUTS), SEL_COUNT))); wire [NUM_INPUTS-1:0] req_valid_in; wire [NUM_INPUTS-1:0][REQ_DATAW-1:0] req_data_in; @@ -44,7 +45,7 @@ module VX_mem_bus_arb import VX_gpu_pkg::*; #( wire [NUM_OUTPUTS-1:0] req_valid_out; wire [NUM_OUTPUTS-1:0][REQ_DATAW-1:0] req_data_out; - wire [SEL_COUNT-1:0][`UP(LOG_NUM_REQS)-1:0] req_sel_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] req_sel_out; wire [NUM_OUTPUTS-1:0] req_ready_out; for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_req_data_in diff --git a/hw/rtl/mem/VX_mem_req_pending.sv b/hw/rtl/mem/VX_mem_req_pending.sv new file mode 100644 index 0000000000..90ed7e620e --- /dev/null +++ b/hw/rtl/mem/VX_mem_req_pending.sv @@ -0,0 +1,72 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Tracks the requests held between the input and output handshakes of a +// section that neither creates nor drops requests (arbiters, buffers). +module VX_mem_req_pending #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter SIZE = 8 * NUM_OUTPUTS + 2 * NUM_INPUTS // bound on the requests the section can hold +) ( + input wire clk, + input wire reset, + input wire [NUM_INPUTS-1:0] in_fire, + input wire [NUM_OUTPUTS-1:0] out_fire, + output wire empty +); + localparam INCRW = `CLOG2(NUM_INPUTS+1); + localparam DECRW = `CLOG2(NUM_OUTPUTS+1); + localparam SIZEW = `CLOG2(SIZE+1); + + // The handshakes usually end long ready chains, so they are registered + // before counting; the count then lags a cycle and empty adds back the + // registered handshakes, the same value on every cycle. + reg [NUM_INPUTS-1:0] in_fire_r; + reg [NUM_OUTPUTS-1:0] out_fire_r; + always @(posedge clk) begin + if (reset) begin + in_fire_r <= '0; + out_fire_r <= '0; + end else begin + in_fire_r <= in_fire; + out_fire_r <= out_fire; + end + end + + wire [INCRW-1:0] in_cnt; + wire [DECRW-1:0] out_cnt; + `POP_COUNT(in_cnt, in_fire_r); + `POP_COUNT(out_cnt, out_fire_r); + + wire [SIZEW-1:0] size_lag; + VX_pending_size #( + .SIZE (SIZE), + .INCRW (INCRW), + .DECRW (DECRW) + ) pending ( + .clk (clk), + .reset (reset), + .incr (in_cnt), + .decr (out_cnt), + `UNUSED_PIN (empty), + `UNUSED_PIN (alm_empty), + `UNUSED_PIN (full), + `UNUSED_PIN (alm_full), + .size (size_lag) + ); + + assign empty = ((SIZEW+1)'(size_lag) + (SIZEW+1)'(in_cnt)) == (SIZEW+1)'(out_cnt); + +endmodule diff --git a/hw/rtl/om/VX_om_core.sv b/hw/rtl/om/VX_om_core.sv index 459708e0d6..e1821560c3 100644 --- a/hw/rtl/om/VX_om_core.sv +++ b/hw/rtl/om/VX_om_core.sv @@ -51,7 +51,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( om_rt_dcrs_t om_rt_dcrs [`VX_OM_MAX_RT]; VX_om_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) om_dcr ( .clk (clk), .reset (reset), @@ -88,7 +88,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( `UNUSED_VAR (mem_write_notify) VX_om_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (MEM_TAG_WIDTH) ) om_mem ( @@ -144,7 +144,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [NUM_LANES-1:0] ds_pass_out; VX_om_ds #( - .INSTANCE_ID ($sformatf("%s-ds", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-ds", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (DS_TAG_WIDTH) ) om_ds ( @@ -187,7 +187,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( wire [OM_RT_IDX_BITS-1:0] mem_rsp_rt; VX_om_blend #( - .INSTANCE_ID ($sformatf("%s-blend", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-blend", INSTANCE_ID))), .NUM_LANES (NUM_LANES), .TAG_WIDTH (BLEND_TAG_WIDTH) ) om_blend ( @@ -575,8 +575,8 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( || ~pending_reads_empty || mem_rsp_valid || mem_unit_busy; `ifdef SCOPE -`ifdef DBG_SCOPE_OM `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_OM wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [OCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[OCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -628,7 +628,7 @@ module VX_om_core import VX_gpu_pkg::*; import VX_om_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/om/VX_om_define.vh b/hw/rtl/om/VX_om_define.vh index d6f9d12a8d..045fab1dc5 100644 --- a/hw/rtl/om/VX_om_define.vh +++ b/hw/rtl/om/VX_om_define.vh @@ -18,8 +18,12 @@ `include "VX_define.vh" +// Buffer bases: cache-block indices. Under VM they are virtual addresses, +// which span the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define OM_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define OM_ADDR_BITS 26 `else `define OM_ADDR_BITS 25 `endif diff --git a/hw/rtl/om/VX_om_ingress.sv b/hw/rtl/om/VX_om_ingress.sv index c1344b4227..5aba9c0991 100644 --- a/hw/rtl/om/VX_om_ingress.sv +++ b/hw/rtl/om/VX_om_ingress.sv @@ -52,7 +52,8 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( VX_mem_bus_if.slave mem_bus_if, // aperture writes, peeled by VX_om_steer VX_om_bus_if.master om_bus_if, // fragments -> VX_om_bus_arb -> VX_om_core - output wire busy + output wire busy, + output wire frag_pending ); `UNUSED_SPARAM (INSTANCE_ID) `UNUSED_PARAM (TAG_WIDTH) @@ -307,6 +308,10 @@ module VX_om_ingress import VX_gpu_pkg::*, VX_om_pkg::*; #( assign busy = (open_valid != '0) || rec_valid || om_bus_if.req_valid; + // Work that completes without another beat. An open record waits for its + // depth beat, which may be queued behind a flush token held at the steer. + assign frag_pending = rec_valid || om_bus_if.req_valid; + `RUNTIME_ASSERT(~(s2_fire && alloc && ~has_free), ("%t: *** %s: OM ingress hold table overflow — MAX_OPEN bound is wrong", $time, INSTANCE_ID)) `RUNTIME_ASSERT(~(s2_fire && s2_is_pair && s2_is_depth && ~hit), diff --git a/hw/rtl/om/VX_om_mem.sv b/hw/rtl/om/VX_om_mem.sv index 67cc03cb6f..f2c5e90097 100644 --- a/hw/rtl/om/VX_om_mem.sv +++ b/hw/rtl/om/VX_om_mem.sv @@ -245,7 +245,7 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (MEM_CHANNELS), .WORD_SIZE (4), @@ -271,6 +271,8 @@ module VX_om_mem import VX_gpu_pkg::*; import VX_om_pkg::*; #( .core_req_data (mreq_data_r), .core_req_tag (mreq_tag_r), .core_req_ready (mreq_ready_r), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), .req_queue_rw_notify (write_notify), .req_queue_empty (req_queue_empty_w), diff --git a/hw/rtl/om/VX_om_steer.sv b/hw/rtl/om/VX_om_steer.sv index cb94dce869..26310185c8 100644 --- a/hw/rtl/om/VX_om_steer.sv +++ b/hw/rtl/om/VX_om_steer.sv @@ -23,12 +23,14 @@ // // The select is the is_addr_om attr bit alone. No cache level decodes it -- the // aperture store is presented to the caches as an ordinary IO store (they carry -// attr through verbatim), so this bit arrives untouched. See VX_lsu_slice. +// attr through verbatim), so this bit arrives untouched. See VX_lsu_block. // -// Buffering: only the OM leg is registered. The L2 pass-through stays -// combinational -- it is not a new driver (the socket already registers its -// outgoing bus), and a skid here would sit on the critical L1->L2 path and -// double the switch's cost for nothing. +// Flush ordering: the steer is a fork, so a flush token bound for L2 is held +// here until the OM side has been flushed. Every aperture store this trunk +// carried before the token is past the steer by then, so the OM flush, which +// waits for the OM to drain, covers them; then the token goes on to L2. +// +// Buffering: each leg presents a registered master interface (OUT_BUF). `include "VX_define.vh" @@ -43,7 +45,12 @@ module VX_om_steer import VX_gpu_pkg::*; #( VX_mem_bus_if.slave bus_in_if, // from the socket VX_mem_bus_if.master l2_out_if, // everything else -> L2 - VX_mem_bus_if.master om_out_if // aperture writes -> OM ingress (posted) + VX_mem_bus_if.master om_out_if, // aperture writes -> OM ingress (posted) + + output wire flush_req, // a flush token is held until the OM side is flushed + output wire flush_l1_only, // the held token is an L1-only one + input wire flush_ack, // pulse: the OM side was flushed after flush_req rose + output wire om_pending // an aperture write is still buffered on the OM leg ); `UNUSED_SPARAM (INSTANCE_ID) @@ -53,11 +60,38 @@ module VX_om_steer import VX_gpu_pkg::*; #( wire is_om = bus_in_if.req_data.attr[MEM_ATTR_OM_OFFS]; + // ── flush tokens: held until the OM side is flushed ──────────────────── + wire is_flush = bus_in_if.req_valid && ~is_om && bus_in_if.req_data.attr[MEM_ATTR_FLUSH_OFFS]; + reg flush_req_r; // token held, waiting for the OM flush + reg flush_pass_r; // OM flushed: the token may go to L2 + wire flush_block = is_flush && ~flush_pass_r; + // ── request: demux by is_addr_om ─────────────────────────────────────── wire l2_ready; wire om_ready; - assign bus_in_if.req_ready = is_om ? om_ready : l2_ready; + assign bus_in_if.req_ready = is_om ? om_ready : (l2_ready && ~flush_block); + + wire flush_fire = is_flush && flush_pass_r && l2_ready; + always @(posedge clk) begin + if (reset) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b0; + end else begin + if (flush_req_r && flush_ack) begin + flush_req_r <= 1'b0; + flush_pass_r <= 1'b1; + end else if (flush_block) begin + flush_req_r <= 1'b1; + end + if (flush_fire) begin + flush_pass_r <= 1'b0; + end + end + end + assign flush_req = flush_req_r; + assign flush_l1_only = (| bus_in_if.req_data.byteen); + assign om_pending = om_out_if.req_valid; // Both legs go through an OUT_BUF elastic buffer so the socket->L2 trunk and // the OM aperture path each present a fully-registered master interface. @@ -71,7 +105,7 @@ module VX_om_steer import VX_gpu_pkg::*; #( ) l2_req_buf ( .clk (clk), .reset (reset), - .valid_in (bus_in_if.req_valid && ~is_om), + .valid_in (bus_in_if.req_valid && ~is_om && ~flush_block), .ready_in (l2_ready), .data_in (l2_req_data_in), .data_out (l2_req_data_out), diff --git a/hw/rtl/raster/VX_raster_be.sv b/hw/rtl/raster/VX_raster_be.sv index e33ca50d9c..8426c46cf0 100644 --- a/hw/rtl/raster/VX_raster_be.sv +++ b/hw/rtl/raster/VX_raster_be.sv @@ -119,7 +119,7 @@ module VX_raster_be import VX_raster_pkg::*; #( wire [PER_BLOCK_QUADS-1:0][`VX_RASTER_DIM_BITS-1:0] qe_yloc; VX_raster_qe #( - .INSTANCE_ID ($sformatf("%s-qe", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-qe", INSTANCE_ID))), .NUM_QUADS (PER_BLOCK_QUADS) ) quad_evaluator ( .clk (clk), diff --git a/hw/rtl/raster/VX_raster_core.sv b/hw/rtl/raster/VX_raster_core.sv index 470e8ea205..372dd007a0 100644 --- a/hw/rtl/raster/VX_raster_core.sv +++ b/hw/rtl/raster/VX_raster_core.sv @@ -17,7 +17,6 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter NUM_SLICES = 1, // number of slices parameter TILE_LOGSIZE = 5, // tile log size @@ -32,6 +31,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // PERF `ifdef PERF_ENABLE VX_raster_perf_if.master perf_raster_if, @@ -79,7 +80,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( raster_dcrs_t raster_dcrs; VX_raster_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)) + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))) ) raster_dcr ( .clk (clk), .reset (reset), @@ -134,14 +135,14 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // Memory unit VX_raster_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), - .INSTANCE_IDX (INSTANCE_IDX), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .NUM_INSTANCES (NUM_INSTANCES), .TILE_LOGSIZE (TILE_LOGSIZE), .QUEUE_SIZE (MEM_FIFO_DEPTH) ) raster_mem ( .clk (clk), .reset (reset), + .instance_idx (instance_idx), .start (mem_unit_start), .busy (mem_unit_busy), @@ -330,7 +331,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `endif VX_raster_slice #( - .INSTANCE_ID ($sformatf("%s-slice%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-slice%d", INSTANCE_ID, slice_id))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), @@ -372,7 +373,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // depth read from the ocache (elastic, variable-latency). Pass-through // when the per-draw earlyz_safe DCR is clear. VX_raster_earlyz #( - .INSTANCE_ID ($sformatf("%s-earlyz%d", INSTANCE_ID, slice_id)), + .INSTANCE_ID (`SFORMATF(("%s-earlyz%d", INSTANCE_ID, slice_id))), .OUTPUT_QUADS (OUTPUT_QUADS) ) raster_earlyz ( .clk (clk), @@ -529,7 +530,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( // no-work case so such an engine drains immediately. localparam LOG2_NUM_INSTANCES = `CLOG2(NUM_INSTANCES); wire [`RASTER_TILE_BITS-1:0] my_tile_count = - (raster_dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; + (raster_dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; wire has_no_tiles = (my_tile_count == '0); always @(posedge clk) begin @@ -559,8 +560,8 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( assign busy = armed_r | frame_kick; `ifdef SCOPE -`ifdef DBG_SCOPE_RASTER `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_RASTER wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire raster_bus_fire = raster_bus_tmp_if[0].req_valid && raster_bus_tmp_if[0].req_ready; @@ -614,7 +615,7 @@ module VX_raster_core import VX_gpu_pkg::*; import VX_raster_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/raster/VX_raster_dcr.sv b/hw/rtl/raster/VX_raster_dcr.sv index 6297c29a87..f535112ae6 100644 --- a/hw/rtl/raster/VX_raster_dcr.sv +++ b/hw/rtl/raster/VX_raster_dcr.sv @@ -32,7 +32,9 @@ module VX_raster_dcr import VX_gpu_pkg::*, VX_raster_pkg::*; output raster_dcrs_t raster_dcrs ); `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_RASTER_EARLYZ_ENABLE `UNUSED_VAR (reset) +`endif // Decode write strobe from the req-style DCR bus wire write_valid = dcr_bus_if.req_valid && dcr_bus_if.req_data.rw; @@ -47,8 +49,16 @@ module VX_raster_dcr import VX_gpu_pkg::*, VX_raster_pkg::*; // DCR registers raster_dcrs_t dcrs; - // DCRs write + // DCRs write. The early-Z gate resets off: a draw that never programs the + // OM depth state must not cull, or read a depth buffer at a power-on + // address. always @(posedge clk) begin + `ifdef VX_CFG_RASTER_EARLYZ_ENABLE + if (reset) begin + dcrs.earlyz_safe <= 1'b0; + dcrs.depth_func <= OM_DEPTH_FUNC_BITS'(`VX_OM_DEPTH_FUNC_ALWAYS); + end else + `endif if (write_valid) begin case (write_addr) `VX_DCR_RASTER_TBUF_ADDR: begin diff --git a/hw/rtl/raster/VX_raster_define.vh b/hw/rtl/raster/VX_raster_define.vh index b8920cba7f..b4c95d2ba2 100644 --- a/hw/rtl/raster/VX_raster_define.vh +++ b/hw/rtl/raster/VX_raster_define.vh @@ -18,8 +18,12 @@ `include "VX_define.vh" +// Buffer bases: cache-block indices. Under VM they are virtual addresses, +// which span the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define RASTER_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define RASTER_ADDR_BITS 26 `else `define RASTER_ADDR_BITS 25 `endif diff --git a/hw/rtl/raster/VX_raster_earlyz.sv b/hw/rtl/raster/VX_raster_earlyz.sv index 2cb0529650..c95061aa05 100644 --- a/hw/rtl/raster/VX_raster_earlyz.sv +++ b/hw/rtl/raster/VX_raster_earlyz.sv @@ -297,7 +297,7 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_PIX), .MEM_CHANNELS (OCACHE_NUM_REQS), .WORD_SIZE (OCACHE_WORD_SIZE), @@ -324,6 +324,8 @@ module VX_raster_earlyz import VX_gpu_pkg::*; import VX_raster_pkg::*; import VX .core_req_data ('0), .core_req_tag (SCHED_TAG_WIDTH'(0)), .core_req_ready (mreq_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/raster/VX_raster_mem.sv b/hw/rtl/raster/VX_raster_mem.sv index d4e5a82bda..b677ce96d5 100644 --- a/hw/rtl/raster/VX_raster_mem.sv +++ b/hw/rtl/raster/VX_raster_mem.sv @@ -23,7 +23,6 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter INSTANCE_IDX = 0, parameter NUM_INSTANCES = 1, parameter TILE_LOGSIZE = 5, parameter QUEUE_SIZE = 8 @@ -31,6 +30,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( input wire clk, input wire reset, + input wire [`LOG2UP(NUM_INSTANCES)-1:0] instance_idx, + // Device configurations raster_dcrs_t dcrs, @@ -144,8 +145,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (th_pids_count) // calculate tile start info - wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + `RASTER_TILE_BITS'(NUM_INSTANCES - 1 - INSTANCE_IDX)) >> LOG2_NUM_INSTANCES; - wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + W_ADDR_BITS'(INSTANCE_IDX * TILE_HEADER_SIZEW); + wire [`RASTER_TILE_BITS-1:0] start_tile_count = (dcrs.tile_count + (`RASTER_TILE_BITS'(NUM_INSTANCES - 1) - `RASTER_TILE_BITS'(instance_idx))) >> LOG2_NUM_INSTANCES; + wire [W_ADDR_BITS-1:0] start_tbuf_addr = {dcrs.tbuf_addr, 4'b0} + (W_ADDR_BITS'(instance_idx) * W_ADDR_BITS'(TILE_HEADER_SIZEW)); // The sorted-pid array is a single dense block that follows ALL bin headers // (num_bins == dcrs.tile_count of them, 3 words each), shared across the N @@ -345,7 +346,7 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `UNUSED_VAR (sched_rsp_tag) VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (NUM_REQS), .MEM_CHANNELS (RCACHE_NUM_REQS), .WORD_SIZE (`RASTER_DATA_BITS / 8), @@ -372,6 +373,8 @@ module VX_raster_mem import VX_gpu_pkg::*; import VX_raster_pkg::*; #( .core_req_data ('0), .core_req_tag (sched_req_tag), .core_req_ready (mem_req_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/raster/VX_raster_slice.sv b/hw/rtl/raster/VX_raster_slice.sv index ae1e0a243f..4f1f920660 100644 --- a/hw/rtl/raster/VX_raster_slice.sv +++ b/hw/rtl/raster/VX_raster_slice.sv @@ -79,7 +79,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_te #( - .INSTANCE_ID ($sformatf("%s-te", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-te", INSTANCE_ID))), .TILE_LOGSIZE (TILE_LOGSIZE), .BLOCK_LOGSIZE (BLOCK_LOGSIZE) ) tile_evaluator ( @@ -147,7 +147,7 @@ module VX_raster_slice import VX_raster_pkg::*; #( `endif VX_raster_be #( - .INSTANCE_ID ($sformatf("%s-be", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-be", INSTANCE_ID))), .BLOCK_LOGSIZE (BLOCK_LOGSIZE), .OUTPUT_QUADS (OUTPUT_QUADS), .QUAD_FIFO_DEPTH (QUAD_FIFO_DEPTH) diff --git a/hw/rtl/rtu/VX_rtu_box_pe.sv b/hw/rtl/rtu/VX_rtu_box_pe.sv index 446e42a1e6..b8638e656b 100644 --- a/hw/rtl/rtu/VX_rtu_box_pe.sv +++ b/hw/rtl/rtu/VX_rtu_box_pe.sv @@ -62,7 +62,10 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // it instead of doing so on the cycle the result lands output wire [TAG_WIDTH-1:0] tag_out_pre, output wire hit, - output wire [31:0] t_near + output wire [31:0] t_near, + // the next result's t_near, one cycle ahead (meaningful when that result + // is valid) + output wire [31:0] t_near_pre ); localparam F = LATENCY_FMA; localparam LAT_FMA = 3 * F; // origin - ro, + corner, * inv_d @@ -318,12 +321,13 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // ── stage 7: hit = lo <= hi; t_near = lo ───────────────────────── // lo and hi are never NaN; t_near is non-negative for t_min >= 0 and +0 for // a zero, so the consumer may order it as an unsigned integer. + wire [31:0] t_near_w = (lo_all_r[30:0] == 31'd0) ? 32'd0 : lo_all_r; reg hit_r; reg [31:0] t_near_r; always_ff @(posedge clk) begin if (enable) begin hit_r <= f32_le(lo_all_r, hi_all_r); - t_near_r <= (lo_all_r[30:0] == 31'd0) ? 32'd0 : lo_all_r; + t_near_r <= t_near_w; end end @@ -368,5 +372,6 @@ module VX_rtu_box_pe import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign tag_out_pre = tag_out_pre_w; assign hit = hit_r; assign t_near = t_near_r; + assign t_near_pre = t_near_w; endmodule diff --git a/hw/rtl/rtu/VX_rtu_scheduler.sv b/hw/rtl/rtu/VX_rtu_scheduler.sv index dab898a532..62f0af6453 100644 --- a/hw/rtl/rtu/VX_rtu_scheduler.sv +++ b/hw/rtl/rtu/VX_rtu_scheduler.sv @@ -322,14 +322,22 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [CTX_TAG_W-1:0] sel_q; reg fresh_q; reg rewalk_q; - // candidate key order, precomputed at ALIGN off the store row: the key of - // the record under test ({instance, record offset}) against the floor and - // the staged candidate, so EXEC only adds the t compares - reg key_gt_floor_q; - reg key_lt_yld_q; + // EXEC's compares, precomputed at ALIGN off the sources its operands are + // captured from. Only states a fresh or re-walk pass never enters read + // them, so word_x is word_q there. *_before_q leaves out yld_q, which + // EXEC reads live. + reg t0_lt_best_q; // procedural t below the committed hit + reg t0_above_q; // ... above the resume floor + reg t0_before_q; // ... ahead of the staged candidate + reg tri_above_q; + reg tri_before_q; + reg tri_occl_q; // a hit at trit occludes the staged candidate + reg tri_more_q; // a further triangle in the loop + reg inst_last_q; // the last instance of the leaf + reg [31:0] inst_next_off_q; // the next instance's record offset ctx_state_t word_q; lane_ray_t ray_q; - reg [BUF_BITS-1:0] fbuf_q; + reg [BUF_BITS-1:0] f_aligned_q; // fetched lines, byte-aligned to the structure reg [STK_ENTW-1:0] stacktop_q; reg [PATHW-1:0] pathv_q; // rank path entry below the current node reg [ADDRW-1:0] structaddr_q; @@ -526,6 +534,24 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( reg [COLL_SIZE-1:0] coll_prochit; wire [31:0] cand_ki_al = cs_word.in_blas ? cs_word.inst_id : 32'd0; + wire key_gt_floor_al = {cand_ki_al, cs_word.cur_off} > {cs_word.floor_ki, cs_word.floor_ko}; + wire key_lt_yld_al = {cand_ki_al, cs_word.cur_off} < {cs_word.yld_ki, cs_word.yld_ko}; + wire [31:0] coll_t0_al = coll_ordt[cs_word.coll_id][0]; + wire [31:0] trit_al = trires_rdata[95:64]; + wire [31:0] inst_next_al = cs_word.inst_idx + 32'd1; + function automatic logic above_floor_al(input logic [31:0] t); + above_floor_al = !cs_word.has_floor + || (t > cs_word.floor_t) + || ((t == cs_word.floor_t) && key_gt_floor_al); + endfunction + function automatic logic before_yld_al(input logic [31:0] t); + before_yld_al = (t < cs_word.yld_t) + || ((t == cs_word.yld_t) && key_lt_yld_al); + endfunction + // the structure's byte offset in its line: the low bits of structaddr_q, + // which only the low bits of its two addends reach + wire [RTU_LINE_SEL_BITS-1:0] f_off_al = slot_scene[s1_slot][RTU_LINE_SEL_BITS-1:0] + + ((s1_fresh || s1_rewalk) ? RTU_LINE_SEL_BITS'(0) : cs_word.cur_off[RTU_LINE_SEL_BITS-1:0]); // ═══════════════════════ stage advance ════════════════════════════ always_ff @(posedge clk) begin @@ -547,11 +573,18 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( sel_q <= s1_sel; fresh_q <= s1_fresh; rewalk_q <= s1_rewalk; - key_gt_floor_q <= {cand_ki_al, cs_word.cur_off} > {cs_word.floor_ki, cs_word.floor_ko}; - key_lt_yld_q <= {cand_ki_al, cs_word.cur_off} < {cs_word.yld_ki, cs_word.yld_ko}; + t0_lt_best_q <= (coll_t0_al < cs_word.best_t); + t0_above_q <= above_floor_al(coll_t0_al); + t0_before_q <= before_yld_al(coll_t0_al); + tri_above_q <= above_floor_al(trit_al); + tri_before_q <= before_yld_al(trit_al); + tri_occl_q <= (cs_word.yld_t >= trit_al); + tri_more_q <= ((cs_word.tri_i + 32'd1) < cs_word.tri_n); + inst_last_q <= (inst_next_al == cs_word.inst_cnt); + inst_next_off_q <= cs_word.inst_base + (inst_next_al * 32'(RTU_INST_STRIDE)); word_q <= cs_word; ray_q <= lane_ray_t'(ray_rdata); - fbuf_q <= fbuf; + f_aligned_q <= fbuf >> {f_off_al, 3'b000}; stacktop_q <= stk_rdata; pathv_q <= path_rdata; // a fresh context's store row is stale and a re-walk restarts: @@ -572,9 +605,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end // ═══════════════════════ EXEC decode ══════════════════════════════ - wire [RTU_LINE_SEL_BITS-1:0] f_off = structaddr_q[RTU_LINE_SEL_BITS-1:0]; - wire [RTU_LINE_SEL_BITS+2:0] f_shift = {f_off, 3'b000}; - wire [BUF_BITS-1:0] f_aligned = fbuf_q >> f_shift; + wire [RTU_LINE_SEL_BITS-1:0] f_off = structaddr_q[RTU_LINE_SEL_BITS-1:0]; + wire [BUF_BITS-1:0] f_aligned = f_aligned_q; wire [31:0] f_off32 = 32'(f_off); wire [LB-1:0] tri_rec_lines = @@ -702,6 +734,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wire box_feed, box_feed_raw; wire box_valid_out, box_hit; wire [31:0] box_t_near; + wire [31:0] box_t_near_pre; wire [COLL_IDW+32-1:0] box_tag_out; wire [COLL_IDW+32-1:0] box_tag_pre; wire [COLL_IDW-1:0] box_coll = box_tag_out[COLL_IDW+32-1 : 32]; @@ -733,12 +766,14 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .tag_out (box_tag_out), .tag_out_pre (box_tag_pre), .hit (box_hit), - .t_near (box_t_near) + .t_near (box_t_near), + .t_near_pre (box_t_near_pre) ); end else begin : g_no_box_pe assign box_valid_out = 1'b0; assign box_hit = 1'b0; assign box_t_near = '0; + assign box_t_near_pre = '0; assign box_tag_out = '0; assign box_tag_pre = '0; `UNUSED_VAR ({box_feed, box_feed_raw, feed_ci, walk_inv_d}) @@ -746,10 +781,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( `UNUSED_VAR ({node, node_kind, node_lines, leaf_lines, stacktop_q, pathv_q}) end - // Row select for the insertion read. It depends only on the tag, which the - // box PE delivers a cycle ahead of its result, so the decode is registered - // here rather than repeated on the result cycle -- where it fans out across - // every row and lands on the path that ends at the ordering registers. + // Row decode for the collector write, registered off the early tag. wire [COLL_IDW-1:0] box_coll_pre = box_tag_pre[COLL_IDW+32-1 : 32]; `UNUSED_VAR (box_tag_pre[31:0]) reg [COLL_SIZE-1:0] box_coll_hot; @@ -761,43 +793,59 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end - wire [COLL_SIZE-1:0][NODE_W*32+RTU_CHILD_BITS-1:0] coll_row_in; - for (genvar c = 0; c < COLL_SIZE; ++c) begin : g_coll_row_in - assign coll_row_in[c] = {coll_ordt[c], coll_ordcnt[c]}; - end + wire coll_pushable = box_hit && (box_childoff != 32'd0); - wire [NODE_W-1:0][31:0] coll_ordt_sel; - wire [RTU_CHILD_BITS-1:0] coll_ordcnt_sel; - VX_onehot_mux #( - .DATAW (NODE_W * 32 + RTU_CHILD_BITS), - .N (COLL_SIZE) - ) coll_row_mux ( - .data_in (coll_row_in), - .sel_in (box_coll_hot), - .data_out ({coll_ordt_sel, coll_ordcnt_sel}) - ); + // Insertion t-compares, taken a cycle ahead off the PE's early row and + // t_near: each entry of that row, and the result landing now, against the + // next result's t. If the result landing now inserts into the same row, the + // registered select picks the compare of the entry each slot shifts to. + wire [NODE_W-1:0] ins_t_le; + if (RTU_BVH_WIDTH != 0) begin : g_ins_pre + reg [NODE_W-1:0] pre_le_q; + reg new_le_q; + reg [NODE_W-1:0] take_new_q; + reg [NODE_W-1:0] take_prev_q; + + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_le_sel + localparam PREV = (oc != 0) ? (oc - 1) : 0; + assign ins_t_le[oc] = take_new_q[oc] ? new_le_q + : take_prev_q[oc] ? pre_le_q[PREV] + : pre_le_q[oc]; + end - // Insertion decisions off the admit thermometer: the collected list is - // sorted ascending and the count mask is a prefix, so the admit bits are a - // prefix of ones and the insertion slot is the first zero. Neighbouring-bit - // decode -- no popcount and no index compare on the ordering-write path. - wire [NODE_W-1:0] ins_le, ins_here, ins_shift; - if (RTU_BVH_WIDTH != 0) begin : g_ins_slots - for (genvar oc = 0; oc < RTU_BVH_WIDTH; ++oc) begin : g_le - assign ins_le[oc] = (RTU_CHILD_BITS'(oc) < coll_ordcnt_sel) - && (coll_ordt_sel[oc] <= box_t_near); + wire [NODE_W-1:0][31:0] pre_ordt = coll_ordt[box_coll_pre]; + wire [RTU_CHILD_BITS-1:0] pre_ordcnt = coll_ordcnt[box_coll_pre]; + wire pre_fwd = box_valid_out && (box_coll == box_coll_pre); + wire pre_proc = coll_proc[box_coll_pre]; + wire fwd_proc = pre_fwd && pre_proc; + wire fwd_ins = pre_fwd && ~pre_proc && coll_pushable; + + // this cycle's insertion into the forwarded row + wire [NODE_W-1:0] cur_le; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_cur_le + assign cur_le[oc] = (RTU_CHILD_BITS'(oc) < pre_ordcnt) && ins_t_le[oc]; end - assign ins_here[0] = ~ins_le[0]; - assign ins_shift[0] = 1'b0; - for (genvar oc = 1; oc < RTU_BVH_WIDTH; ++oc) begin : g_slot - assign ins_here[oc] = ~ins_le[oc] && ins_le[oc-1]; - assign ins_shift[oc] = ~ins_le[oc-1]; + + wire [NODE_W-1:0] pre_le, take_new, take_prev; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_pre_le + localparam PREV = (oc != 0) ? (oc - 1) : 0; + wire cur_here = ~cur_le[oc] && ((oc == 0) || cur_le[PREV]); + wire cur_shift = (oc != 0) && ~cur_le[PREV]; + assign take_new[oc] = ((oc == 0) && fwd_proc) || (fwd_ins && cur_here); + assign take_prev[oc] = fwd_ins && cur_shift; + assign pre_le[oc] = (pre_ordt[oc] <= box_t_near_pre); + end + + always @(posedge clk) begin + pre_le_q <= pre_le; + new_le_q <= (box_t_near <= box_t_near_pre); + take_new_q <= take_new; + take_prev_q <= take_prev; end - end else begin : g_no_ins_slots - `UNUSED_VAR ({coll_ordt_sel, coll_ordcnt_sel}) - assign {ins_le, ins_here, ins_shift} = '0; + end else begin : g_no_ins_pre + assign ins_t_le = '0; + `UNUSED_VAR ({box_t_near_pre, box_coll_pre}) end - wire coll_pushable = box_hit && (box_childoff != 32'd0); // ── tri PE ──────────────────────────────────────────────────────── wire tri_feed; @@ -1259,16 +1307,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // Candidate order: ascending (t, key), key = {instance id, record offset}. // A candidate is staged only above the floor and ahead of the staged one. wire [31:0] cand_ki = word_q.in_blas ? word_q.inst_id : 32'd0; - function automatic logic above_floor(input logic [31:0] t); - above_floor = !word_q.has_floor - || (t > word_q.floor_t) - || ((t == word_q.floor_t) && key_gt_floor_q); - endfunction - function automatic logic before_yld(input logic [31:0] t); - before_yld = !yld_q[sel_q] - || (t < word_q.yld_t) - || ((t == word_q.yld_t) && key_lt_yld_q); - endfunction + wire yld_sel = yld_q[sel_q]; // EXEC outcome (combinational) ctx_state_t word_n; @@ -1573,9 +1612,9 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( CS_PROC_WAIT: begin // woken by the collector: the raw AABB result landed if (coll_hit_q - && (coll_t0_q < word_x.best_t) - && above_floor(coll_t0_q) - && before_yld(coll_t0_q)) begin + && t0_lt_best_q + && t0_above_q + && (!yld_sel || t0_before_q)) begin cf_din_r.kind = CK_YLDP; cf_din_r.t = coll_t0_q; cf_din_r.prim = word_x.prim_base; @@ -1662,13 +1701,13 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( exec_hit_set = 1'b1; word_n.best_t = trit_q; // a closer opaque hit occludes a farther candidate - if (yld_q[sel_q] && (word_x.yld_t >= trit_q)) begin + if (yld_sel && tri_occl_q) begin exec_yld_clr = 1'b1; end if (term_first) begin word_n.cstate = CS_DONE; exec_done = 1'b1; - end else if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + end else if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1679,8 +1718,8 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end end else if (tri_pass - && above_floor(trit_q) - && before_yld(trit_q)) begin + && tri_above_q + && (!yld_sel || tri_before_q)) begin cf_din_r.kind = CK_YLDA; cf_din_r.t = trit_q; cf_din_r.u = triu_q; @@ -1702,7 +1741,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( word_n.yld_t = trit_q; word_n.yld_ki = cand_ki; word_n.yld_ko = word_x.cur_off; - if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1712,7 +1751,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( wake_self = 1'b1; end end else begin - if ((word_x.tri_i + 32'd1) < word_x.tri_n) begin + if (tri_more_q) begin word_n.tri_i = word_x.tri_i + 32'd1; word_n.cur_off = word_x.cur_off + 32'(RTU_TRI_STRIDE); word_n.cstate = CS_LTRI_REQ0; @@ -1851,7 +1890,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // path ends at the instance it resumed. word_n.in_blas = 1'b0; word_n.follow = 1'b0; - if ((word_x.inst_idx + 32'd1) == word_x.inst_cnt) begin + if (inst_last_q) begin if (FLAT) begin word_n.cstate = CS_DONE; exec_done = 1'b1; @@ -1861,8 +1900,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end else begin word_n.inst_idx = word_x.inst_idx + 32'd1; - word_n.cur_off = word_x.inst_base - + ((word_x.inst_idx + 32'd1) * 32'(RTU_INST_STRIDE)); + word_n.cur_off = inst_next_off_q; word_n.cstate = CS_INST_REQ; wake_self = 1'b1; end @@ -1970,8 +2008,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( done_q <= '0; mask_q <= '0; hit_q <= '0; - yld_q <= '0; - objv_q <= '0; attr_q <= '0; running <= '0; finalised <= '0; @@ -1994,7 +2030,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( mask_q[32'(slot_start_slot)*NUM_LANES + k] <= slot_start_mask[k]; done_q[32'(slot_start_slot)*NUM_LANES + k] <= ~slot_start_mask[k]; hit_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; - yld_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; attr_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; live_q[32'(slot_start_slot)*NUM_LANES + k] <= slot_start_mask[k]; seed_v_q[32'(slot_start_slot)*NUM_LANES + k] <= 1'b0; @@ -2019,14 +2054,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( if (exec_hit_set) begin hit_q[sel_q] <= 1'b1; end - if (exec_yld_set) begin - yld_q[sel_q] <= 1'b1; - cbtype_q[sel_q] <= exec_cbtype; - objv_q[sel_q] <= exec_objv; - end - if (exec_yld_clr) begin - yld_q[sel_q] <= 1'b0; - end lvl_q_arr[sel_q] <= word_n.lvl; if (fresh_q || rewalk_q || sp_clr) begin sp_q_arr[sel_q] <= '0; @@ -2064,7 +2091,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( attr_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; seed_v_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; end - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; live_q[32'(bw_slot)*NUM_LANES + k] <= res_rewalk[k]; if (res_rewalk[k]) begin done_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; @@ -2076,18 +2102,6 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( finalised[bw_slot] <= 1'b0; end end else begin - for (k = 0; k < NUM_LANES; k = k + 1) begin - if (bw_copy_mask[k]) begin - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; - cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_CHS); - objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; - end - if (bw_miss_mask[k]) begin - yld_q[32'(bw_slot)*NUM_LANES + k] <= 1'b1; - cbtype_q[32'(bw_slot)*NUM_LANES + k] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_MISS); - objv_q[32'(bw_slot)*NUM_LANES + k] <= 1'b0; - end - end finalised[bw_slot] <= 1'b1; end end @@ -2103,6 +2117,49 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( end end + // Yield flags, per context: the barrier walker's per-lane writes decode to + // each context instead of chaining behind the EXEC write. + for (genvar i = 0; i < NUM_CTX; ++i) begin : g_yld_upd + localparam SLOT = i / NUM_LANES; + localparam LANE = i % NUM_LANES; + wire start_i = slot_start_valid && (slot_start_slot == SLOT_W'(SLOT)); + wire exec_i = x_valid && (sel_q == CTX_TAG_W'(i)); + wire bw_i = bw_job_done && (bw_slot == SLOT_W'(SLOT)); + wire res_i = bw_i && bw_is_res; + wire chs_i = bw_i && ~bw_is_res && bw_copy_mask[LANE]; + wire miss_i = bw_i && ~bw_is_res && bw_miss_mask[LANE]; + always_ff @(posedge clk) begin + if (reset) begin + yld_q[i] <= 1'b0; + objv_q[i] <= 1'b0; + end else begin + if (res_i) begin + yld_q[i] <= 1'b0; + end else if (chs_i || miss_i) begin + yld_q[i] <= 1'b1; + end else if (exec_i && exec_yld_clr) begin + yld_q[i] <= 1'b0; + end else if (exec_i && exec_yld_set) begin + yld_q[i] <= 1'b1; + end else if (start_i) begin + yld_q[i] <= 1'b0; + end + if (miss_i) begin + cbtype_q[i] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_MISS); + end else if (chs_i) begin + cbtype_q[i] <= RTU_CB_TYPE_BITS'(`VX_RT_CB_TYPE_CHS); + end else if (exec_i && exec_yld_set) begin + cbtype_q[i] <= exec_cbtype; + end + if (chs_i || miss_i) begin + objv_q[i] <= 1'b0; + end else if (exec_i && exec_yld_set) begin + objv_q[i] <= exec_objv; + end + end + end + end + // a resume's committed t seeds the lane's re-walk always_ff @(posedge clk) begin if (bw_wr_gnt && (bw_state == BW_WR) && bw_is_res && (bw_field == 3'd0)) begin @@ -2127,37 +2184,76 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( // ── collector update ────────────────────────────────────────────── always_ff @(posedge clk) begin - if (reset) begin - coll_busy <= '0; - end else begin - if (x_valid && coll_alloc_r) begin - coll_busy[coll_alloc_idx] <= 1'b1; - coll_proc[coll_alloc_idx] <= box_raw_r; - coll_ctx[coll_alloc_idx] <= sel_q; - coll_cnt[coll_alloc_idx] <= '0; - coll_last[coll_alloc_idx] <= box_raw_r ? RTU_CHILD_BITS'(0) : last_child; - coll_ordcnt[coll_alloc_idx] <= '0; - end - if (x_valid && coll_free_r) begin - coll_busy[word_x.coll_id] <= 1'b0; + if (~reset && x_valid && coll_alloc_r) begin + coll_ctx[coll_alloc_idx] <= sel_q; + coll_last[coll_alloc_idx] <= box_raw_r ? RTU_CHILD_BITS'(0) : last_child; + end + end + + // Each row decodes its own write enables and insertion slot, so the + // ordering write is one mux per entry. The admit bits are a prefix of ones + // (the list is sorted ascending and the count mask is a prefix), so the + // insertion slot is the first zero: a neighbouring-bit decode. + for (genvar c = 0; c < COLL_SIZE; ++c) begin : g_coll_upd + wire alloc_c = x_valid && coll_alloc_r && (coll_alloc_idx == COLL_IDW'(c)); + wire free_c = x_valid && coll_free_r && (word_x.coll_id == COLL_IDW'(c)); + wire box_c = box_valid_out && box_coll_hot[c]; + wire proc_c = box_c && coll_proc[c]; + wire tree_c = box_c && ~coll_proc[c]; + wire ins_c = tree_c && coll_pushable; + + always_ff @(posedge clk) begin + if (reset) begin + coll_busy[c] <= 1'b0; + end else begin + if (free_c) begin + coll_busy[c] <= 1'b0; + end else if (alloc_c) begin + coll_busy[c] <= 1'b1; + end + if (alloc_c) begin + coll_proc[c] <= box_raw_r; + end + if (tree_c) begin + coll_cnt[c] <= coll_cnt[c] + RTU_CHILD_BITS'(1); + end else if (alloc_c) begin + coll_cnt[c] <= '0; + end + if (ins_c) begin + coll_ordcnt[c] <= coll_ordcnt[c] + RTU_CHILD_BITS'(1); + end else if (alloc_c) begin + coll_ordcnt[c] <= '0; + end + if (proc_c) begin + coll_prochit[c] <= box_hit; + end end - if (box_valid_out) begin - if (coll_proc[box_coll]) begin - coll_prochit[box_coll] <= box_hit; - coll_ordt[box_coll][0] <= box_t_near; - end else begin - coll_cnt[box_coll] <= coll_cnt[box_coll] + RTU_CHILD_BITS'(1); - if (coll_pushable) begin - for (integer oc = 0; oc < RTU_BVH_WIDTH; oc = oc + 1) begin - if (ins_here[oc]) begin - coll_ordoff[box_coll][oc[IDXW-1:0]] <= box_childoff; - coll_ordt[box_coll][oc[IDXW-1:0]] <= box_t_near; - end else if (ins_shift[oc]) begin - coll_ordoff[box_coll][oc[IDXW-1:0]] <= coll_ordoff[box_coll][oc[IDXW-1:0]-1]; - coll_ordt[box_coll][oc[IDXW-1:0]] <= coll_ordt[box_coll][oc[IDXW-1:0]-1]; - end - end - coll_ordcnt[box_coll] <= coll_ordcnt[box_coll] + RTU_CHILD_BITS'(1); + end + + wire [NODE_W-1:0] ins_le; + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_le + assign ins_le[oc] = (RTU_CHILD_BITS'(oc) < coll_ordcnt[c]) && ins_t_le[oc]; + end + + // a procedural box stores its t in slot 0 + for (genvar oc = 0; oc < NODE_W; ++oc) begin : g_slot + localparam PREV = (oc != 0) ? (oc - 1) : 0; + wire ins_here = ~ins_le[oc] && ((oc == 0) || ins_le[PREV]); + wire ins_shift = (oc != 0) && ~ins_le[PREV]; + wire set_t = ((oc == 0) && proc_c) || (ins_c && ins_here); + wire set_off = ins_c && ins_here; + wire shift_c = ins_c && ins_shift; + always_ff @(posedge clk) begin + if (~reset) begin + if (set_t) begin + coll_ordt[c][oc] <= box_t_near; + end else if (shift_c) begin + coll_ordt[c][oc] <= coll_ordt[c][PREV]; + end + if (set_off) begin + coll_ordoff[c][oc] <= box_childoff; + end else if (shift_c) begin + coll_ordoff[c][oc] <= coll_ordoff[c][PREV]; end end end @@ -2290,7 +2386,7 @@ module VX_rtu_scheduler import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( assign obj_vld = objv_q; assign attr_vld = attr_q; - `UNUSED_VAR ({f_aligned, s1_fresh, ray_q, ins_le, ins_here, ins_shift}) + `UNUSED_VAR ({f_aligned, s1_fresh, ray_q}) `ifdef DBG_TRACE_RTU always_ff @(posedge clk) begin diff --git a/hw/rtl/rtu/VX_rtu_unit.sv b/hw/rtl/rtu/VX_rtu_unit.sv index cfeb22a315..ab93aceecf 100644 --- a/hw/rtl/rtu/VX_rtu_unit.sv +++ b/hw/rtl/rtu/VX_rtu_unit.sv @@ -38,13 +38,14 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS, parameter RTU_TAG_WIDTH = 1 ) ( input wire clk, input wire reset, + input wire [SFU_BLOCK_W-1:0] block_idx, + // SFU PE-style request/response interfaces VX_execute_if.slave execute_if, VX_result_if.master result_if, @@ -56,9 +57,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( VX_sched_unlock_if.master sched_unlock_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) - localparam RAM_SIZE = `VX_CFG_NUM_WARPS * RTUW_SLOT_COUNT; + // This SFU block serves the warps with wid mod NUM_SFU_BLOCKS == block_idx; + // its per-warp state is indexed by the block-local warp id. + localparam NLW = SFU_BLOCK_WARPS; + localparam RAM_SIZE = NLW * RTUW_SLOT_COUNT; + localparam RAM_ADDRW = `LOG2UP(RAM_SIZE); localparam RAM_DATAW = 32 * NUM_LANES; // The address concatenates {warp, slot}; the slot stride must be a power of @@ -80,6 +84,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire [2:0] uop = execute_if.data.op_args.rtuw.uop; wire [RTUW_SLOT_BITS-1:0] slot = execute_if.data.op_args.rtuw.slot[RTUW_SLOT_BITS-1:0]; wire [NW_WIDTH-1:0] wid = execute_if.data.header.wid; + wire [SFU_BLOCK_NW_W-1:0] lid = wid_to_sfu_local(wid); // Register the outgoing RTU bus at this module boundary so the core->socket // seam launches at a flop (see VX_rtu_bus_slice). We drive the internal @@ -107,7 +112,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // Written by the RTU, read by the shader. Neither port is ever contended, so // the RTU is never made to wait for a write and a read never waits for it. wire win_v = rtu_bus_w.win_valid; - wire [NW_WIDTH-1:0] win_wid = rtu_bus_w.win_data.wid; + wire [SFU_BLOCK_NW_W-1:0] win_lid = wid_to_sfu_local(rtu_bus_w.win_data.wid); wire [RTUW_SLOT_BITS-1:0] win_slot = rtu_bus_w.win_data.slot; `UNUSED_VAR (rtu_bus_w.win_data.tag) @@ -115,7 +120,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( wire core_rden; wire [RTUW_SLOT_BITS-1:0] core_slot; - wire [NW_WIDTH-1:0] core_wid; + wire [SFU_BLOCK_NW_W-1:0] core_lid; wire [NUM_LANES-1:0][31:0] core_rdata; VX_dp_ram #( @@ -130,9 +135,9 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( .read (core_rden), .write (win_v), .wren (rtu_bus_w.win_data.mask), - .waddr ({win_wid, win_slot}), + .waddr (RAM_ADDRW'({win_lid, win_slot})), .wdata (rtu_bus_w.win_data.data), - .raddr ({core_wid, core_slot}), + .raddr (RAM_ADDRW'({core_lid, core_slot})), .rdata (core_rdata) ); @@ -147,7 +152,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // longer one of them — see the parked-WAIT table below. wire is_read = (op == RTUW_OP_GETWF) || (op == RTUW_OP_GETW); - reg [`VX_CFG_NUM_WARPS-1:0] response_ready, trace_open; + reg [NLW-1:0] response_ready, trace_open; // ── the parked WAIT — a long-latency op MUST NOT hold the execute stage ──── // A WAIT used to sit at execute_if until its record landed, and execute_if is @@ -165,8 +170,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // WAIT and the unlock below is what releases it — so the semantics are unchanged; // only the execute port is freed. A warp holds at most one trace, so one entry // per warp is exact, not a heuristic. - reg [`VX_CFG_NUM_WARPS-1:0] wait_pend; - sfu_header_t wait_hdr [`VX_CFG_NUM_WARPS]; + reg [NLW-1:0] wait_pend; + sfu_header_t wait_hdr [NLW]; // ── result stage (the RAM read is synchronous) ───────────────────────── // One op is accepted per cycle into s1; its word arrives from the RAM on the @@ -227,7 +232,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // // A CONTINUE with no open candidate retires as a no-op (defensive — correct // kernels only issue one inside the yield loop). - wire cont_want = is_cont && trace_open[wid]; + wire cont_want = is_cont && trace_open[lid]; wire is_stream = is_ray || cont_want; wire is_2beat = cont_want || (is_trace && (uop == RTUW_UOP_ARM)); @@ -281,20 +286,22 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // the two arbitrate for the result stage. The parked WAIT wins: it has already // retired from execute_if and the warp behind it is stalled on nothing else, // while a GETW at execute_if can simply wait a cycle. - wire [`VX_CFG_NUM_WARPS-1:0] wait_done = wait_pend & response_ready; + wire [NLW-1:0] wait_done = wait_pend & response_ready; - wire [NW_WIDTH-1:0] wake_wid; - wire wake_valid; - wire [`VX_CFG_NUM_WARPS-1:0] wake_1h; + wire [`LOG2UP(NLW)-1:0] wake_index; + wire wake_valid; + wire [NLW-1:0] wake_1h; VX_priority_encoder #( - .N (`VX_CFG_NUM_WARPS) + .N (NLW) ) wake_picker ( .data_in (wait_done), .onehot_out (wake_1h), - .index_out (wake_wid), + .index_out (wake_index), .valid_out (wake_valid) ); `UNUSED_VAR (wake_1h) + wire [SFU_BLOCK_NW_W-1:0] wake_lid = SFU_BLOCK_NW_W'(wake_index); + wire [NW_WIDTH-1:0] wake_wid = sfu_local_to_wid(wake_lid, block_idx); assign wake_fire = wake_valid && s1_ready; @@ -302,12 +309,12 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // A stream op retires with its last beat; everything else needs only the // result stage. A WAIT retires the moment it is parked — it never blocks. wire read_fire = execute_if.valid && is_read && s1_grant; - wire wait_fire = execute_if.valid && is_wait && ~wait_pend[wid]; + wire wait_fire = execute_if.valid && is_wait && ~wait_pend[lid]; wire stream_fire = rb_fire && rb_end; - wire cont_nop = execute_if.valid && is_cont && ~trace_open[wid] && s1_grant; + wire cont_nop = execute_if.valid && is_cont && ~trace_open[lid] && s1_grant; wire op_fire = read_fire || wait_fire || stream_fire || cont_nop || arm_fire; - `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[wid]), + `RUNTIME_ASSERT(~(execute_if.valid && is_wait && wait_pend[lid]), ("%t: *** %s: wid=%0d issued a second WAIT with one already parked", $time, INSTANCE_ID, wid)) @@ -331,7 +338,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // read reads the slot its op names. assign core_rden = read_fire || wake_fire; assign core_slot = wake_fire ? RTUW_SLOT_BITS'(`VX_RT_STATUS) : slot; - assign core_wid = wake_fire ? wake_wid : wid; + assign core_lid = wake_fire ? wake_lid : lid; // ── sequential state ─────────────────────────────────────────────────── always @(posedge clk) begin @@ -350,7 +357,7 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( if (wake_fire) begin s1_valid <= 1'b1; s1_from_ram <= 1'b1; - s1_header <= wait_hdr[wake_wid]; + s1_header <= wait_hdr[wake_lid]; end else if (op_fire && ~wait_fire) begin s1_valid <= 1'b1; s1_from_ram <= is_read; @@ -359,8 +366,8 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // park the WAIT: it retires now, and answers later if (wait_fire) begin - wait_pend[wid] <= 1'b1; - wait_hdr[wid] <= execute_if.data.header; + wait_pend[lid] <= 1'b1; + wait_hdr[lid] <= execute_if.data.header; end // The last beat retires the op, so the counter always lands back at 0. @@ -370,18 +377,18 @@ module VX_rtu_unit import VX_gpu_pkg::*, VX_rtu_pkg::*; #( // The waking WAIT consumes the record, so the next one waits again. if (wake_fire) begin - wait_pend[wake_wid] <= 1'b0; - response_ready[wake_wid] <= 1'b0; + wait_pend[wake_lid] <= 1'b0; + response_ready[wake_lid] <= 1'b0; end // A CONTINUE resolves the open candidate once its last beat is away. if (stream_fire && cont_want) begin - trace_open[wid] <= 1'b0; + trace_open[lid] <= 1'b0; end // Last, so a record landing in the same cycle as the op it unblocks // wins: the record is the newer event. if (status_wr) begin - response_ready[win_wid] <= 1'b1; - trace_open[win_wid] <= rtu_bus_w.win_data.is_cand; + response_ready[win_lid] <= 1'b1; + trace_open[win_lid] <= rtu_bus_w.win_data.is_cand; end end end diff --git a/hw/rtl/tcu/VX_tcu_abuf.sv b/hw/rtl/tcu/VX_tcu_abuf.sv index ceca424587..30efc1cc92 100644 --- a/hw/rtl/tcu/VX_tcu_abuf.sv +++ b/hw/rtl/tcu/VX_tcu_abuf.sv @@ -238,7 +238,6 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // and each request/response accumulates the row stride, keeping the // multiplier off the LMEM request-address and response-lane paths. localparam ROW_OFF_W = LDM_W + FETCH_CTR_W + 4; - wire [3:0] row_k_words = slot_is_sparse_r ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [3:0] row_k_words_alloc = req_is_sparse ? 4'(TCU_TC_K) : 4'(TCU_WG_FEDP_K); wire [ROW_OFF_W-1:0] row_word_off_init = @@ -357,8 +356,11 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Each LMEM response writes one bank-row at offset rsp_ctr_r. // ----------------------------------------------------------------------- - logic [A_STRIPE_WORDS*32-1:0] storage_wdata; - logic [A_STRIPE_WORDS-1:0] storage_wren; + wire [A_STRIPE_WORDS-1:0][31:0] storage_wdata; + wire [A_STRIPE_WORDS-1:0] storage_wren; + + wire storage_write = in_fetch && tcu_lmem_if.rsp_valid; + wire [BANK_ROW_WORDS-1:0][31:0] rsp_words = tcu_lmem_if.rsp_data.data; // Row-major per-response decoding: the row being written is rsp_ctr_r, // and its tcK source words start at lane = row_word_off & (BANK_ROW_WORDS-1) @@ -366,36 +368,36 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [BANK_ROW_WORDS_LOG2:0] row_lane_rsp = (BANK_ROW_WORDS_LOG2+1)'( row_word_off_rsp_r & ROW_OFF_W'(BANK_ROW_WORDS - 1)); - always_comb begin - storage_wdata = '0; - storage_wren = '0; - if (in_fetch && tcu_lmem_if.rsp_valid) begin - if (slot_row_major_r) begin - // Row-major: write k_words of row rsp_ctr_r into - // storage[rsp_ctr_r * k_words .. + k_words). Source words start - // at row_lane_rsp inside the LMEM response. - for (int k = 0; k < TCU_WG_FEDP_K; ++k) begin - automatic int dst = int'(rsp_ctr_r) * int'(row_k_words) + k; - automatic int src = int'(row_lane_rsp) + k; - if (k < int'(row_k_words) && dst < A_STRIPE_WORDS && src < BANK_ROW_WORDS) begin - storage_wren[dst] = 1'b1; - storage_wdata[dst * 32 +: 32] = - tcu_lmem_if.rsp_data.data[src * 32 +: 32]; - end - end - end else begin - // Block-major: one LMEM response carries BANK_ROW_WORDS - // 32-bit words (= NUM_BANKS * XLEN_RATIO). Store all of them - // starting at the current LMEM-row offset within the stripe. - for (int b = 0; b < BANK_ROW_WORDS; ++b) begin - if (int'(rsp_ctr_r) * BANK_ROW_WORDS + b < A_STRIPE_WORDS) begin - storage_wren[int'(rsp_ctr_r) * BANK_ROW_WORDS + b] = 1'b1; - storage_wdata[(int'(rsp_ctr_r) * BANK_ROW_WORDS + b) * 32 +: 32] = - tcu_lmem_if.rsp_data.data[b * 32 +: 32]; - end - end - end - end + // Source word k of the current row. A lane past the response is never + // written, so its index may wrap. + wire [TCU_WG_FEDP_K-1:0][31:0] row_src_words; + for (genvar k = 0; k < TCU_WG_FEDP_K; ++k) begin : g_row_src + wire [`UP(BANK_ROW_WORDS_LOG2)-1:0] src_idx = `UP(BANK_ROW_WORDS_LOG2)'(row_lane_rsp) + + `UP(BANK_ROW_WORDS_LOG2)'(k); + assign row_src_words[k] = rsp_words[src_idx]; + end + + // Stripe word w has one source per mode: response word w % BANK_ROW_WORDS + // (block-major) or row word w % k_words (row-major). The write data is + // therefore fixed wiring, and only the enables decode rsp_ctr_r. + for (genvar w = 0; w < A_STRIPE_WORDS; ++w) begin : g_storage_wr + localparam DN_ROW = w / TCU_WG_FEDP_K; + localparam DN_K = w % TCU_WG_FEDP_K; + localparam SP_ROW = w / TCU_TC_K; + localparam SP_K = w % TCU_TC_K; + localparam BM_ROW = w / BANK_ROW_WORDS; + localparam BM_B = w % BANK_ROW_WORDS; + + wire dn_wren = (int'(rsp_ctr_r) == DN_ROW) && (int'(row_lane_rsp) + DN_K < BANK_ROW_WORDS); + wire sp_wren = (int'(rsp_ctr_r) == SP_ROW) && (int'(row_lane_rsp) + SP_K < BANK_ROW_WORDS); + wire bm_wren = (int'(rsp_ctr_r) == BM_ROW); + + assign storage_wren[w] = storage_write + && (slot_row_major_r ? (slot_is_sparse_r ? sp_wren : dn_wren) + : bm_wren); + assign storage_wdata[w] = slot_row_major_r + ? (slot_is_sparse_r ? row_src_words[SP_K] : row_src_words[DN_K]) + : rsp_words[BM_B]; end wire [A_STRIPE_WORDS-1:0][31:0] storage_rdata; @@ -411,7 +413,7 @@ module VX_tcu_abuf import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .clk (clk), .reset (reset), .read (1'b1), - .write (in_fetch && tcu_lmem_if.rsp_valid), + .write (storage_write), .wren (storage_wren), .waddr (1'b0), .wdata (storage_wdata), diff --git a/hw/rtl/tcu/VX_tcu_agu.sv b/hw/rtl/tcu/VX_tcu_agu.sv index 6690cf17b1..79baf58dc8 100644 --- a/hw/rtl/tcu/VX_tcu_agu.sv +++ b/hw/rtl/tcu/VX_tcu_agu.sv @@ -15,187 +15,137 @@ `ifdef TCU_META_ENABLE -// Warp-level AGU for TCU_LD instructions. Each TCU block can hold one -// transaction, while the LSU request and metadata write ports are shared. +// A TCU block's TCU_LD unit: one warp-wide load in flight through an LSU +// scheduler client port, issued in NUM_TCU_LANES / NUM_LSU_LANES passes of +// consecutive words, like the LSU's own packets. Response fragments are +// written straight into the block's metadata RAM, lane by lane, so nothing is +// buffered here. The load cannot overwrite metadata an older MMA still needs: +// TCU_LD enters the block in order behind that MMA, which reads the RAM when +// it issues. module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS, parameter NUM_LANES = `VX_CFG_NUM_TCU_LANES ) ( - input wire clk, - input wire reset, + input wire clk, + input wire reset, - // Per-block dispatch observation. The unit feeds INST_TCU_LD - // instructions here (gated by op_type at the wrapper); other - // op_types are masked so the AGU never sees them on .valid. - input wire [BLOCK_SIZE-1:0] per_block_ld_valid, - input tcu_execute_t per_block_ld_data [BLOCK_SIZE], - output wire [BLOCK_SIZE-1:0] per_block_ld_ready, + input wire ld_valid, + input tcu_execute_t ld_data, + output wire ld_ready, - // Memory client connection (to VX_lsu_scheduler at VX_core). - VX_lsu_sched_if.master client_if, + VX_lsu_sched_if.master client_if, - // Meta SRAM write port (broadcast — same data lands in every - // block's VX_tcu_meta so subsequent wmma_sp reads see it - // regardless of which block the warp resides on). - output wire meta_wr_en, - output wire [NW_WIDTH-1:0] meta_wr_wid, - output wire [4:0] meta_wr_idx, - output wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] meta_wr_data, + output wire meta_wr_en, + output wire [NW_WIDTH-1:0] meta_wr_wid, + output wire [4:0] meta_wr_idx, + output wire [TCU_BLOCK_CAP-1:0] meta_wr_mask, + output wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data, - // Result_if hand for the originating block. The wrapper muxes this - // with tcu_core's normal result_if path. - output wire [BLOCK_SIZE-1:0] result_valid, - output tcu_result_t result_data [BLOCK_SIZE], - input wire [BLOCK_SIZE-1:0] result_ready + // releases the TCU_LD's metadata scoreboard bit + output wire result_valid, + output tcu_result_t result_data, + input wire result_ready ); `UNUSED_SPARAM (INSTANCE_ID) - localparam BLOCK_IDX_BITS = (BLOCK_SIZE > 1) ? $clog2(BLOCK_SIZE) : 1; - logic [BLOCK_SIZE-1:0] busy_r; - logic [BLOCK_SIZE-1:0] issued_r; - logic [BLOCK_SIZE-1:0] complete_r; - tcu_header_t header_r [BLOCK_SIZE]; - logic [4:0] slot_r [BLOCK_SIZE]; - logic [`VX_CFG_XLEN-1:0] addr_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0][(LSU_WORD_SIZE*8)-1:0] - rsp_data_r [BLOCK_SIZE]; - logic [NUM_LANES-1:0] rsp_lane_done_r [BLOCK_SIZE]; + localparam LSU_LANES = `VX_CFG_NUM_LSU_LANES; + localparam NUM_PASSES = NUM_LANES / LSU_LANES; + localparam PASS_BITS = `CLOG2(NUM_PASSES); - wire [BLOCK_SIZE-1:0] req_pending = busy_r & ~issued_r; - wire [BLOCK_IDX_BITS-1:0] req_block; - wire [BLOCK_SIZE-1:0] req_grant; - wire req_valid; + // An LSU word holds XLEN/32 metadata words; lane i reads word i. + localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; + localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) req_arb ( - .clk (clk), - .reset (reset), - .requests (req_pending), - .grant_index (req_block), - .grant_onehot (req_grant), - .grant_valid (req_valid), - .grant_ready (client_if.req_ready) - ); + `STATIC_ASSERT ((NUM_LANES % LSU_LANES) == 0, ("TCU_LD requires NUM_TCU_LANES to be a multiple of NUM_LSU_LANES")) + // each pass starts on an LSU word, so lane i keeps reading word i + `STATIC_ASSERT ((NUM_PASSES == 1) || ((LSU_LANES % LSU_W32_RATIO) == 0), ("TCU_LD passes must be LSU-word aligned")) + `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= PASS_BITS, ("LSU client tag cannot encode the TCU_LD pass")) - wire [BLOCK_SIZE-1:0] commit_pending = complete_r; - wire [BLOCK_IDX_BITS-1:0] commit_block; - wire [BLOCK_SIZE-1:0] commit_grant; - wire commit_valid; - wire commit_ready = result_ready[commit_block]; + reg busy_r, issued_r, done_r; + tcu_header_t header_r; + reg [4:0] slot_r; + reg [`VX_CFG_XLEN-1:0] addr_r; - VX_generic_arbiter #( - .NUM_REQS (BLOCK_SIZE), - .TYPE ("R") - ) commit_arb ( - .clk (clk), - .reset (reset), - .requests (commit_pending), - .grant_index (commit_block), - .grant_onehot (commit_grant), - .grant_valid (commit_valid), - .grant_ready (commit_ready) - ); + wire ld_fire = ld_valid && ld_ready; + wire req_fire = client_if.req_valid && client_if.req_ready; + wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; + wire result_fire = result_valid && result_ready; - wire [BLOCK_IDX_BITS-1:0] rsp_block = - client_if.rsp_data.tag[BLOCK_IDX_BITS-1:0]; - wire rsp_fire = client_if.rsp_valid && client_if.rsp_ready; - wire req_fire = req_valid && client_if.req_ready; - wire commit_fire = commit_valid && commit_ready; - `UNUSED_VAR (req_grant) - `UNUSED_VAR (commit_grant) + // the pass being requested, and whether it and the response are the last + wire [`UP(PASS_BITS)-1:0] req_pass; + wire [`UP(PASS_BITS)-1:0] rsp_pass; + wire req_last, rsp_last; always @(posedge clk) begin if (reset) begin - busy_r <= '0; - issued_r <= '0; - complete_r <= '0; - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - header_r[bi] <= '0; - slot_r[bi] <= '0; - addr_r[bi] <= '0; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; end else begin - for (int bi = 0; bi < BLOCK_SIZE; ++bi) begin - if (per_block_ld_valid[bi] && per_block_ld_ready[bi]) begin - busy_r[bi] <= 1'b1; - issued_r[bi] <= 1'b0; - complete_r[bi] <= 1'b0; - header_r[bi] <= per_block_ld_data[bi].header; - slot_r[bi] <= per_block_ld_data[bi].op_args.tcu.fmt_d; - addr_r[bi] <= per_block_ld_data[bi].rs1_data[0]; - rsp_data_r[bi] <= '0; - rsp_lane_done_r[bi] <= '0; - end + if (ld_fire) begin + busy_r <= 1'b1; end - if (req_fire) begin - issued_r[req_block] <= 1'b1; + if (req_fire && req_last) begin + issued_r <= 1'b1; end - if (rsp_fire) begin - for (int t = 0; t < NUM_LANES; ++t) begin - if (client_if.rsp_data.mask[t]) begin - rsp_data_r[rsp_block][t] <= client_if.rsp_data.data[t]; - rsp_lane_done_r[rsp_block][t] <= 1'b1; - end - end - if (client_if.rsp_data.eop - && ((rsp_lane_done_r[rsp_block] | client_if.rsp_data.mask) - == {NUM_LANES{1'b1}})) begin - complete_r[rsp_block] <= 1'b1; - end + if (rsp_fire && client_if.rsp_data.eop && rsp_last) begin + done_r <= 1'b1; end - if (commit_fire) begin - busy_r[commit_block] <= 1'b0; - issued_r[commit_block] <= 1'b0; - complete_r[commit_block] <= 1'b0; - rsp_lane_done_r[commit_block] <= '0; + if (result_fire) begin + busy_r <= 1'b0; + issued_r <= 1'b0; + done_r <= 1'b0; end end + if (ld_fire) begin + header_r <= ld_data.header; + slot_r <= ld_data.op_args.tcu.fmt_d; + addr_r <= ld_data.rs1_data[0]; + end else if (req_fire && (NUM_PASSES > 1)) begin + addr_r <= addr_r + `VX_CFG_XLEN'(LSU_LANES * 4); + end end - `UNUSED_VAR (client_if.rsp_data.sop) - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_ld_ready - assign per_block_ld_ready[bi] = ~busy_r[bi]; + if (NUM_PASSES > 1) begin : g_passes + reg [PASS_BITS-1:0] req_pass_r; + reg [PASS_BITS-1:0] rsp_cnt_r; + always @(posedge clk) begin + if (reset || ld_fire) begin + req_pass_r <= '0; + rsp_cnt_r <= '0; + end else begin + if (req_fire) begin + req_pass_r <= req_pass_r + PASS_BITS'(1); + end + if (rsp_fire && client_if.rsp_data.eop) begin + rsp_cnt_r <= rsp_cnt_r + PASS_BITS'(1); + end + end + end + assign req_pass = req_pass_r; + assign req_last = (req_pass_r == PASS_BITS'(NUM_PASSES-1)); + assign rsp_last = (rsp_cnt_r == PASS_BITS'(NUM_PASSES-1)); + // responses of different passes may return out of order + assign rsp_pass = client_if.rsp_data.tag[PASS_BITS-1:0]; + `UNUSED_VAR (client_if.rsp_data.tag[LSU_CLIENT_TAG_WIDTH-1:PASS_BITS]) + end else begin : g_no_passes + assign req_pass = '0; + assign req_last = 1'b1; + assign rsp_last = 1'b1; + assign rsp_pass = '0; + `UNUSED_VAR (client_if.rsp_data.tag) end + assign ld_ready = ~busy_r; + `UNUSED_VAR (ld_data) + // ----------------------------------------------------------------------- - // Memory request: address from rs1 (warp-broadcast). NUM_LANES mask - // all-ones — we fetch the metadata tile in one warp-level request. - // - // Per-lane offset matches the host's pack_metadata_wg layout - // (sgemm_tcu_wg_sp/main.cpp): the host writes bank `b` and word `c` - // at index `b * stride + c`. The AGU writes wr_data[T] into - // VX_tcu_meta at (bank=T%PWD, col_in_group=T/PWD), so lane T must - // load `meta_base[(T % PWD) * stride + (T / PWD)]`. + // Request: one pass loads LSU_LANES consecutive 32-bit words. // ----------------------------------------------------------------------- - wire [`VX_CFG_XLEN-1:0] base_addr = addr_r[req_block]; - `UNUSED_VAR (base_addr[`CLOG2(LSU_WORD_SIZE)-1:0]) + wire [LSU_ADDR_WIDTH-1:0] base_word_addr = LSU_ADDR_WIDTH'(addr_r >> `CLOG2(LSU_WORD_SIZE)); + `UNUSED_VAR (addr_r[`CLOG2(LSU_WORD_SIZE)-1:0]) - wire [LSU_ADDR_WIDTH-1:0] base_word_addr = - LSU_ADDR_WIDTH'(base_addr[`VX_CFG_XLEN-1:`CLOG2(LSU_WORD_SIZE)]); - - localparam PWD = TCU_META_PER_WARP_DEPTH; - // Per-thread metadata layout: lane T loads h_meta[slot*NT + T]. The - // host packs h_meta so the 32-bit word at offset (slot*NT + T) holds - // the metadata destined for SRAM cell (bank = T%PWD, col = T/PWD). - // One formula serves both WMMA-SP and WGMMA-SP. - - // LSU bus carries LSU_WORD_SIZE-byte chunks per lane (= XLEN/8 bytes). - // The metadata layout is contiguous 32-bit words, so when XLEN > 32 - // each LSU response carries XLEN/32 logical 32-bit words and we need - // to pick the right one. LSU_W32_RATIO = bytes/4, so XLEN=64 → 2. - localparam LSU_W32_RATIO = LSU_WORD_SIZE / 4; - localparam LG_W32_RATIO = (LSU_W32_RATIO > 1) ? $clog2(LSU_W32_RATIO) : 0; - - // is_addr_local route bit — LMEM range check. The metadata buffer is - // per-warp inside shared memory so LMEM detection is uniform across - // lanes; compute once from the base address. `ifdef VX_CFG_LMEM_ENABLE // Equality on the word-address bits above the window: the base is asserted // aligned to the capacity, so the prefix compare is exact and needs no end @@ -211,103 +161,70 @@ module VX_tcu_agu import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire base_is_local = 1'b0; `endif - // Per-lane sub-word selector: which 32-bit slice of the LSU response - // carries the desired metadata word. For XLEN=32 (LSU_W32_RATIO=1) - // every lane reads exactly one 32-bit word, so half_sel is always 0. - // For XLEN=64 (LSU_W32_RATIO=2) we round the 32-bit word offset down - // to an 8-byte LSU address and use the LSB to select low/high half. - logic [NUM_LANES-1:0][`UP(LG_W32_RATIO)-1:0] half_sel_w; - // For XLEN=32, LG_W32_RATIO=0 and half_sel_w is never read downstream; - // tag it unused so Verilator's UNUSEDSIGNAL doesn't fail the build. - if (LG_W32_RATIO == 0) begin : g_half_sel_unused - `UNUSED_VAR (half_sel_w) + mem_bus_attr_t req_attr; + always_comb begin + req_attr = '0; + req_attr.is_addr_local = base_is_local; end - lsu_req_data_t req_w; + lsu_req_data_t req_data; always_comb begin - req_w = '0; - req_w.rw = 1'b0; // load - req_w.mask = {NUM_LANES{1'b1}}; - half_sel_w = '0; - for (int i = 0; i < NUM_LANES; i++) begin - automatic mem_bus_attr_t a; - automatic logic [LSU_ADDR_WIDTH-1:0] word_off_32; - a = '0; - a.is_addr_local = base_is_local; - // base_addr is pre-advanced per TCU_LD slot by the kernel - // (slot s passes base + s*NT*sizeof(float)), so lane i reads word i - // relative to base_word_addr. - word_off_32 = LSU_ADDR_WIDTH'(i); - // word_off_32 is in 32-bit-word units. The LSU bus addresses - // LSU_WORD_SIZE-byte chunks (= XLEN/32 × 32-bit words), so we - // shift down to bus stride and keep the sub-word half for the - // response extraction below. - req_w.addr[i] = base_word_addr + (word_off_32 >> LG_W32_RATIO); - if (LG_W32_RATIO > 0) begin - half_sel_w[i] = `UP(LG_W32_RATIO)'(word_off_32[`UP(LG_W32_RATIO)-1:0]); - end - req_w.byteen[i] = {LSU_WORD_SIZE{1'b1}}; - req_w.data[i] = '0; - req_w.attr[i] = a; + req_data = '0; + req_data.rw = 1'b0; + req_data.mask = {LSU_LANES{1'b1}}; + req_data.tag = LSU_CLIENT_TAG_WIDTH'(req_pass); + for (int i = 0; i < LSU_LANES; ++i) begin + req_data.addr[i] = base_word_addr + (LSU_ADDR_WIDTH'(i) >> LG_W32_RATIO); + req_data.byteen[i] = {LSU_WORD_SIZE{1'b1}}; + req_data.attr[i] = req_attr; end - req_w.tag = '0; - req_w.tag[BLOCK_IDX_BITS-1:0] = req_block; end - `UNUSED_PARAM (PWD) - `STATIC_ASSERT (LSU_CLIENT_TAG_WIDTH >= BLOCK_IDX_BITS, ("LSU client tag cannot encode TCU block")) + + assign client_if.req_valid = busy_r && ~issued_r; + assign client_if.req_data = req_data; + assign client_if.rsp_ready = 1'b1; + `UNUSED_VAR (client_if.rsp_data.sop) + + `RUNTIME_ASSERT(~client_if.rsp_valid || (issued_r && ~done_r), + ("%t: *** %s unexpected TCU_LD response", $time, INSTANCE_ID)) + `ifdef VX_CFG_LSU_STACK_INTERLEAVE_ENABLE // Metadata is read as a linear warp-wide tile, which an interleaved // per-thread stack cannot provide. if (`VX_CFG_NUM_THREADS > 1) begin : g_meta_stack_check - wire [`VX_CFG_XLEN-1:0] meta_end_addr = base_addr + `VX_CFG_XLEN'(NUM_LANES * 4 - 1); - `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (base_addr >= STACK_WINDOW_TOP), - ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, base_addr)) + wire [`VX_CFG_XLEN-1:0] meta_end_addr = addr_r + `VX_CFG_XLEN'(LSU_LANES * 4 - 1); + `RUNTIME_ASSERT(~req_fire || (meta_end_addr < STACK_WINDOW_BOTTOM) || (addr_r >= STACK_WINDOW_TOP), + ("%t: *** %s TCU metadata inside the thread stack window: addr=0x%0h", $time, INSTANCE_ID, addr_r)) end `endif - assign client_if.req_valid = req_valid; - assign client_if.req_data = req_w; - assign client_if.rsp_ready = busy_r[rsp_block] && issued_r[rsp_block] - && !complete_r[rsp_block]; // ----------------------------------------------------------------------- - // VX_tcu_meta write (broadcast to all blocks' meta instances). + // Metadata RAM write: each response fragment writes its own pass's lanes. // ----------------------------------------------------------------------- - assign meta_wr_en = commit_valid; - assign meta_wr_wid = header_r[commit_block].wid; - assign meta_wr_idx = slot_r[commit_block]; - // Repack response data into TCU_BLOCK_CAP × XLEN. The downstream - // meta SRAM consumes only the low 32 bits, so for XLEN=64 we mux the - // half of the LSU response that holds this lane's logical 32-bit word. - for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_data - if (i < NUM_LANES) begin : g_in - if (LG_W32_RATIO == 0) begin : g_w32_passthru - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_data_r[commit_block][i]); - end else begin : g_w32_pick - wire [LSU_W32_RATIO-1:0][31:0] rsp_w32_v = rsp_data_r[commit_block][i]; - assign meta_wr_data[i] = `VX_CFG_XLEN'(rsp_w32_v[half_sel_w[i]]); - end + assign meta_wr_en = rsp_fire; + assign meta_wr_wid = header_r.wid; + assign meta_wr_idx = slot_r; + + for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_meta_wr + if (i < NUM_LANES) begin : g_lane + wire [LSU_W32_RATIO-1:0][31:0] rsp_w32 = client_if.rsp_data.data[i % LSU_LANES]; + assign meta_wr_mask[i] = client_if.rsp_data.mask[i % LSU_LANES] + && (rsp_pass == `UP(PASS_BITS)'(i / LSU_LANES)); + assign meta_wr_data[i] = rsp_w32[i % LSU_W32_RATIO]; + `UNUSED_VAR (rsp_w32) end else begin : g_pad + assign meta_wr_mask[i] = 1'b0; assign meta_wr_data[i] = '0; end end // ----------------------------------------------------------------------- - // Result drive (commits to scoreboard via writeback, releasing - // wr_xregs[0]). Only the owner block's result_if fires. - // tcu_result_t = { header, data[NUM_LANES][XLEN] }. - // The header is propagated from the latched execute_t header — it - // already carries wr_xregs and wb=0 from the decoder. + // Result: no register writeback; its header releases the scoreboard bit. // ----------------------------------------------------------------------- - tcu_result_t commit_data_w; + assign result_valid = done_r; always_comb begin - commit_data_w = '0; - commit_data_w.header = header_r[commit_block]; - commit_data_w.data = '0; - end - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result - assign result_valid[bi] = commit_valid && (commit_block == BLOCK_IDX_BITS'(bi)); - assign result_data[bi] = commit_data_w; + result_data = '0; + result_data.header = header_r; end endmodule diff --git a/hw/rtl/tcu/VX_tcu_bbuf.sv b/hw/rtl/tcu/VX_tcu_bbuf.sv index 11a4624a26..e87fb7e9c6 100644 --- a/hw/rtl/tcu/VX_tcu_bbuf.sv +++ b/hw/rtl/tcu/VX_tcu_bbuf.sv @@ -18,7 +18,7 @@ // // TB-shared B buffer (block-major SMEM, 1 bank-row storage). // -// Single instance per VX_tcu_unit. Holds the bank-row of B that contains +// Single instance per core (in VX_tcu_shared). Holds the bank-row of B that contains // the current (step_k, step_n) block. All Q tcu_cores read from the // same buffer (structural fan-out, not arbitrated). // diff --git a/hw/rtl/tcu/VX_tcu_block.sv b/hw/rtl/tcu/VX_tcu_block.sv new file mode 100644 index 0000000000..1e493f9d83 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_block.sv @@ -0,0 +1,141 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// One TCU block: the MMA datapath and the TCU_LD unit that fills its metadata +// RAM, with WGMMA operands supplied by the core-wide tile buffers +// (VX_tcu_shared). +module VX_tcu_block import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter `STRING BLOCK_STR = "" +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + // MMA execute handshake observed by the WGMMA tile buffers + output wire exec_valid, + output wire exec_ready, + output tcu_execute_t exec_data, + input wire [TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire tbuf_ready, +`endif + +`ifdef TCU_META_ENABLE + // TCU_LD memory client of an LSU scheduler + VX_lsu_sched_if.master tcu_mem_if, +`endif + + VX_execute_if.slave execute_if, + VX_result_if.master result_if +); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_BLOCKS == `VX_CFG_ISSUE_WIDTH, ("must be full issue execution")); + `STATIC_ASSERT (`VX_CFG_NUM_TCU_LANES == `VX_CFG_NUM_THREADS, ("must be full warp execution")); + + VX_execute_if #( + .data_t (tcu_execute_t) + ) core_execute_if(); + + VX_result_if #( + .data_t (tcu_result_t) + ) core_result_if(); + +`ifdef TCU_META_ENABLE + wire agu_ld_valid, agu_ld_ready; + wire agu_result_valid, agu_result_ready; + tcu_result_t agu_result_data; + wire meta_wr_en; + wire [NW_WIDTH-1:0] meta_wr_wid; + wire [4:0] meta_wr_idx; + wire [TCU_BLOCK_CAP-1:0] meta_wr_mask; + wire [TCU_BLOCK_CAP-1:0][31:0] meta_wr_data; + + VX_tcu_agu #( + .INSTANCE_ID (`SFORMATF(("%s-agu%s", INSTANCE_ID, BLOCK_STR))) + ) agu ( + .clk (clk), + .reset (reset), + .ld_valid (agu_ld_valid), + .ld_data (execute_if.data), + .ld_ready (agu_ld_ready), + .client_if (tcu_mem_if), + .meta_wr_en (meta_wr_en), + .meta_wr_wid (meta_wr_wid), + .meta_wr_idx (meta_wr_idx), + .meta_wr_mask (meta_wr_mask), + .meta_wr_data (meta_wr_data), + .result_valid (agu_result_valid), + .result_data (agu_result_data), + .result_ready (agu_result_ready) + ); + + // TCU_LD goes to the AGU, every MMA op to the core + wire is_tcu_ld = (execute_if.data.op_type == INST_TCU_LD); + + assign agu_ld_valid = execute_if.valid && is_tcu_ld; + assign core_execute_if.valid = execute_if.valid && !is_tcu_ld; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = is_tcu_ld ? agu_ld_ready : core_execute_if.ready; + + // AGU wins same-cycle conflicts; the core stalls and retries next cycle. + assign result_if.valid = agu_result_valid || core_result_if.valid; + assign result_if.data = agu_result_valid ? agu_result_data : core_result_if.data; + assign agu_result_ready = result_if.ready; + assign core_result_if.ready = result_if.ready && !agu_result_valid; +`else + assign core_execute_if.valid = execute_if.valid; + assign core_execute_if.data = execute_if.data; + assign execute_if.ready = core_execute_if.ready; + + assign result_if.valid = core_result_if.valid; + assign result_if.data = core_result_if.data; + assign core_result_if.ready = result_if.ready; +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + assign exec_valid = core_execute_if.valid; + assign exec_ready = core_execute_if.ready; + assign exec_data = core_execute_if.data; +`endif + + VX_tcu_core #( + .INSTANCE_ID (`SFORMATF(("%s-fused%s", INSTANCE_ID, BLOCK_STR))) + ) tcu_core ( + `ifdef SCOPE + .scope_reset (scope_reset), + .scope_bus_in (scope_bus_in), + .scope_bus_out (scope_bus_out), + `endif + .clk (clk), + .reset (reset), + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready (tbuf_ready), + `endif + `ifdef TCU_META_ENABLE + .ext_meta_wr_en (meta_wr_en), + .ext_meta_wr_wid (meta_wr_wid), + .ext_meta_wr_idx (meta_wr_idx), + .ext_meta_wr_mask (meta_wr_mask), + .ext_meta_wr_data (meta_wr_data), + `endif + .execute_if (core_execute_if), + .result_if (core_result_if) + ); + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_core.sv b/hw/rtl/tcu/VX_tcu_core.sv index f1a3c91b19..cee0c772dc 100644 --- a/hw/rtl/tcu/VX_tcu_core.sv +++ b/hw/rtl/tcu/VX_tcu_core.sv @@ -27,12 +27,13 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire tbuf_ready, `endif - // External metadata write port from the shared VX_tcu_agu. + // Metadata write port from the block's VX_tcu_agu. `ifdef TCU_META_ENABLE input wire ext_meta_wr_en, input wire [NW_WIDTH-1:0] ext_meta_wr_wid, input wire [4:0] ext_meta_wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] ext_meta_wr_data, + input wire [TCU_BLOCK_CAP-1:0] ext_meta_wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] ext_meta_wr_data, `endif // Inputs @@ -41,6 +42,11 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( // Outputs VX_result_if.master result_if ); + +`ifdef SCOPE + `SCOPE_IO_SWITCH (1); + `SCOPE_IO_UNUSED (0) +`endif `UNUSED_SPARAM (INSTANCE_ID); // FEDP dot-product width in 32-bit words (doubles under FEDP2K). @@ -147,6 +153,10 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire setup_enqueue = execute_fire && is_wgmma_setup; wire fedp_enqueue = execute_fire && !is_wgmma_setup; + // The accumulator is fp32 (int32 for integer inputs); no other fmt_d exists. + `RUNTIME_ASSERT (~fedp_enqueue || (fmt_d == (tcu_fmt_is_int(fmt_s) ? TCU_FMT_WIDTH'(TCU_I32_ID) : TCU_FMT_WIDTH'(TCU_FP32_ID))), + ("%t: *** %s unsupported TCU output format: fmt_s=%0d, fmt_d=%0d (fp32 only)", $time, INSTANCE_ID, fmt_s, fmt_d)) + // ----------------------------------------------------------------------- // Sparse metadata: VX_tcu_sp_meta (for WMMA_SP) + optional tile-buffer mux // ----------------------------------------------------------------------- @@ -303,14 +313,17 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( `endif `ifdef TCU_META_ENABLE + // a TCU block only executes one residue class of warps modulo NUM_TCU_BLOCKS VX_tcu_meta #( - .INSTANCE_ID (INSTANCE_ID) + .INSTANCE_ID (INSTANCE_ID), + .WID_STRIDE (`VX_CFG_NUM_TCU_BLOCKS) ) tcu_meta ( .clk (clk), .reset (reset), .wr_en (ext_meta_wr_en), .wr_wid (ext_meta_wr_wid), .wr_idx (ext_meta_wr_idx), + .wr_mask(ext_meta_wr_mask), .wr_data(ext_meta_wr_data), .rd_wid (execute_if.data.header.wid) `ifdef VX_CFG_TCU_SPARSE_ENABLE @@ -480,14 +493,15 @@ module VX_tcu_core import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar k_idx = 0; k_idx < TCU_TC_K; ++k_idx) begin : g_sparse_slice_assign localparam J_SP = SYM_SPARSE ? (j % (TCU_TC_N / 2)) : j; // rs2_data sparse-pair layout differs by op: - // WGMMA_SP: source is tbuf (shared mem), K-major → - // idx = k_idx*(TC_N*2) + J_SP*2 + lane - // WMMA_SP : source is the register file, J-major → + // WGMMA_SP: source is tbuf (shared mem), K-major, all columns → + // idx = k_idx*(TC_N*2) + j*2 + lane + // WMMA_SP : source is the register file, J-major, folded to + // column pairs when SYM_SPARSE → // idx = J_SP*(TC_K*2) + k_idx*2 + lane // The two layouts are incompatible; separate formulas are required. `ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [31:0] b_col_1_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2]); - wire [31:0] b_col_2_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + J_SP * 2 + 1]); + wire [31:0] b_col_1_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + j * 2]); + wire [31:0] b_col_2_wg = 32'(rs2_data[k_idx * TCU_TC_N * 2 + j * 2 + 1]); wire [31:0] b_col_1_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2]); wire [31:0] b_col_2_wm = 32'(rs2_data[b_off_wm + J_SP * TCU_TC_K * 2 + k_idx * 2 + 1]); assign b_col_1[k_idx] = is_wgmma ? b_col_1_wg : b_col_1_wm; diff --git a/hw/rtl/tcu/VX_tcu_dsm.sv b/hw/rtl/tcu/VX_tcu_dsm.sv index a8f88a88b7..cebc13b747 100644 --- a/hw/rtl/tcu/VX_tcu_dsm.sv +++ b/hw/rtl/tcu/VX_tcu_dsm.sv @@ -26,7 +26,7 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( output wire [TCU_MAX_INPUTS-1:0] vld_mask ); - logic [N-1:0][7:0] vld_mask_per_k; + wire [N-1:0][7:0] vld_mask_per_k; wire [TCU_MAX_INPUTS-1:0] vld_mask_w = vld_mask_per_k; for (genvar k = 0; k < N; ++k) begin : g_k @@ -56,47 +56,48 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( end `UNUSED_VAR({a_tf32_nz, b_tf32_nz, a_f16_nz, b_f16_nz, a_i8_nz, b_i8_nz, a_i4_nz, b_i4_nz}) + logic [7:0] vld_mask_k; always_comb begin - vld_mask_per_k[k] = '1; + vld_mask_k = '1; case (fmt_s) `ifdef VX_CFG_TCU_TF32_ENABLE TCU_TF32_ID: begin - vld_mask_per_k[k] = {8{a_tf32_nz && b_tf32_nz}}; + vld_mask_k = {8{a_tf32_nz && b_tf32_nz}}; end `endif `ifdef VX_CFG_TCU_FP16_ENABLE TCU_FP16_ID, TCU_BF16_ID: begin - vld_mask_per_k[k][0 +: 4] = {4{a_f16_nz[0] && b_f16_nz[0]}}; - vld_mask_per_k[k][4 +: 4] = {4{a_f16_nz[1] && b_f16_nz[1]}}; + vld_mask_k[0 +: 4] = {4{a_f16_nz[0] && b_f16_nz[0]}}; + vld_mask_k[4 +: 4] = {4{a_f16_nz[1] && b_f16_nz[1]}}; end `endif `ifdef VX_CFG_TCU_FP8_ENABLE TCU_FP8_ID, TCU_BF8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `ifdef VX_CFG_TCU_MX_ENABLE TCU_MXFP8_ID, TCU_MXBF8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `ifdef VX_CFG_TCU_FP4_ENABLE `ifdef VX_CFG_TCU_MXFP4_ENABLE TCU_MXFP4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif `ifdef VX_CFG_TCU_NVFP4_ENABLE TCU_NVFP4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif @@ -107,7 +108,7 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( TCU_I8_ID, TCU_U8_ID: begin for (int e = 0; e < 4; ++e) begin - vld_mask_per_k[k][e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; + vld_mask_k[e * 2 +: 2] = {2{a_i8_nz[e] && b_i8_nz[e]}}; end end `endif @@ -115,15 +116,16 @@ module VX_tcu_dsm import VX_tcu_pkg::*; #( TCU_I4_ID, TCU_U4_ID: begin for (int e = 0; e < 8; ++e) begin - vld_mask_per_k[k][e] = a_i4_nz[e] && b_i4_nz[e]; + vld_mask_k[e] = a_i4_nz[e] && b_i4_nz[e]; end end `endif default: begin - vld_mask_per_k[k] = '1; + vld_mask_k = '1; end endcase end + assign vld_mask_per_k[k] = vld_mask_k; end // Optional output register (aligns vld_mask with the FEDP operand pipe). diff --git a/hw/rtl/tcu/VX_tcu_meta.sv b/hw/rtl/tcu/VX_tcu_meta.sv index 5c55e58bb0..46a00f459b 100644 --- a/hw/rtl/tcu/VX_tcu_meta.sv +++ b/hw/rtl/tcu/VX_tcu_meta.sv @@ -19,7 +19,7 @@ // the MX scale-factor stores (A/B) behind a single warp-indexed block RAM. // // All regions share one warp-indexed address space (SIZE = NUM_WARPS) and the -// one broadcast write port from VX_tcu_agu. wr_idx[4] selects the namespace +// one write port from the block's VX_tcu_agu. wr_idx[4] selects the namespace // (0 = sparse, 1 = MX); within MX wr_idx[0] selects the axis (0 = A, 1 = B); // within sparse wr_idx[3:0] is the column-group index. This matches the // TCU_LD destination-register encoding emitted by vx_tensor.h (x0/x1 = SP, @@ -27,20 +27,24 @@ // // Read latency contract: the read address (rd_wid) is the already-registered // execute_if header wid, so the RAM is read combinationally in the issue -// cycle (OUT_REG=0, RADDR_REG=1). The sparse mux / MX scale extraction in +// cycle (OUT_REG=0). The sparse mux / MX scale extraction in // VX_tcu_core consume the result in that same cycle. module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( - parameter `STRING INSTANCE_ID = "" + parameter `STRING INSTANCE_ID = "", + // stores one residue class of warps modulo WID_STRIDE, by wid / WID_STRIDE; + // the parent only presents writes for that class + parameter WID_STRIDE = 1 ) ( input wire clk, input wire reset, - // Unified broadcast write port (from VX_tcu_agu). + // Write port from the block's VX_tcu_agu; wr_mask selects the lanes written. input wire wr_en, input wire [NW_WIDTH-1:0] wr_wid, input wire [4:0] wr_idx, - input wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] wr_data, + input wire [TCU_BLOCK_CAP-1:0] wr_mask, + input wire [TCU_BLOCK_CAP-1:0][31:0] wr_data, // Unified read port (warp id from the FEDP path). input wire [NW_WIDTH-1:0] rd_wid @@ -74,8 +78,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( localparam MERGED_COLS = SP_COLS + 2 * MX_COLS; localparam MERGED_BITS = MERGED_COLS * 32; - localparam META_DEPTH = `VX_CFG_NUM_WARPS; - localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam META_DEPTH = `VX_CFG_NUM_WARPS / WID_STRIDE; + localparam META_ADDRW = `CLOG2(META_DEPTH); + localparam STRIDE_BITS = `CLOG2(WID_STRIDE); + + `STATIC_ASSERT (`IS_POW2(WID_STRIDE), ("invalid parameter")) // Namespace decode (mirrors the AGU / vx_tensor.h slot encoding). The // per-region write strobes are defined inside each region's block below so @@ -85,8 +92,8 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire [MERGED_BITS-1:0] merged_wdata; wire [MERGED_BITS-1:0] merged_rdata; - wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid); - wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid); + wire [`UP(META_ADDRW)-1:0] wr_addr = `UP(META_ADDRW)'(wr_wid >> STRIDE_BITS); + wire [`UP(META_ADDRW)-1:0] rd_addr = `UP(META_ADDRW)'(rd_wid >> STRIDE_BITS); // ----------------------------------------------------------------------- // Sparse region: per-thread metadata packed into PER_WARP_DEPTH banks of @@ -156,15 +163,11 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( for (genvar b = 0; b < PER_WARP_DEPTH; ++b) begin : g_meta_banks for (genvar c = 0; c < NUM_COLS; ++c) begin : g_col localparam int COL = b * NUM_COLS + c; - if (STORES_PER_COL > 1) begin : g_wr - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[b % BANKS_PER_STORE]); - end else begin : g_wr - // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b - localparam int COL_IN_GROUP = c % COLS_PER_LOAD; - localparam int THREAD_OFF = COL_IN_GROUP * PER_WARP_DEPTH + b; - assign merged_wdata[COL * 32 +: 32] = 32'(wr_data[THREAD_OFF]); - end - assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b]; + // Thread offset for column c within its group: (c % CPL) * PER_WARP_DEPTH + b + localparam int THREAD_OFF = (STORES_PER_COL > 1) ? (b % BANKS_PER_STORE) + : ((c % COLS_PER_LOAD) * PER_WARP_DEPTH + b); + assign merged_wdata[COL * 32 +: 32] = wr_data[THREAD_OFF]; + assign merged_wren[COL] = sp_wr && col_wren[c] && meta_wr_bank_en[b] && wr_mask[THREAD_OFF]; end end @@ -190,10 +193,10 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( wire mx_wr_b = wr_en && wr_idx[4] && wr_idx[0]; for (genvar i = 0; i < TCU_BLOCK_CAP; ++i) begin : g_mx_pack - assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = 32'(wr_data[i]); - assign merged_wren[MXA_COL_BASE + i] = mx_wr_a; - assign merged_wren[MXB_COL_BASE + i] = mx_wr_b; + assign merged_wdata[(MXA_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wdata[(MXB_COL_BASE + i) * 32 +: 32] = wr_data[i]; + assign merged_wren[MXA_COL_BASE + i] = mx_wr_a && wr_mask[i]; + assign merged_wren[MXB_COL_BASE + i] = mx_wr_b && wr_mask[i]; end assign meta_a = merged_rdata[(MXA_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; assign meta_b = merged_rdata[(MXB_COL_BASE * 32) +: (TCU_BLOCK_CAP * 32)]; @@ -208,8 +211,7 @@ module VX_tcu_meta import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .WRENW (MERGED_COLS), .LUTRAM (0), .OUT_REG (0), - .RDW_MODE ("W"), - .RADDR_REG(1) // rd_wid is registered! + .RDW_MODE ("W") ) meta_ram ( .clk (clk), .reset (reset), diff --git a/hw/rtl/tcu/VX_tcu_pkg.sv b/hw/rtl/tcu/VX_tcu_pkg.sv index 7c0c76d7fa..5ae866aefc 100644 --- a/hw/rtl/tcu/VX_tcu_pkg.sv +++ b/hw/rtl/tcu/VX_tcu_pkg.sv @@ -113,9 +113,9 @@ package VX_tcu_pkg; localparam TCU_WG_B_BLOCK_SIZE = TCU_WG_FEDP_K * TCU_TC_N; localparam TCU_WG_B_SUB_BLOCKS = TCU_BLOCK_CAP / TCU_WG_B_BLOCK_SIZE; - // Symmetric sparse flag (NT=4, NT=16: block_em == block_en) - // WGMMA always uses full interleaved layout, so SYM_SPARSE is forced off. - localparam SYM_SPARSE = `VX_CFG_TCU_WGMMA_ENABLED ? 0 : (TCU_BLOCK_EM == TCU_BLOCK_EN); + // Symmetric sparse flag (NT=4, NT=16: block_em == block_en). It selects the + // WMMA_SP layout only: WGMMA_SP always reads the full interleaved layout. + localparam SYM_SPARSE = (TCU_BLOCK_EM == TCU_BLOCK_EN); // B micro-tiling (sparse 2:4) // NT=8/32: standard interleaved layout (tcK × tcN × 2 = NT lanes per block) @@ -172,6 +172,222 @@ package VX_tcu_pkg; localparam TCU_MAX_INPUTS = TCU_TC_K * TCU_MAX_ELT_RATIO; + localparam TCU_LG_M = $clog2(TCU_M_STEPS); + localparam TCU_LG_N = $clog2(TCU_N_STEPS); + localparam TCU_LG_K = $clog2(TCU_K_STEPS); + localparam TCU_LG_B_SB = $clog2(TCU_B_SUB_BLOCKS); + + // Register offsets of the WMMA operands of step (m, n, k), permuted so the + // three operands of a step never share a register bank. + // one B sub-block: A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} + // two B sub-blocks: A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} + // asymmetric sparse: B={n[hi],~(n[0]^k),~k}, C={n[hi],m,~(m^n[0])} + function automatic logic [2:0] tcu_ra_off(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_K)-1:0] k); + if (TCU_LG_B_SB == 0) begin + return 3'({m[0], ~m[`UP(TCU_LG_M)-1], k[0]}); + end else begin + return 3'({k[0], ~m[0], m[0] ^ k[`UP(TCU_LG_K)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rb_off(input logic [`UP(TCU_LG_N)-1:0] n, + input logic [`UP(TCU_LG_K)-1:0] k); + if (TCU_LG_B_SB == 0) begin + return 3'({n[0] ^ k[0], ~k[0]}); + end else begin + return 3'({k[0], k[`UP(TCU_LG_K)-1] ^ n[`UP(TCU_LG_N)-1], ~n[`UP(TCU_LG_N)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rc_off(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_N)-1:0] n); + if (TCU_LG_B_SB == 0) begin + return 3'({m[0], ~m[`UP(TCU_LG_M)-1], ~(m[`UP(TCU_LG_M)-1] ^ n[0])}); + end else begin + return 3'({n[0], ~m[0], n[`UP(TCU_LG_N)-1]}); + end + endfunction + + function automatic logic [2:0] tcu_rb_off_sp(input logic [`UP(TCU_LG_N)-1:0] n, + input logic [`UP(TCU_LG_K)-1:0] k); + return 3'({n[`UP(TCU_LG_N)-1], ~(n[0] ^ k[0]), ~k[0]}); + endfunction + + function automatic logic [2:0] tcu_rc_off_sp(input logic [`UP(TCU_LG_M)-1:0] m, + input logic [`UP(TCU_LG_N)-1:0] n); + return 3'({n[`UP(TCU_LG_N)-1], m[0], ~(m[0] ^ n[0])}); + endfunction + + // Matrix fragment load/store (vx_ldm/vx_stm) shapes: {ctx, role[1:0], trans, sparse}. + // Uop e moves fragment register fd + e between it and x[rs1 | grp] + + // (imm[11:5] | col), the element the WMMA/WGMMA expansion reads from it. + localparam TCU_LDM_A = 5'b00000; // WMMA A, row-major + localparam TCU_LDM_A_SP = 5'b00001; // WMMA sparse A, row-major + localparam TCU_LDM_B = 5'b00110; // WMMA B, col-major + localparam TCU_LDM_B_SP = 5'b00111; // WMMA symmetric sparse B, col-major + localparam TCU_LDM_C = 5'b01000; // WMMA C, row-major + localparam TCU_LDM_WG_C = 5'b11000; // WGMMA C, row-major, one 8-register chunk + + // Decode replaces the shape in imm[4:0] with its index, so the per-warp + // expander decodes 3 bits instead of 5, and marks an 8-register fragment + // in bytesel. + localparam TCU_LDM_IDX_A = 3'd0; + localparam TCU_LDM_IDX_A_SP = 3'd1; + localparam TCU_LDM_IDX_B = 3'd2; + localparam TCU_LDM_IDX_B_SP = 3'd3; + localparam TCU_LDM_IDX_C = 3'd4; + localparam TCU_LDM_IDX_WG_C = 3'd5; + + // Byte strides between the words a lane holds along a fragment row + localparam TCU_LDM_LG_CS_K = $clog2(TCU_TC_K * 4); + localparam TCU_LDM_LG_CS_N = $clog2(TCU_TC_N * 4); + localparam TCU_LDM_LG_K_SP = (TCU_LG_K > 0) ? (TCU_LG_K - 1) : 0; + localparam TCU_LDM_COL_W = TCU_LDM_LG_CS_K + 4; + + typedef struct packed { + logic [1:0] grp; + logic [TCU_LDM_COL_W-1:0] col; + } tcu_ldm_uop_t; + + function automatic logic tcu_ldm_valid(input logic [4:0] shape); + case (shape) + TCU_LDM_A, + TCU_LDM_B, + TCU_LDM_C: return 1'b1; + TCU_LDM_A_SP: return 1'(`VX_CFG_TCU_SPARSE_ENABLED); + TCU_LDM_B_SP: return 1'(`VX_CFG_TCU_SPARSE_ENABLED && SYM_SPARSE); + TCU_LDM_WG_C: return 1'(`VX_CFG_TCU_WGMMA_ENABLED); + default: return 1'b0; + endcase + endfunction + + function automatic logic [4:0] tcu_ldm_code(input logic [4:0] shape); + case (shape) + TCU_LDM_A: return 5'(TCU_LDM_IDX_A); + TCU_LDM_A_SP: return 5'(TCU_LDM_IDX_A_SP); + TCU_LDM_B: return 5'(TCU_LDM_IDX_B); + TCU_LDM_B_SP: return 5'(TCU_LDM_IDX_B_SP); + TCU_LDM_C: return 5'(TCU_LDM_IDX_C); + default: return 5'(TCU_LDM_IDX_WG_C); + endcase + endfunction + + function automatic int unsigned tcu_ldm_count(input logic [2:0] idx); + case (idx) + TCU_LDM_IDX_A_SP: return 4; + TCU_LDM_IDX_B, + TCU_LDM_IDX_B_SP: return TCU_NRB; + default: return 8; + endcase + endfunction + + function automatic logic tcu_ldm_is8(input logic [4:0] shape); + return (tcu_ldm_count(3'(tcu_ldm_code(shape))) == 8); + endfunction + + // Uop e moves register fd + e, so the counter feeds the register add + // directly; the shape only maps e to the row group and column of the element + // the WMMA/WGMMA expansion reads from that register, by inverting + // tcu_ra_off/tcu_rb_off/tcu_rc_off (a few inverted and XORed bits of e). + // VX_uop_packld checks the inverse against them at elaboration. + function automatic tcu_ldm_uop_t tcu_ldm_uop(input logic [2:0] idx, input logic [2:0] e); + tcu_ldm_uop_t u; + logic [2:0] m, n, k; + u = '0; + m = '0; + n = '0; + k = '0; + case (idx) + TCU_LDM_IDX_A: begin + if (TCU_LG_B_SB == 0) begin // A={m[0],~m[1],k} + m = {1'b0, ~e[1], e[2]}; + k = {2'b0, e[0]}; + end else begin // A={k[0],~m,m^k[1]} + m = {2'b0, ~e[1]}; + k = {1'b0, e[0] ^ ~e[1], e[2]}; + end + u.grp = 2'(m); + u.col = TCU_LDM_COL_W'(k) << TCU_LDM_LG_CS_K; + end + // registers in element order {m, k} over half the K steps + TCU_LDM_IDX_A_SP: begin + u.grp = 2'(e >> TCU_LDM_LG_K_SP); + u.col = TCU_LDM_COL_W'(e & 3'((1 << TCU_LDM_LG_K_SP) - 1)) << TCU_LDM_LG_CS_K; + end + // a col-major B row is one of its columns (a pair with two sub-blocks) + TCU_LDM_IDX_B: begin + if (TCU_LG_B_SB == 0) begin // B={n^k,~k} + k = {2'b0, ~e[0]}; + n = {2'b0, e[1] ^ ~e[0]}; + end else begin // B={k[0],k[1]^np,~np} + n = {2'b0, ~e[0]}; + k = {1'b0, e[1] ^ ~e[0], e[2]}; + end + u.grp = 2'(n); + u.col = TCU_LDM_COL_W'(k) << TCU_LDM_LG_CS_K; + end + // symmetric sparse: one row per register + TCU_LDM_IDX_B_SP: begin + u.grp = 2'(e); + end + TCU_LDM_IDX_C: begin + if (TCU_LG_B_SB == 0) begin // C={m[0],~m[1],~(m[1]^n)} + m = {1'b0, ~e[1], e[2]}; + n = {2'b0, e[0] ^ e[1]}; + end else begin // C={n[0],~m,n[1]} + m = {2'b0, ~e[1]}; + n = {1'b0, e[0], e[2]}; + end + u.grp = 2'(m); + u.col = TCU_LDM_COL_W'(n) << TCU_LDM_LG_CS_N; + end + // n-major registers {n, m}, two row groups + default: begin + u.grp = 2'(e[0]); + u.col = TCU_LDM_COL_W'(e >> 1) << TCU_LDM_LG_CS_N; + end + endcase + return u; + endfunction + + // The register the WMMA/WGMMA expansion reads element (grp, col) of a + // fragment from, through the forward permutations; -1 when (grp, col) is not + // an element of the fragment. + function automatic int tcu_ldm_reg(input logic [2:0] idx, input tcu_ldm_uop_t u); + int unsigned g, c, ck, cn; + g = int'(u.grp); + c = int'(u.col); + ck = c >> TCU_LDM_LG_CS_K; + cn = c >> TCU_LDM_LG_CS_N; + case (idx) + TCU_LDM_IDX_A: begin + if ((ck << TCU_LDM_LG_CS_K) != c || g >= TCU_M_STEPS || ck >= TCU_K_STEPS) return -1; + return int'(tcu_ra_off(`UP(TCU_LG_M)'(g), `UP(TCU_LG_K)'(ck))); + end + TCU_LDM_IDX_A_SP: begin + if ((ck << TCU_LDM_LG_CS_K) != c || g >= TCU_M_STEPS || ck >= (1 << TCU_LDM_LG_K_SP)) return -1; + return int'((g << TCU_LDM_LG_K_SP) | ck); + end + TCU_LDM_IDX_B: begin + if ((ck << TCU_LDM_LG_CS_K) != c || (g << TCU_LG_B_SB) >= `MAX(TCU_N_STEPS, 1 << TCU_LG_B_SB) || ck >= TCU_K_STEPS) return -1; + return int'(tcu_rb_off(`UP(TCU_LG_N)'(g << TCU_LG_B_SB), `UP(TCU_LG_K)'(ck))); + end + TCU_LDM_IDX_B_SP: begin + if (c != 0 || g >= TCU_NRB) return -1; + return int'(g); + end + TCU_LDM_IDX_C: begin + if ((cn << TCU_LDM_LG_CS_N) != c || g >= TCU_M_STEPS || cn >= TCU_N_STEPS) return -1; + return int'(tcu_rc_off(`UP(TCU_LG_M)'(g), `UP(TCU_LG_N)'(cn))); + end + default: begin + if ((cn << TCU_LDM_LG_CS_N) != c || g >= 2 || cn >= 4) return -1; + return int'((cn << 1) | g); + end + endcase + endfunction + function automatic int unsigned mx_fedp_sf_count( input int unsigned data_bits, input int unsigned block_elems @@ -216,10 +432,13 @@ package VX_tcu_pkg; localparam TCU_MX_MAX_SF = `MAX(`MAX(TCU_MX_SF_FP8, TCU_MX_SF_MXFP4), TCU_MX_SF_NVFP4); + // MX block scales span 2^-254 .. 2^256 on top of the elements' own range. `ifdef VX_CFG_TCU_TF32_ENABLE localparam TCU_EXP_BITS = 10; `elsif VX_CFG_TCU_FP16_ENABLE localparam TCU_EXP_BITS = 10; + `elsif VX_CFG_TCU_MX_ENABLE + localparam TCU_EXP_BITS = 10; `else localparam TCU_EXP_BITS = 9; `endif diff --git a/hw/rtl/tcu/VX_tcu_shared.sv b/hw/rtl/tcu/VX_tcu_shared.sv new file mode 100644 index 0000000000..0b232fa3f4 --- /dev/null +++ b/hw/rtl/tcu/VX_tcu_shared.sv @@ -0,0 +1,74 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// TCU state shared by every TCU block: the WGMMA tile buffers (a warpgroup's +// B operand is common to its blocks). +module VX_tcu_shared import VX_gpu_pkg::*, VX_tcu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter BLOCK_SIZE = `VX_CFG_NUM_TCU_BLOCKS +) ( +`ifdef PERF_ENABLE + output tcu_perf_t tcu_perf, +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + input wire [BLOCK_SIZE-1:0] exec_valid, + input wire [BLOCK_SIZE-1:0] exec_ready, + input tcu_execute_t exec_data [BLOCK_SIZE], + output wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + output wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + output wire [BLOCK_SIZE-1:0] tbuf_ready_eff, + // Bank-parallel LMEM read port + VX_mem_bus_if.master tcu_lmem_if, +`endif + + input wire clk, + input wire reset +); + `UNUSED_SPARAM (INSTANCE_ID) +`ifndef VX_CFG_TCU_WGMMA_ENABLE + `UNUSED_PARAM (BLOCK_SIZE) + `UNUSED_VAR ({clk, reset}) +`endif + +`ifdef VX_CFG_TCU_WGMMA_ENABLE + VX_tcu_wgmma #( + .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), + .BLOCK_SIZE (BLOCK_SIZE) + ) wgmma ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + .tcu_perf (tcu_perf), + `endif + .exec_valid (exec_valid), + .exec_ready (exec_ready), + .exec_data (exec_data), + .tcu_lmem_if (tcu_lmem_if), + .tbuf_rs1_data (tbuf_rs1_data), + .tbuf_rs2_data (tbuf_rs2_data), + .tbuf_ready_eff (tbuf_ready_eff) + ); +`else +`ifdef PERF_ENABLE + assign tcu_perf.tbuf_stalls = '0; + assign tcu_perf.tbuf_cache_hits = '0; + assign tcu_perf.lmem_reads = '0; + assign tcu_perf.wgmma_instrs = '0; + assign tcu_perf.wgmma_stalls = '0; +`endif +`endif + +endmodule diff --git a/hw/rtl/tcu/VX_tcu_unit.sv b/hw/rtl/tcu/VX_tcu_unit.sv index 593e39eee7..b9547072a7 100644 --- a/hw/rtl/tcu/VX_tcu_unit.sv +++ b/hw/rtl/tcu/VX_tcu_unit.sv @@ -21,18 +21,19 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( input wire clk, input wire reset, -`ifdef PERF_ENABLE - output tcu_perf_t tcu_perf, -`endif - `ifdef VX_CFG_TCU_WGMMA_ENABLE - // Bank-parallel LMEM read port - VX_mem_bus_if.master tcu_lmem_if, + // per-block ports of the core-wide WGMMA tile buffers (VX_tcu_shared) + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_valid, + output wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] exec_ready, + output tcu_execute_t exec_data [`VX_CFG_NUM_TCU_BLOCKS], + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data, + input wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tbuf_ready, `endif `ifdef TCU_META_ENABLE - // TCU_LD memory client connection to VX_lsu_scheduler at VX_core. - VX_lsu_sched_if.master tcu_mem_if, + // TCU_LD memory clients of the LSU blocks' schedulers + VX_lsu_sched_if.master tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS], `endif // Inputs @@ -59,6 +60,7 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( ) lane_dispatch ( .clk (clk), .reset (reset), + .port_base ('0), .dispatch_if(dispatch_if), .execute_if (per_block_execute_if) ); @@ -67,185 +69,108 @@ module VX_tcu_unit import VX_gpu_pkg::*, VX_tcu_pkg::*; #( .data_t (tcu_result_t) ) per_block_result_if[BLOCK_SIZE](); - // ----------------------------------------------------------------------- - // Split each per_block_execute_if between two consumers: - // - VX_tcu_agu: handles INST_TCU_LD (warp-level memory load). - // - VX_tcu_core: handles every MMA op_type. - // The ready signal is muxed by op_type so only one consumer drives at a time. - // ----------------------------------------------------------------------- - VX_execute_if #( - .data_t (tcu_execute_t) - ) core_execute_if[BLOCK_SIZE](); - - VX_result_if #( - .data_t (tcu_result_t) - ) core_result_if[BLOCK_SIZE](); - -`ifdef TCU_META_ENABLE - wire [BLOCK_SIZE-1:0] agu_ld_valid; - wire [BLOCK_SIZE-1:0] agu_ld_ready; - tcu_execute_t agu_ld_data [BLOCK_SIZE]; - - wire [BLOCK_SIZE-1:0] agu_result_valid; - tcu_result_t agu_result_data [BLOCK_SIZE]; - wire [BLOCK_SIZE-1:0] agu_result_ready; -`endif - - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_split - `ifdef TCU_META_ENABLE - wire is_tcu_ld = (per_block_execute_if[bi].data.op_type == INST_TCU_LD); - - // To AGU when TCU_LD - assign agu_ld_valid[bi] = per_block_execute_if[bi].valid && is_tcu_ld; - assign agu_ld_data[bi] = per_block_execute_if[bi].data; - - // To tcu_core when NOT TCU_LD - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid && !is_tcu_ld; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - - // Parent .ready: route to AGU on TCU_LD, otherwise to tcu_core - assign per_block_execute_if[bi].ready = is_tcu_ld - ? agu_ld_ready[bi] - : core_execute_if[bi].ready; - `else - // No sparse: pass-through to tcu_core - assign core_execute_if[bi].valid = per_block_execute_if[bi].valid; - assign core_execute_if[bi].data = per_block_execute_if[bi].data; - assign per_block_execute_if[bi].ready = core_execute_if[bi].ready; - `endif - end - - // ----------------------------------------------------------------------- - // Result_if merge: AGU result and tcu_core result are mutually exclusive - // in time per block; OR-mux with priority arbiter (AGU wins: TCU_LD is rare). - // ----------------------------------------------------------------------- `ifdef TCU_META_ENABLE - // AGU wins same-cycle conflicts; tcu_core stalls (ready=0) and retries next cycle. - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_merge - assign per_block_result_if[bi].valid = agu_result_valid[bi] || core_result_if[bi].valid; - assign per_block_result_if[bi].data = agu_result_valid[bi] - ? agu_result_data[bi] - : core_result_if[bi].data; - assign agu_result_ready[bi] = per_block_result_if[bi].ready; - assign core_result_if[bi].ready = per_block_result_if[bi].ready && !agu_result_valid[bi]; - end -`else - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_result_passthru - assign per_block_result_if[bi].valid = core_result_if[bi].valid; - assign per_block_result_if[bi].data = core_result_if[bi].data; - assign core_result_if[bi].ready = per_block_result_if[bi].ready; - end -`endif - - // ----------------------------------------------------------------------- - // WGMMA feature (orchestrator): VX_tcu_tbuf + VX_tcu_lockstep + perf. - // ----------------------------------------------------------------------- + VX_lsu_sched_if block_mem_if [BLOCK_SIZE](); + + if (`VX_CFG_NUM_LSU_BLOCKS == BLOCK_SIZE) begin : g_mem_per_block + // each TCU block loads through the LSU block serving the same warps + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_port + assign tcu_mem_if[b].req_valid = block_mem_if[b].req_valid; + assign tcu_mem_if[b].req_data = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[b].req_ready; + assign block_mem_if[b].rsp_valid = tcu_mem_if[b].rsp_valid; + assign block_mem_if[b].rsp_data = tcu_mem_if[b].rsp_data; + assign tcu_mem_if[b].rsp_ready = block_mem_if[b].rsp_ready; + end + end else begin : g_mem_shared + // the TCU blocks share LSU block 0's client: the block index takes the + // tag's low bits and the block's own tag moves above it + localparam BLOCK_IDX_BITS = `CLOG2(BLOCK_SIZE); + localparam BLOCK_TAG_BITS = LSU_CLIENT_TAG_WIDTH - BLOCK_IDX_BITS; + `STATIC_ASSERT (BLOCK_IDX_BITS > 0, ("invalid parameter")) + `STATIC_ASSERT (BLOCK_TAG_BITS > 0, ("LSU client tag cannot encode TCU block")) + // a block's TCU_LD pass number rides in its own tag bits + `STATIC_ASSERT (BLOCK_TAG_BITS >= `CLOG2(NUM_LANES / `VX_CFG_NUM_LSU_LANES), ("LSU client tag cannot encode TCU block and TCU_LD pass")) + + wire [BLOCK_SIZE-1:0] req_valids; + wire [`UP(BLOCK_IDX_BITS)-1:0] req_idx; + wire [BLOCK_SIZE-1:0] req_grant; + wire req_valid; + lsu_req_data_t req_datas [BLOCK_SIZE]; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_req + assign req_valids[b] = block_mem_if[b].req_valid; + assign req_datas[b] = block_mem_if[b].req_data; + assign block_mem_if[b].req_ready = tcu_mem_if[0].req_ready && req_grant[b]; + end + + VX_generic_arbiter #( + .NUM_REQS (BLOCK_SIZE), + .TYPE ("R") + ) req_arb ( + .clk (clk), + .reset (reset), + .requests (req_valids), + .grant_index (req_idx), + .grant_onehot (req_grant), + .grant_valid (req_valid), + .grant_ready (tcu_mem_if[0].req_ready) + ); -`ifdef VX_CFG_TCU_WGMMA_ENABLE - wire [BLOCK_SIZE-1:0] exec_valid_w; - wire [BLOCK_SIZE-1:0] exec_ready_w; - tcu_execute_t exec_data_w [BLOCK_SIZE]; - for (genvar bi = 0; bi < BLOCK_SIZE; ++bi) begin : g_obs - assign exec_valid_w[bi] = core_execute_if[bi].valid; - assign exec_ready_w[bi] = core_execute_if[bi].ready; - assign exec_data_w[bi] = core_execute_if[bi].data; + lsu_req_data_t req_data; + always_comb begin + req_data = req_datas[req_idx]; + req_data.tag = {req_datas[req_idx].tag[BLOCK_TAG_BITS-1:0], req_idx}; + end + assign tcu_mem_if[0].req_valid = req_valid; + assign tcu_mem_if[0].req_data = req_data; + + wire [`UP(BLOCK_IDX_BITS)-1:0] rsp_idx = tcu_mem_if[0].rsp_data.tag[`UP(BLOCK_IDX_BITS)-1:0]; + lsu_rsp_data_t rsp_data; + always_comb begin + rsp_data = tcu_mem_if[0].rsp_data; + rsp_data.tag = LSU_CLIENT_TAG_WIDTH'(tcu_mem_if[0].rsp_data.tag[LSU_CLIENT_TAG_WIDTH-1:BLOCK_IDX_BITS]); + end + wire [BLOCK_SIZE-1:0] rsp_readys; + for (genvar b = 0; b < BLOCK_SIZE; ++b) begin : g_rsp + assign block_mem_if[b].rsp_valid = tcu_mem_if[0].rsp_valid && (rsp_idx == `UP(BLOCK_IDX_BITS)'(b)); + assign block_mem_if[b].rsp_data = rsp_data; + assign rsp_readys[b] = block_mem_if[b].rsp_ready; + end + assign tcu_mem_if[0].rsp_ready = rsp_readys[rsp_idx]; + + for (genvar b = 1; b < `VX_CFG_NUM_LSU_BLOCKS; ++b) begin : g_tieoff + assign tcu_mem_if[b].req_valid = 1'b0; + assign tcu_mem_if[b].req_data = '0; + assign tcu_mem_if[b].rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if[b].req_ready) + `UNUSED_VAR (tcu_mem_if[b].rsp_valid) + `UNUSED_VAR (tcu_mem_if[b].rsp_data) + end end - - wire [BLOCK_SIZE-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tbuf_rs1_data; - wire [BLOCK_SIZE-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tbuf_rs2_data; - wire [BLOCK_SIZE-1:0] tbuf_ready_eff; - - VX_tcu_wgmma #( - .INSTANCE_ID (`SFORMATF(("%s-wgmma", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE) - ) wgmma ( - .clk (clk), - .reset (reset), - `ifdef PERF_ENABLE - .tcu_perf (tcu_perf), - `endif - .exec_valid (exec_valid_w), - .exec_ready (exec_ready_w), - .exec_data (exec_data_w), - .tcu_lmem_if (tcu_lmem_if), - .tbuf_rs1_data (tbuf_rs1_data), - .tbuf_rs2_data (tbuf_rs2_data), - .tbuf_ready_eff (tbuf_ready_eff) - ); - -`else // !VX_CFG_TCU_WGMMA_ENABLE - -`ifdef PERF_ENABLE - assign tcu_perf.tbuf_stalls = '0; - assign tcu_perf.tbuf_cache_hits = '0; - assign tcu_perf.lmem_reads = '0; - assign tcu_perf.wgmma_instrs = '0; - assign tcu_perf.wgmma_stalls = '0; `endif -`endif // VX_CFG_TCU_WGMMA_ENABLE - - // ----------------------------------------------------------------------- - // VX_tcu_agu — warp-level AGU for TCU_LD instructions. - // Drives meta_wr signals broadcast to every tcu_core so wmma_sp on - // any block sees the loaded metadata. - // ----------------------------------------------------------------------- -`ifdef TCU_META_ENABLE - // The AGU streams NUM_TCU_LANES operands over the LSU memory client, whose - // request/response mask and data are sized by NUM_LSU_LANES; the two lane - // counts must match for the per-lane completion tracking to be correct. - `STATIC_ASSERT (NUM_LANES == `VX_CFG_NUM_LSU_LANES, ("VX_tcu_agu requires NUM_TCU_LANES == NUM_LSU_LANES")); - - wire agu_meta_wr_en; - wire [NW_WIDTH-1:0] agu_meta_wr_wid; - wire [4:0] agu_meta_wr_idx; - wire [TCU_BLOCK_CAP-1:0][`VX_CFG_XLEN-1:0] agu_meta_wr_data; - - VX_tcu_agu #( - .INSTANCE_ID (`SFORMATF(("%s-agu", INSTANCE_ID))), - .BLOCK_SIZE (BLOCK_SIZE), - .NUM_LANES (NUM_LANES) - ) agu ( - .clk (clk), - .reset (reset), - .per_block_ld_valid (agu_ld_valid), - .per_block_ld_data (agu_ld_data), - .per_block_ld_ready (agu_ld_ready), - .client_if (tcu_mem_if), - .meta_wr_en (agu_meta_wr_en), - .meta_wr_wid (agu_meta_wr_wid), - .meta_wr_idx (agu_meta_wr_idx), - .meta_wr_data (agu_meta_wr_data), - .result_valid (agu_result_valid), - .result_data (agu_result_data), - .result_ready (agu_result_ready) - ); -`endif - - // ----------------------------------------------------------------------- - // TCU core instances - // ----------------------------------------------------------------------- - for (genvar block_idx = 0; block_idx < BLOCK_SIZE; ++block_idx) begin : g_blocks - VX_tcu_core #( - .INSTANCE_ID (`SFORMATF(("%s-fused%0d", INSTANCE_ID, block_idx))) - ) tcu_core ( + + VX_tcu_block #( + .INSTANCE_ID (INSTANCE_ID), + .BLOCK_STR (`SFORMATF(("%0d", block_idx))) + ) tcu_block ( `SCOPE_IO_BIND (block_idx) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tbuf_rs1_data (tbuf_rs1_data[block_idx]), - .tbuf_rs2_data (tbuf_rs2_data[block_idx]), - .tbuf_ready (tbuf_ready_eff[block_idx]), + .exec_valid (exec_valid[block_idx]), + .exec_ready (exec_ready[block_idx]), + .exec_data (exec_data[block_idx]), + .tbuf_rs1_data (tbuf_rs1_data[block_idx]), + .tbuf_rs2_data (tbuf_rs2_data[block_idx]), + .tbuf_ready (tbuf_ready[block_idx]), `endif `ifdef TCU_META_ENABLE - .ext_meta_wr_en (agu_meta_wr_en), - .ext_meta_wr_wid (agu_meta_wr_wid), - .ext_meta_wr_idx (agu_meta_wr_idx), - .ext_meta_wr_data (agu_meta_wr_data), + .tcu_mem_if (block_mem_if[block_idx]), `endif - .execute_if (core_execute_if[block_idx]), - .result_if (core_result_if[block_idx]) + .execute_if (per_block_execute_if[block_idx]), + .result_if (per_block_result_if[block_idx]) ); end diff --git a/hw/rtl/tcu/VX_tcu_uops.sv b/hw/rtl/tcu/VX_tcu_uops.sv index 974866a235..437c89c4ad 100644 --- a/hw/rtl/tcu/VX_tcu_uops.sv +++ b/hw/rtl/tcu/VX_tcu_uops.sv @@ -75,7 +75,6 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( `ifdef VX_CFG_TCU_SPARSE_ENABLE localparam LG_A_SB = $clog2(TCU_A_SUB_BLOCKS); `endif - localparam LG_B_SB = $clog2(TCU_B_SUB_BLOCKS); `UNUSED_VAR ({clk, reset, start, advance, uop_idx}) @@ -277,21 +276,11 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( logic [`UP(CTR_W)-1:0] rs2_offset; logic [`UP(CTR_W)-1:0] rs3_offset; - // Bank-conflict-free register offset formulas. - // Permutes A, B, C offsets so all three operands always land in - // different GPR banks, eliminating all RF read-port stalls. - // - Pattern A (LG_B_SB==0): A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} - // - Pattern B (LG_B_SB>0): A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} - wire [`UP(CTR_W)-1:0] ra_off, rb_off, rc_off; - if (LG_B_SB == 0) begin : g_bcfree_bsub1 - assign ra_off = `UP(CTR_W)'({m_index[0], ~m_index[`UP(LG_M)-1], k_index[0]}); - assign rb_off = `UP(CTR_W)'({n_index[0] ^ k_index[0], ~k_index[0]}); - assign rc_off = `UP(CTR_W)'({m_index[0], ~m_index[`UP(LG_M)-1], ~(m_index[`UP(LG_M)-1] ^ n_index[0])}); - end else begin : g_bcfree_bsub2 - assign ra_off = `UP(CTR_W)'({k_index[0], ~m_index[0], m_index[0] ^ k_index[`UP(LG_K)-1]}); - assign rb_off = `UP(CTR_W)'({k_index[0], k_index[`UP(LG_K)-1] ^ n_index[`UP(LG_N)-1], ~n_index[`UP(LG_N)-1]}); - assign rc_off = `UP(CTR_W)'({n_index[0], ~m_index[0], n_index[`UP(LG_N)-1]}); - end + // Bank-conflict-free register offsets: all three operands of a step land + // in different GPR banks, so the RF read ports never stall. + wire [`UP(CTR_W)-1:0] ra_off = `UP(CTR_W)'(tcu_ra_off(m_index, k_index)); + wire [`UP(CTR_W)-1:0] rb_off = `UP(CTR_W)'(tcu_rb_off(n_index, k_index)); + wire [`UP(CTR_W)-1:0] rc_off = `UP(CTR_W)'(tcu_rc_off(m_index, n_index)); `ifdef VX_CFG_TCU_SPARSE_ENABLE if (SYM_SPARSE) begin : g_sym_off @@ -301,12 +290,9 @@ module VX_tcu_uops import VX_tcu_pkg::*, VX_gpu_pkg::*; ( assign rs2_offset = is_sparse ? `UP(CTR_W)'(n_sp) : rb_off; assign rs3_offset = is_sparse ? (`UP(CTR_W)'(eff_ctr) >> 1) : rc_off; end else begin : g_def_off - // Bank-conflict-free sparse register offsets (non-sym sparse). - // A stays identity; B and C are permuted for 0 stalls. - // B_off_sp = {n[hi], ~(n[0]^k[0]), ~k[0]} - // C_off_sp = {n[hi], m[0], ~(m[0]^n[0])} - wire [`UP(CTR_W)-1:0] rb_off_sp = `UP(CTR_W)'({n_index[`UP(LG_N)-1], ~(n_index[0] ^ k_index[0]), ~k_index[0]}); - wire [`UP(CTR_W)-1:0] rc_off_sp = `UP(CTR_W)'({n_index[`UP(LG_N)-1], m_index[0], ~(m_index[0] ^ n_index[0])}); + // Non-symmetric sparse: A stays identity; B and C are permuted. + wire [`UP(CTR_W)-1:0] rb_off_sp = `UP(CTR_W)'(tcu_rb_off_sp(n_index, k_index)); + wire [`UP(CTR_W)-1:0] rc_off_sp = `UP(CTR_W)'(tcu_rc_off_sp(m_index, n_index)); assign rs1_offset = is_sparse ? ((`UP(CTR_W)'(m_index) >> LG_A_SB) << (LG_K / 2)) | `UP(CTR_W)'(k_index) : ra_off; diff --git a/hw/rtl/tcu/VX_tcu_wgmma.sv b/hw/rtl/tcu/VX_tcu_wgmma.sv index 671a1a8d61..4df2e33579 100644 --- a/hw/rtl/tcu/VX_tcu_wgmma.sv +++ b/hw/rtl/tcu/VX_tcu_wgmma.sv @@ -20,8 +20,8 @@ // CTA lockstep gate, and WGMMA-specific perf counters. // // Observation-only on the dispatch path: this module does not drive -// .ready. The wrapper (VX_tcu_unit) feeds tbuf_ready_eff into tcu_core, -// which produces the actual handshake .ready going back to dispatch. +// .ready. VX_tcu_shared exports tbuf_ready_eff per block; the block's +// tcu_core produces the actual handshake .ready going back to dispatch. module VX_tcu_wgmma import VX_gpu_pkg::*, VX_tcu_pkg::*; #( parameter `STRING INSTANCE_ID = "", diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv index b42dfa5393..e9dfcfa2cf 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_acc.sv @@ -77,19 +77,32 @@ module VX_tcu_tfr_acc import VX_tcu_pkg::*; #( `UNUSED_PIN (cout) ); - // -(a+b) = ~a + ~b + 2: cin=1 yields ~(a+b), the +1 completes the negate. - wire [WO-1:0] sig_neg_raw; + // -(a+b) = ~(a+b-1): compressing the all-ones (-1) operand in first keeps + // the negate to one adder, with no increment behind it. + wire [WO-1:0] dec_sum, dec_carry; + VX_csa_32 #( + .N (WO), + .WIDTH_O (WO) + ) dec_csa ( + .a (sum_vec), + .b (carry_vec), + .c ({WO{1'b1}}), + .sum (dec_sum), + .carry (dec_carry) + ); + + wire [WO-1:0] sig_dec; VX_ks_adder #( .N (WO), .BYPASS (`FORCE_BUILTIN_ADDER(WO)) ) neg_adder ( - .cin (1'b1), - .dataa (~sum_vec), - .datab (~carry_vec), - .sum (sig_neg_raw), + .cin (1'b0), + .dataa (dec_sum), + .datab (dec_carry), + .sum (sig_dec), `UNUSED_PIN (cout) ); - wire [WO-1:0] sig_neg = sig_neg_raw + WO'(1); + wire [WO-1:0] sig_neg = ~sig_dec; assign sign_out = sig_pos[WO-1]; assign sig_out = sign_out ? sig_neg : sig_pos; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv index a62e5a3a12..3b36574d76 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_align.sv @@ -34,7 +34,7 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( input wire [N-1:0][WI-1:0] sigs_in, input wire is_int, - output logic [TCU_EXP_BITS-1:0] max_exp, + output wire [TCU_EXP_BITS-1:0] max_exp, output wire [N-1:0][WO-1:0] sigs_out, output wire [N-1:0] sticky_bits, output wire [N-1:0] fp_negs @@ -44,31 +44,30 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( localparam MAX_PRE_SHIFT = WI - 23; localparam SHIFT_MAG_W = (WI - 1) + MAX_PRE_SHIFT; + localparam SHIFT_W = `CLOG2(SHIFT_MAG_W + 1); + localparam TZ_W = `LOG2UP(SHIFT_MAG_W); - wire [TCU_EXP_BITS-1:0] or_red[N:0]; - wire [N-1:0][7:0] shift_amts; + `STATIC_ASSERT (TCU_EXP_BITS > SHIFT_W, ("invalid shift width: %0d", SHIFT_W)) - // Determine maximum exponent via OR-Reduction Tree - assign or_red[0] = {TCU_EXP_BITS{1'b0}}; - for (genvar i = 0; i < N; i++) begin : g_or_red - assign or_red[i+1] = or_red[i] | (sel_exp[i] ? exponents[i] : {TCU_EXP_BITS{1'b0}}); - end - assign max_exp = or_red[N]; + VX_onehot_mux #( + .DATAW (TCU_EXP_BITS), + .N (N) + ) max_exp_mux ( + .data_in (exponents), + .sel_in (sel_exp), + .data_out (max_exp) + ); // Per-lane right-shift distance from the maximum exponent (max_exp is the // largest input, so the difference is non-negative). - for (genvar i = 0; i < N; i++) begin : g_shift_amts - wire [TCU_EXP_BITS-1:0] shift_full = max_exp - exponents[i]; - if (TCU_EXP_BITS > 8) begin : g_sat - assign shift_amts[i] = (|shift_full[TCU_EXP_BITS-1:8]) ? 8'hFF : shift_full[7:0]; - end else begin : g_no_sat - assign shift_amts[i] = 8'(shift_full); - end + wire [N-1:0][TCU_EXP_BITS-1:0] shift_amts; + for (genvar i = 0; i < N; ++i) begin : g_shift_amts + assign shift_amts[i] = max_exp - exponents[i]; end - // Align significands and form the accumulator operands for (genvar i = 0; i < N; ++i) begin : g_align_lanes - wire [7:0] shift_amt = shift_amts[i]; + wire shift_hi = (| shift_amts[i][TCU_EXP_BITS-1:SHIFT_W]); + wire [SHIFT_W-1:0] shift_lo = shift_amts[i][SHIFT_W-1:0]; wire lane_en; if (i == N-1) begin : g_c_en @@ -90,13 +89,24 @@ module VX_tcu_tfr_align import VX_tcu_pkg::*; #( end // 3. Shift adjustment - wire is_overshift = (shift_amt >= 8'(SHIFT_MAG_W)); - wire [SHIFT_MAG_W-1:0] shift_res_full = mag_shifted >> shift_amt; + wire is_overshift = shift_hi || (shift_lo >= SHIFT_W'(SHIFT_MAG_W)); + wire [SHIFT_MAG_W-1:0] shift_res_full = mag_shifted >> shift_lo; wire [WI:0] adj_mag = is_overshift ? '0 : shift_res_full[WI:0]; - // 4. Sticky Calculation - wire [SHIFT_MAG_W-1:0] sticky_check_shift = mag_shifted << (8'(SHIFT_MAG_W) - shift_amt); - assign sticky_bits[i] = lane_en & (is_overshift ? (|mag_shifted) : (|sticky_check_shift)); + // 4. Sticky: a set bit shifts out iff the shift exceeds the trailing + // zero count, which comes straight from the registered magnitude and + // so resolves in parallel with the exponent difference. + wire [TZ_W-1:0] mag_tz; + wire mag_nz; + VX_lzc #( + .N (SHIFT_MAG_W), + .REVERSE (1) + ) mag_tzc ( + .data_in (mag_shifted), + .data_out (mag_tz), + .valid_out (mag_nz) + ); + assign sticky_bits[i] = lane_en && mag_nz && (shift_hi || (shift_lo > SHIFT_W'(mag_tz))); // 5. Operand formation: masked, negated (fp) or sign-extended (int). wire fp_neg = in_sign & lane_en & ~is_int; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv index bdd2b2354e..8f65a751ac 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f4.sv @@ -94,6 +94,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][EXP_TERM_W_MXFP4-1:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -102,6 +103,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -119,13 +121,11 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign b_exp[0] = raw_b[2] & ~raw_b[1]; assign b_exp[1] = raw_b[2] & raw_b[1]; - wire signed [9:0] sf_exp_a = $signed({1'b0, sf_a}) - 10'sd127; - wire signed [9:0] sf_exp_b = $signed({1'b0, sf_b}) - 10'sd127; - wire signed [5:0] exp_biased_raw = 6'(10'(EXP_ADJ_MXFP4) - + sf_exp_a - + sf_exp_b - + 10'(a_exp) - + 10'(b_exp)); + // the block scales are common to the lane's terms: they join its + // exponent, not this alignment field + wire [EXP_TERM_W_MXFP4-1:0] exp_biased_raw = EXP_ADJ_MXFP4 + + EXP_TERM_W_MXFP4'(a_exp) + + EXP_TERM_W_MXFP4'(b_exp); assign term_exp_biased[j] = term_valid[j] ? exp_biased_raw : '0; end @@ -250,10 +250,16 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_sig_mxfp4[i] = {sum_sign & ~is_zero_out, abs_sum[23:0]}; assign sig_zero_mxfp4[i] = is_zero_out; - assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED_MXFP4); + + // The E8M0 scales (2^-254 .. 2^256 together) join the lane exponent. + // A lane with a term stays within [15, 529]; an empty one wraps, but + // its exponent is cleared with its zero significand. + assign result_exp_mxfp4[i] = EXP_W'(max_exp_biased) + EXP_BASE_BIASED_MXFP4 + + EXP_W'(sf_a) + EXP_W'(sf_b) - EXP_W'(254); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_mxfp4[i].is_nan = 1'b0; + // An E8M0 scale of 0xFF is NaN, and so is every element it scales. + assign exceptions_mxfp4[i].is_nan = ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|term_on); assign exceptions_mxfp4[i].is_inf = 1'b0; assign exceptions_mxfp4[i].sign = 1'b0; end @@ -284,13 +290,14 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; for (genvar i = 0; i < TCK; ++i) begin : g_lane_nvfp4 localparam K_WORD = i / 2; - `UNUSED_VAR ({sf_a[7], sf_b[7]}) - // e4m3 scale factor mantissa mul + // UE4M3 scale factors (unsigned: bit 7 is padding); 0x7F is NaN. + `UNUSED_VAR ({sf_a[7], sf_b[7]}) wire [3:0] sf_man_a = {1'b1, sf_a[2:0]}; wire [3:0] sf_man_b = {1'b1, sf_b[2:0]}; wire [3:0] sf_exp_a = sf_a[6:3]; wire [3:0] sf_exp_b = sf_b[6:3]; + wire sf_nan = (sf_a[6:0] == 7'h7F) || (sf_b[6:0] == 7'h7F); wire [7:0] sf_man_prod; VX_tcu_tfr_wmul #( @@ -307,6 +314,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; wire [3:0][23:0] term_mag_shifted; wire [3:0][5:0] term_exp_biased; wire [3:0] term_valid; + wire [3:0] term_on; wire [3:0] term_sign; wire [3:0][1:0] a_man, b_man; wire [3:0][3:0] f4_man_prod; @@ -316,6 +324,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; localparam OFF = (i % 2) * 16 + j * 4; wire lane_valid = vld_mask[i * 4 + j]; + assign term_on[j] = lane_valid; wire [3:0] raw_a = a_row[K_WORD][OFF +: 4]; wire [3:0] raw_b = b_col[K_WORD][OFF +: 4]; @@ -511,7 +520,7 @@ module VX_tcu_tfr_mul_f4 import VX_tcu_pkg::*; assign result_exp_nvfp4[i] = EXP_W'(max_exp_biased) + EXP_W'(EXP_BASE_BIASED); // The sign field is only consumed for infinity lanes downstream. - assign exceptions_nvfp4[i].is_nan = 1'b0; + assign exceptions_nvfp4[i].is_nan = sf_nan && (|term_on); assign exceptions_nvfp4[i].is_inf = 1'b0; assign exceptions_nvfp4[i].sign = 1'b0; end diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv index 60d3c41503..dedd90d8d7 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_f8.sv @@ -194,24 +194,28 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire [7:0] bias_sel = is_bfloat ? BIAS_CONST_BF8 : BIAS_CONST_FP8; - wire [EXP_W-1:0] max_pre_sum_cpa, bias_sel_cpa; - wire [EXP_W-1:0] final_exp; + // One bit wider than the exponent, signed: the E8M0 scales (2^-254 .. + // 2^256 together) can take a lane below the exponent range, where it + // only adds to the sticky bit. + localparam SX = EXP_W + 1; + wire [SX-1:0] max_pre_sum_cpa, bias_sel_cpa; + wire [SX-1:0] final_exp_s; `ifdef VX_CFG_TCU_MX_ENABLE - wire [EXP_W-1:0] bias_cpa_sum, bias_cpa_carry; - wire [3*EXP_W-1:0] sf_comp = fmt_f[3] ? {EXP_W'(sf_a), EXP_W'(sf_b), -EXP_W'(254)} : (3*EXP_W)'(0); + wire [SX-1:0] bias_cpa_sum, bias_cpa_carry; + wire [3*SX-1:0] sf_comp = fmt_f[3] ? {SX'(sf_a), SX'(sf_b), -SX'(254)} : (3*SX)'(0); VX_csa_tree #( .N(4), - .W(EXP_W), - .S(EXP_W) + .W(SX), + .S(SX) ) exp_sf_csa ( - .operands ({EXP_W'(bias_sel), sf_comp}), + .operands ({SX'(bias_sel), sf_comp}), .sum (bias_cpa_sum), .carry (bias_cpa_carry) ); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_bias_add ( .dataa(bias_cpa_sum), .datab(bias_cpa_carry), @@ -220,19 +224,20 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; `UNUSED_PIN(cout) ); `else - assign bias_sel_cpa = EXP_W'(bias_sel); + assign bias_sel_cpa = SX'(bias_sel); `endif - assign max_pre_sum_cpa = EXP_W'(max_pre_sum); + assign max_pre_sum_cpa = SX'(max_pre_sum); VX_ks_adder #( - .N(EXP_W), - .BYPASS(`FORCE_BUILTIN_ADDER(EXP_W)) + .N(SX), + .BYPASS(`FORCE_BUILTIN_ADDER(SX)) ) exp_final_add ( - .dataa(EXP_W'(max_pre_sum_cpa)), - .datab(EXP_W'(bias_sel_cpa)), + .dataa(max_pre_sum_cpa), + .datab(bias_sel_cpa), .cin(1'b0), - .sum(final_exp), + .sum(final_exp_s), `UNUSED_PIN(cout) ); + wire [EXP_W-1:0] final_exp = final_exp_s[SX-1] ? '0 : final_exp_s[EXP_W-1:0]; // The two fp8 mantissa products in a lane are independent unsigned 4x4 // multiplies — small enough to pack into a single DSP48 (USE_DSP), @@ -309,8 +314,22 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; `UNUSED_PIN(cout) ); + // The reverse difference runs beside the forward one, so the magnitude + // is a select rather than a negate behind the subtract. + wire [24:0] sub_rev; + VX_ks_adder #( + .N(25), + .BYPASS(`FORCE_BUILTIN_ADDER(25)) + ) sub_rev_adder ( + .dataa(sig_min_ext), + .datab(~sig_max_ext), + .cin(1'b1), + .sum(sub_rev), + `UNUSED_PIN(cout) + ); + wire sub_neg = sub_raw[24]; - wire [24:0] sub_abs = sub_neg ? -sub_raw : sub_raw; + wire [24:0] sub_abs = sub_neg ? sub_rev : sub_raw; wire [24:0] sig_add_raw = do_sub ? sub_abs : add_raw; // Scaling by 1 avoids renormalization. @@ -335,7 +354,13 @@ module VX_tcu_tfr_mul_f8 import VX_tcu_pkg::*; wire add_nan = (pos_inf_0 && neg_inf_1) || (neg_inf_0 && pos_inf_1); - wire any_nan = (|(nan_sel & lane_valid)) || add_nan; + `ifdef VX_CFG_TCU_MX_ENABLE + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + wire sf_nan = fmt_f[3] && ((sf_a == 8'hFF) || (sf_b == 8'hFF)) && (|lane_valid); + `else + wire sf_nan = 1'b0; + `endif + wire any_nan = (|(nan_sel & lane_valid)) || add_nan || sf_nan; wire any_inf = (|(inf_sel & lane_valid)) && ~any_nan; wire final_sign_inf = inf_sel[0] ? sign_sel[0] : sign_sel[1]; diff --git a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv index 14715ff4a4..738145e053 100644 --- a/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv +++ b/hw/rtl/tcu/tfr/VX_tcu_tfr_mul_i4.sv @@ -55,6 +55,7 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( for (genvar i = 0; i < TCK; ++i) begin : g_lane wire signed [3:0][9:0] y_prod_i4; + wire [9:0] y_corr; // Shared per-element extraction. wire [3:0][3:0] raw_a, raw_b; @@ -127,11 +128,16 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( .data_in (psign), .data_out (psign_r) ); + // -m = ~m + 1: the negated products enter the tree as ones' + // complements and their +1s as one count operand, so no negate + // sits between the DSP and the tree. + wire [3:0] pneg = psign_r & pvalid_r; for (genvar j = 0; j < 4; ++j) begin : g_sign - wire signed [9:0] pf = psign_r[j] ? -$signed({2'b0, magp[j]}) - : $signed({2'b0, magp[j]}); - assign y_prod_i4[j] = pf & {10{pvalid_r[j]}}; + wire [9:0] pm = {2'b0, magp[j]} & {10{pvalid_r[j]}}; + assign y_prod_i4[j] = pneg[j] ? ~pm : pm; end + wire [2:0] pneg_cnt = 3'(pneg[0]) + 3'(pneg[1]) + 3'(pneg[2]) + 3'(pneg[3]); + assign y_corr = 10'(pneg_cnt); end else begin : g_lut for (genvar j = 0; j < 4; ++j) begin : g_mul wire signed [4:0] s_a = is_signed_int ? $signed({raw_a[j][3], raw_a[j]}) : $signed({1'b0, raw_a[j]}); @@ -150,15 +156,16 @@ module VX_tcu_tfr_mul_i4 import VX_tcu_pkg::*; #( ); assign y_prod_i4[j] = prod_r & {10{pvalid_r[j]}}; end + assign y_corr = '0; end wire [9:0] y_i4_sum, y_i4_carry; VX_csa_tree #( - .N (4), + .N (5), .W (10), .S (10) ) i4_csa ( - .operands (y_prod_i4), + .operands ({y_corr, y_prod_i4}), .sum (y_i4_sum), .carry (y_i4_carry) ); diff --git a/hw/rtl/tex/VX_tex_core.sv b/hw/rtl/tex/VX_tex_core.sv index 7ffec13f8c..490b48965f 100644 --- a/hw/rtl/tex/VX_tex_core.sv +++ b/hw/rtl/tex/VX_tex_core.sv @@ -48,7 +48,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( tex_dcrs_t tex_dcrs; VX_tex_dcr #( - .INSTANCE_ID ($sformatf("%s-dcr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-dcr", INSTANCE_ID))), .NUM_STAGES (`VX_TEX_STAGE_COUNT) ) tex_dcr ( .clk (clk), @@ -132,7 +132,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( // sampler, for the same reason the format does: the state a sample merges // under is the state it was issued under. VX_tex_addr #( - .INSTANCE_ID ($sformatf("%s-addr", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-addr", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32), .NUM_LANES (NUM_LANES) ) tex_addr ( @@ -176,7 +176,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire mem_rsp_ready; VX_tex_mem #( - .INSTANCE_ID ($sformatf("%s-mem", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-mem", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH + TEX_FORMAT_BITS + 32 + BLEND_FRAC_W + BORDER_W + LODFRAC_W), .NUM_LANES (NUM_LANES) ) tex_mem ( @@ -218,7 +218,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( wire [TAG_WIDTH-1:0] mem_rsp_tag = mem_rsp_info[(LODFRAC_W + BORDER_W + BLEND_FRAC_W + 32 + TEX_FORMAT_BITS) +: TAG_WIDTH]; VX_tex_sampler #( - .INSTANCE_ID ($sformatf("%s-sampler", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-sampler", INSTANCE_ID))), .REQ_TAGW (TAG_WIDTH), .NUM_LANES (NUM_LANES) ) tex_sampler ( @@ -259,8 +259,8 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ); `ifdef SCOPE -`ifdef DBG_SCOPE_TEX `SCOPE_IO_SWITCH (1); +`ifdef DBG_SCOPE_TEX wire cache_bus_req_fire_0 = cache_bus_if[0].req_valid && cache_bus_if[0].req_ready; wire cache_bus_rsp_fire_0 = cache_bus_if[0].rsp_valid && cache_bus_if[0].rsp_ready; wire [TCACHE_TAG_WIDTH-UUID_WIDTH-1:0] cache_bus_req_tag = cache_bus_if[0].req_data.tag[TCACHE_TAG_WIDTH-UUID_WIDTH-1:0]; @@ -324,7 +324,7 @@ module VX_tex_core import VX_gpu_pkg::*; import VX_tex_pkg::*; #( reset_negedge, 1'b0, 4096 ); `else - `SCOPE_IO_UNUSED() + `SCOPE_IO_UNUSED(0) `endif `endif `ifdef CHIPSCOPE diff --git a/hw/rtl/tex/VX_tex_define.vh b/hw/rtl/tex/VX_tex_define.vh index 7748778be7..d8e95e985a 100644 --- a/hw/rtl/tex/VX_tex_define.vh +++ b/hw/rtl/tex/VX_tex_define.vh @@ -25,8 +25,12 @@ `define TEX_FXD_HALF (`TEX_FXD_ONE >> 1) `define TEX_FXD_MASK (`TEX_FXD_ONE - 1) +// Texture base: a cache-block index. Under VM it is a virtual address, which +// spans the whole 32-bit space on Sv32. `ifdef VX_CFG_XLEN_64 `define TEX_ADDR_BITS 32 +`elsif VX_CFG_VM_ENABLE +`define TEX_ADDR_BITS 26 `else `define TEX_ADDR_BITS 25 `endif diff --git a/hw/rtl/tex/VX_tex_mem.sv b/hw/rtl/tex/VX_tex_mem.sv index d712174204..597c45cd95 100644 --- a/hw/rtl/tex/VX_tex_mem.sv +++ b/hw/rtl/tex/VX_tex_mem.sv @@ -128,7 +128,7 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( ) lsu_mem_if(); VX_mem_scheduler #( - .INSTANCE_ID ($sformatf("%s-memsched", INSTANCE_ID)), + .INSTANCE_ID (`SFORMATF(("%s-memsched", INSTANCE_ID))), .CORE_REQS (TEX_MEM_REQS), .MEM_CHANNELS(TCACHE_NUM_REQS), .WORD_SIZE (4), @@ -159,6 +159,8 @@ module VX_tex_mem import VX_gpu_pkg::*; import VX_tex_pkg::*; #( .core_req_data ('0), .core_req_tag (mem_req_tag), .core_req_ready (mem_req_ready), + `UNUSED_PIN (core_req_ready_rd), + `UNUSED_PIN (core_req_ready_wr), `UNUSED_PIN (req_queue_empty), `UNUSED_PIN (req_queue_rw_notify), diff --git a/hw/rtl/tex/VX_tex_unit.sv b/hw/rtl/tex/VX_tex_unit.sv index 56c19b2be8..d49391f3e1 100644 --- a/hw/rtl/tex/VX_tex_unit.sv +++ b/hw/rtl/tex/VX_tex_unit.sv @@ -32,7 +32,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( parameter `STRING INSTANCE_ID = "", - parameter CORE_ID = 0, parameter NUM_LANES = `VX_CFG_NUM_THREADS ) ( input wire clk, @@ -46,7 +45,6 @@ module VX_tex_unit import VX_gpu_pkg::*, VX_tex_pkg::*; #( VX_tex_bus_if.master tex_bus_if ); `UNUSED_SPARAM (INSTANCE_ID) - `UNUSED_PARAM (CORE_ID) localparam REQ_QUEUE_BITS = `LOG2UP(`VX_CFG_TEX_REQ_QUEUE_SIZE); localparam PID_W = `LOG2UP(`VX_CFG_NUM_THREADS / NUM_LANES); diff --git a/hw/rtl/vm/VX_mmu.sv b/hw/rtl/vm/VX_mmu.sv index 9166421400..5b4979afc3 100644 --- a/hw/rtl/vm/VX_mmu.sv +++ b/hw/rtl/vm/VX_mmu.sv @@ -28,6 +28,16 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, parameter REPLAY_DEPTH = 2, parameter EXEC_SIDE = 0, + // 0: the clients only read, so a parked request keeps no write data or + // byte enables (don't-care on a read). + parameter WRITE_ENABLE = 1, + // Response stage per lane toward the client (TO_OUT_BUF encoding). Keeps + // the injected-response select off the client's response cone. + parameter RSP_OUT_BUF = 0, + // 1: a parking lane is first registered, and allocates in the miss + // station the next cycle. Takes the lane arbitration off the miss + // station's update cone; a lane on the staged page holds meanwhile. + parameter PARK_REG = 0, parameter DATA_SIZE = DCACHE_WORD_SIZE, parameter TAG_WIDTH = DCACHE_TAG_WIDTH_BASE, parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE), @@ -49,10 +59,39 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( VX_tlb_flush_if.slave flush_if, VX_mmu_fault_if.master fault_if, + // Per lane: nothing of the lane's is in the MMU (front stages, fault slot, + // and, where flushes are ordered, the miss station). + output wire [NUM_REQS-1:0] lane_idle, output wire empty ); `UNUSED_PARAM (INSTANCE_ID) + // The client's bus as the MMU drives it: requests pass straight through to + // the lookup (the client registers its own request output), responses + // through an optional MMU-owned stage. + VX_mem_bus_if #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH) + ) req_if [NUM_REQS] (); + + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_rsp_buf + VX_mem_bus_slice #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH), + .ATTR_WIDTH (ATTR_WIDTH), + .REQ_OUT_BUF (0), + .RSP_OUT_BUF (RSP_OUT_BUF) + ) rsp_buf ( + .clk (clk), + .reset (reset), + .bus_in_if (core_bus_if[l]), + .bus_out_if (req_if[l]) + ); + end + localparam DATA_WIDTH = DATA_SIZE * 8; localparam LANE_W = `UP(`CLOG2(NUM_REQS)); localparam PAGE_OFFSET_W = `VX_VM_PAGE_LOG2_SIZE - `CLOG2(DATA_SIZE); @@ -65,7 +104,9 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam F_ADDR_LO = F_DATA_LO + DATA_WIDTH; localparam F_RW_LO = F_ADDR_LO + ADDR_WIDTH; localparam FIELDS_W = F_RW_LO + 1; - localparam PAYLOAD_W = LANE_W + FIELDS_W; + // Parked fields: a read-only client's drop {data, byteen}. + localparam PFIELDS_W = (WRITE_ENABLE != 0) ? FIELDS_W : (FIELDS_W - DATA_WIDTH - DATA_SIZE); + localparam PAYLOAD_W = LANE_W + PFIELDS_W; function automatic tlb_access_e access_of (input logic rw); @@ -94,17 +135,17 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] mshr_match; for (genvar l = 0; l < NUM_REQS; ++l) begin : g_decode - assign req_valid[l] = core_bus_if[l].req_valid; - assign req_rw[l] = core_bus_if[l].req_data.rw; - assign req_addr[l] = core_bus_if[l].req_data.addr; - assign req_attr[l] = core_bus_if[l].req_data.attr[ATTR_WIDTH-1:0]; + assign req_valid[l] = req_if[l].req_valid; + assign req_rw[l] = req_if[l].req_data.rw; + assign req_addr[l] = req_if[l].req_data.addr; + assign req_attr[l] = req_if[l].req_data.attr[ATTR_WIDTH-1:0]; assign req_fields[l] = { - core_bus_if[l].req_data.rw, - core_bus_if[l].req_data.addr, - core_bus_if[l].req_data.data, - core_bus_if[l].req_data.byteen, - core_bus_if[l].req_data.attr[ATTR_WIDTH-1:0], - core_bus_if[l].req_data.tag[TAG_WIDTH-1:0] + req_if[l].req_data.rw, + req_if[l].req_data.addr, + req_if[l].req_data.data, + req_if[l].req_data.byteen, + req_if[l].req_data.attr[ATTR_WIDTH-1:0], + req_if[l].req_data.tag[TAG_WIDTH-1:0] }; assign req_bypass[l] = ~vm_active || req_attr[l][MEM_ATTR_FLUSH_OFFS] @@ -150,6 +191,7 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .MSHR_SIZE (MSHR_SIZE), .REPLAY_DEPTH (REPLAY_DEPTH), .PAYLOAD_W (PAYLOAD_W), + .ALLOC_PROBE (PARK_REG), .ID_WIDTH (ID_WIDTH) ) tlb ( .clk (clk), @@ -195,13 +237,20 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] cat_park; wire [NUM_REQS-1:0] cat_hit; wire [NUM_REQS-1:0] cat_pfault; + wire [NUM_REQS-1:0] stage_hold; // the lane's page is in the park stage + wire [NUM_REQS-1:0] pf_accept; // permission fault taken this cycle + wire [NUM_REQS-1:0] pf_rsp_busy; // a fault response is owed on the lane + wire [NUM_REQS-1:0] lane_parked; // the lane has requests in the miss station for (genvar l = 0; l < NUM_REQS; ++l) begin : g_cat + // A flush token holds behind its lane's parked requests (see g_parked). + wire flush_hold = req_attr[l][MEM_ATTR_FLUSH_OFFS] && lane_parked[l]; + wire xlat = req_valid[l] && !req_bypass[l] && !stage_hold[l]; assign cam_acc[l] = req_acc[l]; - assign cat_bypass[l] = req_valid[l] && req_bypass[l]; - assign cat_park[l] = req_valid[l] && !req_bypass[l] && (mshr_match[l] || !cam_hit[l]); - assign cat_hit[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && cam_perm[l]; - assign cat_pfault[l] = req_valid[l] && !req_bypass[l] && !mshr_match[l] && cam_hit[l] && !cam_perm[l]; + assign cat_bypass[l] = req_valid[l] && req_bypass[l] && !flush_hold; + assign cat_park[l] = xlat && (mshr_match[l] || !cam_hit[l]); + assign cat_hit[l] = xlat && !mshr_match[l] && cam_hit[l] && cam_perm[l]; + assign cat_pfault[l] = xlat && !mshr_match[l] && cam_hit[l] && !cam_perm[l]; end // Park arbitration: at most one lane parks a miss per cycle (lowest lane). @@ -210,13 +259,16 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( localparam PARK_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + FIELDS_W; wire [NUM_REQS-1:0] park_sel; + wire sel_valid; + wire sel_ready; + wire [LANE_W-1:0] sel_lane; VX_priority_encoder #( .N (NUM_REQS) ) park_enc ( .data_in (cat_park), .onehot_out (park_sel), - .index_out (park_lane), - .valid_out (park_valid) + .index_out (sel_lane), + .valid_out (sel_valid) ); wire [NUM_REQS-1:0][PARK_W-1:0] park_src; @@ -232,16 +284,79 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .sel_in (park_sel), .data_out (park_out) ); - assign park_vpn = park_out[PARK_W-1 -: TLB_VPN_WIDTH]; - assign park_access = tlb_access_e'(park_out[FIELDS_W+1 +: $bits(tlb_access_e)]); - assign park_amo = park_out[FIELDS_W]; - assign park_payload = {park_lane, park_out[FIELDS_W-1:0]}; + wire [TLB_VPN_WIDTH-1:0] sel_vpn = park_out[PARK_W-1 -: TLB_VPN_WIDTH]; // --------------------------------------------------------------------- // Replay decode + translation splice // --------------------------------------------------------------------- - wire [LANE_W-1:0] replay_lane = replay_payload[FIELDS_W +: LANE_W]; - wire [FIELDS_W-1:0] replay_fields = replay_payload[FIELDS_W-1:0]; + wire [LANE_W-1:0] replay_lane = replay_payload[PFIELDS_W +: LANE_W]; + wire [PFIELDS_W-1:0] replay_pfields = replay_payload[PFIELDS_W-1:0]; + wire [PFIELDS_W-1:0] park_pfields; + wire [FIELDS_W-1:0] replay_fields; + + if (WRITE_ENABLE != 0) begin : g_park_wr + assign park_pfields = park_out[FIELDS_W-1:0]; + assign replay_fields = replay_pfields; + end else begin : g_park_ro + // A read's data and byte enables are don't-care below the L1; the + // replay presents it as a full-line read. + assign park_pfields = {park_out[FIELDS_W-1:F_ADDR_LO], park_out[F_BYTEEN_LO-1:0]}; + assign replay_fields = {replay_pfields[PFIELDS_W-1:F_BYTEEN_LO], DATA_WIDTH'(0), {DATA_SIZE{1'b1}}, + replay_pfields[F_BYTEEN_LO-1:0]}; + `UNUSED_VAR (park_out[F_ADDR_LO-1:F_BYTEEN_LO]) + `RUNTIME_ASSERT (~(sel_valid && sel_ready) || ~park_out[F_RW_LO], + ("%t: *** %s read-only MMU parked a write, addr=0x%0h", $time, INSTANCE_ID, + park_out[F_ADDR_LO +: ADDR_WIDTH])) + end + + // The selected lane's park, straight into the miss station or through a + // one-entry stage. The stage accepts only while empty, so its ready is a + // flop; it allocates the next cycle against its own VPN (ALLOC_PROBE). + localparam STAGE_W = TLB_VPN_WIDTH + $bits(tlb_access_e) + 1 + LANE_W + PFIELDS_W; + wire [STAGE_W-1:0] sel_park = {sel_vpn, park_out[FIELDS_W+1 +: $bits(tlb_access_e)], + park_out[FIELDS_W], sel_lane, park_pfields}; + wire [STAGE_W-1:0] stage_park; + wire stage_valid; + + if (PARK_REG != 0) begin : g_park_reg + reg stage_valid_r; + reg [STAGE_W-1:0] stage_r; + always @(posedge clk) begin + if (reset) begin + stage_valid_r <= 1'b0; + end else if (sel_valid && sel_ready) begin + stage_valid_r <= 1'b1; + end else if (park_ready) begin + stage_valid_r <= 1'b0; + end + end + always @(posedge clk) begin + if (sel_ready) begin + stage_r <= sel_park; + end + end + assign sel_ready = ~stage_valid_r; + assign stage_valid = stage_valid_r; + assign stage_park = stage_r; + // Until it allocates, a later request on the staged page holds: it would + // miss the miss-station probe and could overtake the staged request. + wire [TLB_VPN_WIDTH-1:0] stage_vpn = stage_r[STAGE_W-1 -: TLB_VPN_WIDTH]; + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_hold + assign stage_hold[l] = stage_valid_r && (req_vpn[l] == stage_vpn); + end + end else begin : g_park_direct + assign sel_ready = park_ready; + assign stage_valid = sel_valid; + assign stage_park = sel_park; + assign stage_hold = '0; + end + + assign park_valid = stage_valid; + assign park_vpn = stage_park[STAGE_W-1 -: TLB_VPN_WIDTH]; + assign park_access = tlb_access_e'(stage_park[LANE_W + PFIELDS_W + 1 +: $bits(tlb_access_e)]); + assign park_amo = stage_park[LANE_W + PFIELDS_W]; + assign park_lane = stage_park[PFIELDS_W +: LANE_W]; + assign park_payload = stage_park[LANE_W + PFIELDS_W - 1 : 0]; wire replay_rw = replay_fields[F_RW_LO]; wire [ADDR_WIDTH-1:0] replay_raddr = replay_fields[F_ADDR_LO +: ADDR_WIDTH]; wire [ATTR_WIDTH-1:0] replay_rattr = replay_fields[F_ATTR_LO +: ATTR_WIDTH]; @@ -334,38 +449,117 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( assign cam_access_hit[l] = cat_hit[l] && !replay_to_lane[l] && pipe_ready[l]; + // A permission fault on a hit never reaches memory, but a data-side + // load or atomic still owes its requester a response, as a kill does: + // it waits in a per-lane slot and is answered with zeroed data. + wire pf_rsp_valid; + wire [TAG_WIDTH-1:0] pf_rsp_tag; + if (EXEC_SIDE == 0) begin : g_pf_rsp + wire pf_needs_rsp = !req_rw[l] || req_amo[l]; + reg pf_rsp_valid_r; + reg [TAG_WIDTH-1:0] pf_rsp_tag_r; + wire pf_rsp_fire = pf_rsp_valid_r && !kill_to_lane[l] && req_if[l].rsp_ready; + // An empty slot loads unconditionally so the tag's enable stays off + // the CAM lookup cone; only the valid bit depends on the fault. + always @(posedge clk) begin + if (reset) begin + pf_rsp_valid_r <= 1'b0; + end else if (pf_rsp_valid_r) begin + pf_rsp_valid_r <= ~pf_rsp_fire; + end else begin + pf_rsp_valid_r <= cat_pfault[l] && pf_needs_rsp; + end + end + always @(posedge clk) begin + if (~pf_rsp_valid_r) begin + pf_rsp_tag_r <= req_fields[l][F_TAG_LO +: TAG_WIDTH]; + end + end + assign pf_accept[l] = cat_pfault[l] && !(pf_needs_rsp && pf_rsp_valid_r); + assign pf_rsp_valid = pf_rsp_valid_r; + assign pf_rsp_tag = pf_rsp_tag_r; + end else begin : g_pf_no_rsp + // A fetch is answered by the fault report alone. + assign pf_accept[l] = cat_pfault[l]; + assign pf_rsp_valid = 1'b0; + assign pf_rsp_tag = '0; + end + assign pf_rsp_busy[l] = pf_rsp_valid; + // Request accept back to the client. - assign core_bus_if[l].req_ready = + assign req_if[l].req_ready = cat_bypass[l] ? (!replay_to_lane[l] && pipe_ready[l]) : cat_hit[l] ? (!replay_to_lane[l] && pipe_ready[l]) - : cat_park[l] ? (park_sel[l] && park_ready) - : cat_pfault[l] ? 1'b1 + : cat_park[l] ? (park_sel[l] && sel_ready) + : cat_pfault[l] ? pf_accept[l] : 1'b0; - // Response path wires straight back, except a kill teardown injects a - // zeroed response ahead of the cache reply for its lane. - assign core_rsp_ready[l] = core_bus_if[l].rsp_ready; - assign core_bus_if[l].rsp_valid = kill_to_lane[l] ? 1'b1 : mem_bus_if[l].rsp_valid; - assign core_bus_if[l].rsp_data.data = kill_to_lane[l] ? '0 : mem_bus_if[l].rsp_data.data; - assign core_bus_if[l].rsp_data.tag = kill_to_lane[l] ? kill_tag : mem_bus_if[l].rsp_data.tag; - assign mem_bus_if[l].rsp_ready = core_bus_if[l].rsp_ready && ~kill_to_lane[l]; + // Response path wires straight back, except a kill teardown or a + // permission fault injects a zeroed response ahead of the cache reply. + // The injected tag is chosen ahead of the one select on the reply. + wire inj_rsp = kill_to_lane[l] || pf_rsp_valid; + wire [TAG_WIDTH-1:0] inj_tag = kill_to_lane[l] ? kill_tag : pf_rsp_tag; + assign core_rsp_ready[l] = req_if[l].rsp_ready; + assign req_if[l].rsp_valid = inj_rsp ? 1'b1 : mem_bus_if[l].rsp_valid; + assign req_if[l].rsp_data.data = inj_rsp ? '0 : mem_bus_if[l].rsp_data.data; + assign req_if[l].rsp_data.tag = inj_rsp ? inj_tag : mem_bus_if[l].rsp_data.tag; + assign mem_bus_if[l].rsp_ready = req_if[l].rsp_ready && ~inj_rsp; end assign replay_ready = replay_valid && (replay_perm ? pipe_ready[replay_lane] : kill_ready); + // --------------------------------------------------------------------- + // Per-lane parked count. A flush token bypasses translation, so it would + // overtake its lane's parked requests; it holds until they have left the + // miss station instead, reaching the cache behind them as a lane's + // requests always do (a FLUSH_FORWARD cache forwards it behind that + // port's writes). Only a data-side MMU with writing clients needs it: a + // read overtaken by a flush token is still answered correctly. + // --------------------------------------------------------------------- + localparam PCNT_W = `CLOG2(MSHR_SIZE * REPLAY_DEPTH + 2); + // A park leaves its lane when the stage (or the miss station) takes it. + wire park_take = sel_valid && sel_ready; + wire [LANE_W-1:0] park_take_lane = sel_lane; + wire drain_take = replay_ready || (kill_valid && kill_ready); + + if ((WRITE_ENABLE != 0) && (EXEC_SIDE == 0)) begin : g_flush_order + for (genvar l = 0; l < NUM_REQS; ++l) begin : g_parked + reg [PCNT_W-1:0] parked_cnt_r; + wire inc = park_take && (park_take_lane == LANE_W'(l)); + wire dec = drain_take && (replay_lane == LANE_W'(l)); + always @(posedge clk) begin + if (reset) begin + parked_cnt_r <= '0; + end else if (inc && !dec) begin + parked_cnt_r <= parked_cnt_r + PCNT_W'(1); + end else if (dec && !inc) begin + parked_cnt_r <= parked_cnt_r - PCNT_W'(1); + end + end + `RUNTIME_ASSERT (reset || ~(dec && !inc) || (parked_cnt_r != '0), + ("%t: *** %s lane %0d drained more than it parked", $time, INSTANCE_ID, l)) + assign lane_parked[l] = (parked_cnt_r != '0); + end + end else begin : g_no_flush_order + assign lane_parked = '0; + `UNUSED_VAR (park_take) + `UNUSED_VAR (park_take_lane) + `UNUSED_VAR (drain_take) + end + // --------------------------------------------------------------------- // Fault sideband to the fault-latch surface. Structural faults come from // the walker via the MSHR; permission faults are detected here on hits // and replays. // --------------------------------------------------------------------- - wire perm_fault_any = (| cat_pfault) || replay_drop; + wire perm_fault_any = (| pf_accept) || replay_drop; reg [TLB_VPN_WIDTH-1:0] perm_fault_vpn; reg perm_fault_amo; always @(*) begin perm_fault_vpn = replay_vpn; perm_fault_amo = replay_amo; for (int l = NUM_REQS-1; l >= 0; --l) begin - if (cat_pfault[l]) begin + if (pf_accept[l]) begin perm_fault_vpn = req_vpn[l]; perm_fault_amo = req_amo[l]; end @@ -398,8 +592,10 @@ module VX_mmu import VX_gpu_pkg::*, VX_tlb_pkg::*; #( wire [NUM_REQS-1:0] pipe_busy; for (genvar l = 0; l < NUM_REQS; ++l) begin : g_busy assign pipe_busy[l] = core_bus_if[l].req_valid // incoming (client held) - || mem_bus_if[l].req_valid; // output stage + || mem_bus_if[l].req_valid // output stage + || pf_rsp_busy[l]; // owed fault response + assign lane_idle[l] = ~pipe_busy[l] && ~lane_parked[l]; end - assign empty = ~(| pipe_busy) && tlb_empty; + assign empty = ~(| pipe_busy) && tlb_empty && ~(PARK_REG != 0 && stage_valid); endmodule diff --git a/hw/rtl/vm/VX_mmu_snoop.sv b/hw/rtl/vm/VX_mmu_snoop.sv index 0d442636dc..fb9179a40a 100644 --- a/hw/rtl/vm/VX_mmu_snoop.sv +++ b/hw/rtl/vm/VX_mmu_snoop.sv @@ -17,7 +17,9 @@ // device VX_mmu_dcr one level up) — forwarding DCR downstream to the cores — // while snooping the SATP writes to derive the translation-enable bit and to // self-time the L1 TLB flush. This keeps satp/flush off the socket boundary: -// only the TLB miss fabric crosses it. +// only the TLB miss fabric crosses it. With a unit MMU, the same write also +// flushes its TLB and invalidates the unit caches, which are virtually indexed +// and tagged (they sit in front of the unit MMU). module VX_mmu_snoop import VX_gpu_pkg::*; ( input wire clk, input wire reset, @@ -30,6 +32,11 @@ module VX_mmu_snoop import VX_gpu_pkg::*; ( // L1 dTLB / iTLB flush. VX_tlb_flush_if.master dmmu_flush_if, +`ifdef SOCKET_UNIT_MMU_ENABLE + // Unit MMU TLB flush and unit cache invalidation. + VX_tlb_flush_if.master ummu_flush_if, + VX_dcr_flush_if.master ucache_flush_if, +`endif VX_tlb_flush_if.master immu_flush_if ); wire dcr_wr = dcr_bus_in_if.req_valid && dcr_bus_in_if.req_data.rw; @@ -55,22 +62,44 @@ module VX_mmu_snoop import VX_gpu_pkg::*; ( assign vm_active = vm_active_r; // Self-timed flush: the SATP_HI write (the address-space commit) requests a - // flush of both L1 TLBs, held until they drain and flash-clear. satp changes + // flush of every L1 TLB, held until they drain and flash-clear. satp changes // only on an address-space change, issued between kernels once the TLBs are // already drained, so completion needs no handshake back to the device. - wire flush_done = dmmu_flush_if.done && immu_flush_if.done; +`ifdef SOCKET_UNIT_MMU_ENABLE + wire tlb_flush_done = dmmu_flush_if.done && immu_flush_if.done && ummu_flush_if.done; +`else + wire tlb_flush_done = dmmu_flush_if.done && immu_flush_if.done; +`endif reg tlb_flush_r; always @(posedge clk) begin if (reset) begin tlb_flush_r <= 1'b0; end else if (is_satp_hi) begin tlb_flush_r <= 1'b1; - end else if (flush_done) begin + end else if (tlb_flush_done) begin tlb_flush_r <= 1'b0; end end assign dmmu_flush_if.req = tlb_flush_r; assign immu_flush_if.req = tlb_flush_r; +`ifdef SOCKET_UNIT_MMU_ENABLE + assign ummu_flush_if.req = tlb_flush_r; + + // The unit-cache invalidation is its own leg: a cache flush waits for its + // outstanding misses, which may need a unit-TLB park, so it must not hold + // the TLB legs (whose flush refuses parks) for its walk. + reg ucache_flush_r; + always @(posedge clk) begin + if (reset) begin + ucache_flush_r <= 1'b0; + end else if (is_satp_hi) begin + ucache_flush_r <= 1'b1; + end else if (ucache_flush_if.done) begin + ucache_flush_r <= 1'b0; + end + end + assign ucache_flush_if.req = ucache_flush_r; +`endif // Forward the DCR downstream verbatim: the socket has no MMU readback, so // requests pass through and responses return unchanged. diff --git a/hw/rtl/vm/VX_tlb_cam.sv b/hw/rtl/vm/VX_tlb_cam.sv index 950d974626..18c15614c5 100644 --- a/hw/rtl/vm/VX_tlb_cam.sv +++ b/hw/rtl/vm/VX_tlb_cam.sv @@ -63,10 +63,19 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( // Selectable fields packed once so each lane picks the winning entry with a // one-hot mux rather than encoding a hit index and re-indexing entries_r. - localparam SEL_W = TLB_PPN_WIDTH + TLB_FLAGS_WIDTH + TLB_LEVEL_WIDTH; + // Each entry also carries its superpage splice mask, decoded from its level + // ahead of the mux, so a hit only ANDs it in rather than decoding the + // selected level after the select. + localparam LOW_W = `UP((TLB_PT_LEVELS - 1) * TLB_LEVEL_BITS); + localparam SEL_W = TLB_PPN_WIDTH + TLB_FLAGS_WIDTH + TLB_LEVEL_WIDTH + LOW_W; wire [TLB_SIZE-1:0][SEL_W-1:0] entry_sel; for (genvar i = 0; i < TLB_SIZE; ++i) begin : g_entry_sel - assign entry_sel[i] = {entries_r[i].ppn, entries_r[i].flags, entries_r[i].level}; + wire [TLB_VPN_WIDTH-1:0] low_mask = ~tlb_vpn_mask(entries_r[i].level); + assign entry_sel[i] = {entries_r[i].ppn, entries_r[i].flags, entries_r[i].level, + low_mask[LOW_W-1:0]}; + if (TLB_VPN_WIDTH > LOW_W) begin : g_hi + `UNUSED_VAR (low_mask[TLB_VPN_WIDTH-1:LOW_W]) // a leaf is below the root level + end end for (genvar l = 0; l < NUM_REQS; ++l) begin : g_lookup @@ -106,11 +115,12 @@ module VX_tlb_cam import VX_tlb_pkg::*; #( end assign lookup_perm[l] = (| (hit_onehot & perm_vec)); - wire [TLB_PPN_WIDTH-1:0] sel_ppn = sel[TLB_LEVEL_WIDTH+TLB_FLAGS_WIDTH +: TLB_PPN_WIDTH]; - wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; - wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[0 +: TLB_LEVEL_WIDTH]; + wire [TLB_PPN_WIDTH-1:0] sel_ppn = sel[LOW_W+TLB_LEVEL_WIDTH+TLB_FLAGS_WIDTH +: TLB_PPN_WIDTH]; + wire [TLB_FLAGS_WIDTH-1:0] sel_flags = sel[LOW_W+TLB_LEVEL_WIDTH +: TLB_FLAGS_WIDTH]; + wire [TLB_LEVEL_WIDTH-1:0] sel_level = sel[LOW_W +: TLB_LEVEL_WIDTH]; + wire [LOW_W-1:0] sel_low = sel[0 +: LOW_W]; - wire [TLB_VPN_WIDTH-1:0] low = lookup_vpn[l] & ~tlb_vpn_mask(sel_level); + wire [TLB_VPN_WIDTH-1:0] low = TLB_VPN_WIDTH'(lookup_vpn[l][LOW_W-1:0] & sel_low); // Splice the superpage's intra-page index from the VPN into the PPN. assign lookup_ppn[l] = sel_ppn | TLB_PPN_WIDTH'(low); assign lookup_flags[l] = sel_flags; diff --git a/hw/rtl/vm/VX_tlb_l1.sv b/hw/rtl/vm/VX_tlb_l1.sv index b9416d2964..32ff7092bb 100644 --- a/hw/rtl/vm/VX_tlb_l1.sv +++ b/hw/rtl/vm/VX_tlb_l1.sv @@ -27,6 +27,7 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( parameter MSHR_SIZE = `VX_CFG_L1_TLB_MSHR_SIZE, parameter REPLAY_DEPTH = 2, parameter PAYLOAD_W = 1, + parameter ALLOC_PROBE = 0, parameter ID_WIDTH = `CLOG2(MSHR_SIZE) ) ( input wire clk, @@ -140,6 +141,7 @@ module VX_tlb_l1 import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .QDATA_W (PAYLOAD_W), .QDEPTH (REPLAY_DEPTH), .DEDUP_LIVE_EXCLUDES_FAULT (1), + .ALLOC_PROBE (ALLOC_PROBE), .ID_WIDTH (ID_WIDTH) ) mshr ( .clk (clk), diff --git a/hw/rtl/vm/VX_tlb_mshr.sv b/hw/rtl/vm/VX_tlb_mshr.sv index 32c8cb47da..0dd99d2ba0 100644 --- a/hw/rtl/vm/VX_tlb_mshr.sv +++ b/hw/rtl/vm/VX_tlb_mshr.sv @@ -24,6 +24,11 @@ // per-lane program order for same-page accesses, which the requester relies on. // DEDUP_LIVE_EXCLUDES_FAULT drops faulted entries from dedup (a new same-VPN // request re-walks); with it clear the request joins and shares the fault. +// ALLOC_PROBE dedups the allocate against its own VPN instead: for a requester +// that registers its park, whose lane probe is a cycle stale by then. An entry +// that filled and freed while the park was registered still matches (retired_r): +// the allocate re-arms it as filled, replaying from its kept translation rather +// than walking the page again. // // The associative state (valid/vpn/control) is held in flip-flops — an all-entry // parallel compare cannot come from an addressable RAM. The wide single-read @@ -39,6 +44,7 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( parameter QDATA_W = 1, parameter QDEPTH = 2, parameter DEDUP_LIVE_EXCLUDES_FAULT = 0, + parameter ALLOC_PROBE = 0, parameter ID_WIDTH = `CLOG2(MSHR_SIZE) ) ( input wire clk, @@ -137,9 +143,32 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( ); end - // Allocate reuses the probing lane's dedup result. - wire has_match = probe_match[alloc_lane]; - wire [IDX_W-1:0] match_slot = probe_slot[alloc_lane]; + wire has_match; + wire [IDX_W-1:0] match_slot; + wire rearm; // the allocate re-arms a retired entry + wire [MSHR_SIZE-1:0] retired; // freed after a good fill, translation kept + if (ALLOC_PROBE != 0) begin : g_alloc_probe + wire [MSHR_SIZE-1:0] m; + for (genvar e = 0; e < MSHR_SIZE; ++e) begin : g_m + assign m[e] = (live[e] || retired[e]) && (vpn_r[e] == alloc_vpn); + end + VX_priority_encoder #( + .N (MSHR_SIZE) + ) alloc_enc ( + .data_in (m), + `UNUSED_PIN (onehot_out), + .index_out (match_slot), + .valid_out (has_match) + ); + assign rearm = has_match && retired[match_slot]; + `UNUSED_VAR (alloc_lane) + `UNUSED_VAR (probe_slot) + end else begin : g_lane_probe + // Allocate reuses the probing lane's dedup result. + assign has_match = probe_match[alloc_lane]; + assign match_slot = probe_slot[alloc_lane]; + assign rearm = 1'b0; + end wire has_free; wire [IDX_W-1:0] free_slot; @@ -401,6 +430,10 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( fault_r <= '0; amo_r <= '0; end else begin + if (alloc_fire && rearm) begin + valid_r[match_slot] <= 1'b1; + filled_r[match_slot] <= 1'b1; + end if (alloc_fire && !has_match) begin valid_r[free_slot] <= 1'b1; issued_r[free_slot] <= 1'b0; @@ -450,6 +483,29 @@ module VX_tlb_mshr import VX_tlb_pkg::*; #( end end + // Retired entries: only a registered park can find its entry freed between + // its lane's probe and its allocate. The entry keeps issued_r set when it + // frees, so a re-arm restores valid and filled only. + if (ALLOC_PROBE != 0) begin : g_retired + reg [MSHR_SIZE-1:0] retired_r; + always @(posedge clk) begin + if (reset || flush) begin + retired_r <= '0; + end else begin + if (drain_free && !fault_r[drain_sel_r]) begin + retired_r[drain_sel_r] <= 1'b1; + end + if (alloc_fire) begin + retired_r[alloc_target] <= 1'b0; + end + end + end + assign retired = retired_r; + end else begin : g_no_retired + assign retired = '0; + `UNUSED_VAR (retired) + end + assign empty = ~(| valid_r); endmodule diff --git a/hw/scripts/afu_mem_ports.sh b/hw/scripts/afu_mem_ports.sh new file mode 100755 index 0000000000..df9b7031cd --- /dev/null +++ b/hw/scripts/afu_mem_ports.sh @@ -0,0 +1,89 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Prints how many AXI memory masters the AFU exposes for a configuration. +# +# Usage: afu_mem_ports.sh [-c ] +# +# The count follows the GPU's memory ports, which only the elaborated RTL +# package knows; a platform top must nevertheless spell its ports out +# literally. Elaborating a one-line probe against the same flags as the build +# hands the build the exact value VX_afu_wrap checks it against. +# +# With -c, the result is cached under , keyed on the flags and on +# the contents of every package and configuration header they name, so a +# Makefile can resolve it on every parse without re-running Verilator. + +set -eo pipefail + +cache_dir="" +if [ "$1" = "-c" ]; then + cache_dir=$2 + shift 2 +fi + +VERILATOR=$1 +shift + +if [ -n "$cache_dir" ]; then + key=$({ + printf '%s\n' "$@" + for arg in "$@"; do + case "$arg" in + -I*) for h in VX_config.vh VX_types.vh vortex_afu.vh; do + if [ -f "${arg#-I}/$h" ]; then cat "${arg#-I}/$h"; fi + done ;; + *) if [ -f "$arg" ]; then cat "$arg"; fi ;; + esac + done + } | sha1sum | cut -d' ' -f1) + cache_file="$cache_dir/afu_mem_ports.$key" + if [ -s "$cache_file" ]; then + cat "$cache_file" + exit 0 + fi +fi + +tmp_dir=$(mktemp -d) +trap 'rm -rf "$tmp_dir"' EXIT + +cat > "$tmp_dir/VX_afu_mem_ports_probe.sv" <<'EOF_PROBE' +`include "VX_define.vh" +`include "vortex_afu.vh" +module VX_afu_mem_ports_probe import VX_gpu_pkg::*; (); + $info("AFU_MEM_PORTS=%0d", `AFU_MEM_PORTS_OF(VX_MEM_PORTS)); +endmodule +EOF_PROBE + +if ! out=$("$VERILATOR" --lint-only -Wno-fatal "$@" --top-module VX_afu_mem_ports_probe \ + "$tmp_dir/VX_afu_mem_ports_probe.sv" 2>&1); then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe did not elaborate" >&2 + exit 1 +fi + +count=$(echo "$out" | grep -o 'AFU_MEM_PORTS=[0-9][0-9]*' | head -1 | cut -d= -f2 || true) +if [ -z "$count" ]; then + echo "$out" >&2 + echo "afu_mem_ports.sh: the probe reported no port count" >&2 + exit 1 +fi + +if [ -n "$cache_dir" ]; then + mkdir -p "$cache_dir" + echo "$count" > "$cache_file.$$" + mv "$cache_file.$$" "$cache_file" +fi +echo "$count" diff --git a/hw/scripts/beacon_calibration_u55c.json b/hw/scripts/beacon_calibration_u55c.json new file mode 100644 index 0000000000..51be12a671 --- /dev/null +++ b/hw/scripts/beacon_calibration_u55c.json @@ -0,0 +1 @@ +{"features":["lev","lev_mean","log2_bits","frac_deep","log2_nbits_deep","k_d","k_wd","k_wctl","k_rd","k_port","s_reg","s_port","s_bram","s_ip","s_other","log2_sfo","log2_sfomax","log2_in_fo","log2_in_fo_deep","cross","xmax","hier_depth","log2_lca_bits","lca_up","p_lut","p_carry","p_cmp","p_lutram","p_bram","p_dsp","n_carry","n_cmp","p_maxfo","p_sumlog_fo","p_nfo64","p_ninst","p_nsrc","hand"],"path_levels":8,"near":0.3,"censor":1.0,"sklearn":"1.9.1","builds":[{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c0_origin/beacon","group":"coreA","registers":2289,"failing":104},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c1_ct_it1/beacon","group":"coreA","registers":2290,"failing":25},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c3_ct_it2/beacon","group":"coreA","registers":2290,"failing":92},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c6_lsuagu/beacon","group":"coreA","registers":2304,"failing":709},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c2_lw_v1/beacon","group":"coreB","registers":1687,"failing":93},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c4_lw_v2/beacon","group":"coreB","registers":1690,"failing":112},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c5_lw/beacon","group":"coreB","registers":1685,"failing":98},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/c7_lsulmem/beacon","group":"coreB","registers":1686,"failing":114},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/k1_p2erase/beacon","group":"cache","registers":775,"failing":10},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l0_merge3/beacon","group":"lmem","registers":163,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l1_r1/beacon","group":"lmem","registers":460,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l2_r2/beacon","group":"lmem","registers":471,"failing":0},{"run":"/home/blaisetine/dev/vortex_beacon/build32/claude/corpus/l3_final/beacon","group":"lmem","registers":477,"failing":0}],"slack":{"link":"identity","base":0.7503466214366413,"trees":[{"f":[2,0,21,-1,1,-1,-1,32,23,32,-1,24,-1,-1,-1,37,-1,-1,37,9,21,22,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,11.775312925643524,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,11,0,13,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.01009299,0,0.007409015,-0.001117181,0,0,0,-0.008838531,0,-0.02555701,-0.04193794,-0.00553998,0,-0.003177707,-0.01679376,0,0,0,0,0.0003354531,-0.01197272,-0.02062047,0.01242137,0,-0.03062118,-0.06987605]},{"f":[2,0,21,20,-1,-1,1,-1,-1,32,23,32,-1,24,-1,-1,-1,37,-1,-1,37,9,21,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0.5,0,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,0,14,0,0,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[20,9,6,5,0,0,8,0,0,17,16,13,0,15,0,0,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01015324,0.00311568,0,0.007038721,-0.001062828,0,0,0,-0.008398592,0,-0.02428303,-0.03984905,-0.005264266,0,-0.003018921,-0.01595749,0,0,0,-0.006789299,-0.01959018,0.01180336,0,-0.02909147,-0.06644545]},{"f":[2,0,21,-1,1,-1,-1,32,23,32,-1,-1,-1,16,-1,22,-1,-1,37,9,21,-1,22,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,3.5,3.9474088816539714,0,0,0,6.044394119358453,0,17.07907323966382,0,0,6.970000000000001,0.5,5.5,0,8.32532955673354,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,14,0,16,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,13,12,11,0,0,0,15,0,17,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.009109054,0,0.006686935,-0.001011141,0,0,0,-0.00798055,-0.02966998,-0.005002274,0,-0.001011141,0,-0.007254686,-0.0369035,0,0,0,-0.006450212,0,-0.002746125,-0.01999366,0.01121609,0,-0.02763817,-0.06318328]},{"f":[2,0,21,-1,1,-1,-1,32,21,21,32,-1,-1,-1,-1,37,-1,-1,37,9,21,22,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,5.5,0,4.145833333333334,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,8.34,0,0,6.970000000000001,0.5,5.5,11.775312925643524,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,13,12,0,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.008653654,0,0.00635273,-0.0009534893,0,0,0,0,-0.00826245,-0.02606419,-0.0009534893,-0.05729316,0,-0.002631972,-0.01481627,0,0,0,0,0.0009788293,-0.01071442,-0.01768159,0.01065804,0,-0.02625748,-0.06008126]},{"f":[2,0,16,-1,-1,32,21,21,32,-1,-1,-1,-1,16,-1,22,-1,-1,37,9,21,-1,-1,-1,22,23,-1,-1,-1],"t":[5.228690439536267,6.5,11.620219825507487,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,6.044394119358453,0,17.07907323966382,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,1.5,0,0,0],"m":[1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,0,0,14,0,16,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,5,4,0,0,13,12,11,10,0,0,0,0,15,0,17,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.007646069,0.0002043261,0,0,0,0,-0.007851071,-0.02476316,-0.0009035235,-0.05450668,0,-0.0009035235,0,-0.006611584,-0.03497537,0,0,0,-0.005822199,-0.01679813,0.01012775,0,0,-0.0185915,-0.02751632,-0.05713154]},{"f":[2,0,1,-1,16,-1,-1,32,23,32,-1,24,-1,-1,-1,-1,19,9,37,23,21,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,4.208333333333334,0,7.266786540694901,0,0,9.271599002932193,3.5,3.9474088816539714,0,7.5,0,0,0,0,0.5,0.5,6.970000000000001,2.5,3.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,4,0,6,0,0,15,14,11,0,13,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.007703011,0,0.00711944,-0.0008404133,0,0,0,-0.006781262,0,-0.01905988,-0.03367628,-0.003831588,-0.003822916,0,0,0,0,0,-0.001641005,-0.01474928,-0.0005012965,-0.02175248,0.009990955,0,-0.02559911,-0.05844007]},{"f":[2,0,21,19,-1,-1,-1,32,21,21,32,-1,-1,-1,-1,-1,19,9,37,4,16,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,3.5,0.5,0,0,0,9.271599002932193,9.5,7.5,3.9474088816539714,0,0,0,0,0,0.5,0.5,6.970000000000001,6.569535826647579,4.523561956057013,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,6,5,0,0,0,15,14,13,12,0,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0.008377874,-0.0008047369,0.004674679,0,0,0,0,-0.007131571,-0.0223218,-0.0008047369,-0.0509243,-0.003631874,0,0,0,0,0,0.008127961,-0.008601503,-0.01357791,-0.02066651,0.009493772,0,-0.02432059,-0.05558786]},{"f":[2,0,16,-1,-1,37,23,-1,-1,16,19,23,-1,-1,-1,-1,19,9,37,23,21,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,11.696967526234287,0,0,7.05,0.5,0,0,8.618385502258606,2.5,5.5,0,0,0,0,0.5,0.5,6.970000000000001,2.5,3.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,1,0,0,0,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,12,0,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,5,4,0,0,9,8,0,0,15,14,13,0,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.006556431,-0.0007025786,0,0,-0.008177587,-0.0007631222,0,0,0,-0.02454818,-0.008721901,-0.008721901,-0.008721901,0,0,0,0,0,-0.001211505,-0.01346069,-0.0002570622,-0.01963477,0.009021331,0,-0.02310591,-0.05287487]},{"f":[2,0,1,-1,16,-1,-1,37,23,-1,-1,16,19,23,-1,-1,-1,-1,37,9,21,-1,-1,-1,22,23,-1,-1,-1],"t":[5.228690439536267,6.5,4.208333333333334,0,7.266786540694901,0,0,7.05,0.5,0,0,8.618385502258606,2.5,5.5,0,0,0,0,6.970000000000001,0.5,5.5,0,0,0,17.094543568094064,1.5,0,0,0],"m":[1,1,1,0,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,12,13,14,0,0,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,7,4,0,6,0,0,11,10,0,0,17,16,15,0,0,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.006664328,0,0.006088676,-0.0006931119,0,0,-0.007769287,-0.0006931119,0,0,0,-0.02332295,-0.00821929,-0.00821929,-0.00821929,0,0,0,-0.004418344,-0.01380193,0.008205545,0,0,-0.01516843,-0.02278726,-0.04745487]},{"f":[2,0,21,-1,-1,37,23,-1,-1,21,22,-1,23,-1,-1,-1,19,9,2,37,25,-1,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0,7.05,0.5,0,0,9.5,12.613886328427084,0,1.5,0,0,0,0.5,0.5,10.510264838663748,6.970000000000001,6.5,0,0,0,0,0,17.07907323966382,0,0],"m":[1,1,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,0,13,0,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,5,4,0,0,9,8,0,0,15,12,0,14,0,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.006324955,0.002321033,0,0,-0.007381373,-0.0006285159,0,0,-0.007743111,0,-0.02348585,-0.00796087,-0.04611934,0,0,0,0,0,-0.006690392,0.01385997,-0.01760044,-0.03040429,0.008210748,0,-0.02108128,-0.04797821]},{"f":[2,1,1,-1,16,-1,-1,23,22,0,-1,21,-1,-1,25,-1,-1,-1,19,1,-1,22,2,16,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.196924603174603,4.208333333333334,0,7.924812503605781,0,0,4.5,12.278990886161445,9.5,0,5.5,0,0,0.5,0,0,0,0.5,0.8611111111111112,0,13.87915215758784,8.9274737487839,4.523561956057013,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,15,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,7,4,0,6,0,0,17,14,11,0,13,0,0,16,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.006037799,0,0.004552922,-0.003502164,0,0,0,-0.001289892,0,-0.008345555,-0.03867145,0,-0.02430005,-0.01160658,0.000666555,0,0,0.004763966,0,0,0,0.006605067,-0.006528114,-0.01764439,-0.01770441,-0.02119219]},{"f":[2,1,16,-1,1,-1,-1,23,22,0,-1,21,-1,-1,-1,22,-1,-1,19,9,2,22,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,6.196924603174603,6.930737337562887,0,4.208333333333334,0,0,4.5,12.278990886161445,9.5,0,5.5,0,0,0,17.094543568094064,0,0,0.5,0.5,10.510264838663748,13.87915215758784,0,0,0,0,1.5,0,0],"m":[1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,12,0,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,7,4,0,6,0,0,15,14,11,0,13,0,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.006029469,0,0.004683011,-0.0014152,0,0,0,-0.001225502,0,-0.00793004,-0.03677452,-0.01443037,0,0.003006789,-0.01327387,0,0,0,0,-0.00530783,-0.01590745,-0.02822703,0.007581285,0,-0.007342449,-0.0224426]},{"f":[2,0,21,20,-1,-1,-1,37,21,-1,-1,21,22,-1,23,-1,-1,-1,37,1,2,-1,-1,21,-1,-1,22,-1,-1],"t":[5.228690439536267,6.5,6.5,0.5,0,0,0,7.05,4.5,0,0,9.5,12.613886328427084,0,1.5,0,0,0,6.970000000000001,1.9735450749013448,9.051177541662383,0,0,3.5,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,0,19,20,21,0,0,24,0,0,27,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,0,26,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.005899725,0.0002879526,0.001736505,0,0,-0.006203783,-0.0005244386,0,0,-0.006625508,0,-0.0207102,-0.006628663,-0.04119741,0,0,0,0.003812352,-0.01242497,0,-0.002328426,-0.0113312,0,-0.01676075,-0.04116548]},{"f":[2,1,1,-1,4,-1,21,-1,-1,23,22,-1,-1,17,-1,-1,19,2,37,1,-1,-1,-1,-1,23,-1,22,-1,-1],"t":[5.228690439536267,6.196924603174603,2.68018018018018,0,3.084962500721156,0,2.5,0,0,3.5,15.212004812197344,0,0,7.65814076587311,0,0,0.5,8.9274737487839,6.970000000000001,2.9861172120945896,0,0,0,0,1.5,0,17.07907323966382,0,0],"m":[1,1,0,0,1,0,0,0,0,1,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,14,0,0,17,18,19,20,0,0,0,0,25,0,27,0,0],"r":[16,9,4,0,6,0,8,0,0,13,12,0,0,15,0,0,24,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0.005847049,0,0.004039003,0,0.004687338,-0.005289506,0,0,-0.007283113,-0.020264,0,0.001437225,-0.01705139,0,0,0,0,0.003890422,-0.005437378,-0.01396088,-0.01538907,0,-0.006343413,0,-0.01932653,-0.04278667]},{"f":[2,32,1,-1,22,-1,23,-1,22,-1,17,-1,-1,21,33,-1,-1,-1,19,1,-1,22,2,16,-1,-1,-1,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,12.738408632106918,0,7.723832212141186,0,0,9.5,22.05815630213804,0,0,0,0.5,0.8611111111111112,0,13.87915215758784,8.9274737487839,4.523561956057013,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,0,11,0,0,14,15,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,13,4,0,6,0,8,0,10,0,12,0,0,17,16,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.004970444,0,0.004405795,0,-0.01045918,0,-0.0110591,0,0.004143893,-0.02416683,0,0,-0.01178025,-0.003587426,-0.03904595,0,0,0.004448402,0,0,0,0.006428722,-0.005202046,-0.01493739,-0.01466711,-0.01747733]},{"f":[2,32,1,4,-1,-1,22,-1,23,-1,22,-1,22,-1,-1,21,4,-1,-1,-1,37,1,2,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,4.0437314206251695,0,0,12.199665042528586,0,0.5,0,12.738408632106918,0,17.094543568094064,0,0,9.5,2.453445297804259,0,0,0,6.970000000000001,2.01685992705101,8.9274737487839,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,13,0,0,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,15,6,5,0,0,8,0,10,0,12,0,14,0,0,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.005258508,0.0003526751,0,0.004185645,0,-0.00993719,0,-0.01050879,0,0.004239104,-0.01781273,0,0,-0.004826696,-0.0139212,-0.03714693,0,0,0,0.002982685,-0.009066476,-0.008790707,0,-0.0144571,-0.03659876]},{"f":[2,32,1,-1,22,-1,23,-1,17,16,-1,-1,-1,21,33,21,-1,-1,-1,-1,20,2,37,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,7.723832212141186,7.924812503605781,0,0,0,9.5,22.05815630213804,7.5,0,0,0,0,0.5,8.9274737487839,6.970000000000001,0,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,10,0,0,0,14,15,16,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,13,4,0,6,0,8,0,12,11,0,0,0,19,18,17,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.004485881,0,0.003976495,0,-0.009441247,0,0,0.004725008,-0.006421602,-0.02080143,0,0,0,-0.01234529,0.01628808,-0.003109551,-0.03534027,0,0,0,0.000392129,-0.01290246,-0.01322363,0,-0.008515076,-0.01818194]},{"f":[2,32,1,-1,22,-1,23,-1,22,-1,22,-1,-1,21,4,-1,-1,-1,20,4,22,2,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,2.69615871138938,4.208333333333334,0,12.199665042528586,0,0.5,0,12.738408632106918,0,17.094543568094064,0,0,9.5,2.453445297804259,0,0,0,0.5,10.510264838663748,15.439474430023438,8.9274737487839,0,0,0,0,14.116600814324018,0,0],"m":[1,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,0,9,0,11,0,0,14,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,13,4,0,6,0,8,0,10,0,12,0,0,17,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.004261611,0,0.003777797,0,-0.008970055,0,-0.009646439,0,0.003884403,-0.01652813,0,0,-0.004287279,-0.01275551,-0.03362148,0,0,0,0,0.0007934911,-0.008767322,-0.01127094,-0.02392104,0,-0.01001035,-0.01952533]},{"f":[2,37,1,4,-1,0,-1,-1,22,-1,-1,22,1,-1,21,-1,-1,-1,32,2,37,-1,22,-1,-1,-1,4,-1,-1],"t":[5.228690439536267,7.05,4.208333333333334,4.0437314206251695,0,1.5,0,0,12.199665042528586,0,0,12.613886328427084,9.107199999999999,0,5.5,0,0,0,6.474913355379556,8.9274737487839,3.97,0,13.747003318603246,0,0,0,6.16978583919235,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,0,0,12,13,0,15,0,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,11,8,5,0,7,0,0,10,0,0,17,14,0,16,0,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.004558292,0,0.004096726,-0.002796903,0,0.003598501,-0.001821469,0,0,-0.002796903,0,-0.005840014,-0.03047734,-0.01300204,0,0,0,0.002492647,0,-0.003283161,-0.0115814,-0.01332112,0,-0.01481442,-0.03537206]},{"f":[2,37,1,4,-1,-1,22,-1,-1,22,1,-1,21,-1,-1,-1,1,-1,22,2,32,25,-1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,7.05,4.208333333333334,4.0437314206251695,0,0,12.199665042528586,0,0,12.613886328427084,9.107199999999999,0,5.5,0,0,0,0.8611111111111112,0,13.87915215758784,10.510264838663748,6.474913355379556,6.5,0,0,0,0,17.094543568094064,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,0,17,0,19,20,21,22,0,0,0,0,27,0,0],"r":[16,9,6,5,0,0,8,0,0,15,12,0,14,0,0,0,18,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0.004330405,-0.0001684425,0,0.003418689,-0.001730433,0,0,-0.002630842,0,-0.005549247,-0.02898235,-0.01235304,0,0.004733279,0,0,0,0,-0.00408254,0.0137212,-0.01792513,-0.02215867,0,-0.01297751,-0.03181022]},{"f":[2,32,1,-1,22,16,-1,2,23,-1,-1,-1,-1,15,23,-1,-1,-1,20,4,22,2,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,3.9474088816539714,4.208333333333334,0,17.094543568094064,7.266786540694901,0,1.292481250360578,0.5,0,0,0,0,8.618385502258606,3.5,0,0,0,0.5,10.510264838663748,15.439474430023438,8.9274737487839,0,0,0,0,1.5,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,0,0,0,0,14,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,13,4,0,12,7,0,11,10,0,0,0,0,17,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.003646412,0,0,0.002874395,0,0,-0.004794221,0.00615717,-0.006353009,-0.01592655,0,0,-0.01367633,0.0005443959,-0.002331011,0,0,0,0,0.0008808274,-0.007476909,-0.009479863,-0.02111318,0,-0.006971701,-0.01517641]},{"f":[30,21,22,0,-1,-1,24,33,2,-1,-1,-1,-1,-1,2,21,1,16,-1,-1,-1,2,22,-1,-1,15,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,7.5,20.752541988539424,4.80643424864552,0,0,0,0,0,8.9274737487839,6.5,6.015625,12.683433292835403,0,0,0,3.39067985676233,17.094543568094064,0,0,10.501837184902296,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,1,1,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,0,0,15,16,17,18,0,0,0,22,23,0,0,26,0,0,0],"r":[14,13,6,5,0,0,12,11,10,0,0,0,0,0,28,21,20,19,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001229939,-0.00736486,0,0,0,0.01325826,-0.008519137,-0.01633367,-0.01513572,-0.03641548,0,0,0,0,0.00384211,-0.004528237,-0.003734257,0,0,0.001578458,-0.01257666,0,-0.007824104,0.004617118,-0.01150193]},{"f":[30,21,22,0,-1,-1,24,-1,-1,-1,2,21,20,16,-1,-1,22,-1,-1,2,22,16,-1,-1,-1,1,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,7.5,0,0,0,8.9274737487839,6.5,0.5,12.683433292835403,0,0,12.199665042528586,0,0,3.39067985676233,17.094543568094064,7.774787059601174,0,0,0,1.000936329588015,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,0,11,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[10,9,6,5,0,0,8,0,0,0,28,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0.001168594,-0.006996992,0,-0.005562268,-0.01438006,-0.03462948,0,0,0,0,0.003888598,-0.004302426,0,0.00368211,-0.005466634,0,0,0,0.003531525,-0.002169117,-0.01195347,0,0.002802234,-0.008142368,-0.01092768]},{"f":[30,21,22,0,-1,-1,4,-1,-1,-1,2,21,19,16,-1,-1,22,-1,-1,2,22,16,-1,-1,-1,15,-1,-1,-1],"t":[1.5,9.5,14.03140051979143,8.5,0,0,6.08730682361797,0,0,0,8.9274737487839,6.5,0.5,12.683433292835403,0,0,12.199665042528586,0,0,3.39067985676233,17.094543568094064,7.774787059601174,0,0,0,10.501837184902296,0,0,0],"m":[0,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,0,0,0,11,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[10,9,6,5,0,0,8,0,0,0,28,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0.001110308,-0.006647498,0,-0.009672096,-0.02687236,-0.03293108,0,0,0,0,0.003682637,-0.004087876,0,0.003498318,-0.005499489,0,0,0,0.003355056,-0.002060779,-0.01136116,0,-0.007140122,0.004390287,-0.01038209]},{"f":[2,32,21,-1,1,4,-1,22,-1,-1,23,-1,22,16,-1,-1,-1,18,-1,-1,1,-1,22,2,32,-1,-1,-1,-1],"t":[5.228690439536267,3.9474088816539714,2.5,0,4.208333333333334,4.0437314206251695,0,15.756821747580634,0,0,0.5,0,17.094543568094064,7.924812503605781,0,0,0,7.980000966034041,0,0,0.8611111111111112,0,13.87915215758784,10.510264838663748,6.474913355379556,0,0,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,13,14,0,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,17,4,0,10,7,0,9,0,0,12,0,16,15,0,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0.003975376,0,0,0.003007934,0,-0.0004294658,-0.01563132,0,-0.004459527,0,0,0.004172534,-0.002212977,-0.01304426,0,-0.006546589,0.008278223,0,0.004560486,0,0,0,-0.002481657,-0.01537794,-0.01839156,-0.01108207]},{"f":[37,2,1,16,-1,-1,22,-1,23,-1,22,-1,-1,9,-1,-1,4,21,22,4,-1,-1,15,-1,33,-1,-1,-1,-1],"t":[7.05,5.065928480304397,4.208333333333334,10.486835021563051,0,0,12.199665042528586,0,3.5,0,16.819976309430963,0,0,0.5,0,0,6.16978583919235,9.5,12.613886328427084,2.453445297804259,0,0,5.044394119358453,0,20.752541988539424,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,0,14,0,0,17,18,19,20,0,0,23,0,25,0,0,0,0],"r":[16,13,6,5,0,0,8,0,10,0,12,0,0,15,0,0,28,27,22,21,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0.003284177,-0.0008901339,0,0.002873133,0,-0.003307776,0,0.007570542,-0.0009312196,0,-0.003307776,0.005985948,0,0,0,0,-0.003307776,-0.007348128,0,-0.01736287,0,-0.003307776,-0.01083695,-0.02550077,-0.02684005]},{"f":[2,32,21,-1,1,4,-1,22,-1,-1,23,-1,22,-1,-1,18,-1,-1,20,4,33,21,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,3.9474088816539714,2.5,0,4.208333333333334,4.0437314206251695,0,15.756821747580634,0,0,0.5,0,17.094543568094064,0,0,7.980000966034041,0,0,0.5,10.510264838663748,26.99635432609064,7.5,0,0,0,0,1.5,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,13,0,0,16,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[18,15,4,0,10,7,0,9,0,0,12,0,14,0,0,17,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.003633094,0,0,0.002720713,0,-0.0005149661,-0.01500218,0,-0.004169768,0,0.001908659,-0.01235126,0,-0.005860888,0.008086778,0,0,0,0,0.0008364615,-0.006063507,-0.0121728,-0.0174813,0,-0.005100672,-0.01212888]},{"f":[2,37,21,-1,1,4,-1,-1,21,-1,-1,22,21,-1,-1,21,-1,-1,20,2,1,-1,2,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,7.05,2.5,0,4.208333333333334,4.0437314206251695,0,0,8.5,0,0,15.439474430023438,8.5,0,0,7.5,0,0,0.5,10.510264838663748,0.8611111111111112,0,5.538407798525416,0,0,0,14.116600814324018,0,0],"m":[1,1,0,0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,0,0,12,13,0,0,16,0,0,19,20,21,0,23,0,0,0,27,0,0],"r":[18,11,4,0,8,7,0,0,10,0,0,15,14,0,0,17,0,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.003496748,0,0,0.002584704,-0.001699859,0,-0.001261285,0.007513785,0,0,-0.002491027,-0.02276248,0,-0.01420882,-0.001699859,0,0,0,0.005680821,0,0.01287367,-0.003544033,-0.01563138,0,-0.00624597,-0.0131535]},{"f":[30,21,22,0,-1,-1,4,15,-1,33,18,-1,-1,-1,-1,-1,2,16,22,-1,16,-1,-1,2,-1,0,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,6.16978583919235,5.044394119358453,0,20.752541988539424,1.292481250360578,0,0,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,0,6.539158811108031,0,0,1.292481250360578,0,4.5,0,0,0],"m":[0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,11,0,0,0,0,0,17,18,19,0,21,0,0,24,0,26,0,0,0],"r":[16,15,6,5,0,0,14,9,0,13,12,0,0,0,0,0,28,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.004036614,-0.005061085,0,0,-0.01573609,0,0,-0.003675963,0.01174653,-0.009628935,-0.02249793,-0.02839957,0,0,0,0.00258567,0,-0.00165613,-0.02315412,0,0.002367423,0,0.0002966802,-0.00550038,-0.008660792]},{"f":[30,21,22,0,-1,-1,4,21,-1,2,-1,-1,-1,-1,2,21,1,16,-1,-1,-1,2,-1,1,-1,17,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,6.16978583919235,7.5,0,5.065928480304397,0,0,0,0,8.9274737487839,6.5,6.015625,12.683433292835403,0,0,0,3.245926548164837,0,1.000936329588015,0,8.496347487943302,0,0,0],"m":[0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,0,0,0,0,15,16,17,18,0,0,0,22,0,24,0,26,0,0,0],"r":[14,13,6,5,0,0,12,9,0,11,0,0,0,0,28,21,20,19,0,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0.003835476,-0.004808298,0,0,-0.009573816,0,0.008457708,-0.006238086,-0.02139129,-0.02700671,0,0,0,0,0.002724847,-0.003548554,-0.002803576,0,0.0004969368,0,0.002569624,0,-0.008739302,-0.001020736,-0.008228388]},{"f":[30,21,22,0,-1,-1,24,36,-1,-1,22,-1,-1,-1,2,16,22,-1,16,-1,-1,2,-1,0,-1,-1,1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,7.5,3.5,0,0,17.094543568094064,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,0,6.539158811108031,0,0,1.292481250360578,0,4.5,0,0,2.01685992705101,0,0],"m":[0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,0,15,16,17,0,19,0,0,22,0,24,0,0,27,0,0],"r":[14,13,6,5,0,0,10,9,0,0,12,0,0,0,26,21,18,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.00364436,-0.004568138,0,0,-0.01152198,0.001375796,0,-0.008921283,-0.02371989,-0.02568217,0,0,0,0.00235863,0,-0.001610228,-0.02204955,0,0.002201136,0,0.0002461764,-0.005154955,0,-0.004627697,-0.01275135]},{"f":[2,1,16,-1,22,-1,-1,4,1,21,-1,-1,-1,-1,21,4,-1,-1,1,-1,22,23,1,-1,-1,-1,22,-1,-1],"t":[5.228690439536267,6.196924603174603,11.620219825507487,0,7.25627765006036,0,0,2.453445297804259,8.375,8.5,0,0,0,0,3.5,3.753897320099348,0,0,0.6824793747286149,0,12.121442107629743,2.5,8.046875,0,0,0,17.094543568094064,0,0],"m":[1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,0,15,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[14,7,4,0,6,0,0,13,12,11,0,0,0,0,18,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.002054476,0,-0.0105251,-0.000386021,0,0,0,-9.511392e-06,0.0102789,-0.005045263,-0.008013701,0,0,0.01337196,0.001247589,0,0.003447043,0,0,0,-0.003356382,-0.01246699,0.003929056,0,-0.007934483,-0.02228827]},{"f":[2,1,21,-1,23,-1,-1,4,15,-1,15,-1,15,-1,37,-1,-1,15,-1,-1,21,-1,1,2,21,-1,-1,-1,-1],"t":[5.228690439536267,5.3560606060606055,7.5,0,4.5,0,0,3.084962500721156,5.044394119358453,0,6.044394119358453,0,6.820178962415188,0,9.205,0,0,6.882643049361842,0,0,3.5,0,4.92309205350118,8.9274737487839,7.5,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,11,0,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,7,4,0,6,0,0,17,10,0,12,0,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0.002275021,0,-0.004177678,0.003519658,0,0,-0.006222774,0,0.01119996,0,-0.008562748,0,0.001360111,-0.006921017,0,-0.001487876,-0.009120422,0,0.003203332,0,0,0,0.002151788,-0.003518152,-0.008850262,-0.008606413]},{"f":[2,1,21,22,-1,2,-1,-1,23,22,-1,-1,-1,4,18,-1,-1,-1,21,-1,1,-1,22,23,-1,-1,15,-1,-1],"t":[5.228690439536267,5.489664082687339,7.5,16.819976309430963,0,3.245926548164837,0,0,4.5,2.084962500721156,0,0,0,3.084962500721156,2.160964047443681,0,0,0,3.5,0,0.6824793747286149,0,11.488914557581817,2.5,0,0,11.696967526234287,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,0,14,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[18,13,8,5,0,7,0,0,12,11,0,0,0,17,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.002383514,0,-0.0004580709,-0.02308646,0,0,0.005687868,-0.005240605,0.003344041,0,0,-0.003091417,0.001533092,-0.005513615,0,0.003043517,0,0.003283517,0,0,-0.003618434,0.004543914,0,-0.008223074,-0.001120926]},{"f":[30,21,22,0,-1,-1,-1,-1,2,16,22,2,-1,-1,16,-1,-1,15,-1,2,-1,1,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,0,0,8.9274737487839,6.930737337562887,17.094543568094064,7.754129516000594,0,0,6.539158811108031,0,0,7.451211111832329,0,1.292481250360578,0,4.208333333333334,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,11,12,0,0,15,0,0,18,0,20,0,22,0,0,25,26,0,0,0],"r":[8,7,6,5,0,0,0,0,24,17,14,13,0,0,16,0,0,19,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0.00365745,-0.003729314,-0.006227947,-0.02355188,0,0,0,0,0.001917375,0.01242073,0,-0.001593158,-0.02063145,0,-0.003767092,0,0.002647624,0,0.0007469128,-0.004844642,0,0,-0.007496685,0.004457542,-0.01123601]},{"f":[2,32,23,22,-1,-1,-1,21,-1,4,1,-1,-1,22,-1,-1,21,-1,1,2,21,-1,-1,22,-1,-1,22,-1,-1],"t":[5.228690439536267,10.463013322934223,3.5,14.158358850608169,0,0,0,2.5,0,4.0437314206251695,4.208333333333334,0,0,14.247293309538719,0,0,3.5,0,4.92309205350118,8.9274737487839,7.5,0,0,10.896650035079205,0,0,17.094543568094064,0,0],"m":[1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,10,11,0,0,14,0,0,17,0,19,20,21,0,0,24,0,0,27,0,0],"r":[16,7,6,5,0,0,0,9,0,13,12,0,0,15,0,0,18,0,26,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.003060156,-0.009889142,0.001226775,0,0.00254678,0,0,0.001816088,-0.0004061465,0,-0.001092494,-0.0150007,0,0.002937343,0,0,0,0.002029989,-0.00300024,0,-0.00177435,-0.01062384,0,-0.006879078,-0.02079873]},{"f":[2,32,16,22,-1,-1,22,-1,-1,18,1,-1,-1,-1,21,-1,1,-1,22,21,16,-1,22,-1,-1,-1,15,-1,-1],"t":[5.228690439536267,3.9474088816539714,11.620219825507487,17.094543568094064,0,0,7.25627765006036,0,0,7.980000966034041,12.069111969111969,0,0,0,3.5,0,0.6824793747286149,0,11.488914557581817,8.5,6.882643049361842,0,7.95646805166752,0,0,0,11.696967526234287,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,0,15,0,17,0,19,20,21,0,23,0,0,0,27,0,0],"r":[14,9,6,5,0,0,8,0,0,13,12,0,0,0,16,0,18,0,26,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001509454,-0.007707629,0,-0.00995907,-0.0006758821,0,0,-0.003248729,-0.0150121,0.007828831,0,0.002790798,0,0.003100529,0,0,0,0.007240898,0,-0.01625131,-0.002213952,-0.009468699,0,-0.007304109,-0.0007449547]},{"f":[30,21,22,0,-1,-1,15,-1,15,-1,-1,-1,2,21,22,-1,21,1,-1,2,-1,-1,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,13.747003318603246,8.5,0,0,5.044394119358453,0,6.169925001442312,0,0,0,8.9274737487839,6.5,13.87915215758784,0,4.5,2.532051282051282,0,3.084962500721156,0,0,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,1,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,0,0,13,14,15,0,17,18,0,20,0,0,0,0,25,26,0,0,0],"r":[12,11,6,5,0,0,8,0,10,0,0,0,24,23,16,0,22,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0.003656149,-0.003272386,0,-0.01223436,0,0.008137118,-0.006057241,-0.02189131,0,0,0,0.002054349,0,0,0.003741753,0,-0.003677748,-0.01116781,0.002291743,-0.0007441683,0,0,-0.006614021,0.004533797,-0.01007423]},{"f":[2,1,-1,4,1,22,-1,22,-1,15,-1,15,-1,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,8.55,12.199665042528586,0,12.738408632106918,0,6.044394119358453,0,7.118941072723508,0,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,2.321928094887362,0,0,17.094543568094064,0,0],"m":[1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,10,0,12,0,0,0,0,17,18,0,0,21,0,23,24,0,0,27,0,0],"r":[16,3,0,15,14,7,0,9,0,11,0,13,0,0,0,0,20,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0.001926072,0,0,0,0.003446589,0,-0.006563533,0,0.005993552,0,-0.01395208,0.0003140402,-0.003877671,-0.00659403,0,0,0.001216004,0.01293189,0,0.002577709,0,0,0.003963392,-0.003341359,0,-0.006555259,-0.0183946]},{"f":[2,1,-1,4,37,22,-1,2,-1,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,1,16,-1,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,8.05,12.199665042528586,0,1.792481250360578,0,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,2.321928094887362,0,9.107199999999999,5.78135971352466,0,0,0,17.094543568094064,0,0],"m":[1,1,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,0,0,0,13,14,0,0,17,0,19,20,0,22,23,0,0,0,27,0,0],"r":[12,3,0,11,10,7,0,9,0,0,0,0,16,15,0,0,18,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0.001829781,0,0,0,0.003274413,0,-0.001785509,0.002536214,-0.004220216,-0.006265041,0,0,0.001155242,0.01229396,0,0.002449089,0,0,0.00376596,0,0,-0.007013876,-0.001399042,-0.009909512,0,-0.006227685,-0.01749064]},{"f":[2,1,-1,4,15,2,-1,-1,15,-1,15,16,-1,-1,2,-1,-1,-1,21,25,-1,-1,1,-1,22,16,-1,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,5.044394119358453,1.792481250360578,0,0,6.044394119358453,0,6.857980995127572,6.539158811108031,0,0,1.292481250360578,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,1.584962500721156,0,0,0],"m":[1,1,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,20,0,0,23,0,25,26,0,0,0],"r":[18,3,0,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,22,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0.001738303,0,0,0,-0.006360621,0.001795956,0,0.006298109,0,0,-0.002986104,-0.0205756,0,0.001939921,-0.001931708,-0.005952465,0,0,0.001097517,0.01168749,0,0.002326888,0,0,0.006134287,-0.002733068,-0.006268198]},{"f":[2,1,16,19,-1,22,-1,23,-1,-1,22,-1,-1,-1,21,25,-1,-1,1,-1,22,25,21,-1,-1,17,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,1.5,0,12.199665042528586,0,1.5,0,0,7.25627765006036,0,0,0,3.5,4.5,0,0,0.6824793747286149,0,12.199665042528586,0.5,7.5,0,0,7.781300095563516,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,11,0,0,0,15,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[14,13,10,5,0,7,0,9,0,0,12,0,0,0,18,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001640322,0,0.004152283,0,-0.005264568,0.002792203,0,-0.00897769,-0.0006652157,-0.003776245,0,0,0.001042676,0.01111094,0,0.002210783,0,0,0,0.00110064,-0.003403931,0,-0.01976826,-0.004592869,-0.005954963]},{"f":[2,1,16,19,-1,22,-1,23,-1,-1,-1,-1,21,-1,17,2,-1,-1,2,18,22,-1,2,-1,-1,23,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,1.5,0,12.199665042528586,0,1.5,0,0,0,0,3.5,0,1.292481250360578,6.08730682361797,0,0,9.883406784235138,6.4178278454199,12.613886328427084,0,5.065928480304397,0,0,3.5,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,0,15,16,0,0,19,20,21,0,23,0,0,26,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,14,0,18,17,0,0,28,25,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0.001558314,0,0.003945174,0,-0.005001719,0.002653361,-0.002269258,-0.003587737,0,0.001423325,0,0,0.003996343,-0.002016102,0,0,0,-0.002608369,0,-0.01364276,-0.005637507,0,-0.002159899,0.005270627,-0.01238126]},{"f":[2,1,-1,4,15,2,-1,-1,15,-1,15,16,-1,-1,2,-1,-1,-1,21,-1,1,-1,22,37,16,-1,-1,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,5.044394119358453,1.792481250360578,0,0,6.044394119358453,0,6.857980995127572,6.539158811108031,0,0,1.292481250360578,0,0,0,3.5,0,0.6824793747286149,0,12.121442107629743,9.075,1.584962500721156,0,0,0,0],"m":[1,1,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,0,21,0,23,24,25,0,0,0,0],"r":[18,3,0,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.001507727,0,0,0,-0.005724268,0.00186293,0,0.005867675,0,0,-0.002749277,-0.01968999,0,0.001801844,-0.001918824,-0.005379656,0,0.001352201,0,0.002106287,0,0,0,0.005811512,-0.001737976,-0.01453875,-0.005204944]},{"f":[2,1,16,-1,21,-1,-1,-1,21,32,-1,-1,17,2,16,-1,-1,-1,2,18,2,-1,22,-1,-1,23,-1,-1,-1],"t":[4.0437314206251695,8.7375,11.620219825507487,0,7.5,0,0,0,3.5,4.60106191191523,0,0,1.292481250360578,6.08730682361797,7.679480099505446,0,0,0,9.883406784235138,6.4178278454199,5.40929108874043,0,17.094543568094064,0,0,3.5,0,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,5,0,0,0,9,10,0,0,13,14,15,0,0,0,19,20,21,0,23,0,0,26,0,0,0],"r":[8,7,4,0,6,0,0,0,12,11,0,0,18,17,16,0,0,0,28,25,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0.001147308,0,-0.0005396475,-0.008503063,-0.003314915,0,0,0.01042591,0.0008518863,0,0,0,0.01191455,0.001421286,-0.001861348,0,0,0,-0.007418408,0,-0.002801377,-0.01597622,0,-0.001912025,0.005118711,-0.01161514]},{"f":[30,21,4,27,22,-1,-1,-1,16,-1,-1,-1,2,16,2,-1,-1,21,-1,15,22,-1,-1,22,-1,-1,16,-1,-1],"t":[1.5,9.5,6.16978583919235,0.5,16.98413870862295,0,0,0,7.442943495848728,0,0,0,8.9274737487839,2.584962500721156,7.640240405159187,0,0,2.5,0,5.169925001442312,16.819976309430963,0,0,17.07907323966382,0,0,11.868050853745158,0,0],"m":[0,1,1,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,0,0,0,13,14,15,0,0,18,0,20,21,0,0,24,0,0,27,0,0],"r":[12,11,8,7,6,0,0,0,10,0,0,0,26,17,16,0,0,19,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.001859192,0.0122916,-0.008121665,0,-0.0001857106,-0.01809779,-0.01931443,0,0,0,0.001923977,0.0143707,0,0.001656344,0,0,-0.002135974,-0.01338066,0,0.0003097886,-0.0116041,0,-0.005389397,0.004822595]},{"f":[30,21,-1,-1,2,16,4,22,-1,15,23,-1,-1,17,-1,-1,-1,15,-1,21,-1,15,-1,-1,1,2,-1,-1,-1],"t":[1.5,9.5,0,0,8.9274737487839,6.930737337562887,7.754129516000594,16.819976309430963,0,6.0,2.5,0,0,7.6181566135214025,0,0,0,7.451211111832329,0,7.5,0,7.774787059601174,0,0,2.01685992705101,9.266718101290426,0,0,0],"m":[0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,11,0,0,14,0,0,0,18,0,20,0,22,0,0,25,26,0,0,0],"r":[4,3,0,0,24,17,16,9,0,13,12,0,0,15,0,0,0,19,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.002178626,-0.01836716,0,0,0,0,0.001460641,0,0,-0.016822,-0.004470897,0,0.001257074,-0.008033794,0.01366689,0,-0.00284892,0,0.001061453,0,0.001485448,-0.005577142,0,0,-0.004974966,0.004824269,-0.007869908]},{"f":[2,1,-1,4,18,4,23,-1,15,-1,-1,-1,-1,-1,21,-1,22,2,1,-1,21,4,-1,-1,-1,-1,22,-1,-1],"t":[4.0437314206251695,4.208333333333334,0,3.084962500721156,2.160964047443681,1.792481250360578,3.5,0,8.076815597050832,0,0,0,0,0,3.5,0,16.765402642760364,9.883406784235138,0.3808851224105462,0,8.5,9.193524834446507,0,0,0,0,16.786010507999382,0,0],"m":[1,1,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,0,0,15,0,17,18,19,0,21,22,0,0,0,0,27,0,0],"r":[14,3,0,13,12,11,8,0,10,0,0,0,0,0,16,0,26,25,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0.001296431,0,0,0,0,-0.004742594,0,0.007903301,-0.0013694,0.001704019,0.001328091,-0.005045252,0,0.001183818,0,0,0,0.004328119,0,0,-0.001635213,0.00848965,-0.007257641,-0.009286119,0,-0.03042415,-0.004559684]},{"f":[37,4,19,35,21,-1,4,22,-1,-1,16,-1,-1,-1,22,2,-1,-1,-1,15,11,-1,-1,-1,36,24,-1,-1,-1],"t":[8.05,3.9534452978042594,1.5,1.5,6.5,0,1.792481250360578,2.084962500721156,0,0,10.501837184902296,0,0,0,12.199665042528586,1.792481250360578,0,0,0,10.376125389276174,0.5,0,0,0,5.5,7.5,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,0,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[24,19,14,13,6,0,10,9,0,0,12,0,0,0,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0.001413787,0,0,0.004463777,-0.001504616,0,0.006182006,-0.0004171584,0.009903235,0,0,0.004909221,-0.001744468,-0.001744468,0,0,-0.001744468,0.006379062,0.001664868,0,0,-0.01167812,-0.002473844,-0.02423254]},{"f":[1,21,-1,16,22,-1,-1,22,-1,-1,4,33,-1,33,-1,22,-1,-1,22,24,21,-1,-1,-1,32,17,-1,-1,-1],"t":[2.01685992705101,7.5,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,13.249860669831508,0,17.460839126440824,0,17.094543568094064,0,0,12.199665042528586,9.5,5.5,0,0,0,6.208926257442949,2.69615871138938,0,0,0],"m":[0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,0,11,12,0,14,0,16,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[10,3,0,7,6,0,0,9,0,0,18,13,0,15,0,17,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0.001514115,0,0,0.00614862,-0.002591541,0,-0.01551936,0.001354442,0,0,-0.005173294,0,0.01158753,0,0.0005208784,-0.007487303,0,0,0,0.001581861,-0.002530958,-0.009962296,0,0,0.004169317,-0.003195852,-0.005922454]},{"f":[1,21,19,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,33,-1,33,-1,-1,22,24,1,-1,-1,-1,32,-1,-1],"t":[2.01685992705101,7.5,1.5,9.193524834446507,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,13.249860669831508,0,17.460839126440824,0,0,12.199665042528586,9.5,2.9733542319749215,0,0,0,6.208926257442949,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,16,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,20,17,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001435946,0.00930363,-0.006581797,0,0,0.005843148,-0.002462223,0,-0.0147512,0.001288402,0,0,-0.004915742,0,0.01101505,0.0003566795,0,0,0,-0.007407482,0.0003047911,-0.009467931,0,-0.0003277002,-0.005626538]},{"f":[30,21,-1,-1,2,21,23,22,17,-1,-1,1,-1,21,17,18,-1,-1,1,-1,-1,-1,2,-1,-1,3,-1,-1,-1],"t":[1.5,9.5,0,0,10.510264838663748,6.5,3.5,13.87915215758784,11.578476357671452,0,0,2.532051282051282,0,4.5,6.9827117829050644,5.065928480304397,0,0,5.015625,0,0,0,4.128693921346326,0,0,0.026802550058364013,0,0,0],"m":[0,1,0,0,1,1,1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,9,0,0,12,0,14,15,16,0,0,19,0,0,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,25,22,11,10,0,0,13,0,21,18,17,0,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,-0.001793758,-0.01657933,0,0,0,0,0,0.001144265,-0.009538767,0,0.00284544,0,0,0,-0.004935764,0.00393552,0,-0.02279681,-0.007585419,0.0006713617,0,0.002871791,0.01086618,0,-0.005913337,-0.0004683477,-0.008264888]},{"f":[1,21,2,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,24,1,-1,-1,-1,32,17,-1,0,-1,-1,-1],"t":[2.01685992705101,7.5,9.250296472946918,9.075473573191719,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,9.5,2.9733542319749215,0,0,0,6.208926257442949,2.69615871138938,0,8.5,0,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,16,0,22,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.001337367,-0.005673278,0.009029398,0,0,0.00557612,-0.002261276,0,-0.01392979,0.001256718,0,0.0003046911,0,0,0,-0.007013127,0.0002690045,-0.008861565,0,0,0.004069179,0,0.008365866,-0.00466798,-0.005258962]},{"f":[25,4,0,-1,-1,-1,2,33,16,17,2,-1,-1,-1,22,15,22,21,-1,-1,-1,16,22,-1,-1,-1,-1,-1,-1],"t":[2.5,6.40929108874043,11.5,0,0,0,9.051177541662383,12.53085435233031,2.584962500721156,11.503187353537466,7.640240405159187,0,0,0,17.07907323966382,5.169925001442312,16.819976309430963,3.5,0,0,0,6.930737337562887,16.96990787537721,0,0,0,0,0,0],"m":[0,1,1,0,0,0,1,1,0,1,1,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,0,0,0,15,16,17,18,0,0,0,22,23,0,0,0,0,0,0],"r":[6,5,4,0,0,0,28,27,14,13,12,0,0,0,26,21,20,19,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,-0.001094566,-0.006290324,-0.01483648,0,0,0,0,0,0.001593409,0.01308531,-0.01030069,0,0,0,0,0.0007053689,-0.002403899,-0.0117903,0,0,0.001589859,-0.01049135,-0.0001711212,-0.01119273,0.007886674,-0.003873363]},{"f":[37,4,19,33,17,-1,-1,16,-1,-1,22,2,-1,-1,-1,15,-1,-1,36,33,-1,32,-1,15,-1,22,-1,-1,-1],"t":[8.05,3.9534452978042594,1.5,27.893589438525005,10.482680779086738,0,0,9.862637357558794,0,0,12.199665042528586,1.792481250360578,0,0,0,10.376125389276174,0,0,5.5,13.895530557358477,0,8.110850196876331,0,9.92481250360578,0,16.765402642760364,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,0,0,0,16,0,0,19,20,0,22,0,24,0,26,0,0,0],"r":[18,15,10,7,6,0,0,9,0,0,14,13,0,0,0,17,0,0,28,21,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0.007673583,0.00101322,0,0.0009128435,-0.001187236,0,0,0.004615235,-0.001405008,-0.001405008,0,-0.001405008,0.001652688,0,0,-0.007351764,0,-0.001405008,0,-0.01226727,0,-0.001405008,-0.007735816,-0.02189021]},{"f":[1,21,2,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,21,16,-1,-1,-1,32,19,20,-1,-1,-1,15,-1,-1],"t":[2.01685992705101,7.5,9.221514256038809,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,8.5,4.523561956057013,0,0,0,6.208926257442949,0.5,0.5,0,0,0,7.451211111832329,0,0],"m":[0,1,1,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,20,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.001061363,0.008722559,0,0,0.005233079,-0.002092173,0,-0.01310443,0.001273905,0,0.0002759241,0,0,0,0.007124086,-0.000793602,-0.01333961,0,0,0,-0.0001593836,-0.01286875,0.008387075,0,-0.006772819,-0.002805109]},{"f":[21,-1,2,1,-1,4,32,23,22,33,-1,15,-1,-1,-1,-1,-1,20,22,-1,2,-1,-1,16,-1,18,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.92309205350118,0,3.084962500721156,10.463013322934223,3.5,12.199665042528586,29.75825500004914,0,9.92481250360578,0,0,0,0,0,0.5,12.613886328427084,0,5.065928480304397,0,0,6.0,0,8.03615005796417,0,0,0],"m":[0,0,1,1,0,1,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,0,0,18,19,0,21,0,0,24,0,26,0,0,0],"r":[2,0,28,5,0,17,16,15,14,11,0,13,0,0,0,0,0,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0.001156793,0,0,0.0004807002,0,0,0,0,0,0.002556343,0,-0.0206506,-0.001238944,-0.003218045,0.001143224,0.006810297,0,0,0.002983373,0,-0.01382329,-0.001115191,0,0.003678295,0,-0.006872176,0.003732961,-0.00401651]},{"f":[33,2,16,17,-1,-1,15,22,21,-1,-1,-1,32,-1,22,33,32,-1,-1,-1,17,-1,-1,-1,15,-1,2,-1,-1],"t":[29.297756302860655,10.510264838663748,2.584962500721156,11.78143940991899,0,0,5.169925001442312,14.116600814324018,3.5,0,0,0,3.753897320099348,0,17.094543568094064,27.408661170593152,7.602820271428284,0,0,0,8.729619742794132,0,0,0,9.92481250360578,0,2.453445297804259,0,0],"m":[1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[24,23,6,5,0,0,12,11,10,0,0,0,14,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0.001505028,-0.01082044,0,0,0,0.0006799388,-0.002203598,-0.005162837,0,0.0139484,0,0,0,0.0003750346,-0.004413337,0.007187929,0,-0.01493915,0.004211367,-0.007211902,0,-0.02132823,0,0.001733127,-0.004824015]},{"f":[1,21,2,2,-1,-1,-1,16,22,-1,-1,22,-1,-1,4,-1,22,21,23,-1,-1,-1,32,17,-1,0,-1,-1,-1],"t":[2.01685992705101,7.5,9.221514256038809,9.075473573191719,0,0,0,11.221587121264806,8.32532955673354,0,0,11.722778233171695,0,0,3.084962500721156,0,12.199665042528586,8.5,2.5,0,0,0,6.208926257442949,2.69615871138938,0,8.5,0,0,0],"m":[0,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,16,0,22,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.001071841,-0.005260897,0.008515776,0,0,0.004903847,-0.001911219,0,-0.01235507,0.001180854,0,0.0002477719,0,0,0,-0.0006684176,0.007248656,-0.01228555,0,0,0.003810277,0,0.007786059,-0.004079266,-0.004405532]},{"f":[33,2,16,17,-1,-1,15,22,21,-1,-1,-1,32,-1,22,33,32,-1,-1,-1,17,-1,-1,-1,15,-1,2,-1,-1],"t":[29.297756302860655,10.510264838663748,2.584962500721156,11.78143940991899,0,0,5.169925001442312,16.819976309430963,3.5,0,0,0,3.753897320099348,0,17.094543568094064,27.408661170593152,7.602820271428284,0,0,0,8.729619742794132,0,0,0,9.92481250360578,0,2.453445297804259,0,0],"m":[1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[24,23,6,5,0,0,12,11,10,0,0,0,14,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0.001388901,-0.01025748,0,0,0,0.0004850435,-0.002437618,-0.01139385,0,0.01321672,0,0,0,0.0003587073,-0.004109776,0.006823695,0,-0.01411461,0.004072127,-0.006751438,0,-0.02017845,0,0.001634424,-0.004481073]},{"f":[21,-1,2,1,-1,4,22,25,-1,-1,23,25,-1,-1,15,-1,15,-1,15,-1,-1,16,2,22,-1,-1,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.92309205350118,0,4.80643424864552,12.121442107629743,0.5,0,0,3.5,1.5,0,0,6.044394119358453,0,6.584962500721156,0,8.618385502258606,0,0,6.882643049361842,5.8698903048866296,14.03140051979143,0,0,0,0,0],"m":[0,0,1,1,0,1,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,12,0,0,15,0,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[2,0,28,5,0,21,10,9,0,0,14,13,0,0,16,0,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0.0009996875,0,0,0.0004428828,0,0,0,0.002592051,-0.009257566,0,0,-0.003094432,0.000810815,0,0.007034391,0,-0.008505137,0,0.004004897,-0.0009194988,0,0,0,0.0009088086,-0.01101109,0.007242427,-0.006014867,-0.003538759]},{"f":[2,1,2,2,21,-1,-1,-1,-1,4,18,4,36,-1,-1,-1,33,-1,32,-1,22,-1,-1,16,32,-1,-1,-1,-1],"t":[10.510264838663748,5.417948717948718,9.266718101290426,9.051177541662383,7.5,0,0,0,0,3.084962500721156,2.160964047443681,1.792481250360578,4.5,0,0,0,17.67212302715558,0,9.271599002932193,0,17.094543568094064,0,0,6.882643049361842,3.9474088816539714,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,12,13,0,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[28,9,8,7,6,0,0,0,0,23,16,15,14,0,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0.0007532304,-0.0009515619,-0.004837303,0.008293815,0,0,0,0,-0.003903181,0.002120943,0.002219091,0,0.01341943,0,-0.00402969,0,0.002114557,-0.007333309,0,0,0.004962241,-0.00316557,-0.003819077,-0.006248942]},{"f":[33,2,1,-1,16,-1,15,22,21,-1,-1,-1,3,-1,32,-1,22,2,-1,2,-1,-1,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,2.584962500721156,0,5.169925001442312,16.819976309430963,3.5,0,0,0,0.030393996247654785,0,5.845435504646156,0,17.07907323966382,1.292481250360578,0,1.792481250360578,0,0,8.729619742794132,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,0,0,13,0,15,0,17,18,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,6,0,12,11,10,0,0,0,14,0,16,0,22,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0.002790976,0,0.001128073,0,0,0,0.0003492307,-0.002246819,-0.01085951,0,-0.006975157,0,0.00886741,0,0,0.0009542799,0,-0.00368098,-0.0001658797,0,-0.01387025,0.001953063,-0.005938513,0,-0.01866556,-0.001272873]},{"f":[33,2,4,2,25,33,-1,22,27,-1,16,-1,15,-1,-1,16,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,10.510264838663748,9.266718101290426,9.051177541662383,6.5,22.05815630213804,0,17.094543568094064,0.5,0,6.930737337562887,0,7.451211111832329,0,0,6.539158811108031,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,11,0,13,0,0,16,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,20,19,18,7,0,15,10,0,12,0,14,0,0,17,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.001523565,0,0,0.003308941,0,0.0007008705,0,-0.002298461,0.0001921743,0,-0.0003739234,-0.01337694,0.009514322,-0.004055407,0.006891412,-0.005643505,0,-0.01774742,0,0,0.007150196,-0.001131879,-0.003900971]},{"f":[4,25,19,-1,2,36,-1,33,-1,33,-1,21,-1,-1,-1,-1,1,-1,1,-1,22,32,-1,17,-1,-1,32,-1,-1],"t":[3.9534452978042594,5.5,1.5,0,3.084962500721156,2.5,0,11.327499782570463,0,17.833701323482934,0,4.5,0,0,0,0,2.01685992705101,0,2.9733542319749215,0,12.199665042528586,8.341935934311302,0,10.543869317296533,0,0,6.208926257442949,0,0],"m":[1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,10,0,12,0,0,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[16,15,4,0,14,7,0,9,0,11,0,13,0,0,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,0.000446794,0,0,0.005358727,0,-0.008957329,0,0.01143143,0,-0.002009784,0.001966057,-0.00426696,0.00872164,0,0.0008400854,0,-0.006513873,0,0,-0.004725644,0,0.002213686,-0.003893726,0,-0.0005258926,-0.004154154]},{"f":[33,2,1,-1,4,2,25,16,-1,15,22,-1,-1,32,-1,22,2,-1,-1,17,-1,-1,-1,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,9.266718101290426,9.034782574103549,6.5,2.584962500721156,0,5.169925001442312,16.819976309430963,0,0,6.021684556978844,0,17.07907323966382,1.292481250360578,0,0,8.729619742794132,0,0,0,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,10,11,0,0,14,0,16,17,0,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,4,0,24,23,22,9,0,13,12,0,0,15,0,19,18,0,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0.002612715,0,0,0,0,0.001061694,0,0,-0.001243468,-0.01001092,0,0.005271949,0,0,0.000839201,-0.0005695343,0,-0.0127569,0.002292555,0.008616354,-0.004972703,0.006566009,-0.005231688,0,-0.01685008,-0.00114727]},{"f":[37,21,-1,22,21,2,17,-1,33,-1,-1,-1,23,-1,22,-1,22,-1,-1,0,-1,23,-1,2,-1,-1,21,-1,-1],"t":[11.26,2.5,0,12.199665042528586,6.5,6.451187557243013,9.075473573191719,0,35.98439522565276,0,0,0,1.5,0,8.133841411710264,0,11.415473622904775,0,0,2.5,0,3.5,0,4.0437314206251695,0,0,8.5,0,0],"m":[1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,13,0,15,0,17,0,0,20,0,22,0,24,0,0,27,0,0],"r":[26,3,0,19,12,11,8,0,10,0,0,0,14,0,16,0,18,0,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0.0007869015,0,0,0,0,0.00168839,0,0.004044465,-0.001560101,0.005703986,0,-0.001816922,0,0.01276758,0,-0.002226985,0.002175376,0,0.0008173675,0,-0.002226985,0,-0.0001956399,-0.001908261,0,-0.002226985,-0.02416666]},{"f":[33,2,1,-1,4,2,25,16,-1,15,22,-1,-1,32,-1,22,2,-1,-1,17,-1,-1,-1,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,10.510264838663748,0.39811320754716983,0,9.266718101290426,9.034782574103549,6.5,2.584962500721156,0,5.169925001442312,16.819976309430963,0,0,5.845435504646156,0,17.07907323966382,1.292481250360578,0,0,8.729619742794132,0,0,0,0,0,0,9.92481250360578,0,0],"m":[1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,10,11,0,0,14,0,16,17,0,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,4,0,24,23,22,9,0,13,12,0,0,15,0,19,18,0,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0.002451664,0,0,0,0,0.0009746228,0,0,-0.001158073,-0.009431339,0,0.006930941,0,0,0.000825467,-0.0004877418,0,-0.01207556,0.002231383,0.008054735,-0.004623655,0.006203605,-0.004884785,0,-0.01553315,-0.001107599]},{"f":[1,21,-1,20,16,16,33,-1,0,-1,-1,-1,-1,15,-1,-1,16,16,-1,-1,22,25,-1,16,-1,-1,25,-1,-1],"t":[5.489664082687339,7.5,0,0.5,13.56045225766691,10.794415866350105,23.76434662895582,0,2.5,0,0,0,0,7.774787059601174,0,0,6.108524456778169,5.044394119358453,0,0,12.199665042528586,0.5,0,8.618385502258606,0,0,1.5,0,0],"m":[1,1,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,0,0,14,0,0,17,18,0,0,21,22,0,24,0,0,27,0,0],"r":[16,3,0,13,12,11,8,0,10,0,0,0,0,15,0,0,20,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0.0005680782,0,0,0,0,0.007122008,0,0.0003006681,-0.004260023,-0.006761044,0.006968803,0,0.001605728,-0.003212158,0,0,-0.001012244,0.007182276,0,0,0.001109443,0,-0.01789238,-0.00258908,0,-0.004105288,0.001269927]},{"f":[37,32,21,-1,22,21,17,4,-1,-1,22,-1,-1,-1,18,0,-1,-1,17,-1,32,-1,-1,-1,36,33,-1,-1,-1],"t":[8.05,10.41931383093672,2.5,0,12.199665042528586,6.5,9.075473573191719,5.599836172418183,0,0,11.876328901062104,0,0,0,7.519373778108656,1.5,0,0,8.571721503073015,0,8.341935934311302,0,0,0,5.5,13.895530557358477,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,11,0,0,0,15,16,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[24,23,4,0,14,13,10,9,0,0,12,0,0,0,18,17,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.0007275386,0,0,0,0,0.001690162,0.007337166,0,-0.0008277944,0.003586534,-0.0007600075,0,0,0.001105116,-0.0008277944,0,0.002775703,0,0.01170319,-0.0008277944,0.006034722,0,0,-0.005809022,-0.0008277944,-0.01536133]},{"f":[4,25,-1,-1,1,22,-1,22,-1,22,-1,-1,1,-1,22,21,17,-1,-1,-1,32,-1,2,1,22,-1,-1,-1,-1],"t":[3.9534452978042594,5.5,0,0,2.01685992705101,7.499052785859982,0,8.133841411710264,0,8.882387211598338,0,0,2.9733542319749215,0,12.199665042528586,8.5,10.543869317296533,0,0,0,6.208926257442949,0,6.011183906514227,4.031939779270633,16.819976309430963,0,0,0,0],"m":[1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,0,8,0,10,0,0,13,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[4,3,0,0,12,7,0,9,0,11,0,0,14,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.0001986951,0.007290981,0,0,-0.007019721,0,0.01019044,0,-0.007760467,0.001094859,0,-0.005800614,0,0,0,0.001646995,-0.002989115,-0.0105119,0,-0.0002854441,0,0,0,-0.001541281,-0.01183116,-0.009389766,-0.002030217]},{"f":[33,2,4,2,25,22,-1,1,-1,16,15,-1,-1,18,-1,15,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,10.510264838663748,9.266718101290426,9.051177541662383,6.5,12.199665042528586,0,4.92309205350118,0,6.044394119358453,5.044394119358453,0,0,1.292481250360578,0,7.118941072723508,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,11,0,0,14,0,16,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,20,19,18,7,0,9,0,13,12,0,0,15,0,17,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0.0007124532,0,0.0003326575,0,0,-0.002615865,0.002131132,0,-0.0008685852,0,-0.009300819,-0.002681429,0.007193294,-0.003279583,0.005917901,-0.004404349,0,-0.01397297,0,0,0.006508585,-0.001023271,-0.003387104]},{"f":[22,37,23,1,-1,22,4,-1,-1,-1,4,2,19,-1,-1,-1,-1,21,-1,-1,34,36,17,17,-1,-1,-1,-1,-1],"t":[17.094543568094064,11.26,3.5,2.01685992705101,0,12.199665042528586,4.951187557243013,0,0,0,6.451187557243013,4.0437314206251695,0.5,0,0,0,0,8.5,0,0,1.5,4.5,8.729619742794132,8.076805701955248,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,0,0,0,0,18,0,0,21,22,23,24,0,0,0,0,0],"r":[20,17,10,5,0,9,8,0,0,0,16,15,14,0,0,0,0,19,0,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0.0004954022,0,0,0.000848469,-0.001507728,-0.001709019,0,0,0,0.0004988783,0.003394667,-0.00165446,0.007306584,0,-0.001933888,-0.02086708,0,0,0,0,0.0005464116,-0.02393084,0.00536996,0.008909601,-0.0129082]},{"f":[1,-1,2,23,22,21,-1,-1,-1,2,-1,-1,22,21,16,2,-1,-1,-1,24,22,-1,-1,-1,22,-1,32,-1,-1],"t":[0.39811320754716983,0,4.0437314206251695,3.5,16.978341770814396,6.5,0,0,0,2.69615871138938,0,0,16.765402642760364,8.5,1.584962500721156,8.9274737487839,0,0,0,5.5,10.224917391507956,0,0,0,16.786010507999382,0,6.337096134072842,0,0],"m":[0,0,1,1,1,1,0,0,0,1,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,12,9,8,7,0,0,0,11,0,0,24,19,18,17,0,0,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.00191388,0,0,0,0,0.0002957072,-0.001729166,-0.006930577,0,0.0008949916,0.006127296,0,0,0,0,0.006992286,-0.002910239,-0.0003879226,0,0,-0.01088958,-0.0004722067,-0.01683201,0,-0.02744455,0,0.004263812,-0.003101888]},{"f":[21,-1,37,22,16,-1,-1,23,1,-1,-1,4,2,33,-1,17,19,21,-1,-1,-1,-1,-1,3,-1,-1,21,-1,-1],"t":[2.5,0,11.26,12.199665042528586,10.501837184902296,0,0,3.5,1.5277777777777777,0,0,6.451187557243013,4.0437314206251695,22.152182868153965,0,8.496347487943302,0.5,7.5,0,0,0,0,0,0.99414994940175,0,0,7.5,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,14,0,16,17,18,0,0,0,0,0,24,0,0,27,0,0],"r":[2,0,26,7,6,0,0,11,10,0,0,23,22,15,0,21,20,19,0,0,0,0,0,25,0,0,28,0,0],"v":[0,0.0006483386,0,0,0,0.0008827369,-0.001535239,0,0,0.0008352234,-0.002051928,0,0,0,0.01117172,0,0,0,0.001963069,-0.001110176,0.003162222,-0.002200939,-0.002011756,0,-0.0002649433,0.01182671,0,-0.002200939,-0.01805595]},{"f":[22,2,4,2,25,37,32,23,1,-1,-1,4,-1,-1,-1,21,-1,-1,-1,-1,-1,-1,32,2,16,-1,-1,-1,-1],"t":[17.094543568094064,10.510264838663748,9.266718101290426,9.051177541662383,6.5,11.26,10.463013322934223,3.5,5.489664082687339,0,0,6.451187557243013,0,0,0,6.5,0,0,0,0,0,0,8.341935934311302,1.292481250360578,6.539158811108031,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,12,0,0,0,16,0,0,0,0,0,0,23,24,25,0,0,0,0],"r":[22,21,20,19,18,15,14,11,10,0,0,13,0,0,0,17,0,0,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0002171799,-0.001321003,0,0.0006050795,0.007868288,0.004187485,0,-0.001975396,-0.01806708,0.006458903,-0.003053104,0.005508971,-0.004069214,0,0,0,6.061457e-06,-0.01021453,0.00533298,-0.01104303]},{"f":[33,23,4,-1,22,15,16,15,15,17,-1,-1,-1,-1,-1,4,-1,25,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,8.5,4.0437314206251695,0,16.725912049439096,10.376125389276174,7.266786540694901,5.78135971352466,5.672425341971495,10.269270673170688,0,0,0,0,0,4.128693921346326,0,0.5,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,9,10,0,0,0,0,0,16,0,18,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,4,0,20,15,14,13,12,11,0,0,0,0,0,17,0,19,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0.0002478288,0,0,0,0,0,0,0.0003624386,-0.004614507,-0.006174394,0.003151608,-0.003116141,0,-0.005455446,0,0.01123074,0.001970949,-0.003764736,0.005835956,0,-0.01197659,0,0,0.00606068,-0.00105788,-0.003042919]},{"f":[22,2,4,2,25,37,32,23,-1,4,-1,-1,-1,21,-1,-1,-1,-1,-1,-1,18,32,-1,17,26,-1,-1,-1,-1],"t":[17.094543568094064,10.510264838663748,9.266718101290426,9.051177541662383,6.5,11.26,10.463013322934223,3.5,0,6.451187557243013,0,0,0,6.5,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0.5,0,0,0,0],"m":[1,1,1,1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,0,0,14,0,0,0,0,0,0,21,22,0,24,25,0,0,0,0],"r":[20,19,18,17,16,13,12,9,0,11,0,0,0,15,0,0,0,0,0,0,28,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-9.325821e-05,0,0.0005834576,0.007419645,0.003912122,0,-0.00182454,-0.01673545,0.0061287,-0.002883515,0.005282127,-0.003836576,0,0,0.003162166,0,0,-0.009270798,0.0007116455,-0.01929915,0.005504659]},{"f":[1,22,-1,-1,4,25,-1,-1,22,24,-1,21,16,2,-1,-1,15,16,-1,17,-1,-1,-1,37,-1,-1,4,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,4.0437314206251695,6.5,0,0,16.725912049439096,3.5,0,8.5,1.584962500721156,8.9274737487839,0,0,10.376125389276174,7.266786540694901,0,8.025009334276167,0,0,0,6.05,0,0,6.434604125275413,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,13,14,0,0,17,18,0,20,0,0,0,24,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,11,0,23,16,15,0,0,22,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0.001304559,0.01016061,0,0,0.0001081318,0.006181318,0,0,0.007538666,0,0,0,0.006555191,-0.002557596,0,0,-0.0001690259,0,-0.005093689,-0.0005222063,0.001068374,0,-0.002514307,-0.01132932,0,-0.004450536,0.005251337]},{"f":[21,-1,2,4,1,-1,16,16,-1,-1,16,17,-1,17,16,-1,-1,-1,4,-1,22,-1,-1,-1,17,16,-1,-1,-1],"t":[2.5,0,8.9274737487839,7.997154617983022,4.92309205350118,0,6.044394119358453,5.78135971352466,0,0,6.832890014164741,6.434604125275413,0,8.729619742794132,6.247927513443585,0,0,0,1.292481250360578,0,11.406459975416917,0,0,0,12.18494601563336,11.868050853745158,0,0,0],"m":[0,0,1,1,1,0,0,1,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,12,0,14,15,0,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,6,0,10,9,0,0,18,13,0,17,16,0,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0.0006011699,0,0,0,0.0002071042,0,0,-0.0003854929,0.005081981,0,0,-0.001190481,0,0,-0.006331103,-0.02530961,0.003152736,0,0.001589913,0,0.0001301891,-0.003396627,0.0097837,0,0,-0.002266734,0.006799971,-0.009500039]},{"f":[33,23,22,33,16,-1,1,-1,-1,-1,22,15,16,22,36,-1,-1,-1,-1,-1,-1,-1,15,-1,2,36,-1,-1,-1],"t":[29.297756302860655,8.5,12.199665042528586,23.925614337747046,10.462502272597265,0,1.000936329588015,0,0,0,12.921158001713904,10.457380879072534,4.247927513443585,12.613886328427084,2.5,0,0,0,0,0,0,0,9.92481250360578,0,2.453445297804259,3.5,0,0,0],"m":[1,1,0,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,13,14,15,0,0,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,21,10,9,6,0,8,0,0,0,20,19,18,17,16,0,0,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0.0007762797,0,-0.010548,-0.000545887,0.006018649,0,0,0,0,0,0.00556791,-0.002257897,-0.00718133,-0.00560397,0.003895453,-3.079856e-05,0.005495829,0,-0.01075617,0,0,0.005631557,-0.001046427,-0.002839746]},{"f":[1,22,-1,-1,4,25,-1,-1,22,24,-1,21,16,2,-1,-1,15,16,-1,-1,-1,22,-1,37,-1,-1,4,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,4.0437314206251695,6.5,0,0,16.725912049439096,3.5,0,8.5,1.584962500721156,8.9274737487839,0,0,10.376125389276174,7.266786540694901,0,0,0,10.224917391507956,0,6.05,0,0,6.434604125275413,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,13,14,0,0,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,11,0,21,16,15,0,0,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0.001245427,0.009588393,0,0,9.64203e-05,0.005846403,0,0,0.007183351,0,0,0,0.006121296,-0.002214316,0,0,-0.0001676299,-0.002872553,0.001068845,0,-0.01054629,0,-0.0004329909,-0.01014548,0,-0.004197824,0.004970097]},{"f":[9,4,-1,2,18,-1,-1,32,15,15,16,3,-1,17,21,-1,-1,-1,-1,-1,16,-1,-1,25,22,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,5.918419679874358,0,0,6.208926257442949,5.78135971352466,5.672425341971495,6.409390936137702,0.5803571428571428,0,10.269270673170688,8.5,0,0,0,0,0,6.6865005271832185,0,0,1.5,15.439474430023438,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,0,14,15,0,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,23,20,19,18,13,0,17,16,0,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0.0001500812,0,0,-0.005064084,0.002398397,0,0,0,0,0,0.004552382,0,0,0.0006636755,-0.004530538,-0.003800189,-0.002865784,-0.005597185,0,0.005666625,0.0001265824,0,0,-0.0003221986,-0.008724003,-0.01189795,0.002598201]},{"f":[21,17,-1,-1,23,1,-1,22,21,-1,4,-1,36,-1,15,-1,-1,-1,4,2,2,17,-1,-1,-1,-1,3,-1,-1],"t":[2.5,10.090748425878532,0,0,3.5,2.01685992705101,0,12.199665042528586,3.5,0,1.292481250360578,0,3.5,0,9.92481250360578,0,0,0,6.451187557243013,4.0437314206251695,2.69615871138938,8.496347487943302,0,0,0,0,0.8775684931506849,0,0],"m":[0,1,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,0,8,9,0,11,0,13,0,15,0,0,0,19,20,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,18,7,0,17,10,0,12,0,14,0,16,0,0,0,26,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0.0004265545,0.004202454,0,0,0.0003790881,0,0,0.003885634,0,0.001199864,0,-0.0006461387,0,-0.01269688,-0.002636263,-0.002047483,0,0,0,0,0.001088166,-0.004428085,0.00549888,-0.001631657,0,-0.002583763,0.007803228]},{"f":[23,22,1,-1,3,22,22,-1,-1,-1,23,22,1,-1,22,-1,-1,15,-1,-1,-1,34,18,15,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,0.39811320754716983,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,3.5,16.978341770814396,5.489664082687339,0,15.717370311433935,0,0,6.0,0,0,0,1.5,8.729619742794132,6.584962500721156,0,0,0,0,0],"m":[1,1,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,13,0,15,0,0,18,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,4,0,10,9,8,0,0,0,20,17,14,0,16,0,0,19,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.001508723,0,0,0,-0.001610429,0.004631914,-0.02330886,0,0,0,0.0001542587,0,-0.0007078582,-0.00654202,0,-0.01135952,0.005269829,0.0008519239,0,0,0,-0.01059023,0.002315926,0.006558252,-0.01062329,0.004780946]},{"f":[21,-1,2,4,2,4,23,16,22,4,-1,-1,15,-1,32,18,-1,-1,-1,-1,-1,-1,22,3,-1,-1,-1,-1,-1],"t":[2.5,0,10.510264838663748,9.266718101290426,8.9274737487839,7.997154617983022,0.5,10.501837184902296,13.415167820124083,7.044021426237043,0,0,6.0,0,7.014643613484123,1.292481250360578,0,0,0,0,0,0,16.765402642760364,0.008766718459616732,0,0,0,0,0],"m":[0,0,1,1,1,1,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,10,0,0,13,0,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,21,20,19,12,11,0,0,14,0,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0.0005243369,0,0,0,0,0,0,0,0,0.0002513836,-0.009223684,0,0.00157535,0,0,-0.002771734,-0.00809283,0.003625687,-0.004581997,0.0001705055,0.009019764,0,0,0.002139303,-0.003640743,-0.01084763,0.006449432,-0.003942209]},{"f":[33,33,33,33,24,-1,22,-1,-1,25,1,-1,-1,24,-1,4,-1,-1,33,-1,-1,22,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,12.613886328427084,0,0,0.5,7.0202978971962615,0,0,6.5,0,3.9534452978042594,0,0,21.16864017939033,0,0,16.838461650489297,7.781300095563516,0,0,0,9.92481250360578,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,9,6,0,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,5.929308e-05,0,-0.0001950208,-0.005759816,0,0,0.002807465,-0.005453767,0,0.001351511,0,0.01936757,0.003057229,0,-0.009129101,-0.001318068,0,0,0.002198293,0.007807967,-0.00260431,0,-0.009764884,-0.0007906171]},{"f":[23,22,9,2,25,37,-1,-1,-1,22,17,22,-1,-1,17,-1,22,-1,-1,-1,-1,34,18,15,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,0.5,9.051177541662383,6.5,11.26,0,0,0,16.752707246237605,9.229707804263573,11.681676585676911,0,0,10.460454442916898,0,12.457649623491124,0,0,0,0,1.5,8.729619742794132,6.584962500721156,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,15,0,17,0,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,20,9,8,7,0,0,0,19,14,13,0,0,16,0,18,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,5.271143e-05,-0.003322685,0.005068241,0,0,0,-0.0008451611,-0.006473262,0,0.008714518,0,-0.006371915,0.0004759029,-0.01040257,0.002322836,0,0,0,-0.009988319,0.002066623,0.005950914,-0.01002393,0.004625229]},{"f":[23,21,23,22,-1,1,-1,15,17,-1,-1,33,-1,23,15,-1,1,-1,-1,22,-1,-1,27,-1,-1,37,-1,-1,-1],"t":[8.5,6.5,3.5,13.87915215758784,0,2.532051282051282,0,5.044394119358453,4.583709072915869,0,0,17.460839126440824,0,0.5,6.0,0,6.015625,0,0,16.978341770814396,0,0,0.5,0,0,11.05,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,0,14,15,0,17,0,0,20,0,0,23,0,0,26,0,0,0],"r":[28,25,22,5,0,7,0,11,10,0,0,13,0,19,16,0,18,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0.0003278223,0,0.002096721,0,0,-0.003957058,-0.01296868,0,0.00876085,0,0,0.004368465,0,-0.008734542,-0.00317443,0,0.0006059423,-0.006422286,0,0.003237538,-0.006090678,0,-0.0004226292,-0.01514355,0.004396751]},{"f":[9,4,-1,2,18,-1,-1,2,-1,32,16,-1,1,15,16,15,-1,-1,-1,-1,33,-1,-1,25,22,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,5.918419679874358,0,0,4.80643424864552,0,6.208926257442949,1.584962500721156,0,6.241071428571429,10.043027283594547,7.651051691178929,6.022367813028454,0,0,0,0,8.065928480304397,0,0,1.5,15.439474430023438,0,0,0,0],"m":[1,1,0,0,1,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,10,11,0,13,14,15,16,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,9,0,23,12,0,20,19,18,17,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0.0001399263,0,0,-0.004712361,0.002222478,0,0.003396663,0,0,0.002633634,0,0,0,0,-0.000602017,0.003283105,-0.002578898,0.001622494,0,-0.01016229,-0.001963616,0,0,-0.0004682529,-0.007479389,-0.01072496,0.002275016]},{"f":[1,22,-1,-1,3,22,22,-1,-1,-1,4,-1,22,-1,22,-1,20,22,-1,17,-1,-1,15,21,17,-1,-1,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,4.951187557243013,0,7.499052785859982,0,8.32532955673354,0,0.5,8.882387211598338,0,11.78143940991899,0,0,9.92481250360578,7.5,7.080487187463968,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,0,13,0,15,0,17,18,0,20,0,0,23,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,12,0,14,0,16,0,22,19,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0.0009699434,0.00866803,0,0,0,-0.001526242,0.004514474,-0.02185101,0,0.0001533866,0,-0.006224256,0,0.008686094,0,0,-0.008579685,0,0.0005215116,-0.007499898,0,0,0,-0.00212912,-0.009693096,-0.001318355,0.002333094]},{"f":[33,33,33,33,24,-1,22,-1,-1,25,1,-1,-1,24,-1,4,-1,-1,33,-1,-1,22,17,-1,-1,-1,15,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,12.613886328427084,0,0,0.5,7.0202978971962615,0,0,6.5,0,3.9534452978042594,0,0,21.16864017939033,0,0,16.838461650489297,7.781300095563516,0,0,0,9.92481250360578,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,9,6,0,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,4.700283e-05,0,-5.867216e-05,-0.005430246,0,0,0.002668604,-0.004832824,0,0.001331974,0,0.01809867,0.002932088,0,-0.008466775,-0.001194389,0,0,0.002156275,0.007429345,-0.002351484,0,-0.008954511,-0.0007174627]},{"f":[23,22,2,1,2,-1,-1,3,22,22,-1,-1,-1,23,22,-1,-1,-1,-1,32,-1,17,17,26,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,10.655783256170313,0.39811320754716983,6.08730682361797,0,0,0.05927318295739348,16.765402642760364,16.73920708776685,0,0,0,3.5,16.978341770814396,0,0,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0.5,0,0,0,0,0],"m":[1,1,1,0,1,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,0,0,0,14,15,0,0,0,0,20,0,22,23,24,0,0,0,0,0],"r":[28,19,18,7,6,0,0,13,12,11,0,0,0,17,16,0,0,0,0,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.0007651724,0.007164905,0,0,0,-0.001447314,0.004309099,-0.02077402,0,0,-8.806794e-05,-0.005350104,0.0007921799,-0.003309924,0,0.003772032,0,0,0,-0.007594293,0.0006331756,-0.01997443,0.001945778,0.004216414]},{"f":[21,17,-1,-1,23,17,22,-1,15,-1,32,-1,-1,-1,18,17,22,-1,21,-1,19,-1,-1,17,-1,22,-1,-1,-1],"t":[2.5,10.090748425878532,0,0,0.5,10.543869317296533,13.415167820124083,0,6.0,0,7.014643613484123,0,0,0,7.033254845462222,4.0437314206251695,8.133841411710264,0,7.5,0,0.5,0,0,5.918419679874358,0,14.116600814324018,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,0,0,15,16,17,0,19,0,21,0,0,24,0,26,0,0,0],"r":[4,3,0,0,14,13,8,0,10,0,12,0,0,0,28,23,18,0,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0.0003418093,0.003862594,0,0,0,0.0001616235,0,0.001334532,0,-0.00480974,0.00327652,-0.004224149,0,0,0,0.01078442,0,0.001105006,0,-0.002661307,0.004180352,0,-0.003270458,0,0.000643998,-0.001990601,0.001083208]},{"f":[33,33,33,33,24,-1,-1,33,2,-1,-1,33,-1,-1,-1,22,21,-1,-1,32,-1,23,-1,-1,15,-1,22,-1,-1],"t":[29.297756302860655,23.58371303411409,20.752541988539424,13.895530557358477,4.5,0,0,17.460839126440824,5.538407798525416,0,0,18.93170853178854,0,0,0,15.439474430023438,6.5,0,0,6.337096134072842,0,7.5,0,0,9.92481250360578,0,9.945253865134848,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,12,0,0,0,16,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[24,15,14,7,6,0,0,11,10,0,0,13,0,0,0,19,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,4.314559e-05,-0.002764437,0,0,0.01014657,0.001390744,0,-0.001941549,0.006371583,-0.001907509,0,0,0.007328932,0.0004279037,0,0.002354332,0,-0.009641126,-0.0007013178,0,-0.008591856,0,0.002336876,-0.001814009]},{"f":[22,22,22,37,32,16,15,-1,33,-1,-1,2,-1,-1,-1,21,-1,-1,-1,15,-1,1,-1,-1,17,17,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,11.26,10.463013322934223,7.266786540694901,5.78135971352466,0,19.167172160526107,0,0,1.292481250360578,0,0,0,7.5,0,0,0,6.0,0,4.008411214953271,0,0,8.729619742794132,8.571721503073015,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,0,16,0,0,0,20,0,22,0,0,25,26,0,0,0],"r":[24,19,18,15,14,11,8,0,10,0,0,13,0,0,0,17,0,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.000155136,0,0.006947173,0.0008623198,0,0.0006292189,-0.0008782106,0.003601141,0,-0.001392002,-0.01448979,-0.005453681,0,-0.007613949,0,-2.627749e-06,0.00361854,0,0,-0.001379502,-0.01120737,0.002764111]},{"f":[23,21,23,22,-1,-1,-1,37,22,22,-1,-1,22,-1,18,18,18,4,0,-1,-1,-1,-1,-1,22,-1,-1,-1,-1],"t":[8.5,6.5,3.5,13.87915215758784,0,0,0,11.05,7.5621616439616925,6.671801068872866,0,0,8.133841411710264,0,10.651680794759452,6.584884225259038,5.303665156874805,1.792481250360578,2.5,0,0,0,0,0,10.305472761501687,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,13,0,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[28,7,6,5,0,0,0,27,12,11,0,0,14,0,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0.0002824937,-0.0008827823,0.002382819,0,0,0,0.001112901,-0.005268795,0,0.008136677,0,0,0,0,0,0.001587054,-0.002304918,0.0005089214,-0.003515407,0.001417443,0,0.001283972,-0.004576182,-0.01311168,0.00401676]},{"f":[9,4,-1,2,22,-1,-1,15,16,17,4,-1,20,15,15,-1,-1,-1,35,-1,-1,-1,32,-1,-1,16,-1,-1,-1],"t":[0.5,4.0437314206251695,0,4.128693921346326,11.876328901062104,0,0,10.376125389276174,7.651051691178929,11.503187353537466,4.80643424864552,0,1.5,5.78135971352466,5.672425341971495,0,0,0,2.5,0,0,0,6.474913355379556,0,0,10.486835021563051,0,0,0],"m":[1,1,0,0,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,0,13,14,15,0,0,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,7,6,0,0,25,22,21,12,0,18,17,16,0,0,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.0001224165,0,0,0.001455868,-0.004570556,0,0,0,0,0.003760708,0,0,0,-0.0001049058,-0.004247879,0.003187398,0,-0.007661725,-0.001041668,-0.008511198,0,-0.002015208,-0.01035709,0,0.005428551,0.0001698345,0.002017734]},{"f":[33,32,33,33,33,33,-1,25,-1,24,-1,4,-1,-1,-1,33,-1,22,17,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[37.98858650139573,10.463013322934223,29.569691584072146,23.58371303411409,20.752541988539424,13.895530557358477,0,0.5,0,6.5,0,3.9534452978042594,0,0,0,24.145246550704165,0,15.439474430023438,7.198837316474133,0,0,0,9.107199999999999,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,0,12,0,0,0,16,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,15,14,7,0,9,0,11,0,13,0,0,0,17,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-2.022886e-05,0,-0.0005912325,0,0.001127476,0,0.01660061,0.002576072,-0.002039654,0,0.007614003,0,0,0.001107133,0.008363542,-0.002015461,0,-0.0002151488,-0.007497097,0.004744692,0,0.0003507168,-0.004360782]},{"f":[23,22,22,22,16,-1,2,17,-1,-1,1,-1,-1,22,-1,1,-1,-1,22,-1,-1,32,-1,17,17,-1,-1,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.978341770814396,7.266786540694901,0,1.292481250360578,10.620215166896223,0,0,4.92309205350118,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,14,0,16,0,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,18,13,6,0,10,9,0,0,12,0,0,15,0,17,0,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0.0002578127,0,0,0.001029445,-0.003076566,0,-7.239991e-06,-0.00211741,0,-0.005784198,0,-0.002436457,0.003022392,0,0.004025647,-0.0001478317,0,0.003215843,0,0,-0.001927052,-0.01814494,0.001914057,0.003857838]},{"f":[9,2,4,2,25,4,4,-1,22,-1,22,-1,22,16,-1,-1,1,-1,-1,18,-1,22,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,10.510264838663748,9.266718101290426,9.051177541662383,6.5,7.950787011501063,4.951187557243013,0,7.499052785859982,0,8.133841411710264,0,9.40390385294334,9.207014320177532,0,0,6.241071428571429,0,0,1.292481250360578,0,12.35227706330556,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,0,13,14,0,0,17,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,19,8,0,10,0,12,0,16,15,0,0,18,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,9.675194e-05,0,-0.004952584,0,0.007284744,0,0,-0.01057729,-0.0007084206,0,0.0002194433,-0.00314305,0,0.008738726,0,-0.005298821,0.005292597,0.004901166,-0.002534666,0.004478233,-0.003200173,0.001916448]},{"f":[23,22,16,-1,37,15,22,-1,-1,25,16,22,-1,-1,22,-1,-1,-1,-1,32,-1,17,17,26,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,1.584962500721156,0,12.52,5.78135971352466,14.116600814324018,0,0,2.5,6.108524456778169,16.964783007086943,0,0,16.786010507999382,0,0,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,0.5,0,0,0,0,0],"m":[1,1,0,0,1,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,12,0,0,15,0,0,0,0,20,0,22,23,24,0,0,0,0,0],"r":[28,19,4,0,18,9,8,0,0,17,14,13,0,0,16,0,0,0,0,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0.0007184988,0,0,0,-0.0002756943,-0.002381241,0,0,0,0.002751195,-0.002592785,0,-0.000156794,0.001520778,0.008629519,-0.005130602,0,0.0031251,0,0,0,-0.006838514,0.0007578724,-0.01722602,0.001859946,0.003682919]},{"f":[21,16,-1,-1,23,17,22,-1,15,-1,32,18,-1,1,-1,-1,-1,-1,18,17,22,-1,21,-1,19,-1,-1,-1,-1],"t":[2.5,10.066089190457772,0,0,0.5,10.543869317296533,13.415167820124083,0,6.0,0,7.014643613484123,1.292481250360578,0,5.015625,0,0,0,0,6.65017628016411,4.0437314206251695,8.133841411710264,0,7.5,0,0.5,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,12,0,14,0,0,0,0,19,20,21,0,23,0,25,0,0,0,0],"r":[4,3,0,0,18,17,8,0,10,0,16,13,0,15,0,0,0,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,0.0002885846,0.003985366,0,0,0,0.0001313876,0,0.001221423,0,0,-0.002147747,0,-0.01250567,-0.004725385,0.003115967,-0.003894374,0,0,0,0.00962093,0,0.00100448,0,-0.002432377,0.003791176,-0.001118853,0.0008390306]},{"f":[22,15,2,-1,24,-1,17,-1,-1,17,1,-1,-1,-1,22,15,16,36,22,-1,-1,-1,-1,-1,4,-1,3,-1,-1],"t":[12.199665042528586,10.501837184902296,1.292481250360578,0,9.5,0,7.5507416557826375,0,0,10.269270673170688,1.3336158192090395,0,0,0,12.921158001713904,10.457380879072534,9.047123912114026,2.5,12.613886328427084,0,0,0,0,0,7.950787011501063,0,0.9640400250156348,0,0],"m":[0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,0,15,16,17,18,19,0,0,0,0,0,25,0,27,0,0],"r":[14,9,4,0,6,0,8,0,0,13,12,0,0,0,24,23,22,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0.001301026,0,0.0002070964,0,-0.00859763,-0.0007894666,0,0,-0.01114649,0.002022127,-0.00754695,0,0,0,0,0,0.005519477,-0.008970404,-0.002985038,-0.01081462,0.003236348,0,-7.603559e-05,0,-0.001200621,0.007371188]},{"f":[32,-1,37,32,1,36,-1,21,23,3,-1,1,-1,-1,22,2,-1,-1,-1,-1,21,-1,-1,-1,21,2,-1,-1,-1],"t":[2.69615871138938,0,11.26,10.463013322934223,5.417948717948718,4.5,0,7.5,0.5,0.9992038216560509,0,3.354166666666667,0,0,16.819976309430963,2.453445297804259,0,0,0,0,9.5,0,0,0,8.5,2.453445297804259,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,12,0,0,15,16,0,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,20,7,0,19,14,11,0,13,0,0,18,17,0,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0.004040335,0,0,0,0,0.002027492,0,0,0,0.006140259,0,0.0001323856,-0.001241057,0,0,0.001839137,0.0004036889,-0.001241057,-0.000989651,0,-0.0006685517,0.00336795,0.003258862,0,0,-0.001241057,-0.004854798,-0.01207521]},{"f":[23,22,22,22,16,-1,2,-1,1,-1,-1,22,-1,1,-1,-1,22,-1,1,-1,-1,25,17,17,-1,-1,-1,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.978341770814396,7.266786540694901,0,1.292481250360578,0,4.92309205350118,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,3.966632962588473,0,0,1.5,8.729619742794132,8.571721503073015,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,12,0,14,0,0,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[28,21,16,11,6,0,8,0,10,0,0,13,0,15,0,0,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.0002362128,0,0.000586864,0,-1.089922e-05,-0.001923253,0,-0.005473053,0,-0.002314521,0.002857811,0,0.003816348,0,-0.002907632,0.0016019,0,0,0,-0.0009098778,-0.01615407,0.002260491,0.002948029,0.003503356]},{"f":[22,15,16,-1,-1,17,1,-1,-1,2,-1,-1,22,15,15,22,36,-1,-1,-1,17,-1,-1,-1,2,25,-1,-1,-1],"t":[12.199665042528586,10.501837184902296,12.711021616791836,0,0,10.269270673170688,1.3336158192090395,0,0,1.292481250360578,0,0,12.921158001713904,10.457380879072534,4.247927513443585,12.613886328427084,2.5,0,0,0,1.792481250360578,0,0,0,9.293467526905953,2.5,0,0,0],"m":[0,1,1,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,14,15,16,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,24,23,20,19,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0.0005259108,-0.004051535,0,0,-0.01057788,0.001909341,0,-0.01115206,-0.002049552,0,0,0,0,0,0.005267462,-0.001952802,-0.00548722,0,-0.003240812,-0.008761213,0.003036129,0,0,-9.638163e-05,0.006034494,0.005287267]},{"f":[1,22,-1,-1,3,22,4,-1,24,-1,-1,-1,25,37,23,-1,-1,21,-1,-1,17,-1,36,22,-1,-1,3,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,4.5,0,0,0,2.5,11.05,3.5,0,0,5.5,0,0,1.292481250360578,0,2.5,12.613886328427084,0,0,0.4843597262952102,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,15,0,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,20,17,16,0,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0.000745041,0.007679012,0,0,0,9.807641e-05,0,1.89675e-05,-0.006747879,-0.007630329,0,0,0,-0.0001500136,0.0005397238,0,-0.001844344,-0.009719756,0,0.01133967,0,0,0.003224214,-0.008242932,0,0.001339661,-0.005935615]},{"f":[23,21,22,23,-1,4,4,21,-1,-1,21,-1,15,-1,-1,21,-1,-1,32,-1,17,17,33,-1,-1,-1,-1,-1,-1],"t":[8.5,10.5,17.094543568094064,3.5,0,6.451187557243013,3.9534452978042594,8.5,0,0,7.5,0,13.56045225766691,0,0,7.5,0,0,8.341935934311302,0,8.729619742794132,8.076805701955248,21.80651201897376,0,0,0,0,0,0],"m":[1,1,1,1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,11,0,13,0,0,16,0,0,19,0,21,22,23,0,0,0,0,0,0],"r":[28,27,18,5,0,15,10,9,0,0,12,0,14,0,0,17,0,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-9.504106e-05,0,0,0,0.0005573634,0.003985367,0,-0.005227653,0,-0.0007727259,0.00592756,0,0.005967394,-0.002183809,0,0.003036698,0,0,0,0.000791736,-0.006525412,-0.01564245,0.001636235,-0.005161431,0.003344097]},{"f":[22,21,18,2,-1,-1,-1,15,4,-1,-1,-1,22,15,16,36,22,-1,-1,17,-1,-1,-1,-1,2,25,-1,-1,-1],"t":[12.199665042528586,7.5,10.616089240373544,9.08214658581559,0,0,0,10.690871009292314,5.918813966585702,0,0,0,12.921158001713904,10.457380879072534,9.047123912114026,2.5,12.613886328427084,0,0,1.292481250360578,0,0,0,0,9.293467526905953,2.5,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,0,13,14,15,16,17,0,0,20,0,0,0,0,25,26,0,0,0],"r":[12,7,6,5,0,0,0,11,10,0,0,0,24,23,22,19,18,0,0,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0.0005815968,0.005846891,-0.004611869,0,0,0.0007024086,-0.002745542,-0.006745449,0,0,0,0,0,0.004851495,-0.007835567,0,0.0002944513,-0.003711812,-0.009855806,0.002884939,0,0,-9.334699e-05,0.005463333,0.005002932]},{"f":[9,22,-1,23,15,16,-1,-1,22,1,-1,-1,-1,17,-1,18,2,17,17,-1,-1,-1,-1,2,17,-1,-1,-1,-1],"t":[0.5,14.03140051979143,0,1.5,7.118941072723508,6.247927513443585,0,0,15.717370311433935,4.008411214953271,0,0,0,4.128693921346326,0,7.383290334558488,5.459058925515887,7.252657678068118,6.531360383582522,0,0,0,0,1.292481250360578,8.496347487943302,0,0,0,0],"m":[1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,10,0,0,0,14,0,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[28,3,0,13,8,7,0,0,12,11,0,0,0,15,0,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0.0001107018,0,0,0,-0.003004553,-0.01626281,0,0,-0.00231143,0.005390633,-0.009899776,0,0.0009732961,0,0,0,0,-0.004612622,0.001332836,-0.0114008,-0.0005078957,0,0,0.001072481,-0.002369264,0.003175984,0.001683987]},{"f":[33,33,12,32,-1,24,32,-1,17,-1,-1,-1,-1,22,21,33,-1,-1,-1,32,-1,23,-1,-1,22,-1,22,-1,-1],"t":[29.569691584072146,23.58371303411409,0.5,2.69615871138938,0,7.5,6.6765734127490415,0,2.69615871138938,0,0,0,0,15.439474430023438,6.5,27.807475161786478,0,0,0,6.337096134072842,0,7.5,0,0,9.945253865134848,0,11.876328901062104,0,0],"m":[1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,9,0,0,0,0,14,15,16,0,0,0,20,0,22,0,0,25,0,27,0,0],"r":[24,13,12,5,0,11,8,0,10,0,0,0,0,19,18,17,0,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0.004771361,0,0,-1.669453e-06,0,0.001359434,0.01562034,-0.001672822,-0.003645227,0,0,0,0.009285386,0.002627192,0.0004316513,0,0.001982987,0,-0.007689853,-0.0004381716,0,0.002410789,0,-0.0094077,-0.001599303]},{"f":[22,-1,21,23,22,-1,16,0,-1,33,17,-1,-1,-1,-1,4,22,-1,18,-1,-1,-1,22,16,-1,-1,37,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,8.618385502258606,2.5,0,12.738373101969731,6.9827117829050644,0,0,0,0,5.89476682248518,16.978341770814396,0,2.160964047443681,0,0,0,7.382047033558342,10.372865060112588,0,0,11.05,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,0,0,0,0,16,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,15,6,0,14,9,0,13,12,0,0,0,0,21,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0.002509542,0,0,0,0.0001689463,0,0,0.001489259,0,0,-0.003937566,-0.01850241,-0.001589133,0.0005244863,0,0,0.001664629,0,0.004105862,-0.0062718,0.006022016,0,0,-0.002887106,-0.009624479,0,-6.152185e-05,-0.009506687]},{"f":[23,21,22,22,22,23,-1,-1,22,-1,1,-1,-1,22,-1,1,-1,-1,33,15,17,-1,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.094543568094064,17.069460217922263,16.978341770814396,0.5,0,0,16.98920034885663,0,4.008411214953271,0,0,17.078739781790915,0,3.966632962588473,0,0,21.80651201897376,6.584962500721156,8.729619742794132,0,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,9,0,11,0,0,14,0,16,0,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,18,13,8,7,0,0,10,0,12,0,0,15,0,17,0,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0002193993,0.0002808209,0,-0.005153638,0,-0.002171466,0.002625696,0,0.003593046,0,-0.002738257,0.001457345,0,0,0,-0.009859065,0.003686049,0.002248802,-0.006735072,-0.004865929,0,0.00647607,-0.0003697883]},{"f":[33,32,33,36,-1,15,2,2,-1,-1,-1,22,17,-1,-1,15,37,-1,22,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[37.98858650139573,10.463013322934223,29.807475161786478,3.5,0,6.539158811108031,3.245926548164837,1.792481250360578,0,0,0,15.439474430023438,7.198837316474133,0,0,13.56045225766691,7.92,0,16.765402642760364,0,0,0,9.107199999999999,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,0,12,13,0,0,16,17,0,19,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,5,0,11,10,9,0,0,0,15,14,0,0,21,18,0,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,-3.90737e-05,0,0,0,0.004088189,-0.004097751,0.01216514,0,0,0.001069895,0.007280478,0,0,0.001692154,0,-0.001470018,-0.008369518,0.005583218,0,7.052354e-05,-0.008957745,0.004215948,0,0.0003076701,-0.003549707]},{"f":[21,17,-1,-1,22,-1,22,-1,22,-1,22,18,2,-1,-1,17,-1,-1,22,17,1,-1,-1,36,22,-1,-1,-1,-1],"t":[2.5,10.090748425878532,0,0,6.779867784305249,0,6.930301023985015,0,7.082728314749225,0,12.199665042528586,10.609627990977145,1.292481250360578,0,0,10.682994316786338,0,0,12.921158001713904,1.292481250360578,6.015625,0,0,2.5,12.613886328427084,0,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,0,7,0,9,0,11,12,13,0,0,16,0,0,19,20,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,6,0,8,0,10,0,18,15,14,0,0,17,0,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0.000241853,0.003320879,0,0.001195796,0,-0.01128507,0,-0.003501632,0,0,0,0.001515495,0.0001468721,0,-0.01120566,-0.0003871476,0,0,0,-0.001470656,0.006088181,0,0,0.004598721,-0.00747814,-0.004152115,-0.0001544794]},{"f":[1,22,-1,-1,3,22,4,-1,24,-1,-1,-1,23,17,18,-1,19,-1,-1,32,1,17,-1,-1,17,-1,-1,-1,-1],"t":[0.39811320754716983,14.247293309538719,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,4.5,0,0,0,0.5,6.924507036847409,5.108372929097653,0,0.5,0,0,7.014643613484123,2.997611464968153,11.651826559171445,0,0,7.430398005367313,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,15,0,17,0,0,20,21,22,0,0,25,0,0,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,28,19,16,0,18,0,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0.0006703231,0.007003,0,0,0,7.198779e-05,0,-3.043659e-05,-0.006337773,-0.007256128,0,0,0,-0.0002708806,0,0.001131137,0.008299261,0,0,0,0.001442156,-0.006275574,0,-0.005792673,-0.002304107,0.003688949,0.0002229141]},{"f":[33,32,33,1,15,15,36,-1,-1,22,22,-1,-1,-1,15,1,22,-1,-1,-1,-1,-1,34,-1,-1,24,-1,-1,-1],"t":[37.98858650139573,10.41931383093672,29.807475161786478,12.64516129032258,5.78135971352466,3.321928094887362,2.5,0,0,16.819976309430963,7.25627765006036,0,0,0,6.169925001442312,6.069196428571429,16.96990787537721,0,0,0,0,0,2.5,0,0,9.5,0,0,0],"m":[1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,0,0,0,15,16,17,0,0,0,0,0,23,0,0,26,0,0,0],"r":[28,25,22,21,14,9,8,0,0,13,12,0,0,0,20,19,18,0,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0.000445985,-0.003238144,0,0,-0.003413072,-0.0005217705,-0.007644091,0,0,0,0.001546206,-0.006528232,0.005051286,1.845813e-05,0.003632365,0,-0.00954444,3.28917e-05,0,0.006434276,-0.001307442,-0.001743188]},{"f":[9,16,17,16,-1,-1,-1,16,22,17,-1,-1,-1,22,22,-1,-1,22,15,-1,-1,22,22,-1,-1,4,-1,-1,-1],"t":[0.5,7.924812503605781,11.78143940991899,7.741466986401147,0,0,0,8.060695931687555,15.717370311433935,6.219136028062415,0,0,0,6.930301023985015,6.5685756301391525,0,0,16.759160379664443,13.56045225766691,0,0,16.786010507999382,16.765402642760364,0,0,3.9534452978042594,0,0,0],"m":[1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,0,14,15,0,0,18,19,0,0,22,23,0,0,26,0,0,0],"r":[28,7,6,5,0,0,0,13,12,11,0,0,0,17,16,0,0,21,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,7.090654e-05,0.001592563,-0.006537682,0,0,0,-0.00400971,0.001696953,-0.009340993,0,0,0.001775594,-0.01071731,0,0,-2.521112e-05,0.007061904,0,0,-0.001481205,-0.02068258,0,0.003575407,-0.001792585,0.001555201]},{"f":[22,1,16,-1,-1,16,-1,-1,33,22,-1,17,21,36,-1,3,-1,26,18,-1,-1,-1,16,-1,-1,-1,15,-1,-1],"t":[11.36027577861459,1.13125,10.794415866350105,0,0,12.711021616791836,0,0,29.807475161786478,11.424679641461715,0,12.18494601563336,8.5,3.5,0,0.3162042875157629,0,0.5,7.198837316474133,0,0,0,10.794415866350105,0,0,0,9.92481250360578,0,0],"m":[0,0,1,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,14,0,16,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,11,0,25,22,15,0,17,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.0002211887,-0.01049953,0,0.0007783227,-0.003367836,0,0,-0.003128726,0,0,0,-0.0001124732,0,0.009205188,0,0,-0.0005832406,0.006751133,0.006355479,0,0.002943048,-0.003190434,-0.00245371,0,-0.009374101,-0.001510996]},{"f":[1,1,-1,18,-1,-1,2,5,20,-1,-1,22,22,22,23,22,-1,-1,-1,-1,-1,32,-1,17,10,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,0,4.245926548164837,0,0,9.068775776507199,0.5,0.5,0,0,17.094543568094064,16.98920034885663,16.98413870862295,3.5,15.717370311433935,0,0,0,0,0,8.341935934311302,0,8.729619742794132,0.5,0,0,0,0],"m":[1,1,0,1,0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,0,0,12,13,14,15,16,0,0,0,0,0,22,0,24,25,0,0,0,0],"r":[6,3,0,5,0,0,28,11,10,0,0,21,20,19,18,17,0,0,0,0,0,23,0,27,26,0,0,0,0],"v":[0,0,6.721089e-05,0,0.001274698,0.008402274,0,0,0,-0.0006467162,-0.005010159,0,0,0,0,0,-0.0001689153,-0.004791333,0.001106801,-0.005977261,0.002961956,0,0.002946815,0,0,-0.00247976,-0.01510932,0.001831796,-0.006106365]},{"f":[22,-1,21,23,22,-1,16,0,-1,33,17,-1,-1,-1,-1,4,22,-1,18,-1,-1,-1,22,-1,37,18,-1,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,8.618385502258606,2.5,0,12.738373101969731,6.9827117829050644,0,0,0,0,5.89476682248518,16.96990787537721,0,1.292481250360578,0,0,0,7.382047033558342,0,11.05,10.620215166896223,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,0,0,0,0,16,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[2,0,22,15,6,0,14,9,0,13,12,0,0,0,0,21,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0.002326739,0,0,0,0.0001523433,0,0,0.001426153,0,0,-0.003654725,-0.01734284,-0.001387002,0.0004467789,0,0,0.001541864,0,0.003775341,-0.004943877,0.005612922,0,-0.003633471,0,0,0.0001112383,-0.002000224,-0.008264276]},{"f":[25,1,24,22,2,-1,-1,-1,21,-1,20,16,1,22,2,-1,-1,-1,17,-1,-1,21,-1,0,-1,-1,-1,-1,-1],"t":[6.5,5.417948717948718,7.5,16.725912049439096,6.584884225259038,0,0,0,7.5,0,0.5,9.250298417906333,2.876915188080302,8.32532955673354,5.14960400919364,0,0,0,7.344289078557851,0,0,8.5,0,2.5,0,0,0,0,0],"m":[1,1,0,1,1,0,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,11,12,13,14,15,0,0,0,19,0,0,22,0,24,0,0,0,0,0],"r":[28,27,8,7,6,0,0,0,10,0,26,21,18,17,16,0,0,0,20,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0.001216514,0.01158271,-0.004069929,0,0.0002255081,0,0,0,0,0,0.0006705586,0.01059693,-0.0006432798,0,-0.004488196,0.001017563,0,-0.005102984,0,0.004771253,-0.003224207,0.0004715571,-0.0003864214,0.003054236]},{"f":[23,21,22,-1,17,16,-1,-1,15,15,21,-1,25,-1,-1,17,-1,-1,21,-1,22,22,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,3.39067985676233,5.672425341971495,0,0,6.857980995127572,6.539158811108031,4.5,0,0.5,0,0,8.729619742794132,0,0,4.5,0,16.786010507999382,16.759160379664443,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,0,0,0,0,27,0,0],"r":[26,25,4,0,8,7,0,0,18,15,12,0,14,0,0,17,0,0,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0.0001138627,0,0,-0.0009515492,0.001370636,0,0,0,-0.003844874,0,-0.001342534,0.006583699,0,-0.01722518,0.004264006,0,0.00651735,0,0,-0.0009888649,-0.0101666,0.0005828408,-0.004336184,0,0.006059314,-0.0004896122]},{"f":[33,32,23,17,18,-1,19,-1,-1,1,17,2,-1,-1,-1,1,17,-1,-1,15,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,0.5,6.924507036847409,5.108372929097653,0,0.5,0,0,2.997611464968153,11.651826559171445,9.20088558729989,0,0,0,6.646978021978022,7.430398005367313,0,0,6.247927513443585,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,0,1,1,0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,12,0,0,0,16,17,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,22,9,6,0,8,0,0,15,14,13,0,0,0,19,18,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.0002505383,0,0.001020107,0.007837077,0,0,0,0.0007260958,0.006451031,-0.005697246,0,0,-0.008742818,-0.002563929,0,-0.00271697,0.007363325,0.000186562,0,0.006014992,-0.001022474,0,0.0003591262,-0.003272633]},{"f":[32,-1,1,-1,33,2,-1,17,-1,-1,33,-1,15,15,17,1,-1,-1,-1,-1,15,17,15,-1,-1,-1,22,-1,-1],"t":[2.69615871138938,0,5.417948717948718,0,13.895530557358477,1.292481250360578,0,2.69615871138938,0,0,17.460839126440824,0,6.169925001442312,4.169925001442312,1.792481250360578,6.015625,0,0,0,0,6.820178962415188,8.729619742794132,6.539158811108031,0,0,0,16.98920034885663,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,0,3,0,5,6,0,8,0,0,11,0,13,14,15,16,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[2,0,4,0,10,7,0,9,0,0,12,0,20,19,18,17,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0.003266462,0,0.0001111334,0,0,-0.004611595,0,0.003048882,-0.002267021,0,0.00588738,0,0,0,0,-0.002743834,0.0046181,-0.004541984,0.004605265,0,0,0,-0.002504499,-0.01182707,0.004155974,0,-0.0005920309,0.001440559]},{"f":[22,1,16,-1,-1,-1,33,22,-1,22,18,-1,22,-1,-1,22,16,16,-1,18,-1,-1,-1,2,-1,-1,15,-1,-1],"t":[11.36027577861459,1.13125,10.794415866350105,0,0,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.680359255641626,0,11.683871801979876,0,0,12.921158001713904,10.462502272597265,4.247927513443585,0,1.792481250360578,0,0,0,9.293467526905953,0,0,9.92481250360578,0,0],"m":[0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,11,0,13,0,0,16,17,18,0,20,0,0,0,24,0,0,27,0,0],"r":[6,5,4,0,0,0,26,9,0,15,12,0,14,0,0,23,22,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.0002392599,-0.00961246,0.0006318935,0,0,-0.00298601,0,0,0.001099376,0,-0.01210346,-0.0005968885,0,0,0,-0.0008285111,0,-0.002724494,-0.008463136,0.002555135,0,-4.642508e-05,0.004703631,0,-0.008613534,-0.00137451]},{"f":[22,-1,21,23,22,-1,1,-1,15,-1,33,-1,-1,4,22,-1,18,-1,-1,-1,22,15,-1,-1,37,22,-1,-1,-1],"t":[2.084962500721156,0,6.5,2.5,14.03140051979143,0,2.532051282051282,0,5.044394119358453,0,19.167172160526107,0,0,5.89476682248518,16.96990787537721,0,1.292481250360578,0,0,0,7.5621616439616925,9.060695931687555,0,0,11.05,8.133841411710264,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,0,11,0,0,14,15,0,17,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,20,13,6,0,8,0,10,0,12,0,0,19,16,0,18,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.002209604,0,0,0,0.0001347799,0,0.001613741,0,-0.004349427,0,0.006890206,-0.002002828,0,0,0.001405166,0,0.003550636,-0.004666927,0.005284749,0,0,-0.002307635,-0.008837002,0,0,0.006668917,-0.0001234677,-0.007799607]},{"f":[23,21,25,1,24,22,2,-1,-1,-1,21,17,-1,-1,20,-1,-1,4,21,-1,-1,25,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,6.5,5.417948717948718,7.5,16.725912049439096,6.584884225259038,0,0,0,7.5,11.969133019829592,0,0,0.5,0,0,6.499295214872664,8.5,0,0,0.5,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,1,1,0,0,0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,19,0,0,22,0,0,0,0,27,0,0],"r":[26,25,24,17,10,9,8,0,0,0,14,13,0,0,16,0,0,21,20,0,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.001119559,0.01095692,-0.003983497,0,0,0.0002432535,-0.00297402,0,-0.001418748,0.0004017068,0,0,-0.0003979971,0.002303538,0,-0.0005404246,-0.01110717,0.003049287,-0.00412,0,0.005716597,-0.0004705366]},{"f":[22,-1,33,23,1,-1,22,0,-1,-1,15,22,-1,33,24,36,-1,-1,-1,22,-1,1,-1,-1,-1,-1,34,-1,-1],"t":[11.36027577861459,0,29.807475161786478,0.5,1.588659147869674,0,11.722778233171695,3.5,0,0,9.38586240064146,11.775312925643524,0,19.167172160526107,6.5,2.5,0,0,0,14.03140051979143,0,5.015625,0,0,0,0,2.5,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,12,0,14,15,16,0,0,0,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,25,6,0,10,9,0,0,24,13,0,19,18,17,0,0,0,21,0,23,0,0,0,0,28,0,0],"v":[0,0.0003383055,0,0,0,0.0003005355,0,0,-0.002632079,-0.009234796,0,0,0.003468886,0,0,0,0.004214829,-0.001859661,0.009879484,0,-0.001806677,0,-0.01468731,-0.002836158,0.003476943,0.0002366284,0,-0.006525231,-0.001078527]},{"f":[22,-1,23,17,17,17,3,3,-1,-1,22,17,18,-1,16,17,-1,-1,-1,-1,15,-1,32,-1,-1,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,11.651826559171445,10.69085411329249,10.616089240373544,0.9992038216560509,0.4226190476190476,0,0,13.827286379004683,9.845435504646158,5.108372929097653,0,12.225508720236002,7.980000966034041,0,0,0,0,6.0,0,7.014643613484123,0,0,0,0,0,0],"m":[0,0,0,1,1,1,0,1,0,0,1,1,1,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,12,13,0,15,16,0,0,0,0,21,0,23,0,0,0,0,0,0],"r":[2,0,28,27,26,25,10,9,0,0,20,19,14,0,18,17,0,0,0,0,22,0,24,0,0,0,0,0,0],"v":[0,0.002124718,0,0,0,0,0,0,0.000899093,0.006933367,0,0,0,-0.0002734773,0,0,0.001111434,0.005347789,-0.003174254,-0.001748001,0,0.001115941,0,-0.003479098,0.002844743,-0.006915884,0.003088691,-0.005107263,0.0001640008]},{"f":[4,-1,2,18,16,22,-1,-1,-1,-1,15,16,4,-1,-1,32,15,17,-1,-1,-1,-1,16,-1,22,15,-1,-1,-1],"t":[4.0437314206251695,0,4.128693921346326,5.918419679874358,7.118941072723508,16.819976309430963,0,0,0,0,10.376125389276174,7.651051691178929,4.80643424864552,0,0,6.474913355379556,9.756556322524087,7.723832212141186,0,0,0,0,10.486835021563051,0,16.759160379664443,13.56045225766691,0,0,0],"m":[1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,0,0,0,11,12,13,0,0,16,17,18,0,0,0,0,23,0,25,26,0,0,0],"r":[2,0,10,9,8,7,0,0,0,0,22,15,14,0,0,21,20,19,0,0,0,0,24,0,28,27,0,0,0],"v":[0,9.72807e-05,0,0,0,0,0.0005393365,-0.00649996,-0.007236603,0.001963577,0,0,0,0.003126606,-0.0002115249,0,0,0,-0.003281519,0.0002860037,-0.006205648,-0.007595421,0,0.004622934,0,0,0.0003168406,0.007026837,-0.006354625]},{"f":[23,21,22,23,-1,-1,17,16,-1,-1,15,4,22,33,-1,15,4,-1,-1,-1,-1,2,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,3.5,0,0,3.39067985676233,5.672425341971495,0,0,7.366322214245816,4.951187557243013,16.819976309430963,17.460839126440824,0,6.247927513443585,3.39067985676233,0,0,0,0,5.8698903048866296,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,12,13,14,0,16,17,0,0,0,0,22,0,0,0,0,27,0,0],"r":[26,25,6,5,0,0,10,9,0,0,24,21,20,15,0,19,18,0,0,0,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,2.525409e-05,0.001471628,0,0,-0.0008500213,0.001176147,0,0,0,0,0.007483575,0,0,-0.00132291,0.003509125,-0.009210508,-0.005401652,0,-0.01004521,-0.0003642403,-0.000122171,-0.003956237,0,0.005459312,-0.000471872]},{"f":[15,15,36,-1,16,-1,-1,22,22,-1,-1,-1,32,-1,15,22,-1,22,-1,-1,0,26,1,-1,-1,-1,22,-1,-1],"t":[5.78135971352466,3.321928094887362,2.5,0,11.031356596255709,0,0,16.819976309430963,7.25627765006036,0,0,0,3.753897320099348,0,6.169925001442312,16.975010474608837,0,17.069460217922263,0,0,11.5,0.5,4.92309205350118,0,0,0,10.558733410935197,0,0],"m":[0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,0,15,16,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,14,0,20,17,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0.0003826748,0,-0.006157388,-3.276973e-05,0,0,-0.002918107,-0.0003977363,-0.006525911,0,0.009982826,0,0,0.00178727,0,-0.00639011,0.002065656,0,0,0,0.000328874,-0.0004170614,0.003289278,0,-0.001863945,-0.008852942]},{"f":[22,5,3,-1,-1,-1,22,22,36,-1,0,-1,17,-1,2,-1,-1,-1,2,2,4,25,-1,-1,-1,17,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.921158001713904,12.613886328427084,2.5,0,3.5,0,1.792481250360578,0,1.292481250360578,0,0,0,9.293467526905953,9.051177541662383,7.950787011501063,2.5,0,0,0,9.44286754736589,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,9,0,11,0,13,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[6,5,4,0,0,0,18,17,10,0,12,0,14,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.000149462,-0.005989927,0.0003259359,0,0,0,0.004295521,0,0.0007139778,0,-0.001314398,0,-0.008289724,-0.001467687,-0.003350586,0,0,0,0,-6.939782e-05,0.006119372,0.007099082,0,-0.008962983,-0.0002786624,0.004348776]},{"f":[23,4,1,-1,-1,22,15,21,16,22,-1,-1,15,-1,17,-1,22,-1,-1,-1,-1,4,15,-1,-1,-1,15,-1,-1],"t":[8.5,4.0437314206251695,14.266666666666666,0,0,16.725912049439096,13.0003521774803,9.5,1.584962500721156,11.488914557581817,0,0,10.376125389276174,0,9.27958770372691,0,11.722778233171695,0,0,0,0,6.434604125275413,9.002815015607053,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,1,1,0,1,0,0,1,0,1,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,13,0,15,0,17,0,0,0,0,22,23,0,0,0,27,0,0],"r":[26,5,4,0,0,21,20,19,12,11,0,0,14,0,16,0,18,0,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,6.865711e-05,0.003754149,0,0,0,0,0,0.004269172,-0.001900848,0,-0.0005608302,0,-8.100782e-05,0,-0.0002282675,0.006496031,-0.00300301,0.006159161,0,0,-0.001514693,-0.006082287,0.003652847,0,0.005300254,-0.0004584858]},{"f":[23,18,22,18,-1,16,-1,-1,1,-1,1,22,-1,-1,15,-1,32,-1,-1,21,17,-1,-1,15,-1,2,-1,-1,-1],"t":[0.5,9.845435504646158,13.87915215758784,5.108372929097653,0,12.225508720236002,0,0,1.000936329588015,0,5.015625,14.03140051979143,0,0,5.906890595608519,0,7.014643613484123,0,0,4.5,10.090748425878532,0,0,9.264442600226602,0,1.292481250360578,0,0,0],"m":[0,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,0,11,12,0,0,15,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,19,8,5,0,7,0,0,10,0,14,13,0,0,16,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,-0.0001255309,0,0.001353687,-0.002898473,0,0.001865353,0,0,-0.003076776,-0.01364371,0,0.004347118,0,-0.00258723,0.002684676,0,0,-0.003101084,0.001729881,0,-0.001257143,0,-0.01107256,-0.002440401,0.0001610808]},{"f":[3,22,15,4,22,3,-1,-1,-1,-1,-1,-1,23,18,17,22,16,18,-1,-1,-1,15,-1,33,-1,-1,-1,-1,-1],"t":[0.034940494354592615,16.765402642760364,10.113742166049189,2.453445297804259,16.73920708776685,0.007093253968253968,0,0,0,0,0,0,0.5,9.845435504646158,9.27958770372691,13.87915215758784,11.696967526234287,6.65017628016411,0,0,0,6.0,0,18.059836468389907,0,0,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,17,18,0,0,0,22,0,24,0,0,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,27,26,21,20,19,0,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003940323,-0.001229854,0.004673495,-0.003234182,-0.005490523,-0.007922677,0,0,0,0,0,0,-1.447437e-05,0.00203141,-0.003255773,0,0.001062179,0,0.004619193,-0.002914414,0.004380231,-0.001501637,0.0002181123]},{"f":[22,-1,21,23,22,-1,32,-1,19,-1,-1,4,22,-1,4,22,-1,-1,2,-1,-1,-1,22,-1,22,-1,37,-1,-1],"t":[2.084962500721156,0,6.5,2.5,15.717370311433935,0,2.69615871138938,0,0.5,0,0,5.89476682248518,16.819976309430963,0,2.160964047443681,16.978341770814396,0,0,2.453445297804259,0,0,0,7.5621616439616925,0,8.32532955673354,0,11.05,0,0],"m":[0,0,1,1,1,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,0,9,0,0,12,13,0,15,16,0,0,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,22,11,6,0,8,0,10,0,0,21,14,0,18,17,0,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0.00199048,0,0,0,9.128872e-05,0,-0.000316437,0,-0.00291417,-0.01124682,0,0,0.001822686,0,0,0.002678696,-0.003640472,0,-0.01409231,0.0001573585,0.004803204,0,-0.002766351,0,0.005773383,0,-0.0001311288,-0.007123591]},{"f":[33,32,4,4,-1,-1,33,2,22,-1,-1,15,15,16,-1,22,-1,-1,-1,21,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,4.0437314206251695,3.5221970596792267,0,0,27.44694185105858,4.128693921346326,11.876328901062104,0,0,10.043027283594547,9.269126679149418,7.651051691178929,0,7.770639075561187,0,0,0,4.5,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,12,13,14,0,16,0,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,22,11,10,0,0,19,18,15,0,17,0,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,1.408731e-05,0.0012715,0,0,0,0.00145741,-0.003540538,0,0,0,6.612074e-05,0,-0.008140946,-0.0009289235,-0.005634328,0,0.005324949,0.000323571,-0.004950917,0,0.005666531,-0.0008281219,0,0.0004467666,-0.00301843]},{"f":[22,-1,22,22,22,-1,-1,16,-1,-1,22,-1,33,22,-1,22,18,-1,22,-1,-1,22,22,-1,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,0,5.78135971352466,0,0,11.36027577861459,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.680359255641626,0,11.683871801979876,0,0,12.921158001713904,12.613886328427084,0,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,0,13,14,0,16,17,0,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,12,0,26,15,0,21,18,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0.001014989,0,0,0,-0.003566221,0.0009036857,0,-0.002507699,-0.008121796,0,0.0004799374,0,0,-0.00257013,0,0,0.0009894666,0,-0.01050575,-0.0003519621,0,0,-0.0004909778,-0.003194478,-4.076873e-06,0,-0.006016436,-0.0009754483]},{"f":[3,22,15,4,22,3,-1,-1,-1,-1,-1,-1,23,18,17,22,-1,15,-1,33,-1,18,-1,1,-1,-1,-1,-1,-1],"t":[0.034940494354592615,16.765402642760364,10.113742166049189,2.453445297804259,16.73920708776685,0.007093253968253968,0,0,0,0,0,0,0.5,9.845435504646158,9.27958770372691,13.87915215758784,0,6.0,0,18.059836468389907,0,1.292481250360578,0,5.015625,0,0,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,0,18,0,20,0,22,0,24,0,0,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,27,26,17,0,19,0,21,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003737364,-0.001207884,0.004449675,-0.003099621,-0.005185214,-0.007545809,0,0,0,0,4.848573e-05,0,0.001009865,0,0.004388053,0,-0.001190317,0,-0.00953042,-0.002870919,0.004142386,-0.001424595,0.0002143892]},{"f":[15,15,22,-1,4,-1,-1,22,-1,16,16,22,-1,-1,15,-1,-1,-1,32,-1,1,21,-1,16,-1,-1,21,-1,-1],"t":[5.78135971352466,4.087462841250339,16.765402642760364,0,2.160964047443681,0,0,10.63843335887114,0,6.409390936137702,5.169925001442312,12.613886328427084,0,0,5.491853096329675,0,0,0,3.753897320099348,0,11.278421650655924,8.5,0,11.98370619265935,0,0,5.5,0,0],"m":[0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,0,10,11,12,0,0,15,0,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[18,7,4,0,6,0,0,9,0,17,14,13,0,0,16,0,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,0.0001740755,0,-0.007310146,-0.0008007952,0,-0.003169831,0,0,0,0.0002312804,-0.002729693,0,0.001717169,-0.001112345,-0.00254345,0,0.009407714,0,0,5.793254e-05,0,0.002333521,-0.00462228,0,-0.0009006177,-0.009033052]},{"f":[23,21,4,21,-1,17,-1,-1,22,24,-1,21,18,-1,-1,-1,22,-1,22,-1,2,-1,16,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,4.0437314206251695,8.5,0,6.781300095563516,0,0,16.725912049439096,3.5,0,9.5,8.03615005796417,0,0,0,16.786010507999382,0,16.819976309430963,0,5.065928480304397,0,7.774787059601174,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,0,0,0,17,0,19,0,21,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,16,11,0,15,14,0,0,0,18,0,20,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,4.638912e-05,0,0.00279699,-0.00127028,0,0,0.004944411,0,0,-0.0003494766,0.0009899167,-0.002768984,0,-0.00574986,0,0.002527957,0,-0.006558926,0,0.002865991,-0.002898646,-0.003635073,0,0.005062457,-0.000537745]},{"f":[1,22,1,-1,2,-1,-1,-1,3,22,4,-1,-1,-1,25,37,-1,21,-1,-1,17,-1,36,22,-1,-1,16,-1,-1],"t":[0.39811320754716983,14.247293309538719,0.01933482600988898,0,4.95344529780426,0,0,0,0.05277777777777778,16.759160379664443,1.292481250360578,0,0,0,2.5,11.05,0,5.5,0,0,1.292481250360578,0,2.5,12.613886328427084,0,0,2.807354922057604,0,0],"m":[0,1,1,0,0,0,0,0,0,1,1,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,0,5,0,0,0,9,10,11,0,0,0,15,16,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[8,7,4,0,6,0,0,0,14,13,12,0,0,0,20,17,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.0004415881,0,-0.0003182101,0.003490246,0.006250146,0,0,0,-8.737013e-05,-0.00333385,-0.006017934,0,0,-3.94821e-06,0,-0.001310489,-0.006720971,0,0.009020025,0,0,0.002509324,-0.006581241,0,-0.004672639,0.0004232735]},{"f":[32,33,33,12,32,-1,37,32,-1,-1,-1,-1,33,-1,22,17,-1,-1,22,-1,22,-1,-1,1,-1,-1,2,-1,-1],"t":[10.463013322934223,29.807475161786478,23.58371303411409,0.5,2.69615871138938,0,7.92,6.6765734127490415,0,0,0,0,24.145246550704165,0,15.439474430023438,7.198837316474133,0,0,16.98920034885663,0,17.083799521744545,0,0,9.107199999999999,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,15,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[26,23,12,11,6,0,10,9,0,0,0,0,14,0,18,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0.003394457,0,0,-6.755307e-06,0.004409221,-0.00108217,-0.003710138,0,0.005911191,0,0,0.0006396892,0.005730768,0,-0.003249621,0,0.0041483,-0.003445515,0,-4.920701e-05,-0.0042937,0,0.003904603,-0.001197929]},{"f":[22,-1,23,17,3,-1,36,-1,1,15,-1,-1,21,22,22,-1,-1,2,33,-1,17,17,-1,-1,-1,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,0.9992038216560509,0,2.5,0,3.354166666666667,11.683433292835403,0,0,3.5,12.199665042528586,11.67903991803059,0,0,2.453445297804259,18.059836468389907,0,5.281121212110536,1.292481250360578,0,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,1,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,0,0,13,14,15,0,0,18,19,0,21,22,0,0,0,0,0,0,0],"r":[2,0,28,27,6,0,8,0,12,11,0,0,26,17,16,0,0,25,20,0,24,23,0,0,0,0,0,0,0],"v":[0,0.001827714,0,0,0,0.001816488,0,0.002251916,0,0,7.22592e-05,-0.004063938,0,0,0,3.427187e-05,0.006371652,0,0,0.006265002,0,0,-0.0003079778,0.006578411,-0.002741484,-0.003268674,-0.002557759,-0.006289677,0.0001489423]},{"f":[23,22,-1,17,16,37,-1,-1,-1,15,4,22,4,18,-1,-1,-1,-1,2,-1,-1,22,22,-1,-1,-1,15,-1,-1],"t":[8.5,14.03140051979143,0,3.39067985676233,5.672425341971495,6.890000000000001,0,0,0,7.366322214245816,4.951187557243013,16.819976309430963,3.39067985676233,7.344289078557851,0,0,0,0,5.8698903048866296,0,0,16.786010507999382,16.759160379664443,0,0,0,6.539158811108031,0,0],"m":[1,1,0,0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,12,13,14,0,0,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,9,8,7,0,0,0,21,18,17,16,15,0,0,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,8.80438e-05,0,0,0,0.0003240069,-0.002697001,0.001020087,0,0,0,0,0,-0.002148216,0.001678976,0.003944622,-0.00487594,0,-0.009377606,-0.0003912682,0,0,-0.0005805953,-0.009430026,0.0006740557,0,0.00490136,-0.0005182563]},{"f":[33,32,16,17,-1,-1,16,22,17,-1,-1,-1,22,18,-1,22,-1,17,-1,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,7.924812503605781,11.78143940991899,0,0,8.060695931687555,15.717370311433935,6.219136028062415,0,0,0,11.722778233171695,0.5,0,11.406459975416917,0,10.680359255641626,0,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,14,0,16,0,18,0,0,0,22,0,0,25,0,27,0,0],"r":[24,21,6,5,0,0,12,11,10,0,0,0,20,15,0,17,0,19,0,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0.0001110503,-0.005348278,0,0,0,-0.00323884,0.001838966,-0.00838379,0,0,-0.006660641,0,8.427872e-05,0,-0.001954677,-0.00840656,0.0002552836,0,0.005190857,-0.000747262,0,0.0004666525,0,-0.0003238991,-0.004187938]},{"f":[15,15,22,21,-1,-1,4,-1,-1,22,-1,16,-1,-1,32,-1,15,1,22,-1,-1,-1,0,26,-1,-1,22,-1,-1],"t":[5.78135971352466,4.087462841250339,16.765402642760364,8.5,0,0,2.160964047443681,0,0,10.63843335887114,0,6.409390936137702,0,0,3.753897320099348,0,6.169925001442312,6.069196428571429,16.96990787537721,0,0,0,11.5,0.5,0,0,10.558733410935197,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,0,12,0,0,15,0,17,18,19,0,0,0,23,24,0,0,27,0,0],"r":[14,9,6,5,0,0,8,0,0,11,0,13,0,0,16,0,22,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.0002437255,-0.002327731,0,-0.006942959,-0.00072086,0,-0.003057063,0,-0.0002749562,-0.002402105,0,0.008611349,0,0,0,0.001111783,-0.004837908,0.003089435,0,0,3.196332e-06,0.002941384,0,-0.00158424,-0.007566388]},{"f":[36,-1,22,2,-1,-1,22,16,16,22,36,-1,-1,-1,18,-1,-1,-1,2,2,4,25,-1,-1,-1,17,-1,-1,-1],"t":[1.5,0,12.199665042528586,1.292481250360578,0,0,12.921158001713904,10.462502272597265,4.247927513443585,12.613886328427084,2.5,0,0,0,1.792481250360578,0,0,0,9.293467526905953,9.051177541662383,7.950787011501063,2.5,0,0,0,9.44286754736589,0,0,0],"m":[0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,6,5,0,0,18,17,14,13,12,0,0,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.00255039,0,0,0.000728523,-4.815958e-05,0,0,0,0,0,0.003848819,-0.001122114,-0.003735257,0,-0.002178583,-0.007648388,0.0020939,0,0,0,0,-5.999851e-05,0.005012247,0.006468246,0,-0.008250555,-0.0004346035,0.003964258]},{"f":[23,3,-1,36,-1,1,15,17,17,-1,-1,-1,16,-1,-1,21,-1,32,19,2,-1,-1,22,21,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,2.5,0,3.354166666666667,11.683433292835403,11.78143940991899,10.69085411329249,0,0,0,11.756139298372927,0,0,1.5,0,6.337096134072842,0.5,1.292481250360578,0,0,12.199665042528586,3.5,0,0,0,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,8,9,0,0,0,13,0,0,16,0,18,19,20,0,0,23,24,0,0,0,0,0],"r":[28,3,0,5,0,15,12,11,10,0,0,0,14,0,0,17,0,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0.001726954,0,0.00208007,0,0,0,0,2.366071e-05,0.002836776,-0.005019183,0,-0.01061265,0.0006011262,0,0.000727132,0,0,0,-0.00985168,-0.001685075,0,0,0.006674446,-0.000408383,-0.002023575,0.002314806,0.000145443]},{"f":[33,32,16,17,-1,-1,16,22,21,-1,-1,-1,22,18,-1,22,-1,17,-1,-1,32,-1,-1,24,-1,-1,2,-1,-1],"t":[37.98858650139573,10.41931383093672,7.924812503605781,11.78143940991899,0,0,8.060695931687555,15.717370311433935,5.5,0,0,0,11.722778233171695,0.5,0,11.406459975416917,0,1.792481250360578,0,0,4.999295214872664,0,0,9.5,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,14,0,16,0,18,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,12,11,10,0,0,0,20,15,0,17,0,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.0001071198,-0.004812333,0,0,0,0.002057969,-0.002844087,-0.007976935,0,0,-0.006183116,0,6.7705e-05,0,0.000455753,-0.005138359,0,-0.005441178,0.0003175409,0,0.00485426,-0.0007388556,0,0.000404014,-0.002674635]},{"f":[36,-1,22,2,-1,24,-1,17,-1,-1,22,22,36,-1,0,15,-1,-1,17,-1,2,24,-1,-1,-1,17,-1,-1,-1],"t":[1.5,0,12.199665042528586,1.292481250360578,0,9.5,0,7.5507416557826375,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,2.0,0,0,1.792481250360578,0,1.292481250360578,4.5,0,0,0,3.084962500721156,0,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,0,15,16,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[2,0,10,5,0,7,0,9,0,0,28,25,14,0,18,17,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.002508185,0,0,0.0006939558,0,9.094653e-05,0,-0.00690872,-0.0001562637,0,0,0,0.003594141,0,0,-0.001041918,0.001923645,0,-0.001041918,0,0,-0.004205539,-0.01139327,-0.001041918,0,-0.00203818,-0.006635357,-2.064016e-05]},{"f":[22,-1,17,16,-1,22,-1,-1,32,-1,33,2,-1,-1,17,17,-1,19,-1,-1,21,-1,18,4,-1,-1,18,-1,-1],"t":[14.03140051979143,0,3.39067985676233,5.672425341971495,0,16.903290731167747,0,0,5.22766361015228,0,20.752541988539424,4.80643424864552,0,0,7.392087364565857,6.864066550153108,0,1.5,0,0,5.5,0,8.496347487943302,5.065928480304397,0,0,9.26091905453048,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,0,11,12,0,0,15,16,0,18,0,0,21,0,23,24,0,0,27,0,0],"r":[2,0,8,5,0,7,0,0,10,0,14,13,0,0,20,17,0,19,0,0,22,0,26,25,0,0,28,0,0],"v":[0,8.798333e-05,0,0,-0.0006454961,0,0.0007231869,0.004766882,0,-0.007975177,0,0,0.01016804,-0.002557033,0,0,-0.0013518,0,-0.01271856,-0.00198334,0,0.003696092,0,0,0.0007465461,-0.001742686,0,-0.005661424,-0.0002768557]},{"f":[23,3,-1,18,36,-1,17,16,15,-1,-1,-1,-1,18,15,-1,-1,-1,22,2,-1,1,-1,24,-1,1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,9.845435504646158,2.5,0,9.27958770372691,11.696967526234287,10.342074667999139,0,0,0,0,10.69085411329249,10.501837184902296,0,0,0,14.03140051979143,1.292481250360578,0,5.417948717948718,0,11.5,0,6.951219512195122,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,0,7,8,9,0,0,0,0,14,15,0,0,0,19,20,0,22,0,24,0,26,0,0,0],"r":[18,3,0,13,6,0,12,11,10,0,0,0,0,17,16,0,0,0,28,21,0,23,0,25,0,27,0,0,0],"v":[0,0,0.001664041,0,0,0.002844147,0,0,0,-0.0002009058,0.003418599,-0.002784184,0.003386913,0,0,-0.00141477,-0.007496345,0.0006935442,0,0,0.001495873,0,0.0005189424,0,-0.0004706574,0,-0.001019517,-0.007720936,-0.0001876315]},{"f":[36,-1,33,32,15,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,1,19,-1,-1,-1,24,-1,-1,2,-1,-1],"t":[1.5,0,37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,8.135583053967,1.5,0,0,0,9.5,0,0,2.453445297804259,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,0,12,13,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,23,6,0,16,9,0,11,0,15,14,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.002462274,0,0,0,8.098606e-05,0,0,-0.006796089,0,0.0007374584,0,0,0.0004913761,-0.004055702,-0.008385,0,0.002753377,0,0,-0.0002735123,0.002435681,-0.004810857,0,0.00462602,-0.0006741461,0,0.0003244513,-0.002560796]},{"f":[22,22,16,-1,2,36,-1,-1,-1,16,-1,-1,22,1,22,16,-1,-1,-1,22,-1,-1,32,-1,17,-1,18,-1,-1],"t":[16.98920034885663,16.978341770814396,6.882643049361842,0,1.292481250360578,3.5,0,0,0,5.044394119358453,0,0,17.078739781790915,4.145833333333334,17.069460217922263,7.741466986401147,0,0,0,17.02942582914998,0,0,8.341935934311302,0,8.571721503073015,0,8.729619742794132,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,0,0,23,0,25,0,27,0,0],"r":[12,9,4,0,8,7,0,0,0,11,0,0,22,19,18,17,0,0,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0.0001606567,0,0,0.00684168,0.0003925926,-0.0005865316,0,-0.005100752,-0.0009147154,0,0,0,0,-0.005145517,0.000330927,0.002053248,0,0.002260854,0.005774756,0,0.003585817,0,-0.0003572266,0,-0.008953791,0.0009740543]},{"f":[23,3,-1,1,-1,-1,37,19,35,17,22,-1,21,2,-1,22,-1,-1,-1,-1,-1,17,21,15,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,3.354166666666667,0,0,7.92,1.5,1.5,1.292481250360578,8.133841411710264,0,7.5,2.160964047443681,0,16.819976309430963,0,0,0,0,0,7.430398005367313,5.5,6.247927513443585,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,0,13,14,0,16,0,0,0,0,0,22,23,24,0,0,0,0,0],"r":[6,3,0,5,0,0,28,21,20,19,12,0,18,15,0,17,0,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0.001607007,0,-3.848277e-06,-0.0006972725,0,0,0,0,0,0.008107373,0,0,0.00229567,0,0.0009929986,-0.0001696198,-0.0001696198,-0.0001280188,0.003616468,0,0,0,0.00163203,0.00837214,0.002345359,-0.0001573003,-0.0006037852]},{"f":[23,21,22,-1,15,17,1,-1,23,-1,-1,18,-1,-1,17,-1,17,-1,33,2,-1,-1,15,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,4.128693921346326,0,5.597993649014255,0,20.752541988539424,4.80643424864552,0,0,7.366322214245816,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,0,0,15,0,17,0,19,20,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,14,11,8,0,10,0,0,13,0,0,16,0,18,0,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,8.595028e-05,0,0,0,0.003745452,0,-0.003869236,-0.0003355775,0,-0.01150837,-0.001168371,0,0.0005564687,0,-0.003653729,0,0,0.009474035,-0.004341657,0,-0.002737734,-3.747142e-05,-0.00336997,0,0.004598386,-0.000430929]},{"f":[36,-1,25,37,2,2,4,23,-1,-1,18,-1,22,-1,-1,-1,4,-1,-1,21,-1,-1,17,-1,22,36,-1,-1,-1],"t":[1.5,0,2.5,11.05,9.266718101290426,9.051177541662383,7.675607660927803,0.5,0,0,1.292481250360578,0,12.35227706330556,0,0,0,9.328332883600808,0,0,5.5,0,0,1.292481250360578,0,12.613886328427084,2.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,0,0,20,0,0,23,0,25,26,0,0,0],"r":[2,0,22,19,16,15,10,9,0,0,12,0,14,0,0,0,18,0,0,21,0,0,24,0,28,27,0,0,0],"v":[0,-0.002416572,0,0,0,0,0,0,-0.0002280844,0.0001807928,0,0.006358582,0,-0.001073983,0.004070212,-0.001073983,0,0.006983041,-0.0005830859,0,-0.001073983,-0.005462382,0,0.008695283,0,0,0.002317816,-0.003307292,-0.003819489]},{"f":[22,-1,15,15,15,15,-1,3,-1,16,-1,-1,-1,18,-1,-1,32,-1,15,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[2.084962500721156,0,5.78135971352466,5.491853096329675,5.169925001442312,4.087462841250339,0,0.49242424242424243,0,6.409390936137702,0,0,0,5.491496787347155,0,0,3.753897320099348,0,6.409390936137702,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,0,0,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,0,14,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,16,13,12,7,0,9,0,11,0,0,0,15,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0.001684919,0,0,0,0,5.651923e-05,0,0.0002384111,0,-0.001550403,-0.004783213,0.001551055,0,-0.001706121,0.002144406,0,0.007864098,0,0.0008903465,0,0,-6.225777e-05,0,0.01057109,0.0002528688,0,-0.004895285,-0.001105996]},{"f":[22,5,3,-1,-1,21,-1,15,-1,-1,22,22,36,-1,0,15,-1,-1,17,-1,2,1,-1,-1,-1,17,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,7.5,0,10.690871009292314,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,2.0,0,0,1.792481250360578,0,1.292481250360578,7.0046728971962615,0,0,0,3.084962500721156,0,0,0],"m":[0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,8,0,0,11,12,13,0,15,16,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[10,5,4,0,0,7,0,9,0,0,28,25,14,0,18,17,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.0001468307,-0.005404109,0,0.0004421178,0,-6.288428e-05,-0.003692913,0,0,0,0.003230283,0,0,-0.001010346,0.001701726,0,-0.001010346,0,0,-0.01082138,-0.003816077,-0.001010346,0,-0.001927472,-0.006145998,-2.365662e-05]},{"f":[23,22,22,22,22,22,23,22,-1,1,-1,18,-1,-1,-1,-1,-1,15,-1,-1,22,-1,-1,33,-1,-1,15,-1,-1],"t":[8.5,17.094543568094064,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,14.03140051979143,0,1.596471647869674,0,6.434604125275413,0,0,0,0,0,6.0,0,0,17.078739781790915,0,0,21.80651201897376,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,12,0,0,0,0,0,18,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,20,17,16,15,14,9,0,11,0,13,0,0,0,0,0,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,4.225677e-05,0,0.001017944,0,-0.002686682,-0.000542882,0.001250202,-0.01894445,0.002401259,0,-0.003825596,-0.0006120831,0,0.003193012,-0.0001735555,0,-0.0002682975,-0.005530896,0,0.004385427,-0.0004145359]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,21,16,2,-1,-1,-1,17,-1,-1,33,22,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.36027577861459,9.5,1.584962500721156,6.451187557243013,0,0,0,1.292481250360578,0,0,29.807475161786478,11.424679641461715,0,0,2.5,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,14,15,16,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,22,19,18,17,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0.0008498236,0,0,0,-0.003274849,0,0.0002938391,0.004450577,0,-0.002053523,-0.006466291,0,0,0,0,0.0009426283,0.007958921,0.0003427967,0,-0.005450453,0.0007309898,0,0,-0.002213943,3.829187e-06,0,-0.005273507,-0.0008666482]},{"f":[23,-1,16,22,-1,22,22,4,23,-1,-1,16,-1,-1,-1,18,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[0.5,0,7.924812503605781,16.819976309430963,0,16.98920034885663,16.978341770814396,2.160964047443681,7.5,0,0,6.930737337562887,0,0,0,8.645081439392136,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,12,0,0,0,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,5,0,15,14,11,10,0,0,13,0,0,0,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.0001499175,0,0,0.000645102,0,0,0,0,0.002247252,-0.001744662,0,-0.001684299,-0.01348697,-0.004151574,0,0.001145494,-0.003734569,0,0,-0.00627844,0,0.004317572,-0.001910574,0,0.0002096801,0,-0.01752395,0.000782237]},{"f":[23,21,22,15,21,22,16,16,17,-1,-1,-1,17,16,-1,-1,-1,-1,-1,25,-1,-1,0,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.105450904090162,5.78135971352466,9.5,16.819976309430963,5.491853096329675,5.357552004618084,11.78143940991899,0,0,0,9.267956609922159,11.031356596255709,0,0,0,0,0,2.5,0,0,7.5,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,1,1,1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,14,0,0,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,22,19,18,17,12,11,10,0,0,0,16,15,0,0,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0001148219,-0.004662423,0.006367097,0,0,-0.0008971953,-0.00388907,0.0004099928,-0.003699911,-0.006177444,0,0.0001001513,0.004713483,0,5.767251e-05,-0.005293038,-0.003275385,0,0.00426665,-0.0003626714]},{"f":[36,-1,4,2,4,4,25,-1,17,-1,-1,22,-1,-1,17,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,2.5,0,1.292481250360578,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,13.054613167076562,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.002255962,0,0,0,0,0,-1.234583e-05,0,0.00824318,-0.0002052668,0,-0.007114486,0.0001055721,0,0.007744476,0,-0.003161132,0.00357478,0,-0.0003137307,0,-0.00627311,0.0004237805,0,0.006310347,0,-0.002065601,0.004293481]},{"f":[22,-1,23,17,2,21,22,-1,15,-1,-1,16,-1,22,1,-1,18,-1,1,-1,-1,-1,16,-1,16,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,7.6934196157931645,6.5,13.747003318603246,0,6.392317422778761,0,0,2.807354922057604,0,11.612768260520431,1.13125,0,3.084962500721156,0,2.9055555555555554,0,0,0,1.584962500721156,0,9.862637357558794,0,0,0,0],"m":[0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,14,15,0,17,0,19,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,27,22,11,8,0,10,0,0,13,0,21,16,0,18,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0.001561701,0,0,0,0,0,7.341584e-06,0,0.0003634437,-0.002377779,0,-0.008746844,0,0,-0.003438381,0,0.004262687,0,0.0007781277,-0.005742483,-0.007257761,0,0.00739253,0,-0.001099774,0.003676724,-0.004797887,0.000121491]},{"f":[23,21,22,-1,17,-1,15,4,22,33,-1,15,4,-1,-1,-1,-1,2,-1,-1,15,18,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,3.39067985676233,0,7.366322214245816,4.951187557243013,16.819976309430963,17.460839126440824,0,6.247927513443585,3.39067985676233,0,0,0,0,5.8698903048866296,0,0,10.457380879072534,9.135888799148791,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,0,0,1,1,0,0,1,1,0,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,12,13,0,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,4,0,6,0,20,17,16,11,0,15,14,0,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,8.263441e-05,0,0.0002976935,0,0,0,0,0.006657913,0,0,-0.000768946,0.003152687,-0.00745933,-0.004121281,0,-0.008406182,-2.940014e-05,0,0,0.0001978888,0.005689106,-0.001304733,-0.003107577,0,0.004050013,-0.000350304]},{"f":[22,22,-1,18,-1,-1,22,1,22,16,-1,-1,-1,22,-1,-1,32,-1,18,18,22,-1,0,-1,22,-1,-1,-1,-1],"t":[16.98920034885663,16.98413870862295,0,1.292481250360578,0,0,17.078739781790915,4.145833333333334,17.069460217922263,7.741466986401147,0,0,0,17.02942582914998,0,0,8.341935934311302,0,8.645081439392136,7.383290334558488,17.07907323966382,0,3.5,0,17.094543568094064,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,0,0,14,0,0,17,0,19,20,21,0,23,0,25,0,0,0,0],"r":[6,3,0,5,0,0,16,13,12,11,0,0,0,15,0,0,18,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-3.376369e-06,0,-0.006774533,-0.002406454,0,0,0,0,-0.00476065,0.0002941662,0.001778816,0,0.002066418,0.005164478,0,0.003064534,0,0,0,-0.003792044,0,0.004653216,0,0.0003845733,-0.004461048,0.001004062,-0.003895381]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,-1,33,22,-1,22,16,-1,15,-1,-1,-1,23,33,-1,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.406459975416917,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.462502272597265,0,10.681238411777805,0,0,0,5.5,41.975696368331896,0,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,0,18,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,24,17,0,23,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0.0007900656,0,0,0,-0.003060302,0,0.0002926971,0.004292125,0,-0.001865108,-0.005913039,0,0.0003571797,0,0,-0.002168099,0,0,0.001124263,0,-0.004251059,-0.0001168441,-9.849168e-05,0,0,-0.00395163,-0.0001293913,0.000315461]},{"f":[23,18,2,15,17,18,18,-1,2,22,-1,24,-1,-1,-1,19,-1,-1,33,-1,-1,1,-1,-1,-1,21,-1,-1,-1],"t":[0.5,9.845435504646158,9.266718101290426,10.113742166049189,6.924507036847409,5.108372929097653,1.792481250360578,0,1.292481250360578,7.5621616439616925,0,4.5,0,0,0,0.5,0,0,16.993854061608424,0,0,1.9735450749013448,0,0,0,3.5,0,0,0],"m":[0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,10,0,12,0,0,0,16,0,0,19,0,0,22,0,0,0,26,0,0,0],"r":[28,25,24,21,18,15,8,0,14,11,0,13,0,0,0,17,0,0,20,0,0,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001100094,0,0,-0.0004201162,0,-0.003618756,-0.009923715,0.00233278,0,0.0006796708,0.006542802,0,-0.002360472,0.00111584,0,-0.003755078,0.002661486,0.004320589,0,0.0007939761,-0.001772633,0.0001214759]},{"f":[36,-1,4,2,4,4,2,-1,-1,22,-1,-1,18,-1,22,-1,-1,15,18,1,-1,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,10.462502272597265,1.292481250360578,1.3897752394988947,0,0,0,0,11.683871801979876,0,13.054613167076562,0,0],"m":[0,0,1,1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,0,18,19,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,17,12,9,8,0,0,11,0,0,14,0,16,0,0,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.002158154,0,0,0,0,0,-2.957378e-05,0.0006850247,0,-0.006563889,8.34556e-05,0,0.006617335,0,-0.003116315,0.003158495,0,0,0,-0.001053707,-0.004691999,0.002632037,-0.005669632,0,0.005783454,0,-0.00194664,0.004050389]},{"f":[33,32,15,15,15,-1,-1,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,12,0,14,15,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,8,7,6,0,0,0,18,11,0,13,0,17,16,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,4.02069e-05,-0.002209834,0.001020459,0,0,-0.006030399,0,0.0006427194,0,0,0.0005280114,-0.003647899,-0.007735929,0,0.002482945,-0.0001947889,0,0.004316062,-0.0004726686,0,0.0003527961,0,0.0001197512,-0.003662279]},{"f":[15,15,-1,22,-1,16,22,1,-1,-1,16,1,-1,-1,-1,-1,32,-1,15,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[5.78135971352466,4.087462841250339,0,10.63843335887114,0,6.409390936137702,11.775312925643524,4.008411214953271,0,0,6.392317422778761,1.5277777777777777,0,0,0,0,3.753897320099348,0,6.409390936137702,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,8,0,0,11,12,0,0,0,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[16,3,0,5,0,15,10,9,0,0,14,13,0,0,0,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,7.750111e-05,0,-0.002673763,0,0,0,0.0005947175,0.005859088,0,0,0.001053874,-0.000919968,0.002386065,-0.002100026,0,0.007090718,0,0.0007969204,0,0,-4.87447e-05,0,0.009857951,0.0002760498,0,-0.004559945,-0.001018229]},{"f":[22,-1,21,15,17,1,-1,23,-1,-1,18,-1,-1,33,2,-1,22,2,-1,-1,-1,22,15,-1,15,-1,-1,-1,-1],"t":[14.03140051979143,0,9.5,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,17.460839126440824,1.792481250360578,0,16.725912049439096,5.065928480304397,0,0,0,14.158358850608169,6.539158811108031,0,7.266786540694901,0,0,0,0],"m":[1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,0,14,15,0,17,18,0,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,13,10,7,0,9,0,0,12,0,0,21,16,0,20,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,7.386097e-05,0,0,0,0,0.003542722,0,-0.003501637,-0.0002069972,0,-0.0107353,-0.001112915,0,0,0.008680086,0,0,-0.001579797,0.006045088,-0.002815618,0,0,-0.0005727615,0,-0.01017883,-0.00196966,-7.224743e-05,0.00235909]},{"f":[23,-1,16,22,-1,22,22,4,23,-1,-1,16,-1,-1,-1,18,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[0.5,0,7.924812503605781,16.819976309430963,0,16.98920034885663,16.978341770814396,2.160964047443681,7.5,0,0,6.930737337562887,0,0,0,8.645081439392136,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,12,0,0,0,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,5,0,15,14,11,10,0,0,13,0,0,0,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.0001369256,0,0,0.0006094291,0,0,0,0,0.002141028,-0.001678012,0,-0.001525937,-0.01284968,-0.003799672,0,0.001029466,-0.003370063,0,0,-0.005942722,0,0.004038239,-0.001767516,0,0.0001978382,0,-0.01663751,0.0006432123]},{"f":[23,21,22,-1,15,17,1,-1,23,-1,-1,18,-1,-1,33,2,-1,-1,22,15,-1,21,-1,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,14.03140051979143,0,5.169925001442312,5.740899715832876,1.4587811431362971,0,1.5,0,0,7.67595272073847,0,0,17.460839126440824,1.792481250360578,0,0,14.158358850608169,6.700439718141092,0,7.5,0,0,0,0,6.539158811108031,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,0,0,15,16,0,0,19,20,0,22,0,0,0,0,27,0,0],"r":[26,25,4,0,14,11,8,0,10,0,0,13,0,0,18,17,0,0,24,21,0,23,0,0,0,0,28,0,0],"v":[0,0,0,7.703898e-05,0,0,0,0.003367635,0,-0.003353729,-0.0002132143,0,-0.01022805,-0.001035555,0,0,0.00823629,1.849241e-05,0,0,-0.0005091394,0,-0.007789537,0.001081442,-1.012293e-05,-0.002990583,0,0.003821322,-0.0002921859]},{"f":[23,3,-1,32,22,16,15,16,-1,-1,-1,16,-1,22,15,1,1,-1,-1,-1,-1,-1,15,-1,-1,2,-1,-1,-1],"t":[0.5,0.9992038216560509,0,7.014643613484123,14.03140051979143,6.882643049361842,5.882643049361842,5.491853096329675,0,0,0,7.614709844115208,0,11.876328901062104,9.264442600226602,3.9935774037849088,1.13125,0,0,0,0,0,6.247927513443585,0,0,1.792481250360578,0,0,0],"m":[0,0,0,1,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,0,0,0,12,0,14,15,16,17,0,0,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,25,22,11,10,9,0,0,0,13,0,21,20,19,18,0,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.001425263,0,0,0,0,0,1.875726e-05,-0.001628508,0.0007070939,0,-0.002263027,0,0,0,0,-0.002670426,0.002657354,-0.001510439,-0.002030759,0.001330384,0,-0.001548538,-0.008793492,0,0.004513271,0.000302646,0.0001177519]},{"f":[36,-1,4,2,4,4,2,-1,21,-1,-1,22,-1,-1,17,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,5.5,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.00209783,0,0,0,0,0,-2.786771e-05,0,0.002735719,0.0002540653,0,-0.006259572,0.000124987,0,0.006847725,0,-0.002689067,0.003130108,0,-0.0002232753,0,-0.005860855,0.0002805306,0,0.005567352,0,-0.002592908,0.00267481]},{"f":[22,-1,23,18,2,-1,-1,17,16,-1,-1,18,-1,-1,16,22,-1,-1,16,36,-1,18,-1,-1,22,-1,22,-1,-1],"t":[2.084962500721156,0,0.5,10.543869317296533,9.266718101290426,0,0,10.682994316786338,9.271463027904375,0,0,11.651826559171445,0,0,7.924812503605781,16.819976309430963,0,0,8.618385502258606,4.5,0,1.292481250360578,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,0,1,1,0,0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,0,0,15,16,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,14,7,6,0,0,11,10,0,0,13,0,0,18,17,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,0.001446506,0,0,0,-0.0001203019,0.004447053,0,0,-0.001707415,-0.008834474,0,0.00164285,-0.002757866,0,0,0.0005789364,-0.0003464138,0,0,-0.005645803,0,0.003860772,-0.001688811,0,0.0001822606,0,-0.0158303,0.0005827458]},{"f":[22,-1,22,-1,15,22,21,16,16,-1,-1,-1,-1,-1,32,-1,15,-1,22,-1,37,1,-1,37,-1,-1,34,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,5.78135971352466,16.819976309430963,9.5,5.491853096329675,5.357552004618084,0,0,0,0,0,3.753897320099348,0,6.409390936137702,0,6.930301023985015,0,10.469999999999999,8.046875,0,8.2,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,0,0,0,0,0,15,0,17,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,4,0,14,13,12,11,10,0,0,0,0,0,16,0,18,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0.0007542944,0,-0.002886802,0,0,0,0,0,4.152573e-05,0.005982891,-0.0006653283,-0.004336417,-0.002994154,0,0.006652959,0,0.0007267351,0,-0.004439108,0,0,-1.939365e-05,0,0.00931669,0.0002897662,0,-0.0042452,-0.0009273663]},{"f":[1,1,-1,18,-1,-1,2,22,22,4,4,4,19,15,-1,-1,-1,20,-1,36,-1,-1,-1,-1,-1,22,-1,-1,-1],"t":[5.417948717948718,5.015625,0,4.245926548164837,0,0,9.068775776507199,16.98920034885663,16.98413870862295,6.499295214872664,5.89476682248518,2.453445297804259,0.5,6.6293566200796095,0,0,0,0.5,0,2.5,0,0,0,0,0,17.07907323966382,0,0,0],"m":[1,1,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,12,13,14,0,0,0,18,0,20,0,0,0,0,0,26,0,0,0],"r":[6,3,0,5,0,0,28,25,24,23,22,17,16,15,0,0,0,19,0,21,0,0,0,0,0,27,0,0,0],"v":[0,0,3.054826e-05,0,0.000956512,0.007231326,0,0,0,0,0,0,0,0,-0.004528667,-0.00010785,0.000724614,0,0.0002231075,0,-0.005093931,-0.001245992,0.002981015,-0.003742265,-0.004722568,0,0.003446213,-0.0007078999,-0.005587921]},{"f":[32,33,1,1,-1,2,17,2,17,19,-1,-1,-1,16,-1,-1,-1,-1,-1,1,17,22,-1,-1,-1,-1,4,-1,-1],"t":[10.463013322934223,29.75825500004914,12.64516129032258,5.417948717948718,0,9.068775776507199,2.160964047443681,1.292481250360578,1.792481250360578,0.5,0,0,0,6.523561956057013,0,0,0,0,0,9.107199999999999,7.6181566135214025,16.978341770814396,0,0,0,0,2.453445297804259,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,0,14,0,0,0,0,0,20,21,22,0,0,0,0,27,0,0],"r":[26,19,18,5,0,17,16,13,12,11,0,0,0,15,0,0,0,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,7.402612e-05,0,0,0,0,0,-0.003259259,0.0006726001,-0.006742363,0,0.001708553,-0.001261852,0.0002075453,-0.005314258,0.002664027,0,0,0,0.003615468,-0.000872668,-0.001348,-0.003562318,0,0.002818553,-0.001098805]},{"f":[23,21,22,22,22,22,22,23,-1,-1,-1,-1,15,-1,21,-1,4,-1,-1,22,-1,-1,0,-1,-1,-1,15,-1,-1],"t":[8.5,10.5,17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,0,0,0,0,6.0,0,5.5,0,1.792481250360578,0,0,17.078739781790915,0,0,7.5,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,0,15,0,17,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,22,19,12,11,10,9,0,0,0,0,14,0,16,0,18,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,-4.996044e-05,0.001230839,-0.01605699,0.002119695,0,-0.003204934,0,0.00122844,0,-0.0003060574,-0.002779332,0,0.002747903,-0.0003260818,0,7.280016e-05,-0.004831306,-0.002812094,0,0.003648395,-0.0002760084]},{"f":[22,5,3,-1,-1,19,-1,-1,22,22,36,-1,0,22,-1,-1,17,1,-1,-1,2,1,-1,-1,-1,4,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0.5,0,0,12.738408632106918,12.613886328427084,2.5,0,3.5,12.457649623491124,0,0,1.792481250360578,6.015625,0,0,1.292481250360578,7.0046728971962615,0,0,0,1.792481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,28,25,12,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-5.634817e-05,-0.004902321,0,0.0001012255,0.000845752,0,0,0,0.002915845,0,0,-0.0006431069,0.002321333,0,0,-0.00322891,-0.000904178,0,0,-0.009654057,-0.003144261,-0.0009863841,0,-0.001518393,-0.006126326,-1.48295e-05]},{"f":[22,-1,22,22,22,-1,21,-1,-1,16,-1,-1,22,-1,33,22,-1,22,16,15,-1,-1,15,-1,-1,-1,34,-1,-1],"t":[4.926654777701837,0,7.082728314749225,6.779867784305249,5.9525054311951475,0,4.5,0,0,5.78135971352466,0,0,11.406459975416917,0,29.807475161786478,11.424679641461715,0,12.199665042528586,10.462502272597265,10.342074667999139,0,0,10.681238411777805,0,0,0,2.5,0,0],"m":[0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,26,17,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0.0007103753,0,0,0,-0.002754747,0,0.0002539278,0.004153117,0,-0.001665104,-0.005173911,0,0.0003252665,0,0,-0.001954678,0,0,0,0.0006016322,0.003108028,0,-0.004028684,-1.427674e-06,-8.553815e-05,0,-0.004791301,-0.0007288458]},{"f":[23,3,-1,32,22,16,-1,16,-1,22,15,0,-1,-1,15,-1,17,2,-1,-1,17,-1,-1,-1,15,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0,7.014643613484123,14.03140051979143,6.882643049361842,0,7.614709844115208,0,11.876328901062104,9.264442600226602,4.5,0,0,10.376125389276174,0,10.482680779086738,1.292481250360578,0,0,10.69085411329249,0,0,0,6.247927513443585,0,0,0,0],"m":[0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,8,0,10,11,12,0,0,15,0,17,18,0,0,21,0,0,0,25,0,0,0,0],"r":[28,3,0,27,24,7,0,9,0,23,14,13,0,0,16,0,20,19,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0.001367944,0,0,0,2.505001e-05,0,-0.002135941,0,0,0,0.001347011,-0.002243548,0,-0.003537883,0,0,0.002157003,-0.001692774,0,-0.007562946,-0.0004471385,0.001250076,0,-0.001488116,-0.008369004,0.001610831,0.0001091043]},{"f":[36,-1,4,2,4,4,2,-1,21,-1,-1,22,-1,-1,18,-1,22,-1,-1,1,-1,1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,6.451187557243013,0,5.5,0,0,8.658207063149977,0,0,1.292481250360578,0,12.35227706330556,0,0,1.9735450749013448,0,4.757007376185458,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,24,19,14,11,8,0,10,0,0,13,0,0,16,0,18,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.001958907,0,0,0,0,0,-2.906243e-05,0,0.002575849,0.00026768,0,-0.005863769,0.000130918,0,0.005974723,0,-0.002689116,0.002799475,0,-0.0001705474,0,-0.005583306,0.0004162532,0,0.005227215,0,-0.002406732,0.002711458]},{"f":[4,-1,18,16,15,15,18,-1,-1,-1,36,-1,-1,4,-1,18,-1,-1,16,16,17,-1,-1,-1,21,-1,16,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.266786540694901,5.78135971352466,5.672425341971495,5.108372929097653,0,0,0,3.5,0,0,4.2849278041654735,0,5.065928480304397,0,0,9.862637357558794,4.247927513443585,10.166483743581448,0,0,0,5.5,0,11.620219825507487,0,0],"m":[1,0,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,14,0,16,0,0,19,20,21,0,0,0,25,0,27,0,0],"r":[2,0,18,13,10,9,8,0,0,0,12,0,0,15,0,17,0,0,24,23,22,0,0,0,26,0,28,0,0],"v":[0,6.40233e-05,0,0,0,0,0,-0.0002381315,0.002703239,-0.00247674,0,-0.001987354,0.002018326,0,-0.005125966,0,6.249178e-06,-0.002227198,0,0,0,0.004892178,-0.001049616,-0.001827069,0,0.003764637,0,-0.002423252,0.002004484]},{"f":[22,22,22,23,2,-1,22,22,22,-1,-1,-1,22,2,-1,-1,-1,-1,-1,-1,22,1,-1,-1,32,-1,18,-1,-1],"t":[16.98920034885663,16.978341770814396,16.975010474608837,7.5,1.292481250360578,0,16.819976309430963,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,2.453445297804259,0,0,0,0,0,0,17.078739781790915,4.008411214953271,0,0,8.341935934311302,0,8.645081439392136,0,0],"m":[1,1,1,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,14,0,0,0,0,0,0,21,22,0,0,25,0,27,0,0],"r":[20,19,18,17,6,0,12,11,10,0,0,0,16,15,0,0,0,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0.0002207142,0,0,0,-6.072964e-05,-0.01512676,0.002558282,0,0,-0.01208095,-0.002155289,0.001014508,-0.003961666,0.002136193,-0.00297184,0,0,7.771822e-05,0.002857635,0,0.002805432,0,-0.0003595626,-0.003550199]},{"f":[23,21,1,33,22,22,-1,17,-1,-1,15,-1,-1,-1,4,22,-1,17,15,-1,-1,-1,24,-1,16,-1,-1,-1,-1],"t":[8.5,10.5,5.417948717948718,27.680602362796474,16.759160379664443,12.027747761539269,0,10.482680779086738,0,0,9.002815015607053,0,0,0,6.499295214872664,17.094543568094064,0,8.729619742794132,6.584962500721156,0,0,0,13.5,0,7.774787059601174,0,0,0,0],"m":[1,1,1,0,1,1,0,1,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,0,0,0,15,16,0,18,19,0,0,0,23,0,25,0,0,0,0],"r":[28,27,14,13,10,7,0,9,0,0,12,0,0,0,22,17,0,21,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0.0004262974,0,0.004656002,0.00107158,0,0.0008269194,-0.006394442,1.79123e-05,0,0,-4.327794e-05,0,0,-0.01080287,-0.001415317,0.002344932,0,-0.008710743,0,0.002373557,-0.003212886,-0.002655626,0.001703714]},{"f":[23,3,4,-1,-1,32,22,16,18,-1,-1,16,-1,22,15,0,-1,-1,-1,-1,33,0,-1,33,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,3.753897320099348,0,0,7.014643613484123,14.03140051979143,6.882643049361842,3.6427011094311244,0,0,7.614709844115208,0,11.876328901062104,9.264442600226602,4.5,0,0,0,0,21.257266042509173,1.5,0,15.968216440900115,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,12,0,14,15,16,0,0,0,0,21,22,0,24,0,0,0,0,0],"r":[28,5,4,0,0,27,20,11,10,0,0,13,0,19,18,17,0,0,0,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0.0003820315,0.003918013,0,0,0,0,-0.000172795,0.0006588985,0,-0.00202995,0,0,0,0.001287546,-0.002131702,-0.001788794,0.001159494,0,0,0.001666224,0,-0.01018026,-0.002799454,0.001588703,0.001529847,0.0001074958]},{"f":[33,32,15,15,-1,-1,22,1,-1,18,-1,22,2,-1,-1,-1,21,-1,1,19,-1,-1,-1,36,-1,-1,3,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,0,0,11.722778233171695,1.3336158192090395,0,2.903677461028802,0,11.67903991803059,1.792481250360578,0,0,0,3.5,0,8.135583053967,1.5,0,0,0,3.5,0,0,0.5187999999999999,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,10,0,12,13,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,16,9,0,11,0,15,14,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-8.62322e-06,0.0008990051,0,0,-0.005460063,0,0.0007143942,0,0,0.0007202362,-0.003114002,-0.007094419,0,0.002120899,0,0,-0.0002511627,0.002048729,-0.004018284,0,0.004112792,-8.696082e-05,0,-0.002552372,-3.201305e-05]},{"f":[22,22,-1,17,-1,-1,22,1,22,16,-1,-1,18,-1,-1,-1,32,-1,18,18,22,-1,0,-1,22,-1,-1,-1,-1],"t":[16.98920034885663,16.98413870862295,0,1.292481250360578,0,0,17.078739781790915,4.008411214953271,17.069460217922263,7.741466986401147,0,0,6.076776015854056,0,0,0,8.341935934311302,0,8.645081439392136,7.383290334558488,17.07907323966382,0,3.5,0,17.094543568094064,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,0,13,0,0,0,17,0,19,20,21,0,23,0,25,0,0,0,0],"r":[6,3,0,5,0,0,16,15,12,11,0,0,14,0,0,0,18,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-2.512974e-06,0,-0.00591185,-0.001773398,0,0,0,0,-0.004517477,-2.608488e-05,0,0.003900517,0.0001681185,0.002721446,0,0.002734387,0,0,0,-0.003533662,0,0.004406116,0,0.0004792144,-0.004006911,0.0009765292,-0.003252435]},{"f":[22,-1,23,18,17,32,22,-1,33,0,-1,33,-1,-1,-1,18,-1,-1,-1,21,17,-1,-1,15,-1,2,-1,-1,-1],"t":[2.084962500721156,0,0.5,9.314012697712199,9.27958770372691,7.014643613484123,14.03140051979143,0,21.257266042509173,1.5,0,15.968216440900115,0,0,0,1.292481250360578,0,0,0,4.5,10.009557117932596,0,0,9.264442600226602,0,1.292481250360578,0,0,0],"m":[0,0,0,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,0,12,0,0,0,16,0,0,0,20,21,0,0,24,0,26,0,0,0],"r":[2,0,28,19,18,15,8,0,14,11,0,13,0,0,0,17,0,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0.001297662,0,0,0,0,0,-7.061512e-05,0,0,0.001585537,0,-0.009681878,-0.002660953,0.00151122,0,-0.0004690263,0.004820542,0.003425479,0,0,-0.002553522,0.0007660554,0,-0.0007135855,0,-0.008094388,-0.001788625,0.0001037762]},{"f":[33,32,15,-1,22,1,-1,16,18,-1,-1,-1,23,-1,36,-1,15,23,-1,-1,-1,24,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,11.193525360501216,10.463013322934223,0,0,0,0.5,0,3.5,0,10.066089190457772,1.5,0,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,8,9,0,0,0,13,0,15,0,17,18,0,0,0,22,0,0,25,0,27,0,0],"r":[24,21,4,0,12,7,0,11,10,0,0,0,14,0,16,0,20,19,0,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,5.259997e-05,0,0,-0.005192361,0,0,-0.001151076,-0.006546692,0.001132571,0,0.00213737,0,-0.005321718,0,0,7.600255e-05,-0.00658604,5.294126e-05,0,0.0035702,-0.000429869,0,0.0003584224,0,0.0004522913,-0.003477152]},{"f":[15,15,-1,22,-1,16,22,1,-1,-1,16,-1,15,-1,22,22,22,-1,-1,-1,-1,-1,32,-1,1,21,-1,-1,-1],"t":[5.78135971352466,4.087462841250339,0,10.63843335887114,0,6.409390936137702,11.775312925643524,4.008411214953271,0,0,5.169925001442312,0,5.357552004618084,0,13.747003318603246,13.689894219688643,13.129761635363703,0,0,0,0,0,3.753897320099348,0,12.069111969111969,8.5,0,0,0],"m":[0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,7,8,0,0,11,0,13,0,15,16,17,0,0,0,0,0,23,0,25,26,0,0,0],"r":[22,3,0,5,0,21,10,9,0,0,12,0,14,0,20,19,18,0,0,0,0,0,24,0,28,27,0,0,0],"v":[0,0,6.450417e-05,0,-0.002406574,0,0,0,0.0005449012,0.005206974,0,-0.001272448,0,0.001170977,0,0,0,-0.003463913,0.001278928,-0.003091262,0.001125258,-0.001924615,0,0.006520018,0,0,3.295861e-05,0.0012329,-0.002360277]},{"f":[36,-1,4,2,4,4,2,-1,-1,22,-1,-1,17,-1,18,-1,-1,15,17,23,-1,-1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,9.266718101290426,9.051177541662383,7.950787011501063,7.408266855279292,7.005613627711627,0,0,8.658207063149977,0,0,1.292481250360578,0,8.14580120349141,0,0,10.794415866350105,9.44286754736589,1.5,0,0,0,0,11.683871801979876,0,9.862637357558794,0,0],"m":[0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,0,18,19,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,17,12,9,8,0,0,11,0,0,14,0,16,0,0,23,22,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.001856066,0,0,0,0,0,-6.606193e-06,0.001380305,0,-0.005576411,0.0001794978,0,0.006216567,0,-0.002902851,0.002331707,0,0,0,-0.0004296602,-0.005558775,0.001476597,-0.005253501,0,0.004999836,0,-0.00221527,0.002628043]},{"f":[1,2,-1,-1,3,22,22,3,-1,2,-1,-1,-1,-1,25,37,-1,25,-1,-1,17,-1,1,-1,22,36,-1,-1,-1],"t":[0.39811320754716983,6.08730682361797,0,0,0.05277777777777778,16.759160379664443,16.73920708776685,0.007093253968253968,0,8.612669628062957,0,0,0,0,2.5,9.629999999999999,0,1.5,0,0,1.292481250360578,0,5.417948717948718,0,12.613886328427084,2.5,0,0,0],"m":[0,1,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,0,0,15,16,0,18,0,0,21,0,23,0,25,26,0,0,0],"r":[4,3,0,0,14,13,12,9,0,11,0,0,0,0,20,17,0,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,6.107634e-05,0.003117912,0,0,0,0,0.00197917,0,-0.001145052,-0.004067211,0.003267748,-0.003782064,0,0,8.086662e-06,0,-0.0007867851,-0.00540863,0,0.00624983,0,0.002134897,0,0,0.002428852,-0.00260524,-0.00520821]},{"f":[4,-1,18,16,15,15,22,-1,-1,1,-1,-1,32,-1,-1,4,-1,18,-1,15,-1,-1,16,-1,21,-1,16,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.266786540694901,5.78135971352466,3.584962500721156,13.747003318603246,0,0,3.822916666666667,0,0,6.208926257442949,0,0,4.2849278041654735,0,5.065928480304397,0,10.457380879072534,0,0,9.862637357558794,0,5.5,0,11.620219825507487,0,0],"m":[1,0,1,0,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,0,16,0,18,0,20,0,0,23,0,25,0,27,0,0],"r":[2,0,22,15,12,9,8,0,0,11,0,0,14,0,0,17,0,19,0,21,0,0,24,0,26,0,28,0,0],"v":[0,5.946979e-05,0,0,0,0,0,5.320867e-05,0.004290571,0,-0.002526286,7.059604e-05,0,0.001968663,-0.001686263,0,-0.004828448,0,-2.154447e-06,0,-0.002842297,4.353141e-05,0,-0.0003605585,0,0.003417106,0,-0.002070403,0.001919204]},{"f":[23,-1,21,17,-1,22,36,20,-1,-1,17,-1,33,-1,3,-1,-1,37,15,22,-1,-1,-1,-1,37,18,-1,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,3.5,0.5,0,0,5.918419679874358,0,27.680602362796474,0,0.5187999999999999,0,0,7.92,6.0,16.819976309430963,0,0,0,0,11.73,6.562222230004293,0,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,0,15,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,17,10,9,0,0,12,0,14,0,16,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.0001155589,0,0,0.0009379896,0,0,0,-0.0007366881,-0.004166354,0,-0.003333163,0,0.005354046,0,-0.0005402187,0.001323115,0,0,0,0.002023685,-0.003614597,0.00394732,-0.008514864,0,0,-0.0003350666,0.0004423764,-0.004667042]},{"f":[22,22,22,23,2,-1,22,22,22,23,-1,-1,-1,-1,22,2,-1,-1,-1,-1,-1,-1,22,1,-1,-1,32,-1,-1],"t":[16.98920034885663,16.978341770814396,16.975010474608837,7.5,1.292481250360578,0,16.819976309430963,16.786010507999382,16.765402642760364,4.5,0,0,0,0,16.903290731167747,2.453445297804259,0,0,0,0,0,0,17.078739781790915,4.145833333333334,0,0,8.341935934311302,0,0],"m":[1,1,1,1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,10,0,0,0,0,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[22,21,20,19,6,0,14,13,12,11,0,0,0,0,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0.000202116,0,0,0,0,-0.0001536971,0.001148689,-0.01416969,0.002395941,0,0,-0.01149336,-0.00200838,0.0009905481,-0.003800441,0.00204389,-0.002698425,0,0,0.0002791404,0.002819429,0,0.002588877,-0.000691701]},{"f":[2,2,21,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,1,18,2,4,-1,-1,-1,22,16,-1,16,-1,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,0,11.98370619265935,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,8.992857142857144,9.229707804263573,9.025136904799169,7.997154617983022,0,0,0,12.921158001713904,10.372865060112588,0,12.683433292835403,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,0,0,11,12,0,0,15,16,17,18,0,0,0,22,23,0,25,0,0,0,0],"r":[10,7,4,0,6,0,0,9,0,0,14,13,0,0,28,21,20,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,0,0,-6.15553e-05,0,0.001543059,-0.003638125,0,-0.0001740254,-0.008541512,0,0,0.0005738365,0.007923623,0,0,0,0,5.390971e-05,0.002387452,-0.001055135,0,0,-0.001194376,0,0.004187256,0.000543837,0.005935314,-0.003006729]},{"f":[22,-1,22,-1,23,18,2,33,33,2,-1,-1,-1,22,-1,15,33,-1,-1,-1,-1,17,17,-1,-1,17,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,17.997194725799872,9.573602462471115,1.292481250360578,0,0,0,14.03140051979143,0,6.247927513443585,21.257266042509173,0,0,0,0,10.682994316786338,10.609627990977145,0,0,11.651826559171445,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,0,0,0,14,0,16,17,0,0,0,0,22,23,0,0,26,0,0,0],"r":[2,0,4,0,28,21,20,13,12,11,0,0,0,15,0,19,18,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0.0006658727,0,-0.002580365,0,0,0,0,0,0,-0.002358538,0.002123783,0.00588597,0,-9.564289e-05,0,0,-0.002266898,0.001443234,-0.007393831,0.003968681,0,0,-0.001516227,-0.00686993,0,0.001255861,-0.002241447,9.827273e-05]},{"f":[1,1,22,-1,-1,18,-1,-1,2,15,2,-1,-1,25,4,-1,18,4,-1,-1,17,-1,-1,16,22,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,0,8.03615005796417,6.499295214872664,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,13,12,0,0,23,16,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,1.693447e-05,0.004057391,0,0.0008255357,0.006630055,0,0,0,-0.003554225,0.002638148,0,0,0.0001340823,0,0,-0.0007256883,-0.004629477,0,0.006554687,-0.0006097988,0,0,0.002679938,-0.004815045,0.006748637,-0.004610451]},{"f":[2,2,-1,15,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,22,-1,-1,22,16,-1,16,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,8.9274737487839,7.997154617983022,2.0,7.95646805166752,0,0,0,0,16.759160379664443,0,0,12.921158001713904,10.372865060112588,0,12.683433292835403,0,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,20,0,0,23,24,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,22,19,18,17,16,0,0,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,-2.317966e-05,0,-0.0001711093,-0.008124201,0,0,0.0004799543,0.007458138,0,0,0,0,0,-0.0003741903,-0.005434918,0.0002268172,0.003927336,0,-0.0006717228,-0.005137457,0,0,-0.001069266,0,0.00401817,0.0005110619,0.004920637]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,3,3,-1,-1,32,24,17,17,-1,21,-1,-1,-1,2,33,-1,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.32902480033627574,0.1873015873015873,0,0,10.463013322934223,9.5,12.18494601563336,11.26283925556662,0,7.5,0,0,0,1.292481250360578,32.267154600431816,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,1,0,0,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,0,15,16,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,14,13,0,0,28,23,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.002619128,-0.0009450154,0,0,0,-0.001098516,0.003155767,-0.004358264,0,0,0.0002742404,0.002502351,0,0,0,0,-1.475879e-05,0,0.003020009,-0.001226802,-0.001165221,0,0,0.002790453,-0.002792572,-0.007617853,0.001357107]},{"f":[1,2,-1,-1,3,22,22,3,-1,2,-1,-1,-1,-1,4,2,4,-1,-1,21,-1,-1,16,15,17,-1,-1,-1,-1],"t":[0.39811320754716983,6.08730682361797,0,0,0.05277777777777778,16.759160379664443,16.73920708776685,0.007093253968253968,0,8.612669628062957,0,0,0,0,9.266718101290426,9.034782574103549,7.675607660927803,0,0,8.5,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,0,0,15,16,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[4,3,0,0,14,13,12,9,0,11,0,0,0,0,22,19,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,7.285914e-05,0.002860556,0,0,0,0,0.001845954,0,-0.001032008,-0.003526576,0.003092576,-0.00321563,0,0,0,-1.183859e-05,0.001273564,0,-0.003819814,-0.0003171247,0,0,0,0.003575633,-0.0004490782,-0.002975847,0.005230289]},{"f":[15,15,15,-1,22,-1,-1,4,23,-1,-1,-1,15,33,-1,18,-1,-1,21,-1,18,-1,1,17,-1,23,-1,-1,-1],"t":[9.756556322524087,8.005624549193879,7.924812503605781,0,15.676331898425197,0,0,3.6427011094311244,0.5,0,0,0,10.066089190457772,29.807475161786478,0,8.03615005796417,0,0,3.5,0,10.620215166896223,0,3.9993913572732804,10.69085411329249,0,1.5,0,0,0],"m":[1,1,1,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,14,0,16,0,0,19,0,21,0,23,24,0,26,0,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,18,15,0,17,0,0,20,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,3.791362e-05,0,-0.00110294,-0.005971119,0,0,0.0004893282,0.002747139,-0.0008965329,0,0,0.0009774974,0,-0.004262534,-0.0005981094,0,0.002113823,0,-1.815557e-05,0,0,-0.004105838,0,-0.001195748,0.003823355,-0.007146985]},{"f":[22,-1,22,16,-1,-1,22,21,4,4,4,18,19,-1,-1,2,-1,2,-1,-1,-1,-1,22,-1,-1,17,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,8.00281227459694,0.5,0,0,1.792481250360578,0,3.245926548164837,0,0,0,0,10.029977181607594,0,0,1.292481250360578,0,0,0],"m":[0,0,0,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,13,0,0,16,0,18,0,0,0,0,23,0,0,26,0,0,0],"r":[2,0,6,5,0,0,28,25,22,21,20,15,14,0,0,17,0,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0.0004961836,0,0,-0.0009555952,-0.004593545,0,0,0,0,0,0,0,0.000276634,0.003150145,0,0.001061889,0,-0.004150169,0.0002003943,0.003971824,-0.005698365,0,0.006090632,0.0002009888,0,-0.004197153,0.0004718811,-5.600173e-05]},{"f":[1,33,22,22,-1,-1,15,-1,-1,21,2,-1,16,-1,-1,15,17,1,-1,-1,-1,16,-1,22,-1,-1,2,-1,-1],"t":[5.417948717948718,27.680602362796474,16.759160379664443,12.027747761539269,0,0,9.002815015607053,0,0,7.5,9.221514256038809,0,10.681238411777805,0,0,10.043027283594547,5.83200138144325,2.9733542319749215,0,0,0,10.372865060112588,0,15.10428213920758,0,0,9.068775776507199,0,0],"m":[1,0,1,1,0,0,1,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,16,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,15,12,0,14,0,0,21,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.0003448861,0.003090753,0,0.0006100274,-0.005808981,0,0,7.320514e-05,0,-0.0003145785,0.006275479,0,0,0,0.000142257,-0.004375167,0.0004914042,0,-0.007765623,0,-0.002247783,0.003056793,0,-0.0001333758,-0.004270565]},{"f":[23,22,22,23,-1,22,-1,22,-1,22,-1,-1,22,18,-1,-1,15,17,-1,-1,4,2,-1,-1,-1,24,-1,-1,-1],"t":[8.5,17.105450904090162,17.069460217922263,7.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,0,17.078739781790915,7.588709768994619,0,0,6.584962500721156,8.729619742794132,0,0,4.0437314206251695,2.69615871138938,0,0,0,8.5,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,10,0,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[28,25,12,5,0,7,0,9,0,11,0,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,1.423734e-06,0,-0.004697508,0,0.002432356,0,-0.005008482,0.001118318,0,0,0.003350518,0.0003939379,0,0,-0.006490984,0.00339545,0,0,0.0002819396,0.003980147,-0.001745674,0,-0.004191935,8.871429e-05,0.001627721]},{"f":[23,4,-1,2,22,-1,-1,2,-1,1,22,15,21,2,-1,-1,-1,21,-1,-1,-1,1,-1,21,-1,-1,15,-1,-1],"t":[8.5,4.0437314206251695,0,4.128693921346326,11.876328901062104,0,0,4.80643424864552,0,6.241071428571429,10.27595502635321,2.807354922057604,8.5,7.894792293346235,0,0,0,7.5,0,0,0,6.970655725248841,0,8.5,0,0,6.539158811108031,0,0],"m":[1,1,0,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,0,4,5,0,0,8,0,10,11,12,13,14,0,0,0,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,3,0,7,6,0,0,9,0,21,20,17,16,15,0,0,0,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,5.31212e-05,0,0,0.001582812,-0.002772966,0,0.001817281,0,0,0,0,0,-0.0001153289,0.004591304,-0.002788111,0,-0.008150073,-0.0008501973,0.0002206146,0,-0.004858338,0,-0.0004280567,-0.005071434,0,0.003151179,-0.0001727772]},{"f":[23,18,15,15,2,17,18,18,-1,-1,19,-1,-1,33,17,-1,1,-1,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1],"t":[0.5,9.314012697712199,10.113742166049189,9.756556322524087,9.266718101290426,6.924507036847409,3.6427011094311244,2.903677461028802,0,0,0.5,0,0,16.993854061608424,7.252657678068118,0,2.484375,0,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0],"m":[0,1,1,1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,0,0,14,15,0,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,22,21,20,13,10,9,0,0,12,0,0,19,16,0,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-8.493683e-05,-0.007255619,0,0.0005923105,0.005697045,0,0,-0.008417187,0,0.0005833898,-0.002071509,0.000635696,0.003615761,-0.002742794,0,0.004417772,0,-0.003022916,0.002052523,-0.0008419496,9.90589e-05]},{"f":[22,-1,22,16,-1,-1,22,-1,33,25,-1,24,-1,33,4,-1,-1,18,16,-1,-1,-1,23,33,1,-1,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,0,29.807475161786478,0.5,0,6.5,0,20.752541988539424,3.9534452978042594,0,0,1.792481250360578,10.46352437327118,0,0,0,5.5,41.975696368331896,9.107199999999999,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,0,12,0,14,15,0,0,18,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,22,11,0,13,0,17,16,0,0,21,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0.0004872621,0,0,-0.0009074962,-0.004248215,0,0.0002642563,0,0,-7.433526e-05,0,-0.0001957651,0,0,0.009572544,0.001937052,0,0,0.005958728,0.0003511154,-0.001649135,0,0,0,-0.002054344,-0.006182216,-4.113954e-05,0.0003373343]},{"f":[33,32,15,-1,22,1,-1,16,19,23,-1,-1,-1,17,-1,-1,23,-1,36,-1,-1,36,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,0,11.722778233171695,1.3336158192090395,0,10.794415866350105,0.5,0.5,0,0,0,10.457893746154532,0,0,0.5,0,3.5,0,0,3.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,0,0,14,0,0,17,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,4,0,16,7,0,13,12,11,0,0,0,15,0,0,18,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,5.216949e-05,0,0,-0.004686107,0,0,0,-0.004469003,-0.0004960295,0.0001897807,0,0.0023035,-0.001424896,0,0.001804464,0,-0.00454338,-9.952322e-05,0,0.003576452,-0.0001422549,0,0.0003916925,0,0.0003960844,-0.003214692]},{"f":[23,-1,21,17,-1,22,3,3,-1,-1,17,-1,19,-1,-1,37,15,22,-1,-1,-1,-1,37,18,-1,-1,23,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,0.8293269230769231,0.49242424242424243,0,0,5.918419679874358,0,0.5,0,0,7.92,6.0,16.819976309430963,0,0,0,0,10.780000000000001,6.531360383582522,0,0,2.5,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,0,0,16,17,18,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,5,0,15,10,9,0,0,12,0,14,0,0,21,20,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.000105438,0,0,0.0008648204,0,0,0,-0.000337902,-0.003266267,0,-0.003001724,0,0.0006302115,0.002825696,0,0,0,0.001936129,-0.003330319,0.003583202,-0.007767065,0,0,-0.0002894373,0.0003875717,0,-0.006347622,-0.001718469]},{"f":[22,-1,22,-1,18,18,4,2,2,4,23,-1,-1,-1,1,1,-1,-1,-1,-1,16,-1,15,-1,-1,21,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,12.18494601563336,11.26283925556662,9.328332883600808,9.293467526905953,9.051177541662383,8.049190845056867,0.5,0,0,0,3.172945205479452,1.99834912080135,0,0,0,0,4.247927513443585,0,10.486835021563051,0,0,7.5,0,0,0],"m":[0,0,0,0,1,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,11,0,0,0,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,4,0,28,25,20,19,14,13,12,0,0,0,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0.0006235917,0,-0.002470516,0,0,0,0,0,0,0,-0.000159454,0.000135301,0.002150293,0,0,-0.0006825407,-0.005605432,0.002346222,0.004313871,0,0.0003335978,0,-0.005070971,-0.0003859173,0,0.002565679,-0.001727665,-0.0009754503]},{"f":[23,1,-1,4,22,22,22,22,22,-1,2,2,-1,-1,-1,-1,-1,-1,17,15,-1,18,-1,-1,-1,22,-1,-1,-1],"t":[8.5,5.417948717948718,0,6.499295214872664,17.094543568094064,16.98920034885663,16.98413870862295,16.975010474608837,15.717370311433935,0,3.084962500721156,1.292481250360578,0,0,0,0,0,0,8.729619742794132,6.584962500721156,0,6.400854179458231,0,0,0,13.005894632618489,0,0,0],"m":[1,1,0,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,8,9,0,11,12,0,0,0,0,0,0,19,20,0,22,0,0,0,26,0,0,0],"r":[28,3,0,25,18,17,16,15,10,0,14,13,0,0,0,0,0,0,24,21,0,23,0,0,0,27,0,0,0],"v":[0,0,5.72748e-05,0,0,0,0,0,0,3.794952e-05,0,0,-0.002782658,0.001115292,-0.009220088,0.002550025,-0.004593088,0.001600853,0,0,-0.01000241,0,-0.00425015,0.002704006,0.002087418,0,-0.005063325,0.001906893,0.001478506]},{"f":[2,2,21,22,26,35,23,-1,19,-1,21,-1,-1,-1,-1,22,-1,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,2.5,0.5,0,1.5,0,5.5,0,0,0,0,15.756821747580634,0,0,11.98370619265935,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,0],"m":[1,1,1,1,1,1,1,0,1,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,0,0,0,0,16,0,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,21,18,15,14,13,8,0,10,0,12,0,0,0,0,17,0,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001610973,0,0.0002075735,0,0.003281557,-0.0004554032,-0.00169941,0.002622239,0,-0.008395882,-0.0003217358,0,0.001461231,-0.003381918,0,-0.0001947923,-0.007644105,0,0,0.0004304302,0.006831269,0.0001087806]},{"f":[22,-1,22,16,-1,-1,22,21,4,4,4,18,19,-1,-1,2,-1,2,-1,-1,-1,-1,22,-1,17,-1,-1,-1,-1],"t":[6.671801068872866,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,8.00281227459694,0.5,0,0,1.792481250360578,0,3.245926548164837,0,0,0,0,10.029977181607594,0,1.292481250360578,0,0,0,0],"m":[0,0,0,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,13,0,0,16,0,18,0,0,0,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,27,22,21,20,15,14,0,0,17,0,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0.0004652803,0,0,-0.0008552314,-0.003902105,0,0,0,0,0,0,0,0.0002362414,0.00290352,0,0.0009914479,0,-0.003872716,0.0001348835,0.003622151,-0.005432537,0,0.005671494,0,-0.001377714,0.002016696,-0.001876631,-4.964883e-05]},{"f":[15,-1,15,21,-1,-1,21,-1,17,22,35,-1,22,-1,-1,37,-1,-1,17,-1,15,22,22,33,-1,-1,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,6.5,0,0,3.5,0,2.160964047443681,16.725912049439096,2.5,0,15.439474430023438,0,0,6.890000000000001,0,0,3.245926548164837,0,13.0003521774803,16.975010474608837,16.765402642760364,29.297756302860655,0,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,8,0,18,15,12,0,14,0,0,17,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,5.180619e-05,0,0,-0.0008886015,-0.005584603,0,0.00190195,0,0,0,0.006423794,0,6.925315e-05,0.004174005,0,0.0004952473,-0.002798607,0,-0.004778292,0,0,0,0,0.002983937,-0.0005585038,-0.004761788,0.0009628373,-0.004362057]},{"f":[22,-1,18,18,4,2,2,4,23,17,-1,-1,-1,-1,1,1,-1,-1,-1,-1,16,-1,15,-1,-1,15,-1,-1,-1],"t":[2.084962500721156,0,12.18494601563336,11.26283925556662,9.328332883600808,9.293467526905953,9.051177541662383,8.049190845056867,0.5,6.924507036847409,0,0,0,0,3.172945205479452,1.99834912080135,0,0,0,0,4.247927513443585,0,10.486835021563051,0,0,11.683433292835403,0,0,0],"m":[0,0,1,1,1,1,1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,9,10,0,0,0,0,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,28,25,20,19,14,13,12,11,0,0,0,0,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0.001122688,0,0,0,0,0,0,0,0,-2.584994e-05,-0.0008151712,0.0001274444,0.001984379,0,0,-0.0006379274,-0.005364049,0.002340735,0.004064017,0,0.0003016229,0,-0.004812568,-0.0003651372,0,0.002600089,-0.001313079,-0.000926264]},{"f":[1,1,22,-1,-1,18,-1,-1,2,22,22,22,4,4,4,-1,-1,-1,-1,-1,-1,17,15,-1,18,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,17.094543568094064,16.98920034885663,16.98413870862295,6.499295214872664,5.89476682248518,2.453445297804259,0,0,0,0,0,0,8.729619742794132,6.584962500721156,0,6.400854179458231,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,14,15,0,0,0,0,0,0,22,23,0,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,21,20,19,18,17,16,0,0,0,0,0,0,27,24,0,26,0,0,0,0],"v":[0,0,0,1.448704e-05,0.00398657,0,0.0007049818,0.006090449,0,0,0,0,0,0,0,0.0001812338,-0.0007516088,0.002446126,-0.003012204,-0.003620395,0.001823743,0,0,-0.008807832,0,-0.004028206,0.003181795,0.001867725,-0.003978094]},{"f":[2,2,21,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,25,-1,-1,18,-1,15,21,-1,17,-1,-1,22,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,0,9.862637357558794,6.5,0,10.620215166896223,0,0,11.876328901062104,0,0],"m":[1,1,1,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,7,0,0,0,0,12,0,0,15,16,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[14,11,4,0,10,9,8,0,0,0,0,13,0,0,18,17,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,-5.516303e-05,0,0,0,0.001410478,-0.004718034,0.002934674,-0.003192754,0,-0.0001922784,-0.007275283,0,0,0.000431974,0.006348174,0,-7.018888e-05,0,0,-0.001309989,0,0.004858994,0.0002115158,0,0.0008724626,0.005013563]},{"f":[15,-1,15,17,-1,-1,21,-1,3,-1,18,18,17,22,35,-1,-1,-1,17,-1,-1,-1,1,17,-1,23,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,1.292481250360578,0,0,3.5,0,0.034940494354592615,0,10.620215166896223,10.45532575654158,2.160964047443681,16.725912049439096,2.5,0,0,0,3.245926548164837,0,0,0,3.9993913572732804,10.69085411329249,0,1.5,0,0,0],"m":[1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,0,0,7,0,9,0,11,12,13,14,15,0,0,0,19,0,0,0,23,24,0,26,0,0,0],"r":[2,0,6,5,0,0,8,0,10,0,22,21,18,17,16,0,0,0,20,0,0,0,28,25,0,27,0,0,0],"v":[0,5.046411e-05,0,0,-0.004032092,-0.0005160069,0,0.001786177,0,-0.003563951,0,0,0,0,0,0.006104158,0.0006066575,-0.0006295331,0,-0.00454419,-0.0003028042,0.00162581,0,0,-0.003830243,0,-0.001033698,0.003171959,-0.006569098]},{"f":[22,5,3,-1,-1,-1,22,23,2,2,17,1,-1,-1,37,-1,-1,22,1,-1,-1,-1,-1,4,-1,-1,4,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.738408632106918,1.5,8.049190845056867,1.292481250360578,1.792481250360578,6.015625,0,0,5.84,0,0,12.613886328427084,3.822916666666667,0,0,0,0,1.792481250360578,0,0,7.997154617983022,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,12,0,0,15,0,0,18,19,0,0,0,0,24,0,0,27,0,0],"r":[6,5,4,0,0,0,26,23,22,17,14,13,0,0,16,0,0,21,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-1.383821e-05,-0.004091174,0.0001882931,0,0,0,0,0,0,-0.001499788,0.005042337,0,-0.0001476961,-0.006221774,0,0,0.0001668783,0.00292483,-0.003527749,-0.003157403,0,-0.001267139,-0.006774015,0,-2.489274e-05,0.001304103]},{"f":[15,22,-1,-1,32,-1,15,22,3,1,-1,-1,-1,22,-1,-1,15,-1,37,1,-1,37,-1,35,-1,-1,34,-1,-1],"t":[5.78135971352466,16.819976309430963,0,0,3.753897320099348,0,6.409390936137702,16.96990787537721,0.5898428571428571,2.354717912147707,0,0,0,17.069460217922263,0,0,6.523561956057013,0,10.469999999999999,8.046875,0,8.2,0,2.5,0,0,3.5,0,0],"m":[0,1,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,0,0,14,0,0,17,0,19,20,0,22,0,24,0,0,27,0,0],"r":[4,3,0,0,6,0,16,13,12,11,0,0,0,15,0,0,18,0,26,21,0,23,0,25,0,0,28,0,0],"v":[0,0,-8.25407e-05,-0.002384293,0,0.005572919,0,0,0,0,0.0004928958,-0.005121378,0.0009820348,0,-0.004690085,0.00144013,0,-0.001843335,0,0,-9.640797e-06,0,0.00800198,0,0.001951461,-0.0007411463,0,-0.00355355,-0.0007411463]},{"f":[23,21,2,-1,16,-1,-1,37,20,22,2,22,21,-1,-1,-1,-1,-1,16,-1,16,18,-1,-1,17,-1,-1,-1,-1],"t":[8.5,7.5,9.221514256038809,0,10.696098170955864,0,0,11.05,0.5,16.98920034885663,1.292481250360578,10.224917391507956,8.5,0,0,0,0,0,6.044394119358453,0,7.330916878114617,8.729619742794132,0,0,7.198837316474133,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,12,13,0,0,0,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,7,4,0,6,0,0,27,18,17,16,15,14,0,0,0,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,2.426437e-05,0,-0.0004231369,0.006008079,0,0,0,0,0,0,-0.00237252,0.004510084,-0.00237252,-0.0003618477,0.0009552398,0,0.001487537,0,0,-0.00237252,0.003611851,0,0.001957837,7.909303e-06,-0.004568192,0.001426121]},{"f":[22,5,3,-1,-1,-1,22,23,15,16,22,16,1,0,-1,-1,17,-1,2,-1,-1,-1,-1,-1,-1,4,-1,-1,-1],"t":[12.199665042528586,0.5,0.3162042875157629,0,0,0,12.738408632106918,1.5,6.523561956057013,4.247927513443585,12.613886328427084,4.169925001442312,6.015625,2.5,0,0,1.292481250360578,0,1.292481250360578,0,0,0,0,0,0,1.792481250360578,0,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,0,7,8,9,10,11,12,13,14,0,0,17,0,19,0,0,0,0,0,0,26,0,0,0],"r":[6,5,4,0,0,0,28,25,24,23,22,21,16,15,0,0,18,0,20,0,0,0,0,0,0,27,0,0,0],"v":[0,0,0,-2.808752e-05,-0.003888591,0.0001835416,0,0,0,0,0,0,0,0,-0.00027575,-0.00374682,0,0.004400697,0,-0.003960331,0.002883514,0.003657035,-0.004277495,-0.00321784,0.001475009,0,-0.001194264,-0.006465769,-2.464022e-06]},{"f":[2,2,17,17,21,21,-1,17,-1,-1,-1,-1,1,-1,17,-1,16,-1,16,-1,-1,15,-1,-1,2,25,-1,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.496347487943302,7.6181566135214025,8.5,7.5,0,5.14960400919364,0,0,0,0,1.000936329588015,0,8.729619742794132,0,10.46352437327118,0,10.696098170955864,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,0],"m":[1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,0,13,0,15,0,17,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,21,12,11,10,7,0,9,0,0,0,0,14,0,16,0,18,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,4.269253e-05,0,-0.00197257,0.0002767582,0.001349798,0.001259447,0,-0.005360318,0,-0.006611341,0,0.00046452,0,-0.003639731,-0.000506154,0,-0.0001873653,-0.00686644,0,0,0.0004488192,0.005986565,0.0001055307]},{"f":[22,-1,23,17,1,16,15,-1,-1,-1,17,18,4,-1,16,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,-1,-1,-1],"t":[2.084962500721156,0,0.5,12.18494601563336,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0,5.169925001442312,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,0,0,0],"m":[0,0,0,1,1,1,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,12,13,0,15,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,10,9,8,0,0,0,20,17,14,0,16,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0.001114683,0,0,0,0,0,4.977636e-05,-0.005250238,0.003653365,0,0,0,-0.001570292,0,0.002797736,-0.0004623542,0,0.0003613881,0.006019331,0,0,-0.007635847,-0.001524005,0,0.003675583,-0.0003076365,-0.003715362,8.672043e-05]},{"f":[22,16,-1,-1,22,16,-1,-1,15,15,-1,22,-1,22,-1,17,-1,3,-1,-1,32,-1,15,-1,15,22,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,5.169925001442312,4.087462841250339,0,11.681676585676911,0,13.747003318603246,0,5.740899715832876,0,0.5803571428571428,0,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,0,14,0,16,0,18,0,0,21,0,23,0,25,26,0,0,0],"r":[4,3,0,0,8,7,0,0,20,11,0,13,0,15,0,17,0,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0.0002917329,0.002453899,0,0,-0.0008497275,-0.003556628,0,0,6.114679e-06,0,-0.003622733,0,1.378719e-05,0,-0.0003805548,0,-0.001799642,-0.008599622,0,0.005259014,0,0.001215117,0,0,0.0001545303,-0.002880906,5.930321e-05]},{"f":[1,1,22,-1,-1,18,-1,-1,2,15,2,-1,-1,25,4,-1,18,18,-1,-1,17,-1,-1,16,22,-1,-1,-1,-1],"t":[5.417948717948718,5.015625,17.094543568094064,0,0,4.245926548164837,0,0,9.068775776507199,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,0,8.03615005796417,7.252657678068118,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0],"m":[1,1,1,0,0,1,0,0,1,0,1,0,0,1,1,0,1,1,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,13,12,0,0,23,16,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,1.49751e-05,0.00379999,0,0.0006499638,0.005608509,0,0,0,-0.003184064,0.002171383,0,0,0.0001262873,0,0,-0.0006033285,-0.003891125,0,0.005808203,-0.000405687,0,0,0.002307846,-0.004091085,0.005803455,-0.003972226]},{"f":[2,2,21,22,26,35,-1,-1,-1,22,-1,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,25,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,2.5,0,0,0,15.756821747580634,0,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,1.0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,0,13,14,15,0,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,12,9,8,7,0,0,0,11,0,0,18,17,16,0,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0,3.350749e-05,-0.001583483,0.002483219,0,-0.007929158,-0.0002895361,0,0,0,0.001279168,-0.004456484,0.002744426,-0.003078512,0,-0.0001860581,-0.006534201,0,0,0.0003614753,0.005595703,0,-0.003015959,0.0001560262]},{"f":[23,21,23,-1,17,-1,-1,37,20,22,2,22,21,-1,-1,-1,-1,-1,16,-1,16,18,-1,-1,17,-1,-1,-1,-1],"t":[8.5,7.5,2.5,0,4.128693921346326,0,0,11.05,0.5,16.98920034885663,1.292481250360578,10.224917391507956,8.5,0,0,0,0,0,6.044394119358453,0,7.330916878114617,8.729619742794132,0,0,7.198837316474133,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,12,13,0,0,0,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,7,4,0,6,0,0,27,18,17,16,15,14,0,0,0,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-2.725351e-05,0,0.001295237,-0.0001830231,0,0,0,0,0,0,-0.002299356,0.004150256,-0.002299356,-0.0003295456,0.0009401356,0,0.001347748,0,0,-0.002299356,0.003415504,0,0.001801719,-1.318236e-05,-0.00442298,0.001371301]},{"f":[22,22,22,22,22,22,15,22,-1,-1,-1,3,-1,-1,-1,-1,-1,22,-1,1,17,17,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.838461650489297,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,15.717370311433935,0,0,0,0.05927318295739348,0,0,0,0,0,17.078739781790915,0,7.9752100840336135,8.729619742794132,8.571721503073015,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,12,0,0,0,0,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[26,17,16,15,14,11,10,9,0,0,0,13,0,0,0,0,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,3.710794e-05,-0.0008976315,0.00421366,0,0.004789747,-0.00256763,-0.01337963,0.001206083,-0.0006498525,0,0.002206736,0,0,0,-0.0001773264,-0.008370778,0.001660131,0.002569259,0,5.209415e-05,-0.003598832]},{"f":[2,2,-1,15,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,-1,15,21,-1,17,-1,-1,22,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,5.538407798525416,7.5,0,0,9.229707804263573,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,0,0,0,0,9.862637357558794,6.5,0,10.620215166896223,0,0,11.876328901062104,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,10,9,0,0,20,19,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-2.320754e-05,0,-0.0001834199,-0.006208414,0,0,0.0003981423,0.005315308,0,0,0,0,0,0.0003695009,-0.004701783,0.0002181119,0.003439896,-0.0009222464,0,0,-0.001171096,0,0.004582615,0.0001633835,0,0.0008389653,0.004688211]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,12,2,-1,22,1,22,3,-1,-1,-1,26,-1,4,-1,-1,3,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.5,6.451187557243013,0,16.975010474608837,5.417948717948718,16.832741648970774,0.09303583502366464,0,0,0,0.5,0,6.912876916441556,0,0,0.11587329992841804,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,0,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,25,20,19,18,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0.002401719,-0.001045312,0,0,0,-0.001005405,0.002983247,-0.003909601,0,0,-2.42268e-05,0,0,0,0,0.003713816,0.0005372382,-0.003709887,0,-0.005617071,0,0.001393748,-0.002907903,0,-0.0003851057,0.005486475,-0.001474831]},{"f":[22,22,22,22,22,23,22,-1,22,-1,-1,-1,-1,-1,23,-1,-1,22,-1,1,17,17,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,4.5,15.717370311433935,0,15.756821747580634,0,0,0,0,0,2.5,0,0,17.078739781790915,0,7.9752100840336135,8.729619742794132,8.571721503073015,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,0,0,0,15,0,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[26,17,14,13,12,11,8,0,10,0,0,0,0,0,16,0,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,1.073721e-05,0,-0.007090796,-0.0003762788,0.0007290118,-0.01273792,0.001673877,0,-0.003159788,-0.0003428538,0,0.002092554,0,0,0,-0.0001824375,-0.007966264,0.001569636,0.002435838,0,4.503003e-05,-0.003424499]},{"f":[3,22,-1,-1,3,16,22,-1,-1,-1,12,2,-1,22,1,22,24,-1,2,-1,-1,-1,26,-1,-1,3,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,8.67595703294175,16.73920708776685,0,0,0,0.5,6.451187557243013,0,16.975010474608837,5.417948717948718,16.832741648970774,2.5,0,6.584884225259038,0,0,0,0.5,0,0,0.11587329992841804,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,25,22,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0.002291248,-0.0008694702,0,0,0,-0.0009652741,0.002925212,-0.003704524,0,0,-2.386906e-05,0,0,0,0,0.0004033116,0,0.0002709234,0.004832473,-0.003505679,0,-0.005330595,-0.0006507567,0,-0.0003467609,0.005223171,-0.001404839]},{"f":[21,-1,20,22,2,21,16,-1,15,-1,-1,0,-1,15,-1,-1,22,17,-1,2,-1,-1,-1,-1,16,-1,37,-1,-1],"t":[7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,0,10.066089190457772,0,0,2.5,0,4.247927513443585,0,0,16.844368517741394,6.199638962254484,0,6.055241654908112,0,0,0,0,6.044394119358453,0,10.015,0,0],"m":[1,0,0,1,0,1,1,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,0,14,0,0,17,18,0,20,0,0,0,0,25,0,27,0,0],"r":[2,0,24,23,16,11,8,0,10,0,0,13,0,15,0,0,22,19,0,21,0,0,0,0,26,0,28,0,0],"v":[0,4.459584e-05,0,0,0,0,0,-0.00130775,0,-0.009758163,-0.00156189,0,0.002346582,0,5.880169e-05,-0.005027393,0,0,-0.0004490952,0,0.007268317,3.417251e-05,-0.003414384,0.0008008714,0,0.001294076,0,4.447286e-05,-0.003452668]},{"f":[22,16,-1,-1,22,16,-1,-1,22,21,16,2,-1,-1,22,22,16,-1,-1,1,-1,-1,3,-1,33,-1,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,11.406459975416917,9.5,1.584962500721156,6.451187557243013,0,0,9.087343393278285,8.530347965843777,6.523561956057013,0,0,1.000936329588015,0,0,0.8516483516483516,0,36.27694515242878,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,1,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,0,0,15,16,17,0,0,20,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,28,27,14,13,0,0,22,19,18,0,0,21,0,0,24,0,26,0,0,0,0],"v":[0,0,0.0002907949,0.00234504,0,0,-0.0007950777,-0.003382913,0,0,0,0,0.0007993489,0.005803045,0,0,0,0.004357616,-8.796538e-05,0,-0.00792498,-0.002019864,0,-0.0008539805,0,0.0005192194,0.002707214,-0.001891717,-4.53513e-05]},{"f":[23,3,3,-1,-1,1,-1,32,21,-1,19,2,19,-1,-1,1,17,17,-1,-1,-1,-1,15,18,-1,-1,-1,-1,-1],"t":[0.5,0.9992038216560509,0.4226190476190476,0,0,3.354166666666667,0,7.014643613484123,1.5,0,1.5,1.292481250360578,0.5,0,0,5.015625,9.26326872279839,4.208926257442949,0,0,0,0,6.0,2.160964047443681,0,0,0,0,0],"m":[0,0,1,0,0,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,13,0,0,16,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,7,0,27,10,0,22,15,14,0,0,21,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0.0002372097,0.0035608,0,1.848862e-05,0,0,0.0003924019,0,0,0,-0.008219651,-0.0007887033,0,0,0,-0.0002563617,-0.00642948,-0.000814085,0.0004448979,0,0,-3.967381e-05,0.004429961,-0.001070356,0.001183511,8.499547e-05]},{"f":[4,-1,2,22,-1,15,22,-1,-1,18,-1,-1,18,17,3,22,26,-1,-1,-1,22,-1,-1,4,16,-1,-1,-1,-1],"t":[4.0437314206251695,0,4.128693921346326,11.876328901062104,0,4.087462841250339,16.819976309430963,0,0,4.245926548164837,0,0,8.03615005796417,6.022197059679227,0.7083333333333333,16.725912049439096,2.5,0,0,0,15.212004812197344,0,0,9.037514954341955,6.0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,0,13,14,15,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,12,5,0,9,8,0,0,11,0,0,28,23,20,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,4.893153e-05,0,0,0.001599818,0,0,0.001710986,-0.003728545,0,-0.005987783,-0.001017,0,0,0,0,0,0.005276386,0.00189246,-0.003144359,0,-0.0002200253,0.003511108,0,0,0.0005735376,-0.001171411,0.002406942,0.000753473]},{"f":[2,3,22,-1,-1,3,22,15,-1,-1,-1,4,2,2,-1,21,-1,-1,1,1,-1,-1,-1,18,-1,22,-1,-1,-1],"t":[10.680359255641626,0.007093253968253968,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.20945336562895,0,0,0,9.266718101290426,9.034782574103549,6.451187557243013,0,8.5,0,0,3.172945205479452,1.9735450749013448,0,0,0,9.293471190846834,0,12.457649623491124,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,0,12,13,14,0,16,0,0,19,20,0,0,0,24,0,26,0,0,0],"r":[28,5,4,0,0,11,10,9,0,0,0,23,18,15,0,17,0,0,22,21,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0.003564556,-0.0008340175,0,0,0,-0.0007561786,-0.003387509,0.002024364,0,0,0,-2.171396e-05,0,0.0007446645,-0.001851714,0,0,-0.000634513,-0.004829356,0.001450535,0,0.004574692,0,-0.002500406,0.001114539,-0.001318829]},{"f":[22,16,-1,-1,22,16,-1,-1,22,5,2,-1,-1,-1,22,23,2,2,17,1,-1,-1,-1,-1,-1,4,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,5.882643049361842,0,0,12.199665042528586,0.5,6.033380966193453,0,0,0,12.738408632106918,1.5,8.049190845056867,1.292481250360578,1.792481250360578,6.015625,0,0,0,0,0,1.792481250360578,0,0,0],"m":[0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,0,0,0,15,16,17,18,19,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,8,7,0,0,14,13,12,0,0,0,28,25,24,23,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,0.0002894852,0.00222781,0,0,-0.0007540835,-0.003185139,0,0,0,-0.002724958,0.00036517,0.000219924,0,0,0,0,0,0,-0.001185108,0.004851965,-0.002640886,0.0008430185,-0.002956571,0,-0.001093986,-0.006161894,-5.106164e-06]},{"f":[23,-1,21,17,-1,22,3,36,20,-1,24,-1,-1,-1,17,-1,19,-1,-1,18,22,-1,22,-1,-1,-1,37,-1,-1],"t":[0.5,0,5.5,3.753897320099348,0,15.717370311433935,0.8293269230769231,3.5,0.5,0,7.5,0,0,0,5.918419679874358,0,0.5,0,0,5.065928480304397,16.819976309430963,0,17.069460217922263,0,0,0,10.780000000000001,0,0],"m":[0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,9,0,11,0,0,0,15,0,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,19,14,13,10,0,12,0,0,0,16,0,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,-9.371156e-05,0,0,0.0007655581,0,0,0,0,-0.0002876986,0,-0.005671316,-0.001715398,0.001948459,0,-0.00268842,0,0.000528445,0.002484792,0,0,0.001930658,0,-0.002575768,0.001917232,-0.005836149,0,-3.53695e-05,-0.00288812]},{"f":[1,1,16,-1,9,-1,-1,2,7,-1,-1,-1,22,-1,23,24,-1,33,15,-1,-1,20,-1,22,33,-1,-1,-1,-1],"t":[0.39811320754716983,0.01933482600988898,10.372865060112588,0,0.5,0,0,5.8698903048866296,0.5,0,0,0,14.03140051979143,0,2.5,6.5,0,19.167172160526107,5.044394119358453,0,0,0.5,0,15.717370311433935,21.257266042509173,0,0,0,0],"m":[0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,1,1,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,0,13,0,15,16,0,18,19,0,0,22,0,24,25,0,0,0,0],"r":[12,7,4,0,6,0,0,11,10,0,0,0,14,0,28,17,0,21,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0.0003725158,0,-0.004130278,-0.001029871,0,0,0.001815941,-0.0005044768,0.005077811,0,4.098783e-05,0,0,-0.00559702,0,0,-0.00125552,0.00623192,0,-0.0003336909,0,0,-0.003417441,0.001338367,-0.007705466,3.689923e-05]},{"f":[22,22,22,22,22,22,-1,-1,-1,-1,-1,22,18,-1,-1,4,17,15,-1,-1,-1,4,4,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.819976309430963,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,17.078739781790915,7.65814076587311,0,0,1.292481250360578,8.729619742794132,6.584962500721156,0,0,0,4.0437314206251695,2.69615871138938,0,0,0,0.5,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,12,13,0,0,16,17,18,0,0,0,22,23,0,0,0,27,0,0],"r":[26,11,10,9,8,7,0,0,0,0,0,15,14,0,0,21,20,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-2.583778e-06,0.001877448,-0.0121006,0.001509886,-0.0005140415,0,0,0.002867953,0.0001637776,0,0,0,-0.006816056,-7.304717e-05,0.002144043,0,0,0.0003502251,0.00396651,-0.0008182621,0,1.493725e-05,-0.003270677]},{"f":[23,18,18,15,15,18,18,18,16,-1,-1,-1,33,-1,33,-1,19,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1,-1],"t":[0.5,9.314012697712199,9.267956609922159,10.342074667999139,9.756556322524087,7.298618922788638,3.5221970596792267,2.903677461028802,7.614709844115208,0,0,0,19.167172160526107,0,21.16864017939033,0,1.5,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0,0],"m":[0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,0,15,0,17,0,0,0,0,22,0,24,0,0,0,0,0],"r":[28,27,26,21,20,19,12,11,10,0,0,0,14,0,16,0,18,0,0,0,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.0001761862,0.000866238,-0.007947775,0,0.005494381,0,-0.003310037,0,0.0003431178,0.003281967,-0.002127285,-0.002338672,0,0.003882627,0,-0.002395308,0.001867202,0.003384091,-0.000721473,8.217465e-05]},{"f":[22,-1,22,-1,23,18,2,22,-1,0,-1,17,17,1,-1,2,-1,-1,-1,2,-1,-1,-1,17,15,-1,-1,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,13.689894219688643,0,1.5,0,3.5221970596792267,1.292481250360578,2.0015923566878984,0,2.453445297804259,0,0,0,1.292481250360578,0,0,0,10.682994316786338,9.264442600226602,0,0,0,0],"m":[0,0,0,0,0,1,1,1,0,0,0,1,1,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,10,0,12,13,14,0,16,0,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,4,0,28,23,22,9,0,11,0,19,18,15,0,17,0,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0.0005421899,0,-0.002414802,0,0,0,0,7.582181e-05,0,0.0001621002,0,0,0,-0.002727973,0,-0.0005203504,-0.002928067,-0.006959846,0,-0.0005203504,-0.001832499,0.002980426,0,0,-0.0009883918,-0.005065854,5.905962e-05,7.806636e-05]},{"f":[33,33,33,33,33,33,12,1,-1,-1,-1,25,19,-1,-1,-1,-1,33,-1,-1,0,22,-1,-1,-1,-1,4,-1,-1],"t":[37.98858650139573,35.98606694718995,29.75825500004914,27.27007187651612,26.622428940176302,23.58371303411409,0.5,8.046875,0,0,0,1.5,0.5,0,0,0,0,27.807475161786478,0,0,8.5,16.978341770814396,0,0,0,0,3.753897320099348,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,12,13,0,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,20,17,16,11,10,9,0,0,0,15,14,0,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,-1.522162e-05,0.0007554014,-0.002908381,0,0,0.002291168,-0.001374452,0.005383822,-0.002358459,0,0.004598028,6.771728e-05,0,0,0.00195842,-0.0008969489,-0.002389769,0.001909219,0,-0.001571287,0.0007211925]},{"f":[4,-1,18,16,18,17,16,15,-1,-1,1,-1,-1,23,-1,-1,-1,4,-1,18,-1,15,19,-1,-1,22,-1,-1,-1],"t":[4.0437314206251695,0,8.03615005796417,7.651051691178929,6.434604125275413,6.4178278454199,6.022367813028454,3.584962500721156,0,0,3.9935774037849088,0,0,1.5,0,0,0,4.2849278041654735,0,6.118903736861568,0,10.457380879072534,0.5,0,0,15.782972563647261,0,0,0],"m":[1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,0,0,18,0,20,0,22,23,0,0,26,0,0,0],"r":[2,0,28,17,16,13,10,9,0,0,12,0,0,15,0,0,0,19,0,21,0,25,24,0,0,27,0,0,0],"v":[0,4.790949e-05,0,0,0,0,0,0,0.0001461957,-0.001630439,0,0.004795219,0.0009288887,0,-0.003042312,-6.438843e-05,0.00208001,0,-0.004658157,0,-5.930549e-05,0,0,-0.009169392,-0.001223174,0,0.001667768,-0.003055979,0.000647258]},{"f":[22,22,22,22,22,-1,-1,-1,-1,22,18,-1,-1,1,17,17,22,18,-1,-1,0,-1,-1,-1,-1,-1,25,-1,-1],"t":[17.105450904090162,17.069460217922263,16.838461650489297,16.786010507999382,16.765402642760364,0,0,0,0,17.078739781790915,7.65814076587311,0,0,7.9752100840336135,8.729619742794132,8.571721503073015,17.07907323966382,7.383290334558488,0,0,6.5,0,0,0,0,0,0.5,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,0,0,14,15,16,17,18,0,0,21,0,0,0,0,0,27,0,0],"r":[26,9,8,7,6,0,0,0,0,13,12,0,0,25,24,23,20,19,0,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,1.696999e-05,-0.01150829,0.0009789716,-0.0005422665,0,0,0.002713771,0.0001446614,0,0,0,0,0,-0.003177422,0.0004218909,0,0.003834523,-0.0001118748,-0.007345586,0.001456651,0.002280492,0,1.907716e-06,-0.003130227]},{"f":[2,2,21,22,26,-1,-1,22,-1,3,-1,-1,16,16,15,-1,-1,-1,-1,15,-1,-1,2,17,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,0,0,15.756821747580634,0,0.5187999999999999,0,0,11.98370619265935,7.679480099505446,7.6653359171851765,0,0,0,0,5.672425341971495,0,0,5.538407798525416,1.292481250360578,0,0,1.0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,0,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,12,7,6,0,0,9,0,11,0,0,18,17,16,0,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,-2.750893e-06,0.002336563,0,-0.006768533,0,-0.004493435,-0.0002090814,0,0,0,0.001167651,-0.004269232,0.002604742,-0.002853839,0,-0.0002321347,-0.005814671,0,0,0.004027894,-0.0005345162,0,-0.002843354,0.0001468747]},{"f":[23,1,22,-1,-1,2,15,2,-1,-1,25,4,18,-1,-1,18,18,-1,-1,17,-1,-1,15,22,-1,-1,-1,-1,-1],"t":[8.5,5.417948717948718,17.083799521744545,0,0,8.049190845056867,2.807354922057604,1.292481250360578,0,0,2.5,3.084962500721156,7.980000966034041,0,0,8.03615005796417,7.252657678068118,0,0,9.075473573191719,0,0,4.523561956057013,12.613886328427084,0,0,0,0,0],"m":[1,1,1,0,0,1,0,1,0,0,1,1,1,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,12,13,0,0,16,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,10,9,0,0,22,15,14,0,0,19,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,4.091925e-05,0.003562964,0,0,0,-0.002985911,0.001996266,0,0,0,0.0002633189,-0.001622072,0,0,-0.0005702189,-0.004598052,0,0.006691126,-0.0004914829,0,0,0.002150164,-0.003919364,0.005209785,-0.003390583,0.001254584]},{"f":[2,2,17,17,15,16,-1,-1,-1,-1,1,-1,17,-1,16,-1,16,-1,-1,21,-1,-1,2,25,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.496347487943302,7.6181566135214025,10.457380879072534,9.756556322524087,0,0,0,0,1.000936329588015,0,8.729619742794132,0,10.46352437327118,0,10.696098170955864,0,0,7.5,0,0,5.538407798525416,7.5,0,0,1.0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,17,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,10,9,8,7,0,0,0,0,12,0,14,0,16,0,18,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-1.03425e-05,-0.002162359,0.000709723,0.001065938,0,-0.004889359,0,-0.005728363,0,0.0004839481,0,-0.003434714,-0.0004512763,0,-0.005528154,-0.0002226544,0,0,0.0003757008,0.004586315,0,-0.002706148,0.0001470388]},{"f":[21,-1,20,22,2,21,16,17,-1,-1,15,-1,-1,22,-1,0,-1,22,-1,-1,22,22,-1,16,-1,-1,-1,-1,-1],"t":[7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,1.292481250360578,0,0,10.066089190457772,0,0,10.224917391507956,0,2.5,0,14.116600814324018,0,0,16.844368517741394,12.738408632106918,0,7.330916878114617,0,0,0,0,0],"m":[1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,16,0,18,0,0,21,22,0,24,0,0,0,0,0],"r":[2,0,28,27,20,13,10,9,0,0,12,0,0,15,0,17,0,19,0,0,26,23,0,25,0,0,0,0,0],"v":[0,4.282642e-05,0,0,0,0,0,0,-0.003757856,-0.0001512461,0,-0.00922063,-0.001534316,0,0.003843341,0,0.001152676,0,-0.005989992,-0.0007517035,0,0,-0.0005687404,0,0.005975231,8.973639e-05,-0.003211293,0.0007171386,0.0002032182]},{"f":[2,3,22,-1,-1,4,2,4,4,35,-1,-1,-1,16,17,-1,-1,-1,17,23,-1,-1,-1,16,1,-1,-1,-1,-1],"t":[10.680359255641626,0.007093253968253968,16.725912049439096,0,0,9.266718101290426,9.034782574103549,7.675607660927803,7.408266855279292,2.5,0,0,0,7.614709844115208,11.303485185891478,0,0,0,9.44286754736589,0.5,0,0,0,10.486835021563051,2.0015923566878984,0,0,0,0],"m":[1,0,0,0,0,1,1,1,1,1,0,0,0,1,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,0,14,15,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[28,5,4,0,0,23,18,13,12,11,0,0,0,17,16,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0.003316928,-0.0007130086,0,0,0,0,0,2.273724e-05,-0.0006666659,-0.003303114,0,0,0.006578861,-0.002912815,-0.001041211,0,0,-0.00104875,-0.004496265,0.001686783,0,0,0.0009398023,-0.002598756,0.00494728,-0.001279562]},{"f":[22,-1,22,-1,23,18,2,-1,-1,17,17,-1,-1,-1,21,17,-1,15,22,-1,22,-1,-1,15,-1,-1,37,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,0.5,10.543869317296533,9.266718101290426,0,0,10.682994316786338,10.609627990977145,0,0,0,5.5,3.753897320099348,0,6.523561956057013,16.96990787537721,0,17.069460217922263,0,0,10.462502272597265,0,0,10.780000000000001,0,0],"m":[0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,0,5,6,7,0,0,10,11,0,0,0,15,16,0,18,19,0,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,14,9,8,0,0,13,12,0,0,0,26,17,0,23,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0.0005305529,0,-0.002274515,0,0,0,-7.5219e-05,0.0028057,0,0,-0.001150688,-0.005647575,6.137202e-05,0,0,0.0007054219,0,0,-0.0004906985,0,-0.00721989,-0.0007997264,0,0.0007140637,-0.001636966,0,-3.657703e-05,-0.002722391]},{"f":[23,22,2,16,-1,-1,24,1,16,-1,1,-1,22,-1,1,-1,-1,4,23,-1,-1,-1,17,-1,-1,24,-1,-1,-1],"t":[8.5,17.105450904090162,1.292481250360578,5.614709844115208,0,0,9.5,8.102912599421547,6.882643049361842,0,4.994215473608098,0,16.819976309430963,0,5.015625,0,0,4.80643424864552,1.5,0,0,0,6.584884225259038,0,0,8.5,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,11,0,13,0,15,0,0,18,19,0,0,0,23,0,0,26,0,0,0],"r":[28,25,6,5,0,0,22,17,10,0,12,0,14,0,16,0,0,21,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.0002625687,0.0003622905,0,0,0,0.0001603206,0,-3.281596e-05,0,-0.0008253029,0,-0.01097344,-0.0008834916,0,0,0.002238568,0.007660542,-0.0007253174,0,-0.005431207,0.0002264183,0,-0.002997108,-0.0002472393,0.001236166]},{"f":[22,16,-1,-1,22,1,-1,-1,15,15,-1,22,-1,22,-1,17,-1,-1,32,-1,15,-1,15,22,22,-1,-1,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,1.000936329588015,0,0,5.169925001442312,4.087462841250339,0,11.681676585676911,0,14.03140051979143,0,5.740899715832876,0,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,13.129761635363703,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,0,12,0,14,0,16,0,0,19,0,21,0,23,24,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,18,11,0,13,0,15,0,17,0,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0.0002777021,0.002129667,0,0,-0.002066991,-0.0002764294,0,0,-1.807745e-06,0,-0.003191819,0,1.846245e-05,0,-0.0001204575,-0.008049883,0,0.004560338,0,0.00111685,0,0,0,-0.003110715,0.001328955,-0.002582603,4.626319e-05]},{"f":[2,2,-1,16,-1,-1,2,25,-1,-1,18,2,4,15,22,-1,-1,-1,-1,-1,15,21,-1,16,-1,-1,2,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,7.330916878114617,0,0,5.538407798525416,7.5,0,0,9.273793716589307,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,0,0,0,0,9.862637357558794,6.5,0,10.681238411777805,0,0,10.41668595978746,0,0],"m":[1,1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,10,9,0,0,20,19,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-2.036912e-05,0,-0.0002148726,-0.0052646,0,0,0.0003883331,0.004310387,0,0,0,0,0,0.0005886734,-0.004237397,0.0002258678,0.003004743,-0.0008694977,0,0,-0.00108897,0,0.004422683,5.006568e-05,0,0.00568535,-0.0004277223]},{"f":[33,32,15,15,15,-1,-1,4,23,-1,24,-1,-1,2,-1,22,-1,-1,22,1,-1,-1,-1,36,-1,-1,4,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,3.6427011094311244,0.5,0,8.5,0,0,6.139515068224824,0,16.838461650489297,0,0,11.722778233171695,1.3336158192090395,0,0,0,3.5,0,0,3.753897320099348,0,0],"m":[1,1,1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,0,0,14,0,16,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,7,6,0,0,13,10,0,12,0,0,15,0,17,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,2.240954e-05,-0.001422935,0,0,0.0004498425,0,0.003274682,5.023885e-05,0,-0.002705903,0,0.001991154,-0.00292824,0,0,-0.003794542,-0.0007131117,-1.670109e-05,0,0.003008605,-0.0002080645,0,-0.001515366,0.0007313202]},{"f":[22,16,-1,-1,22,1,-1,-1,18,18,2,4,4,15,-1,-1,-1,-1,19,22,22,-1,-1,-1,-1,-1,15,-1,-1],"t":[6.671801068872866,7.721099188707185,0,0,7.082728314749225,1.000936329588015,0,0,12.18494601563336,11.43339081173756,9.341519573083566,9.221342183932917,9.05390992190079,5.169925001442312,0,0,0,0,0.5,12.457649623491124,11.685405419153785,0,0,0,0,0,11.385323176175872,0,0],"m":[0,1,0,0,0,1,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,13,14,0,0,0,0,19,20,21,0,0,0,0,0,27,0,0],"r":[4,3,0,0,8,7,0,0,26,25,18,17,16,15,0,0,0,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0.0002633285,0.002046508,0,0,-0.001937941,-0.0002615277,0,0,0,0,0,0,-0.0001327019,7.873903e-05,-0.002534419,0.002892354,0,0,0,0.0002538245,-0.003350836,0.001494384,-0.003298603,0.001740909,0,9.590898e-05,-0.001852343]},{"f":[2,2,21,22,26,-1,-1,22,-1,3,-1,2,-1,-1,16,16,-1,-1,-1,15,-1,-1,2,17,-1,-1,15,-1,-1],"t":[5.40929108874043,5.2849278041654735,8.5,15.717370311433935,0.5,0,0,15.756821747580634,0,0.5187999999999999,0,4.0437314206251695,0,0,11.98370619265935,8.321928094887362,0,0,0,5.672425341971495,0,0,5.538407798525416,1.292481250360578,0,0,1.0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,0,0,0,20,0,0,23,24,0,0,27,0,0],"r":[22,19,14,7,6,0,0,9,0,11,0,13,0,0,18,17,0,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0,-5.315927e-06,0.002223832,0,-0.006322245,0,-0.00427046,0,-7.023735e-05,-0.002317307,0,0,0.000647073,0.002959595,-0.00266012,0,-0.0001582303,-0.005130643,0,0,0.00341067,-0.0004645526,0,-0.002504274,0.0001461899]},{"f":[1,1,33,32,1,-1,17,2,-1,-1,16,-1,-1,-1,-1,16,-1,-1,17,21,-1,15,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,2.160964047443681,1.292481250360578,0,0,10.826548487290914,0,0,0,0,6.539158811108031,0,0,2.160964047443681,4.5,0,8.361943773735241,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,11,0,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,15,14,13,6,0,10,9,0,0,12,0,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,4.297021e-05,0,0,-0.001826854,-0.0002726229,0,0.0001708888,-0.004266228,0.001452962,-0.001717808,0,-0.0004320023,0.005570151,0,0,0.00357933,0,0.0002077861,-0.003824659,0,0,-0.000356735,-0.005042263,0.0002174225]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,23,18,15,15,17,17,18,18,-1,-1,17,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.007093253968253968,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.700439718141092,0,0,0,0.5,9.845435504646158,10.113742166049189,9.756556322524087,9.267956609922159,7.2941523604036345,3.8571227588330617,2.903677461028802,0,0,5.281121212110536,0,0,0,0,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,13,14,15,16,17,18,0,0,21,0,0,0,0,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,27,26,25,24,23,20,19,0,0,22,0,0,0,0,0,0,0,0],"v":[0,0,0.002010181,-0.0007007415,0,0,0,-0.0008078068,-0.003190581,0.00202823,0,0,0,0,0,0,0,0,-8.728396e-05,-0.006083099,0,0.003183532,0.0003283167,-0.002040793,0.002320903,-0.002264961,0.001450276,-0.0007076312,9.536087e-05]},{"f":[1,1,33,32,1,-1,-1,-1,-1,16,18,-1,36,-1,33,-1,-1,-1,17,21,-1,15,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,0,0,0,6.539158811108031,1.792481250360578,0,2.5,0,15.895530557358477,0,0,0,2.160964047443681,4.5,0,8.361943773735241,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,11,0,13,0,15,0,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,9,8,7,6,0,0,0,0,17,12,0,14,0,16,0,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,4.21271e-05,-0.0002058462,0.001383255,-0.001622772,0,0,-0.001136124,0,0.00216843,0,-0.001793616,0.0009711755,0.005295917,0,0,0.003384492,0,0.000193415,-0.003643244,0,0,-0.0003437731,-0.004793476,0.0001749777]},{"f":[33,33,33,33,12,3,15,22,-1,-1,-1,-1,-1,36,-1,-1,-1,33,-1,-1,34,-1,17,36,-1,1,-1,-1,-1],"t":[29.807475161786478,27.27007187651612,26.622428940176302,23.76434662895582,0.5,0.05277777777777778,10.113742166049189,16.765402642760364,0,0,0,0,0,3.5,0,0,0,27.807475161786478,0,0,2.5,0,9.26326872279839,3.5,0,9.107199999999999,0,0,0],"m":[1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,0,14,0,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[20,17,16,13,12,11,10,9,0,0,0,0,0,15,0,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0,0.0001294717,-0.004129104,-0.003740739,2.363393e-05,-0.002525047,0,-0.001325146,0.003595616,-0.002249899,0,0.004334032,8.357521e-05,0,-0.003464471,0,0,0.002862551,0,0.0002476847,-0.001668891,-0.002612866]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,22,23,-1,18,16,18,-1,-1,18,1,-1,-1,-1,16,2,-1,-1,-1,-1],"t":[0.008766718459616732,16.725912049439096,0,0,0.034940494354592615,16.73920708776685,5.20945336562895,0,0,0,17.105450904090162,0.5,0,10.009557117932596,7.924812503605781,8.645081439392136,0,0,1.292481250360578,9.107199999999999,0,0,0,10.462502272597265,4.128693921346326,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,0,14,15,16,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,13,0,23,18,17,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.001902584,-0.000618959,0,0,0,-0.0007203709,-0.002962289,0.002011953,0,0,-8.39777e-05,0,0,0,0.0002785796,-0.002618957,0,0,0.0006245305,-0.003127962,-0.000668681,0,0,0.001440703,0.004430964,-0.0006344552,-0.001242767]},{"f":[22,-1,21,-1,20,22,2,21,16,17,-1,-1,15,-1,-1,22,-1,21,-1,-1,16,-1,-1,-1,16,-1,16,-1,-1],"t":[2.084962500721156,0,7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,1.292481250360578,0,0,10.066089190457772,0,0,10.224917391507956,0,9.5,0,0,1.0,0,0,0,6.044394119358453,0,7.330916878114617,0,0],"m":[0,0,1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,0,0,13,0,0,16,0,18,0,0,21,0,0,0,25,0,27,0,0],"r":[2,0,4,0,24,23,20,15,12,11,0,0,14,0,0,17,0,19,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0.0009380822,0,3.599063e-05,0,0,0,0,0,0,-0.003513158,-0.0001490788,0,-0.008735808,-0.0015121,0,0.003621623,0,0.0002065552,-0.003598943,0,-0.003403331,-0.000138371,0.0007111375,0,0.001120367,0,-0.001227711,0.0001943527]},{"f":[23,22,16,4,17,4,22,22,33,22,1,-1,-1,-1,-1,17,-1,-1,-1,-1,-1,-1,22,-1,-1,-1,15,-1,-1],"t":[8.5,17.105450904090162,1.584962500721156,7.950787011501063,8.148458103439644,4.128693921346326,16.98920034885663,16.98413870862295,23.925614337747046,16.975010474608837,6.196924603174603,0,0,0,0,1.292481250360578,0,0,0,0,0,0,5.247427792245599,0,0,0,6.539158811108031,0,0],"m":[1,1,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,9,10,11,0,0,0,0,16,0,0,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,22,21,20,19,18,15,14,13,12,0,0,0,0,17,0,0,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0,0.0002996164,-0.002556968,0.002713615,0.003100286,0,-0.004852153,-0.001295333,0.001248061,-0.002097814,-0.002061569,0.002432094,0,0.00123395,-4.362446e-05,-0.001258116,0,0.002409768,-0.0001491651]},{"f":[33,33,33,33,12,19,20,-1,-1,23,-1,-1,-1,33,-1,-1,-1,33,-1,-1,34,-1,17,36,-1,34,-1,-1,-1],"t":[29.807475161786478,27.27007187651612,26.622428940176302,23.58371303411409,0.5,1.5,1.5,0,0,1.5,0,0,0,24.145246550704165,0,0,0,27.807475161786478,0,0,2.5,0,9.26326872279839,3.5,0,3.5,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,0,14,0,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[20,17,16,13,12,9,8,0,0,11,0,0,0,15,0,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,-2.01323e-05,-0.004442101,0,-0.0001524279,0.001092607,-0.002338185,0,0.003555767,-0.0002923288,-0.002109673,0,0.004130243,0.0001036654,0,-0.003254503,0,0,0.002722517,0,-0.00187298,0.00014145,-0.002456667]},{"f":[33,33,33,33,1,1,36,-1,1,-1,-1,4,-1,-1,-1,33,-1,-1,0,22,-1,-1,-1,-1,2,-1,10,-1,-1],"t":[37.98858650139573,35.98606694718995,29.807475161786478,27.27007187651612,12.18570183086312,10.402747650036153,3.5,0,8.046875,0,0,4.0437314206251695,0,0,0,27.807475161786478,0,0,8.5,16.978341770814396,0,0,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,0,16,0,0,19,20,0,0,0,0,25,0,27,0,0],"r":[24,23,18,15,14,11,8,0,10,0,0,13,0,0,0,17,0,0,22,21,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0,0,-2.236905e-05,0,0.0002112417,0.005245944,0,-0.004523497,4.456945e-05,0.001865114,0,0.003926375,9.850556e-05,0,0,0.001877912,-0.000861626,-0.002298059,0.001642455,0,0.000563252,0,0.0003216953,-0.002857752]},{"f":[2,2,3,-1,2,-1,-1,16,22,4,19,-1,-1,22,-1,-1,-1,23,16,18,3,-1,-1,3,-1,-1,-1,-1,-1],"t":[5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,7.330916878114617,16.819976309430963,4.951187557243013,0.5,0,0,8.643851681338631,0,0,0,2.5,10.681238411777805,4.128693921346326,0.33771929824561403,0,0,0.41565040650406504,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[28,7,4,0,6,0,0,17,16,13,12,0,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.002599443,0,3.744863e-05,0.002386558,0,0,0,0,0.0002696812,0.006214275,0,-0.003036742,-5.755344e-05,-0.002621101,0,0,0,0,0.0001854384,-0.006075585,0,-0.002501005,0.001251536,0.0006595234,-0.005147438,0.0001620232]},{"f":[15,-1,15,21,-1,-1,21,-1,17,1,-1,22,24,-1,22,-1,-1,37,-1,-1,17,-1,2,2,21,-1,-1,-1,-1],"t":[9.756556322524087,0,10.066089190457772,6.5,0,0,3.5,0,2.160964047443681,0.01933482600988898,0,16.725912049439096,9.5,0,15.439474430023438,0,0,6.890000000000001,0,0,3.245926548164837,0,5.065928480304397,1.792481250360578,5.5,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,0,12,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,20,11,0,17,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,4.053846e-05,0,0,-0.0007636693,-0.005058632,0,0.00148321,0,0,-0.001881966,0,0,0.006157233,0,0.0006386893,0.003928121,0,0.000538298,-0.00221681,0,-0.004005609,0,0,0,0.001542942,-0.0006678246,-0.002052594,0.0004312116]},{"f":[23,1,22,22,-1,16,-1,-1,-1,2,19,-1,4,25,-1,24,-1,-1,2,22,-1,-1,2,-1,-1,-1,15,-1,-1],"t":[8.5,5.417948717948718,17.083799521744545,17.074218326240924,0,7.741466986401147,0,0,0,8.049190845056867,1.5,0,2.453445297804259,0.5,0,7.5,0,0,3.903677461028802,14.158358850608169,0,0,4.2849278041654735,0,0,0,6.539158811108031,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,0,10,11,0,13,14,0,16,0,0,19,20,0,0,23,0,0,0,27,0,0],"r":[26,9,8,5,0,7,0,0,0,25,12,0,18,15,0,17,0,0,22,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,5.15555e-05,0,-0.00528663,-0.000537238,0.00342264,0,0,-0.0003501315,0,0,-0.0008858694,0,0.0008927227,0.004554065,0,0,-0.00406089,9.465915e-06,0,0.005613958,-0.0002177147,-0.002974077,0,0.002326904,-0.000138359]},{"f":[23,1,16,15,-1,-1,-1,17,18,4,19,-1,17,-1,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,33,-1,-1,-1],"t":[0.5,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0.5,0,1.792481250360578,0,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,21.16864017939033,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,7,6,5,0,0,0,19,16,15,12,0,14,0,0,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,4.665429e-05,-0.004447261,0.002847583,0,0,0,0,-0.003836855,0,0.001008895,-0.003880797,9.550321e-05,0,0.0002539539,0.005111669,0,0,-0.006783125,-0.001222005,0,0.003059342,0,-0.003139286,0.0006396841,7.255589e-05]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,15,-1,32,-1,15,-1,15,22,22,-1,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,5.169925001442312,0,3.753897320099348,0,5.491853096329675,0,5.614709844115208,13.689894219688643,13.129761635363703,0,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,0,19,0,21,0,23,24,25,0,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,18,0,20,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0.0001748357,0,0.003856175,0,-0.002136453,0.001247996,-0.001181963,0.001447152,0,0,-0.001824515,-0.0001809701,0,-0.0001148762,0,0.004156895,0,0.001030172,0,0,0,-0.002964868,0.00125604,-0.002461064,3.992807e-05]},{"f":[33,32,15,15,15,-1,-1,4,-1,2,-1,-1,15,18,33,-1,-1,-1,36,-1,-1,24,-1,-1,2,-1,11,-1,-1],"t":[37.98858650139573,10.41931383093672,9.756556322524087,8.005624549193879,7.924812503605781,0,0,3.6427011094311244,0,6.139515068224824,0,0,10.066089190457772,8.03615005796417,29.807475161786478,0,0,0,3.5,0,0,9.5,0,0,2.453445297804259,0,0.5,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,12,7,6,0,0,9,0,11,0,0,18,17,16,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,1.833335e-05,-0.001288839,0,0.001074107,0,-0.002581587,0.0007635017,0,0,0,-0.0004135182,-0.003301587,0.000587599,0,-0.003603936,1.29789e-06,0,0.002396917,-0.000511807,0,0.0005317938,0,-0.002660718,0.0002884979]},{"f":[2,2,3,-1,2,-1,-1,16,22,4,19,-1,-1,22,-1,-1,-1,23,16,18,3,-1,-1,3,-1,-1,-1,-1,-1],"t":[5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,7.330916878114617,16.819976309430963,4.951187557243013,0.5,0,0,8.643851681338631,0,0,0,2.5,10.681238411777805,4.128693921346326,0.33771929824561403,0,0,0.41565040650406504,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,11,0,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[28,7,4,0,6,0,0,17,16,13,12,0,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.002436959,0,3.45659e-05,0.002246299,0,0,0,0,0.0002695934,0.005657506,0,-0.00288125,-5.888126e-05,-0.002434113,0,0,0,0,0.000114365,-0.005747671,0,-0.002373634,0.00125603,0.0006192777,-0.004877933,0.0001547705]},{"f":[23,1,16,15,-1,-1,-1,17,18,4,19,-1,17,-1,-1,-1,22,-1,-1,1,17,-1,-1,33,-1,33,-1,-1,-1],"t":[0.5,3.354166666666667,11.868050853745158,11.683433292835403,0,0,0,6.734566509914796,5.108372929097653,1.292481250360578,0.5,0,1.792481250360578,0,0,0,11.36027577861459,0,0,6.646978021978022,7.344289078557851,0,0,21.011466140970732,0,21.16864017939033,0,0,0],"m":[0,1,1,1,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,0,17,0,0,20,21,0,0,24,0,26,0,0,0],"r":[28,7,6,5,0,0,0,19,16,15,12,0,14,0,0,0,18,0,0,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,4.312619e-05,-0.004192155,0.002724354,0,0,0,0,-0.003627221,0,0.0009554082,-0.003678735,9.510937e-05,0,0.0002347807,0.004814898,0,0,-0.006408491,-0.001181223,0,0.00286422,0,-0.002989306,0.0005928982,7.086195e-05]},{"f":[23,1,-1,18,19,-1,22,4,22,-1,25,-1,23,-1,-1,37,22,-1,36,-1,-1,-1,22,-1,-1,2,-1,-1,-1],"t":[8.5,5.417948717948718,0,9.755304827902876,1.5,0,15.717370311433935,2.453445297804259,11.775312925643524,0,0.5,0,0.5,0,0,11.26,11.775312925643524,0,3.5,0,0,0,16.759160379664443,0,0,1.792481250360578,0,0,0],"m":[1,1,0,1,1,0,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,7,8,9,0,11,0,13,0,0,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,3,0,25,6,0,22,15,10,0,12,0,14,0,0,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,4.240011e-05,0,0,-0.0003399727,0,0,0,0.003327886,0,-0.001046529,0,0.0003839998,0.003792318,0,0,0.003465999,0,-0.002113518,0.001319155,-0.004510246,0,-0.0104009,0.001623688,0,0.001178703,-0.004267208,0.001082709]},{"f":[2,2,-1,21,-1,-1,2,-1,18,2,4,15,22,-1,4,-1,-1,-1,-1,-1,15,2,4,-1,-1,-1,2,-1,-1],"t":[5.40929108874043,5.2849278041654735,0,7.5,0,0,5.538407798525416,0,9.273793716589307,8.9274737487839,7.997154617983022,2.0,7.5621616439616925,0,6.16978583919235,0,0,0,0,0,9.862637357558794,9.645601091186107,6.044228501743113,0,0,0,10.41668595978746,0,0],"m":[1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,0,9,10,11,12,13,0,15,0,0,0,0,0,21,22,23,0,0,0,27,0,0],"r":[6,3,0,5,0,0,8,0,20,19,18,17,14,0,16,0,0,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,-1.838432e-05,0,-0.004677318,-7.308613e-05,0,0.001912573,0,0,0,0,0,0.000692915,0,-0.001794664,-0.00619789,0.0001818501,0.002779106,-0.0008095989,0,0,0,-0.0004992548,0.002506361,-0.002597052,0,0.005416312,-0.0002441138]},{"f":[4,4,2,2,3,-1,2,-1,-1,-1,18,22,16,-1,-1,-1,15,-1,-1,2,-1,-1,16,15,17,-1,-1,-1,-1],"t":[9.266718101290426,9.05390992190079,5.40929108874043,4.0437314206251695,0.5187999999999999,0,3.903677461028802,0,0,0,10.090748425878532,8.32532955673354,7.651051691178929,0,0,0,8.618385502258606,0,0,9.20088558729989,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,13,0,0,0,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,19,10,9,6,0,8,0,0,0,16,15,14,0,0,0,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.002326614,0,3.23055e-05,0.002145682,-0.0003816402,0,0,0,0.003984154,-0.002818845,-2.215655e-05,0,1.281569e-05,0.005241677,0,-0.002419263,-0.0003903762,0,0,0,0.002757849,-0.0003250171,-0.002229412,0.00292931]},{"f":[2,2,4,4,4,22,-1,23,24,-1,-1,-1,22,-1,-1,16,-1,-1,23,16,-1,-1,-1,4,-1,16,-1,-1,-1],"t":[10.680359255641626,9.266718101290426,8.950330235806282,7.950787011501063,7.408266855279292,14.03140051979143,0,2.5,6.5,0,0,0,8.658207063149977,0,0,4.087462841250339,0,0,0.5,2.321928094887362,0,0,0,9.328332883600808,0,4.247927513443585,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,0,13,0,0,16,0,0,19,20,0,0,0,24,0,26,0,0,0],"r":[28,23,18,15,12,7,0,11,10,0,0,0,14,0,0,17,0,0,22,21,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0,6.382708e-05,0,0,-0.0053878,-0.0004587916,3.746395e-05,0,-0.003927722,0.0002876398,0,0.007970478,-0.002981898,0,0,-0.001633402,0.000322287,-0.002810532,0,0.004035511,0,0.0006566655,-0.001842715,-0.001098238]},{"f":[22,-1,21,-1,20,22,2,21,16,22,22,-1,-1,-1,15,-1,-1,22,-1,21,-1,-1,16,-1,-1,-1,16,-1,-1],"t":[2.084962500721156,0,7.5,0,0.5,16.98920034885663,1.292481250360578,8.5,3.169925001442312,16.978341770814396,16.975010474608837,0,0,0,10.066089190457772,0,0,10.224917391507956,0,9.5,0,0,1.0,0,0,0,6.044394119358453,0,0],"m":[0,0,1,0,0,1,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,10,11,0,0,0,15,0,0,18,0,20,0,0,23,0,0,0,27,0,0],"r":[2,0,4,0,26,25,22,17,14,13,12,0,0,0,16,0,0,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0.0008381778,0,3.507826e-05,0,0,0,0,0,0,0,-0.00194298,0.002541671,-0.003227874,0,-0.00827699,-0.001467008,0,0.00340747,0,0.0001909591,-0.003431059,0,-0.003105776,-0.0001114047,0.0006840911,0,0.001050062,-7.81902e-05]},{"f":[23,-1,1,-1,1,2,-1,-1,18,16,17,-1,18,-1,1,-1,-1,17,22,24,-1,-1,-1,-1,16,4,-1,-1,-1],"t":[0.5,0,0.01933482600988898,0,0.3808851224105462,5.14960400919364,0,0,10.009557117932596,7.924812503605781,8.571721503073015,0,8.729619742794132,0,2.0015923566878984,0,0,1.292481250360578,16.725912049439096,7.5,0,0,0,0,10.462502272597265,4.128693921346326,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,0,9,10,11,0,13,0,15,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,4,0,8,7,0,0,24,17,12,0,14,0,16,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-7.686503e-05,0,-0.0012729,0,0,0.0002960431,0.004442453,0,0,0,0.0002279005,0,-0.006407772,0,-0.004240404,0.001151778,0,0,0,0.004764506,0.0009115996,-0.003520044,-0.0005418304,0,0,0.001300601,0.004096608,-0.0006332122]},{"f":[22,23,22,-1,15,22,-1,-1,-1,4,1,18,18,-1,-1,17,-1,-1,2,22,-1,-1,-1,16,-1,-1,25,-1,-1],"t":[17.105450904090162,3.5,16.978341770814396,0,6.0,17.069460217922263,0,0,0,6.451187557243013,8.046875,8.496347487943302,7.467398296089201,0,0,9.523560624409399,0,0,4.128693921346326,15.212004812197344,0,0,0,6.569855608330948,0,0,0.5,0,0],"m":[1,1,1,0,1,0,0,0,0,1,1,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,0,10,11,12,13,0,0,16,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,9,4,0,8,7,0,0,0,23,18,15,14,0,0,17,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-1.47207e-05,0,0,-0.005988447,-0.001435857,0.001493757,0,0,0,0,-0.0001868329,0.0007150649,0,-0.003119494,0.0009949551,0,0,0.0007205396,0.005812916,-0.0004397406,0,0.004026567,0.0003949163,0,1.855839e-05,-0.002730191]},{"f":[3,22,-1,-1,3,22,15,-1,-1,-1,23,18,15,18,15,18,18,-1,-1,-1,-1,-1,2,-1,4,-1,-1,-1,-1],"t":[0.007093253968253968,16.752707246237605,0,0,0.034940494354592615,16.73920708776685,4.247927513443585,0,0,0,0.5,9.845435504646158,10.342074667999139,9.267956609922159,9.756556322524087,7.298618922788638,3.6427011094311244,0,0,0,0,0,1.292481250360578,0,2.160964047443681,0,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,0,11,12,13,14,15,16,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,10,9,8,0,0,0,28,27,22,21,20,19,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0.001543184,-0.001384043,0,0,0,-0.0006819662,-0.002661056,0.001858178,0,0,0,0,0,0,0,-0.0001374177,0.0004715885,-0.00193807,-0.001840134,0.002604949,0,0.003311182,0,-0.001952484,0.001724845,-0.0006390429,8.659154e-05]},{"f":[15,17,2,2,2,-1,-1,-1,16,-1,-1,-1,23,21,-1,33,-1,22,1,-1,19,2,-1,-1,-1,17,-1,-1,-1],"t":[1.584962500721156,9.293471190846834,7.894792293346235,5.065928480304397,4.128693921346326,0,0,0,1.584962500721156,0,0,0,0.5,1.5,0,37.33732815674957,0,14.03140051979143,3.354166666666667,0,0.5,1.292481250360578,0,0,0,2.160964047443681,0,0,0],"m":[0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,0,0,13,14,0,16,0,18,19,0,21,22,0,0,0,26,0,0,0],"r":[12,11,8,7,6,0,0,0,10,0,0,0,28,15,0,17,0,25,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0.0002740867,0.002284059,-0.003260396,0,0.00477731,-5.891899e-05,-0.002234908,0,0,0.0001147015,0,0.0004915625,0,0,-2.838205e-05,0,0,-0.007402583,-0.000641399,-0.0001684462,0,0.001475448,-0.00779603,6.558655e-05]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,21,23,22,-1,15,-1,-1,17,-1,4,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,7.5,2.5,16.819976309430963,0,5.044394119358453,0,0,3.245926548164837,0,6.451187557243013,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,18,19,0,21,0,0,24,0,26,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,28,23,20,0,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.0001421931,0,0.003724013,0,-0.001976003,0.001295521,-0.001070436,0.001380165,0,0,-0.001664304,-9.275003e-05,0,0,0,-1.745877e-06,0,-0.00450495,0.000601354,0,0.001129113,0,-0.0005056882,0.002430005,-0.0001588322]},{"f":[22,22,4,16,-1,15,-1,22,-1,-1,-1,-1,22,1,-1,-1,22,21,4,4,4,-1,-1,-1,22,-1,-1,-1,-1],"t":[6.671801068872866,6.228305549787459,4.614409345247941,2.807354922057604,0,3.169925001442312,0,2.903677461028802,0,0,0,0,7.082728314749225,1.000936329588015,0,0,11.406459975416917,9.5,6.451187557243013,6.118903736861568,5.169368191286958,0,0,0,10.029977181607594,0,0,0,0],"m":[0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,0,0,13,14,0,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[12,11,10,5,0,7,0,9,0,0,0,0,16,15,0,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0.0001351045,0,0.003541131,0,-0.001879664,0.001231545,-0.001017256,0.001311741,0,0,-0.001581684,-8.814013e-05,0,0,0,0,0,0.0001345128,0.002311601,-0.004396169,0,0.004906134,9.191799e-05,-0.001546391,-3.83021e-05]},{"f":[1,1,33,32,1,-1,17,-1,16,-1,-1,-1,-1,16,18,-1,-1,-1,17,22,33,-1,-1,-1,34,20,-1,-1,-1],"t":[9.107199999999999,8.102912599421547,37.98858650139573,10.41931383093672,5.417948717948718,0,2.160964047443681,0,10.826548487290914,0,0,0,0,6.539158811108031,1.792481250360578,0,0,0,2.160964047443681,15.439474430023438,25.701296736393502,0,0,0,3.5,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,0,0,14,15,0,0,0,19,20,21,0,0,0,25,26,0,0,0],"r":[18,13,12,11,6,0,8,0,10,0,0,0,0,17,16,0,0,0,24,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,3.79439e-05,0,-0.0007529528,0,0.000147932,-0.003873806,0.001254217,-0.001377039,0,0,-0.001041091,0.0004313479,0.004696546,0,0,0,0.005360682,-0.0002945195,-0.0028101,0,0,-0.0003009543,-0.004304437,0.0002054138]},{"f":[23,22,16,17,16,17,-1,2,22,-1,-1,2,-1,-1,-1,-1,1,-1,1,15,-1,-1,-1,0,-1,-1,16,-1,-1],"t":[8.5,17.105450904090162,6.409390936137702,11.78143940991899,6.392317422778761,7.6181566135214025,0,7.128362343532883,16.978341770814396,0,0,9.427971070948235,0,0,0,0,0.012999172356714255,0,0.3808851224105462,10.462502272597265,0,0,0,7.5,0,0,0.0,0,0],"m":[1,1,0,1,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,17,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,16,15,14,7,0,11,10,0,0,13,0,0,0,0,18,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-1.044766e-05,0,0,0.0006770313,-0.001758846,0,0.005545325,-0.001097066,0.00144893,-0.003285362,0,-0.002729147,0,0,0.002858411,0.000108037,-0.0001001002,0,-0.0001478776,-0.002591172,0,-3.71773e-05,0.002394061]},{"f":[4,4,2,2,-1,15,-1,-1,18,22,16,-1,-1,15,-1,-1,15,-1,-1,2,-1,-1,16,15,17,-1,-1,-1,-1],"t":[9.266718101290426,9.05390992190079,5.40929108874043,5.2849278041654735,0,5.672425341971495,0,0,10.090748425878532,8.32532955673354,7.651051691178929,0,0,2.0,0,0,8.618385502258606,0,0,9.20088558729989,0,0,10.696098170955864,4.247927513443585,9.293471190846834,0,0,0,0],"m":[1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,14,0,0,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,19,8,5,0,7,0,0,16,13,12,0,0,15,0,0,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-1.480689e-05,0,-9.378831e-05,-0.004437224,0,0,0,0.003643262,-0.002501573,0,-0.001517064,5.20129e-05,0,2.914181e-05,0.004893838,0,-0.002309288,-0.0003880981,0,0,0,0.002527737,-0.000294979,-0.002091427,0.00279867]}]},"p_near":{"link":"logistic","base":-1.7083696958261418,"trees":[{"f":[2,37,0,-1,23,-1,-1,22,25,-1,-1,23,-1,18,-1,-1,19,37,2,20,16,-1,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,7.05,6.5,0,0.5,0,0,12.613886328427084,0.5,0,0,1.5,0,6.434604125275413,0,0,0.5,6.970000000000001,8.9274737487839,0.5,11.407798850221974,0,0,0,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,0,12,0,14,0,0,17,18,19,20,21,0,0,0,0,0,27,0,0],"r":[16,7,4,0,6,0,0,11,10,0,0,13,0,15,0,0,26,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.05134991,0,0.02850011,-0.03064597,0,0,0.03896673,0.1367045,0,0.2659932,0,0.1553289,0.03896673,0,0,0,0,0,-0.02919536,0.06537082,0.1157193,0.1211529,0.1970584,0,0.1136987,0.2807844]},{"f":[2,34,22,-1,23,21,-1,-1,37,-1,-1,37,-1,-1,19,37,2,20,22,-1,22,-1,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,3.5,0,0,7.92,0,0,6.890000000000001,0,0,0.5,6.970000000000001,8.9274737487839,0.5,10.735831902194036,0,14.247293309538719,0,0,0,0,0,1.5,0,0],"m":[1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,0,9,0,0,12,0,0,15,16,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[14,11,4,0,8,7,0,0,10,0,0,13,0,0,26,25,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0.03933735,0,0,0.1317519,0.2439659,0,-0.04342557,0.1180851,0,-0.05040726,-0.006948983,0,0,0,0,0,0.05438295,0,-0.04454419,0.0734199,0.1018529,0.1032284,0.1639021,0,0.1004456,0.2212872]},{"f":[2,37,0,-1,23,-1,-1,22,25,-1,22,-1,-1,23,-1,-1,19,37,2,20,-1,-1,-1,33,-1,-1,23,-1,-1],"t":[5.228690439536267,7.05,6.5,0,0.5,0,0,12.613886328427084,0.5,0,11.876328901062104,0,0,4.5,0,0,0.5,6.970000000000001,6.692122061121539,0.5,0,0,0,26.99635432609064,0,0,1.5,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,0,11,0,0,14,0,0,17,18,19,20,0,0,0,24,0,0,27,0,0],"r":[16,7,4,0,6,0,0,13,10,0,12,0,0,15,0,0,26,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04990029,0,0.02537912,-0.02975497,0,0,0.02542094,0,0.1973522,0.05091504,0,0.1675345,0.02542094,0,0,0,0,-0.03252746,0.090436,0.08535718,0,0.1165523,0.2076601,0,0.08974753,0.1846235]},{"f":[2,34,22,-1,23,-1,37,-1,-1,37,-1,2,-1,15,-1,-1,21,-1,19,2,0,-1,-1,2,18,-1,-1,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,0,7.92,0,0,6.890000000000001,0,2.903677461028802,0,7.118941072723508,0,0,3.5,0,0.5,6.85792738526389,6.5,0,0,10.510264838663748,9.229707804263573,0,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,1,1,1,0,0,1,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,0,12,0,14,0,0,17,0,19,20,21,0,0,24,25,0,0,0,0],"r":[16,9,4,0,6,0,8,0,0,11,0,13,0,15,0,0,18,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0.03268055,0,0.1432333,0,-0.04315071,0.09192994,0,-0.04897868,0,-0.02256726,0,-0.02210076,0.09475449,0,-0.03384799,0,0,0,-0.01155765,0.09359161,0,0,0.121633,0.02142336,0.2204496,0.1539528]},{"f":[2,37,0,21,-1,-1,-1,33,-1,25,-1,-1,21,-1,9,22,2,1,21,-1,-1,-1,-1,17,5,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,23.168926553414675,0,0.5,0,0,3.5,0,0.5,12.15291175877056,9.883406784235138,8.046875,7.5,0,0,0,0,6.864066550153108,0.5,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,0,0,13,0,15,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[12,7,6,5,0,0,0,9,0,11,0,0,14,0,28,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.05334797,-0.02888053,-0.009250229,0,0.1181088,0,0.01730229,0.1059986,0,-0.03293861,0,0,0,0,0,-0.004513102,0.06153231,0.1268148,0.170874,0,0,-0.0365463,0.09638796,0.1425742,-0.05553876]},{"f":[2,32,23,32,-1,22,-1,-1,-1,37,-1,4,-1,15,-1,-1,21,-1,9,22,23,-1,-1,1,-1,17,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,14.158358850608169,0,0,0,6.890000000000001,0,2.903677461028802,0,7.118941072723508,0,0,3.5,0,0.5,12.15291175877056,2.5,0,0,0.512280701754386,0,6.864066550153108,0,0,0],"m":[1,0,1,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,14,0,0,17,0,19,20,21,0,0,24,0,26,0,0,0],"r":[16,9,8,5,0,7,0,0,0,11,0,13,0,15,0,0,18,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0,0.01098206,0,0.07999277,0.1516447,-0.005184722,0,-0.04760259,0,-0.02242395,0,-0.02105073,0.08940546,0,-0.03204336,0,0,0,0.06671569,-0.01711626,0,-0.02781527,0,0.07818029,0.1311688,-0.0550242]},{"f":[2,34,22,-1,23,-1,37,-1,-1,37,21,-1,-1,2,-1,-1,21,-1,19,2,0,-1,-1,15,-1,22,-1,-1,-1],"t":[5.228690439536267,2.5,12.613886328427084,0,1.5,0,7.92,0,0,6.890000000000001,6.5,0,0,2.903677461028802,0,0,3.5,0,0.5,6.85792738526389,5.5,0,0,11.696967526234287,0,16.725912049439096,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,14,0,0,17,0,19,20,21,0,0,24,0,26,0,0,0],"r":[16,9,4,0,6,0,8,0,0,13,12,0,0,15,0,0,18,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,0,0,0.024374,0,0.1072201,0,-0.04385857,0.07284668,0,0,-0.0523185,-0.02657467,0,-0.02090259,0.03661775,0,-0.03116183,0,0,0,-0.0217389,0.06285095,0,0.1006363,0,0.01197789,0.1238998,0.1128529]},{"f":[2,37,0,21,-1,-1,-1,33,-1,25,-1,-1,21,-1,9,22,2,1,21,-1,-1,-1,-1,1,-1,17,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,23.168926553414675,0,0.5,0,0,3.5,0,0.5,12.15291175877056,9.883406784235138,8.046875,7.5,0,0,0,0,0.512280701754386,0,6.864066550153108,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,0,0,13,0,15,16,17,18,19,0,0,0,0,24,0,26,0,0,0],"r":[12,7,6,5,0,0,0,9,0,11,0,0,14,0,28,23,22,21,20,0,0,0,0,25,0,27,0,0,0],"v":[0,0,0,0,-0.05179555,-0.02754394,-0.008589689,0,0.09122144,0,0.0101004,0.08629266,0,-0.0302937,0,0,0,0,0,-0.007914038,0.0470884,0.1010863,0.1319321,0,-0.02933038,0,0.06636024,0.1097159,-0.054343]},{"f":[2,32,23,32,-1,-1,-1,37,21,-1,-1,4,-1,15,-1,-1,21,17,-1,-1,9,22,23,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,0,7.118941072723508,0,0,3.5,1.292481250360578,0,0,0.5,12.15291175877056,2.5,0,0,0.512280701754386,0,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,14,0,0,17,18,0,0,21,22,23,0,0,26,0,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,13,0,15,0,0,20,19,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.007352915,0.08435759,-0.008120347,0,0,-0.05128953,-0.02548231,0,-0.02102126,0,-0.02128574,0.07526447,0,0,0.05232471,-0.05455402,0,0,0,0.05231447,-0.01824147,0,-0.02838343,0.09101017,-0.05387992]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,-1,23,-1,-1,21,17,-1,-1,19,2,37,1,-1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,0,4.5,0,0,3.5,1.292481250360578,0,0,0.5,8.9274737487839,6.05,0.6824793747286149,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,14,0,0,17,18,0,0,21,22,23,24,0,0,0,0,0],"r":[16,9,6,5,0,0,8,0,0,13,12,0,0,15,0,0,20,19,0,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.05078965,-0.02663234,0,-0.02663234,0.006679134,0,0,0.006679134,0.06715705,0,0.08983186,0.006679134,0,0,0.04824232,-0.05400376,0,0,0,0,-0.05275006,0.01385535,0.07156054,0.08616012,0.09100117]},{"f":[2,32,23,22,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,15,-1,-1,21,-1,19,2,37,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,7.118941072723508,0,0,2.5,0,0.5,8.9274737487839,6.775,0,0,2.01685992705101,0,0,0],"m":[1,0,1,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,15,14,0,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.04265729,0.09956031,-0.009130187,0,0,-0.05030417,-0.02468826,0,0,-0.02468826,0.08997507,0,-0.0204007,0.06688811,0,-0.04428007,0,0,0,0.00318417,0.06634833,0,0.05205172,0.1134234,0.08448426]},{"f":[2,37,0,21,-1,-1,-1,22,25,-1,-1,23,-1,-1,21,20,2,-1,-1,-1,9,22,2,-1,-1,1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,0,12.613886328427084,0.5,0,0,4.5,0,0,4.5,1.5,8.612669628062957,0,0,0,0.5,12.15291175877056,9.138550601137563,0,0,0.512280701754386,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,0,0,15,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[14,7,6,5,0,0,0,11,10,0,0,13,0,0,20,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.04982098,-0.02574493,-0.007612804,0,0,0.004189322,0.05897685,0,0.07956892,0.004189322,0,0,0,-0.04984219,0.02885224,0.07324884,0,0,0,0.02625169,0.07944841,0,-0.03038773,0.07947236,-0.05303717]},{"f":[2,32,23,32,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,-1,19,21,-1,2,37,22,-1,17,-1,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,3.9474088816539714,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,0,0.5,2.5,0,8.9274737487839,6.775,10.735831902194036,0,6.686432530056294,0,0,0,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,0,17,18,0,20,21,22,0,24,0,0,0,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,15,14,0,0,0,28,19,0,27,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0.003079002,0.0653839,-0.00994937,0,0,-0.04935017,-0.02397531,0,0,-0.02397531,0.08162515,0.03232094,0,0,-0.05502644,0,0,0,0.02965982,0,-0.04723008,0.01395375,0.05799928,0.07048127,0.07531179]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,22,-1,-1,-1,19,21,-1,1,2,21,-1,-1,1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,11.876328901062104,0,0,0,0.5,2.5,0,6.147321428571429,8.9274737487839,7.5,0,0,2.01685992705101,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,13,0,0,0,17,18,0,20,21,22,0,0,25,0,0,0,0],"r":[16,9,6,5,0,0,8,0,0,15,12,0,14,0,0,0,28,19,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04887828,-0.02495121,0,-0.02495121,0.002258802,0,0,0.002258802,0,0.1010497,0.01253451,0.06378338,0,0,-0.05460514,0,0,0,-0.01766827,0.03432182,0,0.03994211,0.09355377,0.07824164,0.07124798]},{"f":[2,32,23,22,-1,-1,24,-1,-1,37,21,-1,-1,23,-1,22,-1,-1,19,21,-1,2,1,21,-1,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,9.5,0,0,6.890000000000001,6.5,0,0,0.5,0,17.094543568094064,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,0,0,0,0,0],"m":[1,0,1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,14,0,16,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,9,6,5,0,0,8,0,0,13,12,0,0,15,0,17,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0.03254898,0.08262798,0,-0.04053136,0.05733405,0,0,-0.04841712,-0.02320139,0,0.02090364,0,-0.02320139,0.080959,0,0,-0.05420203,0,0,0,-0.003615538,0.03817374,0.07340287,0.09828975,0.0675898]},{"f":[2,37,0,21,-1,-1,16,-1,-1,22,25,-1,-1,-1,19,21,-1,2,1,21,2,-1,-1,2,-1,-1,-1,-1,-1],"t":[5.228690439536267,7.05,6.5,6.5,0,0,6.044394119358453,0,0,12.613886328427084,0.5,0,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,8.767235038274581,0,0,6.4675825248018475,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,0,0,0,15,16,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[14,9,6,5,0,0,8,0,0,13,12,0,0,0,28,17,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.04795167,-0.02415789,0,-0.02415789,0.0008313582,0,0,0.0008313582,0.04731106,0.05767906,0,0,-0.05381553,0,0,0,0,-0.01684763,0.03015068,0,0.01007363,0.0501089,0.06910251,0.09257225,0.06427027]},{"f":[2,32,23,22,-1,-1,-1,37,21,-1,-1,4,22,-1,-1,-1,21,20,2,-1,-1,-1,1,21,-1,-1,22,-1,-1],"t":[5.228690439536267,10.019277334827486,3.5,14.158358850608169,0,0,0,6.890000000000001,6.5,0,0,2.903677461028802,17.094543568094064,0,0,0,4.5,1.5,8.612669628062957,0,0,0,0.6824793747286149,7.5,0,0,12.121442107629743,0,0],"m":[1,0,1,1,0,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,0,0,17,18,19,0,0,0,23,24,0,0,27,0,0],"r":[16,7,6,5,0,0,0,11,10,0,0,15,14,0,0,0,22,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,0.02906729,0.07601426,-0.01245716,0,0,-0.04749573,-0.02249502,0,0,-0.02249502,0.06858353,0.02841536,0,0,0,-0.04812345,0.02159476,0.05082435,0,0,-0.0550579,0.05747865,0,0.02587045,0.0627123]},{"f":[2,0,21,-1,19,-1,-1,15,15,35,-1,-1,19,15,-1,-1,-1,-1,19,21,-1,2,1,21,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0.5,0,0,9.92481250360578,6.169925001442312,1.5,0,0,2.5,7.118941072723508,0,0,0,0,0.5,2.5,0,10.510264838663748,6.147321428571429,7.5,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,13,14,0,0,0,0,19,20,0,22,23,24,0,0,0,0,0],"r":[18,7,4,0,6,0,0,17,12,11,0,0,16,15,0,0,0,0,28,21,0,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.04703249,0,-0.01259492,-0.04356861,0,0,0,0.0632249,-0.01259492,0,0,0.08957084,0.04209667,-0.01259492,-0.01259492,0,0,-0.05327294,0,0,0,-0.003818762,0.0314443,0.06287891,0.08533607,0.05880398]},{"f":[2,0,21,-1,-1,15,23,22,15,-1,25,-1,-1,-1,-1,-1,19,21,-1,2,22,20,-1,-1,-1,-1,23,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0,9.92481250360578,3.5,14.158358850608169,6.0,0,0.5,0,0,0,0,0,0.5,2.5,0,8.9274737487839,16.725912049439096,0.5,0,0,0,0,1.5,0,0],"m":[1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,11,0,0,0,0,0,17,18,0,20,21,22,0,0,0,0,27,0,0],"r":[16,5,4,0,0,15,14,13,10,0,12,0,0,0,0,0,26,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,-0.04657328,-0.01484132,0,0,0,0,0.002813312,0,-0.01274856,0.05220433,0.06986092,-0.008226951,-0.01274856,0,0,-0.05292087,0,0,0,-0.007739295,0.03141063,0.05582555,0.05229711,0,0.01389942,0.06341743]},{"f":[2,0,21,-1,19,-1,-1,15,23,22,-1,-1,-1,-1,32,21,20,2,-1,-1,-1,9,2,18,-1,-1,-1,-1,-1],"t":[5.228690439536267,6.5,6.5,0,0.5,0,0,9.92481250360578,3.5,14.158358850608169,0,0,0,0,6.474913355379556,5.5,1.5,8.612669628062957,0,0,0,0.5,9.138550601137563,8.03615005796417,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,0,1,0,1,1,0,0,0,1,1,1,0,0,0,0,0],"l":[1,2,3,0,5,0,0,8,9,10,0,0,0,0,15,16,17,18,0,0,0,22,23,24,0,0,0,0,0],"r":[14,7,4,0,6,0,0,13,12,11,0,0,0,0,28,21,20,19,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.04611315,0,-0.01283797,-0.04271988,0,0,0,0.02233582,0.06681988,-0.007861901,-0.01283797,0,0,0,0,-0.03767678,0.0198463,0.04844246,0,0,0,0.03390737,-0.008316357,0.06128327,-0.05148846,0.06797863]},{"f":[2,37,21,-1,1,-1,15,-1,-1,22,25,-1,-1,-1,22,2,24,15,5,-1,23,-1,-1,-1,-1,-1,1,-1,-1],"t":[5.228690439536267,7.05,2.5,0,4.208333333333334,0,9.92481250360578,0,0,12.613886328427084,0.5,0,0,0,13.87915215758784,10.510264838663748,8.5,10.457380879072534,0.5,0,2.5,0,0,0,0,0,0.6824793747286149,0,0],"m":[1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,0,0,0,15,16,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[14,9,4,0,6,0,8,0,0,13,12,0,0,0,26,25,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.05276047,0,-0.03525455,0,-0.001089294,-0.03602129,0,0,-0.001089294,0.04007175,0.04734464,0,0,0,0,0,-0.04191718,0,0.02412223,-0.03027988,-0.03890672,0.06477139,0.07493377,0,-0.04498801,0.05252825]},{"f":[2,0,21,-1,-1,15,23,22,-1,-1,37,-1,-1,-1,21,-1,32,9,22,2,1,21,-1,-1,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,1.5,14.158358850608169,0,0,9.34,0,0,0,2.5,0,6.474913355379556,0.5,13.64082212194147,10.510264838663748,6.610817307692308,7.5,0,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,0,0,0,15,0,17,18,19,20,21,22,0,0,0,0,0,0,0],"r":[14,5,4,0,0,13,10,9,0,0,12,0,0,0,16,0,28,27,26,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.04556921,-0.01538865,0,0,0,0.0223368,0.07318662,0,-0.01451039,0.07066788,-0.01451039,0,-0.04387378,0,0,0,0,0,0,-0.01631393,0.01887548,0.0361375,0.07320277,0.04513764,-0.05214398,0.06655788]},{"f":[2,0,21,-1,-1,15,19,37,15,-1,-1,-1,-1,-1,21,-1,32,2,1,-1,16,-1,22,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,2.5,8.05,6.044394119358453,0,0,0,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,0,0,0,15,0,17,18,19,0,21,0,23,0,0,26,0,0,0],"r":[14,5,4,0,0,13,12,11,10,0,0,0,0,0,16,0,28,25,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.04511189,-0.01481516,0,0,0,0,-0.01453377,0.03593709,0.06755506,-0.01453377,-0.01453377,0,-0.0432549,0,0,0,-0.05111178,0,-0.02900328,0,0.01070052,0.04107791,0,0.07094377,0.02969833,0.06400766]},{"f":[2,0,21,-1,-1,23,22,-1,-1,17,-1,-1,21,-1,32,2,1,-1,16,-1,22,3,-1,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,1.5,15.212004812197344,0,0,7.6181566135214025,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0.023705703211517166,0,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,19,0,21,22,0,0,0,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,28,25,18,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,-0.04465106,-0.01454237,0,0,0.01293482,0.06945275,0,-0.01454237,0.05480059,0,-0.04263479,0,0,0,-0.05055317,0,-0.02806512,0,0,-0.05852134,0.01383713,0.03908987,0,0.06799596,0.02814566,0.06164277]},{"f":[2,0,21,-1,-1,15,23,22,-1,-1,37,-1,-1,-1,21,-1,32,2,1,-1,16,-1,22,1,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,8.618385502258606,1.5,14.158358850608169,0,0,9.34,0,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,6.610817307692308,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,0,0,0,15,0,17,18,19,0,21,0,23,24,0,0,0,0,0],"r":[14,5,4,0,0,13,10,9,0,0,12,0,0,0,16,0,28,27,20,0,22,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04418606,-0.01449264,0,0,0,0.02158059,0.06753908,0,-0.01449264,0.06384168,-0.01449264,0,-0.042013,0,0,0,-0.04999921,0,-0.02714755,0,0,-0.0003798591,0.03384598,0.03721916,0.04321895,0.05943802]},{"f":[2,0,21,-1,-1,15,-1,-1,21,-1,1,21,-1,-1,22,33,21,22,-1,22,-1,-1,15,23,-1,-1,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,9.92481250360578,0,0,2.5,0,0.6824793747286149,7.5,0,0,13.64082212194147,34.58978958281367,4.5,12.199665042528586,0,12.637102544927284,0,0,11.696967526234287,2.5,0,0,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,0,11,12,0,0,15,16,17,18,0,20,0,0,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,10,0,14,13,0,0,28,27,22,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04371632,-0.01440975,0,0.01971736,-0.01440975,0,-0.04138908,0,0,-0.04938147,0.04136651,0,0,0,0,-0.03738437,0,0.03851791,-0.04357449,0,0,0.02984819,-0.01322046,-0.03759537,0.07082867,0.04363319]},{"f":[2,0,21,-1,-1,23,22,-1,-1,17,-1,-1,21,-1,32,2,1,-1,16,-1,22,-1,2,-1,-1,3,-1,-1,-1],"t":[4.95344529780426,6.5,6.5,0,0,1.5,15.212004812197344,0,0,7.6181566135214025,0,0,2.5,0,6.474913355379556,8.9274737487839,0.8611111111111112,0,3.584962500721156,0,13.87915215758784,0,6.139515068224824,0,0,0.9640400250156348,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,0,13,0,15,16,17,0,19,0,21,0,23,0,0,26,0,0,0],"r":[12,5,4,0,0,9,8,0,0,11,0,0,14,0,28,25,18,0,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.04324129,-0.0143338,0,0,0.01096677,0.06527487,0,-0.0143338,0.05075265,0,-0.0407627,0,0,0,-0.04925367,0,-0.0264701,0,0.008459378,0,0.05214475,0.01718813,0,0.06282177,0.0233736,0.05593669]},{"f":[21,-1,2,0,16,-1,23,-1,-1,-1,1,2,-1,-1,22,1,21,-1,2,-1,-1,-1,18,33,-1,-1,15,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0.5,0,0,0,1.000936329588015,6.8391900325545585,0,0,12.121442107629743,12.028571428571428,4.5,0,9.299206200930918,0,0,0,7.252657678068118,19.167172160526107,0,0,2.0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,0,0,15,16,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,10,9,6,0,8,0,0,0,14,13,0,0,22,21,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.04920136,0,0,0,-0.04326393,0,-0.002404876,-0.02615403,0.0210241,0,0,-0.04093877,0.02401701,0,0,0,-0.04815723,0,0.007642388,0.0671335,0.05389912,0,0,0.02022347,0.04703901,0,0.06250508,-0.01189632]},{"f":[21,-1,2,0,16,-1,-1,-1,1,21,-1,-1,32,2,16,-1,18,3,22,-1,-1,16,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0,0,0.9444444444444444,7.5,0,0,6.474913355379556,8.9274737487839,3.584962500721156,0,7.252657678068118,0.40731707317073174,16.725912049439096,0,0,6.930737337562887,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,12,11,0,0,28,25,16,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04889217,0,0,0,-0.04268026,-0.01582361,0.01988705,0,0,-0.04756305,0.03886991,0,0,0,-0.02312053,0,0,0,-0.01464019,0.03094376,0,0.01424678,0.04244139,-0.009668511,0,0.0699555,0.01919585,0.04943752]},{"f":[21,-1,2,1,16,-1,-1,15,23,22,-1,1,-1,-1,17,-1,-1,-1,32,2,1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,0,9.92481250360578,3.5,14.03140051979143,0,5.015625,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,0.8611111111111112,0,13.64082212194147,10.457380879072534,0,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,12,0,0,15,0,0,0,19,20,21,0,23,24,0,0,0,0,0],"r":[2,0,18,7,6,0,0,17,14,11,0,13,0,0,16,0,0,0,28,27,22,0,26,25,0,0,0,0,0],"v":[0,-0.04858154,0,0,0,-0.03907538,-0.004557052,0,0,0,0.01218236,0,0.2859055,0.03979016,0,-0.03248448,0.05093697,-0.02404769,0,0,0,-0.0417707,0,0,0.01107016,-0.04122675,0.03098393,0.06204166,0.05156832]},{"f":[21,-1,2,0,16,-1,-1,-1,1,21,-1,-1,32,2,16,-1,18,3,22,-1,-1,16,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,2.0,0,0,0,0.9444444444444444,7.5,0,0,6.474913355379556,8.9274737487839,3.584962500721156,0,7.252657678068118,0.40731707317073174,16.725912049439096,0,0,6.930737337562887,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,12,11,0,0,28,25,16,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04826885,0,0,0,-0.04166863,-0.0148518,0.01849157,0,0,-0.04651086,0.03662241,0,0,0,-0.02245313,0,0,0,-0.01475288,0.02819657,0,0.01276393,0.03942293,-0.009015289,0,0.06700576,0.01746681,0.0462802]},{"f":[21,-1,2,1,-1,15,23,2,4,22,-1,-1,-1,-1,17,-1,-1,-1,32,2,1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,0.8611111111111112,0,13.64082212194147,10.457380879072534,0,0,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,0,0,0,15,0,0,0,19,20,21,0,23,24,0,0,0,0,0],"r":[2,0,18,5,0,17,14,13,12,11,0,0,0,0,16,0,0,0,28,27,22,0,26,25,0,0,0,0,0],"v":[0,-0.04795351,0,0,-0.03081808,0,0,0,0,0,0.004400261,0.04639005,0.06592963,-0.05208316,0,-0.03149088,0.04742329,-0.02308424,0,0,0,-0.04014378,0,0,0.01014232,-0.04010457,0.02833,0.05930186,0.04859849]},{"f":[21,-1,2,0,-1,-1,1,-1,33,2,16,18,-1,15,-1,-1,18,3,-1,-1,-1,3,-1,-1,25,22,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.5,0,0,0.9444444444444444,0,20.752541988539424,8.9274737487839,7.118941072723508,3.6427011094311244,0,5.78135971352466,0,0,8.03615005796417,0.33771929824561403,0,0,0,0.9640400250156348,0,0,0.5,12.121442107629743,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,0,14,0,0,17,18,0,0,0,22,0,0,25,26,0,0,0],"r":[2,0,6,5,0,0,8,0,24,21,16,13,0,15,0,0,20,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,-0.04763494,0,0,-0.01909251,0.01722565,0,-0.03247331,0,0,0,0,-0.02932899,0,0.02323193,-0.03170618,0,0,0.0005019516,0.03863599,-0.01375857,0,0.06437264,0.01589839,0,0,-0.03414619,0.03560709,0.05658904]},{"f":[21,-1,2,1,16,-1,-1,15,23,2,4,22,-1,-1,-1,-1,17,-1,-1,-1,32,2,1,-1,2,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,7.723832212141186,0,0,0,6.337096134072842,8.9274737487839,0.8611111111111112,0,7.754129516000594,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,12,0,0,0,0,17,0,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,20,7,6,0,0,19,16,15,14,13,0,0,0,0,18,0,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.04731258,0,0,0,-0.03738272,-0.001975644,0,0,0,0,0,0.004700065,0.04450977,0.06103039,-0.05044497,0,-0.03050973,0.04511656,-0.0223045,0,0,0,-0.04683938,0,0.01234415,-0.04777094,0.03164205,0.04435139]},{"f":[21,-1,2,1,-1,15,2,19,-1,-1,-1,-1,1,-1,33,2,16,18,-1,-1,18,3,-1,-1,-1,3,-1,-1,-1],"t":[2.5,0,2.453445297804259,4.54375,0,9.92481250360578,1.792481250360578,2.5,0,0,0,0,0.9444444444444444,0,20.752541988539424,8.9274737487839,7.118941072723508,3.6427011094311244,0,0,8.03615005796417,0.33771929824561403,0,0,0,0.9640400250156348,0,0,0],"m":[0,0,1,1,0,1,1,1,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,0,0,13,0,15,16,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,12,5,0,11,10,9,0,0,0,0,14,0,28,25,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.04698592,0,0,-0.02924094,0,0,0,0.02662478,-0.03210394,-0.0240471,-0.02950968,0,-0.0313425,0,0,0,0,-0.02842189,0.006977665,0,0,-0.0001296144,0.03595058,-0.01357567,0,0.06185342,0.01354981,0.03848262]},{"f":[21,-1,2,1,16,-1,21,-1,-1,15,23,2,4,22,-1,-1,-1,-1,-1,-1,32,2,15,22,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,0,7.5,0,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0,0,0,0,0,6.474913355379556,10.510264838663748,9.92481250360578,13.64082212194147,0,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,12,13,14,0,0,0,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,9,6,0,8,0,0,19,18,17,16,15,0,0,0,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04665443,0,0,0,-0.03634106,0,-0.02485282,0.05085381,0,0,0,0,0,0.003855855,0.04203111,0.05666671,-0.04962,-0.01546787,-0.02106922,0,0,0,0,0.006719832,0.03069862,-0.01349788,0.05591427,0.04342491]},{"f":[21,-1,2,1,-1,15,23,2,4,22,-1,25,-1,-1,-1,-1,17,-1,-1,-1,32,2,15,22,-1,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,0,9.92481250360578,3.5,3.39067985676233,2.453445297804259,14.03140051979143,0,0.5,0,0,0,0,7.723832212141186,0,0,0,6.474913355379556,10.510264838663748,10.043027283594547,13.64082212194147,0,0,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,0,1,0,0,0,0,1,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,0,17,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,5,0,19,16,15,14,11,0,13,0,0,0,0,18,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04631764,0,0,-0.02777808,0,0,0,0,0,0.003655738,0,0.07466978,0.01109733,0.0538774,-0.04845789,0,-0.02898236,0.04262988,-0.0203451,0,0,0,0,0.006355353,0.02938993,-0.01301426,0.05445227,0.04209795]},{"f":[21,-1,2,37,16,-1,23,-1,-1,-1,1,21,-1,-1,37,22,21,32,-1,17,-1,-1,23,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,8.05,2.0,0,0.5,0,0,0,0.9444444444444444,7.5,0,0,9.075,13.64082212194147,4.5,4.167695177346962,0,11.207174616271544,0,0,3.5,0,0,7.252657678068118,0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,0,0,15,16,17,18,0,20,0,0,23,0,0,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,14,13,0,0,28,25,22,19,0,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.04597508,0,0,0,-0.03876072,0,0.002182574,-0.02193025,0.02006405,0,0,-0.04453599,0.03415506,0,0,0,0,0.02791156,0,-0.0378967,0.03107904,0,0.01510551,-0.02715591,0,0.02842011,-0.02093516,0.0495966]},{"f":[21,-1,2,1,21,-1,-1,15,2,19,37,-1,-1,-1,-1,-1,1,-1,33,22,21,16,-1,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,4.54375,6.5,0,0,9.92481250360578,1.792481250360578,2.5,8.05,0,0,0,0,0,0.9444444444444444,0,34.58978958281367,13.415167820124083,4.5,5.78135971352466,0,0,0,7.519373778108656,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,0,0,0,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,16,7,6,0,0,15,14,13,12,0,0,0,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.0456263,0,0,0,-0.04352707,-0.007658946,0,0,0,0,0.0144468,0.05367531,-0.03055248,-0.02329862,-0.02716471,0,-0.03018577,0,0,0,0,0.005269386,-0.03518827,0.01191654,0,0.02718519,-0.01751452,0.0539777]},{"f":[21,-1,2,1,16,4,-1,-1,21,-1,-1,15,16,16,-1,-1,16,22,-1,15,-1,-1,23,-1,-1,-1,33,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,9.92481250360578,4.0437314206251695,0,0,7.5,0,0,9.92481250360578,6.044394119358453,5.044394119358453,0,0,7.118941072723508,12.121442107629743,0,6.584962500721156,0,0,3.5,0,0,0,29.297756302860655,0,0],"m":[0,0,1,1,1,1,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,9,0,0,12,13,14,0,0,17,18,0,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,11,8,7,0,0,10,0,0,25,16,15,0,0,22,19,0,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04527088,0,0,0,0,-0.03795573,0.05443006,0,-0.02240816,0.05033973,0,0,0,0.01165059,-0.05502919,0,0,-0.03418476,0,0.05688752,0.1828984,0,0.01769152,-0.05311149,-0.01854299,0,0.01127519,0.05012653]},{"f":[21,-1,37,2,16,-1,23,32,-1,-1,21,-1,17,-1,-1,1,2,-1,-1,15,-1,21,-1,-1,21,4,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,2.0,0,0.5,7.014643613484123,0,0,6.5,0,1.792481250360578,0,0,0.6824793747286149,6.8391900325545585,0,0,10.376125389276174,0,4.5,0,0,9.5,1.292481250360578,0,0,0],"m":[0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,0,13,0,0,16,17,0,0,20,0,22,0,0,25,26,0,0,0],"r":[2,0,24,15,6,0,10,9,0,0,12,0,14,0,0,19,18,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,-0.04490843,0,0,0,-0.03737126,0,0,0.007712535,-0.04573087,0,-0.03647454,0,0.01384044,-0.0199533,0,0,-0.05361513,0.01188741,0,0.01384044,0,-0.05306828,0.004451765,0,0,0.01384044,0.0348549,0.08400629]},{"f":[21,-1,2,22,1,16,-1,23,15,22,-1,-1,-1,-1,25,-1,-1,-1,1,-1,33,22,21,-1,-1,18,-1,-1,-1],"t":[2.5,0,2.453445297804259,17.094543568094064,8.375,2.0,0,0.5,10.501837184902296,14.03140051979143,0,0,0,0,2.5,0,0,0,0.9444444444444444,0,34.58978958281367,13.415167820124083,4.5,0,0,7.519373778108656,0,0,0],"m":[0,0,1,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,0,0,0,15,0,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[2,0,18,17,14,7,0,13,12,11,0,0,0,0,16,0,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,-0.04453855,0,0,0,0,-0.04285045,0,0,0,-0.0124953,0.05135107,0.08373111,-0.02359198,0,0.02722831,-0.02621211,0.05897048,0,-0.02875853,0,0,0,-0.01558627,0.009967963,0,0.02480846,-0.01703299,0.05086243]},{"f":[21,-1,2,1,21,-1,-1,15,16,16,-1,-1,16,22,-1,15,-1,-1,23,-1,-1,-1,32,2,15,-1,-1,-1,-1],"t":[2.5,0,4.80643424864552,4.208333333333334,6.5,0,0,9.92481250360578,6.044394119358453,5.044394119358453,0,0,7.118941072723508,12.121442107629743,0,6.584962500721156,0,0,3.5,0,0,0,6.474913355379556,10.510264838663748,9.92481250360578,0,0,0,0],"m":[0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,0,13,14,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,7,6,0,0,21,12,11,0,0,18,15,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.0441609,0,0,0,-0.03835686,-0.007375393,0,0,0,0.01066815,-0.05425878,0,0,-0.0329907,0,0.05275929,0.1615261,0,0.01679704,-0.05269746,-0.01720838,0,0,0,0.0108168,-0.01361279,0.05181233,0.03676989]},{"f":[21,-1,37,4,16,0,16,-1,-1,23,-1,22,-1,-1,-1,1,21,17,-1,-1,-1,-1,21,4,19,-1,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,10.501837184902296,4.5,5.169925001442312,0,0,0.5,0,17.094543568094064,0,0,0,2.01685992705101,7.5,10.269270673170688,0,0,0,0,9.5,1.292481250360578,0.5,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,12,0,0,0,16,17,18,0,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,15,14,9,8,0,0,11,0,13,0,0,0,21,20,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04377516,0,0,0,0,0,-0.02059419,-0.0431916,0,0.00839848,0,-0.02059419,0.01230817,0.01230817,0,0,0,-0.03625751,0.01230817,0.01060766,0.01230817,0,0,0,0.05215809,0.01230817,0.03183008,0.07773629]},{"f":[21,-1,2,16,-1,16,2,-1,-1,15,-1,16,15,-1,-1,18,21,-1,-1,-1,37,1,-1,22,21,-1,-1,-1,-1],"t":[2.5,0,2.453445297804259,2.0,0,5.169925001442312,1.792481250360578,0,0,6.108524456778169,0,6.832890014164741,6.569855608330948,0,0,2.160964047443681,7.5,0,0,0,9.075,0.9444444444444444,0,12.199665042528586,4.5,0,0,0,0],"m":[0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,12,13,0,0,16,17,0,0,0,21,22,0,24,25,0,0,0,0],"r":[2,0,20,5,0,9,8,0,0,11,0,15,14,0,0,19,18,0,0,0,28,23,0,27,26,0,0,0,0],"v":[0,-0.043381,0,0,-0.0324052,0,0,0.03908272,-0.03865623,0,-0.05145791,0,0,0.01360851,0.09799413,0,0,-0.004173689,0.06160731,-0.02526535,0,0,-0.02741526,0,0,-0.03178637,0.006311698,0.01654803,0.04187049]},{"f":[21,-1,37,2,16,-1,23,32,-1,-1,21,-1,17,-1,17,-1,-1,15,-1,-1,21,4,19,-1,-1,24,-1,-1,-1],"t":[2.5,0,8.05,3.084962500721156,2.0,0,0.5,7.014643613484123,0,0,6.5,0,1.792481250360578,0,8.496347487943302,0,0,8.618385502258606,0,0,9.5,1.292481250360578,0.5,0,0,9.5,0,0,0],"m":[0,0,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,0,11,0,13,0,15,0,0,18,0,0,21,22,23,0,0,26,0,0,0],"r":[2,0,20,17,6,0,10,9,0,0,12,0,14,0,16,0,0,19,0,0,28,25,24,0,0,27,0,0,0],"v":[0,-0.04297815,0,0,0,-0.03491544,0,0,0.007463702,-0.04469251,0,-0.03391638,0,0.01141863,0,-0.039468,0.01141863,0,0.01078732,-0.009093951,0,0,0,0.05048926,0.01141863,0,0.01981933,0.05550001,0.07445053]},{"f":[21,-1,2,37,21,15,15,-1,-1,-1,-1,37,22,-1,16,-1,-1,21,4,-1,-1,-1,3,18,-1,-1,17,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,5.087462841250339,3.169925001442312,0,0,0,0,8.05,12.199665042528586,0,6.044394119358453,0,0,9.5,1.292481250360578,0,0,0,0.9640400250156348,9.011202871982467,0,0,10.460454442916898,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,15,0,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,22,11,10,9,8,0,0,0,0,17,14,0,16,0,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.04256635,0,0,0,0,0,-0.05131789,-0.004097161,-0.04841785,-0.004097161,0,0,-0.004097161,0,-0.004097161,0.01167757,0,0,0.01167757,0.02786755,0.07381561,0,0,0.05948354,-0.01135936,0,-0.001675911,0.03707276]},{"f":[21,-1,4,37,21,-1,20,-1,-1,18,16,-1,21,-1,-1,-1,1,22,-1,-1,1,-1,34,22,21,-1,-1,-1,-1],"t":[2.5,0,3.084962500721156,6.890000000000001,6.5,0,0.5,0,0,2.160964047443681,2.807354922057604,0,3.5,0,0,0,2.01685992705101,12.457649623491124,0,0,2.969223484848485,0,3.5,12.121442107629743,5.5,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,0,13,0,0,0,17,18,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,16,9,6,0,8,0,0,15,12,0,14,0,0,0,20,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.04214538,0,0,0,-0.0296257,0,-0.005772898,-0.03525213,0,0,-0.005772898,0,-0.005772898,0.03645334,-0.005772898,0,0,0.002012647,-0.03589173,0,0.07415715,0,0,0,-0.02932495,0.00852195,0.01846851,0.05320863]},{"f":[21,-1,2,37,21,-1,-1,16,35,22,-1,-1,-1,22,25,-1,17,-1,-1,15,3,-1,-1,22,-1,-1,3,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,1.5,12.613886328427084,0,0,0,12.199665042528586,0.5,0,9.26326872279839,0,0,7.118941072723508,0.5803571428571428,0,0,12.738408632106918,0,0,0.9640400250156348,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,0,0,14,15,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[2,0,26,7,6,0,0,13,12,11,0,0,0,19,16,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,-0.04171503,0,0,0,-0.03356613,-0.00385435,0,0,0,-0.00385435,0.03885311,-0.00385435,0,0,-0.00385435,0,0.05082708,0.0047808,0,0,0.005784174,0.06458847,0,0.05119786,0.004693344,0,0.04619199,0.01033541]},{"f":[21,-1,4,16,0,-1,-1,16,22,-1,-1,-1,1,21,-1,-1,1,-1,34,22,21,1,-1,-1,-1,2,-1,-1,-1],"t":[2.5,0,3.084962500721156,11.620219825507487,4.5,0,0,11.868050853745158,11.683871801979876,0,0,0,2.01685992705101,7.5,0,0,2.969223484848485,0,3.5,12.121442107629743,5.5,5.546070460704607,0,0,0,6.483793263648036,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,0,13,14,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[2,0,12,7,6,0,0,11,10,0,0,0,16,15,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.04127515,0,0,0,-0.02706231,-0.002621483,0,0,0.1443441,0.03665415,0.000915704,0,0,-0.02502298,0.009438208,0,0.0706893,0,0,0,0,-0.009956547,-0.06767161,0.008048767,0,0.02466861,0.001073646,0.0509581]},{"f":[21,-1,2,37,21,15,-1,-1,-1,16,35,22,-1,-1,-1,22,25,-1,17,-1,-1,15,3,-1,-1,-1,22,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,5.087462841250339,0,0,0,6.044394119358453,1.5,12.613886328427084,0,0,0,12.199665042528586,0.5,0,9.26326872279839,0,0,7.118941072723508,0.5803571428571428,0,0,0,11.681676585676911,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,12,0,0,0,16,17,0,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,9,8,7,0,0,0,15,14,13,0,0,0,21,18,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.04082558,0,0,0,0,-0.01443879,-0.04763503,-0.003833449,0,0,0,-0.003833449,0.03675042,-0.003833449,0,0,-0.003833449,0,0.04852314,0.004139135,0,0,0.005120311,0.06168576,0.008175539,0,0.001021283,0.03529796]},{"f":[21,-1,2,37,16,-1,1,-1,4,4,23,-1,21,-1,-1,-1,15,-1,22,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,8.05,2.321928094887362,0,0.4794055810397554,0,3.084962500721156,1.792481250360578,0.5,0,6.5,0,0,0,10.457380879072534,0,16.725912049439096,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,11,0,13,0,0,0,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,21,6,0,8,0,16,15,12,0,14,0,0,0,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04993984,0,0,0,-0.0320502,0,-0.05364995,0,0,0,0.006711889,0,-0.03269655,0.0005045657,-0.0288941,0,0.006711889,0,-0.02810975,0.006711889,0.01801191,0,0,0,0.02271667,-0.04546493,0.04741198,0.07439243]},{"f":[21,-1,2,37,21,22,-1,-1,-1,37,22,-1,16,-1,-1,21,33,-1,27,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,4.470000000000001,6.5,16.819976309430963,0,0,0,8.05,12.199665042528586,0,6.044394119358453,0,0,9.5,13.895530557358477,0,0.5,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,13,0,0,16,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,9,8,7,0,0,0,15,12,0,14,0,0,21,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04985466,0,0,0,0,-0.03749304,-0.004439844,-0.004439844,0,0,-0.004439844,0,-0.004439844,0.008773839,0,0,0.05255991,0,0.008773839,0.04488811,0.06799054,0,0,0,0.02162829,-0.0435164,0.04626432,0.07398263]},{"f":[21,-1,2,1,21,-1,22,22,-1,-1,-1,4,37,2,-1,-1,-1,34,22,22,-1,-1,2,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,2.532051282051282,6.5,0,7.082728314749225,6.671801068872866,0,0,0,3.084962500721156,8.05,1.792481250360578,0,0,0,3.5,13.747003318603246,12.637102544927284,0,0,6.483793263648036,0,0,0,1.5,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,1,1,0,0,0,1,1,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,8,0,0,0,12,13,14,0,0,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,11,6,0,10,9,0,0,0,17,16,15,0,0,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04976949,0,0,0,-0.04299488,0,0,-0.03097717,0.08727394,-0.007417513,0,0,0,-0.004850413,-0.03354984,0.0128789,0,0,0,0.003078168,-0.04929808,0,0.02815026,-0.007554823,0.04817462,0,0.01526761,0.07361517]},{"f":[21,-1,2,16,26,-1,-1,1,-1,15,22,21,-1,-1,-1,15,-1,1,21,-1,-1,15,16,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,2.584962500721156,0.5,0,0,0.4794055810397554,0,5.78135971352466,16.819976309430963,4.5,0,0,0,6.169925001442312,0,4.92309205350118,7.5,0,0,6.857980995127572,6.6865005271832185,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,12,0,0,0,16,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,7,6,0,0,9,0,15,14,13,0,0,0,17,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.04968413,0,0,0,0.01309162,-0.03546466,0,-0.05299355,0,0,0,-0.006853767,0.02121278,0.1357665,0,-0.04971195,0,0,-0.02951811,0.002683889,0,0,0.02933642,0.1377346,0.005932405,0,0.01454233,0.07328184]},{"f":[21,-1,2,1,-1,16,26,-1,-1,15,22,-1,-1,15,-1,1,-1,18,15,19,-1,-1,2,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,0,5.78135971352466,16.819976309430963,0,0,6.169925001442312,0,4.92309205350118,0,8.729619742794132,6.857980995127572,1.5,0,0,1.292481250360578,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,0,0,14,0,16,0,18,19,20,0,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,13,12,0,0,15,0,17,0,25,22,21,0,0,24,0,0,0,28,0,0],"v":[0,-0.04959839,0,0,-0.05228592,0,0,0.01262366,-0.03453981,0,0,0.009208582,0.1212337,0,-0.04848066,0,-0.01171547,0,0,0,0.1280567,0.02248596,0,-0.0116267,0.01806142,-0.0412347,0,0.0138511,0.07297461]},{"f":[21,-1,2,1,-1,16,26,2,-1,-1,21,-1,-1,37,4,4,-1,-1,15,-1,22,-1,-1,16,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,1.792481250360578,0,0,8.5,0,0,8.05,3.084962500721156,1.792481250360578,0,0,10.457380879072534,0,16.725912049439096,0,0,5.044394119358453,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,0,0,1,1,1,0,0,1,0,1,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,0,14,15,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[2,0,26,5,0,13,10,9,0,0,12,0,0,23,18,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,-0.04951208,0,0,-0.05196165,0,0,0,-0.01487707,0.04028857,0,-0.03830456,0.03716279,0,0,0,-0.002612464,-0.0267502,0,0.008896182,0,-0.02690315,0.008896182,0,0.06076272,0.01241319,0,0.01319227,0.07268553]},{"f":[21,-1,2,1,-1,16,26,2,-1,-1,-1,37,4,4,-1,-1,15,-1,22,-1,-1,-1,20,2,4,-1,-1,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,1.792481250360578,0,0,0,8.05,3.084962500721156,1.792481250360578,0,0,10.457380879072534,0,16.725912049439096,0,0,0,1.5,10.510264838663748,9.266718101290426,0,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,1,1,1,0,0,1,0,1,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,0,12,13,14,0,0,17,0,19,0,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,11,10,9,0,0,0,21,16,15,0,0,18,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,-0.04942501,0,0,-0.05163247,0,0,0,-0.01428203,0.03879967,-0.03290268,0,0,0,-0.002486539,-0.02592857,0,0.008544841,0,-0.02580445,0.008544841,0.01684772,0,0,0,0.01788594,-0.04408516,0.0437903,0.07240675]},{"f":[21,-1,2,1,-1,2,22,16,-1,16,22,-1,21,-1,-1,15,-1,22,-1,32,-1,-1,-1,34,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,0.6824793747286149,0,2.453445297804259,17.094543568094064,2.0,0,5.169925001442312,13.747003318603246,0,4.5,0,0,6.108524456778169,0,6.930301023985015,0,6.337096134072842,0,0,0,3.5,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,0,13,0,0,16,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,5,0,23,22,9,0,15,12,0,14,0,0,17,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.04933698,0,0,-0.04004954,0,0,0,-0.03300616,0,0,0.003495727,0,0.1795929,0.04468963,0,-0.04984272,0,0.07056401,0,0.0346409,-0.009048814,0.04844247,0,0.004168946,0.044918,0,0.01347157,0.08675308]},{"f":[21,-1,2,1,-1,2,22,23,21,1,-1,15,-1,-1,15,-1,1,-1,-1,-1,-1,37,22,21,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,2.453445297804259,17.094543568094064,4.5,7.5,4.54375,0,10.457380879072534,0,0,11.683433292835403,0,2.0015923566878984,0,0,0,0,12.52,12.121442107629743,4.5,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,15,0,17,0,0,0,0,22,23,24,0,0,0,0,0],"r":[2,0,28,5,0,21,20,19,14,11,0,13,0,0,16,0,18,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,-0.04924782,0,0,-0.04218048,0,0,0,0,0,-0.0311681,0,0.006909913,-0.02777852,0,0.007931393,0,0.1439446,-0.00551275,-0.03277736,0.04510326,0,0,0,-0.03886386,0.002805203,0.0100585,0.05958475,0.04233455]},{"f":[21,-1,2,1,-1,16,26,-1,-1,15,22,-1,-1,15,-1,1,21,-1,-1,18,15,16,-1,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,0,5.78135971352466,16.819976309430963,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,8.729619742794132,6.857980995127572,6.554588851677638,0,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,0,0,14,0,16,17,0,0,20,21,22,0,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,13,12,0,0,15,0,19,18,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,-0.04915733,0,0,-0.05075186,0,0,0.0114666,-0.03167116,0,0,0.008024315,0.1027502,0,-0.04759378,0,0,-0.03882388,-0.002491575,0,0,0,0.0197086,0.1134931,0.007650019,-0.04020552,0,0.01129196,0.07135309]},{"f":[21,-1,2,1,-1,37,18,2,2,15,-1,-1,18,-1,-1,-1,17,-1,21,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,8.05,7.519373778108656,3.084962500721156,1.792481250360578,2.0,0,0,4.357122758833062,0,0,0,8.571721503073015,0,5.5,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,0,13,0,0,0,17,0,19,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,5,0,21,16,15,12,11,0,0,14,0,0,0,18,0,20,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04906534,0,0,-0.04091493,0,0,0,0,0,-0.04492389,-5.010009e-05,0,-0.03565973,0.006471874,0.006471874,0,-0.03856837,0,-0.02334742,0.001892959,0,0,0.04753227,0,0.006471874,0.04082209,0.06384489,0.041051]},{"f":[21,-1,2,1,-1,23,22,4,-1,24,21,2,-1,-1,2,-1,-1,-1,15,1,-1,-1,15,-1,20,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.5,14.03140051979143,1.292481250360578,0,9.5,5.5,8.612669628062957,0,0,5.40929108874043,0,0,0,5.700439718141092,2.532051282051282,0,0,6.169925001442312,0,0.5,0,0,0,0],"m":[0,0,1,0,0,1,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,12,0,0,15,0,0,0,19,20,0,0,23,0,25,0,0,0,0],"r":[2,0,28,5,0,27,18,9,0,17,14,13,0,0,16,0,0,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,-0.04897165,0,0,-0.04005608,0,0,0,-0.01500493,0,0,0,-0.01449538,0.03366804,0,0.02270486,-0.0008874588,0.04713837,0,0,0.004658667,0.05422136,0,-0.04333924,0,-0.005988206,0.02611942,-0.02902094,0.0400781]},{"f":[21,-1,2,1,-1,18,17,2,37,21,18,-1,22,-1,-1,-1,-1,-1,33,-1,17,22,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.430398005367313,7.198837316474133,9.034782574103549,6.970000000000001,6.5,1.792481250360578,0,12.199665042528586,0,0,0,0,0,31.279747787828097,0,7.392087364565857,9.998977010081727,0,0,0,10.5,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,13,0,0,0,0,0,19,0,21,22,0,0,0,26,0,0,0],"r":[2,0,28,5,0,25,18,17,16,15,12,0,14,0,0,0,0,0,20,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.04887609,0,0,-0.03920008,0,0,0,0,0,0,-0.03412893,0,-0.03893961,-0.001668505,-0.001668505,0.006719302,0.02656833,0,0.00160113,0,0,-0.008771029,0.1731208,0.02888569,0,-0.01185115,0.03325099,0.0391244]},{"f":[21,-1,2,1,-1,23,22,4,23,-1,21,-1,-1,24,-1,-1,15,1,-1,-1,15,-1,20,22,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.5,14.03140051979143,1.292481250360578,0.5,0,9.5,0,0,9.5,0,0,5.700439718141092,2.532051282051282,0,0,6.169925001442312,0,0.5,16.844368517741394,0,0,0,0,0],"m":[0,0,1,0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,11,0,0,14,0,0,17,18,0,0,21,0,23,24,0,0,0,0,0],"r":[2,0,28,5,0,27,16,13,10,0,12,0,0,15,0,0,20,19,0,0,22,0,26,25,0,0,0,0,0],"v":[0,-0.04877848,0,0,-0.03834743,0,0,0,0,-0.0007415783,0,-0.0367286,0.03408687,0,0.0002519168,0.04472044,0,0,0.004238806,0.05138736,0,-0.04189957,0,0,-0.01406468,0.02643565,0.02459002,-0.02811762,0.03818848]},{"f":[21,-1,2,1,-1,16,26,-1,21,-1,-1,16,22,-1,-1,16,4,-1,-1,1,21,-1,22,-1,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.4794055810397554,0,2.321928094887362,0.5,0,8.5,0,0,5.169925001442312,16.819976309430963,0,0,6.108524456778169,4.951187557243013,0,0,4.92309205350118,7.5,0,8.32532955673354,0,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,13,0,0,16,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[2,0,26,5,0,11,8,0,10,0,0,15,14,0,0,19,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,-0.04867862,0,0,-0.04927712,0,0,0.009803803,0,-0.03497796,0.04020465,0,0,0.01038503,0.1138281,0,0,-0.05221024,0.007619351,0,0,-0.02399654,0,0.06939004,0.001728215,0.008708279,0,0.009344453,0.0708777]},{"f":[21,-1,2,1,-1,18,17,16,16,16,-1,22,-1,-1,-1,15,16,-1,-1,-1,33,-1,17,22,-1,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,0.4794055810397554,0,7.430398005367313,7.198837316474133,6.930737337562887,5.78135971352466,2.584962500721156,0,15.756821747580634,0,0,0,10.457380879072534,7.924812503605781,0,0,0,31.279747787828097,0,7.392087364565857,9.998977010081727,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,0,16,17,0,0,0,21,0,23,24,0,0,0,0,0],"r":[2,0,28,5,0,27,20,15,14,11,0,13,0,0,0,19,18,0,0,0,22,0,26,25,0,0,0,0,0],"v":[0,-0.04857634,0,0,-0.03701495,0,0,0,0,0,-0.01451733,0,0.007137679,0.04689465,-0.0300675,0,0,0.001465424,0.02380207,-0.009367041,0,0.001123717,0,0,-0.008267491,0.1491712,0.02719548,-0.008604041,0.03702496]},{"f":[21,-1,4,16,0,21,22,-1,-1,-1,-1,16,22,-1,-1,-1,1,21,-1,-1,1,-1,2,2,-1,22,-1,-1,-1],"t":[1.5,0,3.084962500721156,11.620219825507487,5.5,6.5,14.03140051979143,0,0,0,0,11.868050853745158,11.683871801979876,0,0,0,2.01685992705101,7.5,0,0,2.969223484848485,0,9.299206200930918,3.245926548164837,0,16.819976309430963,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,0,0,17,18,0,0,21,0,23,24,0,26,0,0,0],"r":[2,0,16,11,10,9,8,0,0,0,0,15,14,0,0,0,20,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,-0.04847146,0,0,0,0,0,-0.04272313,-0.01004036,-0.01004036,-0.001066744,0,0,0.1128854,0.02220971,0.001119293,0,0,-0.02465371,0.008690264,0,0.06157739,0,0,0.03410772,0,-0.000170169,0.02720208,0.05347866]},{"f":[21,-1,22,23,21,1,-1,22,25,22,-1,-1,-1,2,23,-1,27,-1,-1,-1,37,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,6.5,2.0015923566878984,0,11.775312925643524,0.5,7.25627765006036,0,0,0,9.012591414317754,2.5,0,0.5,0,0,0,9.075,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,1,1,0,0,0,1,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,0,0,14,15,0,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,20,7,0,13,12,11,0,0,0,19,16,0,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.0483638,0,0,0,0,-0.03197678,0,0,0,-0.001837247,-0.04632334,0.02483216,0,0,0.006511403,0,-0.03851128,0.03396099,0.06753089,0,0.004626959,0.0442567,-0.04067866,0,0,-0.01064013,0.07598303,-0.03523382]},{"f":[21,-1,2,1,-1,16,26,-1,21,-1,-1,15,-1,15,-1,1,-1,22,25,-1,-1,15,-1,2,-1,-1,20,-1,-1],"t":[1.5,0,8.9274737487839,0.39811320754716983,0,2.321928094887362,0.5,0,8.5,0,0,5.78135971352466,0,6.169925001442312,0,4.92309205350118,0,12.199665042528586,0.5,0,0,7.118941072723508,0,1.292481250360578,0,0,1.5,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,14,0,16,0,18,19,0,0,22,0,24,0,0,27,0,0],"r":[2,0,26,5,0,11,8,0,10,0,0,13,0,15,0,17,0,21,20,0,0,23,0,25,0,0,28,0,0],"v":[0,-0.04825318,0,0,-0.0506809,0,0,0.009075243,0,-0.03346517,0.03873662,0,0.008781625,0,-0.04434036,0,-0.01051462,0,0,-0.02129964,0.03662667,0,0.04646605,0,-0.01880435,0.01871891,0,0.008388415,0.06931155]},{"f":[21,-1,2,1,-1,22,23,21,22,22,1,-1,25,-1,-1,-1,0,-1,23,-1,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.4794055810397554,0,17.094543568094064,7.5,6.5,11.775312925643524,10.558733410935197,1.000936329588015,0,0.5,0,0,0,1.5,0,2.5,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,13,0,0,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,23,22,21,16,15,12,0,14,0,0,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.04813942,0,0,-0.03468243,0,0,0,0,0,0,0.122574,0,-0.05685315,-0.004615148,-0.05216004,0,-0.04552942,0,0.007554467,-0.02042955,0.005470228,-0.03986527,0,0,-0.01127363,0.07149181,-0.03502767,0.04864151]},{"f":[21,-1,4,37,21,-1,-1,22,-1,22,24,-1,16,-1,-1,-1,15,16,16,16,-1,-1,-1,-1,22,25,-1,-1,-1],"t":[1.5,0,3.9534452978042594,4.470000000000001,6.5,0,0,12.199665042528586,0,16.73920708776685,7.5,0,6.044394119358453,0,0,0,10.376125389276174,7.266786540694901,5.78135971352466,5.491853096329675,0,0,0,0,16.725912049439096,0.5,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,13,0,0,0,17,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,16,7,6,0,0,9,0,15,12,0,14,0,0,0,24,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04802234,0,0,0,-0.0354209,-0.007442192,0,-0.007442192,0,0,-0.006477223,0,-0.004008228,0.02756081,-0.007442192,0,0,0,0,0.002898538,0.04779405,-0.01691177,0.02251224,0,0,-0.06015028,-0.008365658,0.03199672]},{"f":[21,-1,2,1,-1,27,11,4,2,2,16,-1,-1,-1,-1,15,16,3,-1,-1,-1,22,25,-1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.4794055810397554,0,0.5,0.5,4.951187557243013,8.612669628062957,6.994342343386083,2.0,0,0,0,0,10.376125389276174,6.882643049361842,0.5803571428571428,0,0,0,16.725912049439096,0.5,0,0,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,0,0,0,0,16,17,18,0,0,0,22,23,0,0,0,0,0,0],"r":[2,0,28,5,0,27,26,15,14,13,12,0,0,0,0,21,20,19,0,0,0,25,24,0,0,0,0,0,0],"v":[0,-0.04790176,0,0,-0.03337599,0,0,0,0,0,0,-0.02823457,-0.001001615,-0.0545662,0.05364832,0,0,0,-0.04866518,0.005421228,0.02437115,0,0,-0.05713768,-0.01045506,0.03168308,-0.02858791,0.02654387,0.04792086]},{"f":[21,-1,2,37,15,-1,15,-1,22,16,18,16,-1,17,-1,-1,-1,-1,0,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,8.05,5.78135971352466,0,6.169925001442312,0,12.199665042528586,10.501837184902296,10.37503779525057,7.651051691178929,0,1.292481250360578,0,0,0,0,3.5,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,9,10,11,12,0,14,0,0,0,0,19,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,21,6,0,8,0,18,17,16,13,0,15,0,0,0,0,20,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04777751,0,0,0,0.004186739,0,-0.03728867,0,0,0,0,-0.0424349,0,0.004825962,-0.01841777,-0.0527865,0.004825962,0,-0.02140017,0.004825962,0,0,0.04403683,0,0.004825962,0.03603353,0.06005426,0.04725857]},{"f":[21,-1,2,22,23,1,-1,37,15,-1,15,-1,22,21,-1,-1,0,-1,-1,15,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.510264838663748,17.094543568094064,7.5,0.4794055810397554,0,9.34,5.78135971352466,0,6.169925001442312,0,12.199665042528586,5.5,0,0,3.5,0,0,6.108524456778169,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,11,0,13,14,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,7,0,19,10,0,12,0,16,15,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04764942,0,0,0,0,-0.03228648,0,0,0.006048445,0,-0.03599126,0,0,-0.03099685,-0.003728494,0,-0.01776544,0.007138946,0,0.007305945,0.02808422,-0.03863075,0,0,-0.0110596,0.06802305,-0.03379511,0.03298558]},{"f":[21,-1,2,22,23,1,-1,25,2,32,15,-1,23,21,-1,-1,-1,-1,37,-1,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,17.094543568094064,7.5,0.4794055810397554,0,6.5,2.453445297804259,10.463013322934223,2.0,0,3.5,7.5,0,0,0,0,11.57,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,11,0,13,14,0,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,7,0,21,18,17,12,0,16,15,0,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.0475173,0,0,0,0,-0.03146479,0,0,0,0,-0.04430363,0,0,-0.000898385,0.02602013,-0.01751566,-0.0441493,0,0.003163307,0.05615568,-0.05006505,-0.03786854,0,0,-0.01059428,0.064088,-0.03277483,0.04615613]},{"f":[21,-1,2,1,-1,21,-1,2,4,-1,15,16,-1,-1,22,-1,-1,-1,20,3,-1,17,2,-1,-1,22,-1,-1,-1],"t":[1.5,0,8.9274737487839,0.39811320754716983,0,2.5,0,7.754129516000594,4.951187557243013,0,10.113742166049189,7.118941072723508,0,0,16.725912049439096,0,0,0,1.5,0.9640400250156348,0,10.460454442916898,9.266718101290426,0,0,12.457649623491124,0,0,0],"m":[0,0,1,0,0,0,0,1,1,0,1,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0],"l":[1,0,3,4,0,6,0,8,9,0,11,12,0,0,15,0,0,0,19,20,0,22,23,0,0,26,0,0,0],"r":[2,0,18,5,0,7,0,17,10,0,14,13,0,0,16,0,0,0,28,21,0,25,24,0,0,27,0,0,0],"v":[0,-0.047381,0,0,-0.0493066,0,-0.03416496,0,0,-0.001896749,0,0,-0.002479757,0.028629,0,-0.0205434,0.03636186,-0.03807373,0,0,0.03004712,0,0,0.006748572,-0.05723914,0,0.04717709,-0.02008235,0.06842665]},{"f":[21,-1,2,27,11,1,15,-1,-1,25,22,4,15,-1,17,-1,-1,24,21,-1,-1,-1,17,-1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,0.5,0.4794055810397554,10.462502272597265,0,0,6.5,12.199665042528586,1.292481250360578,10.501837184902296,0,10.269270673170688,0,0,9.5,4.5,0,0,0,0.5,0,0,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,12,13,0,15,0,0,18,19,0,0,0,23,0,0,0,0,0,0],"r":[2,0,28,27,26,9,8,0,0,25,22,17,14,0,16,0,0,21,20,0,0,0,24,0,0,0,0,0,0],"v":[0,-0.04724033,0,0,0,0,0,-0.0513696,0.01210377,0,0,0,0,-0.02390886,0,-0.02567579,0.1044394,0,0,-0.02750984,0.003003729,0.04093375,0,-0.03689284,0.006436161,-0.04921654,-0.02667362,0.0242645,0.04541875]},{"f":[21,-1,2,21,-1,25,18,17,-1,33,33,-1,-1,2,-1,-1,18,18,17,23,-1,-1,-1,-1,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,2.5,0,6.5,7.383290334558488,7.198837316474133,0,31.279747787828097,10.073602462471115,0,0,3.39067985676233,0,0,10.620215166896223,9.273793716589307,8.571721503073015,2.5,0,0,0,0,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,8,0,10,11,0,0,14,0,0,17,18,19,20,0,0,0,0,0,0,27,0,0],"r":[2,0,26,5,0,25,16,9,0,13,12,0,0,15,0,0,24,23,22,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.04709515,0,0,-0.03360134,0,0,0,-0.0002862654,0,0,0.04170918,-0.004472949,0,0.1089442,0.02745428,0,0,0,0,0.01415397,-0.0431679,0.01125457,-0.03154277,0.01080276,-0.05689081,0,0.002429439,0.08094656]},{"f":[21,-1,2,37,32,16,17,22,4,-1,-1,-1,-1,15,-1,15,-1,1,-1,-1,-1,21,33,-1,27,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,8.05,10.41931383093672,2.584962500721156,9.135888799148791,17.105450904090162,5.65757478112815,0,0,0,0,5.78135971352466,0,6.169925001442312,0,4.643065068493151,0,0,0,9.5,13.895530557358477,0,0.5,0,0,0,0],"m":[0,0,1,1,1,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,0,0,14,0,16,0,18,0,0,0,22,23,0,25,0,0,0,0],"r":[2,0,28,21,20,13,12,11,10,0,0,0,0,15,0,17,0,19,0,0,0,27,24,0,26,0,0,0,0],"v":[0,-0.04694528,0,0,0,0,0,0,0,-0.03485982,-0.006402535,0.004194797,0.004194797,0,0.004194797,0,-0.03419248,0,-0.008795806,0.004194797,-0.03366648,0,0,0.04317016,0,0.004194797,0.03352374,0.05855175,0.04446806]},{"f":[21,-1,22,23,2,1,-1,21,-1,25,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,15,0,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,17,16,0,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04679056,0,0,0,0,-0.04889546,0,-0.03368229,0,0.0008906734,-0.05598086,0,0,0,0.04895621,-0.01033175,0,0.01272672,0,-0.05821599,0.02459681,0.07352587,-0.03598477,0,0,-0.01069441,0.06109605,-0.03230747]},{"f":[21,-1,2,33,15,22,4,21,-1,-1,-1,1,-1,21,-1,-1,15,-1,1,21,-1,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,5.78135971352466,14.03140051979143,3.8571227588330617,3.5,0,0,0,2.532051282051282,0,3.5,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,0,14,0,0,17,0,19,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,16,11,10,9,0,0,0,13,0,15,0,0,18,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04663085,0,0,0,0,0,0,-0.04727671,-0.004257542,0.008204,0,0.002253581,0,0.1308596,0.03311993,0,-0.03873337,0,0,-0.03635847,-0.001119192,0.004309659,0,0,-0.04994089,0.01296594,0.03135386,0.04377937]},{"f":[21,-1,2,1,-1,21,-1,25,4,4,32,15,-1,-1,-1,-1,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1],"t":[1.5,0,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,3.084962500721156,2.69615871138938,10.463013322934223,2.0,0,0,0,0,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0],"m":[0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,10,11,12,0,0,0,0,0,0,19,20,21,0,0,24,0,26,0,0,0],"r":[2,0,18,5,0,7,0,17,16,15,14,13,0,0,0,0,0,0,28,23,22,0,0,25,0,27,0,0,0],"v":[0,-0.04646598,0,0,-0.04832233,0,-0.03239399,0,0,0,0,0,-0.04189867,0.0002409953,-0.03865575,-0.04846429,0.00399874,-0.05483847,0,0,0,0.04778974,-0.01017131,0,0.01109382,0,-0.05598712,0.02398447,0.06591128]},{"f":[21,-1,2,22,23,37,15,-1,15,-1,22,16,18,-1,-1,-1,0,-1,-1,15,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,17.094543568094064,7.5,9.34,5.78135971352466,0,6.169925001442312,0,12.199665042528586,10.501837184902296,10.37503779525057,0,0,0,3.5,0,0,6.108524456778169,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,11,12,13,0,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,19,8,0,10,0,16,15,14,0,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04629583,0,0,0,0,0,0.005002243,0,-0.03335805,0,0,0,-0.01585172,-0.05135809,0.006004001,0,-0.01843826,0.005908812,0,0.006004001,0.02574206,-0.03512231,0,0,-0.01048606,0.0572009,-0.031323,0.04312844]},{"f":[21,-1,2,21,-1,25,18,-1,18,18,17,2,1,-1,-1,-1,-1,-1,23,22,-1,16,-1,-1,-1,-1,1,-1,-1],"t":[1.5,0,9.341519573083566,2.5,0,6.5,7.383290334558488,0,10.620215166896223,9.273793716589307,9.265614570460752,8.049190845056867,4.145833333333334,0,0,0,0,0,0.5,11.228801658478353,0,5.044394119358453,0,0,0,0,3.015625,0,0],"m":[0,0,1,0,0,1,1,0,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,12,13,0,0,0,0,0,19,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,5,0,25,8,0,18,17,16,15,14,0,0,0,0,0,24,21,0,23,0,0,0,0,28,0,0],"v":[0,-0.04612025,0,0,-0.03201163,0,0,0.002326566,0,0,0,0,0,-0.04225896,-0.004834584,0.02993111,0.03145768,-0.02948948,0,0,-0.007581322,0,0.01726435,0.1520849,-0.001625747,-0.05400262,0,0.001435862,0.07788863]},{"f":[21,-1,2,33,15,22,4,16,-1,-1,20,-1,-1,23,-1,-1,15,-1,22,-1,22,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,5.78135971352466,14.03140051979143,2.453445297804259,5.087462841250339,0,0,1.5,0,0,2.5,0,0,6.169925001442312,0,6.930301023985015,0,12.199665042528586,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,16,13,10,9,0,0,12,0,0,15,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.0459391,0,0,0,0,0,0,-0.007404275,-0.0555345,0,0.002213119,0.04520265,0,0.05246906,0.01045252,0,-0.03662119,0,0.05297465,0,-0.01338931,0.002638506,0,0,-0.04867302,0.01171209,0.02950625,0.04222579]},{"f":[21,-1,22,23,2,1,-1,21,-1,25,-1,-1,20,3,17,-1,-1,2,-1,18,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,6.5,0,0,1.5,0.9640400250156348,9.44286754736589,0,0,9.266718101290426,0,10.457893746154532,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,15,0,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,17,16,0,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04575228,0,0,0,0,-0.04776658,0,-0.03216827,0,0.0006915878,-0.05293562,0,0,0,0.04619806,-0.01121868,0,0.01107355,0,-0.05392078,0.02276939,0.0704734,-0.03414641,0,0,0.05484038,-0.009619258,-0.0302163]},{"f":[21,-1,2,18,17,26,32,22,16,-1,1,-1,-1,0,-1,-1,-1,-1,33,-1,2,1,-1,-1,-1,-1,17,-1,-1],"t":[1.5,0,10.510264838663748,7.383290334558488,7.198837316474133,1.5,3.9474088816539714,10.27595502635321,10.462502272597265,0,1.0491071428571428,0,0,2.5,0,0,0,0,31.279747787828097,0,3.753897320099348,2.532051282051282,0,0,0,0,1.792481250360578,0,0],"m":[0,0,1,1,1,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,0,0,14,0,0,0,0,19,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,18,17,16,13,10,0,12,0,0,15,0,0,0,0,20,0,24,23,0,0,0,0,28,0,0],"v":[0,-0.04555965,0,0,0,0,0,0,0,0.000737844,0,0.08367819,-0.002749327,0,-0.03104963,-0.003448997,0.008426521,-0.04119531,0,-0.002168649,0,0,-0.02578424,0.1123493,0.02434705,-0.006941192,0,-0.01313254,0.05084248]},{"f":[21,-1,22,23,21,1,-1,2,23,22,-1,15,-1,33,-1,16,-1,-1,24,-1,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,6.5,2.0015923566878984,0,9.025136904799169,2.5,14.03140051979143,0,5.044394119358453,0,19.167172160526107,0,8.618385502258606,0,0,12.5,0,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,0,12,0,14,0,16,0,0,19,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,22,7,0,21,18,11,0,13,0,15,0,17,0,0,20,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.04536112,0,0,0,0,-0.02725008,0,0,0,-0.003150558,0,0.06041647,0,-0.04690705,0,0.03665075,-0.009247334,0,0.01001764,-0.04910839,0.05583604,0.00379833,-0.03336142,0,0,0.05254067,-0.009303683,-0.02908685]},{"f":[21,-1,2,27,16,17,-1,-1,15,22,3,-1,-1,1,-1,21,-1,-1,15,-1,1,-1,15,-1,18,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,2.584962500721156,9.070120320271018,0,0,5.78135971352466,14.03140051979143,0.023705703211517166,0,0,2.532051282051282,0,3.5,0,0,6.169925001442312,0,4.92309205350118,0,6.857980995127572,0,8.03615005796417,0,0,0,0],"m":[0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,0,0,14,0,16,0,0,19,0,21,0,23,0,25,0,0,0,0],"r":[2,0,28,27,8,7,0,0,18,13,12,0,0,15,0,17,0,0,20,0,22,0,24,0,26,0,0,0,0],"v":[0,-0.04515658,0,0,0,0,-0.01905173,0.0186765,0,0,0,-0.05130367,0.004104435,0,0.003810395,0,0.120572,0.03341529,0,-0.03478252,0,-0.007918015,0,0.02653102,0,0.003824314,-0.04146723,0.02019011,0.04136698]},{"f":[21,-1,4,15,37,21,-1,-1,-1,22,-1,-1,15,16,16,16,-1,-1,-1,17,3,-1,-1,-1,22,25,-1,-1,-1],"t":[1.5,0,3.9534452978042594,11.683433292835403,4.470000000000001,6.5,0,0,0,11.683871801979876,0,0,10.457380879072534,7.118941072723508,5.78135971352466,5.491853096329675,0,0,0,9.011202871982467,0.3033788641265277,0,0,0,16.725912049439096,0.5,0,0,0],"m":[0,0,1,1,0,1,0,0,0,0,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,0,13,14,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[2,0,12,9,8,7,0,0,0,11,0,0,24,19,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,-0.04494596,0,0,0,0,-0.03333517,-0.009986201,-0.0005210401,0,0.07262877,0.0005740016,0,0,0,0,0.000459813,0.03733974,-0.01563177,0,0,-0.01478715,0.02934673,0.003006156,0,0,-0.0535174,-0.008191478,0.02782182]},{"f":[21,-1,8,4,15,-1,23,16,22,-1,24,-1,-1,-1,-1,15,16,16,16,-1,-1,-1,-1,22,25,-1,-1,-1,-1],"t":[1.5,0,0.5,3.9534452978042594,1.584962500721156,0,0.5,10.501837184902296,14.03140051979143,0,7.5,0,0,0,0,10.457380879072534,6.882643049361842,5.78135971352466,5.491853096329675,0,0,0,0,16.725912049439096,0.5,0,0,0,0],"m":[0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,9,0,11,0,0,0,0,16,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,15,6,0,14,13,10,0,12,0,0,0,0,23,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.04472917,0,0,0,-0.04128825,0,0,0,-0.004301209,0,0.009477323,0.1079229,0.04615746,-0.00613413,0,0,0,0,0.0004590438,0.03511031,-0.01647268,0.01755462,0,0,-0.05141409,-0.007803508,0.02741095,0.02633406]},{"f":[21,-1,37,32,15,22,-1,-1,15,-1,1,21,-1,-1,22,2,17,-1,-1,-1,33,-1,-1,-1,4,-1,35,-1,-1],"t":[1.5,0,9.34,10.41931383093672,5.78135971352466,4.4828921423310435,0,0,6.169925001442312,0,4.92309205350118,6.5,0,0,12.199665042528586,5.228690439536267,4.128693921346326,0,0,0,18.128588230264874,0,0,0,1.292481250360578,0,2.5,0,0],"m":[0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,0,0,15,16,17,0,0,0,21,0,0,0,25,0,27,0,0],"r":[2,0,24,23,8,7,0,0,10,0,14,13,0,0,20,19,18,0,0,0,22,0,0,0,26,0,28,0,0],"v":[0,-0.04450615,0,0,0,0,-0.03336007,0.005301483,0,-0.03034243,0,0,-0.03325438,-0.0007562919,0,0,0,-0.002801849,-0.04448712,0.005393992,0,-0.02876742,0.005393992,-0.03048752,0,0.005393992,0,0.01088568,0.07411001]},{"f":[21,-1,25,2,37,32,16,17,-1,-1,15,-1,16,-1,1,21,-1,-1,22,-1,-1,-1,4,-1,35,-1,-1,-1,-1],"t":[1.5,0,6.5,10.680359255641626,9.34,10.41931383093672,2.584962500721156,9.135888799148791,0,0,5.78135971352466,0,6.108524456778169,0,4.92309205350118,6.5,0,0,12.199665042528586,0,0,0,1.292481250360578,0,2.5,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,15,16,0,0,19,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,27,22,21,10,9,0,0,12,0,14,0,18,17,0,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,-0.04427684,0,0,0,0,0,0,-0.01701763,0.005361211,0,0.005361211,0,-0.03390987,0,0,-0.03084639,-0.00191216,0,-0.01200581,0.005361211,-0.02938185,0,0.005361211,0,0.01203113,0.07248287,0.04045006,-0.04519658]},{"f":[21,-1,22,23,2,1,-1,21,-1,21,-1,-1,20,2,-1,2,4,-1,-1,17,-1,-1,-1,-1,18,32,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.051177541662383,0.39811320754716983,0,2.5,0,9.5,0,0,1.5,9.193524834446507,0,10.510264838663748,9.266718101290426,0,0,1.792481250360578,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,10,0,0,13,14,0,16,17,0,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,12,7,0,9,0,11,0,0,22,15,0,19,18,0,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04404121,0,0,0,0,-0.04689441,0,-0.03174546,0,-0.000439466,0.01719139,0,0,0.03074281,0,0,0.005180268,-0.03862947,0,-0.01563769,0.04997608,0.06751223,-0.03205908,0,0,-0.01130819,0.04983677,-0.02871723]},{"f":[21,-1,8,22,21,33,33,-1,-1,-1,-1,17,22,-1,-1,22,-1,22,-1,15,-1,15,-1,15,-1,2,-1,-1,-1],"t":[1.5,0,0.5,12.199665042528586,5.5,44.79706586676442,15.103049306997898,0,0,0,0,0.5,15.676331898425197,0,0,12.921158001713904,0,13.415167820124083,0,5.700439718141092,0,6.108524456778169,0,7.118941072723508,0,2.453445297804259,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,13,0,0,16,0,18,0,20,0,22,0,24,0,26,0,0,0],"r":[2,0,28,11,10,9,8,0,0,0,0,15,14,0,0,17,0,19,0,21,0,23,0,25,0,27,0,0,0],"v":[0,-0.04379923,0,0,0,0,0,0.03142173,-0.02352737,0.02339872,0.002154243,0,0,-0.0441131,0.07472403,0,0.0169751,0,-0.03399584,0,0.01826115,0,-0.03469834,0,0.02046308,0,-0.0157063,0.006375886,0.02634362]},{"f":[21,-1,27,11,4,2,16,17,-1,-1,16,-1,-1,-1,20,22,-1,24,-1,-1,2,16,-1,22,-1,-1,-1,-1,-1],"t":[1.5,0,0.5,0.5,4.951187557243013,9.075473573191719,2.0,2.160964047443681,0,0,5.169925001442312,0,0,0,0.5,6.930301023985015,0,2.5,0,0,9.28539480853161,5.906890595608519,0,10.316277538115674,0,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,0,15,16,0,18,0,0,21,22,0,24,0,0,0,0,0],"r":[2,0,28,27,14,13,10,9,0,0,12,0,0,0,20,17,0,19,0,0,26,23,0,25,0,0,0,0,0],"v":[0,-0.0435509,0,0,0,0,0,0,-0.04940988,0.02385243,0,0.01378884,-0.004566912,0.04270859,0,0,0.0576218,0,-0.0008057243,-0.04217749,0,0,-0.01578926,0,-0.0018102,0.02614339,0.07704892,-0.02177644,0.02013569]},{"f":[21,-1,25,33,11,22,22,1,-1,-1,18,-1,-1,17,22,-1,-1,-1,22,17,-1,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,6.5,29.297756302860655,0.5,12.199665042528586,10.27595502635321,1.13125,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,17.094543568094064,9.135888799148791,0,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,14,15,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,18,13,10,9,0,0,12,0,0,17,16,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.0432962,0,0,0,0,0,0,0.01970439,-0.002737218,0,-0.0186567,0.06141893,0,0,-0.04382727,0.0789862,0.004999762,0,0,-0.03450104,0.04000203,0.02326237,0,0,-0.04629939,0.01091659,0.02651226,-0.04465514]},{"f":[21,-1,25,22,-1,22,1,-1,1,-1,17,-1,-1,22,-1,22,18,4,-1,-1,23,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,6.5,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,1.292481250360578,0,0,0.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,0,14,0,16,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,13,8,0,10,0,12,0,0,15,0,23,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04303516,0,0,-0.02877073,0,0,0.0481367,0,-0.03650455,0,-0.0123332,0.02737821,0,-0.04346265,0,0,0,-0.02179873,-0.001909484,0,0.06288032,-0.001714988,0,0,-0.04348262,0.07406342,0.004145708,-0.04365731]},{"f":[21,-1,33,11,4,2,-1,-1,15,16,3,-1,-1,-1,22,18,-1,-1,-1,22,17,-1,-1,-1,25,36,-1,-1,-1],"t":[1.5,0,29.297756302860655,0.5,4.951187557243013,9.075473573191719,0,0,10.376125389276174,7.266786540694901,0.5803571428571428,0,0,0,16.725912049439096,1.292481250360578,0,0,0,17.094543568094064,9.135888799148791,0,0,0,0.5,3.5,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,0,0,0,15,16,0,0,0,20,21,0,0,0,25,26,0,0,0],"r":[2,0,24,19,8,7,0,0,14,13,12,0,0,0,18,17,0,0,0,23,22,0,0,0,28,27,0,0,0],"v":[0,-0.04276782,0,0,0,0,-0.003244542,0.04136913,0,0,0,-0.04011176,0.003969951,0.018208,0,0,-0.04945447,-0.005916833,0.0251378,0,0,-0.03367199,0.03804658,0.02170831,0,0,-0.04544727,0.01115122,0.02541649]},{"f":[21,-1,2,25,37,32,16,17,-1,-1,15,-1,15,-1,1,21,-1,-1,-1,-1,4,35,-1,-1,35,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,6.5,9.34,10.41931383093672,2.584962500721156,9.135888799148791,0,0,5.78135971352466,0,6.169925001442312,0,4.92309205350118,6.5,0,0,0,0,1.292481250360578,2.5,0,0,2.5,0,0,0,0],"m":[0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,15,16,0,0,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,20,19,10,9,0,0,12,0,14,0,18,17,0,0,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04249422,0,0,0,0,0,0,-0.01540029,0.005132975,0,0.005132975,0,-0.02749546,0,0,-0.03054733,-0.00171548,0.005132975,-0.02803948,0,0,0.0321742,0.005132975,0,0.01040127,0.07168238,-0.04282359,0.03903024]},{"f":[21,-1,27,11,4,2,-1,-1,20,22,2,22,22,-1,-1,-1,-1,-1,2,22,16,-1,23,-1,-1,-1,-1,-1,-1],"t":[1.5,0,0.5,0.5,4.951187557243013,9.075473573191719,0,0,0.5,10.27595502635321,7.661668144640085,8.133841411710264,7.5621616439616925,0,0,0,0,0,9.28539480853161,14.03140051979143,7.118941072723508,0,1.5,0,0,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,0,1,1,1,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,11,12,13,0,0,0,0,0,19,20,21,0,23,0,0,0,0,0,0],"r":[2,0,28,27,8,7,0,0,18,17,16,15,14,0,0,0,0,0,26,25,22,0,24,0,0,0,0,0,0],"v":[0,-0.04221442,0,0,0,0,-0.002624558,0.0406923,0,0,0,0,0,0.02380553,-0.04679734,0.06314779,-0.006816104,-0.007912248,0,0,0,-0.01833373,0,0.001581558,0.04182561,0.03225132,0.0746659,-0.01958289,0.01831579]},{"f":[21,-1,25,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,18,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,6.5,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,13,14,0,0,0,18,0,20,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,17,8,0,10,0,12,0,16,15,0,0,0,19,0,23,22,0,0,27,26,0,0,0,0],"v":[0,-0.04192852,0,0,-0.02784962,0,0,0.04438512,0,-0.03583653,0,-0.01183342,0,0,0.09617002,0.01973489,0.006666726,0,-0.04213585,0,0,-0.006458389,0.01838362,0,0,-0.04295816,0.07020756,0.004006022,-0.04210723]},{"f":[21,-1,2,33,22,22,15,-1,-1,18,-1,-1,17,22,-1,-1,23,2,-1,-1,18,-1,-1,25,36,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,29.297756302860655,12.199665042528586,10.27595502635321,11.054604317960672,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,3.5,9.012591414317754,0,0,6.499823868264185,0,0,0.5,3.5,0,0,0,0],"m":[0,0,1,1,0,1,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,14,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,12,9,8,0,0,11,0,0,16,15,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04163659,0,0,0,0,0,0.0007908027,0.0451383,0,-0.01899406,0.05603306,0,0,-0.0425941,0.06590589,0,0,0.005107633,0.04624051,0,0.0043691,-0.02441895,0,0,-0.04453443,0.01048237,0.02414156,0.0382888]},{"f":[21,-1,22,23,21,2,1,-1,21,-1,-1,20,1,4,-1,-1,-1,-1,0,1,-1,-1,-1,-1,18,1,-1,-1,-1],"t":[1.5,0,17.094543568094064,7.5,9.5,9.051177541662383,0.6824793747286149,0,2.5,0,0,1.5,2.01685992705101,9.266718101290426,0,0,0,0,7.5,6.015625,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,7,0,9,0,0,12,13,14,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,11,8,0,10,0,0,17,16,15,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04133877,0,0,0,0,0,-0.0441684,0,-0.03057852,-0.0004097421,0,0,0,0.007245895,-0.03038825,0.02615113,0.06391196,0,0,0.01264546,-0.03005297,0.0554922,-0.02970498,0,0,0.04889499,-0.01019332,-0.02786179]},{"f":[21,-1,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,-1,17,22,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,0,13.415167820124083,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,0,8,0,10,0,12,13,0,0,0,17,0,19,0,21,22,0,0,25,0,27,0,0],"r":[2,0,4,0,16,7,0,9,0,11,0,15,14,0,0,0,18,0,20,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.04103517,0,-0.02748127,0,0,0.04090813,0,-0.03524016,0,-0.01133647,0,0,0.09153008,0.01857239,0.006178016,0,-0.04132925,0,-0.003993234,0,0,-0.04214797,0.06238161,0,0.0144967,0,-0.03227029,0.002567573]},{"f":[21,-1,2,8,1,-1,22,23,21,22,-1,23,33,-1,-1,-1,0,1,-1,-1,-1,-1,18,1,-1,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,0.5,0.3126984126984127,0,17.094543568094064,7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,0,0,0,7.5,6.015625,0,0,0,0,8.729619742794132,7.9752100840336135,0,0,0,0,0],"m":[0,0,1,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,9,10,0,12,13,0,0,0,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,6,0,22,21,16,11,0,15,14,0,0,0,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04072594,0,0,0,-0.05055082,0,0,0,0,-0.002814897,0,0,0.04748269,0.006466959,-0.0008773881,0,0,0.01230905,-0.02898986,0.05481169,-0.02901288,0,0,0.0465744,-0.009905904,-0.02708573,0.02443185,0.03770173]},{"f":[21,-1,25,37,32,23,1,-1,-1,4,2,2,22,19,23,-1,-1,-1,-1,-1,-1,-1,-1,4,-1,35,-1,-1,-1],"t":[1.5,0,6.5,9.34,10.41931383093672,3.5,2.01685992705101,0,0,6.451187557243013,4.0437314206251695,1.792481250360578,17.094543568094064,0.5,7.5,0,0,0,0,0,0,0,0,1.292481250360578,0,2.5,0,0,0],"m":[0,0,1,1,1,1,0,0,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,12,13,14,15,0,0,0,0,0,0,0,0,24,0,26,0,0,0],"r":[2,0,28,23,22,9,8,0,0,21,20,19,18,17,16,0,0,0,0,0,0,0,0,25,0,27,0,0,0],"v":[0,-0.04041125,0,0,0,0,0,-0.004797754,0.004197415,0,0,0,0,0,0,0.004817313,-0.03168209,-0.03438584,0.004817313,-0.05133093,0.004817313,-0.04170566,-0.02704965,0,0.004817313,0,0.009675346,0.07083378,-0.04150261]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,15,-1,16,-1,22,16,-1,-1,-1,4,25,-1,-1,22,-1,-1,17,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,5.169925001442312,0,6.108524456778169,0,12.199665042528586,10.501837184902296,0,0,0,1.292481250360578,0.5,0,0,11.876328901062104,0,0,1.792481250360578,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,15,16,0,0,0,20,21,0,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,19,12,0,14,0,18,17,0,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04635256,0.02629075,0,0.05758389,-0.03982065,0,0,0,0.005909675,0,-0.01455159,0,0,-0.01511334,0.007675853,0.003623878,0,0,0.007675853,0.05174116,0,0.008595503,0.0498464,0,-0.05933105,-0.001125528]},{"f":[21,-1,22,-1,22,1,-1,1,-1,17,-1,21,16,-1,-1,-1,22,-1,22,18,-1,18,-1,-1,17,22,-1,-1,-1],"t":[1.5,0,4.926654777701837,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,4.583709072915869,0,7.5,10.372865060112588,0,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,0,10.682994316786338,0,0,0.5,15.717370311433935,0,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,8,0,10,0,12,13,0,0,0,17,0,19,20,0,22,0,0,25,26,0,0,0],"r":[2,0,4,0,16,7,0,9,0,11,0,15,14,0,0,0,18,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,-0.03981271,0,-0.02690138,0,0,0.03843373,0,-0.03444583,0,-0.01101633,0,0,0.0872553,0.01711465,0.006302162,0,-0.04049558,0,0,-0.005889104,0,0.07673547,0.003012865,0,0,-0.04175453,0.05837514,0.003158565]},{"f":[8,4,15,-1,-1,15,16,16,16,-1,0,-1,-1,-1,21,-1,16,2,18,-1,-1,-1,-1,22,25,-1,-1,-1,-1],"t":[0.5,3.9534452978042594,1.584962500721156,0,0,10.457380879072534,7.118941072723508,5.78135971352466,5.491853096329675,0,3.5,0,0,0,3.5,0,11.620219825507487,6.033380966193453,6.531360383582522,0,0,0,0,16.725912049439096,0.5,0,0,0,0],"m":[1,1,0,0,0,1,1,1,1,0,1,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,11,0,0,0,15,0,17,18,19,0,0,0,0,24,25,0,0,0,0],"r":[28,5,4,0,0,23,14,13,10,0,12,0,0,0,16,0,22,21,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03899899,-0.002298874,0,0,0,0,0.0006494024,0,0.003678795,0.05515755,-0.01364798,0,-0.01781177,0,0,0,0.05129896,-0.00280212,0.01403365,-0.0009492856,0,0,-0.04672495,-0.006810343,0.02479433,0.02292819]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,15,4,-1,17,-1,-1,16,-1,-1,4,25,-1,-1,35,-1,-1,18,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,5.169925001442312,9.266718101290426,0,9.845435504646158,0,0,6.108524456778169,0,0,1.292481250360578,0.5,0,0,2.5,0,0,1.292481250360578,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,14,0,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,26,19,16,13,0,15,0,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,-0.04609355,0.02545893,0,0.0519442,-0.03931514,0,0,0,0,0.007281549,0,-0.04083408,0.007281549,0,-0.01405551,-4.769928e-05,0,0,0.007281549,0.05103897,0,0.01723915,0.06565749,0,-0.05920916,-0.001254031]},{"f":[2,22,22,27,11,21,-1,4,2,-1,-1,18,-1,-1,-1,-1,22,18,-1,-1,2,-1,-1,18,32,-1,-1,-1,-1],"t":[10.680359255641626,17.094543568094064,17.069460217922263,0.5,0.5,2.5,0,9.266718101290426,9.051177541662383,0,0,10.457893746154532,0,0,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,16,15,14,7,0,11,10,0,0,13,0,0,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.01988046,0,0,6.293282e-05,0.01693904,0,-0.05649365,0.02197476,-0.02393363,0.02389892,0,0,-0.0723663,-0.01157007,0,-0.02839885,0.02769368,0,0,-0.01350405,0.0432997,-0.02630726,0.03728505]},{"f":[2,27,8,1,-1,32,33,11,4,2,-1,-1,20,22,2,-1,-1,-1,-1,17,-1,-1,-1,33,-1,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.5,0.3126984126984127,0,10.463013322934223,29.297756302860655,0.5,4.951187557243013,8.612669628062957,0,0,0.5,10.27595502635321,7.661668144640085,0,0,0,0,9.135888799148791,0,0,0,41.64100553123272,0,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,13,14,15,0,0,0,0,20,0,0,0,24,0,0,0,0,0],"r":[28,27,26,5,0,23,22,19,12,11,0,0,18,17,16,0,0,0,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.05001371,0,0,0,0,0,-0.003193292,0.0514572,0,0,0,0.02700366,-0.006611672,-0.00806361,0.01331996,0,-0.02566665,0.03819604,0.01865378,0,-0.04484214,0.002602119,0.02246143,0.01653092,0.03669659]},{"f":[22,22,22,22,22,-1,-1,4,-1,22,-1,-1,23,-1,-1,22,18,-1,-1,2,-1,-1,18,1,17,-1,-1,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,3.753897320099348,0,16.819976309430963,0,0,7.5,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,7.9752100840336135,7.588709768994619,0,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,0,0,16,17,0,0,20,0,0,23,24,25,0,0,0,0],"r":[22,15,12,7,6,0,0,9,0,11,0,0,14,0,0,19,18,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,6.61179e-06,0.05759227,0,-0.03096453,0,-0.02248856,0.02572651,0,0.03168952,-0.02299863,0,0,-0.06922217,-0.01128777,0,-0.02765806,0.02659547,0,0,0,0.008723347,0.06358994,-0.009342047,-0.02546829]},{"f":[21,-1,2,22,-1,22,-1,22,16,-1,-1,22,-1,22,18,2,4,-1,-1,-1,23,-1,-1,17,22,-1,-1,-1,-1],"t":[1.5,0,10.680359255641626,4.926654777701837,0,5.9525054311951475,0,7.770639075561187,10.462502272597265,0,0,8.32532955673354,0,12.199665042528586,10.609627990977145,9.266718101290426,1.292481250360578,0,0,0,0.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,15,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,5,0,7,0,11,10,0,0,13,0,23,20,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.03894383,0,0,-0.02591774,0,0.07389122,0,0,0.0005112473,0.03779606,0,-0.04018153,0,0,0,0,-0.0194922,9.134581e-05,-0.04471223,0,0.05623916,-0.00377298,0,0,-0.04125589,0.05493933,0.002987771,0.03611181]},{"f":[25,21,-1,15,37,36,23,-1,2,21,-1,-1,18,-1,-1,17,-1,-1,33,15,-1,-1,21,22,-1,-1,-1,-1,-1],"t":[6.5,1.5,0,13.56045225766691,7.92,3.5,0.5,0,4.0437314206251695,7.5,0,0,9.229707804263573,0,0,7.198837316474133,0,0,15.81450217830399,5.169925001442312,0,0,9.5,16.838461650489297,0,0,0,0,0],"m":[1,0,0,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,9,10,0,0,13,0,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,3,0,27,18,15,8,0,12,11,0,0,14,0,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,-0.0386027,0,0,0,0,0.003043284,0,0,-0.02336767,0.001949692,0,0.003043284,-0.01638389,0,-0.006704644,-0.05539274,0,0,0.05456151,0.0090468,0,0,0.003043284,0.02206681,0.04491113,-0.02113175,-0.03906335]},{"f":[22,22,22,22,22,-1,-1,-1,23,16,22,-1,-1,2,-1,-1,-1,22,18,-1,-1,2,-1,-1,18,32,-1,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,0,7.5,6.0,16.98920034885663,0,0,5.538407798525416,0,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,8.341935934311302,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,0,14,0,0,0,18,19,0,0,22,0,0,25,26,0,0,0],"r":[24,17,8,7,6,0,0,0,16,13,12,0,0,15,0,0,0,21,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,2.879763e-05,0.05604016,-0.013476,0,0,0,0.08787637,0.009391198,0,-0.02395913,0.03445103,-0.02257724,0,0,-0.06627058,-0.01105752,0,-0.02712769,0.02545373,0,0,-0.01371785,0.04020514,-0.02465916]},{"f":[2,8,1,-1,33,32,33,33,37,-1,-1,4,15,16,-1,-1,-1,-1,21,2,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,1.292481250360578,0,9.107199999999999,0,0,0,0,0,0,0],"m":[1,1,0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,9,0,0,12,13,14,0,0,0,0,19,20,0,22,0,0,0,0,0,0,0],"r":[28,27,4,0,26,25,18,11,10,0,0,17,16,15,0,0,0,0,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.04914958,0,0,0,0,0,-0.0003484918,0.03864565,0,0,0,-0.00420483,-0.07289036,-0.01140236,0.02850373,0,0,-0.008816227,0,0.005908955,0.04515286,0.05212909,-0.05412855,0.02036192,0.0219938,0.03549929]},{"f":[21,-1,22,-1,22,-1,22,16,-1,-1,22,-1,22,-1,17,22,-1,-1,22,36,-1,-1,22,-1,15,23,-1,-1,-1],"t":[1.5,0,4.926654777701837,0,5.9525054311951475,0,7.770639075561187,10.462502272597265,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,2.5,0,0,13.415167820124083,0,5.169925001442312,3.5,0,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,0,13,0,15,16,0,0,19,20,0,0,23,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,12,0,14,0,18,17,0,0,22,21,0,0,24,0,28,27,0,0,0],"v":[0,-0.03825098,0,-0.02536491,0,0.06905526,0,0,0.0002927825,0.03531875,0,-0.04425602,0,-0.003689371,0,0,-0.04081468,0.05212172,0,0,-0.007164225,0.02117701,0,-0.03106824,0,0,0.03410449,-0.0001712189,-0.0006686058]},{"f":[21,4,22,-1,-1,16,-1,-1,25,37,-1,4,25,-1,-1,35,-1,-1,17,-1,22,36,16,-1,-1,-1,22,-1,-1],"t":[2.5,4.951187557243013,13.87915215758784,0,0,5.78135971352466,0,0,2.5,9.205,0,1.292481250360578,0.5,0,0,2.5,0,0,1.792481250360578,0,12.613886328427084,2.5,4.247927513443585,0,0,0,12.637102544927284,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,0,0,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,18,11,0,15,14,0,0,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,-0.04564456,0.02393552,0,0.04905142,-0.03890582,0,0,0.0002782962,0,0,0.006534391,0.04935628,0,0.01528042,0.06359917,0,-0.05591899,0,0,0,-0.03365172,-0.003175018,0.02347732,0,0.05093611,-0.02491219]},{"f":[2,22,22,22,22,22,-1,-1,22,-1,-1,23,-1,-1,22,18,-1,-1,2,-1,-1,18,1,17,-1,-1,-1,-1,-1],"t":[10.680359255641626,17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,16.819976309430963,0,0,7.5,0,0,17.078739781790915,6.076776015854056,0,0,4.0437314206251695,0,0,8.729619742794132,7.9752100840336135,7.588709768994619,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,12,0,0,15,16,0,0,19,0,0,22,23,24,0,0,0,0,0],"r":[28,21,14,11,8,7,0,0,10,0,0,13,0,0,18,17,0,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.000137097,0.054663,0,-0.03506131,0.003082815,0,0.02884207,-0.02215727,0,0,-0.06312083,-0.01061445,0,-0.02667727,0.02456631,0,0,0,0.0063784,0.06140676,-0.009093028,-0.02365658,0.03489929]},{"f":[22,-1,22,16,-1,-1,1,17,21,22,17,-1,-1,-1,16,-1,22,-1,16,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,6.5,10.092754721228424,1.792481250360578,0,0,0,1.0,0,16.25454052700782,0,7.924812503605781,0,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,0,15,0,17,0,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,6,5,0,0,22,21,14,13,12,0,0,0,16,0,18,0,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.03877638,0,0,0.002128306,0.04237145,0,0,0,0,0,0.0663754,-0.03566395,-0.03770366,0,0.0426125,0,-0.01107746,0,-0.0003484879,0.05806144,0.02698556,0,0,0.004212369,0.06687305,0,-0.0004285725,0.03520718]},{"f":[15,15,22,22,21,4,-1,16,-1,-1,25,-1,-1,16,22,-1,2,-1,-1,-1,18,32,-1,17,-1,-1,-1,-1,-1],"t":[13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,2.5,4.951187557243013,0,5.78135971352466,0,0,2.5,0,0,7.774787059601174,17.078739781790915,0,4.0437314206251695,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0,0,0,0,0],"m":[1,1,1,1,0,1,0,1,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,0,0,14,15,0,17,0,0,0,21,22,0,24,0,0,0,0,0],"r":[28,27,20,13,10,7,0,9,0,0,12,0,0,19,16,0,18,0,0,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.03741092,0,0.04601529,-0.03849768,0,0.001328845,-0.007745085,0,0,-0.05638241,0,-0.03092265,0.01915006,0.01062877,0,0,-0.01443564,0,0.01798947,0.0633233,-0.02247111,0.02680592,-0.02021767]},{"f":[22,-1,22,1,-1,1,-1,18,-1,-1,22,-1,21,17,37,-1,-1,-1,15,-1,18,3,-1,22,-1,-1,22,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,0,0,8.133841411710264,0,2.5,1.292481250360578,3.97,0,0,0,10.457380879072534,0,10.620215166896223,0.26136363636363635,0,16.725912049439096,0,0,11.722778233171695,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,0,13,14,15,0,0,0,19,0,21,22,0,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,12,0,18,17,16,0,0,0,20,0,26,23,0,25,0,0,28,0,0],"v":[0,-0.03838517,0,0,0.03163942,0,-0.02815111,0,-0.01202527,0.01782893,0,-0.04341897,0,0,0,-0.03761462,0.01906616,-0.04380175,0,0.00130031,0,0,0.03565261,0,-0.01698605,0.01783731,0,0.06921919,-0.0034232]},{"f":[33,32,33,23,22,-1,15,-1,17,25,-1,-1,-1,18,21,-1,-1,1,-1,22,-1,-1,33,37,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,3.5,14.116600814324018,0,5.044394119358453,0,6.199638962254484,2.5,0,0,0,6.400854179458231,6.5,0,0,1.981615336266499,0,17.083799521744545,0,0,39.9785486130134,9.629999999999999,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,9,10,0,0,0,14,15,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,22,13,6,0,8,0,12,11,0,0,0,17,16,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.001039466,0,0.0387713,0,0,0.03186536,-0.04233575,0.0113977,0,0,-0.04612823,0.006035849,0,0.06917963,0,-0.03337164,0.008957684,0,0,0.01008027,0.04995612,-0.02450837,-0.05303169,0.01959925]},{"f":[2,21,-1,1,-1,21,2,-1,17,15,-1,22,-1,-1,-1,22,-1,-1,3,22,-1,-1,16,15,2,-1,-1,-1,-1],"t":[9.051177541662383,2.5,0,0.39811320754716983,0,9.5,7.005613627711627,0,11.364132993990529,10.113742166049189,0,16.725912049439096,0,0,0,15.756821747580634,0,0,0.9640400250156348,13.005894632618489,0,0,11.696967526234287,4.247927513443585,9.266718101290426,0,0,0,0],"m":[1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,10,0,12,0,0,0,16,0,0,19,20,0,0,23,24,25,0,0,0,0],"r":[18,3,0,5,0,15,8,0,14,11,0,13,0,0,0,17,0,0,22,21,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03115147,0,-0.04547573,0,0,0.0004238226,0,0,-0.03344622,0,-0.0170306,0.02108246,0.01772352,0,0.0185142,-0.03842968,0,0,0.004159251,0.05352754,0,0,0,0.00730741,-0.02474082,0.02932175,-0.04087207]},{"f":[2,27,23,-1,18,21,-1,18,18,15,-1,-1,-1,22,-1,22,-1,-1,1,-1,22,15,-1,-1,18,-1,-1,-1,-1],"t":[10.680359255641626,0.5,3.5,0,6.400854179458231,6.5,0,5.303665156874805,1.292481250360578,13.56045225766691,0,0,0,17.069460217922263,0,17.074218326240924,0,0,1.981615336266499,0,17.094543568094064,3.4594316186372973,0,0,8.729619742794132,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,0,0,14,0,16,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,18,7,0,13,12,11,0,0,0,15,0,17,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0.0008688135,0,0,-0.04530512,0,0,0,0.01267497,-0.01909423,-0.01047629,0,0.04587989,0,-0.03299907,0.03870555,0,0.06557318,0,0,0.006409351,-0.03748134,0,0.0233713,-0.02952561,0.01483354,0.03406799]},{"f":[22,-1,22,15,-1,-1,22,-1,22,-1,17,22,-1,-1,22,36,22,-1,-1,-1,22,-1,15,23,2,-1,-1,-1,-1],"t":[2.69615871138938,0,7.770639075561187,10.066089190457772,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,12.921158001713904,2.5,12.613886328427084,0,0,0,13.415167820124083,0,5.169925001442312,2.5,1.292481250360578,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,0,11,12,0,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,10,0,14,13,0,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.03798241,0,0,0.00438222,0.04970522,0,-0.04254784,0,-0.003646481,0,0,-0.04092188,0.0497635,0,0,0,-0.0308517,0.04979545,0.01966191,0,-0.02916723,0,0,0,0.07359063,0.01449303,0.002055427,-0.0006468962]},{"f":[4,32,16,17,2,19,15,17,15,-1,-1,-1,15,-1,15,-1,-1,-1,37,-1,33,-1,-1,-1,22,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,10.501837184902296,10.37503779525057,1.792481250360578,2.5,5.169925001442312,1.792481250360578,4.087462841250339,0,0,0,6.108524456778169,0,7.118941072723508,0,0,0,6.890000000000001,0,18.188502736593104,0,0,0,11.683871801979876,0,0,0,0],"m":[1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,0,15,0,0,0,19,0,21,0,0,0,25,0,0,0,0],"r":[28,27,24,23,18,17,12,11,10,0,0,0,14,0,16,0,0,0,20,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.02327791,0.03037042,0.05166419,0,-0.04850989,0,0.02297538,-0.001783744,-0.02128623,0,-0.03892199,0,-0.005302947,0.04110297,-0.03265456,0,0.04364554,0.001078311,-0.03696068,0.001923163]},{"f":[33,32,33,33,37,21,-1,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,33,-1,-1,24,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,2.5,0,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,27.893589438525005,0,0,9.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,0,0,17,18,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,16,9,8,7,0,0,0,15,14,13,0,0,0,0,26,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.02179544,0.0001554081,0.0370031,0,0,0,-0.003090679,-0.07086655,-0.01082045,0.02677882,0,0,0.03708092,0,0,-0.006006292,-0.03985529,0,0.004339499,0.05840513,0.05044537,-0.05214598,0.01916364]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,23,-1,4,2,17,17,21,-1,18,-1,-1,-1,-1,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,3.5,0,6.451187557243013,4.0437314206251695,8.496347487943302,6.434604125275413,7.5,0,1.292481250360578,0,0,0,0,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,0,14,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,13,0,25,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.02416019,0,0.0636792,0,0,0,-0.02405238,0.02371367,-0.04329052,0,0,0.002772439,0,0,0,0,0,-0.02974168,0,0.02807088,-0.02682687,-0.04017566,0.01919947,0.006469301,-0.03581546,0,-0.05222685,-0.00501312]},{"f":[2,21,22,-1,-1,1,-1,21,2,4,-1,22,-1,-1,-1,22,-1,-1,22,4,-1,-1,18,1,-1,-1,4,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,0,0,0.39811320754716983,0,9.5,6.959990297524481,4.80643424864552,0,14.116600814324018,0,0,0,15.756821747580634,0,0,11.681676585676911,9.266718101290426,0,0,9.273793716589307,4.835403151065801,0,0,10.510264838663748,0,0],"m":[1,0,1,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,10,0,12,0,0,0,16,0,0,19,20,0,0,23,24,0,0,27,0,0],"r":[18,5,4,0,0,7,0,15,14,11,0,13,0,0,0,17,0,0,22,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.03832385,0.02393021,0,-0.04490813,0,0,0,-0.001329259,0,0.002892213,0.02317814,-0.008625237,0,0.01707988,-0.03763073,0,0,0.006467514,-0.03953748,0,0,0.04863647,-0.007007478,0,-0.02456756,0.04456657]},{"f":[25,22,-1,23,22,-1,-1,4,2,16,17,3,-1,17,16,-1,-1,-1,-1,21,17,-1,-1,-1,21,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,3.5,14.116600814324018,0,0,6.451187557243013,4.0437314206251695,7.330916878114617,7.723832212141186,0.80625,0,1.292481250360578,2.807354922057604,0,0,0,0,9.5,0.5,0,0,0,7.5,0,0,0,0],"m":[1,0,0,1,1,0,0,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,0,14,15,0,0,0,0,20,21,0,0,0,25,0,0,0,0],"r":[28,3,0,7,6,0,0,27,24,19,18,13,0,17,16,0,0,0,0,23,22,0,0,0,26,0,0,0,0],"v":[0,0,-0.03733523,0,0,-8.63422e-05,0.008762544,0,0,0,0,0,0.04189127,0,0,-0.0119619,0.03718052,-0.02608683,0.0287739,0,0,0.02879585,-0.04921855,0.03637455,0,0.02945256,0.001609076,-0.03200583,-0.03620637]},{"f":[2,21,22,-1,-1,1,-1,21,-1,0,22,-1,16,2,-1,-1,-1,-1,22,4,-1,-1,18,1,-1,-1,4,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,0,0,0.39811320754716983,0,9.5,0,7.5,10.735831902194036,0,11.98370619265935,5.065928480304397,0,0,0,0,11.681676585676911,9.266718101290426,0,0,9.273793716589307,4.835403151065801,0,0,10.510264838663748,0,0],"m":[1,0,1,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,0,10,11,0,13,14,0,0,0,0,19,20,0,0,23,24,0,0,27,0,0],"r":[18,5,4,0,0,7,0,9,0,17,12,0,16,15,0,0,0,0,22,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.037856,0.0231667,0,-0.0445055,0,-0.0005473865,0,0,0.02649672,0,0,-0.04126792,0.002072523,0.02755521,0.04942722,0,0,0.006229683,-0.03773779,0,0,0.04747722,-0.006457718,0,-0.02275057,0.04408524]},{"f":[8,1,15,-1,-1,22,21,1,22,-1,-1,33,19,33,23,-1,-1,-1,-1,-1,-1,17,23,-1,-1,2,-1,-1,-1],"t":[0.5,0.3126984126984127,1.584962500721156,0,0,12.199665042528586,5.5,1.000936329588015,9.505613627711627,0,0,44.79706586676442,2.5,32.199935296219856,1.5,0,0,0,0,0,0,0.5,1.5,0,0,9.012591414317754,0,0,0],"m":[1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,9,0,0,12,13,14,15,0,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,21,20,11,10,0,0,19,18,17,16,0,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0.02199107,-0.04797761,0,0,0,0,0.1185201,-0.006499067,0,0,0,0,-0.01103942,-0.04410049,-0.05355518,0.01082353,0.01819656,0.0009895513,0,0,-0.0410158,0.03106066,0,0.001505373,0.02136791,0.01939352]},{"f":[22,-1,22,16,-1,-1,1,17,21,15,22,21,-1,-1,-1,22,-1,17,-1,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,7.5,7.417852514885898,10.194756329264862,5.5,0,0,0,14.247293309538719,0,2.453445297804259,0,0,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,0,0,0,16,0,18,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,6,5,0,0,22,21,20,15,14,13,0,0,0,17,0,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.0369412,0,0,0.001639016,0.03571176,0,0,0,0,0,0,0.08520961,-0.02146925,-0.04369329,0,-0.01923165,0,-0.03895147,0.0549225,0.0003756083,0.02367122,0,0,0.002402603,0.06378397,0,-0.0002677128,0.03301319]},{"f":[33,33,37,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,-1,35,2,2,-1,-1,22,-1,-1,0,-1,-1,-1],"t":[20.752541988539424,13.895530557358477,8.05,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,0,2.5,3.753897320099348,2.903677461028802,0,0,16.832741648970774,0,0,9.5,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,0,0,13,14,0,16,0,18,19,20,0,0,23,0,0,26,0,0,0],"r":[12,5,4,0,0,11,10,9,0,0,0,0,28,15,0,17,0,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.0002714585,0.03621296,0,0,0,-0.002837578,-0.06947134,-0.01032719,0.02512787,0,0,0.03577378,0,-0.0113362,0,0,0,-0.001377679,0.03135811,0,-0.01695878,0.0153108,0,0.01301329,0.06334502,0.04845068]},{"f":[15,37,36,23,-1,2,21,22,23,-1,-1,-1,-1,18,-1,22,-1,-1,17,-1,-1,33,15,-1,-1,21,-1,-1,-1],"t":[13.56045225766691,7.92,3.5,0.5,0,3.5799356683891945,7.5,16.964783007086943,2.5,0,0,0,0,9.229707804263573,0,12.457649623491124,0,0,7.198837316474133,0,0,13.895530557358477,5.169925001442312,0,0,9.5,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,0,0,14,0,16,0,0,19,0,0,22,23,0,0,26,0,0,0],"r":[28,21,18,5,0,13,12,11,10,0,0,0,0,15,0,17,0,0,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0.002374337,0,0,0,0,-0.01831063,-0.04897854,0.002374337,0.002297135,0,0.002374337,0,-0.004599243,-0.0413623,0,-0.00531649,-0.05323033,0,0,0.05180018,0.00721344,0,0.002374337,0.03907194,-0.01816584]},{"f":[22,-1,22,-1,21,-1,18,18,18,18,18,4,2,-1,-1,-1,-1,-1,-1,15,15,-1,-1,21,-1,-1,18,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,0,10.651680794759452,9.273793716589307,9.265614570460752,9.26091905453048,9.135888799148791,9.266718101290426,9.051177541662383,0,0,0,0,0,0,8.618385502258606,5.044394119358453,0,0,7.5,0,0,10.682994316786338,0,0],"m":[0,0,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0],"l":[1,0,3,0,5,0,7,8,9,10,11,12,13,0,0,0,0,0,0,20,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,6,0,26,19,18,17,16,15,14,0,0,0,0,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,-0.02314795,0,0.0605996,0,-0.02031821,0,0,0,0,0,0,0,0.0001227477,0.01649975,-0.02945901,0.04058596,-0.03882634,0.02707376,0,0,-0.006317172,0.05819963,0,-0.04570328,0.004292899,0,0.07613289,0.004180338]},{"f":[25,27,23,-1,21,-1,18,22,-1,18,23,16,-1,0,-1,-1,-1,22,-1,-1,17,-1,2,-1,-1,33,-1,-1,-1],"t":[6.5,0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,4.5,10.501837184902296,0,4.5,0,0,0,17.069460217922263,0,0,8.496347487943302,0,1.292481250360578,0,0,32.199935296219856,0,0,0],"m":[1,1,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,0,10,11,12,0,14,0,0,0,18,0,0,21,0,23,0,0,26,0,0,0],"r":[28,25,4,0,6,0,20,9,0,17,16,13,0,15,0,0,0,19,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0,0.001134893,0,-0.03385489,0,0,-0.02273701,0,0,0,0.04682833,0,-0.02662347,0.03881443,-0.003071736,0,0.04098085,0.004223899,0,-0.03833507,0,0.02962506,-0.02383783,0,0.03083834,-0.001440019,-0.03546117]},{"f":[8,2,1,-1,15,15,32,15,-1,-1,19,-1,-1,22,-1,16,-1,17,-1,-1,18,17,-1,-1,0,-1,-1,-1,-1],"t":[0.5,10.680359255641626,0.3126984126984127,0,10.457380879072534,9.756556322524087,7.491051153478379,8.076815597050832,0,0,1.5,0,0,10.12257207114862,0,9.92481250360578,0,10.415240582199587,0,0,10.620215166896223,9.27958770372691,0,0,3.5,0,0,0,0],"m":[1,1,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,11,0,0,14,0,16,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,20,13,10,9,0,0,12,0,0,15,0,17,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.04729026,0,0,0,0,-0.0001991041,-0.01622991,0,-0.01703066,0.03232403,0,0.0726373,0,0.002465464,0,0.08133166,-0.003246547,0,0,-0.005307552,-0.03497255,0,0.007487656,0.0792107,0.03928104,0.01863938]},{"f":[33,32,33,33,37,23,-1,-1,-1,4,15,16,-1,-1,-1,-1,21,33,-1,2,33,-1,-1,24,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,20.752541988539424,13.895530557358477,8.05,0.5,0,0,0,4.80643424864552,7.266786540694901,5.044394119358453,0,0,0,0,9.5,21.16864017939033,0,1.292481250360578,27.893589438525005,0,0,9.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,12,0,0,0,0,17,18,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,16,9,8,7,0,0,0,15,14,13,0,0,0,0,26,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0.003577248,-0.002376671,0.03438303,0,0,0,-0.00214203,-0.06805924,-0.00969207,0.02384082,0,0,0.03473951,0,0,-0.005273238,-0.03741992,0,0.003833323,0.054813,0.04690393,-0.05073755,0.01757133]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,23,-1,4,2,17,17,-1,-1,-1,22,22,-1,-1,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,3.5,0,6.451187557243013,4.0437314206251695,8.496347487943302,6.434604125275413,0,0,0,17.078739781790915,17.069460217922263,0,0,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,0,14,15,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,13,0,25,20,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,-0.02272625,0,0.05668928,0,0,0,-0.02254475,0.02224281,-0.04233469,0,0,0.002482434,0,0,0,0,-0.005700716,-0.03821915,0.01737904,0,0,0.005565345,-0.04110728,0.04160162,-0.03058842,0,-0.05182257,-0.004855111]},{"f":[22,-1,15,15,22,22,22,22,22,-1,15,-1,-1,4,-1,-1,-1,16,22,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,16.978341770814396,16.73920708776685,14.116600814324018,0,10.376125389276174,0,0,3.753897320099348,0,0,0,7.774787059601174,17.078739781790915,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,0,0,14,0,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,17,16,13,10,0,12,0,0,15,0,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.03611249,0,0,0,0,0,0,0,-0.001037258,0,0.01519008,-0.007656955,0,-0.0237626,0.007648706,0.01774712,0,0,-0.05406541,-0.009209838,0.008465644,0,0,-0.01592934,0.03762167,-0.02386461,0.02318288,-0.01717751]},{"f":[25,22,-1,22,16,-1,-1,1,17,21,15,22,21,-1,-1,-1,22,-1,17,-1,-1,-1,-1,1,1,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.01685992705101,11.651826559171445,7.5,7.417852514885898,10.194756329264862,5.5,0,0,0,14.247293309538719,0,2.453445297804259,0,0,0,0,2.969223484848485,2.532051282051282,0,0,0,0],"m":[1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,10,11,12,13,0,0,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,3,0,7,6,0,0,23,22,21,16,15,14,0,0,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,-0.03570673,0,0,0.00137873,0.03366261,0,0,0,0,0,0,0.07896909,-0.02110503,-0.04260297,0,-0.0178005,0,-0.03810966,0.05149144,1.244076e-05,0.02250619,0,0,0.00149073,0.06162453,0.0004161218,-0.03397166]},{"f":[33,32,33,22,22,1,16,-1,-1,1,-1,-1,18,-1,-1,17,22,-1,-1,-1,33,22,37,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,12.199665042528586,10.27595502635321,1.13125,10.46352437327118,0,0,2.01685992705101,0,0,10.609627990977145,0,0,0.5,15.717370311433935,0,0,0,39.9785486130134,16.98920034885663,9.629999999999999,0,0,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,13,0,0,16,17,0,0,0,21,22,23,0,0,0,0,0,0],"r":[28,27,20,15,12,9,8,0,0,11,0,0,14,0,0,19,18,0,0,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0.004270737,0.04692356,0,-0.03020038,0.001639145,0,-0.01461203,0.04533336,0,0,-0.03929696,0.04715554,0.002448613,0,0,0,0.01650566,0.04842568,-0.01015498,-0.02285446,-0.04992887,0.01664914]},{"f":[22,-1,25,15,37,36,23,-1,2,21,22,-1,-1,19,-1,-1,-1,17,-1,-1,33,15,-1,-1,36,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,13.56045225766691,7.92,3.5,0.5,0,3.5799356683891945,6.5,16.964783007086943,0,0,0.5,0,0,0,7.198837316474133,0,0,15.81450217830399,5.169925001442312,0,0,5.5,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,0,0,14,0,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,27,20,17,8,0,16,13,12,0,0,15,0,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.0352937,0,0,0,0,0,0.002413458,0,0,0,-0.03520479,0.002413458,0,0.002413458,-0.016439,0.0004723459,0,-0.005069178,-0.05217482,0,0,0.05170425,0.004999724,0,0.002413458,0.03809208,-0.01652327,-0.03319]},{"f":[2,8,1,-1,22,21,1,-1,1,-1,-1,-1,22,18,1,-1,-1,36,22,-1,-1,15,-1,-1,22,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,12.199665042528586,5.5,1.000936329588015,0,3.354166666666667,0,0,0,12.921158001713904,1.292481250360578,6.806765389082463,0,0,2.5,12.613886328427084,0,0,10.457380879072534,0,0,13.415167820124083,0,0,0,0],"m":[1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,0,13,14,15,0,0,18,19,0,0,22,0,0,25,0,0,0,0],"r":[28,27,4,0,12,11,8,0,10,0,0,0,24,17,16,0,0,21,20,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.04658577,0,0,0,0.05527156,0,-0.04178605,-0.008136349,0.0007364046,0,0,0,0.009859745,-0.04234059,0,0,-0.02946419,0.04828823,0,0.03026439,-0.02794717,0,-0.02978623,0.001300487,0.01919271,0.03072546]},{"f":[2,21,22,22,-1,-1,-1,1,-1,4,4,1,-1,-1,-1,4,-1,-1,3,3,-1,17,-1,-1,16,16,-1,-1,-1],"t":[9.051177541662383,2.5,13.827286379004683,6.106096185954236,0,0,0,0.39811320754716983,0,7.950787011501063,7.408266855279292,4.92309205350118,0,0,0,8.950330235806282,0,0,0.9640400250156348,0.00554530545305453,0,9.44286754736589,0,0,11.031356596255709,4.247927513443585,0,0,0],"m":[1,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,0,8,0,10,11,12,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[18,7,6,5,0,0,0,9,0,15,14,13,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0.01606443,-0.04576444,0.02207317,0,-0.04322778,0,0,0,-0.002904912,0.002317431,0.02181977,0,-0.04028141,0.002935224,0,0,-0.0120038,0,0.06067694,0.003668424,0,0,-0.003980463,0.02937165,-0.03344942]},{"f":[22,-1,22,15,4,21,-1,-1,22,-1,22,0,-1,-1,15,-1,-1,-1,22,-1,22,-1,17,22,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.770639075561187,10.066089190457772,1.292481250360578,6.5,0,0,5.9525054311951475,0,6.671801068872866,6.5,0,0,8.361943773735241,0,0,0,8.133841411710264,0,12.199665042528586,0,0.5,15.717370311433935,0,0,9.012591414317754,0,0],"m":[0,0,0,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,0,0,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,18,17,8,7,0,0,10,0,14,13,0,0,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.0348474,0,0,0,0,-0.03992865,0.004595526,0,0.04870009,0,0,-0.04307294,-0.0003253425,0,0.01270228,-0.03122984,0.03873067,0,-0.04189229,0,-0.00295863,0,0,-0.03859241,0.0452805,0,0.001277438,0.01955989]},{"f":[33,32,33,3,2,-1,2,21,-1,-1,-1,23,-1,21,-1,18,-1,-1,33,22,37,-1,-1,22,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,29.297756302860655,0.32902480033627574,6.055241654908112,0,9.075473573191719,7.5,0,0,0,3.5,0,6.5,0,6.499823868264185,0,0,39.9785486130134,16.98920034885663,9.629999999999999,0,0,17.078739781790915,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,12,0,14,0,16,0,0,19,20,21,0,0,24,0,0,0,0,0],"r":[28,27,18,11,6,0,10,9,0,0,0,13,0,15,0,17,0,0,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.04444242,0,0,-0.03354237,-0.002899533,0.03339488,0,0.002084142,0,-0.02795478,0,0.006215366,-0.0176704,0,0,0,0.0155726,0.04674452,0,-0.03162319,0.01221448,-0.02201819,-0.04932598,0.01616105]},{"f":[22,23,21,4,32,15,15,22,-1,-1,-1,-1,-1,2,22,-1,-1,24,-1,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,3.084962500721156,10.463013322934223,7.924812503605781,7.721099188707185,14.03140051979143,0,0,0,0,0,3.245926548164837,13.415167820124083,0,0,9.5,0,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,0,14,15,0,0,18,0,0,0,0,23,24,0,26,0,0,0],"r":[22,21,20,13,12,11,10,9,0,0,0,0,0,17,16,0,0,19,0,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.01061683,0.007784138,-0.04411773,0.005209784,-0.03398798,0,0,0.0008644373,0.0406258,0,-0.0001781357,0.03884771,0.04138301,-0.02440777,0,0,-0.01488895,0,0.01316324,0.05681217,-0.02277068]},{"f":[27,15,22,-1,-1,15,-1,1,18,-1,22,22,-1,-1,-1,15,25,-1,-1,18,19,-1,-1,24,-1,-1,33,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,0,0,6.108524456778169,0,4.92309205350118,10.620215166896223,0,12.457649623491124,11.228801658478353,0,0,0,6.857980995127572,1.5,0,0,6.864066550153108,2.5,0,0,9.5,0,0,32.199935296219856,0,0],"m":[1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,0,0,0,16,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[26,5,4,0,0,7,0,15,10,0,14,13,0,0,0,19,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0.001035242,0.04265881,0,-0.03188543,0,0,-0.009290596,0,0,-0.005644319,0.06624457,-0.006614333,0,0,0.003197538,0.03481111,0,0,0.002501974,0.05039567,0,-0.05072342,-0.00585167,0,0.02879898,-0.001896365]},{"f":[2,8,1,-1,15,15,-1,22,-1,16,-1,17,-1,-1,21,33,15,15,-1,-1,-1,-1,23,-1,22,-1,-1,-1,-1],"t":[10.680359255641626,0.5,0.3126984126984127,0,10.457380879072534,9.756556322524087,0,10.12257207114862,0,9.92481250360578,0,10.415240582199587,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,0.5,0,16.73920708776685,0,0,0,0],"m":[1,1,0,0,1,1,0,0,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,4,0,14,7,0,9,0,11,0,13,0,0,22,21,20,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,-0.04593672,0,0,-0.0004167475,0,0.06683724,0,0.002277786,0,0.07594551,-0.00400826,0,0,0,0,-0.03010192,0.0142536,-0.05115349,0.01516818,0,0.03467046,0,-0.008392041,0.02457925,0.01800276,0.03007879]},{"f":[22,-1,15,15,22,22,22,22,22,-1,-1,4,-1,22,-1,-1,-1,18,-1,18,-1,-1,18,1,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,13.56045225766691,13.323758479706308,17.094543568094064,17.069460217922263,16.978341770814396,16.73920708776685,16.25454052700782,0,0,3.753897320099348,0,16.819976309430963,0,0,0,6.076776015854056,0,7.383290334558488,0,0,8.729619742794132,7.9752100840336135,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,12,0,14,0,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[2,0,28,27,22,17,16,11,10,0,0,13,0,15,0,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.03450026,0,0,0,0,0,0,0,0.0002936113,0.02296362,0,-0.02263418,0,-0.01793057,0.02168535,0.01732284,0,-0.03491052,0,0.03177174,-0.02586941,0,0,0.03693904,-0.008089789,-0.02241954,0.02159721,-0.01561999]},{"f":[22,-1,25,2,8,1,-1,15,15,-1,22,-1,16,-1,-1,18,17,-1,-1,0,15,-1,-1,22,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,10.680359255641626,0.5,0.3126984126984127,0,10.457380879072534,9.756556322524087,0,10.12257207114862,0,9.92481250360578,0,0,10.620215166896223,9.27958770372691,0,0,2.5,11.683433292835403,0,0,13.589275375459419,0,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,11,0,13,0,0,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,28,27,26,7,0,15,10,0,12,0,14,0,0,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,-0.03409807,0,0,0,0,-0.04539693,0,0,-0.0001328235,0,0.06194904,0,0.002144281,0.0266071,0,0,-0.004803607,-0.0339822,0,0,-0.0429613,0.01509952,0,0.08312032,0.02578596,0.01715231,0.02952665,-0.03189176]},{"f":[4,32,4,15,-1,16,17,17,16,15,18,-1,-1,-1,-1,-1,-1,22,-1,-1,-1,-1,33,2,15,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,2.903677461028802,2.0,0,10.501837184902296,10.37503779525057,9.075473573191719,7.930737337562887,7.118941072723508,1.792481250360578,0,0,0,0,0,0,11.683871801979876,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,11,0,0,0,0,0,0,18,0,0,0,0,23,24,25,0,0,0,0],"r":[22,21,20,5,0,17,16,15,14,13,12,0,0,0,0,0,0,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-0.03642264,0,0,0,0,0,0,-0.008782032,0.01208001,-0.01539432,0.01115627,0.03854479,-0.0304719,0,0.03617102,-0.0002818038,-0.04304733,-0.03282471,0,0,0,0.02929073,-0.04577661,0.0001435927,0.03697725]},{"f":[27,15,22,-1,-1,15,-1,1,21,-1,-1,15,33,17,-1,-1,-1,18,-1,24,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,0,0,6.108524456778169,0,4.92309205350118,6.5,0,0,6.857980995127572,21.257266042509173,8.729619742794132,0,0,0,6.864066550153108,0,9.5,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,0,1,0,0,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,14,0,0,0,18,0,20,0,0,23,0,25,0,27,0,0],"r":[22,5,4,0,0,7,0,11,10,0,0,17,16,15,0,0,0,19,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0.001030086,0.03907527,0,-0.0308897,0,0,-0.02259926,-0.001792785,0,0,0,0.0416159,-0.01868563,0.0006853423,0,0.003672269,0,-0.04876935,-0.005876308,0,0.02794371,0,-0.02058915,0,0.02694352,-0.01717436]},{"f":[4,2,2,-1,-1,3,-1,-1,15,16,3,-1,23,22,15,-1,-1,-1,-1,26,-1,-1,22,18,-1,18,-1,-1,-1],"t":[4.951187557243013,8.612669628062957,7.005613627711627,0,0,0.00554530545305453,0,0,10.376125389276174,6.882643049361842,0.5803571428571428,0,4.5,14.03140051979143,5.78135971352466,0,0,0,0,0.5,0,0,16.725912049439096,1.292481250360578,0,9.273793716589307,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,14,15,0,0,0,0,20,0,0,23,24,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,19,12,0,18,17,16,0,0,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,-0.00146533,-0.03427049,0,-0.01129313,0.05030356,0,0,0,-0.03543203,0,0,0,0.006005043,-0.01800662,0.04110833,-0.02266304,0,0.0411355,0.009645151,0,0,-0.03721798,0,0.002133525,-0.03131865,0.01883526]},{"f":[22,-1,22,-1,21,17,4,-1,-1,-1,30,37,15,-1,16,-1,22,16,-1,-1,0,-1,-1,24,-1,-1,21,-1,-1],"t":[4.926654777701837,0,5.9525054311951475,0,2.5,1.292481250360578,4.951187557243013,0,0,0,1.5,11.26,5.169925001442312,0,6.108524456778169,0,12.199665042528586,7.6653359171851765,0,0,4.5,0,0,12.5,0,0,3.5,0,0],"m":[0,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,0,0,11,12,13,0,15,0,17,18,0,0,21,0,0,24,0,0,27,0,0],"r":[2,0,4,0,10,9,8,0,0,0,26,23,14,0,16,0,20,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,-0.0203116,0,0.05136133,0,0,0,-0.02133435,0.02094251,-0.04151091,0,0,0,0.003956935,0,-0.01284733,0,0,-0.02160912,-0.000993997,0,-0.008085439,0.00685698,0,0.03807201,0.007949522,0,-0.04891911,-0.004511346]},{"f":[21,17,32,-1,16,-1,22,-1,-1,4,22,16,21,-1,-1,16,22,2,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1],"t":[9.5,1.292481250360578,3.753897320099348,0,3.169925001442312,0,16.786010507999382,0,0,10.41668595978746,12.121442107629743,10.46352437327118,4.5,0,0,11.868050853745158,11.67903991803059,2.453445297804259,0,0,0,0,0,0,15.756821747580634,7.5,0,0,0],"m":[1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,12,13,0,0,16,17,18,0,0,0,0,0,0,25,26,0,0,0],"r":[24,9,4,0,6,0,8,0,0,23,22,15,14,0,0,21,20,19,0,0,0,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.03969274,0,-0.01581501,0,0.002800823,-0.04287218,0,0,0,0,-0.04186954,-0.007481976,0,0,0,-0.03025258,0.0302263,0.0651291,-0.004238073,0.003730774,0.04607086,0,0,0.007644117,0.04667794,-0.03674767]},{"f":[27,21,16,17,15,-1,-1,-1,4,2,2,-1,16,-1,-1,-1,-1,22,0,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,9.5,1.584962500721156,2.160964047443681,3.4594316186372973,0,0,0,9.328332883600808,9.293467526905953,9.075473573191719,0,10.826548487290914,0,0,0,0,15.756821747580634,7.5,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,0,1,1,0,0,0,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,13,0,0,0,0,18,19,0,0,0,23,0,25,0,27,0,0],"r":[22,17,8,7,6,0,0,0,16,15,12,0,14,0,0,0,0,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0,0,-0.03887765,-0.002588995,0.009320053,0,0,0,-0.0008234847,0,0.003289547,0.05154509,-0.04013943,0.02040004,0,0,0.007252843,0.04612885,-0.03596093,0,0.02704894,0,-0.0194308,0,0.026615,-0.01599409]},{"f":[27,15,22,15,21,18,-1,-1,-1,16,17,-1,22,22,16,-1,-1,-1,-1,18,-1,-1,-1,15,-1,-1,33,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,4.087462841250339,7.5,1.792481250360578,0,0,0,6.409390936137702,1.292481250360578,0,13.054613167076562,12.613886328427084,4.247927513443585,0,0,0,0,3.084962500721156,0,0,0,6.108524456778169,0,0,32.199935296219856,0,0],"m":[1,0,1,1,1,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,13,14,15,0,0,0,0,20,0,0,0,24,0,0,27,0,0],"r":[26,23,22,9,8,7,0,0,0,19,12,0,18,17,16,0,0,0,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.02381931,0.005381404,0.003122376,0,0,0.02190813,0,0,0,-0.02637937,0.005978022,0.04486072,-0.01902183,0,0.009360407,0.05838593,0.03693146,0,-0.0294905,-0.0006045397,0,0.02576842,-0.001151966]},{"f":[22,23,21,4,32,15,-1,17,15,22,-1,-1,-1,-1,-1,24,3,-1,-1,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,2.453445297804259,10.463013322934223,2.0,0,0.5,9.862637357558794,8.643851681338631,0,0,0,0,0,9.5,0.0339080459770115,0,0,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,8,9,10,0,0,0,0,0,16,17,0,0,0,0,0,23,24,0,26,0,0,0],"r":[22,21,20,15,14,7,0,13,12,11,0,0,0,0,0,19,18,0,0,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.03771356,0,0,0,0.01735183,-0.0390538,0.04532453,-0.002345566,-0.03041858,0,0,0.03684941,0.0003128534,0.03673299,0.03830928,-0.02334418,0,0,-0.01258525,0,0.01218302,0.05320149,-0.02233006]},{"f":[25,4,4,32,15,-1,4,2,18,17,16,-1,-1,-1,-1,37,-1,-1,-1,-1,-1,33,2,15,-1,-1,-1,-1,-1],"t":[6.5,3.084962500721156,2.903677461028802,10.463013322934223,2.0,0,2.453445297804259,1.792481250360578,3.245926548164837,1.792481250360578,7.930737337562887,0,0,0,0,6.335,0,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0,0,0,0,0],"m":[1,1,1,1,0,0,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,10,11,0,0,0,0,16,0,0,0,0,0,22,23,24,0,0,0,0,0],"r":[28,21,20,19,6,0,18,15,14,13,12,0,0,0,0,17,0,0,0,0,0,27,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03501134,0,0,0,0,0,-0.007854834,0.020726,0.02703403,-0.005615446,0,-0.03310545,0.001009192,0.02404281,-0.03046935,-0.04253979,0,0,0,0.02753548,-0.045023,0.0002251664,0.03502533,-0.03084417]},{"f":[22,23,21,15,-1,3,-1,21,33,-1,-1,23,-1,22,-1,-1,-1,22,33,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,10.457380879072534,0,0.26136363636363635,0,5.5,37.46769166150554,0,0,0.5,0,16.759160379664443,0,0,0,16.98920034885663,26.622428940176302,0,0,0,8.729619742794132,8.341935934311302,0,8.571721503073015,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,0,14,0,0,0,18,19,0,0,0,23,24,0,26,0,0,0],"r":[22,17,16,5,0,7,0,11,10,0,0,13,0,15,0,0,0,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0.0007866337,0,0.02917926,0,0,-0.03165145,0.0002664779,0,0.02907745,0,-0.00933823,0.03652526,0.03578635,0,0,-0.02635081,0.02512382,-0.03843597,0,0,-0.0114683,0,0.01295004,0.0541117,-0.02146951]},{"f":[4,2,2,-1,-1,3,-1,-1,15,16,-1,18,2,1,-1,-1,1,-1,-1,18,-1,-1,22,18,-1,18,-1,-1,-1],"t":[4.951187557243013,8.612669628062957,7.005613627711627,0,0,0.00554530545305453,0,0,10.376125389276174,7.651051691178929,0,6.076776015854056,6.033380966193453,8.102912599421547,0,0,1.987429289754871,0,0,8.03615005796417,0,0,16.725912049439096,1.292481250360578,0,9.273793716589307,0,0,0],"m":[1,1,1,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,14,0,0,17,0,0,20,0,0,23,24,0,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,11,0,19,16,15,0,0,18,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,-0.001348156,-0.03259894,0,-0.01118427,0.04722752,0,0,0.0002006886,0,0,0,0.06392461,0.0005316531,0,0.01392467,-0.02292956,0,0.0406101,0.003673919,0,0,-0.03586346,0,0.001891895,-0.03084241,0.01771647]},{"f":[27,22,22,-1,-1,22,-1,17,16,-1,-1,23,22,-1,-1,1,-1,1,-1,33,-1,16,22,-1,-1,-1,33,-1,-1],"t":[0.5,16.786010507999382,16.765402642760364,0,0,16.807008825964132,0,3.39067985676233,3.321928094887362,0,0,2.5,16.838461650489297,0,0,1.282051282051282,0,3.015625,0,23.450447293378375,0,7.924812503605781,17.078739781790915,0,0,0,32.199935296219856,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,0,0,16,0,18,0,20,0,22,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,11,10,0,0,15,14,0,0,17,0,19,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0,-4.580222e-05,0.05116745,0,-0.03499084,0,0,-0.003094783,-0.04449653,0,0,0.01057429,0.05996108,0,0.04495647,0,-0.03503925,0,-0.02139206,0,0,-0.02403295,0.01145945,0.02192115,0,0.0251988,-0.001252562]},{"f":[25,22,4,15,-1,23,15,-1,-1,32,-1,-1,35,22,22,-1,22,-1,23,-1,-1,-1,-1,17,22,-1,-1,-1,-1],"t":[6.5,12.199665042528586,1.292481250360578,9.862637357558794,0,0.5,10.681238411777805,0,0,10.463013322934223,0,0,2.5,9.40390385294334,8.61260371847175,0,8.643851681338631,0,0.5,0,0,0,0,0.5,15.717370311433935,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,12,5,0,9,8,0,0,11,0,0,22,21,16,0,18,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02845477,0,0,0.07609298,0.005413449,0,-0.001088143,-0.0434042,0,0,0,-0.0002853763,0,0.05841484,0,-0.02240015,0.03698431,-0.007016873,0.02880697,0,0,-0.03750356,0.03998551,0.002322317,-0.03046169]},{"f":[2,37,32,23,36,-1,-1,2,21,22,23,21,-1,-1,-1,-1,19,2,-1,-1,-1,-1,18,-1,-1,21,-1,-1,-1],"t":[10.680359255641626,7.92,6.6765734127490415,0.5,2.5,0,0,4.0437314206251695,7.5,16.978341770814396,2.5,5.5,0,0,0,0,1.5,1.792481250360578,0,0,0,0,2.160964047443681,0,0,9.5,0,0,0],"m":[1,1,1,0,0,0,0,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,11,12,0,0,0,0,17,18,0,0,0,0,23,0,0,26,0,0,0],"r":[28,25,22,7,6,0,0,21,16,15,14,13,0,0,0,0,20,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.02812402,0.00132493,0,0,0,0,0,-0.03188234,-0.002490756,-0.04846759,0.00132493,0,0,0.00132493,-0.04103839,-0.03618565,-0.0003603635,0,-0.004077653,-0.04622951,0,0.002575392,0.03504305,0.02787434]},{"f":[22,-1,22,1,-1,15,-1,16,-1,-1,1,17,18,17,21,15,-1,-1,-1,-1,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,1.000936329588015,0,3.169925001442312,0,5.614709844115208,0,0,2.01685992705101,11.651826559171445,11.26283925556662,11.207174616271544,7.5,9.92481250360578,0,0,0,0,0,0,2.9611191860465116,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,14,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,22,21,20,19,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.03239367,0,0,0.02018731,0,-0.03587803,0,0.0634071,0.00104425,0,0,0,0,0,0,-0.02581246,-0.0005323232,-0.0001515717,0.0443586,-0.05209633,0.01835752,0,0,-0.0005926011,0.06243899,0,-0.0001253904,0.03017675]},{"f":[25,22,4,15,-1,23,15,-1,-1,33,-1,-1,35,1,-1,22,22,-1,-1,-1,36,-1,-1,17,22,-1,-1,-1,-1],"t":[6.5,12.199665042528586,1.292481250360578,9.862637357558794,0,0.5,10.681238411777805,0,0,44.805779984159756,0,0,2.5,0.01933482600988898,0,9.40390385294334,8.61260371847175,0,0,0,3.5,0,0,0.5,15.717370311433935,0,0,0,0],"m":[1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,0,16,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,12,5,0,9,8,0,0,11,0,0,20,15,0,19,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02755856,0,0,0.06891497,0.004914856,0,-0.001197858,-0.04355167,0,0,0.04063529,0,0,-0.0003675541,0.02587335,-0.008045766,0,0.008725815,0.04978249,0,0,-0.03675614,0.03842542,0.002311892,-0.02998506]},{"f":[2,1,-1,21,22,22,-1,-1,-1,4,4,-1,-1,4,-1,-1,20,2,16,-1,-1,18,21,-1,-1,15,-1,-1,-1],"t":[9.051177541662383,0.39811320754716983,0,2.5,13.827286379004683,6.106096185954236,0,0,0,7.950787011501063,7.408266855279292,0,0,8.950330235806282,0,0,1.5,9.241970852671834,10.372865060112588,0,0,10.457893746154532,7.5,0,0,9.862637357558794,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,0,0,14,0,0,17,18,19,0,0,22,23,0,0,26,0,0,0],"r":[16,3,0,9,8,7,0,0,0,13,12,0,0,15,0,0,28,21,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,-0.04186884,0,0,0,0.01769125,-0.04515822,0.02227478,0,0,0.0001868565,0.01840334,0,-0.03962944,0.003375661,0,0,0,0.005754393,0.0399687,0,0,-0.0407683,0.002770523,0,0.04105563,-0.01852386,0.04236688]},{"f":[22,-1,33,21,36,4,-1,-1,-1,24,18,-1,-1,33,-1,18,-1,-1,33,-1,37,32,-1,-1,2,-1,35,-1,-1],"t":[2.69615871138938,0,20.752541988539424,3.5,3.5,4.128693921346326,0,0,0,7.5,6.531360383582522,0,0,13.895530557358477,0,4.245926548164837,0,0,21.16864017939033,0,9.205,10.41931383093672,0,0,1.292481250360578,0,2.5,0,0],"m":[0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,0,14,0,16,0,0,19,0,21,22,0,0,25,0,27,0,0],"r":[2,0,18,9,8,7,0,0,0,13,12,0,0,15,0,17,0,0,20,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.03195651,0,0,0,0,-0.06372255,0.006175873,-0.0077718,0,0,-0.00163461,-0.02773952,0,0.04538962,0,-0.01592748,0.02601571,0,0.03076367,0,0,0.0004107926,-0.01870973,0,0.00528633,0,0.01051736,0.05750603]},{"f":[27,23,-1,21,-1,18,22,-1,18,23,16,-1,-1,-1,22,-1,22,-1,-1,1,-1,22,16,-1,-1,-1,33,-1,-1],"t":[0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,4.5,10.501837184902296,0,0,0,17.069460217922263,0,17.074218326240924,0,0,1.99834912080135,0,17.094543568094064,8.618385502258606,0,0,0,32.199935296219856,0,0],"m":[1,1,0,0,0,1,0,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,10,11,0,0,0,15,0,17,0,0,20,0,22,23,0,0,0,27,0,0],"r":[26,3,0,5,0,19,8,0,14,13,12,0,0,0,16,0,18,0,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0.000824726,0,-0.02963807,0,0,-0.02040781,0,0,0,0.04244725,-0.00792386,-0.003179793,0,0.03775435,0,-0.02945221,0.03636092,0,0.03644093,0,0,-0.04708406,-0.003223382,0.003048554,0,0.02440747,-0.001498673]},{"f":[4,4,21,22,22,-1,-1,-1,2,1,-1,4,4,30,-1,21,-1,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,13.87915215758784,6.106096185954236,0,0,0,9.051177541662383,0.39811320754716983,0,7.754129516000594,7.408266855279292,1.5,0,3.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,12,13,14,0,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,7,6,0,0,0,22,11,0,19,18,15,0,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0.01694725,-0.04493482,0.02167449,0,0,-0.04129783,0,0,0,0.0007716419,0,-0.04556525,-0.005025756,0.01990549,0,-0.03398605,0.003076303,0,0.00313436,0.03812477,-0.01562184,0,-0.007925638,0.04221917]},{"f":[22,-1,22,1,-1,15,-1,16,-1,-1,1,22,-1,22,21,-1,-1,21,-1,17,-1,-1,1,1,-1,-1,2,-1,-1],"t":[2.69615871138938,0,7.082728314749225,1.000936329588015,0,3.169925001442312,0,5.614709844115208,0,0,2.01685992705101,8.133841411710264,0,9.087343393278285,7.5,0,0,3.5,0,11.364132993990529,0,0,2.969223484848485,2.532051282051282,0,0,9.299206200930918,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,0,14,15,0,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,22,13,0,17,16,0,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.03158437,0,0,0.01914193,0,-0.03533647,0,0.06007071,0.0009043419,0,0,-0.03723484,0,0,0.09265711,0.008415869,0,-0.04255008,0,-0.005447855,0.01721488,0,0,-0.0004495164,0.05519666,0,-0.00016906,0.02901499]},{"f":[22,-1,22,18,1,-1,-1,36,22,-1,-1,15,16,2,1,-1,-1,22,17,-1,-1,-1,-1,-1,22,-1,15,-1,-1],"t":[12.199665042528586,0,12.921158001713904,1.292481250360578,6.806765389082463,0,0,2.5,12.613886328427084,0,0,10.457380879072534,8.321928094887362,1.292481250360578,7.0046728971962615,0,0,12.679695715883515,11.364132993990529,0,0,0,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,14,15,0,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[2,0,24,7,6,0,0,11,10,0,0,23,22,17,16,0,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,-0.001813116,0,0,0,0.007316444,-0.04011865,0,0,-0.02735308,0.04557229,0,0,0,0,0.07023691,0.01833549,0,0,-0.01525961,0.0266005,0.03245897,0.06241558,-0.02431888,0,-0.02783277,0,0.006726407,-0.002320807]},{"f":[27,23,-1,21,-1,18,22,-1,18,-1,-1,1,-1,22,16,-1,-1,17,-1,-1,22,22,0,-1,-1,-1,22,-1,-1],"t":[0.5,3.5,0,6.5,0,6.499823868264185,13.179405754316653,0,5.303665156874805,0,0,1.99834912080135,0,17.094543568094064,8.618385502258606,0,0,8.729619742794132,0,0,16.98920034885663,16.978341770814396,8.5,0,0,0,17.078739781790915,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,0,4,0,6,7,0,9,0,0,12,0,14,15,0,0,18,0,0,21,22,23,0,0,0,27,0,0],"r":[20,3,0,5,0,11,8,0,10,0,0,13,0,17,16,0,0,19,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0.0007819701,0,-0.02857849,0,0,-0.01943657,0,0.001000318,0.02340619,0,0.03517011,0,0,-0.04644577,-0.002915101,0,0.01959187,-0.0281862,0,0,0,-0.02775744,0.0186644,0.07011168,0,-0.02898187,0.01858902]},{"f":[25,1,15,22,22,-1,-1,-1,15,-1,18,1,-1,2,16,23,-1,-1,-1,2,-1,-1,1,-1,-1,1,-1,-1,-1],"t":[6.5,4.92309205350118,5.78135971352466,16.819976309430963,16.73920708776685,0,0,0,7.266786540694901,0,10.620215166896223,1.987429289754871,0,3.245926548164837,7.741466986401147,1.5,0,0,0,6.139515068224824,0,0,3.015625,0,0,4.949128919860627,0,0,0],"m":[1,1,0,1,1,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,0,11,12,0,14,15,16,0,0,0,20,0,0,23,0,0,26,0,0,0],"r":[28,25,8,7,6,0,0,0,10,0,22,13,0,19,18,17,0,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,0.002199906,-0.02030799,0.03018442,0,-0.02550686,0,0,0.01059011,0,0,0,-0.02162832,0.05591538,-0.04292744,0,0.01165886,-0.008210649,0,0.004111009,0.06448319,0,0.04286975,0.001379564,-0.02920992]},{"f":[22,-1,22,1,-1,1,-1,18,0,-1,-1,22,-1,16,-1,-1,22,-1,15,-1,3,-1,18,22,-1,-1,0,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,6.5,0,0,6.870153907126159,0,6.6865005271832185,0,0,8.133841411710264,0,10.457380879072534,0,0.26136363636363635,0,10.620215166896223,16.725912049439096,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,16,5,0,7,0,11,10,0,0,13,0,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03115883,0,0,0.01786936,0,-0.02449956,0,0,-0.03491359,0.0003025789,0,0.05139884,0,-0.002607587,0.0210302,0,-0.04041813,0,0.0008830353,0,0.02683725,0,0,-0.01340833,0.01318489,0,-0.004594597,0.04559774]},{"f":[33,32,33,33,3,2,-1,2,21,-1,-1,3,-1,-1,18,21,-1,-1,-1,22,37,-1,-1,22,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,29.297756302860655,0.32902480033627574,6.055241654908112,0,8.612669628062957,7.5,0,0,0.00554530545305453,0,0,1.792481250360578,7.5,0,0,0,16.98920034885663,9.629999999999999,0,0,17.078739781790915,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,15,16,0,0,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,26,19,14,7,0,11,10,0,0,13,0,0,18,17,0,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.04155677,0,0,-0.04578689,-0.002924064,0,-0.01310025,0.04458219,0,0,-0.007510666,0.005314759,0.002786034,0,0,0.01424358,0.04329861,0,-0.0296961,0.01285596,-0.02273425,-0.04610887,0.01356457]},{"f":[22,-1,22,16,22,36,-1,18,22,-1,-1,2,24,-1,-1,17,-1,-1,15,-1,-1,15,-1,-1,22,-1,15,-1,-1],"t":[12.199665042528586,0,12.921158001713904,7.930737337562887,12.613886328427084,2.5,0,1.792481250360578,12.278990886161445,0,0,1.292481250360578,4.5,0,0,11.364132993990529,0,0,4.247927513443585,0,0,10.457380879072534,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,1,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,0,12,13,0,0,16,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,24,21,18,7,0,11,10,0,0,15,14,0,0,17,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,-0.00177763,0,0,0,0,-0.02647477,0,0,-0.03421813,0.02023357,0,0,0.02128815,0.06526815,0,-0.03923757,0.02538865,0,0.04554109,-0.0008853601,0,0.03822801,-0.02403069,0,-0.02658656,0,0.006403141,-0.002166651]},{"f":[1,-1,1,-1,16,1,-1,16,16,-1,-1,4,-1,-1,15,33,4,17,-1,-1,26,-1,1,-1,-1,23,-1,-1,-1],"t":[4.92309205350118,0,4.949128919860627,0,6.108524456778169,5.417948717948718,0,5.044394119358453,4.523561956057013,0,0,4.80643424864552,0,0,10.457380879072534,20.752541988539424,4.0437314206251695,6.636169092079312,0,0,0.5,0,6.147321428571429,0,0,3.5,0,0,0],"m":[1,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,0,8,9,0,0,12,0,0,15,16,17,18,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,4,0,14,7,0,11,10,0,0,13,0,0,28,25,20,19,0,0,22,0,24,0,0,27,0,0,0],"v":[0,-0.001578708,0,0.04227986,0,0,-0.04340054,0,0,0.0009605669,0.04145586,0,-0.0425108,-0.0009419996,0,0,0,0,-0.007750911,-0.06940017,0,0.04069515,0,-0.03090671,0.006581522,0,0.01221195,-0.001705005,-0.005287951]},{"f":[33,32,33,33,3,2,-1,2,21,-1,-1,3,-1,-1,37,-1,33,-1,21,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,0.32902480033627574,6.055241654908112,0,8.612669628062957,7.5,0,0,0.00554530545305453,0,0,8.05,0,18.39964081076096,0,9.5,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,15,0,17,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,14,7,0,11,10,0,0,13,0,0,16,0,18,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.04052649,0,0,-0.03510759,-0.002695856,0,-0.01241086,0.04347556,0,-3.426364e-05,0,0.02765346,0,0.004394954,0.04323707,0,0,0.01386919,0.05107421,-0.0261251,-0.02171692,-0.04553307,0.01325531]},{"f":[22,-1,22,1,-1,1,-1,18,0,-1,-1,22,-1,16,-1,-1,22,-1,15,-1,3,-1,18,18,-1,-1,0,-1,-1],"t":[2.69615871138938,0,7.770639075561187,1.000936329588015,0,2.0015923566878984,0,3.084962500721156,6.5,0,0,6.870153907126159,0,6.6865005271832185,0,0,8.133841411710264,0,10.457380879072534,0,0.26136363636363635,0,10.620215166896223,9.273793716589307,0,0,2.5,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,12,0,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,16,5,0,7,0,11,10,0,0,13,0,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03076566,0,0,0.01711783,0,-0.02383297,0,0,-0.03415549,0.000101996,0,0.04784915,0,-0.002492942,0.01969234,0,-0.03972442,0,0.0007864858,0,0.0247203,0,0,-0.007160826,-0.04125801,0,-0.004229439,0.04274598]},{"f":[27,22,22,-1,-1,22,17,-1,-1,15,-1,17,-1,15,18,-1,22,-1,17,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,16.786010507999382,16.765402642760364,0,0,16.819976309430963,8.378590816408359,0,0,5.044394119358453,0,3.39067985676233,0,10.113742166049189,6.4178278454199,0,17.083799521744545,0,8.729619742794132,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,10,0,12,0,14,15,0,17,0,19,0,0,0,23,0,25,0,27,0,0],"r":[22,5,4,0,0,9,8,0,0,11,0,13,0,21,16,0,18,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,-2.660957e-05,0.04997001,0,0,-0.04619802,-0.01273923,0,0.03178635,0,-0.03118849,0,0,0.008800678,0,-0.03303198,0,0.01807895,-0.02746346,0.04874634,0,0.02290535,0,-0.01848068,0,0.0245664,-0.01491572]},{"f":[33,32,33,33,33,33,22,22,-1,-1,22,-1,15,-1,18,-1,-1,-1,17,-1,-1,37,33,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,29.297756302860655,27.27007187651612,24.56142327888218,16.786010507999382,16.765402642760364,0,0,16.819976309430963,0,5.044394119358453,0,3.39067985676233,0,0,0,2.903677461028802,0,0,9.629999999999999,31.22997243441001,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,0,13,0,15,0,0,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,18,17,10,9,0,0,12,0,14,0,16,0,0,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-2.141054e-05,0.04876181,0,-0.03306207,0,0.02990965,0,-0.03366025,0.002082608,0.02399528,0,0.001728813,-0.03549486,0,0,0.02187664,-0.001662582,0.03706144,-0.0210734,-0.0448693,0.01281724]},{"f":[22,22,22,22,22,-1,-1,22,17,-1,-1,2,-1,-1,23,16,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.083799521744545,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,16.819976309430963,8.378590816408359,0,0,2.453445297804259,0,0,7.5,6.0,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,11,10,0,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,4.392446e-05,0.04757847,0,0,-0.04490171,-0.01103917,0,-0.02467231,0.01469652,0,0,0,0.07095646,0.005795883,0.01031241,-0.01924533,0,-0.03550463,0,0.0283056,-0.02079809,0,0.002422244,0.03877211]},{"f":[22,-1,2,1,16,18,21,-1,-1,15,18,-1,-1,-1,21,15,-1,4,-1,-1,-1,25,16,-1,-1,-1,18,-1,-1],"t":[12.199665042528586,0,9.012591414317754,4.92309205350118,5.169925001442312,2.160964047443681,7.5,0,0,4.857980995127572,4.128693921346326,0,0,0,7.5,8.361943773735241,0,2.453445297804259,0,0,0,0.5,6.409390936137702,0,0,0,9.229707804263573,0,0],"m":[0,0,1,1,0,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,7,0,0,10,11,0,0,0,15,16,0,18,0,0,0,22,23,0,0,0,27,0,0],"r":[2,0,26,21,14,9,8,0,0,13,12,0,0,0,20,17,0,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,-0.001752997,0,0,0,0,0,-0.03641152,0.01504698,0,0,0.07218105,0.0005244503,0.08311255,0,0,-0.04334181,0,-0.03998563,0.005120862,-0.00194471,0,0,-0.005389022,0.01200471,-0.00289504,0,0.03760807,-0.01055258]},{"f":[22,-1,22,16,-1,-1,1,17,2,22,-1,2,1,-1,-1,-1,2,2,-1,-1,-1,-1,1,-1,2,-1,0,-1,-1],"t":[2.69615871138938,0,7.082728314749225,10.681238411777805,0,0,2.532051282051282,11.651826559171445,4.0437314206251695,15.782972563647261,0,1.292481250360578,1.000936329588015,0,0,0,9.266718101290426,9.075473573191719,0,0,0,0,2.9611191860465116,0,9.299206200930918,0,3.5,0,0],"m":[0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,12,13,0,0,0,17,18,0,0,0,0,23,0,25,0,27,0,0],"r":[2,0,6,5,0,0,22,21,16,11,0,15,14,0,0,0,20,19,0,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.03036152,0,0,0.001600217,0.02160278,0,0,0,0,-0.03467802,0,0,0.07536121,-0.004443392,-0.03775268,0,0,-0.005489362,0.01248007,-0.02118905,0.01663449,0,0.05815262,0,-0.0001906665,0,0.0006184862,0.05224698]},{"f":[22,16,22,22,16,36,-1,18,22,-1,-1,18,-1,-1,-1,-1,-1,-1,23,33,1,-1,-1,22,15,-1,-1,-1,-1],"t":[14.116600814324018,6.894817763307944,12.921158001713904,12.613886328427084,6.569855608330948,2.5,0,1.792481250360578,10.27595502635321,0,0,5.108372929097653,0,0,0,0,0,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,0,0,0,19,20,21,0,0,24,25,0,0,0,0],"r":[18,17,16,15,14,7,0,11,10,0,0,13,0,0,0,0,0,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02619252,0,0,0.003911938,-0.01855909,0,0.02714492,-0.004212059,-0.03410418,0.0385494,-0.02583539,0.001574441,0,0,0,-0.007791871,0.06246263,0,0,0.007744395,-0.0312847,0.04225412,-0.001058465]},{"f":[21,22,23,-1,18,18,0,-1,-1,-1,-1,23,33,1,-1,-1,22,15,-1,-1,-1,18,-1,-1,22,0,-1,-1,-1],"t":[9.5,14.116600814324018,3.5,0,5.374934713698423,5.303665156874805,6.5,0,0,0,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,9.229707804263573,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,0,0,1,1,0,0,0,0,0,1,1,0,0,1,1,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,0,12,13,14,0,0,17,18,0,0,0,22,0,0,25,26,0,0,0],"r":[24,11,4,0,10,9,8,0,0,0,0,21,16,15,0,0,20,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,-0.0008115927,0,0,0,-0.02406884,0.01047196,0.03843496,-0.03364889,0,0,0,-0.00749714,0.06001579,0,0,0.007407136,-0.02958025,0.04175003,0,0.0005027684,-0.02152111,0,0,0.0062299,0.04418973,-0.03477343]},{"f":[33,21,36,4,-1,-1,-1,24,18,22,-1,-1,-1,3,-1,33,-1,-1,33,-1,37,32,-1,-1,2,21,-1,-1,-1],"t":[20.752541988539424,3.5,3.5,4.128693921346326,0,0,0,7.5,6.531360383582522,16.73920708776685,0,0,0,0.4330891330891331,0,18.39964081076096,0,0,21.16864017939033,0,9.205,10.41931383093672,0,0,2.453445297804259,5.5,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,10,0,0,0,14,0,16,0,0,19,0,21,22,0,0,25,26,0,0,0],"r":[18,7,6,5,0,0,0,13,12,11,0,0,0,15,0,17,0,0,20,0,24,23,0,0,28,27,0,0,0],"v":[0,0,0,0,-0.0597028,0.003906886,-0.007406011,0,0,0,-0.003151613,0.02633197,-0.02418927,0,-0.02090006,0,0.04088733,-0.008182004,0,0.02869483,0,0,0.0002344734,-0.0166037,0,0,0.004504352,0.02322543,0.02435416]},{"f":[4,32,4,-1,-1,-1,2,16,-1,17,-1,-1,24,22,15,15,-1,16,1,-1,16,-1,-1,18,-1,-1,-1,-1,-1],"t":[3.084962500721156,10.463013322934223,2.903677461028802,0,0,0,3.245926548164837,8.005624549193879,0,9.265614570460752,0,0,9.5,17.083799521744545,5.78135971352466,4.087462841250339,0,6.569855608330948,1.5994791666666668,0,5.78135971352466,0,0,1.292481250360578,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,0,0,13,14,15,16,0,18,19,0,21,0,0,24,0,0,0,0,0],"r":[6,5,4,0,0,0,12,9,0,11,0,0,28,27,26,17,0,23,20,0,22,0,0,25,0,0,0,0,0],"v":[0,0,0,-0.0006461299,-0.04051701,-0.02890446,0,0,0.03464158,0,-0.01852052,0.02423861,0,0,0,0,-0.0004624868,0,0,0.05812915,0,0.007691341,-0.01549938,0,0.008451238,0.05645556,-0.003720987,0.0395611,0.03116522]},{"f":[22,-1,2,4,21,-1,2,1,-1,4,1,22,22,-1,-1,23,-1,-1,-1,4,-1,-1,22,-1,-1,1,-1,-1,-1],"t":[2.69615871138938,0,10.680359255641626,9.266718101290426,2.5,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,1.13125,9.087343393278285,8.133841411710264,0,0,4.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,2.997611464968153,0,0,0],"m":[0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,8,0,10,11,12,13,0,0,16,0,0,0,20,0,0,23,0,0,26,0,0,0],"r":[2,0,28,25,6,0,22,9,0,19,18,15,14,0,0,17,0,0,0,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.0300226,0,0,0,-0.02509467,0,0,-0.04026436,0,0,0,0,0.007613368,0.04327431,0,-0.01799437,0.0293027,-9.672435e-05,0,-0.03777843,0.001849039,0,0.003260246,0.03586905,0,-0.02931022,0.01837179,0.02535304]},{"f":[25,4,32,15,-1,17,-1,22,-1,19,23,1,-1,2,21,-1,-1,-1,36,-1,-1,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,10.463013322934223,2.0,0,0.5,0,9.40390385294334,0,1.5,0.5,2.68018018018018,0,1.292481250360578,3.5,0,0,0,3.5,0,0,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,0,10,11,12,0,14,15,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,22,5,0,7,0,9,0,21,18,13,0,17,16,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.03446682,0,0.01583025,0,-0.02335122,0,0,0,-0.01118463,0,0,0.02344095,0.1070385,-0.003384102,0,0.03944749,-0.002020414,-0.005994974,-0.02727702,0,0,0.03599253,0.0004344727,0.03939422,-0.02818043]},{"f":[22,-1,17,22,-1,1,22,-1,15,-1,-1,15,-1,-1,22,36,22,-1,-1,-1,22,-1,23,1,23,-1,-1,-1,-1],"t":[12.199665042528586,0,1.292481250360578,12.278990886161445,0,2.532051282051282,15.782972563647261,0,10.066089190457772,0,0,5.78135971352466,0,0,12.921158001713904,2.5,12.613886328427084,0,0,0,13.415167820124083,0,1.5,5.015625,0.5,0,0,0,0],"m":[0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,0,0,21,0,23,24,25,0,0,0,0],"r":[2,0,14,5,0,11,8,0,10,0,0,13,0,0,20,19,18,0,0,0,22,0,28,27,26,0,0,0,0],"v":[0,-0.001686009,0,0,-0.03301135,0,0,-0.03490858,0,0.04807254,-0.02909895,0,0.02451571,-0.002959219,0,0,0,-0.02428826,0.04225951,0.01936212,0,-0.02678958,0,0,0,0.05905133,-0.001730838,0.005120272,-0.0006524599]},{"f":[22,22,22,22,22,22,15,-1,-1,17,-1,-1,-1,22,-1,2,15,-1,-1,-1,23,-1,-1,18,-1,-1,18,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,16.752707246237605,13.56045225766691,0,0,2.453445297804259,0,0,0,16.807008825964132,0,3.903677461028802,6.169925001442312,0,0,0,7.5,0,0,6.076776015854056,0,0,8.729619742794132,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,7,0,0,10,0,0,0,14,0,16,17,0,0,0,21,0,0,24,0,0,27,0,0],"r":[26,23,20,13,12,9,8,0,0,11,0,0,0,15,0,19,18,0,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.0006829172,-0.02608594,0,0.02050267,-0.04245286,0.04631768,0,-0.02921383,0,0,0.02048107,-0.03918855,0.01157325,0,0.02759683,-0.01871247,0,-0.03177584,0.008538379,0,0.01935821,-0.02065751]},{"f":[27,15,15,-1,15,-1,16,22,22,22,16,-1,-1,-1,-1,-1,18,-1,-1,15,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,5.78135971352466,4.087462841250339,0,4.169925001442312,0,6.409390936137702,15.756821747580634,13.054613167076562,12.613886328427084,4.247927513443585,0,0,0,0,0,3.084962500721156,0,0,6.108524456778169,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,0,1,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,11,0,0,0,0,0,17,0,0,20,0,0,23,0,25,0,27,0,0],"r":[22,19,4,0,6,0,16,15,14,13,12,0,0,0,0,0,18,0,0,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,-0.001238172,0,0.06780113,0,0,0,0,0,-0.02270872,0.007770407,0.03863289,-0.01697488,0.0403669,0,0.007864233,0.05566339,0,-0.02736884,-0.0004334135,0,0.0222906,0,-0.01789004,0,0.02328286,-0.01365444]},{"f":[4,4,21,22,-1,22,-1,-1,2,1,-1,4,4,30,-1,21,-1,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,13.87915215758784,0,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,7.408266855279292,1.5,0,3.5,0,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,14,0,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,22,11,0,19,18,15,0,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01890487,0,-0.04426829,0.01818911,0,0,-0.03972482,0,0,0,0.0007249247,0,-0.04166343,-0.004630165,0.01647553,0,-0.03674189,0.001444212,0,0.002758468,0.03484661,-0.0152927,0,-0.006973507,0.04078479]},{"f":[22,4,15,16,15,-1,-1,-1,23,15,-1,-1,33,-1,-1,35,1,-1,21,17,-1,4,-1,16,-1,-1,-1,-1,-1],"t":[12.199665042528586,1.292481250360578,9.862637357558794,7.741466986401147,7.266786540694901,0,0,0,0.5,10.681238411777805,0,0,44.805779984159756,0,0,2.5,0.01933482600988898,0,4.5,0.5,0,2.903677461028802,0,5.491853096329675,0,0,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,0,13,0,0,16,17,0,19,20,0,22,0,24,0,0,0,0,0],"r":[28,15,8,7,6,0,0,0,12,11,0,0,14,0,0,27,18,0,26,21,0,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.02483366,0.03713004,-0.04729705,0,0,0.06087465,0.005373672,0,-0.001119741,-0.04281668,0,0,0.03771278,0,0,0.05233549,0,-0.05088953,0,-0.04809056,-0.001642014,0.0002699809,0.02506441,0.001325468]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,17,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,2.69615871138938,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02529036,0,0,0,0,-0.0284073,0,-0.04444668,0.01742876,0,0.01613508,0,-0.03248316,0,-0.04495773,0,0,0,-0.02565915,0,-0.005284666,0.02752015,0.02094187,0.02532219,-0.03402766,0,0.02781116,0.0004082212]},{"f":[33,32,33,33,18,21,15,-1,22,-1,2,1,-1,25,-1,-1,-1,-1,37,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,1.792481250360578,7.5,3.584962500721156,0,10.12257207114862,0,8.767235038274581,5.985044642857143,0,1.5,0,0,0,0,11.57,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,11,12,0,14,0,0,0,0,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,18,17,8,0,10,0,16,13,0,15,0,0,0,0,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.0201183,0,0.02392245,0,0,-0.03568305,0,0.006096749,-0.01804667,0.03441589,0.002825529,0,0.0006935193,0.03130436,0,0,0.01299401,0.04836138,-0.02280869,-0.02143972,-0.0436607,0.01208763]},{"f":[22,23,21,15,15,32,-1,-1,33,-1,18,3,-1,-1,-1,-1,-1,22,33,-1,-1,-1,18,32,-1,17,-1,-1,-1],"t":[17.094543568094064,7.5,10.5,10.457380879072534,9.756556322524087,7.98281324063098,0,0,37.46769166150554,0,8.03615005796417,0.11826564442602569,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,0,8.729619742794132,8.341935934311302,0,7.588709768994619,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,11,12,0,0,0,0,0,18,19,0,0,0,23,24,0,26,0,0,0],"r":[22,17,16,15,8,7,0,0,10,0,14,13,0,0,0,0,0,21,20,0,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.000834941,0.01938192,0,0.0007856535,0,0,-0.001288467,0.06259131,-0.001634458,-0.00392608,0.03296644,0,0,-0.02434379,0.02153605,-0.03790291,0,0,-0.01218666,0,0.009104323,0.04948617,-0.02000539]},{"f":[4,4,21,22,-1,37,-1,-1,2,1,-1,4,2,15,-1,-1,-1,4,-1,-1,16,15,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,9.075473573191719,0.39811320754716983,0,7.950787011501063,7.894792293346235,5.169925001442312,0,0,0,8.950330235806282,0,0,10.826548487290914,2.0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,12,13,14,0,0,0,18,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,20,11,0,17,16,15,0,0,0,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0.02013509,0,-0.0437893,0.01871182,0,0,-0.04319745,0,0,0,0.003375057,-0.001080397,0.03315488,0,-0.03607525,0.002658986,0,0,0.006490786,-0.02571648,0.03915046,-0.01479455,0,-0.006471147,0.04034967]},{"f":[22,23,-1,21,-1,2,-1,-1,22,16,2,17,-1,1,-1,-1,22,2,-1,-1,15,-1,-1,15,-1,-1,22,-1,-1],"t":[12.199665042528586,2.5,0,7.5,0,5.459058925515887,0,0,12.921158001713904,7.930737337562887,1.292481250360578,1.792481250360578,0,7.0046728971962615,0,0,12.613886328427084,8.049190845056867,0,0,4.247927513443585,0,0,10.457380879072534,0,0,13.415167820124083,0,0],"m":[0,1,0,0,0,1,0,0,0,1,0,1,0,1,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,14,0,0,17,18,0,0,21,0,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,26,23,16,13,0,15,0,0,20,19,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.0006528354,0,-0.04593677,0,-0.01976725,0.02243094,0,0,0,0,-0.02405458,0,0.05767292,0.01747531,0,0,-0.02515652,0.02113054,0,0.04130097,-0.00257966,0,0.03345486,-0.02218937,0,-0.02351257,0.0008620913]},{"f":[36,-1,22,23,-1,21,-1,2,-1,-1,2,1,16,18,-1,15,18,-1,-1,-1,21,15,-1,-1,-1,-1,18,-1,-1],"t":[1.5,0,12.199665042528586,2.5,0,7.5,0,5.459058925515887,0,0,9.012591414317754,4.92309205350118,5.169925001442312,2.160964047443681,0,4.857980995127572,4.128693921346326,0,0,0,7.5,8.361943773735241,0,0,0,0,9.229707804263573,0,0],"m":[0,0,0,1,0,0,0,1,0,0,1,1,0,1,0,1,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,0,0,11,12,13,14,0,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,10,5,0,7,0,9,0,0,26,25,20,15,0,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.02476074,0,0,-0.0008353313,0,-0.04558936,0,-0.01894191,0.02122613,0,0,0,0,-0.01438286,0,0,0.06300734,8.286513e-05,0.0712922,0,0,-0.04187867,-0.00997961,-0.001467274,0.003327807,0,0.03436003,-0.008900435]},{"f":[4,15,-1,21,-1,15,-1,-1,2,21,21,22,-1,-1,-1,-1,20,4,-1,23,-1,16,-1,22,-1,-1,22,-1,-1],"t":[4.951187557243013,11.683433292835403,0,7.5,0,12.27029532647204,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,0.5,5.505613627711627,0,3.5,0,8.321928094887362,0,16.759160379664443,0,0,10.316277538115674,0,0],"m":[1,1,0,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,0,0,0,17,18,0,20,0,22,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,16,15,14,13,0,0,0,0,26,19,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,-0.001797476,0,-0.005139253,0,0.03744125,-0.01748509,0,0,0,0,0.07575251,-0.003019467,0.06641454,0.0006618821,0,0,-0.03592881,0,0.0009868638,0,-0.03500572,0,-0.00962263,0.0301016,0,-0.0260699,0.01568162]},{"f":[3,22,-1,-1,2,4,21,22,-1,37,-1,-1,2,1,-1,4,-1,4,-1,-1,16,-1,-1,16,21,-1,-1,-1,-1],"t":[0.007093253968253968,16.725912049439096,0,0,10.680359255641626,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,9.034782574103549,0.39811320754716983,0,7.950787011501063,0,8.950330235806282,0,0,4.087462841250339,0,0,10.462502272597265,6.5,0,0,0,0],"m":[0,0,0,0,1,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,10,0,0,13,14,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,12,9,0,11,0,0,20,15,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.02394067,0.02091734,0,0,0,0,0.01887879,0,-0.04336437,0.0185444,0,0,-0.03752422,0,0.0006043857,0,-0.03485586,0.0007657266,0,0.000833088,0.02398852,0,0,0.02351993,-0.01488787,-0.04070603,0.03398669]},{"f":[22,-1,25,4,4,32,15,18,-1,-1,21,-1,-1,-1,-1,33,2,15,19,-1,-1,-1,22,15,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,6.5,3.084962500721156,2.903677461028802,10.463013322934223,11.683433292835403,10.69085411329249,0,0,7.5,0,0,0,0,41.64100553123272,3.245926548164837,8.005624549193879,0.5,0,0,0,17.083799521744545,5.78135971352466,0,0,0,0,0],"m":[0,0,1,1,1,1,1,1,0,0,1,0,0,0,0,1,0,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,0,0,0,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,15,14,13,10,9,0,0,12,0,0,0,0,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.02914437,0,0,0,0,0,0,-0.0008054098,-0.04709604,0,-0.004998145,0.02858695,-0.02663439,-0.03996095,0,0,0,0,0.002941973,0.03489168,-0.0419578,0,0,0.002896637,-0.003213114,0.03371058,0.03234353,-0.02749096]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,33,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,8.065928480304397,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02438049,0,0,0,0,-0.02692898,0,-0.04324688,0.01710071,0,0.01467326,0,-0.03117858,0,-0.04383846,0,0,0,-0.02240988,0,-0.004995817,0.03780037,0.02021388,0.02337153,-0.03292273,0,0.02697186,0.0003744369]},{"f":[33,23,21,15,-1,15,-1,15,23,-1,-1,22,22,-1,17,-1,-1,-1,-1,-1,3,-1,33,-1,33,18,-1,-1,-1],"t":[29.297756302860655,8.5,10.5,10.457380879072534,0,10.462502272597265,0,10.681238411777805,0.5,0,0,16.73920708776685,11.722778233171695,0,5.597993649014255,0,0,0,0,0,0.33771929824561403,0,32.199935296219856,0,44.79706586676442,2.160964047443681,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,13,0,15,0,0,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,19,18,5,0,7,0,11,10,0,0,17,14,0,16,0,0,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0.0005003071,0,-0.04315276,0,0,0.06633598,0.009673874,0,0,0.008603091,0,-0.02751981,-0.003949825,0.01668754,0.02864369,-0.02368592,0,0.02752875,0,0.0216597,0,0,-0.0007092938,-0.0391465,0.007496616]},{"f":[4,4,15,-1,-1,16,-1,-1,2,21,21,22,-1,-1,-1,-1,20,4,-1,23,-1,16,-1,22,-1,-1,22,-1,-1],"t":[4.951187557243013,4.554262228389085,1.584962500721156,0,0,10.462502272597265,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,0.5,5.505613627711627,0,3.5,0,8.321928094887362,0,16.759160379664443,0,0,10.316277538115674,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,0,0,0,17,18,0,20,0,22,0,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,16,15,14,13,0,0,0,0,26,19,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.02750557,-0.0002960292,0,-0.03990733,-0.003064654,0,0,0,0,0.0700733,-0.002983254,0.06527874,0.0005435246,0,0,-0.03496801,0,0.000936177,0,-0.03367515,0,-0.008198478,0.02851581,0,-0.02540437,0.01497287]},{"f":[22,22,22,22,22,22,-1,17,-1,-1,-1,22,17,-1,-1,2,-1,-1,23,-1,-1,18,22,-1,-1,-1,18,-1,-1],"t":[17.094543568094064,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,16.752707246237605,0,2.453445297804259,0,0,0,16.819976309430963,8.378590816408359,0,0,2.453445297804259,0,0,7.5,0,0,6.076776015854056,17.078739781790915,0,0,0,8.729619742794132,0,0],"m":[1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,12,13,0,0,16,0,0,19,0,0,22,23,0,0,0,27,0,0],"r":[26,21,18,11,10,7,0,9,0,0,0,15,14,0,0,17,0,0,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0.0003650238,0,0.0172546,-0.04069183,0.04438824,0,0,-0.04364489,-0.009579366,0,-0.02370568,0.01369038,0,0.02639753,-0.01828363,0,0,-0.04791408,-0.01062507,0.007771705,0,0.01814341,-0.01950769]},{"f":[22,-1,25,4,4,21,22,22,-1,-1,-1,2,1,-1,4,-1,4,-1,-1,16,15,-1,-1,-1,-1,17,-1,-1,-1],"t":[2.69615871138938,0,6.5,10.510264838663748,9.266718101290426,2.5,13.827286379004683,6.106096185954236,0,0,0,9.075473573191719,0.39811320754716983,0,7.950787011501063,0,8.950330235806282,0,0,10.826548487290914,2.0,0,0,0,0,2.160964047443681,0,0,0],"m":[0,0,1,1,1,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,13,0,15,0,17,0,0,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,25,24,11,10,9,0,0,0,19,14,0,16,0,18,0,0,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.028622,0,0,0,0,0,0,0.01832962,-0.04297113,0.02121797,0,0,-0.04223473,0,0.0006578646,0,-0.03358727,0.001731551,0,0,0.005971122,-0.02420703,0.03755015,-0.01442419,0,-0.006386911,0.0397233,-0.02686291]},{"f":[36,-1,22,-1,22,16,2,17,-1,24,-1,-1,22,2,-1,-1,15,-1,-1,15,17,-1,-1,-1,22,-1,15,-1,-1],"t":[1.5,0,12.199665042528586,0,12.921158001713904,7.930737337562887,1.292481250360578,1.792481250360578,0,4.5,0,0,12.613886328427084,8.049190845056867,0,0,4.247927513443585,0,0,10.457380879072534,3.084962500721156,0,0,0,13.415167820124083,0,7.118941072723508,0,0],"m":[0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,10,0,0,13,14,0,0,17,0,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,4,0,24,19,12,9,0,11,0,0,16,15,0,0,18,0,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.0239651,0,-0.00175032,0,0,0,0,-0.02348125,0,0.01566499,0.05391111,0,0,-0.02384504,0.01917566,0,0.03946719,-0.00246655,0,0,0.01573934,0.05490786,-0.02088897,0,-0.02144794,0,0.005356133,-0.002048346]},{"f":[4,-1,2,22,1,22,-1,-1,-1,-1,1,17,2,22,22,-1,-1,-1,-1,16,16,15,-1,-1,-1,-1,22,-1,-1],"t":[4.951187557243013,0,5.40929108874043,14.03140051979143,2.6153846153846154,13.129761635363703,0,0,0,0,7.40625,6.734566509914796,6.793888758164105,9.087343393278285,8.133841411710264,0,0,0,0,11.756139298372927,4.247927513443585,2.0,0,0,0,0,12.121442107629743,0,0],"m":[1,0,0,1,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,0,11,12,13,14,15,0,0,0,0,20,21,22,0,0,0,0,27,0,0],"r":[2,0,10,9,8,7,0,0,0,0,26,19,18,17,16,0,0,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,-0.001042679,0,0,0,0,0.07394285,-0.0113962,-0.0004553394,0.03943833,0,0,0,0,0,-0.0222494,0.02154863,-0.03320664,-0.0001373231,0,0,0,0.01184813,-0.03197164,0.01193328,-0.008677325,0,0.0006820325,0.03395331]},{"f":[8,3,22,-1,-1,2,15,15,-1,1,-1,21,-1,-1,21,33,15,15,-1,-1,-1,-1,23,22,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,10.457380879072534,9.756556322524087,0,1.987429289754871,0,5.5,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,0.5,11.406459975416917,0,0,0,0,0],"m":[1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,14,9,0,11,0,13,0,0,22,21,20,19,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03569901,0.02035782,0,0,0,-0.0003756614,0,0.06504405,0,0.02548947,0.002341145,0,0,0,0,-0.02753561,0.02011086,-0.0477024,0.0112779,0,0,0.004604285,0.04721073,-0.002865922,0.03336892,0.01290231]},{"f":[21,4,2,2,1,-1,2,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,22,-1,23,2,-1,0,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,10.735831902194036,0,4.5,5.065928480304397,0,1.5,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,0,0,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,13,12,9,6,0,8,0,0,11,0,0,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03429811,0,0.0001403677,-0.008758438,0,0.002900409,0.03926632,-0.04038251,0,0.0002093894,0.04874062,0,0,0,0.01590879,0,0,-0.02967046,0,-0.01735912,0.01227165,0.0245434,0.04333462,-0.03536052]},{"f":[4,32,15,-1,19,-1,32,21,17,15,-1,-1,16,2,25,21,-1,-1,-1,-1,1,-1,-1,-1,-1,-1,37,-1,-1],"t":[2.453445297804259,10.463013322934223,2.0,0,1.5,0,7.491051153478379,7.5,1.792481250360578,6.523561956057013,0,0,6.539158811108031,1.292481250360578,0.5,3.5,0,0,0,0,6.015625,0,0,0,0,0,12.52,0,0],"m":[1,1,0,0,1,0,1,1,0,0,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,14,15,16,0,0,0,0,21,0,0,0,0,0,27,0,0],"r":[26,25,4,0,6,0,24,23,12,11,0,0,20,19,18,17,0,0,0,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.03293005,0,0.002191644,0,0,0,0,-0.03254394,-0.004242374,0,0,0,0,-0.02619942,0.04539413,0.006165644,-0.01786685,0,0.008691074,-0.04285326,-0.03327073,0.01489119,-0.02510917,0,0.0006700538,0.03656825]},{"f":[25,4,32,15,-1,17,-1,22,18,-1,-1,19,23,1,-1,15,-1,-1,36,-1,-1,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,10.463013322934223,2.0,0,0.5,0,9.40390385294334,3.084962500721156,0,0,1.5,0.5,4.008411214953271,0,7.118941072723508,0,0,3.5,0,0,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,1,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,0,8,9,0,0,12,13,14,0,16,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[28,23,22,5,0,7,0,11,10,0,0,21,18,15,0,17,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.03246613,0,0.01357387,0,0,-0.04266774,-6.309872e-05,0,0,0,0.005990131,0,0.08702033,0.01705688,0,0.03738086,-0.001913596,-0.005479172,-0.02432844,0,0,0.03271053,0.0004103088,0.03662969,-0.02648385]},{"f":[21,4,2,2,1,-1,2,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,22,-1,23,2,-1,0,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,10.735831902194036,0,4.5,5.065928480304397,0,1.5,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,0,0,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,13,12,9,6,0,8,0,0,11,0,0,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03377754,0,0.0001383288,-0.008384941,0,0.002711103,0.03838246,-0.0393618,0,0.0001497505,0.04795249,0,0,0,0.01530604,0,0,-0.02868962,0,-0.01682956,0.01172211,0.02419863,0.04271399,-0.03457889]},{"f":[8,3,22,-1,-1,2,15,15,-1,-1,21,33,15,15,-1,-1,-1,-1,18,18,-1,-1,1,15,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,10.457380879072534,9.756556322524087,0,0,5.5,44.79706586676442,10.46352437327118,10.462502272597265,0,0,0,0,9.265614570460752,9.229707804263573,0,0,2.9055555555555554,11.683433292835403,0,0,0,0,0],"m":[1,0,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,0,0,11,12,13,14,0,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,10,9,0,0,18,17,16,15,0,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03449238,0.01914682,0,0,0,-0.0003517633,0.009682495,0,0,0,0,-0.02678963,0.02004963,-0.04666453,0.009976811,0,0,-0.0002666043,-0.04659687,0,0,-0.04563641,0.0123061,0.03246251,0.03266598,0.01258721]},{"f":[22,-1,15,22,22,22,18,4,15,-1,-1,-1,18,-1,-1,1,21,-1,-1,-1,-1,-1,15,-1,37,-1,34,-1,-1],"t":[2.69615871138938,0,5.78135971352466,16.819976309430963,16.73920708776685,15.439474430023438,1.792481250360578,1.292481250360578,2.584962500721156,0,0,0,6.459338463407563,0,0,4.008411214953271,7.5,0,0,0,0,0,6.108524456778169,0,9.205,0,3.5,0,0],"m":[0,0,0,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,0,0,13,0,0,16,17,0,0,0,0,0,23,0,25,0,27,0,0],"r":[2,0,22,21,20,15,12,11,10,0,0,0,14,0,0,19,18,0,0,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.02783737,0,0,0,0,0,0,0,0.02484232,-0.03488583,4.039765e-05,0,0.01005282,-0.002200887,0,0,-0.03658613,0.0165868,0.05256053,-0.01853958,0.02462665,0,-0.02632126,0,-0.0009494897,0,0.02156601,0.005128151]},{"f":[23,21,22,-1,23,33,1,-1,-1,22,15,-1,-1,-1,17,-1,15,-1,-1,-1,22,22,33,-1,-1,1,-1,-1,-1],"t":[7.5,10.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,5.9770981551934375,0,7.366322214245816,0,0,0,17.105450904090162,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,0,15,0,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[20,19,4,0,14,9,8,0,0,13,12,0,0,0,16,0,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.0009708463,0,0,0,-0.004988053,0.05440767,0,0,0.005884021,-0.02897938,0.04008004,0,-0.005981048,0,0.02399114,-0.001627852,0.02476031,0,0,0,-0.02340315,0.02058087,0,0.01249541,-0.04666244,0.01048642]},{"f":[16,-1,21,-1,17,15,3,-1,16,0,-1,22,-1,-1,22,-1,-1,-1,17,-1,17,-1,17,23,-1,-1,17,-1,-1],"t":[7.924812503605781,0,3.5,0,9.26091905453048,10.457380879072534,0.3033788641265277,0,9.92481250360578,2.5,0,9.945253865134848,0,0,11.681676585676911,0,0,0,9.265614570460752,0,9.27958770372691,0,10.620215166896223,1.5,0,0,10.68167758836394,0,0],"m":[1,0,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,12,0,0,15,0,0,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,4,0,18,17,8,0,14,11,0,13,0,0,16,0,0,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.001259405,0,-0.02948889,0,0,0,-0.0136455,0,0,-0.02607614,0,0.04960621,0.008238833,0,0.01344415,0.05283981,-0.001083745,0,-0.03060703,0,0.02640547,0,0,-0.0433397,-0.003500445,0,0.04556726,0.0003782666]},{"f":[4,4,32,-1,-1,-1,2,16,-1,17,-1,-1,22,24,22,22,22,22,15,-1,-1,-1,-1,-1,18,-1,-1,-1,-1],"t":[3.084962500721156,2.903677461028802,10.463013322934223,0,0,0,3.245926548164837,8.005624549193879,0,9.265614570460752,0,0,17.083799521744545,9.5,17.069460217922263,16.819976309430963,16.786010507999382,16.73920708776685,5.78135971352466,0,0,0,0,0,6.076776015854056,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,10,0,0,13,14,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[6,5,4,0,0,0,12,9,0,11,0,0,28,27,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,-0.0006138507,-0.02425995,-0.03962292,0,0,0.03199152,0,-0.01758537,0.02164423,0,0,0,0,0,0,0,0.002411892,-0.004072813,0.02219211,-0.01757543,0.01508234,0,-0.03540113,0.01049434,0.02951154,0.03376621]},{"f":[8,2,1,-1,4,21,22,-1,22,-1,-1,2,21,-1,0,-1,-1,16,-1,3,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,13.827286379004683,0,0,9.034782574103549,9.5,0,7.5,0,0,4.087462841250339,0,0.8775684931506849,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,1,0,0,1,0,0,0,1,0,0,1,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,13,0,15,0,0,18,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,4,0,22,11,8,0,10,0,0,17,14,0,16,0,0,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.04094138,0,0,0,0.01756101,0,-0.04256792,0.01792846,0,0,-0.0004901151,0,0.004788067,0.0415163,0,0.0003354334,0,0.04325274,0.007762232,0,0,0.02098427,-0.01461658,-0.0389493,0.03176173,0.01249203]},{"f":[25,4,15,-1,17,-1,22,1,18,-1,17,-1,-1,17,-1,-1,22,17,-1,-1,18,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,2.0,0,0.5,0,11.67903991803059,6.015625,9.938949842906478,0,10.37503779525057,0,0,6.584884225259038,0,0,11.722778233171695,1.792481250360578,0,0,9.229707804263573,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,11,0,0,14,0,0,17,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,16,13,10,0,12,0,0,15,0,0,20,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03198243,0,0.01308648,0,0,0,-0.03460838,0,0.05912069,-0.01481275,0,0.04288732,-0.03850048,0,0,-0.01825382,0.04499281,0,-0.0001959819,-0.01919032,0,0,0.03163215,0.0004356065,0.03533668,-0.02632281]},{"f":[16,16,-1,2,-1,-1,21,-1,17,15,3,-1,16,0,-1,22,-1,-1,-1,-1,17,-1,17,-1,17,23,-1,-1,-1],"t":[7.924812503605781,7.741466986401147,0,6.055241654908112,0,0,3.5,0,9.26091905453048,10.457380879072534,0.3033788641265277,0,9.92481250360578,2.5,0,9.945253865134848,0,0,0,0,9.265614570460752,0,9.27958770372691,0,10.620215166896223,1.5,0,0,0],"m":[1,1,0,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,0,9,10,11,0,13,14,0,16,0,0,0,0,21,0,23,0,25,26,0,0,0],"r":[6,3,0,5,0,0,8,0,20,19,12,0,18,15,0,17,0,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,-0.0004032148,0,-0.04433486,0.003336594,0,-0.02862102,0,0,0,-0.01278204,0,0,-0.02516528,0,0.04691202,0.007705458,0.03390429,-0.001079247,0,-0.02979412,0,0.02430044,0,0,-0.04220717,-0.003209401,0.00208029]},{"f":[22,21,1,22,-1,-1,15,-1,25,-1,37,15,-1,-1,15,-1,-1,-1,37,17,21,22,-1,-1,15,-1,-1,-1,-1],"t":[12.121442107629743,5.5,1.000936329588015,10.092754721228424,0,0,3.169925001442312,0,0.5,0,6.775,5.614709844115208,0,0,5.906890595608519,0,0,0,4.875,3.39067985676233,7.5,13.054613167076562,0,0,10.066089190457772,0,0,0,0],"m":[0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,12,0,0,15,0,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[18,17,6,5,0,0,8,0,10,0,14,13,0,0,16,0,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,0,0.04831649,-0.0265408,0,-0.04476444,0,-0.0369535,0,0,-0.003097653,-0.03915964,0,-0.003097653,0.03178391,6.678472e-05,0,0,0,0,-0.0006865153,-0.03468089,0,-0.0006865153,-0.03959899,-0.0006865153,0.002627289]},{"f":[25,22,23,-1,21,-1,2,-1,-1,37,17,21,4,-1,-1,15,-1,-1,-1,16,-1,15,3,-1,16,-1,-1,-1,-1],"t":[6.5,12.121442107629743,2.5,0,7.5,0,5.459058925515887,0,0,4.875,3.39067985676233,7.5,6.209321624687949,0,0,10.066089190457772,0,0,0,6.044394119358453,0,7.118941072723508,0.3033788641265277,0,6.584962500721156,0,0,0,0],"m":[1,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,0,10,11,12,13,0,0,16,0,0,0,20,0,22,23,0,25,0,0,0,0],"r":[28,9,4,0,6,0,8,0,0,19,18,15,14,0,0,17,0,0,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,0,-0.0006370621,0,-0.04437701,0,-0.01786025,0.01900329,0,0,0,0,-0.03339679,-0.0005356923,0,-0.0005356923,-0.03913497,-0.0005356923,0,-0.0005356923,0,0,-0.0005356923,0,0.01347476,0.04558825,0.0007340236,-0.0258347]},{"f":[23,21,22,-1,23,33,1,-1,-1,22,15,-1,-1,-1,-1,0,1,-1,-1,-1,22,22,33,-1,-1,1,-1,-1,-1],"t":[7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,15.676331898425197,6.247927513443585,0,0,0,0,7.5,6.015625,0,0,0,17.105450904090162,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,0,0,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[20,15,4,0,14,9,8,0,0,13,12,0,0,0,0,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.001379787,0,0,0,-0.004185043,0.05156556,0,0,0.005056465,-0.02928403,0.03898462,-0.0001120287,0,0,0.008283445,-0.02311807,0.04114284,0,0,0,-0.02335391,0.01966944,0,0.01206375,-0.04620541,0.0100656]},{"f":[16,17,18,17,17,-1,2,-1,-1,1,21,-1,-1,33,-1,-1,-1,18,-1,15,21,-1,-1,-1,16,-1,21,-1,-1],"t":[7.930737337562887,8.571721503073015,7.430398005367313,7.198837316474133,6.434604125275413,0,6.4675825248018475,0,0,6.9941860465116275,3.5,0,0,21.011466140970732,0,0,0,8.729619742794132,0,7.6653359171851765,7.5,0,0,0,8.060695931687555,0,3.5,0,0],"m":[1,1,1,1,1,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,0,0,0,18,0,20,21,0,0,0,25,0,27,0,0],"r":[24,17,16,9,6,0,8,0,0,13,12,0,0,15,0,0,0,19,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0,0,-0.001418591,0,-0.03599597,0.01940146,0,0,0.057784,0.003426188,0,-0.02894766,0.006610314,-0.02763173,0,0.03280716,0,0,0.006588754,-0.02345091,0.02966597,0,0.018686,0,-0.02760588,0.001050784]},{"f":[8,3,22,-1,-1,2,1,-1,4,21,22,-1,37,-1,-1,15,15,-1,21,-1,-1,16,-1,-1,1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,3.5,0,0,6.108524456778169,0,0,2.997611464968153,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,0,13,0,0,16,17,0,19,0,0,22,0,0,25,0,0,0,0],"r":[28,5,4,0,0,27,8,0,24,15,12,0,14,0,0,21,18,0,20,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.03272105,0.01841225,0,0,-0.03056911,0,0,0,0.0182483,0,-0.04205205,0.01742445,0,0,0.0004192113,0,0.04974772,0.009153413,0,-0.009879473,0.0005216367,0,-0.02518899,0.01526023,0.03105576,0.01211619]},{"f":[25,22,-1,22,22,21,-1,-1,-1,22,16,15,-1,-1,1,16,1,-1,21,-1,-1,-1,21,-1,18,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,12.121442107629743,1.584962500721156,1.0,0,0,1.13125,10.696098170955864,0.9444444444444444,0,6.5,0,0,0,3.5,0,10.609627990977145,0,0,0,0],"m":[1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,0,0,10,11,12,0,0,15,16,17,0,19,0,0,0,23,0,25,0,0,0,0],"r":[28,3,0,9,8,7,0,0,0,27,14,13,0,0,22,21,18,0,20,0,0,0,24,0,26,0,0,0,0],"v":[0,0,-0.02704887,0,0,0,-0.02930824,0.01334777,0.04521775,0,0,0,0.01902397,-0.03613128,0,0,0,-0.0431467,0,0.05180143,0.0002334168,0.02854442,0,-0.04222893,0,-0.0032152,0.00927358,0.001095341,-0.02559633]},{"f":[4,15,15,15,-1,24,-1,-1,-1,21,-1,15,-1,-1,2,21,21,22,-1,-1,-1,-1,1,2,-1,-1,22,-1,-1],"t":[4.951187557243013,11.683433292835403,10.690871009292314,10.462502272597265,0,7.5,0,0,0,7.5,0,12.27029532647204,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,7.40625,5.459058925515887,0,0,12.121442107629743,0,0],"m":[1,1,1,1,0,1,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,0,15,16,17,18,0,0,0,0,23,24,0,0,27,0,0],"r":[14,9,8,5,0,7,0,0,0,11,0,13,0,0,22,21,20,19,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,-0.001817789,0,0.03803736,-0.008559667,-0.02164495,0,-0.004515661,0,0.03054417,-0.01638557,0,0,0,0,0.06059782,-0.003225312,0.06082254,0.0003011362,0,0,-0.03844007,-0.001004055,0,0.0004700966,0.03247342]},{"f":[27,15,22,22,22,18,-1,-1,1,21,-1,-1,-1,-1,-1,15,4,-1,-1,-1,22,22,0,-1,-1,-1,22,-1,-1],"t":[0.5,5.78135971352466,16.819976309430963,16.73920708776685,15.439474430023438,1.792481250360578,0,0,4.008411214953271,7.5,0,0,0,0,0,6.108524456778169,4.80643424864552,0,0,0,16.98920034885663,16.978341770814396,8.5,0,0,0,17.078739781790915,0,0],"m":[1,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,10,0,0,0,0,0,16,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,15,14,13,8,7,0,0,12,11,0,0,0,0,0,19,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.003235987,0.00413684,0,0,-0.03597985,0.01601078,0.04796945,-0.0170894,0.02259043,0,0,-0.03603506,-0.00288004,-0.0003639394,0,0,0,-0.02823613,0.01662817,0.05993406,0,-0.0285014,0.0168168]},{"f":[4,-1,4,15,22,-1,33,-1,-1,-1,21,23,22,21,-1,15,-1,-1,2,17,-1,-1,17,15,-1,-1,-1,-1,-1],"t":[4.951187557243013,0,5.89476682248518,7.417852514885898,14.03140051979143,0,15.968216440900115,0,0,0,7.5,3.5,11.681676585676911,4.5,0,1.584962500721156,0,0,7.255412984064776,9.265614570460752,0,0,11.26283925556662,10.457380879072534,0,0,0,0,0],"m":[1,0,0,1,1,0,1,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,14,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,28,27,18,15,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.0009132264,0,0,0,-0.0001751799,0,0.04808792,-0.001581857,0.05279631,0,0,0,0,-0.04291609,0,-0.02080327,0.009811948,0,0,-0.01052853,0.008834626,0,0,0.05281529,0.004345018,-0.007790232,-0.02689716,0.004052124]},{"f":[22,-1,22,22,21,-1,-1,-1,21,17,22,-1,-1,-1,4,15,-1,2,-1,22,-1,-1,18,2,-1,-1,15,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,2.5,1.292481250360578,12.027747761539269,0,0,0,9.266718101290426,1.0,0,9.051177541662383,0,13.005894632618489,0,0,10.457893746154532,10.680359255641626,0,0,9.862637357558794,0,0],"m":[0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,0,15,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,8,7,6,0,0,0,14,13,12,0,0,0,22,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.02657932,0,0,0,-0.02889629,0.01252947,0.042177,0,0,0,-0.02814726,0.01226751,-0.03642527,0,0,0.02015033,0,-0.0001881156,0,0.002778319,0.02767714,0,0,-0.03615105,0.02904504,0,0.0371524,-0.01354331]},{"f":[33,33,23,-1,-1,-1,33,22,-1,-1,3,33,22,-1,3,-1,-1,-1,17,33,33,33,33,-1,-1,-1,-1,-1,-1],"t":[24.416594167368267,23.58371303411409,8.5,0,0,0,26.99635432609064,12.121442107629743,0,0,0.9218559218559219,38.67649696721479,16.838461650489297,0,0.08766192532385064,0,0,0,7.980000966034041,37.56709924177028,35.37421412138842,32.199935296219856,28.819877645289402,0,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,0,11,12,13,0,15,0,0,0,19,20,21,22,23,0,0,0,0,0,0],"r":[6,5,4,0,0,0,10,9,0,0,18,17,14,0,16,0,0,0,28,27,26,25,24,0,0,0,0,0,0],"v":[0,0,0,5.598831e-05,-0.02696004,-0.01840146,0,0,0.00508871,0.03684558,0,0,0,-0.004811488,0,-0.005468393,0.03706093,0.04711068,0,0,0,0,0,-0.003725716,0.02228911,-0.02331346,0.02495639,-0.01304203,-0.03588906]},{"f":[25,22,-1,22,1,4,21,-1,-1,22,-1,22,0,-1,-1,-1,-1,22,-1,22,18,22,-1,-1,18,-1,-1,-1,-1],"t":[6.5,2.69615871138938,0,7.770639075561187,5.489664082687339,1.292481250360578,7.5,0,0,5.9525054311951475,0,6.671801068872866,6.5,0,0,0,0,8.133841411710264,0,12.121442107629743,10.609627990977145,10.27595502635321,0,0,10.682994316786338,0,0,0,0],"m":[1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,0,0,10,0,12,13,0,0,0,0,18,0,20,21,22,0,0,25,0,0,0,0],"r":[28,3,0,17,16,9,8,0,0,11,0,15,14,0,0,0,0,19,0,27,24,23,0,0,26,0,0,0,0],"v":[0,0,-0.0262417,0,0,0,0,-0.03768972,0.01197017,0,0.03824131,0,0,-0.04050405,0.0005897113,0.006449357,0.02782908,0,-0.03658672,0,0,0,0.001107241,-0.009685326,0,0.04904049,0.00055213,0.001018606,-0.02499145]},{"f":[33,33,23,-1,-1,-1,33,22,-1,-1,3,33,22,-1,3,-1,-1,-1,17,33,22,33,-1,-1,22,-1,-1,-1,-1],"t":[24.416594167368267,23.58371303411409,8.5,0,0,0,26.99635432609064,12.121442107629743,0,0,0.9218559218559219,38.67649696721479,16.838461650489297,0,0.08766192532385064,0,0,0,7.980000966034041,37.56709924177028,16.98920034885663,29.807475161786478,0,0,17.083799521744545,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,0,0,7,8,0,0,11,12,13,0,15,0,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[6,5,4,0,0,0,10,9,0,0,18,17,14,0,16,0,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,3.970423e-05,-0.02625631,-0.01753681,0,0,0.005134633,0.03587205,0,0,0,-0.00463181,0,-0.005306734,0.0360213,0.04575587,0,0,0,0,-0.0007939868,0.02214049,0,-0.03402329,0.003480547,-0.01252444,-0.03477807]},{"f":[16,17,18,-1,-1,16,17,-1,17,-1,-1,22,-1,18,-1,-1,16,22,17,-1,-1,-1,21,-1,1,22,-1,-1,-1],"t":[7.930737337562887,8.496347487943302,7.6181566135214025,0,0,5.044394119358453,9.229707804263573,0,10.269270673170688,0,0,15.676331898425197,0,8.729619742794132,0,0,8.060695931687555,15.676331898425197,6.219136028062415,0,0,0,3.5,0,1.4859550561797752,15.756821747580634,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,0,9,0,0,12,0,14,0,0,17,18,19,0,0,0,23,0,25,26,0,0,0],"r":[16,5,4,0,0,11,8,0,10,0,0,13,0,15,0,0,22,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,-0.001378356,-0.04167064,0,0,0.01960716,0,-0.04588864,0.001237215,0,0.06547288,0,0.04692287,-0.02603725,0,0,0,0.02064676,-0.008731698,0.03856093,0,-0.02623483,0,0,0.005461226,0.04140703,2.372956e-05]},{"f":[4,4,15,-1,-1,16,-1,-1,2,21,21,22,-1,-1,-1,-1,1,17,2,22,15,2,-1,-1,-1,-1,-1,-1,-1],"t":[4.951187557243013,4.554262228389085,1.584962500721156,0,0,10.462502272597265,0,0,5.40929108874043,7.5,3.5,13.129761635363703,0,0,0,0,7.40625,6.734566509914796,6.793888758164105,13.054613167076562,5.044394119358453,5.459058925515887,0,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,0,0,0,17,18,19,20,21,22,0,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,16,15,14,13,0,0,0,0,28,27,26,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,-0.02494754,-0.0002139714,0,-0.03807139,-0.002140533,0,0,0,0,0.05606312,-0.003243222,0.05835315,0.0001669682,0,0,0,0,0,0,-0.0374641,0.01527579,-0.02687666,-0.04115644,-0.0002324589,0.002772271,0.01661506]},{"f":[25,4,19,15,-1,-1,36,-1,16,17,-1,-1,18,0,-1,-1,32,-1,33,-1,21,-1,-1,1,3,-1,-1,-1,-1],"t":[6.5,2.453445297804259,0.5,3.169925001442312,0,0,2.5,0,7.118941072723508,1.792481250360578,0,0,2.160964047443681,9.5,0,0,7.491051153478379,0,44.805779984159756,0,3.5,0,0,12.18570183086312,0.0339080459770115,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,10,0,0,13,14,0,0,17,0,19,0,21,0,0,24,25,0,0,0,0],"r":[28,23,6,5,0,0,8,0,12,11,0,0,16,15,0,0,18,0,20,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02697864,0.003248511,0,-0.03079984,0,0,-0.02387205,0.01201437,0,0,-0.002865558,0.03631147,0,-0.05237024,0,0.003880128,0,0.003349352,-0.03890548,0,0,0.03030723,0.0004141825,0.03408797,-0.02450605]},{"f":[21,4,2,2,1,-1,2,22,-1,23,-1,-1,15,17,-1,-1,-1,22,-1,-1,-1,0,-1,-1,22,0,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,7.005613627711627,14.116600814324018,0,1.5,0,0,10.113742166049189,10.620215166896223,0,0,0,13.005894632618489,0,0,0,3.5,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,7,8,0,10,0,0,13,14,0,0,0,18,0,0,0,22,0,0,25,26,0,0,0],"r":[24,21,20,17,6,0,12,9,0,11,0,0,16,15,0,0,0,19,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03230585,0,0,-0.001304598,0,0.01527706,0.0001979062,0,0,-0.02298694,0.006158863,0.003690036,0,0.00307307,0.03642986,-0.03650131,0,0.0005006791,0.04558712,0,0,0.005482793,0.04064822,-0.03301573]},{"f":[8,3,22,-1,-1,2,1,-1,4,21,22,-1,37,-1,-1,15,15,-1,-1,16,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,0,6.108524456778169,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,0,13,0,0,16,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,8,0,22,15,12,0,14,0,0,19,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03173148,0.01704593,0,0,-0.0299514,0,0,0,0.01760053,0,-0.0414468,0.01733499,0,0,0.0004933107,0.01126183,0,-0.009502414,0.0004843427,0,0,0.02019691,-0.01312088,-0.03649686,0.02988037,0.01116644]},{"f":[33,33,21,2,-1,33,-1,-1,2,22,-1,-1,30,4,-1,20,22,17,-1,22,-1,-1,-1,-1,-1,-1,33,-1,-1],"t":[20.752541988539424,18.891308976303876,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,15.212004812197344,0,0,2.5,1.792481250360578,0,1.5,16.725912049439096,2.69615871138938,0,12.613886328427084,0,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,12,11,0,0,24,15,0,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.02452999,0,-0.04127794,0.01576108,0,0,0.005046867,0.03760152,0,0,-0.03820311,0,0,0,-0.02002746,0,-0.004205467,0.02378399,0.01871024,0.02319121,-0.02687431,-0.03383524,0,0.02456691,0.000360695]},{"f":[16,-1,16,-1,21,-1,16,4,-1,-1,20,15,22,15,17,-1,22,-1,-1,-1,4,-1,-1,-1,22,22,-1,-1,-1],"t":[7.930737337562887,0,8.618385502258606,0,3.5,0,9.271463027904375,3.245926548164837,0,0,0.5,10.681238411777805,12.35227706330556,10.066089190457772,2.903677461028802,0,8.658207063149977,0,0,0,5.89476682248518,0,0,0,16.73920708776685,16.25454052700782,0,0,0],"m":[1,0,0,0,0,0,0,1,0,0,1,0,1,1,1,0,1,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,12,13,14,15,0,17,0,0,0,21,0,0,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,24,23,20,19,16,0,18,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.001063955,0,0.01333103,0,-0.02688735,0,0,-0.04511819,0.0001526749,0,0,0,0,0,0.010377,0,0.004737869,-0.03250155,0.04380135,0,0.04847562,0.01819511,-0.001277052,0,0,-0.002613306,0.02752653,-0.02741783]},{"f":[21,4,2,2,1,-1,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,22,-1,16,4,-1,16,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0.6824793747286149,0,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,10.735831902194036,0,11.98370619265935,5.065928480304397,0,7.266786540694901,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,0,12,0,14,0,0,17,18,19,0,21,22,0,24,0,0,0,0,0],"r":[16,11,10,7,6,0,0,9,0,0,0,13,0,15,0,0,28,27,20,0,26,23,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03171008,-0.0004766024,0,0.002888615,0.03560998,-0.03450842,0,-0.009820952,0,0.04111999,-0.000496728,0,0,0,0.01374746,0,0,-0.03303478,0,-0.01365763,0.01660294,0.02269609,0.04013746,-0.03237197]},{"f":[4,4,21,18,22,-1,-1,2,-1,-1,2,1,-1,4,15,-1,25,-1,-1,4,-1,-1,22,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,3.903677461028802,0,0,9.051177541662383,0.39811320754716983,0,7.950787011501063,5.169925001442312,0,2.5,0,0,8.950330235806282,0,0,13.005894632618489,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,0,14,15,0,17,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,22,13,0,19,16,0,18,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.04113697,0.01756147,0,-0.02222392,0.05127317,0,0,-0.03508693,0,0,0.003001537,0,-0.0004957098,-0.0305543,0,-0.03321926,0.001836429,0,0.00251609,0.02686963,-0.01371772,0,-0.005371275,0.03747273]},{"f":[22,23,21,4,-1,22,22,23,2,16,15,-1,-1,22,-1,-1,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,10.5,4.951187557243013,0,17.074218326240924,17.069460217922263,6.5,5.40929108874043,5.700439718141092,3.584962500721156,0,0,14.03140051979143,0,0,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,0,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,11,0,0,14,0,0,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,20,5,0,19,18,17,16,13,12,0,0,15,0,0,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.0009997352,0,0,0,0,0,0,0.007620789,0.07040175,0,0.0003360312,0.03682267,-0.0004932758,0.03414001,-0.03060701,0.02619212,0.02167428,0,0,-0.0215016,0.01894759,0,0.01481926,-0.04532985,0.0117433]},{"f":[23,22,22,-1,-1,17,-1,-1,21,24,-1,-1,18,22,-1,22,-1,18,23,-1,-1,-1,17,19,-1,-1,18,-1,-1],"t":[2.5,16.819976309430963,16.25454052700782,0,0,2.453445297804259,0,0,6.5,12.5,0,0,6.499823868264185,8.133841411710264,0,15.676331898425197,0,6.076776015854056,3.5,0,0,0,8.496347487943302,0.5,0,0,9.229707804263573,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,0,13,14,0,16,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,12,11,0,0,22,15,0,17,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0.0006735762,-0.03532002,0,-0.007554074,0.03523631,0,0,0.006886978,-0.0390611,0,0,-0.03701584,0,0.01029627,0,0,-0.04855926,-5.871044e-05,0.03292958,0,0,-0.004715374,-0.04341279,0,0.0144666,-0.00938543]},{"f":[8,3,22,-1,-1,2,1,-1,23,22,35,32,22,-1,33,0,-1,-1,-1,36,33,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,0.3126984126984127,0,1.5,15.676331898425197,2.5,7.014643613484123,14.03140051979143,0,21.257266042509173,2.5,0,0,0,3.5,37.63181656149436,0,0,0,0,0,0,0,0],"m":[1,0,0,0,0,1,0,0,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,11,12,13,0,15,16,0,0,0,20,21,0,0,0,0,0,0,0,0],"r":[28,5,4,0,0,27,8,0,26,25,24,19,14,0,18,17,0,0,0,23,22,0,0,0,0,0,0,0,0],"v":[0,0,0,-0.03118631,0.01630555,0,0,-0.02888304,0,0,0,0,0,-0.0002720803,0,0,-0.0006481759,0.03508697,-0.008311087,0,0,-0.05644862,4.340041e-05,-0.00086682,0.01809704,0.03783801,-0.001507857,0.02931823,0.01105729]},{"f":[16,17,17,17,-1,-1,-1,-1,21,-1,16,-1,16,4,-1,-1,20,15,22,15,17,-1,22,-1,-1,-1,-1,-1,-1],"t":[7.930737337562887,11.78143940991899,10.682994316786338,8.496347487943302,0,0,0,0,3.5,0,8.618385502258606,0,9.271463027904375,3.245926548164837,0,0,0.5,10.681238411777805,12.35227706330556,10.066089190457772,2.903677461028802,0,8.658207063149977,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,1,0,1,1,1,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,0,11,0,13,14,0,0,17,18,19,20,21,0,23,0,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,10,0,12,0,16,15,0,0,28,27,26,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.001975277,0.008311174,-0.04418727,0.02003739,0,-0.02478347,0,0.01357791,0,0,-0.04474785,0.0002253555,0,0,0,0,0,0.009894216,0,0.004440165,-0.03099192,0.04150549,0.03165619,-0.001204267,-0.002186118]},{"f":[22,-1,33,22,33,-1,-1,-1,33,-1,22,22,22,22,-1,4,16,-1,-1,-1,17,22,-1,-1,-1,18,-1,-1,-1],"t":[2.69615871138938,0,24.416594167368267,16.98920034885663,23.58371303411409,0,0,0,26.99635432609064,0,17.078739781790915,17.069460217922263,16.978341770814396,16.73920708776685,0,3.753897320099348,9.92481250360578,0,0,0,6.781300095563516,16.98920034885663,0,0,0,6.076776015854056,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,1,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,0,11,12,13,14,0,16,17,0,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,8,7,6,0,0,0,10,0,28,25,20,15,0,19,18,0,0,0,24,23,0,0,0,27,0,0,0],"v":[0,-0.02547825,0,0,0,-0.0002476696,-0.01604304,-0.02717801,0,0.01936865,0,0,0,0,0.0007040368,0,0,-0.02797233,0.02074528,0.003387655,0,0,0.02338323,-0.03301097,0.03359794,0,-0.04653524,-0.003239492,0.01124988]},{"f":[22,-1,23,22,22,-1,-1,17,-1,-1,21,24,-1,-1,18,22,-1,22,-1,18,-1,-1,17,19,-1,-1,18,-1,-1],"t":[2.69615871138938,0,2.5,16.819976309430963,16.25454052700782,0,0,2.453445297804259,0,0,6.5,12.5,0,0,6.499823868264185,8.133841411710264,0,15.676331898425197,0,6.076776015854056,0,0,8.496347487943302,0.5,0,0,9.229707804263573,0,0],"m":[0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,0,0,15,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,14,13,0,0,22,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,-0.02515423,0,0,0,0.000704717,-0.03488848,0,-0.006973633,0.03300325,0,0,0.006510812,-0.03826711,0,0,-0.03652593,0,0.009660491,0,-0.001743891,0.03179637,0,0,-0.004727726,-0.04293511,0,0.0133122,-0.009550406]},{"f":[25,23,22,22,-1,-1,-1,21,24,-1,-1,18,18,22,-1,22,-1,-1,-1,1,-1,16,22,-1,18,-1,-1,-1,-1],"t":[6.5,2.5,16.819976309430963,16.25454052700782,0,0,0,6.5,12.5,0,0,6.400854179458231,6.076776015854056,8.133841411710264,0,15.10428213920758,0,0,0,1.99834912080135,0,10.681238411777805,17.094543568094064,0,8.729619742794132,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,14,0,16,0,0,0,20,0,22,23,0,25,0,0,0,0],"r":[28,7,6,5,0,0,0,11,10,0,0,19,18,15,0,17,0,0,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,0,0,0.0006823822,-0.03417564,0.02017578,0,0,0.006218864,-0.03751466,0,0,0,-0.03607472,0,0.01203601,-0.001452403,0.03581099,0,0.02141946,0,0,-0.02716145,0,0.01481245,-0.02622359,0.01039911,-0.0232857]},{"f":[36,-1,33,33,15,22,17,22,-1,-1,22,-1,1,-1,-1,-1,15,4,-1,-1,22,20,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.891308976303876,5.169925001442312,15.212004812197344,2.69615871138938,13.054613167076562,0,0,12.613886328427084,0,6.276785714285714,0,0,0,7.266786540694901,1.792481250360578,0,0,16.725912049439096,0.5,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,18,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,16,15,10,9,0,0,12,0,14,0,0,0,20,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.02050397,0,0,0,0,0,0,0.002142837,-0.04767578,0,-0.002264973,0,-0.003538711,0.03637511,0.03925254,0,0,-0.05528219,-0.01087847,0,0,-0.01976376,-0.0008804921,0.01868639,-0.03369838,0,0.02288101,0.000318746]},{"f":[25,37,23,15,-1,-1,-1,4,25,-1,-1,35,2,-1,18,-1,-1,-1,17,-1,22,36,16,-1,-1,-1,22,-1,-1],"t":[2.5,9.205,8.5,5.78135971352466,0,0,0,1.292481250360578,0.5,0,0,2.5,2.453445297804259,0,1.292481250360578,0,0,0,1.292481250360578,0,12.613886328427084,2.5,4.247927513443585,0,0,0,12.637102544927284,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[18,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0.001951522,-0.001060956,-0.02363016,0,0,0.003567369,0.03744002,0,0,0.003567369,0,0.003567369,0.028041,0.0456299,0,-0.03416587,0,0,0,-0.02033631,-0.0008586856,0.008761099,0,0.03451626,-0.0121327]},{"f":[1,16,-1,-1,1,-1,22,21,22,15,-1,21,-1,-1,21,-1,23,24,-1,18,-1,2,-1,-1,-1,33,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,5.5,8.723206471664755,4.087462841250339,0,3.5,0,0,3.5,0,0.5,12.5,0,5.065928480304397,0,2.903677461028802,0,0,0,29.807475161786478,0,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,12,0,0,15,0,17,18,0,20,0,22,0,0,0,26,0,0,0],"r":[4,3,0,0,6,0,28,25,14,11,0,13,0,0,16,0,24,19,0,21,0,23,0,0,0,27,0,0,0],"v":[0,0,-0.03400401,0.04537052,0,-0.03789964,0,0,0,0,-0.007184911,0,0.06628803,-0.0007320487,0,-0.04653108,0,0,-0.04477798,0,0.04281073,0,-0.04643455,0.004750472,-0.04433968,0,-0.0002987713,0.03029624,0.0009458898]},{"f":[36,-1,33,33,21,2,-1,33,-1,-1,2,-1,23,-1,4,-1,20,22,33,-1,22,-1,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,3.5,2.453445297804259,0,15.856946627097248,0,0,1.292481250360578,0,0.5,0,1.792481250360578,0,1.5,16.725912049439096,8.065928480304397,0,12.613886328427084,0,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,0,13,0,15,0,17,18,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,10,7,0,9,0,0,12,0,14,0,16,0,24,23,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.02044277,0,0,0,0,-0.02257426,0,-0.0394709,0.01284038,0,0.01162218,0,-0.02722273,0,-0.03707243,0,0,0,-0.01786116,0,-0.004003238,0.03405199,0.01764761,0.01851457,-0.03492638,0,0.02221299,0.00027235]},{"f":[22,-1,21,4,2,2,-1,-1,-1,16,-1,16,-1,-1,22,0,17,2,17,-1,-1,15,22,-1,-1,-1,-1,-1,-1],"t":[2.69615871138938,0,9.5,9.328332883600808,9.293467526905953,9.051177541662383,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,6.781300095563516,1.292481250360578,1.792481250360578,0,0,7.679480099505446,10.735831902194036,0,0,0,0,0,0],"m":[0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,12,0,0,15,16,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[2,0,14,9,8,7,0,0,0,11,0,13,0,0,28,27,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,-0.02485101,0,0,0,0,-0.0005769298,0.007877116,-0.03295867,0,-0.009814891,0,0.04109197,-0.001516849,0,0,0,0,0,-0.001697724,-0.03655869,0,0,0.01019393,-0.01758902,0.02174561,0.02202161,0.03911782,-0.03137057]},{"f":[22,23,21,22,-1,23,33,1,-1,-1,-1,-1,24,-1,1,16,15,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,14.116600814324018,0,1.5,16.993854061608424,2.0015923566878984,0,0,0,0,13.5,0,6.015625,7.6653359171851765,6.0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,0,0,0,0,13,0,15,16,17,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,5,0,11,10,9,0,0,0,0,14,0,20,19,18,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.001213727,0,0,0,-0.00379562,0.04623889,-0.001725897,-0.0003100634,0,0.03864936,0,0,0,0.01213902,-0.01607928,0.01427906,-0.02032466,0,0,-0.02003348,0.01851451,0,0.01572385,-0.04412042,0.01089214]},{"f":[16,17,17,17,18,-1,-1,-1,-1,-1,21,-1,16,-1,16,3,4,-1,-1,-1,1,15,22,-1,-1,16,-1,-1,-1],"t":[7.930737337562887,11.78143940991899,10.682994316786338,8.496347487943302,7.6181566135214025,0,0,0,0,0,3.5,0,8.618385502258606,0,9.271463027904375,0.9816932380428807,3.6427011094311244,0,0,0,1.4859550561797752,10.066089190457772,9.178103463238745,0,0,11.221587121264806,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,0,0,11,0,13,0,15,16,17,0,0,0,21,22,23,0,0,26,0,0,0],"r":[10,9,8,7,6,0,0,0,0,0,12,0,14,0,20,19,18,0,0,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.001339186,-0.04041629,0.007801152,-0.04355901,0.01889013,0,-0.02376542,0,0.01244373,0,0,0,-0.008579983,0.0212306,-0.03469697,0,0,0,0.01037786,0.05008463,0,-0.02308971,0.01988139,0.0004778028]},{"f":[8,3,22,-1,-1,2,4,21,18,22,-1,-1,22,-1,-1,15,-1,16,21,-1,-1,-1,16,21,-1,-1,-1,-1,-1],"t":[0.5,0.007093253968253968,16.725912049439096,0,0,10.680359255641626,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,11.36027577861459,0,0,5.169925001442312,0,6.108524456778169,4.5,0,0,0,10.462502272597265,6.5,0,0,0,0,0],"m":[1,0,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,0,13,0,0,16,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,5,4,0,0,27,22,15,12,11,0,0,14,0,0,17,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,-0.03014898,0.01538133,0,0,0,0,0,-0.04066202,0.01613425,0,0.04676049,-0.02137157,0,0.00243548,0,0,-0.02427941,-0.003852762,0.0003589115,0,0,0.01893244,-0.01295922,-0.03506403,0.02797847,0.01049968]},{"f":[4,4,21,18,22,-1,-1,2,-1,-1,25,2,4,-1,-1,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,3.903677461028802,0,0,2.5,9.051177541662383,7.754129516000594,0,0,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,13,0,0,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,16,15,14,0,0,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.04031961,0.01569653,0,-0.02149531,0.04596397,0,0,0,0.0006965278,-0.01188369,0.00810894,0,-0.04416768,0,0,0,-0.001686983,-0.01925433,0.007898811,0.01319782,-0.01340446,0,-0.005718443,0.03690042]},{"f":[22,-1,22,1,21,1,-1,17,2,-1,-1,-1,-1,-1,22,-1,22,18,22,20,-1,-1,21,-1,-1,18,-1,-1,-1],"t":[2.69615871138938,0,7.770639075561187,5.489664082687339,6.5,3.015625,0,9.265614570460752,4.882435795368045,0,0,0,0,0,8.133841411710264,0,12.121442107629743,10.609627990977145,10.27595502635321,0.5,0,0,4.5,0,0,10.682994316786338,0,0,0],"m":[0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,8,9,0,0,0,0,0,15,0,17,18,19,20,0,0,23,0,0,26,0,0,0],"r":[2,0,14,13,12,7,0,11,10,0,0,0,0,0,16,0,28,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,-0.02453138,0,0,0,0,-0.04049141,0,0,-0.01873646,0.007323555,0.01894908,0.006562314,0.0246449,0,-0.03416198,0,0,0,0,0.00876268,-0.004773854,0,-0.04713527,-0.005165242,0,0.04511654,0.0006395994,0.0007737751]},{"f":[22,-1,22,22,21,-1,-1,-1,16,17,18,-1,-1,16,22,22,-1,-1,-1,22,-1,18,-1,-1,21,-1,16,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,7.930737337562887,8.496347487943302,7.6181566135214025,0,0,5.044394119358453,16.978341770814396,12.35227706330556,0,0,0,15.676331898425197,0,8.729619742794132,0,0,3.5,0,8.618385502258606,0,0],"m":[0,0,0,1,1,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,14,15,16,0,0,0,20,0,22,0,0,25,0,27,0,0],"r":[2,0,8,7,6,0,0,0,24,13,12,0,0,19,18,17,0,0,0,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02422063,0,0,0,-0.02753878,0.01134654,0.03485806,0,0,0,-0.001513643,-0.04000898,0,0,0,0.004981419,-0.02006701,0.03306218,0,0.06030667,0,0.04459808,-0.02604242,0,-0.02295241,0,0.01175113,0.0006843663]},{"f":[22,23,21,4,2,-1,-1,0,22,-1,-1,-1,0,1,16,4,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,9.328332883600808,9.293467526905953,0,0,3.5,12.457649623491124,0,0,0,7.5,6.015625,7.6653359171851765,3.084962500721156,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,14,15,16,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,7,6,0,0,11,10,0,0,0,20,19,18,17,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.000185369,-0.03137401,0,0,0.008606427,-0.0206178,0.043221,0,0,0,0,-0.02775815,0.005387595,0.01383997,-0.01944011,0.03823548,0,0,-0.01941925,0.01791244,0,0.01508148,-0.04342878,0.01049646]},{"f":[22,-1,23,22,15,15,-1,-1,15,-1,4,1,15,-1,-1,-1,-1,-1,18,2,-1,4,-1,-1,17,-1,17,-1,-1],"t":[2.69615871138938,0,1.5,15.676331898425197,10.462502272597265,10.043027283594547,0,0,10.46352437327118,0,5.065928480304397,4.008411214953271,10.794415866350105,0,0,0,0,0,9.229707804263573,9.012591414317754,0,9.037514954341955,0,0,9.265614570460752,0,10.009557117932596,0,0],"m":[0,0,1,1,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,11,12,13,0,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,18,17,8,7,0,0,10,0,16,15,14,0,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02391201,0,0,0,0,0.0004602681,-0.02114284,0,0.04489037,0,0,0,0.04586316,0.01055539,-0.008378484,-0.02838705,0.03640799,0,0,-0.00117253,0,0.04571736,-0.004295459,0,-0.04516115,0,0.01892066,-0.01268308]},{"f":[33,23,21,15,15,-1,-1,15,-1,15,23,-1,-1,-1,-1,-1,3,22,-1,-1,33,-1,33,-1,33,22,-1,-1,-1],"t":[29.807475161786478,8.5,10.5,10.457380879072534,9.756556322524087,0,0,10.462502272597265,0,10.681238411777805,0.5,0,0,0,0,0,0.33771929824561403,17.02942582914998,0,0,32.199935296219856,0,34.58978958281367,0,37.56709924177028,16.98920034885663,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,11,0,0,0,0,0,17,18,0,0,21,0,23,0,25,26,0,0,0],"r":[16,15,14,7,6,0,0,9,0,13,12,0,0,0,0,0,20,19,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.0003790543,0.008291678,0,-0.040128,0,0,0.05803811,0.009232172,-0.002909215,0.0190022,-0.01801611,0,0,0.03093176,-0.001897351,0,0.02445234,0,-0.0322784,0,0,0.0266269,-0.01597224,-0.01197799]},{"f":[33,32,33,33,33,33,1,24,33,33,-1,-1,15,-1,4,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,12.64516129032258,9.5,13.895530557358477,11.716303571902237,0,0,6.169925001442312,0,4.128693921346326,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,13,0,15,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,12,11,0,0,14,0,16,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001607452,0.02579381,0,-0.02174558,0,-0.00410909,0.008030233,0.01722699,-0.01199041,0.02344105,-0.01712159,0,0,0.009329272,0.04356282,-0.01929627,-0.02319546,-0.03854973,0.009643183]},{"f":[1,17,18,16,21,22,22,-1,-1,-1,18,18,-1,-1,22,-1,-1,16,-1,22,-1,2,-1,-1,-1,-1,1,-1,-1],"t":[2.532051282051282,11.651826559171445,11.26283925556662,9.756556322524087,6.5,8.723206471664755,7.5621616439616925,0,0,0,6.4178278454199,1.292481250360578,0,0,10.35974204013634,0,0,10.060695931687555,0,9.024458927812285,0,2.453445297804259,0,0,0,0,2.9611191860465116,0,0],"m":[0,1,1,1,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,0,20,0,22,0,0,0,0,27,0,0],"r":[26,25,24,17,10,9,8,0,0,0,14,13,0,0,16,0,0,19,0,21,0,23,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,-0.03509058,0.04877611,-0.03626321,0,0,-0.003711011,-0.03929201,0,-0.006199358,0.04543044,0,0.0484611,0,0.01275903,0,-0.03512803,-0.0005994074,-0.0465203,0.0128422,0,0.05052738,0.0001371739]},{"f":[33,32,33,33,33,33,23,21,15,-1,15,-1,15,23,-1,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,8.5,10.5,10.457380879072534,0,10.462502272597265,0,10.681238411777805,0.5,0,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,11,0,13,14,0,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,10,0,12,0,16,15,0,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0003446176,0,-0.03950454,0,0,0.05447682,0.008849646,-0.002559884,0.01820507,-0.01766443,0.0224661,-0.01647238,0,0,0.008887926,0.04202946,-0.01854242,-0.02235574,-0.03810222,0.009178049]},{"f":[27,23,22,22,-1,-1,-1,21,0,-1,-1,18,22,-1,17,-1,-1,17,19,-1,-1,-1,33,-1,33,-1,22,-1,-1],"t":[0.5,2.5,16.819976309430963,16.25454052700782,0,0,0,6.5,6.5,0,0,6.499823868264185,8.882387211598338,0,0.5,0,0,8.496347487943302,0.5,0,0,0,32.199935296219856,0,35.37421412138842,0,17.02942582914998,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,0,0,12,13,0,15,0,0,18,19,0,0,0,23,0,25,0,27,0,0],"r":[22,7,6,5,0,0,0,11,10,0,0,17,14,0,16,0,0,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,0,0,0,0.0004583834,-0.03360559,0.01921091,0,0,-0.04184666,-0.006392555,0,0,-0.02195227,0,0.04321053,0.002300021,0,0,-0.004263506,-0.04158171,0.001277264,0,0.01728573,0,-0.0147515,0,0.0176754,-0.01021025]},{"f":[1,16,-1,-1,1,-1,22,22,21,1,-1,-1,4,16,-1,-1,2,-1,-1,21,-1,17,17,-1,-1,18,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,9.087343393278285,3.5,3.4375,0,0,6.434604125275413,3.169925001442312,0,0,6.894613978429,0,0,3.5,0,10.609627990977145,3.245926548164837,0,0,10.682994316786338,0,0,0],"m":[0,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,10,0,0,13,14,0,0,17,0,0,20,0,22,23,0,0,26,0,0,0],"r":[4,3,0,0,6,0,28,19,12,11,0,0,16,15,0,0,18,0,0,21,0,25,24,0,0,27,0,0,0],"v":[0,0,-0.03262876,0.04355081,0,-0.03658632,0,0,0,0,0.05832828,-0.02110503,0,0,-0.02163607,0.0001817776,0,0.0282215,-0.0002435664,0,-0.04566864,0,0,0.0002466937,-0.009592551,0,0.0404941,0.004142735,0.0009244092]},{"f":[1,17,18,1,16,1,-1,16,22,-1,-1,15,-1,-1,-1,17,4,-1,-1,23,22,-1,-1,-1,-1,-1,1,-1,-1],"t":[2.532051282051282,11.651826559171445,11.26283925556662,1.4859550561797752,11.221587121264806,0.39811320754716983,0,9.862637357558794,10.896650035079205,0,0,10.066089190457772,0,0,0,7.2941523604036345,3.753897320099348,0,0,1.5,10.224917391507956,0,0,0,0,0,2.9611191860465116,0,0],"m":[0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,12,0,0,0,16,17,0,0,20,21,0,0,0,0,0,27,0,0],"r":[26,25,24,15,14,7,0,11,10,0,0,13,0,0,0,19,18,0,0,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0428421,0,0,0.002349578,-0.009495544,0,0.03904236,-0.0120723,0.01950438,0,0,-0.03403728,-0.01021274,0,0,0.00432344,-0.032232,0.02828478,-0.04577896,0.01215068,0,0.04902383,0.0001374588]},{"f":[22,22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,23,17,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.07907323966382,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,6.208926257442949,0,3.903677461028802,0,16.819976309430963,0,0,7.5,6.781300095563516,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,9,0,11,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-5.862762e-05,0.04289036,0,0.0140831,0,-0.03705638,0,-0.02073352,0.01322906,0,0,0,0.02111441,-0.02308533,0.03847408,-0.01464109,0,-0.0329202,0,0.02074971,-0.01973395,0,0.0008027369,0.02722976]},{"f":[33,32,33,33,33,33,1,24,33,37,-1,-1,32,-1,4,-1,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,12.64516129032258,9.5,13.895530557358477,8.05,0,0,5.845435504646156,0,4.80643424864552,0,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,13,0,15,0,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,18,17,12,11,0,0,14,0,16,0,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001954926,0.02753784,0,-0.03135453,0,-0.004277736,0.008928019,0.0166291,-0.01156325,0.02131871,-0.01547853,0,0,0.008351983,0.04044206,-0.01713424,-0.02216533,-0.03763772,0.008883194]},{"f":[27,22,22,-1,-1,24,-1,22,-1,22,-1,18,-1,22,-1,23,-1,-1,22,0,-1,-1,22,-1,22,-1,17,-1,-1],"t":[0.5,16.73920708776685,16.725912049439096,0,0,6.5,0,16.765402642760364,0,16.786010507999382,0,3.39067985676233,0,16.819976309430963,0,2.5,0,0,16.978341770814396,8.5,0,0,16.98920034885663,0,17.078739781790915,0,4.128693921346326,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,0,10,0,12,0,14,0,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[18,5,4,0,0,7,0,9,0,11,0,13,0,15,0,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,0,0,3.248992e-06,0.02236146,0,0.02307242,0,-0.03285547,0,0.05056606,0,-0.02511892,0,-0.01335278,0,0.02781915,0.0004492083,0,0,-0.02858647,0.0130073,0,0.05153679,0,-0.02638865,0,0.00437282,0.03353279]},{"f":[22,23,21,4,2,-1,-1,16,-1,16,-1,-1,0,1,16,15,-1,-1,-1,-1,-1,22,33,-1,-1,1,-1,-1,-1],"t":[17.105450904090162,7.5,9.5,9.328332883600808,9.293467526905953,0,0,4.247927513443585,0,10.696098170955864,0,0,7.5,6.015625,7.6653359171851765,6.0,0,0,0,0,0,16.98920034885663,26.622428940176302,0,0,6.981101934362095,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,0,13,14,15,16,0,0,0,0,0,22,23,0,0,26,0,0,0],"r":[28,21,12,7,6,0,0,9,0,11,0,0,20,19,18,17,0,0,0,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.0001841701,-0.03015335,0,-0.009023451,0,0.04011197,-0.001152659,0,0,0,0,0.01078874,-0.01486718,0.01308399,-0.01831711,0.0375592,0,0,-0.01784648,0.01689643,0,0.01574567,-0.04233949,0.01021732]},{"f":[1,16,-1,-1,1,-1,22,-1,37,16,-1,21,-1,17,-1,-1,25,23,1,-1,15,19,-1,-1,-1,-1,33,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,0,4.875,10.372865060112588,0,6.5,0,3.245926548164837,0,0,0.5,4.5,4.949128919860627,0,7.118941072723508,1.5,0,0,0,0,38.67649696721479,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,0,7,0,9,10,0,12,0,14,0,0,17,18,19,0,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,6,0,8,0,16,11,0,13,0,15,0,0,26,25,20,0,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.03068195,0.04247626,0,-0.0356296,0,-0.001420239,0,0,-0.000690105,0,-0.04008498,0,-0.03850944,-0.000690105,0,0,0,0.03783063,0,0,0.04405953,0.002963187,0.002522635,-0.000690105,0,-0.000690105,0.04045227]},{"f":[22,-1,22,22,21,-1,-1,-1,21,17,22,-1,-1,-1,4,15,-1,15,22,-1,-1,-1,18,2,-1,-1,15,-1,-1],"t":[2.69615871138938,0,5.9525054311951475,4.926654777701837,6.5,0,0,0,2.5,1.292481250360578,12.027747761539269,0,0,0,9.266718101290426,1.0,0,1.584962500721156,10.194756329264862,0,0,0,10.457893746154532,10.680359255641626,0,0,9.862637357558794,0,0],"m":[0,0,0,1,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,0,0,15,16,0,18,19,0,0,0,23,24,0,0,27,0,0],"r":[2,0,8,7,6,0,0,0,14,13,12,0,0,0,22,17,0,21,20,0,0,0,26,25,0,0,28,0,0],"v":[0,-0.0234169,0,0,0,-0.02706728,0.01061899,0.03262572,0,0,0,-0.0245002,0.01103377,-0.03368796,0,0,0.01756277,0,0,-0.02728704,0.006751426,0.0003492933,0,0,-0.03092196,0.02498004,0,0.03214731,-0.01101499]},{"f":[4,4,21,18,22,-1,-1,15,-1,-1,25,15,15,-1,-1,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,5.065928480304397,13.827286379004683,0,0,5.614709844115208,0,0,2.5,5.169925001442312,4.087462841250339,0,0,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[1,1,0,1,1,0,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,12,13,0,0,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[26,25,10,7,6,0,0,9,0,0,16,15,14,0,0,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,-0.0397427,0.01358932,0,0.04231364,-0.02036749,0,0,0,0.0006554825,0.01238794,-0.0004639606,0,-0.04330753,0,0,0,-0.001427181,-0.0184904,0.00780679,0.0128989,-0.0121621,0,-0.005054549,0.03565654]},{"f":[36,-1,33,33,15,22,17,22,-1,-1,22,-1,1,-1,-1,-1,15,4,-1,-1,22,20,-1,-1,-1,-1,33,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,5.169925001442312,15.212004812197344,2.69615871138938,13.054613167076562,0,0,12.613886328427084,0,6.276785714285714,0,0,0,7.266786540694901,1.792481250360578,0,0,16.725912049439096,0.5,0,0,0,0,21.16864017939033,0,0],"m":[0,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,0,13,0,0,0,17,18,0,0,21,22,0,0,0,0,27,0,0],"r":[2,0,26,25,16,15,10,9,0,0,12,0,14,0,0,0,20,19,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0.01926051,0,0,0,0,0,0,0.002176211,-0.04699047,0,-0.001525107,0,-0.003768597,0.03342055,0.03662304,0,0,-0.05354741,-0.008933978,0,0,-0.01827848,-0.000178523,0.01637283,-0.03455966,0,0.02182082,0.0002470965]},{"f":[22,22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,23,17,22,-1,-1,-1,-1,18,-1,18,-1,-1,2,-1,-1],"t":[17.07907323966382,17.069460217922263,16.978341770814396,16.786010507999382,16.765402642760364,0,0,6.208926257442949,0,3.903677461028802,0,16.819976309430963,0,0,7.5,6.781300095563516,16.98920034885663,0,0,0,0,6.076776015854056,0,7.383290334558488,0,0,4.0437314206251695,0,0],"m":[1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,0,8,0,10,0,12,0,0,15,16,17,0,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,14,7,6,0,0,9,0,11,0,13,0,0,20,19,18,0,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-8.274457e-05,0.04065332,0,0.01380531,0,-0.03618138,0,-0.01884859,0.01191364,0,0,0,0.01931848,-0.02192679,0.03689126,-0.01378302,0,-0.0305744,0,0.02004292,-0.01903441,0,0.001066688,0.02610127]},{"f":[22,-1,4,4,21,18,22,-1,-1,22,-1,-1,15,-1,25,-1,18,-1,22,36,18,-1,-1,-1,-1,-1,17,-1,-1],"t":[2.69615871138938,0,10.510264838663748,9.283086494496427,2.5,5.065928480304397,13.827286379004683,0,0,11.36027577861459,0,0,1.0,0,2.5,0,1.292481250360578,0,12.613886328427084,2.5,1.792481250360578,0,0,0,0,0,2.160964047443681,0,0],"m":[0,0,1,1,0,1,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,0,15,0,17,0,19,20,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,12,9,8,0,0,11,0,0,14,0,16,0,18,0,24,23,22,0,0,0,0,0,28,0,0],"v":[0,-0.0231273,0,0,0,0,0,-0.03937696,0.01280615,0,0.04117672,-0.02027805,0,0.01489144,0,0.0004949228,0,-0.04241776,0,0,0,-0.001286757,-0.01768719,0.007447166,0.01169727,-0.01228884,0,-0.004851012,0.03523205]},{"f":[36,-1,33,33,-1,-1,33,-1,22,22,22,22,22,-1,-1,22,-1,22,-1,24,15,-1,-1,22,-1,-1,-1,-1,-1],"t":[1.5,0,20.752541988539424,18.99840809449714,0,0,21.16864017939033,0,17.078739781790915,17.069460217922263,16.978341770814396,16.73920708776685,16.25454052700782,0,0,16.765402642760364,0,16.786010507999382,0,6.5,6.169925001442312,0,0,16.838461650489297,0,0,0,0,0],"m":[0,0,0,0,0,0,0,0,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,13,0,0,16,0,18,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,6,5,0,0,8,0,28,27,26,15,14,0,0,17,0,19,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0.01904812,0,0,-0.001386145,-0.0338623,0,0.02115922,0,0,0,0,0,0.0001416421,0.01828222,0,-0.03150567,0,0.04845059,0,0,0.0009769029,-0.03671669,0,-0.008649636,0.02874892,0.01836524,-0.02889569,0.008258235]},{"f":[2,4,18,17,15,-1,22,-1,15,-1,-1,27,-1,15,-1,-1,15,16,-1,-1,-1,17,-1,22,-1,-1,33,-1,-1],"t":[2.453445297804259,1.792481250360578,3.245926548164837,1.792481250360578,7.6653359171851765,0,10.735831902194036,0,10.113742166049189,0,0,0.5,0,8.618385502258606,0,0,10.501837184902296,6.832890014164741,0,0,0,2.69615871138938,0,11.775312925643524,0,0,38.67649696721479,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,5,0,7,0,9,0,0,12,0,14,0,0,17,18,0,0,0,22,0,24,0,0,27,0,0],"r":[26,21,16,11,6,0,8,0,10,0,0,13,0,15,0,0,20,19,0,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.009714011,0,0.03074234,0,0.007601305,-0.01988504,0,0.005662974,0,0.0561421,-0.02759225,0,0,0.001642888,-0.02223094,0.01108274,0,-0.0232225,0,-0.04498874,0.03277145,0,0.0002996279,0.03187001]},{"f":[33,32,33,33,33,33,33,33,23,-1,-1,19,-1,-1,19,22,-1,-1,-1,-1,-1,22,0,-1,-1,-1,-1,-1,-1],"t":[44.79706586676442,10.482680779086738,39.9785486130134,35.37421412138842,32.199935296219856,29.807475161786478,27.27007187651612,24.56142327888218,1.5,0,0,0.5,0,0,0.5,16.978341770814396,0,0,0,0,0,17.02942582914998,9.5,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,12,0,0,15,16,0,0,0,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,21,20,19,14,11,10,0,0,13,0,0,18,17,0,0,0,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.001053212,-0.001994669,0,0.002408304,0.0368105,0,0,-0.004734427,0.01843955,-0.03337248,0.02041653,-0.01548377,0,0,0.007549821,0.03852867,-0.0156064,-0.02379339,-0.03698385,0.008782827]},{"f":[1,16,-1,-1,1,-1,22,21,15,-1,22,21,-1,-1,21,-1,24,-1,23,17,-1,-1,-1,-1,37,16,-1,-1,-1],"t":[0.01933482600988898,11.031356596255709,0,0,0.3126984126984127,0,12.121442107629743,5.5,3.169925001442312,0,8.723206471664755,3.5,0,0,3.5,0,12.5,0,0.5,7.980000966034041,0,0,0,0,4.875,10.372865060112588,0,0,0],"m":[0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,0,7,8,9,0,11,12,0,0,15,0,17,0,19,20,0,0,0,0,25,26,0,0,0],"r":[4,3,0,0,6,0,24,23,10,0,14,13,0,0,16,0,18,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,0,-0.03005443,0.04154321,0,-0.03509368,0,0,0,-0.04281656,0,0,0.05290852,-0.0003390853,0,-0.0445974,0,-0.04502883,0,0,0.02607024,-0.002710098,-0.03981304,-4.747419e-05,0,0,-0.0005959057,-0.01383349,0.002150883]},{"f":[21,4,2,2,2,22,-1,22,-1,-1,15,17,-1,-1,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,7.005613627711627,14.116600814324018,0,16.73920708776685,0,0,10.113742166049189,10.620215166896223,0,0,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,0,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,11,12,0,0,0,16,0,0,0,20,0,22,0,0,25,26,0,0,0],"r":[24,19,18,15,10,7,0,9,0,0,14,13,0,0,0,17,0,0,0,21,0,23,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.00137234,0,0.006864906,-0.002328207,0,0,-0.02088969,0.004742376,0.003412793,0,0.001778179,0.03235362,-0.02802522,0,-0.00838907,0,0.03976938,-0.001021682,0,0,0.004647844,0.03688829,-0.02771817]},{"f":[4,19,15,-1,-1,36,-1,16,17,-1,33,4,-1,0,-1,-1,-1,18,37,-1,-1,32,-1,33,-1,-1,37,-1,-1],"t":[2.453445297804259,0.5,3.169925001442312,0,0,2.5,0,7.118941072723508,1.792481250360578,0,21.257266042509173,1.292481250360578,0,7.5,0,0,0,2.160964047443681,6.890000000000001,0,0,7.491051153478379,0,44.805779984159756,0,0,12.52,0,0],"m":[1,1,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,11,12,0,14,0,0,0,18,19,0,0,22,0,24,0,0,27,0,0],"r":[26,5,4,0,0,7,0,17,10,0,16,13,0,15,0,0,0,21,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.02378909,0.002966342,0,-0.02705645,0,0,-0.02253216,0,0,0.02094642,0,-0.02305267,0.01930283,-0.004778912,0,0,-0.01891737,0.007030573,0,-0.04785072,0,0.003618846,-0.02354262,0,0.0004568836,0.03109942]},{"f":[22,-1,36,-1,33,22,-1,-1,33,22,-1,-1,22,22,22,-1,17,22,15,-1,-1,-1,-1,-1,18,17,-1,-1,-1],"t":[2.69615871138938,0,1.5,0,24.416594167368267,16.98920034885663,0,0,26.99635432609064,12.121442107629743,0,0,17.078739781790915,17.069460217922263,16.978341770814396,0,6.781300095563516,16.98920034885663,8.618385502258606,0,0,0,0,0,8.729619742794132,7.6181566135214025,0,0,0],"m":[0,0,0,0,0,1,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,0,5,6,0,0,9,10,0,0,13,14,15,0,17,18,19,0,0,0,0,0,25,26,0,0,0],"r":[2,0,4,0,8,7,0,0,12,11,0,0,24,23,16,0,22,21,20,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.02258769,0,0.01870917,0,0,-0.001163067,-0.02515026,0,0,0.00219386,0.03294481,0,0,0,-0.0001417385,0,0,0,-0.011951,0.0349717,-0.03003484,0.03119447,-0.02653055,0,0,0.0008072368,0.03555081,-0.01353254]},{"f":[21,4,2,2,-1,22,-1,-1,-1,16,-1,16,-1,-1,22,0,17,2,17,-1,-1,15,22,-1,-1,-1,-1,-1,-1],"t":[9.5,9.328332883600808,9.293467526905953,9.051177541662383,0,13.005894632618489,0,0,0,4.247927513443585,0,10.696098170955864,0,0,15.756821747580634,7.5,6.781300095563516,1.292481250360578,1.792481250360578,0,0,7.679480099505446,10.735831902194036,0,0,0,0,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,0,12,0,0,15,16,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[14,9,8,5,0,7,0,0,0,11,0,13,0,0,28,27,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0005573471,0,0.001674208,0.03163256,-0.02700685,0,-0.008037009,0,0.03859088,-0.0009962494,0,0,0,0,0,-5.211721e-05,-0.0355644,0,0,0.007644839,-0.01557479,0.02041412,0.01979599,0.03645675,-0.02683742]},{"f":[4,4,21,22,-1,37,-1,-1,15,15,-1,-1,16,22,2,33,-1,-1,-1,22,-1,-1,15,-1,-1,-1,17,-1,-1],"t":[10.510264838663748,9.266718101290426,2.5,6.106096185954236,0,5.25,0,0,5.169925001442312,4.087462841250339,0,0,6.108524456778169,16.98413870862295,4.80643424864552,26.622428940176302,0,0,0,16.98920034885663,0,0,5.700439718141092,0,0,0,2.160964047443681,0,0],"m":[1,1,0,0,0,1,0,0,0,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,15,16,0,0,0,20,0,0,23,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,12,11,0,0,22,19,18,17,0,0,0,21,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0.01525134,0,-0.03902519,0.01520265,0,0,-0.0001732564,0.009336089,0,0,0,0,-0.03819535,0.007397454,-0.004184137,0,0.03345825,-0.004526526,0,0.03207897,0.000147787,-0.01179276,0,-0.00450298,0.03475647]}]},"p_fail":{"link":"logistic","base":-2.9214699960499186,"trees":[{"f":[37,2,4,22,-1,-1,20,-1,-1,21,-1,-1,22,36,-1,23,-1,-1,4,32,-1,32,-1,19,36,-1,-1,-1,-1],"t":[7.05,6.247802615595391,5.065928480304397,17.094543568094064,0,0,0.5,0,0,3.5,0,0,12.613886328427084,4.5,0,1.5,0,0,6.16978583919235,5.450056031488728,0,6.208926257442949,0,2.5,2.5,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,0,10,0,0,13,14,0,16,0,0,19,20,0,22,0,24,25,0,0,0,0],"r":[12,9,6,5,0,0,8,0,0,11,0,0,18,15,0,17,0,0,28,21,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.04769787,0.0934937,0,-0.02507065,0.09083067,0,-0.04844851,0.0934937,0,0,0.0934937,0,0.0934937,0.3504119,0,0,0.4894688,0,0.0934937,0,0,0.0934937,0.3642448,0.0934937,0.6431695]},{"f":[37,2,22,-1,-1,21,-1,-1,22,21,-1,37,4,-1,-1,-1,4,19,22,22,36,-1,-1,-1,32,-1,-1,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0,12.613886328427084,5.5,0,9.075,3.245926548164837,0,0,0,6.16978583919235,2.5,16.25454052700782,15.439474430023438,3.5,0,0,0,9.271599002932193,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,13,0,0,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[8,5,4,0,0,7,0,0,16,11,0,15,14,0,0,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,-0.04745094,0.0681254,0,-0.05016321,0.0681254,0,0,0.0681254,0,0,0.0681254,0.139018,0.3786697,0,0,0,0,0,0.2480394,0.0681254,0.3715553,0,0.08642291,0.2383047,0.0681254,0.3876677]},{"f":[37,2,22,-1,-1,21,-1,-1,22,21,-1,33,-1,-1,19,32,-1,15,-1,22,22,33,-1,-1,-1,-1,2,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0,12.613886328427084,5.5,0,27.893589438525005,0,0,2.5,5.450056031488728,0,6.169925001442312,0,17.094543568094064,16.98920034885663,17.460839126440824,0,0,0,0,1.292481250360578,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,0,0,15,16,0,18,0,20,21,22,0,0,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,14,11,0,13,0,0,26,17,0,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,0,-0.0471081,0.05273543,0,-0.04993683,0.05273543,0,0,0.05273543,0,0.06704428,0.3058953,0,0,0.2611051,0,0.05273543,0,0,0,0.06439798,0.1988733,0.05273543,0.3195874,0,0.05273543,0.1830133]},{"f":[37,2,22,-1,-1,21,-1,9,-1,-1,22,21,-1,1,-1,3,-1,-1,4,19,22,15,-1,-1,22,-1,-1,-1,-1],"t":[7.05,5.228690439536267,17.094543568094064,0,0,3.5,0,0.5,0,0,12.613886328427084,5.5,0,8.980632551528075,0,0.9197769197769198,0,0,6.16978583919235,2.5,16.98920034885663,13.56045225766691,0,0,17.094543568094064,0,0,0,0],"m":[1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0],"l":[1,2,3,0,0,6,0,8,0,0,11,12,0,14,0,16,0,0,19,20,21,22,0,0,25,0,0,0,0],"r":[10,5,4,0,0,7,0,9,0,0,18,13,0,15,0,17,0,0,28,27,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.0467624,0.04229328,0,-0.04971282,0,0.04229328,-0.04846064,0,0,0.04229328,0,0.04229328,0,0.09199477,0.259104,0,0,0,0,0.1681462,0.04229328,0,0.04229328,0.1356608,0.04229328,0.2279098]},{"f":[37,2,-1,21,-1,-1,32,19,21,22,-1,15,-1,15,-1,22,22,-1,-1,-1,-1,-1,3,-1,17,-1,17,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,10.41931383093672,2.5,9.5,14.03140051979143,0,5.044394119358453,0,6.169925001442312,0,17.094543568094064,16.98920034885663,0,0,0,0,0,0.01140250391999155,0,7.980000966034041,0,8.729619742794132,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,10,0,12,0,14,0,16,17,0,0,0,0,0,23,0,25,0,27,0,0],"r":[6,3,0,5,0,0,22,21,20,11,0,13,0,15,0,19,18,0,0,0,0,0,24,0,26,0,28,0,0],"v":[0,0,-0.04758055,0,-0.04961903,0.0231401,0,0,0,0,0.06343669,0,0.2028083,0,0.0231401,0,0,0.1305835,0.02508399,0.211893,0.2532368,0.0231401,0,0.2139762,0,0.0231401,0,0.3284481,0.0231401]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,21,22,-1,-1,-1,4,-1,-1,2,22,-1,23,-1,-1,18,2,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,9.5,12.613886328427084,0,0,0,2.69615871138938,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,6.8391900325545585,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,0,0,1,1,0,1,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,0,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,18,15,14,13,0,0,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.0472698,0,-0.04940234,0,0.01861817,-0.04935216,0,0,0,0,0.05051997,0.1155308,0.2080956,0,0.01861817,0.09211612,0,0,0.01861817,0,0.2278524,0.01861817,0,0,0.1058997,0.2163142,0.01861817]},{"f":[37,2,-1,21,-1,-1,32,19,21,22,16,-1,-1,1,-1,-1,-1,33,-1,-1,2,22,-1,23,-1,-1,18,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,10.41931383093672,2.5,9.5,15.439474430023438,6.0,0,0,7.6141732283464565,0,0,0,24.04238488811619,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,0,0],"m":[1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,9,10,11,0,0,14,0,0,0,18,0,0,21,22,0,24,0,0,27,0,0],"r":[6,3,0,5,0,0,20,17,16,13,12,0,0,15,0,0,0,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-0.04695588,0,-0.04918672,0.01489223,0,0,0,0,0,0.01489223,0.08088957,0,0.07379098,0.1299393,0.1775134,0,0.01489223,0.0841136,0,0,0.01489223,0,0.1894964,0.01489223,0,0.1321402,0.01489223]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,21,22,-1,15,-1,-1,-1,4,-1,-1,2,22,-1,23,-1,-1,22,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,9.5,12.613886328427084,0,5.044394119358453,0,0,0,2.69615871138938,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,16.765402642760364,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,1,0,0,1,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,0,14,0,0,0,18,0,0,21,22,0,24,0,0,27,0,0],"r":[8,3,0,5,0,7,0,0,20,17,16,13,0,15,0,0,0,19,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,-0.04663823,0,-0.04897172,0,0.01189007,-0.04915927,0,0,0,0,0.04010872,0,0.128136,0.08068234,0.1553895,0,0.01189007,0.07539508,0,0,0.01189007,0,0.1613723,0.01189007,0,0.08239529,0.2135406]},{"f":[32,37,-1,19,-1,-1,2,2,22,-1,16,-1,-1,0,-1,22,2,-1,-1,2,-1,-1,1,19,-1,-1,20,-1,-1],"t":[10.41931383093672,6.970000000000001,0,2.5,0,0,9.138550601137563,5.228690439536267,17.094543568094064,0,6.539158811108031,0,0,4.5,0,14.03140051979143,6.942466823974881,0,0,5.40929108874043,0,0,2.0761752136752136,1.5,0,0,0.5,0,0],"m":[0,0,0,1,0,0,1,1,1,0,1,0,0,1,0,1,1,0,0,0,0,0,1,1,0,0,1,0,0],"l":[1,2,0,4,0,0,7,8,9,0,11,0,0,14,0,16,17,0,0,20,0,0,23,24,0,0,27,0,0],"r":[6,3,0,5,0,0,22,13,10,0,12,0,0,15,0,19,18,0,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,-0.009080677,0,0.07817652,0.03068926,0,0,0,-0.04601527,0,-0.0302457,0.1278346,0,-0.01969078,0,0,0.01363114,0.118908,0,0.308648,0.05017494,0,0,0.01796037,0.1290202,0,0.3238385,0.0726058]},{"f":[37,2,-1,21,-1,9,-1,-1,32,19,4,-1,-1,33,-1,24,-1,-1,2,22,-1,23,-1,-1,18,22,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0.5,0,0,10.41931383093672,2.5,6.08730682361797,0,0,24.04238488811619,0,8.5,0,0,6.118903736861568,17.094543568094064,0,7.5,0,0,8.03615005796417,16.759160379664443,0,0,0],"m":[1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,1,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,0,0,14,0,16,0,0,19,20,0,22,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,18,13,12,0,0,15,0,17,0,0,24,21,0,23,0,0,28,27,0,0,0],"v":[0,0,-0.04600386,0,-0.04871792,0,0.007965274,-0.04873968,0,0,0,0.06515144,0.128155,0,0.007965274,0,0.007965274,0.1174494,0,0,0.007965274,0,0.1249281,0.007965274,0,0,0.0732324,0.1747689,0.007965274]},{"f":[32,37,17,-1,-1,19,-1,-1,2,2,22,-1,16,-1,-1,0,-1,22,-1,23,-1,-1,1,22,-1,-1,2,-1,-1],"t":[10.41931383093672,7.125,4.2849278041654735,0,0,2.5,0,0,9.138550601137563,5.228690439536267,17.094543568094064,0,6.539158811108031,0,0,4.5,0,11.775312925643524,0,4.5,0,0,2.01685992705101,16.725912049439096,0,0,10.510264838663748,0,0],"m":[0,0,1,0,0,1,0,0,1,1,1,0,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,0,16,0,18,0,20,0,0,23,24,0,0,27,0,0],"r":[8,5,4,0,0,7,0,0,22,15,12,0,14,0,0,17,0,19,0,21,0,0,26,25,0,0,28,0,0],"v":[0,0,0,-0.03423149,0.02865827,0,0.06654153,0.03003279,0,0,0,-0.04538788,0,-0.02954708,0.1011357,0,-0.01893109,0,0.0106163,0,0.09251107,0.02714556,0,0,0.02421608,0.1225957,0,0.09327025,0.2530409]},{"f":[32,37,17,-1,-1,19,4,-1,-1,-1,2,22,-1,17,-1,-1,4,4,3,18,-1,-1,-1,17,3,-1,-1,-1,-1],"t":[10.41931383093672,7.205,4.2849278041654735,0,0,2.5,6.08730682361797,0,0,0,5.228690439536267,17.094543568094064,0,7.588709768994619,0,0,10.510264838663748,5.40929108874043,0.9300813008130082,1.292481250360578,0,0,0,11.78143940991899,0.9148166023166023,0,0,0,0],"m":[0,0,1,0,0,1,1,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,0,11,12,0,14,0,0,17,18,19,20,0,0,0,24,25,0,0,0,0],"r":[10,5,4,0,0,9,8,0,0,0,16,13,0,15,0,0,28,23,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03362229,0.0267103,0,0,0.0559467,0.1115538,0.02782049,0,0,-0.04502868,0,-0.0291198,0.09162809,0,0,0,0,0.0830128,-0.009944685,0.225705,0,0,0.09089829,-0.01522781,0.117968,0.1732332]},{"f":[37,2,-1,21,-1,-1,22,21,-1,-1,23,22,18,22,-1,4,22,22,-1,-1,-1,-1,33,-1,-1,18,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,14.116600814324018,8.5,0,0,7.5,17.094543568094064,2.160964047443681,15.782972563647261,0,2.453445297804259,16.765402642760364,16.73920708776685,0,0,0,0,20.752541988539424,0,0,8.729619742794132,0,0,0],"m":[1,1,0,0,0,0,0,1,0,0,1,1,0,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,0,16,17,18,0,0,0,0,23,0,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,28,25,22,15,0,21,20,19,0,0,0,0,24,0,0,27,0,0,0],"v":[0,0,-0.0450083,0,-0.04848159,0.004115474,0,0,0.004115474,0.09244651,0,0,0,0,0.1137074,0,0,0,0.07352224,0.004115474,0.1430886,0.01691213,0,0.004115474,0.0478023,0,0.1306,0.004115474,0.004115474]},{"f":[2,32,19,23,37,-1,-1,-1,-1,22,-1,-1,2,37,-1,22,-1,22,-1,22,3,-1,32,-1,-1,-1,0,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,8.05,0,0,0,0,17.094543568094064,0,0,10.100798745059642,4.875,0,14.116600814324018,0,14.158358850608169,0,17.094543568094064,0.026802550058364013,0,8.599793042091783,0,0,0,2.5,0,0],"m":[1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,0,0,10,0,0,13,14,0,16,0,18,0,20,21,0,23,0,0,0,27,0,0],"r":[12,9,8,7,6,0,0,0,0,11,0,0,26,15,0,17,0,19,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0.03467511,0.07517031,-0.05267765,-0.05050093,0,-0.04429315,0.03881418,0,0,-0.01054975,0,0.02064976,0,0.196239,0,0,0.07833555,0,0.01853533,0.05589292,0.1184446,0,0.007721112,0.152295]},{"f":[2,32,19,23,37,22,-1,-1,-1,-1,-1,22,-1,-1,2,37,-1,22,18,-1,-1,22,-1,22,2,-1,-1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,8.05,12.613886328427084,0,0,0,0,0,17.094543568094064,0,0,10.510264838663748,4.875,0,14.116600814324018,7.980000966034041,0,0,14.158358850608169,0,17.094543568094064,8.049190845056867,0,0,0,0],"m":[1,0,1,1,1,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,12,0,0,15,16,0,18,19,0,0,22,0,24,25,0,0,0,0],"r":[14,11,10,9,8,7,0,0,0,0,0,13,0,0,28,17,0,21,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.01037496,0.04517335,0.06944228,-0.05197606,-0.05014537,0,-0.04390751,0.03608557,0,0,-0.00982346,0,0,0.01749402,0.06173516,0,0.1673379,0,0,0.03146814,0.08225331,0.1105438,0.1242269]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,2,37,-1,22,-1,22,-1,22,3,22,-1,-1,-1,36,-1,-1,0,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,10.100798745059642,4.875,0,14.116600814324018,0,14.158358850608169,0,16.759160379664443,0.12256006006006007,16.73920708776685,0,0,0,2.5,0,0,2.5,0,0],"m":[1,0,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,0,14,0,16,0,18,19,20,0,0,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,26,13,0,15,0,17,0,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.04323287,-0.05129791,-0.04979607,0,-0.04351432,0.03361381,0,0,-0.009722018,0,0.01774984,0,0.1459777,0,0,0,0.06562953,0.01546931,0.01546931,0,0.01546931,0.08406263,0,0.002805423,0.1184653]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,2,1,21,-1,-1,17,32,-1,-1,17,-1,21,-1,22,-1,-1,1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,8.9274737487839,4.92309205350118,7.5,0,0,7.198837316474133,6.337096134072842,0,0,7.781300095563516,0,4.5,0,17.083799521744545,0,0,2.01685992705101,0,0],"m":[1,0,1,1,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,0,16,17,0,0,20,0,22,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,26,15,14,0,0,19,18,0,0,21,0,23,0,25,0,0,28,0,0],"v":[0,0,0,0,0.04019397,-0.05064109,-0.04945217,0,-0.04311322,0.03136577,0,0,0,-0.03874625,0.01260896,0,0,-0.02822746,0.06900363,0,0.1011399,0,-0.04572707,0,0.03976827,0.101486,0,0.03259705,0.09209488]},{"f":[37,2,-1,21,-1,-1,22,21,-1,-1,23,22,22,18,22,22,-1,-1,-1,22,23,-1,-1,-1,-1,18,-1,-1,-1],"t":[6.970000000000001,5.065928480304397,0,3.5,0,0,14.116600814324018,8.5,0,0,7.5,17.094543568094064,16.98920034885663,2.160964047443681,16.765402642760364,15.782972563647261,0,0,0,16.844368517741394,1.5,0,0,0,0,8.729619742794132,0,0,0],"m":[1,1,0,0,0,0,0,1,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,0,4,0,0,7,8,0,0,11,12,13,14,15,16,0,0,0,20,21,0,0,0,0,26,0,0,0],"r":[6,3,0,5,0,0,10,9,0,0,28,25,24,19,18,17,0,0,0,23,22,0,0,0,0,27,0,0,0],"v":[0,0,-0.04329869,0,-0.04809694,0.00108665,0,0,0.00108665,0.07404429,0,0,0,0,0,0,0.08492338,0.02684222,0.1149836,0,0,0.0408302,0.00108665,0.08200073,0.00108665,0,0.1011915,0.00108665,0.00108665]},{"f":[2,32,19,23,-1,-1,-1,22,-1,-1,21,-1,2,1,21,-1,-1,17,32,22,-1,-1,-1,24,-1,-1,1,-1,-1],"t":[5.228690439536267,10.019277334827486,2.5,4.5,0,0,0,17.094543568094064,0,0,3.5,0,8.9274737487839,4.92309205350118,7.5,0,0,7.198837316474133,6.337096134072842,15.782972563647261,0,0,0,5.5,0,0,2.01685992705101,0,0],"m":[1,0,1,1,0,0,0,1,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,0,13,14,15,0,0,18,19,20,0,0,0,24,0,0,27,0,0],"r":[10,7,6,5,0,0,0,9,0,0,12,0,26,17,16,0,0,23,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.03544255,-0.05079942,-0.04916817,0,-0.04238859,0.02788346,0,-0.04983604,0,0,0,-0.03669358,0.01109883,0,0,0,-0.04248968,0.05992567,0.06142792,0,0.1085325,0.04500142,0,0.03381769,0.08928647]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,2,32,4,21,-1,-1,-1,36,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,1.282051282051282,0,9.068775776507199,6.337096134072842,5.714180086352146,7.5,0,0,0,4.5,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,0,14,0,0,17,0,19,20,21,22,0,0,0,26,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,16,13,0,15,0,0,18,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,-0.02120774,0.04750176,-0.02658132,0,-0.04620745,0.09207463,0,0,-0.03297489,0,0.0432284,-0.04898179,0,-0.01578352,0,0,0,0,0.1064438,0.01990581,-0.007052136,0,0.08881526,0.04026965,0.07190423]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,1,-1,2,32,4,21,-1,-1,-1,-1,-1],"t":[4.5,7.5,1.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,1.282051282051282,0,9.068775776507199,6.337096134072842,5.714180086352146,7.5,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,13,0,0,16,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,18,15,12,0,14,0,0,17,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.01489534,0.03607625,0,-0.04587588,0.08355575,0,0,0,-0.03804654,0,0.05311217,-0.04043443,0,0.04045386,-0.04866499,0,-0.01522208,0,0,0,0,0.09582695,0.01864698,-0.006744542,0.05059549,0.06568794]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,0,-1,2,1,-1,22,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,2.5,0,7.661668144640085,4.92309205350118,0,17.094543568094064,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,1,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,15,0,0,18,0,0,21,0,23,24,0,26,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,20,17,14,0,16,0,0,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,0,0,-0.02069481,0.04262807,-0.02611898,0,-0.04553799,0.07617299,0,0,0,-0.03750723,0,0.04942457,-0.0397771,0,0.03791869,-0.04835057,0,-0.01324508,0,0,0.00989946,0,0.03356092,0.08526907,0.0618625]},{"f":[21,24,1,-1,15,-1,-1,17,-1,-1,2,24,22,-1,-1,35,-1,-1,1,-1,2,1,3,-1,-1,-1,1,-1,-1],"t":[4.5,7.5,6.241071428571429,0,6.247927513443585,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,0,0,0.7457446808510639,0,9.068775776507199,4.92309205350118,0.02996265189107678,0,0,0,2.01685992705101,0,0],"m":[0,0,0,0,1,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,0,8,0,0,11,12,13,0,0,16,0,0,19,0,21,22,23,0,0,0,27,0,0],"r":[10,7,4,0,6,0,0,9,0,0,18,15,14,0,0,17,0,0,20,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,-0.03749565,0,0.03327541,-0.04072238,0,-0.0451934,0.06972815,0,0,0,-0.03695925,0.01076815,0,0.03559294,-0.04803796,0,-0.03453626,0,0,0,0.04762514,-0.0022136,0.03765599,0,0.02828295,0.07306647]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,-1,-1,35,15,-1,-1,-1,0,-1,2,22,2,21,-1,-1,-1,-1,-1],"t":[4.5,7.5,0.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,8.618385502258606,0,0,0,2.5,0,7.640240405159187,17.094543568094064,6.011183906514227,7.5,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,0,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,0,0,14,15,0,0,0,19,0,21,22,23,24,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,18,13,12,0,0,17,16,0,0,0,20,0,28,27,26,25,0,0,0,0,0],"v":[0,0,0,-0.01566103,0.03023375,0,-0.04484172,0.06406368,0,0,0,-0.03640272,0.01019892,0,0,0.04343813,-0.003260204,-0.0477266,0,-0.01264021,0,0,0,0,0.09407714,0.008180154,0.0106975,0.08078468,0.05320676]},{"f":[21,24,1,-1,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,22,2,4,21,4,-1,16,-1,-1,-1,-1,-1,-1],"t":[4.5,7.5,6.241071428571429,0,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,4.0437314206251695,0,6.022367813028454,0,0,0,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,0,12,0,0,15,0,17,18,19,20,21,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,14,11,0,13,0,0,16,0,28,27,26,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.03685333,0.02180261,0,-0.0444826,0.05905361,0,0,-0.02985524,0,0.03159317,-0.04741601,0,-0.03399038,0,0,0,0,0,0.01851626,0,0.02402876,0.1257916,0.01502488,-0.001678763,0.04544176,0.07722198]},{"f":[21,24,-1,17,-1,-1,2,24,-1,35,-1,-1,1,-1,22,2,4,21,22,4,-1,15,-1,-1,-1,-1,-1,-1,-1],"t":[4.5,7.5,0,11.578476357671452,0,0,4.882435795368045,7.5,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,16.98920034885663,4.0437314206251695,0,6.0,0,0,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,0,7,8,0,10,0,0,13,0,15,16,17,18,19,20,0,22,0,0,0,0,0,0,0],"r":[6,3,0,5,0,0,12,9,0,11,0,0,14,0,28,27,26,25,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0.01204038,0,-0.04411569,0.05459654,0,0,-0.02917017,0,0.02975699,-0.0471057,0,-0.03334568,0,0,0,0,0,0,0.03599023,0,0.02607497,0.1228093,-0.02388121,0.01414081,-0.001597897,0.04221126,0.07469412]},{"f":[21,24,16,22,-1,-1,-1,17,-1,-1,2,24,22,-1,15,-1,-1,35,-1,-1,0,-1,2,22,4,-1,-1,-1,-1],"t":[4.5,7.5,6.392317422778761,12.613886328427084,0,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,6.584962500721156,0,0,2.5,0,0,2.5,0,7.640240405159187,17.094543568094064,5.89476682248518,0,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,0,0,0,8,0,0,11,12,13,0,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[10,7,6,5,0,0,0,9,0,0,20,17,14,0,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,-0.02161302,0.03444387,-0.0249776,0,-0.04374068,0.0506102,0,0,0,-0.03488475,0,0.04865143,-0.03849569,0,0.0280519,-0.04679523,0,-0.01251703,0,0,0,0.02208245,0.00594982,0.07343997,0.0444141]},{"f":[21,24,-1,17,-1,-1,2,24,22,-1,-1,35,-1,-1,1,-1,22,2,4,21,22,-1,-1,-1,25,-1,-1,-1,-1],"t":[4.5,7.5,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,0,0,2.5,0,0,0.7457446808510639,0,17.094543568094064,9.068775776507199,5.89476682248518,7.5,16.98920034885663,0,0,0,0.5,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,4,0,0,7,8,9,0,0,12,0,0,15,0,17,18,19,20,21,0,0,0,25,0,0,0,0],"r":[6,3,0,5,0,0,14,11,10,0,0,13,0,0,16,0,28,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0.01077333,0,-0.04335725,0.04702737,0,0,0,-0.03429881,0.01213996,0,0.02646501,-0.0464842,0,-0.03273892,0,0,0,0,0,0.05840486,-0.02381983,0.0120509,0,-0.008741902,0.0477524,0.03715148,0.07030573]},{"f":[21,24,20,-1,-1,17,-1,-1,2,24,22,16,-1,16,-1,-1,-1,35,15,-1,-1,-1,1,-1,22,2,-1,-1,-1],"t":[4.5,7.5,0.5,0,0,11.578476357671452,0,0,4.882435795368045,7.5,17.094543568094064,7.930737337562887,0,7.948367231584678,0,0,0,2.5,8.618385502258606,0,0,0,0.7457446808510639,0,17.094543568094064,10.246900367172017,0,0,0],"m":[0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,0,14,0,0,0,18,19,0,0,0,23,0,25,26,0,0,0],"r":[8,5,4,0,0,7,0,0,22,17,16,13,0,15,0,0,0,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,-0.01620981,0.02469631,0,-0.04296512,0.04379272,0,0,0,0,-0.04605363,0,0.1118448,-0.01814951,0.01148982,0,0,0.03450058,-0.008189892,-0.04617224,0,-0.03209099,0,0,0.01515762,0.04674731,0.06834029]},{"f":[21,22,-1,-1,2,32,23,-1,-1,22,-1,17,-1,-1,1,-1,22,23,-1,-1,15,-1,22,20,-1,36,-1,-1,-1],"t":[3.5,14.03140051979143,0,0,4.95344529780426,9.271599002932193,3.5,0,0,17.094543568094064,0,7.588709768994619,0,0,0.7457446808510639,0,12.199665042528586,2.5,0,0,2.0,0,17.094543568094064,1.5,0,5.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,0,12,0,0,15,0,17,18,0,0,21,0,23,24,0,26,0,0,0],"r":[4,3,0,0,14,9,8,0,0,11,0,13,0,0,16,0,20,19,0,0,22,0,28,25,0,27,0,0,0],"v":[0,0,-0.0492693,0.01850451,0,0,0,0.0297388,-0.03108552,0,-0.0300383,0,-0.03732463,0.05620476,0,-0.03248162,0,0,0.01040253,-0.03359049,0,0.07708978,0,0,0.01205445,0,0.07895045,0.01385546,0.0675537]},{"f":[21,-1,2,32,23,37,-1,-1,-1,22,-1,23,-1,-1,1,-1,22,2,22,23,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,3.5,8.05,0,0,0,17.094543568094064,0,7.5,0,0,1.282051282051282,0,17.094543568094064,7.997154617983022,14.116600814324018,2.5,0,0,6.011183906514227,6.247927513443585,0,0,0,0,0],"m":[0,0,1,0,1,1,0,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,0,10,0,12,0,0,15,0,17,18,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,9,8,7,0,0,0,11,0,13,0,0,16,0,28,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04718663,0,0,0,0,0.006771033,0.03678638,-0.03012814,0,-0.0351069,0,0.04960633,-0.04001062,0,-0.01580857,0,0,0,0,0.007391575,-0.03172959,0,0,0.0191489,0.1123809,0.01037857,0.03187703,0.06580984]},{"f":[21,-1,2,32,23,-1,-1,22,-1,-1,1,21,-1,-1,22,2,4,18,-1,-1,22,1,-1,-1,16,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,3.5,0,0,17.094543568094064,0,0,1.282051282051282,7.5,0,0,17.094543568094064,7.997154617983022,5.89476682248518,5.303665156874805,0,0,16.759160379664443,8.985394456289978,0,0,8.060695931687555,0,0,0,0],"m":[0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,0,0,15,16,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,10,7,6,0,0,9,0,0,14,13,0,0,28,27,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04694051,0,0,0,0.01720874,-0.02918956,0,-0.03453166,0.02455382,0,0,-0.03967469,0.008688512,0,0,0,0,0.01007621,0.04274037,0,0,-0.02485978,0.03548785,0,-0.01045169,0.09564767,0.02987968,0.06417699]},{"f":[21,-1,2,32,23,-1,-1,22,-1,-1,22,2,1,15,-1,-1,22,-1,-1,-1,22,-1,22,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,4.5,0,0,17.094543568094064,0,0,16.765402642760364,10.510264838663748,4.008411214953271,10.043027283594547,0,0,16.73920708776685,0,0,0,16.786010507999382,0,16.838461650489297,0,16.975010474608837,0,17.094543568094064,0,0],"m":[0,0,1,0,1,0,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,0,8,0,0,11,12,13,14,0,0,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,10,7,6,0,0,9,0,0,20,19,16,15,0,0,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,-0.0466876,0,0,0,0.01441173,-0.05029048,0,-0.03394833,0.02309867,0,0,0,0,0.0001123449,-0.03173058,0,0.02017585,-0.03624753,0.04229864,0,0.122649,0,-0.02831197,0,0.08257443,0,0.0003875773,0.06264172]},{"f":[21,-1,2,32,10,-1,37,-1,-1,22,-1,-1,1,-1,22,2,23,-1,-1,-1,2,16,-1,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,0.5,0,8.05,0,0,17.094543568094064,0,0,0.8611111111111112,0,14.116600814324018,10.510264838663748,2.5,0,0,0,6.011183906514227,6.247927513443585,0,0,16.765402642760364,0,16.786010507999382,0,0],"m":[0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,0,15,16,17,0,0,0,21,22,0,0,25,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,14,0,20,19,18,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,-0.04642763,0,0,0,-0.0483602,0,0.0004197282,0.03627079,0,-0.03335729,0.02174877,0,-0.03121126,0,0,0,0.006779979,-0.02569572,0.04693949,0,0,0.01533142,0.1008213,0,0.005871671,0,0.1136719,0.01681201]},{"f":[21,-1,2,32,23,19,37,-1,-1,-1,-1,22,-1,-1,22,1,-1,2,16,17,-1,-1,1,2,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,4.5,2.5,8.05,0,0,0,0,17.094543568094064,0,0,17.094543568094064,0.8611111111111112,0,8.9274737487839,6.0,7.6181566135214025,0,0,6.241071428571429,5.40929108874043,0,0,0,0,0],"m":[0,0,1,0,1,1,1,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,0,12,0,0,15,16,0,18,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,11,10,9,8,0,0,0,0,13,0,0,28,17,0,27,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04616034,0,0,0,0,0,0.004349323,0.03509822,-0.04451945,-0.04941121,0,-0.03275894,0.02049418,0,0,-0.03056908,0,0,0,0.005214205,-0.03269304,0,0,0.05043723,0.002658791,0.02975996,0.0245357,0.0607468]},{"f":[21,-1,2,32,23,-1,-1,22,-1,23,-1,-1,22,9,2,22,23,-1,-1,2,16,-1,-1,0,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,9.271599002932193,3.5,0,0,17.094543568094064,0,7.5,0,0,17.094543568094064,0.5,10.510264838663748,14.116600814324018,2.5,0,0,5.8698903048866296,6.247927513443585,0,0,10.5,0,0,0,0,0],"m":[0,0,1,0,1,0,0,1,0,1,0,0,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,0,0,13,14,15,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[2,0,12,7,6,0,0,9,0,11,0,0,28,27,26,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,-0.04588547,0,0,0,0.01812057,-0.02792353,0,-0.02898088,0,0.04048717,-0.04056214,0,0,0,0,0,0.004344982,-0.0255992,0,0,0.01477525,0.0990032,0,0.009703885,0.05599894,0.03638467,-0.04495106,0.05939642]},{"f":[21,-1,2,32,10,-1,37,-1,-1,22,-1,-1,22,2,1,-1,20,-1,6,-1,-1,-1,22,-1,22,-1,17,-1,-1],"t":[2.5,0,4.95344529780426,10.019277334827486,0.5,0,8.05,0,0,17.094543568094064,0,0,16.765402642760364,10.042900245509953,0.8611111111111112,0,0.5,0,0.5,0,0,0,16.786010507999382,0,16.844368517741394,0,8.571721503073015,0,0],"m":[0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,0,7,0,0,10,0,0,13,14,15,0,17,0,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,12,9,6,0,8,0,0,11,0,0,22,21,16,0,18,0,20,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.04560277,0,0,0,-0.04729053,0,-1.370328e-05,0.03239601,0,-0.03160328,0.01823889,0,0,0,-0.04492289,0,-0.003970515,0,0.01107503,0.08406212,0.03206478,0,0.1042065,0,-0.02511207,0,0.01125656,0.06441634]},{"f":[21,-1,2,1,-1,15,19,17,22,22,37,15,-1,16,-1,16,-1,-1,-1,-1,-1,-1,-1,-1,22,9,-1,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,9.92481250360578,2.5,8.729619742794132,17.094543568094064,16.25454052700782,10.26,5.044394119358453,0,6.044394119358453,0,6.832890014164741,0,0,0,0,0,0,0,0,17.094543568094064,0.5,0,0,0],"m":[0,0,1,1,0,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,11,12,0,14,0,16,0,0,0,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,22,21,20,19,18,13,0,15,0,17,0,0,0,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.045312,0,0,-0.03704434,0,0,0,0,0,0,0,0.03114977,0,-0.04344409,0,0.03093049,-0.0009472042,0.06598908,-0.0446415,0.07566763,-0.03917708,-0.04533979,-0.03784864,0,0,0.009485297,-0.04435298,0.0572878]},{"f":[21,22,-1,-1,0,2,-1,-1,2,32,25,-1,-1,-1,22,2,22,-1,2,16,-1,-1,22,-1,1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,2.5,6.692122061121539,0,0,5.022197059679227,8.041220073122355,1.5,0,0,0,17.094543568094064,9.341519573083566,14.116600814324018,0,5.8698903048866296,6.247927513443585,0,0,16.759160379664443,0,3.354166666666667,0,0,0,0],"m":[0,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,0,0,0,15,16,17,0,19,20,0,0,23,0,25,0,0,0,0],"r":[4,3,0,0,8,7,0,0,14,13,12,0,0,0,28,27,18,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,0,-0.04941795,0.02040811,0,0,-0.03471049,-0.007214486,0,0,0,0.0179928,-0.005743234,-0.007214486,0,0,0,-0.002035699,0,0,0.01385001,0.09164148,0,0.001010537,0,0.0697626,0.006305775,0.03437896,0.05677849]},{"f":[21,-1,2,1,-1,10,-1,15,37,16,33,-1,-1,16,-1,32,-1,-1,-1,-1,22,9,2,1,-1,-1,-1,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,0.5,0,10.457380879072534,10.26,6.832890014164741,21.257266042509173,0,0,7.930737337562887,0,8.230739723643078,0,0,0,0,17.094543568094064,0.5,8.9274737487839,4.92309205350118,0,0,0,0,0],"m":[0,0,1,1,0,0,0,1,1,0,1,0,0,1,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,9,10,11,0,0,14,0,16,0,0,0,0,21,22,23,24,0,0,0,0,0],"r":[2,0,20,5,0,7,0,19,18,13,12,0,0,15,0,17,0,0,0,0,28,27,26,25,0,0,0,0,0],"v":[0,-0.04476815,0,0,-0.0362323,0,-0.0458471,0,0,0,0,0.02693624,-0.01336106,0,-0.03647401,0,0.02646223,-0.01483124,0.05658854,-0.03682662,0,0,0,0,-0.004279183,0.01234558,0.01975654,-0.04380781,0.05499583]},{"f":[21,22,-1,-1,2,1,-1,10,-1,15,37,25,15,17,-1,-1,4,-1,22,-1,-1,-1,-1,-1,22,9,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,4.95344529780426,4.208333333333334,0,0.5,0,10.457380879072534,10.26,1.5,6.857980995127572,8.729619742794132,0,0,1.292481250360578,0,15.676331898425197,0,0,0,0,0,17.094543568094064,0.5,0,0,0],"m":[0,1,0,0,1,1,0,0,0,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,0,0,5,6,0,8,0,10,11,12,13,14,0,0,17,0,19,0,0,0,0,0,25,26,0,0,0],"r":[4,3,0,0,24,7,0,9,0,23,22,21,16,15,0,0,18,0,20,0,0,0,0,0,28,27,0,0,0],"v":[0,0,-0.04927137,0.02153649,0,0,-0.03570202,0,-0.04543541,0,0,0,0,0,0.03003618,-0.03446454,0,-0.02258577,0,0.03079846,-0.02124878,-0.02108944,0.0541399,-0.03631026,0,0,0.008151014,-0.04349749,0.05389784]},{"f":[21,-1,23,0,21,-1,-1,22,21,-1,-1,22,15,-1,-1,22,-1,15,-1,22,2,-1,22,-1,-1,18,-1,-1,-1],"t":[2.5,0,7.5,2.5,7.5,0,0,12.199665042528586,8.5,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,0,5.044394119358453,0,17.094543568094064,3.084962500721156,0,14.158358850608169,0,0,8.729619742794132,0,0,0],"m":[0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,0,16,0,18,0,20,21,0,23,0,0,26,0,0,0],"r":[2,0,28,7,6,0,0,11,10,0,0,15,14,0,0,17,0,19,0,25,22,0,24,0,0,27,0,0,0],"v":[0,-0.04419066,0,0,0,-0.03259207,-0.006138191,0,0,-0.006138191,0.04357473,0,0,0.0212662,0.125602,0,-0.006138191,0,0.0339114,0,0,-0.006138191,0,0.03899336,0.006318712,0,0.06261998,-0.006138191,-0.04853153]},{"f":[21,-1,2,1,-1,10,-1,17,22,-1,-1,-1,22,2,1,-1,22,-1,-1,-1,22,-1,22,-1,22,-1,22,-1,-1],"t":[2.5,0,4.95344529780426,4.208333333333334,0,0.5,0,8.729619742794132,17.07907323966382,0,0,0,16.765402642760364,10.510264838663748,4.008411214953271,0,16.73920708776685,0,0,0,16.786010507999382,0,16.838461650489297,0,16.975010474608837,0,17.094543568094064,0,0],"m":[0,0,1,1,0,0,0,1,1,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,0,8,9,0,0,0,13,14,15,0,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,12,5,0,7,0,11,10,0,0,0,20,19,16,0,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,-0.04385387,0,0,-0.0349297,0,-0.04484473,0,0,0.00600041,0.06816357,-0.04190301,0,0,0,-0.007255128,0,0.01373254,-0.03810842,0.02839899,0,0.09229408,0,-0.02894215,0,0.07035562,0,-0.003676729,0.05189646]},{"f":[21,-1,23,21,17,18,-1,-1,15,16,-1,-1,-1,37,1,2,16,-1,-1,22,2,4,-1,-1,-1,-1,-1,-1,-1],"t":[2.5,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,6.247927513443585,0,0,0,11.73,8.992857142857144,4.0437314206251695,6.569855608330948,0,0,16.765402642760364,10.510264838663748,5.714180086352146,0,0,0,0,0,0,0],"m":[0,0,1,0,1,1,0,0,1,1,0,0,0,1,1,1,0,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,0,14,15,16,17,0,0,20,21,22,0,0,0,0,0,0,0],"r":[2,0,28,13,8,7,0,0,12,11,0,0,0,27,26,19,18,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,-0.0435077,0,0,0,0,-0.04187091,-0.007272224,0,0,0.01563269,0.07788661,-0.03591708,0,0,0,0,-0.03691492,-0.001781169,0,0,0,0.008281345,-0.0131824,0.02680986,0.02282309,0.02093767,0.05843752,-0.04805242]},{"f":[21,22,-1,-1,23,21,17,18,-1,-1,15,16,-1,3,-1,-1,-1,37,2,1,21,-1,-1,22,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,6.247927513443585,0,0.5803571428571428,0,0,0,11.73,7.640240405159187,4.92309205350118,7.5,0,0,12.199665042528586,0,0,0,0,0],"m":[0,1,0,0,1,0,1,1,0,0,1,1,0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,0,11,12,0,14,0,0,0,18,19,20,21,0,0,24,0,0,0,0,0],"r":[4,3,0,0,28,17,10,9,0,0,16,13,0,15,0,0,0,27,26,23,22,0,0,25,0,0,0,0,0],"v":[0,0,-0.04897354,0.02675818,0,0,0,0,-0.04141658,-0.006956498,0,0,0.01479017,0,0.01785389,0.1227712,-0.03521335,0,0,0,0,-0.03332217,0.001615612,0,-0.00205188,0.01489583,0.01739247,0.05616515,-0.04767729]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,-1,22,-1,-1,37,15,22,-1,-1,1,2,-1,-1,17,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,0,12.613886328427084,0,0,11.73,5.044394119358453,12.583618810773686,0,0,4.92309205350118,7.005613627711627,0,0,7.430398005367313,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,13,0,0,16,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[4,3,0,0,28,15,8,0,10,0,12,0,14,0,0,27,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,-0.04888828,0.02546888,0,0,0,-0.04170173,0,-0.03974655,0,0.04629398,0,-0.04470534,0.009147239,0,0,0,0.001289558,0.02951269,0,0,-0.01946889,0.01071366,0,-0.002258804,0.0173456,0.05401979,-0.04729785]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,36,-1,-1,22,-1,4,-1,-1,37,1,2,16,-1,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,2.5,0,0,12.613886328427084,0,3.084962500721156,0,0,11.73,8.992857142857144,4.0437314206251695,7.930737337562887,0,0,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,12,0,0,15,0,17,0,0,20,21,22,23,0,0,0,0,0,0],"r":[4,3,0,0,28,19,8,0,10,0,14,13,0,0,16,0,18,0,0,27,26,25,24,0,0,0,0,0,0],"v":[0,0,-0.04880087,0.02425331,0,0,0,-0.04126047,0,-0.03924992,0,0,-0.01910794,0.06144132,0,-0.04415405,0,-0.007312026,0.0228358,0,0,0,0,-0.02553072,0.003437068,0.006317643,0.01894109,0.05198711,-0.04691395]},{"f":[21,-1,23,0,21,-1,-1,22,21,-1,-1,22,15,-1,-1,22,16,-1,-1,15,16,-1,4,-1,-1,0,-1,-1,-1],"t":[2.5,0,7.5,2.5,7.5,0,0,12.199665042528586,8.5,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,10.462502272597265,0,0,7.266786540694901,6.247927513443585,0,3.245926548164837,0,0,10.5,0,0,0],"m":[0,0,1,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,0,16,17,0,0,20,21,0,23,0,0,26,0,0,0],"r":[2,0,28,7,6,0,0,11,10,0,0,15,14,0,0,19,18,0,0,25,22,0,24,0,0,27,0,0,0],"v":[0,-0.04224213,0,0,0,-0.03050701,-0.005608156,0,0,-0.005608156,0.03809337,0,0,0.01822794,0.1093571,0,0,-0.005608156,0.07042198,0,0,0.007620166,0,0.01794224,0.06913722,0,-0.0002947462,0.05159234,-0.04652544]},{"f":[21,-1,23,21,17,18,-1,-1,15,-1,-1,1,16,2,22,-1,17,-1,-1,-1,16,-1,22,-1,-1,2,-1,-1,-1],"t":[2.5,0,7.5,4.5,7.198837316474133,1.792481250360578,0,0,6.857980995127572,0,0,8.992857142857144,7.930737337562887,4.80643424864552,17.094543568094064,0,8.729619742794132,0,0,0,7.948367231584678,0,16.759160379664443,0,0,5.2849278041654735,0,0,0],"m":[0,0,1,0,1,1,0,0,1,0,0,1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,0,9,0,0,12,13,14,15,0,17,0,0,0,21,0,23,0,0,26,0,0,0],"r":[2,0,28,11,8,7,0,0,10,0,0,25,20,19,16,0,18,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,-0.04184212,0,0,0,0,-0.0400542,-0.007071741,0,0.02153407,-0.03367574,0,0,0,0,-0.04198294,0,0.08426537,-0.04637398,0.002076236,0,0.09779169,0,0.002774478,0.02704839,0,0.01441048,0.06510857,-0.0461322]},{"f":[21,22,-1,-1,23,21,22,-1,17,-1,22,-1,22,-1,-1,37,15,22,-1,-1,22,22,2,-1,-1,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,0,12.613886328427084,0,0,11.73,5.044394119358453,12.583618810773686,0,0,17.078739781790915,17.02942582914998,1.292481250360578,0,0,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,11,0,13,0,0,16,17,18,0,0,21,22,23,0,0,0,0,0,0],"r":[4,3,0,0,28,15,8,0,10,0,12,0,14,0,0,27,20,19,0,0,26,25,24,0,0,0,0,0,0],"v":[0,0,-0.04855028,0.02712687,0,0,0,-0.0403746,0,-0.03806783,0,0.03874685,0,-0.04367935,0.00748707,0,0,0,0.001227466,0.02607162,0,0,0,-0.02032093,0.003960392,-0.047245,0.02348072,0.04782709,-0.04573414]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,22,-1,17,-1,22,36,-1,-1,22,-1,2,-1,17,-1,-1,37,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,4.5,12.199665042528586,0,1.292481250360578,0,12.278990886161445,2.5,0,0,12.613886328427084,0,3.084962500721156,0,4.208926257442949,0,0,11.73,0,0,0],"m":[0,1,0,0,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,0,13,0,15,16,0,0,19,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,12,0,14,0,18,17,0,0,20,0,22,0,24,0,0,27,0,0,0],"v":[0,0,-0.04845345,0.02581625,0,0,0,-0.0475481,0.02935341,0,0,-0.03883403,0,-0.03746568,0,0,-0.01974357,0.05223118,0,-0.04303301,0,-0.006108139,0,-0.02547879,0.02754156,0,0.004929074,0.04609694,-0.04533118]},{"f":[21,22,-1,-1,23,0,21,-1,-1,21,22,-1,22,15,-1,-1,22,15,-1,-1,18,32,-1,-1,-1,0,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,2.5,7.5,0,0,8.5,12.199665042528586,0,12.921158001713904,7.930737337562887,0,0,17.094543568094064,5.044394119358453,0,0,8.729619742794132,8.341935934311302,0,0,0,10.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,0,13,14,0,0,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,12,0,16,15,0,0,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,-0.0483537,0.02458094,0,0,0,-0.02917053,-0.005283171,0,0,-0.005283171,0,0,0.01639215,0.09511894,0,0,0.01999178,-0.001952071,0,0,0.0005619419,0.07348316,-0.005283171,0,0.008752542,0.06373888,-0.04492329]},{"f":[21,22,-1,-1,2,16,-1,1,21,-1,1,-1,-1,17,22,35,22,18,-1,33,-1,-1,-1,-1,-1,-1,22,-1,-1],"t":[2.5,13.87915215758784,0,0,4.95344529780426,2.0,0,4.208333333333334,6.5,0,1.13125,0,0,8.729619742794132,17.07907323966382,2.5,16.25454052700782,2.160964047443681,0,21.011466140970732,0,0,0,0,0,0,17.094543568094064,0,0],"m":[0,1,0,0,1,0,0,1,1,0,0,0,0,1,1,1,1,1,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,0,8,9,0,11,0,0,14,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[4,3,0,0,26,7,0,13,10,0,12,0,0,25,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,-0.04825086,0.02341546,0,0,-0.04576753,0,0,-0.04319346,0,0.1066285,-0.02206816,0,0,0,0,0,0.01974611,0,-0.02803953,0.006045121,-0.04009057,-0.04314444,0.05530916,-0.04066525,0,0.004515912,0.04894231]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,-1,15,18,22,4,-1,22,-1,33,-1,-1,-1,-1,-1,-1],"t":[2.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,0,12.199665042528586,4.5,0,0,12.738408632106918,0,7.266786540694901,8.729619742794132,16.765402642760364,1.292481250360578,0,14.03140051979143,0,21.257266042509173,0,0,0,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,0,14,0,16,17,18,19,0,21,0,23,0,0,0,0,0,0],"r":[2,0,28,7,6,0,0,9,0,13,12,0,0,15,0,27,26,25,20,0,22,0,24,0,0,0,0,0,0],"v":[0,-0.04002015,0,0,0,-0.04276778,-0.008689074,0,-0.008471613,0,0,-0.008689074,0.02910244,0,0.07308604,0,0,0,0,-0.0008009462,0,-0.0006805364,0,0.0770557,0.007129929,0.08080158,-0.008689074,0.001663858,0.0141975]},{"f":[21,24,-1,-1,23,1,21,-1,-1,21,22,22,22,21,5,-1,-1,-1,1,1,-1,-1,-1,-1,-1,0,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,0.9444444444444444,7.5,0,0,8.5,14.116600814324018,12.921158001713904,12.199665042528586,5.5,0.5,0,0,0,7.772727272727273,2.969223484848485,0,0,0,0,0,10.5,0,0,0],"m":[0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,13,14,15,0,0,0,19,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,24,23,18,17,16,0,0,0,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,0.02483755,-0.04828474,0,0,0,-0.04716233,0.02722272,0,0,0,0,0,0,0.01679259,-0.04112182,0.001563527,0,0,0.01306856,0.06467286,-0.0007962808,-0.03095008,0.006670292,0,0.007506407,0.0615206,-0.04432304]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,2,33,-1,22,17,16,18,-1,-1,21,-1,-1,-1,-1,-1,0,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,8.5,3.084962500721156,21.16864017939033,0,17.094543568094064,2.160964047443681,7.930737337562887,1.792481250360578,0,0,5.5,0,0,0,0,0,10.5,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,1,0,0,1,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,0,14,15,16,17,0,0,20,0,0,0,0,0,26,0,0,0],"r":[4,3,0,0,28,9,8,0,0,25,24,13,0,23,22,19,18,0,0,21,0,0,0,0,0,27,0,0,0],"v":[0,0,-0.04795446,0.02502213,0,0,0,-0.04694082,0.02577395,0,0,0,0.009318887,0,0,0,0,-0.03690457,0.06536049,0,-0.0356707,0.03524162,-0.03065157,0.02379027,0.00454146,0,0.007110988,0.06012012,-0.04389734]},{"f":[21,24,-1,-1,23,2,37,21,-1,-1,22,16,-1,22,-1,22,-1,-1,18,32,-1,-1,-1,22,18,-1,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,8.9274737487839,4.470000000000001,6.5,0,0,17.094543568094064,6.044394119358453,0,12.199665042528586,0,15.676331898425197,0,0,8.729619742794132,8.341935934311302,0,0,0,16.759160379664443,10.460454442916898,0,0,0,0],"m":[0,0,0,0,1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,2,0,0,5,6,7,8,0,0,11,12,0,14,0,16,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,10,9,0,0,18,13,0,15,0,17,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0.02452211,-0.0480799,0,0,0,0,-0.04201388,-0.007746937,0,0,-0.00718756,0,-0.003774955,0,0.02277743,-0.002076101,0,0,-0.003835715,0.06897731,-0.007746937,0,0,0.001869323,0.02938925,0.05522759,-0.0434666]},{"f":[21,22,-1,-1,23,1,21,-1,-1,21,2,1,-1,15,19,-1,-1,-1,22,-1,22,-1,22,-1,33,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,7.5,0.9444444444444444,7.5,0,0,10.5,4.882435795368045,4.208333333333334,0,10.457380879072534,2.5,0,0,0,16.765402642760364,0,16.786010507999382,0,16.844368517741394,0,28.295013863458657,0,0,0,0],"m":[0,1,0,0,1,0,1,0,0,1,1,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,0,14,15,0,0,0,19,0,21,0,23,0,25,0,0,0,0],"r":[4,3,0,0,28,9,8,0,0,27,18,13,0,17,16,0,0,0,20,0,22,0,24,0,26,0,0,0,0],"v":[0,0,-0.04773352,0.02462824,0,0,0,-0.04663599,0.0244132,0,0,0,-0.0287097,0,0,0.005634083,-0.04359395,-0.04076002,0,0.002659729,0,0.07997755,0,-0.02762028,0,0.07424978,0.006695558,0.08385683,-0.04303091]},{"f":[21,24,-1,-1,23,21,22,-1,17,-1,-1,1,16,2,17,17,-1,-1,-1,-1,22,-1,24,-1,-1,2,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,4.5,12.199665042528586,0,1.292481250360578,0,0,8.992857142857144,7.930737337562887,4.80643424864552,8.496347487943302,0.5,0,0,0,0,16.844368517741394,0,7.5,0,0,5.2849278041654735,0,0,0],"m":[0,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,1,0,0,1,0,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,12,13,14,15,16,0,0,0,0,21,0,23,0,0,26,0,0,0],"r":[4,3,0,0,28,11,8,0,10,0,0,25,20,19,18,17,0,0,0,0,22,0,24,0,0,27,0,0,0],"v":[0,0,0.02419855,-0.04786217,0,0,0,-0.03808526,0,-0.03572993,0.005870632,0,0,0,0,0,0.03862579,-0.04792835,0.01637885,0.0003100423,0,0.004470237,0,0.007539149,0.08173934,0,0.01125712,0.05923616,-0.04259034]},{"f":[21,22,-1,-1,16,27,-1,22,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,2,-1,-1,-1,18,-1,-1,-1,-1],"t":[2.5,13.87915215758784,0,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.903677461028802,0,0,0,8.729619742794132,0,0,0,0],"m":[0,1,0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,0,8,0,0,11,12,13,14,15,16,17,0,0,0,21,0,0,0,25,0,0,0,0],"r":[4,3,0,0,10,7,0,9,0,0,28,27,24,23,20,19,18,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,-0.04749917,0.02423347,0,0,-0.04686202,0,0.05621368,-0.005436559,0,0,0,0,0,0,0,0.0004997464,0.08364889,-0.03217078,0,-0.03242364,0.07578061,-0.0460498,0,0.04698117,-0.04408739,0.03197118,-0.03342913]},{"f":[21,24,-1,-1,23,1,21,-1,-1,21,2,1,21,-1,17,15,-1,-1,-1,15,19,24,-1,-1,-1,-1,-1,-1,-1],"t":[2.5,6.5,0,0,7.5,0.9444444444444444,7.5,0,0,10.5,4.882435795368045,4.208333333333334,6.5,0,0.5,1.0,0,0,0,10.457380879072534,2.5,7.5,0,0,0,0,0,0,0],"m":[0,0,0,0,1,0,1,0,0,1,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,0,5,6,7,0,0,10,11,12,13,0,15,16,0,0,0,20,21,22,0,0,0,0,0,0,0],"r":[4,3,0,0,28,9,8,0,0,27,26,19,14,0,18,17,0,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0.02386832,-0.04763062,0,0,0,-0.04636445,0.02292783,0,0,0,0,-0.0424868,0,0,-0.02165982,0.1055114,-0.02353047,0,0,0,-0.008981635,0.01098317,-0.04289208,-0.04013795,0.00496509,0.08177919,-0.04192196]},{"f":[21,22,-1,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,1,-1,-1,15,18,22,-1,-1,-1,-1,22,-1,-1],"t":[2.5,13.87915215758784,0,0,8.9274737487839,4.470000000000001,6.5,0,0,6.044394119358453,0,12.199665042528586,4.5,0,0,12.738408632106918,6.806765389082463,0,0,7.266786540694901,8.729619742794132,16.765402642760364,0,0,0,0,16.765402642760364,0,0],"m":[0,1,0,0,1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,7,0,0,10,0,12,13,0,0,16,17,0,0,20,21,22,0,0,0,0,27,0,0],"r":[4,3,0,0,26,9,8,0,0,11,0,15,14,0,0,19,18,0,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.04725055,0.02383873,0,0,0,-0.04093104,-0.007546297,0,-0.007546297,0,0,-0.007546297,0.02396932,0,0,0.01910986,0.09846133,0,0,0,0.01396939,0.0720513,-0.007546297,7.156398e-05,0,0.00795188,0.05227254]},{"f":[21,-1,23,21,22,21,-1,16,-1,-1,33,20,-1,-1,33,-1,-1,1,16,2,17,-1,-1,-1,-1,2,-1,-1,-1],"t":[1.5,0,7.5,4.5,14.03140051979143,3.5,0,5.78135971352466,0,0,12.738373101969731,0.5,0,0,19.167172160526107,0,0,8.992857142857144,7.930737337562887,4.80643424864552,8.496347487943302,0,0,0,0,5.2849278041654735,0,0,0],"m":[0,0,1,0,1,1,0,0,0,0,1,1,0,0,0,0,0,1,1,1,1,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,0,8,0,0,11,12,0,0,15,0,0,18,19,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,17,10,7,0,9,0,0,14,13,0,0,16,0,0,25,24,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.046071,0,0,0,0,-0.0435097,0,0.006826579,-0.02956261,0,0,-0.005971999,0.1224262,0,-0.04402448,0.009464642,0,0,0,0,-0.04075631,0.01413626,5.431809e-07,0.005686057,0,0.01053436,0.05721801,-0.04159016]},{"f":[21,-1,23,1,21,-1,-1,21,22,21,-1,23,-1,-1,15,-1,15,17,-1,-1,15,4,-1,-1,0,-1,-1,-1,-1],"t":[1.5,0,7.5,0.9444444444444444,7.5,0,0,10.5,14.116600814324018,3.5,0,3.5,0,0,5.044394119358453,0,6.108524456778169,7.67595272073847,0,0,7.266786540694901,3.084962500721156,0,0,10.5,0,0,0,0],"m":[0,0,1,0,1,0,0,1,1,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,0,12,0,0,15,0,17,18,0,0,21,22,0,0,25,0,0,0,0],"r":[2,0,28,7,6,0,0,27,14,11,0,13,0,0,16,0,20,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04588038,0,0,0,-0.04607412,0.02125406,0,0,0,-0.04249296,0,0.003742941,-0.02438879,0,0.02290859,0,0,-0.04536661,0.02687842,0,0,0.003454589,0.04807964,0,-0.002520004,0.0412374,0.07973202,-0.041128]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,36,-1,-1,22,1,-1,-1,22,22,-1,1,-1,-1,-1,22,17,-1,-1,-1],"t":[1.5,0,8.9274737487839,4.470000000000001,6.5,0,0,6.108524456778169,0,12.199665042528586,4.5,0,0,12.738408632106918,6.806765389082463,0,0,15.676331898425197,14.116600814324018,0,6.431742323097463,0,0,0,16.765402642760364,12.18494601563336,0,0,0],"m":[0,0,1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,0,14,15,0,0,18,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,7,6,0,0,9,0,13,12,0,0,17,16,0,0,23,20,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.04568415,0,0,0,-0.04086517,-0.007595534,0,-0.006430627,0,0,-0.007595534,0.0220775,0,0,0.01827047,0.09160768,0,0,-0.007595534,0,0.08459528,0.01453342,-0.0005234241,0,0,0.003334381,0.03665177,0.04993822]},{"f":[21,-1,23,21,22,22,4,-1,22,22,22,21,2,-1,-1,-1,-1,-1,16,-1,-1,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,15.439474430023438,14.158358850608169,1.292481250360578,0,14.116600814324018,13.054613167076562,12.613886328427084,5.5,7.038658222940513,0,0,0,0,0,6.247927513443585,0,0,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,12,13,0,0,0,0,0,19,0,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,21,8,0,18,17,16,15,14,0,0,0,0,0,20,0,0,0,0,27,26,0,0,0,0],"v":[0,-0.0454822,0,0,0,0,0,-0.02043748,0,0,0,0,0,-0.02337914,0.01722297,0.006105582,0.02885044,-0.02568061,0,0.002582041,0.0639365,-0.04359709,0.005296599,0,0,-0.0006533681,0.07682648,0.05743497,-0.04079216]},{"f":[21,-1,23,21,0,22,-1,-1,4,22,-1,15,25,23,-1,4,-1,-1,-1,-1,23,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,2.5,12.278990886161445,0,0,9.427971070948235,14.116600814324018,0,7.266786540694901,0.5,0.5,0,4.951187557243013,0,0,0,0,0.5,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,12,13,14,0,16,0,0,0,0,21,0,0,24,25,0,0,0,0],"r":[2,0,28,23,8,7,0,0,20,11,0,19,18,15,0,17,0,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,-0.04527443,0,0,0,0,-0.007049605,-0.03442445,0,0,-0.007049605,0,0,0,0.1660129,0,0.01137514,0.04552523,-0.001928244,-0.0007938151,0,0.04430979,-0.002838154,0,0,-0.0006220144,0.0743123,0.05622959,-0.04031933]},{"f":[21,-1,23,21,15,22,21,-1,23,4,-1,-1,-1,-1,22,18,3,-1,-1,22,-1,-1,-1,36,15,-1,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,14.116600814324018,3.5,0,3.5,4.0437314206251695,0,0,0,0,16.759160379664443,10.680359255641626,0.23857586512866016,0,0,11.722778233171695,0,0,0,3.5,12.000704269011246,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,0,0,0,0,15,16,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,14,13,8,0,12,11,0,0,0,0,22,19,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.04506075,0,0,0,0,0,-0.04211861,0,0,-0.004151898,0.01314008,-0.02995186,0.006288763,0,0,0,0.007543494,-0.03079766,0,0.04910407,-0.03419864,0.05698289,0,0,-0.0005921548,0.07185534,0.05507834,-0.03984266]},{"f":[21,-1,23,1,21,-1,-1,21,22,22,22,16,-1,22,22,22,-1,-1,-1,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,0.9444444444444444,7.5,0,0,10.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,1,0,1,0,0,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,10,11,12,0,14,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,28,7,6,0,0,27,22,21,20,13,0,19,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04484107,0,0,0,-0.04542495,0.01968913,0,0,0,0,0,-0.05351182,0,0,0,0.002015116,-0.02453143,0.07684184,-0.03141265,0.03571493,-0.02150517,0,0,-0.008228402,0.05958407,-0.04216529,0.07462045,-0.03936237]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,22,22,22,22,-1,-1,-1,-1,22,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,17.074218326240924,16.98413870862295,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,16.903290731167747,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,13,14,15,16,0,0,0,0,21,0,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,24,23,20,19,18,17,0,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,-0.04461533,0,0,-0.0450234,0,0.05262886,-0.005716507,0,0,0,0,0,0,0,0,0.00119906,-0.02361084,0.07351184,-0.03079958,0,0.09085205,-0.001532166,-0.04142191,0,0.04371251,-0.05037161,0.02569157,-0.0331999]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,-1,-1,22,-1,22,16,-1,-1,15,-1,15,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,6.108524456778169,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,0,0,17,0,19,20,0,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,16,15,0,0,18,0,22,21,0,0,24,0,26,0,0,0,0],"v":[0,-0.04438347,0,0,-0.04515581,0.01865035,0,0,0,0,-0.02031184,0.05099774,0,0,-0.02694096,0.0002096226,0,0.02066217,0,0,-0.04717985,-0.009431105,0,0.02047234,0,-0.03165113,0.003890518,0.07214702,-0.03855637]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,22,22,22,22,-1,-1,-1,-1,22,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,17.074218326240924,16.98413870862295,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,16.903290731167747,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,13,14,15,16,0,0,0,0,21,0,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,24,23,20,19,18,17,0,0,0,0,22,0,0,0,26,0,0,0,0],"v":[0,-0.04414543,0,0,-0.04437951,0,0.0502634,-0.005598849,0,0,0,0,0,0,0,0,0.001080079,-0.02296416,0.07040045,-0.03005782,0,0.08488696,-0.001347097,-0.04086561,0,0.04141813,-0.04961409,0.02494974,-0.03183053]},{"f":[21,-1,2,37,21,-1,-1,22,21,-1,-1,22,18,-1,1,21,-1,-1,-1,-1,22,15,1,-1,21,-1,-1,-1,-1],"t":[1.5,0,7.128362343532883,4.21,7.5,0,0,14.116600814324018,8.5,0,0,16.73920708776685,2.160964047443681,0,6.431742323097463,4.5,0,0,0,0,16.759160379664443,11.683433292835403,2.01685992705101,0,6.5,0,0,0,0],"m":[0,0,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,9,0,0,12,13,0,15,16,0,0,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,20,7,6,0,0,11,10,0,0,19,14,0,18,17,0,0,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.0439012,0,0,0,-0.03199786,-0.007626234,0,0,-0.00609195,0.02598802,0,0,0.0238801,0,0,0.1116578,0.01334135,-0.00652092,-0.004581889,0,0,0,-0.002878923,0,0.0422152,0.0102295,-0.03061198,0.04311101]},{"f":[21,-1,23,21,15,16,22,21,-1,-1,-1,17,15,-1,-1,-1,22,18,3,-1,-1,22,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,9.92481250360578,14.116600814324018,3.5,0,0,0,10.415240582199587,6.169925001442312,0,0,0,16.759160379664443,10.680359255641626,0.23857586512866016,0,0,11.722778233171695,0,0,0,10.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,0,12,13,0,0,0,17,18,19,0,0,22,0,0,0,26,0,0,0],"r":[2,0,28,25,16,11,10,9,0,0,0,15,14,0,0,0,24,21,20,0,0,23,0,0,0,27,0,0,0],"v":[0,-0.04365075,0,0,0,0,0,0,-0.0411592,-0.004603287,0.004460966,0,0,0.007009178,0.06733153,0.001853978,0,0,0,0.008107928,-0.02943402,0,0.0460579,-0.03354471,0.0487388,0,0.004866312,0.05174681,-0.03787428]},{"f":[21,-1,16,27,-1,22,-1,-1,33,33,22,22,-1,-1,22,21,-1,-1,22,-1,22,16,-1,-1,15,-1,-1,-1,-1],"t":[1.5,0,1.0,0.5,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,7.95646805166752,6.870153907126159,0,0,12.199665042528586,6.5,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,0,0,0],"m":[0,0,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,12,0,0,15,16,0,0,19,0,21,22,0,0,25,0,0,0,0],"r":[2,0,8,5,0,7,0,0,28,27,14,13,0,0,18,17,0,0,20,0,24,23,0,0,26,0,0,0,0],"v":[0,-0.04339407,0,0,-0.04378851,0,0.04819539,-0.005406306,0,0,0,0,-0.02389523,0.04210003,0,0,-0.03065252,0.0002217809,0,0.01886576,0,0,-0.04687731,-0.00558849,0,0.01937173,0.0006606143,0.02320266,-0.03121608]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,3,-1,-1,-1,22,1,-1,-1,22,16,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0.15991874474642756,0,0,0,12.921158001713904,7.772727272727273,0,0,14.116600814324018,6.832890014164741,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,18,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.04313119,0,0,-0.04459568,0.01791357,0,0,0,0,-0.01911264,0.04536785,0,0,0,0.0001918975,-0.03945259,0.0003018653,0,0,0.03004077,-0.002107936,0,0,-0.04646361,-0.008638938,0.003875568,0.06930752,-0.03704108]},{"f":[21,-1,4,16,-1,16,2,-1,-1,16,-1,21,-1,-1,22,15,-1,-1,22,-1,22,-1,22,-1,22,3,-1,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,1.792481250360578,0,0,6.108524456778169,0,8.5,0,0,16.759160379664443,11.683433292835403,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,17.078739781790915,0.9876439904368615,0,0,0],"m":[0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,6,7,0,0,10,0,12,0,0,15,16,0,0,19,0,21,0,23,0,25,26,0,0,0],"r":[2,0,14,5,0,9,8,0,0,11,0,13,0,0,18,17,0,0,20,0,22,0,24,0,28,27,0,0,0],"v":[0,-0.04286212,0,0,-0.0348447,0,0,0.02531216,-0.02378263,0,-0.0451174,0,-0.003173148,0.0281381,0,0,0.003821338,-0.02271719,0,0.06354274,0,-0.03527543,0,0.06521234,0,0,0.01636762,-0.03980294,0.02923775]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,-1,-1,18,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,0,13,14,15,16,17,18,19,0,0,0,0,0,25,0,0,0,0],"r":[2,0,6,5,0,0,12,9,0,11,0,0,28,27,24,23,22,21,20,0,0,0,0,0,26,0,0,0,0],"v":[0,-0.04258691,0,0,-0.04431565,0.01697249,0,0,-0.05040285,0,0.04385041,-0.004967678,0,0,0,0,0,0,0,0.0002371222,0.06538128,-0.02883743,0.03477109,-0.04185014,0,0.03867437,-0.04186403,0.02395807,-0.02946598]},{"f":[21,-1,1,21,-1,-1,23,21,22,22,22,16,-1,22,22,22,-1,-1,-1,-1,-1,-1,18,32,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,1,1,1,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,12,0,14,15,16,0,0,0,0,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,22,21,20,13,0,19,18,17,0,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.04230561,0,0,-0.04403862,0.01614924,0,0,0,0,0,0,-0.05099021,0,0,0,0.001699628,-0.02265345,0.06315547,-0.02799189,0.02845052,-0.02020206,0,0,-0.01347737,0.05533891,-0.03881414,0.06573984,-0.03602711]},{"f":[21,-1,4,16,-1,16,-1,16,-1,21,35,16,-1,-1,-1,-1,22,15,21,-1,-1,-1,22,-1,22,-1,22,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,0,6.108524456778169,0,8.5,2.5,6.832890014164741,0,0,0,0,16.759160379664443,11.683433292835403,3.5,0,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,0],"m":[0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,10,11,12,0,0,0,0,17,18,19,0,0,0,23,0,25,0,27,0,0],"r":[2,0,16,5,0,7,0,9,0,15,14,13,0,0,0,0,22,21,20,0,0,0,24,0,26,0,28,0,0],"v":[0,-0.04201831,0,0,-0.03356758,0,0.01781394,0,-0.04488894,0,0,0,0.0133181,-0.005131029,-0.04254985,0.02616599,0,0,0,-0.02346069,0.005221255,-0.02194071,0,0.05834848,0,-0.03289196,0,0.0591334,0.006902691]},{"f":[21,-1,2,16,-1,17,2,-1,-1,32,1,17,-1,-1,17,17,22,22,-1,2,-1,-1,-1,-1,-1,-1,22,-1,-1],"t":[1.5,0,5.022197059679227,2.0,0,2.160964047443681,1.792481250360578,0,0,8.230739723643078,4.208333333333334,10.620215166896223,0,0,8.729619742794132,7.467398296089201,15.676331898425197,12.613886328427084,0,2.453445297804259,0,0,0,0,0,0,17.094543568094064,0,0],"m":[0,0,1,0,0,0,1,0,0,1,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,6,7,0,0,10,11,12,0,0,15,16,17,18,0,20,0,0,0,0,0,0,27,0,0],"r":[2,0,26,5,0,9,8,0,0,25,14,13,0,0,24,23,22,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,-0.04172508,0,0,-0.04152036,0,0,0.01495722,-0.01907947,0,0,0,-0.03524169,0.01970894,0,0,0,0,-0.0286238,0,-0.007675352,0.02086252,-0.0270904,0.05987334,-0.03318593,-0.0273981,0,0.001951025,0.04283957]},{"f":[21,-1,22,22,-1,-1,0,2,-1,-1,15,22,-1,23,-1,-1,22,18,-1,-1,22,1,21,-1,17,-1,-1,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,0,2.5,6.209321624687949,0,0,3.321928094887362,12.278990886161445,0,1.5,0,0,14.116600814324018,10.680359255641626,0,0,14.247293309538719,6.431742323097463,3.5,0,7.67595272073847,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,0,11,12,0,14,0,0,17,18,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,10,9,0,0,16,13,0,15,0,0,20,19,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.04142605,0,0,-0.02680442,0.03720654,0,0,-0.03620511,-0.005810674,0,0,0.002106607,0,0.05187278,0.01047778,0,0,-0.005810674,0.03292529,0,0,0,0.1443347,0,0.003374921,0.05566415,0.001708498,-9.389661e-05]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,22,-1,-1,33,33,22,22,-1,-1,18,36,18,-1,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,35.98606694718995,29.297756302860655,17.078739781790915,16.98413870862295,0,0,8.729619742794132,4.5,6.076776015854056,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,12,0,0,15,16,17,18,0,0,21,22,23,0,0,0,0,0,0],"r":[2,0,6,5,0,0,14,9,0,11,0,13,0,0,28,27,20,19,0,0,26,25,24,0,0,0,0,0,0],"v":[0,-0.04112132,0,0,-0.04362534,0.01553306,0,0,-0.04957439,0,0.03938465,0,-0.01187725,0.03417886,0,0,0,0,0.0009343161,-0.03924989,0,0,0,0.005950492,0.06892507,-0.01285721,-0.04734982,0.02094184,-0.02831571]},{"f":[21,-1,2,37,21,-1,-1,16,-1,22,22,-1,2,-1,-1,-1,22,24,-1,1,21,2,-1,-1,-1,21,-1,-1,-1],"t":[1.5,0,7.128362343532883,4.21,6.5,0,0,6.108524456778169,0,16.73920708776685,12.199665042528586,0,1.292481250360578,0,0,0,16.759160379664443,3.5,0,2.997611464968153,7.5,9.221514256038809,0,0,0,6.5,0,0,0],"m":[0,0,1,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,1,1,1,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,8,0,10,11,0,13,0,0,0,17,18,0,20,21,22,0,0,0,26,0,0,0],"r":[2,0,16,7,6,0,0,9,0,15,12,0,14,0,0,0,28,19,0,25,24,23,0,0,0,27,0,0,0],"v":[0,-0.04081105,0,0,0,-0.03721462,-0.006585851,0,-0.006059933,0,0,-0.003568792,0,-0.005748575,0.01747117,-0.006585851,0,0,-0.03273532,0,0,0,-0.0009418384,-0.03870391,0.01235918,0,0.0485052,0.006571797,0.03660347]},{"f":[21,-1,22,22,-1,15,-1,-1,22,21,8,-1,-1,-1,22,18,-1,-1,22,16,-1,-1,15,-1,15,-1,15,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,2.584962500721156,0,0,12.199665042528586,5.5,0.5,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,0,6.169925001442312,0,7.266786540694901,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,0,9,10,11,0,0,0,15,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[2,0,8,5,0,7,0,0,14,13,12,0,0,0,18,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,-0.04049537,0,0,-0.02597528,0,-0.004696531,0.04611787,0,0,0,-0.03694939,0.001057964,0.0008961766,0,0,0.006140275,0.03742557,0,0,-0.0460469,-0.005878225,0,0.01572396,0,-0.02693714,0,0.01747372,-0.002085951]},{"f":[21,-1,1,21,-1,2,16,-1,16,22,2,-1,-1,-1,21,25,-1,-1,15,22,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,15.782972563647261,1.792481250360578,0,0,0,7.5,1.5,0,0,7.930737337562887,17.07907323966382,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,1,0,0,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,0,0,0,15,16,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,14,13,12,0,0,0,18,17,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.04017447,0,0,-0.03922691,0,0,-0.04462307,0,0,0,0.0334882,-0.01707655,-0.009494072,0,0,-0.004410805,-0.02888808,0,0,-0.04237766,0.01583686,0.06561047,0.001422746,0,0,0.006760758,-0.03675581,0.0511419]},{"f":[21,-1,22,22,-1,-1,0,2,22,-1,-1,-1,18,22,-1,18,16,-1,22,-1,15,15,4,-1,-1,-1,-1,-1,-1],"t":[1.5,0,7.95646805166752,6.870153907126159,0,0,2.5,6.209321624687949,16.25454052700782,0,0,0,10.680359255641626,12.199665042528586,0,10.326824911906375,6.108524456778169,0,13.054613167076562,0,6.857980995127572,6.820178962415188,2.453445297804259,0,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,1,0,0,0,1,0,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,0,0,0,13,14,0,16,17,0,19,0,21,22,23,0,0,0,0,0,0],"r":[2,0,6,5,0,0,12,11,10,0,0,0,28,15,0,27,18,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,-0.03984853,0,0,-0.02514894,0.03237616,0,0,0,-0.04287186,-0.005324636,-0.005324636,0,0,-0.005324636,0,0,-0.005324636,0,0.03212414,0,0,0,1.81118e-05,0.02190174,0.1113206,-0.001228513,0.03197323,0.01902973]},{"f":[21,-1,1,21,-1,2,16,-1,16,-1,15,-1,21,16,33,-1,-1,-1,15,22,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,0,6.169925001442312,0,7.5,6.832890014164741,21.257266042509173,0,0,0,7.930737337562887,17.07907323966382,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,11,0,13,14,15,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,10,0,12,0,18,17,16,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03951774,0,0,-0.03881549,0,0,-0.04428515,0,0.01423833,0,-0.04066374,0,0,0,0.02291585,-0.01859955,-0.02507965,0,0,-0.03841224,0.0151285,0.05897641,0.00127933,0,0,0.006624721,-0.0363166,0.04985753]},{"f":[21,-1,23,21,15,16,16,-1,-1,17,-1,-1,22,22,16,-1,-1,1,-1,22,3,-1,-1,-1,-1,0,-1,-1,-1],"t":[1.5,0,7.5,8.5,10.457380879072534,7.930737337562887,7.679480099505446,0,0,1.792481250360578,0,0,16.759160379664443,11.722778233171695,10.501837184902296,0,0,4.87351462522852,0,15.782972563647261,0.9367573126142597,0,0,0,0,10.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,4,5,6,7,0,0,10,0,0,13,14,15,0,0,18,0,20,21,0,0,0,0,26,0,0,0],"r":[2,0,28,25,12,9,8,0,0,11,0,0,24,17,16,0,0,19,0,23,22,0,0,0,0,27,0,0,0],"v":[0,-0.03918232,0,0,0,0,0,-0.001366369,-0.03538726,0,0.02999083,0.002842506,0,0,0,-0.03953442,0.0304412,0,-0.03393047,0,0,0.001194571,-0.04739987,0.03898419,0.04005036,0,0.002693387,0.04864266,-0.03419142]},{"f":[21,-1,1,21,-1,-1,22,15,-1,-1,22,21,3,-1,-1,-1,22,18,-1,-1,22,16,16,-1,3,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,0.15991874474642756,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,11.98370619265935,6.832890014164741,0,0.9300813008130082,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,0,11,12,13,0,0,0,17,18,0,0,21,22,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,10,9,0,0,16,15,14,0,0,0,20,19,0,0,28,27,24,0,26,0,0,0,0],"v":[0,-0.03884249,0,0,-0.04292969,0.01353767,0,0,-0.01714962,0.03484716,0,0,0,-0.0003613278,-0.03737172,3.735105e-05,0,0,0.005750901,0.03482473,0,0,0,-0.04546503,0,0.009922641,-0.02616948,0.06059164,0.002679587]},{"f":[21,-1,23,22,22,22,16,-1,22,22,22,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,18,32,-1,-1,-1,-1],"t":[1.5,0,7.5,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,8.729619742794132,8.341935934311302,0,0,0,0],"m":[0,0,1,1,1,1,0,0,1,1,1,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,9,10,11,0,13,0,0,0,0,18,0,20,0,0,0,24,25,0,0,0,0],"r":[2,0,28,23,22,17,8,0,16,15,12,0,14,0,0,0,0,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,-0.03849849,0,0,0,0,0,-0.049562,0,0,0,0.001178517,0,-0.04966474,0.005874608,0.05880673,-0.02701998,0,0.07267422,0,-0.03866225,0.02365846,-0.01854536,0,0,-0.01484886,0.05234519,-0.03759737,-0.03370726]},{"f":[21,-1,1,21,-1,2,16,-1,16,22,15,-1,-1,-1,1,15,-1,16,-1,15,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,5.044394119358453,15.782972563647261,3.169925001442312,0,0,0,8.046875,6.169925001442312,0,6.832890014164741,0,7.930737337562887,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,11,0,0,0,15,16,0,18,0,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,14,13,12,0,0,0,22,17,0,19,0,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03815056,0,0,-0.03827172,0,0,-0.04384255,0,0,0,0.0005435638,0.03826474,-0.009580161,0,0,-0.039842,0,0.01300223,0,-0.04751183,0.002614804,-0.03753913,0.001173686,0,0,0.005904612,-0.0351082,0.04823457]},{"f":[21,-1,1,21,-1,-1,23,21,22,15,-1,-1,22,21,-1,23,-1,-1,15,21,-1,-1,15,17,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.5,10.5,7.95646805166752,3.584962500721156,0,0,14.116600814324018,3.5,0,3.5,0,0,5.044394119358453,4.5,0,0,6.108524456778169,7.67595272073847,0,0,0,0,0],"m":[0,0,0,1,0,0,1,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,14,0,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,28,27,12,11,0,0,18,15,0,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,-0.03779896,0,0,-0.04262449,0.01279318,0,0,0,0,-0.01602039,0.03201368,0,0,-0.03762463,0,0.001230399,-0.02089674,0,0,0.05675212,0.008726838,0,0,-0.0435526,0.0276931,0.002808998,0.05620512,-0.03288507]},{"f":[21,-1,4,16,-1,16,-1,16,-1,21,-1,-1,22,15,-1,-1,22,-1,22,-1,22,-1,22,3,-1,-1,18,-1,-1],"t":[1.5,0,3.753897320099348,2.0,0,5.044394119358453,0,6.108524456778169,0,9.5,0,0,16.759160379664443,11.683433292835403,0,0,16.807008825964132,0,16.844368517741394,0,16.975010474608837,0,17.078739781790915,0.9876439904368615,0,0,6.076776015854056,0,0],"m":[0,0,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,0,6,0,8,0,10,0,0,13,14,0,0,17,0,19,0,21,0,23,24,0,0,27,0,0],"r":[2,0,12,5,0,7,0,9,0,11,0,0,16,15,0,0,18,0,20,0,22,0,26,25,0,0,28,0,0],"v":[0,-0.03744396,0,0,-0.03045687,0,0.01377087,0,-0.04319898,0,-0.002704366,0.02789751,0,0,0.002808301,-0.02100878,0,0.04992055,0,-0.03180911,0,0.05401418,0,0,0.01384821,-0.03774676,0,-0.007790685,0.0528454]},{"f":[21,-1,1,21,-1,-1,16,27,-1,22,-1,22,-1,-1,33,33,22,22,22,-1,-1,-1,18,36,-1,-1,-1,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,35.98606694718995,29.297756302860655,17.078739781790915,16.98413870862295,16.844368517741394,0,0,0,8.729619742794132,4.5,0,0,0,0,0],"m":[0,0,0,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,1,0,0,0,1,1,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,0,10,0,12,0,0,15,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,6,5,0,0,14,9,0,11,0,13,0,0,28,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,-0.03708583,0,0,-0.04231704,0.0121534,0,0,-0.04856811,0,0.035951,0,-0.01112444,0.03447137,0,0,0,0,0,2.717095e-05,0.03182792,-0.03782661,0,0,0.04872084,-0.01389883,-0.04687394,0.01867221,-0.02695284]},{"f":[21,-1,1,21,-1,2,16,-1,17,21,-1,1,-1,22,22,17,-1,37,-1,-1,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,3.084962500721156,2.0,0,2.160964047443681,3.5,0,1.000936329588015,0,15.782972563647261,12.679695715883515,1.792481250360578,0,6.335,0,0,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,0,9,10,0,12,0,14,15,16,0,18,0,0,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,23,8,0,22,11,0,13,0,21,20,17,0,19,0,0,0,0,0,0,28,27,0,0,0],"v":[0,-0.03672484,0,0,-0.03760351,0,0,-0.04313957,0,0,-0.03837817,0,0.06688621,0,0,0,-0.03342332,0,-0.005676927,0.05641691,0.03497029,-0.01645368,-0.01225245,0.001093049,0,0,0.005374607,-0.03449066,0.04695946]},{"f":[21,-1,22,22,22,16,-1,22,22,22,15,-1,-1,2,-1,-1,-1,-1,2,15,-1,-1,-1,-1,18,25,-1,-1,-1],"t":[1.5,0,17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,4.128693921346326,6.0,0,0,0,0,8.729619742794132,1.5,0,0,0],"m":[0,0,1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,0,8,9,10,11,0,0,14,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,7,0,17,16,13,12,0,0,15,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03636128,0,0,0,0,-0.04837939,0,0,0,0,0.001609291,-0.03994322,0,-0.04851931,0.005053669,0.0562097,-0.02591685,0,0,0.03638483,-0.03839813,0.0334195,-0.01920342,0,0,0.05081489,-0.03454055,-0.03633062]},{"f":[21,-1,1,21,-1,-1,16,16,23,-1,21,15,-1,-1,18,22,2,-1,-1,17,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[1.5,0,0.9444444444444444,7.5,0,0,7.930737337562887,7.679480099505446,2.5,0,6.5,9.92481250360578,0,0,8.729619742794132,17.094543568094064,2.453445297804259,0,0,7.65814076587311,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[0,0,0,1,0,0,1,1,1,0,0,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,0,11,12,0,0,15,16,17,0,0,20,0,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,10,0,14,13,0,0,22,19,18,0,0,21,0,0,0,0,26,0,28,0,0],"v":[0,-0.03599544,0,0,-0.04200938,0.01146663,0,0,0,0.003823118,0,0,-0.04964319,0.005125839,0,0,0,-0.03314527,-0.0005861092,0,-0.02612062,0.05682441,-0.0383852,-0.03368746,0,-0.02718161,0,0.005421687,-0.03514925]},{"f":[1,21,17,-1,-1,22,22,-1,-1,-1,2,21,15,-1,22,36,-1,22,18,-1,-1,-1,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,0,0,7.640240405159187,10.5,10.457380879072534,0,16.759160379664443,4.5,0,15.782972563647261,1.792481250360578,0,0,0,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,0,0,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,0,0,11,12,13,0,15,16,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[10,5,4,0,0,9,8,0,0,0,24,23,14,0,22,17,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,-0.04528932,0.01756331,0,0,0.03143234,-0.006623206,0.04526575,0,0,0,0.0008692184,0,0,-0.03860163,0,0,0.0002389873,-0.03456944,0.03461777,0.04594592,0.05394909,0,0.03810514,0,0.01485232,-0.01298895]},{"f":[21,-1,1,21,-1,16,23,16,-1,-1,21,-1,2,17,-1,-1,-1,1,22,21,-1,-1,-1,-1,2,15,-1,-1,-1],"t":[1.5,0,9.107199999999999,2.5,0,7.930737337562887,1.5,6.832890014164741,0,0,6.5,0,5.065928480304397,8.496347487943302,0,0,0,8.046875,17.078739781790915,4.5,0,0,0,0,6.118903736861568,9.92481250360578,0,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0],"l":[1,0,3,4,0,6,7,8,0,0,11,0,13,14,0,0,0,18,19,20,0,0,0,0,25,26,0,0,0],"r":[2,0,24,5,0,17,10,9,0,0,12,0,16,15,0,0,0,23,22,21,0,0,0,0,28,27,0,0,0],"v":[0,-0.03528415,0,0,-0.03709063,0,0,0,0.006460179,-0.01776892,0,-0.03630059,0,0,-0.04648903,0.00888371,-0.001897841,0,0,0,-0.03582028,0.003567248,0.05102468,-0.04818414,0,0,0.005150059,-0.03339474,0.04575383]},{"f":[21,-1,22,22,22,22,22,10,-1,22,15,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,18,25,-1,-1,-1],"t":[1.5,0,17.094543568094064,16.98920034885663,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,8.729619742794132,1.5,0,0,0],"m":[0,0,1,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,0,3,4,5,6,7,8,0,10,11,0,0,14,0,0,0,0,19,0,21,0,0,0,25,26,0,0,0],"r":[2,0,24,23,18,17,16,9,0,13,12,0,0,15,0,0,0,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,-0.03491332,0,0,0,0,0,0,-0.04757049,0,0,0.001557939,-0.0388918,0,-0.04737802,0.004484664,0.05331158,-0.02499392,0,0.06610248,0,-0.03955096,0.01904964,-0.01843418,0,0,0.04902665,-0.03298333,-0.03517368]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,-1,-1,-1,-1,2,21,-1,36,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,0,0,0,0,7.640240405159187,9.5,0,3.5,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[18,5,4,0,0,17,8,0,12,11,0,0,16,15,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,0,0,-0.04494275,0.01654138,0,0,0.02905408,0,0,-0.03748118,0.03376526,0,0,-0.01394865,0.01366294,-0.02382422,0.04282743,0,0,-0.0008830709,0,0.0002338056,0.04609576,0,0.03603105,0,0.01396907,-0.01254279]},{"f":[21,-1,22,-1,22,-1,22,21,18,15,-1,-1,-1,-1,22,18,-1,-1,22,16,-1,16,3,-1,-1,-1,15,-1,-1],"t":[1.5,0,6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,1.292481250360578,10.462502272597265,0,0,0,0,12.921158001713904,5.022197059679227,0,0,14.116600814324018,6.832890014164741,0,11.98370619265935,0.9300813008130082,0,0,0,5.044394119358453,0,0],"m":[0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,0,7,8,9,10,0,0,0,0,15,16,0,0,19,20,0,22,23,0,0,0,27,0,0],"r":[2,0,4,0,6,0,14,13,12,11,0,0,0,0,18,17,0,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,-0.0341861,0,-0.02351237,0,0.02462828,0,0,0,0,-0.01773833,0.03118704,-0.03899074,-0.0002496866,0,0,0.003995168,0.03164629,0,0,-0.04490937,0,0,0.008538227,-0.02407242,0.04324245,0,0.01298894,-0.0001194084]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,22,-1,-1,-1,-1,-1,2,21,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,15.212004812197344,0,0,0,0,0,7.640240405159187,10.5,0,0,7.950787011501063,0,12.921158001713904,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,0,0,0,21,22,0,0,25,0,27,0,0],"r":[20,5,4,0,0,19,8,0,12,11,0,0,18,17,16,0,0,0,0,0,24,23,0,0,26,0,28,0,0],"v":[0,0,0,-0.04460296,0.01437523,0,0,0.02658723,0,0,-0.03699775,0.03288056,0,0,0,-0.0289936,-0.001027945,0.01262732,-0.02310523,0.04112329,0,0,-0.0003411152,0.04991078,0,0.03411911,0,0.01281603,-0.01197847]},{"f":[16,16,22,21,-1,23,-1,-1,16,16,21,-1,-1,-1,4,17,22,22,-1,-1,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,14.116600814324018,3.5,0,2.5,0,0,6.108524456778169,5.700439718141092,3.5,0,0,0,3.084962500721156,8.729619742794132,16.765402642760364,16.725912049439096,0,0,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[1,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,16,17,18,0,0,0,0,0,0,25,0,27,0,0],"r":[24,23,8,5,0,7,0,0,14,13,12,0,0,0,22,21,20,19,0,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,-0.03741457,0,0.003553897,-0.02065787,0,0,0,0.05755425,-0.002051408,-0.03739278,0,0,0,0,0.003816632,-0.0369634,0.04237707,-0.04461457,0.03660426,-0.03312169,0,-0.02630323,0,0.004572917,-0.03472151]},{"f":[1,21,-1,25,22,22,-1,-1,17,-1,-1,1,37,-1,-1,21,-1,-1,2,21,15,-1,15,-1,15,-1,-1,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,0,0,6.118903736861568,9.5,3.0,0,8.076815597050832,0,9.92481250360578,0,0,0,0],"m":[1,0,0,1,1,1,0,0,1,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,0,0,12,13,0,0,16,0,0,19,20,21,0,23,0,25,0,0,0,0],"r":[18,3,0,11,8,7,0,0,10,0,0,15,14,0,0,17,0,0,28,27,22,0,24,0,26,0,0,0,0],"v":[0,0,-0.040988,0,0,0,0.000550166,-0.01916023,0,0.04655631,-0.01094139,0,0,-0.003065214,0.0512748,0,-0.007546844,-0.04293696,0,0,0,0.01074388,0,-0.03818886,0,0.009734258,-0.03237232,0.02280608,0.04362413]},{"f":[1,21,17,-1,-1,22,22,-1,2,22,-1,-1,22,21,22,-1,-1,-1,-1,-1,2,21,-1,15,-1,-1,21,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,6.247802615595391,15.782972563647261,0,0,16.73920708776685,7.5,15.212004812197344,0,0,0,0,0,9.025136904799169,8.5,0,7.774787059601174,0,0,6.5,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,1,1,0,1,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,0,0,0,21,22,0,24,0,0,27,0,0],"r":[20,5,4,0,0,19,8,0,12,11,0,0,18,17,16,0,0,0,0,0,26,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.0442258,0.01364621,0,0,0.02466354,0,0,-0.03642663,0.031885,0,0,0,-0.02839138,-0.001217117,0.01187416,-0.0218931,0.03938221,0,0,-0.0006539971,0,0.005666048,0.06171031,0,0.03685784,0.003633558]},{"f":[23,22,16,33,-1,-1,33,24,22,22,22,22,22,-1,-1,-1,22,-1,-1,-1,18,32,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,41.975696368331896,0,0,35.98606694718995,9.5,17.078739781790915,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,0,0,0,8.729619742794132,6.6765734127490415,0,0,0,0,0,0,0],"m":[1,1,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,12,13,0,0,0,17,0,0,0,21,22,0,0,0,0,0,0,0],"r":[28,27,6,5,0,0,26,25,20,19,16,15,14,0,0,0,18,0,0,0,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.03043149,0.01091065,0,0,0,0,0,0,0,-0.0008118916,0.0500112,-0.02424359,0,0.06320059,0.003731958,-0.0377676,0,0,0.04668406,-0.007788845,-0.03854966,0.01903378,-0.02537439,0.03074929,-0.03088151]},{"f":[16,16,22,21,-1,23,-1,-1,16,16,21,-1,-1,-1,4,17,22,22,-1,-1,-1,-1,-1,-1,21,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,14.116600814324018,3.5,0,2.5,0,0,6.108524456778169,5.700439718141092,3.5,0,0,0,3.084962500721156,8.729619742794132,16.765402642760364,16.725912049439096,0,0,0,0,0,0,4.5,0,35.98606694718995,0,0],"m":[1,1,1,1,0,1,0,0,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,16,17,18,0,0,0,0,0,0,25,0,27,0,0],"r":[24,23,8,5,0,7,0,0,14,13,12,0,0,0,22,21,20,19,0,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,0,-0.03657603,0,0.003325339,-0.0201145,0,0,0,0.05470296,-0.001652724,-0.03654839,0,0,0,0,0.00360352,-0.03577841,0.03907122,-0.04327172,0.03496011,-0.03241937,0,-0.02546191,0,0.004310161,-0.03382166]},{"f":[23,22,1,21,-1,33,11,22,22,-1,-1,16,-1,-1,-1,1,4,-1,22,35,-1,-1,-1,-1,2,-1,-1,-1,-1],"t":[7.5,17.105450904090162,9.107199999999999,2.5,0,21.80651201897376,0.5,16.759160379664443,16.73920708776685,0,0,8.321928094887362,0,0,0,8.980632551528075,3.6427011094311244,0,16.838461650489297,2.5,0,0,0,0,6.118903736861568,0,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,1,1,0,1,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,16,17,0,19,20,0,0,0,0,25,0,0,0,0],"r":[28,27,24,5,0,15,14,11,10,0,0,13,0,0,0,23,18,0,22,21,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04023759,0,0,0,0,0.001358292,-0.03618354,0,-0.009776591,0.0355936,-0.03692126,0,0,-0.01468595,0,0,-0.03988866,0.00531139,0.02147941,-0.04617929,0,0.002510107,0.04279764,0.02976307,-0.0303814]},{"f":[22,-1,22,-1,22,21,18,15,-1,-1,-1,-1,1,22,-1,22,-1,-1,22,15,-1,-1,22,16,-1,16,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,1.292481250360578,10.462502272597265,0,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,6.832890014164741,0,11.98370619265935,0,0,0],"m":[0,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,8,0,0,0,0,13,14,0,16,0,0,19,20,0,0,23,24,0,26,0,0,0],"r":[2,0,4,0,12,11,10,9,0,0,0,0,18,15,0,17,0,0,22,21,0,0,28,25,0,27,0,0,0],"v":[0,-0.02276697,0,0.02221951,0,0,0,0,-0.016648,0.02865262,-0.03818693,-0.0004490657,0,0,-0.04458753,0,0.08471516,-0.01926575,0,0,0.005400982,0.03671182,0,0,-0.04329496,0,-0.006526605,0.04715056,0.002032241]},{"f":[23,1,21,2,22,-1,-1,-1,18,22,15,-1,22,-1,-1,16,22,-1,15,-1,-1,-1,0,-1,-1,18,-1,-1,-1],"t":[7.5,4.92309205350118,5.5,8.612669628062957,16.978341770814396,0,0,0,10.460454442916898,16.759160379664443,11.054604317960672,0,11.722778233171695,0,0,8.96289600533726,17.078739781790915,0,7.774787059601174,0,0,0,2.5,0,0,10.415240582199587,0,0,0],"m":[1,1,0,1,1,0,0,0,1,1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,0,13,0,0,16,17,0,19,0,0,0,23,0,0,26,0,0,0],"r":[28,25,8,7,6,0,0,0,22,15,12,0,14,0,0,21,18,0,20,0,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0,0,-0.03767375,0.01594472,0.007147679,0,0,0,-0.001518122,0,0.01289474,-0.03476661,0,0,-0.01853461,0,-0.004875199,0.05593625,0.04626422,0,-0.01660668,0.01803495,0,0.002552575,-0.03136239,-0.02989691]},{"f":[23,22,16,33,-1,-1,37,33,1,21,2,-1,-1,-1,18,33,-1,22,-1,-1,36,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,41.975696368331896,0,0,11.73,35.98606694718995,4.92309205350118,5.5,8.612669628062957,0,0,0,8.729619742794132,29.297756302860655,0,11.775312925643524,0,0,3.5,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,0,0,0,15,16,0,18,0,0,21,0,0,0,25,0,0,0,0],"r":[28,27,6,5,0,0,24,23,14,13,12,0,0,0,20,17,0,19,0,0,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.02914945,0.009270846,0,0,0,0,0,-0.03159743,0.007054867,-0.0003126234,0,0,0.0002963171,0,-0.02988421,0.01102159,0,0.00659777,-0.04445348,-0.04488339,0,0.01102159,0.04055271,0.02851139,-0.02938429]},{"f":[4,16,-1,18,2,-1,21,-1,22,22,-1,-1,-1,33,-1,18,-1,17,22,22,25,-1,-1,-1,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,2.160964047443681,1.792481250360578,0,6.5,0,16.765402642760364,16.73920708776685,0,0,0,20.752541988539424,0,6.531360383582522,0,8.729619742794132,17.074218326240924,15.676331898425197,1.5,0,0,0,0,0,17.094543568094064,0,0],"m":[1,0,0,0,1,0,1,0,1,1,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,0,7,0,9,10,0,0,0,14,0,16,0,18,19,20,21,0,0,0,0,0,27,0,0],"r":[26,3,0,13,6,0,8,0,12,11,0,0,0,15,0,17,0,25,24,23,22,0,0,0,0,0,28,0,0],"v":[0,0,-0.04367882,0,0,0.01110924,0,-0.02880601,0,0,0.007411336,-0.03054416,0.04028828,0,-0.04550984,0,-0.02372651,0,0,0,0,0.01622035,-0.01738184,-0.0253585,0.0460922,-0.01962101,0,0.001104154,0.04361174]},{"f":[22,15,-1,-1,22,21,4,-1,18,-1,-1,-1,1,22,-1,22,-1,-1,15,18,-1,-1,0,-1,16,-1,22,-1,-1],"t":[7.95646805166752,3.584962500721156,0,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,2.0,6.515678298127854,0,0,3.5,0,6.108524456778169,0,15.676331898425197,0,0],"m":[0,1,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0],"l":[1,2,0,0,5,6,7,0,9,0,0,0,13,14,0,16,0,0,19,20,0,0,23,0,25,0,27,0,0],"r":[4,3,0,0,12,11,8,0,10,0,0,0,18,15,0,17,0,0,22,21,0,0,24,0,26,0,28,0,0],"v":[0,0,-0.01527696,0.02382192,0,0,0,-0.04556211,0,0.007736276,-0.03174542,-0.0004746289,0,0,-0.04404701,0,0.081333,-0.0189889,0,0,-0.002649186,0.03303319,0,-0.02078143,0,-0.005306992,0,0.01282279,-0.002211129]},{"f":[22,22,22,16,-1,22,22,22,15,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,13.56045225766691,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,0,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,16,5,0,15,14,11,10,0,0,13,0,0,0,0,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,-0.04476454,0,0,0,0,0.001080863,-0.03809541,0,-0.0441257,0.006368085,0.04626344,-0.02279396,0,0.06074829,0,-0.0377506,0.01751778,-0.01600387,0,0.05086283,0,0,0.04200197,-0.0343183,-0.04092094]},{"f":[16,16,23,3,-1,-1,21,22,-1,-1,0,25,22,-1,2,17,-1,-1,-1,-1,-1,-1,21,22,-1,-1,33,-1,-1],"t":[7.930737337562887,7.679480099505446,1.5,0.46415770609318996,0,0,6.5,16.98413870862295,0,0,10.5,1.5,8.882387211598338,0,5.065928480304397,7.723832212141186,0,0,0,0,0,0,4.5,16.964783007086943,0,0,35.98606694718995,0,0],"m":[1,1,1,0,0,0,0,1,0,0,1,1,0,0,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,12,13,0,15,16,0,0,0,0,0,0,23,24,0,0,27,0,0],"r":[22,21,6,5,0,0,10,9,0,0,20,19,14,0,18,17,0,0,0,0,0,0,26,25,0,0,28,0,0],"v":[0,0,0,0,-0.03177998,0.003675534,0,0,-0.03967718,0.008399244,0,0,0,-0.03784429,0,0,-0.03253222,0.01074341,0.003486562,-0.04012532,0.03481599,-0.0311672,0,0,-0.03489667,0.009353348,0,0.003875825,-0.0320983]},{"f":[37,4,16,-1,18,-1,23,22,22,21,-1,-1,-1,-1,22,21,-1,-1,17,-1,-1,18,-1,32,-1,-1,21,-1,-1],"t":[11.73,3.245926548164837,2.0,0,2.160964047443681,0,1.5,15.676331898425197,14.03140051979143,5.5,0,0,0,0,17.07907323966382,8.5,0,0,8.729619742794132,0,0,5.303665156874805,0,6.208926257442949,0,0,6.5,0,0],"m":[1,1,0,0,0,0,0,1,1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,0,0,15,16,0,0,19,0,0,22,0,24,0,0,27,0,0],"r":[26,21,4,0,6,0,14,13,12,11,0,0,0,0,18,17,0,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,0,-0.04211611,0,0.002510851,0,0,0,0,-0.04140454,-0.001043257,0.007682738,-0.02352092,0,0,-0.04447206,0.005765152,0,0.007682738,-0.0411484,0,-0.001431787,0,-0.02643618,0.007682738,0,0.007682738,0.04430404]},{"f":[22,22,22,16,-1,22,22,22,22,-1,-1,2,-1,-1,-1,-1,22,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.844368517741394,0.0,0,16.786010507999382,16.765402642760364,16.73920708776685,16.725912049439096,0,0,6.515505953718852,0,0,0,0,16.903290731167747,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,0,0,0,17,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,16,5,0,15,14,11,10,0,0,13,0,0,0,0,18,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,-0.04380095,0,0,0,0,-0.0001549932,0.02517291,0,-0.04320023,0.0058056,0.04487524,-0.02193033,0,0.05881776,0,-0.03687298,0.01685398,-0.01530467,0,0.05004955,0,0,0.04061515,-0.03358007,-0.03986542]},{"f":[1,21,17,-1,-1,22,22,-1,21,22,-1,-1,22,22,21,-1,-1,21,2,-1,-1,-1,-1,-1,3,-1,15,-1,-1],"t":[2.01685992705101,6.5,11.503187353537466,0,0,16.765402642760364,7.082728314749225,0,7.5,16.725912049439096,0,0,16.725912049439096,9.024458927812285,8.5,0,0,8.5,6.451187557243013,0,0,0,0,0,0.0339080459770115,0,5.044394119358453,0,0],"m":[0,1,1,0,0,1,0,0,1,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,9,10,0,0,13,14,15,0,0,18,19,0,0,0,0,0,25,0,27,0,0],"r":[24,5,4,0,0,23,8,0,12,11,0,0,22,17,16,0,0,21,20,0,0,0,0,0,26,0,28,0,0],"v":[0,0,0,-0.04303826,0.01113399,0,0,0.0220408,0,0,-0.02425201,0.00917943,0,0,0,-0.03410327,0.02014267,0,0,0.07436227,0.01887429,-0.001948031,-0.03128686,0.03579711,0,0.0184512,0,0.005072573,-0.001178746]},{"f":[22,-1,22,1,22,-1,-1,-1,22,21,4,-1,18,16,-1,-1,-1,-1,1,22,-1,22,-1,-1,15,-1,0,-1,-1],"t":[6.870153907126159,0,7.95646805166752,4.008411214953271,7.082728314749225,0,0,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,5.78135971352466,0,0,0,0,1.282051282051282,15.717370311433935,0,16.786010507999382,0,0,1.584962500721156,0,2.5,0,0],"m":[0,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,0,0,0,9,10,11,0,13,14,0,0,0,0,19,20,0,22,0,0,25,0,27,0,0],"r":[2,0,8,7,6,0,0,0,18,17,12,0,16,15,0,0,0,0,24,21,0,23,0,0,26,0,28,0,0],"v":[0,-0.02116289,0,0,0,0.02297743,-0.02048075,0.04364325,0,0,0,-0.04510033,0,0,0.05777522,0.001559113,-0.03086705,-0.0006664709,0,0,-0.0435371,0,0.07724595,-0.01786724,0,0.02649964,0,-0.02387062,0.001588241]},{"f":[1,21,-1,25,22,22,22,-1,2,-1,-1,22,-1,4,-1,-1,17,-1,-1,1,37,-1,-1,21,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.844368517741394,0,4.128693921346326,0,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,9,0,0,12,0,14,0,0,17,0,0,20,21,0,0,24,0,0,27,0,0],"r":[26,3,0,19,16,11,8,0,10,0,0,13,0,15,0,0,18,0,0,23,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.0386659,0,0,0,0,-0.0003573252,0,-0.02966444,0.02904536,0,-0.03488498,0,-0.02546538,0.04227216,0,0.04159356,-0.008051253,0,0,-0.003009605,0.04804979,0,-0.007907642,-0.04089777,0,0.001718148,0.04139564]},{"f":[23,21,15,16,-1,2,22,-1,-1,22,-1,-1,22,23,18,-1,-1,3,-1,4,-1,22,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,16.765402642760364,0,0,16.759160379664443,0.5,10.680359255641626,0,0,0.15991874474642756,0,6.749860669831508,0,15.782972563647261,11.054604317960672,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,0,18,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,17,16,0,0,19,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0009200338,0,0,0.008159861,0.1231972,0,0.001272801,0.03394909,0,0,0,-0.01198311,0.04409055,0,0.00455494,0,-0.03746531,0,0,-0.004066924,-0.03726454,0.0364301,0.02711662,0.03326431,-0.02804092]},{"f":[22,-1,22,-1,22,21,4,-1,18,-1,-1,-1,22,15,-1,-1,22,16,-1,-1,15,21,-1,-1,15,17,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,0,0,0,12.921158001713904,7.930737337562887,0,0,14.116600814324018,6.832890014164741,0,0,5.044394119358453,4.5,0,0,6.169925001442312,7.67595272073847,0,0,0],"m":[0,0,0,0,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,6,7,0,9,0,0,0,13,14,0,0,17,18,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,4,0,12,11,8,0,10,0,0,0,16,15,0,0,20,19,0,0,24,23,0,0,28,27,0,0,0],"v":[0,-0.02062812,0,0.01833599,0,0,0,-0.04480601,0,0.007366949,-0.03020402,-0.0006440062,0,0,0.004048895,0.03323765,0,0,-0.04389083,-0.00314322,0,0,0.05044376,0.006581648,0,0,-0.03496036,0.02819683,0.001320179]},{"f":[37,25,22,22,22,22,33,10,-1,21,-1,22,22,15,37,-1,-1,-1,-1,-1,-1,18,-1,-1,-1,-1,-1,-1,-1],"t":[12.52,6.5,17.078739781790915,17.02942582914998,16.838461650489297,16.786010507999382,35.98606694718995,0.5,0,2.5,0,16.759160379664443,16.73920708776685,13.56045225766691,7.92,0,0,0,0,0,0,5.065928480304397,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,10,0,12,13,14,15,0,0,0,0,0,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,21,20,9,0,11,0,19,18,17,16,0,0,0,0,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.04357399,0,-0.03755064,0,0,0,0,-2.248409e-05,0.006643442,-0.03723074,-0.03202099,0.01044043,-0.04380336,0,-0.03155387,0.01044043,0.01044043,-0.04047122,0.01000917,-0.03271139,0.0211646]},{"f":[4,19,37,21,-1,-1,-1,-1,22,3,-1,2,16,16,-1,-1,22,16,-1,17,-1,-1,-1,2,33,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,4.470000000000001,6.5,0,0,0,0,17.078739781790915,0.13721264367816094,0,5.8698903048866296,6.0,5.78135971352466,0,0,14.03140051979143,7.266786540694901,0,6.531360383582522,0,0,0,6.793888758164105,29.297756302860655,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,0,12,13,14,0,0,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,11,0,23,16,15,0,0,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.0380358,-0.007650061,-0.0001614822,-0.03841138,0,0,0.03346782,0,0,0,-0.0008485986,-0.03832504,0,0,-0.0235098,0,0.05852162,0.004494898,0.04696023,0,0,-0.02474213,0.01527734,0.0004130082,0.03833289]},{"f":[23,21,15,16,-1,2,22,-1,-1,22,-1,-1,22,22,16,-1,22,-1,-1,1,-1,22,33,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,16.765402642760364,0,0,16.759160379664443,11.722778233171695,10.501837184902296,0,11.683871801979876,0,0,4.87351462522852,0,15.782972563647261,35.98606694718995,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,19,16,0,18,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0008511883,0,0,0.007479311,0.1078587,0,0.001225443,0.0323811,0,0,0,-0.03458501,0,0.04082287,0.002329716,0,-0.02985581,0,0,0.01404621,-0.03889483,0.03340085,0.02550476,0.03178215,-0.02772391]},{"f":[4,19,37,21,-1,-1,-1,-1,22,22,22,22,22,15,21,-1,-1,22,-1,1,-1,-1,16,-1,-1,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,4.470000000000001,6.5,0,0,0,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,16.759160379664443,11.683433292835403,3.5,0,0,15.782972563647261,0,3.354166666666667,0,0,9.047123912114026,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,1,1,1,1,1,0,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,11,12,13,14,15,0,0,18,0,20,0,0,23,0,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,27,26,25,22,17,16,0,0,19,0,21,0,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.03757822,-0.007338955,-0.0001679753,-0.03801239,0,0,0,0,0,0,0,-0.01731596,0.004275734,0,-0.03406439,0,-0.02891757,0.03534956,0,-0.01282505,0.04227748,-0.03985752,0.03319051,-0.03371939,0.03642016]},{"f":[23,21,15,16,-1,2,22,-1,-1,-1,23,18,-1,-1,22,3,-1,4,-1,22,15,-1,18,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,11.775312925643524,0,0,0,0.5,10.680359255641626,0,0,16.759160379664443,0.15991874474642756,0,6.749860669831508,0,15.782972563647261,11.054604317960672,0,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,1,0,0,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,0,0,15,16,0,18,0,20,21,0,23,0,0,0,0,0,0],"r":[28,27,10,5,0,9,8,0,0,0,14,13,0,0,26,17,0,19,0,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0008137362,0,0,0.007287805,0.09818403,0.003022284,0,0,-0.01129833,0.04118405,0,0,0.004279094,0,-0.03575305,0,0,-0.002947206,0,-0.04610209,-0.00293657,0.03055063,0.02284039,0.0304822,-0.02741312]},{"f":[37,25,4,16,-1,18,2,-1,21,-1,22,-1,22,-1,-1,23,22,-1,-1,22,21,-1,-1,17,-1,-1,-1,-1,-1],"t":[12.52,6.5,3.245926548164837,2.0,0,2.160964047443681,1.792481250360578,0,6.5,0,16.73920708776685,0,16.765402642760364,0,0,1.5,15.676331898425197,0,0,17.07907323966382,8.5,0,0,8.729619742794132,0,0,0,0,0],"m":[1,1,1,0,0,0,1,0,1,0,1,0,1,0,0,0,1,0,0,1,1,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,9,0,11,0,13,0,0,16,17,0,0,20,21,0,0,24,0,0,0,0,0],"r":[28,27,26,5,0,15,8,0,10,0,12,0,14,0,0,19,18,0,0,23,22,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.04151075,0,0,0.009953302,0,-0.02590938,0,0.00598079,0,-0.02769539,0.009953302,0,0,0.001489379,-0.02270373,0,0,-0.04360947,0.004622311,0,0.009953302,-0.04004547,0.001737412,-0.0325325,0.02011708]},{"f":[16,27,-1,22,-1,22,-1,-1,22,22,22,22,22,37,33,-1,-1,-1,-1,-1,2,-1,-1,-1,18,33,-1,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,17.094543568094064,0,0,17.078739781790915,16.98920034885663,16.844368517741394,16.786010507999382,16.765402642760364,11.73,35.98606694718995,0,0,0,0,0,3.903677461028802,0,0,0,8.729619742794132,20.091102901552432,0,0,0],"m":[0,1,0,0,0,1,0,0,1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,1,1,0,0,0],"l":[1,2,0,4,0,6,0,0,9,10,11,12,13,14,15,0,0,0,0,0,21,0,0,0,25,26,0,0,0],"r":[8,3,0,5,0,7,0,0,24,23,20,19,18,17,16,0,0,0,0,0,22,0,0,0,28,27,0,0,0],"v":[0,0,-0.04429889,0,0.03156798,0,-0.01058458,0.03037179,0,0,0,0,0,0,0,-0.0002285236,-0.04265765,0.02819952,0.04045087,-0.02026709,0,-0.007706993,0.04584722,-0.03390756,0,0,0.03885594,-0.01162431,-0.0311866]},{"f":[22,-1,22,-1,22,21,22,0,-1,-1,33,18,33,-1,-1,-1,-1,15,21,-1,-1,16,-1,-1,22,-1,22,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,6.5,11.775312925643524,15.5,0,0,35.98606694718995,4.128693921346326,35.98439522565276,0,0,0,0,7.774787059601174,7.5,0,0,11.756139298372927,0,0,12.921158001713904,0,14.116600814324018,0,0],"m":[0,0,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,0,3,0,5,6,7,8,0,0,11,12,13,0,0,0,0,18,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,4,0,24,17,10,9,0,0,16,15,14,0,0,0,0,21,20,0,0,23,0,0,26,0,28,0,0],"v":[0,-0.01936995,0,0.01718378,0,0,0,0,-0.0418689,0.01613514,0,0,0,-0.002151339,0.03980001,-0.02614875,-0.04150753,0,0,-0.02486643,9.604249e-05,0,0.02612519,-0.01506922,0,0.009020859,0,-0.01422753,0.001590904]},{"f":[23,21,15,4,37,-1,-1,22,16,-1,-1,-1,22,22,33,-1,22,-1,-1,1,-1,22,33,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,4.470000000000001,0,0,17.078739781790915,9.047123912114026,0,0,0,16.759160379664443,11.722778233171695,73.23848001446603,0,11.683871801979876,0,0,4.87351462522852,0,15.782972563647261,35.98606694718995,0,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,14,15,0,17,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,12,7,6,0,0,11,10,0,0,0,26,19,16,0,18,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,-0.01972546,-0.0002675474,0,0,-0.0006489014,0.01220754,0.03416047,0,0,0,-0.03458928,0,0.03713988,0.001487132,0,-0.02772609,0,0,0.01260747,-0.03719497,0.02834835,0.02143715,0.02936902,-0.02640155]},{"f":[21,4,16,-1,16,36,22,2,-1,-1,-1,-1,16,-1,-1,22,18,-1,32,-1,20,-1,-1,-1,36,18,-1,-1,-1],"t":[9.5,3.245926548164837,2.0,0,5.044394119358453,3.5,15.782972563647261,1.292481250360578,0,0,0,0,6.108524456778169,0,0,17.094543568094064,5.303665156874805,0,6.208926257442949,0,0.5,0,0,0,3.5,1.792481250360578,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,0,0,13,0,0,16,17,0,19,0,21,0,0,0,25,26,0,0,0],"r":[24,15,4,0,12,11,10,9,0,0,0,0,14,0,0,23,18,0,20,0,22,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.04162805,0,0,0,0,0.02214346,-0.02123699,-0.01447639,0.03134874,0,-0.03748368,-0.003091021,0,0,-0.002335299,0,-0.02587865,0,0.001476428,0.02051819,0.0377323,0,0,0.01253233,-0.02183775,0.04059007]},{"f":[9,16,16,-1,-1,21,-1,17,4,22,-1,-1,22,1,-1,-1,-1,17,-1,17,-1,22,18,-1,-1,18,-1,-1,-1],"t":[0.5,7.930737337562887,7.679480099505446,0,0,4.5,0,1.792481250360578,5.108372929097653,12.199665042528586,0,0,16.725912049439096,1.987429289754871,0,0,0,5.065928480304397,0,6.219136028062415,0,16.838461650489297,7.519373778108656,0,0,6.400854179458231,0,0,0],"m":[1,1,1,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,0,0,6,0,8,9,10,0,0,13,14,0,0,0,18,0,20,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,7,0,17,12,11,0,0,16,15,0,0,0,19,0,21,0,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.000579045,-0.029974,0,-0.02252365,0,0,0,0.01185023,0.04560286,0,0,0.01400594,-0.03960625,0.01487129,0,-0.01468594,0,0.04292927,0,0,-0.002576344,0.008704066,0,0.03685414,-0.01384072,-0.02850907]},{"f":[23,21,15,22,21,-1,23,-1,-1,-1,22,22,-1,22,-1,1,-1,-1,-1,0,22,-1,22,1,-1,-1,-1,-1,-1],"t":[7.5,8.5,11.054604317960672,14.116600814324018,3.5,0,3.5,0,0,0,16.759160379664443,11.683871801979876,0,15.782972563647261,0,4.008411214953271,0,0,0,10.5,10.224917391507956,0,13.589275375459419,1.99834912080135,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,12,0,14,0,16,0,0,0,20,21,0,23,24,0,0,0,0,0],"r":[28,19,10,9,6,0,8,0,0,0,18,13,0,15,0,17,0,0,0,27,22,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03420563,0,0.0006637714,-0.02101308,0.002952378,0,0,0.02603522,0,-0.03269707,0,-0.02747008,0.02436748,0.02184548,0,0,0.02157967,0,0,0.001063255,-0.02295418,0.01402991,0.04204404,-0.025924]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,-1,-1,24,33,22,22,22,22,-1,-1,-1,-1,18,-1,-1,-1,21,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,0,0,10.5,35.98606694718995,17.078739781790915,16.98920034885663,16.844368517741394,16.786010507999382,0,0,0,0,8.729619742794132,0,0,0,5.5,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,0,6,0,8,0,10,0,0,13,14,15,16,17,18,0,0,0,0,23,0,0,0,27,0,0],"r":[12,3,0,5,0,7,0,9,0,11,0,0,26,25,22,21,20,19,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,-0.04377156,0,0.03053834,0,-0.02920295,0,0.03069325,0,-0.01955002,0.02907301,0,0,0,0,0,0,0.0005554394,-0.01967355,0.02611749,-0.03348163,0,0.02371023,-0.03055981,-0.04003184,0,-0.01000152,0.03326369]},{"f":[4,19,25,0,-1,-1,-1,-1,22,3,-1,22,22,33,32,22,36,-1,-1,21,-1,-1,-1,1,-1,-1,-1,-1,-1],"t":[4.0437314206251695,2.5,5.5,10.5,0,0,0,0,17.078739781790915,0.13721264367816094,0,17.02942582914998,16.844368517741394,25.24368171001516,6.474913355379556,16.759160379664443,3.5,0,0,4.5,0,0,0,8.992857142857144,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,0,1,1,1,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,0,0,9,10,0,12,13,14,15,16,17,0,0,20,0,0,0,24,0,0,0,0,0],"r":[8,7,6,5,0,0,0,0,28,11,0,27,26,23,22,19,18,0,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,-0.001815253,0.01304727,-0.02896619,-0.03684765,0,0,0.0297223,0,0,0,0,0,0,0.0002567637,-0.03104826,0,-0.02157197,0.03593005,0.04592138,0,-0.03830531,0.01434913,0.03024525,-0.03241794,0.02917006]},{"f":[9,4,-1,22,22,22,22,4,15,19,17,-1,-1,-1,17,19,-1,-1,-1,18,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,5.108372929097653,6.0,0.5,2.69615871138938,0,0,0,7.5507416557826375,0.5,0,0,0,7.980000966034041,0,6.015625,0,0,0,0,0,0,0],"m":[1,1,0,1,1,1,1,0,1,1,1,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,9,10,11,0,0,0,15,16,0,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,3,0,27,26,25,24,19,14,13,12,0,0,0,18,17,0,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,-0.001503967,0,0,0,0,0,0,0,0,-0.03121455,0.01661443,-0.03221111,0,0,0.06524631,0.01974229,-0.006954385,0,-0.003771089,0,0.002669158,0.03777952,-0.03805878,0.0288783,-0.03182133,0.02766429,-0.02796035]},{"f":[23,21,15,16,-1,2,-1,-1,23,18,18,-1,-1,-1,2,-1,22,17,22,-1,-1,22,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,0,0,0.5,10.680359255641626,1.292481250360578,0,0,0,6.247802615595391,0,16.759160379664443,1.292481250360578,16.725912049439096,0,0,15.782972563647261,11.683433292835403,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,0,0,15,0,17,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,14,13,12,0,0,0,16,0,26,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0007520889,0,0.05558977,0.002451594,0,0,0,0.01486154,-0.0278513,0.03671718,0,-0.03396625,0,0,0,-0.04229276,-0.001433588,0,0,0.001545216,-0.02844528,0.02556977,0.02046043,0.02680156,-0.02552626]},{"f":[22,-1,22,-1,22,21,4,-1,18,16,-1,-1,-1,-1,25,22,1,-1,17,-1,-1,22,16,18,-1,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,5.5,3.753897320099348,0,4.128693921346326,5.78135971352466,0,0,0,0,1.5,12.921158001713904,6.806765389082463,0,1.292481250360578,0,0,17.094543568094064,6.108524456778169,5.303665156874805,0,0,0,0,0],"m":[0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,1,0,0,0,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,0,0,0,15,16,17,0,19,0,0,22,23,24,0,0,0,0,0],"r":[2,0,4,0,14,13,8,0,12,11,0,0,0,0,28,21,18,0,20,0,0,27,26,25,0,0,0,0,0],"v":[0,-0.01841266,0,0.01459139,0,0,0,-0.04367686,0,0,0.05738918,0.000560006,-0.02817846,-0.0009530716,0,0,0,0.007338378,0,-0.02402383,0.05596067,0,0,0,-0.004180405,-0.0427599,0.002307904,0.02380197,-0.007199142]},{"f":[1,21,15,-1,-1,22,1,4,-1,2,-1,-1,22,21,-1,-1,22,-1,-1,-1,3,-1,23,-1,21,22,-1,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,4.128693921346326,0,6.434604125275413,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,0.0339080459770115,0,2.5,0,6.5,16.975010474608837,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,0,13,14,0,0,17,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,5,4,0,0,19,12,9,0,11,0,0,16,15,0,0,18,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0.008588529,-0.04193537,0,0,0,0.03205036,0,-0.0350176,0.006684227,0,0,-0.03962562,0.01110695,0,0.03484633,-0.006362182,0.03373683,0,0.01576775,0,0.00230263,0,0,-0.04627764,0.004661067,-0.0006622719]},{"f":[23,21,15,22,-1,-1,23,18,18,-1,-1,-1,2,-1,22,17,22,-1,-1,22,15,-1,18,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,14.116600814324018,0,0,0.5,10.680359255641626,1.292481250360578,0,0,0,6.247802615595391,0,16.759160379664443,1.292481250360578,16.725912049439096,0,0,15.782972563647261,11.696967526234287,0,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,1,0,0,0,0,1,0,1,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,0,0,13,0,15,16,17,0,0,20,21,0,23,0,0,0,0,0,0],"r":[28,27,6,5,0,0,12,11,10,0,0,0,14,0,26,19,18,0,0,25,22,0,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.001886748,0.003116849,0,0,0,0.01387099,-0.02690405,0.03527857,0,-0.03328413,0,0,0,-0.04186564,-0.001427556,0,0,-1.031038e-06,0,-0.04459475,0.001154378,0.02398799,0.01957031,0.02561695,-0.02505729]},{"f":[1,21,-1,25,22,22,22,-1,-1,22,-1,4,-1,-1,17,-1,-1,1,37,-1,-1,21,22,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,0,0,6.431742323097463,5.539999999999999,0,0,4.5,12.613886328427084,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,0,1,0,1,0,0,1,0,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,0,0,10,0,12,0,0,15,0,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,17,14,9,8,0,0,11,0,13,0,0,16,0,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.03641567,0,0,0,0,0.0003281725,0.02919956,0,-0.03197466,0,-0.02475006,0.02386182,0,0.03859508,-0.009820765,0,0,-0.002454756,0.04179329,0,0,-0.02178736,0.0008623797,-0.03993215,0,0.001344532,0.03696772]},{"f":[22,-1,22,15,-1,-1,22,16,17,4,15,-1,-1,2,16,-1,-1,-1,-1,16,17,-1,22,-1,1,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,2.584962500721156,0,0,12.199665042528586,10.462502272597265,4.128693921346326,6.16978583919235,5.491853096329675,0,0,7.661668144640085,6.554588851677638,0,0,0,0,11.756139298372927,9.075473573191719,0,11.67903991803059,0,3.892276422764228,0,0,0,0],"m":[0,0,0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,0,14,15,0,0,0,0,20,21,0,23,0,25,0,0,0,0],"r":[2,0,6,5,0,0,28,19,18,13,12,0,0,17,16,0,0,0,0,27,22,0,24,0,26,0,0,0,0],"v":[0,-0.01779532,0,0,-0.008206224,0.01869656,0,0,0,0,0,-0.0402469,0.000275312,0,0,0.07768203,0.002324341,-0.00286437,-0.01895322,0,0,-0.008548105,0,-0.02602594,0,0.002616477,0.04744119,-0.002304271,0.001001513]},{"f":[1,21,-1,25,22,22,22,23,-1,1,-1,-1,2,-1,-1,22,-1,-1,17,-1,-1,2,-1,32,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,16.98920034885663,16.844368517741394,5.5,0,4.92309205350118,0,0,4.128693921346326,0,0,17.078739781790915,0,0,8.729619742794132,0,0,3.245926548164837,0,5.954008630531401,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,1,0,0,0,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,0,10,0,0,13,0,0,16,0,0,19,0,0,22,0,24,0,0,27,0,0],"r":[26,3,0,21,18,15,12,9,0,11,0,0,14,0,0,17,0,0,20,0,0,23,0,25,0,0,28,0,0],"v":[0,0,-0.0359931,0,0,0,0,0,0.0008008813,0,-0.0009490432,-0.038977,0,-0.02876386,0.02426914,0,-0.03122566,-0.001167428,0,0.03733335,-0.009503803,0,-0.02298416,0,-0.005793546,0.02086369,0,0.001407464,0.03598735]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,-1,-1,25,37,33,1,18,21,-1,22,15,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,0,0,5.5,11.26,35.98606694718995,4.92309205350118,10.482680779086738,6.5,0,16.759160379664443,11.054604317960672,0,0,0,0,0,0,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,1,1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,0,8,0,10,0,0,13,14,15,16,17,18,0,20,21,0,0,0,0,0,0,0,0],"r":[12,3,0,5,0,7,0,9,0,11,0,0,28,27,26,25,24,19,0,23,22,0,0,0,0,0,0,0,0],"v":[0,0,-0.04258086,0,0.02917803,0,-0.02908089,0,0.02741263,0,-0.01765123,0.02726299,0,0,0,0,0,0,-0.01992403,0,0,-0.002557998,-0.02161626,0.01069455,0.008184686,0.002707297,-0.0403115,0.02111565,-0.01934854]},{"f":[23,21,4,16,-1,0,25,16,17,-1,-1,21,-1,22,-1,-1,-1,-1,22,2,-1,21,15,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,3.245926548164837,2.0,0,10.5,1.5,6.832890014164741,8.729619742794132,0,0,4.5,0,16.786010507999382,0,0,0,0,17.094543568094064,3.39067985676233,0,3.5,6.108524456778169,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,0,0,12,0,14,0,0,0,0,19,20,0,22,23,0,0,0,0,0,0],"r":[28,27,18,5,0,17,16,11,10,0,0,13,0,15,0,0,0,0,26,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-0.0386289,0,0,0,0,0.008781062,-0.032185,0,-0.03881174,0,-0.0009756952,-0.03293237,-0.02158317,0.02042736,0,0,0.02358997,0,0,-0.03254938,0.007182342,0.001180854,0.03511917,0.02463128,-0.02427073]},{"f":[4,19,22,22,7,3,-1,-1,-1,-1,22,-1,1,-1,22,-1,15,-1,-1,-1,32,35,2,-1,-1,-1,33,-1,-1],"t":[4.0437314206251695,2.5,16.73920708776685,16.725912049439096,0.5,0.013290990355210539,0,0,0,0,16.752707246237605,0,1.3897752394988947,0,16.765402642760364,0,5.044394119358453,0,0,0,6.208926257442949,1.5,4.128693921346326,0,0,0,20.091102901552432,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,17,0,0,0,21,22,23,0,0,0,27,0,0],"r":[20,19,10,9,8,7,0,0,0,0,12,0,14,0,16,0,18,0,0,0,26,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0.01915891,-0.0004800527,-0.02061663,0.02235007,0,-0.04101381,0,0.01770038,0,-0.03612755,0,0.04305543,-0.007149387,-0.0360606,0,0,0,0.04771961,0.000543631,-0.02792749,0,0.0488383,0.006023567]},{"f":[9,4,19,22,-1,22,-1,-1,-1,22,22,22,22,22,15,-1,22,18,-1,-1,1,-1,-1,-1,-1,-1,-1,-1,-1],"t":[0.5,4.0437314206251695,2.5,16.73920708776685,0,16.752707246237605,0,0,0,17.078739781790915,17.02942582914998,16.844368517741394,16.819976309430963,16.759160379664443,11.683433292835403,0,15.782972563647261,9.265614570460752,0,0,3.354166666666667,0,0,0,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,1,1,1,1,1,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,0,10,11,12,13,14,15,0,17,18,0,0,21,0,0,0,0,0,0,0,0],"r":[28,9,8,5,0,7,0,0,0,27,26,25,24,23,16,0,20,19,0,0,22,0,0,0,0,0,0,0,0],"v":[0,0,0,0,0.0007239242,0,-0.04001536,-0.004028935,-0.03567426,0,0,0,0,0,0,0.002721333,0,0,-0.04405957,-0.003748194,0,-0.02339982,0.02501184,0.02286986,-0.03704645,0.02779309,-0.03041822,0.02432198,-0.02635812]},{"f":[22,-1,22,-1,22,16,17,4,15,-1,-1,2,16,-1,-1,-1,36,-1,-1,22,3,-1,-1,2,17,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,0,12.199665042528586,10.462502272597265,4.128693921346326,6.16978583919235,5.491853096329675,0,0,7.661668144640085,6.554588851677638,0,0,0,4.5,0,0,11.67903991803059,0.9197769197769198,0,0,6.959990297524481,9.075473573191719,0,0,0,0],"m":[0,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,1,0,0,0,0],"l":[1,0,3,0,5,6,7,8,9,0,0,12,13,0,0,0,17,0,0,20,21,0,0,24,25,0,0,0,0],"r":[2,0,4,0,28,19,16,11,10,0,0,15,14,0,0,0,18,0,0,23,22,0,0,27,26,0,0,0,0],"v":[0,-0.01704002,0,0.01241154,0,0,0,0,0,-0.03979355,0.0002104061,0,0,0.07158735,0.002002327,-0.002840142,0,-0.02403978,-0.0009302347,0,0,0.0007774116,-0.03592212,0,0,-0.02350598,0.05231824,0.001323554,0.0009511319]},{"f":[37,25,22,21,16,-1,-1,33,23,-1,-1,-1,22,-1,22,16,-1,-1,15,33,-1,-1,22,15,-1,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,4.5,5.78135971352466,0,0,35.98606694718995,3.5,0,0,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,0,8.076815597050832,21.225056427472516,0,0,15.782972563647261,11.054604317960672,0,0,0,0,0],"m":[1,1,0,0,1,0,0,1,1,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,0,0,13,0,15,16,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,12,7,6,0,0,11,10,0,0,0,14,0,18,17,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.008487619,-0.03903273,0,0,0.0006991581,-0.02184652,-0.03923225,0,0.008266468,0,0,-0.04261478,-0.001457769,0,0,0.007622474,-0.003871938,0,0,-0.007779817,-0.0404934,0.0005805188,-0.03073089,0.01718633]},{"f":[1,21,15,-1,-1,22,1,2,-1,2,-1,-1,22,21,-1,-1,22,-1,-1,-1,3,-1,23,-1,21,32,-1,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,4.128693921346326,0,6.434604125275413,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,0.0339080459770115,0,2.5,0,6.5,8.599793042091783,0,0,0],"m":[0,1,0,0,0,1,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,0,0,13,14,0,0,17,0,0,0,21,0,23,0,25,26,0,0,0],"r":[20,5,4,0,0,19,12,9,0,11,0,0,16,15,0,0,18,0,0,0,22,0,24,0,28,27,0,0,0],"v":[0,0,0,0.00792157,-0.04131482,0,0,0,0.03507159,0,-0.03373966,0.006709944,0,0,-0.03863128,0.01174717,0,0.03275735,-0.006369885,0.03204121,0,0.01527975,0,0.002081461,0,0,-0.04578793,0.003438653,-0.0006419702]},{"f":[23,21,4,19,37,21,-1,-1,22,-1,-1,-1,32,35,4,20,16,-1,-1,-1,-1,-1,33,-1,37,-1,-1,-1,-1],"t":[7.5,10.5,4.0437314206251695,2.5,4.470000000000001,6.5,0,0,16.73920708776685,0,0,0,6.208926257442949,1.5,4.2849278041654735,0.5,5.169925001442312,0,0,0,0,0,20.091102901552432,0,8.2,0,0,0,0],"m":[1,1,1,1,1,1,0,0,1,0,0,0,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,0,13,14,15,16,17,0,0,0,0,0,23,0,25,0,0,0,0],"r":[28,27,12,11,8,7,0,0,10,0,0,0,22,21,20,19,18,0,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.03485482,-0.006660176,0,0.001313949,-0.006194045,-0.03498043,0,0,0,0,0,-0.003048922,0.06191166,-0.0198233,0.0002484178,-0.02984766,0,0.04666257,0,-0.006764908,0.01602912,0.02366009,-0.02375298]},{"f":[23,22,16,37,-1,22,-1,-1,24,33,25,0,21,-1,25,-1,36,2,-1,-1,-1,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,10.5,35.98606694718995,5.5,10.5,2.5,0,1.5,0,3.5,2.453445297804259,0,0,0,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,0,15,0,17,18,0,0,0,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,23,22,21,14,0,16,0,20,19,0,0,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04288964,0,0.009648156,-0.02152431,0,0,0,0,0,-0.03454881,0,0.0009965807,0,0,-0.01686766,0.003565301,-0.03923718,0.008033139,-0.01783608,-0.03821387,0,-0.01095233,0.03096692,0.02194829,-0.02330577]},{"f":[0,32,9,23,22,21,-1,23,18,17,22,-1,-1,25,19,-1,-1,-1,-1,16,-1,1,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,0.5,7.5,14.116600814324018,3.5,0,3.5,7.519373778108656,2.160964047443681,13.129761635363703,0,0,0.5,0.5,0,0,0,0,6.882643049361842,0,3.015625,0,0,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,8,9,10,11,0,0,14,15,0,0,0,0,20,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,7,0,19,18,13,12,0,0,17,16,0,0,0,0,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,-0.03150349,0,0,0,0,0.006522176,-0.03361463,0,0,-0.006741272,-0.03121529,0.0008433752,0.008393098,0,-0.03047496,0,-0.03273121,0.008122041,0.001713299,-0.02286527,-0.02551937,-0.02891637,0.02210169]},{"f":[1,21,15,-1,-1,22,1,22,22,4,-1,2,-1,-1,-1,-1,22,21,-1,-1,22,-1,-1,-1,4,-1,22,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,15.676331898425197,11.722778233171695,5.169368191286958,0,6.434604125275413,0,0,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,0,0,4.0437314206251695,0,17.083799521744545,0,0],"m":[0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,12,0,0,0,0,17,18,0,0,21,0,0,0,25,0,27,0,0],"r":[24,5,4,0,0,23,16,15,14,11,0,13,0,0,0,0,20,19,0,0,22,0,0,0,26,0,28,0,0],"v":[0,0,0,0.00756629,-0.04069803,0,0,0,0,0,0.02737264,0,-0.03770863,0.00626781,-0.03556869,0.0558781,0,0,-0.03805363,0.01126003,0,0.0310247,-0.006160298,0.03109916,0,-0.001095643,0,0.002450967,0.03863816]},{"f":[23,21,15,4,-1,32,-1,-1,22,33,-1,-1,2,16,-1,-1,22,15,33,-1,22,21,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,0,6.208926257442949,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,16.759160379664443,13.56045225766691,35.98606694718995,0,15.782972563647261,6.5,0,0,0,0,0,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,0,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,0,17,18,19,0,21,22,0,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,12,11,0,0,16,15,0,0,26,25,20,0,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.001197142,0,0.001433055,0.01543327,0,0,-0.0336332,0.03119191,0,0,-0.0416607,0.000948247,0,0,0,0.009266374,0,0,-0.04592799,-0.00639504,0.02064763,-0.03296686,0.01719661,0.02276484,-0.02259417]},{"f":[37,25,22,21,16,15,-1,-1,-1,33,23,-1,-1,-1,1,-1,22,-1,22,16,-1,0,-1,-1,15,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,4.5,5.78135971352466,5.357552004618084,0,0,0,35.98606694718995,3.5,0,0,0,0.4906446161274747,0,12.921158001713904,0,14.116600814324018,6.832890014164741,0,3.5,0,0,7.266786540694901,0,0,0,0],"m":[1,1,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,0,0,15,0,17,0,19,20,0,22,0,0,25,0,0,0,0],"r":[28,27,14,9,8,7,0,0,0,13,12,0,0,0,16,0,18,0,24,21,0,23,0,0,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.0300412,0.00803332,-0.03831544,0,0,0.0003267036,-0.02060435,-0.03760148,0,-0.03252169,0,0.007678009,0,0,-0.04183475,0,-0.02896185,0.004066075,0,0.006057357,-0.001391861,-0.02950608,0.01629964]},{"f":[23,21,22,16,37,-1,22,-1,-1,25,37,33,4,18,-1,23,22,-1,-1,22,-1,-1,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,5.5,11.26,35.98606694718995,3.245926548164837,2.160964047443681,0,1.5,15.676331898425197,0,0,17.07907323966382,0,0,0,0,0,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,1,1,1,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,12,13,14,0,16,17,0,0,20,0,0,0,0,0,0,0,0,0],"r":[28,27,26,9,6,0,8,0,0,25,24,23,22,15,0,19,18,0,0,21,0,0,0,0,0,0,0,0,0],"v":[0,0,0,0,0,-0.04249957,0,0.009164167,-0.02137413,0,0,0,0,0,0.001237015,0,0,0.002077751,-0.02193729,0,-0.03611581,-0.001112154,0.002213939,-0.03769628,0.01861395,-0.01851134,0.02086601,0.02186465,-0.02222705]},{"f":[1,18,21,2,1,-1,-1,-1,22,15,4,4,4,-1,-1,-1,-1,-1,16,22,-1,-1,-1,-1,18,36,-1,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,8.767235038274581,4.208333333333334,0,0,0,16.759160379664443,13.56045225766691,9.266718101290426,9.037514954341955,8.049190845056867,0,0,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,1.5,0,0,0],"m":[1,1,1,1,1,0,0,0,1,1,1,1,1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,11,12,13,0,0,0,0,0,19,20,0,0,0,0,25,26,0,0,0],"r":[24,23,8,7,6,0,0,0,18,17,16,15,14,0,0,0,0,0,22,21,0,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03450696,0.005165675,0.005018082,0,0,0,0,0,-0.001807297,-0.02985526,0.01412054,-0.01711252,-0.03887372,0,0,-0.01390294,0.01595285,0.034365,0.007370148,0,0,0.008056233,-6.709972e-06,-0.02956845]},{"f":[0,32,22,-1,22,-1,22,-1,25,21,-1,23,22,21,-1,-1,-1,16,-1,-1,0,36,-1,-1,22,-1,-1,-1,-1],"t":[18.5,10.482680779086738,6.870153907126159,0,7.95646805166752,0,8.133841411710264,0,1.5,2.5,0,0.5,11.612768260520431,8.5,0,0,0,7.930737337562887,0,0,9.5,3.5,0,0,12.613886328427084,0,0,0,0],"m":[1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,0,7,0,9,10,0,12,13,14,0,0,0,18,0,0,21,22,0,0,25,0,0,0,0],"r":[28,27,4,0,6,0,8,0,20,11,0,17,16,15,0,0,0,19,0,0,24,23,0,0,26,0,0,0,0],"v":[0,0,0,-0.01570901,0,0.01035319,0,-0.02821645,0,0,-0.03239921,0,0,0,-0.02438695,0.00623479,0.01035319,0,-0.004199505,0.001930141,0,0,-0.00782354,-0.03934819,0,-0.006457531,0.01035319,-0.02808919,0.02091893]},{"f":[1,21,2,22,16,1,-1,-1,-1,-1,-1,-1,18,36,18,18,2,-1,-1,16,-1,19,-1,-1,-1,21,-1,-1,-1],"t":[4.92309205350118,5.5,8.612669628062957,16.978341770814396,5.044394119358453,2.9861172120945896,0,0,0,0,0,0,10.415240582199587,1.5,8.729619742794132,7.198837316474133,5.40929108874043,0,0,5.906890595608519,0,1.5,0,0,0,9.5,0,0,0],"m":[1,0,1,1,0,1,0,0,0,0,0,0,1,1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,0,0,13,14,15,16,17,0,0,20,0,22,0,0,0,26,0,0,0],"r":[12,11,10,9,8,7,0,0,0,0,0,0,28,25,24,19,18,0,0,21,0,23,0,0,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.03549645,0.007193742,-0.04507015,0.01794875,0.004350499,-0.0005569191,0,0,0,0,0,0.01018342,-0.009695928,0,-0.0280299,0,0.06885403,-0.01830041,-0.04010735,0,-0.0008610118,0.02381008,-0.02885752]},{"f":[23,22,22,22,22,22,22,22,22,-1,-1,-1,-1,4,-1,-1,-1,-1,22,-1,25,17,18,-1,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,3.9534452978042594,0,0,0,0,17.078739781790915,0,0.5,8.729619742794132,6.076776015854056,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,1,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,0,14,0,0,0,0,19,0,21,22,23,0,0,0,0,0,0],"r":[28,27,18,17,16,13,12,11,10,0,0,0,0,15,0,0,0,0,20,0,26,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0.0003543207,-0.01598054,0.03555436,-0.01776992,0,-0.02670447,0.04565068,-0.02232468,0.0279993,0,-0.02612518,0,0,0,-0.005342238,0.04186882,-0.02262198,-0.01986869,0.02062469,-0.02170787]},{"f":[16,16,-1,-1,21,22,-1,-1,17,4,22,22,-1,-1,-1,22,22,-1,-1,-1,17,-1,17,-1,22,-1,18,-1,-1],"t":[7.930737337562887,7.679480099505446,0,0,4.5,16.964783007086943,0,0,1.792481250360578,5.108372929097653,12.199665042528586,9.40390385294334,0,0,0,16.725912049439096,12.606173770495205,0,0,0,5.065928480304397,0,6.219136028062415,0,16.838461650489297,0,6.400854179458231,0,0],"m":[1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0],"l":[1,2,0,0,5,6,0,0,9,10,11,12,0,0,0,16,17,0,0,0,21,0,23,0,25,0,27,0,0],"r":[4,3,0,0,8,7,0,0,20,15,14,13,0,0,0,19,18,0,0,0,22,0,24,0,26,0,28,0,0],"v":[0,0,-0.0004320748,-0.02834351,0,0,-0.03112944,0.007461188,0,0,0,0,0.03855456,0.000242866,0.03845999,0,0,0.00230292,-0.04334327,0.01143947,0,-0.01354915,0,0.03238742,0,-0.0003485959,0,0.03154377,-0.01343238]},{"f":[0,32,23,1,18,21,1,-1,-1,22,15,4,22,20,-1,16,-1,-1,-1,-1,-1,16,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.5,4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,0,16.759160379664443,13.56045225766691,9.266718101290426,16.73920708776685,0.5,0,6.247927513443585,0,0,0,0,0,7.924812503605781,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,12,13,14,0,16,0,0,0,0,0,22,0,0,0,0,0,0,0],"r":[28,27,26,25,24,9,8,0,0,21,20,19,18,15,0,17,0,0,0,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.03768246,-0.007392102,0,0,0,0,0,0.004503932,0,-0.03340966,-0.002889244,-0.01826908,-0.01632148,-0.03791676,0,-0.0156111,0.01038487,0.006770207,0.001294866,-0.0214632,-0.02727234,0.02097505]},{"f":[37,25,22,33,21,16,15,-1,-1,-1,23,-1,-1,-1,1,-1,25,-1,25,2,-1,-1,22,2,-1,-1,-1,-1,-1],"t":[12.52,6.5,12.199665042528586,35.98606694718995,4.5,5.78135971352466,5.357552004618084,0,0,0,3.5,0,0,0,0.4906446161274747,0,1.5,0,3.5,3.245926548164837,0,0,12.613886328427084,1.292481250360578,0,0,0,0,0],"m":[1,1,0,1,0,1,1,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,0,0,0,15,0,17,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,14,13,10,9,8,0,0,0,12,0,0,0,16,0,18,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.02927179,0.007505689,-0.03694912,0,0.0002151429,-0.01976116,-0.03723897,0,-0.03155182,0,0.002171845,0,0,-0.02736368,0.003736286,0,0,0.007505689,-0.03515427,0.007505689,-0.02853124,0.01524558]},{"f":[1,21,-1,25,22,22,-1,22,-1,4,-1,-1,17,17,-1,-1,-1,1,37,-1,-1,21,22,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,1.5,17.094543568094064,17.02942582914998,0,17.078739781790915,0,5.83200138144325,0,0,8.729619742794132,6.8199529586472405,0,0,0,6.431742323097463,5.539999999999999,0,0,4.5,12.613886328427084,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,0,1,0,1,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,8,0,10,0,0,13,14,0,0,0,18,19,0,0,22,23,0,0,0,27,0,0],"r":[26,3,0,17,12,7,0,9,0,11,0,0,16,15,0,0,0,21,20,0,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.03269254,0,0,0,0.0005384438,0,-0.04033622,0,-0.02165308,0.01790456,0,0,-0.007525335,0.04604144,-0.009022261,0,0,-0.001766978,0.03560762,0,0,-0.02014275,0.001126372,-0.03694107,0,0.0007174215,0.03238949]},{"f":[1,18,21,1,-1,15,-1,-1,22,15,4,4,4,4,-1,-1,-1,-1,-1,-1,16,22,-1,-1,-1,-1,18,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,6.022367813028454,0,0,16.759160379664443,13.56045225766691,9.266718101290426,9.037514954341955,8.049190845056867,7.005613627711627,0,0,0,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,0,0],"m":[1,1,1,1,0,1,0,0,1,1,1,1,1,1,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,14,0,0,0,0,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,8,5,0,7,0,0,20,19,18,17,16,15,0,0,0,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.03749273,0,0.003683369,-0.02439378,0,0,0,0,0,0,-0.003096391,0.01841876,-0.02901573,0.01330212,-0.01571817,-0.03749282,0,0,-0.01281117,0.01299959,0.03211413,0.006339923,0,0.001489835,-0.0279138]},{"f":[23,22,16,37,-1,22,-1,-1,24,33,8,22,21,-1,22,-1,16,-1,-1,16,-1,-1,-1,-1,21,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,17.02942582914998,0,0,10.5,35.98606694718995,0.5,14.116600814324018,3.5,0,13.129761635363703,0,6.832890014164741,0,0,3.4594316186372973,0,0,0,0,5.5,0,0,0,0],"m":[1,1,0,1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,12,13,0,15,0,17,0,0,20,0,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,23,22,19,14,0,16,0,18,0,0,21,0,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.04186564,0,0.008329338,-0.01989297,0,0,0,0,0,-0.02970353,0,0.0003803036,0,-0.04187925,-0.004100064,0,0.0389723,0.001321569,0.01145128,-0.03548271,0,-0.01053596,0.02873495,0.01977488,-0.02099798]},{"f":[9,4,16,-1,17,-1,0,22,22,16,-1,-1,18,-1,23,-1,-1,15,17,-1,18,-1,-1,-1,-1,22,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,16.73920708776685,11.67903991803059,12.225508720236002,0,0,2.160964047443681,0,1.5,0,0,7.6653359171851765,6.8199529586472405,0,8.729619742794132,0,0,0,0,17.094543568094064,0,0,0],"m":[1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,0,15,0,0,18,19,0,21,0,0,0,0,26,0,0,0],"r":[28,25,4,0,6,0,24,17,12,11,0,0,14,0,16,0,0,23,20,0,22,0,0,0,0,27,0,0,0],"v":[0,0,0,-0.03900973,0,0.02836884,0,0,0,0,-0.03795652,-0.001703966,0,0.005269366,0,-0.0006369295,-0.02834131,0,0,-0.02331221,0,0.007903221,-0.0372977,-0.02931036,0.01849179,0,0.0008018865,0.03370499,-0.02348772]},{"f":[0,32,9,17,16,2,16,-1,-1,-1,22,21,-1,-1,24,-1,-1,17,16,-1,-1,2,21,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,0.5,2.160964047443681,2.584962500721156,6.434604125275413,2.0,0,0,0,15.782972563647261,3.5,0,0,7.5,0,0,2.69615871138938,5.672425341971495,0,0,1.792481250360578,8.5,0,0,0,0,0,0],"m":[1,1,1,0,0,1,1,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,15,0,0,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,26,17,10,9,8,0,0,0,14,13,0,0,16,0,0,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.03818005,0.009599584,0.001160176,0,0,-0.01767583,0.009599584,0,0.008443298,-0.01841312,0,0,0.005619404,-0.03998831,0,0,-0.01019473,0.009599584,0.0005668617,-0.02319468,-0.02674319,0.01939121]},{"f":[1,4,21,-1,22,4,22,-1,2,22,-1,-1,16,-1,18,22,2,-1,-1,-1,-1,-1,22,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,16.765402642760364,9.037514954341955,7.082728314749225,0,6.16978583919235,15.782972563647261,0,0,2.0,0,1.792481250360578,16.73920708776685,6.515505953718852,0,0,0,0,0,16.807008825964132,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,1,1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,10,0,0,13,0,15,16,17,0,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,4,0,22,21,8,0,12,11,0,0,14,0,20,19,18,0,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03270746,0,0,0,0.01174115,0,0,-0.03505356,0.02998449,0,0.01641878,0,0,0,0.01073702,-0.004477631,-0.01530491,-0.02335988,0.01200775,0,0.03661017,-0.01697719,-0.02348055,0,0.01388563,0.0002607881]},{"f":[36,-1,22,21,22,16,-1,-1,16,17,-1,-1,33,-1,-1,15,2,-1,-1,15,-1,-1,25,22,-1,22,-1,-1,-1],"t":[1.5,0,12.199665042528586,6.5,11.775312925643524,10.681238411777805,0,0,5.78135971352466,4.128693921346326,0,0,35.98606694718995,0,0,9.269126679149418,9.266718101290426,0,0,10.681238411777805,0,0,1.5,13.054613167076562,0,13.589275375459419,0,0,0],"m":[0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,13,0,0,16,17,0,0,20,0,0,23,24,0,26,0,0,0],"r":[2,0,22,15,8,7,0,0,12,11,0,0,14,0,0,19,18,0,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0.01731195,0,0,0,0,-0.03784749,0.009337127,0,0,0.06113893,0.00208247,0,-0.001135055,-0.04093453,0,0,0.0006372041,-0.02105346,0,0.03746657,0.001760726,0,0,0.01187996,0,-0.02643743,0.00122301,-0.006109866]},{"f":[1,18,21,1,-1,15,15,-1,17,-1,-1,22,-1,-1,22,15,15,-1,-1,-1,16,22,-1,-1,-1,-1,18,-1,-1],"t":[4.92309205350118,10.482680779086738,6.5,2.0015923566878984,0,6.022367813028454,3.321928094887362,0,3.5221970596792267,0,0,16.844368517741394,0,0,16.759160379664443,11.054604317960672,8.361943773735241,0,0,0,8.96289600533726,17.078739781790915,0,0,0,0,10.415240582199587,0,0],"m":[1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,1,1,0,0,0,1,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,0,0,21,22,0,0,0,0,27,0,0],"r":[26,25,14,5,0,11,8,0,10,0,0,13,0,0,20,19,18,0,0,0,24,23,0,0,0,0,28,0,0],"v":[0,0,0,0,-0.0363659,0,0,-0.01925466,0,0.04742671,0.001625,0,-0.03816592,0.01154999,0,0,0,-0.003439671,0.011905,-0.01926756,0,0,-0.01235609,0.01243254,0.03035654,0.005938423,0,0.001456313,-0.02718615]},{"f":[16,27,-1,22,-1,22,-1,22,-1,22,33,-1,-1,-1,24,33,8,22,21,-1,-1,16,-1,-1,-1,-1,21,-1,-1],"t":[0.0,0.5,0,16.975010474608837,0,16.978341770814396,0,16.98920034885663,0,17.094543568094064,30.6384348520704,0,0,0,10.5,35.98606694718995,0.5,14.116600814324018,3.5,0,0,3.4594316186372973,0,0,0,0,5.5,0,0],"m":[0,1,0,0,0,0,0,0,0,1,0,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,0,6,0,8,0,10,11,0,0,0,15,16,17,18,19,0,0,22,0,0,0,0,27,0,0],"r":[14,3,0,5,0,7,0,9,0,13,12,0,0,0,26,25,24,21,20,0,0,23,0,0,0,0,28,0,0],"v":[0,0,-0.04065578,0,0.02660504,0,-0.027938,0,0.02466142,0,0,0.01366091,-0.01946498,0.02483497,0,0,0,0,0,-0.0284631,-0.0009673824,0,0.0371967,0.001257417,0.01080124,-0.03444043,0,-0.01016611,0.02744654]},{"f":[0,32,23,-1,21,36,-1,-1,22,-1,24,16,-1,-1,4,3,22,22,16,-1,-1,2,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,2.5,0,6.5,3.5,0,0,8.882387211598338,0,7.5,8.321928094887362,0,0,9.221342183932917,0.9973346627775462,16.765402642760364,16.73920708776685,6.247927513443585,0,0,9.012591414317754,0,0,0,0,0,0,0],"m":[1,1,1,0,0,0,0,0,0,0,0,1,0,0,1,0,1,1,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,0,0,9,0,11,12,0,0,15,16,17,18,19,0,0,22,0,0,0,0,0,0,0],"r":[28,27,4,0,8,7,0,0,10,0,14,13,0,0,26,25,24,21,20,0,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0.001008796,0,0,0.007734436,-0.03733756,0,-0.03371288,0,0,-0.02804208,-0.001402643,0,0,0,0,0,-0.0369591,0.00412716,0,-0.04195115,0.003241869,0.006791216,0.009016185,-0.01645094,-0.02584637,0.01899083]},{"f":[36,-1,24,37,32,33,33,-1,-1,-1,1,-1,-1,-1,25,36,22,22,-1,16,-1,-1,17,17,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,19.167172160526107,0,0,0,7.992857142857143,0,0,0,2.5,5.5,17.094543568094064,17.02942582914998,0,7.774787059601174,0,0,8.729619742794132,6.8199529586472405,0,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,0,20,0,0,23,24,0,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,22,19,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0.01709646,0,0,0,0,0,-0.003657788,0.00903672,-0.01527097,0,0.0006681056,-0.03995246,0.02224554,0,0,0,0,0.0008774713,0,-0.02764402,0.006217401,0,0,-0.005967939,0.04481716,-0.01709764,0.03368201,-0.02593326]},{"f":[9,4,16,-1,17,-1,0,25,0,-1,22,22,-1,33,-1,-1,22,-1,-1,15,-1,-1,-1,22,2,-1,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,1.5,2.5,0,16.73920708776685,12.199665042528586,0,21.80651201897376,0,0,16.765402642760364,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,0,0,0,0],"m":[1,1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,0,11,12,0,14,0,0,17,0,0,20,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,22,19,10,0,16,13,0,15,0,0,18,0,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03793159,0,0.02656791,0,0,0,-0.02725224,0,0,-0.007877083,0,0.004830291,-0.01395248,0,-0.01395248,0.007264611,0,-0.0008221642,-0.02808961,0.01727183,0,0,0.02346592,0.0002589914,0.03273515,-0.02265144]},{"f":[0,32,23,-1,16,16,27,-1,22,-1,-1,-1,18,4,18,22,-1,-1,-1,15,-1,22,18,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,1.5,0,6.523561956057013,5.700439718141092,0.5,0,8.882387211598338,0,0,0,10.45532575654158,4.0437314206251695,2.160964047443681,16.73920708776685,0,0,0,11.054604317960672,0,15.782972563647261,9.265614570460752,0,0,0,0,0,0],"m":[1,1,1,0,0,1,0,0,0,0,0,0,1,1,0,1,0,0,0,1,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,0,13,14,15,16,0,0,0,20,0,22,23,0,0,0,0,0,0],"r":[28,27,4,0,12,11,8,0,10,0,0,0,26,19,18,17,0,0,0,21,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0.001314337,0,0,0,-0.03683438,0,-0.03332304,-0.0004380951,-0.03663239,0,0,0,0,0.005932363,-0.008496944,-0.01519748,0,0.008806047,0,0,-0.04249566,0.009029194,0.005896337,-0.02125673,-0.02544133,0.01824841]},{"f":[22,22,22,22,22,10,-1,22,-1,2,-1,-1,-1,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,0,6.515505953718852,0,0,0,0,16.903290731167747,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,1,0,1,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,10,0,0,0,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,14,13,12,7,0,9,0,11,0,0,0,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03716428,0,0.0005899204,0,-0.03705176,0.004892647,0.03329184,-0.01729782,0,0.04992859,0.00295869,0,-0.03563697,0,-0.01896676,0.01634258,0,0.04366789,0,0,0.03543847,-0.03006035,-0.03616866]},{"f":[1,22,18,1,21,-1,17,22,3,-1,-1,-1,22,-1,-1,22,-1,-1,16,-1,2,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,10.482680779086738,2.9733542319749215,4.5,0,5.83200138144325,16.725912049439096,0.8516483516483516,0,0,0,10.029977181607594,0,0,8.658207063149977,0,0,2.0,0,1.292481250360578,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,0,0,16,0,0,19,0,21,0,0,24,0,0,27,0,0],"r":[26,23,18,15,6,0,12,11,10,0,0,0,14,0,0,17,0,0,20,0,22,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.02496745,0,0,0,0.02276873,0.0006980129,-0.01957281,0,-0.03273084,-0.003985078,0,0.001181061,-0.02268705,0,0.01780436,0,0.02586372,-0.003349068,0,-0.01616854,0.02322389,0,0.001226166,-0.02681659]},{"f":[36,-1,24,37,32,33,33,-1,-1,-1,1,-1,-1,-1,25,36,22,22,-1,22,-1,1,-1,-1,17,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,19.167172160526107,0,0,0,7.992857142857143,0,0,0,2.5,5.5,17.094543568094064,17.02942582914998,0,17.078739781790915,0,3.966632962588473,0,0,8.729619742794132,0,0,0,0],"m":[0,0,0,1,0,0,0,0,0,0,0,0,0,0,1,1,1,1,0,1,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,0,20,0,22,0,0,25,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,24,19,0,21,0,23,0,0,26,0,0,0,0],"v":[0,0.01661246,0,0,0,0,0,-0.003599391,0.008643721,-0.01477682,0,0.0004175491,-0.03906081,0.02130811,0,0,0,0,0.0008321283,0,-0.03653109,0,0.01288627,-0.02185493,0,0.03364766,-0.01678479,0.03303076,-0.02503573]},{"f":[1,21,16,-1,-1,22,1,22,22,4,-1,2,-1,-1,-1,-1,22,21,-1,-1,22,-1,0,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,6.5,1.584962500721156,0,0,16.765402642760364,1.13125,15.676331898425197,11.722778233171695,5.169368191286958,0,6.434604125275413,0,0,0,0,15.212004812197344,7.5,0,0,15.782972563647261,0,4.5,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,8,9,10,0,12,0,0,0,0,17,18,0,0,21,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,25,16,15,14,11,0,13,0,0,0,0,20,19,0,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0.01329827,-0.03444596,0,0,0,0,0,0.02182062,0,-0.03592064,0.004812239,-0.03425778,0.0523529,0,0,-0.0366883,0.009704955,0,0.02773332,0,-0.03141702,-0.004063269,0.03000068,0,0.01348583,0.0002424909]},{"f":[23,21,15,16,23,-1,21,-1,-1,2,22,-1,-1,22,21,-1,-1,-1,22,33,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,1.5,0,6.5,0,0,5.065928480304397,12.613886328427084,0,0,16.765402642760364,6.5,0,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,18,9,6,0,8,0,0,13,12,0,0,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.002125205,0,-0.02203343,-0.0007829523,0,0,0.007681896,0.07611699,0,0,0.02733753,-0.001467127,0.02761282,0,0,-0.03208947,0.0284722,0,0,-0.04059994,0.002950779,-0.003783727,0.01911772,-0.01981439]},{"f":[4,16,-1,0,25,23,-1,21,-1,18,22,22,-1,-1,-1,21,-1,-1,15,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0.5,0,4.5,0,8.729619742794132,16.765402642760364,16.73920708776685,0,0,0,8.5,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,1,1,0,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,5,6,0,8,0,10,11,12,0,0,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,21,18,7,0,9,0,15,14,13,0,0,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03727764,0,0,0,0.007426856,0,-0.02991588,0,0,0,0.0003004174,-0.02058601,0.007519704,0,-0.04265443,0.007519704,0,-0.0008667786,-0.02675799,0.01688877,0,0,0,0.008609003,0.05568028,0.0001737734,0.03153949]},{"f":[22,22,22,22,22,10,-1,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,10,9,8,7,0,0,0,0,12,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03646398,-5.729218e-05,0.03130665,-0.01676685,0,0.04844019,0,-0.03382156,0,-0.005869855,0.02223841,0,-0.03460722,0,-0.01789892,0.01464121,0,0.04265708,0,0,0.03413462,-0.02906111,-0.03490866]},{"f":[0,32,23,1,22,15,18,17,22,3,-1,22,-1,18,-1,-1,-1,17,-1,-1,-1,-1,16,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.5,4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,6.199638962254484,16.73920708776685,0.8516483516483516,0,12.921158001713904,0,3.39067985676233,0,0,0,7.335383440499621,0,0,0,0,7.924812503605781,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,12,0,14,0,0,0,18,0,0,0,0,23,0,0,0,0,0,0],"r":[28,27,26,25,22,21,20,17,16,11,0,13,0,15,0,0,0,19,0,0,0,0,24,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.008735198,0,-2.308258e-05,0,-0.03493561,0.008735198,-0.02711705,0,-0.03183461,-0.008406527,0.005657192,-0.03662844,0,-0.01564824,0.008735198,0.001027786,-0.01937589,-0.02454708,0.01762184]},{"f":[4,16,-1,17,-1,0,25,0,-1,22,22,-1,33,-1,-1,22,-1,0,-1,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,0,10.5,1.5,2.5,0,16.73920708776685,12.199665042528586,0,21.80651201897376,0,0,16.765402642760364,0,6.5,0,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,0,6,7,8,0,10,11,0,13,0,0,16,0,18,0,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,5,0,21,20,9,0,15,12,0,14,0,0,17,0,19,0,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03670403,0,0.02349386,0,0,0,-0.02638664,0,0,-0.007642003,0,0.004293565,-0.01355315,0,-0.01355315,0,-0.01355315,0.006912449,-0.01652606,0.0162454,0,0,0,0.008259998,0.05349254,0.0002081394,0.03038432]},{"f":[36,-1,22,21,22,16,-1,-1,16,17,-1,-1,33,-1,-1,15,22,21,-1,-1,0,1,-1,-1,-1,15,-1,-1,-1],"t":[1.5,0,12.199665042528586,6.5,11.775312925643524,10.681238411777805,0,0,5.78135971352466,4.128693921346326,0,0,35.98606694718995,0,0,9.269126679149418,10.27595502635321,9.5,0,0,8.5,1.99834912080135,0,0,0,10.681238411777805,0,0,0],"m":[0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,0,0,9,10,0,0,13,0,0,16,17,18,0,0,21,22,0,0,0,26,0,0,0],"r":[2,0,28,15,8,7,0,0,12,11,0,0,14,0,0,25,20,19,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0.01576173,0,0,0,0,-0.03595106,0.008220654,0,0,0.06180165,0.001541605,0,-0.00115046,-0.04011966,0,0,0,0.0005735295,0.02754618,0,0,-0.005525349,-0.03227311,0.008919784,0,0.03452866,0.001671597,0.0007395262]},{"f":[25,0,21,-1,33,11,36,-1,-1,-1,25,22,2,22,36,15,-1,-1,-1,-1,-1,-1,-1,21,16,-1,-1,-1,-1],"t":[6.5,10.5,2.5,0,21.16864017939033,0.5,4.5,0,0,0,1.5,16.838461650489297,6.977234485717126,16.73920708776685,3.5,7.118941072723508,0,0,0,0,0,0,0,8.5,2.807354922057604,0,0,0,0],"m":[1,1,0,0,1,1,1,0,0,0,0,1,1,1,0,1,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,13,14,15,16,0,0,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,10,9,8,0,0,0,22,21,20,19,18,17,0,0,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03035081,0,0,0,0.001537552,-0.02012441,-0.03121628,0,0,0,0,0,0,-0.005252742,-0.03704239,0.00215211,-0.01553366,-0.03344279,0.00215211,-0.02881779,0,0,0.03126858,0.00215211,0.03325288,-0.01752567]},{"f":[22,22,22,22,16,-1,-1,-1,22,2,-1,-1,32,-1,22,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,3.245926548164837,0,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,0,0,0,9,10,0,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,8,7,6,0,0,0,12,11,0,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,-0.03342047,0.0001751879,-0.03575353,0,0,-0.01602722,0.04106375,0,-0.03312424,0,-0.005891778,0.02088217,0,-0.03411187,0,-0.01741116,0.01383589,0,0.04156113,0,0,0.03342582,-0.02811128,-0.03452368]},{"f":[9,4,16,-1,17,-1,0,22,22,16,-1,-1,18,-1,-1,15,18,17,-1,-1,-1,-1,-1,22,2,-1,-1,-1,-1],"t":[0.5,3.245926548164837,2.0,0,0.5,0,10.5,16.73920708776685,11.67903991803059,12.225508720236002,0,0,2.160964047443681,0,0,7.451211111832329,8.729619742794132,6.781300095563516,0,0,0,0,0,17.094543568094064,3.39067985676233,0,0,0,0],"m":[1,1,0,0,0,0,1,1,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,0,7,8,9,10,0,0,13,0,0,16,17,18,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,6,0,22,15,12,11,0,0,14,0,0,21,20,19,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.03612379,0,0.02204615,0,0,0,0,-0.03653742,-0.00106871,0,0.004026285,-0.00432336,0,0,0,-0.02031299,0.006390081,-0.03437425,-0.02537835,0.01511048,0,0,0.02125922,0.0003183325,0.02926431,-0.02132668]},{"f":[1,4,21,-1,22,4,-1,15,-1,-1,22,-1,-1,-1,3,-1,23,33,33,33,-1,-1,-1,22,1,-1,-1,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,16.765402642760364,9.037514954341955,0,1.584962500721156,0,0,16.807008825964132,0,0,0,0.0339080459770115,0,0.5,73.23848001446603,21.16864017939033,19.167172160526107,0,0,0,14.116600814324018,5.489664082687339,0,0,0,0],"m":[0,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,0,0,11,0,0,0,15,0,17,18,19,20,0,0,0,24,25,0,0,0,0],"r":[14,13,4,0,10,7,0,9,0,0,12,0,0,0,16,0,28,23,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.0307465,0,0,-0.003405211,0,-0.001765521,0.02048834,0,0.03373604,-0.01420917,-0.02148532,0,0.01300618,0,0,0,0,-0.01576698,0.01834916,-0.01550049,0,0,0.003353618,0.02570128,0.1075703,-0.0007620793]},{"f":[36,-1,22,33,-1,-1,15,18,20,-1,-1,-1,15,2,-1,-1,15,15,33,-1,-1,18,-1,-1,18,-1,15,-1,-1],"t":[1.5,0,12.613886328427084,8.532371382375128,0,0,5.044394119358453,3.39067985676233,1.5,0,0,0,6.169925001442312,4.80643424864552,0,0,6.857980995127572,6.820178962415188,21.257266042509173,0,0,1.292481250360578,0,0,6.864066550153108,0,9.756556322524087,0,0],"m":[0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,1,0,0,0,0,0,1,0,1,0,0],"l":[1,0,3,4,0,0,7,8,9,0,0,0,13,14,0,0,17,18,19,0,0,22,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,12,11,10,0,0,0,16,15,0,0,24,21,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0.01541521,0,0,-0.03884941,-0.0007086158,0,0,0,-0.002280015,0.02577748,0.02119771,0,0,-0.04271015,-0.005538247,0,0,0,0.01500467,-0.00729789,0,-0.008228481,0.122688,0,0.0006314057,0,-0.04561954,-0.001444666]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,4,-1,-1,-1,32,-1,-1,4,-1,22,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.9534452978042594,0,0,0,2.69615871138938,0,0,5.83200138144325,0,17.074218326240924,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,0,0,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,14,13,10,9,8,0,0,0,12,0,0,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002766423,0.03031757,-0.01562612,0,-0.02706208,0.04061381,-0.02174086,0,-0.02019812,0.03073861,0,-0.02140525,0,-0.02297876,0.01378555,0,0.04054511,0,0,0.03262991,-0.02693838,-0.03372325]},{"f":[23,21,15,16,23,-1,21,-1,-1,2,22,-1,-1,22,21,-1,-1,-1,22,33,-1,-1,2,16,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,1.5,0,6.5,0,0,5.065928480304397,12.613886328427084,0,0,16.765402642760364,6.5,0,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,10,11,0,0,14,15,0,0,0,19,20,0,0,23,24,0,0,0,0,0],"r":[28,27,18,9,6,0,8,0,0,13,12,0,0,17,16,0,0,0,22,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,0.00187755,0,-0.02094084,-0.000621669,0,0,0.007334077,0.06921221,0,0,0.02629596,-0.00122017,0.02579276,0,0,-0.03107945,0.02620881,0,0,-0.03989516,0.003599346,-0.003719058,0.01815634,-0.01883694]},{"f":[32,32,37,25,22,22,22,22,16,-1,21,-1,-1,-1,2,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,2.5,0,0,0,4.128693921346326,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,11,0,0,0,15,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,14,13,10,0,12,0,0,0,16,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.03045751,0,-0.02942563,0.0003410311,-0.03481996,0,-0.0153244,0.009290385,0,-0.03816857,0,-0.03081084,0.008635513,0.009256029,-0.02682189,0.01865143,-0.0108869,0,0.01800078,-0.02030181]},{"f":[1,22,15,18,17,22,3,17,-1,-1,15,16,-1,-1,1,-1,-1,-1,17,-1,-1,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,6.199638962254484,16.73920708776685,0.8516483516483516,1.792481250360578,0,0,5.78135971352466,3.321928094887362,0,0,1.987429289754871,0,0,0,7.335383440499621,0,0,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,1,0,0,0,0,1,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,22,21,18,17,10,9,0,0,14,13,0,0,16,0,0,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.03332434,0.002419703,0,0,-0.00562872,0.008798187,0,-0.004553145,-0.03949348,-0.02612626,0,-0.03090538,-0.008088934,0.005194101,-0.03617331,0,-0.01499009,0.02115338,0,0.001198715,-0.02556]},{"f":[36,-1,24,37,32,33,20,-1,-1,-1,1,-1,-1,-1,25,36,22,16,18,-1,-1,-1,23,22,-1,-1,-1,-1,-1],"t":[1.5,0,7.5,8.05,6.337096134072842,21.16864017939033,1.5,0,0,0,7.992857142857143,0,0,0,2.5,5.5,12.199665042528586,10.462502272597265,5.491496787347155,0,0,0,0.5,14.116600814324018,0,0,0,0,0],"m":[0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,0,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,5,6,7,0,0,0,11,0,0,0,15,16,17,18,19,0,0,0,23,24,0,0,0,0,0],"r":[2,0,14,13,10,9,8,0,0,0,12,0,0,0,28,27,22,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0.0151101,0,0,0,0,0,-0.003926233,0.008031238,-0.01297366,0,4.835463e-05,-0.03719461,0.01974195,0,0,0,0,0,-0.001984422,-0.0229716,0.005120954,0,0,0.008744671,0.08830057,0.0006895943,0.03172311,-0.02289858]},{"f":[1,21,-1,24,36,4,33,33,21,-1,-1,-1,-1,1,-1,-1,1,-1,-1,21,22,-1,17,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,7.5,3.5,4.128693921346326,21.16864017939033,19.167172160526107,3.5,0,0,0,0,6.241071428571429,0,0,7.992857142857143,0,0,3.5,14.03140051979143,0,2.453445297804259,0,0,0,6.118903736861568,0,0],"m":[1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,9,0,0,0,0,14,0,0,17,0,0,20,21,0,23,0,0,0,27,0,0],"r":[26,3,0,19,16,13,12,11,10,0,0,0,0,15,0,0,18,0,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,-0.02959937,0,0,0,0,0,0,-0.04328795,-0.002594844,0.01766977,-0.01330622,0,-0.01123939,0.0192136,0,-0.006471855,-0.03025158,0,0,-0.02058904,0,8.650688e-05,0.08909283,0.0002672712,0,0.0004468031,0.02934524]},{"f":[22,22,22,22,22,10,-1,22,-1,-1,-1,-1,22,-1,24,-1,-1,22,-1,4,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,17.02942582914998,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,16.73920708776685,0,0,0,0,16.903290731167747,0,8.5,0,0,17.078739781790915,0,5.83200138144325,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,8,0,0,0,0,13,0,15,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,12,11,10,7,0,9,0,0,0,0,14,0,16,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.03459883,0,0.0005389407,-0.01168945,0.02892919,-0.01486639,0,0.04579073,0,0.007121963,-0.02775931,0,-0.03275164,0,-0.01552009,0.01169231,0,0.0400122,0,0,0.03171197,-0.02567927,-0.03339427]},{"f":[9,16,16,-1,-1,21,22,-1,-1,17,4,22,22,-1,-1,-1,1,-1,22,-1,-1,17,19,-1,-1,17,-1,-1,-1],"t":[0.5,7.930737337562887,7.679480099505446,0,0,4.5,16.844368517741394,0,0,1.792481250360578,5.108372929097653,12.199665042528586,9.40390385294334,0,0,0,2.782861472075366,0,16.725912049439096,0,0,5.065928480304397,0.5,0,0,6.219136028062415,0,0,0],"m":[1,1,1,0,0,0,1,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,0,0,6,7,0,0,10,11,12,13,0,0,0,17,0,19,0,0,22,23,0,0,26,0,0,0],"r":[28,5,4,0,0,9,8,0,0,21,16,15,14,0,0,0,18,0,20,0,0,25,24,0,0,27,0,0,0],"v":[0,0,0,-0.0002254485,-0.02604705,0,0,-0.0297874,0.004244366,0,0,0,0,0.03017014,-0.0008129859,0.03137929,0,0.01396942,0,-0.03410699,0.006646086,0,0,-0.002872328,-0.02871541,0,0.0276251,0.0006187469,-0.02074276]},{"f":[1,21,-1,32,33,33,33,21,22,-1,0,-1,-1,-1,15,-1,-1,37,33,-1,-1,-1,1,-1,-1,-1,2,-1,-1],"t":[9.107199999999999,2.5,0,10.463013322934223,21.225056427472516,17.737700742794445,13.660441030622824,3.5,14.03140051979143,0,2.5,0,0,0,7.266786540694901,0,0,7.92,20.752541988539424,0,0,0,8.980632551528075,0,0,0,6.118903736861568,0,0],"m":[1,0,0,1,1,1,1,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0],"l":[1,2,0,4,5,6,7,8,9,0,11,0,0,0,15,0,0,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,3,0,25,22,17,14,13,10,0,12,0,0,0,16,0,0,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,-0.02914395,0,0,0,0,0,0,-0.02014119,0,-0.008700306,0.07864425,-4.540569e-05,0,-0.03124685,-0.002390379,0,0,-0.03399046,0.01579079,0.04109544,0,-0.001328528,-0.03205171,0.01555589,0,0.0004704381,0.02890218]},{"f":[23,21,15,16,-1,2,-1,-1,22,33,-1,-1,2,16,-1,-1,22,16,-1,18,4,-1,-1,-1,17,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,0,0,11.683871801979876,73.23848001446603,0,0,6.247802615595391,11.98370619265935,0,0,15.782972563647261,11.031356596255709,0,9.265614570460752,6.749860669831508,0,0,0,2.453445297804259,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,0,0,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,0,0,13,14,0,0,17,18,0,20,21,0,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,12,11,0,0,16,15,0,0,24,19,0,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.0005990482,0,0.03877404,0.001554693,0,0,-0.03079736,0.02424184,0,0,-0.03947643,0.003284262,0,0,0.002016877,0,0,0.0004564276,-0.04186025,-0.003473469,0,-0.006637059,0.02020296,0.01677374,-0.01872448]},{"f":[22,22,22,22,22,22,22,10,-1,-1,-1,-1,4,15,-1,-1,-1,-1,32,-1,-1,-1,25,-1,17,2,-1,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0.5,0,0,0,0,3.9534452978042594,7.6653359171851765,0,0,0,0,2.69615871138938,0,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,14,0,0,0,0,19,0,0,0,23,0,25,26,0,0,0],"r":[22,21,18,17,12,11,10,9,0,0,0,0,16,15,0,0,0,0,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,0,-0.03384561,-6.219846e-05,0.02796024,-0.01420096,0,0,0.01173548,-0.04562418,0.0386632,-0.02157325,0,-0.01938285,0.02931987,-0.01040311,0,0.03933645,0,0,0.03064014,-0.02489054,-0.03289275]},{"f":[1,4,21,-1,15,2,22,-1,16,-1,-1,16,22,-1,-1,18,-1,-1,15,-1,22,-1,16,-1,-1,-1,3,-1,-1],"t":[2.01685992705101,9.266718101290426,3.5,0,9.269126679149418,6.434604125275413,7.082728314749225,0,9.250298417906333,0,0,2.0,9.024458927812285,0,0,1.292481250360578,0,0,10.342074667999139,0,11.722778233171695,0,11.031356596255709,0,0,0,0.0339080459770115,0,0],"m":[0,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,9,0,0,12,13,0,0,16,0,0,19,0,21,0,23,0,0,0,27,0,0],"r":[26,25,4,0,18,11,8,0,10,0,0,15,14,0,0,17,0,0,20,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.0297471,0,0,0,0.005446147,0,-0.03597124,0.01151041,0,0,-0.009590379,0.02101905,0,-0.0005372139,-0.0194222,0,0.05442632,0,0.0142029,0,0.004948389,-0.03325814,-0.02087975,0,0.01219149,0.0002699354]},{"f":[36,-1,22,33,-1,-1,15,18,1,21,-1,-1,22,-1,-1,-1,16,27,17,-1,23,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,12.613886328427084,8.532371382375128,0,0,5.044394119358453,3.39067985676233,6.241071428571429,7.5,0,0,15.782972563647261,0,0,0,6.108524456778169,0.5,7.67595272073847,0,4.5,0,0,0,15.676331898425197,0,15.717370311433935,0,0],"m":[0,0,0,0,0,0,0,1,1,1,0,0,1,0,0,0,0,1,1,0,0,0,0,0,1,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,0,13,0,0,0,17,18,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,16,15,12,11,0,0,14,0,0,0,24,23,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.01488951,0,0,-0.03820586,-0.0006625095,0,0,0,0,-0.03913077,-0.001805941,0,0.02311472,-0.008063909,0.01970324,0,0,0,-0.03744182,0,0.0338147,-0.03602174,0.001631269,0,0.005500774,0,-0.04863694,0.000902261]},{"f":[22,22,22,22,16,-1,37,-1,-1,-1,32,-1,2,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,11.73,0,0,0,6.337096134072842,0,4.128693921346326,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,10,9,6,0,8,0,0,0,12,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.03035862,0,-0.0001485749,0.01899811,-0.03375386,0,0.03242157,0,-0.02567146,0.007893181,0,-0.03673554,0,-0.02998947,0.008397111,0,0.04289446,0,0,0,-0.01223014,0.03694283,-0.01767824,-0.03306512]},{"f":[23,21,15,4,-1,-1,22,33,-1,-1,2,-1,22,16,15,-1,-1,18,4,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,4.0437314206251695,0,0,11.683871801979876,73.23848001446603,0,0,3.903677461028802,0,15.782972563647261,11.031356596255709,10.462502272597265,0,0,9.265614570460752,5.169368191286958,0,0,0,4.208926257442949,16.725912049439096,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,13,14,15,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,27,6,5,0,0,10,9,0,0,12,0,22,17,16,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.000954109,0.002807875,0,0,-0.0305722,0.0227498,0,-0.02905386,0,0,0,-0.01173313,0.01044533,0,0,0.005051833,-0.04199783,-0.003314509,0,0,-0.03508307,0.004414403,0.02159654,0.01599363,-0.0184517]},{"f":[1,22,15,18,17,22,3,4,-1,-1,15,15,-1,-1,1,-1,-1,-1,2,-1,-1,-1,-1,16,-1,-1,18,-1,-1],"t":[4.031939779270633,16.759160379664443,13.56045225766691,10.482680779086738,5.83200138144325,16.73920708776685,0.8516483516483516,3.753897320099348,0,0,5.78135971352466,2.584962500721156,0,0,1.987429289754871,0,0,0,5.065928480304397,0,0,0,0,7.924812503605781,0,0,10.415240582199587,0,0],"m":[1,1,1,1,1,1,0,1,0,0,1,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,22,21,18,17,10,9,0,0,14,13,0,0,16,0,0,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.00454647,0.04033168,0,0,-0.003402182,0.01209681,0,-0.004514952,-0.03867778,-0.02527137,0,-0.03548722,-0.007789007,0.004439769,-0.03470454,0,-0.01421519,0.01986672,0,0.001108231,-0.02520512]},{"f":[36,-1,22,21,-1,-1,22,2,25,21,-1,15,16,17,-1,-1,-1,-1,33,-1,33,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,0,15.676331898425197,6.321815375530864,0.5,4.5,0,8.005624549193879,7.651051691178929,7.67595272073847,0,0,0,0,19.167172160526107,0,21.16864017939033,0,0,0,15.717370311433935,0,15.756821747580634,0,0],"m":[0,0,1,0,0,0,0,1,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,0,12,13,14,0,0,0,0,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,18,11,0,17,16,15,0,0,0,0,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.0144912,0,0,-0.02526506,-0.0007067007,0,0,0,0,0.05785902,0,0,0,-0.01237391,0.02727261,0.02820652,-0.01798242,0,-0.03044807,0,0.01480105,-0.009015611,-0.01362962,0,-0.04744306,0,0.02700996,-0.000275287]},{"f":[1,21,-1,24,36,-1,22,22,-1,-1,-1,21,22,1,-1,-1,0,-1,-1,-1,2,16,-1,21,-1,33,-1,-1,-1],"t":[9.107199999999999,2.5,0,7.5,3.5,0,16.765402642760364,16.725912049439096,0,0,0,3.5,14.03140051979143,6.069196428571429,0,0,2.5,0,0,0,6.118903736861568,2.807354922057604,0,5.5,0,22.152182868153965,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,0,4,5,0,7,8,0,0,0,12,13,14,0,0,17,0,0,0,21,22,0,24,0,26,0,0,0],"r":[20,3,0,11,6,0,10,9,0,0,0,19,16,15,0,0,18,0,0,0,28,23,0,25,0,27,0,0,0],"v":[0,0,-0.02818292,0,0,-0.001124767,0,0,-0.02180734,0.01282015,-0.03191553,0,0,0,-0.0331883,0.005449223,0,-0.008514059,0.06932958,0.0001982928,0,0,0.02081909,0,-0.02956655,0,-0.002693192,0.01096304,0.0276481]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,4,15,-1,-1,-1,-1,-1,25,-1,22,-1,-1,36,-1,18,2,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,3.9534452978042594,7.6653359171851765,0,0,0,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,0,0,0,18,0,20,0,0,23,0,25,26,0,0,0],"r":[22,17,16,15,10,9,8,0,0,0,14,13,0,0,0,0,0,19,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002942564,0.02709838,-0.01374291,0,0,0.01176798,-0.04466282,0.03702947,-0.02104984,0.01341006,0,-0.03593011,0,-0.02975049,0.007513909,0,0.04190738,0,0,0.0309028,-0.01202722,-0.03258933]},{"f":[23,22,22,22,22,-1,-1,22,17,-1,22,-1,-1,22,-1,-1,25,-1,22,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0,0,16.903290731167747,3.39067985676233,0,16.844368517741394,0,0,16.978341770814396,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,0,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,7,6,0,0,13,10,0,12,0,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,-3.52445e-05,-0.03300605,0,0,-0.01457058,0,0.005093953,0.05099494,0,-0.009183819,0.01375261,0,-0.03414116,0,-0.02933781,0.007421387,0,0.0422119,0,0,0.03518917,-0.01151126,-0.03208034,-0.01827722]},{"f":[4,16,-1,0,25,-1,-1,-1,32,19,20,-1,-1,-1,33,-1,2,16,-1,22,17,21,-1,-1,-1,-1,1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0,0,0,6.208926257442949,0.5,1.5,0,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,13.747003318603246,5.918419679874358,4.5,0,0,0,0,8.992857142857144,0,0],"m":[1,0,0,1,1,0,0,0,0,1,1,0,0,0,0,0,0,1,0,1,1,1,0,0,0,0,1,0,0],"l":[1,2,0,4,5,0,0,0,9,10,11,0,0,0,15,0,17,18,0,20,21,22,0,0,0,0,27,0,0],"r":[8,3,0,7,6,0,0,0,14,13,12,0,0,0,16,0,26,19,0,25,24,23,0,0,0,0,28,0,0],"v":[0,0,-0.0345645,0,0,-0.0007873226,-0.01430889,0.01534646,0,0,0,-0.003207907,0.01700523,-0.02698362,0,0.02531228,0,0,-2.235858e-05,0,0,0,-0.008603364,0.03685534,-0.02775145,0.06462431,0,-0.0004173437,0.0217643]},{"f":[17,19,21,-1,22,22,24,-1,4,2,22,33,-1,22,-1,-1,-1,-1,-1,-1,-1,4,-1,-1,17,22,-1,-1,-1],"t":[2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,4.5,0,7.856894618075133,7.543917866187989,15.782972563647261,13.249860669831508,0,13.64082212194147,0,0,0,0,0,0,0,4.128693921346326,0,0,2.69615871138938,16.752707246237605,0,0,0],"m":[0,1,0,0,1,1,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,3,0,5,6,7,0,9,10,11,12,0,14,0,0,0,0,0,0,0,22,0,0,25,26,0,0,0],"r":[24,21,4,0,20,19,8,0,18,17,16,13,0,15,0,0,0,0,0,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0,-0.02612836,0,0,0,-0.03591819,0,0,0,0,0.02135707,0,0.003933521,-0.03595072,-0.01079651,0.02500678,-0.007310479,0.0209034,-0.02511985,0,0.007800726,0.06144135,0,0,-0.04062761,0.002623101,-0.0002987259]},{"f":[4,16,-1,0,25,23,15,-1,-1,21,22,-1,-1,18,-1,21,-1,-1,15,-1,-1,-1,22,2,15,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,10.5,1.5,0.5,6.857980995127572,0,0,4.5,17.069460217922263,0,0,8.729619742794132,0,8.5,0,0,2.807354922057604,0,0,0,17.094543568094064,3.39067985676233,6.820178962415188,0,0,0,0],"m":[1,0,0,1,1,0,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,1,0,0,0,0],"l":[1,2,0,4,5,6,7,0,0,10,11,0,0,14,0,16,0,0,19,0,0,0,23,24,25,0,0,0,0],"r":[22,3,0,21,18,9,8,0,0,13,12,0,0,15,0,17,0,0,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,-0.03407333,0,0,0,0,0.006434462,-0.01134249,0,0,-0.03729104,0.006434462,0,-0.0005879263,0,-0.04057551,0.006434462,0,0.0002733105,-0.02273548,0.01450832,0,0,0,0.005917388,0.04730994,0.0002249151,0.02664937]},{"f":[22,22,22,22,22,22,22,-1,-1,-1,23,16,-1,-1,-1,-1,32,-1,-1,25,-1,-1,36,-1,18,2,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0],"m":[1,1,1,1,1,1,1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,11,12,0,0,0,0,17,0,0,20,0,0,23,0,25,26,0,0,0],"r":[22,19,16,15,10,9,8,0,0,0,14,13,0,0,0,0,18,0,0,21,0,0,24,0,28,27,0,0,0],"v":[0,0,0,0,0,0,0,-0.0002689262,0.02652874,-0.01289692,0,0,0.01255175,-0.03653024,0.03988442,-0.01983948,0,-0.01911446,0.02693024,0,-0.02439305,-0.0004113575,0,0.03957781,0,0,0.02952421,-0.01125484,-0.03117568]},{"f":[25,0,21,-1,33,11,22,22,-1,-1,-1,-1,25,2,22,36,32,-1,-1,-1,-1,-1,-1,21,16,-1,-1,-1,-1],"t":[6.5,10.5,2.5,0,21.16864017939033,0.5,11.67903991803059,10.27595502635321,0,0,0,0,1.5,4.699371845969097,15.676331898425197,3.5,6.337096134072842,0,0,0,0,0,0,8.5,2.807354922057604,0,0,0,0],"m":[1,1,0,0,1,1,0,1,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,0,5,6,7,8,0,0,0,0,13,14,15,16,17,0,0,0,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,12,11,10,9,0,0,0,0,22,21,20,19,18,0,0,0,0,0,0,27,26,0,0,0,0],"v":[0,0,0,-0.02738359,0,0,0,0,0.0009149088,-0.02071366,0.001871156,-0.02812344,0,0,0,0,0,-0.007981284,-0.04038152,0.001871156,-0.0209522,0.001679905,-0.0241815,0,0,0.02876611,0.001871156,0.03091676,-0.01585394]},{"f":[22,22,22,22,22,22,-1,-1,23,16,-1,-1,-1,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,0,0,5.5,6.0,0,0,0,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,9,10,0,0,0,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,14,13,8,7,0,0,12,11,0,0,0,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0.0001397589,-0.01247359,0,0,0.01216983,-0.03579127,0.03879286,-0.01911321,0.01186519,0,-0.03404331,0,-0.02917847,0.006650592,0,0.03857237,0,0,0,-0.01275578,0.03537248,-0.01576702,-0.03077856]},{"f":[32,15,22,20,-1,-1,1,16,-1,-1,22,-1,-1,15,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,-1,-1,-1],"t":[6.208926257442949,5.044394119358453,12.613886328427084,0.5,0,0,6.241071428571429,7.417852514885898,0,0,15.782972563647261,0,0,6.169925001442312,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,0,0,0],"m":[0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,14,0,0,17,18,0,20,21,0,0,0,25,26,0,0,0],"r":[16,13,6,5,0,0,10,9,0,0,12,0,0,15,0,0,24,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,-0.03800288,0.0001975778,0,0,-0.03606196,0.003321249,0,0.01774582,-0.008016322,0,-0.03009095,-0.00198731,0,0,-0.006672178,0,0,0.00967076,-0.01403249,0.03567468,0,0,-0.0002387131,-0.01990208,0.01798672]},{"f":[4,18,16,-1,-1,23,1,22,-1,15,17,-1,-1,-1,25,-1,-1,21,22,35,-1,-1,17,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.160964047443681,2.0,0,0,0.5,5.015625,11.67903991803059,0,6.820178962415188,3.5221970596792267,0,0,0,1.5,0,0,8.5,17.094543568094064,1.5,0,0,8.729619742794132,0,0,1.292481250360578,0,0,0],"m":[1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,1,0,0,0],"l":[1,2,3,0,0,6,7,8,0,10,11,0,0,0,15,0,0,18,19,20,0,0,23,0,0,26,0,0,0],"r":[28,5,4,0,0,17,14,9,0,13,12,0,0,0,16,0,0,25,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,-0.02965453,0.001710067,0,0,0,-0.01496299,0,0,0.0374878,-0.02034454,0.07531467,0,0.0004620555,-0.02440106,0,0,0,-0.004668929,-0.04032762,0,0.03136271,-0.03021892,0,0.02509902,-0.02314448,0.0007884743]},{"f":[32,22,33,22,22,18,-1,-1,-1,22,-1,-1,1,-1,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,-1,-1,-1],"t":[6.208926257442949,16.838461650489297,13.660441030622824,15.782972563647261,14.03140051979143,1.292481250360578,0,0,0,16.725912049439096,0,0,8.245481927710843,0,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,0,0,0],"m":[0,1,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,0,13,0,0,0,17,18,0,20,21,0,0,0,25,26,0,0,0],"r":[16,15,12,9,8,7,0,0,0,11,0,0,14,0,0,0,24,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,0,0,0,-0.01497974,-0.001061518,0.01936158,0,-0.01711024,0.004103512,0,-0.03231887,-7.980924e-05,0.01958767,0,0,-0.006422545,0,0,0.009201105,-0.01279095,0.03420512,0,0,-0.0002290338,-0.01876179,0.01713414]},{"f":[23,21,15,16,-1,-1,22,33,-1,-1,2,-1,22,16,15,-1,-1,18,4,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,0,11.683871801979876,73.23848001446603,0,0,3.903677461028802,0,15.782972563647261,11.031356596255709,10.462502272597265,0,0,9.265614570460752,5.169368191286958,0,0,0,4.208926257442949,16.725912049439096,0,0,0,0,0],"m":[1,1,1,1,0,0,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,13,14,15,0,0,18,19,0,0,0,23,24,0,0,0,0,0],"r":[28,27,6,5,0,0,10,9,0,0,12,0,22,17,16,0,0,21,20,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,-0.0006087073,0.003411418,0,0,-0.03076971,0.02150358,0,-0.02745154,0,0,0,-0.01127498,0.01007587,0,0,0.004301979,-0.04139095,-0.003193003,0,0,-0.03325891,0.002609342,0.02016668,0.01511057,-0.0176329]},{"f":[4,16,-1,18,18,21,24,-1,-1,-1,1,-1,-1,16,16,-1,-1,15,-1,17,1,-1,1,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.0,0,2.160964047443681,1.792481250360578,4.5,6.5,0,0,0,8.046875,0,0,5.044394119358453,3.321928094887362,0,0,6.169925001442312,0,10.620215166896223,4.54375,0,5.015625,0,0,1.292481250360578,0,0,0],"m":[1,0,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,0,0,11,0,0,14,15,0,0,18,0,20,21,0,23,0,0,26,0,0,0],"r":[28,3,0,13,10,9,8,0,0,0,12,0,0,17,16,0,0,19,0,25,22,0,24,0,0,27,0,0,0],"v":[0,0,-0.03320279,0,0,0,0,0.01760705,-0.03602775,0.001729622,0,0.02838924,-0.002256033,0,0,-0.01611914,0.03850315,0,-0.03690258,0,0,-0.0378286,0,0.03244263,-0.007398219,0,0.03264725,-0.02160085,0.0007683509]},{"f":[22,22,22,22,22,-1,23,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0002440042,0,0,0.01807521,-0.03117506,0.03770076,-0.01839873,0,-0.01860192,0.0252875,0,-0.02328365,0,-0.02625701,0.006448386,0,0.03765662,0,0,0,-0.01223394,0.0343222,-0.01510918,-0.02981399]},{"f":[32,22,33,22,22,-1,-1,22,-1,-1,1,-1,-1,-1,33,20,-1,2,25,-1,-1,-1,0,25,4,-1,-1,-1,-1],"t":[6.208926257442949,16.838461650489297,13.660441030622824,15.782972563647261,14.03140051979143,0,0,16.725912049439096,0,0,8.245481927710843,0,0,0,21.16864017939033,1.5,0,2.453445297804259,1.5,0,0,0,11.5,1.5,3.084962500721156,0,0,0,0],"m":[0,1,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,1,0,0,0,0,1,1,1,0,0,0,0],"l":[1,2,3,4,5,0,0,8,0,0,11,0,0,0,15,16,0,18,19,0,0,0,23,24,25,0,0,0,0],"r":[14,13,10,7,6,0,0,9,0,0,12,0,0,0,22,17,0,21,20,0,0,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.002888478,0.01852643,0,-0.01618932,0.003759426,0,-0.03154887,-3.582095e-05,0.01857665,0,0,-0.006275131,0,0,0.009095738,-0.01214284,0.03281602,0,0,0,-0.002590961,0.001066631,-0.01786807,0.01649808]},{"f":[1,22,25,-1,-1,16,-1,-1,18,22,22,1,22,-1,23,2,-1,-1,18,-1,16,-1,20,-1,-1,-1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,0,7.924812503605781,0,0,10.415240582199587,15.676331898425197,15.439474430023438,7.772727272727273,11.775312925643524,0,0.5,2.160964047443681,0,0,5.339740049752724,0,7.442943495848728,0,0.5,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,1,1,1,1,0,0,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,0,15,16,0,0,19,0,21,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,27,26,25,14,0,18,17,0,0,20,0,22,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.02225745,-0.001783778,0,-0.01416877,0.01830887,0,0,0,0,0,-0.007821517,0,0,0.009316643,0.07134654,0,-0.0003091048,0,-0.0238768,0,0.01152414,0.05313715,-0.002465099,0.01802416,-0.002181439,-0.02429624]},{"f":[22,22,22,22,22,-1,4,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,0,3.9534452978042594,7.6653359171851765,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0002095625,0,0,0.01113905,-0.0422334,0.0326599,-0.01765068,0,-0.02058615,0.01557893,0,-0.03286851,0,-0.02861173,0.005806409,0,0.03646748,0,0,0,-0.006789821,0.03728963,-0.0101863,-0.02937043]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,18,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,6.118903736861568,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,12,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,9,8,7,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02935135,0.0001593727,-0.03105675,0,0.02826367,0,-0.02837601,0,-0.006207359,0.01523199,0,-0.02958067,0,0.006355457,-0.02579155,0,0.03699422,0,0,0.03228821,-0.009744283,-0.02890147,-0.01753425]},{"f":[23,21,15,16,-1,2,22,-1,-1,4,-1,-1,33,0,16,-1,22,-1,22,21,-1,16,-1,-1,-1,-1,-1,-1,-1],"t":[7.5,10.5,10.457380879072534,9.92481250360578,0,5.065928480304397,12.613886328427084,0,0,8.049190845056867,0,0,35.98606694718995,9.5,10.501837184902296,0,11.683871801979876,0,15.782972563647261,6.5,0,11.031356596255709,0,0,0,0,0,0,0],"m":[1,1,1,1,0,0,1,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,10,0,0,13,14,15,0,17,0,19,20,0,22,0,0,0,0,0,0,0],"r":[28,27,12,5,0,9,8,0,0,11,0,0,26,25,16,0,18,0,24,21,0,23,0,0,0,0,0,0,0],"v":[0,0,0,0,-0.0005924223,0,0,0.004955928,0.0555374,0,0.002679129,-0.02121981,0,0,0,-0.0358792,0,0.002420917,0,0,-0.03996886,0,0.002420917,-0.01410616,0.002420917,0.01065069,-0.02238214,0.0144151,-0.01720937]},{"f":[25,17,19,21,-1,22,4,-1,-1,-1,2,2,17,32,-1,-1,1,-1,-1,-1,24,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,7.856894618075133,0,0,0,2.69615871138938,1.792481250360578,1.292481250360578,2.69615871138938,0,0,8.046875,0,0,0,11.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,14,0,0,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,10,5,0,9,8,0,0,0,20,19,16,15,0,0,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.0248033,0,0,0.005262183,-0.006870847,-0.02500953,0,0,0,0,-0.03779576,0.02527195,0,0.0500981,0.00290769,-0.01647936,0,-0.0002554258,0.06434647,0,0,0.004598065,-0.03823436,-0.0002876117,-0.01515024]},{"f":[22,22,22,22,22,-1,23,15,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,1,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,7.6141732283464565,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001988777,0,0,0.01686162,-0.03011173,0.03576517,-0.01668208,0,-0.01769169,0.02298584,0,-0.02219739,0,-0.02528363,0.005642732,0,0.03485524,0,0,0,-0.01180912,0.03301565,-0.01388192,-0.02858068]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,-1,32,-1,22,-1,-1,22,-1,18,-1,-1,36,-1,18,2,-1,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,0,2.69615871138938,0,16.978341770814396,0,0,17.078739781790915,0,6.118903736861568,0,0,4.5,0,8.729619742794132,1.292481250360578,0,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,1,0,0,0,0,1,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,0,12,0,14,0,0,17,0,19,0,0,22,0,24,25,0,0,0,0],"r":[28,21,16,9,8,7,0,0,0,11,0,13,0,15,0,0,18,0,20,0,0,23,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,-0.02897547,0.0001624062,-0.03044849,0,0.02682469,0,-0.02757954,0,-0.005928615,0.01427496,0,-0.02873263,0,0.006076068,-0.02516378,0,0.03526522,0,0,0.03122327,-0.009199561,-0.02812298,-0.0170479]},{"f":[23,22,21,15,4,-1,-1,18,22,3,-1,21,-1,22,-1,-1,17,22,-1,22,-1,-1,-1,23,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,10.5,10.457380879072534,4.128693921346326,0,0,10.680359255641626,15.782972563647261,0.15991874474642756,0,6.5,0,15.212004812197344,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,0,0,0,0,0],"m":[1,1,1,1,1,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,0,12,0,14,0,0,17,18,0,20,0,0,0,24,0,0,0,0,0],"r":[28,27,26,7,6,0,0,23,16,11,0,13,0,15,0,0,22,19,0,21,0,0,0,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.0009888626,0.002588299,0,0,0,0.003848945,0,-0.04326574,0,-0.01326948,0.01495182,0,0,-0.03343784,0,0.01086603,-0.02286115,0.01973809,0,0.02481359,-0.01719959,0.01373303,0.01639651,-0.01673274]},{"f":[36,22,33,-1,-1,22,-1,22,3,-1,-1,25,16,16,-1,-1,22,2,25,-1,-1,-1,22,-1,-1,-1,1,-1,-1],"t":[5.5,12.613886328427084,8.532371382375128,0,0,12.921158001713904,0,13.589275375459419,0.05277777777777778,0,0,3.5,6.108524456778169,5.700439718141092,0,0,15.676331898425197,5.8698903048866296,0.5,0,0,0,15.717370311433935,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,9,0,0,12,13,14,0,0,17,18,19,0,0,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,11,10,0,0,25,16,15,0,0,22,21,20,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03365481,-0.0008517176,0,0.009899635,0,0,0.03172948,-0.02746437,0,0,0,-0.00222406,-0.03203401,0,0,0,0.01932599,-0.001266322,-0.0114306,0,-0.04593816,0.001437757,0.02613119,0,-0.007694907,0.03033718]},{"f":[25,17,19,21,-1,22,4,-1,-1,-1,2,2,17,32,-1,-1,33,-1,-1,-1,24,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,7.856894618075133,0,0,0,2.69615871138938,1.792481250360578,1.292481250360578,2.69615871138938,0,0,9.184798673139337,0,0,0,9.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,0,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,0,0,0,11,12,13,14,0,0,17,0,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,10,5,0,9,8,0,0,0,20,19,16,15,0,0,18,0,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02413015,0,0,0.004997364,-0.00677334,-0.02353784,0,0,0,0,-0.03727882,0.0241627,0,0.04977472,0.004352348,-0.01598046,0,-0.001347914,0.05937283,0,0,0.004449632,-0.03759374,-0.0002935407,-0.01526205]},{"f":[36,-1,18,-1,18,-1,15,22,-1,-1,16,16,-1,22,-1,-1,2,22,21,-1,-1,-1,33,-1,17,17,-1,-1,-1],"t":[1.5,0,7.723832212141186,0,8.03615005796417,0,2.0,11.722778233171695,0,0,9.92481250360578,6.584962500721156,0,11.424679641461715,0,0,1.292481250360578,14.247293309538719,6.5,0,0,0,34.303028789183955,0,9.270294853526895,9.265614570460752,0,0,0],"m":[0,0,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,1,0,0,0],"l":[1,0,3,0,5,0,7,8,0,0,11,12,0,14,0,0,17,18,19,0,0,0,23,0,25,26,0,0,0],"r":[2,0,4,0,6,0,10,9,0,0,16,13,0,15,0,0,22,21,20,0,0,0,24,0,28,27,0,0,0],"v":[0,0.01303046,0,-0.0005658297,0,0.0413735,0,0,-0.002869389,0.01978187,0,0,-0.005232505,0,0.01085328,-0.03648865,0,0,0,-0.01069603,0.05452268,-0.01665529,0,0.01951536,0,0,-0.008433096,0.03822161,-0.01375019]},{"f":[25,17,19,21,-1,22,22,22,4,15,-1,22,-1,-1,-1,25,-1,-1,-1,-1,2,2,-1,-1,-1,17,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,15.782972563647261,7.856894618075133,6.022367813028454,0,13.64082212194147,0,0,0,0.5,0,0,0,0,2.69615871138938,1.792481250360578,0,0,0,2.69615871138938,0,0,0],"m":[1,0,1,0,0,1,1,1,1,0,0,1,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,12,0,0,0,16,0,0,0,0,21,22,0,0,0,26,0,0,0],"r":[28,25,20,5,0,19,18,15,14,11,0,13,0,0,0,17,0,0,0,0,24,23,0,0,0,27,0,0,0],"v":[0,0,0,0,-0.02361886,0,0,0,0,0,0.01633615,0,0.003789626,-0.03399917,-0.00644698,0,-0.01879727,0.02903499,0.0182728,-0.0226865,0,0,0.01126917,-0.01551672,0.04500507,0,-0.02469458,-0.0002707668,-0.01491889]},{"f":[22,22,22,22,22,-1,4,-1,-1,-1,32,-1,-1,25,33,-1,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,3.9534452978042594,0,0,0,2.69615871138938,0,0,3.5,34.128137685572966,0,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,0,0,0,11,0,0,14,15,0,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,13,10,9,6,0,8,0,0,0,12,0,0,17,16,0,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001952333,0,-0.02350132,0.03068452,-0.01569596,0,-0.01694218,0.02149412,0,0,-0.03554215,-0.006445065,0,-0.02472433,0.005319961,0,0.03330506,0,0,0,-0.006927613,0.0355758,-0.008751248,-0.0277367]},{"f":[1,22,25,-1,18,18,-1,22,21,15,-1,17,-1,-1,-1,-1,-1,16,-1,1,5,-1,-1,-1,18,1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,10.482680779086738,5.491496787347155,0,15.212004812197344,7.5,12.711021616791836,0,9.265614570460752,0,0,0,0,0,7.924812503605781,0,3.172945205479452,0.5,0,0,0,10.415240582199587,4.145833333333334,0,0,0],"m":[1,1,0,0,1,1,0,1,1,1,0,0,0,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,0,8,9,10,0,12,0,0,0,0,0,18,0,20,21,0,0,0,25,26,0,0,0],"r":[24,17,4,0,16,7,0,15,14,11,0,13,0,0,0,0,0,19,0,23,22,0,0,0,28,27,0,0,0],"v":[0,0,0,-0.0217266,0,0,-0.001840272,0,0,0,-0.03843723,0,-0.02837423,0.01957583,0.0002296663,0.01793702,0.004021918,0,-0.01360556,0,0,0.007667737,0.04612837,0.00746323,0,0,0.02217102,0.0007627109,-0.02319038]},{"f":[36,-1,32,32,37,22,22,18,4,-1,21,22,2,-1,-1,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,18,-1,-1],"t":[1.5,0,10.463013322934223,10.41931383093672,12.52,17.083799521744545,17.02942582914998,7.723832212141186,5.505613627711627,0,7.5,15.782972563647261,7.128362343532883,0,0,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,3.5221970596792267,0,0],"m":[0,0,1,1,1,1,1,1,1,0,0,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,9,0,11,12,13,0,0,0,0,0,19,0,21,0,0,0,0,0,27,0,0],"r":[2,0,26,25,24,23,18,17,10,0,16,15,14,0,0,0,0,0,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0.01285817,0,0,0,0,0,0,0,0.0004645854,0,0,0,-0.03973472,-0.002708061,0.008167785,-0.001001435,0.00374324,0,-0.03042655,0,-0.02708706,0.005065974,0.008167785,0.01650307,-0.009464405,0,0.01482556,-0.01859576]},{"f":[4,16,-1,18,17,-1,0,15,-1,-1,-1,23,1,22,-1,-1,33,-1,-1,21,22,35,-1,-1,-1,2,-1,-1,-1],"t":[3.084962500721156,2.0,0,2.160964047443681,0.5,0,5.5,7.930737337562887,0,0,0,0.5,5.015625,11.67903991803059,0,0,17.938048609286376,0,0,8.5,17.07907323966382,1.5,0,0,0,1.292481250360578,0,0,0],"m":[1,0,0,0,0,0,1,1,0,0,0,0,1,0,0,0,1,0,0,1,1,0,0,0,0,1,0,0,0],"l":[1,2,0,4,5,0,7,8,0,0,0,12,13,14,0,0,17,0,0,20,21,22,0,0,0,26,0,0,0],"r":[28,3,0,11,6,0,10,9,0,0,0,19,16,15,0,0,18,0,0,25,24,23,0,0,0,27,0,0,0],"v":[0,0,-0.03224973,0,0,0.01857331,0,0,-0.03967426,-0.007577726,0.002390075,0,0,0,-0.01324479,0.05386171,0,-0.0359599,-0.001833894,0,0,0,-0.003690879,-0.03904348,0.009167576,0,0.02151365,-0.02134234,0.0007247974]},{"f":[25,17,19,21,-1,22,22,22,4,15,-1,-1,-1,25,-1,-1,-1,-1,2,2,-1,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,1.5,4.5,0,16.765402642760364,16.725912049439096,15.782972563647261,7.856894618075133,6.022367813028454,0,0,0,0.5,0,0,0,0,2.69615871138938,1.792481250360578,0,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,7,8,9,10,0,0,0,14,0,0,0,0,19,20,0,0,0,24,25,0,0,0,0],"r":[28,23,18,5,0,17,16,13,12,11,0,0,0,15,0,0,0,0,22,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0,-0.02304816,0,0,0,0,0,0.01546698,-6.83899e-05,-0.006040092,0,-0.0179384,0.0281367,0.01774904,-0.02181204,0,0,0.01067945,-0.01517634,0.04229225,0,0,0.00524275,-0.03684096,-0.0002902153,-0.0147579]},{"f":[1,22,25,-1,-1,16,-1,-1,18,22,22,1,22,22,-1,-1,18,23,2,-1,-1,-1,15,-1,-1,-1,-1,-1,-1],"t":[4.031939779270633,16.759160379664443,1.5,0,0,7.924812503605781,0,0,10.415240582199587,15.676331898425197,15.439474430023438,7.772727272727273,11.775312925643524,8.530347965843777,0,0,5.339740049752724,0.5,2.453445297804259,0,0,0,6.820178962415188,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,0,6,0,0,9,10,11,12,13,14,0,0,17,18,19,0,0,0,23,0,0,0,0,0,0],"r":[8,5,4,0,0,7,0,0,28,27,26,25,16,15,0,0,22,21,20,0,0,0,24,0,0,0,0,0,0],"v":[0,0,0,-0.02118453,-0.001659795,0,-0.01302399,0.01645274,0,0,0,0,0,0,0.001446704,-0.03127928,0,0,0,0.006392764,0.0578416,-0.0001175846,0,-0.01194249,0.04073813,-0.002241079,0.01611131,-0.002102261,-0.02272082]},{"f":[36,-1,22,21,-1,-1,22,2,25,15,21,-1,17,1,-1,-1,-1,-1,33,-1,33,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,0,15.676331898425197,6.321815375530864,0.5,8.005624549193879,4.5,0,7.67595272073847,4.949128919860627,0,0,0,0,19.167172160526107,0,21.16864017939033,0,0,0,15.717370311433935,0,15.756821747580634,0,0],"m":[0,0,1,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,0,7,8,9,10,11,0,13,14,0,0,0,0,19,0,21,0,0,0,25,0,27,0,0],"r":[2,0,6,5,0,0,24,23,18,17,12,0,16,15,0,0,0,0,20,0,22,0,0,0,26,0,28,0,0],"v":[0,0.01261551,0,0,-0.0248341,-0.0004877547,0,0,0,0,0,0.04680855,0,0,-0.02774555,0.01956807,0.03765899,-0.01748867,0,-0.02682827,0,0.01328253,-0.007032233,-0.0122124,0,-0.04446373,0,0.0261126,-0.0003434513]},{"f":[4,16,-1,15,17,22,-1,-1,37,18,-1,23,-1,-1,18,22,22,18,-1,-1,-1,-1,-1,22,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,12.000704269011246,0.5,15.717370311433935,0,0,5.05,1.292481250360578,0,3.5,0,0,2.160964047443681,16.765402642760364,16.73920708776685,1.792481250360578,0,0,0,0,0,13.930814526604713,0,0,17.094543568094064,0,0],"m":[1,0,0,1,0,1,0,0,1,0,0,1,0,0,1,1,1,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,9,10,0,12,0,0,15,16,17,18,0,0,0,0,0,24,0,0,27,0,0],"r":[26,3,0,23,8,7,0,0,14,11,0,13,0,0,22,21,20,19,0,0,0,0,0,25,0,0,28,0,0],"v":[0,0,-0.03188048,0,0,0,0.005489266,0.04767831,0,0,-0.03808363,0,-0.006245923,-0.03046873,0,0,0,0,0.002037554,0.01483946,-0.006245923,0.01997592,-0.005020823,0,0.04281369,-0.01350057,0,0.0005774775,0.02346037]},{"f":[25,17,3,-1,1,21,2,-1,-1,22,0,-1,-1,21,-1,-1,1,-1,22,22,-1,-1,24,-1,-1,17,-1,-1,-1],"t":[6.5,2.160964047443681,0.12256006006006007,0,5.3560606060606055,6.5,6.209321624687949,0,0,10.27595502635321,2.5,0,0,7.5,0,0,6.015625,0,15.782972563647261,15.439474430023438,0,0,6.5,0,0,2.69615871138938,0,0,0],"m":[1,0,0,0,1,1,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,14,0,0,17,0,19,20,0,0,23,0,0,26,0,0,0],"r":[28,25,4,0,16,9,8,0,0,13,12,0,0,15,0,0,18,0,22,21,0,0,24,0,0,27,0,0,0],"v":[0,0,0,0.02018299,0,0,0,-0.0386854,0.004698478,0,0,0.002300318,0.03185257,0,-0.02725084,-0.002151736,0,0.03151299,0,0,0.0009147154,0.01617029,0,0.0261104,-0.01387582,0,-0.02352998,-0.0002845665,-0.01441598]},{"f":[4,16,-1,15,17,22,-1,-1,0,-1,22,22,7,3,-1,23,-1,-1,-1,-1,16,-1,-1,22,-1,-1,22,-1,-1],"t":[3.245926548164837,2.0,0,12.000704269011246,0.5,15.717370311433935,0,0,2.5,0,16.73920708776685,16.725912049439096,0.5,0.013290990355210539,0,3.5,0,0,0,0,5.169925001442312,0,0,13.930814526604713,0,0,17.094543568094064,0,0],"m":[1,0,0,1,0,1,0,0,0,0,1,1,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,0,4,5,6,0,0,9,0,11,12,13,14,0,16,0,0,0,0,21,0,0,24,0,0,27,0,0],"r":[26,3,0,23,8,7,0,0,10,0,20,19,18,15,0,17,0,0,0,0,22,0,0,25,0,0,28,0,0],"v":[0,0,-0.03143667,0,0,0,0.005285641,0.04639147,0,-0.02653058,0,0,0,0,0.01213223,0,-0.0003657774,-0.01070984,-0.0139461,0.02048465,0,0.02962974,-0.01197582,0,0.04022961,-0.01317249,0,0.0005694225,0.02282868]},{"f":[32,32,37,25,22,22,22,22,16,-1,-1,-1,2,15,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,4.128693921346326,4.087462841250339,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,0,0,0,13,14,0,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,12,11,10,0,0,0,16,15,0,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,-0.02589777,0.0001678156,-0.03004744,0,0,0.008212787,-0.03194411,0.008212787,0,-0.0297227,0,-0.02693562,0.004827883,0.008212787,-0.02376206,0.01650675,-0.009061229,0,0.01421445,-0.01823891]},{"f":[36,-1,22,21,-1,22,22,33,-1,23,-1,-1,21,-1,-1,16,-1,22,-1,2,22,-1,-1,-1,22,-1,22,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,2.5,0,0,7.5,0,0,6.832890014164741,0,13.589275375459419,0,2.69615871138938,13.930814526604713,0,0,0,15.676331898425197,0,15.717370311433935,0,0],"m":[0,0,1,0,0,1,1,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,0,10,0,0,13,0,0,16,0,18,0,20,21,0,0,0,25,0,27,0,0],"r":[2,0,24,5,0,15,12,9,0,11,0,0,14,0,0,17,0,19,0,23,22,0,0,0,26,0,28,0,0],"v":[0,0.01220917,0,0,-0.02409089,0,0,0,-0.03618505,0,0.0006346911,-0.01362974,0,0.01533413,-0.002257378,0,-0.03969738,0,-0.0229445,0,0,0.03573857,-0.01733003,-0.003465642,0,0.004971389,0,-0.04288695,0.001041905]},{"f":[32,15,22,20,-1,-1,1,16,-1,-1,22,-1,-1,15,-1,-1,33,20,-1,2,-1,-1,0,1,-1,37,-1,-1,-1],"t":[6.208926257442949,5.044394119358453,12.613886328427084,0.5,0,0,6.241071428571429,7.417852514885898,0,0,15.782972563647261,0,0,6.169925001442312,0,0,21.16864017939033,1.5,0,2.453445297804259,0,0,11.5,8.980632551528075,0,9.205,0,0,0],"m":[0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,0,0,11,0,0,14,0,0,17,18,0,20,0,0,23,24,0,26,0,0,0],"r":[16,13,6,5,0,0,10,9,0,0,12,0,0,15,0,0,22,19,0,21,0,0,28,25,0,27,0,0,0],"v":[0,0,0,0,-0.0362492,0.0001357379,0,0,-0.03398597,0.002689093,0,0.01479019,-0.005806546,0,-0.02727202,-0.001577577,0,0,-0.005802042,0,0.002663439,0.03022486,0,0,-7.066511e-06,0,-0.02770138,0.005836702,0.01512042]},{"f":[36,22,33,-1,-1,22,-1,22,-1,25,16,16,-1,-1,22,2,25,-1,-1,-1,22,-1,22,-1,-1,-1,1,-1,-1],"t":[5.5,12.613886328427084,8.532371382375128,0,0,13.129761635363703,0,13.589275375459419,0,3.5,6.108524456778169,5.700439718141092,0,0,15.676331898425197,5.8698903048866296,0.5,0,0,0,15.717370311433935,0,15.756821747580634,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,0,0,0,0,0,1,0,1,0,0,0,1,1,0,0,0,0,0,0,0,0,0,1,0,0],"l":[1,2,3,0,0,6,0,8,0,10,11,12,0,0,15,16,17,0,0,0,21,0,23,0,0,0,27,0,0],"r":[26,5,4,0,0,7,0,9,0,25,14,13,0,0,20,19,18,0,0,0,22,0,24,0,0,0,28,0,0],"v":[0,0,0,-0.03182236,-0.0007828596,0,0.008368893,0,-0.02736078,0,0,0,-0.002160545,-0.03025718,0,0,0,0.01682621,-0.001269217,-0.01057397,0,-0.04136889,0,0.02518297,-0.0004440288,0.02486866,0,-0.007279599,0.0293479]},{"f":[16,-1,22,22,22,19,16,4,-1,-1,1,25,-1,-1,16,-1,-1,33,-1,2,-1,-1,-1,25,-1,-1,2,-1,-1],"t":[9.92481250360578,0,16.759160379664443,16.73920708776685,16.725912049439096,0.5,10.681238411777805,6.266299719336492,0,0,4.030980086372361,0.5,0,0,12.225508720236002,0,0,35.98606694718995,0,6.942466823974881,0,0,0,1.5,0,0,6.011183906514227,0,0],"m":[1,0,1,1,1,1,0,1,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,5,6,7,8,0,0,11,12,0,0,15,0,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[2,0,26,23,22,17,10,9,0,0,14,13,0,0,16,0,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,-0.0008160621,0,0,0,0,0,0,0.04764705,-0.002022995,0,0,-0.03386935,-0.00233711,0,0.0379485,-0.0005453786,0,0.003127101,0,-0.04038036,-0.01183358,0.03146734,0,-0.03823973,0.0001627594,0,0.06018883,0.007404572]},{"f":[4,16,-1,17,22,-1,-1,0,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,21,-1,-1,22,17,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,3.5,0,0,14.03140051979143,5.918419679874358,0,0,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,1,0,0,1,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,0,10,0,0,13,14,0,0,17,0,19,20,21,0,0,24,25,0,0,0,0],"r":[12,3,0,7,6,0,0,9,0,11,0,0,16,15,0,0,18,0,28,23,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.03098857,0,0,0.00506188,0.04363966,0,-0.02341522,0,-0.001336491,0.02620222,0,0,-0.001033395,-0.02291685,0,0.01871044,0,0,0,-0.01406246,0.007581998,0,0,0.02065052,-0.026709,0.05568416,0.0001094698]},{"f":[22,-1,22,15,-1,-1,22,-1,15,23,18,18,-1,-1,21,-1,-1,-1,15,2,-1,-1,18,16,18,-1,-1,-1,-1],"t":[6.870153907126159,0,7.95646805166752,2.584962500721156,0,0,8.133841411710264,0,2.0,2.5,10.269270673170688,2.453445297804259,0,0,6.5,0,0,0,2.584962500721156,1.292481250360578,0,0,11.694748780085186,9.92481250360578,7.383290334558488,0,0,0,0],"m":[0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,1,1,1,0,0,0,0],"l":[1,0,3,4,0,0,7,0,9,10,11,12,0,0,15,0,0,0,19,20,0,0,23,24,25,0,0,0,0],"r":[2,0,6,5,0,0,8,0,18,17,14,13,0,0,16,0,0,0,22,21,0,0,28,27,26,0,0,0,0],"v":[0,-0.01048992,0,0,-0.008350996,0.009516204,0,-0.02540252,0,0,0,0,0.0002736101,-0.02411812,0,0.01510917,0.0005031756,0.0257099,0,0,0.02439665,-0.02427584,0,0,0,-0.0004306658,-0.01468085,0.00174774,-0.02108148]},{"f":[22,22,22,22,22,22,22,22,-1,-1,-1,-1,23,16,-1,-1,-1,-1,32,-1,-1,22,-1,-1,25,-1,27,-1,-1],"t":[17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,16.786010507999382,16.765402642760364,16.73920708776685,0,0,0,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,1.5,0,0],"m":[1,1,1,1,1,1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,0,0,0,0,13,14,0,0,0,0,19,0,0,22,0,0,25,0,27,0,0],"r":[24,21,18,17,12,11,10,9,0,0,0,0,16,15,0,0,0,0,20,0,0,23,0,0,26,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0.0002952252,-0.01181247,0.02571841,-0.01107796,0,0,0.01034804,-0.03309592,0.03414202,-0.01536872,0,-0.01614956,0.02062544,0,-0.01907575,0.0003986176,0,0.03382643,0,-0.02742342,0.006084136]},{"f":[9,22,21,-1,22,22,33,-1,-1,-1,16,-1,22,-1,15,16,-1,25,-1,-1,-1,22,-1,22,-1,22,-1,-1,-1],"t":[0.5,14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,0,0,6.832890014164741,0,13.589275375459419,0,10.457380879072534,7.330916878114617,0,1.5,0,0,0,15.676331898425197,0,15.717370311433935,0,15.756821747580634,0,0,0],"m":[1,1,0,0,1,1,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,0,13,0,15,16,0,18,0,0,0,22,0,24,0,26,0,0,0],"r":[28,21,4,0,10,9,8,0,0,0,12,0,14,0,20,17,0,19,0,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,-0.02420477,0,0,0,-0.0309049,0.000112628,0.008335214,0,-0.03891524,0,-0.02096234,0,0,0.01538258,0,0.00301665,-0.0222228,0.02796338,0,0.005047686,0,-0.03997375,0,0.02366556,-0.0002778811,-0.01735849]},{"f":[36,22,22,22,22,22,-1,23,16,-1,-1,-1,-1,32,-1,-1,22,-1,-1,25,-1,17,2,-1,-1,-1,1,-1,-1],"t":[5.5,17.094543568094064,16.98920034885663,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,8.729619742794132,1.292481250360578,0,0,0,8.992857142857144,0,0],"m":[1,1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,1,0,0,0,0,1,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,8,9,0,0,0,0,14,0,0,17,0,0,20,0,22,23,0,0,0,27,0,0],"r":[26,19,16,13,12,7,0,11,10,0,0,0,0,15,0,0,18,0,0,21,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.0003551799,0,0,0.01005279,-0.03240906,0.03324267,-0.01476409,0,-0.01597074,0.01981775,0,-0.01837746,0.0004733124,0,0.03325039,0,0,0.0238726,-0.01811125,-0.02684913,0,-0.006804508,0.02892849]},{"f":[23,22,22,22,22,16,-1,-1,-1,22,22,-1,-1,24,-1,-1,22,-1,18,-1,-1,32,-1,33,-1,18,-1,-1,-1],"t":[7.5,17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0.0,0,0,0,16.903290731167747,16.844368517741394,0,0,8.5,0,0,17.078739781790915,0,6.118903736861568,0,0,8.341935934311302,0,37.46769166150554,0,5.491496787347155,0,0,0],"m":[1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,0,0,0,10,11,0,0,14,0,0,17,0,19,0,0,22,0,24,0,26,0,0,0],"r":[28,21,16,9,8,7,0,0,0,13,12,0,0,15,0,0,18,0,20,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,0,-0.02731352,0.0001382179,-0.02931118,0,0,0.002317593,0.03775648,0,0.004102936,-0.02275056,0,-0.02682704,0,0.005481593,-0.02327126,0,-0.02149697,0,0.04152171,0,-0.01528678,0.01024062,-0.01575288]},{"f":[36,22,22,4,-1,-1,-1,23,33,15,18,-1,2,-1,-1,15,-1,18,-1,-1,-1,18,16,-1,-1,-1,1,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,1.292481250360578,0,0,6.857980995127572,0,10.680359255641626,0,0,0,10.45532575654158,6.523561956057013,0,0,0,8.992857142857144,0,0],"m":[1,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,16,0,18,0,0,0,22,23,0,0,0,27,0,0],"r":[26,7,6,5,0,0,0,21,20,15,12,0,14,0,0,17,0,19,0,0,0,25,24,0,0,0,28,0,0],"v":[0,0,0,0,-0.0256417,0.001478922,-0.0189827,0,0,0,0,-0.02067456,0,0.03733521,0.003650071,0,0.05576668,0,-0.008857605,0.0240074,-0.002760774,0,0,-0.004792323,0.001468052,-0.01964785,0,-0.00657326,0.02843799]},{"f":[4,16,-1,19,0,22,23,16,-1,21,-1,22,-1,22,-1,-1,-1,22,-1,1,-1,22,-1,0,-1,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,2.5,10.5,16.73920708776685,4.5,6.832890014164741,0,4.5,0,15.676331898425197,0,15.717370311433935,0,0,0,16.752707246237605,0,1.3897752394988947,0,16.765402642760364,0,6.5,0,0,0,0,0],"m":[1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0],"l":[1,2,0,4,5,6,7,8,0,10,0,12,0,14,0,0,0,18,0,20,0,22,0,24,0,0,0,0,0],"r":[28,3,0,27,26,17,16,9,0,11,0,13,0,15,0,0,0,19,0,21,0,23,0,25,0,0,0,0,0],"v":[0,0,-0.03038959,0,0,0,0,0,0.002713899,0,-0.03660111,0,0.001212375,0,-0.03944028,-0.001942143,0.00571377,0,-0.03324837,0,0.00571377,0,-0.02701678,0,-0.02382006,0.00571377,0.01252382,-0.02823429,0.0008039568]},{"f":[36,-1,22,21,-1,23,22,22,33,-1,-1,-1,17,-1,-1,16,-1,1,-1,16,-1,-1,22,17,18,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,3.5,13.129761635363703,12.613886328427084,8.532371382375128,0,0,0,6.9827117829050644,0,0,6.882643049361842,0,3.015625,0,9.047123912114026,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,0,0,0],"m":[0,0,1,0,0,1,1,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,0,0,0,13,0,0,16,0,18,0,20,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,15,12,11,10,0,0,0,14,0,0,17,0,19,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0.01194332,0,0,-0.02269799,0,0,0,0,-0.03396925,0.0003604977,0.009013224,0,-0.03511601,-0.002739511,0,-0.02483694,0,-0.02673286,0,-0.00130045,0.02491257,0,0,0,0.03003735,-0.002905559,0.03995877,-0.0004125795]},{"f":[36,22,22,22,16,0,-1,-1,33,-1,-1,32,-1,-1,22,-1,-1,25,-1,17,24,-1,16,-1,-1,-1,1,-1,-1],"t":[5.5,17.094543568094064,17.02942582914998,16.978341770814396,0.0,9.5,0,0,35.98606694718995,0,0,2.69615871138938,0,0,17.078739781790915,0,0,1.5,0,8.729619742794132,8.5,0,3.321928094887362,0,0,0,8.992857142857144,0,0],"m":[1,1,1,1,0,1,0,0,1,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0,0,1,0,0],"l":[1,2,3,4,5,6,0,0,9,0,0,12,0,0,15,0,0,18,0,20,21,0,23,0,0,0,27,0,0],"r":[26,17,14,11,8,7,0,0,10,0,0,13,0,0,16,0,0,19,0,25,22,0,24,0,0,0,28,0,0],"v":[0,0,0,0,0,0,-0.02856119,-0.002359248,0,0.0001545208,-0.02120544,0,-0.01758649,0.01251943,0,-0.02654596,0.0003993916,0,0.03229379,0,0,-0.02228186,0,-0.01118246,0.02984381,-0.02668656,0,-0.006006715,0.02777579]},{"f":[0,32,18,4,18,4,19,-1,-1,22,16,22,-1,33,-1,-1,-1,-1,-1,21,22,2,-1,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,7.723832212141186,5.505613627711627,7.383290334558488,4.951187557243013,2.5,0,0,14.03140051979143,7.266786540694901,7.082728314749225,0,15.910824717275187,0,0,0,0,0,7.5,15.782972563647261,7.128362343532883,0,0,0,0,0,0,0],"m":[1,1,1,1,1,1,1,0,0,1,1,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,10,11,12,0,14,0,0,0,0,0,20,21,22,0,0,0,0,0,0,0],"r":[28,27,26,19,18,9,8,0,0,17,16,13,0,15,0,0,0,0,0,25,24,23,0,0,0,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.0001258966,-0.026812,0,0,0,0.00635591,0,-0.04119804,-0.007854937,0.003298861,0.00635591,-0.02563621,0,0,0,-0.03878438,-0.002032912,0.00635591,-6.868587e-06,0.002483203,-0.01990555,0.01284458]},{"f":[32,32,37,25,22,22,22,22,22,7,-1,-1,-1,21,-1,-1,-1,25,-1,22,-1,-1,-1,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,12.52,6.5,17.083799521744545,17.02942582914998,16.759160379664443,16.73920708776685,16.725912049439096,0.5,0,0,0,7.5,0,0,0,3.5,0,17.069460217922263,0,0,0,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,1,1,1,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,8,9,10,0,0,0,14,0,0,0,18,0,20,0,0,0,0,0,0,27,0,0],"r":[26,25,24,23,22,17,16,13,12,11,0,0,0,15,0,0,0,19,0,21,0,0,0,0,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0,0,0,0.0001385361,-0.01379701,0.007405428,0,0.001339232,-0.03356032,0.00394992,0,-0.02819549,0,-0.02547243,0.005162231,0.007405428,-0.02265052,0.01488203,-0.00852197,0,0.01346689,-0.01740567]},{"f":[9,17,3,-1,1,21,2,-1,-1,22,0,-1,-1,22,-1,-1,1,23,-1,22,-1,-1,-1,17,22,-1,-1,-1,-1],"t":[0.5,2.160964047443681,0.23857586512866016,0,5.489664082687339,6.5,5.065928480304397,0,0,10.27595502635321,2.5,0,0,16.759160379664443,0,0,6.015625,0.5,0,12.679695715883515,0,0,0,2.69615871138938,16.752707246237605,0,0,0,0],"m":[1,0,0,0,1,1,1,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,10,11,0,0,14,0,0,17,18,0,20,0,0,0,24,25,0,0,0,0],"r":[28,23,4,0,16,9,8,0,0,13,12,0,0,15,0,0,22,19,0,21,0,0,0,27,26,0,0,0,0],"v":[0,0,0,0.0197193,0,0,0,-0.03778464,0.002768378,0,0,0.002179115,0.02985171,0,-0.009191311,0.01285769,0,0,0.06122814,0,-0.02238117,0.02633824,0.002300494,0,0,-0.03724582,0.004473968,-0.0002673413,-0.01687313]},{"f":[36,-1,22,21,-1,22,22,22,22,33,-1,23,-1,-1,21,-1,-1,-1,-1,16,-1,-1,22,17,18,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,0,13.930814526604713,13.589275375459419,13.129761635363703,12.613886328427084,8.532371382375128,0,2.5,0,0,7.5,0,0,0,0,7.266786540694901,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,0,0,0],"m":[0,0,1,0,0,1,1,1,1,0,0,1,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0],"l":[1,0,3,4,0,6,7,8,9,10,0,12,0,0,15,0,0,0,0,20,0,0,23,24,25,0,0,0,0],"r":[2,0,22,5,0,19,18,17,14,11,0,13,0,0,16,0,0,0,0,21,0,0,28,27,26,0,0,0,0],"v":[0,0.01150582,0,0,-0.02193139,0,0,0,0,0,-0.03342411,0,0.0005177747,-0.01198808,0,0.01277901,-0.002932467,-0.0242891,0.01892417,0,-0.03346829,-0.004215131,0,0,0,0.02885655,-0.002810389,0.03838764,-0.0003660619]},{"f":[36,22,22,4,-1,-1,-1,23,33,15,18,-1,2,-1,-1,15,-1,22,-1,4,-1,-1,-1,18,-1,-1,18,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,1.292481250360578,0,0,6.857980995127572,0,11.683871801979876,0,2.69615871138938,0,0,0,10.45532575654158,0,0,4.245926548164837,0,0],"m":[1,0,1,0,0,0,0,0,1,1,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,0,8,9,10,11,0,13,0,0,16,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[26,7,6,5,0,0,0,23,22,15,12,0,14,0,0,17,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02502607,0.001249596,-0.01804626,0,0,0,0,-0.01978315,0,0.0346484,0.003056412,0,0.05186416,0,0.02782874,0,-0.02941145,0.009223243,-0.002488165,0,-2.828262e-05,-0.01902705,0,-0.003607992,0.03061127]},{"f":[36,22,22,4,-1,-1,21,-1,-1,23,33,33,33,-1,-1,1,16,-1,15,-1,-1,-1,-1,18,-1,-1,1,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,7.5,0,0,0.5,21.257266042509173,19.167172160526107,16.11799658182957,0,0,2.969223484848485,1.584962500721156,0,9.756556322524087,0,0,0,0,10.45532575654158,0,0,8.992857142857144,0,0],"m":[1,0,1,0,0,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,13,0,0,16,17,0,19,0,0,0,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,23,22,15,14,0,0,21,18,0,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02458856,0.001186713,0,-0.02883821,-0.006469457,0,0,0,0,0.0003889512,-0.03885435,0,0,0.01584202,0,-0.02486421,0.02039624,0.01608246,-0.009620983,0,-2.687333e-05,-0.01838971,0,-0.005373443,0.02672298]},{"f":[36,-1,16,-1,22,22,21,-1,33,16,-1,22,0,-1,-1,-1,-1,2,-1,-1,22,-1,22,-1,2,-1,2,-1,-1],"t":[1.5,0,9.92481250360578,0,16.725912049439096,15.756821747580634,4.5,0,35.98606694718995,10.681238411777805,0,15.212004812197344,3.5,0,0,0,0,7.071452393754301,0,0,16.73920708776685,0,16.752707246237605,0,6.011183906514227,0,9.068775776507199,0,0],"m":[0,0,1,0,1,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0],"l":[1,0,3,0,5,6,7,0,9,10,0,12,13,0,0,0,0,18,0,0,21,0,23,0,25,0,27,0,0],"r":[2,0,4,0,20,17,8,0,16,11,0,15,14,0,0,0,0,19,0,0,22,0,24,0,26,0,28,0,0],"v":[0,0.01161462,0,-0.0008253242,0,0,0,-0.02252791,0,0,0.01407398,0,0,-0.01189688,0.001976899,0.01824497,-0.03182736,0,-0.002351529,-0.01818838,0,0.02945295,0,-0.02906311,0,0.05459514,0,-0.003660772,0.02043778]},{"f":[0,32,32,33,-1,15,-1,-1,33,-1,2,33,-1,16,-1,23,-1,15,17,-1,-1,22,32,-1,-1,-1,-1,-1,-1],"t":[18.5,10.482680779086738,6.208926257442949,13.660441030622824,0,6.169925001442312,0,0,21.16864017939033,0,3.084962500721156,24.416594167368267,0,2.0,0,0.5,0,7.930737337562887,7.344289078557851,0,0,12.613886328427084,10.019277334827486,0,0,0,0,0,0],"m":[1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0,1,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,0,11,12,0,14,0,16,0,18,19,0,0,22,23,0,0,0,0,0,0],"r":[28,27,8,5,0,7,0,0,10,0,26,13,0,15,0,17,0,21,20,0,0,25,24,0,0,0,0,0,0],"v":[0,0,0,0,-6.000791e-05,0,-0.02859716,0.0002913545,0,0.006210531,0,0,-0.0163167,0,-0.02717173,0,0.006210531,0,0,-0.03449295,0.001884348,0,0,-0.0008964449,-0.03220954,0.006210531,0.0007825591,-0.01923655,0.01252153]},{"f":[36,22,22,4,-1,-1,21,-1,-1,23,33,15,18,-1,17,-1,-1,15,-1,18,-1,-1,-1,18,-1,-1,18,-1,-1],"t":[5.5,11.67903991803059,10.27595502635321,1.292481250360578,0,0,7.5,0,0,0.5,9.184798673139337,5.672425341971495,1.292481250360578,0,10.594763897338481,0,0,6.857980995127572,0,10.680359255641626,0,0,0,10.45532575654158,0,0,4.245926548164837,0,0],"m":[1,0,1,0,0,0,1,0,0,0,1,1,1,0,1,0,0,0,0,1,0,0,0,1,0,0,1,0,0],"l":[1,2,3,4,0,0,7,0,0,10,11,12,13,0,15,0,0,18,0,20,0,0,0,24,0,0,27,0,0],"r":[26,9,6,5,0,0,8,0,0,23,22,17,14,0,16,0,0,19,0,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,-0.02405035,0.001054004,0,-0.02818671,-0.006121747,0,0,0,0,-0.01936885,0,0.01949042,-0.001847633,0,0.04673717,0,-0.007678575,0.02161485,-0.002203193,0,-2.937709e-05,-0.01774363,0,-0.00333549,0.02992343]},{"f":[21,15,-1,22,22,21,18,-1,-1,-1,-1,17,22,-1,22,-1,-1,-1,25,18,22,2,-1,-1,-1,18,-1,-1,-1],"t":[9.5,11.054604317960672,0,15.782972563647261,11.876328901062104,7.5,9.265614570460752,0,0,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,1.792481250360578,13.589275375459419,6.554507209719489,0,0,0,6.781300095563516,0,0,0],"m":[1,1,0,1,0,1,1,0,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,0,0,0,12,13,0,15,0,0,0,19,20,21,22,0,0,0,26,0,0,0],"r":[18,3,0,11,10,9,8,0,0,0,0,17,14,0,16,0,0,0,28,25,24,23,0,0,0,27,0,0,0],"v":[0,0,0.0001306395,0,0,0,0,-0.03109817,0.003737936,0.01280989,-0.03928888,0,0,-0.03102462,0,0.01023138,-0.02207271,0.01811167,0,0,0,0,-0.01176288,0.01604863,0.01944935,0,-0.02709823,0.003339791,0.03560569]},{"f":[36,-1,22,21,1,-1,2,-1,-1,-1,22,17,18,-1,21,33,-1,-1,-1,-1,22,21,-1,-1,4,22,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,4.5,15.968216440900115,0,0,0,0,15.439474430023438,7.5,0,0,8.049190845056867,15.676331898425197,0,0,0],"m":[0,0,1,0,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,0,15,16,0,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,20,19,14,0,18,17,0,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.01127949,0,0,0,-0.03491008,0,-0.02441836,0.01295538,-0.0005645311,0,0,0,0.02758678,0,0,0.03529548,-0.001435192,-0.03072509,0.03703045,0,0,-0.0382676,-0.002350667,0,0,0.01078785,-0.001398656,0.02464549]},{"f":[23,21,15,-1,22,22,21,-1,-1,-1,17,22,-1,22,-1,-1,-1,25,18,22,2,-1,-1,-1,17,-1,-1,-1,-1],"t":[7.5,9.5,11.054604317960672,0,15.782972563647261,11.876328901062104,7.5,0,0,0,4.491496787347155,16.725912049439096,0,16.765402642760364,0,0,0,0.5,1.792481250360578,13.589275375459419,6.554507209719489,0,0,0,6.781300095563516,0,0,0,0],"m":[1,1,1,0,1,0,1,0,0,0,1,0,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0,0,0],"l":[1,2,3,0,5,6,7,0,0,0,11,12,0,14,0,0,0,18,19,20,21,0,0,0,25,0,0,0,0],"r":[28,17,4,0,10,9,8,0,0,0,16,13,0,15,0,0,0,27,24,23,22,0,0,0,26,0,0,0,0],"v":[0,0,0,0.0001717044,0,0,0,-0.01119803,0.01226257,-0.03870008,0,0,-0.03061427,0,0.009821786,-0.02115368,0.01720715,0,0,0,0,-0.01129842,0.01528353,0.01951297,0,-0.02385384,0.0005770256,0.03512267,-0.01510579]},{"f":[25,17,15,16,15,1,2,-1,-1,-1,-1,1,16,-1,-1,0,-1,15,-1,-1,21,-1,-1,17,16,-1,-1,-1,-1],"t":[6.5,2.160964047443681,10.066089190457772,7.930737337562887,7.679480099505446,5.489664082687339,6.434604125275413,0,0,0,0,2.9265316718587746,10.681238411777805,0,0,4.5,0,8.361943773735241,0,0,7.5,0,0,2.69615871138938,5.672425341971495,0,0,0,0],"m":[1,0,1,1,1,1,1,0,0,0,0,1,1,0,0,0,0,1,0,0,1,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,0,0,0,12,13,0,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[28,23,20,11,10,9,8,0,0,0,0,15,14,0,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,0,0,0,0,0,-0.02063876,0.003318765,0.005779536,-0.03163007,0,0,0.05206223,0.009809757,0,-0.01829017,0,0.01952295,0.0001341528,0,-0.03550612,-0.001097156,0,0,0.006273604,-0.03495474,-0.0002904841,-0.01335855]},{"f":[4,16,-1,17,22,-1,-1,0,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,-1,22,17,-1,-1,-1,1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,14.03140051979143,5.918419679874358,0,0,0,8.992857142857144,0,0],"m":[1,0,0,0,1,0,0,0,0,1,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,1,0,0],"l":[1,2,0,4,5,0,0,8,0,10,0,0,13,14,0,0,17,0,19,20,0,22,23,0,0,0,27,0,0],"r":[12,3,0,7,6,0,0,9,0,11,0,0,16,15,0,0,18,0,26,21,0,25,24,0,0,0,28,0,0],"v":[0,0,-0.02898804,0,0,0.004734501,0.04041734,0,-0.02164402,0,-0.001253426,0.02184157,0,0,-0.001138326,-0.02161096,0,0.01822622,0,0,7.838059e-05,0,0,0.01908821,-0.02560621,0.04961958,0,-0.0002946053,0.01803736]},{"f":[32,32,18,18,-1,-1,23,33,18,19,15,-1,-1,33,-1,-1,22,18,-1,-1,-1,-1,17,-1,-1,-1,18,-1,-1],"t":[10.463013322934223,10.41931383093672,7.198837316474133,6.864066550153108,0,0,3.5,21.257266042509173,8.03615005796417,1.5,6.820178962415188,0,0,21.011466140970732,0,0,12.664623226706265,9.265614570460752,0,0,0,0,8.571721503073015,0,0,0,3.5221970596792267,0,0],"m":[1,1,1,1,0,0,1,1,0,0,1,0,0,1,0,0,1,0,0,0,0,0,1,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,10,11,0,0,14,0,0,17,18,0,0,0,0,23,0,0,0,27,0,0],"r":[26,25,6,5,0,0,22,21,16,13,12,0,0,15,0,0,20,19,0,0,0,0,24,0,0,0,28,0,0],"v":[0,0,0,0,-0.0002646311,-0.03273447,0,0,0,0,0,-0.01250424,0.05171404,0,-0.0240908,0.01130802,0,0,-0.01969439,0.0018294,0.01516394,-0.007125205,0,-0.03164153,-0.001045833,-0.008383151,0,0.01258778,-0.01683235]},{"f":[22,21,-1,22,22,33,-1,-1,21,21,15,-1,-1,-1,22,-1,-1,16,-1,22,-1,15,16,-1,25,-1,-1,-1,-1],"t":[14.116600814324018,3.5,0,13.129761635363703,12.613886328427084,8.532371382375128,0,0,6.5,4.5,4.247927513443585,0,0,0,12.679695715883515,0,0,6.832890014164741,0,13.589275375459419,0,10.457380879072534,7.330916878114617,0,1.5,0,0,0,0],"m":[1,0,0,1,1,0,0,0,0,1,0,0,0,0,0,0,0,0,0,0,0,1,0,0,0,0,0,0,0],"l":[1,2,0,4,5,6,0,0,9,10,11,0,0,0,15,0,0,18,0,20,0,22,23,0,25,0,0,0,0],"r":[28,3,0,17,8,7,0,0,14,13,12,0,0,0,16,0,0,19,0,21,0,27,24,0,26,0,0,0,0],"v":[0,0,-0.02170537,0,0,0,-0.02895092,2.788778e-05,0,0,0,0.008283474,-0.02295847,0.02530636,0,-0.02848512,0.003626055,0,-0.03735856,0,-0.01844812,0,0,0.01410868,0,0.003503246,-0.02135924,0.02433066,0.0008761896]},{"f":[4,4,21,22,16,17,2,-1,2,-1,-1,2,-1,-1,22,-1,-1,-1,25,20,-1,-1,-1,16,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,9.5,14.116600814324018,10.066089190457772,5.83200138144325,5.357122758833062,0,6.209321624687949,0,0,5.065928480304397,0,0,12.606173770495205,0,0,0,0.5,0.5,0,0,0,3.321928094887362,0,0,10.510264838663748,0,0],"m":[1,1,1,1,1,1,1,0,1,0,0,1,0,0,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,6,7,0,9,0,0,12,0,0,15,0,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,17,14,11,8,0,10,0,0,13,0,0,16,0,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,0,0,0.0006240543,0,-0.03967037,0.002036687,0,-0.0407844,-0.006726319,0,0.0008870068,0.02398334,0.0008842878,0,0,0.01100079,-0.008142506,0.0346951,0,0.00238734,0.02084421,0,-0.022301,0.001155364]},{"f":[36,-1,22,21,1,-1,2,-1,-1,-1,22,17,18,-1,21,33,-1,-1,-1,-1,22,21,-1,-1,4,22,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,0,14.158358850608169,7.67595272073847,3.5221970596792267,0,4.5,15.968216440900115,0,0,0,0,15.439474430023438,7.5,0,0,8.049190845056867,15.676331898425197,0,0,0],"m":[0,0,1,0,1,0,1,0,0,0,0,1,0,0,1,1,0,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,0,11,12,13,0,15,16,0,0,0,0,21,22,0,0,25,26,0,0,0],"r":[2,0,10,9,6,0,8,0,0,0,20,19,14,0,18,17,0,0,0,0,24,23,0,0,28,27,0,0,0],"v":[0,0.01121195,0,0,0,-0.03408446,0,-0.02380469,0.01279845,-0.000541759,0,0,0,0.02648216,0,0,0.03230134,-0.001455192,-0.02950427,0.03577266,0,0,-0.03758023,-0.002185425,0,0,0.01013767,-0.001380137,0.0231694]},{"f":[23,22,33,16,37,-1,-1,33,18,-1,33,18,33,-1,15,-1,1,-1,-1,-1,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,35.98606694718995,0.0,8.2,0,0,32.32039757950231,7.198837316474133,0,21.257266042509173,8.03615005796417,19.167172160526107,0,6.820178962415188,0,4.757007376185458,0,0,0,0,0,7.859614184977944,16.975010474608837,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,0,1,0,0,0,1,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,0,11,12,13,0,15,0,17,0,0,0,0,0,23,24,0,0,0,0,0],"r":[28,27,22,7,6,0,0,21,10,0,20,19,14,0,16,0,18,0,0,0,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.032848,0.004623132,0,0,-0.0004886829,0,0,0,-0.02078839,0,0.005895375,0,0.0002273617,0.05470741,0.000675328,-0.00702401,0.008440921,0,0,0.01555368,-0.006539068,-0.03257316,0.0149061,-0.01447693]},{"f":[17,3,-1,15,15,15,3,-1,1,22,-1,-1,2,-1,-1,-1,1,-1,15,-1,-1,16,-1,-1,17,22,-1,-1,-1],"t":[2.160964047443681,0.23857586512866016,0,10.066089190457772,7.930737337562887,7.679480099505446,0.8758561643835616,0,5.3560606060606055,12.921158001713904,0,0,2.453445297804259,0,0,0,2.6153846153846154,0,8.005624549193879,0,0,11.696967526234287,0,0,2.69615871138938,16.752707246237605,0,0,0],"m":[0,0,0,1,1,1,0,0,1,0,0,0,1,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0],"l":[1,2,0,4,5,6,7,0,9,10,0,0,13,0,0,0,17,0,19,0,0,22,0,0,25,26,0,0,0],"r":[24,3,0,21,16,15,8,0,12,11,0,0,14,0,0,0,18,0,20,0,0,23,0,0,28,27,0,0,0],"v":[0,0,0.01836958,0,0,0,0,-0.02171847,0,0,-4.068974e-05,-0.0330322,0,0.002727684,0.0475376,-0.03045184,0,0.04304723,0,0.01636597,-0.002306708,0,-0.03636079,-0.0008601216,0,0,-0.03577964,0.004556282,-0.000299649]},{"f":[36,-1,22,21,1,-1,2,-1,-1,19,22,22,22,22,33,-1,-1,21,-1,-1,-1,-1,16,-1,15,-1,-1,-1,-1],"t":[1.5,0,14.116600814324018,3.5,6.069196428571429,0,2.453445297804259,0,0,2.5,13.930814526604713,13.589275375459419,13.129761635363703,12.613886328427084,8.532371382375128,0,0,7.5,0,0,0,0,7.266786540694901,0,5.672425341971495,0,0,0,0],"m":[0,0,1,0,1,0,1,0,0,1,1,1,1,1,0,0,0,1,0,0,0,0,1,0,0,0,0,0,0],"l":[1,0,3,4,5,0,7,0,0,10,11,12,13,14,15,0,0,18,0,0,0,0,23,0,25,0,0,0,0],"r":[2,0,28,9,6,0,8,0,0,27,22,21,20,17,16,0,0,19,0,0,0,0,24,0,26,0,0,0,0],"v":[0,0.01082348,0,0,0,-0.03366455,0,-0.02360236,0.01093684,0,0,0,0,0,0,-0.03235467,0.0002680472,0,0.01111827,-0.00324784,-0.02162411,0.01683858,0,-0.03239394,0,0.002812655,-0.02752625,-0.01208649,0.0008053466]},{"f":[4,4,21,22,16,-1,22,16,-1,22,-1,17,-1,-1,16,-1,-1,-1,25,20,-1,-1,-1,16,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,9.5,14.116600814324018,10.066089190457772,0,12.199665042528586,10.462502272597265,0,11.67903991803059,0,9.265614570460752,0,0,10.681238411777805,0,0,0,0.5,0.5,0,0,0,3.321928094887362,0,0,10.510264838663748,0,0],"m":[1,1,1,1,1,0,1,0,0,1,0,1,0,0,0,0,0,0,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,10,0,12,0,0,15,0,0,0,19,20,0,0,0,24,0,0,27,0,0],"r":[26,23,18,17,6,0,14,9,0,11,0,13,0,0,16,0,0,0,22,21,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0,-0.005004915,0,0,-0.0131936,0,-0.0005356274,0,-0.001144047,0.04454448,0,0.03158605,-0.003986986,0.0008175038,0,0,0.01036369,-0.007614103,0.03419294,0,0.002318613,0.01946517,0,-0.02165032,0.001080498]},{"f":[4,35,16,-1,17,22,-1,-1,0,15,-1,-1,15,0,22,22,-1,-1,22,-1,2,-1,-1,-1,-1,-1,22,-1,-1],"t":[3.245926548164837,2.5,2.0,0,0.5,15.717370311433935,0,0,2.5,11.683433292835403,0,0,12.000704269011246,10.5,16.73920708776685,15.717370311433935,0,0,16.765402642760364,0,6.055241654908112,0,0,0,0,0,17.094543568094064,0,0],"m":[1,1,0,0,0,1,0,0,0,1,0,0,1,1,1,1,0,0,0,0,1,0,0,0,0,0,1,0,0],"l":[1,2,3,0,5,6,0,0,9,10,0,0,13,14,15,16,0,0,19,0,21,0,0,0,0,0,27,0,0],"r":[26,25,4,0,8,7,0,0,12,11,0,0,24,23,18,17,0,0,20,0,22,0,0,0,0,0,28,0,0],"v":[0,0,0,-0.02626633,0,0,0.004897532,0.0383355,0,0,-0.03231743,-0.01072318,0,0,0,0,-0.002584429,0.003200933,0,-0.01072318,0,-0.01072318,0.005145272,0.01177902,0.01886583,-0.02829571,0,0.0005011922,0.02026176]},{"f":[36,-1,18,1,-1,1,23,-1,22,-1,-1,1,-1,-1,19,18,19,-1,17,22,-1,22,-1,-1,-1,18,-1,-1,-1],"t":[1.5,0,2.160964047443681,5.992857142857143,0,6.015625,0.5,0,12.679695715883515,0,0,6.241071428571429,0,0,2.5,5.303665156874805,0.5,0,7.67595272073847,15.717370311433935,0,15.782972563647261,0,0,0,5.425874520708029,0,0,0],"m":[0,0,1,1,0,0,0,0,0,0,0,0,0,0,1,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,0,3,4,0,6,7,0,9,0,0,12,0,0,15,16,17,0,19,20,0,22,0,0,0,26,0,0,0],"r":[2,0,14,5,0,11,8,0,10,0,0,13,0,0,28,25,18,0,24,21,0,23,0,0,0,27,0,0,0],"v":[0,0.01073131,0,0,-0.001111509,0,0,0.05844935,0,-0.02074106,0.01821854,0,-0.02834994,0.003399589,0,0,0,-0.001118159,0,0,-0.04656284,0,0.01881109,-0.02877042,0.01343702,0,0.01547718,9.411942e-05,-0.03204478]},{"f":[22,22,22,22,22,-1,23,16,-1,-1,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,16.975010474608837,16.844368517741394,0,5.5,6.0,0,0,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,0,0],"m":[1,1,1,1,1,0,1,0,0,0,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,0,0,0,0,13,0,0,16,0,18,0,0,21,0,23,24,25,0,0,0,0],"r":[20,15,12,11,6,0,10,9,0,0,0,0,14,0,0,17,0,19,0,0,22,0,28,27,26,0,0,0,0],"v":[0,0,0,0,0,-0.0001741651,0,0,0.00989552,-0.03189508,0.0320004,-0.01397096,0,-0.01707441,0.01199941,0,-0.02750653,0,-0.02528663,0.004844778,0,0.03048668,0,0,0,-0.006123794,0.03543431,-0.0102573,-0.02666742]},{"f":[23,22,33,16,37,-1,-1,33,22,22,22,-1,-1,21,-1,-1,16,-1,-1,0,-1,-1,17,22,-1,-1,-1,-1,-1],"t":[7.5,17.105450904090162,35.98606694718995,0.0,8.2,0,0,29.297756302860655,16.759160379664443,16.73920708776685,16.725912049439096,0,0,7.5,0,0,9.047123912114026,0,0,10.5,0,0,7.859614184977944,16.975010474608837,0,0,0,0,0],"m":[1,1,1,0,1,0,0,1,1,1,1,0,0,0,0,0,1,0,0,1,0,0,1,1,0,0,0,0,0],"l":[1,2,3,4,5,0,0,8,9,10,11,0,0,14,0,0,17,0,0,20,0,0,23,24,0,0,0,0,0],"r":[28,27,22,7,6,0,0,19,16,13,12,0,0,15,0,0,18,0,0,21,0,0,26,25,0,0,0,0,0],"v":[0,0,0,0,0,-0.03239246,0.004305551,0,0,0,0,-0.0003872956,0.017824,0,0.001909178,-0.03249436,0,-0.00309557,0.01620558,0,-0.0003302304,0.01144512,0,0,0.01442817,-0.006189615,-0.0319793,0.01432902,-0.01436649]},{"f":[22,22,22,16,-1,33,-1,17,-1,-1,32,-1,-1,25,-1,22,-1,-1,36,-1,18,2,17,-1,-1,1,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.978341770814396,0.0,0,35.98606694718995,0,7.5507416557826375,0,0,2.69615871138938,0,0,3.5,0,17.069460217922263,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,7.9752100840336135,0,0,0],"m":[1,1,1,0,0,1,0,1,0,0,1,0,0,0,0,0,0,0,0,0,1,1,0,0,0,1,0,0,0],"l":[1,2,3,4,0,6,0,8,0,0,11,0,0,14,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,13,10,5,0,7,0,9,0,0,12,0,0,15,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,-0.01053498,0,0.0002413101,0,0.004028045,-0.03155643,0,-0.01675781,0.0116292,0,-0.02682762,0,-0.02491908,0.004774713,0,0.02968877,0,0,0,-0.005968699,0.03492938,0,-2.200378e-05,-0.02569682,-0.0262785]},{"f":[23,22,16,37,-1,22,-1,-1,33,33,18,-1,33,18,33,-1,15,-1,-1,-1,-1,0,-1,-1,17,-1,-1,-1,-1],"t":[7.5,17.105450904090162,0.0,8.2,0,16.975010474608837,0,0,35.98606694718995,29.297756302860655,7.198837316474133,0,21.257266042509173,8.03615005796417,19.167172160526107,0,6.820178962415188,0,0,0,0,10.5,0,0,7.5507416557826375,0,0,0,0],"m":[1,1,0,1,0,0,0,0,1,1,1,0,1,0,0,0,1,0,0,0,0,1,0,0,1,0,0,0,0],"l":[1,2,3,4,0,6,0,0,9,10,11,0,13,14,15,0,17,0,0,0,0,22,0,0,25,0,0,0,0],"r":[28,27,8,5,0,7,0,0,24,21,12,0,20,19,16,0,18,0,0,0,0,23,0,0,26,0,0,0,0],"v":[0,0,0,0,-0.03187602,0,0.01414252,-0.003959792,0,0,0,-0.00060122,0,0,0,-0.0202799,0,0.005676263,0.03932839,0.000549893,-0.007307062,0,-0.0003293615,0.01088567,0,0.003897222,-0.0311391,0.01389528,-0.01425653]},{"f":[4,4,1,1,-1,-1,21,22,22,-1,16,-1,2,-1,-1,-1,17,17,23,-1,-1,-1,-1,2,-1,-1,2,-1,-1],"t":[9.266718101290426,9.12919100246427,0.4906446161274747,0.00982698026493647,0,0,9.5,14.116600814324018,13.129761635363703,0,7.118941072723508,0,2.453445297804259,0,0,0,6.781300095563516,6.584884225259038,5.5,0,0,0,0,9.20088558729989,0,0,10.510264838663748,0,0],"m":[1,1,0,0,0,0,1,1,1,0,1,0,1,0,0,0,1,1,1,0,0,0,0,1,0,0,0,0,0],"l":[1,2,3,4,0,0,7,8,9,0,11,0,13,0,0,0,17,18,19,0,0,0,0,24,0,0,27,0,0],"r":[26,23,6,5,0,0,16,15,10,0,12,0,14,0,0,0,22,21,20,0,0,0,0,25,0,0,28,0,0],"v":[0,0,0,0,0.002054828,-0.03081351,0,0,0,-0.0006639095,0,-0.03913285,0,-0.02991954,0.0004621483,0.0009034818,0,0,0,0.01040111,-0.01834043,-0.01957415,0.02989109,0,0.01719827,0.0007603995,0,-0.02110157,0.001049236]},{"f":[23,22,22,22,16,-1,22,22,22,-1,-1,-1,22,-1,-1,-1,22,-1,2,-1,-1,32,-1,33,-1,18,-1,-1,-1],"t":[7.5,17.083799521744545,16.98920034885663,16.978341770814396,0.0,0,16.844368517741394,16.786010507999382,16.765402642760364,0,0,0,16.903290731167747,0,0,0,17.078739781790915,0,6.011183906514227,0,0,8.341935934311302,0,37.46769166150554,0,5.491496787347155,0,0,0],"m":[1,1,1,1,0,0,1,1,1,0,0,0,1,0,0,0,1,0,1,0,0,0,0,0,0,0,0,0,0],"l":[1,2,3,4,5,0,7,8,9,0,0,0,13,0,0,0,17,0,19,0,0,22,0,24,0,26,0,0,0],"r":[28,21,16,15,6,0,12,11,10,0,0,0,14,0,0,0,18,0,20,0,0,23,0,25,0,27,0,0,0],"v":[0,0,0,0,0,-0.009827781,0,0,0,-0.0001587768,0.02427942,-0.01061054,0,0.03689536,-0.01406602,0.01427867,0,-0.01617316,0,-0.02042728,0.005927456,0,-0.02141124,0,0.0396384,0,-0.01433463,0.009490787,-0.01429643]},{"f":[36,-1,18,1,21,19,-1,-1,-1,1,23,-1,22,-1,-1,1,-1,25,22,22,-1,22,-1,-1,-1,-1,19,-1,-1],"t":[1.5,0,2.160964047443681,5.992857142857143,4.5,1.5,0,0,0,6.015625,0.5,0,12.679695715883515,0,0,6.241071428571429,0,1.5,15.782972563647261,14.03140051979143,0,15.676331898425197,0,0,0,0,2.5,0,0],"m":[0,0,1,1,0,1,0,0,0,0,0,0,0,0,0,0,0,0,1,1,0,0,0,0,0,0,1,0,0],"l":[1,0,3,4,5,6,0,0,0,10,11,0,13,0,0,16,0,18,19,20,0,22,0,0,0,0,27,0,0],"r":[2,0,26,9,8,7,0,0,0,15,12,0,14,0,0,17,0,25,24,21,0,23,0,0,0,0,28,0,0],"v":[0,0.01059414,0,0,0,0,-0.02948498,0.003453432,-0.0005478812,0,0,0.05639407,0,-0.02046758,0.01741503,0,-0.02779637,0,0,0,0.001934643,0,0.03481605,0.01123725,-0.004170599,-0.001578983,0,-0.0006724874,-0.03158055]},{"f":[4,16,-1,17,22,-1,-1,0,15,-1,-1,15,-1,-1,32,19,-1,-1,33,-1,2,16,-1,22,17,-1,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,15.717370311433935,0,0,2.5,11.683433292835403,0,0,12.000704269011246,0,0,6.208926257442949,0.5,0,0,21.16864017939033,0,5.357122758833062,6.820178962415188,0,14.03140051979143,1.292481250360578,0,0,0,0],"m":[1,0,0,0,1,0,0,0,1,0,0,1,0,0,0,1,0,0,0,0,0,1,0,1,1,0,0,0,0],"l":[1,2,0,4,5,0,0,8,9,0,0,12,0,0,15,16,0,0,19,0,21,22,0,24,25,0,0,0,0],"r":[14,3,0,7,6,0,0,11,10,0,0,13,0,0,18,17,0,0,20,0,28,23,0,27,26,0,0,0,0],"v":[0,0,-0.02771688,0,0,0.004936615,0.03698473,0,0,-0.03191672,-0.01059708,0,-0.001198192,0.01731528,0,0,-0.001015727,-0.0198412,0,0.01674298,0,0,0.0005543455,0,0,0.0232747,-0.01521563,0.04493106,3.031976e-05]},{"f":[24,22,-1,-1,15,17,3,-1,15,15,15,3,-1,-1,-1,1,-1,15,-1,-1,16,-1,-1,17,22,-1,-1,-1,-1],"t":[2.5,16.725912049439096,0,0,13.323758479706308,2.160964047443681,0.23857586512866016,0,10.066089190457772,7.930737337562887,7.679480099505446,0.8758561643835616,0,0,0,2.6153846153846154,0,8.005624549193879,0,0,11.696967526234287,0,0,2.69615871138938,16.752707246237605,0,0,0,0],"m":[0,1,0,0,1,0,0,0,1,1,1,0,0,0,0,0,0,0,0,0,1,0,0,0,1,0,0,0,0],"l":[1,2,0,0,5,6,7,0,9,10,11,12,0,0,0,16,0,18,0,0,21,0,0,24,25,0,0,0,0],"r":[4,3,0,0,28,23,8,0,20,15,14,13,0,0,0,17,0,19,0,0,22,0,0,27,26,0,0,0,0],"v":[0,0,-0.02468129,0.001955816,0,0,0,0.01795559,0,0,0,0,-0.02093395,0.001593168,-0.02974698,0,0.039573,0,0.01485571,-0.002378856,0,-0.03567978,0.008174359,0,0,-0.03469858,0.004563038,-0.0003286931,0.02301642]},{"f":[4,16,-1,17,-1,0,-1,22,22,2,3,-1,22,36,15,-1,-1,-1,33,22,-1,-1,-1,-1,-1,16,-1,-1,-1],"t":[3.245926548164837,2.0,0,0.5,0,2.5,0,16.73920708776685,16.725912049439096,6.515505953718852,0.013290990355210539,0,15.676331898425197,4.5,12.000704269011246,0,0,0,13.660441030622824,15.782972563647261,0,0,0,0,0,5.169925001442312,0,0,0],"m":[1,0,0,0,0,0,0,1,1,1,0,0,1,1,1,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"l":[1,2,0,4,0,6,0,8,9,10,11,0,13,14,15,0,0,0,19,20,0,0,0,0,0,26,0,0,0],"r":[28,3,0,5,0,7,0,25,24,23,12,0,18,17,16,0,0,0,22,21,0,0,0,0,0,27,0,0,0],"v":[0,0,-0.02730848,0,0.01382737,0,-0.01978536,0,0,0,0,0.01593739,0,0,0,-0.00194808,0.02317662,0.01090724,0,0,0.01430373,-0.008809323,-0.01030266,-0.01030266,0.01603699,0,0.02892904,-0.01022444,0.0006918482]},{"f":[22,22,22,22,-1,-1,32,-1,2,-1,22,-1,-1,22,-1,2,-1,-1,36,-1,18,2,17,-1,-1,25,-1,-1,-1],"t":[17.083799521744545,17.02942582914998,16.838461650489297,16.819976309430963,0,0,6.337096134072842,0,4.128693921346326,0,16.978341770814396,0,0,17.078739781790915,0,6.011183906514227,0,0,4.5,0,8.729619742794132,1.292481250360578,6.8199529586472405,0,0,1.5,0,0,0],"m":[1,1,1,1,0,0,0,0,1,0,1,0,0,1,0,1,0,0,0,0,1,1,0,0,0,0,0,0,0],"l":[1,2,3,4,0,0,7,0,9,0,11,0,0,14,0,16,0,0,19,0,21,22,23,0,0,26,0,0,0],"r":[18,13,6,5,0,0,8,0,10,0,12,0,0,15,0,17,0,0,20,0,28,25,24,0,0,27,0,0,0],"v":[0,0,0,0,3.589778e-06,-0.02867953,0,0.02596594,0,-0.02250964,0,-0.003091871,0.01277347,0,-0.02474156,0,-0.02036325,0.005026073,0,0.02861016,0,0,0,-0.005674536,0.03452174,0,0.002328657,-0.02091042,-0.02588668]}]}} \ No newline at end of file diff --git a/hw/scripts/beacon_summary.py b/hw/scripts/beacon_summary.py new file mode 100644 index 0000000000..60c117e3cd --- /dev/null +++ b/hw/scripts/beacon_summary.py @@ -0,0 +1,319 @@ +#!/usr/bin/env python3 +"""Summarize a Beacon run (foldsim-estimate-timing, foldsim-estimate-area) +for logic_gate. + +Beacon writes one row per end point bit: the logic levels of its longest +path, its start and its end; and one row per instance: its resources and its +subtree's. ci/logic_gate.py gates depth per RTL unit and the design's area, +so this collapses the tables into what it compares: + + beacon_summary.csv one row: the deepest end point, the end point bits at or + above the fix and watch thresholds, all end points, and + the design's LUTs, flip-flops, distributed RAM LUTs, + block RAMs, UltraRAMs and DSPs (area.csv) + beacon_units.json the deepest end point of every unit at or above the + watch threshold, the deepest paths (diagnostics), and + `area`: each unit's resources, one copy's subtree + beacon_paths.txt if Beacon wrote paths.jsonl: every path at or above + the watch threshold, op by op, by unit (write_paths) + beacon_area.txt if Beacon wrote area.csv: the area summary, the units + by LUTs, and where each memory goes (write_area) + +If Beacon wrote ports.csv (BEACON_INTERFACES), beacon_units.json also holds +`interfaces`: per output port of those modules' instances, folded like +units, the bits a combinational path from outside the instance reaches. + +A unit is an end point's instance path with generate indices folded +(g_banks_3_bank -> g_banks_*_bank), so one deep bank reads as one unit. + + usage: beacon_summary.py --top [--output-dir ] +""" + +import argparse +import collections +import csv +import json +import os +import re + +# 300 MHz on the V80: failing end points have a median of 7-8 levels, and +# about 55-60% of the registers at 12 or more fail +# (docs/designs/beacon.md §7). +FIX_LEVELS = 12 +WATCH_LEVELS = 8 +NUM_PATHS = 10 + +COLUMNS = ("top", "max_levels", "deep_endpoints", "watch_endpoints", + "endpoints") +AREA = ("lut", "ff", "lutram", "bram", "uram", "dsp") +# Units of at least this share of the design's LUTs are listed in +# beacon_area.txt. +AREA_UNIT_SHARE = 0.01 + +INDEX = re.compile(r"_\d+(?=_|/|$)") + + +def norm(name): + return name.replace(".", "_").replace("[", "_").replace("]", "") + + +def instance_paths(path): + """Instance id -> path from the top, from a run's instances.csv.""" + rows = {} + with open(path) as fh: + for row in csv.DictReader(fh): + rows[int(row["id"])] = (int(row["parent"]), row["instance"]) + paths = {0: ""} + + def get(ctx): + if ctx not in paths: + parent, name = rows[ctx] + p = get(parent) + paths[ctx] = "%s/%s" % (p, name) if p else name + return paths[ctx] + + for ctx in rows: + get(ctx) + return paths + + +def unit(path): + """The unit of an end point: its directory, generate indices folded.""" + return INDEX.sub("_*", norm(path).rpartition("/")[0]) or "" + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + ap.add_argument("run_dir") + ap.add_argument("--top", required=True) + ap.add_argument("--output-dir", default=".") + args = ap.parse_args() + + paths = instance_paths(os.path.join(args.run_dir, "instances.csv")) + units = collections.defaultdict(int) + counts = collections.Counter() + deepest = [] + with open(os.path.join(args.run_dir, "endpoints.csv")) as fh: + for row in csv.DictReader(fh): + levels = int(row["levels"]) + counts["endpoints"] += 1 + if levels < WATCH_LEVELS: + continue + counts["watch_endpoints"] += 1 + counts["deep_endpoints"] += levels >= FIX_LEVELS + ip = paths[int(row["end_inst"])] + end = "%s/%s" % (ip, row["end"]) if ip else row["end"] + u = unit(end) + units[u] = max(units[u], levels) + sp = paths[int(row["start_inst"])] + deepest.append((levels, "%s/%s" % (sp, row["start"]) if sp + else row["start"], end)) + # Keep the list short: one sort at the end of a few thousand rows. + if len(deepest) > 50 * NUM_PATHS: + deepest = sorted(deepest, key=lambda d: -d[0])[:NUM_PATHS] + deepest = sorted(deepest, key=lambda d: -d[0])[:NUM_PATHS] + + summary = {"top": args.top, + "max_levels": max(units.values(), default=0), + "deep_endpoints": counts["deep_endpoints"], + "watch_endpoints": counts["watch_endpoints"], + "endpoints": counts["endpoints"]} + if not units: + # Nothing reaches the watch threshold; the deepest is still recorded. + with open(os.path.join(args.run_dir, "endpoints.csv")) as fh: + summary["max_levels"] = max( + (int(r["levels"]) for r in csv.DictReader(fh)), default=0) + + area = read_area(args.run_dir) + columns = COLUMNS + if area: + summary.update(area["total"]) + columns = COLUMNS + AREA + with open(os.path.join(args.output_dir, "beacon_summary.csv"), "w", + newline="") as fh: + w = csv.DictWriter(fh, fieldnames=columns) + w.writeheader() + w.writerow(summary) + doc = {"fix_levels": FIX_LEVELS, "watch_levels": WATCH_LEVELS, + "units": dict(sorted(units.items())), + "deepest_paths": ["%d %s -> %s" % d for d in deepest]} + if area: + doc["area"] = area["units"] + ports = os.path.join(args.run_dir, "ports.csv") + if os.path.exists(ports): + doc["interfaces"] = interfaces(ports) + print("beacon: %d interface outputs reached combinationally from " + "outside" % len(doc["interfaces"])) + with open(os.path.join(args.output_dir, "beacon_units.json"), "w") as fh: + json.dump(doc, fh, indent=2) + fh.write("\n") + print("beacon: %s max %d levels, %d end point bits >= %d, %d units >= %d" + % (args.top, summary["max_levels"], summary["deep_endpoints"], + FIX_LEVELS, len(units), WATCH_LEVELS)) + if os.path.exists(os.path.join(args.run_dir, "paths.jsonl")): + with open(os.path.join(args.output_dir, "beacon_paths.txt"), "w") as fh: + n, u = write_paths(fh, args.run_dir, WATCH_LEVELS, paths=paths) + print("beacon: %d paths >= %d levels in %d units -> beacon_paths.txt" + % (n, WATCH_LEVELS, u)) + if area: + with open(os.path.join(args.output_dir, "beacon_area.txt"), "w") as fh: + write_area(fh, args.run_dir, area) + t = area["total"] + print("beacon: %s %d LUTs (%d distributed RAM), %d FFs, %g block " + "RAMs, %d UltraRAMs, %d DSPs -> beacon_area.txt" + % (args.top, t["lut"], t["lutram"], t["ff"], t["bram"], + t["uram"], t["dsp"])) + + +def read_area(run_dir): + """The design's resources and each unit's from a run's area.csv, or None. + `lut` counts every LUT, distributed RAM included, as Vivado's utilization + does. A unit's resources are one copy's subtree; `copies` counts its + instances.""" + path = os.path.join(run_dir, "area.csv") + if not os.path.exists(path): + return None + total, units = None, {} + with open(path) as fh: + for row in csv.DictReader(fh): + res = {k: float(row["total_" + k]) for k in AREA} + res["lut"] += res["lutram"] + res = {k: (round(v, 1) if k == "bram" else int(round(v))) + for k, v in res.items()} + if total is None: + total = res + continue + u = fold(row["path"]) + if u in units: + units[u]["copies"] += 1 + else: + units[u] = dict(res, module=row["module"], copies=1) + return {"total": total, "units": dict(sorted(units.items()))} + + +def write_area(out, run_dir, area): + """The area report, the units holding at least AREA_UNIT_SHARE of the + LUTs, and every memory with its placement.""" + with open(os.path.join(run_dir, "area.txt")) as fh: + out.write(fh.read()) + t = area["total"] + floor = AREA_UNIT_SHARE * t["lut"] + out.write("\nUnits with at least %g%% of the LUTs, one copy's subtree " + "(x copies):\n" % (100 * AREA_UNIT_SHARE)) + out.write(" %9s %8s %7s %7s %5s %5s %s\n" + % ("lut", "ff", "lutram", "bram", "uram", "dsp", "unit")) + rows = sorted(area["units"].items(), key=lambda kv: -kv[1]["lut"]) + for u, r in rows: + if r["lut"] * r["copies"] < floor: + continue + out.write(" %9d %8d %7d %7g %5d %5d %s (%s) x%d\n" + % (r["lut"], r["ff"], r["lutram"], r["bram"], r["uram"], + r["dsp"], u, r["module"], r["copies"])) + mems = os.path.join(run_dir, "area_memories.csv") + if not os.path.exists(mems): + return + out.write("\nMemories (depth x width, read latency, ram_style -> " + "placement and cost, x copies):\n") + groups = collections.OrderedDict() + with open(mems) as fh: + for r in csv.DictReader(fh): + name = fold("%s/%s" % (r["path"], r["name"]) if r["path"] + else r["name"]) + key = (name, r["depth"], r["width"], r["latency"], r["style"], + r["placement"]) + if key in groups: + groups[key][1] += 1 + else: + groups[key] = [r, 1] + for (name, depth, width, latency, style, placement), (r, n) in \ + groups.items(): + cost = ("%s LUTs" % r["lutram"] if placement == "lutram" else + "%s blocks" % r["bram"] if placement == "bram" else + "%s UltraRAMs" % r["uram"] if placement == "uram" else + "%s FFs" % r["ff"]) + out.write(" %6sx%-5s lat %s %-11s -> %-6s %-12s %s x%d\n" + % (depth, width, latency, style, placement, cost, name, n)) + +def fold(name): + return INDEX.sub("_*", norm(name)) + + +def interfaces(path): + """'.' -> bits, for every output port of a kept module's + instance that a combinational path from outside reaches; generate copies + folded, keeping the most bits.""" + out = {} + with open(path) as fh: + for row in csv.DictReader(fh): + bits = int(row["through"]) + if row["direction"] != "out" or not bits: + continue + key = "%s.%s" % (fold(row["instance"]), row["port"]) + out[key] = max(out.get(key, 0), bits) + return dict(sorted(out.items())) + + +def write_paths(out, run_dir, min_levels, unit_filter=None, paths=None): + """Write a run's deep paths (paths.jsonl) by unit, op by op. + + Beacon lists one path per pair of start and end registers; this folds + the pairs whose names differ only in generate indices and bits, as units + fold them, keeping the deepest. Returns (paths, units) written. + """ + if paths is None: + paths = instance_paths(os.path.join(run_dir, "instances.csv")) + + def full(ctx, name): + return "%s/%s" % (paths[ctx], name) if paths[ctx] else name + + units = collections.defaultdict(dict) + with open(os.path.join(run_dir, "paths.jsonl")) as fh: + for line in fh: + p = json.loads(line) + if p["levels"] < min_levels: + continue + start = full(p["start_inst"], p["start"]) + end = full(p["end_inst"], p["end"]) + u = unit(end) + if unit_filter and unit_filter not in u: + continue + group = units[u] + key = (fold(start), fold(end)) + if key in group: + group[key]["copies"] += 1 + group[key]["bits"] += p["bits"] + else: + group[key] = {"path": p, "copies": 1, "bits": p["bits"], + "start": start, "end": end} + + def deepest(u): + return max(g["path"]["levels"] for g in units[u].values()) + + order = sorted(units, key=lambda u: (-deepest(u), u)) + count = sum(len(g) for g in units.values()) + out.write("%d paths at or above %d levels in %d units, deepest unit " + "first.\n" % (count, min_levels, len(units))) + out.write("A path is its deepest end point bit, op by op from the end " + "point back to its\nstart: levels, kind, fanout, source line, " + "instance/name. Bits of the same pair\nof registers, and " + "generate copies (_0, _1, ...), are folded into it.\n") + for u in order: + group = sorted(units[u].values(), key=lambda g: -g["path"]["levels"]) + out.write("\n== %s: %d levels, %d paths\n" % (u, deepest(u), + len(group))) + for g in group: + p = g["path"] + retimed = (", %d after retiming" % p["retimed"] + if p["retimed"] != p["levels"] else "") + copies = " in %d copies" % g["copies"] if g["copies"] > 1 else "" + out.write("\n %d levels%s, %d %s bits%s\n from %s\n to %s\n" + % (p["levels"], retimed, g["bits"], p["kind"], copies, + g["start"], g["end"])) + for levels, kind, fanout, ctx, name, source in p["ops"]: + out.write(" %4d %-6s fo %-5d %-24s %s\n" + % (levels, kind, fanout, source, full(ctx, name))) + return count, len(units) + + +if __name__ == "__main__": + main() diff --git a/hw/scripts/check_ip_contracts.py b/hw/scripts/check_ip_contracts.py new file mode 100755 index 0000000000..c20ff7f8c6 --- /dev/null +++ b/hw/scripts/check_ip_contracts.py @@ -0,0 +1,136 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Check the pipeline contracts of the Xilinx floating-point cores. + +Each black box in xilinx_ip_stubs.sv declares the latency it is used at +(`fixed_latency`). That latency must be the core's C_Latency in +xilinx_ip_gen.tcl, the latency the RTL around it asserts, its simulation +model's (sim/foldsim/xilinx_fp_models.sv) and, with --cflags, the +VX_CFG_*_LATENCY the configuration resolves to for it. + +usage: check_ip_contracts.py [--cflags '<-D flags of the build>'] +""" +import argparse +import os +import re +import subprocess +import sys + +HERE = os.path.dirname(os.path.realpath(__file__)) +ROOT = os.path.realpath(os.path.join(HERE, '..', '..')) + +# Where the RTL states the latency it uses each core at. +RTL_ASSERTS = { + 'xil_fma': ('hw/rtl/fpu/VX_fma_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fdiv': ('hw/rtl/fpu/VX_fdiv_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fsqrt': ('hw/rtl/fpu/VX_fsqrt_unit.sv', r'USE_VENDOR_IP\s*\|\|\s*\(LATENCY\s*==\s*(\d+)\)'), + 'xil_fmul': ('hw/rtl/tcu/dsp/VX_tcu_fedp_dsp.sv', r'localparam\s+FMUL_LATENCY\s*=\s*(\d+)'), + 'xil_fadd': ('hw/rtl/tcu/dsp/VX_tcu_fedp_dsp.sv', r'localparam\s+FADD_LATENCY\s*=\s*(\d+)'), +} + +# The configuration's latency of each core, when the configuration selects it. +CONFIG_LATENCIES = { + 'xil_fma': [('VX_CFG_FMA_LATENCY', 'VX_CFG_FPU_TYPE_DSP')], + 'xil_fdiv': [('VX_CFG_FDIV_LATENCY', 'VX_CFG_FPU_TYPE_DSP'), ('VX_CFG_RTU_FDIV_LATENCY', None)], + 'xil_fsqrt': [('VX_CFG_FSQRT_LATENCY', 'VX_CFG_FPU_TYPE_DSP')], +} + + +def stub_contracts(path): + """module -> its attributes, from the `(* ... *)` before each module.""" + text = open(path).read() + contracts = {} + for attrs, name in re.findall(r'\(\*(.*?)\*\)\s*module\s+(\w+)', text, re.S): + values = {} + for item in re.findall(r'(\w+)(?:\s*=\s*("[^"]*"|\w+))?', attrs): + values[item[0]] = item[1].strip('"') if item[1] else '' + contracts[name] = values + return contracts + + +def tcl_latencies(path): + """module -> C_Latency, from each core's set_property dictionary.""" + text = open(path).read() + latencies = {} + for body, name in re.findall(r'set_property -dict \[list (.*?)\] \[get_ips (\w+)\]', text, re.S): + m = re.search(r'CONFIG\.C_Latency \{(\d+)\}', body) + latencies[name] = int(m.group(1)) if m else None + return latencies + + +def model_latencies(path): + """module -> the latency of its simulation model's pipeline.""" + text = open(path).read() + return {name: int(lat) for name, lat in + re.findall(r'module (\w+) \(.*?xilinx_fp_model_pipe #\(\.LATENCY\((\d+)\)', text, re.S)} + + +def config_values(cflags): + out = subprocess.run([sys.executable, os.path.join(ROOT, 'ci', 'gen_config.py'), + '--config', os.path.join(ROOT, 'VX_config.toml'), '--cflags', cflags], + check=True, capture_output=True, text=True).stdout + values = {} + for flag in out.split(): + if flag.startswith('-D'): + name, _, value = flag[2:].partition('=') + values[name] = value + return values + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split('\n')[0]) + ap.add_argument('--cflags', help="the build's -D flags, to check its resolved latencies") + a = ap.parse_args() + errors = [] + contracts = stub_contracts(os.path.join(HERE, 'xilinx_ip_stubs.sv')) + tcl = tcl_latencies(os.path.join(HERE, 'xilinx_ip_gen.tcl')) + config = config_values(a.cflags) if a.cflags is not None else None + models = model_latencies(os.path.join(ROOT, 'sim', 'foldsim', 'xilinx_fp_models.sv')) + for name, rtl in RTL_ASSERTS.items(): + c = contracts.get(name) + if c is None or 'fixed_latency' not in c: + errors.append(f'{name}: no fixed_latency contract in xilinx_ip_stubs.sv') + continue + latency = int(c['fixed_latency']) + if tcl.get(name) != latency: + errors.append(f'{name}: contract latency {latency}, C_Latency {tcl.get(name)} in xilinx_ip_gen.tcl') + if models.get(name) != latency: + errors.append(f'{name}: contract latency {latency}, sim/foldsim/xilinx_fp_models.sv models {models.get(name)}') + path, pattern = rtl + m = re.search(pattern, open(os.path.join(ROOT, path)).read()) + if not m: + errors.append(f'{name}: {path} no longer states the latency it uses the core at') + elif int(m.group(1)) != latency: + errors.append(f'{name}: contract latency {latency}, {path} uses {m.group(1)}') + if config is None: + continue + for knob, selector in CONFIG_LATENCIES.get(name, []): + if selector and selector not in config: + continue + # The RTU uses the vendor divider on the Vivado flow only. + if knob == 'VX_CFG_RTU_FDIV_LATENCY' and not re.search(r'-DVIVADO\b', a.cflags): + continue + if knob in config and int(config[knob]) != latency: + errors.append(f'{name}: contract latency {latency}, the configuration resolves {knob}={config[knob]}') + for e in errors: + print(f'ERROR: {e}', file=sys.stderr) + if errors: + sys.exit(1) + latencies = ", ".join(f"{n} {contracts[n]['fixed_latency']}" for n in RTL_ASSERTS) + print(f'xilinx IP contracts: {latencies} consistent') + + +if __name__ == '__main__': + main() diff --git a/hw/scripts/circt_elaborate.sh b/hw/scripts/circt_elaborate.sh new file mode 100755 index 0000000000..d6f86f95ec --- /dev/null +++ b/hw/scripts/circt_elaborate.sh @@ -0,0 +1,178 @@ +#!/bin/bash + +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Elaborates SystemVerilog sources with CIRCT and writes the design as one +# file, the netlist Yosys, Beacon and Foldsim read (Vivado reads the sources +# themselves). Asynchronous RAM reads are made synchronous before this, in the +# sources (gen_sources.sh -F); here: +# +# - memories are detected, and a RAM that asks for block RAM with its +# synthesis attribute (ram_style = "block", ...) reads synchronously, so +# that it maps to a block RAM or an SRAM macro (seq-retime-firmem-reads). +# A request that cannot be honored stops the build with the reason. -m +# decides for RAMs without a hint: off (the default) leaves them, auto +# converts those it can. A report lists every asynchronous-read RAM; +# - black-box modules (vendor IP, SRAM macros) stay instances of +# themselves, with their parameter values, for the tool to bind. -x names +# a file of their declarations (ports only), read for their interfaces; +# -b names modules declared in the sources themselves; +# - a conditional write of one element of an array at a dynamic index +# becomes a select per element, as the statement means; +# - a RAM whose read port is registered states its read-under-write +# behavior to Vivado (rw_addr_collision), which otherwise leaves a read +# colliding with a write undefined in block RAM; other tools ignore it; +# - -f sv (the default) writes SystemVerilog, for Vivado and Quartus; -f +# v2005 writes Verilog-2005 for tools with a Verilog frontend (yosys): no +# packed arrays or structs, no local variables, no size casts; -f mlir +# writes the HW/Comb/Seq netlist itself, for tools built on CIRCT (the +# Beacon), and exports nothing. +# +# The sources are a filelist (-l, as gen_sources.sh writes it: +define+, +# +incdir+ and files in compile order), or include directories whose sources +# are all read (-I). CIRCT_VERILOG, CIRCT_OPT and FIRTOOL name the tools +# (hw/syn/common.mk). + +CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" +CIRCT_OPT="${CIRCT_OPT:-circt-opt}" +FIRTOOL="${FIRTOOL:-firtool}" + +set -eo pipefail + +top="" +filelist="" +format=sv +mem_mode=off +report="" +includes=() +defines=() +blackboxes=() +stubs=() +output_file=out.sv + +usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } +[ $# -eq 0 ] && usage +while getopts "t:l:o:I:D:b:x:f:m:r:h" arg; do + case $arg in + t) # top module + top=${OPTARG} + ;; + l) # filelist (+define+, +incdir+, files) + filelist=${OPTARG} + ;; + o) # output file + output_file=${OPTARG} + ;; + I) # include directory, whose sources are read too + includes+=("${OPTARG}") + ;; + D) # macro definition + defines+=("-D${OPTARG}") + ;; + b) # black-box modules declared in the sources, comma-separated + IFS=',' read -r -a names <<< "${OPTARG}" + for name in "${names[@]}"; do + [ -n "$name" ] && blackboxes+=("--blackbox=$name") + done + ;; + x) # file of black-box declarations (vendor IP) + stubs+=("${OPTARG}") + ;; + f) # output format: sv, v2005 or mlir + format=${OPTARG} + ;; + m) # RAMs without a hint: off or auto + mem_mode=${OPTARG} + ;; + r) # memory report (default: next to the output) + report=${OPTARG} + ;; + h | *) + usage + exit 0 + ;; + esac +done + +if [ -z "$top" ]; then + echo "circt_elaborate.sh: error: no top module (-t)" >&2 + exit 1 +fi +case "$format" in + sv|v2005|mlir) ;; + *) echo "circt_elaborate.sh: error: format must be sv, v2005 or mlir, not '$format'" >&2 + exit 1 ;; +esac +[ -z "$report" ] && report="${output_file%.*}.memories.rpt" + +args=() +files=() +if [ -n "$filelist" ]; then + while IFS= read -r line; do + case "$line" in + "") ;; + +define+*) args+=("-D${line#+define+}") ;; + +incdir+*) args+=("-I${line#+incdir+}") ;; + *) files+=("$line") ;; + esac + done < "$filelist" +fi +for dir in "${includes[@]}"; do + args+=("-I$dir") + while IFS= read -r file; do + files+=("$file") + done < <(find "$dir" -maxdepth 1 -type f \( -name '*.v' -o -name '*.sv' \) | sort) +done +for stub in "${stubs[@]}"; do + files+=("$stub") + while IFS= read -r name; do + blackboxes+=("--blackbox=$name") + done < <(sed -nE 's/^[[:space:]]*module[[:space:]]+([A-Za-z_][A-Za-z0-9_$]*).*/\1/p' "$stub") +done + +stem="${output_file%.*}" +# The netlist is the output itself in mlir format; keep the import beside it, +# and every op's source location: the tools built on the netlist name logic +# that has no name of its own by its file and line. +debuginfo=() +if [ "$format" = mlir ]; then + stem="$stem.import" + debuginfo=(--mlir-print-debuginfo) +fi +"$CIRCT_VERILOG" --ir-hw --top="$top" --detect-memories --allow-use-before-declare \ + --allow-hierarchical-const --ignore-uninstantiated-modules "${blackboxes[@]}" \ + "${defines[@]}" "${args[@]}" "${files[@]}" "${debuginfo[@]}" -o "$stem.mlir" + +# Memories are detected on import; what remains of a conditional write at a +# dynamic index (`if (c) x[i] = v`) is expanded into per-element selects. +passes=(--hw-expand-array-writes "--seq-retime-firmem-reads=mode=$mem_mode report=$report") +if [ "$format" = v2005 ]; then + passes+=(--seq-lower-aggregate-regs --hw-aggregate-ports-to-int --hw-convert-bitcasts + --hw-aggregate-to-comb) + lowering=noAlwaysComb,disallowLocalVariables,disallowPackedArrays,disallowPackedStructAssignments,disallowSizeCasts,svVarDeclStyle=reg,locationInfoStyle=none,omitVersionComment +else + lowering=locationInfoStyle=none,omitVersionComment +fi +# A module body is a graph region, where CSE cannot replace a duplicate whose +# users come first; sorting the ops first lets it merge every duplicate. +if [ "$format" = mlir ]; then + "$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ + "${debuginfo[@]}" -o "$output_file" + exit 0 +fi +"$CIRCT_OPT" "$stem.mlir" "${passes[@]}" --canonicalize --topological-sort --cse \ + -o "$stem.lowered.mlir" +"$FIRTOOL" --format=mlir "$stem.lowered.mlir" --disable-all-randomization \ + --add-vivado-ram-address-conflict-synthesis-bug-workaround \ + --lowering-options="$lowering" -o "$output_file" diff --git a/hw/scripts/gen_platform.py b/hw/scripts/gen_platform.py new file mode 100644 index 0000000000..84dbd31a0b --- /dev/null +++ b/hw/scripts/gen_platform.py @@ -0,0 +1,375 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Resolve a Xilinx card descriptor and emit what a build flow consumes. + +Card facts live in hw/syn/xilinx/platforms/.toml, one file per card. A +flow names the card the way it already does -- the XRT flow by platform XSA +and FPGA family, the AVED flow by device part -- and this script resolves the +descriptor and emits one artifact: + + mk make fragment for hw/syn/xilinx/{xrt,aved}/Makefile + cfg slashkit linker config (AVED flow) + firesim-mk make fragment for hw/syn/firesim and sim/firesim + scala the card's FireSim platform config class + card the resolved card name + family the FPGA family of a device part, from any card on it (the DUT + synthesis flow, which builds for a part rather than a card) +""" + +from __future__ import annotations + +import argparse +import fnmatch +import os +import sys +from dataclasses import dataclass, field +from typing import List, Optional + +_ROOT = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..")) +PLATFORM_DIR = os.path.join(_ROOT, "hw", "syn", "xilinx", "platforms") + +# The configuration generator's TOML reader: these builds run under whatever +# python3 the tool environment provides (FireSim's conda env is 3.10, before +# tomllib), and descriptors stay within the subset it reads. +sys.path.insert(0, os.path.join(_ROOT, "ci")) +from gen_config import TinyTOML # noqa: E402 + +FLOWS = ("xrt", "aved") + +# The AFU kernel instance name each linker assigns. +KERNEL_INSTANCE = {"xrt": "vortex_afu_1", "aved": "vortex_afu_0"} + + +# Width of the PLATFORM_MEMORY_OFFSET literal the AFU adds to device addresses. +OFFSET_BITS = 40 + + +class DescriptorError(Exception): + pass + + +def verilog_offset(base: int) -> str: + # Sized hex, escaped for make and the shell: Vivado truncates an oversized + # decimal constant to 0 with only a warning. + return f"{OFFSET_BITS}\\'h{base:X}" + + +@dataclass +class Card: + name: str + path: str + flow: str + part: str + family: str + match_xsa: List[str] + match_family: List[str] + match_device: List[str] + banks: int + addr_width: int + base: int + mem_tag: str + host_tag: str + host_refuse: List[str] + host_refuse_why: str + kernel_mhz: int + axil_block_bytes: int + firesim: dict = field(default_factory=dict) + + def specificity(self) -> int: + # An exact XSA or device pattern beats a family match, which beats a + # catch-all, so a card-specific descriptor always wins. + if any(p != "*" for p in self.match_xsa + self.match_device): + return 2 + if self.match_family: + return 1 + return 0 + + +def _require(d: dict, key: str, path: str, ctx: str): + if key not in d: + raise DescriptorError(f"{path}: missing [{ctx}] {key}") + return d[key] + + +def load_card(path: str) -> Card: + with open(path, "r", encoding="utf-8") as f: + t = TinyTOML().parse(f.read()) + card = _require(t, "card", path, "card") + mem = _require(t, "memory", path, "memory") + host = t.get("host", {}) + clock = t.get("clock", {}) + control = t.get("control", {}) + flow = _require(card, "flow", path, "card") + if flow not in FLOWS: + raise DescriptorError(f"{path}: flow must be one of {FLOWS} (got '{flow}')") + c = Card( + name=_require(card, "name", path, "card"), + path=path, + flow=flow, + part=card.get("part", ""), + family=card.get("family", ""), + match_xsa=list(card.get("match_xsa", [])), + match_family=list(card.get("match_family", [])), + match_device=list(card.get("match_device", [])), + banks=int(_require(mem, "banks", path, "memory")), + addr_width=int(_require(mem, "addr_width", path, "memory")), + base=int(mem.get("base", 0)), + mem_tag=str(mem.get("tag", "")), + host_tag=str(host.get("tag", "")), + host_refuse=list(host.get("refuse", [])), + host_refuse_why=str(host.get("refuse_why", "")), + kernel_mhz=int(clock.get("kernel_mhz", 300)), + axil_block_bytes=int(control.get("axil_block_bytes", 0)), + firesim=dict(t.get("firesim", {})), + ) + if not 0 <= c.base < (1 << OFFSET_BITS): + raise DescriptorError(f"{path}: [memory] base 0x{c.base:x} does not fit {OFFSET_BITS} bits") + if c.host_tag and c.host_tag in c.host_refuse: + raise DescriptorError(f"{path}: host tag '{c.host_tag}' is in its own refuse list") + return c + + +def load_all(platform_dir: str) -> List[Card]: + cards = [] + for fn in sorted(os.listdir(platform_dir)): + if fn.endswith(".toml"): + cards.append(load_card(os.path.join(platform_dir, fn))) + names = [c.name for c in cards] + dups = {n for n in names if names.count(n) > 1} + if dups: + raise DescriptorError(f"duplicate card names: {sorted(dups)}") + return cards + + +def resolve(cards: List[Card], flow: str, xsa: str, family: str, device: str, name: str) -> Card: + pool = [c for c in cards if not flow or c.flow == flow] + if name: + hits = [c for c in pool if c.name == name] + if not hits: + raise DescriptorError(f"no {flow or 'Xilinx'} card named '{name}'; known: {sorted(c.name for c in pool)}") + return hits[0] + + def matches(c: Card) -> bool: + if xsa and any(fnmatch.fnmatchcase(xsa, p) for p in c.match_xsa): + return True + if device and any(fnmatch.fnmatchcase(device, p) for p in c.match_device): + return True + if family and family in c.match_family: + return True + return False + + hits = [c for c in pool if matches(c)] + if not hits: + what = xsa or device or family or "" + raise DescriptorError(f"no {flow} card descriptor matches '{what}'; known: " + f"{sorted(c.name for c in pool)} (add one under {PLATFORM_DIR})") + best = max(c.specificity() for c in hits) + top = [c for c in hits if c.specificity() == best] + if len(top) > 1: + raise DescriptorError(f"ambiguous card for '{xsa or device or family}': " + f"{sorted(c.name for c in top)}") + return top[0] + + +def family_of(cards: List[Card], device: str) -> str: + families = {c.family for c in cards + if c.family and (device == c.part or + any(fnmatch.fnmatchcase(device, p) for p in c.match_device))} + if len(families) != 1: + raise DescriptorError(f"no single family for device '{device}': " + f"{sorted(families) or 'no card has this part'}") + return families.pop() + + +def emit_mk(c: Card) -> str: + # override: a command-line CONFIGS=... must not drop the card settings. + ov = "override " + inst = KERNEL_INSTANCE[c.flow] + lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + f"PLATFORM_CARD := {c.name}"] + if c.part: + lines.append(f"PLATFORM_PART := {c.part}") + lines.append(f"PLATFORM_AXIL_BLOCK_BYTES := {c.axil_block_bytes}") + lines.append(f"PLATFORM_HOST_REFUSE := {' '.join(c.host_refuse)}") + lines.append(f"PLATFORM_HOST_REFUSE_WHY := {c.host_refuse_why}") + + lines.append(f"{ov}CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS={c.banks} " + f"-DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH={c.addr_width}") + if c.base: + lines.append(f"{ov}CONFIGS += -DPLATFORM_MEMORY_OFFSET={verilog_offset(c.base)}") + + if c.flow == "xrt": + if c.host_tag: + lines.append(f"HOST_TAG = {c.host_tag}") + lines.append(f"VPP_FLAGS += --connectivity.sp {inst}.m_axi_host:$(HOST_TAG)") + # The flow binds each AFU memory master to a contiguous group of the + # card's memory channels. + if c.mem_tag: + lines.append(f"PLATFORM_MEMORY_TAG := {c.mem_tag}") + lines.append(f"PLATFORM_MEMORY_CHANNELS := {c.banks}") + else: + if not c.mem_tag: + raise DescriptorError(f"{c.path}: the aved flow binds its memory port to a [memory] tag") + lines.append(f"MEM_TAG = {c.mem_tag}") + lines.append(f"HOST_TAG = {c.host_tag}") + if c.part: + lines.append(f"DEVICE ?= {c.part}") + lines.append(f"KERNEL_FREQ ?= {c.kernel_mhz}") + return "\n".join(lines) + "\n" + + +def emit_cfg(c: Card, mem_tag: str, host_tag: str, kernel_mhz: int, + kernel: str = "vortex_afu", mem_ports: int = 1) -> str: + """The linker config of `kernel`; one with no host-memory port has no host tag. + A kernel with several device-memory masters (m_axi_mem_0..mem_ports-1) binds + each to the memory tag: the linker gives each its own way into it.""" + if c.flow != "aved": + raise DescriptorError(f"card '{c.name}' is an {c.flow} card; cfg is for the aved flow") + if host_tag: + check_host_tag(c, host_tag) + inst = KERNEL_INSTANCE["aved"] if kernel == "vortex_afu" else f"{kernel}_0" + return "\n".join([ + f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + "[connectivity]", + "# Bind to the compute-only static shell. The service shell adds the DCMAC", + "# ethernet stack the AFU has no use for.", + "shell=compute", + f"nk={kernel}:1:{inst}", + ] + [f"sp={inst}.m_axi_mem_{i}:{mem_tag}" for i in range(mem_ports) + ] + ([f"sp={inst}.m_axi_host:{host_tag}"] if host_tag else []) + [ + "", + "# Kernel clock, in Hz rather than via the linker's --clock-hz flag, whose", + "# name and help text disagree on the unit.", + "[clock]", + f"krnl={inst}", + f"freqhz={kernel_mhz * 1000000}", + "", + ]) + + +# FireSim platform -> the midasexamples config every card on it builds on. +FIRESIM_BASE_CONFIG = {"vitis": "DefaultVitisConfig", "aved": "DefaultAvedConfig"} + + +def firesim_config_class(c: Card) -> str: + return f"{c.name.upper()}PlatformConfig" + + +def _firesim(c: Card) -> dict: + fs = c.firesim + if not fs: + raise DescriptorError(f"{c.path}: no [firesim] section; card '{c.name}' has no FireSim platform") + platform = _require(fs, "platform", c.path, "firesim") + if platform not in FIRESIM_BASE_CONFIG: + raise DescriptorError(f"{c.path}: [firesim] platform must be one of {sorted(FIRESIM_BASE_CONFIG)}") + _require(fs, "host_mem_sp", c.path, "firesim") + return fs + + +def emit_firesim_mk(c: Card) -> str: + fs = _firesim(c) + lines = [f"# Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + f"FIRESIM_CARD := {c.name}", + f"FIRESIM_PLATFORM := {fs['platform']}", + f"PLATFORM_CONFIG := {firesim_config_class(c)}", + f"FPGA_PART := {c.part}", + f"HOST_MEM_SP := {fs['host_mem_sp']}"] + if "device" in fs: + lines.append(f"DEVICE := {fs['device']}") + return "\n".join(lines) + "\n" + + +def emit_scala(c: Card) -> str: + fs = _firesim(c) + return "\n".join([ + f"// Generated by gen_platform.py from {os.path.relpath(c.path, PLATFORM_DIR)}. Do not edit.", + "package firesim.midasexamples", + "", + "import org.chipsalliance.cde.config.Config", + "", + f"class {firesim_config_class(c)} extends Config(", + f" new midas.WithHostMemBase(BigInt(\"{c.base:x}\", 16)) ++ new {FIRESIM_BASE_CONFIG[fs['platform']]})", + "", + ]) + + +def check_host_tag(c: Card, host_tag: str) -> None: + if host_tag in c.host_refuse: + raise DescriptorError(f"host tag '{host_tag}' is refused on card '{c.name}': {c.host_refuse_why}") + + +def main(argv: Optional[List[str]] = None) -> int: + ap = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + ap.add_argument("--flow", default="", choices=("",) + FLOWS, + help="restrict matching to one flow; required unless --card is given") + ap.add_argument("--xsa", default="", help="XRT platform name (xrt flow)") + ap.add_argument("--family", default="", help="FPGA family reported by platforminfo (xrt flow)") + ap.add_argument("--device", default="", help="device part (aved flow)") + ap.add_argument("--card", default="", help="select a card by name instead of matching") + ap.add_argument("--emit", required=True, choices=("mk", "cfg", "firesim-mk", "scala", "card", "family")) + ap.add_argument("--mem-tag", default="") + ap.add_argument("--host-tag", default="") + ap.add_argument("--kernel-mhz", type=int, default=0) + ap.add_argument("--kernel", default="vortex_afu", help="kernel the cfg links (aved flow)") + ap.add_argument("--no-host", action="store_true", help="the kernel has no host-memory port") + ap.add_argument("--mem-ports", type=int, default=1, + help="device-memory masters of the kernel the cfg links (aved flow)") + ap.add_argument("--platform-dir", default=PLATFORM_DIR) + ap.add_argument("-o", "--output", default="-") + a = ap.parse_args(argv) + + if a.emit == "family" and not a.device: + ap.error("--emit=family takes --device") + if not a.flow and not a.card and a.emit != "family": + ap.error("--flow is required unless --card names the card") + try: + cards = load_all(a.platform_dir) + if a.emit == "family": + out = family_of(cards, a.device) + "\n" + else: + c = resolve(cards, a.flow, a.xsa, a.family.strip(), a.device, a.card) + if a.emit == "mk": + out = emit_mk(c) + elif a.emit == "cfg": + out = emit_cfg(c, a.mem_tag or c.mem_tag, + "" if a.no_host else (a.host_tag or c.host_tag), + a.kernel_mhz or c.kernel_mhz, a.kernel, a.mem_ports) + elif a.emit == "firesim-mk": + out = emit_firesim_mk(c) + elif a.emit == "scala": + out = emit_scala(c) + else: + out = c.name + "\n" + except DescriptorError as e: + print(f"gen_platform: error: {e}", file=sys.stderr) + return 1 + + if a.output == "-": + sys.stdout.write(out) + else: + # Rewrite only on change so dependents are not needlessly invalidated. + old = None + if os.path.exists(a.output): + with open(a.output) as f: + old = f.read() + if old != out: + os.makedirs(os.path.dirname(os.path.abspath(a.output)), exist_ok=True) + with open(a.output, "w") as f: + f.write(out) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/scripts/gen_sources.sh b/hw/scripts/gen_sources.sh index 26bfcda304..06b23d0c35 100755 --- a/hw/scripts/gen_sources.sh +++ b/hw/scripts/gen_sources.sh @@ -19,8 +19,9 @@ SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" &> /dev/null && pwd )" # every stage of a pipeline. set -eo pipefail -# Resolve verilator path at runtime, allowing it to be overridden by environment variable. -VERILATOR="${VERILATOR:-verilator}" +# The preprocessor is CIRCT's SystemVerilog frontend; hw/syn/common.mk exports +# its absolute path. +CIRCT_VERILOG="${CIRCT_VERILOG:-circt-verilog}" defines=() includes=() @@ -32,6 +33,9 @@ define_header="" top_module="" copy_folder="" preprocessor=0 +elaborate=0 +fold=0 +stubs=() defines_str="" params_str="" @@ -47,7 +51,7 @@ add_option() { } # Parse command arguments -while getopts D:G:T:I:J:O:H:C:Ph flag +while getopts D:G:T:I:J:O:H:C:X:PEFh flag do case "${flag}" in D) defines+=( ${OPTARG} ) @@ -71,7 +75,13 @@ do ;; P) preprocessor=1 ;; - h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-h help] [files...]" + E) elaborate=1 + ;; + F) fold=1 + ;; + X) stubs+=( "${OPTARG}" ) + ;; + h) echo "Usage: [-D] [-G=] [-T] [-I] [-J] [-O] [-C: copy to] [-H] [-P: macro preprocessing] [-F: fold asynchronous RAM reads into block RAM] [-E: elaborate into one netlist] [-X] [-h help] [files...]" exit 0 ;; \?) echo "Invalid option: -$OPTARG" 1>&2 @@ -121,17 +131,16 @@ copy_one_file() { "$SCRIPT_DIR"/repl_params.py $params_str -T"$top_module" "$file" > "$temp_file" input="$temp_file" fi - # Trust verilator's exit status, not the output size: a missing tool, - # unresolved include, or preprocessor error returns non-zero and is caught - # here (this is the failure that otherwise resurfaced far downstream as - # sv2v "Could not find top module"). An *empty but successful* output is - # legitimate — config-gated modules (e.g. VX_decompressor.sv when RVC is + # Trust the preprocessor's exit status, not the output size: a missing + # tool, unresolved include, or preprocessor error returns non-zero and is + # caught here rather than far downstream. An *empty but successful* output + # is legitimate — config-gated modules (e.g. VX_decompressor.sv when RVC is # disabled) preprocess to nothing — so emptiness alone must not fail. - if ! "$VERILATOR" $defines_str $includes_str -E -P "$input" > "$dest"; then + if ! "$CIRCT_VERILOG" -E $defines_str $includes_str "$input" -o "$dest"; then if [ -n "$temp_file" ]; then rm -f "$temp_file"; fi - echo "gen_sources.sh: error: verilator failed to preprocess '$file'" >&2 - echo " using VERILATOR='$VERILATOR' — ensure it is reachable" \ - "(set VERILATOR / VERILATOR_PATH, see hw/syn/common.mk) and that" \ + echo "gen_sources.sh: error: circt-verilog failed to preprocess '$file'" >&2 + echo " using CIRCT_VERILOG='$CIRCT_VERILOG' — ensure it is reachable" \ + "(set CIRCT_VERILOG / CIRCT_PATH, see hw/syn/common.mk) and that" \ "the include paths and defines resolve." >&2 exit 1 fi @@ -186,6 +195,28 @@ if [ -n "$copy_folder" ]; then fi fi +# Optional RAM fold (-F): patch the preprocessed copies so that every +# block-RAM-sized VX_dp_ram that reads combinationally reads a registered next +# address instead, which every synthesis tool maps to block RAM. The report +# and the patch land in the copy folder. RAM_FOLD_PYTHON is a python3 with +# pyslang 11 or 12. +if [ "$fold" != 0 ]; then + if [ -z "$copy_folder" ] || [ "$preprocessor" = 0 ] || [ -z "$top_module" ]; then + echo "gen_sources.sh: error: -F needs preprocessed copies (-P -C) and a top module (-T)" >&2 + exit 1 + fi + fold_files=() + while IFS= read -r f; do + fold_files+=( "$f" ) + done < <(find "$(realpath "$copy_folder")" -maxdepth 1 -type f \( -name "*.v" -o -name "*.sv" \) | sort) + if ! "${RAM_FOLD_PYTHON:-python3}" "$SCRIPT_DIR"/ram_fold.py --top "$top_module" $defines_str \ + --report "$copy_folder/ram_fold.rpt" --diff "$copy_folder/ram_fold.diff" "${fold_files[@]}"; then + echo "gen_sources.sh: error: ram_fold.py failed (RAM_FOLD_PYTHON='${RAM_FOLD_PYTHON:-python3}'," \ + "which needs pyslang 11 or 12)" >&2 + exit 1 + fi +fi + # Optional filelist generation # Order-sensitive consumers (Questa, VCS) need every package compiled before # its importers. Detect packages by content, not filename: vendored IP ships @@ -268,4 +299,25 @@ if [ "$output_file" != "" ]; then done fi } > "$output_file" -fi \ No newline at end of file +fi + +# Optional elaboration (-E): CIRCT elaborates the listed sources into the one +# netlist synthesis reads (circt_elaborate.sh), and the filelist names it. The +# RTL filelist stays next to it as .rtl.f; the memory report as +# .memories.rpt. -X files declare the black boxes (vendor IP) the +# synthesis tool binds. +if [ "$elaborate" != 0 ]; then + if [ -z "$top_module" ] || [ -z "$output_file" ]; then + echo "gen_sources.sh: error: -E needs a top module (-T) and a filelist (-O)" >&2 + exit 1 + fi + stem="${output_file%.*}" + mv "$output_file" "$stem.rtl.f" + stub_args=() + for stub in "${stubs[@]}"; do + stub_args+=( -x "$(realpath "$stub")" ) + done + "$SCRIPT_DIR"/circt_elaborate.sh -t "$top_module" -l "$stem.rtl.f" "${stub_args[@]}" \ + -r "$stem.memories.rpt" -o "$stem.elab.sv" + realpath "$stem.elab.sv" > "$output_file" +fi diff --git a/hw/scripts/ram_fold.py b/hw/scripts/ram_fold.py new file mode 100644 index 0000000000..ecedc8e46d --- /dev/null +++ b/hw/scripts/ram_fold.py @@ -0,0 +1,1078 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""RAM fold: block RAM for asynchronous reads, as a patch of the sources. + +A block RAM reads synchronously. A VX_dp_ram of block-RAM size that reads +combinationally (OUT_REG=0) becomes one when its read address is computed one +cycle early: the RAM then registers that next address itself (OUT_REG=1, +RDW_MODE "W"), which reads exactly what the combinational read did on every +cycle (docs/hardware_library.md, read forms). + +The next address is the address logic evaluated on the next values of the +registers that feed it, wherever in the hierarchy they are. The pass elaborates +the design with slang, the front end CIRCT uses, follows the address back to +those registers, and patches the sources it was given: + + - every signal on that cone gets a twin __nx carrying its next value, + driven by a copy of the signal's own driver with its operands renamed; + - a register's twin is a combinational copy of its always_ff block that keeps + only the assignments to that register (their right-hand sides still read + current values, as non-blocking assignments do); + - a port on the cone gets a twin port, connected at every instantiation; + - an interface signal gets a twin member; + - the RAM takes the twin address and the registered-address form, reading + every cycle. + +Edits only add code, except the RAM's own parameters and address connection. +The patched sources are elaborated again; a patch that adds an error is dropped +whole and every RAM keeps its original, combinational read. Nothing is folded across a +construct the pass does not handle: it reports the RAM and leaves it alone. +""" + +import argparse +import difflib +import os +import re +import shlex +import sys + +import pyslang +from pyslang import analysis, ast, driver + +SFX = "__nx" +SK = ast.SymbolKind +EK = ast.ExpressionKind + +RAM_MODULES = ("VX_dp_ram",) + +CONST_KINDS = {SK.Parameter, SK.TypeParameter, SK.EnumValue, SK.Genvar, SK.Specparam} + +PURE_SYSTEM_CALLS = {"$clog2", "$bits", "$signed", "$unsigned", "$countones", "$size", + "$high", "$low", "$left", "$right", "$onehot", "$onehot0", "$isunknown"} + + +class Unsupported(Exception): + pass + + +def force_bram(depth, width): + # `FORCE_BRAM in VX_platform.vh + return depth >= 64 and depth * width >= 2048 + + +def const_int(v): + s = str(v) + m = re.match(r"^(?:\d+)?'s?[dhbo]?([0-9a-fA-F_xXzZ]+)$", s) + if s.isdigit() or (s.startswith("-") and s[1:].isdigit()): + return int(s) + if m: + base = {"d": 10, "h": 16, "b": 2, "o": 8} + b = re.search(r"'s?([dhbo])", s) + return int(m.group(1).replace("_", ""), base[b.group(1)] if b else 10) + if s.startswith('"'): + return s + raise ValueError(s) + + +class Sources: + """File contents and text edits, by byte offset.""" + + def __init__(self, sm): + self.sm = sm + self.data = {} + + def file(self, loc): + return os.path.realpath(self.sm.getFullPath(loc.buffer)) + + def bytes(self, path): + if path not in self.data: + with open(path, "rb") as f: + self.data[path] = f.read() + return self.data[path] + + def span(self, rng): + path = self.file(rng.start) + return path, rng.start.offset, rng.end.offset + + def text(self, rng): + path, s, e = self.span(rng) + return self.bytes(path)[s:e].decode() + + def indent(self, path, offset): + b = self.bytes(path) + line = b.rfind(b"\n", 0, offset) + 1 + m = re.match(rb"[ \t]*", b[line:]) + return m.group(0).decode() + + +class Edits: + """Keyed insertions and replacements; the same key must carry the same edit.""" + + def __init__(self): + self.items = {} + self.order = [] + + def add(self, key, path, start, end, text): + e = (path, start, end, text) + if key in self.items: + if self.items[key] != e: + raise Unsupported("conflicting edits at %s:%d" % (os.path.basename(path), start)) + return + self.items[key] = e + self.order.append(key) + + def merge(self, other): + for k in other.order: + self.add(k, *other.items[k]) + + def apply(self, srcs): + out = {} + by_file = {} + for n, k in enumerate(self.order): + path, s, e, t = self.items[k] + by_file.setdefault(path, []).append((s, e, n, t)) + for path, lst in by_file.items(): + data = srcs.bytes(path) + # replacements must not overlap each other or contain insertions + reps = [(s, e) for s, e, _, _ in lst if e > s] + for s, e in reps: + for s2, e2, _, _ in lst: + if (s2, e2) != (s, e) and s < s2 < e or (e2 > s2 and s2 < e and s < e2 and (s2, e2) != (s, e)): + raise Unsupported("overlapping edits in %s at %d" % (os.path.basename(path), s)) + # apply back to front; at one offset, insertions keep their order + lst.sort(key=lambda x: (x[0], x[2])) + res = bytearray() + pos = 0 + for s, e, _, t in lst: + res += data[pos:s] + res += t.encode() + pos = max(pos, e) if e > s else max(pos, s) + res += data[pos:] + out[path] = bytes(res) + return out + + +class Design: + """An elaborated design and its indices.""" + + def __init__(self, args, files): + self.dr = driver.Driver() + self.dr.addStandardArgs() + # no error limit: slang stops its driver analysis once the limit is reached + cmd = ["slang", "--allow-use-before-declare", "--allow-hierarchical-const", + "--ignore-unknown-modules", "--disable-instance-caching", "--error-limit=0"] + args + files + if not self.dr.parseCommandLine(shlex.join(cmd), driver.CommandLineOptions()): + raise SystemExit("ram_fold: bad slang arguments") + if not (self.dr.processOptions() and self.dr.parseAllSources()): + raise SystemExit("ram_fold: cannot parse the sources") + self.comp = self.dr.createCompilation() + self.root = self.comp.getRoot() + self.errors = self._errors() + self.am = analysis.AnalysisManager() + self.am.analyze(self.comp) + self.srcs = Sources(self.dr.sourceManager) + self.parent = {} # instance path -> parent instance + self.owner = {} # value symbol path -> instance + self.by_path = {} # value symbol path -> symbol + self.insts = {} # instance path -> instance + self.iface_maps = {} + self.iface_users = {} # interface instance path -> [(instance, port)] + self.rams = [] + self.arrayed = set() # instances that are elements of an instance array + for top in self.root.topInstances: + self.parent[top.hierarchicalPath] = None + self.insts[top.hierarchicalPath] = top + self._index(top.body, top) + + + def _errors(self): + de = pyslang.DiagnosticEngine(self.dr.sourceManager) + cl = pyslang.TextDiagnosticClient() + de.addClient(cl) + errs = set() + for d in self.comp.getAllDiagnostics(): + if d.isError(): + cl.clear() + de.issue(d) + errs.add(re.sub(r":\d+:\d+:", ":", cl.getString().split("\n")[0])) + return errs + + def _index(self, scope, inst): + for m in scope: + k = m.kind + if k == SK.Instance: + self.parent[m.hierarchicalPath] = inst + self.insts[m.hierarchicalPath] = m + self._ifaces(m) + if m.definition.name in RAM_MODULES: + self.rams.append(m) + self._index(m.body, m) + elif k in (SK.GenerateBlock, SK.GenerateBlockArray): + if k == SK.GenerateBlock and m.isUninstantiated: + continue + self._index(m, inst) + elif k in (SK.Net, SK.Variable): + self.owner[m.hierarchicalPath] = inst + self.by_path[m.hierarchicalPath] = m + elif k == SK.InstanceArray: + self._index_array(m, inst) + + def _ifaces(self, inst): + for p in inst.body.portList: + if p.kind == SK.InterfacePort: + c = inst.getPortConnection(p) + if c is not None and c.ifaceConn is not None and c.ifaceConn[0] is not None: + self.iface_users.setdefault(c.ifaceConn[0].hierarchicalPath, []).append((inst, p)) + + def _index_array(self, arr, inst): + for e in arr: + if e.kind == SK.InstanceArray: + self._index_array(e, inst) + elif e.kind == SK.Instance: + self.parent[e.hierarchicalPath] = inst + self.insts[e.hierarchicalPath] = e + self.arrayed.add(e.hierarchicalPath) + self._ifaces(e) + self._index(e.body, e) + + def param(self, inst, name): + for p in inst.body.parameters: + if p.name == name: + return p.value + raise KeyError(name) + + def port(self, inst, name): + for p in inst.body.portList: + if p.name == name: + return p + raise KeyError(name) + + def port_of(self, inst, sym): + for p in inst.body.portList: + if p.kind == SK.Port and p.internalSymbol is not None and \ + p.internalSymbol.hierarchicalPath == sym.hierarchicalPath: + return p + return None + + +def leaf_name_span(srcs, node, name): + """The byte span of the last identifier of a value reference.""" + path, s, e = srcs.span(node.sourceRange) + txt = srcs.bytes(path)[s:e].decode() + m = None + for m in re.finditer(r"\b%s\b" % re.escape(name), txt): + pass + if m is None: + raise Unsupported("cannot locate '%s' in '%s'" % (name, txt)) + return path, s + m.start(), s + m.end() + + +def chain_of(node): + """(root value expression, selector path) of a select/member chain.""" + path = [] + while True: + k = node.kind + if k == EK.ElementSelect: + c = node.selector.constant + path.append(("idx", str(c) if c is not None else None)) + node = node.value + elif k == EK.RangeSelect: + path.append(("rng", None)) + node = node.value + elif k == EK.MemberAccess: + path.append(("mem", node.member.name)) + node = node.value + elif k == EK.Conversion and node.isImplicit if hasattr(node, "isImplicit") else False: + node = node.operand + else: + break + if node.kind in (EK.NamedValue, EK.HierarchicalValue): + return node, tuple(reversed(path)) + return None, () + + +def reroot(p, a, b): + """p re-rooted from the hierarchy under a to the one under b, or None.""" + if p == a or p.startswith(a + ".") or p.startswith(a + "["): + return b + p[len(a):] + return None + + +def path_text(path): + return "".join("[%s]" % (v if v is not None else "*") if k != "mem" else "." + v for k, v in path) + + +def compatible(a, b): + for x, y in zip(a, b): + if x[0] != y[0]: + return True + if x[0] == "mem" and x[1] != y[1]: + return False + if x[0] == "idx" and x[1] is not None and y[1] is not None and x[1] != y[1]: + return False + return True + + +class Fold: + """Folds one RAM; its edits are merged only if it succeeds.""" + + def __init__(self, design): + self.d = design + self.srcs = design.srcs + self.edits = Edits() + self.declared = set() + self.done = set() + + # ---- canonical bodies ------------------------------------------------- + # + # slang analyzes one body per set of identically parameterized instances, so + # the drivers of a signal in any other instance are found on its counterpart + # in that canonical body, and whatever those drivers reference is mapped back. + + def canon_path(self, path): + """(path of the analyzed counterpart of path, mapping back).""" + tr = [] + for _ in range(64): + x = self.d.insts.get(path) or self.d.owner.get(path) + moved = False + while x is not None: + cb = x.canonicalBody if not x.isInterface else None + ci = cb.parentInstance if cb is not None else None + if ci is not None and ci.hierarchicalPath != x.hierarchicalPath: + tr.append((ci.hierarchicalPath, x.hierarchicalPath)) + path = reroot(path, x.hierarchicalPath, ci.hierarchicalPath) + moved = True + break + x = self.d.parent.get(x.hierarchicalPath) + if not moved: + break + return path, tuple(tr) + + def canon(self, sym): + """(counterpart of sym that carries drivers, mapping back).""" + path, tr = self.canon_path(sym.hierarchicalPath) + csym = self.d.by_path.get(path) + if csym is None: + raise Unsupported("no canonical counterpart of '%s'" % sym.hierarchicalPath) + return csym, tr + + def iface_drivers(self, sym, inst): + """Drivers of an interface signal in the modules connected to it whose + bodies slang analyzed only as another instance: [(driver, csym, tr)].""" + rel = sym.hierarchicalPath[len(inst.hierarchicalPath):] + found = [] + for user, port in self.d.iface_users.get(inst.hierarchicalPath, []): + upath, tr = self.canon_path(user.hierarchicalPath) + if not tr: + continue # analyzed as itself: its drivers are on sym already + cuser = self.d.insts.get(upath) + if cuser is None: + continue + cport = next((p for p in cuser.body.portList if p.name == port.name), None) + c = cuser.getPortConnection(cport) if cport is not None else None + if c is None or c.ifaceConn is None or c.ifaceConn[0] is None: + continue + csym = self.d.by_path.get(c.ifaceConn[0].hierarchicalPath + rel) + if csym is None: + continue + for drv in self.d.am.getDrivers(csym): + if reroot(drv.containingSymbol.hierarchicalPath, upath, upath) is not None: + found.append((drv, csym, tr)) + return found + + def back(self, path, tr): + for cp, ap in reversed(tr): + q = reroot(path, cp, ap) + if q is None: + # a signal of an interface the shared body reaches through a port + for ic, ia in self.iface_map(cp, ap): + q = reroot(path, ic, ia) + if q is not None: + break + if q is None: + raise Unsupported("a shared instance body reads '%s' outside itself" % path) + path = q + return path + + def iface_map(self, cp, ap): + """(interface instance path, its counterpart) for the interface ports of + instance cp and its copy ap.""" + if (cp, ap) in self.d.iface_maps: + return self.d.iface_maps[(cp, ap)] + ci, ai = self.d.insts.get(cp), self.d.insts.get(ap) + res = [] + if ci is not None and ai is not None: + for cport, aport in zip(ci.body.portList, ai.body.portList): + if cport.kind != SK.InterfacePort: + continue + cc, ac = ci.getPortConnection(cport), ai.getPortConnection(aport) + if cc is None or ac is None or cc.ifaceConn is None or ac.ifaceConn is None: + continue + c0, a0 = cc.ifaceConn[0], ac.ifaceConn[0] + if c0 is not None and a0 is not None: + res.append((c0.hierarchicalPath, a0.hierarchicalPath)) + self.d.iface_maps[(cp, ap)] = res + return res + + def act(self, sym, tr): + if not tr: + return sym + s = self.d.by_path.get(self.back(sym.hierarchicalPath, tr)) + if s is None: + raise Unsupported("no instance counterpart of '%s'" % sym.hierarchicalPath) + return s + + def act_inst(self, inst, tr): + if not tr: + return inst + i = self.d.insts.get(self.back(inst.hierarchicalPath, tr)) + if i is None: + raise Unsupported("no instance counterpart of '%s'" % inst.hierarchicalPath) + return i + + def scope_key(self, sym, tr): + p = sym.hierarchicalPath + for cp, ap in reversed(tr): + p = reroot(p, cp, ap) or p + return p + + def clock_root(self, expr, tr=()): + """Where a clock comes from: its net, followed up through input ports.""" + while True: + while expr.kind == EK.Conversion: + expr = expr.operand + if expr.kind != EK.NamedValue: + return None + sym = self.act(self.value_symbol(expr), tr) + tr = () + inst = self.d.owner.get(sym.hierarchicalPath) + port = self.d.port_of(inst, sym) if inst is not None else None + if port is None or port.direction != ast.ArgumentDirection.In or \ + self.d.parent.get(inst.hierarchicalPath) is None: + return sym.hierarchicalPath + conn = inst.getPortConnection(port) + if conn is None or conn.expression is None: + return None + expr = conn.expression + + # ---- references ------------------------------------------------------- + + def value_symbol(self, node): + sym = node.symbol + if sym.kind == SK.ModportPort: + sym = sym.internalSymbol + return sym + + def leaves(self, expr, tr=()): + """Non-constant value references in expr: (node, symbol, path).""" + paths = {} + found = [] + + def visit(n): + if not isinstance(n, ast.Expression): + return ast.VisitAction.Advance + k = n.kind + if k in (EK.ElementSelect, EK.RangeSelect, EK.MemberAccess): + root, p = chain_of(n) + if root is not None: + key = root.sourceRange.start.offset + if key not in paths or len(p) > len(paths[key]): + paths[key] = p + elif k in (EK.NamedValue, EK.HierarchicalValue): + sym = self.value_symbol(n) + if sym.kind in CONST_KINDS: + return ast.VisitAction.Advance + if sym.kind not in (SK.Net, SK.Variable): + raise Unsupported("reference to %s '%s'" % (sym.kind.name, sym.name)) + found.append((n, self.act(sym, tr))) + elif k == EK.Call: + self.check_pure(n) + elif k == EK.Assignment: + raise Unsupported("assignment inside an expression") + return ast.VisitAction.Advance + + expr.visit(visit) + return [(n, s, paths.get(n.sourceRange.start.offset, ())) for n, s in found] + + def check_pure(self, call): + sub = call.subroutine + name = call.subroutineName + if call.isSystemCall: + if name not in PURE_SYSTEM_CALLS: + raise Unsupported("system call %s" % name) + return + if sub.kind != SK.Subroutine: + raise Unsupported("call to %s" % name) + + def visit(n): + if isinstance(n, ast.Expression) and n.kind in (EK.NamedValue, EK.HierarchicalValue): + s = n.symbol + if s.kind in (SK.Net, SK.Variable) and not self.inside(s, sub): + raise Unsupported("function %s reads '%s'" % (name, s.name)) + return ast.VisitAction.Advance + + sub.visit(visit) + + @staticmethod + def inside(sym, scope_sym): + p = sym.lexicalPath if hasattr(sym, "lexicalPath") else "" + return p.startswith(scope_sym.lexicalPath) if hasattr(scope_sym, "lexicalPath") else False + + def nx_expr(self, expr, extra=(), tr=()): + """The source text of expr evaluated on next values.""" + path, s, e = self.srcs.span(expr.sourceRange) + reps = [] + for node, sym, p in self.leaves(expr, tr): + if extra and node.sourceRange.start.offset == s: + p = p + tuple(extra) + name = self.twin(sym, p) + fp, ls, le = leaf_name_span(self.srcs, node, sym.name) + if fp != path: + raise Unsupported("expression spans files") + reps.append((ls, le, name)) + return self.substitute(path, s, e, reps) + + def substitute(self, path, s, e, reps): + data = self.srcs.bytes(path) + out, pos = [], s + for ls, le, t in sorted(set(reps)): + out.append(data[pos:ls].decode()) + out.append(t) + pos = le + out.append(data[pos:e].decode()) + return "".join(out) + + def rename_root(self, lhs, tr=()): + """The text of an lvalue with its root signal renamed to its twin.""" + root, _ = chain_of(lhs) + if root is None: + raise Unsupported("lvalue '%s'" % self.srcs.text(lhs.sourceRange)) + for n, _, _ in self.leaves(lhs, tr): + if n.sourceRange.start.offset != root.sourceRange.start.offset: + raise Unsupported("lvalue with a variable index '%s'" % self.srcs.text(lhs.sourceRange)) + sym = self.value_symbol(root) + path, s, e = self.srcs.span(lhs.sourceRange) + _, ls, le = leaf_name_span(self.srcs, root, sym.name) + return self.substitute(path, s, e, [(ls, le, sym.name + SFX)]), sym + + # ---- twins ------------------------------------------------------------ + + def twin(self, sym, path=()): + key = sym.hierarchicalPath + name = sym.name + SFX + inst = self.d.owner.get(key) + if inst is None: + raise Unsupported("no owner for '%s'" % key) + port = self.d.port_of(inst, sym) if inst.isModule or inst.isInterface else None + if key not in self.declared: + self.declared.add(key) + self.declare(sym, inst, port) + if port is not None and port.direction == ast.ArgumentDirection.In: + if ("in", key, path) not in self.done: + self.done.add(("in", key, path)) + self.input_port(sym, inst, port, path) + return name + if sym.kind == SK.Net and sym.initializer is not None: + return name # driven by its declaration, which declare() copied + # every part read must be driven: a twin read without a driver would be + # undriven logic that still elaborates + csym, tr = self.canon(sym) + drivers = [(drv, csym, tr) for drv in self.d.am.getDrivers(csym)] + if inst.isInterface: + drivers += self.iface_drivers(sym, inst) + if not drivers: + raise Unsupported("'%s' is undriven" % sym.hierarchicalPath) + blocks = {(self.scope_key(drv.containingSymbol, t), drv.containingSymbol.syntax.sourceRange.start.offset) + for drv, c, t in drivers if drv.containingSymbol.kind == SK.ProceduralBlock} + if len(blocks) > 1: + raise Unsupported("'%s' is driven by %d procedural blocks" % (sym.hierarchicalPath, len(blocks))) + if not any([self.driver(sym, inst, c, drv, path, t) for drv, c, t in drivers]): + raise Unsupported("no driver of '%s'%s" % (sym.hierarchicalPath, path_text(path))) + return name + + def declare(self, sym, inst, port): + name = sym.name + SFX + if port is not None: + self.declare_port(port, name) + return + decl = sym.syntax + while decl is not None and decl.kind not in (pyslang.syntax.SyntaxKind.DataDeclaration, + pyslang.syntax.SyntaxKind.NetDeclaration): + decl = decl.parent + if decl is None: + raise Unsupported("no declaration for '%s'" % sym.name) + path, ds, de = self.srcs.span(decl.sourceRange) + _, vs, ve = self.srcs.span(sym.syntax.sourceRange) + data = self.srcs.bytes(path) + prefix = data[ds:self.srcs.span(decl.declarators[0].sourceRange)[1]].decode() + dims = "".join(str(x) for x in sym.syntax.dimensions).strip() + init = "" + if sym.kind == SK.Net and sym.initializer is not None: + init = " = " + self.nx_expr(sym.initializer) + ind = self.srcs.indent(path, ds) + text = "\n%s%s%s%s%s;" % (ind, prefix.strip() + " ", name, (" " + dims) if dims else "", init) + self.edits.add(("decl", path, de, name), path, de, de, text) + if inst.isInterface: + self.modports(sym, inst, name) + + def declare_port(self, port, name): + syn = port.syntax + if syn is not None and syn.kind == pyslang.syntax.SyntaxKind.Declarator: + syn = syn.parent + if syn is None or syn.kind != pyslang.syntax.SyntaxKind.ImplicitAnsiPort: + raise Unsupported("port '%s' is not an ANSI port" % port.name) + header = syn + while header is not None and not str(header.header).strip(): + header = self.prev_port(header) + if header is None: + raise Unsupported("port '%s' has no header" % port.name) + dims = "".join(str(x) for x in syn.declarator.dimensions).strip() + # appended after the last port, so an instantiation that connects by + # position elsewhere keeps its ports + ports = [p for p in syn.parent.ports if isinstance(p, pyslang.syntax.SyntaxNode)] + path, ps, _ = self.srcs.span(syn.sourceRange) + _, _, pe = self.srcs.span(ports[-1].sourceRange) + ind = self.srcs.indent(path, ps) + text = ",\n%s%s %s%s" % (ind, str(header.header).strip(), name, (" " + dims) if dims else "") + self.edits.add(("port", path, pe, name), path, pe, pe, text) + + @staticmethod + def prev_port(syn): + lst = syn.parent.ports if hasattr(syn.parent, "ports") else syn.parent + prev = None + for item in lst: + if item is syn or (hasattr(item, "sourceRange") and item.sourceRange.start.offset == syn.sourceRange.start.offset): + return prev + if isinstance(item, pyslang.syntax.SyntaxNode) and item.kind == pyslang.syntax.SyntaxKind.ImplicitAnsiPort: + prev = item + return None + + def modports(self, sym, inst, name): + for m in inst.body: + if m.kind != SK.Modport: + continue + for p in m: + if p.name == sym.name: + path, s, e = self.srcs.span(p.syntax.sourceRange) + self.edits.add(("modport", path, e, name), path, e, e, ", " + name) + + def input_port(self, sym, inst, port, path): + parent = self.d.parent.get(inst.hierarchicalPath) + if parent is None: + raise Unsupported("'%s' is a top-level input" % sym.hierarchicalPath) + conn = inst.getPortConnection(port) + if conn is None or conn.expression is None: + raise Unsupported("input '%s' of %s is unconnected" % (port.name, inst.hierarchicalPath)) + text = self.nx_expr(conn.expression, path) + self.connect(inst, port.name + SFX, text) + + def connect(self, inst, port_name, text): + if inst.hierarchicalPath in self.d.arrayed: + raise Unsupported("instance array %s on the address cone" % inst.hierarchicalPath) + syn = inst.syntax + conns = [c for c in syn.connections if isinstance(c, pyslang.syntax.SyntaxNode)] + if not conns or any(c.kind != pyslang.syntax.SyntaxKind.NamedPortConnection for c in conns): + raise Unsupported("instance %s does not connect its ports by name" % inst.name) + path, s, e = self.srcs.span(conns[-1].sourceRange) + ind = self.srcs.indent(path, s) + self.edits.add(("conn", path, e, port_name), path, e, e, + ",\n%s.%s (%s)" % (ind, port_name, text)) + + def driver(self, sym, inst, csym, drv, path, tr): + cs = drv.containingSymbol + k = cs.kind + if k == SK.ContinuousAssign: + lhs, rhs = cs.assignment.left, cs.assignment.right + if lhs.kind == EK.Concatenation: + return self.concat_assign(sym, csym, cs, lhs, rhs, path, tr) + root, lpath = chain_of(lhs) + if root is None: + raise Unsupported("assign to '%s'" % self.srcs.text(lhs.sourceRange)) + if not compatible(lpath, path): + return False + key = ("assign", self.scope_key(cs, tr), cs.syntax.sourceRange.start.offset, + self.srcs.file(cs.syntax.sourceRange.start)) + if key in self.done: + return True + self.done.add(key) + ltxt, _ = self.rename_root(lhs, tr) + rtxt = self.nx_expr(rhs, (), tr) + stmt = cs.syntax.parent + while stmt is not None and stmt.kind != pyslang.syntax.SyntaxKind.ContinuousAssign: + stmt = stmt.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + self.edits.add(("assign", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s;" % (ind, ltxt, rtxt)) + return True + elif k == SK.ProceduralBlock: + self.register(sym, csym, cs, tr) + return True + elif k == SK.Instance: + return self.child_output(sym, inst, self.act_inst(cs, tr), path) + else: + raise Unsupported("'%s' is driven by a %s" % (sym.name, k.name)) + + def concat_assign(self, sym, csym, cs, lhs, rhs, path, tr): + """assign {.., sym, ..} = e: the twin of e, sliced.""" + ops = list(lhs.operands) + mine = [] + for i, op in enumerate(ops): + root, p = chain_of(op) + if root is None: + raise Unsupported("assign to '%s'" % self.srcs.text(lhs.sourceRange)) + if self.value_symbol(root).hierarchicalPath == csym.hierarchicalPath and compatible(p, path): + mine.append(i) + if not mine: + return False + key = ("cassign", self.scope_key(cs, tr), cs.syntax.sourceRange.start.offset, + self.srcs.file(cs.syntax.sourceRange.start), sym.hierarchicalPath) + if key in self.done: + return True + self.done.add(key) + stmt = cs.syntax.parent + while stmt is not None and stmt.kind != pyslang.syntax.SyntaxKind.ContinuousAssign: + stmt = stmt.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + whole = self.srcs.text(lhs.sourceRange) + tname = "%s_cat%d%s" % (sym.name, cs.syntax.sourceRange.start.offset, SFX) + rtxt = self.nx_expr(rhs, (), tr) + self.edits.add(("catw", fpath, se, tname), fpath, se, se, + "\n%swire [$bits(%s)-1:0] %s = %s;" % (ind, whole, tname, rtxt)) + for i in mine: + later = [self.srcs.text(o.sourceRange) for o in ops[i + 1:]] + off = "$bits({%s})" % ", ".join(later) if later else "0" + ltxt, _ = self.rename_root(ops[i], tr) + self.edits.add(("cats", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s[%s +: $bits(%s)];" % (ind, ltxt, tname, off, self.srcs.text(ops[i].sourceRange))) + return True + + # ---- registers -------------------------------------------------------- + + def register(self, sym, csym, block, tr): + key = ("reg", self.scope_key(block, tr), block.syntax.sourceRange.start.offset, + self.srcs.file(block.syntax.sourceRange.start), sym.name) + if key in self.done: + return + self.done.add(key) + pk = block.procedureKind + if pk not in (ast.ProceduralBlockKind.AlwaysFF, ast.ProceduralBlockKind.Always): + raise Unsupported("'%s' is driven by an %s block" % (sym.name, pk.name)) + timed = block.body + if timed.kind != ast.StatementKind.Timed: + raise Unsupported("'%s' is driven by an untimed block" % sym.name) + tc = self.srcs.text(timed.timing.sourceRange) + if not re.fullmatch(r"(@\s*\(\s*)?posedge\s+\w+(\s*\))?", tc.strip()): + raise Unsupported("'%s' is clocked by '%s'" % (sym.name, tc)) + ev = getattr(timed.timing, "expr", None) + clk = self.clock_root(ev, tr) if ev is not None else None + if clk is None or clk != self.clock: + raise Unsupported("'%s' is clocked by '%s', not the RAM's clock" % (sym.hierarchicalPath, clk or tc)) + body = timed.stmt + path, bs, be = self.srcs.span(body.sourceRange) + reps = [] + self.keep_only(csym, body, reps) + btxt = self.substitute(path, bs, be, reps) + name = sym.name + SFX + bpath, s, e = self.srcs.span(block.syntax.sourceRange) + ind = self.srcs.indent(bpath, s) + body_ind = "\n".join((" " + l) if l.strip() else l for l in btxt.split("\n")) + text = "\n%salways @(*) begin\n%s %s = %s;\n%s %s\n%send" % (ind, ind, name, sym.name, ind, body_ind.lstrip(), ind) + self.edits.add(key[2:], bpath, e, e, text) + + def keep_only(self, sym, st, reps): + """Rewrite a clocked statement to compute sym's twin and nothing else.""" + k = st.kind + SKd = ast.StatementKind + if k == SKd.Block: + self.keep_only(sym, st.body, reps) + elif k == SKd.List: + for s in st.list: + self.keep_only(sym, s, reps) + elif k == SKd.Conditional: + self.keep_only(sym, st.ifTrue, reps) + if st.ifFalse is not None: + self.keep_only(sym, st.ifFalse, reps) + elif k == SKd.Case: + for it in st.items: + self.keep_only(sym, it.stmt, reps) + if st.defaultCase is not None: + self.keep_only(sym, st.defaultCase, reps) + elif k == SKd.ForLoop: + self.keep_only(sym, st.body, reps) + elif k in (SKd.Empty, SKd.VariableDeclaration): + pass + elif k == SKd.ExpressionStatement: + e = st.expr + path, s, en = self.srcs.span(st.sourceRange) + if e.kind == EK.Assignment: + root, _ = chain_of(e.left) + rsym = self.value_symbol(root) if root is not None else None + if e.isNonBlocking: + if rsym is None and any(x.hierarchicalPath == sym.hierarchicalPath + for _, x, _ in self.leaves(e.left)): + raise Unsupported("'%s' is assigned as part of '%s'" + % (sym.name, self.srcs.text(e.left.sourceRange))) + if rsym is not None and rsym.hierarchicalPath == sym.hierarchicalPath: + _, ls, le = leaf_name_span(self.srcs, root, sym.name) + reps.append((ls, le, sym.name + SFX)) + op = e.syntax.operatorToken.range + reps.append((op.start.offset, op.end.offset, "=")) + else: + reps.append((s, en, ";")) + return + raise Unsupported("blocking assignment in the block driving '%s'" % sym.name) + if e.kind == EK.Call and e.isSystemCall: + reps.append((s, en, ";")) + return + raise Unsupported("statement '%s' in the block driving '%s'" % (self.srcs.text(st.sourceRange)[:40], sym.name)) + else: + raise Unsupported("%s statement in the block driving '%s'" % (k.name, sym.name)) + + # ---- child outputs ---------------------------------------------------- + + def child_output(self, sym, inst, child, path): + covered = False + for conn in child.portConnections: + port = conn.port + if port.kind != SK.Port or port.direction != ast.ArgumentDirection.Out or conn.expression is None: + continue + lv = conn.expression + if lv.kind == EK.Assignment: + lv = lv.left + refs = [(n, s, p) for n, s, p in self.leaves(lv) if s.hierarchicalPath == sym.hierarchicalPath] + if not refs: + continue + key = ("out", child.hierarchicalPath, port.name, sym.hierarchicalPath, path) + if key in self.done: + covered = True + continue + self.done.add(key) + self.twin(port.internalSymbol) + pname = port.name + SFX + if lv.kind == EK.Concatenation: + covered = self.concat_output(sym, child, pname, lv, path) or covered + else: + root, p = chain_of(lv) + if root is None or not compatible(p, path): + continue + ltxt, _ = self.rename_root(lv) + self.connect(child, pname, ltxt) + covered = True + return covered + + def concat_output(self, sym, child, pname, lv, path): + ops = list(lv.operands) + fname = "%s_%s" % (child.name, pname) + stmt = child.syntax.parent + fpath, ss, se = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(fpath, ss) + whole = self.srcs.text(lv.sourceRange) + self.edits.add(("fwire", fpath, ss, fname), fpath, ss, ss, + "wire [$bits(%s)-1:0] %s;\n%s" % (whole, fname, ind)) + self.connect(child, pname, fname) + covered = False + for i, op in enumerate(ops): + root, p = chain_of(op) + if root is None or self.value_symbol(root).hierarchicalPath != sym.hierarchicalPath: + continue + if not compatible(p, path): + continue + later = [self.srcs.text(o.sourceRange) for o in ops[i + 1:]] + off = "$bits({%s})" % ", ".join(later) if later else "0" + ltxt, _ = self.rename_root(op) + self.edits.add(("fslice", fpath, se, ltxt), fpath, se, se, + "\n%sassign %s = %s[%s +: $bits(%s)];" % (ind, ltxt, fname, off, self.srcs.text(op.sourceRange))) + covered = True + return covered + + # ---- the RAM ---------------------------------------------------------- + + def ram(self, ram, check): + parent = self.d.parent[ram.hierarchicalPath] + syn = ram.syntax + stmt = syn.parent + raddr = ram.getPortConnection(self.d.port(ram, "raddr")) + if raddr is None or raddr.expression is None: + raise Unsupported("raddr unconnected") + clk = ram.getPortConnection(self.d.port(ram, "clk")) + self.clock = self.clock_root(clk.expression) if clk is not None and clk.expression is not None else None + if self.clock is None: + raise Unsupported("cannot follow the RAM's clock") + nx = self.nx_expr(raddr.expression) + # parameters + params = stmt.parameters + if params is None: + raise Unsupported("RAM without a parameter list") + named = {} + for p in params.parameters: + if not isinstance(p, pyslang.syntax.SyntaxNode): + continue + if p.kind != pyslang.syntax.SyntaxKind.NamedParamAssignment: + raise Unsupported("RAM parameters not assigned by name") + named[p.name.valueText] = p + for pname, val in (("OUT_REG", "1"), ("RDW_MODE", '"W"')): + if pname in named: + rng = named[pname].expr.sourceRange + path, s, e = self.srcs.span(rng) + self.edits.add(("rparam", path, s, pname), path, s, e, val) + else: + path, s, e = self.srcs.span(params.openParen.range) + self.edits.add(("rparam+", path, e, pname), path, e, e, "\n .%s (%s)," % (pname, val)) + path, s, e = self.srcs.span(raddr.expression.sourceRange) + self.edits.add(("raddr", path, s), path, s, e, nx) + read = ram.getPortConnection(self.d.port(ram, "read")) + if read is not None and read.expression is not None: + path, s, e = self.srcs.span(read.expression.sourceRange) + self.edits.add(("read", path, s), path, s, e, "1'b1") + else: + self.connect(ram, "read", "1'b1") + if check: + self.check_assert(ram, raddr.expression, nx) + return nx + + def check_assert(self, ram, orig, nx): + clk = ram.getPortConnection(self.d.port(ram, "clk")).expression + rst = ram.getPortConnection(self.d.port(ram, "reset")).expression + stmt = ram.syntax.parent + path, s, e = self.srcs.span(stmt.sourceRange) + ind = self.srcs.indent(path, s) + n = ram.name + o = self.srcs.text(orig.sourceRange) + c, r = self.srcs.text(clk.sourceRange), self.srcs.text(rst.sourceRange) + text = ("\n%sreg [$bits(%s)-1:0] %s_fold_raddr_r;\n%sreg %s_fold_ok_r = 0;\n" + "%salways @(posedge %s) begin\n%s %s_fold_raddr_r <= %s;\n%s %s_fold_ok_r <= !%s;\n" + "%s if (%s_fold_ok_r && !%s && %s_fold_raddr_r !== %s)\n" + "%s $error(\"ram_fold: %%m.%s next address %%h, address %%h\", %s_fold_raddr_r, %s);\n%send" + % (ind, o, n, ind, n, ind, c, ind, n, nx, ind, n, r, ind, n, r, n, o, ind, n, n, o, ind)) + self.edits.add(("check", path, e, n), path, e, e, text) + + +def run(args): + files = list(args.files) + design = Design(args.slang, files) + report = [] + edits = Edits() + targets = {} + for ram in design.rams: + try: + d = const_int(design.param(ram, "SIZE")) + w = const_int(design.param(ram, "DATAW")) + oreg = const_int(design.param(ram, "OUT_REG")) + lut = const_int(design.param(ram, "LUTRAM")) + except (KeyError, ValueError) as ex: + report.append("skip %s: parameters (%s)" % (ram.hierarchicalPath, ex)) + continue + if oreg != 0 or not args.all and (lut != 0 or not force_bram(d, w)): + continue + targets[ram.hierarchicalPath] = (ram, d, w) + # A fold edits the RAM's instantiation statement, which every instance of its + # module shares: all RAMs it makes fold together or none does. + groups = {} + for p, (ram, d, w) in sorted(targets.items()): + syn = ram.syntax.parent.sourceRange + groups.setdefault((design.srcs.file(syn.start), syn.start.offset), []) + for ram in design.rams: + syn = ram.syntax.parent.sourceRange + key = (design.srcs.file(syn.start), syn.start.offset) + if key in groups: + groups[key].append(ram) + folded = [] + for key in sorted(groups): + rams = groups[key] + f = Fold(design) + cur = rams[0] + try: + for cur in rams: + if const_int(design.param(cur, "OUT_REG")) != 0: + raise Unsupported("its instantiation also makes a RAM that registers its read") + nxs = [] + for cur in rams: + nxs.append(f.ram(cur, args.check)) + trial = Edits() + trial.merge(edits) + trial.merge(f.edits) + trial.apply(design.srcs) + edits = trial + for ram, nx in zip(rams, nxs): + folded.append(ram.hierarchicalPath) + report.append("fold %s (%sx%s): raddr <- %s" % ( + ram.hierarchicalPath, design.param(ram, "SIZE"), design.param(ram, "DATAW"), nx)) + except Exception as ex: + if not isinstance(ex, Unsupported): + ex = "internal error: %s: %s" % (type(ex).__name__, ex) + for ram in rams: + why = str(ex) if ram is cur else "%s cannot fold: %s" % (cur.hierarchicalPath, ex) + report.append("keep %s (%sx%s): %s" % ( + ram.hierarchicalPath, design.param(ram, "SIZE"), design.param(ram, "DATAW"), why)) + patched = edits.apply(design.srcs) if folded else {} + return design, report, patched, folded + + +def main(): + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--top", action="append", required=True, help="top module (slang --top)") + ap.add_argument("--report", help="write the fold report here") + ap.add_argument("--diff", help="write the patch as a unified diff here") + ap.add_argument("--check", action="store_true", + help="add a simulation check that each folded RAM's next address is its address a cycle later") + ap.add_argument("--all", action="store_true", + help="fold every combinational VX_dp_ram read, whatever its size (testing the pass)") + ap.add_argument("--dry-run", action="store_true", help="report only, do not patch") + ap.add_argument("files", nargs="*") + args, slang = ap.parse_known_args() + args.slang = [x for t in args.top for x in ("--top", t)] + [a for a in slang if not a.endswith((".sv", ".v"))] + args.files = [a for a in args.files] + [a for a in slang if a.endswith((".sv", ".v"))] + + design, report, patched, folded = run(args) + + diff = [] + if patched and not args.dry_run: + originals = {p: design.srcs.bytes(p) for p in patched} + for p, b in patched.items(): + with open(p, "wb") as f: + f.write(b) + new = Design(args.slang, args.files) + added = new.errors - design.errors + if added: + for p, b in originals.items(): + with open(p, "wb") as f: + f.write(b) + report.append("REVERTED: the patched sources do not elaborate:") + report += [" " + e for e in sorted(added)] + folded = [] + else: + still = [r.hierarchicalPath for r in new.rams + if r.hierarchicalPath in folded and const_int(new.param(r, "OUT_REG")) != 1] + if still: + raise SystemExit("ram_fold: internal error, not folded after patching: %s" % still) + for p, b in patched.items(): + diff += difflib.unified_diff(originals[p].decode().splitlines(True), + b.decode().splitlines(True), fromfile=p, tofile=p) + if args.diff: + with open(args.diff, "w") as f: + f.writelines(diff) + + text = "\n".join(report) + ("\n" if report else "") + if args.report: + with open(args.report, "w") as f: + f.write(text) + sys.stdout.write(text) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/hw/scripts/sv2v.sh b/hw/scripts/sv2v.sh deleted file mode 100755 index 3f2f928b8f..0000000000 --- a/hw/scripts/sv2v.sh +++ /dev/null @@ -1,80 +0,0 @@ -#!/bin/bash - -# Copyright © 2019-2023 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# this script uses sv2v and yosys tools to run. -# sv2v: https://github.com/zachjs/sv2v -# yosys: http://www.clifford.at/yosys/ - -# Tool binaries: prefer caller-supplied env vars (set by hw/syn/common.mk -# to absolute $(TOOLDIR)/{sv2v,yosys}/bin/...) so the build is self- -# contained. Fall back to PATH lookup for callers that don't set them. -SV2V="${SV2V:-sv2v}" -YOSYS="${YOSYS:-yosys}" - -# exit when any command fails -set -e - -source="" -includes=() -macro_args="" -output_file=out.v - -usage() { echo "$0 usage:" && grep " .)\ #" $0; exit 0; } -[ $# -eq 0 ] && usage -while getopts "t:s:o:I:D:h" arg; do - case $arg in - t) # source - top=--top=${OPTARG} - ;; - s) # source - source=${OPTARG} - ;; - o) # output-file - output_file=${OPTARG} - ;; - I) # include directory - includes+=(${OPTARG}) - ;; - D) # macro definition - macro_args="$macro_args -D${OPTARG}" - ;; - h | *) - usage - exit 0 - ;; - esac -done - -# process include paths -inc_args="" -for dir in "${includes[@]}" -do - inc_args="$inc_args -I$dir" -done - -# process source files -file_args=$source -for dir in "${includes[@]}" -do - for file in $(find $dir -maxdepth 1 -name '*.v' -o -name '*.sv' -type f) - do - echo "file: $file" - file_args="$file_args $file" - done -done - -# system-verilog to verilog conversion -echo "$SV2V $top $macro_args $inc_args $file_args -v -w $output_file" -"$SV2V" $top $macro_args $inc_args $file_args -v -w $output_file \ No newline at end of file diff --git a/hw/scripts/xilinx_async_bram_patch.tcl b/hw/scripts/xilinx_async_bram_patch.tcl deleted file mode 100644 index f145035899..0000000000 --- a/hw/scripts/xilinx_async_bram_patch.tcl +++ /dev/null @@ -1,563 +0,0 @@ -# Copyright © 2019-2023 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -namespace eval vortex { - -variable debug 0 -variable start_time [clock seconds] -variable cpu_start_time [clock clicks -milliseconds] - -# --- Standardized Logging Wrappers --- - -proc print_msg {level msg} { - variable debug - set prefix "\[AsyncBRAM Patch\]" - switch $level { - "ERROR" { puts "ERROR: $prefix $msg" } - "CRITICAL" { puts "CRITICAL WARNING: $prefix $msg" } - "WARNING" { puts "WARNING: $prefix $msg" } - "INFO" { puts "INFO: $prefix $msg" } - "DEBUG" { if {$debug} { puts "DEBUG: $prefix $msg" } } - } -} - -proc log_error {msg {exit_code -1}} { - print_msg "ERROR" $msg - if {$exit_code != 0} { exit $exit_code } -} - -proc log_critical {msg} { - print_msg "CRITICAL" $msg -} - -proc log_warning {msg} { - print_msg "WARNING" $msg -} - -proc log_info {msg} { - print_msg "INFO" $msg -} - -proc log_debug {msg} { - print_msg "DEBUG" $msg -} - -# --- Time Formatting Utils --- - -proc format_time_val {total_seconds} { - set s [expr {int($total_seconds)}] - set ms [expr {int(($total_seconds - $s) * 100)}] - set h [expr {$s / 3600}] - set m [expr {($s % 3600) / 60}] - set s [expr {$s % 60}] - - if {$ms > 0} { - return [format "%02d:%02d:%02d.%02d" $h $m $s $ms] - } else { - return [format "%02d:%02d:%02d" $h $m $s] - } -} - -proc log_time {} { - variable start_time - variable cpu_start_time - - set now [clock seconds] - set cpu_now [clock clicks -milliseconds] - - set elapsed_s [expr {$now - $start_time}] - set cpu_s [expr {($cpu_now - $cpu_start_time) / 1000.0}] - - set cpu_fmt [format_time_val $cpu_s] - set elapsed_fmt [format_time_val $elapsed_s] - - puts "INFO: \[AsyncBRAM Patch\] Time (s): cpu = $cpu_fmt ; elapsed = $elapsed_fmt ." -} - -# --- String & Regex Utilities --- - -proc str_replace {str match repl} { - set result "" - regsub $match $str $repl result - return $result -} - -proc regex_escape {str} { - return [string map { - \\ \\\\ - ^ \\^ - . \\. - \[ \\\[ - \] \\\] - \$ \\\$ - \( \\\( - \) \\\) - | \\| - * \\* - + \\+ - ? \\? - \{ \\\{ - \} \\\} - } $str] -} - -proc unique_cell_name {name} { - if {[get_cells -quiet $name] == {}} { return $name } - set index 0 - while {[get_cells -quiet ${name}_${index}] != {}} { incr index } - return ${name}_${index} -} - -proc unique_net_name {name} { - if {[get_nets -quiet $name] == {}} { return $name } - set index 0 - while {[get_nets -quiet ${name}_${index}] != {}} { incr index } - return ${name}_${index} -} - -# --- Net Finding Utilities --- - -# Fast retrieval of the nets one level inside $cell (i.e. PARENT_CELL == $cell). -# Scoping with current_instance is O(nets-in-cell); the equivalent -# 'get_nets -hierarchical -filter {PARENT_CELL == $cell}' rescans the entire -# design's net database on every call (O(total_nets)), which dominated runtime -# on large multi-core designs. The NAME property is still the full hierarchical -# name, so all downstream regexp/name matching is unchanged. -proc cell_nets {cell} { - current_instance -quiet $cell - set nets [get_nets -quiet *] - current_instance -quiet - return $nets -} - -proc find_cell_nets {cell name_match {required 1}} { - set matching_nets {} - foreach net [cell_nets $cell] { - set name [get_property NAME $net] - if {[regexp $name_match $name]} { - lappend matching_nets $net - } - } - - if {[llength $matching_nets] == 0 && $required} { - log_error "No matching net found for '$cell' matching '$name_match'." - } - return $matching_nets -} - -proc find_cell_net {cell name_match {required 1}} { - set nets [find_cell_nets $cell $name_match $required] - - if {[llength $nets] == 0} { - return "" - } elseif {[llength $nets] > 1} { - log_error "Multiple matching nets found for '$cell' matching '$name_match'." - } - return [lindex $nets 0] -} - -proc get_cell_net {cell name} { - set net {} - foreach n [cell_nets $cell] { - if {[get_property NAME $n] == $name} { set net $n; break } - } - if {[llength $net] == 0} { - log_error "No matching net found for '$cell' matching '$name'." - } - return $net -} - -proc get_cell_pin {cell name} { - set pin [get_pins -of_objects $cell -filter "NAME == $name"] - if {[llength $pin] == 0} { - log_error "No matching pin found for '$cell' matching '$name'." - } - return $pin -} - -# --- Modification Utilities --- - -proc remove_cell_from_netlist {cell} { - foreach pin [get_pins -quiet -of_objects $cell] { - foreach net [get_nets -quiet -of_objects $pin] { - disconnect_net -net $net -objects $pin - log_debug "Disconnected net '$net' from pin '$pin'." - } - } - remove_cell $cell - log_debug "Cell '$cell' was removed successfully." -} - -proc find_net_driver {target_net {required 1}} { - set driverPins [get_pins -quiet -leaf -of_objects $target_net -filter {DIRECTION == "OUT"}] - - if {[llength $driverPins] == 0} { - set driverPorts [get_ports -quiet -of_objects $target_net -filter {DIRECTION == "IN"}] - - if {[llength $driverPorts] == 0} { - if {$required} { log_error "No driver found for '$target_net'." } - return "" - } elseif {[llength $driverPorts] > 1} { - log_warning "Multiple driver ports found for '$target_net'. Using first." - return [lindex $driverPorts 0] - } - return $driverPorts - } elseif {[llength $driverPins] > 1} { - log_warning "Multiple driver pins found for '$target_net'. Using first." - return [lindex $driverPins 0] - } - return $driverPins -} - -proc find_pin_driver {target_pin {required 1}} { - set net [get_nets -quiet -of_objects $target_pin] - - if {[llength $net] == 0} { - if {$required} { log_error "No net connected to pin '$target_pin'." } - return "" - } elseif {[llength $net] > 1} { - log_error "Multiple nets connected to pin '$target_pin'." - } - return [find_net_driver $net] -} - -# --- Core Logic --- - -proc create_register_next {parent reg_cell raddr_reset} { - set hier_sep [get_hierarchy_separator] - - set reg_d_pin [get_pins "${reg_cell}${hier_sep}D"] - if {[llength $reg_d_pin] != 1} { - log_error "Found [llength $reg_d_pin] D pins on register cell '$reg_cell'. Expected 1." - } - - log_debug "reg_d_pin: '$reg_d_pin'" - - set reg_d_src_pin [find_pin_driver $reg_d_pin] - log_debug "reg_d_src_pin: '$reg_d_src_pin'" - - # [Optimization] 1. Check if reset is requested by the patch instance - if {$raddr_reset == ""} { - log_debug "No reset requested by patch. Skipping LUT." - return $reg_d_src_pin - } - - set register_type [get_property REF_NAME $reg_cell] - set reg_r_pin "" - set lut_init "" - - if {$register_type == "FDRE"} { - set reg_r_pin [get_pins -quiet "${reg_cell}${hier_sep}R"] - set lut_init "4'b0010" ;# Reset dominates - } elseif {$register_type == "FDSE"} { - set reg_r_pin [get_pins -quiet "${reg_cell}${hier_sep}S"] - set lut_init "4'b1110" ;# Set dominates - } else { - log_error "Unsupported register type: '$register_type'." - } - - if {[llength $reg_r_pin] == 0} { - log_error "No Reset/Set pin found on $register_type cell '$reg_cell'." - } - - set reg_r_src_pin [find_pin_driver $reg_r_pin 0] - - # [Optimization] 2. Check if the register's reset pin is unconnected or grounded - if {$reg_r_src_pin == ""} { - log_debug "Reset pin unconnected. Optimization: Skipping LUT." - return $reg_d_src_pin - } else { - set driver_cell [get_cells -quiet -of_objects $reg_r_src_pin] - if {[llength $driver_cell] > 0} { - set driver_ref [get_property REF_NAME $driver_cell] - if {$driver_ref == "GND"} { - log_debug "Reset driven by GND. Optimization: Skipping LUT." - return $reg_d_src_pin - } - } - } - - log_debug "reg_r_src_pin: '$reg_r_src_pin'" - - set reg_d_src_net [get_nets -of_objects $reg_d_src_pin] - set reg_r_src_net [get_nets -of_objects $reg_r_src_pin] - - # Create LUT2 MUX - set lut_name [unique_cell_name "${parent}${hier_sep}raddr_next"] - set lut_cell [create_cell -reference LUT2 $lut_name] - set_property INIT $lut_init $lut_cell - - log_debug "Created LUT cell: '$lut_cell' (Init: $lut_init)" - - set lut_i0_pin [get_pins "${lut_cell}${hier_sep}I0"] - set lut_i1_pin [get_pins "${lut_cell}${hier_sep}I1"] - set lut_o_pin [get_pins "${lut_cell}${hier_sep}O"] - - connect_net -net $reg_d_src_net -objects $lut_i0_pin -hierarchical - connect_net -net $reg_r_src_net -objects $lut_i1_pin -hierarchical - - return $lut_o_pin -} - -proc getOrCreateVCCPin {parent} { - set hier_sep [get_hierarchy_separator] - set cell_name "${parent}${hier_sep}VCC" - - set vcc_cell [get_cells -quiet $cell_name] - if {[llength $vcc_cell] == 0} { - set vcc_cell [create_cell -reference VCC $cell_name] - log_debug "Created VCC cell: '$vcc_cell'" - } - - set vcc_pin [get_pins "${vcc_cell}${hier_sep}P"] - if {[llength $vcc_pin] == 0} { - log_error "No VCC pin found on VCC cell '$vcc_cell'." - } - return $vcc_pin -} - -proc getOrCreateGNDPin {parent} { - set hier_sep [get_hierarchy_separator] - set cell_name "${parent}${hier_sep}GND" - - set gnd_cell [get_cells -quiet $cell_name] - if {[llength $gnd_cell] == 0} { - set gnd_cell [create_cell -reference GND $cell_name] - log_debug "Created GND cell: '$gnd_cell'" - } - - set gnd_pin [get_pins "${gnd_cell}${hier_sep}G"] - if {[llength $gnd_pin] == 0} { - log_error "No GND pin found on GND cell '$gnd_cell'." - } - return $gnd_pin -} - -proc find_net_sinks {source_net {required 1}} { - set sink_pins {} - - foreach pin [get_pins -quiet -of_objects $source_net] { - set direction [get_property DIRECTION $pin] - # Input pins of nested cells - if {$direction == "IN"} { - lappend sink_pins $pin - } - # Output pins of the parent cell - set pin_cell [get_cells -of_objects $pin] - set is_primitive [get_property IS_PRIMITIVE $pin_cell] - if {$direction == "OUT" && !$is_primitive} { - lappend sink_pins $pin - } - } - - foreach port [get_ports -quiet -of_objects $source_net -filter {DIRECTION == "OUT"}] { - lappend sink_pins $port - } - - if {[llength $sink_pins] == 0 && $required} { - log_error "No sink found for '$source_net'." - } - return $sink_pins -} - -proc find_matching_nets {cell nets match repl} { - set matching_nets {} - foreach net $nets { - set net_name [str_replace $net $match $repl] - set matching_net [get_cell_net $cell $net_name] - if {$matching_net != ""} { - lappend matching_nets $matching_net - } - } - - if {[llength $matching_nets] != [llength $nets]} { - log_error "Mismatch in number of matching nets. Found [llength $matching_nets], expected [llength $nets]." - } - return $matching_nets -} - -proc replace_net_source {net source_pin} { - foreach pin [find_net_sinks $net 0] { - # 1. Disconnect - disconnect_net -net $net -objects $pin - log_debug "Disconnected net '$net' from pin '$pin'." - - # 2. Find/Create Source Net - set source_net [get_nets -quiet -of_objects $source_pin] - - if {[llength $source_net] == 0} { - set source_cell [get_cells -of_objects $source_pin] - set net_name [unique_net_name "${source_cell}_tmp_net"] - set source_net [create_net $net_name] - connect_net -net $source_net -objects $source_pin -hierarchical - log_debug "Created new source_net '$source_net' and connected to '$source_pin'." - } elseif {[llength $source_net] > 1} { - log_error "Multiple nets connected to pin '$source_pin'." - } - - # 3. Connect Sink - set external_net [get_nets -quiet -of_objects $pin] - - if {[llength $external_net] == 0} { - connect_net -net $source_net -objects $pin -hierarchical - log_debug "Connected net '$source_net' to pin '$pin'." - } elseif {[llength $external_net] == 1} { - foreach external_pin [get_pins -of_objects $external_net] { - disconnect_net -net $external_net -objects $pin - replace_net_source $external_net $source_pin - } - } else { - log_error "Multiple nets connected to pin '$pin'." - } - } -} - -# --- Main Patching Routine --- - -proc resolve_async_bram {inst} { - log_info "Resolving asynchronous BRAM patch: '$inst'." - - set hier_sep [get_hierarchy_separator] - - # 1. Find Nets - set raddr_w_nets [find_cell_nets $inst "raddr_w(\\\[\\d+\\\])?$"] - set raddr_s_nets [find_matching_nets $inst $raddr_w_nets "raddr_w(\\\[\\d+\\\])?$" "raddr_s\\1"] - - set read_s_net [find_cell_net $inst "read_s$"] - log_debug "read_s_net: '$read_s_net'" - - set is_raddr_reg_net [find_cell_net $inst "g_async_ram.is_raddr_reg$" 0] - set raddr_reset_net [find_cell_net $inst "raddr_reset$" 0] - log_debug "raddr_reset: '$raddr_reset_net'" - - set reg_next_pins {} - set reg_ce_src_pin "" - - # 2. Analyze Drivers & Create Logic - foreach raddr_w_net $raddr_w_nets { - log_debug "Processing raddr_w net: '$raddr_w_net'" - - set raddr_src_pin [find_net_driver $raddr_w_net] - - if {[get_ports -quiet $raddr_src_pin] ne ""} { - log_warning "Net '$raddr_w_net' driven by port. Skipping patch for this instance." - break - } - - set raddr_src_cell [get_cells -of_objects $raddr_src_pin] - set driver_type [get_property REF_NAME $raddr_src_cell] - - if {$driver_type != "FDRE" && $driver_type != "FDSE"} { - log_warning "Net '$raddr_w_net' driven by '$driver_type' (not FDRE/FDSE). Skipping." - break - } - - # Create Next Logic (with optimization) - set reg_next_pin [create_register_next $inst $raddr_src_cell $raddr_reset_net] - if {$reg_next_pin == ""} { - log_error "Failed to create register next value for '$raddr_src_cell'." - } - - lappend reg_next_pins $reg_next_pin - - # Capture CE Pin (once) - if {$reg_ce_src_pin == ""} { - set reg_ce_pin [get_pins "${raddr_src_cell}${hier_sep}CE"] - if {[llength $reg_ce_pin] != 1} { - log_error "Expected 1 CE pin on '$raddr_src_cell', found [llength $reg_ce_pin]." - } - set reg_ce_src_pin [find_pin_driver $reg_ce_pin] - log_debug "reg_ce_src_pin: '$reg_ce_src_pin'" - } - } - - set addr_width [llength $raddr_w_nets] - - # 3. Apply Patch or Fallback - if {[llength $reg_next_pins] == $addr_width} { - log_debug "Fully registered read address detected. Patching..." - - # Connect Address - for {set i 0} {$i < $addr_width} {incr i} { - replace_net_source [lindex $raddr_s_nets $i] [lindex $reg_next_pins $i] - } - - # Connect CE - replace_net_source $read_s_net $reg_ce_src_pin - - # Connect Const 1 (Valid Patch) - if {$is_raddr_reg_net != ""} { - set vcc_pin [getOrCreateVCCPin $inst] - replace_net_source $is_raddr_reg_net $vcc_pin - } - } else { - log_critical "Read address not fully registered. Falling back to Asynchronous (Grounding)." - - set gnd_pin [getOrCreateGNDPin $inst] - - # Ground Address & Control - foreach net $raddr_s_nets { replace_net_source $net $gnd_pin } - replace_net_source $read_s_net $gnd_pin - - # Connect Const 0 (Invalid Patch) - if {$is_raddr_reg_net != ""} { - replace_net_source $is_raddr_reg_net $gnd_pin - } - } - - # 4. Cleanup (Optimized for Speed) - remove_cell_from_netlist [get_cells -quiet "${inst}${hier_sep}placeholder1"] - if {$is_raddr_reg_net != ""} { - remove_cell_from_netlist [get_cells -quiet "${inst}${hier_sep}*placeholder2*"] - } -} - -proc resolve_async_brams {} { - variable debug - set bram_patch_cells [get_cells -hierarchical -filter {REF_NAME =~ "*VX_async_ram_patch*"}] - if {[llength $bram_patch_cells] != 0} { - foreach cell $bram_patch_cells { - resolve_async_bram $cell - } - } else { - log_info "No async BRAM patch cells found in the design." - } -} - -proc dump_cell_hierarchy {parent_cell indent} { - set children [get_cells -quiet -hierarchical -filter "PARENT_CELL == $parent_cell"] - foreach child $children { - set type [get_property REF_NAME $child] - log_info "${indent}child cell: '$child', type: '$type'" - dump_cell_hierarchy $child "${indent} " - } -} - -proc dump_async_bram_cells {} { - set bram_patch_cells [get_cells -hierarchical -filter {REF_NAME =~ "*VX_async_ram_patch*"}] - if {[llength $bram_patch_cells] != 0} { - foreach cell $bram_patch_cells { - log_info "Found async BRAM patch cell: '$cell'." - dump_cell_hierarchy $cell " " - } - } else { - log_info "No async BRAM patch cells found in the design." - } -} - -} - -# Run -vortex::resolve_async_brams -# vortex::dump_async_bram_cells -vortex::log_time \ No newline at end of file diff --git a/hw/scripts/xilinx_cone_dump.tcl b/hw/scripts/xilinx_cone_dump.tcl new file mode 100644 index 0000000000..7fc60dce0b --- /dev/null +++ b/hw/scripts/xilinx_cone_dump.tcl @@ -0,0 +1,59 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_cone_dump.tcl ...: for each pattern, the worst +# setup path into the matching cells, cell by cell, whatever its slack. When the +# path starts at a retimed register (_fret/_bret), the worst path into that +# register follows, up to four hops, and the levels are summed: the depth of +# the RTL cone as Vivado built it. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +open_checkpoint $dcp +set f [open $out w] + +proc dump_path {f p indent} { + puts $f "${indent}slack [get_property SLACK $p] levels [get_property LOGIC_LEVELS $p] logic [get_property DATAPATH_LOGIC_DELAY $p] net [get_property DATAPATH_NET_DELAY $p]" + puts $f "${indent} from [get_property STARTPOINT_PIN $p]" + puts $f "${indent} to [get_property ENDPOINT_PIN $p]" + foreach c [get_cells -of_objects $p] { + puts $f "${indent} [get_property REF_NAME $c] [get_property NAME $c]" + } +} + +foreach pat [lrange $argv 2 end] { + puts $f "=== $pat" + set cells [get_cells -quiet -hier -filter "IS_SEQUENTIAL && NAME =~ $pat"] + if {[llength $cells] == 0} { + set cells [get_cells -quiet -hier -filter "PRIMITIVE_LEVEL != MACRO && NAME =~ $pat"] + } + puts $f "cells [llength $cells]" + if {[llength $cells] == 0} continue + set pins [get_pins -quiet -of_objects $cells -filter "DIRECTION == IN && IS_CLOCK == 0"] + set p [get_timing_paths -quiet -setup -max_paths 1 -nworst 1 -to $pins] + if {[llength $p] == 0} { puts $f "no timed path"; continue } + set total 0 + set indent "" + for {set hop 0} {$hop < 5} {incr hop} { + dump_path $f $p $indent + set total [expr {$total + [get_property LOGIC_LEVELS $p]}] + set start [get_cells -quiet -of_objects [get_property STARTPOINT_PIN $p]] + if {$start eq "" || ![regexp {_[fb]ret} [get_property NAME $start]]} break + set spins [get_pins -quiet -of_objects $start -filter "DIRECTION == IN && IS_CLOCK == 0"] + set p [get_timing_paths -quiet -setup -max_paths 1 -nworst 1 -to $spins] + if {[llength $p] == 0} break + set indent "$indent " + puts $f "${indent}<- into retimed start [get_property NAME $start]" + } + puts $f "summed levels $total" +} +close $f diff --git a/hw/scripts/xilinx_cone_probe.tcl b/hw/scripts/xilinx_cone_probe.tcl new file mode 100644 index 0000000000..861f6dc210 --- /dev/null +++ b/hw/scripts/xilinx_cone_probe.tcl @@ -0,0 +1,64 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_cone_probe.tcl ...: for each from|to pair of cell +# patterns ("-" for any), up to 500 setup paths between them; prints the +# level histogram, the endpoints of the deepest and the deepest path cell by +# cell. Answers whether a cone Beacon reports exists in Vivado's netlist, +# and how deep Vivado built it. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +open_checkpoint $dcp +set f [open $out w] + +proc cells_of {pat} { + return [get_cells -quiet -hier -filter "IS_SEQUENTIAL && NAME =~ $pat"] +} + +foreach {from to} [lrange $argv 2 end] { + puts $f "=== from $from to $to" + set args [list -quiet -setup -max_paths 500 -nworst 1] + if {$from ne "-"} { + set fc [cells_of $from] + puts $f "from cells [llength $fc]" + if {[llength $fc] == 0} continue + lappend args -from $fc + } + if {$to ne "-"} { + set tc [cells_of $to] + puts $f "to cells [llength $tc]" + if {[llength $tc] == 0} continue + lappend args -to [get_pins -quiet -of_objects $tc -filter "DIRECTION == IN && IS_CLOCK == 0"] + } + set paths [get_timing_paths {*}$args] + puts $f "paths [llength $paths]" + if {[llength $paths] == 0} continue + array unset hist + set deepest "" + set dl -1 + foreach p $paths { + set l [get_property LOGIC_LEVELS $p] + incr hist($l) + if {$l > $dl} { set dl $l; set deepest $p } + } + set h "" + foreach l [lsort -integer [array names hist]] { append h " $l:$hist($l)" } + puts $f "levels$h" + puts $f "deepest slack [get_property SLACK $deepest] levels $dl" + puts $f " from [get_property STARTPOINT_PIN $deepest]" + puts $f " to [get_property ENDPOINT_PIN $deepest]" + foreach c [get_cells -of_objects $deepest] { + puts $f " [get_property REF_NAME $c] [get_property NAME $c]" + } +} +close $f diff --git a/hw/scripts/xilinx_ip_gen.tcl b/hw/scripts/xilinx_ip_gen.tcl index 16bb3541a4..119cd8fec3 100644 --- a/hw/scripts/xilinx_ip_gen.tcl +++ b/hw/scripts/xilinx_ip_gen.tcl @@ -41,10 +41,10 @@ create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -modu set_property -dict [list CONFIG.Component_Name {xil_fma} CONFIG.Operation_Type {FMA} CONFIG.Add_Sub_Value {Add} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Has_UNDERFLOW {true} CONFIG.C_Has_OVERFLOW {true} CONFIG.C_Has_INVALID_OP {true} CONFIG.Has_A_TUSER {false} CONFIG.A_Precision_Type {Single} CONFIG.C_A_Exponent_Width {8} CONFIG.C_A_Fraction_Width {24} CONFIG.Result_Precision_Type {Single} CONFIG.C_Result_Exponent_Width {8} CONFIG.C_Result_Fraction_Width {24} CONFIG.C_Mult_Usage {Medium_Usage} CONFIG.Has_RESULT_TREADY {false} CONFIG.C_Latency {16} CONFIG.C_Rate {1} CONFIG.A_TUSER_Width {1}] [get_ips xil_fma] create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -module_name xil_fmul -dir ${ip_dir} -set_property -dict [list CONFIG.Operation_Type {Multiply} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage}] [get_ips xil_fmul] +set_property -dict [list CONFIG.Operation_Type {Multiply} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage} CONFIG.C_Latency {8}] [get_ips xil_fmul] create_ip -name floating_point -vendor xilinx.com -library ip -version 7.1 -module_name xil_fadd -dir ${ip_dir} -set_property -dict [list CONFIG.Operation_Type {Add_Subtract} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage}] [get_ips xil_fadd] +set_property -dict [list CONFIG.Operation_Type {Add_Subtract} CONFIG.A_Precision_Type {Single} CONFIG.Result_Precision_Type {Single} CONFIG.Has_RESULT_TREADY {false} CONFIG.Flow_Control {NonBlocking} CONFIG.Has_ACLKEN {true} CONFIG.C_Rate {1} CONFIG.C_Mult_Usage {Full_Usage} CONFIG.C_Latency {11}] [get_ips xil_fadd] generate_target all [get_ips] diff --git a/hw/scripts/xilinx_ip_stubs.sv b/hw/scripts/xilinx_ip_stubs.sv new file mode 100644 index 0000000000..4b5860da1a --- /dev/null +++ b/hw/scripts/xilinx_ip_stubs.sv @@ -0,0 +1,111 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Black-box declarations of the Xilinx floating-point cores that +// xilinx_ip_gen.tcl generates: their ports as configured there, no contents. +// The CIRCT elaboration reads them to know each core's interface and keeps +// every instance as an instance; Vivado binds it to the generated core. +// Synthesis never reads this file. Keep it in step with xilinx_ip_gen.tcl. +// +// Each declares the contract the RTL uses it under: a fixed-latency +// pipeline. Its data_out outputs are a function of its data_in inputs of +// exactly fixed_latency cycles of `clock` with `clock_enable` high before, +// and of nothing else (pure_pipeline: non-blocking flow control, no reset, no other state); the `tie` +// inputs are driven with those constants, its other outputs are unused. +// fixed_latency is the core's C_Latency in xilinx_ip_gen.tcl and the latency +// the RTL asserts (hw/scripts/check_ip_contracts.py checks both). + +(* black_box, fixed_latency = 28, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1", pure_pipeline *) +module xil_fdiv ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [3:0] m_axis_result_tuser // underflow, overflow, invalid, divide by zero +); +endmodule + +(* black_box, fixed_latency = 28, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1", pure_pipeline *) +module xil_fsqrt ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [0:0] m_axis_result_tuser // invalid +); +endmodule + +(* black_box, fixed_latency = 16, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata s_axis_c_tdata", + data_out = "m_axis_result_tdata m_axis_result_tuser", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1 s_axis_c_tvalid=1", pure_pipeline *) +module xil_fma ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_c_tvalid, + input wire [31:0] s_axis_c_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [2:0] m_axis_result_tuser // underflow, overflow, invalid +); +endmodule + +(* black_box, fixed_latency = 8, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata", + data_out = "m_axis_result_tdata", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1", pure_pipeline *) +module xil_fmul ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); +endmodule + +(* black_box, fixed_latency = 11, clock = "aclk", clock_enable = "aclken", + data_in = "s_axis_a_tdata s_axis_b_tdata s_axis_operation_tdata", + data_out = "m_axis_result_tdata", + tie = "s_axis_a_tvalid=1 s_axis_b_tvalid=1 s_axis_operation_tvalid=1", pure_pipeline *) +module xil_fadd ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_operation_tvalid, + input wire [7:0] s_axis_operation_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); +endmodule diff --git a/hw/scripts/xilinx_slr_pblocks.tcl b/hw/scripts/xilinx_slr_pblocks.tcl index 3001d908a5..3bd5540103 100644 --- a/hw/scripts/xilinx_slr_pblocks.tcl +++ b/hw/scripts/xilinx_slr_pblocks.tcl @@ -186,7 +186,10 @@ proc slr_pblocks_run {} { # Per-cluster reconciliation: everything sizeable inside a cluster must be # captured by an atom or a cluster cache. If it is not, the atom model is # stale (RTL changed) and we must NOT floorplan on a partial view. - set clusters [cells_re {.*/g_clusters\[[0-9]+\]\.cluster}] + # + # A generate-loop instance is g_x[i].inst read from the RTL, and g_x_i_inst + # read from the CIRCT netlist, which names generate scopes with `_`. + set clusters [cells_re {.*/g_clusters(\[[0-9]+\]\.|_[0-9]+_)cluster}] if {[llength $clusters] == 0} { # Non-clustered top (single cluster inlined): treat vortex as one region. set clusters [cells_re {.*/(Vortex|vortex)$}] @@ -199,9 +202,9 @@ proc slr_pblocks_run {} { set acc 0 # sockets (cores + private L1s), whole. - foreach s [cells_re "[slr_re_escape $clp]/g_sockets\\\[\[0-9\]+\\\]\\.socket"] { + foreach s [cells_re "[slr_re_escape $clp]/g_sockets(\\\[\[0-9\]+\\\]\\.|_\[0-9\]+_)socket"] { set g {} - foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/g_cores\\\[\[0-9\]+\\\]\\.core"] { lappend g $c } + foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/g_cores(\\\[\[0-9\]+\\\]\\.|_\[0-9\]+_)core"] { lappend g $c } foreach c [cells_re "[slr_re_escape [get_property NAME $s]]/(icache|dcache)"] { lappend g $c } if {[llength $g] == 0} { set g [list $s] } set f {0 0 0 0} @@ -212,10 +215,10 @@ proc slr_pblocks_run {} { # extension core BUNDLED with its private cache. foreach {corere cachere} { - {.*\.tex_core} {.*/tcache} - {.*\.raster_core} {.*/rcache} - {.*\.om_core} {.*/ocache} - {.*\.rtu_core} {.*/rtcache} + {.*[._]tex_core} {.*/tcache} + {.*[._]raster_core} {.*/rcache} + {.*[._]om_core} {.*/ocache} + {.*[._]rtu_core} {.*/rtcache} {.*/dxa_core} {} } { foreach core [get_cells -quiet -hierarchical -regexp $corere] { diff --git a/hw/scripts/xilinx_wire_probe.tcl b/hw/scripts/xilinx_wire_probe.tcl new file mode 100644 index 0000000000..3fec054a0f --- /dev/null +++ b/hw/scripts/xilinx_wire_probe.tcl @@ -0,0 +1,62 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# xilinx_wire_probe.tcl [max paths]: for the worst setup path of up +# to N end points, the placement the wire delay comes from: the Manhattan +# length of the path in tiles (sum over consecutive cells), the bounding box +# of its cells, and the largest fanout of a net on it. Feature 5's upper +# bound: how much of the net delay a perfect placement estimate explains. +set dcp [lindex $argv 0] +set out [lindex $argv 1] +set n [expr {[llength $argv] > 2 ? [lindex $argv 2] : 5000}] +open_checkpoint $dcp +set f [open $out w] +puts $f "slack,levels,logic,net,length,bbox,fanout,start,end" + +proc tile_xy {c} { + set site [get_sites -quiet -of_objects $c] + if {$site eq ""} { return "" } + set t [get_tiles -quiet -of_objects $site] + if {$t eq ""} { return "" } + return [list [get_property COLUMN $t] [get_property ROW $t]] +} + +foreach p [get_timing_paths -setup -max_paths $n -nworst 1 -unique_pins] { + set len 0 + set prev "" + set xs {} + set ys {} + foreach c [get_cells -quiet -of_objects $p] { + set xy [tile_xy $c] + if {$xy eq ""} continue + lappend xs [lindex $xy 0] + lappend ys [lindex $xy 1] + if {$prev ne ""} { + set len [expr {$len + abs([lindex $xy 0] - [lindex $prev 0]) + abs([lindex $xy 1] - [lindex $prev 1])}] + } + set prev $xy + } + set bbox 0 + if {[llength $xs]} { + set xs [lsort -integer $xs] + set ys [lsort -integer $ys] + set bbox [expr {[lindex $xs end] - [lindex $xs 0] + [lindex $ys end] - [lindex $ys 0]}] + } + set fo 0 + foreach net [get_nets -quiet -of_objects $p] { + set k [get_property FLAT_PIN_COUNT $net] + if {$k > $fo} { set fo $k } + } + puts $f "[get_property SLACK $p],[get_property LOGIC_LEVELS $p],[get_property DATAPATH_LOGIC_DELAY $p],[get_property DATAPATH_NET_DELAY $p],$len,$bbox,[expr {$fo - 1}],[get_property STARTPOINT_PIN $p],[get_property ENDPOINT_PIN $p]" +} +close $f diff --git a/hw/syn/altera/dut/catalog.mk b/hw/syn/altera/dut/catalog.mk index 2bca1ee61b..233d580c83 100644 --- a/hw/syn/altera/dut/catalog.mk +++ b/hw/syn/altera/dut/catalog.mk @@ -9,7 +9,7 @@ # Add a DUT: append its name to DUTS and define its variables. No new directory, # no dispatcher edit. -DUTS := cache core fpu issue lmem mem_unit scope top unittest vortex +DUTS := cache core fpu issue lmem mem_unit scope top vortex # --------------------------------------------------------------------------- # shared include fragments @@ -58,8 +58,8 @@ issue_PRJ := VX_issue_top issue_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) \ -I$(IP_CACHE_DIR) -I$(UNITTEST_DIR)/issue -lmem_PRJ := VX_local_mem_top -lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(UNITTEST_DIR)/local_mem +lmem_PRJ := VX_lmem_unit_top +lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(UNITTEST_DIR)/lmem_unit mem_unit_PRJ := VX_mem_unit_top mem_unit_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/core -I$(RTL_DIR)/fpu \ @@ -75,10 +75,6 @@ top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RT -I$(AFU_DIR) -I$(AFU_DIR)/ccip -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) top_PKG = $(TCU_PKG) $(DXA_PKG) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ - -I$(IP_CACHE_DIR) $(FPU_INC) - vortex_PRJ := Vortex vortex_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(IP_CACHE_DIR) $(FPU_INC) $(TCU_INC) $(DXA_INC) diff --git a/hw/syn/altera/dut/common.mk b/hw/syn/altera/dut/common.mk index 962d005e46..d0d786d1df 100644 --- a/hw/syn/altera/dut/common.mk +++ b/hw/syn/altera/dut/common.mk @@ -54,7 +54,7 @@ all: gen-sources $(PROJECT).sta.rpt $(PROJECT).pow.rpt gen-sources: src src: mkdir -p src - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Csrc $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Csrc $(RTL_PKGS) syn: $(PROJECT).syn.rpt diff --git a/hw/syn/common.mk b/hw/syn/common.mk index 24883c7502..333f6a10d1 100644 --- a/hw/syn/common.mk +++ b/hw/syn/common.mk @@ -21,17 +21,29 @@ ifndef TOOLDIR $(error TOOLDIR not set — include $$(ROOT_DIR)/config.mk before hw/syn/common.mk) endif -# Tool install prefixes (overridable; default under $(TOOLDIR)). -SV2V_PATH ?= $(TOOLDIR)/sv2v +# Tool install prefixes (overridable; default under $(TOOLDIR)). CIRCT is the +# SystemVerilog frontend of every flow: its preprocessor (gen_sources.sh -P) +# feeds Vivado, and its elaboration into one netlist (circt_elaborate.sh; +# gen_sources.sh -E) feeds Yosys, Beacon and Foldsim. Before either, the RAM +# fold (gen_sources.sh -F, hw/scripts/ram_fold.py) patches the preprocessed +# sources so that asynchronous RAM reads map to block RAM. +CIRCT_PATH ?= $(TOOLDIR)/circt YOSYS_PATH ?= $(TOOLDIR)/yosys STA_PATH ?= $(TOOLDIR)/sta VERILATOR_PATH ?= $(TOOLDIR)/verilator # Absolute tool binaries exported via env var so builds are self-contained # and multiple trees can coexist without a sourced env polluting PATH. -SV2V ?= $(SV2V_PATH)/bin/sv2v +CIRCT_VERILOG ?= $(CIRCT_PATH)/bin/circt-verilog +CIRCT_OPT ?= $(CIRCT_PATH)/bin/circt-opt +FIRTOOL ?= $(CIRCT_PATH)/bin/firtool +FOLDSIM ?= $(FOLDSIM_PATH)/bin/foldsim +FOLDSIM_OPT ?= $(FOLDSIM_PATH)/bin/foldsim-opt +FOLDSIM_LIFT_TOOL ?= $(FOLDSIM_PATH)/bin/foldsim-lift YOSYS ?= $(YOSYS_PATH)/bin/yosys STA ?= $(STA_PATH)/bin/sta VERILATOR ?= $(VERILATOR_PATH)/bin/verilator +# a python3 with pyslang 11 or 12 (pip install "pyslang>=11,<13") +RAM_FOLD_PYTHON ?= python3 -export SV2V_PATH YOSYS_PATH STA_PATH VERILATOR_PATH SV2V YOSYS STA VERILATOR +export CIRCT_PATH FOLDSIM_PATH FOLDSIM_SHELL FOLDSIM_LIFT_TOOL YOSYS_PATH STA_PATH VERILATOR_PATH CIRCT_VERILOG CIRCT_OPT FIRTOOL FOLDSIM FOLDSIM_OPT YOSYS STA VERILATOR RAM_FOLD_PYTHON diff --git a/hw/syn/firesim/Makefile b/hw/syn/firesim/Makefile index 10e96cd472..e94cb94ccd 100644 --- a/hw/syn/firesim/Makefile +++ b/hw/syn/firesim/Makefile @@ -1,18 +1,28 @@ -# FireSim platform backend. +# FireSim image build for the Xilinx cards, driven exactly as the native flows: # -# The Chisel targets and the upstream patches live here, in the Vortex tree, so a -# fresh checkout can reproduce a build. The FireSim checkout itself is a working -# directory, not a source of truth: `stage` copies the targets into it and applies -# the patches, and anything left only in the checkout is expected to be lost. +# make PLATFORM= [TARGET=hw|hw_emu] an XRT card, as hw/syn/xilinx/xrt +# make [DEVICE=] [TARGET=hw|sim] an aved card, as hw/syn/xilinx/aved +# +# with PREFIX, XLEN, CONFIGS and PERF=1 as there too. The image lands in +# $(BUILD_DIR)/bin as firesim.xclbin or firesim.vbin, beside firesim.card, which +# names the card for run-firesim; the runtime's xrtsim/avedsim targets need no +# image at all. +# +# FireSim itself is the vortexgpgpu/firesim fork, consumed through FIRESIM_PATH +# and pinned by FIRESIM_REV in VERSION: every change Vortex needs in FireSim is a +# commit there, and the checkout must contain the pinned one. The Chisel targets, +# harnesses and the card's platform config live here and are staged into it. The +# card's descriptor (hw/syn/xilinx/platforms/.toml) supplies the FireSim +# platform, the FPGA part and device, the host-memory binding and its base. ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk +include $(VORTEX_HOME)/hw/syn/common.mk # configure copies this Makefile into the build tree, so ROOT_DIR resolves there # while sources are read from VORTEX_HOME. SRC_DIR := $(VORTEX_HOME)/hw/syn/firesim -PATCH_DIR := $(SRC_DIR)/patches # Targets are compiled as part of FireSim's midasexamples project, which is where # its harness and bridge helpers are resolvable from. @@ -27,53 +37,161 @@ CC_DEST := $(FIRESIM_PATH)/sim/src/main/cc/midasexamples RTL_DIR := $(VORTEX_HOME)/hw/rtl SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts -BUILD_DIR ?= $(ROOT_DIR)/hw/syn/firesim +XLEN ?= 32 +PREFIX ?= build$(XLEN) + +# Nothing given here on the command line reaches FireSim's own makes. A +# command-line variable propagates to every sub-make, and FireSim uses several +# of these names for its own purposes: PLATFORM is its platform (vitis/aved), +# not an XSA; TARGET reached v++ as `--target sim`; BUILD_DIR locates its +# elaborator jar. Each sub-make below is handed exactly what it needs instead. +MAKEOVERRIDES := + +# The card, named as the native flow for its kind names it. The runtime has an +# internal need to point the flattening elsewhere (FIRESIM_CARD, FIRESIM_WORK_DIR); +# those are not user knobs. +ifneq ($(FIRESIM_CARD),) +PLATFORM_GEN_ARGS := --card=$(FIRESIM_CARD) +else ifneq ($(PLATFORM),) +XSA := $(strip $(patsubst %.xpfm,%,$(notdir $(PLATFORM)))) +DEV_ARCH := $(shell platforminfo -p $(PLATFORM) 2>/dev/null | grep 'FPGA Family' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') +PLATFORM_GEN_ARGS := --flow=xrt --xsa=$(XSA) --family='$(DEV_ARCH)' +else +DEVICE ?= xcv80-lsva4737-2MHP-e-S +PLATFORM_GEN_ARGS := --flow=aved --device=$(DEVICE) +endif +PLATFORM_GEN := python3 $(SCRIPT_DIR)/gen_platform.py $(PLATFORM_GEN_ARGS) + +# The card is resolved before BUILD_DIR exists, because BUILD_DIR is named after +# it. $(shell) discards exit codes, so an empty answer means the generator +# failed, and it is asked again for its message. +FIRESIM_CARD := $(shell $(PLATFORM_GEN) --emit=card 2>/dev/null) +ifeq ($(FIRESIM_CARD),) +$(error $(shell $(PLATFORM_GEN) --emit=card 2>&1 >/dev/null)) +endif +FIRESIM_PLATFORM := $(shell python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk 2>&1 | sed -n 's/^FIRESIM_PLATFORM := //p') +ifeq ($(FIRESIM_PLATFORM),) +$(error $(shell python3 $(SCRIPT_DIR)/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk 2>&1 >/dev/null)) +endif + +# The image each card can take, as its native flow names it. +TARGET ?= hw +ifeq ($(FIRESIM_PLATFORM), aved) +VALID_TARGETS := hw sim +else +VALID_TARGETS := hw hw_emu +endif +ifeq ($(FIRESIM_WORK_DIR),) +ifeq ($(filter $(TARGET),$(VALID_TARGETS)),) +$(error TARGET=$(TARGET) is not an image target for card $(FIRESIM_CARD): use one of $(VALID_TARGETS) \ + (xrtsim and avedsim are run targets and need no image)) +endif +endif -# Flattened Vortex sources for the BlackBox. Golden Gate takes a file list, not -# include paths, so the tree is copied into one directory with the macros already -# inlined (-P) and emitted in elaboration order: packages, interfaces, modules. -VIVADO_SRCS_DIR := $(BUILD_DIR)/vivado_srcs -VIVADO_SRCS_LIST := $(BUILD_DIR)/vivado_srcs.f +# Named as the native flows name theirs, so FPGA_BIN_DIR means the same thing. +ifeq ($(FIRESIM_PLATFORM), aved) +BUILD_DIR := $(abspath $(PREFIX)_aved_$(TARGET)) +else +BUILD_DIR := $(abspath $(PREFIX)_$(if $(XSA),$(XSA),$(FIRESIM_CARD))_$(TARGET)) +endif +# The runtime's metasimulator reuses the simulation flattening from here, in a +# directory of its own. +ifneq ($(FIRESIM_WORK_DIR),) +BUILD_DIR := $(abspath $(FIRESIM_WORK_DIR)) +endif +BIN_DIR := $(BUILD_DIR)/bin + +FIRESIM_MK := $(BUILD_DIR)/firesim_platform.mk +$(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=firesim-mk -o $(FIRESIM_MK)) +include $(FIRESIM_MK) + +# A user-named XSA is the one to build for; the descriptor's is the default. +ifneq ($(XSA),) +DEVICE := $(XSA) +endif + +VORTEX_TARGET_CONFIG := VortexConfig + +# The FireSim flow's make variables for this card's build. +FIRESIM_TARGET_ARGS := PLATFORM=$(FIRESIM_PLATFORM) TARGET_PROJECT=midasexamples \ + DESIGN=VortexTarget TARGET_CONFIG=$(VORTEX_TARGET_CONFIG) PLATFORM_CONFIG=$(PLATFORM_CONFIG) +QUINTUPLET := $(FIRESIM_PLATFORM)-midasexamples-VortexTarget-$(VORTEX_TARGET_CONFIG)-$(PLATFORM_CONFIG) + +# Vortex for the BlackBox. Golden Gate takes a file list, not include paths, so +# the tree is copied into one directory with the macros already inlined (-P) and +# emitted in elaboration order: packages, interfaces, modules. CIRCT then +# elaborates it into the one netlist the image reads, as every synthesis flow +# does (circt_elaborate.sh); the list names that netlist, and the RTL list stays +# next to it as $(VIVADO_SRCS_NAME).rtl.f. +# +# A simulation image (TARGET=sim or hw_emu) gets its own flattening, under +# -DSIMULATION, as the native xrt flow does for hw_emu. +ifneq ($(filter $(TARGET),sim hw_emu),) +IMAGE_KIND := sim +IMAGE_SUFFIX := _sim +endif +VIVADO_SRCS_NAME := vivado_srcs$(IMAGE_SUFFIX) +VIVADO_SRCS_DIR := $(BUILD_DIR)/$(VIVADO_SRCS_NAME) +VIVADO_SRCS_LIST := $(BUILD_DIR)/$(VIVADO_SRCS_NAME).f +# The same tree flattened for simulation instead. gen_sources.sh writes a +# directory, not a stamp, so the rule records one to hang the dependency on. SRCS_DIR := $(BUILD_DIR)/vortex_srcs SRCS_LIST := $(BUILD_DIR)/vortex_srcs.f +SIM_SRCS_STAMP := $(SRCS_DIR)/.sources.stamp -.PHONY: stage patch sources check-firesim clean-staged +.PHONY: all help clean elaborate stage sources sim-sources vivado-sources xilinx-ips kernel-env image \ + metasim run-metasim check-firesim clean-staged +.DEFAULT_GOAL := all + +help: + @echo "FireSim image build (same tokens as hw/syn/xilinx/xrt and hw/syn/xilinx/aved):" + @echo " make PLATFORM= [TARGET=hw|hw_emu] XRT card" + @echo " make [DEVICE=] [TARGET=hw|sim] aved card (default: the V80)" + @echo " options: PREFIX= XLEN=32|64 CONFIGS=\"-DVX_CFG_...\" PERF=1" + @echo " image: __/bin/firesim.{xclbin,vbin}" + @echo " run: make -C tests/regression/ run-firesim TARGET= FPGA_BIN_DIR=" + @echo " (TARGET=xrtsim|avedsim need no image)" + @echo " developer goals: metasim run-metasim sources clean" + +# Copies only what changed: FireSim's make judges its elaboration by these +# files' timestamps, so rewriting identical files would redo Golden Gate (and +# relink the metasimulator) on every run. stage: check-firesim @mkdir -p $(TARGET_DEST) $(CC_DEST) @for f in $(SCALA_SRCS); do \ - cp -f "$$f" $(TARGET_DEST)/ || exit 1; \ - echo "staged $$(basename $$f)"; \ + cmp -s "$$f" $(TARGET_DEST)/$$(basename $$f) || { cp -f "$$f" $(TARGET_DEST)/ && echo "staged $$(basename $$f)"; } || exit 1; \ done @for f in $(CC_SRCS); do \ - cp -f "$$f" $(CC_DEST)/ || exit 1; \ - echo "staged $$(basename $$f)"; \ + cmp -s "$$f" $(CC_DEST)/$$(basename $$f) || { cp -f "$$f" $(CC_DEST)/ && echo "staged $$(basename $$f)"; } || exit 1; \ done + @$(PLATFORM_GEN) --emit=scala -o $(BUILD_DIR)/$(PLATFORM_CONFIG).scala + @cmp -s $(BUILD_DIR)/$(PLATFORM_CONFIG).scala $(TARGET_DEST)/$(PLATFORM_CONFIG).scala || { \ + cp -f $(BUILD_DIR)/$(PLATFORM_CONFIG).scala $(TARGET_DEST)/ && \ + echo "staged $(PLATFORM_CONFIG).scala ($(FIRESIM_CARD))"; } -# Applies in series against a pristine checkout. Ordering matters: 0003 and 0005 -# both build on 0001. Re-running against an already-patched checkout will report -# reversed hunks; that is the intended signal, not a failure to hide. -patch: check-firesim - @cd $(FIRESIM_PATH) && for p in $(sort $(wildcard $(PATCH_DIR)/[0-9]*.patch)); do \ - echo "applying $$(basename $$p)"; \ - patch -p2 --no-backup-if-mismatch < "$$p" || exit 1; \ - done - -XLEN ?= 32 -CONFIGS ?= $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='-DVX_CFG_XLEN=$(XLEN)') +# Architecture overrides, as the other synthesis flows take them, e.g. +# CONFIGS="-DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE". XCONFIGS is the fully +# resolved set, used only to select extension sources and type selectors. +CONFIGS ?= +XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') # VX_config.vh and VX_types.vh are generated into the build tree, so those come # from ROOT_DIR while the RTL itself comes from VORTEX_HOME. RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ - -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/xrt -I$(RTL_DIR)/afu/firesim + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(RTL_DIR)/afu/firesim + +RTL_PKGS := +include $(VORTEX_HOME)/hw/syn/extensions.mk -# Only XLEN and the TCU/FPU selectors are passed. Every other scalar is computed -# by the generated VX_config.vh behind an ifndef, so defining them here replaces -# those computations with the generator's report values -- which sets the memory -# port counts to zero and yields a design with no memory interface at all. -TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(CONFIGS)) +# Only the overrides and the TCU/FPU selectors are passed. Every other scalar is +# computed by the generated VX_config.vh behind an ifndef, so passing the whole +# resolved XCONFIGS here replaces those computations with the generator's report +# values -- which sets the memory port counts to zero and yields a design with no +# memory interface at all. +TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) # The same defines every other synthesis flow passes (hw/syn/xilinx, hw/syn/altera). # VX_dp_ram and VX_sp_ram gate all block-RAM inference behind SYNTHESIS, and @@ -82,6 +200,9 @@ TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(CONFIGS)) # design lands in distributed RAM. A U55C has 600960 LUTs usable as memory and # this design asks for 753744 of them without it. SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG +ifeq ($(IMAGE_KIND), sim) +SYNTH_DEFINES += -DSIMULATION +endif # This wrapper exposes a single AXI memory port, so give Vortex a single # platform memory bank to match. Left at the default of 2, Vortex builds two @@ -117,20 +238,64 @@ PLATFORM_DEFINES := -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 # checked, not just the overflow avoided. PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=4 +# On aved the simulator's clock gates are turned into clock enables by +# synthesis (the V80's reconfigurable partition has no BUFGCE, and a fabric gate +# is not safe; see FireSimFabricClockGate). That conversion cannot enter a +# vendor core: each is synthesized out of context behind a kept hierarchy, so +# it would stay on a logic-driven clock and the platform build rejects it. So +# no unit may take one here: the FPU is the STD one, as in the metasimulator, +# its units' copies in the flattened tree take their soft path (VENDOR_FP_UNITS +# below), and the RTU uses the soft FMA and divider, with the soft divider's +# latency in place of xil_fdiv's. +ifeq ($(FIRESIM_PLATFORM), aved) +PLATFORM_DEFINES += -DVX_CFG_FPU_TYPE_STD -DVX_CFG_RTU_USE_DSP=0 -DVX_CFG_RTU_FDIV_LATENCY=17 +VENDOR_FP_UNITS := VX_fma_unit.sv VX_fdiv_unit.sv VX_fsqrt_unit.sv +endif + # Vortex's performance counters are the only view inside the core that does not # require re-elaborating the design: at a stall they say whether warps are still # resident, whether they are waiting at a barrier, and which pipeline is holding # them. The AXI boundary cannot show any of that -- a core waiting on a response # it never retired looks exactly like a core with nothing left to do. They are # read over the DCR response path, which both the harness and the driver already -# drive, but VX_csr_data.sv gates them behind PERF_ENABLE. Off by default because -# they cost area; build with PERF_DEFINES=-DPERF_ENABLE to read them, and note -# that without it every counter reads zero, which is indistinguishable from an -# idle core rather than an absent instrument. -PERF_DEFINES ?= -PLATFORM_DEFINES += $(PERF_DEFINES) +# drive, but VX_csr_mpm.sv gates them behind PERF_ENABLE. Off by default because +# they cost area; build with PERF=1 to read them, as in the other synthesis +# flows, and note that without it every counter reads zero, which is +# indistinguishable from an idle core rather than an absent instrument. +ifdef PERF +PLATFORM_DEFINES += -DPERF_ENABLE +endif -GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) +# Synthesis resolves the unit types as a synthesis build: TYPE_DEFINES comes from +# a resolution without SYNTHESIS, which picks the simulation FPU (STD) over the +# FPGA one (DSP) and would hand synthesis a different design than Vortex's. +SYNTH_TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ + --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)')) +GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(SYNTH_TYPE_DEFINES) $(PLATFORM_DEFINES) $(SYNTH_DEFINES) $(RTL_INCLUDE) + +# Defines arrive on the command line, which no file timestamp reflects, so a +# changed CONFIGS would otherwise reuse the previous flattened tree. The stamp is +# rewritten only when the flags differ, which keeps an unchanged build current. +CONFIG_STAMP := $(BUILD_DIR)/gen_cflags$(IMAGE_SUFFIX).stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(GEN_CFLAGS)' > $(CONFIG_STAMP).tmp && \ + (cmp -s $(CONFIG_STAMP).tmp $(CONFIG_STAMP) && rm -f $(CONFIG_STAMP).tmp || mv -f $(CONFIG_STAMP).tmp $(CONFIG_STAMP))) + +# The metasimulator needs the opposite of both synthesis defines, so it gets its +# own flattening rather than reusing the Vivado one. +# +# -DVIVADO selects the Xilinx floating_point cores (xil_fma, xil_fdiv) over the +# soft FPU. Those are generated .xci IP with no Verilog to elaborate, so a +# metasim built from Vivado-flattened sources fails on the missing modules. +# +# -DSYNTHESIS gates VX_dp_ram/VX_sp_ram onto their BRAM-inference branch; the +# behavioural models are what a simulator wants, and they carry the read/write +# collision semantics the inference branch leaves to the tool. +SIM_DEFINES := -DNDEBUG +SIM_GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) $(SIM_DEFINES) $(RTL_INCLUDE) + +SIM_CONFIG_STAMP := $(BUILD_DIR)/sim_cflags.stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(SIM_GEN_CFLAGS)' > $(SIM_CONFIG_STAMP).tmp && \ + (cmp -s $(SIM_CONFIG_STAMP).tmp $(SIM_CONFIG_STAMP) && rm -f $(SIM_CONFIG_STAMP).tmp || mv -f $(SIM_CONFIG_STAMP).tmp $(SIM_CONFIG_STAMP))) # Only the packages and the wrapper are named; every other Vortex module is # resolved by library search over RTL_INCLUDE. Handing the whole tree over as a @@ -139,12 +304,19 @@ GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(TYPE_DEFINES) $(PLAT # Packages have to be parsed before the modules that import them, so they are # listed rather than left to library search, in dependency order. Vortex.sv # imports gpu, trace and tlb; fpu is pulled in below the core. -EXPLICIT_NAMES := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv \ - VX_firesim_wrap.sv -EXPLICIT_SRCS := $(addprefix $(VIVADO_SRCS_DIR)/,$(EXPLICIT_NAMES)) +# Extension packages follow in dependency order, which is not extensions.mk's +# append order: raster imports om. +EXT_PKG_ORDER := VX_tcu_pkg.sv VX_dxa_pkg.sv VX_rtu_pkg.sv VX_om_pkg.sv \ + VX_raster_pkg.sv VX_tex_pkg.sv +EXT_PKG_NAMES := $(filter $(notdir $(RTL_PKGS)),$(EXT_PKG_ORDER)) +PKG_ORDER := VX_gpu_pkg.sv VX_fpu_pkg.sv VX_tlb_pkg.sv VX_trace_pkg.sv $(EXT_PKG_NAMES) +EXPLICIT_NAMES := $(PKG_ORDER) VX_firesim_wrap.sv +EXPLICIT_SRCS := $(addprefix $(SRCS_DIR)/,$(EXPLICIT_NAMES)) +empty := +space := $(empty) $(empty) sources: $(SRCS_LIST) -$(SRCS_LIST): $(VIVADO_SRCS_LIST) +$(SRCS_LIST): $(SIM_SRCS_STAMP) @mkdir -p $(BUILD_DIR) @printf '%s\n' $(EXPLICIT_SRCS) > $@ @echo "wrote $@ ($(words $(EXPLICIT_SRCS)) explicit sources; rest via library search)" @@ -161,12 +333,12 @@ incdirs: # the constant-parameter failure that ruled flattening out for Verilator is an # elaboration behaviour, not a synthesis one. -# The XRT AFU is a second kernel top with its own control shell. It arrives here -# only because its directory is on the include path for the shared port macros, -# and leaving it in gives Vivado two candidate tops, which makes it elaborate -# modules standalone and reject their interface ports. The header stays; the -# modules go. -XRT_AFU_MODULES := vortex_afu.v VX_afu_ctrl.sv VX_afu_wrap.sv +# The shared AFU shell (VX_afu_wrap and its control, demux, drain and reset +# modules) is a second kernel top. It arrives here only because its directory is +# on the include path for the shared port macros in vortex_afu.vh, and leaving it +# in gives Vivado two candidate tops, which makes it elaborate modules standalone +# and reject their interface ports. The header stays; the modules go. +AFU_SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv)) # Every input the flattener reads. Without these the rule below has no # prerequisites at all, so make runs it exactly once and then reports "nothing to @@ -175,18 +347,39 @@ XRT_AFU_MODULES := vortex_afu.v VX_afu_ctrl.sv VX_afu_wrap.sv # listed too because GEN_CFLAGS resolves widths out of them. RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ - $(firstword $(MAKEFILE_LIST)) + $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) vivado-sources: $(VIVADO_SRCS_LIST) $(VIVADO_SRCS_LIST): $(RTL_SRCS) @mkdir -p $(BUILD_DIR) + @# gen_sources.sh adds to its directory without clearing it, so a source the + @# previous configuration needed (a TEX file, say) would stay and be listed. + rm -rf $(VIVADO_SRCS_DIR) cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) \ - -TVX_firesim_wrap -Cvivado_srcs -Ovivado_srcs.f - @for f in $(XRT_AFU_MODULES); do \ + -TVX_firesim_wrap -C$(VIVADO_SRCS_NAME) -O$(VIVADO_SRCS_NAME).f + @for f in $(AFU_SHELL_MODULES); do \ rm -f $(VIVADO_SRCS_DIR)/$$f; \ grep -v "/$$f$$" $(VIVADO_SRCS_LIST) > $(VIVADO_SRCS_LIST).tmp && \ mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST); \ done + @for f in $(VENDOR_FP_UNITS); do \ + [ -f $(VIVADO_SRCS_DIR)/$$f ] || continue; \ + grep -q 'localparam VENDOR_OK = 1;' $(VIVADO_SRCS_DIR)/$$f || \ + { echo "ERROR: $$f no longer selects its vendor core by VENDOR_OK"; exit 1; }; \ + sed -i 's/localparam VENDOR_OK = 1;/localparam VENDOR_OK = 0;/' $(VIVADO_SRCS_DIR)/$$f; \ + done + @# the RAM fold runs here, not in gen_sources.sh (-F), so it patches the sources as read + $(RAM_FOLD_PYTHON) $(SCRIPT_DIR)/ram_fold.py --top VX_firesim_wrap --report $(VIVADO_SRCS_DIR)/ram_fold.rpt \ + --diff $(VIVADO_SRCS_DIR)/ram_fold.diff $$(grep -E '\.s?v$$' $(VIVADO_SRCS_LIST)) + @# gen_sources.sh lists packages in find(1) order, which is directory order, + @# not dependency order. Synthesis reorders on its own; a simulator compiles + @# the list as given. Under -DSIMULATION VX_trace_pkg imports VX_gpu_pkg, and + @# with trace first xsim failed on "'VX_gpu_pkg' is not declared". Put the + @# packages in the dependency order the metasim list already uses. + @{ grep '^+' $(VIVADO_SRCS_LIST); \ + for p in $(PKG_ORDER); do grep "/$$p$$" $(VIVADO_SRCS_LIST); done; \ + grep -v '^+' $(VIVADO_SRCS_LIST) | grep -vE '/($(subst $(space),|,$(PKG_ORDER)))$$'; \ + } > $(VIVADO_SRCS_LIST).tmp && mv $(VIVADO_SRCS_LIST).tmp $(VIVADO_SRCS_LIST) @# A design with no memory ports elaborates as zero-sized arrays deep inside @# Vivado, hours from here. Defining the config scalars instead of letting @# VX_config.vh compute them is enough to cause it, so check the result. @@ -195,15 +388,174 @@ $(VIVADO_SRCS_LIST): $(RTL_SRCS) echo " pass only XLEN and the TCU/FPU selectors; VX_config.vh computes the rest"; \ exit 1; \ fi - @echo "wrote $(VIVADO_SRCS_LIST) ($$(grep -cv '^+' $(VIVADO_SRCS_LIST)) files)" + mv $(VIVADO_SRCS_LIST) $(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f + $(SCRIPT_DIR)/circt_elaborate.sh -tVX_firesim_wrap -l$(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f \ + -x$(SCRIPT_DIR)/xilinx_ip_stubs.sv -o$(BUILD_DIR)/$(VIVADO_SRCS_NAME).elab.sv + realpath $(BUILD_DIR)/$(VIVADO_SRCS_NAME).elab.sv > $(VIVADO_SRCS_LIST) + @echo "wrote $(VIVADO_SRCS_LIST): $(VIVADO_SRCS_NAME).elab.sv, elaborated from $$(grep -cv '^+' $(BUILD_DIR)/$(VIVADO_SRCS_NAME).rtl.f) files" +# The same flattening under the simulation defines. Verilator finds the modules +# by library search over this directory, so only the tree has to be complete -- +# the file list the search is seeded with is $(SRCS_LIST) above. +SIM_RTL_SRCS := $(filter-out $(CONFIG_STAMP),$(RTL_SRCS)) $(SIM_CONFIG_STAMP) + +sim-sources: $(SIM_SRCS_STAMP) +$(SIM_SRCS_STAMP): $(SIM_RTL_SRCS) + @mkdir -p $(BUILD_DIR) + @# Cleared for the same reason as the Vivado flattening above. + rm -rf $(SRCS_DIR) + cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(SIM_GEN_CFLAGS) \ + -TVX_firesim_wrap -Cvortex_srcs -Ovortex_srcs_all.f + @# The same zero-memory-port check the Vivado flattening makes: here it would + @# surface as a zero-width AXI port rather than an hours-later elaboration + @# failure, but it is the same defect and cheaper to catch at the source. + @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ + echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; \ + echo " pass only XLEN and the TCU/FPU selectors; VX_config.vh computes the rest"; \ + exit 1; \ + fi + @# The vendor FP branch is a generate-if on a localparam, not an ifdef, so the + @# instantiation survives preprocessing either way. What matters is that the + @# localparam resolves to 0 so the branch is dead -- Verilator then prunes it, + @# as the rtlsim flow has always relied on. Checking for the instantiation text + @# instead reports every build as broken. + @if grep -qE 'localparam +VENDOR_OK *= *1 *;' $(SRCS_DIR)/VX_fma_unit.sv; then \ + echo "ERROR: simulation sources select the Xilinx FP IP (VENDOR_OK=1)"; \ + echo " SIM_DEFINES must not carry -DVIVADO or -DQUARTUS"; \ + exit 1; \ + fi + touch $@ + @echo "wrote $(SRCS_DIR) ($$(ls $(SRCS_DIR) | wc -l) files)" + +# Vendor floating-point cores. On Vivado the FPU and RTU select Xilinx's +# floating_point IP (xil_fma, xil_fdiv, ...) instead of the soft core, and the +# flattened sources only instantiate them, so the kernel packaging must be handed +# the generated cores or synthesis fails on the missing modules. Same script and +# core set as the XRT flow, for the card's part (FPGA_PART, from the descriptor). +XILINX_IP_DIR := $(BUILD_DIR)/ip +XILINX_IP_STAMP := $(XILINX_IP_DIR)/.part-$(FPGA_PART) + +xilinx-ips: $(XILINX_IP_STAMP) +$(XILINX_IP_STAMP): $(SCRIPT_DIR)/xilinx_ip_gen.tcl + rm -rf $(XILINX_IP_DIR) + cd $(BUILD_DIR) && vivado -mode batch -nojournal -nolog -source $(SCRIPT_DIR)/xilinx_ip_gen.tcl \ + -tclargs $(XILINX_IP_DIR) $(FPGA_PART) + @test -n "$$(find $(XILINX_IP_DIR) -name '*.xci')" || { echo "ERROR: no .xci generated under $(XILINX_IP_DIR)"; exit 1; } + touch $@ + +# The environment the FireSim Vitis kernel build reads: the netlist and vendor +# cores package_kernel.tcl attaches. +kernel-env: + @echo 'export VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST)' + @echo 'export VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR)' + +# Simulator clock (MHz): the image is timed at it and the shell programs it. +# V80: the Fmax measured on a FireSim image of Vortex (143.4 MHz). +ifeq ($(FIRESIM_PLATFORM), aved) +FREQUENCY ?= 143.0 +else +FREQUENCY ?= 60.0 +endif +STRATEGY ?= TIMING + +# How the card's FireSim platform spells this TARGET, and where it leaves the +# image. aved builds both images from one goal and selects with its own TARGET, +# putting each in build_dir.$(TARGET); vitis has a separate `sim` goal for +# hw_emu, and must never see TARGET at all -- its Makefile sets +# `bitstream: TARGET = hw` as a target-specific variable, which a passed-in +# TARGET would beat, sending `--target sim` to v++. +PLATFORM_BUILD_DIR := $(FIRESIM_PATH)/platforms/$(FIRESIM_PLATFORM)/cl_$(QUINTUPLET) +ifeq ($(FIRESIM_PLATFORM), aved) +PLATFORM_IMAGE_GOAL := bitstream TARGET=$(TARGET) +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/bitstream/build_dir.$(TARGET)/firesim.vbin +IMAGE_NAME := firesim.vbin +else ifeq ($(TARGET), hw_emu) +# Without debug: a debug hw_emu kernel logs a waveform of the whole shell on +# every run (~800 MB for demo, tens of GB for raycast) and runs far slower. +PLATFORM_IMAGE_GOAL := sim VPP_DEBUG= +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/simulation/build_dir.$(DEVICE)/firesim.xclbin +IMAGE_NAME := firesim.xclbin +else +PLATFORM_IMAGE_GOAL := bitstream +PLATFORM_IMAGE := $(PLATFORM_BUILD_DIR)/bitstream/build_dir.$(DEVICE)/firesim.xclbin +IMAGE_NAME := firesim.xclbin +endif + +# One command from sources to the card's image, collected into $(BIN_DIR) with +# what running it needs beside it. Expects the FireSim environment +# (ci/firesim_env.sh) and the Xilinx tools for the card's platform. +all: image + @mkdir -p $(BIN_DIR) + cp -f $(PLATFORM_IMAGE) $(BIN_DIR)/$(IMAGE_NAME) + @echo $(FIRESIM_CARD) > $(BIN_DIR)/firesim.card +ifeq ($(TARGET), hw_emu) + emconfigutil --platform $(DEVICE) --od $(BIN_DIR) +endif + @echo "image: $(abspath $(BIN_DIR))/$(IMAGE_NAME) (card $(FIRESIM_CARD), TARGET=$(TARGET))" + +# The FireSim platform's own build. $(SRCS_LIST) is deliberately not a +# prerequisite: the recipe hands synthesis the Vivado-flattened list, and pulling +# the simulation one in would flatten the tree a second time under the opposite +# defines for nothing. +image: vivado-sources xilinx-ips stage + $(MAKE) -C $(FIRESIM_PATH)/sim replace-rtl $(FIRESIM_TARGET_ARGS) + VORTEX_VIVADO_SRCS=$(VIVADO_SRCS_LIST) VORTEX_XILINX_IP_DIR=$(XILINX_IP_DIR) \ + $(MAKE) -C $(PLATFORM_BUILD_DIR) $(PLATFORM_IMAGE_GOAL) \ + DEVICE=$(DEVICE) HOST_MEM_SP='$(HOST_MEM_SP)' FREQUENCY=$(FREQUENCY) STRATEGY=$(STRATEGY) + +# Golden Gate's elaboration of the card's target on its own, for the runtime's +# metasimulator (xrtsim/avedsim), which needs the generated RTL and headers but +# no image. An image build elaborates as part of replace-rtl. +elaborate: stage + $(MAKE) -C $(FIRESIM_PATH)/sim verilog $(FIRESIM_TARGET_ARGS) + +clean: + rm -rf $(BUILD_DIR) + +# The metasimulator: the FireSim-transformed design in Verilator, driven through +# the DPI emul simif. No FPGA and no Vivado, so it runs the same target the card +# runs without needing either -- which is what makes it the step before silicon. +# +# Vortex is supplied here rather than inlined into the generated wrapper (see +# VortexTarget.scala): the packages and the wrapper are named and everything else +# resolves by library search over the flattened tree. Naming the whole tree +# instead makes Verilator stop folding the macro-computed widths into constants. +# +# The list is collapsed onto one line because it reaches Verilator through a +# shell, which would otherwise split it on the newlines. +# hw/syn/common.mk exports VERILATOR as a bare path to Vortex's own toolchain, +# and FireSim declares `VERILATOR ?= verilator --main --timing --cc --exe`. A +# `?=` yields to the environment, so the inherited value silently strips every +# flag and Verilator exits asking for --cc. Drop it from the environment instead +# of restating FireSim's flags here, which would rot against its Makefrag. The +# metasim then builds with the Verilator on PATH, which FireSim's env.sh pins. +metasim: sources stage + env -u VERILATOR $(MAKE) -C $(FIRESIM_PATH)/sim verilator $(FIRESIM_TARGET_ARGS) \ + EXTRA_VERILATOR_FLAGS="-Wno-fatal -y $(SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SRCS_LIST))" + +# Runs it. ARGS reaches the harness; the metasim needs no image, since the design +# is the simulator. +# +# Not dependent on `metasim`, so running never rebuilds the 460 MB model behind +# the caller's back. Build with `make metasim` first. +run-metasim: + env -u VERILATOR $(MAKE) -C $(FIRESIM_PATH)/sim run-verilator $(FIRESIM_TARGET_ARGS) ARGS="$(ARGS)" + +# The checkout must contain the pinned fork commit. Being ahead of it is normal +# while developing the fork; being behind it, or on unrelated history, builds +# against FireSim changes this tree does not expect. check-firesim: @test -d $(FIRESIM_PATH)/sim || { \ echo "FireSim checkout not found at $(FIRESIM_PATH)"; \ - echo "set FIRESIM_PATH to a FireSim 1.21.0 checkout"; \ + echo "clone vortexgpgpu/firesim there (see docs/building_toolchain.md)"; \ + exit 1; } + @git -C $(FIRESIM_PATH) merge-base --is-ancestor $(FIRESIM_REV) HEAD 2>/dev/null || { \ + echo "FireSim checkout at $(FIRESIM_PATH) does not contain FIRESIM_REV=$(FIRESIM_REV)"; \ + echo " HEAD is $$(git -C $(FIRESIM_PATH) rev-parse --short HEAD)"; \ + echo " fetch the fork and check out $(FIRESIM_REV) or a descendant"; \ exit 1; } clean-staged: - @for f in $(SCALA_SRCS); do \ + @for f in $(SCALA_SRCS) $(PLATFORM_CONFIG).scala; do \ rm -f $(TARGET_DEST)/$$(basename "$$f"); \ done diff --git a/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch b/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch deleted file mode 100644 index 6527f07302..0000000000 --- a/hw/syn/firesim/patches/0001-rebase-target-cycle-counter.patch +++ /dev/null @@ -1,73 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h.orig 2026-08-02 05:13:08.369209268 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h 2026-08-02 05:16:23.260800446 -0700 -@@ -145,7 +145,18 @@ - midas_time_t end_time; - std::optional start_hcycle; - std::optional end_hcycle; -+ uint64_t start_tcycle = 0; - uint64_t end_tcycle = 0; -+ -+ /// Target cycles consumed by *this* run. -+ /// The counter lives in fabric and persists while the bitstream stays -+ /// programmed, so it must be rebased against the start sample -- host -+ /// cycles are already handled this way. -+ /// Saturates rather than wrapping: a device reprogrammed mid-run can leave -+ /// end < start, and an unsigned wrap would report a nonsense duration. -+ uint64_t elapsed_tcycles() const { -+ return (end_tcycle >= start_tcycle) ? (end_tcycle - start_tcycle) : 0; -+ } - }; - - #endif // __SIMULATION_H ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc.orig 2026-08-02 05:13:08.373209261 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc 2026-08-02 05:16:23.260800446 -0700 -@@ -41,6 +41,7 @@ - - void simulation_t::record_start_times() { - start_hcycle = clock.hcycle(); -+ start_tcycle = clock.tcycle(); - start_time = timestamp(); - } - -@@ -58,10 +59,13 @@ - - const uint64_t hcycles = *end_hcycle - *start_hcycle; - const double sim_time = diff_secs(end_time, start_time); -- const double sim_speed = ((double)end_tcycle) / (sim_time * 1000.0); -+ const uint64_t tcycles = elapsed_tcycles(); -+ const double sim_speed = ((double)tcycles) / (sim_time * 1000.0); - const double measured_host_frequency = - ((double)hcycles) / (sim_time * 1000.0); -- const double fmr = ((double)hcycles / end_tcycle); -+ // A run that steps no target cycles is legal (e.g. fingerprint-only); -+ // reporting inf/nan for it would be worse than reporting zero. -+ const double fmr = tcycles ? ((double)hcycles / tcycles) : 0.0; - - fprintf(stderr, "\nEmulation Performance Summary\n"); - fprintf(stderr, "------------------------------\n"); -@@ -74,7 +78,7 @@ - fprintf(stderr, "%.3f KHz\n", measured_host_frequency); - } - -- fprintf(stderr, "Target Cycles Emulated: %" PRIu64 "\n", end_tcycle); -+ fprintf(stderr, "Target Cycles Emulated: %" PRIu64 "\n", tcycles); - fprintf(stderr, "Effective Target Frequency: "); - if (sim_speed > 1000.0) { - fprintf(stderr, "%.3f MHz\n", sim_speed / 1000.0); -@@ -154,13 +158,13 @@ - fprintf(stderr, - "*** FAILED *** (code = %d) after %" PRIu64 " cycles\n", - exit_code, -- end_tcycle); -+ elapsed_tcycles()); - } else if (timeout) { - fprintf(stderr, - "*** FAILED *** simulation timed out after %" PRIu64 " cycles\n", -- end_tcycle); -+ elapsed_tcycles()); - } else { -- fprintf(stderr, "*** PASSED *** after %" PRIu64 " cycles\n", end_tcycle); -+ fprintf(stderr, "*** PASSED *** after %" PRIu64 " cycles\n", elapsed_tcycles()); - } - - print_simulation_performance_summary(); diff --git a/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch b/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch deleted file mode 100644 index 524e715dd3..0000000000 --- a/hw/syn/firesim/patches/0002-vitis-u55c-build-fixes.patch +++ /dev/null @@ -1,68 +0,0 @@ -Vitis flow fixes required to build a FireSim xclbin for the Alveo U55C -under Vitis 2024.2. Upstream FireSim targets Vitis 2022.1 and a U250. - - 1. host_mem_0 connectivity was hardcoded to DDR[0]; the U55C is HBM-only - (platforminfo reports "Bus SP Tag: HBM" and no DDR entry). Parameterized - as HOST_MEM_SP so non-HBM platforms can override it. - 2. package_kernel.tcl hardcoded the U250 part, so kernel IP was generated - for the wrong device regardless of DEVICE=. - 3. package_xo silently produced no .xo: create_project chdirs into the temp - project and the closing "close_project -delete" removes it, leaving the - process CWD on a deleted path so package_xo dies inside - boost::filesystem::current_path(). Save and restore the CWD. - 4. v++ --temp_dir pointed at the same directory the .xo is written to, and - Vitis 2024.2 clears its temp dir at startup -- deleting the .xo before - the link step reads it. - ---- build_firesim/firesim/platforms/vitis/cl_firesim/Makefile.orig 2026-08-02 05:18:37.916499449 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/Makefile 2026-08-02 05:18:37.948499376 -0700 -@@ -47,6 +47,11 @@ - # current support on firesim1 - DEVICE ?= xilinx_u250_gen3x16_xdma_3_1_202020_1 - -+# Memory resource backing the simulator's host_mem_0 port. Upstream hardcodes -+# DDR[0]; HBM-only cards (Alveo U55C) expose no DDR bank at all, so this must -+# be overridable per platform. -+HOST_MEM_SP ?= HBM[0] -+ - # Provide's the desired simulator frequency. This is generated from the clock - # provided to the kernel using an internal MMCM. - FREQUENCY ?= 90 -@@ -168,7 +173,7 @@ - mkdir -p $(@D) - # Move into temp directory to avoid polluting top-level workdir - cd $(@D)/..; $(VPP) $(vpp_config_overlay_args) $(VPP_TARGET_SPECIFIC_ARGS) $(VPP_COMMON_ARGS) \ -- --connectivity.sp $(PROJECT_NAME)_1.host_mem_0:DDR[0] \ -+ --connectivity.sp $(PROJECT_NAME)_1.host_mem_0:$(HOST_MEM_SP) \ - --link $(VPP_LDFLAGS) -o $@ $< - - ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl.orig 2026-08-02 00:47:00.546532830 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl 2026-08-02 00:47:00.718532277 -0700 -@@ -19,7 +19,9 @@ - set path_to_packaged "./packaged_kernel_${suffix}" - set path_to_tmp_project "./tmp_kernel_pack_${suffix}" - --set projPart "xcu250-figd2104-2L-e" -+# Upstream hardcodes the U250 part; the kernel-packaging project must use -+# the part of the target platform or its IP is generated for the wrong device. -+set projPart "xcu55c-fsvh2892-2L-e" - - set base_filename "FireSim-generated" - set synth_xdc_path "${path_to_hdl}/${base_filename}.synthesis.xdc" ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/gen_xo.tcl.orig 2026-08-02 00:51:18.973709853 -0700 -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/gen_xo.tcl 2026-08-02 00:51:19.061709575 -0700 -@@ -33,7 +33,13 @@ - set script_path [file dirname [file normalize [info script]]] - - #source -notrace ./scripts/package_kernel.tcl -+# Vivado's create_project chdirs into the project dir, and package_kernel.tcl's -+# closing "close_project -delete" then removes it -- leaving the process CWD -+# pointing at a deleted directory. package_xo subsequently dies inside -+# boost::filesystem::current_path(). Save the CWD and restore it below. -+set origin_cwd [pwd] - source ${script_path}/package_kernel.tcl -+cd $origin_cwd - - if {[file exists "${xoname}"]} { - file delete -force "${xoname}" diff --git a/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch b/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch deleted file mode 100644 index 73a23e69bf..0000000000 --- a/hw/syn/firesim/patches/0003-vitis-memcheck-and-buffer-lifetime.patch +++ /dev/null @@ -1,156 +0,0 @@ -Diagnostic and correctness fixes for FireSim on the Alveo U55C. -Applies on top of 0001-rebase-target-cycle-counter.patch. - - 1. simif_vitis.cc: hold the host_mem_0 xrt::bo as a member. Upstream - allocates it as a constructor local, so the device allocation is freed - when the constructor returns and XRT may reassign that memory for the - entire life of the simulation. - - 2. simif_vitis.cc: drop the hardcoded U250 DDR base assertion. The kernel's - host_mem_0_offset register is IP-XACT metadata only -- it is absent from - the generated RTL, and writing it lands in FireSim's own widget MMIO - space. Address mapping is done by the platform memory subsystem via - --connectivity.sp. - - 3. simulation.{h,cc}: add +memcheck, which round-trips a known pattern - through the LoadMem widget before target execution, testing the target - DRAM path independently of the design under test. This is what identified - the U55C HBM memory path as non-functional while the control plane works - (see docs/proposals/firesim_vitis_host_mem_offset.md). - ---- build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc.orig 2026-08-02 03:11:40.536120157 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc 2026-08-02 06:01:44.813949894 -0700 -@@ -42,16 +42,16 @@ - xrt::uuid uuid; - xrt::ip kernel_handle; - xrt::run run_handle; -+ /// The device-side memory backing host_mem_0. Must outlive the -+ /// simulation: as a constructor local its allocation is freed on -+ /// return, leaving the window pointing at memory XRT may reuse. -+ xrt::bo fpga_mem_0; - }; - --constexpr size_t u250_dram_channel_size_bytes = 16ULL * 1024 * 1024 * 1024; --/** -- * We currently support only a single FPGA DRAM channel. When the buffer is -- * allocated in XRT, we'd generally have to write the offset back to the kernel -- * using MMIO. For now we've hardcoded the offset for a U250 with no other -- * concurrently running kernel. -- */ --constexpr uint64_t u250_dram_expected_offset = 0x4000000000L; -+/// Size of the single device-memory window backing host_mem_0. -+/// One U55C HBM pseudo-channel is 512 MB; a U250 DDR bank is larger, but a -+/// single channel is all this simif supports either way. -+constexpr size_t host_mem_0_size_bytes = 512ULL * 1024 * 1024; - - simif_vitis_t::simif_vitis_t(const TargetConfig &config, - const std::vector &args) -@@ -92,22 +92,21 @@ - kernel_handle = xrt::ip(device_handle, uuid, "firesim"); - - // Intialize FPGA-DRAM regions. -- // The final argument here is the bank index for the dram channel. -- // I used xclbinutil to find this -- // https://xilinx.github.io/XRT/master/html/xclbintools.html -- auto fpga_mem_0 = xrt::bo(device_handle, -- u250_dram_channel_size_bytes, -- xrt::bo::flags::device_only, -- 0); -- -- if (fpga_mem_0.address() != u250_dram_expected_offset) { -- std::cerr -- << "Allocated device_only buffer address not match the expected offset." -- << std::endl; -- std::cerr << "Expected: " << u250_dram_expected_offset -- << "Received: " << fpga_mem_0.address() << std::endl; -- exit(1); -- } -+ // The final argument is the bank index for the DRAM channel, found with -+ // xclbinutil: https://xilinx.github.io/XRT/master/html/xclbintools.html -+ fpga_mem_0 = xrt::bo(device_handle, -+ host_mem_0_size_bytes, -+ xrt::bo::flags::device_only, -+ 0); -+ -+ // NOTE: the kernel's host_mem_0_offset register (package_kernel.tcl, s_axi_lite -+ // 0x010) is IP-XACT metadata only -- it does not appear in the generated RTL, -+ // and writing it would land in FireSim's own widget MMIO space. Address -+ // mapping into HBM is done by the platform memory subsystem via -+ // --connectivity.sp, so nothing needs programming here. -+ std::cerr << "[vitis] host_mem_0 base = 0x" << std::hex -+ << fpga_mem_0.address() << " size = 0x" << host_mem_0_size_bytes -+ << std::dec << std::endl; - } - - void simif_vitis_t::write(size_t addr, uint32_t data) { ---- build_firesim/patchbase/simulation.h 2026-08-02 06:12:46.916218880 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h 2026-08-02 06:06:05.805273470 -0700 -@@ -130,6 +130,10 @@ - */ - bool do_zero_out_dram = false; - -+ /// Diagnostic: round-trip a pattern through the LoadMem widget after -+ /// loading, to prove the target DRAM path independently of the DUT. -+ bool do_memcheck = false; -+ - /** - * If set, read the presence register, check it, and exit the simulation - * cleanly ---- build_firesim/patchbase/simulation.cc 2026-08-02 06:12:46.916218880 -0700 -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc 2026-08-02 06:06:05.805273470 -0700 -@@ -24,6 +24,9 @@ - if (arg.find("+loadmem=") == 0) { - load_mem_path = arg.c_str() + 9; - } -+ if (arg.find("+memcheck") == 0) { -+ do_memcheck = true; -+ } - if (arg.find("+zero-out-dram") == 0) { - do_zero_out_dram = true; - } -@@ -189,6 +192,45 @@ - if (!load_mem_path.empty()) { - loadmem->load_mem_from_file(load_mem_path); - } -+ -+ // Diagnostic: write a known pattern through the LoadMem widget and read it -+ // back through the same widget. This isolates "the target DRAM path works" -+ // from any question about the design under test -- if this round-trip -+ // fails, no memory-touching target can possibly pass. -+ if (do_memcheck) { -+ const size_t chunk_bits = loadmem->get_mem_data_chunk() * 8; -+ mpz_t wr, rd; -+ mpz_init(wr); -+ mpz_init(rd); -+ unsigned failures = 0; -+ const size_t probes[] = {0x0, 0x40, 0x1000, 0x40000, 0xF0000}; -+ for (size_t addr : probes) { -+ // A value that is distinct per address and not all-ones/all-zeros. -+ mpz_set_ui(wr, 0); -+ mpz_setbit(wr, 0); -+ mpz_mul_2exp(wr, wr, chunk_bits / 2); -+ mpz_add_ui(wr, wr, (unsigned long)(0xA5A50000u | (addr & 0xFFFF))); -+ loadmem->write_mem(addr, wr); -+ loadmem->read_mem(addr, rd); -+ const bool ok = (mpz_cmp(wr, rd) == 0); -+ gmp_fprintf(stderr, -+ "[memcheck] addr=0x%zx wrote=0x%Zx read=0x%Zx %s\n", -+ addr, -+ wr, -+ rd, -+ ok ? "OK" : "MISMATCH"); -+ if (!ok) { -+ failures++; -+ } -+ } -+ mpz_clear(wr); -+ mpz_clear(rd); -+ fprintf(stderr, -+ "[memcheck] %s (%u/%zu probes failed)\n", -+ failures ? "TARGET DRAM PATH IS BROKEN" : "target DRAM path OK", -+ failures, -+ sizeof(probes) / sizeof(probes[0])); -+ } - } else { - if (do_zero_out_dram || !load_mem_path.empty()) { - fprintf(stderr, diff --git a/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch b/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch deleted file mode 100644 index 273dbe940e..0000000000 --- a/hw/syn/firesim/patches/0004-vitis-host-mem-base-address.patch +++ /dev/null @@ -1,66 +0,0 @@ -Fix the FireSim Vitis shim hardcoding the U250 DDR base address in RTL. - -VitisShim added a literal 0x4000000000 -- the base of DDR[0] on a U250 -- to -every host_mem_0 address, in the generated RTL rather than in any register. -On an HBM-only part such as the Alveo U55C, host_mem_0 binds to HBM[0] at -0x0 with a 512 MB (29-bit) aperture, so every access landed far outside the -bound pseudo-channel: reads returned a fixed bus default independent of -address and writes were dropped, leaving the target DRAM path silently dead -while the MMIO/control plane still worked. - -Replaced with a documented constant in VitisConstants so the base tracks -whatever --connectivity.sp binds host_mem_0 to. A proper upstream fix makes -this a Config field keyed off the platform; it is a constant here because -this tree targets one card, and the code says so. - -Validated under hw_emu, which exercises the same block design without -place-and-route: - - pre-fix (control): +memcheck 5/5 probes fail, address-independent constant - with fix : +memcheck 0/5 probes fail, every probe reads back what - was written - -The regenerated RTL was independently confirmed to contain zero occurrences -of the old constant, down from two. - ---- build_firesim/firesim/sim/midas/src/main/scala/midas/platform/VitisShim.scala.orig 2026-08-02 07:11:46.303359225 -0700 -+++ build_firesim/firesim/sim/midas/src/main/scala/midas/platform/VitisShim.scala 2026-08-02 07:11:46.331359159 -0700 -@@ -21,6 +21,25 @@ - - /** The hardcoded TCL variable name used to specify the simulator's frequency */ - val frequencyVariableName = "frequency" -+ -+ /** Base address added to every host_mem_0 access. -+ * -+ * The shim drives absolute device addresses, so this must match the base of -+ * whatever memory resource `--connectivity.sp` binds host_mem_0 to. -+ * -+ * U250 / U280, DDR[0] -> 0x4000000000 -+ * U55C, HBM[0] (HBM-only) -> 0x0, with a 512 MB (29-bit) aperture -+ * -+ * Upstream hardcoded the U250 DDR base inline. On an HBM part every access -+ * then lands far outside the bound pseudo-channel: reads return a fixed bus -+ * default independent of address and writes are dropped, so the target DRAM -+ * path is silently dead while the control plane still works. -+ * -+ * A proper upstream fix makes this a Config field keyed off the platform -+ * rather than a constant; it is a constant here because this tree targets -+ * one card. -+ */ -+ val hostMemBaseAddress = 0x0L - } - - class VitisShim(implicit p: Parameters) extends PlatformShim { -@@ -89,8 +108,10 @@ - host_mem_cdc.io.s_axi.drivenByStandardAXI4(top.module.mem(0), hostClock, hostSyncReset) - host_mem_cdc.io.s_axi_aclk := hostClock - host_mem_cdc.io.s_axi_aresetn := (!hostSyncReset).asAsyncReset -- host_mem_cdc.io.s_axi.araddr := 0x4000000000L.U(addrWidth) + top.module.mem(0).ar.bits.addr -- host_mem_cdc.io.s_axi.awaddr := 0x4000000000L.U(addrWidth) + top.module.mem(0).aw.bits.addr -+ host_mem_cdc.io.s_axi.araddr := -+ VitisConstants.hostMemBaseAddress.U(addrWidth) + top.module.mem(0).ar.bits.addr -+ host_mem_cdc.io.s_axi.awaddr := -+ VitisConstants.hostMemBaseAddress.U(addrWidth) + top.module.mem(0).aw.bits.addr - host_mem_cdc.io.s_axi.arcache.foreach { _ := AXI4Parameters.CACHE_MODIFIABLE } - host_mem_cdc.io.s_axi.awcache.foreach { _ := AXI4Parameters.CACHE_MODIFIABLE } - diff --git a/hw/syn/firesim/patches/0005-inflight-target-cycle.patch b/hw/syn/firesim/patches/0005-inflight-target-cycle.patch deleted file mode 100644 index 6b8f8b34f1..0000000000 --- a/hw/syn/firesim/patches/0005-inflight-target-cycle.patch +++ /dev/null @@ -1,54 +0,0 @@ -Correct the target-cycle count across the counter's origin. - -The clock bridge counts target cycles *started*, not completed: under FAME-1 -the target accepts the clock token for cycle N+1 and only then stalls on the -channels it needs to finish it, so exactly one cycle is in flight whenever the -target has started at least one. (The clock-token channel is wired to the -target combinationally -- SimWrapper.genClockChannel -- so a token that has -fired has been accepted, not buffered.) - -That in-flight cycle is present at both samples taken by -0001-rebase-target-cycle-counter.patch and cancels -- except across the -counter's origin, where the start sample sees a target that has yet to start -its first cycle. The result was a count that depended on whether the bitstream -had been run before: 14 cycles on the first run after programming and 13 on -every run after it, for a test that provably steps the target exactly 13 times -(TestAdder4: target_reset(5) plus 8 single steps). A fresh metasim is always -the origin case, so upstream reports the +1 on every metasim run. - -Corrects for the in-flight cycle explicitly: cycles completed at a sample are -"started - in flight", and in flight is 0 exactly when the counter reads 0 and -1 otherwise. Validated on the vitis metasim: Adder4 14 -> 13, matching the -counted steps and the card's steady state; PointerChaser 1625 -> 1624, result -still bit-exact at 0x498. Host-side only, no bitstream rebuild needed. - -This deliberately diverges from upstream, which reports the +1. - ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.h -@@ -152,14 +152,23 @@ - uint64_t start_tcycle = 0; - uint64_t end_tcycle = 0; - -- /// Target cycles consumed by *this* run. -+ /// Target cycles completed by *this* run. -+ /// - /// The counter lives in fabric and persists while the bitstream stays - /// programmed, so it must be rebased against the start sample -- host - /// cycles are already handled this way. -+ /// -+ /// It counts cycles *started*, so one cycle is in flight at any sample taken -+ /// after the target's first. That cancels between the two samples, except at -+ /// the counter's origin, where the start sample sees none. -+ /// - /// Saturates rather than wrapping: a device reprogrammed mid-run can leave - /// end < start, and an unsigned wrap would report a nonsense duration. - uint64_t elapsed_tcycles() const { -- return (end_tcycle >= start_tcycle) ? (end_tcycle - start_tcycle) : 0; -+ if (end_tcycle <= start_tcycle) { -+ return 0; -+ } -+ return end_tcycle - start_tcycle - (start_tcycle == 0 ? 1 : 0); - } - }; - diff --git a/hw/syn/firesim/patches/0006-package-design-sources.patch b/hw/syn/firesim/patches/0006-package-design-sources.patch deleted file mode 100644 index 060d7cc9b6..0000000000 --- a/hw/syn/firesim/patches/0006-package-design-sources.patch +++ /dev/null @@ -1,112 +0,0 @@ -Carry design sources that simulation resolves by library search into the kernel. - -Vortex is handed to Verilator as a handful of packages plus a wrapper, with the -remaining modules resolved by library search over the include paths. Vivado has -no equivalent, so without this it synthesizes a design whose submodules are all -missing. - -The sources cannot simply be added to the packaging project. ipx::package_project -infers bus interfaces from everything it imports, and a design carrying -SystemVerilog interfaces has no IP-XACT representation, so packaging fails on -ports it cannot express. They are instead attached to the core after packaging, -where the inference pass never sees them. - -Attaching alone is not enough either. A file referenced outside the component -root does not travel with the IP, leaving the linker to find the instantiation -but not the module, so each file is copied into the component and registered -relatively. It is registered in both the synthesis and behavioural-simulation -file groups: the first builds bitstreams, the second builds hardware emulation, -and a file present in only one is invisible to the other flow. - -The kernel top is named explicitly as well. Vivado infers it from the source set, -and a design large enough to need this hook is large enough for that inference to -settle on one of its internal modules, which then fails to package. - -Absent VORTEX_VIVADO_SRCS the flow is unchanged. - ---- build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl.orig -+++ build_firesim/firesim/platforms/vitis/cl_firesim/scripts/package_kernel.tcl -@@ -46,6 +46,45 @@ - $path_to_hdl/FireSim-generated.post-processed.sv \ - ] - -+# The design under test may resolve most of its RTL by library search during -+# simulation, which Vivado has no equivalent for. When a flattened source list is -+# supplied, every file in it is added explicitly and its search paths recorded. -+if {[info exists ::env(VORTEX_VIVADO_SRCS)]} { -+ set vortex_list $::env(VORTEX_VIVADO_SRCS) -+ if {![file exists $vortex_list]} { -+ puts "ERROR: VORTEX_VIVADO_SRCS points at a missing file: $vortex_list" -+ exit 1 -+ } -+ set fh [open $vortex_list r] -+ set vortex_files {} -+ set vortex_incdirs {} -+ while {[gets $fh line] >= 0} { -+ set line [string trim $line] -+ if {$line eq ""} { continue } -+ if {[string match "+incdir+*" $line]} { -+ lappend vortex_incdirs [string range $line 8 end] -+ } elseif {[string match "+*" $line]} { -+ continue -+ } else { -+ lappend vortex_files $line -+ } -+ } -+ close $fh -+ if {[llength $vortex_files] == 0} { -+ puts "ERROR: no sources listed in $vortex_list" -+ exit 1 -+ } -+ puts "collected [llength $vortex_files] design sources from $vortex_list" -+ # Deliberately not added to this project. ipx::package_project infers bus -+ # interfaces from every file it imports, and a design carrying SystemVerilog -+ # interfaces cannot be expressed in IP-XACT -- packaging fails on ports it -+ # has no way to represent. The files are attached to the packaged core -+ # further down instead, so they ship with the IP without being parsed here. -+ if {[llength $vortex_incdirs] > 0} { -+ set_property include_dirs $vortex_incdirs [get_filesets sources_1] -+ } -+} -+ - set_property USED_IN_SYNTHESIS false [get_files $impl_xdc_path] - set_property USED_IN_IMPLEMENTATION true [get_files $impl_xdc_path] - -@@ -102,6 +141,37 @@ - set_property sdx_kernel_type rtl $core - set_property supported_families { } $core - set_property auto_family_support_level level_2 $core -+# Deliver the design sources with the packaged core. They are added to the -+# synthesis file group directly rather than through the project, so synthesis -+# sees them while the packager's interface inference never does. -+if {[info exists vortex_files] && [llength $vortex_files] > 0} { -+ # Both groups: synthesis builds the bitstream, behavioural simulation builds -+ # hardware emulation, and a file registered in only one is invisible to the -+ # other flow. -+ set target_fgs [list \ -+ [ipx::get_file_groups xilinx_anylanguagesynthesis -of_objects $core] \ -+ [ipx::get_file_groups xilinx_anylanguagebehavioralsimulation -of_objects $core]] -+ # Copied into the core rather than referenced where they lie: a path outside -+ # the component root does not survive IP delivery, so the linker would find -+ # the instantiation but not the module. -+ set vortex_dst $path_to_packaged/src -+ file mkdir $vortex_dst -+ foreach vf $vortex_files { -+ set base [file tail $vf] -+ file copy -force $vf $vortex_dst/$base -+ foreach fg $target_fgs { -+ set added [ipx::add_file src/$base $fg] -+ if {[string match "*.vh" $base] || [string match "*.svh" $base]} { -+ set_property type verilogSource $added -+ set_property is_include true $added -+ } elseif {[string match "*.sv" $base]} { -+ set_property type systemVerilogSource $added -+ } -+ } -+ } -+ puts "attached [llength $vortex_files] design sources to the packaged core" -+} -+ - ipx::create_xgui_files $core - ipx::update_checksums $core - ipx::check_integrity -kernel $core diff --git a/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch b/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch deleted file mode 100644 index 4739bfa6a5..0000000000 --- a/hw/syn/firesim/patches/0007-vitis-target-dram-aperture.patch +++ /dev/null @@ -1,157 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/simif_vitis.cc -@@ -1,4 +1,8 @@ -+#include - #include -+#include -+#include -+#include - - #include - #include -@@ -12,6 +16,7 @@ - #include "experimental/xrt_device.h" - #include "experimental/xrt_ip.h" - #include "experimental/xrt_kernel.h" -+#include "experimental/xrt_xclbin.h" - - class simif_vitis_t final : public simif_t, public FPGAManagedStreamIO { - public: -@@ -48,10 +53,61 @@ - xrt::bo fpga_mem_0; - }; - --/// Size of the single device-memory window backing host_mem_0. --/// One U55C HBM pseudo-channel is 512 MB; a U250 DDR bank is larger, but a --/// single channel is all this simif supports either way. --constexpr size_t host_mem_0_size_bytes = 512ULL * 1024 * 1024; -+/// Size of the device-memory window backing host_mem_0. -+/// -+/// This must cover the target's whole address map, not merely the part of it a -+/// given test happens to touch. A 32-bit target places its stack at the top of -+/// 4 GiB and its program image well above any buffer it allocates, so anything -+/// smaller leaves those accesses unserved: writes are dropped and reads return -+/// an address-independent value, with no error anywhere to say so. -+/// -+/// One U55C HBM pseudo-channel is 512 MB, so 4 GiB spans eight of them and -+/// `--connectivity.sp` must bind host_mem_0 to a matching range (HOST_MEM_SP in -+/// the Vitis kernel Makefile). The two are a pair: widening one alone silently -+/// truncates. -+constexpr size_t host_mem_0_size_bytes = 4ULL * 1024 * 1024 * 1024; -+ -+/// Allocates the device-memory window backing host_mem_0, choosing a memory -+/// region that can hold all of it. Tries the largest usable regions first so a -+/// range group is preferred over any single pseudo-channel inside it. -+static xrt::bo allocate_host_mem(const xrt::device &device, -+ const std::string &binary_file, -+ size_t bytes) { -+ xrt::xclbin xclbin(binary_file); -+ const auto mems = xclbin.get_mems(); -+ -+ std::vector candidates; -+ for (size_t i = 0; i < mems.size(); ++i) { -+ if (mems[i].get_used()) { -+ candidates.push_back(i); -+ } -+ } -+ std::stable_sort(candidates.begin(), candidates.end(), -+ [&mems](size_t a, size_t b) { -+ return mems[a].get_size_kb() > mems[b].get_size_kb(); -+ }); -+ -+ for (size_t idx : candidates) { -+ try { -+ xrt::bo bo(device, bytes, xrt::bo::flags::device_only, idx); -+ std::cerr << "[vitis] host_mem_0 backed by memory index " << idx << " (" -+ << mems[idx].get_tag() << ")" << std::endl; -+ return bo; -+ } catch (const std::exception &) { -+ // Too small, or not allocatable: try the next region. -+ } -+ } -+ -+ std::cerr << "[vitis] no memory region can back a 0x" << std::hex << bytes -+ << std::dec << "-byte host_mem_0 window. Topology:" << std::endl; -+ for (size_t i = 0; i < mems.size(); ++i) { -+ std::cerr << "[vitis] [" << i << "] " << mems[i].get_tag() << " " -+ << mems[i].get_size_kb() << " KB" -+ << (mems[i].get_used() ? " (used)" : "") << std::endl; -+ } -+ throw std::runtime_error("host_mem_0 window does not fit any memory region; " -+ "widen --connectivity.sp (HOST_MEM_SP)"); -+} - - simif_vitis_t::simif_vitis_t(const TargetConfig &config, - const std::vector &args) -@@ -92,12 +148,15 @@ - kernel_handle = xrt::ip(device_handle, uuid, "firesim"); - - // Intialize FPGA-DRAM regions. -- // The final argument is the bank index for the DRAM channel, found with -- // xclbinutil: https://xilinx.github.io/XRT/master/html/xclbintools.html -- fpga_mem_0 = xrt::bo(device_handle, -- host_mem_0_size_bytes, -- xrt::bo::flags::device_only, -- 0); -+ // -+ // The last argument is a memory index into the xclbin's topology. Binding -+ // host_mem_0 to a range of pseudo-channels makes the region a memory *group* -+ // whose index is not knowable from the port name, and xrt::ip -- unlike -+ // xrt::kernel -- exposes no group_id() to ask for it. So try the indices the -+ // xclbin reports until one can back the whole window, and fail loudly with -+ // the topology printed if none can. Quietly settling for a smaller region is -+ // the exact failure this window size exists to prevent. -+ fpga_mem_0 = allocate_host_mem(device_handle, binary_file, host_mem_0_size_bytes); - - // NOTE: the kernel's host_mem_0_offset register (package_kernel.tcl, s_axi_lite - // 0x010) is IP-XACT metadata only -- it does not appear in the generated RTL, ---- build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/core/simulation.cc -@@ -203,7 +203,20 @@ - mpz_init(wr); - mpz_init(rd); - unsigned failures = 0; -- const size_t probes[] = {0x0, 0x40, 0x1000, 0x40000, 0xF0000}; -+ // Probes must span the target's whole address map, not just its bottom. -+ // An earlier list stopped below 1 MB and so passed against a window that -+ // served only its first eighth, while the target's program image and -+ // stack -- which sit at 0x80000000 and just under 4 GiB -- landed -+ // outside it. Proving the path works says nothing about how far it -+ // reaches; these last three are what test the reach. -+ const size_t probes[] = {0x0, -+ 0x40, -+ 0x1000, -+ 0x40000, -+ 0xF0000, -+ 0x20000000, -+ 0x80000000, -+ 0xFFFF0000}; - for (size_t addr : probes) { - // A value that is distinct per address and not all-ones/all-zeros. - mpz_set_ui(wr, 0); ---- build_firesim/firesim/platforms/vitis/cl_firesim/Makefile.orig -+++ build_firesim/firesim/platforms/vitis/cl_firesim/Makefile -@@ -50,7 +50,13 @@ - # Memory resource backing the simulator's host_mem_0 port. Upstream hardcodes - # DDR[0]; HBM-only cards (Alveo U55C) expose no DDR bank at all, so this must - # be overridable per platform. --HOST_MEM_SP ?= HBM[0] -+# -+# The range must span the target's whole address map: a 32-bit target needs -+# 4 GiB, which on a U55C is eight 512 MB HBM pseudo-channels. It is bound -+# together with host_mem_0_size_bytes in simif_vitis.cc -- the port reaches -+# what this binds, and the driver backs what that allocates, so widening -+# either alone silently truncates. -+HOST_MEM_SP ?= HBM[0:7] - - # Provide's the desired simulator frequency. This is generated from the clock - # provided to the kernel using an internal MMCM. -@@ -118,7 +124,7 @@ - VPP_MSG_FLAGS = --message-rules $(MSG_RULES) --log_dir $(LOG_SUBDIR) --report_dir $(REPORT_SUBDIR) - VPP_COMMON_ARGS = \ - --platform $(DEVICE) \ -- --temp_dir $(TEMP_SUBDIR) \ -+ --temp_dir vpp_temp.$(XSA) \ - $(VPP_OTHER_FLAGS) \ - $(VPP_MSG_FLAGS) - diff --git a/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch b/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch deleted file mode 100644 index 9ae0b9aa21..0000000000 --- a/hw/syn/firesim/patches/0008-driver-teardown-before-exit.patch +++ /dev/null @@ -1,18 +0,0 @@ ---- build_firesim/firesim/sim/midas/src/main/cc/entry.cc.orig -+++ build_firesim/firesim/sim/midas/src/main/cc/entry.cc -@@ -73,5 +73,14 @@ - simulation = create_simulation(simif, *widget_registry_ptr, args); - - // Run the simulation with the given implementation. -- return simif.run(*simulation); -+ const int rc = simif.run(*simulation); -+ -+ // Tear these down here rather than leaving them to static destruction. The -+ // platform layer holds handles into libraries with statics of their own -- -+ // XRT's device registry among them -- and by the time exit handlers run those -+ // may already be gone, so destroying the interface then walks freed state. -+ // Order mirrors construction. Nothing below may use `simif`. -+ simulation.reset(); -+ simulator.reset(); -+ return rc; - } diff --git a/hw/syn/firesim/patches/README b/hw/syn/firesim/patches/README deleted file mode 100644 index 4e696af09c..0000000000 --- a/hw/syn/firesim/patches/README +++ /dev/null @@ -1,104 +0,0 @@ -Patches against upstream FireSim (tag 1.21.0) required to build and run a -FireSim simulator on the Alveo U55C under Vitis/Vivado 2024.2. - -The paths in these patches are relative to the Vortex repository root, where -the FireSim clone sits at build_firesim/firesim/. Applying them from the root -of a FireSim checkout therefore strips two components, not one: - - cd - patch -p2 < 0001-rebase-target-cycle-counter.patch - patch -p2 < 0002-vitis-u55c-build-fixes.patch - patch -p2 < 0003-vitis-memcheck-and-buffer-lifetime.patch - patch -p2 < 0004-vitis-host-mem-base-address.patch - patch -p2 < 0005-inflight-target-cycle.patch - patch -p2 < 0006-package-design-sources.patch - patch -p2 < 0007-vitis-target-dram-aperture.patch - patch -p2 < 0008-driver-teardown-before-exit.patch - -Apply in order: 0003 and 0005 both touch simulation.{h,cc} and assume 0001 is -already in. Applying the series in order to a pristine 1.21.0 checkout -reproduces this tree's clone byte-for-byte. - -0001-rebase-target-cycle-counter.patch - Correctness fix. The target-cycle counter lives in FPGA fabric and - persists while the bitstream stays programmed, but simulation_t reported - it as an absolute value. Every driver invocation against a resident - xclbin therefore inherited the previous run's count -- observed as - 13, 26, 39, ... cycles of drift across successive runs of the same test, - with FMR and Effective Target Frequency corrupted alongside it. Host - cycles were already rebased in the same function; target cycles now are - too. Host-side only, no bitstream rebuild needed. - -0002-vitis-u55c-build-fixes.patch - Build-flow fixes. Upstream's Vitis flow targets Vitis 2022.1 and a U250; - neither assumption holds here. See the patch header for the four - independent failures and their causes. - -0003-vitis-memcheck-and-buffer-lifetime.patch - Correctness fix plus the diagnostic that settled the FASED-on-HBM - question. The xrt::bo backing host_mem_0 was a constructor local, so its - device allocation was freed before the simulation ran. +memcheck - round-trips a pattern through the LoadMem widget before target execution, - isolating the target DRAM path from the design under test; on the U55C it - reports an address-independent constant, unchanged by zeroing DRAM -- - i.e. the HBM memory path never accesses memory, while the MMIO/control - plane is fully functional. See - docs/proposals/firesim_vitis_host_mem_offset.md. - -0004-vitis-host-mem-base-address.patch - The fix that makes FASED work on an HBM part. VitisShim added a literal - 0x4000000000 (U250 DDR[0] base) to every host_mem_0 address in RTL. On the - U55C, host_mem_0 binds to HBM[0] at 0x0 with a 512 MB aperture, so every - access landed outside the window: reads returned a fixed bus default - independent of address, writes were dropped, and the target DRAM path was - dead while the control plane still worked. Now a documented constant in - VitisConstants. Validated with +memcheck under hw_emu: 5/5 probes fail - before, 0/5 after. - -0005-inflight-target-cycle.patch - Correctness fix on top of 0001. The clock bridge counts target cycles - *started*, so one cycle is always in flight; it cancels between the two - samples except across the counter's origin, leaving the first run after - programming one cycle longer than every run after it (14 vs 13 for a test - that provably steps 13 times). A fresh metasim is always the origin case, - so upstream reports the +1 on every metasim run. See - docs/proposals/firesim_inflight_target_cycle.md. - -0006-package-design-sources.patch - Build-flow fix. A design whose RTL is resolved by library search in - simulation loses its submodules under Vivado, which has no equivalent. - VORTEX_VIVADO_SRCS names a flattened source list to add to the packaged - kernel; absent it the flow is unchanged. - -0007-vitis-target-dram-aperture.patch - Correctness fix. host_mem_0 was bound to a single HBM pseudo-channel and - backed by a 512 MB allocation, while a 32-bit target spans 4 GB -- its - program image sits at 0x80000000 and its stack just under 4 GB. Uploading a - kernel therefore wrote outside the served window and the target fetched - memory that was never written, with no error anywhere to say so. Widened to - HBM[0:7] plus a matching 4 GiB window; the two are a pair and each carries a - comment saying widening one alone truncates silently. The buffer is now - allocated from whichever memory region can hold the window rather than from - bank 0 -- a range binding appears as a memory *group* (MBG[0:7], index 39 on - this platform) whose index cannot be derived from the port name, and - xrt::ip exposes no group_id(). +memcheck gains probes at 0x20000000, - 0x80000000 and 0xFFFF0000; it passed throughout this defect because it only - probed below 1 MB, which is how the bug reached the card. No Chisel change: - HostMemChannelKey.size is already 16 GiB, so the shim's address path was - never the constraint. See docs/proposals/firesim_target_memory_aperture.md. - -0008-driver-teardown-before-exit.patch - Correctness fix. `simulator` is a global std::unique_ptr, so the platform - interface was destroyed during static destruction -- and on Vitis that runs - xrt::device::~device() after XRT's own statics are gone, segfaulting in the - device registry's deleter. Upstream's comment says the global "must exist - for dpi.cc calls in emulation, must be global to avoid deletion", but a - global unique_ptr *is* deleted at exit; nothing references it once entry() - returns, on either platform. Now torn down explicitly at the end of entry(). - Never observed before because no run had ever reached normal process exit -- - every earlier one timed out. - -All are candidates for upstreaming: only the HOST_MEM_SP default is -U55C-specific, and it is parameterized rather than hardcoded. 0005 changes a -reported number relative to upstream; the patch header records why the new -number is the correct one. diff --git a/hw/syn/foldsim/Makefile b/hw/syn/foldsim/Makefile new file mode 100644 index 0000000000..62df5ff58a --- /dev/null +++ b/hw/syn/foldsim/Makefile @@ -0,0 +1,252 @@ +# foldsim image build for the V80, driven as the native aved flow +# (hw/syn/xilinx/aved): +# +# make [DEVICE=] [PREFIX=...] [CONFIGS="-DVX_CFG_..."] [PERF=1] +# [FOLDSIM_CLASSES="..."] [FOLDSIM_LIFT="..."] [CIRCT_BIN=] [FOLDSIM=] +# +# foldsim compiles Vortex into a simulator, on CIRCT: each class threaded +# through one physical copy (FAME-5), the rest of the target a model of its +# own, and every channel between them in one token fabric. Foldsim's shell +# (FOLDSIM_SHELL, part of the Foldsim installation) adds the memory and +# control endpoints around it; Vortex supplies only the target wrapper +# (hw/rtl/afu/foldsim/target) and this flow. The image lands in +# $(BUILD_DIR)/bin/foldsim.vbin; run it with +# FPGA_BIN_DIR= ./ci/blackbox.sh --driver=foldsim --target=hw ... +# The runtime's avedsim target simulates the same generated design in-process. + +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk +include $(VORTEX_HOME)/hw/syn/common.mk + +SRC_DIR := $(VORTEX_HOME)/hw/syn/foldsim +RTL_DIR := $(VORTEX_HOME)/hw/rtl +SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts +# The shell's RTL, and the wrapper that gives Vortex the shell's target ports. +FOLDSIM_RTL_DIR := $(FOLDSIM_SHELL)/rtl +FOLDSIM_TARGET_DIR := $(RTL_DIR)/afu/foldsim/target +VIVADO := $(XILINX_VIVADO)/bin/vivado +SLASHKIT ?= slashkit +# Parallel link runs; each Vivado of a large simulator needs tens of GB. +SLASHKIT_JOBS ?= + +XLEN ?= 32 +PREFIX ?= build$(XLEN) +TARGET ?= hw +DEVICE ?= xcv80-lsva4737-2MHP-e-S + +ifneq ($(TARGET), hw) +$(error TARGET must be hw: the runtime's avedsim target simulates the generated design itself) +endif + +BUILD_DIR := $(abspath $(PREFIX)_foldsim_$(TARGET)) +# The runtime's in-process simulator generates the design in a directory of +# its own. +ifneq ($(FOLDSIM_WORK_DIR),) +BUILD_DIR := $(abspath $(FOLDSIM_WORK_DIR)) +endif +BIN_DIR := $(BUILD_DIR)/bin +GEN_DIR := $(BUILD_DIR)/gen + +# The classes threaded by FAME-5, and the units given plain-port boundaries +# first so they can be classes (Foldsim's foldsim-lift; [:drop=P,...][;port=P,...]). +FOLDSIM_CLASSES ?= VX_socket__lift +FOLDSIM_LIFT ?= VX_socket:drop=INSTANCE_ID +# Where circt-verilog, circt-opt and firtool are (hw/syn/common.mk's CIRCT); +# the foldsim driver is hw/syn/common.mk's FOLDSIM. +CIRCT_BIN ?= $(CIRCT_PATH)/bin +LIFT_PYTHON ?= python3 +# The kernel clock (MHz): the image is timed at it and the shell programs it. +FOLDSIM_FREQ ?= 250 +# Options of the foldsim driver: by default the set that times s2w4i2 at +# 250 MHz (regions of 2048 loads with their own control, registered state +# committed by select over two host cycles, pipelined multipliers, staged +# memory writes and tokens, folded fixed-latency pipelines); docs/designs/ +# foldsim.md lists each. Large images add --compact-staging; multi-SLR ones +# --domains and --relay. +FOLDSIM_FLAGS ?= --region-loads 2048 --registered-state --commit-select --state-stages 2 \ + --pipeline-multipliers --staged-ram-writes --staged-tokens --fold-pipelines + +# The target: Vortex with its AXI4 port, DCR, start and busy as plain ports. +TARGET_TOP := foldsim_target + +# The same tree the native synthesis flows elaborate. +CONFIGS ?= +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs \ + -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm \ + -I$(RTL_DIR)/cache -I$(RTL_DIR)/fpu -I$(RTL_DIR)/afu/common -I$(FOLDSIM_TARGET_DIR) +SYNTH_DEFINES := -DSYNTHESIS -DVIVADO -DNDEBUG +ifdef PERF +PLATFORM_DEFINES := -DPERF_ENABLE +endif +# Every default is the configuration's (VX_config.toml with CONFIGS). The FPU +# type is chosen as in the native Vivado flows: DSP selects the Xilinx +# floating-point cores, which the simulator keeps as instances: their netlists +# are encrypted, so their state cannot be threaded, and each is simulated by +# one unthreaded copy of itself and a per-thread store of its results (foldsim +# --fold-pipelines, by the pipeline contract xilinx_ip_stubs.sv declares). +XCONFIGS := $(shell python3 $(VORTEX_HOME)/ci/gen_config.py \ + --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN) $(SYNTH_DEFINES) $(PLATFORM_DEFINES)') +TYPE_DEFINES := $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +cfg_value = $(lastword $(patsubst -D$(1)=%,%,$(filter -D$(1)=%,$(XCONFIGS)))) +# One memory port per memory bank of the configuration, each its own memory +# endpoint and DRAM channel. AXI ids are what Vortex drives with its tag +# buffer engaged: the buffer's 4 index bits, and above them the index of the +# GPU's memory port the read came from, of which there are at most as many as +# memory ports. +MEM_PORTS := $(call cfg_value,VX_CFG_PLATFORM_MEMORY_NUM_BANKS) +ifeq ($(filter 1 2 4 8,$(MEM_PORTS)),) +$(error the foldsim kernel has 1, 2, 4 or 8 memory ports (VX_CFG_PLATFORM_MEMORY_NUM_BANKS=$(MEM_PORTS))) +endif +MEM_ID_W := $(shell python3 -c 'print(4 + ($(MEM_PORTS) - 1).bit_length())') +PLATFORM_DEFINES += -DPLATFORM_MEMORY_ID_WIDTH=$(MEM_ID_W) +# The shift that takes an address to its memory port's channel: the ports +# interleave by block, or (INTERLEAVE=0) split the space, where every port's +# channel holds the address as it is. +MEM_CH_SHIFT := $(if $(filter 1,$(call cfg_value,VX_CFG_PLATFORM_MEMORY_INTERLEAVE)), \ + $(shell python3 -c 'print(($(MEM_PORTS) - 1).bit_length())'),0) +# The extensions' packages and include paths, as every synthesis flow finds +# them. +include $(VORTEX_HOME)/hw/syn/extensions.mk +GEN_CFLAGS := -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) $(CONFIGS) $(TYPE_DEFINES) $(PLATFORM_DEFINES) \ + $(SYNTH_DEFINES) $(RTL_INCLUDE) + +# Rewritten only when a setting changes, so an unchanged build stays current. +define stamp +$(shell mkdir -p $(BUILD_DIR) && echo '$(2)' > $(1).tmp && \ + (cmp -s $(1).tmp $(1) && rm -f $(1).tmp || mv -f $(1).tmp $(1))) +endef +CONFIG_STAMP := $(BUILD_DIR)/cflags.stamp +CLASSES_STAMP := $(BUILD_DIR)/classes.stamp +$(call stamp,$(CONFIG_STAMP),$(GEN_CFLAGS)) +$(call stamp,$(CLASSES_STAMP),$(FOLDSIM_CLASSES) | $(FOLDSIM_LIFT)$(if $(FOLDSIM_FLAGS), | $(FOLDSIM_FLAGS))) + +SRCS_NAME := vortex_srcs +SRCS_DIR := $(BUILD_DIR)/$(SRCS_NAME) +SRCS_LIST := $(BUILD_DIR)/$(SRCS_NAME).f +LIFT_LIST := $(BUILD_DIR)/lift/srcs.f +TARGET_SRCS := $(if $(strip $(FOLDSIM_LIFT)),$(LIFT_LIST),$(SRCS_LIST)) +GEN_SV := $(GEN_DIR)/foldsim_sim.sv +KERNEL_VH := $(GEN_DIR)/foldsim_kernel.vh +KERNEL_SRCS := $(BUILD_DIR)/kernel_srcs.f +IP_DIR := $(BUILD_DIR)/foldsim_afu_ip +CFG := $(BUILD_DIR)/config.cfg +VBIN := $(BIN_DIR)/foldsim.vbin + +RTL_SRCS := $(shell find $(RTL_DIR) -name '*.sv' -o -name '*.v' -o -name '*.vh') \ + $(wildcard $(ROOT_DIR)/hw/VX_config.vh) $(wildcard $(ROOT_DIR)/hw/VX_types.vh) \ + $(firstword $(MAKEFILE_LIST)) $(CONFIG_STAMP) +# The AFU shells of the native flows are second tops. +SHELL_MODULES := $(notdir $(wildcard $(RTL_DIR)/afu/common/*.sv)) + +.PHONY: all sources generate package-ip ooc clean FORCE + +all: $(VBIN) + +# The Xilinx floating-point cores, declared as black boxes with their pipeline +# contracts; their contracts agree with the cores' generation and the RTL. +IP_STUBS := $(SCRIPT_DIR)/xilinx_ip_stubs.sv +# The list is written to a temporary and moved into place last, so a failing +# step leaves no list that looks current. +sources: $(SRCS_LIST) +$(SRCS_LIST): $(RTL_SRCS) $(IP_STUBS) $(SCRIPT_DIR)/xilinx_ip_gen.tcl + rm -rf $(SRCS_DIR) $@ $@.tmp + python3 $(SCRIPT_DIR)/check_ip_contracts.py --cflags '$(GEN_CFLAGS)' + cd $(BUILD_DIR) && $(SCRIPT_DIR)/gen_sources.sh -P $(GEN_CFLAGS) -T$(TARGET_TOP) -C$(SRCS_NAME) \ + -O$(SRCS_NAME).f.tmp $(RTL_PKGS) + cp $(IP_STUBS) $(SRCS_DIR)/ && echo "$(SRCS_DIR)/$(notdir $(IP_STUBS))" >> $@.tmp + @for f in $(SHELL_MODULES); do \ + rm -f $(SRCS_DIR)/$$f; \ + grep -v "/$$f$$" $@.tmp > $@.tmp2 && mv $@.tmp2 $@.tmp; \ + done + @# the RAM fold runs here, not in gen_sources.sh (-F), so it patches the sources as read + $(RAM_FOLD_PYTHON) $(SCRIPT_DIR)/ram_fold.py --top $(TARGET_TOP) --report $(SRCS_DIR)/ram_fold.rpt \ + --diff $(SRCS_DIR)/ram_fold.diff $$(grep -E '\.s?v$$' $@.tmp) + @if grep -qE 'localparam +VX_MEM_PORTS *= *0 *;' $(SRCS_DIR)/VX_gpu_pkg.sv; then \ + echo "ERROR: generated sources resolve VX_MEM_PORTS to 0 (no memory interface)"; exit 1; \ + fi + mv $@.tmp $@ + +$(LIFT_LIST): $(SRCS_LIST) $(FOLDSIM_LIFT_TOOL) $(CLASSES_STAMP) + rm -rf $(BUILD_DIR)/lift + $(LIFT_PYTHON) $(FOLDSIM_LIFT_TOOL) --srcs $(SRCS_LIST) --top $(TARGET_TOP) \ + --out $(BUILD_DIR)/lift $(foreach u,$(FOLDSIM_LIFT),--unit '$(u)') + +generate: $(GEN_SV) $(KERNEL_VH) +# The compiler is the driver and the tools it runs: foldsim-opt beside it and +# CIRCT's, so a rebuilt tool regenerates the simulator. +FOLDSIM_TOOLS := $(FOLDSIM) $(wildcard $(dir $(FOLDSIM))foldsim-opt \ + $(addprefix $(CIRCT_BIN)/,circt-verilog circt-opt firtool)) +$(GEN_SV): $(TARGET_SRCS) $(FOLDSIM_TOOLS) $(CLASSES_STAMP) + $(FOLDSIM) --srcs $(TARGET_SRCS) --top $(TARGET_TOP) --out $(GEN_DIR) \ + $(if $(CIRCT_BIN),--circt-bin $(CIRCT_BIN)) $(FOLDSIM_FLAGS) $(FOLDSIM_CLASSES) + +# What the kernel's top needs of the configuration: its memory ports, their id +# width, and FOLDSIM_MEM_REPEAT(f,s), `f(p) for each port p joined by `s, which +# lists the ports whose names carry their index. Spelled out here: Vivado's IP +# packager does not paste a macro's expanded argument into a macro name. +MEM_REPEAT := $(shell python3 -c 'print(" `s ".join("`f(%d)" % p for p in range($(MEM_PORTS))))') +$(KERNEL_VH): FORCE + @mkdir -p $(@D) + @printf '%s\n' '// Generated by hw/syn/foldsim. Do not edit.' \ + '`define FOLDSIM_MEM_PORTS $(MEM_PORTS)' '`define FOLDSIM_MEM_CH_SHIFT $(strip $(MEM_CH_SHIFT))' \ + '`define FOLDSIM_MEM_ID_W $(MEM_ID_W)' '`define FOLDSIM_MEM_REPEAT(f,s) $(MEM_REPEAT)' \ + $(if $(findstring --axi-spec,$(FOLDSIM_FLAGS)),'`define FOLDSIM_MEM_SPEC $(if $(findstring --axi-spec=2,$(FOLDSIM_FLAGS)),2,1)') > $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + +$(KERNEL_SRCS): $(GEN_SV) $(KERNEL_VH) + @{ echo "+incdir+$(FOLDSIM_RTL_DIR)"; echo "+incdir+$(GEN_DIR)"; \ + echo "$(GEN_SV)"; \ + for f in foldsim_token.sv foldsim_mem_bmap.sv foldsim_mem_bank.sv foldsim_mem.sv foldsim_ctrl.sv \ + foldsim_afu.sv; do echo "$(FOLDSIM_RTL_DIR)/$$f"; done; \ + } > $@ + +# The kernel's clock at FOLDSIM_FREQ, for its out-of-context synthesis. +OOC_XDC := $(BUILD_DIR)/foldsim_afu_ooc.xdc +$(OOC_XDC): FORCE + @mkdir -p $(@D) + @printf 'create_clock -name ap_clk -period %s [get_ports ap_clk]\n' \ + $$(python3 -c 'print(round(1000 / $(FOLDSIM_FREQ), 3))') > $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + +package-ip: $(IP_DIR)/component.xml +# The kernel is packaged with the floating-point cores the simulator +# instantiates, generated as the aved flow generates them. +$(IP_DIR)/component.xml: $(KERNEL_SRCS) $(KERNEL_VH) $(OOC_XDC) $(wildcard $(FOLDSIM_RTL_DIR)/*) $(SCRIPT_DIR)/xilinx_ip_gen.tcl + cd $(BUILD_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -nojournal -nolog \ + -source $(VORTEX_HOME)/hw/syn/xilinx/aved/package_ip.tcl \ + -tclargs foldsim_afu $(KERNEL_SRCS) . $(DEVICE) ooc-xdc=$(OOC_XDC) \ + $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) + +# foldsim --slr-groups writes a floorplan that keeps each threaded copy whole +# in one SLR; the implementation applies it before opt_design (floorplan.tcl). +FLOORPLAN_ENV := $(if $(findstring --slr-groups,$(FOLDSIM_FLAGS)),FOLDSIM_FLOORPLAN=$(GEN_DIR)/foldsim_floorplan.tcl) + +# The kernel alone, out of context at FOLDSIM_FREQ: synthesis with global +# retiming, placement and routing, without the shell, its reports in +# $(BUILD_DIR)/ooc. It answers what a change does to the kernel's Fmax in +# about half an image build's time. +OOC_DIR := $(BUILD_DIR)/ooc +ooc: $(KERNEL_SRCS) + rm -rf $(OOC_DIR) && mkdir -p $(OOC_DIR) + cd $(OOC_DIR) && TOOL_DIR=$(SCRIPT_DIR) $(FLOORPLAN_ENV) $(VIVADO) -mode batch -nojournal -log vivado.log \ + -source $(SRC_DIR)/ooc.tcl -tclargs $(KERNEL_SRCS) $(DEVICE) \ + $$(python3 -c 'print(round(1000 / $(FOLDSIM_FREQ), 3))') \ + $$(grep -qE '^ *xil_f(ma|div|sqrt|mul|add) ' $(GEN_SV) || echo no-vendor-ip) + @grep -A2 'WNS(ns)' $(OOC_DIR)/routed_timing_summary.rpt | tail -1 + +$(CFG): FORCE + @python3 $(SCRIPT_DIR)/gen_platform.py --flow=aved --device=$(DEVICE) --emit=cfg --kernel=foldsim_afu \ + --no-host --mem-ports=$(MEM_PORTS) --kernel-mhz=$(FOLDSIM_FREQ) -o $@.tmp + @cmp -s $@.tmp $@ && rm -f $@.tmp || mv -f $@.tmp $@ + +FORCE: + +$(VBIN): $(IP_DIR)/component.xml $(CFG) + mkdir -p $(BIN_DIR) + TOOL_DIR=$(SCRIPT_DIR) $(FLOORPLAN_ENV) $(SLASHKIT) link -c $(CFG) -p hw -o $@ -k $(IP_DIR)/component.xml \ + --vivado $(VIVADO) $(if $(SLASHKIT_JOBS),--jobs $(SLASHKIT_JOBS)) \ + --pre-synth-tcls $(SRC_DIR)/pre_synth_hook.tcl + @echo "image: $@" + +clean: + rm -rf $(BUILD_DIR) diff --git a/hw/syn/foldsim/floorplan.tcl b/hw/syn/foldsim/floorplan.tcl new file mode 100644 index 0000000000..39a4ec11c4 --- /dev/null +++ b/hw/syn/foldsim/floorplan.tcl @@ -0,0 +1,30 @@ +# Applies foldsim's --slr-groups floorplan, $::env(FOLDSIM_FLOORPLAN), to the +# open synthesized design: each threaded copy kept whole in one SLR, the SLR +# left to the placer. A copy split over an SLR boundary pays a crossing on +# paths wholly inside it. Sourced before opt_design by ooc.tcl and by the +# image's implementation (pre_opt_hook.tcl); the Makefile sets the variable +# when FOLDSIM_FLAGS has --slr-groups. The assignments land in floorplan.txt. + +if {[info exists ::env(FOLDSIM_FLOORPLAN)] && $::env(FOLDSIM_FLOORPLAN) ne ""} { + set fs_floorplan $::env(FOLDSIM_FLOORPLAN) + if {![file exists $fs_floorplan]} { + error "foldsim floorplan not found: $fs_floorplan" + } + set fp [open $fs_floorplan r] + set fs_groups [regexp -all {USER_SLR_ASSIGNMENT} [read $fp]] + close $fp + source $fs_floorplan + set fs_cells [get_cells -quiet -hierarchical -filter {USER_SLR_ASSIGNMENT =~ "fs_*"}] + set fp [open floorplan.txt w] + foreach c $fs_cells { puts $fp "$c [get_property USER_SLR_ASSIGNMENT $c]" } + close $fp + # A floorplan that matches nothing would leave the copies to area-only + # placement without a trace in the reports. + if {[llength $fs_cells] == 0} { + error "foldsim floorplan $fs_floorplan matched no cell" + } + if {[llength $fs_cells] < $fs_groups} { + puts "CRITICAL WARNING: foldsim floorplan: [llength $fs_cells] of $fs_groups copies found" + } + puts "INFO: foldsim floorplan: [llength $fs_cells] threaded copies in SLR groups ($fs_floorplan)" +} diff --git a/hw/syn/foldsim/ooc.tcl b/hw/syn/foldsim/ooc.tcl new file mode 100644 index 0000000000..2901cd62b7 --- /dev/null +++ b/hw/syn/foldsim/ooc.tcl @@ -0,0 +1,81 @@ +# Copyright © 2019-2023 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# The foldsim kernel alone, out of context: synthesis with global retiming, +# placement and routing at the kernel clock, as the image's own run times it +# but without the shell around it. Reports land in the current directory. +# +# vivado -mode batch -source ooc.tcl -tclargs [no-vendor-ip] +# +# The Xilinx floating-point cores a simulator instantiates are generated as +# the aved flow generates them (TOOL_DIR's xilinx_ip_gen.tcl) and synthesized +# out of context; no-vendor-ip skips them for a kernel without any. + +if {$argc < 3} { + puts "Usage: $::argv0 \[no-vendor-ip\]" + exit 1 +} +lassign $argv srcs part period options +set vendor_ip [expr {$options ne "no-vendor-ip"}] + +if {$vendor_ip} { + set tool_dir $::env(TOOL_DIR) + set argv [list [pwd]/ip $part] + set argc 2 + source ${tool_dir}/xilinx_ip_gen.tcl +} + +set incdirs {} +set fp [open $srcs r] +foreach line [split [read $fp] "\n"] { + set line [string trim $line] + if {$line eq "" || [string match "#*" $line]} continue + if {[string match "+incdir+*" $line]} { lappend incdirs [string range $line 8 end]; continue } + read_verilog -sv $line +} +close $fp +if {$vendor_ip} { + foreach xci [glob [pwd]/ip/*/*.xci] { read_ip $xci } + generate_target all [get_ips] + synth_ip [get_ips] +} + +# The clock with the uncertainty the image's runs see. +set f [open clk.xdc w] +puts $f "create_clock -name ap_clk -period $period \[get_ports ap_clk\]" +puts $f "set_clock_uncertainty -setup 0.200 \[get_clocks ap_clk\]" +close $f +read_xdc clk.xdc + +set t0 [clock seconds] +synth_design -top foldsim_afu -part $part -mode out_of_context -include_dirs $incdirs -global_retiming on +set t1 [clock seconds] +report_utilization -file synth_util.rpt +report_timing_summary -max_paths 20 -file synth_timing_summary.rpt +source [file join [file dirname [file normalize [info script]]] floorplan.tcl] +opt_design +place_design -directive Explore +phys_opt_design -directive AggressiveExplore +set t2 [clock seconds] +route_design -directive Explore +phys_opt_design -directive AggressiveExplore +set t3 [clock seconds] +report_utilization -file routed_util.rpt +report_utilization -hierarchical -hierarchical_depth 6 -file routed_util_hier.rpt +report_timing_summary -max_paths 50 -file routed_timing_summary.rpt +report_design_analysis -timing -max_paths 100 -file routed_design_analysis.rpt +report_design_analysis -logic_level_distribution -logic_level_dist_paths 5000 -file routed_lld.rpt +report_high_fanout_nets -max_nets 40 -timing -file routed_hfn.rpt +set f [open times.txt w] +puts $f "synth [expr {$t1-$t0}] place [expr {$t2-$t1}] route [expr {$t3-$t2}] total [expr {$t3-$t0}]" +close $f diff --git a/hw/syn/foldsim/pre_opt_hook.tcl b/hw/syn/foldsim/pre_opt_hook.tcl new file mode 100644 index 0000000000..8be17f2aa3 --- /dev/null +++ b/hw/syn/foldsim/pre_opt_hook.tcl @@ -0,0 +1,12 @@ +# Runs on impl_1 immediately before opt_design, attached by pre_synth_hook.tcl +# when the image has a floorplan: the native flow's pre-opt hook, then the +# floorplan. + +set fs_hook_dir [file dirname [file normalize [info script]]] +source [file join $fs_hook_dir .. xilinx aved pre_opt_hook.tcl] +# The variable reaches the run from the linker's environment; a run without +# it would silently place without the floorplan the image was built for. +if {![info exists ::env(FOLDSIM_FLOORPLAN)] || $::env(FOLDSIM_FLOORPLAN) eq ""} { + error "FOLDSIM_FLOORPLAN is not set in the implementation run" +} +source [file join $fs_hook_dir floorplan.tcl] diff --git a/hw/syn/foldsim/pre_synth_hook.tcl b/hw/syn/foldsim/pre_synth_hook.tcl new file mode 100644 index 0000000000..e63b565ae6 --- /dev/null +++ b/hw/syn/foldsim/pre_synth_hook.tcl @@ -0,0 +1,21 @@ +# Sourced by slashkit via --pre-synth-tcls, after the linker has created the +# project and its runs but before it launches them. + +# The native flow's hook: the pre-opt utilization report and post-route +# phys_opt_design -directive AggressiveExplore. +set fs_hook_dir [file dirname [file normalize [info script]]] +source [file join $fs_hook_dir .. xilinx aved pre_synth_hook.tcl] + +# An image with a floorplan (foldsim --slr-groups) applies it before +# opt_design, after the native pre-opt hook. +if {[info exists ::env(FOLDSIM_FLOORPLAN)] && $::env(FOLDSIM_FLOORPLAN) ne ""} { + set_property STEPS.OPT_DESIGN.TCL.PRE [file join $fs_hook_dir pre_opt_hook.tcl] [get_runs impl_1] + puts "INFO: foldsim floorplan $::env(FOLDSIM_FLOORPLAN) attached to impl_1" +} + +# The kernel's out-of-context synthesis sees its clock (the IP's OOC XDC) and +# retimes against it: a threaded model's registered state is retimable. +foreach run [get_runs -quiet -filter {IS_SYNTHESIS} *foldsim_afu*] { + set_property STEPS.SYNTH_DESIGN.ARGS.GLOBAL_RETIMING on $run + puts "INFO: global retiming on for $run" +} diff --git a/hw/syn/synopsys/Makefile b/hw/syn/synopsys/Makefile index 61bdbd8341..f457f6f24c 100644 --- a/hw/syn/synopsys/Makefile +++ b/hw/syn/synopsys/Makefile @@ -154,7 +154,7 @@ all: synthesis gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: - $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -C$(BUILD_DIR)/src -O$(BUILD_DIR)/sources.txt + $(SCRIPT_DIR)/gen_sources.sh -P -F -T$(TOP_LEVEL_ENTITY) $(CFLAGS) -C$(BUILD_DIR)/src -O$(BUILD_DIR)/sources.txt synthesis: $(BUILD_DIR)/sources.txt cd $(BUILD_DIR); \ diff --git a/hw/syn/xilinx/README b/hw/syn/xilinx/README index 7f0c43d38c..d698de9fde 100644 --- a/hw/syn/xilinx/README +++ b/hw/syn/xilinx/README @@ -9,7 +9,7 @@ vivado -mode batch -source xilinx_ip_gen.tcl -tclargs ip/xilinx_u50_gen3x16_xdma # build per-block (dut): Vivado project-mode synthesis of an individual block. # Self-backgrounds; writes //build.log. Default DEVICE: xcu55c. -# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope unittest +# Targets: core vortex top issue cache fpu tcu dxa tex raster om lmem mem_unit scope cd hw/syn/xilinx/dut PREFIX=smoke1 make core PREFIX=smoke1 make vortex @@ -70,9 +70,9 @@ TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vp TARGET=hw PLATFORM=xilinx_u55c_gen3x16_xdma_3_202210_1 VPP_FLAGS="--from_step vpl.impl.route_design" make > build.log 2>&1 & # running test -FPGA_BIN_DIR= TARGET=hw_emu ./ci/blackbox.sh --driver=xrt --app=demo -FPGA_BIN_DIR= TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" -FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 TARGET=hw ./ci/blackbox.sh --driver=xrt --app=sgemm --args="-n1024" +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw_emu --app=demo +FPGA_BIN_DIR= ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" +FPGA_BIN_DIR= XRT_DEVICE_INDEX=1 ./ci/blackbox.sh --driver=xrt --target=hw --app=sgemm --args="-n1024" # build report logs /bin/vortex_afu.xclbin.info diff --git a/hw/syn/xilinx/aved/Makefile b/hw/syn/xilinx/aved/Makefile index f1471e46d8..c7866c4809 100644 --- a/hw/syn/xilinx/aved/Makefile +++ b/hw/syn/xilinx/aved/Makefile @@ -1,6 +1,6 @@ ROOT_DIR := $(realpath ../../../..) include $(ROOT_DIR)/config.mk -# exports VERILATOR/VERILATOR_PATH, which gen_sources.sh resolves from the +# exports CIRCT_VERILOG, the preprocessor gen_sources.sh resolves from the # environment rather than PATH include $(VORTEX_HOME)/hw/syn/common.mk @@ -19,29 +19,14 @@ TARGET ?= hw PREFIX ?= build$(XLEN) -MAX_JOBS ?= 8 - # V80 device. Override for a different speed grade or package. DEVICE ?= xcv80-lsva4737-2MHP-e-S SRC_DIR := $(VORTEX_HOME)/hw/syn/xilinx/aved +AFU_DIR := $(VORTEX_HOME)/hw/rtl/afu/aved -RTL_DIR := $(VORTEX_HOME)/hw/rtl -DPI_DIR := $(VORTEX_HOME)/hw/dpi -AFU_DIR := $(RTL_DIR)/afu/aved -AFU_COMMON_DIR := $(RTL_DIR)/afu/common -SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts - -VIVADO := $(XILINX_VIVADO)/bin/vivado SLASHKIT ?= slashkit -CP = cp -rf -RMDIR = rm -rf -ECHO = @echo - -NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') -JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) - BUILD_DIR := $(PREFIX)_aved_$(TARGET) BIN_DIR := $(BUILD_DIR)/bin @@ -60,89 +45,29 @@ ifneq ($(TARGET),$(findstring $(TARGET), hw sim)) $(error TARGET must be hw or sim (got '$(TARGET)')) endif -# Control RTL debug tracing states -DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE -DBG_TRACE_FLAGS += -DDBG_TRACE_MEM -DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE -DBG_TRACE_FLAGS += -DDBG_TRACE_AFU -DBG_TRACE_FLAGS += -DDBG_TRACE_TCU - -# Control logic analyzer monitors -DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU -DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE -DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH -DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU - -ifdef NUM_CORES -# cluster configuration -CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 -CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 -CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 -CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 -override CONFIGS += $(CONFIGS_$(NUM_CORES)c) -endif - -# `override`, as in platforms.mk: a command-line `make CONFIGS=...` otherwise -# discards these appends, and a build silently missing -DSYNTHESIS elaborates -# the simulation-only paths. -override CONFIGS += -DSYNTHESIS -DVIVADO - ifeq ($(TARGET), sim) override CONFIGS += -DSIMULATION endif -XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') +.DEFAULT_GOAL := all +include $(VORTEX_HOME)/hw/syn/xilinx/common.mk -# include sources -RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv -FPU_INCLUDE = -I$(RTL_DIR)/fpu -ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) - RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv - FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src -endif -RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) -RTL_INCLUDE += $(FPU_INCLUDE) - -# Command Processor sources. RTL_PKGS holds packages only; the CP -# interface files are auto-discovered via the -I path below. -RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv -RTL_INCLUDE += -I$(RTL_DIR)/cp - -# Per-extension RTL sources — single source of truth, appended based on -# the resolved XCONFIGS. -include $(VORTEX_HOME)/hw/syn/extensions.mk - -# load platform settings -include $(SRC_DIR)/platforms.mk - -# Memory connectivity tags resolved into the linker config. -MEM_TAG ?= HBM0 -HOST_TAG ?= HOST - -# HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete -# on the compute shell: the CP hangs on its first ring fetch and reports -# nothing -- the queue reads back armed and correct while head stays at 0. That -# is a full synthesis run to discover, and it has already cost one, so refuse -# the tag at parse time rather than emitting the config. See platforms.mk. -ifeq ($(HOST_TAG),HOST) -$(error HOST_TAG=HOST routes m_axi_host to the QDMA slave bridge, whose reads never complete on the compute shell; the CP would hang on its first ring fetch. Use HOST_TAG=HBM8 (the platforms.mk default)) -endif +# Card settings (memory geometry, connectivity tags, kernel clock), resolved +# from the device part. A tag the card refuses -- on the V80, HOST, whose reads +# never complete on the compute shell -- stops the build at parse time rather +# than after a full synthesis run. +PLATFORM_GEN_ARGS := --flow=aved --device=$(DEVICE) +include $(VORTEX_HOME)/hw/syn/xilinx/platform.mk # MEM covers the 32-bit device space from the HBM base, i.e. HBM0..HBM7; a CP # staging aperture there aliases device memory and large uploads overwrite the -# heap. See platforms.mk. +# heap. See hw/syn/xilinx/platforms/v80.toml. ifneq ($(filter HBM0 HBM1 HBM2 HBM3 HBM4 HBM5 HBM6 HBM7,$(HOST_TAG)),) ifneq ($(filter MEM HBM0,$(MEM_TAG)),) $(error HOST_TAG=$(HOST_TAG) lies inside the device-memory window of MEM_TAG=$(MEM_TAG) (HBM0..HBM7); CP staging would alias the Vortex heap. Use HBM8 or above) endif endif -# platforms.mk states the kernel clock in MHz; the linker config wants Hz. -KERNEL_FREQ_HZ := $(shell echo $$(( $(KERNEL_FREQ) * 1000000 ))) - # Debugging ifdef DEBUG CFLAGS += -DVX_DBG_DEBUG_LEVEL=$(DEBUG) $(DBG_TRACE_FLAGS) @@ -150,70 +75,40 @@ else CFLAGS += -DNDEBUG endif -# Enable scope analyzer -ifdef SCOPE - CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) - SCOPE_JSON += $(BIN_DIR)/scope.json -endif +CFLAGS += $(COMPILE_CFLAGS) -# Enable perf counters -ifdef PERF - CFLAGS += -DPERF_ENABLE +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) endif +# The linker config binds the single device-memory master to MEM_TAG. +ifneq ($(AFU_MEM_PORTS),1) +$(error the V80 link binds one device-memory master, but this configuration has AFU_MEM_PORTS=$(AFU_MEM_PORTS)) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) -# compilation flags -CFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) -# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the -# resolved type selectors so an enum override selects its RTL implementation -# (without this, VX_tcu_fedp elaborates 'no_implementation_selected'). -CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) -CFLAGS += $(RTL_INCLUDE) - -# ast dump flags -XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI - -.PHONY: all clean gen-sources gen-ast scope-json package-ip gen-bin check-axil-blocks - -# Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only -# partially decoded. Measured on V80 silicon: such a block DECERRs on EVERY -# word, including the words that ARE implemented -- the shell -> s_axi_control -# path resolves at 128-bit granularity and rejects a block it cannot fully -# resolve, so the per-word decode never gets consulted. No simulator reproduces -# it; the symptom is a register reading 0xFFFFFFFF on hardware for no reason -# visible in the RTL. Finding that cost ten days. A synthesis run costs ~37 -# minutes, so this check pays for itself the first time it fires. -check-axil-blocks: - @python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv +.PHONY: all clean package-ip gen-bin FORCE all: check-axil-blocks $(VBIN_CONTAINER) -gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt - -gen-ast: $(BUILD_DIR)/vortex.xml -$(BUILD_DIR)/vortex.xml: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml - -scope-json: $(BIN_DIR)/scope.json -$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml - mkdir -p $(BIN_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json +# Regenerated on every run and rewritten only when its content changes, so a +# command-line tag or clock override reaches the linker without invalidating an +# unchanged build. +$(BUILD_DIR)/config.cfg: FORCE + $(PLATFORM_GEN) --emit=cfg --mem-tag='$(MEM_TAG)' --host-tag='$(HOST_TAG)' --kernel-mhz=$(KERNEL_FREQ) -o $@ -$(BUILD_DIR)/config.cfg: $(SRC_DIR)/config.cfg.tmpl - mkdir -p $(BUILD_DIR) - sed -e 's|@MEM_TAG@|$(MEM_TAG)|g' -e 's|@HOST_TAG@|$(HOST_TAG)|g' \ - -e 's|@KERNEL_FREQ_HZ@|$(KERNEL_FREQ_HZ)|g' $< > $@ - @grep -q '@[A-Z_]*@' $@ && { echo "ERROR: unsubstituted placeholder in $@"; grep -n '@[A-Z_]*@' $@; rm -f $@; exit 1; } || true +FORCE: package-ip: $(IP_DIR)/component.xml $(IP_DIR)/component.xml: $(BUILD_DIR)/sources.txt cd $(BUILD_DIR); TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -source $(SRC_DIR)/package_ip.tcl -tclargs vortex_afu sources.txt . $(DEVICE) gen-bin: $(VBIN_CONTAINER) -# TOOL_DIR is consumed by pre_opt_hook.tcl to locate hw/scripts. The pre-synth -# hook is the linker's only opportunity to set per-step properties on impl_1, -# and it is what attaches the pre-opt hook that resolves the async-BRAM -# placeholders. +# The pre-synth hook is the linker's only opportunity to set per-step properties +# on impl_1, and it is what attaches the pre-opt hook (the hierarchical +# utilization report). $(VBIN_CONTAINER): $(IP_DIR)/component.xml $(BUILD_DIR)/config.cfg $(SCOPE_JSON) mkdir -p $(BIN_DIR) TOOL_DIR=$(SCRIPT_DIR) $(SLASHKIT) link -c $(BUILD_DIR)/config.cfg -p $(TARGET) -o $(VBIN_CONTAINER) \ diff --git a/hw/syn/xilinx/aved/config.cfg.tmpl b/hw/syn/xilinx/aved/config.cfg.tmpl deleted file mode 100644 index f401f4702b..0000000000 --- a/hw/syn/xilinx/aved/config.cfg.tmpl +++ /dev/null @@ -1,15 +0,0 @@ -[connectivity] -# Bind to the compute-only static shell. This is slashkit's current default, -# but it selects which abstract shell the AFU links against, so state it -# rather than inherit it. The service shell adds the DCMAC ethernet stack the -# AFU has no use for. -shell=compute -nk=vortex_afu:1:vortex_afu_0 -sp=vortex_afu_0.m_axi_mem_0:@MEM_TAG@ -sp=vortex_afu_0.m_axi_host:@HOST_TAG@ - -# Kernel clock. Stated in Hz here rather than via the linker's --clock-hz -# flag, whose name and help text disagree on the unit. -[clock] -krnl=vortex_afu_0 -freqhz=@KERNEL_FREQ_HZ@ diff --git a/hw/syn/xilinx/aved/package_ip.tcl b/hw/syn/xilinx/aved/package_ip.tcl index 0e6bdb33cc..ada3047e57 100644 --- a/hw/syn/xilinx/aved/package_ip.tcl +++ b/hw/syn/xilinx/aved/package_ip.tcl @@ -14,9 +14,9 @@ # Package the AFU as an IP-XACT component. The V80 linker consumes the # component.xml directly, so there is no intermediate object container. -if { $::argc != 4 } { - puts "ERROR: Program \"$::argv0\" requires 4 arguments!\n" - puts "Usage: $::argv0 \n" +if { $::argc < 4 || $::argc > 6 } { + puts "ERROR: Program \"$::argv0\" requires 4 to 6 arguments!\n" + puts "Usage: $::argv0 \[no-vendor-ip\] \[ooc-xdc=\]\n" exit 1 } @@ -24,6 +24,18 @@ set krnl_name [lindex $::argv 0] set vcs_file [lindex $::argv 1] set build_dir [lindex $::argv 2] set part [lindex $::argv 3] +# A kernel whose RTL instantiates no vendor floating-point core (a foldsim +# simulator of a soft FPU) is packaged without them. +set options [lrange $::argv 4 end] +set vendor_ip [expr {[lsearch -exact $options no-vendor-ip] < 0}] +# The kernel's clock for its out-of-context synthesis, so that synthesis +# times and retimes it: without one the IP is synthesized unconstrained. +set ooc_xdc "" +foreach option $options { + if {[string match "ooc-xdc=*" $option]} { + set ooc_xdc [file normalize [string range $option 8 end]] + } +} set tool_dir $::env(TOOL_DIR) @@ -36,9 +48,11 @@ set path_to_packaged "${build_dir}/${krnl_name}_ip" # and RTL elaboration fails at link time, after the IP looks fine. # xilinx_ip_gen.tcl opens and closes its own in-memory project, so it has to # run before the packaging project below. -set argv [list ${build_dir}/ip $part] -set argc 2 -source ${tool_dir}/xilinx_ip_gen.tcl +if {$vendor_ip} { + set argv [list ${build_dir}/ip $part] + set argc 2 + source ${tool_dir}/xilinx_ip_gen.tcl +} create_project -force kernel_pack $path_to_tmp_project -part $part @@ -63,13 +77,20 @@ foreach line [split $data "\n"] { } } +if {$ooc_xdc ne ""} { + add_files -norecurse -fileset constrs_1 $ooc_xdc + set_property USED_IN {synthesis out_of_context} [get_files $ooc_xdc] +} + # Pull in the floating-point cores generated above so they are packaged # alongside the RTL. The set matches the XRT flow's package_kernel.tcl. -set ip_files [list] -foreach ip {xil_fdiv xil_fma xil_fsqrt xil_fmul xil_fadd} { - lappend ip_files [file normalize "${build_dir}/ip/${ip}/${ip}.xci"] +if {$vendor_ip} { + set ip_files [list] + foreach ip {xil_fdiv xil_fma xil_fsqrt xil_fmul xil_fadd} { + lappend ip_files [file normalize "${build_dir}/ip/${ip}/${ip}.xci"] + } + add_files -verbose -norecurse -fileset [get_filesets sources_1] $ip_files } -add_files -verbose -norecurse -fileset [get_filesets sources_1] $ip_files set_property top $krnl_name [current_fileset] update_compile_order -fileset sources_1 diff --git a/hw/syn/xilinx/aved/platforms.mk b/hw/syn/xilinx/aved/platforms.mk deleted file mode 100644 index 9a6b5b2a90..0000000000 --- a/hw/syn/xilinx/aved/platforms.mk +++ /dev/null @@ -1,74 +0,0 @@ -# Platform specific configurations for the AMD Alveo V80. -# -# `override` so a command-line `make CONFIGS=...` cannot drop the platform -# settings; without it the AFU is packaged for the wrong memory topology. - -override CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 - -# The V80 exposes its HBM stacks through the linker's HBM connectivity tags. -# A single wide master fanned across the channels avoids the per-bank base -# address problem: the linker assigns each memory range independently, and -# those bases are not known at synthesis time. -override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -override CONFIGS += -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 -override CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - -# Device-memory aperture base. The V80 maps the AFU's m_axi_mem_0 port at -# 0x40_0000_0000 in both build flavours -- simulation puts the BRAM model there -# (vbin.prj/run_pre.tcl) and hardware puts HBM_AXI_00 there (slash's generated -# address map). Vortex's device allocator is based at VX_MEM_USER_BASE_ADDR -# (0x10000) and a 32-bit core cannot emit the aperture base itself, so the AFU -# wrapper rebases every device access -- from the cores and from the CP -- by -# this synthesis-time offset. Without it nothing decodes: writes still collect -# a BRESP from the interconnect while the data goes nowhere. -# -# Sized Verilog literal, NOT decimal. A decimal constant this large is silently -# truncated: "WARNING: [VRFC 10-8884] decimal constant 274877906944 should be -# smaller than 2147483648; using 0 instead". -# -# Tied to MEM_TAG=HBM0; a different tag lands on a different aperture base. -override CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 - -# Device-memory connectivity. NOT HBM0: that tag is a single HBM_AXI channel, -# which the compute shell maps as 1 GB at 0x40_0000_0000. Vortex's 32-bit memory -# map puts the stack at the top of a 4 GB space -- vx_start.S sets sp to -# VX_MEM_STACK_BASE_ADDR (0xFFFF0000) and grows down, and that lands just below -# the local-memory window (VX_lsu_slice.sv decodes LMEM *upward* from the same -# base), so it is a global-memory access. Rebased it becomes 0x40_FFFE_Fxxx, -# which a 1 GB aperture cannot answer: the read gets no response, the LSU stalls -# forever, and the AFU never goes idle -- taking the shell's AXI path with it. -# -# The MEM tag routes through the HBM VNOC, which the shell maps at the same -# 0x40_0000_0000 base with 32 GB of range, so it covers the whole 32-bit space -# and PLATFORM_MEMORY_OFFSET above stays correct unchanged. -# -# Set before the Makefile's `MEM_TAG ?= HBM0`, which is why plain `=` suffices. -MEM_TAG = MEM - -# Host-memory connectivity for the CP's command-ring master. NOT HOST: that tag -# routes m_axi_host to the QDMA slave bridge, and on this compute shell a read -# through that bridge never returns a response at all. The CP's very first ring -# fetch then hangs with no error to show for it -- the queue reads back armed -# and correct (Q_CONTROL=0x1, Q_TAIL=0x40, Q_ERROR=0) while head stays at 0 -# forever, and the next process's reset is refused because that read is still -# genuinely outstanding. -# -# HBM1 routes the port to a second HBM aperture instead, and the runtime stages -# the ring there and publishes it on the doorbell. That path works. -# -# This default was carried on the command line for one build and lost on the -# next, which cost a bitstream and several hours of hardware debugging. It -# belongs here next to MEM_TAG, for the same reason and by the same mechanism: -# stated before the Makefile's `HOST_TAG ?= HOST`, so plain `=` suffices. -# The aperture must lie OUTSIDE the device-memory window. Each HBM_AXI port -# addresses its own 512 MB slice of the stack, HBM at 0x40_0000_0000 + -# k*512 MB, and the MEM tag above exposes Vortex's whole 32-bit space from that -# same base -- so HBM0..HBM7 ARE device memory. With HBM1, VRT placed the CP's -# staging buffers at +512 MB, inside the heap: any scene whose buffers grew past -# 512 MB was overwritten by its own upload (wrong images; corrupted BVHs that -# traversal never finished). HBM8 starts at +4 GB, past the last device byte. -HOST_TAG = HBM8 - -# Kernel clock target (MHz). The linker also accepts a frequency request; -# the runtime can retune within the platform's supported range. -KERNEL_FREQ ?= 300 diff --git a/hw/syn/xilinx/aved/pre_opt_hook.tcl b/hw/syn/xilinx/aved/pre_opt_hook.tcl index e1ec4d450f..b79091ad28 100644 --- a/hw/syn/xilinx/aved/pre_opt_hook.tcl +++ b/hw/syn/xilinx/aved/pre_opt_hook.tcl @@ -1,13 +1,5 @@ # Runs on impl_1 immediately before opt_design. # -# VX_async_ram_patch emits VX_placeholder black boxes wherever an -# asynchronous-read RAM has to be realized on a synchronous BRAM. Their -# contents are filled in here, from the synthesized netlist. Without this, -# opt_design aborts with "DRC INBB-3 Black Box Instances ... has undefined -# contents" -- but only once a configuration is large enough to push those -# RAMs out of distributed LUTRAM and into BRAM, so small builds pass and -# large ones do not. -# # Note: the xrt and dut flows source xilinx_slr_pblocks.tcl here, which SPREADS # a large design across SLRs to relieve congestion. That is deliberately still # omitted -- this AFU sits in a DFX reconfigurable partition and does not have @@ -18,7 +10,4 @@ # pre_synth_hook.tcl for the numbers. The SLR split those attempts were # chasing is fixed shell-side, in slash_base.tcl. -set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl - report_utilization -file hier_utilization.rpt -hierarchical -hierarchical_percentages diff --git a/hw/syn/xilinx/aved/pre_synth_hook.tcl b/hw/syn/xilinx/aved/pre_synth_hook.tcl index ccea1e8766..fdacefb3e5 100644 --- a/hw/syn/xilinx/aved/pre_synth_hook.tcl +++ b/hw/syn/xilinx/aved/pre_synth_hook.tcl @@ -10,11 +10,6 @@ if {![file exists $pre_opt]} { error "pre_opt_hook.tcl not found next to pre_synth_hook.tcl: $pre_opt" } -# TOOL_DIR is read by the hook itself; fail here rather than mid-implementation. -if {![info exists ::env(TOOL_DIR)]} { - error "TOOL_DIR is not set -- the pre-opt hook needs it to locate hw/scripts" -} - set_property STEPS.OPT_DESIGN.TCL.PRE $pre_opt [get_runs impl_1] puts "INFO: attached pre-opt hook to impl_1: $pre_opt" diff --git a/hw/syn/xilinx/common.mk b/hw/syn/xilinx/common.mk new file mode 100644 index 0000000000..0a10d046ae --- /dev/null +++ b/hw/syn/xilinx/common.mk @@ -0,0 +1,129 @@ +# Shared by the Xilinx flows (xrt, aved): configuration, RTL sources and the +# source-generation rules. The including flow sets BUILD_DIR, BIN_DIR, AFU_DIR +# and any TARGET-specific CONFIGS before including this, and includes +# platform.mk after it. + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi +AFU_COMMON_DIR := $(RTL_DIR)/afu/common +SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts + +VIVADO := $(XILINX_VIVADO)/bin/vivado + +CP = cp -rf +RMDIR = rm -rf +ECHO = @echo + +MAX_JOBS ?= 8 +NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') +JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) + +# Control RTL debug tracing states +DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE +DBG_TRACE_FLAGS += -DDBG_TRACE_MEM +DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE +DBG_TRACE_FLAGS += -DDBG_TRACE_AFU +DBG_TRACE_FLAGS += -DDBG_TRACE_TEX +DBG_TRACE_FLAGS += -DDBG_TRACE_RASTER +DBG_TRACE_FLAGS += -DDBG_TRACE_OM +DBG_TRACE_FLAGS += -DDBG_TRACE_GBAR +DBG_TRACE_FLAGS += -DDBG_TRACE_TCU + +# Control logic analyzer monitors +DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU +DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE +DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH +DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU + +ifdef NUM_CORES +# cluster configuration +CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 +CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 +CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 +CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 +CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 +CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 +CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 +override CONFIGS += $(CONFIGS_$(NUM_CORES)c) +endif + +# `override` throughout: a command-line `make CONFIGS=...` otherwise discards +# these appends, and a build silently missing -DSYNTHESIS elaborates the +# simulation-only paths. +override CONFIGS += -DSYNTHESIS -DVIVADO + +XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') + +# include sources +RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv +FPU_INCLUDE = -I$(RTL_DIR)/fpu +ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) + RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv + FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src +endif +RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) +RTL_INCLUDE += $(FPU_INCLUDE) + +# Command Processor sources. RTL_PKGS holds packages only; the CP +# interface files are auto-discovered via the -I path below. +RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +RTL_INCLUDE += -I$(RTL_DIR)/cp + +# Per-extension RTL sources (TCU/DXA/V/RTU/TEX/RASTER/OM) — +# single source of truth, appended based on the resolved XCONFIGS. +include $(VORTEX_HOME)/hw/syn/extensions.mk + +# Enable perf counters +ifdef PERF + CFLAGS += -DPERF_ENABLE +endif + +# Enable scope analyzer +ifdef SCOPE + CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) + SCOPE_JSON += $(BIN_DIR)/scope.json +endif + +# compilation flags (the flow adds its DEBUG/NDEBUG flags) +COMPILE_CFLAGS = -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) +# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the +# resolved type selectors so an enum override selects its RTL implementation +# (without this, VX_tcu_fedp elaborates 'no_implementation_selected'). +COMPILE_CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +COMPILE_CFLAGS += $(RTL_INCLUDE) + +# ast dump flags +XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI + +# Fail before synthesis if any 16-byte block in the CP AXI-Lite regfile is only +# partially decoded, on cards whose shell resolves AXI-Lite in blocks (the +# descriptor's axil_block_bytes). Measured on V80 silicon: such a block DECERRs +# on EVERY word, including the words that ARE implemented. No simulator +# reproduces it; the symptom is a register reading 0xFFFFFFFF on hardware for no +# reason visible in the RTL. +check-axil-blocks: + $(if $(filter-out 0,$(PLATFORM_AXIL_BLOCK_BYTES)),@python3 $(SCRIPT_DIR)/check_axil_blocks.py $(RTL_DIR)/cp/VX_cp_axil_regfile.sv) + +# Vivado reads the preprocessed sources (gen_sources.sh -P), so its own +# inference sees the RTL as written; the Xilinx floating-point cores bind to +# the cores xilinx_ip_gen.tcl generates. +gen-sources: $(BUILD_DIR)/sources.txt +$(BUILD_DIR)/sources.txt: $(BUILD_DIR)/afu_mem_ports.stamp + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P -F -Tvortex_afu $(CFLAGS) -Csrc -Osources.txt + +# The preprocessed sources bake in the kernel's memory port count (the flow's +# AFU_MEM_PORTS): regenerate them whenever the configuration changes it. +$(BUILD_DIR)/afu_mem_ports.stamp: force-afu-mem-ports + @mkdir -p $(BUILD_DIR) + @if [ "$$(cat $@ 2>/dev/null)" != "$(AFU_MEM_PORTS)" ]; then echo "$(AFU_MEM_PORTS)" > $@; fi + +gen-ast: $(BUILD_DIR)/vortex.xml +$(BUILD_DIR)/vortex.xml: + mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml + +scope-json: $(BIN_DIR)/scope.json +$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml + mkdir -p $(BIN_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json + +.PHONY: check-axil-blocks gen-sources gen-ast scope-json force-afu-mem-ports +force-afu-mem-ports: diff --git a/hw/syn/xilinx/dut/Makefile b/hw/syn/xilinx/dut/Makefile index 802263a1ad..54188f0227 100644 --- a/hw/syn/xilinx/dut/Makefile +++ b/hw/syn/xilinx/dut/Makefile @@ -23,6 +23,12 @@ $(DUTS): cp build.mk $(PREFIX)_$@/Makefile $(MAKE) -C $(PREFIX)_$@ DUT=$@ clean && $(MAKE) -C $(PREFIX)_$@ DUT=$@ > $(PREFIX)_$@/build.log 2>&1 & +# Beacon on one DUT, in the foreground: make tcu-beacon +%-beacon: + mkdir -p $(PREFIX)_$* + cp build.mk $(PREFIX)_$*/Makefile + $(MAKE) -C $(PREFIX)_$* DUT=$* clean beacon + # Re-run power analysis on an existing build without rebuilding the design. # Usage: make -power SAIF_FILE= [SAIF_INST=] %-power: diff --git a/hw/syn/xilinx/dut/build.mk b/hw/syn/xilinx/dut/build.mk index 878af652d0..dbff6f5ae3 100644 --- a/hw/syn/xilinx/dut/build.mk +++ b/hw/syn/xilinx/dut/build.mk @@ -32,3 +32,14 @@ RTL_PKGS += $($(DUT)_PKG) ifeq ($($(DUT)_EXT),1) include $(VORTEX_HOME)/hw/syn/extensions.mk endif + +# An AFU build reads the platform tops too, which declare one AXI master per +# GPU memory port literally: resolve the count from the elaborated +# configuration, as the xrt flow does. +ifeq ($($(DUT)_AFU),1) +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c .afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J% -DSYNTHESIS -DVIVADO,$(CFLAGS)) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) +endif diff --git a/hw/syn/xilinx/dut/catalog.mk b/hw/syn/xilinx/dut/catalog.mk index 9a2cde4690..172de1c912 100644 --- a/hw/syn/xilinx/dut/catalog.mk +++ b/hw/syn/xilinx/dut/catalog.mk @@ -10,8 +10,8 @@ # Add a DUT: append its name to DUTS and define the five variables. No new # directory, no dispatcher edit. -DUTS := cache core cp dxa fpu gfx issue lmem mem_unit mufu om raster rtu \ - scope tcu tensor tex top unittest vm vortex +DUTS := cache core cp dxa fpu gfx gfxvm issue lmem mem_unit mufu om raster rtu \ + scope tcu tensor tex top vm vortex # --------------------------------------------------------------------------- # shared include fragments @@ -82,13 +82,21 @@ gfx_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/cache -I$ gfx_PKG = $(RTL_DIR)/fpu/VX_fpu_pkg.sv $(RTL_DIR)/dxa/VX_dxa_pkg.sv $(RTL_DIR)/tex/VX_tex_pkg.sv \ $(RTL_DIR)/rtu/VX_rtu_pkg.sv $(RTL_DIR)/raster/VX_raster_pkg.sv $(RTL_DIR)/om/VX_om_pkg.sv +# The graphics slice under VM: the unit MMUs between the graphics caches / DXA +# and memory, which the gfx DUT (VM off) and the vm DUT (no units) never build. +gfxvm_PRJ := VX_gfx_top +gfxvm_IP := 1 +gfxvm_CFG := $(gfx_CFG) -DVX_CFG_VM_ENABLE +gfxvm_INC = $(gfx_INC) +gfxvm_PKG = $(gfx_PKG) $(RTL_DIR)/vm/VX_tlb_pkg.sv + # NOTE: the original issue/Makefile listed $(FPU_INCLUDE) twice; a duplicate -I # is a no-op, so it is listed once here. issue_PRJ := VX_issue_top issue_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm $(FPU_INC) -I$(UNITTEST_DIR)/issue -lmem_PRJ := VX_local_mem_top -lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(UNITTEST_DIR)/local_mem +lmem_PRJ := VX_lmem_unit_top +lmem_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(UNITTEST_DIR)/lmem_unit mem_unit_PRJ := VX_mem_unit_top mem_unit_INC = $(BASE_INC) -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/core -I$(RTL_DIR)/fpu -I$(UNITTEST_DIR)/mem_unit @@ -138,12 +146,10 @@ tex_PKG = $(RTL_DIR)/tex/VX_tex_pkg.sv top_PRJ := VX_afu_wrap top_IP := 1 top_EXT := 1 +top_AFU := 1 top_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache \ -I$(RTL_DIR)/cp -I$(AFU_DIR) -I$(AFU_COMMON_DIR) $(FPU_INC) -unittest_PRJ := Unittest -unittest_INC = $(BASE_INC) -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache $(FPU_INC) - vm_PRJ := VX_vm_top vm_CFG := -DVX_CFG_VM_ENABLE -DVX_CFG_EXT_A_ENABLE -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 \ diff --git a/hw/syn/xilinx/dut/common.mk b/hw/syn/xilinx/dut/common.mk index e76070a987..fe8a851365 100644 --- a/hw/syn/xilinx/dut/common.mk +++ b/hw/syn/xilinx/dut/common.mk @@ -1,8 +1,8 @@ ROOT_DIR := $(realpath ../../../../..) include $(ROOT_DIR)/config.mk -# Exports VERILATOR/VERILATOR_PATH as absolute paths. gen_sources.sh resolves -# verilator from the environment, so without this the flow silently depends on -# verilator being on PATH and fails with "verilator: command not found". +# Exports CIRCT_VERILOG as an absolute path. gen_sources.sh resolves its +# preprocessor from the environment, so without this the flow silently depends +# on circt-verilog being on PATH. include $(VORTEX_HOME)/hw/syn/common.mk DEVICE ?= xcu55c-fsvh2892-2L-e @@ -60,7 +60,7 @@ all: $(PROJECT).xpr gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Cproject_1/src -Oproject_1/sources.txt $(RTL_PKGS) build: $(PROJECT).xpr $(PROJECT).xpr: project_1/sources.txt @@ -83,9 +83,45 @@ power: TOOL_DIR=$(SCRIPT_DIR) SAIF_FILE=$(SAIF_FILE) SAIF_INST=$(SAIF_INST) \ $(VIVADO) -mode batch -source $(SCRIPT_DIR)/xilinx_power_analysis.tcl +# Beacon (foldsim-estimate-timing, foldsim-estimate-area): the logic levels of +# every register-to-register path of the design, counted as Vivado counts them +# on DEVICE's family (hw/syn/xilinx/platforms), by RTL unit, and its LUTs, +# flip-flops, distributed and block RAM and DSPs, in seconds and without +# Vivado. It elaborates the same sources and defines as gen-sources into a +# CIRCT netlist of its own, with the Xilinx floating-point cores as black +# boxes, mapped once for depth and once for area. ci/logic_gate.py gates +# beacon_summary.csv and beacon_units.json; beacon_paths.txt has every path +# of at least BEACON_PATH_LEVELS (beacon_summary.py's watch threshold), op by +# op, by unit, and beacon_area.txt the area by unit and where each memory +# goes. +BEACON_PATH_LEVELS ?= 8 +# The Beacon target: the family of the device, as hw/syn/xilinx/platforms names it. +BEACON_TARGET = $(shell python3 $(SCRIPT_DIR)/gen_platform.py --device=$(DEVICE) --emit=family) +# Modules whose interfaces must be registered (space separated): their +# instances keep their ports through mapping, and beacon/ports.csv counts, per +# output port, the bits a combinational path from outside the instance +# reaches (`through`). beacon_summary.py records them for logic_gate. +BEACON_INTERFACES ?= +beacon_comma := , +beacon_empty := +beacon_space := $(beacon_empty) $(beacon_empty) +beacon_keep = $(subst $(beacon_space),$(beacon_comma),$(strip $(BEACON_INTERFACES))) +BEACON_MAP = $(if $(beacon_keep),foldsim-keep-modules{modules=$(beacon_keep)}$(beacon_comma))foldsim-beacon-map{target=$(BEACON_TARGET)} +BEACON_PORTS = $(if $(beacon_keep),ports=beacon/ports.csv port-modules=$(beacon_keep)) +beacon: + mkdir -p beacon + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -T$(TOP_LEVEL_ENTITY) -P -F -Cbeacon/src -Obeacon/sources.f $(RTL_PKGS) + $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t $(TOP_LEVEL_ENTITY) -l beacon/sources.f \ + -x $(SCRIPT_DIR)/xilinx_ip_stubs.sv -r beacon/memories.rpt -o beacon/netlist.mlir + $(FOLDSIM_OPT) beacon/netlist.mlir --pass-pipeline='builtin.module($(BEACON_MAP))' \ + --emit-bytecode -o beacon/mapped.mlirbc + $(FOLDSIM_OPT) beacon/mapped.mlirbc -o /dev/null --pass-pipeline="builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/endpoints.csv instances=beacon/instances.csv report=beacon/report.txt paths=beacon/paths.jsonl path-levels=$(BEACON_PATH_LEVELS) $(BEACON_PORTS)})" + $(FOLDSIM_OPT) beacon/netlist.mlir -o /dev/null --pass-pipeline="builtin.module(foldsim-beacon-map{target=$(BEACON_TARGET) area},foldsim-estimate-area{top=$(TOP_LEVEL_ENTITY) target=$(BEACON_TARGET) output=beacon/area.csv memories=beacon/area_memories.csv report=beacon/area.txt})" + python3 $(SCRIPT_DIR)/beacon_summary.py beacon --top $(TOP_LEVEL_ENTITY) + clean: ifndef RESUME - rm -rf project_1 + rm -rf project_1 beacon beacon_summary.csv beacon_units.json beacon_paths.txt beacon_area.txt rm -rf .Xil rm -f *.rpt rm -f *.log @@ -95,4 +131,4 @@ else @echo "RESUME is defined, skipping clean." endif -.PHONY: all gen-sources build clean power +.PHONY: all gen-sources build beacon clean power diff --git a/hw/syn/xilinx/dut/pre_opt_hook.tcl b/hw/syn/xilinx/dut/pre_opt_hook.tcl index 0bb8c7bbb7..b4405850c3 100644 --- a/hw/syn/xilinx/dut/pre_opt_hook.tcl +++ b/hw/syn/xilinx/dut/pre_opt_hook.tcl @@ -1,3 +1,2 @@ set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl source ${tool_dir}/xilinx_slr_pblocks.tcl \ No newline at end of file diff --git a/hw/syn/xilinx/platform.mk b/hw/syn/xilinx/platform.mk new file mode 100644 index 0000000000..a2434a3056 --- /dev/null +++ b/hw/syn/xilinx/platform.mk @@ -0,0 +1,19 @@ +# Card facts for the Xilinx flows, resolved from hw/syn/xilinx/platforms/.toml. +# +# The including flow sets PLATFORM_GEN_ARGS (how it names the card) and +# BUILD_DIR before including this. $(shell) discards exit codes, so the +# generator's status is captured explicitly: a descriptor error must stop the +# build here, not surface later as a design built for the wrong card. + +PLATFORM_MK := $(BUILD_DIR)/platform.mk +PLATFORM_GEN := python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py $(PLATFORM_GEN_ARGS) + +PLATFORM_GEN_RC := $(shell mkdir -p $(BUILD_DIR) && $(PLATFORM_GEN) --emit=mk -o $(PLATFORM_MK) 2> $(PLATFORM_MK).err; echo $$?) +ifneq ($(PLATFORM_GEN_RC),0) +$(error $(shell cat $(PLATFORM_MK).err)) +endif +include $(PLATFORM_MK) + +ifneq ($(filter $(HOST_TAG),$(PLATFORM_HOST_REFUSE)),) +$(error HOST_TAG=$(HOST_TAG) is refused on card $(PLATFORM_CARD): $(PLATFORM_HOST_REFUSE_WHY)) +endif diff --git a/hw/syn/xilinx/platforms/u200.toml b/hw/syn/xilinx/platforms/u200.toml new file mode 100644 index 0000000000..be36aa2471 --- /dev/null +++ b/hw/syn/xilinx/platforms/u200.toml @@ -0,0 +1,22 @@ +# AMD Alveo U200: 64 GB DDR4 in four banks of 16 GB. +# +# KNOWN INCOMPLETE: the DDR banks' device base (PLATFORM_MEMORY_OFFSET) is not +# set; confirming it needs the U200 platform's memory map (platforminfo / +# xclbinutil). +[card] +name = "u200" +flow = "xrt" +family = "virtexuplus" +part = "xcu200-fsgd2104-2-e" +match_xsa = ["xilinx_u200_*"] + +[memory] +banks = 4 +addr_width = 36 +tag = "DDR" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u250.toml b/hw/syn/xilinx/platforms/u250.toml new file mode 100644 index 0000000000..10fb0715f9 --- /dev/null +++ b/hw/syn/xilinx/platforms/u250.toml @@ -0,0 +1,20 @@ +# AMD Alveo U250: 64 GB DDR4 in four banks. Only bank 0 is bound: multi-bank +# needs per-bank device-address offsets that are not known at synthesis time. +[card] +name = "u250" +flow = "xrt" +family = "virtexuplus" +part = "xcu250-figd2104-2L-e" +match_xsa = ["xilinx_u250_*"] + +[memory] +banks = 1 +addr_width = 34 +tag = "DDR" +base = 0x4000000000 + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u280.toml b/hw/syn/xilinx/platforms/u280.toml new file mode 100644 index 0000000000..6d1807d60c --- /dev/null +++ b/hw/syn/xilinx/platforms/u280.toml @@ -0,0 +1,19 @@ +# AMD Alveo U280: 8 GB HBM2, 32 pseudo-channels of 256 MB (plus DDR4, unused). +[card] +name = "u280" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu280-fsvh2892-2L-e" +match_xsa = ["xilinx_u280_*"] + +[memory] +banks = 32 +addr_width = 33 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u50.toml b/hw/syn/xilinx/platforms/u50.toml new file mode 100644 index 0000000000..1b2a2b2581 --- /dev/null +++ b/hw/syn/xilinx/platforms/u50.toml @@ -0,0 +1,19 @@ +# AMD Alveo U50: 8 GB HBM2, 32 pseudo-channels of 256 MB. +[card] +name = "u50" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu50-fsvh2104-2-e" +match_xsa = ["xilinx_u50_*"] + +[memory] +banks = 32 +addr_width = 33 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/u55c.toml b/hw/syn/xilinx/platforms/u55c.toml new file mode 100644 index 0000000000..0578213bd6 --- /dev/null +++ b/hw/syn/xilinx/platforms/u55c.toml @@ -0,0 +1,26 @@ +# AMD Alveo U55C: 16 GB HBM2, 32 pseudo-channels of 512 MB. +[card] +name = "u55c" +flow = "xrt" +family = "virtexuplusHBM" +part = "xcu55c-fsvh2892-2L-e" +match_xsa = ["xilinx_u55c_*"] + +[memory] +banks = 32 +addr_width = 34 +# The AFU memory masters split the channels into contiguous groups. +tag = "HBM" + +[host] +# The CP's m_axi_host reaches host DRAM through the XDMA slave bridge. +tag = "HOST[0]" + +[clock] +kernel_mhz = 300 + +[firesim] +platform = "vitis" +device = "xilinx_u55c_gen3x16_xdma_3_202210_1" +# 32-bit target spans 4 GiB: eight 512 MB pseudo-channels. +host_mem_sp = "HBM[0:7]" diff --git a/hw/syn/xilinx/platforms/v80.toml b/hw/syn/xilinx/platforms/v80.toml new file mode 100644 index 0000000000..0004272258 --- /dev/null +++ b/hw/syn/xilinx/platforms/v80.toml @@ -0,0 +1,45 @@ +# AMD Alveo V80 through the SLASH compute shell. The authoritative rationale for +# each value is in docs/designs/aved_driver_architecture.md (§4, §9). +[card] +name = "v80" +flow = "aved" +family = "versal" +part = "xcv80-lsva4737-2MHP-e-S" +match_device = ["xcv80-*"] + +[memory] +# One memory channel caps the AFU at one memory master whatever the GPU's +# memory port count: the linker assigns each memory range independently, and +# per-bank bases are not known at synthesis time. +banks = 1 +addr_width = 34 +# The shell maps m_axi_mem_0 at 0x40_0000_0000 in hardware and simulation, and a +# 32-bit core cannot emit that base, so the AFU rebases every device access. +base = 0x4000000000 +# MEM, not HBM0: HBM0 is a 1 GB aperture and the stack sits just under 4 GB. MEM +# routes through the HBM VNOC at the same base with 32 GB of range. +tag = "MEM" + +[host] +# HBM8 with runtime staging. HOST is the QDMA slave bridge, whose reads never +# complete on the compute shell: the CP hangs on its first ring fetch silently. +# The aperture must lie outside the device-memory window: HBM sits at +# 0x40_0000_0000 + k*512 MB and MEM exposes the whole 32-bit space from that +# base, so HBM0..HBM7 are device memory. With HBM1 the CP's staging buffers +# landed inside the heap and uploads past 512 MB overwrote it; HBM8 starts at +# +4 GB, past the last device byte. +tag = "HBM8" +refuse = ["HOST"] +refuse_why = "m_axi_host through the QDMA slave bridge never completes reads on the compute shell; the CP would hang on its first ring fetch" + +[control] +# A 16-byte AXI-Lite block answers only if all four of its words decode. +axil_block_bytes = 16 + +[clock] +kernel_mhz = 300 + +[firesim] +platform = "aved" +device = "xcv80-lsva4737-2MHP-e-S" +host_mem_sp = "MEM" diff --git a/hw/syn/xilinx/platforms/vck5000.toml b/hw/syn/xilinx/platforms/vck5000.toml new file mode 100644 index 0000000000..3983523988 --- /dev/null +++ b/hw/syn/xilinx/platforms/vck5000.toml @@ -0,0 +1,15 @@ +# AMD VCK5000 (Versal AI Core). Device memory sits at 0xC0_0000_0000. +[card] +name = "vck5000" +flow = "xrt" +family = "versal" +part = "xcvc1902-vsvd1760-2MP-e-S" +match_xsa = ["xilinx_vck5000_*"] + +[memory] +banks = 1 +addr_width = 32 +base = 0xC000000000 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/versal.toml b/hw/syn/xilinx/platforms/versal.toml new file mode 100644 index 0000000000..f3a14d7077 --- /dev/null +++ b/hw/syn/xilinx/platforms/versal.toml @@ -0,0 +1,13 @@ +# Any other Versal XRT platform. +[card] +name = "versal" +flow = "xrt" +family = "versal" +match_family = ["versal"] + +[memory] +banks = 1 +addr_width = 32 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/platforms/zynquplus.toml b/hw/syn/xilinx/platforms/zynquplus.toml new file mode 100644 index 0000000000..2e31d6739f --- /dev/null +++ b/hw/syn/xilinx/platforms/zynquplus.toml @@ -0,0 +1,13 @@ +# Zynq UltraScale+ XRT platforms (embedded; no PCIe host). +[card] +name = "zynquplus" +flow = "xrt" +family = "zynquplus" +match_family = ["zynquplus"] + +[memory] +banks = 1 +addr_width = 32 + +[clock] +kernel_mhz = 300 diff --git a/hw/syn/xilinx/sandbox/Makefile b/hw/syn/xilinx/sandbox/Makefile index ab2cb72cd1..b82cee5f6c 100644 --- a/hw/syn/xilinx/sandbox/Makefile +++ b/hw/syn/xilinx/sandbox/Makefile @@ -68,19 +68,16 @@ $(KERNEL).bin: kernel.bin.coe: $(KERNEL).bin $(SCRIPT_DIR)/bin2coe.py --out=$@ --binfile=8192:$(KERNEL).bin --depth=16384 --wordsize=64 --little_endian -pre_opt_hook.tcl: $(SRC_DIR)/pre_opt_hook.tcl - cp $< $@ - simulate.tcl: $(SRC_DIR)/simulate.tcl cp $< $@ gen-sources: project_1/sources.txt project_1/sources.txt: mkdir -p project_1 - $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -Cproject_1/src -Oproject_1/sources.txt + $(SCRIPT_DIR)/gen_sources.sh $(CFLAGS) -P -F -TVortex_top -Cproject_1/src -Oproject_1/sources.txt build: done.dcp -done.dcp: project_1/sources.txt kernel.bin.coe project.tcl pre_opt_hook.tcl +done.dcp: project_1/sources.txt kernel.bin.coe project.tcl MAX_JOBS=$(JOBS) OPT_LEVEL=$(OPT_LEVEL) TOOL_DIR=$(SCRIPT_DIR) $(VIVADO) -mode batch -source project.tcl -tclargs $(DEVICE) project_1/sources.txt echo done > done.dcp diff --git a/hw/syn/xilinx/sandbox/pre_opt_hook.tcl b/hw/syn/xilinx/sandbox/pre_opt_hook.tcl deleted file mode 100644 index 92d1e94f97..0000000000 --- a/hw/syn/xilinx/sandbox/pre_opt_hook.tcl +++ /dev/null @@ -1,2 +0,0 @@ -set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl \ No newline at end of file diff --git a/hw/syn/xilinx/sandbox/project.tcl.in b/hw/syn/xilinx/sandbox/project.tcl.in index 430c8c85a0..2bc478902f 100644 --- a/hw/syn/xilinx/sandbox/project.tcl.in +++ b/hw/syn/xilinx/sandbox/project.tcl.in @@ -422,7 +422,7 @@ proc run_setup {} { # register compilation hooks #set_property STEPS.SYNTH_DESIGN.TCL.PRE ${script_dir}/pre_synth_hook.tcl [get_runs synth_1] #set_property STEPS.SYNTH_DESIGN.TCL.POST ${script_dir}/post_synth_hook.tcl [get_runs synth_1] - set_property STEPS.OPT_DESIGN.TCL.PRE ${script_dir}/pre_opt_hook.tcl [get_runs impl_1] + #set_property STEPS.OPT_DESIGN.TCL.PRE ${script_dir}/pre_opt_hook.tcl [get_runs impl_1] #set_property STEPS.OPT_DESIGN.TCL.POST ${script_dir}/post_opt_hook.tcl [get_runs impl_1] #set_property STEPS.POWER_OPT_DESIGN.TCL.PRE ${script_dir}/pre_power_opt_hook.tcl [get_runs impl_1] #set_property STEPS.POWER_OPT_DESIGN.TCL.POST ${script_dir}/post_power_opt_hook.tcl [get_runs impl_1] diff --git a/hw/syn/xilinx/xrt/Makefile b/hw/syn/xilinx/xrt/Makefile index 22dc8986d5..32eb40b5ce 100644 --- a/hw/syn/xilinx/xrt/Makefile +++ b/hw/syn/xilinx/xrt/Makefile @@ -1,8 +1,8 @@ ROOT_DIR := $(realpath ../../../..) include $(ROOT_DIR)/config.mk -# Exports VERILATOR/VERILATOR_PATH as absolute paths. gen_sources.sh resolves -# verilator from the environment, so without this the flow silently depends on -# verilator being on PATH and fails with "verilator: command not found". +# Exports CIRCT_VERILOG as an absolute path. gen_sources.sh resolves its +# preprocessor from the environment, so without this the flow silently depends +# on circt-verilog being on PATH. include $(VORTEX_HOME)/hw/syn/common.mk ifneq ($(findstring Makefile, $(MAKEFILE_LIST)), Makefile) @@ -23,7 +23,6 @@ PREFIX ?= build$(XLEN) SAIF_FILE ?= SAIF_INST ?= -MAX_JOBS ?= 8 # Synthesis optimization level (standardized across hw/syn): # 0 -- fastest compile, no optimization (Vitis --optimize 0) @@ -32,28 +31,14 @@ MAX_JOBS ?= 8 OPT_LEVEL ?= 3 SRC_DIR := $(VORTEX_HOME)/hw/syn/xilinx/xrt +AFU_DIR := $(VORTEX_HOME)/hw/rtl/afu/xrt -RTL_DIR := $(VORTEX_HOME)/hw/rtl -DPI_DIR := $(VORTEX_HOME)/hw/dpi -AFU_DIR := $(RTL_DIR)/afu/xrt -AFU_COMMON_DIR := $(RTL_DIR)/afu/common -SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts - -VIVADO := $(XILINX_VIVADO)/bin/vivado VPP := $(XILINX_VITIS)/bin/v++ -CP = cp -rf -RMDIR = rm -rf -ECHO = @echo - -NCPUS := $(shell lscpu | grep "^Core(s) per socket:" | awk '{print $$4}') -JOBS ?= $(shell echo $$(( $(NCPUS) > $(MAX_JOBS) ? $(MAX_JOBS) : $(NCPUS) ))) - PLATFORM_TO_XSA = $(strip $(patsubst %.xpfm, % , $(shell basename $(PLATFORM)))) XSA := $(call PLATFORM_TO_XSA, $(PLATFORM)) DEV_ARCH := $(shell platforminfo -p $(PLATFORM) | grep 'FPGA Family' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') -CPU_TYPE := $(shell platforminfo -p $(PLATFORM) | grep 'CPU Type' | sed 's/.*://' | sed '/ai_engine/d' | sed 's/^[[:space:]]*//') BUILD_DIR := $(PREFIX)_$(XSA)_$(TARGET) BIN_DIR := $(BUILD_DIR)/bin @@ -61,66 +46,16 @@ BIN_DIR := $(BUILD_DIR)/bin XO_CONTAINER := $(BIN_DIR)/vortex_afu.xo XCLBIN_CONTAINER := $(BIN_DIR)/vortex_afu.xclbin -# Control RTL debug tracing states -DBG_TRACE_FLAGS += -DDBG_TRACE_PIPELINE -DBG_TRACE_FLAGS += -DDBG_TRACE_MEM -DBG_TRACE_FLAGS += -DDBG_TRACE_CACHE -DBG_TRACE_FLAGS += -DDBG_TRACE_AFU -DBG_TRACE_FLAGS += -DDBG_TRACE_TEX -DBG_TRACE_FLAGS += -DDBG_TRACE_RASTER -DBG_TRACE_FLAGS += -DDBG_TRACE_OM -DBG_TRACE_FLAGS += -DDBG_TRACE_GBAR -DBG_TRACE_FLAGS += -DDBG_TRACE_TCU - -# Control logic analyzer monitors -DBG_SCOPE_FLAGS += -DDBG_SCOPE_AFU -DBG_SCOPE_FLAGS += -DDBG_SCOPE_ISSUE -DBG_SCOPE_FLAGS += -DDBG_SCOPE_FETCH -DBG_SCOPE_FLAGS += -DDBG_SCOPE_LSU - -ifdef NUM_CORES -# cluster configuration -CONFIGS_1c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=1 -CONFIGS_2c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=2 -CONFIGS_4c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=4 -CONFIGS_8c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=8 -CONFIGS_16c := -DVX_CFG_NUM_CLUSTERS=1 -DVX_CFG_NUM_CORES=16 -CONFIGS_32c := -DVX_CFG_NUM_CLUSTERS=2 -DVX_CFG_NUM_CORES=16 -CONFIGS_64c := -DVX_CFG_NUM_CLUSTERS=4 -DVX_CFG_NUM_CORES=16 -CONFIGS += $(CONFIGS_$(NUM_CORES)c) -endif - -CONFIGS += -DSYNTHESIS -DVIVADO - ifeq ($(TARGET), hw_emu) - CONFIGS += -DSIMULATION + override CONFIGS += -DSIMULATION endif -XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') - -# include sources -RTL_PKGS = $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/fpu/VX_fpu_pkg.sv -FPU_INCLUDE = -I$(RTL_DIR)/fpu -ifneq (,$(filter -DVX_CFG_FPU_TYPE_FPNEW, $(XCONFIGS))) - RTL_PKGS += $(THIRD_PARTY_DIR)/cvfpu/src/fpnew_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src/cf_math_pkg.sv $(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl/defs_div_sqrt_mvp.sv - FPU_INCLUDE += -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/include -J$(THIRD_PARTY_DIR)/cvfpu/src/common_cells/src -J$(THIRD_PARTY_DIR)/cvfpu/src/fpu_div_sqrt_mvp/hdl -J$(THIRD_PARTY_DIR)/cvfpu/src -endif -RTL_INCLUDE = -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(RTL_DIR)/libs -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/core -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(AFU_DIR) -I$(AFU_COMMON_DIR) -RTL_INCLUDE += $(FPU_INCLUDE) - -# Command Processor sources. RTL_PKGS holds packages only; the CP -# interface files are auto-discovered via the -I path below. -RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv -RTL_INCLUDE += -I$(RTL_DIR)/cp - -# Per-extension RTL sources (TCU/DXA/V/RTU/TEX/RASTER/OM) — -# single source of truth, appended based on the resolved XCONFIGS. -include $(VORTEX_HOME)/hw/syn/extensions.mk +.DEFAULT_GOAL := all +include $(VORTEX_HOME)/hw/syn/xilinx/common.mk -# Kernel clock target frequency (MHz). Defaults to the U55C platform native -# 300 MHz. Override on the command line (e.g. KERNEL_FREQ=250) for extra +# Kernel clock target frequency (MHz). The card descriptor supplies the +# default; override on the command line (e.g. KERNEL_FREQ=250) for extra # timing-closure margin. -KERNEL_FREQ ?= 300 # Timing-miss policy. Default (1): a design that misses KERNEL_FREQ is # auto-scaled down to the achieved frequency and the build succeeds with @@ -146,17 +81,15 @@ ifdef IMPL_STRATEGY VPP_FLAGS += --xp "vivado_prop:run.impl_1.STRATEGY=$(IMPL_STRATEGY)" endif -# load platform settings -include $(SRC_DIR)/platforms.mk +# Card settings, resolved from the platform's XSA and FPGA family. +ifneq ($(PLATFORM),) +PLATFORM_GEN_ARGS := --flow=xrt --xsa=$(XSA) --family='$(DEV_ARCH)' +include $(VORTEX_HOME)/hw/syn/xilinx/platform.mk +endif VPP_FLAGS += --report_level 2 VPP_FLAGS += --config $(SRC_DIR)/vitis.ini -# Enable perf counters -ifdef PERF - CFLAGS += -DPERF_ENABLE -endif - # Generates profile summary report ifdef PROFILE VPP_FLAGS += --profile_kernel data:all:all:all @@ -178,42 +111,36 @@ else CFLAGS += -DNDEBUG endif -# Enable scope analyzer -ifdef SCOPE - CFLAGS += -DSCOPE $(DBG_SCOPE_FLAGS) - SCOPE_JSON += $(BIN_DIR)/scope.json -endif - -# compilation flags -CFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) -# raw CONFIGS carries only an enum *value* (e.g. VX_CFG_TCU_TYPE=TFR); append the -# resolved type selectors so an enum override selects its RTL implementation. -CFLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) -CFLAGS += $(RTL_INCLUDE) +CFLAGS += $(COMPILE_CFLAGS) -# ast dump flags -XML_CFLAGS = $(filter-out -DSYNTHESIS -DVIVADO, $(CFLAGS)) $(RTL_PKGS) -I$(DPI_DIR) -DSV_DPI +# One AXI memory master per GPU memory port. The kernel top must declare its +# ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(BUILD_DIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator $(filter-out -J%,$(XML_CFLAGS))) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + +# Port i owns the i-th contiguous 1/N of device memory (VX_afu_wrap). Bind it +# to the matching contiguous group of the platform's memory channels, so each +# port reaches only its own channels and no two ports share one. +ifdef PLATFORM_MEMORY_TAG +VPP_FLAGS += $(shell n=$(AFU_MEM_PORTS); g=$$(($(PLATFORM_MEMORY_CHANNELS) / n)); \ + for i in $$(seq 0 $$((n - 1))); do \ + lo=$$((i * g)); hi=$$((lo + g - 1)); \ + if [ $$g -eq 1 ]; then r=$$lo; else r=$$lo:$$hi; fi; \ + echo "--connectivity.sp vortex_afu_1.m_axi_mem_$$i:$(PLATFORM_MEMORY_TAG)[$$r]"; \ + done) +endif # RTL Kernel only supports Hardware and Hardware Emulation. ifneq ($(TARGET),$(findstring $(TARGET), hw hw_emu)) $(warning WARNING:Application supports only hw hw_emu TARGET. Please use the target for running the application) endif -.PHONY: all clean gen-sources gen-ast emconfig check-devices power - -all: check-devices emconfig $(XCLBIN_CONTAINER) report - -gen-sources: $(BUILD_DIR)/sources.txt -$(BUILD_DIR)/sources.txt: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -Csrc -Osources.txt - -gen-ast: $(BUILD_DIR)/vortex.xml -$(BUILD_DIR)/vortex.xml: - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(VERILATOR_PATH)/bin/verilator --xml-only -O0 $(XML_CFLAGS) vortex_afu.v --xml-output vortex.xml +.PHONY: all clean emconfig check-devices power -scope-json: $(BIN_DIR)/scope.json -$(BIN_DIR)/scope.json: $(BUILD_DIR)/vortex.xml - mkdir -p $(BUILD_DIR); cd $(BUILD_DIR); $(SCRIPT_DIR)/scope.py vortex.xml -o bin/scope.json +all: check-devices check-axil-blocks emconfig $(XCLBIN_CONTAINER) report gen-xo: $(XO_CONTAINER) $(XO_CONTAINER): $(BUILD_DIR)/sources.txt diff --git a/hw/syn/xilinx/xrt/package_kernel.tcl b/hw/syn/xilinx/xrt/package_kernel.tcl index 2f97354835..d9b9777c72 100644 --- a/hw/syn/xilinx/xrt/package_kernel.tcl +++ b/hw/syn/xilinx/xrt/package_kernel.tcl @@ -45,8 +45,7 @@ set vdefines_list [lindex $vlist 2] #puts ${vdefines_list} set chipscope 0 -set num_banks 1 -set merged_mem_if 0 +set afu_mem_ports -1 # parse vdefines_list for configuration parameters foreach def $vdefines_list { @@ -55,16 +54,9 @@ foreach def $vdefines_list { if { $name == "CHIPSCOPE" } { set chipscope 1 } - if { $name == "PLATFORM_MEMORY_NUM_BANKS" } { - set num_banks [lindex $fields 1] + if { $name == "AFU_MEM_PORTS" } { + set afu_mem_ports [lindex $fields 1] } - if { $name == "PLATFORM_MERGED_MEMORY_INTERFACE" } { - set merged_mem_if 1 - } -} - -if { $merged_mem_if == 1 } { - set num_banks 1 } create_project -force kernel_pack $path_to_tmp_project @@ -177,6 +169,14 @@ foreach m $axi_masters { ipx::associate_bus_interfaces -busif $m -clock ap_clk $core } +# The build binds AFU_MEM_PORTS memory masters to device memory; a kernel +# with any other count would leave ports unbound or bind names it lacks. +set num_mem_masters [llength [lsearch -all -glob $axi_masters "m_axi_mem_*"]] +if { $num_mem_masters != $afu_mem_ports } { + puts "ERROR: packaged kernel has $num_mem_masters m_axi_mem masters, but AFU_MEM_PORTS=$afu_mem_ports" + exit 1 +} + set mem_map [::ipx::add_memory_map -quiet "s_axi_ctrl" $core] set addr_block [::ipx::add_address_block -quiet "reg0" $mem_map] diff --git a/hw/syn/xilinx/xrt/platforms.mk b/hw/syn/xilinx/xrt/platforms.mk deleted file mode 100644 index 7b4356bfe0..0000000000 --- a/hw/syn/xilinx/xrt/platforms.mk +++ /dev/null @@ -1,48 +0,0 @@ -# Platform specific configurations -# Add your platform specific configurations here - -CONFIGS += -DPLATFORM_MEMORY_DATA_WIDTH=512 - -ifeq ($(DEV_ARCH), zynquplus) -# zynquplus -CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -else ifeq ($(DEV_ARCH), versal) -# versal -CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -ifneq ($(findstring xilinx_vck5000,$(XSA)),) - CONFIGS += -DPLATFORM_MEMORY_OFFSET=40'hC000000000 -endif -else -# alveo -# The Command Processor's host-memory master (m_axi_host) reaches host DRAM -# through the platform slave-bridge / Host Memory Access aperture. All Alveo -# XDMA shells expose this as the HOST[0] connectivity tag. -VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_host:HOST[0] -ifneq ($(findstring xilinx_u55c,$(XSA)),) - # 16 GB of HBM2 with 32 channels (512 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] - #VPP_FLAGS += $(foreach i,$(shell seq 0 31), --connectivity.sp vortex_afu_1.m_axi_mem_$(i):HBM[$(i)]) -else ifneq ($(findstring xilinx_u50,$(XSA)),) - # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - CONFIGS += -DPLATFORM_MERGED_MEMORY_INTERFACE - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] -else ifneq ($(findstring xilinx_u280,$(XSA)),) - # 8 GB of HBM2 with 32 channels (256 MB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=32 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=33 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:HBM[0:31] -else ifneq ($(findstring xilinx_u250,$(XSA)),) - # 16 GB of DDR4 (single channel, bank 0). Multi-bank requires per-bank XRT - # VA offsets that aren't known at synthesis time. - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=34 - VPP_FLAGS += --connectivity.sp vortex_afu_1.m_axi_mem_0:DDR[0] - CONFIGS += -DPLATFORM_MEMORY_OFFSET=40\'h4000000000 -else ifneq ($(findstring xilinx_u200,$(XSA)),) - # 64 GB of DDR4 with 4 channels (16 GB per channel) - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=4 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=36 -else - CONFIGS += -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 -DVX_CFG_PLATFORM_MEMORY_ADDR_WIDTH=32 -endif -endif diff --git a/hw/syn/xilinx/xrt/pre_opt_hook.tcl b/hw/syn/xilinx/xrt/pre_opt_hook.tcl index 7f1ad925f1..af9d25dffd 100644 --- a/hw/syn/xilinx/xrt/pre_opt_hook.tcl +++ b/hw/syn/xilinx/xrt/pre_opt_hook.tcl @@ -1,5 +1,4 @@ set tool_dir $::env(TOOL_DIR) -source ${tool_dir}/xilinx_async_bram_patch.tcl source ${tool_dir}/xilinx_slr_pblocks.tcl report_utilization -file hier_utilization.rpt -hierarchical -hierarchical_percentages diff --git a/hw/syn/yosys/Makefile b/hw/syn/yosys/Makefile index b3c8a6c225..11cf88bd11 100644 --- a/hw/syn/yosys/Makefile +++ b/hw/syn/yosys/Makefile @@ -1,6 +1,6 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -include $(ROOT_DIR)/hw/syn/common.mk # SV2V_PATH/YOSYS_PATH/STA_PATH + exported SV2V/YOSYS/STA +include $(ROOT_DIR)/hw/syn/common.mk # CIRCT/YOSYS/STA paths + exported tools DELAY_UNC ?= 0.02 DELAY_IO ?= 0.05 @@ -90,6 +90,7 @@ else BB_MODULES ?= VX_dp_ram_asic,VX_sp_ram_asic endif +comma := , SCRIPT_DIR := $(VORTEX_HOME)/hw/scripts RTL_DIR := $(VORTEX_HOME)/hw/rtl @@ -188,7 +189,7 @@ CFLAGS += $(RTL_INCLUDE) # Build targets all: timing -.PHONY: all synthesis techmap timing gate-sim gate-saif normalize-saif asap7 asap7-models clean +.PHONY: all synthesis techmap timing beacon gate-sim gate-saif normalize-saif asap7 asap7-models clean asap7: $(ASAP7_INSTALL) --install-dir=$(ASAP7_DIR) --vt=$(ASAP7_VT) @@ -201,11 +202,15 @@ $(BUILD_DIR): mkdir -p $@ $(BUILD_DIR)/src: | $(BUILD_DIR) - $(SCRIPT_DIR)/gen_sources.sh -P $(CFLAGS) -C$(BUILD_DIR)/src + $(SCRIPT_DIR)/gen_sources.sh -P -F -T$(TOP_LEVEL_ENTITY) $(CFLAGS) -C$(BUILD_DIR)/src -sv2v: $(BUILD_DIR)/project.v +# CIRCT elaborates the design into one Verilog-2005 file; the black-box +# modules (SRAM macros) stay instances with their parameter values, bound to +# their stubs in the sources. +verilog: $(BUILD_DIR)/project.v $(BUILD_DIR)/project.v: $(BUILD_DIR)/src - $(SCRIPT_DIR)/sv2v.sh -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -o$(BUILD_DIR)/project.v + $(SCRIPT_DIR)/circt_elaborate.sh -fv2005 -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ + -o$(BUILD_DIR)/project.v # Remove severity/control system-task lines yosys can't synthesize # ($fatal/$error/$warning/$info/$stop are used by the cvfpu sources). sed -ri '/^[[:space:]]*[$$](finish|display|fatal|error|warning|info|stop)/d' $(BUILD_DIR)/project.v @@ -213,6 +218,9 @@ $(BUILD_DIR)/project.v: $(BUILD_DIR)/src gen-sources: $(BUILD_DIR)/sources.txt $(BUILD_DIR)/sources.txt: $(BUILD_DIR)/project.v echo $(realpath $(BUILD_DIR)/project.v) > $(BUILD_DIR)/sources.txt + for m in $(subst $(comma), ,$(BB_MODULES)); do \ + grep -lE "^[[:space:]]*module[[:space:]]+$$m\b" $(BUILD_DIR)/src/*.v $(BUILD_DIR)/src/*.sv 2>/dev/null; \ + done | xargs -r realpath | sort -u >> $(BUILD_DIR)/sources.txt ifneq ($(strip $(DUT_FILELIST)),) SYN_SRC_FILE := $(abspath $(DUT_FILELIST)) @@ -277,6 +285,22 @@ timing: $(SYN_SRC_DEPS) $(SYN_LIB_DEPS) | $(BUILD_DIR) @$(SRC_DIR)/synth_summary.py --reports $(BUILD_DIR)/reports \ --top $(TOP_LEVEL_ENTITY) --clock-mhz $(CLOCK_FREQ) +# Beacon (foldsim-estimate-timing, foldsim-estimate-area, target=$(PDK)): the +# gate stages of every register-to-register path and the cells, flip-flops and +# SRAM macros of the design, with the same sources and SRAM black boxes as +# synthesis, in seconds and without Yosys; report.txt converts the deepest +# path to a period with the library's delays. docs/designs/beacon.md. +beacon: $(BUILD_DIR)/src + mkdir -p $(BUILD_DIR)/beacon + $(SCRIPT_DIR)/circt_elaborate.sh -f mlir -t$(TOP_LEVEL_ENTITY) -I$(BUILD_DIR)/src -b'$(BB_MODULES)' \ + -o$(BUILD_DIR)/beacon/netlist.mlir + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(PDK)})' \ + --emit-bytecode -o $(BUILD_DIR)/beacon/mapped.mlirbc + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/mapped.mlirbc -o /dev/null --pass-pipeline='builtin.module(foldsim-estimate-timing{top=$(TOP_LEVEL_ENTITY) target=$(PDK) output=$(BUILD_DIR)/beacon/endpoints.csv instances=$(BUILD_DIR)/beacon/instances.csv report=$(BUILD_DIR)/beacon/report.txt})' + $(FOLDSIM_OPT) $(BUILD_DIR)/beacon/netlist.mlir -o /dev/null --pass-pipeline='builtin.module(foldsim-beacon-map{target=$(PDK) area},foldsim-estimate-area{top=$(TOP_LEVEL_ENTITY) target=$(PDK) output=$(BUILD_DIR)/beacon/area.csv memories=$(BUILD_DIR)/beacon/area_memories.csv report=$(BUILD_DIR)/beacon/area.txt})' + @grep "^deepest path" $(BUILD_DIR)/beacon/report.txt + @head -1 $(BUILD_DIR)/beacon/area.txt + gate-sim: techmap asap7-models @if [ "$(PDK)" != "asap7" ]; then echo "ERROR: gate-sim requires PDK=asap7 functional models" >&2; exit 1; fi @if [ -z "$(TB_FILELIST)" ]; then echo "ERROR: TB_FILELIST is required for gate-sim" >&2; exit 1; fi diff --git a/hw/syn/yosys/dut/catalog.mk b/hw/syn/yosys/dut/catalog.mk index 0403566afe..2392cbf40d 100644 --- a/hw/syn/yosys/dut/catalog.mk +++ b/hw/syn/yosys/dut/catalog.mk @@ -9,7 +9,7 @@ # synthesizes, so both gates measure the same modules and a divergence between # them is meaningful rather than an artefact of two DUT lists. -DUTS := cache core tcu gfx tex raster om rtu dxa mufu vm tensor top vortex +DUTS := cache core lmem tcu gfx gfxvm tex raster om rtu dxa mufu vm tensor top vortex UNITTEST_DIR ?= $(VORTEX_HOME)/hw/unittest AFU_RTL_DIR ?= $(VORTEX_HOME)/hw/rtl/afu @@ -22,6 +22,12 @@ core_TOP := VX_core_top core_INC := -I$(UNITTEST_DIR)/core core_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_C_ENABLE -DVX_CFG_EXT_A_ENABLE +# Warp-wide local memory at the configured core shape; its gate builds set the +# shape, as the Xilinx lmem DUT's do. +lmem_TOP := VX_lmem_unit_top +lmem_INC := -I$(UNITTEST_DIR)/lmem_unit +lmem_CFG := + tcu_TOP := VX_tcu_unit_top tcu_INC := -I$(UNITTEST_DIR)/tcu_unit # EXT_F_DISABLE mirrors the Xilinx catalog: this DUT is the tensor datapath, and @@ -40,6 +46,11 @@ gfx_CFG := -DVX_CFG_EXT_TEX_ENABLE -DVX_CFG_EXT_RTU_ENABLE -DVX_CFG_EXT_RASTER_E -DVX_CFG_SOCKET_SIZE=2 -DVX_CFG_PLATFORM_MEMORY_NUM_BANKS=1 \ -DVX_CFG_L2_ENABLE -DVX_CFG_L2_SIZE=262144 +# The graphics slice under VM, as the Xilinx catalog's gfxvm. +gfxvm_TOP := VX_gfx_top +gfxvm_INC := $(gfx_INC) +gfxvm_CFG := $(gfx_CFG) -DVX_CFG_VM_ENABLE + tex_TOP := VX_tex_core_top tex_INC := -I$(UNITTEST_DIR)/tex_core tex_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TEX_ENABLE @@ -87,7 +98,7 @@ tensor_CFG := -DVX_CFG_NUM_THREADS=16 -DVX_CFG_NUM_WARPS=16 -DVX_CFG_EXT_TCU_ENA # Multi-core system top behind the AFU control/memory wrapper, as in the Xilinx # catalog: 2 cores with atomics and virtual memory. top_TOP := VX_afu_wrap -top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/xrt -I$(AFU_RTL_DIR)/common +top_INC := -I$(VORTEX_HOME)/hw/rtl/cp -I$(AFU_RTL_DIR)/common top_CFG := -DVX_CFG_NUM_THREADS=4 -DVX_CFG_NUM_WARPS=4 -DVX_CFG_NUM_CORES=2 \ -DVX_CFG_EXT_A_ENABLE -DVX_CFG_VM_ENABLE -DVX_CFG_DCACHE_LATENCY=3 diff --git a/hw/syn/yosys/liberty_stage.py b/hw/syn/yosys/liberty_stage.py new file mode 100755 index 0000000000..b1ff614ae4 --- /dev/null +++ b/hw/syn/yosys/liberty_stage.py @@ -0,0 +1,171 @@ +#!/usr/bin/env python3 +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""The delays Beacon's ASIC profile (foldsim-estimate-timing target=asap7) +converts stages with, read from a Liberty library. + +usage: liberty_stage.py [--fanout N] [--suffix _ASAP7_75t_R] + +For a representative set of gates, the delay of one stage driving `fanout` +copies of a NAND2's input, at the slew such a stage produces (iterated to a +fixed point), and the flip-flop's clock-to-output delay and setup time at +that slew. Prints one line per cell and the three numbers the asap7 profile +is built with; re-derive them when the library or its corner changes. +""" +import argparse +import bisect +import re +import sys + +GATES = ['INVx1', 'NAND2xp5', 'NOR2xp33', 'AOI22xp5', 'OAI21xp5', 'XOR2xp5', + 'MAJIxp5', 'AO21x1'] +NAND = 'NAND2xp5' +FLOP = 'DFFHQNx1' + + +def cell_block(text, name): + """The text of `cell (name) { ... }`, by brace matching.""" + m = re.search(r'\bcell\s*\(\s*"?%s"?\s*\)\s*\{' % re.escape(name), text) + if not m: + sys.exit(f'no cell {name}') + depth, i = 1, m.end() + while depth: + c = text[i] + depth += c == '{' + depth -= c == '}' + i += 1 + return text[m.end():i - 1] + + +def groups(text, kind): + """(argument, body) of each `kind (arg) { ... }` at any depth.""" + for m in re.finditer(r'\b%s\s*\(\s*"?([^")]*)"?\s*\)\s*\{' % kind, text): + depth, i = 1, m.end() + while depth: + c = text[i] + depth += c == '{' + depth -= c == '}' + i += 1 + yield m.group(1), text[m.end():i - 1] + + +def numbers(s): + return [float(x) for x in re.findall(r'-?[0-9.]+(?:e-?[0-9]+)?', s)] + + +def table(body, kind): + """(index_1, index_2, rows) of the first `kind` table in `body`.""" + for _, t in groups(body, kind): + i1 = numbers(re.search(r'index_1\s*\(\s*"([^"]*)"', t).group(1)) + m2 = re.search(r'index_2\s*\(\s*"([^"]*)"', t) + i2 = numbers(m2.group(1)) if m2 else [0.0] + vals = numbers(re.search(r'values\s*\((.*?)\)\s*;', t, re.S).group(1)) + return i1, i2, [vals[k * len(i2):(k + 1) * len(i2)] + for k in range(len(i1))] + return None + + +def interp(tab, x, y): + i1, i2, v = tab + + def bracket(idx, q): + k = min(max(bisect.bisect_left(idx, q), 1), len(idx) - 1) + lo, hi = idx[k - 1], idx[k] + return k - 1, k, (q - lo) / (hi - lo) if hi != lo else 0.0 + + if len(i2) == 1: + a, b, t = bracket(i1, x) + return v[a][0] + t * (v[b][0] - v[a][0]) + a, b, t = bracket(i1, x) + c, d, u = bracket(i2, y) + top = v[a][c] + u * (v[a][d] - v[a][c]) + bot = v[b][c] + u * (v[b][d] - v[b][c]) + return top + t * (bot - top) + + +def pin_cap(cell, pin): + for name, body in groups(cell, 'pin'): + if name == pin: + return float(re.search(r'\bcapacitance\s*:\s*([0-9.]+)', body).group(1)) + sys.exit(f'no pin {pin}') + + +def arcs(cell, out, related=None): + """Timing groups of output pin `out`, optionally for one related pin.""" + for name, body in groups(cell, 'pin'): + if name != out: + continue + for _, t in groups(body, 'timing'): + rel = re.search(r'related_pin\s*:\s*"?(\w+)"?', t).group(1) + if related is None or rel == related: + yield rel, t + + +def gate_delay(cell, out, slew, load): + """Mean of rise and fall delay and transition over every input arc.""" + d, s, n = 0.0, 0.0, 0 + for _, t in arcs(cell, out): + if 'timing_type' in t and 'combinational' not in t: + continue + for kind, tkind in (('cell_rise', 'rise_transition'), + ('cell_fall', 'fall_transition')): + dt, st = table(t, kind), table(t, tkind) + if dt and st: + d += interp(dt, slew, load) + s += interp(st, slew, load) + n += 1 + return d / n, s / n + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument('lib') + ap.add_argument('--fanout', type=float, default=3) + ap.add_argument('--suffix', default='_ASAP7_75t_R') + args = ap.parse_args() + text = open(args.lib).read() + nand = cell_block(text, NAND + args.suffix) + load = args.fanout * pin_cap(nand, 'A') + + slew = 20.0 + for _ in range(20): + delays = {} + out_slews = [] + for g in GATES: + cell = cell_block(text, g + args.suffix) + d, s = gate_delay(cell, 'Y', slew, load) + delays[g] = d + out_slews.append(s) + new = sum(out_slews) / len(out_slews) + if abs(new - slew) < 0.01: + break + slew = new + + flop = cell_block(text, FLOP + args.suffix) + clk2q = sum(interp(table(t, k), slew, load) + for _, t in arcs(flop, 'QN', 'CLK') + for k in ('cell_rise', 'cell_fall')) / 2 + setup = 0.0 + for _, t in arcs(flop, 'D', 'CLK'): + if 'setup_rising' in t: + setup = (interp(table(t, 'rise_constraint'), slew, slew) + + interp(table(t, 'fall_constraint'), slew, slew)) / 2 + for g in GATES: + print(f'{g:10s} {delays[g]:6.1f} ps') + stage = sum(delays.values()) / len(delays) + print(f'load {load:.2f} fF (fanout {args.fanout:g}), slew {slew:.1f} ps') + print(f'stage {stage:.1f} ps, clk->q {clk2q:.1f} ps, setup {setup:.1f} ps') + + +if __name__ == '__main__': + main() diff --git a/hw/syn/yosys/run_synth.sh b/hw/syn/yosys/run_synth.sh index e0df65e607..a1713ee0d6 100755 --- a/hw/syn/yosys/run_synth.sh +++ b/hw/syn/yosys/run_synth.sh @@ -231,7 +231,9 @@ log "Writing $YS" for l in "${LIB_LIST[@]}"; do printf "read_liberty -lib %q\n" "$l"; done echo "# read sources" - for s in "${SRC_FILES[@]}"; do printf "read_verilog -defer %q\n" "$s"; done + # SystemVerilog mode: the design is Verilog-2005, which it includes, and + # the black-box stubs are SystemVerilog. + for s in "${SRC_FILES[@]}"; do printf "read_verilog -sv -defer %q\n" "$s"; done if [[ -n "$BB_MODULES" ]]; then IFS=',' read -r -a bbmods <<< "$BB_MODULES" @@ -303,7 +305,7 @@ log "Writing $YS" fi # Drop what mapping left behind before anything measures or reads the # netlist. Without it the gate reports dead cells as area, and the written - # netlist carries ~129 undriven `assign w = 'hx` wires -- sv2v function + # netlist carries undriven `assign w = 'hx` wires -- frontend function # scopes that survive as debris. OpenSTA's Verilog reader rejects those # outright once Yosys marks them `signed` (0.69 does, 0.57 did not), which # is how a missing clean read as a Yosys incompatibility. diff --git a/hw/syn/yosys/sta_endpoints.tcl b/hw/syn/yosys/sta_endpoints.tcl new file mode 100644 index 0000000000..7a29f67a60 --- /dev/null +++ b/hw/syn/yosys/sta_endpoints.tcl @@ -0,0 +1,36 @@ +# Copyright © 2019-2026 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# sta_endpoints.tcl: OpenSTA, the worst setup slack at the data input of +# every flip-flop, for comparing Beacon per register with an ASIC flow (a +# CSV in the end point table format of ci/beacon.py). +# env: LIB NETLIST TOP SDC OUT +proc env {name} { return $::env($name) } +read_liberty [env LIB] +read_verilog [env NETLIST] +link_design [env TOP] +# The flow's SDC is in ns, whatever the library's time unit. +set_cmd_units -time ns -capacitance fF +read_sdc [env SDC] +set f [open [env OUT] w] +puts $f "slack,levels,logic,net,start,end" +# Flip-flops by library cell name (ASAP7: DFF*, SDF*); other cells have D +# pins too (OR4). +foreach pin [get_pins -hierarchical */D] { + set cell [get_cells -of_objects $pin] + if {![string match {[DS][DF]F*} [get_property $cell ref_name]]} continue + set slack [get_property $pin slack_max] + if {$slack eq "" || $slack eq "INF"} continue + puts $f "$slack,0,0,0,-,[get_full_name $cell]/D" +} +close $f diff --git a/hw/unittest/Makefile b/hw/unittest/Makefile index 001ab90c2a..f8e4a6ea79 100644 --- a/hw/unittest/Makefile +++ b/hw/unittest/Makefile @@ -4,7 +4,7 @@ all: $(MAKE) -C cache $(MAKE) -C core $(MAKE) -C issue - $(MAKE) -C local_mem + $(MAKE) -C lmem_unit $(MAKE) -C mem_unit $(MAKE) -C tcu_fedp $(MAKE) -C cta_dispatcher @@ -19,8 +19,11 @@ all: $(MAKE) -C cp_axil_regfile $(MAKE) -C afu_axil_demux $(MAKE) -C afu_reset_seq + $(MAKE) -C mm_axi_arb $(MAKE) -C cp_axi_path $(MAKE) -C cp_dma + $(MAKE) -C cp_llc_port + $(MAKE) -C stream_arb $(MAKE) -C cp_core $(MAKE) -C fma_unit $(MAKE) -C fdiv_unit @@ -36,7 +39,7 @@ run: $(MAKE) -C cache run $(MAKE) -C core run $(MAKE) -C issue run - $(MAKE) -C local_mem run + $(MAKE) -C lmem_unit run $(MAKE) -C mem_unit run $(MAKE) -C tcu_fedp run $(MAKE) -C cta_dispatcher run @@ -51,8 +54,11 @@ run: $(MAKE) -C cp_axil_regfile run $(MAKE) -C afu_axil_demux run $(MAKE) -C afu_reset_seq run + $(MAKE) -C mm_axi_arb run $(MAKE) -C cp_axi_path run $(MAKE) -C cp_dma run + $(MAKE) -C cp_llc_port run + $(MAKE) -C stream_arb run $(MAKE) -C cp_core run $(MAKE) -C fma_unit run $(MAKE) -C fdiv_unit run @@ -125,6 +131,50 @@ run-tcu: $(call tcu_int_pair,DPI,INT8,17,18) $(call tcu_int_pair,DPI,INT4,19,20) +# VX_tcu_unit end to end (dispatch, uops, operand routing, commit), bit for bit +# (--ulp=0) against the reference of each datapath: the native TFR one against +# the TfrFedp reference model, the DPI one against its discrete fp32 mul/add. +# One build per tile shape and datapath: 4 threads (2x2) with tf32/fp16/bf16, +# 8 threads (4x2) fp16 on DPI. +TCU_UNIT_TFR := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DVX_CFG_TCU_TF32_ENABLE +TCU_UNIT_DPI := $(TCU_BASE) -DVX_CFG_TCU_TYPE_DPI +run-tcu-unit: + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_TFR)" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_TFR)" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=1 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=2 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_TCU_TF32_ENABLE" OPTS="--fmt=3 --ulp=0 --no-trace" + $(MAKE) -C tcu_unit clean + $(MAKE) -C tcu_unit CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" + $(MAKE) -C tcu_unit run CONFIGS="$(TCU_UNIT_DPI) -DVX_CFG_NUM_THREADS=8" OPTS="--fmt=2 --ulp=0 --no-trace" + $(call tcu_mx_suite,2,1) + $(call tcu_mx_suite,4,2) + +# The native datapath's fp8 and block-scaled formats against the TfrFedp +# reference model, bit for bit: fp8/bf8, MXFP8/MXBF8 (E8M0 scales), MXFP4 (E8M0) +# and NVFP4 (E4M3). VX_tcu_unit takes its block scales from the metadata path, +# so these run on the FEDP. Two shapes: 2 registers on one scale slot, 4 on two. +# Directed legs pin zero products in an fp8 lane, MXFP4 scales far from one +# (2^-34), MXFP8/MXBF8 scales so small the lane underflows, and NaN scales. +TCU_MX := $(TCU_BASE) -DVX_CFG_TCU_TYPE_TFR -DUSE_FEDP -DVX_CFG_TCU_MX_ENABLE -DVX_CFG_TCU_FP8_ENABLE \ + -DVX_CFG_TCU_FP4_ENABLE -DVX_CFG_TCU_MXFP4_ENABLE -DVX_CFG_TCU_NVFP4_ENABLE +# format:sf_a:sf_b +TCU_MX_DIRECTED := 10:110:110 8:0:3 9:4:2 8:0xff:127 11:0x7f:0x38 +tcu_mx_opts = --fmt=$(word 1,$(subst :, ,$(1))) --sfa=$(word 2,$(subst :, ,$(1))) --sfb=$(word 3,$(subst :, ,$(1))) +# $(1)=NUM_REGS $(2)=SF +define tcu_mx_suite + $(MAKE) -C tcu_fedp clean + $(MAKE) -C tcu_fedp NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" + $(foreach f,4 5 8 9 10 11,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --no-trace" &&) true + $(foreach f,4 8 9,$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="--fmt=$(f) --W=25 --ulp=0 --features=zeros --tests=20000 --no-trace" &&) true + $(foreach o,$(TCU_MX_DIRECTED),$(MAKE) -C tcu_fedp run NUM_REGS=$(1) SF=$(2) CONFIGS="$(TCU_MX)" OPTS="$(call tcu_mx_opts,$(o)) --W=25 --ulp=0 --tests=20000 --no-trace" &&) true +endef + # Same native-TFR formats as run-tcu, but with the multipliers mapped onto DSP48 # (USE_DSP=1). USE_DSP is bit-identical to the LUT path by contract, so the same # references apply — a mismatch here is a DSP-packing bug. @@ -138,7 +188,7 @@ clean: $(MAKE) -C cache clean $(MAKE) -C core clean $(MAKE) -C issue clean - $(MAKE) -C local_mem clean + $(MAKE) -C lmem_unit clean $(MAKE) -C mem_unit clean $(MAKE) -C tcu_fedp clean $(MAKE) -C cta_dispatcher clean @@ -153,6 +203,9 @@ clean: $(MAKE) -C cp_axil_regfile clean $(MAKE) -C cp_axi_path clean $(MAKE) -C cp_dma clean + $(MAKE) -C cp_llc_port clean + $(MAKE) -C stream_arb clean + $(MAKE) -C mm_axi_arb clean $(MAKE) -C cp_core clean $(MAKE) -C fma_unit clean $(MAKE) -C fdiv_unit clean diff --git a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv index a0fbbf4d70..c7ad7530af 100644 --- a/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv +++ b/hw/unittest/afu_reset_seq/VX_afu_reset_seq_top.sv @@ -30,6 +30,7 @@ module VX_afu_reset_seq_top #( // AXI handshakes, driven directly by the testbench. input wire aw_fire, + input wire w_fire, input wire w_fire_last, input wire b_fire, input wire ar_fire, @@ -43,6 +44,18 @@ module VX_afu_reset_seq_top #( output wire gate_in_ready, output wire gate_out_valid, + // A real write gate on the same stop_req, the testbench again playing + // both the AFU-internal master (wg_in_*) and the shell (wg_out_*). + input wire wg_in_awvalid, + input wire wg_out_awready, + output wire wg_in_awready, + output wire wg_out_awvalid, + input wire wg_in_wvalid, + input wire wg_in_wlast, + input wire wg_out_wready, + output wire wg_in_wready, + output wire wg_out_wvalid, + output wire stop_req, output wire rst_assert, output wire busy, @@ -60,10 +73,26 @@ module VX_afu_reset_seq_top #( .out_ready (gate_out_ready) ); + VX_afu_wr_gate wr_gate ( + .clk (clk), + .reset (reset), + .stop_req (stop_req), + .in_awvalid (wg_in_awvalid), + .in_awready (wg_in_awready), + .out_awvalid (wg_out_awvalid), + .out_awready (wg_out_awready), + .in_wvalid (wg_in_wvalid), + .in_wlast (wg_in_wlast), + .in_wready (wg_in_wready), + .out_wvalid (wg_out_wvalid), + .out_wready (wg_out_wready) + ); + VX_afu_axi_drain drain ( .clk (clk), .reset (reset), .aw_fire (aw_fire), + .w_fire (w_fire), .w_fire_last (w_fire_last), .b_fire (b_fire), .ar_fire (ar_fire), diff --git a/hw/unittest/afu_reset_seq/main.cpp b/hw/unittest/afu_reset_seq/main.cpp index 6f303d8f3c..3656915e4c 100644 --- a/hw/unittest/afu_reset_seq/main.cpp +++ b/hw/unittest/afu_reset_seq/main.cpp @@ -35,9 +35,11 @@ void tick() { } void clear_fires() { - dut->aw_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; + dut->aw_fire = 0; dut->w_fire = 0; dut->w_fire_last = 0; dut->b_fire = 0; dut->ar_fire = 0; dut->r_fire_last = 0; dut->gate_in_valid = 0; dut->gate_out_ready = 0; + dut->wg_in_awvalid = 0; dut->wg_out_awready = 0; + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; dut->wg_out_wready = 0; } void do_reset() { @@ -203,6 +205,154 @@ int main(int argc, char** argv) { run(8); check(!dut->stop_req, "gate reopens after the sequence"); + // ------------------------------------------------------------------ + // AXI4 lets write data precede its address. Quiescing must neither strand + // data whose AW it then withholds nor withhold the AW of data the shell + // already took: the shell must see whole writes only. + printf("8. quiescing never splits a write\n"); + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; // keep QUIESCE long enough to observe + + // (a) data first: a single-beat W is taken before its AW is presented. + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to accepted data still goes out"); + tick(); + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_in_awready, "and is accepted"); + tick(); + dut->wg_out_awready = 0; + dut->eval(); + check(!dut->wg_out_awvalid, "the next AW is withheld"); + dut->wg_out_awready = 1; + dut->eval(); + check(!dut->wg_in_awready, "and not reported accepted"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid, "its data is withheld too"); + check(!dut->wg_in_wready, "and not reported accepted"); + run(4); + check(!dut->wg_out_awvalid && !dut->wg_out_wvalid, "and both stay withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (b) address first, then a two-beat burst: the data owed to an accepted AW + // goes out in full, and nothing after it. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + tick(); + dut->wg_in_awvalid = 0; + pulse_request(); + run(4); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 0; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the first beat owed to the AW goes out"); + tick(); + dut->wg_in_wlast = 1; + dut->eval(); + check(dut->wg_out_wvalid, "the burst runs to its last beat"); + tick(); + dut->wg_in_wlast = 0; + dut->eval(); + check(!dut->wg_out_wvalid, "data with no accepted AW is withheld"); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_awvalid, "a new AW is withheld"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + check(!dut->stop_req, "gate reopens after the sequence"); + dut->wg_in_awvalid = 1; dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->wg_out_awready = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_awvalid && dut->wg_out_wvalid, "writes flow again once reopened"); + clear_fires(); + + // (c) a slave may hold AWREADY until it has seen the data: an address + // already offered when quiescing begins obliges its data. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_awvalid = 1; dut->wg_out_awready = 0; + tick(); + pulse_request(); + run(4); + check(dut->stop_req, "sequencer is quiescing"); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "data owed to an offered AW goes out"); + tick(); + dut->wg_in_wvalid = 0; dut->wg_in_wlast = 0; + dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "and the slave then takes the AW"); + tick(); + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // (d) a slave may hold WREADY until it has seen the address: data already + // offered when quiescing begins obliges its AW. + do_reset(); + dut->ar_fire = 1; tick(); dut->ar_fire = 0; + dut->wg_in_wvalid = 1; dut->wg_in_wlast = 1; dut->wg_out_wready = 0; + tick(); + pulse_request(); + run(4); + dut->wg_in_awvalid = 1; dut->wg_out_awready = 1; + dut->eval(); + check(dut->wg_out_awvalid, "the AW owed to offered data goes out"); + tick(); + dut->wg_in_awvalid = 0; + dut->wg_out_wready = 1; + dut->eval(); + check(dut->wg_out_wvalid, "and the slave then takes the data"); + tick(); + dut->wg_in_awvalid = 1; + dut->eval(); + check(!dut->wg_out_wvalid && !dut->wg_out_awvalid, "nothing further goes out"); + clear_fires(); + dut->r_fire_last = 1; tick(); dut->r_fire_last = 0; + run(16); + check(saw_assert, "reset proceeds"); + run(8); + + // ------------------------------------------------------------------ + // The drain counters see last beats only. A burst whose first beats went + // out ahead of its AW still owes the interconnect the rest of the burst + // and the AW, so the port is not idle and reset must wait. + printf("9. a write burst in progress keeps the port busy\n"); + do_reset(); + dut->w_fire = 1; tick(); dut->w_fire = 0; // beat 1 of 2, no AW yet + tick(); + check(!dut->masters_idle, "not idle with a burst under way"); + pulse_request(); + run(16); + check(!saw_assert, "rst_assert withheld mid-burst"); + dut->w_fire = 1; dut->w_fire_last = 1; tick(); dut->w_fire = 0; dut->w_fire_last = 0; + dut->aw_fire = 1; tick(); dut->aw_fire = 0; + dut->b_fire = 1; tick(); dut->b_fire = 0; + run(8); + check(dut->masters_idle, "idle once the write completes"); + check(saw_assert, "reset proceeds"); + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", failures, failures == 1 ? "" : "s"); delete dut; diff --git a/hw/unittest/cache/VX_cache_top.sv b/hw/unittest/cache/VX_cache_top.sv index b58bd48dd5..3ba5f9c37b 100644 --- a/hw/unittest/cache/VX_cache_top.sv +++ b/hw/unittest/cache/VX_cache_top.sv @@ -52,6 +52,13 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Enable dirty bytes on writeback parameter DIRTY_BYTES = `VX_CFG_DCACHE_DIRTYBYTES, + // Write misses allocate without a fill (write-validate) + parameter LAZY_FETCH = `VX_CFG_DCACHE_LAZY_FETCH, + + // Same-word read merge across the ports, as the dcache merges the + // channels of one LSU block + parameter READ_MERGE = `VX_CFG_DCACHE_READ_MERGE ? NUM_REQS : 0, + // Bank pipeline depth (capacity-scaled) parameter LATENCY = `VX_CFG_DCACHE_LATENCY, @@ -71,6 +78,9 @@ module VX_cache_top import VX_gpu_pkg::*; #( // Memory request output buffer parameter MEM_OUT_BUF = 3, + // Hashed bank select + parameter BANK_HASH = (NUM_BANKS >= 4), + parameter MEM_TAG_WIDTH = `CACHE_MEM_TAG_WIDTH(MSHR_SIZE, NUM_BANKS, MEM_PORTS, UUID_WIDTH) ) ( input wire clk, @@ -180,8 +190,11 @@ module VX_cache_top import VX_gpu_pkg::*; #( .WRITE_ENABLE (WRITE_ENABLE), .WRITEBACK (WRITEBACK), .DIRTY_BYTES (DIRTY_BYTES), + .LAZY_FETCH (LAZY_FETCH), + .READ_MERGE (READ_MERGE), .AMO_ENABLE (AMO_ENABLE), .IS_LLC (IS_LLC), + .BANK_HASH (BANK_HASH), .CORE_OUT_BUF (CORE_OUT_BUF), .MEM_OUT_BUF (MEM_OUT_BUF) ) cache ( @@ -191,7 +204,8 @@ module VX_cache_top import VX_gpu_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (core_bus_if), - .mem_bus_if (mem_bus_if) + .mem_bus_if (mem_bus_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/core/VX_core_top.sv b/hw/unittest/core/VX_core_top.sv index 1e0621d0a7..04eb76192a 100644 --- a/hw/unittest/core/VX_core_top.sv +++ b/hw/unittest/core/VX_core_top.sv @@ -42,23 +42,23 @@ module VX_core_top import VX_gpu_pkg::*; input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] dcache_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] dcache_rsp_ready, - output wire icache_req_valid, - output wire icache_req_rw, - output wire [ICACHE_WORD_SIZE-1:0] icache_req_byteen, - output wire [ICACHE_ADDR_WIDTH-1:0] icache_req_addr, - output wire [ICACHE_WORD_SIZE*8-1:0] icache_req_data, - output wire [ICACHE_TAG_WIDTH-1:0] icache_req_tag, - input wire icache_req_ready, - - input wire icache_rsp_valid, - input wire [ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, - input wire [ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, - output wire icache_rsp_ready, + output wire [ICACHE_NUM_REQS-1:0] icache_req_valid, + output wire [ICACHE_NUM_REQS-1:0] icache_req_rw, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE-1:0] icache_req_byteen, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_ADDR_WIDTH-1:0] icache_req_addr, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_req_data, + output wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_req_tag, + input wire [ICACHE_NUM_REQS-1:0] icache_req_ready, + + input wire [ICACHE_NUM_REQS-1:0] icache_rsp_valid, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_WORD_SIZE*8-1:0] icache_rsp_data, + input wire [ICACHE_NUM_REQS-1:0][ICACHE_TAG_WIDTH-1:0] icache_rsp_tag, + output wire [ICACHE_NUM_REQS-1:0] icache_rsp_ready, output wire gbar_req_valid, output wire [NB_WIDTH-1:0] gbar_req_id, - output wire [NC_WIDTH-1:0] gbar_req_size_m1, - output wire [NC_WIDTH-1:0] gbar_req_core_id, + output wire [CORE_ID_WIDTH-1:0] gbar_req_size_m1, + output wire [CORE_ID_WIDTH-1:0] gbar_req_core_id, input wire gbar_req_ready, input wire gbar_rsp_valid, input wire [NB_WIDTH-1:0] gbar_rsp_id, @@ -72,6 +72,15 @@ module VX_core_top import VX_gpu_pkg::*; output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Kernel launches: without them the CTA dispatcher is constant and + // synthesis removes it. + input wire kmu_valid, + input wire kmu_kind, + input wire kmu_eop, + input wire [KMU_DEST_W-1:0] kmu_dest, + input wire [KMU_DATAW-1:0] kmu_data, + output wire kmu_ready, + // Status output wire busy ); @@ -136,11 +145,12 @@ module VX_core_top import VX_gpu_pkg::*; `endif VX_kmu_bus_if kmu_bus_if(); - assign kmu_bus_if.valid = 1'b0; - assign kmu_bus_if.kind = KMU_KIND_COMPUTE; - assign kmu_bus_if.eop = 1'b0; - assign kmu_bus_if.dest = '0; - assign kmu_bus_if.data = '0; + assign kmu_bus_if.valid = kmu_valid; + assign kmu_bus_if.kind = kmu_kind; + assign kmu_bus_if.eop = kmu_eop; + assign kmu_bus_if.dest = kmu_dest; + assign kmu_bus_if.data = kmu_data; + assign kmu_ready = kmu_bus_if.ready; VX_dcr_bus_if dcr_bus_if(); @@ -174,21 +184,23 @@ module VX_core_top import VX_gpu_pkg::*; VX_mem_bus_if #( .DATA_SIZE (ICACHE_WORD_SIZE), .TAG_WIDTH (ICACHE_TAG_WIDTH) - ) icache_bus_if(); - - assign icache_req_valid = icache_bus_if.req_valid; - assign icache_req_rw = icache_bus_if.req_data.rw; - assign icache_req_byteen = icache_bus_if.req_data.byteen; - assign icache_req_addr = icache_bus_if.req_data.addr; - assign icache_req_data = icache_bus_if.req_data.data; - assign icache_req_tag = icache_bus_if.req_data.tag; - assign icache_bus_if.req_ready = icache_req_ready; - `UNUSED_VAR (icache_bus_if.req_data.attr) - - assign icache_bus_if.rsp_valid = icache_rsp_valid; - assign icache_bus_if.rsp_data.tag = icache_rsp_tag; - assign icache_bus_if.rsp_data.data = icache_rsp_data; - assign icache_rsp_ready = icache_bus_if.rsp_ready; + ) icache_bus_if[ICACHE_NUM_REQS](); + + for (genvar i = 0; i < ICACHE_NUM_REQS; ++i) begin : g_icache_bus_if + assign icache_req_valid[i] = icache_bus_if[i].req_valid; + assign icache_req_rw[i] = icache_bus_if[i].req_data.rw; + assign icache_req_byteen[i] = icache_bus_if[i].req_data.byteen; + assign icache_req_addr[i] = icache_bus_if[i].req_data.addr; + assign icache_req_data[i] = icache_bus_if[i].req_data.data; + assign icache_req_tag[i] = icache_bus_if[i].req_data.tag; + assign icache_bus_if[i].req_ready = icache_req_ready[i]; + `UNUSED_VAR (icache_bus_if[i].req_data.attr) + + assign icache_bus_if[i].rsp_valid = icache_rsp_valid[i]; + assign icache_bus_if[i].rsp_data.tag = icache_rsp_tag[i]; + assign icache_bus_if[i].rsp_data.data = icache_rsp_data[i]; + assign icache_rsp_ready[i] = icache_bus_if[i].rsp_ready; + end `ifdef PERF_ENABLE sysmem_perf_t mem_perf; @@ -207,13 +219,17 @@ module VX_core_top import VX_gpu_pkg::*; `UNUSED_VAR (scope_bus_out_w) `endif + wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] core_lsu_busy; + wire core_busy; + assign busy = core_busy || (| core_lsu_busy); + VX_core #( - .INSTANCE_ID (`SFORMATF(("core"))), - .CORE_ID (CORE_ID) + .INSTANCE_ID (`SFORMATF(("core"))) ) core ( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .core_id (CORE_ID_WIDTH'(CORE_ID)), `ifdef PERF_ENABLE .sysmem_perf (sysmem_perf), @@ -243,7 +259,8 @@ module VX_core_top import VX_gpu_pkg::*; `endif .kmu_bus_if (kmu_bus_if), - .busy (busy) + .lsu_busy (core_lsu_busy), + .busy (core_busy) ); endmodule diff --git a/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv b/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv index 68a6fe802b..3a0638cf2d 100644 --- a/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv +++ b/hw/unittest/cp_axil_regfile/VX_cp_axil_regfile_top.sv @@ -23,6 +23,8 @@ module VX_cp_axil_regfile_top input wire clk, input wire reset, + output wire [63:0] satp, + // AXI-Lite W/AW/B input wire awvalid, output wire awready, @@ -114,6 +116,7 @@ module VX_cp_axil_regfile_top .q_error (q_error_arr), .last_dcr_rsp (32'd0), .q_state (q_state_arr), + .satp (satp), .q_reset_pulse (q_reset_arr), .dbg_host_w_counts (32'd0), .dbg_host_r_counts (32'd0), diff --git a/hw/unittest/cp_axil_regfile/main.cpp b/hw/unittest/cp_axil_regfile/main.cpp index 7574b9d6d9..6a32120570 100644 --- a/hw/unittest/cp_axil_regfile/main.cpp +++ b/hw/unittest/cp_axil_regfile/main.cpp @@ -9,8 +9,8 @@ // - CP_STATUS reflects the harness-driven cp_busy / cp_error inputs. // - CP_DEV_CAPS returns the configured (NUM_QUEUES, RING_SIZE_LOG2_MAX, // AXI_TID_WIDTH) fields. -// - GPU_DEV_CAPS / GPU_ISA_CAPS read back non-DECERR, decode to sane -// fields, and ignore writes (read-only). +// - GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS read back non-DECERR, +// decode to sane fields, and ignore writes (read-only). // - CP_CYCLE counter actually advances per clock. // - Atomic Q_TAIL commit: writing Q_TAIL_LO alone does NOT advance // q_state.tail; writing Q_TAIL_HI atomically commits both halves. @@ -23,6 +23,7 @@ #include "vl_simulator.h" #include "VVX_cp_axil_regfile_top.h" +#include #include #include #include @@ -152,10 +153,13 @@ template static uint8_t q_profile_en(T* t) { return (uint8_t)read_ static constexpr uint16_t CP_CTRL = 0x000; static constexpr uint16_t CP_STATUS = 0x004; static constexpr uint16_t CP_DEV_CAPS = 0x008; +static constexpr uint16_t GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint16_t CP_CYCLE_LO = 0x010; static constexpr uint16_t CP_CYCLE_HI = 0x014; static constexpr uint16_t GPU_DEV_CAPS_LO = 0x018; static constexpr uint16_t GPU_DEV_CAPS_HI = 0x01C; +static constexpr uint16_t CP_SATP_LO = 0x028; +static constexpr uint16_t CP_SATP_HI = 0x02C; static constexpr uint16_t GPU_ISA_CAPS_LO = 0x020; static constexpr uint16_t GPU_ISA_CAPS_HI = 0x024; @@ -197,6 +201,16 @@ int main(int argc, char** argv) { EXPECT((v & 0xff) == 1, "T1: NUM_QUEUES low byte"); EXPECT(((v >> 8) & 0xff) == 16, "T1: RING_SIZE_LOG2_MAX byte"); EXPECT(((v >> 16) & 0xff) == 6, "T1: AXI_TID_WIDTH byte"); + EXPECT(((v >> 24) & 1) == VX_CFG_VM_ENABLED, "T1: VM_ENABLED bit"); + } + + // ----- Test 1b: CP_SATP holds the DMA page-table root ----- + { + EXPECT(axil_write(sim, tick, CP_SATP_LO, 0x80012345) == 0, "T1b: SATP_LO write"); + EXPECT(axil_write(sim, tick, CP_SATP_HI, 0x00000007) == 0, "T1b: SATP_HI write"); + EXPECT((axil_read(sim, tick, CP_SATP_LO) & 0xffffffffull) == 0x80012345, "T1b: SATP_LO readback"); + EXPECT((axil_read(sim, tick, CP_SATP_HI) & 0xffffffffull) == 0x00000007, "T1b: SATP_HI readback"); + EXPECT(sim->satp == 0x0000000780012345ull, "T1b: satp output"); } // ----- Test 2: CP_CYCLE counter advances ----- @@ -334,8 +348,12 @@ int main(int argc, char** argv) { "T11: caps must not be the DECERR sentinel"); uint64_t dev_caps = ((uint64_t)dhi << 32) | dlo; uint64_t isa_caps = ((uint64_t)ihi << 32) | ilo; - // dev_caps reserved bits [63:42] must be zero. - EXPECT((dev_caps >> 42) == 0, "T11: dev_caps reserved bits set"); + // Exact counts: warps [48:42], clusters [63:57], socket size in the extension word. + EXPECT(((dev_caps >> 42) & 0x7F) == VX_CFG_NUM_WARPS, "T11: dev_caps exact warps"); + EXPECT(((dev_caps >> 57) & 0x7F) == VX_CFG_NUM_CLUSTERS, "T11: dev_caps exact clusters"); + uint64_t de = axil_read(sim, tick, GPU_DEV_CAPS_EXT); + EXPECT((de >> 32) == 0, "T11: GPU_DEV_CAPS_EXT DECERR"); + EXPECT((uint32_t)de == VX_CFG_SOCKET_SIZE, "T11: GPU_DEV_CAPS_EXT socket size"); // isa_caps XLEN field [31:30] = clog2(XLEN)-4, so 1 (RV32) or 2 (RV64). uint32_t xlen_field = (uint32_t)((isa_caps >> 30) & 0x3); EXPECT(xlen_field == 1 || xlen_field == 2, "T11: isa_caps XLEN field"); diff --git a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv index 060b56a28d..1de85c8b61 100644 --- a/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv +++ b/hw/unittest/cp_dcr_proxy/VX_cp_dcr_proxy_top.sv @@ -28,7 +28,8 @@ module VX_cp_dcr_proxy_top output wire [`VX_DCR_ADDR_BITS-1:0] dcr_req_addr, output wire [`VX_DCR_DATA_BITS-1:0] dcr_req_data, input wire dcr_rsp_valid, - input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data + input wire [`VX_DCR_DATA_BITS-1:0] dcr_rsp_data, + input wire gpu_busy ); cmd_t cmd_typed; @@ -46,7 +47,8 @@ module VX_cp_dcr_proxy_top .dcr_req_addr (dcr_req_addr), .dcr_req_data (dcr_req_data), .dcr_rsp_valid (dcr_rsp_valid), - .dcr_rsp_data (dcr_rsp_data) + .dcr_rsp_data (dcr_rsp_data), + .gpu_busy (gpu_busy) ); endmodule : VX_cp_dcr_proxy_top diff --git a/hw/unittest/cp_dcr_proxy/main.cpp b/hw/unittest/cp_dcr_proxy/main.cpp index 56f3e18cf3..171acef369 100644 --- a/hw/unittest/cp_dcr_proxy/main.cpp +++ b/hw/unittest/cp_dcr_proxy/main.cpp @@ -9,6 +9,8 @@ // S_REQ → write: S_DONE; read: S_WAIT_RSP // S_WAIT_RSP → dcr_rsp_valid ⇒ latch rsp_data_r, S_DONE // S_DONE → IDLE +// CMD_CACHE_FLUSH: IDLE → S_START (broadcast write) → per core +// S_REQ/S_WAIT_RSP (read with the core index) → S_DONE after the last. // // Coverage: // 1. Reset: no transitions, dcr_req_valid stays 0, done stays 0. @@ -19,11 +21,21 @@ // S_WAIT_RSP until dcr_rsp_valid arrives; rsp_data is latched // into last_rsp_data and visible while done pulses. // 4. Back-to-back write→read: FSM re-arms cleanly. -// 5. WAIT_RSP hangs if rsp_valid never arrives (no spurious done). +// 5. CMD_CACHE_FLUSH over 3 cores: one broadcast write to +// VX_DCR_BASE_CACHE_FLUSH, then one read per core in order with the +// core index as data, each waiting for its response; done only +// after the last core answers. +// 6. CMD_CACHE_FLUSH over 0 cores retires without any DCR request. +// 7. CMD_CACHE_FLUSH with the L1-only flag: the broadcast write carries +// VX_CACHE_FLUSH_L1_ONLY (test 5 checks a full flush writes 0), the +// per-core read still carries the core index. +// 8. A full CMD_CACHE_FLUSH holds done while Vortex is busy; an L1-only +// one retires regardless. // ============================================================================ #include "vl_simulator.h" #include "VVX_cp_dcr_proxy_top.h" +#include #include #include #include @@ -51,6 +63,7 @@ void sim_trace_enable(bool e) { trace_en = e; } enum CmdOp : uint8_t { OP_DCR_WRITE = 0x04, OP_DCR_READ = 0x05, + OP_CACHE_FLUSH = 0x0A, }; // Same packed-cmd layout as the cp_engine TB: hdr in the MSB word @@ -72,11 +85,14 @@ static void pack_cmd(uint32_t out_words[9], (static_cast(flags) << 8); } +// CMD_CACHE_FLUSH header flag bit 2 (F_FLUSH_L1_ONLY). +constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + template static void set_cmd(T* top, uint8_t opcode, - uint64_t arg0 = 0, uint64_t arg1 = 0) { + uint64_t arg0 = 0, uint64_t arg1 = 0, uint8_t flags = 0) { uint32_t words[9]; - pack_cmd(words, opcode, 0, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); + pack_cmd(words, opcode, flags, arg0, arg1, /*arg2=*/0, /*profile_slot=*/0); for (int i = 0; i < 9; ++i) top->cmd_packed[i] = words[i]; } @@ -97,6 +113,7 @@ int main(int argc, char** argv) { sim->grant = 0; sim->dcr_rsp_valid = 0; sim->dcr_rsp_data = 0; + sim->gpu_busy = 0; set_cmd(sim.operator->(), 0); tick = sim.reset(tick); @@ -194,6 +211,118 @@ int main(int argc, char** argv) { EXPECT(sim->done == 1, "re-arm: done not asserted"); cycle(sim, tick); + // ----- Test 5: CMD_CACHE_FLUSH over 3 cores. arg0 = core count. ----- + constexpr uint32_t NCORES = 3; + set_cmd(sim.operator->(), OP_CACHE_FLUSH, NCORES); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + + // S_START: one write that every core receives. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSH: start must be a write"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: start addr mismatch"); + EXPECT(sim->dcr_req_data == 0, "FLUSH: a full flush starts with value 0"); + EXPECT(sim->done == 0, "FLUSH: done premature in S_START"); + cycle(sim, tick); // S_START → S_REQ + + for (uint32_t cid = 0; cid < NCORES; ++cid) { + // S_REQ: this core's completion read. + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSH: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSH: per-core request must be a read"); + EXPECT(sim->dcr_req_addr == VX_DCR_BASE_CACHE_FLUSH, "FLUSH: read addr mismatch"); + EXPECT((sim->dcr_req_data & 0xffff) == cid, "FLUSH: read must target the cores in order"); + EXPECT(sim->done == 0, "FLUSH: done before every core answered"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + + // Nothing moves until this core answers. + for (int i = 0; i < 3; ++i) { + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request while waiting for a core"); + EXPECT(sim->done == 0, "FLUSH: spurious done while waiting for a core"); + cycle(sim, tick); + } + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → next core's S_REQ, or S_DONE + sim->dcr_rsp_valid = 0; + } + + sim->eval(); + EXPECT(sim->done == 1, "FLUSH: done not asserted after the last core"); + EXPECT(sim->dcr_req_valid == 0, "FLUSH: request after the last core"); + cycle(sim, tick); // S_DONE → IDLE + sim->eval(); + EXPECT(sim->done == 0, "FLUSH: done should pulse only one cycle"); + + // ----- Test 6: CMD_CACHE_FLUSH over 0 cores retires at once. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_DONE + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 0, "FLUSH0: no request expected"); + EXPECT(sim->done == 1, "FLUSH0: done not asserted"); + cycle(sim, tick); + + // ----- Test 7: L1-only CMD_CACHE_FLUSH over 1 core. ----- + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, FLUSH_FLAG_L1_ONLY); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: start write not issued"); + EXPECT(sim->dcr_req_rw == 1, "FLUSHL1: start must be a write"); + EXPECT(sim->dcr_req_data == VX_CACHE_FLUSH_L1_ONLY, "FLUSHL1: start value must carry the L1-only flag"); + cycle(sim, tick); // S_START → S_REQ + sim->eval(); + EXPECT(sim->dcr_req_valid == 1, "FLUSHL1: per-core read not issued"); + EXPECT(sim->dcr_req_rw == 0, "FLUSHL1: per-core request must be a read"); + EXPECT(sim->dcr_req_data == 0, "FLUSHL1: read must carry the core index only"); + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->dcr_rsp_valid = 1; + sim->dcr_rsp_data = 0; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + sim->eval(); + EXPECT(sim->done == 1, "FLUSHL1: done not asserted"); + cycle(sim, tick); + + // ----- Test 8: a full flush waits for busy; an L1-only one does not. ----- + for (int l1_only = 0; l1_only < 2; ++l1_only) { + set_cmd(sim.operator->(), OP_CACHE_FLUSH, 1, 0, l1_only ? FLUSH_FLAG_L1_ONLY : 0); + sim->grant = 1; + cycle(sim, tick); // IDLE → S_START + sim->grant = 0; + cycle(sim, tick); // S_START → S_REQ + cycle(sim, tick); // S_REQ → S_WAIT_RSP + sim->gpu_busy = 1; + sim->dcr_rsp_valid = 1; + cycle(sim, tick); // → S_DONE + sim->dcr_rsp_valid = 0; + if (l1_only) { + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: an L1-only flush must not wait for busy"); + cycle(sim, tick); + sim->gpu_busy = 0; + } else { + for (int i = 0; i < 4; ++i) { + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: a full flush retired while busy"); + EXPECT(sim->dcr_req_valid == 0, "DRAIN: request while draining"); + cycle(sim, tick); + } + sim->gpu_busy = 0; + sim->eval(); + EXPECT(sim->done == 1, "DRAIN: done not asserted once idle"); + cycle(sim, tick); + sim->eval(); + EXPECT(sim->done == 0, "DRAIN: done should pulse only one cycle"); + } + } + std::printf("PASSED\n"); return 0; } diff --git a/hw/unittest/cp_dma/VX_cp_dma_top.sv b/hw/unittest/cp_dma/VX_cp_dma_top.sv index da9faf2c7c..9c438cbaae 100644 --- a/hw/unittest/cp_dma/VX_cp_dma_top.sv +++ b/hw/unittest/cp_dma/VX_cp_dma_top.sv @@ -27,6 +27,7 @@ module VX_cp_dma_top input wire grant, input wire [$bits(cmd_t)-1:0] cmd_packed, output wire done, + input wire [63:0] satp, // ---- Host-memory AXI master (flat) ---- output wire h_awvalid, @@ -181,6 +182,7 @@ module VX_cp_dma_top .grant (grant), .cmd (cmd_typed), .done (done), + .satp (satp), .axi_host (axi_host), .axi_dev (axi_dev) ); diff --git a/hw/unittest/cp_dma/main.cpp b/hw/unittest/cp_dma/main.cpp index 0e725d8ab1..56683f5e27 100644 --- a/hw/unittest/cp_dma/main.cpp +++ b/hw/unittest/cp_dma/main.cpp @@ -19,6 +19,11 @@ // bytes match src bytes byte-for-byte. // 2. Second back-to-back COPY (different addrs / pattern) re-arms // cleanly — DMA returns to IDLE and accepts the next grant. +// 3. With a page-table root in satp, a COPY between two virtual pages +// walks the table for both operands and moves the physical lines. +// 4. A superpage leaf takes the low address bits from the VA. +// 5. F_MEM_PHYSICAL skips the walk. +// 6. An unmapped VA passes through untranslated, as in the C++ model. // ============================================================================ #include "vl_simulator.h" @@ -28,6 +33,10 @@ #include #include #include +#include +#include +#include +#include #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull @@ -66,11 +75,22 @@ static void pack_cmd(uint32_t out_words[9], out_words[8] = (uint32_t)opcode | ((uint32_t)flags << 8); } -// ---- AXI4 slave model ---- +// ---- AXI4 slave model (sparse, cache-line granular) ---- struct AxiSlave { - static constexpr uint64_t MEM_BASE = 0x1000; - static constexpr int MEM_SIZE = 4096; - uint8_t mem[MEM_SIZE] = {0}; + std::unordered_map> lines; + + uint8_t& byte(uint64_t a) { + auto it = lines.find(a >> 6); + if (it == lines.end()) it = lines.emplace(a >> 6, std::array{}).first; + return it->second[a & 63]; + } + uint8_t peek(uint64_t a) const { + auto it = lines.find(a >> 6); + return (it == lines.end()) ? 0 : it->second[a & 63]; + } + void write_word(uint64_t a, uint64_t v, int bytes) { + for (int b = 0; b < bytes; ++b) byte(a + b) = uint8_t(v >> (8 * b)); + } bool r_inflight = false; uint64_t r_addr = 0; @@ -83,28 +103,18 @@ struct AxiSlave { uint8_t b_id = 0; void mem_write_cl(uint64_t addr, const uint8_t* src) { - for (int i = 0; i < 64; ++i) { - int64_t a = (int64_t)addr - (int64_t)MEM_BASE + i; - if (a >= 0 && a < MEM_SIZE) mem[a] = src[i]; - } + for (int i = 0; i < 64; ++i) byte(addr + i) = src[i]; } void mem_read_cl(uint64_t addr, uint32_t* dst) const { for (int w = 0; w < 16; ++w) { uint32_t v = 0; - for (int b = 0; b < 4; ++b) { - int64_t a = (int64_t)addr - (int64_t)MEM_BASE + w*4 + b; - if (a >= 0 && a < MEM_SIZE) v |= (uint32_t)mem[a] << (8 * b); - } + for (int b = 0; b < 4; ++b) v |= (uint32_t)peek(addr + w*4 + b) << (8 * b); dst[w] = v; } } int mem_cmp_cl(uint64_t addr_a, uint64_t addr_b) const { for (int i = 0; i < 64; ++i) { - int64_t aa = (int64_t)addr_a - (int64_t)MEM_BASE + i; - int64_t ab = (int64_t)addr_b - (int64_t)MEM_BASE + i; - uint8_t va = (aa >= 0 && aa < MEM_SIZE) ? mem[aa] : 0; - uint8_t vb = (ab >= 0 && ab < MEM_SIZE) ? mem[ab] : 0; - if (va != vb) return i; + if (peek(addr_a + i) != peek(addr_b + i)) return i; } return -1; } @@ -146,10 +156,7 @@ struct AxiSlave { // Write 64 bytes from wdata[0..15] into memory at aw_addr. for (int w = 0; w < 16; ++w) { uint32_t v = top->d_wdata[w]; - for (int b = 0; b < 4; ++b) { - int64_t a = (int64_t)aw_addr - (int64_t)MEM_BASE + w*4 + b; - if (a >= 0 && a < MEM_SIZE) mem[a] = (uint8_t)(v >> (8 * b)); - } + for (int b = 0; b < 4; ++b) byte(aw_addr + w*4 + b) = (uint8_t)(v >> (8 * b)); } aw_taken = false; b_pending = true; @@ -172,10 +179,54 @@ static void cycle(vl_simulator& sim, AxiSlave& s, uint64_t& tick) { top->eval(); } +// Page-table builder in the slave's memory: Sv32 at XLEN=32, Sv39 at 64. +struct PageTable { + static constexpr int LEVEL_BITS = __builtin_ctz(VX_VM_PT_SIZE / VX_VM_PTE_SIZE); + AxiSlave& mem; + uint64_t root; + uint64_t next_table; + + PageTable(AxiSlave& m, uint64_t root_pa) : mem(m), root(root_pa), next_table(root_pa + VX_VM_PT_SIZE) {} + + uint64_t satp() const { +#if VX_VM_PT_LEVEL == 2 + return (1ull << 31) | (root >> VX_VM_PAGE_LOG2_SIZE); +#else + return (8ull << 60) | (root >> VX_VM_PAGE_LOG2_SIZE); +#endif + } + uint64_t pte_addr(uint64_t table, uint64_t va, int level) const { + uint64_t idx = (va >> (VX_VM_PAGE_LOG2_SIZE + level * LEVEL_BITS)) & ((1ull << LEVEL_BITS) - 1); + return table + idx * VX_VM_PTE_SIZE; + } + uint64_t read_pte(uint64_t a) const { + uint64_t v = 0; + for (int b = 0; b < VX_VM_PTE_SIZE; ++b) v |= uint64_t(mem.peek(a + b)) << (8 * b); + return v; + } + // Map one page (leaf_level 0) or one superpage (leaf_level > 0). + void map(uint64_t va, uint64_t pa, int leaf_level) { + uint64_t table = root; + for (int level = VX_VM_PT_LEVEL - 1; level > leaf_level; --level) { + uint64_t a = pte_addr(table, va, level); + uint64_t pte = read_pte(a); + if ((pte & 1) == 0) { + pte = ((next_table >> VX_VM_PAGE_LOG2_SIZE) << 10) | 0x1; // V, pointer + mem.write_word(a, pte, VX_VM_PTE_SIZE); + next_table += VX_VM_PT_SIZE; + } + table = (pte >> 10) << VX_VM_PAGE_LOG2_SIZE; + } + uint64_t leaf = ((pa >> VX_VM_PAGE_LOG2_SIZE) << 10) | 0xC7; // D A W R V + mem.write_word(pte_addr(table, va, leaf_level), leaf, VX_VM_PTE_SIZE); + } +}; + template static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, - uint64_t src, uint64_t dst, const uint8_t* pattern) { - slave.mem_write_cl(src, pattern); + uint64_t src, uint64_t dst, const uint8_t* pattern, + uint64_t pattern_pa, uint8_t flags = 0) { + slave.mem_write_cl(pattern_pa, pattern); // Drain any leftover state (a previous run_copy returns with the FSM // in S_DONE; one idle cycle takes it back to S_IDLE before we drive @@ -184,7 +235,7 @@ static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, for (int i = 0; i < 2; ++i) cycle(sim, slave, tick); uint32_t c[9]; - pack_cmd(c, /*opcode=*/0x03 /*MEM_COPY*/, 0, /*arg0=dst*/dst, + pack_cmd(c, /*opcode=*/0x03 /*MEM_COPY*/, flags, /*arg0=dst*/dst, /*arg1=src*/src, /*arg2=size*/64); for (int i = 0; i < 9; ++i) sim->cmd_packed[i] = c[i]; @@ -202,11 +253,11 @@ static void run_copy(vl_simulator& sim, AxiSlave& slave, uint64_t& tick, EXPECT(latched, "DMA never asserted arvalid (grant capture failed)"); bool got_done = false; - for (int g = 0; g < 50 && !got_done; ++g) { + for (int g = 0; g < 200 && !got_done; ++g) { cycle(sim, slave, tick); if (sim->done) got_done = true; } - EXPECT(got_done, "DMA did not signal done within 50 cycles"); + EXPECT(got_done, "DMA did not signal done within 200 cycles"); } int main(int argc, char** argv) { @@ -225,7 +276,7 @@ int main(int argc, char** argv) { { uint8_t pat[64]; for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0xA0 + i); - run_copy(sim, slave, tick, /*src=*/0x1000, /*dst=*/0x1100, pat); + run_copy(sim, slave, tick, /*src=*/0x1000, /*dst=*/0x1100, pat, 0x1000); int diff = slave.mem_cmp_cl(0x1000, 0x1100); EXPECT(diff < 0, "T1: dst doesn't match src after copy"); @@ -235,12 +286,55 @@ int main(int argc, char** argv) { { uint8_t pat[64]; for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x5A ^ (i << 1)); - run_copy(sim, slave, tick, /*src=*/0x1200, /*dst=*/0x1300, pat); + run_copy(sim, slave, tick, /*src=*/0x1200, /*dst=*/0x1300, pat, 0x1200); int diff = slave.mem_cmp_cl(0x1200, 0x1300); EXPECT(diff < 0, "T2: second copy mismatch"); } - std::printf("PASSED — 2 scenarios\n"); +#if VX_CFG_VM_ENABLED + PageTable pt(slave, 0x100000); + pt.map(0x80001000, 0x20000, 0); + pt.map(0x80005000, 0x23000, 0); + pt.map(0xC0000000, 0x400000, 1); + sim->satp = pt.satp(); + + // ----- Test 3: both operands walk the table ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x3C + 7 * i); + run_copy(sim, slave, tick, /*src=*/0x80001040, /*dst=*/0x80005080, pat, 0x20040); + EXPECT(slave.mem_cmp_cl(0x20040, 0x23080) < 0, "T3: VA copy did not land at the mapped PA"); + EXPECT(slave.lines.count(0x80005080 >> 6) == 0, "T3: VA copy wrote the untranslated address"); + } + + // ----- Test 4: superpage leaf keeps the VA's low bits ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x91 ^ i); + run_copy(sim, slave, tick, /*src=*/0xC0012340, /*dst=*/0x80005100, pat, 0x412340); + EXPECT(slave.mem_cmp_cl(0x412340, 0x23100) < 0, "T4: superpage copy mismatch"); + } + + // ----- Test 5: F_MEM_PHYSICAL skips the walk ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0x17 + i); + run_copy(sim, slave, tick, /*src=*/0x30000, /*dst=*/0x30400, pat, 0x30000, /*flags=*/0x04); + EXPECT(slave.mem_cmp_cl(0x30000, 0x30400) < 0, "T5: physical copy mismatch"); + } + + // ----- Test 6: an unmapped VA passes through ----- + { + uint8_t pat[64]; + for (int i = 0; i < 64; ++i) pat[i] = (uint8_t)(0xE1 - i); + run_copy(sim, slave, tick, /*src=*/0x00031000, /*dst=*/0x80005140, pat, 0x31000); + EXPECT(slave.mem_cmp_cl(0x31000, 0x23140) < 0, "T6: unmapped source did not pass through"); + } + + std::printf("PASSED — 6 scenarios\n"); +#else + std::printf("PASSED — 2 scenarios (VX_CFG_VM_ENABLE off: no page walk)\n"); +#endif return 0; } diff --git a/hw/unittest/cp_llc_port/Makefile b/hw/unittest/cp_llc_port/Makefile new file mode 100644 index 0000000000..f37bc1700d --- /dev/null +++ b/hw/unittest/cp_llc_port/Makefile @@ -0,0 +1,27 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := cp_llc_port + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) +CXXFLAGS += -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv $(RTL_DIR)/VX_trace_pkg.sv + +RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) \ + -I$(RTL_DIR)/libs -I$(VORTEX_HOME)/hw/unittest/$(PROJECT) +RTL_INCLUDE += -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/fpu \ + -I$(RTL_DIR)/core -I$(RTL_DIR)/cp + +TOP := VX_cp_llc_port_top + +include ../common.mk diff --git a/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv new file mode 100644 index 0000000000..ad339b8342 --- /dev/null +++ b/hw/unittest/cp_llc_port/VX_cp_llc_port_top.sv @@ -0,0 +1,102 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_define.vh" + +// Test wrapper around VX_cp_llc_port: the last-level cache side of the +// port's VX_mem_bus_if is flattened so the C++ harness can play the cache. +module VX_cp_llc_port_top import VX_gpu_pkg::*; #( + parameter DATA_SIZE = 64, + parameter TAG_WIDTH = 4, + parameter SLOT_TAG_WIDTH = UUID_WIDTH + 1 + TAG_WIDTH, + parameter ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(DATA_SIZE) +) ( + input wire clk, + input wire reset, + + input wire req_valid, + input wire req_rw, + input wire req_noalloc, + input wire [ADDR_WIDTH-1:0] req_addr, + input wire [DATA_SIZE*8-1:0] req_data, + input wire [DATA_SIZE-1:0] req_byteen, + input wire [TAG_WIDTH-1:0] req_tag, + output wire req_ready, + + output wire rsp_valid, + output wire [DATA_SIZE*8-1:0] rsp_data, + output wire [TAG_WIDTH-1:0] rsp_tag, + input wire rsp_ready, + + input wire start_in, + output wire start_out, + output wire busy, + + output wire mem_req_valid, + output wire mem_req_rw, + output wire [ADDR_WIDTH-1:0] mem_req_addr, + output wire [DATA_SIZE*8-1:0] mem_req_data, + output wire [DATA_SIZE-1:0] mem_req_byteen, + output wire mem_req_flush, + output wire mem_req_io, + output wire mem_req_noalloc, + output wire [SLOT_TAG_WIDTH-1:0] mem_req_tag, + input wire mem_req_ready, + + input wire mem_rsp_valid, + input wire [DATA_SIZE*8-1:0] mem_rsp_data, + input wire [SLOT_TAG_WIDTH-1:0] mem_rsp_tag, + output wire mem_rsp_ready +); + VX_mem_bus_if #( + .DATA_SIZE (DATA_SIZE), + .TAG_WIDTH (SLOT_TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) mem_bus_if (); + + VX_cp_llc_port #( + .DATA_SIZE (DATA_SIZE), + .SLOT_TAG_WIDTH (SLOT_TAG_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .ADDR_WIDTH (ADDR_WIDTH) + ) dut ( + .clk (clk), + .reset (reset), + .req_valid (req_valid), + .req_rw (req_rw), + .req_noalloc (req_noalloc), + .req_addr (req_addr), + .req_data (req_data), + .req_byteen (req_byteen), + .req_tag (req_tag), + .req_ready (req_ready), + .rsp_valid (rsp_valid), + .rsp_data (rsp_data), + .rsp_tag (rsp_tag), + .rsp_ready (rsp_ready), + .start_in (start_in), + .start_out (start_out), + .busy (busy), + .mem_bus_if (mem_bus_if) + ); + + assign mem_req_valid = mem_bus_if.req_valid; + assign mem_req_rw = mem_bus_if.req_data.rw; + assign mem_req_addr = mem_bus_if.req_data.addr; + assign mem_req_data = mem_bus_if.req_data.data; + assign mem_req_byteen = mem_bus_if.req_data.byteen; + mem_bus_attr_t mem_req_attr; + assign mem_req_attr = mem_bus_attr_t'(mem_bus_if.req_data.attr); + assign mem_req_flush = mem_req_attr.is_flush; + assign mem_req_io = mem_req_attr.is_addr_io; + assign mem_req_noalloc = mem_req_attr.is_noalloc; + `UNUSED_VAR (mem_req_attr) + assign mem_req_tag = mem_bus_if.req_data.tag; + assign mem_bus_if.req_ready = mem_req_ready; + + assign mem_bus_if.rsp_valid = mem_rsp_valid; + assign mem_bus_if.rsp_data.data = mem_rsp_data; + assign mem_bus_if.rsp_data.tag = mem_rsp_tag; + assign mem_rsp_ready = mem_bus_if.rsp_ready; + +endmodule diff --git a/hw/unittest/cp_llc_port/main.cpp b/hw/unittest/cp_llc_port/main.cpp new file mode 100644 index 0000000000..5975b84561 --- /dev/null +++ b/hw/unittest/cp_llc_port/main.cpp @@ -0,0 +1,393 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_cp_llc_port, the command processor's DMA port +// into the last-level cache. +// +// The harness plays the cache: requests enter an in-order input queue that the +// test can stall, and a request takes effect only when it leaves that queue, so +// an accepted DMA write is not yet visible while it sits in the port's buffer +// or in the queue. Every cycle the launch leaves the port (start_out), all DMA +// writes the port accepted must already have taken effect. +// +// Coverage: +// 1. A launch with no DMA write behind it passes straight through. +// 2. DMA reads return data and tag, and do not arm the launch fence. +// 3. A launch behind a DMA write held at the cache input waits for that +// write, then for the fence token's answer; the token is a read with the +// flush attribute and slot tag bit 0 set, and its answer is not forwarded +// to the DMA. The port takes no DMA request while the fence is pending. +// 4. The fence clears: a following launch with no new write passes through. +// 5. Several writes into a cache that accepts and drains in bursts. +// 6. Request attributes: IO aperture, and no-allocate on writes only. + +#include "vl_simulator.h" +#include "VVX_cp_llc_port_top.h" +#include +#include +#include +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +#define EXPECT(cond, msg) do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d (cycle %llu): %s\n", __FILE__, __LINE__, \ + (unsigned long long)cycles, msg); \ + std::exit(1); \ + } \ +} while (0) + +using Top = VVX_cp_llc_port_top; + +static constexpr uint32_t LINE_SIZE = 64; +static constexpr uint32_t TAG_BITS = 4; +static constexpr uint32_t DATA_LINE = 0x100000; // above the IO aperture +static constexpr uint32_t IO_LINE = VX_MEM_IO_BASE_ADDR / LINE_SIZE; + +static_assert(VX_MEM_IO_BASE_ADDR % LINE_SIZE == 0, "IO base must be line aligned"); +static_assert(DATA_LINE >= VX_MEM_IO_END_ADDR / LINE_SIZE, "data line must be outside the IO aperture"); + +struct CacheReq { + bool rw; + bool flush; + uint32_t addr; + uint32_t data; + uint64_t tag; +}; + +struct CacheRsp { + uint32_t data; + uint64_t tag; +}; + +static uint64_t cycles = 0; + +// The last-level cache seen through the port's slot. +struct Cache { + bool accept = true; // input ready + uint32_t period = 0; // with duty: ready duty of every period cycles + uint32_t duty = 0; + uint32_t service = 1; // cycles per request + std::deque input; + std::deque rsps; + std::unordered_map mem; + uint32_t busy_cycles = 0; + uint64_t writes_applied = 0; + + // Retire at most one request from the head of the input queue. + void step() { + if (input.empty()) { + return; + } + if (++busy_cycles < service) { + return; + } + busy_cycles = 0; + CacheReq r = input.front(); + input.pop_front(); + if (r.flush) { + rsps.push_back({0, r.tag}); + } else if (r.rw) { + mem[r.addr] = r.data; + ++writes_applied; + } else { + auto it = mem.find(r.addr); + rsps.push_back({it != mem.end() ? it->second : 0, r.tag}); + } + } +}; + +struct Bench { + vl_simulator sim; + uint64_t tick = 0; + Cache cache; + + uint64_t dma_writes = 0; // DMA writes the port accepted + uint64_t fence_tokens = 0; + uint64_t starts = 0; + std::deque dma_rsps; + + Bench() { + idle_inputs(); + tick = sim.reset(tick); + } + + void idle_inputs() { + sim->req_valid = 0; + sim->req_rw = 0; + sim->req_noalloc = 0; + sim->req_addr = 0; + sim->req_byteen = 0; + sim->req_tag = 0; + sim->rsp_ready = 1; + sim->start_in = 0; + sim->mem_req_ready = 0; + sim->mem_rsp_valid = 0; + sim->mem_rsp_tag = 0; + for (uint32_t i = 0; i < LINE_SIZE / 4; ++i) { + sim->req_data[i] = 0; + sim->mem_rsp_data[i] = 0; + } + } + + // One clock: present the cache's outputs, sample every handshake on the + // settled inputs, then advance the edge and the cache model. + void cycle() { + sim->mem_req_ready = cache.accept && (cache.period == 0 || (cycles % cache.period) < cache.duty); + sim->mem_rsp_valid = !cache.rsps.empty(); + if (!cache.rsps.empty()) { + sim->mem_rsp_tag = cache.rsps.front().tag; + sim->mem_rsp_data[0] = cache.rsps.front().data; + } + sim->eval(); + + if (sim->start_out) { + ++starts; + EXPECT(cache.writes_applied == dma_writes, + "launch left the port before its DMA writes reached the cache"); + } + if (sim->req_valid && sim->req_ready && sim->req_rw) { + ++dma_writes; + } + if (sim->mem_req_valid && sim->mem_req_ready) { + CacheReq r; + r.rw = sim->mem_req_rw; + r.flush = sim->mem_req_flush; + r.addr = sim->mem_req_addr; + r.data = sim->mem_req_data[0]; + r.tag = sim->mem_req_tag; + if (r.flush) { + ++fence_tokens; + EXPECT(!r.rw, "fence token must be a read"); + EXPECT((r.tag & 1) == 1, "fence token must carry slot tag bit 0"); + } else { + EXPECT((r.tag & 1) == 0, "DMA request must clear slot tag bit 0"); + } + cache.input.push_back(r); + } + if (sim->mem_rsp_valid && sim->mem_rsp_ready) { + CacheRsp rsp = cache.rsps.front(); + cache.rsps.pop_front(); + if ((rsp.tag & 1) == 0) { + dma_rsps.push_back({rsp.data, (rsp.tag >> 1) & ((1u << TAG_BITS) - 1)}); + } + } + if (sim->rsp_valid && sim->rsp_ready) { + EXPECT(!dma_rsps.empty(), "the port forwarded a response the DMA never asked for"); + EXPECT(sim->rsp_tag == dma_rsps.front().tag, "DMA response tag mismatch"); + EXPECT(sim->rsp_data[0] == dma_rsps.front().data, "DMA response data mismatch"); + dma_rsps.pop_front(); + } + + tick = sim.step(tick, 2); + ++cycles; + cache.step(); + } + + // Hold one DMA request until the port takes it. + void dma_req(bool rw, uint32_t addr, uint32_t data, uint32_t tag, bool noalloc = false) { + sim->req_valid = 1; + sim->req_rw = rw; + sim->req_noalloc = noalloc; + sim->req_addr = addr; + sim->req_data[0] = data; + sim->req_byteen = ~0ull; + sim->req_tag = tag; + for (int i = 0;; ++i) { + EXPECT(i < 1000, "DMA request never accepted"); + sim->eval(); + bool fire = sim->req_ready; + cycle(); + if (fire) { + break; + } + } + sim->req_valid = 0; + } + + // Pulse start_in for one cycle; returns whether start_out fired with it. + bool launch() { + sim->start_in = 1; + uint64_t before = starts; + cycle(); + sim->start_in = 0; + return starts != before; + } + + void run(int n) { + for (int i = 0; i < n; ++i) { + cycle(); + } + } + + // Wait for the launch to leave the port; returns the cycles waited. + int wait_start(int limit) { + uint64_t before = starts; + for (int i = 0; i < limit; ++i) { + cycle(); + if (starts != before) { + return i + 1; + } + } + EXPECT(false, "launch never left the port"); + return -1; + } +}; + +static void test_launch_without_writes(Bench& b) { + b.cache.accept = true; + b.run(4); + EXPECT(!b.sim->busy, "port busy at idle"); + EXPECT(b.launch(), "launch with no DMA write must pass through in the same cycle"); + b.run(8); + EXPECT(b.fence_tokens == 0, "no fence token expected without a DMA write"); + EXPECT(!b.sim->busy, "port busy after a pass-through launch"); +} + +static void test_reads(Bench& b) { + b.cache.accept = true; + b.cache.mem[DATA_LINE + 7] = 0xcafe0007; + b.cache.mem[DATA_LINE + 9] = 0xcafe0009; + b.dma_req(false, DATA_LINE + 7, 0, 3); + b.dma_req(false, DATA_LINE + 9, 0, 5); + b.run(10); + EXPECT(b.dma_rsps.empty(), "DMA read responses missing"); + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "DMA reads must not hold the launch"); + b.run(4); + EXPECT(b.fence_tokens == tokens, "DMA reads must not arm the fence"); +} + +static void test_fence_holds_launch(Bench& b) { + constexpr int STALL = 64; + b.cache.accept = false; + b.dma_req(true, DATA_LINE + 1, 0x11111111, 1); + EXPECT(b.cache.writes_applied < b.dma_writes, "write should still be held at the cache input"); + + uint64_t starts = b.starts; + EXPECT(!b.launch(), "launch passed the port with a DMA write still in front of the cache"); + for (int i = 0; i < STALL; ++i) { + b.sim->eval(); + EXPECT(b.sim->busy, "port must be busy while the fence is pending"); + EXPECT(!b.sim->req_ready, "port must take no DMA request while the fence is pending"); + b.cycle(); + } + EXPECT(b.starts == starts, "launch left the port while the cache input was stalled"); + + // Release the cache slowly: the write, then the token, each take 6 cycles. + uint64_t tokens = b.fence_tokens; + b.cache.accept = true; + b.cache.service = 6; + int waited = b.wait_start(200); + EXPECT(waited >= 2 * 6, "launch left before the cache answered the fence token"); + EXPECT(b.fence_tokens == tokens + 1, "exactly one fence token expected"); + EXPECT(b.dma_rsps.empty(), "fence answer must not reach the DMA"); + b.run(2); + EXPECT(!b.sim->busy, "port still busy after the fence answered"); + b.cache.service = 1; +} + +static void test_fence_clears(Bench& b) { + uint64_t tokens = b.fence_tokens; + EXPECT(b.launch(), "a launch with no new DMA write must pass through after a fence"); + b.run(8); + EXPECT(b.fence_tokens == tokens, "fence re-armed without a new DMA write"); +} + +static void test_write_bursts(Bench& b) { + for (int round = 0; round < 8; ++round) { + b.cache.service = 1 + (round % 4); + b.cache.accept = true; + b.cache.period = 5 + round; + b.cache.duty = 1 + (round % 3); + for (int i = 0; i < 6; ++i) { + b.dma_req(true, DATA_LINE + 32 * round + i, 0x20000000 + 16 * round + i, i & 0xf); + } + b.cache.period = 0; + b.cache.accept = (round % 2) == 0; + uint64_t starts = b.starts; + bool passed = b.launch(); + EXPECT(!passed || b.cache.writes_applied == b.dma_writes, "launch passed undrained writes"); + if (!passed) { + b.run(round * 5); + b.cache.accept = true; + for (int i = 0; b.starts == starts; ++i) { + EXPECT(i < 500, "launch never left the port"); + b.cycle(); + } + } + b.run(4); + EXPECT(!b.sim->busy, "port busy after a burst launch"); + } + // Every written line reads back through the port. + b.cache.accept = true; + b.cache.service = 1; + b.dma_req(false, DATA_LINE + 32 * 7 + 5, 0, 2); + b.run(10); + EXPECT(b.cache.mem[DATA_LINE + 32 * 7 + 5] == 0x20000000 + 16 * 7 + 5, "burst write lost"); +} + +static void test_attributes(Bench& b) { + b.cache.accept = false; + struct Case { bool rw; uint32_t addr; bool noalloc; bool io; bool exp_noalloc; }; + const Case cases[] = { + {true, IO_LINE, false, true, false}, + {false, IO_LINE, true, true, false}, + {true, DATA_LINE + 3, true, false, true }, + {false, DATA_LINE + 3, true, false, false}, + }; + for (const auto& c : cases) { + b.sim->req_valid = 1; + b.sim->req_rw = c.rw; + b.sim->req_noalloc = c.noalloc; + b.sim->req_addr = c.addr; + b.sim->req_tag = 1; + b.sim->eval(); + EXPECT(b.sim->req_ready, "port buffer should be empty"); + b.cycle(); + b.sim->req_valid = 0; + b.sim->eval(); + EXPECT(b.sim->mem_req_valid, "request missing at the cache input"); + EXPECT(b.sim->mem_req_io == c.io, "IO attribute mismatch"); + EXPECT(b.sim->mem_req_noalloc == c.exp_noalloc, "no-allocate attribute mismatch"); + EXPECT(!b.sim->mem_req_flush, "DMA request carries the flush attribute"); + b.cache.accept = true; + b.cycle(); + b.cache.accept = false; + } + b.cache.accept = true; + b.run(10); + // Leave the fence armed by these writes discharged. + b.launch(); + b.wait_start(100); + b.run(4); +} + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + Bench b; + + test_launch_without_writes(b); + test_reads(b); + test_fence_holds_launch(b); + test_fence_clears(b); + test_write_bursts(b); + test_attributes(b); + + EXPECT(b.cache.input.empty() && b.cache.rsps.empty(), "cache traffic left over"); + std::printf("PASSED\n"); + return 0; +} diff --git a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv index 564634269b..9062fd6110 100644 --- a/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv +++ b/hw/unittest/cta_dispatcher/VX_cta_dispatch_top.sv @@ -101,21 +101,18 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; wire cta_init; wire busy; - // CTA-CSR read-back / schedule lookup ports are unexercised by this bench - // (it only checks warp dispatch via cta_fire/cta_wid), so the read indices - // are tied off and the read-back outputs are left unused. - cta_csrs_t cta_rd_csrs; - cta_lane_t [`VX_CFG_NUM_THREADS-1:0] cta_rd_lane; - wire [NCTA_WIDTH-1:0] schedule_cta_id; + // The CTA-CSR broadcast is unexercised by this bench (it only checks warp + // dispatch via cta_fire/cta_wid). + cta_bcast_t cta_bcast; + cta_lane_bcast_t cta_lane_bcast; `UNUSED_VAR (cta_PC) `UNUSED_VAR (cta_tmask) `UNUSED_VAR (cta_param) `UNUSED_VAR (cta_init) `UNUSED_VAR (busy) - `UNUSED_VAR (cta_rd_csrs) - `UNUSED_VAR (cta_rd_lane) - `UNUSED_VAR (schedule_cta_id) + `UNUSED_VAR (cta_bcast) + `UNUSED_VAR (cta_lane_bcast) VX_cta_dispatch cta_dispatch ( .clk (clk), @@ -130,12 +127,8 @@ module VX_cta_dispatch_top import VX_gpu_pkg::*; .cta_tmask (cta_tmask), .cta_param (cta_param), .cta_init (cta_init), - .csr_rd_wid ('0), - .csr_rd_cta_id ('0), - .cta_rd_csrs (cta_rd_csrs), - .cta_rd_lane (cta_rd_lane), - .schedule_wid ('0), - .schedule_cta_id (schedule_cta_id), + .cta_bcast (cta_bcast), + .cta_lane_bcast(cta_lane_bcast), .busy (busy) ); diff --git a/hw/unittest/gfx/VX_gfx_top.sv b/hw/unittest/gfx/VX_gfx_top.sv index 51aed0b6d5..db0c7c0166 100644 --- a/hw/unittest/gfx/VX_gfx_top.sv +++ b/hw/unittest/gfx/VX_gfx_top.sv @@ -21,10 +21,17 @@ // socket slice: // VX_tex_bus_arb -> VX_tex_core xN -> tcache (VX_cache_cluster) // VX_rtu_bus_arb -> VX_rtu_core xN -> rtcache (VX_cache_cluster) +// VX_mmu : under VM, the socket unit MMU between the tcache / +// rtcache / DXA gmem ports and the arbiter; its tlb_bus, satp +// mode and TLB flush are top ports. The 3-input socket +// TLB arbiter and the snoop's unit-MMU and unit-cache +// flush legs are elaborated only by the full-GPU build // VX_mem_bus_arb : the socket memory arbiter with its full production // input set {icache, dcache, tcache, rtcache, DXA gmem}; // the icache/dcache/DXA slots are driven from top ports // cluster tail: +// VX_mmu : under VM, the cluster unit MMU between the rcache / +// ocache ports and the L2 (tlb_bus and flush as ports) // VX_graphics : the production container — raster launch fork, // VX_raster_core xN + rcache, the OM trunk stage // (VX_om_steer/VX_om_ingress per trunk port), @@ -47,6 +54,9 @@ // top-level port and every DUT output is observed at a top-level port. module VX_gfx_top import VX_gpu_pkg::* +`ifdef VX_CFG_VM_ENABLE + , VX_tlb_pkg::* +`endif `ifdef VX_CFG_EXT_TEX_ENABLE , VX_tex_pkg::* `endif @@ -60,6 +70,11 @@ module VX_gfx_top import VX_gpu_pkg::* parameter SFU_LANES = `VX_CFG_NUM_SFU_LANES, parameter L2_MEM_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(L2_SECTOR_SIZE), parameter EXT_ADDR_W = `VX_CFG_MEM_ADDR_WIDTH - `CLOG2(`VX_CFG_L1_LINE_SIZE) +`ifdef VX_CFG_VM_ENABLE + , parameter UTLB_ID_W = `UP(L1_TLB_ID_WIDTH), + parameter UTLB_REQ_W = UTLB_ID_W + $bits(tlb_access_e) + 1 + TLB_VPN_WIDTH, + parameter UTLB_RSP_W = UTLB_ID_W + 1 + TLB_LEVEL_WIDTH + TLB_PPN_WIDTH + TLB_FLAGS_WIDTH +`endif ) ( input wire clk, input wire reset, @@ -78,7 +93,7 @@ module VX_gfx_top import VX_gpu_pkg::* input wire [NC-1:0] tex_req_valid, input wire [NC-1:0][SFU_LANES-1:0] tex_req_mask, input wire [NC-1:0][1:0][SFU_LANES-1:0][31:0] tex_req_coords, - input wire [NC-1:0][SFU_LANES-1:0][TEX_LOD_BITS-1:0] tex_req_lod, + input wire [NC-1:0][SFU_LANES-1:0][TEX_LODF_BITS-1:0] tex_req_lod, input wire [NC-1:0][TEX_STAGE_BITS-1:0] tex_req_stage, input wire [NC-1:0][TEX_REQ_TAG_WIDTH-1:0] tex_req_tag, output wire [NC-1:0] tex_req_ready, @@ -177,6 +192,38 @@ module VX_gfx_top import VX_gpu_pkg::* input wire flush_req, output wire flush_done, +`ifdef VX_CFG_VM_ENABLE + input wire vm_active, +`endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + // ----------------------------------------------------------------------- + // Cluster unit MMU: the address-space change, drain, and its tlb_bus + // ----------------------------------------------------------------------- + input wire cmmu_flush_req, + output wire cmmu_flush_done, + output wire cmmu_empty, + output wire ctlb_req_valid, + output wire [UTLB_REQ_W-1:0] ctlb_req_data, + input wire ctlb_req_ready, + input wire ctlb_rsp_valid, + input wire [UTLB_RSP_W-1:0] ctlb_rsp_data, + output wire ctlb_rsp_ready, +`endif +`ifdef SOCKET_UNIT_MMU_ENABLE + // ----------------------------------------------------------------------- + // Socket unit MMU: TLB flush, drain, and its tlb_bus + // ----------------------------------------------------------------------- + input wire ummu_flush_req, + output wire ummu_flush_done, + output wire ummu_empty, + output wire utlb_req_valid, + output wire [UTLB_REQ_W-1:0] utlb_req_data, + input wire utlb_req_ready, + input wire utlb_rsp_valid, + input wire [UTLB_RSP_W-1:0] utlb_rsp_data, + output wire utlb_rsp_ready, +`endif + // ----------------------------------------------------------------------- // L2 memory side (master) // ----------------------------------------------------------------------- @@ -292,6 +339,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (tcache_flush_if), + .l1_only (1'b0), .core_bus_if (tcache_bus_if[0]), .cache_bus_if (tcache_flushable_bus_if[0]) ); @@ -322,13 +370,15 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) tcache ( .clk (clk), .reset (reset), .core_bus_if (tcache_flushable_bus_if), - .mem_bus_if (tcache_mem_bus_tmp_if) + .mem_bus_if (tcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[0] = 1'b1; @@ -443,6 +493,7 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .dcr_flush_if (rtcache_flush_if), + .l1_only (1'b0), .core_bus_if (rtcache_bus_if[0]), .cache_bus_if (rtcache_flushable_bus_if[0]) ); @@ -473,18 +524,109 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITEBACK (0), .DIRTY_BYTES (0), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) rtcache ( .clk (clk), .reset (reset), .core_bus_if (rtcache_flushable_bus_if), - .mem_bus_if (rtcache_mem_bus_tmp_if) + .mem_bus_if (rtcache_mem_bus_tmp_if), + `UNUSED_PIN (req_empty) ); `else assign flush_done_w[1] = 1'b1; `endif // VX_CFG_EXT_RTU_ENABLE +`ifdef SOCKET_UNIT_MMU_ENABLE + /////////////////////////////////////////////////////////////////////////// + // Socket unit MMU, wired as VX_socket wires it + /////////////////////////////////////////////////////////////////////////// + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_mem_bus_if[UMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) unit_phys_bus_if[UMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_TCACHE_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF_EX (unit_mem_bus_if[UMMU_RTCACHE_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); +`endif +`ifdef VX_CFG_EXT_DXA_ENABLE + // DXA gmem slot (read-only tile fetches from the ports) + assign unit_mem_bus_if[UMMU_DXA_IDX].req_valid = dxa_mem_req_valid; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.rw = 1'b0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.byteen = '1; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.addr = dxa_mem_req_addr; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.attr = '0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.data = '0; + assign unit_mem_bus_if[UMMU_DXA_IDX].req_data.tag = dxa_mem_req_tag; + assign dxa_mem_req_ready = unit_mem_bus_if[UMMU_DXA_IDX].req_ready; + assign dxa_mem_rsp_valid = unit_mem_bus_if[UMMU_DXA_IDX].rsp_valid; + assign dxa_mem_rsp_data = unit_mem_bus_if[UMMU_DXA_IDX].rsp_data.data; + assign dxa_mem_rsp_tag = unit_mem_bus_if[UMMU_DXA_IDX].rsp_data.tag; + assign unit_mem_bus_if[UMMU_DXA_IDX].rsp_ready = dxa_mem_rsp_ready; +`endif + + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) utlb_bus_if(); + assign utlb_req_valid = utlb_bus_if.req_valid; + assign utlb_req_data = utlb_bus_if.req_data; + assign utlb_bus_if.req_ready = utlb_req_ready; + assign utlb_bus_if.rsp_valid = utlb_rsp_valid; + assign utlb_bus_if.rsp_data = utlb_rsp_data; + assign utlb_rsp_ready = utlb_bus_if.rsp_ready; + + VX_tlb_flush_if ummu_flush_if(); + assign ummu_flush_if.req = ummu_flush_req; + assign ummu_flush_done = ummu_flush_if.done; + +`ifdef PERF_ENABLE + mmu_perf_t ummu_perf; + `UNUSED_VAR (ummu_perf) +`endif + + VX_mmu_fault_if ummu_fault_if(); + `UNUSED_VAR (ummu_fault_if.valid) + `UNUSED_VAR (ummu_fault_if.va) + `UNUSED_VAR (ummu_fault_if.access) + `UNUSED_VAR (ummu_fault_if.amo) + + VX_mmu #( + .INSTANCE_ID ("ummu"), + .NUM_REQS (UMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_UTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (0), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (SOCKET_MEM_TAG_WIDTH) + ) ummu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (ummu_perf), + `endif + .core_bus_if (unit_mem_bus_if), + .mem_bus_if (unit_phys_bus_if), + .tlb_bus_if (utlb_bus_if), + .flush_if (ummu_flush_if), + .fault_if (ummu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (ummu_empty) + ); +`endif + /////////////////////////////////////////////////////////////////////////// // Socket memory arbiter: {icache, dcache, tcache, rtcache, DXA} -> trunk /////////////////////////////////////////////////////////////////////////// @@ -534,13 +676,26 @@ module VX_gfx_top import VX_gpu_pkg::* assign dc_mem_rsp_tag[0] = socket_arb_in_if[DCACHE_MEM_ARB_IDX].rsp_data.tag; assign socket_arb_in_if[DCACHE_MEM_ARB_IDX].rsp_ready = dc_mem_rsp_ready[0]; + `ifdef SOCKET_UNIT_MMU_ENABLE + `ifdef VX_CFG_EXT_TEX_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[TCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_TCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_RTU_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], unit_phys_bus_if[UMMU_RTCACHE_IDX]); + `endif + `ifdef VX_CFG_EXT_DXA_ENABLE + `ASSIGN_VX_MEM_BUS_IF (socket_arb_in_if[DXA_MEM_ARB_IDX], unit_phys_bus_if[UMMU_DXA_IDX]); + `endif + `else `ifdef VX_CFG_EXT_TEX_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[TCACHE_MEM_ARB_IDX], tcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, TCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif `ifdef VX_CFG_EXT_RTU_ENABLE `ASSIGN_VX_MEM_BUS_IF_EX (socket_arb_in_if[RTCACHE_MEM_ARB_IDX], rtcache_mem_bus_tmp_if[0], SOCKET_MEM_TAG_WIDTH, RTCACHE_MEM_TAG_WIDTH, UUID_WIDTH); `endif + `endif `ifdef VX_CFG_EXT_DXA_ENABLE + `ifndef SOCKET_UNIT_MMU_ENABLE // DXA gmem slot (read-only tile fetches from the ports) assign socket_arb_in_if[DXA_MEM_ARB_IDX].req_valid = dxa_mem_req_valid; assign socket_arb_in_if[DXA_MEM_ARB_IDX].req_data.rw = 1'b0; @@ -554,6 +709,7 @@ module VX_gfx_top import VX_gpu_pkg::* assign dxa_mem_rsp_data = socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_data.data; assign dxa_mem_rsp_tag = socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_data.tag; assign socket_arb_in_if[DXA_MEM_ARB_IDX].rsp_ready = dxa_mem_rsp_ready; + `endif `else `UNUSED_VAR (dxa_mem_req_valid) `UNUSED_VAR (dxa_mem_req_addr) @@ -657,11 +813,20 @@ module VX_gfx_top import VX_gpu_pkg::* ) ocache_l2_bus_if(); `endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + VX_dcr_flush_if gfx_mmu_flush_if(); + VX_tlb_flush_if cmmu_flush_if(); + assign gfx_mmu_flush_if.req = cmmu_flush_req; + assign cmmu_flush_if.req = cmmu_flush_req && gfx_mmu_flush_if.done; + assign cmmu_flush_done = cmmu_flush_if.done; +`endif + VX_graphics #( - .CLUSTER_ID (0) + .INSTANCE_ID ("cluster0") ) graphics ( .clk (clk), .reset (reset), + .cluster_id ('0), `ifdef VX_CFG_EXT_RASTER_ENABLE .raster_kmu_bus_if (raster_kmu_bus_if), .rcache_mem_bus_if (rcache_l2_bus_if), @@ -674,6 +839,9 @@ module VX_gfx_top import VX_gpu_pkg::* `endif .dcr_bus_if (gfx_dcr_bus_if), .cluster_flush_if (cluster_flush_if), + `ifdef CLUSTER_UNIT_MMU_ENABLE + .mmu_flush_if (gfx_mmu_flush_if), + `endif .busy (busy) ); @@ -687,12 +855,81 @@ module VX_gfx_top import VX_gpu_pkg::* end `endif +`ifdef CLUSTER_UNIT_MMU_ENABLE + // Cluster unit MMU, wired as VX_cluster wires it. + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_mem_bus_if[CMMU_NUM_REQS](); + + VX_mem_bus_if #( + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) gfx_phys_bus_if[CMMU_NUM_REQS](); + +`ifdef VX_CFG_EXT_RASTER_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_RCACHE_IDX], rcache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_RASTER_IDX], gfx_phys_bus_if[CMMU_RCACHE_IDX]); +`endif +`ifdef VX_CFG_EXT_OM_ENABLE + `ASSIGN_VX_MEM_BUS_IF (gfx_mem_bus_if[CMMU_OCACHE_IDX], ocache_l2_bus_if); + `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_OM_IDX], gfx_phys_bus_if[CMMU_OCACHE_IDX]); +`endif + + VX_tlb_bus_if #( + .ID_WIDTH (L1_TLB_ID_WIDTH) + ) ctlb_bus_if(); + assign ctlb_req_valid = ctlb_bus_if.req_valid; + assign ctlb_req_data = ctlb_bus_if.req_data; + assign ctlb_bus_if.req_ready = ctlb_req_ready; + assign ctlb_bus_if.rsp_valid = ctlb_rsp_valid; + assign ctlb_bus_if.rsp_data = ctlb_rsp_data; + assign ctlb_rsp_ready = ctlb_bus_if.rsp_ready; + + VX_mmu_fault_if cmmu_fault_if(); + `UNUSED_VAR (cmmu_fault_if.valid) + `UNUSED_VAR (cmmu_fault_if.va) + `UNUSED_VAR (cmmu_fault_if.access) + `UNUSED_VAR (cmmu_fault_if.amo) + +`ifdef PERF_ENABLE + mmu_perf_t cmmu_perf; + `UNUSED_VAR (cmmu_perf) +`endif + + VX_mmu #( + .INSTANCE_ID ("cmmu"), + .NUM_REQS (CMMU_NUM_REQS), + .TLB_SIZE (`VX_CFG_CTLB_SIZE), + .EXEC_SIDE (0), + .WRITE_ENABLE (`VX_CFG_EXT_OM_ENABLED), + .RSP_OUT_BUF (2), + .PARK_REG (1), + .DATA_SIZE (`VX_CFG_L1_LINE_SIZE), + .TAG_WIDTH (L2_TAG_WIDTH) + ) cmmu ( + .clk (clk), + .reset (reset), + .vm_active (vm_active), + `ifdef PERF_ENABLE + .mmu_perf (cmmu_perf), + `endif + .core_bus_if (gfx_mem_bus_if), + .mem_bus_if (gfx_phys_bus_if), + .tlb_bus_if (ctlb_bus_if), + .flush_if (cmmu_flush_if), + .fault_if (cmmu_fault_if), + `UNUSED_PIN (lane_idle), + .empty (cmmu_empty) + ); +`else `ifdef VX_CFG_EXT_RASTER_ENABLE `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_RASTER_IDX], rcache_l2_bus_if); `endif `ifdef VX_CFG_EXT_OM_ENABLE `ASSIGN_VX_MEM_BUS_IF (l2_core_if[L2_GFX_OM_IDX], ocache_l2_bus_if); `endif +`endif // CLUSTER_UNIT_MMU_ENABLE `else // !EXT_GFX_CLUSTER_ENABLE // No cluster-resident graphics: the trunk connects straight to the L2. @@ -707,6 +944,12 @@ module VX_gfx_top import VX_gpu_pkg::* end `endif // EXT_GFX_CLUSTER_ENABLE + // Under VM the walker's PTE fetches take an L2 client slot; the walker is + // covered by the vm DUT, so the slot idles here. + if (PTW_ON_L2 != 0) begin : g_ptw_l2_idle + `INIT_VX_MEM_BUS_IF (l2_core_if[L2_PTW_IDX]) + end + assign flush_done = & flush_done_w; /////////////////////////////////////////////////////////////////////////// @@ -752,6 +995,7 @@ module VX_gfx_top import VX_gpu_pkg::* .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -763,7 +1007,8 @@ module VX_gfx_top import VX_gpu_pkg::* .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); endmodule diff --git a/hw/unittest/issue/VX_issue_top.sv b/hw/unittest/issue/VX_issue_top.sv index 0b34f25478..01de117a5e 100644 --- a/hw/unittest/issue/VX_issue_top.sv +++ b/hw/unittest/issue/VX_issue_top.sv @@ -90,7 +90,21 @@ module VX_issue_top import VX_gpu_pkg::*; #( assign decode_if.data.rs1 = decode_rs1; assign decode_if.data.rs2 = decode_rs2; assign decode_if.data.rs3 = decode_rs3; - assign decode_ready = decode_if.ready; + + // issue slice i takes the warps with wid mod ISSUE_WIDTH == i + VX_decode_if slice_decode_if[`VX_CFG_ISSUE_WIDTH](); + wire [ISSUE_ISW_W-1:0] decode_isw = wid_to_isw(decode_wid); + wire [`VX_CFG_ISSUE_WIDTH-1:0] slice_decode_ready; + for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_decode_if + assign slice_decode_if[i].valid = decode_if.valid && (decode_isw == ISSUE_ISW_W'(i)); + assign slice_decode_if[i].data = decode_if.data; + assign slice_decode_ready[i] = slice_decode_if[i].ready; + `UNUSED_VAR (slice_decode_if[i].ibuf_pop) + end + assign decode_if.ready = slice_decode_ready[decode_isw]; + assign decode_if.ibuf_pop = '0; + assign decode_ready = decode_if.ready; + `UNUSED_VAR (decode_if.ibuf_pop) for (genvar i = 0; i < `VX_CFG_ISSUE_WIDTH; ++i) begin : g_writeback_if assign writeback_if[i].valid = writeback_valid[i]; @@ -157,7 +171,7 @@ module VX_issue_top import VX_gpu_pkg::*; #( .issue_perf (issue_perf), `endif - .decode_if (decode_if), + .decode_if (slice_decode_if), .writeback_if (writeback_if), .dispatch_if (dispatch_if), .issue_sched_if (issue_sched_if) diff --git a/hw/unittest/local_mem/Makefile b/hw/unittest/lmem_unit/Makefile similarity index 93% rename from hw/unittest/local_mem/Makefile rename to hw/unittest/lmem_unit/Makefile index e2488d0855..58eeddf360 100644 --- a/hw/unittest/local_mem/Makefile +++ b/hw/unittest/lmem_unit/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk -PROJECT := local_mem +PROJECT := lmem_unit RTL_DIR := $(VORTEX_HOME)/hw/rtl DPI_DIR := $(VORTEX_HOME)/hw/dpi @@ -20,6 +20,6 @@ RTL_PKGS := $(RTL_DIR)/VX_gpu_pkg.sv RTL_INCLUDE := -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs RTL_INCLUDE += -I$(RTL_DIR)/interfaces -I$(RTL_DIR)/mem -I$(RTL_DIR)/vm -I$(RTL_DIR)/cache -I$(SRC_DIR) -TOP := VX_local_mem_top +TOP := VX_lmem_unit_top include ../common.mk \ No newline at end of file diff --git a/hw/unittest/lmem_unit/VX_lmem_unit_top.sv b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv new file mode 100644 index 0000000000..0e6dabe6f2 --- /dev/null +++ b/hw/unittest/lmem_unit/VX_lmem_unit_top.sv @@ -0,0 +1,155 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +// Synthesis wrapper of the warp-wide local memory at the configured core +// shape: one warp request port per LSU block, and the row-wide DMA port. The +// requests arrive through a register each, as from the LSU blocks' local-path +// buffers and the DMA arbiter in the core; out-of-context synthesis leaves +// paths from the ports untimed, so this is what puts the unit's request +// decoding on the timing report. +module VX_lmem_unit_top import VX_gpu_pkg::*; #( + parameter `STRING INSTANCE_ID = "", + parameter SIZE = (1 << `VX_CFG_LMEM_LOG_SIZE), + parameter NUM_BLOCKS = `VX_CFG_NUM_LSU_BLOCKS, + parameter NUM_LANES = `VX_CFG_NUM_LSU_LANES, + parameter NUM_BANKS = `VX_CFG_LMEM_NUM_BANKS, + parameter WORD_SIZE = LSU_WORD_SIZE, + parameter TAG_WIDTH = LSU_TAG_WIDTH, + parameter ADDR_WIDTH = `CLOG2(SIZE / WORD_SIZE), + parameter WORD_WIDTH = WORD_SIZE * 8, + parameter DMA_ADDR_WIDTH = ADDR_WIDTH - `CLOG2(NUM_BANKS), + parameter DMA_DATA_SIZE = NUM_BANKS * WORD_SIZE, + parameter DMA_TAG_WIDTH = LMEM_DMA_TAG_WIDTH +) ( + input wire clk, + input wire reset, + + // warp requests + input wire [NUM_BLOCKS-1:0] req_valid, + input wire [NUM_BLOCKS-1:0] req_rw, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] req_mask, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_SIZE-1:0] req_byteen, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][ADDR_WIDTH-1:0] req_addr, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][MEM_ATTR_WIDTH-1:0] req_attr, + input wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] req_data, + input wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] req_tag, + output wire [NUM_BLOCKS-1:0] req_ready, + + // responses + output wire [NUM_BLOCKS-1:0] rsp_valid, + output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0] rsp_mask, + output wire [NUM_BLOCKS-1:0][NUM_LANES-1:0][WORD_WIDTH-1:0] rsp_data, + output wire [NUM_BLOCKS-1:0][TAG_WIDTH-1:0] rsp_tag, + input wire [NUM_BLOCKS-1:0] rsp_ready, + + // DMA requests + input wire dma_req_valid, + input wire dma_req_rw, + input wire [DMA_ADDR_WIDTH-1:0] dma_req_addr, + input wire [DMA_DATA_SIZE*8-1:0] dma_req_data, + input wire [DMA_DATA_SIZE-1:0] dma_req_byteen, + input wire [DMA_TAG_WIDTH-1:0] dma_req_tag, + output wire dma_req_ready, + + // DMA responses + output wire dma_rsp_valid, + output wire [DMA_DATA_SIZE*8-1:0] dma_rsp_data, + output wire [DMA_TAG_WIDTH-1:0] dma_rsp_tag, + input wire dma_rsp_ready +); + localparam REQ_DATAW = NUM_LANES + 1 + NUM_LANES * (LSU_ADDR_WIDTH + WORD_WIDTH + WORD_SIZE + MEM_ATTR_WIDTH) + TAG_WIDTH; + localparam DMA_REQ_DATAW = 1 + DMA_ADDR_WIDTH + DMA_DATA_SIZE * 8 + DMA_DATA_SIZE + MEM_ATTR_WIDTH + DMA_TAG_WIDTH; + + VX_lsu_mem_if #( + .NUM_LANES (NUM_LANES), + .DATA_SIZE (WORD_SIZE), + .TAG_WIDTH (TAG_WIDTH) + ) lsu_mem_if[NUM_BLOCKS](); + + VX_mem_bus_if #( + .DATA_SIZE (NUM_BANKS * WORD_SIZE), + .TAG_WIDTH (DMA_TAG_WIDTH), + .ADDR_WIDTH (DMA_ADDR_WIDTH) + ) dma_bus_if(); + + for (genvar k = 0; k < NUM_BLOCKS; ++k) begin : g_blocks + wire [NUM_LANES-1:0][LSU_ADDR_WIDTH-1:0] lsu_addr; + for (genvar i = 0; i < NUM_LANES; ++i) begin : g_addr + assign lsu_addr[i] = LSU_ADDR_WIDTH'(req_addr[k][i]); + end + + VX_elastic_buffer #( + .DATAW (REQ_DATAW), + .SIZE (1) + ) req_buf ( + .clk (clk), + .reset (reset), + .valid_in (req_valid[k]), + .ready_in (req_ready[k]), + .data_in ({req_mask[k], req_rw[k], lsu_addr, req_data[k], req_byteen[k], req_attr[k], req_tag[k]}), + .data_out (lsu_mem_if[k].req_data), + .valid_out (lsu_mem_if[k].req_valid), + .ready_out (lsu_mem_if[k].req_ready) + ); + + assign rsp_valid[k] = lsu_mem_if[k].rsp_valid; + assign rsp_mask[k] = lsu_mem_if[k].rsp_data.mask; + assign rsp_data[k] = lsu_mem_if[k].rsp_data.data; + assign rsp_tag[k] = lsu_mem_if[k].rsp_data.tag; + assign lsu_mem_if[k].rsp_ready = rsp_ready[k]; + end + + VX_elastic_buffer #( + .DATAW (DMA_REQ_DATAW), + .SIZE (1) + ) dma_req_buf ( + .clk (clk), + .reset (reset), + .valid_in (dma_req_valid), + .ready_in (dma_req_ready), + .data_in ({dma_req_rw, dma_req_addr, dma_req_data, dma_req_byteen, MEM_ATTR_WIDTH'(0), dma_req_tag}), + .data_out (dma_bus_if.req_data), + .valid_out (dma_bus_if.req_valid), + .ready_out (dma_bus_if.req_ready) + ); + + assign dma_rsp_valid = dma_bus_if.rsp_valid; + assign dma_rsp_data = dma_bus_if.rsp_data.data; + assign dma_rsp_tag = dma_bus_if.rsp_data.tag; + assign dma_bus_if.rsp_ready = dma_rsp_ready; + + VX_lmem_unit #( + .INSTANCE_ID (INSTANCE_ID), + .SIZE (SIZE), + .NUM_BLOCKS (NUM_BLOCKS), + .NUM_LANES (NUM_LANES), + .NUM_BANKS (NUM_BANKS), + .WORD_SIZE (WORD_SIZE), + .ADDR_WIDTH (ADDR_WIDTH), + .TAG_WIDTH (TAG_WIDTH), + .DMA_ENABLE (1), + .DMA_TAG_WIDTH (DMA_TAG_WIDTH), + .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED) + ) lmem_unit ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (lmem_perf), + `endif + .lsu_mem_if (lsu_mem_if), + .dma_bus_if (dma_bus_if) + ); + +endmodule diff --git a/hw/unittest/local_mem/main.cpp b/hw/unittest/lmem_unit/main.cpp similarity index 100% rename from hw/unittest/local_mem/main.cpp rename to hw/unittest/lmem_unit/main.cpp diff --git a/hw/unittest/local_mem/VX_local_mem_top.sv b/hw/unittest/local_mem/VX_local_mem_top.sv deleted file mode 100644 index 3bbf4b9c1f..0000000000 --- a/hw/unittest/local_mem/VX_local_mem_top.sv +++ /dev/null @@ -1,115 +0,0 @@ -// Copyright © 2019-2023 -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -`include "VX_define.vh" - -module VX_local_mem_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - - // Size of cache in bytes - parameter SIZE = (1024*16*8), - - // Number of Word requests per cycle - parameter NUM_REQS = 4, - // Number of banks - parameter NUM_BANKS = 4, - - // Size of a word in bytes - parameter WORD_SIZE = `VX_CFG_XLEN/8, - - // Request tag size - parameter TAG_WIDTH = 16 + UUID_WIDTH, - - // Address width - parameter NUM_WORDS = SIZE / WORD_SIZE, - parameter WORDS_PER_BANK = NUM_WORDS / NUM_BANKS, - parameter BANK_ADDR_WIDTH = `CLOG2(WORDS_PER_BANK), - parameter ADDR_WIDTH = BANK_ADDR_WIDTH + `CLOG2(NUM_BANKS) - ) ( - input wire clk, - input wire reset, - - // Core request - input wire [NUM_REQS-1:0] mem_req_valid, - input wire [NUM_REQS-1:0] mem_req_rw, - input wire [NUM_REQS-1:0][WORD_SIZE-1:0] mem_req_byteen, - input wire [NUM_REQS-1:0][ADDR_WIDTH-1:0] mem_req_addr, - input wire [NUM_REQS-1:0][MEM_ATTR_WIDTH-1:0] mem_req_attr, - input wire [NUM_REQS-1:0][WORD_SIZE*8-1:0] mem_req_data, - input wire [NUM_REQS-1:0][TAG_WIDTH-1:0] mem_req_tag, - output wire [NUM_REQS-1:0] mem_req_ready, - - // Core response - output wire [NUM_REQS-1:0] mem_rsp_valid, - output wire [NUM_REQS-1:0][WORD_SIZE*8-1:0] mem_rsp_data, - output wire [NUM_REQS-1:0][TAG_WIDTH-1:0] mem_rsp_tag, - input wire [NUM_REQS-1:0] mem_rsp_ready -); - VX_mem_bus_if #( - .DATA_SIZE (WORD_SIZE), - .TAG_WIDTH (TAG_WIDTH), - .ADDR_WIDTH(ADDR_WIDTH) - ) lsu_bus_if[NUM_REQS](); - - VX_mem_bus_if #( - .DATA_SIZE (NUM_BANKS * WORD_SIZE), - .TAG_WIDTH (LMEM_DMA_TAG_WIDTH), - .ADDR_WIDTH(BANK_ADDR_WIDTH) - ) dma_bus_if(); - - // LSU request - for (genvar i = 0; i < NUM_REQS; ++i) begin - assign lsu_bus_if[i].req_valid = mem_req_valid[i]; - assign lsu_bus_if[i].req_data.rw = mem_req_rw[i]; - assign lsu_bus_if[i].req_data.byteen = mem_req_byteen[i]; - assign lsu_bus_if[i].req_data.addr = mem_req_addr[i]; - assign lsu_bus_if[i].req_data.attr = mem_req_attr[i]; - assign lsu_bus_if[i].req_data.data = mem_req_data[i]; - assign lsu_bus_if[i].req_data.tag = mem_req_tag[i]; - assign mem_req_ready[i] = lsu_bus_if[i].req_ready; - end - - // LSU response - for (genvar i = 0; i < NUM_REQS; ++i) begin - assign mem_rsp_valid[i] = lsu_bus_if[i].rsp_valid; - assign mem_rsp_data[i] = lsu_bus_if[i].rsp_data.data; - assign mem_rsp_tag[i] = lsu_bus_if[i].rsp_data.tag; - assign lsu_bus_if[i].rsp_ready = mem_rsp_ready[i]; - end - - // DMA port not exercised by this testbench - assign dma_bus_if.req_valid = '0; - assign dma_bus_if.req_data = '0; - assign dma_bus_if.rsp_ready = '1; - `UNUSED_VAR (dma_bus_if.req_ready) - `UNUSED_VAR (dma_bus_if.rsp_valid) - `UNUSED_VAR (dma_bus_if.rsp_data) - - VX_local_mem #( - .INSTANCE_ID(INSTANCE_ID), - .SIZE (SIZE), - .NUM_REQS (NUM_REQS), - .NUM_BANKS (NUM_BANKS), - .WORD_SIZE (WORD_SIZE), - .ADDR_WIDTH (ADDR_WIDTH), - .TAG_WIDTH (TAG_WIDTH), - .AMO_ENABLE (`VX_CFG_EXT_A_ENABLED), - .OUT_BUF (3) - ) local_mem ( - .clk (clk), - .reset (reset), - .lsu_bus_if (lsu_bus_if), - .dma_bus_if (dma_bus_if) - ); - -endmodule diff --git a/hw/unittest/mem_unit/VX_mem_unit_top.sv b/hw/unittest/mem_unit/VX_mem_unit_top.sv index e88c3d87b9..12d186a47d 100644 --- a/hw/unittest/mem_unit/VX_mem_unit_top.sv +++ b/hw/unittest/mem_unit/VX_mem_unit_top.sv @@ -14,29 +14,19 @@ `include "VX_define.vh" module VX_mem_unit_top import VX_gpu_pkg::*; #( - parameter `STRING INSTANCE_ID = "", - parameter LSU_WORD_WIDTH = LSU_WORD_SIZE * 8 + parameter `STRING INSTANCE_ID = "" ) ( // Clock input wire clk, input wire reset, - // LSU memory request + // LSU block memory clients input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_valid, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_rw, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_req_mask, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_SIZE-1:0] lsu_req_byteen, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_ADDR_WIDTH-1:0] lsu_req_addr, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][MEM_ATTR_WIDTH-1:0] lsu_req_user, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_req_data, - input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_req_tag, + input lsu_req_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_data, output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_req_ready, - // LSU memory response output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_valid, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0] lsu_rsp_mask, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][`VX_CFG_NUM_LSU_LANES-1:0][LSU_WORD_WIDTH-1:0] lsu_rsp_data, - output wire [`VX_CFG_NUM_LSU_BLOCKS-1:0][LSU_TAG_WIDTH-1:0] lsu_rsp_tag, + output lsu_rsp_data_t [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_data, input wire [`VX_CFG_NUM_LSU_BLOCKS-1:0] lsu_rsp_ready, // Memory request @@ -55,32 +45,62 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( input wire [DCACHE_NUM_REQS-1:0][DCACHE_TAG_WIDTH-1:0] mem_rsp_tag, output wire [DCACHE_NUM_REQS-1:0] mem_rsp_ready ); +`ifdef VX_CFG_LMEM_ENABLE VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[`VX_CFG_NUM_LSU_BLOCKS](); - - // LSU memory request - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_req - assign lsu_mem_if[i].req_valid = lsu_req_valid[i]; - assign lsu_mem_if[i].req_data.rw = lsu_req_rw[i]; - assign lsu_mem_if[i].req_data.mask = lsu_req_mask[i]; - assign lsu_mem_if[i].req_data.byteen = lsu_req_byteen[i]; - assign lsu_mem_if[i].req_data.addr = lsu_req_addr[i]; - assign lsu_mem_if[i].req_data.user = lsu_req_user[i]; - assign lsu_mem_if[i].req_data.data = lsu_req_data[i]; - assign lsu_mem_if[i].req_data.tag = lsu_req_tag[i]; - assign lsu_req_ready[i] = lsu_mem_if[i].req_ready; - end + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); +`endif - // LSU memory response - for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_rsp - assign lsu_rsp_valid[i] = lsu_mem_if[i].rsp_valid; - assign lsu_rsp_mask[i] = lsu_mem_if[i].rsp_data.mask; - assign lsu_rsp_data[i] = lsu_mem_if[i].rsp_data.data; - assign lsu_rsp_tag[i] = lsu_mem_if[i].rsp_data.tag; - assign lsu_mem_if[i].rsp_ready = lsu_rsp_ready[i]; + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); + + for (genvar i = 0; i < `VX_CFG_NUM_LSU_BLOCKS; ++i) begin : g_lsu_mem_paths + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[i]; + assign client_if.req_data = lsu_req_data[i]; + assign lsu_req_ready[i] = client_if.req_ready; + assign lsu_rsp_valid[i] = client_if.rsp_valid; + assign lsu_rsp_data[i] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[i]; + + `ifdef TCU_META_ENABLE + VX_lsu_sched_if tcu_mem_if(); + assign tcu_mem_if.req_valid = 1'b0; + assign tcu_mem_if.req_data = '0; + assign tcu_mem_if.rsp_ready = 1'b1; + `UNUSED_VAR (tcu_mem_if.req_ready) + `UNUSED_VAR (tcu_mem_if.rsp_valid) + `UNUSED_VAR (tcu_mem_if.rsp_data) + `endif + + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d", INSTANCE_ID, i))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (tcu_mem_if), + `endif + `UNUSED_PIN (sched_empty), + `UNUSED_PIN (store_empty), + `UNUSED_PIN (busy), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if[i]), + `endif + .dcache_bus_if (lsu_dcache_bus_if[i * DCACHE_CHANNELS +: DCACHE_CHANNELS]) + ); end VX_mem_bus_if #( @@ -116,9 +136,6 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( assign dcr_flush_if.req = 1'b0; `UNUSED_VAR (dcr_flush_if.done) - wire mem_unit_empty; - `UNUSED_VAR (mem_unit_empty) - VX_mem_unit #( .INSTANCE_ID (INSTANCE_ID) ) mem_unit ( @@ -127,10 +144,14 @@ module VX_mem_unit_top import VX_gpu_pkg::*; #( `ifdef PERF_ENABLE .lmem_perf (lmem_perf), `endif - .empty (mem_unit_empty), - .lsu_mem_if (lsu_mem_if), + `ifdef VX_CFG_LMEM_ENABLE + .lmem_if (lsu_lmem_if), + `endif + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_if), - .dcache_bus_if (mem_bus_if) + .flush_l1_only (1'b0), + .dcache_bus_if (mem_bus_if), + `UNUSED_PIN (busy) ); endmodule diff --git a/hw/unittest/mm_axi_arb/Makefile b/hw/unittest/mm_axi_arb/Makefile new file mode 100644 index 0000000000..7ff7e7effb --- /dev/null +++ b/hw/unittest/mm_axi_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := mm_axi_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_mm_axi_arb_top + +include ../common.mk diff --git a/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv new file mode 100644 index 0000000000..73f72b6724 --- /dev/null +++ b/hw/unittest/mm_axi_arb/VX_mm_axi_arb_top.sv @@ -0,0 +1,159 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_platform.vh" + +// Harness for the ID-routed, multi-outstanding 2:1 VX_mm_axi_arb; the +// testbench plays both masters and the slave. +module VX_mm_axi_arb_top #( + parameter NUM_INPUTS = 2, + parameter ADDR_WIDTH = 32, + parameter DATA_WIDTH = 32, + parameter ID_WIDTH = 8 +) ( + input wire clk, + input wire reset, + + input wire [NUM_INPUTS-1:0] s_awvalid, + output wire [NUM_INPUTS-1:0] s_awready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_awaddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_awid, + input wire [NUM_INPUTS-1:0][7:0] s_awlen, + + input wire [NUM_INPUTS-1:0] s_wvalid, + output wire [NUM_INPUTS-1:0] s_wready, + input wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_wdata, + input wire [NUM_INPUTS-1:0] s_wlast, + + output wire [NUM_INPUTS-1:0] s_bvalid, + input wire [NUM_INPUTS-1:0] s_bready, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_bid, + + input wire [NUM_INPUTS-1:0] s_arvalid, + output wire [NUM_INPUTS-1:0] s_arready, + input wire [NUM_INPUTS-1:0][ADDR_WIDTH-1:0] s_araddr, + input wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_arid, + input wire [NUM_INPUTS-1:0][7:0] s_arlen, + + output wire [NUM_INPUTS-1:0] s_rvalid, + input wire [NUM_INPUTS-1:0] s_rready, + output wire [NUM_INPUTS-1:0][DATA_WIDTH-1:0] s_rdata, + output wire [NUM_INPUTS-1:0] s_rlast, + output wire [NUM_INPUTS-1:0][ID_WIDTH-1:0] s_rid, + + output wire m_awvalid, + input wire m_awready, + output wire [ADDR_WIDTH-1:0] m_awaddr, + output wire [ID_WIDTH-1:0] m_awid, + output wire [7:0] m_awlen, + + output wire m_wvalid, + input wire m_wready, + output wire [DATA_WIDTH-1:0] m_wdata, + output wire m_wlast, + + input wire m_bvalid, + output wire m_bready, + input wire [ID_WIDTH-1:0] m_bid, + + output wire m_arvalid, + input wire m_arready, + output wire [ADDR_WIDTH-1:0] m_araddr, + output wire [ID_WIDTH-1:0] m_arid, + output wire [7:0] m_arlen, + + input wire m_rvalid, + output wire m_rready, + input wire [DATA_WIDTH-1:0] m_rdata, + input wire m_rlast, + input wire [ID_WIDTH-1:0] m_rid +); + localparam STRB_WIDTH = DATA_WIDTH / 8; + + wire [NUM_INPUTS-1:0][1:0] s_bresp, s_rresp; + wire [STRB_WIDTH-1:0] m_wstrb; + `UNUSED_VAR ({s_bresp, s_rresp, m_wstrb}) + + VX_mm_axi_arb #( + .NUM_INPUTS (NUM_INPUTS), + .ADDR_WIDTH (ADDR_WIDTH), + .DATA_WIDTH (DATA_WIDTH), + .ID_WIDTH (ID_WIDTH), + .ARBITER ("R"), + .MULTI_OUT (1) + ) arb ( + .clk (clk), + .reset (reset), + + .s_awvalid (s_awvalid), + .s_awready (s_awready), + .s_awaddr (s_awaddr), + .s_awid (s_awid), + .s_awlen (s_awlen), + + .s_wvalid (s_wvalid), + .s_wready (s_wready), + .s_wdata (s_wdata), + .s_wstrb ({NUM_INPUTS{{STRB_WIDTH{1'b1}}}}), + .s_wlast (s_wlast), + + .s_bvalid (s_bvalid), + .s_bready (s_bready), + .s_bid (s_bid), + .s_bresp (s_bresp), + + .s_arvalid (s_arvalid), + .s_arready (s_arready), + .s_araddr (s_araddr), + .s_arid (s_arid), + .s_arlen (s_arlen), + + .s_rvalid (s_rvalid), + .s_rready (s_rready), + .s_rdata (s_rdata), + .s_rlast (s_rlast), + .s_rid (s_rid), + .s_rresp (s_rresp), + + .m_awvalid (m_awvalid), + .m_awready (m_awready), + .m_awaddr (m_awaddr), + .m_awid (m_awid), + .m_awlen (m_awlen), + + .m_wvalid (m_wvalid), + .m_wready (m_wready), + .m_wdata (m_wdata), + .m_wstrb (m_wstrb), + .m_wlast (m_wlast), + + .m_bvalid (m_bvalid), + .m_bready (m_bready), + .m_bid (m_bid), + .m_bresp (2'b00), + + .m_arvalid (m_arvalid), + .m_arready (m_arready), + .m_araddr (m_araddr), + .m_arid (m_arid), + .m_arlen (m_arlen), + + .m_rvalid (m_rvalid), + .m_rready (m_rready), + .m_rdata (m_rdata), + .m_rlast (m_rlast), + .m_rid (m_rid), + .m_rresp (2'b00) + ); + +endmodule diff --git a/hw/unittest/mm_axi_arb/main.cpp b/hw/unittest/mm_axi_arb/main.cpp new file mode 100644 index 0000000000..0dfd3bbc7c --- /dev/null +++ b/hw/unittest/mm_axi_arb/main.cpp @@ -0,0 +1,415 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The properties under test, for the ID-routed (MULTI_OUT) 2:1 arbiter: +// * the slave sees each write's data in the order it saw the addresses, +// whole bursts at a time, whichever channel each master offers first; +// * an offer the slave has not taken stays unchanged (AXI4 A3.2.1); +// * no master waits on AWREADY for its data, so a slave that takes the +// data before the address cannot deadlock it; +// * B and R reach the master that issued them, in order; +// * a write whose address and data are offered together moves in one cycle. + +#include +#include +#include +#include +#include "VVX_mm_axi_arb_top.h" +#include "verilated.h" + +namespace { + +constexpr int NUM_MASTERS = 2; +constexpr int ID_SRC_BIT = 7; // the arbiter's source field (ID_WIDTH-1) + +VVX_mm_axi_arb_top* dut = nullptr; +int failures = 0; +uint32_t rng = 1; + +uint32_t rnd() { + rng = rng * 1103515245u + 12345u; + return rng >> 16; +} + +bool chance(int pct) { return int(rnd() % 100) < pct; } + +void check(bool ok, const char* what) { + printf(" %-62s %s\n", what, ok ? "PASS" : "*** FAIL ***"); + if (!ok) { + ++failures; + } +} + +int errors = 0; +void error(const char* msg, long a = 0, long b = 0) { + if (errors < 10) { + printf(" error: %s (%ld, %ld)\n", msg, a, b); + } + ++errors; +} + +// Packed [N][W] port helpers. +template uint32_t get(T v, int i, int w) { return uint32_t((uint64_t(v) >> (i * w)) & ((1ull << w) - 1)); } +template void put(T& v, int i, int w, uint32_t x) { + uint64_t m = ((1ull << w) - 1) << (i * w); + v = T((uint64_t(v) & ~m) | ((uint64_t(x) << (i * w)) & m)); +} + +uint32_t waddr(int m, int k) { return (uint32_t(m) << 28) | (uint32_t(k) << 8); } +uint32_t wdata(int m, int k, int beat) { return waddr(m, k) | uint32_t(beat); } + +enum Style { + STYLE_FREE, // AW and W offered independently; data may lead its address + STYLE_AWFIRST // data only after its address was taken, one write at a time +}; + +struct Master { + int id; + Style style; + int pct; // offer probability per cycle + int num_writes, num_reads; + std::vector len; // per write, beats - 1 + int aw_next = 0, aw_done = 0; + bool aw_offer = false; + int w_next = 0, w_beat = 0, w_done = 0; + bool w_offer = false; + int b_recv = 0; + int ar_next = 0; + bool ar_offer = false; + std::deque> r_exp; // addr, len + int r_beat = 0, r_done = 0; + bool done() const { return b_recv == num_writes && r_done == num_reads; } +}; + +struct SlaveCfg { + int aw_pct, w_pct, ar_pct, b_pct, r_pct; + bool w_first; // AWREADY only once that write's data has begun +}; + +struct AwRec { uint32_t addr; uint32_t id; int len; }; + +bool run(std::vector& ms, const SlaveCfg& sc, int max_cycles, int* cycles_out, + int* writes_out = nullptr, int* last_write_cycle = nullptr) { + std::deque aw_q; + std::deque> w_bursts; + std::vector w_cur; + int w_started = 0, aw_taken = 0; + std::deque b_q; + std::deque> ar_q; // addr|id, beats + std::deque r_q; + int r_beat = 0; + bool b_hold = false, r_hold = false; + int writes = 0; + // offer-stability tracking on the slave side + bool pend_aw = false, pend_w = false, pend_ar = false; + uint64_t pend_aw_v = 0, pend_w_v = 0, pend_ar_v = 0; + + int cycle = 0; + for (; cycle < max_cycles; ++cycle) { + bool all_done = true; + for (auto& m : ms) { + all_done = all_done && m.done(); + } + if (all_done && aw_q.empty() && w_bursts.empty() && b_q.empty() && r_q.empty()) { + break; + } + + // ---- masters drive ---- + dut->s_awvalid = 0; dut->s_wvalid = 0; dut->s_arvalid = 0; + dut->s_wlast = 0; dut->s_bready = 0; dut->s_rready = 0; + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (!m.aw_offer && m.aw_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.aw_next - m.w_done <= 3) + : (m.aw_next == m.w_done); + m.aw_offer = ok && chance(m.pct); + } + if (m.aw_offer) { + dut->s_awvalid |= (1 << i); + put(dut->s_awaddr, i, 32, waddr(i, m.aw_next)); + put(dut->s_awid, i, 8, m.id); + put(dut->s_awlen, i, 8, m.len[m.aw_next]); + } + if (!m.w_offer && m.w_next < m.num_writes) { + bool ok = (m.style == STYLE_FREE) ? (m.w_next <= m.aw_done + 1) + : (m.w_next < m.aw_done); + m.w_offer = ok && (m.w_beat > 0 || chance(m.pct)); + } + if (m.w_offer) { + dut->s_wvalid |= (1 << i); + put(dut->s_wdata, i, 32, wdata(i, m.w_next, m.w_beat)); + if (m.w_beat == m.len[m.w_next]) { + dut->s_wlast |= (1 << i); + } + } + if (!m.ar_offer && m.ar_next < m.num_reads) { + m.ar_offer = chance(m.pct); + } + if (m.ar_offer) { + dut->s_arvalid |= (1 << i); + put(dut->s_araddr, i, 32, (uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80); + put(dut->s_arid, i, 8, m.id); + put(dut->s_arlen, i, 8, m.ar_next % 3); + } + if (chance(70)) { + dut->s_bready |= (1 << i); + } + if (chance(70)) { + dut->s_rready |= (1 << i); + } + } + + // ---- slave drives ---- + dut->m_awready = chance(sc.aw_pct) && (!sc.w_first || w_started > aw_taken); + dut->m_wready = chance(sc.w_pct); + dut->m_arready = chance(sc.ar_pct); + if (!b_hold && !b_q.empty() && chance(sc.b_pct)) { + b_hold = true; + } + dut->m_bvalid = b_hold; + dut->m_bid = b_hold ? b_q.front() : 0; + if (!r_hold && !r_q.empty() && chance(sc.r_pct)) { + r_hold = true; + } + dut->m_rvalid = r_hold; + if (r_hold) { + dut->m_rdata = r_q.front().addr | uint32_t(r_beat); + dut->m_rid = r_q.front().id; + dut->m_rlast = (r_beat == r_q.front().len); + } + + dut->clk = 0; + dut->eval(); + + // ---- offer stability, slave side ---- + uint64_t aw_v = (uint64_t(dut->m_awaddr) << 16) | (uint64_t(dut->m_awid) << 8) | dut->m_awlen; + uint64_t w_v = (uint64_t(dut->m_wdata) << 1) | dut->m_wlast; + uint64_t ar_v = (uint64_t(dut->m_araddr) << 16) | (uint64_t(dut->m_arid) << 8) | dut->m_arlen; + if (pend_aw && (!dut->m_awvalid || aw_v != pend_aw_v)) { + error("AW offer changed before it was taken", cycle); + } + if (pend_w && (!dut->m_wvalid || w_v != pend_w_v)) { + error("W offer changed before it was taken", cycle); + } + if (pend_ar && (!dut->m_arvalid || ar_v != pend_ar_v)) { + error("AR offer changed before it was taken", cycle); + } + pend_aw = dut->m_awvalid && !dut->m_awready; pend_aw_v = aw_v; + pend_w = dut->m_wvalid && !dut->m_wready; pend_w_v = w_v; + pend_ar = dut->m_arvalid && !dut->m_arready; pend_ar_v = ar_v; + + // ---- slave handshakes ---- + if (dut->m_awvalid && dut->m_awready) { + aw_q.push_back({uint32_t(dut->m_awaddr), uint32_t(dut->m_awid), int(dut->m_awlen)}); + ++aw_taken; + } + if (dut->m_wvalid && dut->m_wready) { + if (w_cur.empty()) { + ++w_started; + } + w_cur.push_back(uint32_t(dut->m_wdata)); + if (dut->m_wlast) { + w_bursts.push_back(w_cur); + w_cur.clear(); + } + } + while (!aw_q.empty() && !w_bursts.empty()) { + auto aw = aw_q.front(); auto& bu = w_bursts.front(); + if (int(bu.size()) != aw.len + 1) { + error("burst length differs from its AW", bu.size(), aw.len); + } + for (size_t b = 0; b < bu.size(); ++b) { + if (bu[b] != (aw.addr | uint32_t(b))) { + error("W data does not belong to the AW at its position", bu[b], aw.addr); + } + } + if (((aw.addr >> 28) & 1) != ((aw.id >> ID_SRC_BIT) & 1)) { + error("AW id source field wrong", aw.addr, aw.id); + } + b_q.push_back(aw.id); + aw_q.pop_front(); w_bursts.pop_front(); + ++writes; + if (last_write_cycle) { + *last_write_cycle = cycle; + } + } + if (dut->m_arvalid && dut->m_arready) { + r_q.push_back({uint32_t(dut->m_araddr), uint32_t(dut->m_arid), int(dut->m_arlen)}); + } + bool b_fire = b_hold && dut->m_bready; + bool r_fire = r_hold && dut->m_rready; + + // ---- master handshakes ---- + for (int i = 0; i < NUM_MASTERS; ++i) { + auto& m = ms[i]; + if (m.aw_offer && get(dut->s_awready, i, 1)) { + m.aw_offer = false; + ++m.aw_next; + ++m.aw_done; + } + if (m.w_offer && get(dut->s_wready, i, 1)) { + m.w_offer = false; + if (m.w_beat == m.len[m.w_next]) { + m.w_beat = 0; + ++m.w_next; + ++m.w_done; + } else { + ++m.w_beat; + } + } + if (m.ar_offer && get(dut->s_arready, i, 1)) { + m.ar_offer = false; + m.r_exp.push_back({(uint32_t(i) << 28) | (uint32_t(m.ar_next) << 8) | 0x80, m.ar_next % 3}); + ++m.ar_next; + } + if (get(dut->s_bvalid, i, 1) && get(dut->s_bready, i, 1)) { + if (get(dut->s_bid, i, 8) != uint32_t(m.id)) { + error("B to the wrong master or id", i, get(dut->s_bid, i, 8)); + } + if (m.b_recv >= m.aw_done) { + error("B before its write was issued", i, m.b_recv); + } + ++m.b_recv; + } + if (get(dut->s_rvalid, i, 1) && get(dut->s_rready, i, 1)) { + if (m.r_exp.empty()) { + error("unexpected R", i); + } else { + auto e = m.r_exp.front(); + if (get(dut->s_rid, i, 8) != uint32_t(m.id)) { + error("R to the wrong master or id", i, get(dut->s_rid, i, 8)); + } + if (get(dut->s_rdata, i, 32) != (e.first | uint32_t(m.r_beat))) { + error("R data wrong", i, get(dut->s_rdata, i, 32)); + } + bool last = (m.r_beat == e.second); + if (get(dut->s_rlast, i, 1) != uint32_t(last)) { + error("RLAST wrong", i, m.r_beat); + } + if (last) { + m.r_exp.pop_front(); + m.r_beat = 0; + ++m.r_done; + } else { + ++m.r_beat; + } + } + } + } + + dut->clk = 1; + dut->eval(); + + if (b_fire) { + b_q.pop_front(); + b_hold = false; + } + if (r_fire) { + if (r_beat == r_q.front().len) { + r_q.pop_front(); + r_beat = 0; + } else { + ++r_beat; + } + r_hold = false; + } + } + *cycles_out = cycle; + if (writes_out) { + *writes_out = writes; + } + for (auto& m : ms) { + if (!m.done()) { + return false; + } + } + return true; +} + +void do_reset() { + dut->reset = 1; + for (int i = 0; i < 4; ++i) { + dut->clk = 0; + dut->eval(); + dut->clk = 1; + dut->eval(); + } + dut->reset = 0; + dut->clk = 0; dut->eval(); +} + +Master make_master(int id, Style style, int pct, int writes, int reads, int max_len) { + Master m; + m.id = id; m.style = style; m.pct = pct; + m.num_writes = writes; m.num_reads = reads; + for (int k = 0; k < writes; ++k) { + m.len.push_back(int(rnd() % (max_len + 1))); + } + return m; +} + +} // namespace + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + dut = new VVX_mm_axi_arb_top; + int cycles = 0; + + printf("1. two masters, random traffic and back-pressure\n"); + for (uint32_t seed = 1; seed <= 8; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 60, 400, 300, 3), + make_master(6, STYLE_AWFIRST, 60, 400, 300, 3) }; + SlaveCfg sc = {50, 50, 50, 60, 60, false}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: all writes and reads complete in order (%d cycles)", seed, cycles); + check(ok && errors == 0, what); + } + + printf("2. a slave that takes the data before the address\n"); + for (uint32_t seed = 11; seed <= 14; ++seed) { + rng = seed; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 70, 400, 100, 3), + make_master(6, STYLE_FREE, 70, 400, 100, 3) }; + SlaveCfg sc = {60, 60, 60, 70, 70, true}; + bool ok = run(ms, sc, 200000, &cycles); + char what[96]; + snprintf(what, sizeof(what), "seed %u: no deadlock, data still in address order", seed); + check(ok && errors == 0, what); + } + + printf("3. single-beat writes with address and data offered together\n"); + { + rng = 21; + errors = 0; + do_reset(); + std::vector ms = { make_master(5, STYLE_FREE, 100, 1000, 0, 0), + make_master(6, STYLE_FREE, 0, 0, 0, 0) }; + SlaveCfg sc = {100, 100, 100, 100, 100, false}; + int writes = 0, last = 0; + bool ok = run(ms, sc, 20000, &cycles, &writes, &last); + char what[96]; + snprintf(what, sizeof(what), "one write per cycle (%d writes in %d cycles)", writes, last + 1); + check(ok && errors == 0 && last + 1 <= writes + 16, what); + } + + printf("\n%s (%d failure%s)\n", failures ? "FAILED" : "PASSED", + failures, failures == 1 ? "" : "s"); + delete dut; + return failures ? 1 : 0; +} diff --git a/hw/unittest/raster_core/VX_raster_core_top.sv b/hw/unittest/raster_core/VX_raster_core_top.sv index 9e1665a093..fb64f41c63 100644 --- a/hw/unittest/raster_core/VX_raster_core_top.sv +++ b/hw/unittest/raster_core/VX_raster_core_top.sv @@ -108,7 +108,6 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( VX_raster_core #( .INSTANCE_ID (INSTANCE_ID), - .INSTANCE_IDX (INSTANCE_IDX), .NUM_INSTANCES (NUM_INSTANCES), .NUM_SLICES (NUM_SLICES), .TILE_LOGSIZE (TILE_LOGSIZE), @@ -120,6 +119,7 @@ module VX_raster_core_top import VX_gpu_pkg::*; import VX_raster_pkg::*; #( `SCOPE_IO_BIND (0) .clk (clk), .reset (reset), + .instance_idx (`LOG2UP(NUM_INSTANCES)'(INSTANCE_IDX)), `ifdef PERF_ENABLE .perf_raster_if(perf_raster_if), `endif diff --git a/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv index f89563d573..5ff50ede0a 100644 --- a/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv +++ b/hw/unittest/rtu_box_pe/VX_rtu_box_pe_tb.sv @@ -105,7 +105,8 @@ module VX_rtu_box_pe_tb import VX_gpu_pkg::*, VX_fpu_pkg::*, VX_rtu_pkg::*; #( .tag_out (tag_out), `UNUSED_PIN (tag_out_pre), .hit (hit), - .t_near (t_near) + .t_near (t_near), + `UNUSED_PIN (t_near_pre) ); endmodule diff --git a/hw/unittest/stream_arb/Makefile b/hw/unittest/stream_arb/Makefile new file mode 100644 index 0000000000..ee2f11ccfd --- /dev/null +++ b/hw/unittest/stream_arb/Makefile @@ -0,0 +1,23 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := stream_arb + +RTL_DIR := $(VORTEX_HOME)/hw/rtl +DPI_DIR := $(VORTEX_HOME)/hw/dpi + +SRC_DIR := $(VORTEX_HOME)/hw/unittest/$(PROJECT) + +CXXFLAGS := -I$(SRC_DIR) -I$(VORTEX_HOME)/hw/unittest/common -I$(SW_COMMON_DIR) + +SRCS := $(SRC_DIR)/main.cpp + +DBG_TRACE_FLAGS := + +RTL_PKGS := + +RTL_INCLUDE := -I$(RTL_DIR) -I$(DPI_DIR) -I$(RTL_DIR)/libs -I$(SRC_DIR) + +TOP := VX_stream_arb_top + +include ../common.mk diff --git a/hw/unittest/stream_arb/VX_stream_arb_tb.sv b/hw/unittest/stream_arb/VX_stream_arb_tb.sv new file mode 100644 index 0000000000..d94197bf9e --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_tb.sv @@ -0,0 +1,179 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// Self-checking harness around one VX_stream_arb: every input sends numbered +// packets under random valid and output backpressure, and every packet must +// leave once, on an output its input maps to, with the select of its row. +module VX_stream_arb_tb #( + parameter NUM_INPUTS = 1, + parameter NUM_OUTPUTS = 1, + parameter MAX_FANOUT = 8, + parameter SEED = 1 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [31:0] delivered, + output wire [31:0] errors +); + localparam SRC_W = 8; + localparam SEQ_W = 16; + localparam DATAW = SRC_W + SEQ_W; + localparam NUM_REQS = (NUM_INPUTS > NUM_OUTPUTS) ? `CDIV(NUM_INPUTS, NUM_OUTPUTS) : `CDIV(NUM_OUTPUTS, NUM_INPUTS); + localparam SEL_COUNT = `MIN(NUM_INPUTS, NUM_OUTPUTS); + localparam SEL_W = `UP(`CLOG2(NUM_REQS)); + + `STATIC_ASSERT (NUM_INPUTS < (1 << SRC_W), ("invalid parameter: NUM_INPUTS=%0d", NUM_INPUTS)) + + function automatic logic [31:0] hash32(input logic [31:0] x); + logic [31:0] h; + h = x ^ (x >> 16); + h = h * 32'h7feb352d; + h = h ^ (h >> 15); + h = h * 32'h846ca68b; + return h ^ (h >> 16); + endfunction + + reg [31:0] cycle; + + reg [NUM_INPUTS-1:0] valid_in; + reg [NUM_INPUTS-1:0][SEQ_W-1:0] seq_in; + wire [NUM_INPUTS-1:0][DATAW-1:0] data_in; + wire [NUM_INPUTS-1:0] ready_in; + + wire [NUM_OUTPUTS-1:0] valid_out; + wire [NUM_OUTPUTS-1:0][DATAW-1:0] data_out; + wire [NUM_OUTPUTS-1:0] ready_out; + wire [SEL_COUNT-1:0][SEL_W-1:0] sel_out; + + for (genvar i = 0; i < NUM_INPUTS; ++i) begin : g_data_in + assign data_in[i] = {SRC_W'(i), seq_in[i]}; + end + + // outputs stall at random, with a longer stall now and then so the + // arbiter buffers fill up + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_ready_out + wire [1:0] rnd = 2'(hash32(cycle * 32'd977 + 32'd7919 * 32'(o + 1024) + 32'(SEED))); + assign ready_out[o] = (rnd != 0) && (cycle[9:6] != 4'(SEED)); + end + + VX_stream_arb #( + .NUM_INPUTS (NUM_INPUTS), + .NUM_OUTPUTS (NUM_OUTPUTS), + .DATAW (DATAW), + .ARBITER ("R"), + .MAX_FANOUT (MAX_FANOUT), + .OUT_BUF (3) + ) dut ( + .clk (clk), + .reset (reset), + .valid_in (valid_in), + .data_in (data_in), + .ready_in (ready_in), + .valid_out (valid_out), + .data_out (data_out), + .ready_out (ready_out), + .sel_out (sel_out) + ); + + wire [NUM_OUTPUTS-1:0][SRC_W-1:0] src_out; + wire [NUM_OUTPUTS-1:0][SEQ_W-1:0] seq_out; + + for (genvar o = 0; o < NUM_OUTPUTS; ++o) begin : g_data_out + assign {src_out[o], seq_out[o]} = data_out[o]; + end + + reg [NUM_INPUTS-1:0][31:0] sent_cnt, recv_cnt; + reg [NUM_INPUTS-1:0][31:0] sent_sum, recv_sum; + reg [31:0] errors_r; + + logic [NUM_INPUTS-1:0][31:0] recv_inc, recv_sum_inc; + logic [31:0] errors_inc; + + always_comb begin + recv_inc = '0; + recv_sum_inc = '0; + errors_inc = '0; + for (integer o = 0; o < NUM_OUTPUTS; ++o) begin + if (valid_out[o] && ready_out[o]) begin + if (32'(src_out[o]) >= NUM_INPUTS) begin + errors_inc = errors_inc + 1; + end else begin + recv_inc[src_out[o]] = recv_inc[src_out[o]] + 1; + recv_sum_inc[src_out[o]] = recv_sum_inc[src_out[o]] + 32'(seq_out[o]); + if (NUM_INPUTS >= NUM_OUTPUTS) begin + // an input reaches one output, in order, with its row as select + if ((32'(src_out[o]) % NUM_OUTPUTS) != o + || seq_out[o] != recv_cnt[src_out[o]][SEQ_W-1:0] + || 32'(sel_out[o % SEL_COUNT]) != (32'(src_out[o]) / NUM_OUTPUTS)) begin + errors_inc = errors_inc + 1; + end + end else begin + // an input reaches any output of its column + if ((o % NUM_INPUTS) != 32'(src_out[o])) begin + errors_inc = errors_inc + 1; + end + end + end + end + end + end + + always @(posedge clk) begin + if (reset) begin + cycle <= '0; + valid_in <= '0; + seq_in <= '0; + sent_cnt <= '0; + sent_sum <= '0; + recv_cnt <= '0; + recv_sum <= '0; + errors_r <= '0; + end else begin + cycle <= cycle + 1; + for (integer i = 0; i < NUM_INPUTS; ++i) begin + logic [2:0] rnd; + rnd = 3'(hash32(cycle * 32'd977 + 32'd7919 * 32'(i) + 32'(SEED))); + if (valid_in[i] && ready_in[i]) begin + sent_cnt[i] <= sent_cnt[i] + 1; + sent_sum[i] <= sent_sum[i] + 32'(seq_in[i]); + seq_in[i] <= seq_in[i] + 1; + valid_in[i] <= ~drain && (rnd != 0); + end else if (~valid_in[i]) begin + valid_in[i] <= ~drain && (rnd != 0); + end + end + for (integer i = 0; i < NUM_INPUTS; ++i) begin + recv_cnt[i] <= recv_cnt[i] + recv_inc[i]; + recv_sum[i] <= recv_sum[i] + recv_sum_inc[i]; + end + errors_r <= errors_r + errors_inc; + if (errors_inc != 0) begin + $display("cycle %0d: stream_arb %0dx%0d (fanout %0d): misrouted packet", cycle, NUM_INPUTS, NUM_OUTPUTS, MAX_FANOUT); + end + end + end + + logic [31:0] delivered_w; + logic idle_w; + + always_comb begin + delivered_w = '0; + idle_w = (valid_in == '0); + for (integer i = 0; i < NUM_INPUTS; ++i) begin + delivered_w = delivered_w + recv_cnt[i]; + // every input must have been served, and nothing lost + idle_w = idle_w && (recv_cnt[i] != 0) + && (recv_cnt[i] == sent_cnt[i]) + && (recv_sum[i] == sent_sum[i]); + end + end + + assign idle = idle_w; + assign delivered = delivered_w; + assign errors = errors_r; + +endmodule diff --git a/hw/unittest/stream_arb/VX_stream_arb_top.sv b/hw/unittest/stream_arb/VX_stream_arb_top.sv new file mode 100644 index 0000000000..7698928767 --- /dev/null +++ b/hw/unittest/stream_arb/VX_stream_arb_top.sv @@ -0,0 +1,63 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +`include "VX_platform.vh" + +// VX_stream_arb shapes under test, each in its own self-checking harness. +// Over 1.5x MAX_FANOUT requests an arbiter becomes a tree of slices; the +// shapes cover full slices, a last slice of one input or output, one smaller +// than the other side, one not a whole number of rows, and a join that is a +// tree itself. +module VX_stream_arb_top #( + parameter NUM_CFGS = 21 +) ( + input wire clk, + input wire reset, + input wire drain, + output wire idle, + output wire [NUM_CFGS-1:0] busy, + output wire [31:0] delivered, + output wire [31:0] errors +); + // input select | output select | flat + localparam int CFG_NI [NUM_CFGS] = '{64, 17, 65, 13, 19, 34, 36, 40, 26, 27, 129, 200, 1, 4, 2, 1, 4, 5, 8, 3, 1}; + localparam int CFG_NO [NUM_CFGS] = '{ 1, 1, 1, 1, 1, 4, 4, 4, 2, 2, 1, 1, 17, 34, 27, 64, 32, 2, 8, 5, 1}; + localparam int CFG_MF [NUM_CFGS] = '{ 8, 8, 8, 8, 8, 4, 4, 4, 4, 4, 8, 4, 8, 4, 4, 8, 8, 8, 8, 8, 8}; + + wire [NUM_CFGS-1:0] idle_w; + wire [NUM_CFGS-1:0][31:0] delivered_w; + wire [NUM_CFGS-1:0][31:0] errors_w; + + for (genvar c = 0; c < NUM_CFGS; ++c) begin : g_cfgs + VX_stream_arb_tb #( + .NUM_INPUTS (CFG_NI[c]), + .NUM_OUTPUTS (CFG_NO[c]), + .MAX_FANOUT (CFG_MF[c]), + .SEED (c + 1) + ) tb ( + .clk (clk), + .reset (reset), + .drain (drain), + .idle (idle_w[c]), + .delivered (delivered_w[c]), + .errors (errors_w[c]) + ); + end + + logic [31:0] delivered_s, errors_s; + + always_comb begin + delivered_s = '0; + errors_s = '0; + for (integer c = 0; c < NUM_CFGS; ++c) begin + delivered_s = delivered_s + delivered_w[c]; + errors_s = errors_s + errors_w[c]; + end + end + + assign idle = (& idle_w); + assign busy = ~idle_w; + assign delivered = delivered_s; + assign errors = errors_s; + +endmodule diff --git a/hw/unittest/stream_arb/main.cpp b/hw/unittest/stream_arb/main.cpp new file mode 100644 index 0000000000..740795a1ca --- /dev/null +++ b/hw/unittest/stream_arb/main.cpp @@ -0,0 +1,62 @@ +// Copyright © 2019-2023 +// Licensed under the Apache License, Version 2.0. + +// Verilator unit test for VX_stream_arb over the shapes in VX_stream_arb_top. +// Each shape's harness checks its own traffic: random input valids and output +// stalls for a while, then the inputs stop and every packet must have left +// exactly once, on the right output, with the right select. + +#include "vl_simulator.h" +#include "VVX_stream_arb_top.h" +#include +#include + +#ifndef TRACE_START_TIME +#define TRACE_START_TIME 0ull +#endif +#ifndef TRACE_STOP_TIME +#define TRACE_STOP_TIME -1ull +#endif + +static uint64_t timestamp = 0; +static bool trace_en = false; + +double sc_time_stamp() { return timestamp; } +bool sim_trace_enabled() { return trace_en; } +void sim_trace_enable(bool e) { trace_en = e; } + +int main(int argc, char** argv) { + Verilated::commandArgs(argc, argv); + vl_simulator sim; + uint64_t tick = 0; + + constexpr uint32_t kTrafficCycles = 20000; + constexpr uint32_t kDrainCycles = 20000; + + sim->drain = 0; + sim->reset = 1; + tick = sim.step(tick, 8); + sim->reset = 0; + + tick = sim.step(tick, 2 * kTrafficCycles); + sim->drain = 1; + uint32_t cycles = 0; + while (!sim->idle && sim->errors == 0 && cycles < kDrainCycles) { + tick = sim.step(tick, 2); + ++cycles; + } + timestamp = tick; + + std::printf("delivered=%u, errors=%u, idle=%d\n", sim->delivered, sim->errors, sim->idle); + for (uint32_t c = 0; c < 8 * sizeof(sim->busy); ++c) { + if ((sim->busy >> c) & 1) { + std::printf("shape %u did not drain\n", c); + } + } + if (sim->errors != 0 || !sim->idle) { + std::printf("FAILED!\n"); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} diff --git a/hw/unittest/tcu_fedp/Makefile b/hw/unittest/tcu_fedp/Makefile index 7f095276fe..507bbee44c 100644 --- a/hw/unittest/tcu_fedp/Makefile +++ b/hw/unittest/tcu_fedp/Makefile @@ -26,8 +26,14 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # configured VX_CFG_TCU_LATENCY. LATENCY ?= $(patsubst -DVX_CFG_TCU_LATENCY=%,%,$(filter -DVX_CFG_TCU_LATENCY=%, $(XCONFIGS))) -CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) +# Block-scale slots of an MX datapath: the lanes split evenly across them. +SF ?= 1 + +CXXFLAGS += -DNUM_REGS=$(NUM_REGS) -DLATENCY=$(LATENCY) -DSF=$(SF) PARAMS := -GN=$(NUM_REGS) -GLATENCY=$(LATENCY) +ifneq (,$(filter -DVX_CFG_TCU_MX_ENABLE, $(XCONFIGS))) +PARAMS += -GSF=$(SF) +endif SRCS += $(DPI_DIR)/dpi_util.cpp SRCS += $(DPI_DIR)/dpi_float.cpp diff --git a/hw/unittest/tcu_fedp/main.cpp b/hw/unittest/tcu_fedp/main.cpp index 3e11bd1dde..5e60fbcaa1 100644 --- a/hw/unittest/tcu_fedp/main.cpp +++ b/hw/unittest/tcu_fedp/main.cpp @@ -158,6 +158,19 @@ void init_default_fp_format(uint32_t fmt, int *exp_bits, int *sig_bits) { *exp_bits = 5; *sig_bits = 2; break; + case 8: // mxfp8 (E4M3, E8M0 block scale) + *exp_bits = 4; + *sig_bits = 3; + break; + case 9: // mxbf8 (E5M2, E8M0 block scale) + *exp_bits = 5; + *sig_bits = 2; + break; + case 10: // mxfp4 (E2M1, E8M0 block scale) + case 11: // nvfp4 (E2M1, E4M3 block scale) + *exp_bits = 2; + *sig_bits = 1; + break; default: break; } @@ -187,6 +200,7 @@ struct TestConfig { int ulp = 1; // float precision error bits int sparsity = 0; // % of zero inputs int32_t test_id = -1; // Specific test ID to run (for debugging) + int sf_a = -1, sf_b = -1; // fixed block scales (debugging), random when negative }; std::ostream &operator<<(std::ostream &os, const TestConfig &config) { @@ -255,9 +269,17 @@ static void print_format(const std::string& prefix, std::vector& value static int float_fmt_width(int exp_bits, int sig_bits) { int element_bits = 1 + exp_bits + sig_bits; + if (element_bits <= 4) return 4; // fp4 packs eight to a register return 8 * ((element_bits + 7) / 8); } +// Block-scaled (MX/NV) formats carry the 2nd MSB of the format code. +static bool is_mx_fmt(uint32_t fmt) { return (fmt & 0x18) == 0x8; } + +#ifndef SF +#define SF 1 +#endif + static int int_fmt_width(int fmt) { switch (fmt) { case 16: return 32; // int32 @@ -486,6 +508,20 @@ class Testbench { return value; } + // A block scale: E8M0 (MX) or UE4M3 (NVFP4, NaN at 0x7F). Half the draws + // stay within 2^+-8 of one, the rest span the whole encoding. + uint8_t generate_scale(uint32_t fmt) { + std::uniform_int_distribution coin(0, 1); + if (fmt == 11) { + std::uniform_int_distribution any(0, 0x7f); + std::uniform_int_distribution near(0x30, 0x48); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + std::uniform_int_distribution any(0, 255); + std::uniform_int_distribution near(127 - 8, 127 + 8); + return uint8_t(coin(rng_) ? near(rng_) : any(rng_)); + } + uint32_t generate_fp_value(const std::string &feature, uint32_t exp_bits, uint32_t sig_bits, uint32_t test_id) { const uint32_t all_exp = (exp_bits == 32 ? 0xFFFFFFFFu : ((1u << exp_bits) - 1u)); const uint32_t max_frac = (sig_bits == 32 ? 0xFFFFFFFFu : ((1u << sig_bits) - 1u)); @@ -739,7 +775,7 @@ class Testbench { const uint32_t tests_per_feature = (NT + NF - 1) / NF; #ifdef USE_FEDP - FEDP fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); + TfrFedp fedp(config_.exp_bits, config_.sig_bits, (int)config_.frm, config_.W, config_.renorm); #endif uint32_t skipped = 0; @@ -780,6 +816,24 @@ class Testbench { uint32_t c_value_hex; std::memcpy(&c_value_hex, &c_value_float, sizeof(float)); + // Block scales, one per scale slot; lane l (two per register) takes + // slot l * SF / lanes, as the datapath maps them. + const bool mx = is_mx_fmt(config_.fmt_s); + const int lanes = 2 * NUM_REGS; + uint8_t sf_slot_a[SF] = {}, sf_slot_b[SF] = {}; + std::vector sf_lane_a(lanes), sf_lane_b(lanes); + if (mx) { + for (int k = 0; k < SF; ++k) { + sf_slot_a[k] = generate_scale(config_.fmt_s); + sf_slot_b[k] = generate_scale(config_.fmt_s); + if (config_.sf_a >= 0) sf_slot_a[k] = uint8_t(config_.sf_a); + if (config_.sf_b >= 0) sf_slot_b[k] = uint8_t(config_.sf_b); + } + for (int l = 0; l < lanes; ++l) { + sf_lane_a[l] = sf_slot_a[(l * SF) / lanes]; + sf_lane_b[l] = sf_slot_b[(l * SF) / lanes]; + } + } // skip if not in selected test id if (config_.test_id >= 0 && test_id != config_.test_id) continue; @@ -793,6 +847,11 @@ class Testbench { pack_elements(a_value_hex, element_bits, NUM_REGS, a_packed.data()); pack_elements(b_value_hex, element_bits, NUM_REGS, b_packed.data()); + #ifdef VX_CFG_TCU_MX_ENABLE + std::memcpy(&dut_->sf_a, sf_slot_a, SF); + std::memcpy(&dut_->sf_b, sf_slot_b, SF); + #endif + // Apply to DUT for (int i = 0; i < NUM_REGS; i++) { WRITE_WDATA(dut_->a_row, i, a_packed[i]); @@ -821,7 +880,9 @@ class Testbench { std::memcpy(&dut_result, &dut_result_bits, sizeof(float)); #ifdef USE_FEDP - float expected = fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); + float expected = mx ? fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS, + sf_lane_a.data(), sf_lane_b.data(), config_.fmt_s == 11) + : fedp(a_packed.data(), b_packed.data(), c_value_float, NUM_REGS); #else float expected = dot_product(a_value_hex.data(), b_value_hex.data(), c_value_hex, total_elements, config_.exp_bits, config_.sig_bits, config_.fused); @@ -848,6 +909,13 @@ class Testbench { print_float(" bf_values=", b_values_float, true); print_format(" bx_values=", b_value_hex, true); print_float(" c_value=", c_value_float, true); + if (mx) { + std::cout << " sf_a=" << std::hex; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_a[k]); + std::cout << ", sf_b="; + for (int k = 0; k < SF; ++k) std::cout << " 0x" << int(sf_slot_b[k]); + std::cout << std::dec << std::endl; + } print_float(" expected=", expected, true); print_float(" actual=", dut_result, true); std::cout << " delta=" << delta << std::endl; @@ -927,6 +995,10 @@ TestConfig parse_args(int argc, char **argv) { config_.num_tests = std::stoi(arg.substr(8)); } else if (arg.substr(0, 7) == "--test=") { config_.test_id = std::stoi(arg.substr(7)); + } else if (arg.substr(0, 6) == "--sfa=") { + config_.sf_a = std::stoi(arg.substr(6), nullptr, 0); + } else if (arg.substr(0, 6) == "--sfb=") { + config_.sf_b = std::stoi(arg.substr(6), nullptr, 0); } else if (arg.substr(0, 7) == "--seed=") { config_.random_seed = std::stoi(arg.substr(7)); } else { diff --git a/hw/unittest/tcu_unit/Makefile b/hw/unittest/tcu_unit/Makefile index 25a468a2e5..936f24d4ea 100644 --- a/hw/unittest/tcu_unit/Makefile +++ b/hw/unittest/tcu_unit/Makefile @@ -56,12 +56,14 @@ RTL_INCLUDE += -I$(THIRD_PARTY_DIR)/hardfloat/source/RISCV TOP := VX_tcu_fedp_bhf endif +# the native TFR datapath is checked against tcu_fedp's windowed FEDP model ifneq (,$(filter -DVX_CFG_TCU_TYPE_TFR, $(XCONFIGS))) RTL_INCLUDE += -I$(RTL_DIR)/tcu/tfr +CXXFLAGS += -DUSE_FEDP endif # default test: fp16 -OPTS ?= --fmt=1 +OPTS ?= --fmt=2 TOP := VX_tcu_unit_top diff --git a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv index 192fb19b5a..7542fe1600 100644 --- a/hw/unittest/tcu_unit/VX_tcu_unit_top.sv +++ b/hw/unittest/tcu_unit/VX_tcu_unit_top.sv @@ -13,7 +13,7 @@ `include "VX_define.vh" -// Flat-port testbench wrapper around VX_tcu_unit. +// Flat-port testbench wrapper around VX_tcu_unit and VX_tcu_shared. // All VX_dispatch_if and VX_commit_if signals are exposed as individual // ports so that the Verilator C++ testbench can drive them directly. // Only ISSUE_WIDTH == 1 is supported by this wrapper. @@ -106,13 +106,13 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `endif `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); - assign tcu_mem_req_valid = tcu_mem_if.req_valid; - assign tcu_mem_req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = tcu_mem_req_ready; - assign tcu_mem_if.rsp_valid = tcu_mem_rsp_valid; - assign tcu_mem_if.rsp_data = tcu_mem_rsp_data; - assign tcu_mem_rsp_ready = tcu_mem_if.rsp_ready; + VX_lsu_sched_if tcu_mem_if [`VX_CFG_NUM_LSU_BLOCKS](); + assign tcu_mem_req_valid = tcu_mem_if[0].req_valid; + assign tcu_mem_req_data = tcu_mem_if[0].req_data; + assign tcu_mem_if[0].req_ready = tcu_mem_req_ready; + assign tcu_mem_if[0].rsp_valid = tcu_mem_rsp_valid; + assign tcu_mem_if[0].rsp_data = tcu_mem_rsp_data; + assign tcu_mem_rsp_ready = tcu_mem_if[0].rsp_ready; `endif VX_dispatch_if dispatch_if[`VX_CFG_ISSUE_WIDTH](); @@ -164,20 +164,51 @@ module VX_tcu_unit_top import VX_gpu_pkg::*, VX_tcu_pkg::*; ( `UNUSED_VAR (commit_if[0].data.cta_id) // ---- DUT ------------------------------------------------------------- +`ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; +`endif + VX_tcu_unit #( .INSTANCE_ID ("tcu_unit_top") ) tcu_unit ( `SCOPE_IO_BIND (0) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (tcu_mem_if), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide WGMMA tile buffers the TCU blocks share + VX_tcu_shared #( + .INSTANCE_ID ("tcu_unit_top"), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); endmodule diff --git a/hw/unittest/tcu_unit/main.cpp b/hw/unittest/tcu_unit/main.cpp index 9b52e631e2..847e57b3fc 100644 --- a/hw/unittest/tcu_unit/main.cpp +++ b/hw/unittest/tcu_unit/main.cpp @@ -43,6 +43,9 @@ #include "VX_config.h" #include "softfloat_ext.h" #include +#ifdef USE_FEDP +#include "fedp.h" +#endif bool sim_trace_enabled() { return true; } @@ -74,9 +77,9 @@ constexpr int LANE_WORDS = VX_CFG_XLEN / 32; // TCU format IDs (must match VX_tcu_pkg.sv) constexpr uint32_t TCU_FP32_ID = 0; -constexpr uint32_t TCU_FP16_ID = 1; -constexpr uint32_t TCU_BF16_ID = 2; -constexpr uint32_t TCU_TF32_ID = 5; +constexpr uint32_t TCU_TF32_ID = 1; +constexpr uint32_t TCU_FP16_ID = 2; +constexpr uint32_t TCU_BF16_ID = 3; // WMMA op_type (INST_TCU_WMMA = 4'h0 from VX_gpu_pkg.sv) constexpr uint32_t INST_TCU_WMMA = 0; @@ -114,6 +117,8 @@ struct FmtInfo { int exp_bits; int sig_bits; int width_bits; // 16, 19 (tf32), or 32 + // a 32-bit register packs this many elements, element e in bits [e*width +: width] + int per_word() const { return 32 / width_bits; } }; static FmtInfo get_fmt_info(uint32_t fmt_id) { @@ -129,41 +134,54 @@ static FmtInfo get_fmt_info(uint32_t fmt_id) { } // Convert a packed source-format element to float (for software reference). -static float elem_to_float(uint32_t raw, const FmtInfo &fi) { +[[maybe_unused]] static float elem_to_float(uint32_t raw, const FmtInfo &fi) { return cvt_custom_to_f32(raw, fi.exp_bits, fi.sig_bits, 0 /*RNE*/, nullptr); } // ---- Software WMMA reference ----------------------------------------------- -// Computes D[t] = sum_k(A[i][k] * B[k][j]) + C[t] -// where t = i*TC_N + j, A in row-major, B in col-major (each column consecutive). -// -// rs1_data layout: lane t -> A[i][k] for t = i*TC_K + k (A in row-major) -// BUT each OUTPUT thread t=(i,j) shares the same A row. -// a_row[k] = rs1_data[i * TC_K + k] (all TC_N output threads in the same row share this) -// rs2_data layout: lane t -> B[k][j] for t = j*TC_K + k (B^T in row-major, i.e. B columns packed) -// b_col[k] = rs2_data[j * TC_K + k] -// rs3_data layout: lane t -> C[i][j] where t = i*TC_N + j -// +// D[t] = sum_k(A[i][k] * B[k][j]) + C[t], t = i*TC_N + j. Each register lane +// packs per_word() elements: +// rs1 lane i*TC_K + w holds A[i][w*per_word() ...] (row i of A) +// rs2 lane j*TC_K + w holds B[w*per_word() ...][j] (column j of B) +// rs3 lane t holds C[t] (fp32) +// The native TFR datapath is checked against the windowed FEDP model +// (USE_FEDP, as tcu_fedp); the DPI datapath against its discrete fp32 mul/add. static std::vector wmma_ref( - const std::vector &rs1, // VX_CFG_NUM_THREADS elements (A, row-major) - const std::vector &rs2, // VX_CFG_NUM_THREADS elements (B columns, col-major) - const std::vector &rs3, // VX_CFG_NUM_THREADS elements (C, row-major) - const FmtInfo &fi_s) // source format (for A and B) + const std::vector &rs1, + const std::vector &rs2, + const std::vector &rs3, + const FmtInfo &fi_s, + int window) { std::vector result(VX_CFG_NUM_THREADS); - +#ifdef USE_FEDP + TfrFedp fedp(fi_s.exp_bits, fi_s.sig_bits, 0 /*RNE*/, window); +#else + (void)window; +#endif for (int i = 0; i < TCU_TC_M; i++) { for (int j = 0; j < TCU_TC_N; j++) { int t = i * TCU_TC_N + j; - // Accumulate with fp32 rounding at each multiply-add step (matches DPI model). - float prod = 0.0f; + float c = bit_cast(rs3[t]); +#ifdef USE_FEDP + result[t] = fedp(&rs1[i * TCU_TC_K], &rs2[j * TCU_TC_K], c, TCU_TC_K); +#else + const int epw = fi_s.per_word(); + const uint32_t emask = (fi_s.width_bits == 32) ? ~0u : ((1u << fi_s.width_bits) - 1u); + // the DPI datapath's order: each register's products summed in + // fp32, then the register sums in order, then C + float sum = 0.0f; for (int k = 0; k < TCU_TC_K; k++) { - float a = elem_to_float(rs1[i * TCU_TC_K + k], fi_s); - float b = elem_to_float(rs2[j * TCU_TC_K + k], fi_s); - prod += a * b; + float word_sum = 0.0f; + for (int e = 0; e < epw; e++) { + uint32_t ea = (rs1[i * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + uint32_t eb = (rs2[j * TCU_TC_K + k] >> (e * fi_s.width_bits)) & emask; + word_sum += elem_to_float(ea, fi_s) * elem_to_float(eb, fi_s); + } + sum += word_sum; } - float c = bit_cast(rs3[t]); // C is always fp32 - result[t] = prod + c; + result[t] = sum + c; +#endif } } return result; @@ -208,7 +226,9 @@ struct TestConfig { RoundingMode frm = RoundingMode::RNE; uint32_t num_tests = 1000; int ulp = 1; + int window = 25; // TFR accumulator window (FEDP model) int32_t test_id = -1; + bool directed_only = false; std::vector features = {"infinities","nans","subnormals","normals","zeros"}; }; @@ -250,6 +270,30 @@ static uint32_t gen_fp_value(const std::string &feat, uint32_t eb, uint32_t sb, std::abort(); } +// ---- Directed vectors ------------------------------------------------------- +// Register contents (rs1 = A rows, rs2 = B columns, rs3 = C) on which the native +// datapath's windowed reduction and a discrete fp32 dot product round apart; +// each runs ahead of the random tests for its format and shape. +struct DirectedVec { + const char *name; + uint32_t fmt; + int num_threads; + uint32_t rs1[4], rs2[4], rs3[4]; +}; +static const DirectedVec kDirected[] = { + // fp16 rows of subnormals only: the window aligns on the products' + // exponent fields, so their few significant bits sit low in it + {"fp16 subnormal row", TCU_FP16_ID, 4, + {0x015e005e, 0x024080e2, 0x81fb0228, 0x820c8134}, + {0x41389f40, 0x96d5bc23, 0x285ac58a, 0x7044a57d}, + {0x4feb0a36, 0x4ca6e55a, 0x9a3df577, 0xfa232341}}, + // fp16 products that nearly cancel: the window truncates the residue + {"fp16 cancellation", TCU_FP16_ID, 4, + {0x2f13b5fb, 0x1f98ae04, 0xbdc6632b, 0xc171dfc8}, + {0xb03832e2, 0x2f380ce8, 0x609b8940, 0x7507d768}, + {0x387492a6, 0x2b2843b3, 0xad72f4f6, 0x858ca727}}, +}; + // ---- Testbench class ------------------------------------------------------- class Testbench { public: @@ -300,6 +344,25 @@ class Testbench { << " VX_CFG_NUM_THREADS=" << VX_CFG_NUM_THREADS << "\n"; + // directed vectors for this format and tile shape + uint32_t directed_done = 0; + for (const auto &v : kDirected) { + if (v.fmt != cfg_.fmt_s || v.num_threads != VX_CFG_NUM_THREADS) + continue; + std::vector rs1(v.rs1, v.rs1 + VX_CFG_NUM_THREADS); + std::vector rs2(v.rs2, v.rs2 + VX_CFG_NUM_THREADS); + std::vector rs3(v.rs3, v.rs3 + VX_CFG_NUM_THREADS); + if (!this->check(rs1, rs2, rs3, fi, std::string("Directed (") + v.name + ")")) + return false; + ++directed_done; + } + if (directed_done != 0) { + std::cout << directed_done << " directed test(s) PASSED!\n"; + } + if (cfg_.directed_only) { + return true; + } + uint32_t tests_done = 0; for (uint32_t test_id = 0; test_id < cfg_.num_tests; test_id++) { if (cfg_.test_id >= 0 && (int)test_id != cfg_.test_id) @@ -309,37 +372,30 @@ class Testbench { size_t feat_idx = test_id % cfg_.features.size(); const std::string &feat = cfg_.features[feat_idx]; - // Generate source elements for A and B (VX_CFG_NUM_THREADS values each) - std::vector rs1(VX_CFG_NUM_THREADS), rs2(VX_CFG_NUM_THREADS), rs3(VX_CFG_NUM_THREADS); - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - rs1[t] = gen_fp_value(feat, fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - rs2[t] = gen_fp_value("normals", fi.exp_bits, fi.sig_bits, test_id * VX_CFG_NUM_THREADS + t, rng_); - float c = bit_cast(gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_)); - rs3[t] = bit_cast(c); + // A carries the feature, B and C are normals; each register lane + // packs per_word() source elements. + const int epw = fi.per_word(); + std::vector rs1(VX_CFG_NUM_THREADS, 0), rs2(VX_CFG_NUM_THREADS, 0), rs3(VX_CFG_NUM_THREADS); + for (int t = 0; t < TCU_TC_M * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs1[t] |= gen_fp_value(feat, fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); + } } - - // Compute software reference - std::vector expected = wmma_ref(rs1, rs2, rs3, fi); - // Drive DUT - std::vector result = drive_and_collect(rs1, rs2, rs3, fi); - - bool test_pass = true; - for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { - float act = bit_cast(result[t]); - int delta = approx_equal(act, expected[t]); - if (std::abs(delta) > cfg_.ulp) { - if (test_pass) { - // First failure for this test - std::cout << "Test #" << test_id << " (" << feat << ") FAILED:\n"; - } - std::cout << " thread=" << t - << " expected="; print_float("", expected[t], false); - std::cout << " actual="; print_float("", act, false); - std::cout << " delta=" << delta << "\n"; - test_pass = false; + for (int t = 0; t < TCU_TC_N * TCU_TC_K; t++) { + for (int e = 0; e < epw; e++) { + uint32_t elem_id = (test_id * VX_CFG_NUM_THREADS + t) * epw + e; + rs2[t] |= gen_fp_value("normals", fi.exp_bits, fi.sig_bits, elem_id, rng_) << (e * fi.width_bits); } } - if (!test_pass) return false; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + rs3[t] = gen_fp_value("normals", 8, 23, test_id * VX_CFG_NUM_THREADS + t + 1, rng_); + } + + std::ostringstream label; + label << "Test #" << test_id << " (" << feat << ")"; + if (!this->check(rs1, rs2, rs3, fi, label.str())) + return false; tests_done++; } @@ -354,6 +410,38 @@ class Testbench { } private: + // Drive one packet and compare every thread against the reference. + bool check(const std::vector &rs1, const std::vector &rs2, + const std::vector &rs3, const FmtInfo &fi, const std::string &label) { + std::vector expected = wmma_ref(rs1, rs2, rs3, fi, cfg_.window); + std::vector result = drive_and_collect(rs1, rs2, rs3, fi); + bool pass = true; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) { + float act = bit_cast(result[t]); + int delta = approx_equal(act, expected[t]); + if (std::abs(delta) > cfg_.ulp) { + if (pass) { + std::cout << label << " FAILED:\n"; + } + std::cout << " thread=" << t + << " expected="; print_float("", expected[t], false); + std::cout << " actual="; print_float("", act, false); + std::cout << " delta=" << delta << "\n"; + pass = false; + } + } + if (!pass) { + std::cout << std::hex << " rs1="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs1[t]; + std::cout << "\n rs2="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs2[t]; + std::cout << "\n rs3="; + for (int t = 0; t < VX_CFG_NUM_THREADS; t++) std::cout << " 0x" << rs3[t]; + std::cout << std::dec << "\n"; + } + return pass; + } + TestConfig cfg_; uint64_t cycle_; std::mt19937 rng_; @@ -454,7 +542,8 @@ class Testbench { write_lane(&dut_->dispatch_rs3_data, t, rs3[t]); } - // Clock until dispatch fires (ready goes high) + // The packet fires on the edge that samples valid with ready: test + // ready ahead of each edge, so it is sent exactly once. dut_->commit_ready = 1; uint64_t start_cycle = cycle_; while (true) { @@ -462,10 +551,12 @@ class Testbench { std::cerr << "TIMEOUT waiting for dispatch_ready\n"; std::abort(); } + dut_->clk = 0; + dut_->eval(); + bool fire = dut_->dispatch_ready; tick(); - if (dut_->dispatch_ready) break; + if (fire) break; } - // Dispatch fired: de-assert valid idle_dispatch(); // Wait for commit result @@ -508,6 +599,10 @@ static TestConfig parse_args(int argc, char **argv) { cfg.random_seed = std::stoi(arg.substr(7)); } else if (arg.substr(0, 6) == "--ulp=") { cfg.ulp = std::stoi(arg.substr(6)); + } else if (arg == "--directed") { + cfg.directed_only = true; + } else if (arg.substr(0, 4) == "--W=") { + cfg.window = std::stoi(arg.substr(4)); } else if (arg.substr(0, 14) == "--trace-start=") { cfg.trace_start = std::stoull(arg.substr(14)); } else if (arg.substr(0, 12) == "--trace-end=") { @@ -519,12 +614,14 @@ static TestConfig parse_args(int argc, char **argv) { while (std::getline(ss, item, ',')) cfg.features.push_back(item); } else { std::cout << "Usage: " << argv[0] << " [options]\n" - << " --fmt= Source format (0=fp32,1=fp16,2=bf16,5=tf32)\n" + << " --fmt= Source format (1=tf32,2=fp16,3=bf16)\n" << " --frm= Rounding mode (RNE,RTZ,RDN,RUP,RMM)\n" << " --tests= Number of test cases (default 1000)\n" << " --test= Run a single test by id\n" << " --seed= Random seed\n" << " --ulp= ULP tolerance (default 1)\n" + << " --W= TFR accumulator window of the FEDP model (default 25)\n" + << " --directed Run only the directed vectors\n" << " --no-trace Disable VCD output\n" << " --features=LIST Comma-separated list of features to test\n" << " --trace-start=N Start VCD trace at timestamp N\n" diff --git a/hw/unittest/tensor/VX_tensor_top.sv b/hw/unittest/tensor/VX_tensor_top.sv index 70244c6cc7..741521a4dc 100644 --- a/hw/unittest/tensor/VX_tensor_top.sv +++ b/hw/unittest/tensor/VX_tensor_top.sv @@ -20,10 +20,12 @@ // // per core (VX_CFG_SOCKET_SIZE): // VX_tcu_unit : tensor-core unit (dispatch/commit from top ports) -// VX_lsu_scheduler : per-LSU-block, 2 clients — the ordinary LSU stream -// (top ports) and the TCU AGU metadata-load port -// VX_mem_unit : LMEM + coalescer + LMEM-DMA arb (TCU WGMMA + DXA -// staging converge here) + dcache adapters +// VX_tcu_shared : WGMMA tile buffers +// VX_lsu_mem_path : per-LSU-block scheduler (2 clients — the ordinary LSU +// stream from top ports and the TCU AGU metadata-load +// port), lmem switch, coalescer and adapters +// VX_mem_unit : LMEM + LMEM-DMA arb (TCU WGMMA + DXA staging +// converge here) + dcache flush injectors // per socket: // VX_dxa_core : descriptor-driven tile-transfer engine (DCR-fed) // VX_mem_bus_switch : DXA staging writes back to the owning core's LMEM @@ -35,7 +37,7 @@ // // Memory hierarchy: TCU WGMMA prefetches and DXA staging writes meet at the // real LMEM DMA arbiter inside VX_mem_unit; TCU_LD metadata loads merge with -// the ordinary LSU stream at block-0's VX_lsu_scheduler; global traffic +// the ordinary LSU stream at block-0's scheduler; global traffic // (dcache refills, DXA tile reads) contends at the socket arbiter and the // shared L2, whose memory side exits to top-level ports. // @@ -73,7 +75,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( // ----------------------------------------------------------------------- // Ordinary LSU client streams, flattened [core][lsu-block] - // (client 0 of each VX_lsu_scheduler; the TCU AGU is client 1 on block 0) + // (client 0 of each LSU block's scheduler; the TCU block AGUs are client 1) // ----------------------------------------------------------------------- input wire [LSU_STREAMS-1:0] lsu_req_valid, input lsu_req_data_t [LSU_STREAMS-1:0] lsu_req_data, @@ -120,12 +122,6 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ); localparam TCU_LMEM_BANK_ADDR_W = `VX_CFG_LMEM_LOG_SIZE - `CLOG2(LSU_WORD_SIZE) - `CLOG2(`VX_CFG_LMEM_NUM_BANKS); -`ifdef TCU_META_ENABLE - localparam LSU_SCHED_NUM_CLIENTS = 2; -`else - localparam LSU_SCHED_NUM_CLIENTS = 1; -`endif - // ------------------------------------------------------------------------ // Per-core DXA staging returns + collected dcache core ports // ------------------------------------------------------------------------ @@ -148,7 +144,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( wire [NC-1:0] lsu_sched_empty; // ======================================================================== - // Per-core tensor slice: TCU + LSU schedulers + mem_unit + // Per-core tensor slice: TCU + LSU memory paths + mem_unit // ======================================================================== for (genvar c = 0; c < NC; ++c) begin : g_cores @@ -172,83 +168,111 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( ) tcu_lmem_if(); `ifdef TCU_META_ENABLE - VX_lsu_sched_if tcu_mem_if(); + VX_lsu_sched_if lsu_tcu_mem_if [NUM_LSU_BLKS](); + `endif + + `ifdef VX_CFG_TCU_WGMMA_ENABLE + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_exec_valid, tcu_exec_ready; + tcu_execute_t tcu_exec_data [`VX_CFG_NUM_TCU_BLOCKS]; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_A_DATA_SIZE-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs1_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0][TCU_WG_RS2_WIDTH-1:0][`VX_CFG_XLEN-1:0] tcu_tbuf_rs2_data; + wire [`VX_CFG_NUM_TCU_BLOCKS-1:0] tcu_tbuf_ready; `endif VX_tcu_unit #( .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))) ) tcu_unit ( `SCOPE_IO_BIND (c) - .clk (clk), - .reset (reset), + .clk (clk), + .reset (reset), `ifdef VX_CFG_TCU_WGMMA_ENABLE - .tcu_lmem_if (tcu_lmem_if), + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready (tcu_tbuf_ready), `endif `ifdef TCU_META_ENABLE - .tcu_mem_if (tcu_mem_if), + .tcu_mem_if (lsu_tcu_mem_if), + `endif + .dispatch_if (dispatch_if), + .commit_if (commit_if) + ); + + // the core-wide WGMMA tile buffers the TCU blocks share + VX_tcu_shared #( + .INSTANCE_ID (`SFORMATF(("%s-tcu%0d", INSTANCE_ID, c))), + .BLOCK_SIZE (`VX_CFG_NUM_TCU_BLOCKS) + ) tcu_shared ( + `ifdef VX_CFG_TCU_WGMMA_ENABLE + .exec_valid (tcu_exec_valid), + .exec_ready (tcu_exec_ready), + .exec_data (tcu_exec_data), + .tbuf_rs1_data (tcu_tbuf_rs1_data), + .tbuf_rs2_data (tcu_tbuf_rs2_data), + .tbuf_ready_eff (tcu_tbuf_ready), + .tcu_lmem_if (tcu_lmem_if), `endif - .dispatch_if (dispatch_if), - .commit_if (commit_if) + .clk (clk), + .reset (reset) ); - // --- LSU schedulers: client 0 from top ports, client 1 = TCU AGU --- + // --- LSU memory paths: client 0 from top ports, client 1 = the TCU block AGUs --- + wire [NUM_LSU_BLKS-1:0] block_sched_empty; + wire [NUM_LSU_BLKS-1:0] block_store_empty; + VX_lsu_mem_if #( .NUM_LANES (`VX_CFG_NUM_LSU_LANES), .DATA_SIZE (LSU_WORD_SIZE), .TAG_WIDTH (LSU_TAG_WIDTH) - ) lsu_mem_if[NUM_LSU_BLKS](); - - wire [NUM_LSU_BLKS-1:0] block_sched_empty; - - for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_scheduler - localparam s = c * NUM_LSU_BLKS + block_idx; + ) lsu_lmem_if[`VX_CFG_NUM_LSU_BLOCKS](); - VX_lsu_sched_if sched_client_if [LSU_SCHED_NUM_CLIENTS](); + VX_mem_bus_if #( + .DATA_SIZE (DCACHE_WORD_SIZE), + .TAG_WIDTH (DCACHE_TAG_WIDTH) + ) lsu_dcache_bus_if[DCACHE_NUM_REQS](); - assign sched_client_if[0].req_valid = lsu_req_valid[s]; - assign sched_client_if[0].req_data = lsu_req_data[s]; - assign lsu_req_ready[s] = sched_client_if[0].req_ready; - assign lsu_rsp_valid[s] = sched_client_if[0].rsp_valid; - assign lsu_rsp_data[s] = sched_client_if[0].rsp_data; - assign sched_client_if[0].rsp_ready = lsu_rsp_ready[s]; - `ifdef TCU_META_ENABLE - if (block_idx == 0) begin : g_tcu_client - assign sched_client_if[1].req_valid = tcu_mem_if.req_valid; - assign sched_client_if[1].req_data = tcu_mem_if.req_data; - assign tcu_mem_if.req_ready = sched_client_if[1].req_ready; - assign tcu_mem_if.rsp_valid = sched_client_if[1].rsp_valid; - assign tcu_mem_if.rsp_data = sched_client_if[1].rsp_data; - assign sched_client_if[1].rsp_ready = tcu_mem_if.rsp_ready; - end else begin : g_tcu_client_tieoff - assign sched_client_if[1].req_valid = 1'b0; - assign sched_client_if[1].req_data = '0; - assign sched_client_if[1].rsp_ready = 1'b1; - `UNUSED_VAR (sched_client_if[1].req_ready) - `UNUSED_VAR (sched_client_if[1].rsp_valid) - `UNUSED_VAR (sched_client_if[1].rsp_data) - end - `endif + for (genvar block_idx = 0; block_idx < NUM_LSU_BLKS; ++block_idx) begin : g_lsu_mem_path + localparam s = c * NUM_LSU_BLKS + block_idx; - VX_lsu_scheduler #( - .INSTANCE_ID (`SFORMATF(("%s-lsusched%0d-%0d", INSTANCE_ID, c, block_idx))), - .NUM_CLIENTS (LSU_SCHED_NUM_CLIENTS), - .NUM_LANES (`VX_CFG_NUM_LSU_LANES), - .CORE_QUEUE_SIZE(`VX_CFG_LSU_QUEUE_IN_SIZE), - .MEM_QUEUE_SIZE (LSU_QUEUE_OUT_SIZE), - .PENDING_SIZE (`VX_CFG_LSU_PENDING_SIZE) - ) lsu_scheduler ( - .clk (clk), - .reset (reset), - .client_if (sched_client_if), - .empty (block_sched_empty[block_idx]), - .lsu_mem_if (lsu_mem_if[block_idx]) + VX_lsu_sched_if client_if(); + + assign client_if.req_valid = lsu_req_valid[s]; + assign client_if.req_data = lsu_req_data[s]; + assign lsu_req_ready[s] = client_if.req_ready; + assign lsu_rsp_valid[s] = client_if.rsp_valid; + assign lsu_rsp_data[s] = client_if.rsp_data; + assign client_if.rsp_ready = lsu_rsp_ready[s]; + + VX_lsu_mem_path #( + .INSTANCE_ID (`SFORMATF(("%s-lsu%0d-%0d", INSTANCE_ID, c, block_idx))) + ) mem_path ( + .clk (clk), + .reset (reset), + `ifdef PERF_ENABLE + `UNUSED_PIN (perf_rd_req_fire), + `UNUSED_PIN (perf_wr_req_fire), + `UNUSED_PIN (perf_rsp_fire), + `UNUSED_PIN (coalescer_misses), + `endif + .client_if (client_if), + `ifdef TCU_META_ENABLE + .tcu_mem_if (lsu_tcu_mem_if[block_idx]), + `endif + .sched_empty (block_sched_empty[block_idx]), + .store_empty (block_store_empty[block_idx]), + `UNUSED_PIN (busy), + .lmem_if (lsu_lmem_if[block_idx]), + .dcache_bus_if (lsu_dcache_bus_if[block_idx * DCACHE_CHANNELS +: DCACHE_CHANNELS]) ); end assign lsu_sched_empty[c] = & block_sched_empty; + assign mem_unit_empty[c] = & block_store_empty; - // --- mem_unit: LMEM + DMA arb + dcache adapters --- + // --- mem_unit: LMEM + DMA arb + dcache flush injectors --- VX_txbar_bus_if dxa_txbar_bus_if(); assign txbar_valid[c] = dxa_txbar_bus_if.valid; assign txbar_data[c] = dxa_txbar_bus_if.data; @@ -275,10 +299,12 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .dxa_lmem_bus_if (per_core_dxa_lmem_bus_if[c]), .dxa_txbar_bus_if(dxa_txbar_bus_if), `endif - .empty (mem_unit_empty[c]), - .lsu_mem_if (lsu_mem_if), + .lmem_if (lsu_lmem_if), + .lsu_dcache_bus_if (lsu_dcache_bus_if), .dcr_flush_if (dcr_flush_dcache_if), - .dcache_bus_if (dcache_bus_if) + .flush_l1_only (1'b0), + .dcache_bus_if (dcache_bus_if), + `UNUSED_PIN (busy) ); for (genvar i = 0; i < DCACHE_NUM_REQS; ++i) begin : g_dcache_passthru @@ -384,6 +410,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -394,7 +421,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (all_dc_core_if), - .mem_bus_if (dcache_mem_bus_if) + .mem_bus_if (dcache_mem_bus_if), + `UNUSED_PIN (req_empty) ); // ======================================================================== @@ -502,6 +530,7 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -513,7 +542,8 @@ module VX_tensor_top import VX_gpu_pkg::*, VX_tcu_pkg::*, VX_dxa_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& mem_unit_empty) && (& lsu_sched_empty) && !dxa_busy; diff --git a/hw/unittest/tex_core/VX_tex_core_top.sv b/hw/unittest/tex_core/VX_tex_core_top.sv index ef2c474918..8457fd6053 100644 --- a/hw/unittest/tex_core/VX_tex_core_top.sv +++ b/hw/unittest/tex_core/VX_tex_core_top.sv @@ -28,7 +28,7 @@ module VX_tex_core_top import VX_gpu_pkg::*; import VX_tex_pkg::*; #( input wire tex_req_valid, input wire [NUM_LANES-1:0] tex_req_mask, input wire [1:0][NUM_LANES-1:0][31:0] tex_req_coords, - input wire [NUM_LANES-1:0][TEX_LOD_BITS-1:0] tex_req_lod, + input wire [NUM_LANES-1:0][TEX_LODF_BITS-1:0] tex_req_lod, input wire [TEX_STAGE_BITS-1:0] tex_req_stage, input wire [TAG_WIDTH-1:0] tex_req_tag, output wire tex_req_ready, diff --git a/hw/unittest/tex_core/main.cpp b/hw/unittest/tex_core/main.cpp index 90f094ba3b..ccec730071 100644 --- a/hw/unittest/tex_core/main.cpp +++ b/hw/unittest/tex_core/main.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include "VVX_tex_core_top.h" #include "verilated.h" #ifdef VCD_OUTPUT @@ -49,7 +50,7 @@ int main(int argc, char** argv) { dut->dcr_write_valid = 0; dut->tex_req_valid = 0; dut->tex_rsp_ready = 1; - dut->cache_req_ready = ~0ULL; + dut->cache_req_ready = std::remove_reference_tcache_req_ready)>(~0ULL); dut->cache_rsp_valid = 0; for (int i = 0; i < 4; ++i) clock_cycle(); diff --git a/hw/unittest/vm/VX_vm_top.sv b/hw/unittest/vm/VX_vm_top.sv index 7ac7aec7f0..c578de445c 100644 --- a/hw/unittest/vm/VX_vm_top.sv +++ b/hw/unittest/vm/VX_vm_top.sv @@ -255,7 +255,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( // --- socket miss ports (dTLB, iTLB) into the socket arbiter --- VX_tlb_bus_if #(.ID_WIDTH (L1_TLB_ID_WIDTH)) - socket_tlb_bus_if [2] (); + socket_tlb_bus_if [SOCKET_NUM_MMUS] (); VX_mmu #( .INSTANCE_ID ("dmmu"), @@ -273,6 +273,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .tlb_bus_if (socket_tlb_bus_if[0]), .flush_if (dmmu_flush_if), .fault_if (dmmu_fault_if), + `UNUSED_PIN (lane_idle), .empty (dmmu_empty[s]) ); @@ -305,6 +306,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_DCACHE_WRITEBACK), .DIRTY_BYTES (`VX_CFG_DCACHE_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_DCACHE_LAZY_FETCH), .REPL_POLICY (`VX_CFG_DCACHE_REPL_POLICY), .NC_ENABLE (1), .CORE_OUT_BUF (3), @@ -315,7 +317,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (dmmu_phys_if), - .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]) + .mem_bus_if (all_dc_mem_if[s * L1_MEM_PORTS +: L1_MEM_PORTS]), + `UNUSED_PIN (req_empty) ); if (INCLUDE_IMMU != 0) begin : g_immu @@ -381,6 +384,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .tlb_bus_if (socket_tlb_bus_if[1]), .flush_if (immu_flush_if), .fault_if (immu_fault_if), + `UNUSED_PIN (lane_idle), .empty (immu_empty[s]) ); @@ -410,17 +414,19 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (0), .REPL_POLICY (`VX_CFG_ICACHE_REPL_POLICY), .NC_ENABLE (0), + .FLUSH_FORWARD (0), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3) ) icache ( .clk (clk), .reset (reset), .core_bus_if (immu_phys_if), - .mem_bus_if (ic_mem_if) + .mem_bus_if (ic_mem_if), + `UNUSED_PIN (req_empty) ); end else begin : g_no_immu - // Keep the socket arbiter 2-wide (so TLB_SOCKET_ID_WIDTH matches the - // package) with the iTLB port idle. + // Keep the socket arbiter at the package's width (so + // TLB_SOCKET_ID_WIDTH matches) with the iTLB port idle. assign socket_tlb_bus_if[1].req_valid = 1'b0; assign socket_tlb_bus_if[1].req_data = '0; assign socket_tlb_bus_if[1].rsp_ready = 1'b0; @@ -449,9 +455,20 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( `UNUSED_VAR (ic_mem_rsp_tag[s]) end - // --- socket TLB arbiter: {dTLB, iTLB} -> cluster fabric --- + // A build with socket units adds the unit MMU's port (SOCKET_NUM_MMUS); + // this sandbox has no units, so that port idles. + for (genvar t = 2; t < SOCKET_NUM_MMUS; ++t) begin : g_unit_tlb_idle + assign socket_tlb_bus_if[t].req_valid = 1'b0; + assign socket_tlb_bus_if[t].req_data = '0; + assign socket_tlb_bus_if[t].rsp_ready = 1'b0; + `UNUSED_VAR (socket_tlb_bus_if[t].req_ready) + `UNUSED_VAR (socket_tlb_bus_if[t].rsp_valid) + `UNUSED_VAR (socket_tlb_bus_if[t].rsp_data) + end + + // --- socket TLB arbiter: {dTLB, iTLB[, unit]} -> cluster fabric --- VX_tlb_bus_arb #( - .NUM_INPUTS (2), + .NUM_INPUTS (SOCKET_NUM_MMUS), .ID_WIDTH_IN (L1_TLB_ID_WIDTH), .OUT_BUF (3) ) tlb_socket_arb ( @@ -613,6 +630,7 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .WRITE_ENABLE (1), .WRITEBACK (`VX_CFG_L2_WRITEBACK), .DIRTY_BYTES (`VX_CFG_L2_DIRTYBYTES), + .LAZY_FETCH (`VX_CFG_L2_LAZY_FETCH), .REPL_POLICY (`VX_CFG_L2_REPL_POLICY), .CORE_OUT_BUF (3), .MEM_OUT_BUF (3), @@ -624,7 +642,8 @@ module VX_vm_top import VX_gpu_pkg::*, VX_tlb_pkg::*; #( .clk (clk), .reset (reset), .core_bus_if (l2_core_if), - .mem_bus_if (l2_mem_if) + .mem_bus_if (l2_mem_if), + `UNUSED_PIN (req_empty) ); assign empty = (& dmmu_empty) && (& immu_empty) && l2tlb_empty && ptw_empty; diff --git a/miscs/apptainer/README.md b/miscs/apptainer/README.md index 06e6a4e3f2..541db951f7 100644 --- a/miscs/apptainer/README.md +++ b/miscs/apptainer/README.md @@ -105,7 +105,7 @@ Apptainer> pwd Apptainer> make -C runtime/ clean -Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin TARGET=hw PLATFORM=xilinx_u50_gen3x16_xdma_5_202210_1 ./ci/blackbox.sh --driver=xrt --app=demo +Apptainer> FPGA_BIN_DIR=/home/vortex/build/hw/syn/xilinx/xrt/test1_xilinx_u50_gen3x16_xdma_5_202210_1_hw/bin ./ci/blackbox.sh --driver=xrt --target=hw --app=demo Verify following line being printed: info: device name=xilinx_u50_gen3x16_xdma_base_5, memory_capacity=0x200000000 bytes, memory_banks=32. diff --git a/profiles/a100.json b/profiles/a100.json index 7a6003fe0b..c000279319 100644 --- a/profiles/a100.json +++ b/profiles/a100.json @@ -1,11 +1,35 @@ { "name": "a100", "description": "NVIDIA A100 SXM4 (GA100, Ampere, sm_80) at its 1410 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 16, + "VX_CFG_NUM_LSU_LANES": 8, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4, + "VX_CFG_FCVT_PE_RATIO": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_DCACHE_NUM_BANKS": 4, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 17, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +37,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 64, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -26,10 +51,24 @@ }, "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "108 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA A100 Tensor Core GPU Architecture whitepaper, Table 3)", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and all type conversions not to or from 8/16-bit integers (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0); 16 FPU lanes / 4", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM (A100 whitepaper): 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part", "DCACHE_SIZE": "192 KB combined L1/shared per SM (whitepaper), rounded down to the power of two the cache accepts", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB and 32 banks; 16 ways as in GPGPU-Sim's SM80_A100 dl2", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "40 MB L2 in 80 slices (whitepaper), rounded down to 32 MB; 16 ways as in GPGPU-Sim's SM80_A100 dl2; 64 banks of 64 B (4 KB/clk), the power of two nearest the 80 slices' 5 KB/clk", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM2 stacks x 8 128-bit channels = 40 channels at 2430 MT/s, modelled as 32 channels at 3038 MT/s for a power-of-two interleave: 32 x 16 B x 3038 MT/s = 1555 GB/s, the card's bandwidth; the array timing is in nanoseconds, so latency is unchanged", "PLATFORM_MEMORY_LATENCY, PLATFORM_MEMORY_QUEUE_SIZE": "Fixed DRAM latency and per-channel request queue of GPGPU-Sim's SM80_A100 configuration (accel-sim gpu-simulator/configs/tested-cfgs/SM80_A100/gpgpusim.config)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1410 MHz boost clock, 1555 GB/s (whitepaper, Table 3)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA A100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "164 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 8.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the 192 KB L1/shared pool rounded down: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Third-generation tensor cores: FP16/BF16, TF32, INT8 and INT4, with 2:4 structured sparsity (A100 whitepaper). A100 has cp.async but no TMA, so no DXA", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 8.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GA100 SM (A100 whitepaper, Figure 7): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32" } } diff --git a/profiles/gtx1060.json b/profiles/gtx1060.json new file mode 100644 index 0000000000..419eabbf20 --- /dev/null +++ b/profiles/gtx1060.json @@ -0,0 +1,24 @@ +{ + "name": "gtx1060", + "extends": "gtx1080", + "description": "NVIDIA GeForce GTX 1060 6GB (GP106, Pascal, sm_61) at its 1506 MHz base clock: one Vortex core per SM; the SM is GP104's (gtx1080).", + "chip": { + "VX_CFG_NUM_CORES": 10, + "VX_CFG_L2_SIZE": 1048576, + "VX_CFG_L2_NUM_BANKS": 8 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 8, + "VX_CFG_PLATFORM_CLOCK_RATE": 1506, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 192000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 12000 + }, + "sources": { + "SM (all core knobs)": "Inherited from gtx1080: GP106 uses the same Pascal SM as GP104 (128 CUDA cores, 32 LD/ST, 32 SFU, 4 FP64, 96 KB shared memory, 48 KB L1/texture)", + "NUM_CORES": "10 SMs, 1280 CUDA cores (GTX 1060 6GB)", + "L2_SIZE, L2_NUM_BANKS": "1.5 MB L2 in 12 slices (6 x 32-bit controllers, 2 slices each), rounded down to the power-of-two 1 MB and 8 banks", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "192-bit GDDR5 at 8 Gbps (192 GB/s) modelled as 8 GDDR6 x16 channels at 12000 MT/s for a power-of-two interleave: 8 x 2 B x 12 GT/s = 192 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE": "1506 MHz base clock (1708 MHz boost), base as in gtx1080" + } +} diff --git a/profiles/gtx1080.json b/profiles/gtx1080.json new file mode 100644 index 0000000000..443dd54561 --- /dev/null +++ b/profiles/gtx1080.json @@ -0,0 +1,65 @@ +{ + "name": "gtx1080", + "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 64, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 8, + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 32, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 20, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1607, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", + "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own instruction buffer and warp scheduler (NVIDIA GeForce GTX 1080 (GP104) whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB dedicated shared memory per SM, separate from the 48 KB L1/texture (GTX 1080 whitepaper), rounded down to the power of two the LMEM accepts", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE": "Pascal has no tensor cores and no RT cores", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "NUM_MUFU_LANES": "32 special-function results per clock per SM (8 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GP104 SM (GTX 1080 whitepaper): 4 processing blocks, each with 32 CUDA cores (FP32 and INT32 on the same cores), 8 LD/ST units and 8 SFUs; 4 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 4", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "32 results per clock per SM, 8 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 6.1); 32 FPU lanes / 4" + } +} diff --git a/profiles/h100.json b/profiles/h100.json index d95d2fabda..fb5c6a2f94 100644 --- a/profiles/h100.json +++ b/profiles/h100.json @@ -1,11 +1,37 @@ { "name": "h100", "description": "NVIDIA H100 SXM5 (GH100, Hopper, sm_90) at its 1980 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 262144 + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 8, + "VX_CFG_DCACHE_SIZE": 262144, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 17, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_TCU_WGMMA_ENABLE": true, + "VX_CFG_EXT_DXA_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8, + "VX_CFG_DCACHE_NUM_BANKS": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +39,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 33554432, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 64, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -25,8 +52,22 @@ "sources": { "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "132 SMs (SXM5), 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", "DCACHE_SIZE": "256 KB combined L1/shared per SM (whitepaper)", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways and 32 banks as in the a100 profile", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "50 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways; 64 banks of 64 B, the power of two nearest the part's L2 bandwidth per clock (as a100)", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "5 HBM3 stacks x 16 64-bit channels at 5236 MT/s = 40 128-bit channel pairs, modelled as 32 at 6545 MT/s for a power-of-two interleave: 32 x 16 B x 6545 MT/s = 3.35 TB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1980 MHz boost clock, 3.35 TB/s (H100 SXM5 datasheet)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA H100 Tensor Core GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "228 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 9.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 256 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fourth-generation tensor cores: FP16/BF16, TF32, FP8 and INT8 (no INT4), 2:4 structured sparsity, warpgroup MMA (wgmma); the Tensor Memory Accelerator is the DXA (H100 whitepaper)", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Off: the datacenter part is built for compute (no display outputs; the raster and ROP hardware is reduced or absent), so the profile models its compute machine", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 9.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GH100 SM (H100 whitepaper, Figure 7): 4 processing blocks, each with 32 FP32, 16 INT32, 16 FP64 cores, 8 LD/ST units, 1 SFU and 1 tensor core. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 64", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 9.0", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM: 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part" } } diff --git a/profiles/rtx1080.json b/profiles/rtx1080.json deleted file mode 100644 index 73908bb3d2..0000000000 --- a/profiles/rtx1080.json +++ /dev/null @@ -1,32 +0,0 @@ -{ - "name": "rtx1080", - "description": "NVIDIA GeForce GTX 1080 (GP104, Pascal, sm_61) at its 1607 MHz base clock: one Vortex core per SM.", - "core": { - "VX_CFG_NUM_WARPS": 64, - "VX_CFG_NUM_THREADS": 32, - "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 32768 - }, - "chip": { - "VX_CFG_NUM_CLUSTERS": 1, - "VX_CFG_NUM_CORES": 20, - "VX_CFG_L2_ENABLE": true, - "VX_CFG_L2_SIZE": 2097152, - "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 16 - }, - "platform": { - "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, - "VX_CFG_PLATFORM_CLOCK_RATE": 1607, - "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 320000, - "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", - "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 10000 - }, - "sources": { - "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "20 SMs, 64 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA GeForce GTX 1080 whitepaper)", - "DCACHE_SIZE": "48 KB unified L1/texture per SM (whitepaper), rounded down to the power of two the cache accepts", - "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2 (whitepaper); 16 ways, 16 banks for the 16 memory channels", - "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "256-bit GDDR5X at 10 Gbps modelled as 16 GDDR6 x16 channels at 10000 MT/s: 16 x 2 B x 10 GT/s = 320 GB/s, the card's bandwidth; latency not calibrated", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1607 MHz base clock, 320 GB/s (whitepaper)" - } -} diff --git a/profiles/rtx3060.json b/profiles/rtx3060.json new file mode 100644 index 0000000000..3f8d9dcb14 --- /dev/null +++ b/profiles/rtx3060.json @@ -0,0 +1,67 @@ +{ + "name": "rtx3060", + "description": "NVIDIA GeForce RTX 3060 12GB (GA106, Ampere, sm_86) at its 1320 MHz base clock: one Vortex core per SM.", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 48, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 4, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 28, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 1320, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 360000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 11250 + }, + "sources": { + "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH": "28 SMs (3584 CUDA cores), 48 resident warps per SM (compute capability 8.6), 4 processing blocks each with its own scheduler and L0 instruction cache (NVIDIA Ampere GA102 whitepaper, SM)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GA10x SM processing block: 16 FP32 cores plus 16 cores that run FP32 or INT32, 4 LD/ST units, 1 SFU, 1 tensor core (GA102 whitepaper): 32 FP32, 16 INT32 and 4 LD/ST lanes per block. The shared FP32/INT32 half is modelled as FPU lanes, so an INT and an FP32 instruction can overlap where the part would share those 16 cores. FP64 (2 per SM on the part) is not modelled separately: with XLEN=64 all 128 FPU lanes execute FP64", + "DCACHE_SIZE": "128 KB L1/shared per SM (GA102 whitepaper)", + "LMEM_LOG_SIZE": "100 KB maximum shared memory per SM (compute capability 8.6), rounded down to the power of two the LMEM accepts", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "3 MB L2 (GA106), rounded down to 2 MB; one bank per memory channel; 16 ways as in GPGPU-Sim's SM86_RTX3070", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "192-bit GDDR6 at 15 Gbps: 6 devices x 2 x16 channels = 12 channels, modelled as 16 at 11250 MT/s for a power-of-two interleave: 16 x 2 B x 11.25 GT/s = 360 GB/s, the card's bandwidth; latency not calibrated", + "PLATFORM_CLOCK_RATE": "1320 MHz base clock (1777 MHz boost), base as in rtx5080", + "EXT_TCU_ENABLE, TCU_*": "Third-generation tensor cores, one per processing block: FP16/BF16, TF32, INT8, INT4, 2:4 sparsity", + "EXT_RTU_ENABLE": "Off although the part has one RT core per SM: the RTU needs full-warp ALU lanes (NUM_ALU_LANES == NUM_THREADS), and the part's 16 INT32 lanes per block take precedence", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (compute capability 8.6)", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "As in the other NVIDIA profiles: demand-paged MMU, atomics with CAS, FP16, 16 named barriers, and the graphics pipeline of a GeForce part", + "xlen": "64-bit machine: 64-bit pointers and FP64", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 8.6" + } +} diff --git a/profiles/rtx5080.json b/profiles/rtx5080.json index d90b047d99..fbba2f3bf0 100644 --- a/profiles/rtx5080.json +++ b/profiles/rtx5080.json @@ -1,12 +1,44 @@ { "name": "rtx5080", "description": "NVIDIA GeForce RTX 5080 (GB203, Blackwell, sm_120) at its 2295 MHz base clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 48, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 4, "VX_CFG_DCACHE_SIZE": 131072, - "VX_CFG_DCACHE_MSHR_SIZE": 24 + "VX_CFG_DCACHE_MSHR_SIZE": 24, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_TF32_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_FP4_ENABLE": true, + "VX_CFG_TCU_MX_ENABLE": true, + "VX_CFG_TCU_MXFP4_ENABLE": true, + "VX_CFG_TCU_NVFP4_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_EXT_DXA_ENABLE": true, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_RTU_TLAS_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 8, + "VX_CFG_FSQRT_PE_RATIO": 8, + "VX_CFG_FCVT_PE_RATIO": 8 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -14,8 +46,9 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 67108864, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32, - "VX_CFG_L2_MSHR_SIZE": 192 + "VX_CFG_L2_NUM_BANKS": 64, + "VX_CFG_L2_MSHR_SIZE": 192, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -23,7 +56,7 @@ "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 960000, "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 15000, - "VX_CFG_PLATFORM_MEMORY_LATENCY": 339 + "VX_CFG_PLATFORM_MEMORY_LATENCY": 675 }, "reference": { "clock_mhz": 2295, @@ -52,12 +85,26 @@ "NUM_CORES, NUM_WARPS, NUM_THREADS, ISSUE_WIDTH": "84 SMs, 48 resident warps/SM, 4 schedulers x 32 lanes (NVIDIA RTX Blackwell GPU Architecture whitepaper, App. B; cudaGetDeviceProperties on an RTX 5080)", "DCACHE_SIZE": "128 KB L1/shared per SM (whitepaper App. B)", "L2_SIZE, L2_NUM_WAYS": "64 MB L2 (whitepaper App. B); 16 ways as in GPGPU-Sim's Ampere/Hopper L2 configs", - "L2_NUM_BANKS": "32 banks as the nearest power of two to the 48 L2 slices measured by arXiv 2507.10789", + "L2_NUM_BANKS": "64 banks of 64 B (4 KB/clk): the 48 L2 slices measured by arXiv 2507.10789 deliver about 3.5 KB/clk, nearer 64 banks than 32", "DCACHE_MSHR_SIZE, L2_MSHR_SIZE": "384 L1 MSHR entries per SM (24 x 16 banks) and 192 per L2 slice, GPGPU-Sim SM86_RTX3070 (-gpgpu_cache:dl1 A:384:48, dl2 A:192:4)", "PLATFORM_MEMORY_NUM_BANKS": "8 x 32-bit GDDR7 devices = 32 independent channels (whitepaper App. B)", "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "2295 MHz base clock, 960 GB/s (whitepaper App. B)", "reference": "Nsight Compute on an RTX 5080 at its 2295 MHz base clock with a cold L2, 2026-10-05", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_DATA_RATE": "30 Gbps GDDR7 x8 channels modelled as GDDR6 x16 channels at 15000 MT/s: the same channel count, 32 B burst time and 960 GB/s (32 x 2 B x 15 GT/s)", - "PLATFORM_MEMORY_LATENCY": "339 core cycles of PHY/NoC latency: the 407 calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference', less the 68 cycles JEDEC GDDR6 timing adds to the device's row-conflict latency (derived, not re-run; recalibrate with a pointer chase when the memory model changes)" + "PLATFORM_MEMORY_LATENCY": "675 core cycles of PHY/NoC latency, calibrated in SimX against the 802-cycle pointer-chase DRAM latency in 'reference' with the same chase (random cyclic permutation, 128 B stride, one thread, cold L2 with CP_LLC_CLIENT=0; cycle difference between 128 KiB and 256 KiB footprints over the 1024 extra loads: 802.1). The L2 link latency is not modelled, so an L2 hit is about 79 cycles against the card's 354. Recalibrate when the memory model changes", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA RTX Blackwell GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "100 KB maximum shared memory per SM (compute capability 12.0 device query: 102400 B per SM), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Fifth-generation tensor cores: FP16/BF16, TF32, FP8, FP4 (NVFP4, MXFP4) and the MX formats, INT8, with 2:4 structured sparsity; no wgmma on sm_120. TMA is the DXA; the fourth-generation RT cores are the RTU (RTX Blackwell whitepaper)", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "RTU_TLAS_ENABLE": "RT cores traverse two-level acceleration structures (instance TLAS over BLAS), as DXR/Vulkan ray tracing requires", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 12.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GB203 SM (RTX Blackwell whitepaper, Figure 5 and App. B): 4 processing blocks, each with 32 unified FP32/INT32 CUDA cores (INT32 TOPS = FP32 TFLOPS, Table 4), 4 LD/ST units and 1 SFU, 1 tensor core; 2 FP64 cores per SM. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 128 FP64 lanes per SM against the part's 2", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 32 FPU lanes / 8, compute capability 12.0" } } diff --git a/profiles/rx6600.json b/profiles/rx6600.json new file mode 100644 index 0000000000..1e343cac94 --- /dev/null +++ b/profiles/rx6600.json @@ -0,0 +1,71 @@ +{ + "name": "rx6600", + "description": "AMD Radeon RX 6600 (Navi 23, RDNA 2, gfx1032) at its 2044 MHz game clock: one Vortex core per compute unit (CU).", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 40, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 2, + "VX_CFG_FETCH_WIDTH": 2, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 16, + "VX_CFG_NUM_LSU_BLOCKS": 1, + "VX_CFG_DCACHE_SIZE": 16384, + "VX_CFG_ICACHE_SIZE": 16384, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 28, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 2097152, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 8, + "VX_CFG_L3_ENABLE": true, + "VX_CFG_L3_SIZE": 33554432, + "VX_CFG_L3_NUM_WAYS": 16, + "VX_CFG_L3_NUM_BANKS": 8, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 8, + "VX_CFG_PLATFORM_CLOCK_RATE": 2044, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 224000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 14000 + }, + "sources": { + "NUM_CORES": "28 CUs (14 WGPs), 1792 stream processors (RX 6600)", + "NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH, NUM_WARPS": "2 SIMD32 units per CU, each with its own scheduler, running wave32 at one instruction per clock; 20 waves per SIMD (AMD RDNA architecture whitepaper)", + "NUM_ALU_LANES, NUM_FPU_LANES": "32 VALU lanes per SIMD for FP32 and INT32 (no dual-issue on RDNA 2)", + "NUM_LSU_LANES, NUM_LSU_BLOCKS": "One vector-memory path per CU behind the L0 vector cache, 64 B per clock (16 x 4 B lanes)", + "DCACHE_SIZE": "16 KB L0 vector cache per CU, 128 B lines. The 128 KB graphics L1 per shader array, shared by its CUs, has no Vortex level: cores in a socket would share the L0-sized dcache, so the L1 is left out", + "ICACHE_SIZE": "32 KB instruction cache per WGP (two CUs), i.e. 16 KB per CU", + "LMEM_LOG_SIZE, LMEM_NUM_BANKS": "64 KB LDS per CU in CU mode (128 KB per WGP), 32 banks of 4 B = 128 B per clock per CU; Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "2 MB L2, one slice per 16-bit memory channel, 16-way", + "L3_ENABLE, L3_SIZE, L3_NUM_WAYS, L3_NUM_BANKS": "32 MB Infinity Cache (first generation) behind the L2, modelled as the Vortex L3, one bank per memory channel", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "128-bit GDDR6 at 14 Gbps: 4 devices x 2 x16 channels = 8 channels, 8 x 2 B x 14 GT/s = 224 GB/s; latency not calibrated", + "PLATFORM_CLOCK_RATE": "2044 MHz game clock (2491 MHz boost)", + "NUM_MUFU_LANES": "Transcendentals at a quarter rate per SIMD: 16 per CU per clock", + "EXT_TCU_ENABLE": "Off: RDNA 2 has no matrix units (only packed dot-product instructions)", + "EXT_RTU_ENABLE": "One ray accelerator per CU", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16 instructions and workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/16 of FP32 (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "Transcendentals (reciprocal, reciprocal square root, on which division and square root are built) run at a quarter of the FP32 rate on GCN, RDNA and CDNA: FPU lanes / 4" + } +} diff --git a/profiles/rx9070xt.json b/profiles/rx9070xt.json new file mode 100644 index 0000000000..6f59395d8d --- /dev/null +++ b/profiles/rx9070xt.json @@ -0,0 +1,76 @@ +{ + "name": "rx9070xt", + "description": "AMD Radeon RX 9070 XT (Navi 48, RDNA 4, gfx1201) at its 2400 MHz game clock: one Vortex core per compute unit (CU).", + "xlen": 64, + "core": { + "VX_CFG_NUM_WARPS": 32, + "VX_CFG_NUM_THREADS": 32, + "VX_CFG_ISSUE_WIDTH": 2, + "VX_CFG_FETCH_WIDTH": 2, + "VX_CFG_NUM_ALU_LANES": 32, + "VX_CFG_NUM_FPU_LANES": 32, + "VX_CFG_NUM_LSU_LANES": 16, + "VX_CFG_NUM_LSU_BLOCKS": 1, + "VX_CFG_DCACHE_SIZE": 32768, + "VX_CFG_ICACHE_SIZE": 16384, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_TCU_FP8_ENABLE": true, + "VX_CFG_TCU_INT8_ENABLE": true, + "VX_CFG_TCU_INT4_ENABLE": true, + "VX_CFG_TCU_SPARSE_ENABLE": true, + "VX_CFG_EXT_RTU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4 + }, + "chip": { + "VX_CFG_NUM_CLUSTERS": 1, + "VX_CFG_NUM_CORES": 64, + "VX_CFG_L2_ENABLE": true, + "VX_CFG_L2_SIZE": 8388608, + "VX_CFG_L2_NUM_WAYS": 16, + "VX_CFG_L2_NUM_BANKS": 16, + "VX_CFG_L3_ENABLE": true, + "VX_CFG_L3_SIZE": 67108864, + "VX_CFG_L3_NUM_WAYS": 16, + "VX_CFG_L3_NUM_BANKS": 16, + "VX_CFG_CP_LLC_CLIENT": 1 + }, + "platform": { + "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 16, + "VX_CFG_PLATFORM_CLOCK_RATE": 2400, + "VX_CFG_PLATFORM_MEMORY_PEAK_BW": 640000, + "VX_CFG_PLATFORM_MEMORY_TYPE": "GDDR6", + "VX_CFG_PLATFORM_MEMORY_DATA_RATE": 20000 + }, + "sources": { + "NUM_CORES": "64 CUs in 4 shader engines (AMD RX 9070 XT specifications; Chips and Cheese, RDNA 4 architecture)", + "NUM_THREADS, ISSUE_WIDTH, FETCH_WIDTH, NUM_WARPS": "2 SIMD32 units per CU, each with its own scheduler, running wave32 at one instruction per clock; 16 waves per SIMD (AMD RDNA 4 ISA; zolotukhin.ai RDNA3/RDNA4 reference)", + "NUM_ALU_LANES, NUM_FPU_LANES": "32 lanes per SIMD for FP32 and INT32 (VALU). RDNA 4's dual-issue (VOPD) and the paired FP32 rate behind its 48.7 TFLOPS are not modelled: the profile issues one VALU instruction per SIMD per clock", + "NUM_LSU_LANES, NUM_LSU_BLOCKS": "One vector-memory path per CU behind the 32 KB L0 vector cache, which returns 64 B per clock (16 x 4 B lanes)", + "DCACHE_SIZE": "32 KB L0 vector cache per CU, 64 B lines. RDNA 4 turned the shader-array L1 into a read/write coalescing buffer, so there is no L1 level to model (Chips and Cheese, RDNA 4)", + "ICACHE_SIZE": "32 KB instruction cache per WGP (two CUs), i.e. 16 KB per CU", + "LMEM_LOG_SIZE, LMEM_NUM_BANKS": "64 KB LDS per CU in CU mode (128 KB per WGP), 32 banks of 4 B = 128 B per clock per CU; Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock", + "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "8 MB L2, 2 MB per shader engine, 128 B lines (Chips and Cheese, RDNA 4); one bank per memory channel; 16 ways as GCN/RDNA L2s", + "L3_ENABLE, L3_SIZE, L3_NUM_WAYS, L3_NUM_BANKS": "64 MB third-generation Infinity Cache (MALL) behind the L2, modelled as the Vortex L3, one bank per memory channel", + "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE, PLATFORM_MEMORY_PEAK_BW": "256-bit GDDR6 at 20 Gbps: 8 devices x 2 x16 channels = 16 channels, 16 x 2 B x 20 GT/s = 640 GB/s; latency not calibrated", + "PLATFORM_CLOCK_RATE": "2400 MHz game clock (2970 MHz boost)", + "EXT_MUFU_ENABLE, NUM_MUFU_LANES": "Transcendentals (exp, log, rcp, rsq, sqrt) at a quarter rate per SIMD: 8 per SIMD, 16 per CU per clock", + "EXT_TCU_ENABLE, TCU_*": "Two AI accelerators (WMMA) per CU, one per SIMD as NUM_TCU_BLOCKS=ISSUE_WIDTH: FP16/BF16, FP8/BF8, INT8 and INT4 with 4:2 sparsity; no TF32", + "EXT_RTU_ENABLE": "One ray accelerator per CU", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, NUM_BARRIERS, EXT_TEX_ENABLE, EXT_RASTER_ENABLE, EXT_OM_ENABLE": "GPUVM paging, global and LDS atomics with compare-swap, FP16, workgroup barriers, and the graphics pipeline (4 texture units per CU, rasterizers, ROPs) of a gaming part", + "xlen": "64-bit machine: 64-bit pointers and FP64. FP64 runs at 1/32 of the dual-issue FP32 rate (4 per CU per clock) on the part; with XLEN=64 every FPU lane executes FP64, so the profile has 64 FP64 lanes per CU", + "FDIV_PE_RATIO, FSQRT_PE_RATIO": "Transcendentals (reciprocal, reciprocal square root, on which division and square root are built) run at a quarter of the FP32 rate on GCN, RDNA and CDNA: FPU lanes / 4" + } +} diff --git a/profiles/v100.json b/profiles/v100.json index 451551d910..dc1007341f 100644 --- a/profiles/v100.json +++ b/profiles/v100.json @@ -1,11 +1,33 @@ { "name": "v100", "description": "NVIDIA Tesla V100 SXM2 (GV100, Volta, sm_70) at its 1530 MHz boost clock: one Vortex core per SM.", + "xlen": 64, "core": { "VX_CFG_NUM_WARPS": 64, "VX_CFG_NUM_THREADS": 32, "VX_CFG_ISSUE_WIDTH": 4, - "VX_CFG_DCACHE_SIZE": 131072 + "VX_CFG_NUM_ALU_LANES": 16, + "VX_CFG_NUM_FPU_LANES": 16, + "VX_CFG_NUM_LSU_LANES": 8, + "VX_CFG_DCACHE_SIZE": 131072, + "VX_CFG_FETCH_WIDTH": 4, + "VX_CFG_LMEM_LOG_SIZE": 16, + "VX_CFG_LMEM_NUM_BANKS": 16, + "VX_CFG_VM_ENABLE": true, + "VX_CFG_EXT_A_ENABLE": true, + "VX_CFG_EXT_ZACAS_ENABLE": true, + "VX_CFG_EXT_ZFHMIN_ENABLE": true, + "VX_CFG_EXT_MUFU_ENABLE": true, + "VX_CFG_NUM_MUFU_LANES": 16, + "VX_CFG_EXT_TCU_ENABLE": true, + "VX_CFG_NUM_BARRIERS": 16, + "VX_CFG_EXT_TEX_ENABLE": true, + "VX_CFG_EXT_RASTER_ENABLE": true, + "VX_CFG_EXT_OM_ENABLE": true, + "VX_CFG_FDIV_PE_RATIO": 4, + "VX_CFG_FSQRT_PE_RATIO": 4, + "VX_CFG_FCVT_PE_RATIO": 4, + "VX_CFG_DCACHE_NUM_BANKS": 4 }, "chip": { "VX_CFG_NUM_CLUSTERS": 1, @@ -13,7 +35,8 @@ "VX_CFG_L2_ENABLE": true, "VX_CFG_L2_SIZE": 4194304, "VX_CFG_L2_NUM_WAYS": 16, - "VX_CFG_L2_NUM_BANKS": 32 + "VX_CFG_L2_NUM_BANKS": 32, + "VX_CFG_CP_LLC_CLIENT": 1 }, "platform": { "VX_CFG_PLATFORM_MEMORY_NUM_BANKS": 32, @@ -27,6 +50,20 @@ "DCACHE_SIZE": "128 KB combined L1/shared per SM (whitepaper)", "L2_SIZE, L2_NUM_WAYS, L2_NUM_BANKS": "6 MB L2 (whitepaper), rounded down to the power of two the cache accepts; 16 ways, 32 banks for the 32 memory channels", "PLATFORM_MEMORY_TYPE, PLATFORM_MEMORY_NUM_BANKS, PLATFORM_MEMORY_DATA_RATE": "4 HBM2 stacks x 8 128-bit channels = 32 channels at 1752 MT/s: 32 x 16 B x 1752 MT/s = 897 GB/s (whitepaper)", - "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)" + "PLATFORM_CLOCK_RATE, PLATFORM_MEMORY_PEAK_BW": "1530 MHz boost clock, 900 GB/s (whitepaper, Table 1)", + "FETCH_WIDTH": "4 SM processing blocks, each with its own warp scheduler, dispatch unit and L0 instruction cache (NVIDIA Tesla V100 GPU Architecture whitepaper)", + "CP_LLC_CLIENT": "Host copies reach device memory through the L2 like every other client, so uploaded data is L2-resident at launch", + "LMEM_LOG_SIZE, DCACHE_SIZE": "96 KB maximum shared memory per SM (CUDA Programming Guide, compute capability 7.0), rounded down to the power of two the LMEM accepts. DCACHE_SIZE keeps the whole 128 KB L1/shared pool: the driver gives a kernel the L1 its shared-memory use leaves, so a kernel without shared memory sees the full pool", + "LMEM_NUM_BANKS": "128 B per clock per SM: 32 banks of 4 B on the part (CUDA C++ Programming Guide, shared memory). Vortex banks are XLEN wide (8 B), so 16 banks carry the same bytes per clock; lanes reading one 8 B word share its bank access, so a stride-1 FP32 warp read is conflict-free as on the part", + "VM_ENABLE, EXT_A_ENABLE, EXT_ZACAS_ENABLE, EXT_ZFHMIN_ENABLE, EXT_MUFU_ENABLE": "Every NVIDIA GPU since Pascal has a demand-paged MMU (unified memory), global and shared-memory atomics with a native compare-and-swap (atomicCAS), FP16 storage and conversion, and per-partition special-function units for the transcendentals (MUFU)", + "EXT_TCU_ENABLE, TCU_*, EXT_DXA_ENABLE, EXT_RTU_ENABLE": "Volta tensor cores: FP16 inputs with FP32 accumulation only (Volta whitepaper). The TCU's FP16 format group also carries BF16, which Volta lacks: no knob separates them", + "xlen": "64-bit machine: 64-bit pointers and FP64 (the D extension follows XLEN=64), with a 48-bit device address space against NVIDIA's 49-bit virtual addresses; configure --profile picks a build64 tree and a build32 tree rejects the profile", + "NUM_BARRIERS": "16 named CTA barriers (bar.sync 0-15, PTX ISA) on every NVIDIA GPU", + "EXT_TEX_ENABLE": "Texture units in every SM", + "EXT_RASTER_ENABLE, EXT_OM_ENABLE": "Raster engines in every GPC and ROPs in every memory partition (architecture whitepaper)", + "NUM_MUFU_LANES": "16 special-function results per clock per SM (4 SFUs in each of the 4 processing blocks) for the reciprocal, reciprocal square root, log2, exp2, sine and cosine (CUDA C++ Programming Guide, arithmetic instruction throughput, compute capability 7.0)", + "NUM_ALU_LANES, NUM_FPU_LANES, NUM_LSU_LANES": "GV100 SM (Volta whitepaper, Figure 5): 4 processing blocks, each with 16 FP32, 16 INT32, 8 FP64 cores, 8 LD/ST units, 1 SFU and 2 tensor cores. One Vortex ALU/FPU/LSU block per processing block (ISSUE_WIDTH=4) with the part's INT32, FP32 and LD/ST lanes; a 32-thread warp instruction issues over NUM_THREADS/lanes cycles, as on the part. The TCU stays full-warp (NUM_TCU_LANES == NUM_THREADS). FP64 is not modelled separately: with XLEN=64 each FPU lane also executes FP64, so the profile has 64 FP64 lanes per SM against the part's 32", + "FDIV_PE_RATIO, FSQRT_PE_RATIO, FCVT_PE_RATIO": "16 results per clock per SM, 4 per processing block: FP32 reciprocal and reciprocal square root, on which division and square root are built, and the 32-bit type conversions (CUDA C++ Programming Guide, arithmetic instruction throughput); 16 FPU lanes / 4, compute capability 7.0", + "DCACHE_NUM_BANKS": "128 B per clock of L1 data per SM: 4 banks of the 32 B dcache word. The default (one bank per LSU request channel) gives 8, twice the part (Volta whitepaper: 128 B/clk L1)" } } diff --git a/sim/avedsim/Makefile b/sim/avedsim/Makefile index 863442440d..730e82c183 100644 --- a/sim/avedsim/Makefile +++ b/sim/avedsim/Makefile @@ -146,6 +146,16 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -154,6 +164,7 @@ VL_FLAGS += --x-initial unique --x-assign unique VL_FLAGS += verilator.vlt VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) VL_FLAGS += $(CONFIGS) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/avedsim/vortex_afu_shim.sv b/sim/avedsim/vortex_afu_shim.sv index ecad4997bc..9600a70718 100644 --- a/sim/avedsim/vortex_afu_shim.sv +++ b/sim/avedsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -65,7 +65,7 @@ module vortex_afu_shim #( .ap_clk (ap_clk), .ap_rst_n (ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_ARGS, MP_COMMA), `AXI_HOST_ARGS, diff --git a/sim/common/axi_latency_pipe.h b/sim/common/axi_latency_pipe.h new file mode 100644 index 0000000000..c7591af18b --- /dev/null +++ b/sim/common/axi_latency_pipe.h @@ -0,0 +1,189 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace vortex { + +// Timing of one AXI4 memory port as a latency-bandwidth pipe: a fixed latency +// per transaction and a bound on the reads and writes in flight, one beat per +// cycle on each response channel. Cycle for cycle this is FireSim FASED's +// "Latency Bandwidth Pipe" (LatencyBandwidthPipe.scala over TimingModel.scala's +// SplitTransactionModel and AXI4Releaser); the defaults are FASED's hardware +// defaults. Every output is a function of registered state only, so the caller +// samples the port's request side once per cycle, before the clock edge, and +// calls step() after it. Timing only: responses leave in the order their +// requests were accepted, so the caller keeps the data alongside. +class AxiLatencyPipe { +public: + struct Config { + uint32_t read_latency = 30; + uint32_t write_latency = 30; + uint32_t max_reads = 16; + uint32_t max_writes = 16; + }; + + struct Inputs { + bool ar_valid = false; + uint32_t ar_id = 0; + uint32_t ar_len = 0; + bool aw_valid = false; + uint32_t aw_id = 0; + bool w_valid = false; + bool w_last = false; + bool r_ready = false; + bool b_ready = false; + }; + + AxiLatencyPipe() : cfg_() {} + explicit AxiLatencyPipe(const Config& cfg) : cfg_(cfg) {} + + void reset() { + *this = AxiLatencyPipe(cfg_); + } + + bool ar_ready() const { return pending_reads_ < cfg_.max_reads; } + bool aw_ready() const { return pending_aw_ < cfg_.max_writes; } + bool w_ready() const { return pending_w_ < cfg_.max_writes; } + + bool r_valid() const { return cur_read_.has_value(); } + uint32_t r_id() const { return cur_read_->id; } + bool r_last() const { return cur_read_->beat == cur_read_->len; } + uint32_t r_beat() const { return cur_read_->beat; } + + bool b_valid() const { return cur_write_.has_value(); } + uint32_t b_id() const { return cur_write_->id; } + + // Advances one cycle on the request-side values sampled before the edge. + void step(const Inputs& in) { + bool ar_fire = in.ar_valid && this->ar_ready(); + bool aw_fire = in.aw_valid && this->aw_ready(); + bool wlast_fire = in.w_valid && in.w_last && this->w_ready(); + bool r_fire = this->r_valid() && in.r_ready; + bool rlast_fire = r_fire && this->r_last(); + bool b_fire = this->b_valid() && in.b_ready; + + // Reads: latency pipe (a flow queue) into a one-entry pipe stage that + // drives R. Release is one cycle early, since that stage adds the cycle. + { + std::optional arrived; + if (ar_fire) { + arrived = Read{cycle_ + cfg_.read_latency - 1, in.ar_id, in.ar_len, 0}; + } + Read* head = !read_pipe_.empty() ? &read_pipe_.front() : (arrived ? &*arrived : nullptr); + bool release = head && head->release <= cycle_; + bool stage_ready = !cur_read_ || rlast_fire; + if (r_fire && !rlast_fire) { + ++cur_read_->beat; + } else if (rlast_fire) { + cur_read_.reset(); + } + if (release && stage_ready) { + cur_read_ = *head; + if (!read_pipe_.empty()) { + read_pipe_.pop_front(); + } else { + arrived.reset(); + } + } + if (arrived) { + read_pipe_.push_back(*arrived); + } + } + + // Writes: an AW and its last W beat, in either order, form one write; its + // latency counts from the cycle both have arrived. + { + bool new_write = (pending_w_ > pending_aw_ && aw_fire) + || (pending_w_ < pending_aw_ && wlast_fire) + || (aw_fire && wlast_fire); + std::optional aw_arrived; + if (aw_fire) { + aw_arrived = in.aw_id; + } + std::optional formed; + if (new_write) { + uint32_t id; + if (!aw_queue_.empty()) { + id = aw_queue_.front(); + aw_queue_.pop_front(); + } else { + id = *aw_arrived; + aw_arrived.reset(); + } + formed = Write{cycle_ + cfg_.write_latency - 1, id}; + } + if (aw_arrived) { + aw_queue_.push_back(*aw_arrived); + } + Write* head = !write_pipe_.empty() ? &write_pipe_.front() : (formed ? &*formed : nullptr); + bool release = head && head->release <= cycle_; + bool stage_ready = !cur_write_ || b_fire; + if (b_fire) { + cur_write_.reset(); + } + if (release && stage_ready) { + cur_write_ = *head; + if (!write_pipe_.empty()) { + write_pipe_.pop_front(); + } else { + formed.reset(); + } + } + if (formed) { + write_pipe_.push_back(*formed); + } + } + + pending_reads_ = count(pending_reads_, ar_fire, rlast_fire); + pending_aw_ = count(pending_aw_, aw_fire, b_fire); + pending_w_ = count(pending_w_, wlast_fire, b_fire); + ++cycle_; + } + +private: + struct Read { + uint64_t release; + uint32_t id; + uint32_t len; + uint32_t beat; + }; + + struct Write { + uint64_t release; + uint32_t id; + }; + + static uint32_t count(uint32_t value, bool inc, bool dec) { + if (inc && !dec) return value + 1; + if (dec && !inc && value != 0) return value - 1; + return value; + } + + Config cfg_; + uint64_t cycle_ = 0; + uint32_t pending_reads_ = 0; + uint32_t pending_aw_ = 0; + uint32_t pending_w_ = 0; + std::deque read_pipe_; + std::optional cur_read_; + std::deque aw_queue_; + std::deque write_pipe_; + std::optional cur_write_; +}; + +} // namespace vortex diff --git a/sim/common/cmd_processor.cpp b/sim/common/cmd_processor.cpp index 0972440954..6bb250b452 100644 --- a/sim/common/cmd_processor.cpp +++ b/sim/common/cmd_processor.cpp @@ -15,7 +15,8 @@ namespace vortex { // ============================================================================ // Static GPU capability words // -// GPU_DEV_CAPS / GPU_ISA_CAPS register values exposed as read-only MMIO. +// GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS register values exposed as +// read-only MMIO. // sw/runtime/common/vx_caps.h is the matching decoder. // ============================================================================ namespace { @@ -45,6 +46,10 @@ uint64_t gpu_dev_caps() { | ((uint64_t(cluster_size) & 0xFF) << 49) | ((uint64_t(VX_CFG_NUM_CLUSTERS) & 0x7F) << 57); } +uint32_t gpu_dev_caps_ext() { + static_assert(VX_CFG_SOCKET_SIZE < 256, "GPU_DEV_CAPS_EXT socket size overflows"); + return uint32_t(VX_CFG_SOCKET_SIZE); +} uint64_t gpu_isa_caps() { return (uint64_t(VX_CFG_MISA_EXT) << 32) | ((uint64_t(cp_clog2(VX_CFG_XLEN) - 4) & 0x3) << 30) @@ -78,7 +83,7 @@ void CommandProcessor::mmio_write(uint32_t off, uint32_t value) { case 0x028: satp_ = (satp_ & 0xFFFFFFFF00000000ULL) | uint64_t(value); return; case 0x02C: satp_ = (satp_ & 0x00000000FFFFFFFFULL) | (uint64_t(value) << 32); return; // STATUS / DEV_CAPS / CYCLE / GPU caps are RO; ignore writes. - case 0x004: case 0x008: case 0x010: case 0x014: + case 0x004: case 0x008: case 0x00C: case 0x010: case 0x014: case 0x018: case 0x01C: case 0x020: case 0x024: return; } // Queue 0 (offsets 0x100..0x12F) @@ -110,7 +115,8 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { case 0x000: return cp_ctrl_; case 0x004: return uint32_t(busy() ? 1 : 0); // CP_STATUS bit0 case 0x008: { - // CP_DEV_CAPS: {SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | + // CP_DEV_CAPS: {DMA_LLC_CLIENT:1 @bit28 | MMU_FAULT_REPORT:1 @bit27 | + // SUPPORTS_QMD:1 @bit26 | SUPPORTS_DRAW:1 @bit25 | // VM_ENABLED:1 @bit24 | AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8}. // Defaults: TID=6, RING_LOG2=16, NUM_QUEUES=1. VM_ENABLED reflects // the build-config so the config-agnostic libvortex.so can @@ -125,9 +131,11 @@ uint32_t CommandProcessor::mmio_read(uint32_t off) const { const uint32_t supports_draw = 1u << 25; const uint32_t supports_qmd = 1u << 26; const uint32_t fault_report = hooks_.mmu_fault_report ? (1u << 27) : 0; - return supports_qmd | supports_draw | vm_enabled | fault_report + const uint32_t llc_client = hooks_.dma_llc_client ? (1u << 28) : 0; + return llc_client | supports_qmd | supports_draw | vm_enabled | fault_report | (uint32_t(6) << 16) | (uint32_t(16) << 8) | uint32_t(1); } + case 0x00C: return gpu_dev_caps_ext(); case 0x010: return uint32_t(cycle_counter_ & 0xFFFFFFFF); case 0x014: return uint32_t(cycle_counter_ >> 32); case 0x018: return uint32_t(gpu_dev_caps() & 0xFFFFFFFF); @@ -328,6 +336,26 @@ void CommandProcessor::apply_qmd_(uint64_t qmd_addr) { } } +// One DCR write to VX_DCR_BASE_CACHE_FLUSH reaches every core and starts its +// flush, so the cores flush concurrently and share one walk of the shared +// caches; the per-core reads, answered once that core's flush completes, are +// serial because the DCR response bus cannot carry two responses at once. +void CommandProcessor::flush_cores_(uint32_t num_cores, bool l1_only) { + if (num_cores == 0 || !hooks_.vortex_dcr_read) { + return; + } + if (hooks_.vortex_dcr_write) { + hooks_.vortex_dcr_write(VX_DCR_BASE_CACHE_FLUSH, + l1_only ? VX_CACHE_FLUSH_L1_ONLY : 0); + } + for (uint32_t cid = 0; cid < num_cores; ++cid) { + (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); + } + if (!l1_only && hooks_.vortex_drain) { + hooks_.vortex_drain(); + } +} + // Execute one draw-bundle step. Inline ops (DCR_WRITE/DCR_READ/CACHE_FLUSH) // complete immediately and return false; a launch step (LAUNCH/LAUNCH_QMD) // kicks the launch sub-FSM and returns true so the caller waits for the drain @@ -351,11 +379,8 @@ bool CommandProcessor::exec_inline_cmd_(const Cmd& c) { uint32_t(c.arg0 & 0xFFF), uint32_t(c.arg1 & 0xFFFFFFFF)); return false; case OP_CACHE_FLUSH: - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(c.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } + this->flush_cores_(uint32_t(c.arg0 & 0xFFFFFFFF), + (c.flags & FLUSH_FLAG_L1_ONLY) != 0); return false; default: // NOP / FENCE / unknown step — no-op (draws don't use MEM_*/EVENT_*). @@ -484,15 +509,9 @@ void CommandProcessor::tick_engine() { } eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_CACHE_FLUSH) { - // Sweep a per-core DCR-read of VX_DCR_BASE_CACHE_FLUSH across - // [0, num_cores). arg0 carries num_cores; dcr_read routes each - // call to flush_caches() (drains dcache/L2/L3 to memsim). - if (hooks_.vortex_dcr_read) { - uint32_t n = uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF); - for (uint32_t cid = 0; cid < n; ++cid) { - (void)hooks_.vortex_dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid); - } - } + // arg0 carries num_cores. + this->flush_cores_(uint32_t(cur_cmd_.arg0 & 0xFFFFFFFF), + (cur_cmd_.flags & FLUSH_FLAG_L1_ONLY) != 0); eng_state_ = EngState::Retire; } else if (cur_cmd_.opcode == OP_EVENT_SIG) { // CMD_EVENT_SIGNAL: write arg1 (8-byte value) to arg0 @@ -536,6 +555,8 @@ void CommandProcessor::tick_engine() { src = cp_translate(src, physical); } const uint64_t total = cur_cmd_.arg2; + const bool no_alloc = hooks_.dma_no_alloc_min != 0 + && total >= hooks_.dma_no_alloc_min; constexpr uint64_t CHUNK = 64 * 1024; std::vector buf( std::size_t(total < CHUNK ? total : CHUNK)); @@ -543,7 +564,11 @@ void CommandProcessor::tick_engine() { uint64_t n = total - done; if (n > CHUNK) n = CHUNK; hooks_.dram_read (src + done, buf.data(), n); - hooks_.dram_write(dst + done, buf.data(), n); + if (hooks_.dma_write) { + hooks_.dma_write(dst + done, buf.data(), n, no_alloc); + } else { + hooks_.dram_write(dst + done, buf.data(), n); + } done += n; } } diff --git a/sim/common/cmd_processor.h b/sim/common/cmd_processor.h index 0ccd6fc379..de5199979a 100644 --- a/sim/common/cmd_processor.h +++ b/sim/common/cmd_processor.h @@ -26,6 +26,7 @@ // 0x000 CP_CTRL bit0=enable_global, bit1=reset_all // 0x004 CP_STATUS bit0=busy, bit1=error // 0x008 CP_DEV_CAPS {AXI_TID_W:8 | RING_LOG2:8 | NUM_QUEUES:8} +// 0x00C GPU_DEV_CAPS_EXT device-config caps extension (RO) // 0x010 CP_CYCLE_LO // 0x014 CP_CYCLE_HI // 0x018 GPU_DEV_CAPS_LO device-config caps, low 32 bits (RO) @@ -98,10 +99,26 @@ class CommandProcessor { // before retiring the CMD_LAUNCH. std::function vortex_busy; + // Block until memory has acknowledged every write Vortex issued. A + // full CMD_CACHE_FLUSH retires only then. Unset for a model whose + // writes land as they are issued. + std::function vortex_drain; + // Backend decodes the MMU fault-report DCRs. Published in DEV_CAPS // so the runtime does not read them on a target that would leave // the DCR bus waiting for a response. bool mmu_fault_report = false; + + // Write `bytes` bytes from `src` to `addr` for a CMD_MEM_* transfer. + // `no_alloc` marks a transfer of at least dma_no_alloc_min bytes, + // whose writes should not allocate in the device caches. Unset: + // dram_write. + std::function dma_write; + uint64_t dma_no_alloc_min = 0; + + // Device-memory accesses go through the last-level cache, so host + // transfers need no cache flush. Published in DEV_CAPS. + bool dma_llc_client = false; }; explicit CommandProcessor(const Hooks& hooks); @@ -164,6 +181,10 @@ class CommandProcessor { // mode is its own descriptor field (modern-GPU convention). static constexpr uint8_t MEM_FLAG_PHYSICAL = 0x04; + // CMD_CACHE_FLUSH header flag: flush only the levels above the last-level + // cache. + static constexpr uint8_t FLUSH_FLAG_L1_ONLY = 0x04; + // Decoded cmd record (matches cmd_t struct layout on-wire). struct Cmd { uint8_t opcode; @@ -242,6 +263,9 @@ class CommandProcessor { // CMD_LAUNCH_QMD: read the in-memory KMU descriptor at `qmd_addr` and // replay its {dcr_addr,value} pairs through the DCR-write hook. void apply_qmd_(uint64_t qmd_addr); + // CMD_CACHE_FLUSH across cores [0, num_cores); l1_only leaves the + // last-level cache unwalked. + void flush_cores_(uint32_t num_cores, bool l1_only); // Advance the launch FSM one step using cur_cmd_. void tick_launch(); // Advance the engine FSM one step. diff --git a/sim/common/dram_bank_model.h b/sim/common/dram_bank_model.h new file mode 100644 index 0000000000..0bc8b3c694 --- /dev/null +++ b/sim/common/dram_bank_model.h @@ -0,0 +1,876 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace vortex { + +// One DRAM channel as a reservation-based bank model: each request is +// scheduled once, by placing its precharge, activate and column commands at +// the first DRAM clocks every JEDEC constraint allows -- before or after +// commands reserved earlier -- and its read data departs tCL + tBL after its +// last column. Banks keep their rows open; a request to a bank waits for the +// bank's earlier requests (first come, first served per bank). The controller +// issues one command per clock; on HBM an ACT holds the row-command bus for +// two. +// +// Requests enter the controller's read or write buffer in order, one per +// request's bursts' worth of clocks, while the buffer has room; a write is +// complete once buffered. The controller serves reads, and turns to writes +// when its write bursts reach the high watermark or no read waits, back once +// they fall below the low one with a read waiting -- dram_sim's write drain. A +// batch of writes is placed after every column reserved before it, and the +// reads after it after the batch, so the data bus turns around once per batch. +// Reservations reach at most `horizon` clocks ahead: a request that would +// reach further waits. +// +// Refresh, when on, is an operation of its own: every `refi` clocks it takes +// the clock's turn to reserve. A per-bank refresh closes the next bank in +// turn -- a precharge once the bank's commands allow it, then REF -- and keeps +// it from activating for `rfc` clocks; an all-bank refresh does the same to +// every bank with one precharge and one REF. A refresh that would reach past +// the horizon waits, and the requests with it. +// +// Foldsim's memory endpoint (its shell's rtl/foldsim_mem_bank.sv) implements +// the same decisions; the normative copy of this model is the shell's +// model/dram_bank_model.h, and this one gives Vortex's simulators the same +// timing (rtlsim's AXI_MEM_MODEL=bank, against which ci/testcases/foldsim.yaml +// holds Foldsim cycle-exact). The faces below put it behind DramSim's +// interface and behind an AXI4 port. +class DramBankModel { +public: + enum class Refresh { Off, PerBank, AllBank }; + + struct Config { + dram_timing::geometry_t geom; + dram_timing::clocks_t t; + Refresh refresh = Refresh::Off; // dram_sim models none + uint32_t refi = 0; // clocks from one refresh operation to the next + uint32_t rfc = 0; // clocks a refreshed bank may not activate + uint32_t read_queue = 32; // reserved read bursts whose column has not issued + uint32_t write_queue = 32; // the same for writes + // Write bursts buffered or reserved and not issued at which the channel + // turns to writes, and below which it turns back to waiting reads: + // dram_sim's 80 % and 20 % of its 32-entry write buffer. + uint32_t wr_high = 26; + uint32_t wr_low = 7; + // Clocks a reservation may reach past the current one: foldsim's ring of + // 512 clocks, less the word of 64 it is retiring. + uint32_t horizon = 448; + uint32_t bursts = 2; // 32 B bursts per request + + static Config make(dram_timing::Standard standard, uint32_t data_rate, uint32_t block_size, + Refresh refresh = Refresh::Off) { + Config c; + c.geom = dram_timing::geometry(standard); + c.t = dram_timing::clocks(standard, data_rate); + c.bursts = (block_size > dram_timing::kBurstSize) ? block_size / dram_timing::kBurstSize : 1; + if (c.bursts > (1u << c.geom.log2_visit)) { + throw std::invalid_argument("dram_bank_model: a request spans more than one row visit"); + } + c.set_refresh(refresh, dram_timing::refresh(standard, data_rate)); + return c; + } + + // Per bank, the banks take turns within tREFI; all-bank, the channel + // refreshes once per tREFI. + void set_refresh(Refresh mode, const dram_timing::refresh_t& r) { + refresh = mode; + refi = (mode == Refresh::Off) ? 0 : (mode == Refresh::PerBank) ? r.REFI / this->num_banks() : r.REFI; + rfc = (mode == Refresh::Off) ? 0 : (mode == Refresh::PerBank) ? r.RFCPB : r.RFC; + } + + static Refresh parse_refresh(const std::string& name) { + if (name.empty() || name == "off") return Refresh::Off; + if (name == "bank") return Refresh::PerBank; + if (name == "all") return Refresh::AllBank; + throw std::invalid_argument("dram_bank_model: refresh must be off, bank or all (got '" + name + "')"); + } + + // Column-to-column spacing on one pseudo-channel, and within one bank + // group, by the directions of the earlier and the later command. + uint32_t rtw() const { + int64_t v = int64_t(t.CL) + t.BL + geom.rtw_extra - t.CWL; + return uint32_t(std::max(t.CCDS, v)); + } + uint32_t wtr_s() const { return t.CWL + t.BL + t.WTRS; } + uint32_t wtr_l() const { return t.CWL + t.BL + t.WTRL; } + uint32_t sep_pc(bool from_w, bool to_w) const { + if (from_w == to_w) return t.CCDS; + return from_w ? wtr_s() : rtw(); + } + uint32_t sep_bg(bool from_w, bool to_w) const { + if (from_w == to_w) return t.CCDL; + return from_w ? wtr_l() : 0; + } + // The furthest clock after a command its constraints on later commands + // reach, which must lie within the horizon. + uint32_t col_reach(bool w) const { + uint32_t r = std::max(t.CCDS, t.CCDL); + r = std::max(r, w ? std::max(wtr_s(), wtr_l()) : rtw()); + return r - 1; + } + uint32_t act_reach() const { + uint32_t r = std::max({t.RRDS, t.RRDL, t.FAW}) - 1; + return std::max(r, geom.two_clock_act ? 1 : 0); + } + uint32_t num_banks() const { return 1u << geom.bank_bits(); } + uint32_t pc_of(uint32_t bank) const { return bank & ((1u << geom.log2_pcs) - 1); } + uint32_t bg_of(uint32_t bank) const { + return (bank >> geom.log2_pcs) & ((1u << geom.log2_bgs) - 1); + } + }; + + struct Stats { + uint64_t reads = 0, writes = 0, hits = 0, misses = 0, conflicts = 0, refreshes = 0; + uint64_t deferred = 0; // clocks a request or refresh waited: it reached past the horizon + }; + + explicit DramBankModel(const Config& cfg) : cfg_(cfg) { this->reset(); } + + const Config& config() const { return cfg_; } + const Stats& stats() const { return stats_; } + + void reset() { + d_ = 0; + ingress_.clear(); + ingress_ready_ = 0; + uint32_t nb = cfg_.num_banks(); + open_.assign(nb, false); + row_.assign(nb, 0); + act_.assign(nb, kNever); + col_ready_.assign(nb, 0); + pre_ready_.assign(nb, 0); + cmd_.clear(); + rowbus_.clear(); + cols_.clear(); + acts_.clear(); + colkind_.clear(); + dep_.clear(); + rq_ = 0; + wq_ = 0; + rbuf_.clear(); + wbuf_.clear(); + write_mode_ = false; + frontier_ = 0; + batch_floor_ = 0; + batch_last_ = 0; + read_floor_ = 0; + ref_due_ = cfg_.refi; + ref_bank_ = 0; + stats_ = Stats(); + } + + // The bank and row of a byte address within the channel: a row visit of + // four bursts, then pseudo-channel, bank group and bank from the low end, + // the bank XORed with the row folded to its width (permutation interleave). + void decode(uint64_t addr, uint32_t* bank, uint32_t* row) const { + const auto& g = cfg_.geom; + uint32_t bb = g.bank_bits(); + uint64_t tx = (addr & (dram_timing::kChannelSize - 1)) / dram_timing::kBurstSize; + tx >>= g.log2_visit; + uint64_t b = tx & ((1u << bb) - 1); + tx >>= bb; + uint64_t r = tx >> (g.log2_row_bursts - g.log2_visit); + for (uint64_t x = r; x != 0; x >>= bb) { + b ^= x & ((1u << bb) - 1); + } + *bank = uint32_t(b); + *row = uint32_t(r); + } + + // A request joins the ingress; `tag` comes back when it completes. + void push(uint32_t tag, bool write, uint64_t addr) { + Request r; + r.tag = tag; + r.write = write; + this->decode(addr, &r.bank, &r.row); + ingress_.push_back(r); + } + + bool ingress_empty() const { return ingress_.empty(); } + size_t ingress_size() const { return ingress_.size(); } + uint64_t clock() const { return d_; } + + // One DRAM clock d: columns issued by d leave the controller queues, the + // ingress head enters its buffer if it may, the head of the buffer the + // controller serves is reserved if it can be (its commands no earlier than + // d+1), and a read whose data departs at d+1 completes. + template + void tick(ReadDone&& read_done, WriteDone&& write_done) { + const int64_t d = int64_t(d_); + for (auto it = colkind_.begin(); it != colkind_.end() && it->first <= d;) { + (it->second ? wq_ : rq_) -= 1; + it = colkind_.erase(it); + } + const uint32_t nb = cfg_.bursts; + if (!ingress_.empty() && d >= ingress_ready_) { + const Request& r = ingress_.front(); + if (r.write ? (wq_ + nb * (wbuf_.size() + 1) <= cfg_.write_queue) + : (rq_ + nb * (rbuf_.size() + 1) <= cfg_.read_queue)) { + (r.write ? wbuf_ : rbuf_).push_back(r); + if (r.write) { + write_done(r.tag); + } + ingress_.pop_front(); + ingress_ready_ = d + nb; + } + } + const uint64_t wn = wq_ + nb * wbuf_.size(); + const uint64_t rn = rq_ + nb * rbuf_.size(); + if (!write_mode_) { + if (wn >= cfg_.wr_high || rn == 0) { + write_mode_ = true; + batch_floor_ = std::max(d + 1, frontier_ + 1); + } + } else if (wn < cfg_.wr_low && rn != 0) { + write_mode_ = false; + read_floor_ = batch_last_ + 1; + } + auto& buf = write_mode_ ? wbuf_ : rbuf_; + if (cfg_.refresh != Refresh::Off && cfg_.refi != 0 && d + 1 >= ref_due_) { + if (this->refresh(d + 1, d + int64_t(cfg_.horizon))) { + ref_due_ += cfg_.refi; + ++stats_.refreshes; + } else { + ++stats_.deferred; + } + } else if (!buf.empty()) { + if (this->reserve(buf.front(), d + 1, d + int64_t(cfg_.horizon), + write_mode_ ? batch_floor_ : read_floor_)) { + buf.pop_front(); + } else { + ++stats_.deferred; + } + } + auto it = dep_.find(d + 1); + if (it != dep_.end()) { + read_done(it->second); + dep_.erase(it); + } + this->prune(d); + ++d_; + } + +private: + static constexpr int64_t kNever = INT64_MIN / 4; + static constexpr int64_t kNone = INT64_MAX / 4; + + struct Request { + uint32_t tag; + bool write; + uint32_t bank; + uint32_t row; + }; + + struct Col { + uint32_t pc, pg; + bool w; + }; + + struct Act { + uint32_t pc, pg; + }; + + bool cmd_free(int64_t t) const { return !cmd_.count(t); } + bool row_free(int64_t t) const { return !rowbus_.count(t); } + + // The first clock in [lb, end] a column of direction `w` to (pc, pg) may + // take, or kNone. A clock inside a reserved column's forbidden span skips + // to its end. + int64_t place_col(int64_t lb, int64_t end, uint32_t pc, uint32_t pg, bool w) const { + const int64_t reach = this->max_col_sep(); + int64_t t = lb; + while (t <= end) { + int64_t next = t; + if (!this->cmd_free(t)) { + next = t + 1; + } + for (auto it = cols_.lower_bound(t - reach); it != cols_.end() && it->first <= t + reach; ++it) { + const int64_t c = it->first; + const Col& col = it->second; + if (col.pc != pc || c == t) { + continue; + } + int64_t need_pc, need_bg, after_pc, after_bg; + after_pc = cfg_.sep_pc(col.w, w); + after_bg = cfg_.sep_bg(col.w, w); + if (c < t) { + need_pc = after_pc; + need_bg = after_bg; + } else { + need_pc = cfg_.sep_pc(w, col.w); + need_bg = cfg_.sep_bg(w, col.w); + } + const int64_t dist = (c < t) ? (t - c) : (c - t); + if (dist < need_pc) { + next = std::max(next, c + std::max(after_pc, 1)); + } + if (col.pg == pg && dist < need_bg) { + next = std::max(next, c + std::max(after_bg, 1)); + } + } + if (next == t) { + return t; + } + t = next; + } + return kNone; + } + + // True if an ACT to (pc, pg) may take clock t: the command and row buses + // are free, tRRD holds against every ACT of the pseudo-channel (tRRD_L + // within the bank group) on either side, and no tFAW window holding t + // holds four other ACTs of the pseudo-channel. + bool act_ok(int64_t t, uint32_t pc, uint32_t pg) const { + if (!this->cmd_free(t) || !this->row_free(t)) return false; + if (cfg_.geom.two_clock_act && !this->row_free(t + 1)) return false; + const int64_t F = cfg_.t.FAW; + const int64_t reach = std::max({F, cfg_.t.RRDS, cfg_.t.RRDL}); + std::vector near; + for (auto it = acts_.lower_bound(t - reach); it != acts_.end() && it->first <= t + reach; ++it) { + const int64_t a = it->first; + const Act& act = it->second; + if (act.pc != pc) continue; + const int64_t dist = (a < t) ? (t - a) : (a - t); + if (dist < int64_t(cfg_.t.RRDS)) return false; + if (act.pg == pg && dist < int64_t(cfg_.t.RRDL)) return false; + if (dist < F) near.push_back(a); + } + near.push_back(t); + std::sort(near.begin(), near.end()); + for (size_t i = 0; i + 4 < near.size(); ++i) { + if (near[i + 4] - near[i] < F) return false; + } + return true; + } + + int64_t place_act(int64_t lb, int64_t end, uint32_t pc, uint32_t pg) const { + for (int64_t t = lb; t <= end; ++t) { + if (this->act_ok(t, pc, pg)) return t; + } + return kNone; + } + + int64_t place_pre(int64_t lb, int64_t end) const { + for (int64_t t = lb; t <= end; ++t) { + if (this->cmd_free(t) && this->row_free(t)) return t; + } + return kNone; + } + + int64_t max_col_sep() const { + return std::max({cfg_.t.CCDS, cfg_.t.CCDL, cfg_.rtw(), cfg_.wtr_s(), cfg_.wtr_l()}); + } + + // Places the request's commands; false (and nothing changed) if its queue + // is full or it does not fit within the horizon. + bool reserve(const Request& r, int64_t e, int64_t end, int64_t floor) { + const auto& T = cfg_.t; + const uint32_t nb = cfg_.bursts; + const uint32_t b = r.bank; + const uint32_t pc = cfg_.pc_of(b); + const uint32_t pg = pc * (1u << cfg_.geom.log2_bgs) + cfg_.bg_of(b); + const bool hit = open_[b] && row_[b] == r.row; + const bool conflict = open_[b] && !hit; + int64_t pre = kNone, act = kNone; + int64_t col_ready = col_ready_[b]; + int64_t pre_ready = pre_ready_[b]; + if (!hit) { + int64_t act_lb = e; + if (conflict) { + pre = this->place_pre(std::max(e, pre_ready_[b]), end); + if (pre == kNone) return false; + act_lb = pre + T.RP; + } + act_lb = std::max(act_lb, act_[b] + int64_t(T.RC)); + act = this->place_act(act_lb, end, pc, pg); + if (act == kNone) return false; + col_ready = act + (r.write ? T.RCDWR : T.RCDRD); + pre_ready = act + T.RAS; + } + int64_t col[8]; + for (uint32_t k = 0; k < nb; ++k) { + int64_t lb = std::max({e + int64_t(k), col_ready, floor}); + if (k != 0) { + lb = std::max(lb, col[k - 1] + int64_t(std::max(T.CCDL, 1))); + } + col[k] = this->place_col(lb, end, pc, pg, r.write); + if (col[k] == kNone) return false; + col_ready = std::max(col_ready, col[k] + 1); + pre_ready = std::max(pre_ready, r.write ? col[k] + T.CWL + T.BL + T.WR : col[k] + T.RTP); + } + const int64_t last_col = col[nb - 1]; + const int64_t departure = last_col + T.CL + T.BL; + int64_t last = last_col + cfg_.col_reach(r.write); + if (!r.write) last = std::max(last, departure); + if (act != kNone) last = std::max(last, act + int64_t(cfg_.act_reach())); + if (last > end) { + return false; + } + + // Commit. + if (pre != kNone) { + cmd_.insert(pre); + rowbus_.insert(pre); + } + if (act != kNone) { + cmd_.insert(act); + rowbus_.insert(act); + if (cfg_.geom.two_clock_act) rowbus_.insert(act + 1); + acts_[act] = Act{pc, pg}; + act_[b] = act; + open_[b] = true; + row_[b] = r.row; + } + for (uint32_t k = 0; k < nb; ++k) { + cmd_.insert(col[k]); + cols_[col[k]] = Col{pc, pg, r.write}; + colkind_[col[k]] = r.write; + frontier_ = std::max(frontier_, col[k]); + if (r.write) batch_last_ = std::max(batch_last_, col[k]); + } + (r.write ? wq_ : rq_) += nb; + col_ready_[b] = col_ready; + pre_ready_[b] = pre_ready; + if (!r.write) { + dep_[departure] = r.tag; + } + if (r.write) ++stats_.writes; else ++stats_.reads; + if (hit) ++stats_.hits; else if (conflict) ++stats_.conflicts; else ++stats_.misses; + return true; + } + + // A refresh operation: the bank in turn (all banks), precharged if open -- + // after every command reserved to it allows -- then REF, once the precharge + // and the last activate allow; until rfc after it the bank may not + // activate. Both commands take the row-command bus as a precharge does. + // False (and nothing changed) if they do not fit within the horizon. + bool refresh(int64_t e, int64_t end) { + const auto& T = cfg_.t; + const uint32_t nb = cfg_.num_banks(); + const bool all = (cfg_.refresh == Refresh::AllBank); + bool open = false; + int64_t pre_lb = e, act_lb = e; + for (uint32_t b = 0; b < nb; ++b) { + if (!all && b != ref_bank_) continue; + if (open_[b]) { + open = true; + pre_lb = std::max(pre_lb, pre_ready_[b]); + } + act_lb = std::max(act_lb, act_[b] + int64_t(T.RC)); + } + int64_t pre = kNone, lb = act_lb; + if (open) { + pre = this->place_pre(pre_lb, end); + if (pre == kNone) return false; + lb = std::max(lb, pre + int64_t(T.RP)); + } + const int64_t ref = this->place_pre(lb, end); + if (ref == kNone) return false; + + // Commit. + if (pre != kNone) { + cmd_.insert(pre); + rowbus_.insert(pre); + } + cmd_.insert(ref); + rowbus_.insert(ref); + for (uint32_t b = 0; b < nb; ++b) { + if (!all && b != ref_bank_) continue; + open_[b] = false; + act_[b] = ref + int64_t(cfg_.rfc) - int64_t(T.RC); + } + if (!all) ref_bank_ = (ref_bank_ + 1) % nb; + return true; + } + + // Drops what can no longer constrain a command at d+1 or later. + void prune(int64_t d) { + const int64_t keep_cols = d + 1 - this->max_col_sep(); + while (!cols_.empty() && cols_.begin()->first < keep_cols) cols_.erase(cols_.begin()); + const int64_t keep_acts = d + 1 - std::max({cfg_.t.FAW, cfg_.t.RRDS, cfg_.t.RRDL}); + while (!acts_.empty() && acts_.begin()->first < keep_acts) acts_.erase(acts_.begin()); + while (!cmd_.empty() && *cmd_.begin() <= d) cmd_.erase(cmd_.begin()); + while (!rowbus_.empty() && *rowbus_.begin() <= d) rowbus_.erase(rowbus_.begin()); + } + + Config cfg_; + uint64_t d_; + std::deque ingress_; + int64_t ingress_ready_; + std::vector open_; + std::vector row_; + std::vector act_, col_ready_, pre_ready_; + std::set cmd_, rowbus_; + std::map cols_; + std::map acts_; + std::map colkind_; + std::map dep_; + uint32_t rq_, wq_; + std::deque rbuf_, wbuf_; + bool write_mode_; + int64_t frontier_, batch_floor_, batch_last_, read_floor_; + int64_t ref_due_; + uint32_t ref_bank_; + Stats stats_; +}; + +/////////////////////////////////////////////////////////////////////////////// + +// The bank model behind DramSim's interface (sim/common/dram_sim.h): one +// channel per memory bank, the same core-clock credit loop, one request +// reserved per bursts' worth of DRAM clocks per channel, the fixed latency +// after a read's data departs, and responses in completion order per channel. +class DramBankSim { +public: + typedef bool (*ResponseCallback)(void* arg); + + struct Config { + dram_timing::Standard type; + uint32_t num_channels; + uint32_t block_size; + uint32_t core_clock; // MHz + uint32_t data_rate; // MT/s + uint32_t latency; // core cycles added after a read's data departs + uint32_t queue_size; // requests in flight per channel; 0 = unbounded + DramBankModel::Refresh refresh = DramBankModel::Refresh::Off; + }; + + explicit DramBankSim(const Config& cfg) : DramBankSim(cfg, DramBankModel::Config()) {} + + // `policy` gives the controller's queues, horizon and write policy. + DramBankSim(const Config& cfg, const DramBankModel::Config& policy) + : cfg_(cfg) + , model_cfg_(DramBankModel::Config::make(cfg.type, cfg.data_rate, cfg.block_size, cfg.refresh)) { + model_cfg_.read_queue = policy.read_queue; + model_cfg_.write_queue = policy.write_queue; + model_cfg_.wr_high = policy.wr_high; + model_cfg_.wr_low = policy.wr_low; + model_cfg_.horizon = policy.horizon; + for (uint32_t i = 0; i < cfg.num_channels; ++i) { + channels_.emplace_back(model_cfg_); + } + this->reset(); + } + + void reset() { + cycle_ = 0; + credit_ = 0; + for (auto& ch : channels_) { + ch.model.reset(); + ch.read_rsps.clear(); + ch.write_rsps.clear(); + ch.inflight = 0; + } + requests_.clear(); + free_.clear(); + } + + void tick() { + ++cycle_; + credit_ += cfg_.data_rate; + for (auto& ch : channels_) { + this->drain(ch.read_rsps, ch); + this->drain(ch.write_rsps, ch); + } + while (credit_ >= 2ull * cfg_.core_clock) { + for (auto& ch : channels_) { + ch.model.tick( + [&](uint32_t tag) { ch.read_rsps.push_back({tag, cycle_ + 1 + cfg_.latency}); }, + [&](uint32_t tag) { ch.write_rsps.push_back({tag, cycle_}); }); + } + credit_ -= 2ull * cfg_.core_clock; + } + } + + bool can_accept(uint32_t channel) const { + return (cfg_.queue_size == 0) || (channels_.at(channel).inflight < cfg_.queue_size); + } + + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback cb, void* arg) { + uint32_t tag; + if (!free_.empty()) { + tag = free_.back(); + free_.pop_back(); + requests_[tag] = {cb, arg}; + } else { + tag = uint32_t(requests_.size()); + requests_.push_back({cb, arg}); + } + auto& ch = channels_.at(channel); + ch.model.push(tag, is_write, addr & ~uint64_t(cfg_.block_size - 1)); + ++ch.inflight; + } + + const DramBankModel& channel(uint32_t i) const { return channels_.at(i).model; } + +private: + struct Rsp { + uint32_t tag; + uint64_t ready; + }; + + struct Channel { + explicit Channel(const DramBankModel::Config& c) : model(c) {} + DramBankModel model; + std::deque read_rsps, write_rsps; + uint32_t inflight = 0; + }; + + struct Req { + ResponseCallback cb; + void* arg; + }; + + void drain(std::deque& rsps, Channel& ch) { + while (!rsps.empty() && rsps.front().ready <= cycle_) { + Req& req = requests_[rsps.front().tag]; + if (req.cb && !req.cb(req.arg)) { + break; + } + free_.push_back(rsps.front().tag); + --ch.inflight; + rsps.pop_front(); + } + } + + Config cfg_; + DramBankModel::Config model_cfg_; + std::vector channels_; + std::vector requests_; + std::vector free_; + uint64_t cycle_; + uint64_t credit_; +}; + +/////////////////////////////////////////////////////////////////////////////// + +// The bank model as one AXI4 memory port, in AxiLatencyPipe's style +// (sim/common/axi_latency_pipe.h): every output is a function of registered +// state, the caller samples the request side before the clock edge and calls +// step() after it. Single-beat transactions; a burst sets the error, and from +// the next cycle on the port is ready for nothing. +// +// A read takes the lowest free read slot from acceptance until R takes it; it +// reaches the channel in the cycle after it was accepted, and is ready to +// drive R LATENCY + 1 cycles after the cycle whose DRAM clocks its data +// departed in. An AW and its W, in either order, form a write, which reaches +// the channel in the same way and is acknowledged on B from the cycle after +// the controller buffered it. Requests reach the channel in the order they +// formed, a write before a read of the same cycle. +// R and B take the oldest ready response. A request counts against `queue` +// from acceptance until it is ready. +class DramBankAxi { +public: + struct Config { + DramBankModel::Config model; + uint32_t core_clock = 400; // MHz + uint32_t data_rate = 800; // MT/s + uint32_t latency = 0; // core cycles + uint32_t queue = 0; // requests in flight; 0 = only the slots bound them + uint32_t read_slots = 32; + uint32_t write_slots = 16; // AWs, and Ws, accepted and not acknowledged on B + }; + + struct Inputs { + bool ar_valid = false; + uint32_t ar_id = 0; + uint64_t ar_addr = 0; + uint32_t ar_len = 0; + bool aw_valid = false; + uint32_t aw_id = 0; + uint64_t aw_addr = 0; + uint32_t aw_len = 0; + bool w_valid = false; + bool w_last = true; + bool r_ready = false; + bool b_ready = false; + }; + + explicit DramBankAxi(const Config& cfg) : cfg_(cfg), model_(cfg.model) { this->reset(); } + + const DramBankModel& model() const { return model_; } + + void reset() { + model_.reset(); + cycle_ = 0; + credit_ = 0; + error_ = false; + inflight_ = 0; + slot_busy_.assign(cfg_.read_slots, false); + slot_id_.assign(cfg_.read_slots, 0); + slots_used_ = 0; + pending_aw_ = 0; + pending_w_ = 0; + aw_queue_.clear(); + ready_reads_.clear(); + uncounted_ = 0; + acks_.clear(); + cur_r_valid_ = false; + cur_b_valid_ = false; + full_cycles_ = 0; + } + + bool ar_ready() const { return !error_ && this->room() && slots_used_ < cfg_.read_slots; } + bool aw_ready() const { return !error_ && this->room() && pending_aw_ < cfg_.write_slots; } + bool w_ready() const { return !error_ && pending_w_ < cfg_.write_slots; } + + bool r_valid() const { return cur_r_valid_; } + uint32_t r_id() const { return cur_r_id_; } + uint32_t r_slot() const { return cur_r_slot_; } + bool b_valid() const { return cur_b_valid_; } + uint32_t b_id() const { return cur_b_id_; } + bool error() const { return error_; } + // Target cycles the port could take no read: no slot, or no room. + uint64_t full_cycles() const { return full_cycles_; } + // The slot a read accepted this cycle takes: the lowest free once R has + // taken its read, if it does. + uint32_t ar_slot(bool r_ready) const { + for (uint32_t s = 0; s < cfg_.read_slots; ++s) { + if (!slot_busy_[s] || (r_ready && cur_r_valid_ && s == cur_r_slot_)) return s; + } + return cfg_.read_slots; + } + // Nothing accepted is unanswered. + bool idle() const { + return slots_used_ == 0 && pending_aw_ == 0 && pending_w_ == 0; + } + + void step(const Inputs& in) { + const bool ar_hs = in.ar_valid && this->ar_ready(); + const bool aw_hs = in.aw_valid && this->aw_ready(); + const bool w_hs = in.w_valid && this->w_ready(); + const bool burst = (ar_hs && in.ar_len != 0) || (aw_hs && in.aw_len != 0) || (w_hs && !in.w_last); + const bool ar_fire = ar_hs && in.ar_len == 0; + const bool aw_fire = aw_hs && in.aw_len == 0; + const bool w_fire = w_hs && in.w_last; + const bool r_fire = cur_r_valid_ && in.r_ready; + const bool b_fire = cur_b_valid_ && in.b_ready; + if (!(this->room() && slots_used_ < cfg_.read_slots)) { + ++full_cycles_; + } + + // This cycle's DRAM clocks, on what was accepted before it. + credit_ += cfg_.data_rate; + while (credit_ >= 2ull * cfg_.core_clock) { + model_.tick( + [&](uint32_t slot) { ready_reads_.push_back({slot, cycle_ + 1 + cfg_.latency}); }, + [&](uint32_t id) { acks_.push_back(id); ++uncounted_acks_; }); + credit_ -= 2ull * cfg_.core_clock; + } + + // Responses: the stage driving R and B, then what is ready next cycle. + if (r_fire) { + slot_busy_[cur_r_slot_] = false; + --slots_used_; + cur_r_valid_ = false; + } + if (b_fire) { + cur_b_valid_ = false; + } + const uint64_t next = cycle_ + 1; + while (uncounted_ < ready_reads_.size() && ready_reads_[uncounted_].ready <= next) { + ++uncounted_; + --inflight_; + } + inflight_ -= uncounted_acks_; + uncounted_acks_ = 0; + if (!cur_r_valid_ && uncounted_ != 0) { + cur_r_valid_ = true; + cur_r_slot_ = ready_reads_.front().slot; + cur_r_id_ = slot_id_[cur_r_slot_]; + ready_reads_.pop_front(); + --uncounted_; + } + if (!cur_b_valid_ && !acks_.empty()) { + cur_b_valid_ = true; + cur_b_id_ = acks_.front(); + acks_.pop_front(); + } + + // Requests: writes form from an AW and its W, then join the channel ahead + // of this cycle's read. + const bool new_write = (pending_w_ > pending_aw_ && aw_fire) + || (pending_w_ < pending_aw_ && w_fire) + || (aw_fire && w_fire); + if (aw_fire) { + aw_queue_.push_back({in.aw_id, in.aw_addr}); + } + if (new_write) { + auto aw = aw_queue_.front(); + aw_queue_.pop_front(); + model_.push(aw.id, true, aw.addr); + } + if (ar_fire) { + uint32_t s = 0; + while (slot_busy_[s]) ++s; + slot_busy_[s] = true; + slot_id_[s] = in.ar_id; + ++slots_used_; + model_.push(s, false, in.ar_addr); + } + inflight_ += uint32_t(ar_fire) + uint32_t(aw_fire); + pending_aw_ = pending_aw_ + uint32_t(aw_fire) - uint32_t(b_fire && pending_aw_ != 0); + pending_w_ = pending_w_ + uint32_t(w_fire) - uint32_t(b_fire && pending_w_ != 0); + if (burst) { + error_ = true; + } + ++cycle_; + } + +private: + struct Ready { + uint32_t slot; + uint64_t ready; + }; + + struct Aw { + uint32_t id; + uint64_t addr; + }; + + bool room() const { return cfg_.queue == 0 || inflight_ < cfg_.queue; } + + Config cfg_; + DramBankModel model_; + uint64_t cycle_; + uint64_t credit_; + bool error_; + uint32_t inflight_; + std::vector slot_busy_; + std::vector slot_id_; + uint32_t slots_used_; + uint32_t pending_aw_, pending_w_; + std::deque aw_queue_; + std::deque ready_reads_; // reads departed; the first `uncounted_` are ready next cycle + size_t uncounted_; + std::deque acks_; + uint32_t uncounted_acks_ = 0; + bool cur_r_valid_, cur_b_valid_; + uint32_t cur_r_id_ = 0, cur_r_slot_ = 0, cur_b_id_ = 0; + uint64_t full_cycles_ = 0; +}; + +} // namespace vortex diff --git a/sim/common/dram_sim.cpp b/sim/common/dram_sim.cpp index bc899b89fc..4c019d38ce 100644 --- a/sim/common/dram_sim.cpp +++ b/sim/common/dram_sim.cpp @@ -12,7 +12,11 @@ // limitations under the License. #include "dram_sim.h" +#include "dram_bank_model.h" #include "util.h" +#include +#include +#include #include #include #include @@ -36,57 +40,9 @@ using namespace vortex; namespace { -// Bus timings are DRAM clocks, fixed by the burst; array timings are -// nanoseconds, which Ramulator converts at the configured data rate. -struct clock_timing_t { - const char* name; - int clocks; -}; - -struct ns_timing_t { - const char* name; - float ns; -}; - -// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same -// spacing between column commands to another bank group (four within one). -// nRTW only completes the set Ramulator requires. -const clock_timing_t hbm_bus_timing[] = { - {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, -}; - -// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its -// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); -// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its -// cycles nanoseconds. The array does not speed up with the interface: HBM2E -// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM -// lines are the same 14/33/47 ns at their own clocks). Refresh timings complete -// the set Ramulator requires. -const ns_timing_t hbm_array_timing[] = { - {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, - {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, - {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFCSB", 160}, {"tREFI", 3900}, {"tRREFD", 8}, -}; - -// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a -// x16 channel, is eight clocks, and column commands are two (three within a -// bank group) command clocks apart, a command clock being eight data beats. -const clock_timing_t gddr6_bus_timing[] = { - {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, -}; - -// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset -// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh -// timings complete the set Ramulator requires. -const ns_timing_t gddr6_array_timing[] = { - {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, - {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, - {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, - {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, -}; - template -void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A]) { +void set_timing(YAML::Node timing, uint32_t data_rate, const dram_timing::clock_timing_t (&bus)[B], + const dram_timing::ns_timing_t (&array)[A]) { timing["rate"] = data_rate; for (auto& t : bus) { timing[t.name] = t.clocks; @@ -98,8 +54,51 @@ void set_timing(YAML::Node timing, uint32_t data_rate, const clock_timing_t (&bu } +// A DRAM backend: Ramulator, or the bank model (sim/common/dram_bank_model.h) +// with VORTEX_DRAM_MODEL=bank -- a choice of simulator, not of machine. class DramSim::Impl { +public: + virtual ~Impl() {} + virtual void reset() = 0; + virtual void tick() = 0; + virtual bool can_accept(uint32_t channel) const = 0; + virtual void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback cb, void* arg) = 0; +}; + +namespace { + +class BankDram : public DramSim::Impl { +public: + explicit BankDram(const DramSim::Config& c) + : sim_(DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, + c.latency, c.queue_size, refresh_mode()}) {} + void reset() override { sim_.reset(); } + void tick() override { sim_.tick(); } + bool can_accept(uint32_t channel) const override { return sim_.can_accept(channel); } + void send_request(uint32_t channel, uint64_t addr, bool is_write, DramSim::ResponseCallback cb, + void* arg) override { + sim_.send_request(channel, addr, is_write, cb, arg); + } + +private: + // VORTEX_DRAM_REFRESH=off|bank|all: refresh, which Ramulator here does not + // model, per bank or all banks at once. + static DramBankModel::Refresh refresh_mode() { + const char* s = getenv("VORTEX_DRAM_REFRESH"); + return DramBankModel::Config::parse_refresh(s ? s : ""); + } + + DramBankSim sim_; +}; + +} + +class RamulatorDram : public DramSim::Impl { private: + using Config = DramSim::Config; + using Type = DramSim::Type; + using ResponseCallback = DramSim::ResponseCallback; + // One request (a block); it completes when all its transactions have. struct request_t { ResponseCallback callback; @@ -130,19 +129,15 @@ class DramSim::Impl { uint32_t inflight; }; - // Each standard moves 32 B per column command (prefetch x channel width). - static constexpr uint32_t tx_size_ = 32; - - // Every device here holds 8 Gb per channel; addresses above alias, which - // only affects row hits. - static constexpr uint64_t channel_size_ = uint64_t(1) << 30; + static constexpr uint32_t tx_size_ = dram_timing::kBurstSize; - // Bursts per row visit (minimalist open page, 128 B). - static constexpr uint32_t row_visit_bits_ = 2; + // Addresses above a channel's 8 Gb alias, which only affects row hits. + static constexpr uint64_t channel_size_ = dram_timing::kChannelSize; Config config_; - uint32_t bank_bits_; // pseudo-channel, bank group and bank - uint32_t column_bits_; // bursts per row + uint32_t row_visit_bits_; // bursts per row visit (minimalist open page, 128 B) + uint32_t bank_bits_; // pseudo-channel, bank group and bank + uint32_t column_bits_; // bursts per row std::vector channels_; uint64_t cycle_; // DRAM ticks owed, in units of 1/(2 x core_clock): each core cycle adds the @@ -158,7 +153,7 @@ class DramSim::Impl { dram["org"]["density"] = 8192; dram["org"]["channel"] = 1; dram["org"]["channel_width"] = 128; - set_timing(dram["timing"], config.data_rate, hbm_bus_timing, hbm_array_timing); + set_timing(dram["timing"], config.data_rate, dram_timing::hbm_bus_timing, dram_timing::hbm_array_timing); } static void set_gddr6_device(YAML::Node dram, const Config& config) { @@ -174,7 +169,7 @@ class DramSim::Impl { dram["org"]["column"] = 512; dram["org"]["density"] = 8192; dram["org"]["channel_width"] = 32; - set_timing(dram["timing"], config.data_rate, gddr6_bus_timing, gddr6_array_timing); + set_timing(dram["timing"], config.data_rate, dram_timing::gddr6_bus_timing, dram_timing::gddr6_array_timing); } YAML::Node channel_config(uint32_t ch) const { @@ -276,13 +271,13 @@ class DramSim::Impl { } public: - Impl(const Config& config) + RamulatorDram(const Config& config) : config_(config) - // HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 32 bursts per 1 KB - // row; GDDR6: 4 x 4 banks, 64 bursts per 2 KB row. - , bank_bits_((config.type == Type::GDDR6) ? 4 : 5) - , column_bits_((config.type == Type::GDDR6) ? 6 : 5) , channels_(config.num_channels) { + auto geometry = dram_timing::geometry(config.type); + row_visit_bits_ = geometry.log2_visit; + bank_bits_ = geometry.bank_bits(); + column_bits_ = geometry.log2_row_bursts; for (uint32_t ch = 0; ch < config.num_channels; ++ch) { auto ramulator_config = this->channel_config(ch); auto& channel = channels_[ch]; @@ -294,7 +289,7 @@ class DramSim::Impl { this->reset(); } - ~Impl() { + ~RamulatorDram() override { std::ofstream nullstream("ramulator.stats.log"); auto original_buf = std::cout.rdbuf(); std::cout.rdbuf(nullstream.rdbuf()); @@ -305,7 +300,7 @@ class DramSim::Impl { std::cout.rdbuf(original_buf); } - void reset() { + void reset() override { cycle_ = 0; clock_credit_ = 0; for (auto& channel : channels_) { @@ -316,7 +311,7 @@ class DramSim::Impl { } } - void tick() { + void tick() override { ++cycle_; clock_credit_ += config_.data_rate; for (auto& channel : channels_) { @@ -332,11 +327,11 @@ class DramSim::Impl { } } - bool can_accept(uint32_t channel) const { + bool can_accept(uint32_t channel) const override { return (config_.queue_size == 0) || (channels_.at(channel).inflight < config_.queue_size); } - void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) { + void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg) override { // A block wider than a burst is several transactions. uint32_t n = (config_.block_size > tx_size_) ? (config_.block_size / tx_size_) : 1; uint64_t base = addr & ~uint64_t(config_.block_size - 1); @@ -360,13 +355,9 @@ DramSim::Config DramSim::platform_config() { // companion. const std::string type = DRAM_STR(VX_CFG_PLATFORM_MEMORY_TYPE); Config config; - if (type == "HBM2") { - config.type = Type::HBM2; - } else if (type == "HBM3") { - config.type = Type::HBM3; - } else if (type == "GDDR6") { - config.type = Type::GDDR6; - } else { + try { + config.type = dram_timing::parse_standard(type); + } catch (const std::invalid_argument&) { throw std::invalid_argument("dram_sim: unknown VX_CFG_PLATFORM_MEMORY_TYPE " + type); } config.num_channels = VX_CFG_PLATFORM_MEMORY_NUM_BANKS; @@ -378,8 +369,19 @@ DramSim::Config DramSim::platform_config() { return config; } +static DramSim::Impl* make_backend(const DramSim::Config& config) { + const char* model = getenv("VORTEX_DRAM_MODEL"); + if (model == nullptr || model[0] == '\0' || !strcmp(model, "ramulator")) { + return new RamulatorDram(config); + } + if (!strcmp(model, "bank")) { + return new BankDram(config); + } + throw std::invalid_argument(std::string("dram_sim: unknown VORTEX_DRAM_MODEL ") + model); +} + DramSim::DramSim(const Config& config) - : impl_(new Impl(config)) + : impl_(make_backend(config)) {} DramSim::~DramSim() { diff --git a/sim/common/dram_sim.h b/sim/common/dram_sim.h index 565dede377..cf260c132b 100644 --- a/sim/common/dram_sim.h +++ b/sim/common/dram_sim.h @@ -14,6 +14,7 @@ #pragma once #include +#include namespace vortex { @@ -24,12 +25,8 @@ class DramSim { typedef bool (*ResponseCallback)(void *arg); // The JEDEC standards Ramulator models, each at its own organization and - // timing; the data rate picks the speed. - enum class Type { - HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts - HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts - GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages - }; + // timing (sw/common/dram_timing.h); the data rate picks the speed. + using Type = dram_timing::Standard; struct Config { Type type; @@ -57,8 +54,10 @@ class DramSim { // addr: byte address within the channel. void send_request(uint32_t channel, uint64_t addr, bool is_write, ResponseCallback response_cb, void* arg); -private: + // The backend: Ramulator, or with VORTEX_DRAM_MODEL=bank the bank model. class Impl; + +private: Impl* impl_; }; diff --git a/sim/common/simobject.h b/sim/common/simobject.h index b3c9879dc8..5cc55e36ec 100644 --- a/sim/common/simobject.h +++ b/sim/common/simobject.h @@ -88,6 +88,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -245,6 +246,10 @@ class SimPlatform { return (tl_exec_domain_ == HOST_DOMAIN) ? cycles_ : tl_cycles_; } + // Counts resets: cycle numbers repeat across runs, so per-cycle state is + // tagged with the run it belongs to. + uint64_t reset_gen() const { return reset_gen_; } + // Sends issued outside any object tick (driver/host calls). static constexpr uint32_t HOST_DOMAIN = 0xffffffffu; @@ -261,6 +266,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -276,6 +282,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -327,6 +352,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -387,9 +413,11 @@ class SimPlatform { std::vector> domains_; uint64_t cycles_; + uint64_t reset_gen_ = 0; uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -651,6 +679,7 @@ class SimChannel : public SimChannelBase { : SimChannelBase(other.module_) , storage_(other.storage_.capacity()) , pending_count_(0) + , reg_ready_(other.reg_ready_) , convert_fn_(other.convert_fn_) , tx_cb_(other.tx_cb_) { sink_ = nullptr; @@ -666,6 +695,9 @@ class SimChannel : public SimChannelBase { pop_cb_ = {}; storage_ = RingQueue(other.storage_.capacity()); pending_count_ = 0; + reg_ready_ = other.reg_ready_; + freed_cycle_ = ~uint64_t(0); + freed_count_ = 0; convert_fn_ = other.convert_fn_; tx_cb_ = other.tx_cb_; } @@ -704,9 +736,14 @@ class SimChannel : public SimChannelBase { return sink_->full(); } assert(!SimPlatform::instance().mt_cross_access(module_)); - return this->occupancy() >= storage_.capacity(); + return this->occupancy() + this->freed_now() >= storage_.capacity(); } + // A registered ready: a slot freed by a pop is offered to producers from the + // next cycle, as a buffer whose full flag is a register does, whatever order + // the producer and the consumer tick in. + void set_registered_ready(bool enable) { reg_ready_ = enable; } + void send(const Pkt& pkt, uint64_t delay = 1) { auto& platform = SimPlatform::instance(); // A send whose endpoint lives in another domain is handed to that @@ -849,7 +886,21 @@ class SimChannel : public SimChannelBase { bool queue_empty() const { return storage_.empty(); } uint32_t queue_size() const { return storage_.size(); } const Pkt& queue_front() const { return storage_.front(); } + uint32_t freed_now() const { + auto& platform = SimPlatform::instance(); + return (reg_ready_ && freed_cycle_ == platform.cycles() && freed_gen_ == platform.reset_gen()) + ? freed_count_ : 0; + } void queue_pop() { + if (reg_ready_) { + auto& platform = SimPlatform::instance(); + if (freed_cycle_ != platform.cycles() || freed_gen_ != platform.reset_gen()) { + freed_cycle_ = platform.cycles(); + freed_gen_ = platform.reset_gen(); + freed_count_ = 0; + } + ++freed_count_; + } storage_.pop(); --SimChannelBase::inflight_count(); if (pop_cb_) { @@ -860,6 +911,10 @@ class SimChannel : public SimChannelBase { RingQueue storage_; uint32_t pending_count_; + bool reg_ready_ = false; + uint64_t freed_cycle_ = ~uint64_t(0); + uint64_t freed_gen_ = 0; + uint32_t freed_count_ = 0; SmallFunction convert_fn_; TxCallback tx_cb_; @@ -1151,6 +1206,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1198,6 +1269,7 @@ inline void SimPlatform::reset() { // Reset timing cycles_ = 0; + ++reset_gen_; } inline void SimPlatform::tick() { @@ -1328,7 +1400,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/common/simobject_functional.h b/sim/common/simobject_functional.h index 8074a179a1..11668176e9 100644 --- a/sim/common/simobject_functional.h +++ b/sim/common/simobject_functional.h @@ -103,6 +103,7 @@ class SimObjectBase { std::string name_; bool tick_active_ = true; uint32_t domain_ = 0; + bool launch_kept_ = false; // built under a LaunchKeepScope virtual void do_reset() = 0; virtual void do_tick() = 0; friend class SimPlatform; @@ -276,6 +277,7 @@ class SimPlatform { uint32_t alloc_domain() { return num_domains_++; } uint32_t num_domains() const { return num_domains_; } uint32_t build_domain() const { return build_domain_; } + bool build_launch_kept() const { return build_launch_kept_; } // Cap the parallel thread count (must be called before the first tick). // A limit of 1 forces serial execution — required while the configuration @@ -291,6 +293,25 @@ class SimPlatform { req_workers_ = std::max(1u, num_workers); } + // Objects built under a LaunchKeepScope (the last-level cache and the + // memory below it, when the command processor writes into that cache) + // carry their state and in-flight work across reset_launch(). + class LaunchKeepScope { + public: + explicit LaunchKeepScope(bool keep) { + auto& p = SimPlatform::instance(); + saved_ = p.build_launch_kept_; + p.build_launch_kept_ = saved_ || keep; + } + ~LaunchKeepScope() { + SimPlatform::instance().build_launch_kept_ = saved_; + } + LaunchKeepScope(const LaunchKeepScope&) = delete; + LaunchKeepScope& operator=(const LaunchKeepScope&) = delete; + private: + bool saved_; + }; + class DomainScope { public: explicit DomainScope(uint32_t domain) { @@ -342,6 +363,7 @@ class SimPlatform { void schedule(Func&& func, const Pkt& pkt, uint64_t delay, uint32_t domain); void reset(); + void reset_launch(); void tick(); private: @@ -408,6 +430,7 @@ class SimPlatform { uint32_t num_domains_ = 1; uint32_t build_domain_ = 0; + bool build_launch_kept_ = false; static inline thread_local uint32_t tl_exec_domain_ = HOST_DOMAIN; static inline thread_local SimObjectBase* tl_exec_object_ = nullptr; static inline thread_local uint64_t tl_cross_seq_ = 0; @@ -717,6 +740,10 @@ class SimChannel : public SimChannelBase { template void tx_callback(F&& callback) { tx_cb_ = std::forward(callback); } + // Ready timing is a timing-model property; accepted for API compatibility + // and ignored. + void set_registered_ready(bool /*enable*/) {} + // Back-pressure is disabled: producers never stall. bool full() const override { return false; } @@ -1167,6 +1194,22 @@ inline void SimPlatform::validate_topology() { } } +// A kernel launch resets every object but those built under a +// LaunchKeepScope: the hardware does not reset between a transfer and the +// launch that follows it, so the transfer's line fills may still be in +// flight in the kept memory side when the kernel starts. Their events stay +// scheduled and time keeps counting; the rest of the model is quiescent. +inline void SimPlatform::reset_launch() { + for (auto* object : active_reset_) { + if (!object->launch_kept_) { + object->do_reset(); + } + } + for (auto& object : objects_) { + object->tick_active_ = true; + } +} + inline void SimPlatform::reset() { this->validate_topology(); @@ -1344,7 +1387,8 @@ void SimChannelEvent::fire() { inline SimObjectBase::SimObjectBase(const SimContext&, const std::string& name) : name_(name) - , domain_(SimPlatform::instance().build_domain()) {} + , domain_(SimPlatform::instance().build_domain()) + , launch_kept_(SimPlatform::instance().build_launch_kept()) {} inline SimChannelBase::SimChannelBase(SimObjectBase* module) : module_(module) diff --git a/sim/firesim/Makefile b/sim/firesim/Makefile index 83cc74033a..3f473e92c0 100644 --- a/sim/firesim/Makefile +++ b/sim/firesim/Makefile @@ -8,14 +8,28 @@ PROJECT := firesim SIMIF_DIR := $(FIRESIM_PATH)/sim/midas/src/main/cc -PLATFORM ?= vitis DESIGN ?= VortexTarget TARGET_CONFIG ?= VortexConfig -PLATFORM_CONFIG ?= Default$(shell echo $(PLATFORM) | sed 's/^./\U&/')Config + +# Nothing given on this make's command line reaches FireSim's makes: FireSim +# uses PLATFORM and TARGET for its own purposes. Each sub-make is handed what it +# needs instead. +MAKEOVERRIDES := + +# The card's FireSim platform and platform config come from its descriptor, the +# same ones hw/syn/firesim elaborated the target with. The runtime names the +# card (from the image, or the metasimulator's default). +FIRESIM_CARD ?= u55c +FIRESIM_MK := $(DESTDIR)/firesim_platform.mk +FIRESIM_GEN_RC := $(shell mkdir -p $(DESTDIR) && python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py --card=$(FIRESIM_CARD) --emit=firesim-mk -o $(FIRESIM_MK) 2> $(FIRESIM_MK).err; echo $$?) +ifneq ($(FIRESIM_GEN_RC),0) +$(error $(shell cat $(FIRESIM_MK).err)) +endif +include $(FIRESIM_MK) # Golden Gate writes the generated header here; the driver objects are built # alongside it so a rebuild of the target invalidates them. -GEN_DIR := $(FIRESIM_PATH)/sim/output/$(PLATFORM)/$(PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG)/build +GEN_DIR := $(FIRESIM_PATH)/sim/output/$(FIRESIM_PLATFORM)/$(FIRESIM_PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG)/build # The driver is built as a shared library rather than FireSim's usual # executable, because the Vortex runtime loads it. Upstream's driver.mk only @@ -27,10 +41,20 @@ DRIVER_SRCS := $(SRC_DIR)/firesim_sim.cpp # -fPIC is required for every object that lands in the shared library, so it # has to reach the bridge and platform objects too, not just ours. SIM_CXXFLAGS := -O2 -fPIC -std=c++20 -I$(SRC_DIR) \ - -idirafter $(CONDA_PREFIX)/include -idirafter /usr/include \ - -idirafter $(XILINX_XRT)/include -SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -L$(XILINX_XRT)/lib \ - -luuid -lxrt_coreutil -ldl -lz + -idirafter $(CONDA_PREFIX)/include -idirafter /usr/include +SIM_LDFLAGS := -shared -L$(CONDA_PREFIX)/lib -ldl -lz + +# The platform layer's own library: XRT for vitis cards, VRT for the V80. VRT +# is built against the host glibc and libstdc++, so its platform layer is +# compiled by the host toolchain, and its headers need jsoncpp's and libxml2's. +ifeq ($(FIRESIM_PLATFORM), vitis) +SIM_CXXFLAGS += -idirafter $(XILINX_XRT)/include +SIM_LDFLAGS += -L$(XILINX_XRT)/lib -luuid -lxrt_coreutil +else ifeq ($(FIRESIM_PLATFORM), aved) +SIM_CXX := /usr/bin/g++ +SIM_CXXFLAGS += $(shell pkg-config --cflags jsoncpp libxml-2.0) +SIM_LDFLAGS += -L/usr/lib/x86_64-linux-gnu -lvrt +endif .PHONY: all lib clean check-gen @@ -47,25 +71,99 @@ check-gen: # objects current and link them into a shared library, which fails late with # "final link failed: bad value". The position-independent build gets its own # object directory, seeded with the generated headers it needs. -PIC_DIR := $(DESTDIR)/firesim_pic +# Per platform: the objects embed that platform's generated widths, and make +# judges them by timestamp, so a directory shared between cards would link one +# card's objects into the other's library. +PIC_DIR := $(DESTDIR)/firesim_pic_$(FIRESIM_PLATFORM) $(PIC_DIR)/FireSim-generated.const.h: $(GEN_DIR)/FireSim-generated.const.h @mkdir -p $(PIC_DIR) @cp -f $(GEN_DIR)/FireSim-generated.const.* $(PIC_DIR)/ 2>/dev/null || true @cp -f $(GEN_DIR)/FireSim-generated.const.h $(PIC_DIR)/ +ifneq ($(filter $(TARGET),xrtsim avedsim),) + +# The metasimulator as the library, for either kind of card: the FireSim-transformed target Verilated, +# with the emulated host interface (simif_emul) in place of the card's, linked +# into lib$(PROJECT).so so the runtime loads it exactly as it loads the card's. +# firesim_emul_main.cpp supplies the loop that --main would otherwise generate. +# +# The Vortex RTL comes from hw/syn/firesim's simulation flattening, the same +# sources the standalone metasim builds from; the generated FireSim RTL and +# headers come from the elaborated target. +SYN_DIR := $(ROOT_DIR)/hw/syn/firesim +EMUL_DIR := $(DESTDIR)/firesim_emul_$(FIRESIM_CARD) +WORK_DIR := $(EMUL_DIR)/work +SIM_SRCS_LIST := $(WORK_DIR)/vortex_srcs.f +SIM_SRCS_DIR := $(WORK_DIR)/vortex_srcs +ELAB_DIR := $(FIRESIM_PATH)/sim/generated-src/$(FIRESIM_PLATFORM)/$(FIRESIM_PLATFORM)-midasexamples-$(DESIGN)-$(TARGET_CONFIG)-$(PLATFORM_CONFIG) +EMUL_NAME := firesim_emul + +EMUL_SRCS := $(SRC_DIR)/firesim_sim.cpp $(SRC_DIR)/firesim_emul_main.cpp + +EMUL_VERILATOR_FLAGS := -Wno-fatal -y $(SIM_SRCS_DIR) +libext+.sv+.v $$(tr '\n' ' ' < $(SIM_SRCS_LIST)) + +# EMUL_DEBUG=1: a wave-tracing metasimulator. --trace Verilates the dump hooks +# and +define+DEBUG arms the emul top's +waveformfile/+dump-start plusargs +# (FireSim's top.sv), which VORTEX_FIRESIM_EMUL_ARGS passes at run time. Its own +# build directory: the model rule does not depend on flags, so sharing one would +# hand back whichever build ran first. +ifdef EMUL_DEBUG +EMUL_VERILATOR_FLAGS += --trace --trace-structs +define+DEBUG +EMUL_DIR := $(EMUL_DIR)_dbg +endif + +.PHONY: emul-sources +# The simulation flattening of the Vortex RTL for CONFIGS, and Golden Gate's +# elaboration of the card's target; both are no-ops when current. +emul-sources: + $(MAKE) -C $(SYN_DIR) sources elaborate FIRESIM_CARD=$(FIRESIM_CARD) FIRESIM_WORK_DIR=$(WORK_DIR) \ + XLEN=$(XLEN) CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) + +# Host toolchain, as for the aved platform library: the runtime that loads this +# is built with it. -fPIC reaches the Verilated objects too, through -CFLAGS. +# hw/syn/common.mk exports VERILATOR as a bare path and FireSim's `?=` would +# yield to it, so it is named here in full. +# +# The sub-make's own rule does not know the Vortex sources are inputs -- they +# arrive as flags -- so it would call the model current after a config change. +# This rule does know, so once it decides to rebuild, the old model goes. +$(EMUL_DIR)/V$(EMUL_NAME): $(EMUL_SRCS) $(SIM_SRCS_LIST) $(ELAB_DIR)/FireSim-generated.sv + rm -f $@ + $(MAKE) -C $(SIMIF_DIR) verilator \ + PLATFORM=$(FIRESIM_PLATFORM) \ + DRIVER_NAME=$(EMUL_NAME) \ + GEN_FILE_BASENAME=FireSim-generated \ + GEN_DIR=$(ELAB_DIR) \ + OUT_DIR=$(EMUL_DIR) \ + DRIVER="$(EMUL_SRCS)" \ + CXX=/usr/bin/g++ \ + VERILATOR="verilator --timing --cc --exe" \ + VERILATOR_FLAGS="$(EMUL_VERILATOR_FLAGS)" \ + CXXFLAGS="-O2 -fPIC -std=c++20 -DFIRESIM_EMUL -I$(SRC_DIR)" \ + LDFLAGS="-shared" + +lib: emul-sources + $(MAKE) $(EMUL_DIR)/V$(EMUL_NAME) + cp -f $(EMUL_DIR)/V$(EMUL_NAME) $(DESTDIR)/lib$(PROJECT).so + +else + # DRIVER_NAME is chosen so the sub-make emits lib$(PROJECT).so directly. lib: check-gen $(PIC_DIR)/FireSim-generated.const.h $(MAKE) -C $(SIMIF_DIR) driver \ - MAIN=$(PLATFORM) PLATFORM=$(PLATFORM) \ + MAIN=$(FIRESIM_PLATFORM) PLATFORM=$(FIRESIM_PLATFORM) \ DRIVER_NAME=lib$(PROJECT).so.tmp \ GEN_FILE_BASENAME=FireSim-generated \ GEN_DIR=$(PIC_DIR) \ OUT_DIR=$(DESTDIR) \ DRIVER="$(DRIVER_SRCS)" \ CXXFLAGS="$(SIM_CXXFLAGS)" \ - LDFLAGS="$(SIM_LDFLAGS)" - mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(PLATFORM) $(DESTDIR)/lib$(PROJECT).so + LDFLAGS="$(SIM_LDFLAGS)" \ + $(if $(SIM_CXX),CXX=$(SIM_CXX)) + mv -f $(DESTDIR)/lib$(PROJECT).so.tmp-$(FIRESIM_PLATFORM) $(DESTDIR)/lib$(PROJECT).so + +endif clean: rm -f $(DESTDIR)/lib$(PROJECT).so diff --git a/sim/firesim/firesim_emul_main.cpp b/sim/firesim/firesim_emul_main.cpp new file mode 100644 index 0000000000..8993266acf --- /dev/null +++ b/sim/firesim/firesim_emul_main.cpp @@ -0,0 +1,58 @@ +// The metasimulator's main loop, as a function the runtime calls on its +// simulator thread rather than as a process entry point. +// +// This is Verilator's --main loop for a --timing model. The emul top's initial +// block calls simulator_entry() through the DPI, which reaches entry() and +// spawns the driver thread, exactly as the standalone metasim does. The loop +// ends when the top $finishes, which the DPI tick triggers once the driver has +// returned -- that is, once the runtime has shut the simulation down. + +#include +#include +#include +#include +#include + +#include "verilated.h" +#include "Vemul.h" + +int firesim_emul_run(int argc, char **argv) { + // VORTEX_FIRESIM_EMUL_ARGS appends Verilator runtime arguments, e.g. + // "+verilator+rand+reset+2 +verilator+seed+7" to start every register that + // has no reset from a random value instead of zero -- the closest a 2-state + // simulator gets to a 4-state one's X on an uninitialised flop. + std::vector extra; + if (const char *env = getenv("VORTEX_FIRESIM_EMUL_ARGS")) { + std::istringstream in(env); + for (std::string tok; in >> tok;) { + extra.push_back(tok); + } + } + std::vector args(argv, argv + argc); + for (const auto &tok : extra) { + args.push_back(tok.c_str()); + } + + const std::unique_ptr contextp{new VerilatedContext}; + // Required before time 0 for an EMUL_DEBUG build's $dumpvars to arm; a + // no-op in a build without tracing. (Verilator's own --main emits it.) + contextp->traceEverOn(true); + contextp->commandArgs(static_cast(args.size()), args.data()); + if (!extra.empty()) { + fprintf(stderr, "[emul] extra args:%s -> randReset=%d seed=%d\n", + [&] { std::string s; for (auto &t : extra) s += " " + t; return s; }().c_str(), + contextp->randReset(), contextp->randSeed()); + } + const std::unique_ptr topp{new Vemul{contextp.get()}}; + + while (!contextp->gotFinish()) { + topp->eval(); + if (!topp->eventsPending()) { + break; + } + contextp->time(topp->nextTimeSlot()); + } + + topp->final(); + return 0; +} diff --git a/sim/firesim/firesim_sim.cpp b/sim/firesim/firesim_sim.cpp index 1ff8723c8c..854bbae4b1 100644 --- a/sim/firesim/firesim_sim.cpp +++ b/sim/firesim/firesim_sim.cpp @@ -14,8 +14,11 @@ #include "firesim_sim.h" #include "bridges/clock.h" +#include "bridges/fased_memory_timing_model.h" #include +#include +#include #include #include #include @@ -27,6 +30,7 @@ #include #include #include +#include #include #include @@ -40,6 +44,12 @@ // Provided by the simulator's entry translation unit. extern int entry(int argc, char **argv); +#ifdef FIRESIM_EMUL +// Provided by firesim_emul_main.cpp: runs the Verilated metasimulator, whose +// initial block reaches entry() through the DPI. +extern int firesim_emul_run(int argc, char **argv); +#endif + namespace { bool trace_enabled() { @@ -116,14 +126,21 @@ class vortex_simulation_t; channel_t *g_channel = nullptr; vortex_simulation_t *g_sim = nullptr; +// The exit status a stopped run ends with (128+signal, or 124 for the launch +// watchdog), 0 while it has not been stopped. Once set, the simulation reports +// failure and the process exits with it, whatever the application prints: a +// run cut short must never read as a pass. +std::atomic g_stop_status{0}; + // A simulation whose run loop services requests instead of executing a fixed // script. Everything the target does happens on this thread: the bridges are // not thread-safe and must never be touched by the caller. class vortex_simulation_t final : public simulation_t { public: - vortex_simulation_t(widget_registry_t ®istry, + vortex_simulation_t(simif_t &simif, widget_registry_t ®istry, const std::vector &args) - : simulation_t(registry, args), + : simulation_t(registry, args), simif(simif), + fased(registry.get_bridges()), peek_poke(registry.get_widget()), loadmem(registry.get_widget()), clock(registry.get_widget()) {} @@ -146,7 +163,7 @@ class vortex_simulation_t final : public simulation_t { lock, [] { return g_channel->shutdown || !g_channel->pending.empty(); }); if (g_channel->shutdown && g_channel->pending.empty()) { g_sim = nullptr; - return 0; + return g_stop_status.load() != 0 ? 1 : 0; } req = std::move(g_channel->pending.front().fn); req_label = std::move(g_channel->pending.front().label); @@ -169,6 +186,19 @@ class vortex_simulation_t final : public simulation_t { peek_poke.step(n, true); } + // Only ever called after a blocking step, which has already waited for the + // step to finish. A blocking peek would additionally wait up to 0.1 s for + // PRECISE_PEEKABLE and then return the same value: time lost per peek, and + // dcr_read peeks once per target cycle. + uint32_t peek(const char *name) { + return peek_poke.peek(name, false); + } + + // The DRAM timing models sit between the target's memory port and the + // card's memory. Their request and error counters say whether a request the + // core is waiting on ever left the target, came back, or came back an error. + simif_t &simif; + std::vector fased; peek_poke_t &peek_poke; loadmem_t &loadmem; // True target cycles elapsed. step() requests cycles; this reports how many @@ -185,8 +215,7 @@ std::unique_ptr create_simulation(simif_t &simif, widget_registry_t ®istry, const std::vector &args) { - (void)simif; - return std::make_unique(registry, args); + return std::make_unique(simif, registry, args); } namespace { @@ -365,19 +394,27 @@ class firesim_sim::Impl { g_channel = nullptr; } - int init(const char *bitstream) { + int init(const std::vector &plusargs) { g_channel = &channel_; argv_storage_.emplace_back("firesim"); - if (bitstream != nullptr && *bitstream != '\0') { - argv_storage_.emplace_back(std::string("+binary_file=") + bitstream); + for (const auto &arg : plusargs) { + argv_storage_.push_back(arg); } for (auto &s : argv_storage_) { argv_.push_back(const_cast(s.c_str())); } - thread_ = std::thread( - [this] { entry(static_cast(argv_.size()), argv_.data()); }); + // On a card this thread is the driver. Under the metasimulator it is the + // simulator's own loop instead: entry() is then reached from a Verilog + // initial block, and the driver runs on the thread simif_emul spawns. + thread_ = std::thread([this] { +#ifdef FIRESIM_EMUL + firesim_emul_run(static_cast(argv_.size()), argv_.data()); +#else + entry(static_cast(argv_.size()), argv_.data()); +#endif + }); FSIM_TRACE("waiting for the simulator thread to come up"); { @@ -441,7 +478,90 @@ class firesim_sim::Impl { firesim_sim::firesim_sim() : impl_(new Impl()), step_batch_(kStepBatchFloor), busy_cycles_(0), - stall_reported_(false) {} + stall_reported_(false), awaiting_busy_(false), idle_cycles_(0), + abort_phase_(0), launch_started_s_(0), stopped_(false), reporting_(false) { + install_stop_handlers(); +} + +// Wall-clock seconds a launch may take before the runtime gives up on it; 0 +// (the default) never does. Meant for a card run under an outer timeout: this +// fires first, so the run ends with a report and a normal exit instead of a +// kill. +static double launch_timeout_s() { + static const double value = [] { + if (const char *env = getenv("VORTEX_FIRESIM_TIMEOUT_S")) { + const double v = atof(env); + if (v > 0) { + return v; + } + } + return 0.0; + }(); + return value; +} + +// A first SIGINT or SIGTERM asks the run to stop instead of killing it: the +// launch in flight is reported and completed, nothing more is sent to the +// target, and the application finishes through its own teardown. Killing a +// FireSim run mid-launch is what preceded every host reset on the V80, so +// Ctrl-C and `timeout -s INT` take this path. A second signal kills as usual. +namespace { +std::atomic g_stop_signal{0}; + +extern "C" void on_stop_signal(int sig) { + if (g_stop_signal.exchange(sig) != 0) { + signal(sig, SIG_DFL); + raise(sig); + } +} +} // namespace + +// After the application's own teardown, a stopped run exits with its status. +// Output is flushed first; _exit skips any later atexit handlers. +extern "C" void exit_with_stop_status() { + const int status = g_stop_status.load(); + if (status != 0) { + fflush(nullptr); + fprintf(stderr, "[firesim] the run was stopped; its results are not valid (exit %d)\n", + status); + _exit(status); + } +} + +// Only where the application has not installed its own handler. +void firesim_sim::install_stop_handlers() { + static bool registered = false; + if (!registered) { + atexit(exit_with_stop_status); + registered = true; + } + for (int sig : {SIGINT, SIGTERM}) { + struct sigaction old_action; + if (sigaction(sig, nullptr, &old_action) == 0 && old_action.sa_handler == SIG_DFL) { + struct sigaction action = {}; + action.sa_handler = on_stop_signal; + sigemptyset(&action.sa_mask); + sigaction(sig, &action, nullptr); + } + } +} + +bool firesim_sim::stopped() { + if (!stopped_ && g_stop_signal.load() != 0) { + fprintf(stderr, "[firesim] %s received: stopping the run (send it again to kill)\n", + g_stop_signal.load() == SIGINT ? "SIGINT" : "SIGTERM"); + stopped_ = true; + int expected = 0; + g_stop_status.compare_exchange_strong(expected, 128 + g_stop_signal.load()); + } + return stopped_; +} + +static double wall_now_s() { + return std::chrono::duration( + std::chrono::steady_clock::now().time_since_epoch()) + .count(); +} // Target cycles of continuous busy after which the target is assumed wedged and // its bus activity is reported. Well above any kernel this platform runs today @@ -460,6 +580,13 @@ static uint64_t stall_report_cycles() { } void firesim_sim::report_progress(const char *why) { + // The report's own reads must reach the target even when it is written + // because the run is stopping. + struct reporting_scope { + bool &flag; + explicit reporting_scope(bool &f) : flag(f) { flag = true; } + ~reporting_scope() { flag = false; } + } scope(reporting_); uint64_t tcycle = 0; const int ret = impl_->submit("read target clock", [&tcycle] { tcycle = g_sim->clock.tcycle(); }); @@ -472,6 +599,23 @@ void firesim_sim::report_progress(const char *why) { // than anything the design did. fprintf(stderr, "[firesim] %s: requested=%llu tcycle=%llu\n", why, (unsigned long long)busy_cycles_, (unsigned long long)tcycle); + std::string mem; + impl_->submit("read memory model counters", [&mem] { + static const char *const kRegs[] = { + "totalReads", "totalReadBeats", "totalWrites", "totalWriteBeats", + "rrespError", "brespError", "readMaxReqs", "writeMaxReqs"}; + for (size_t i = 0; i < g_sim->fased.size(); ++i) { + const AddressMap &map = g_sim->fased[i]->get_addr_map(); + mem += " [" + std::to_string(i) + "]"; + for (const char *reg : kRegs) { + if (map.r_reg_exists(reg)) { + mem += std::string(" ") + reg + "=" + + std::to_string(g_sim->simif.read(map.r_addr(reg))); + } + } + } + }); + fprintf(stderr, "[firesim] memory model:%s\n", mem.c_str()); report_core_counters(); } @@ -483,7 +627,7 @@ void firesim_sim::report_progress(const char *why) { // they are parked at a barrier, and which pipeline is holding them. // // Requires the RTL to be built with PERF_ENABLE (see hw/syn/firesim/Makefile); -// without it VX_csr_data.sv ties the counters off and every read returns zero, +// without it VX_csr_mpm.sv ties the counters off and every read returns zero, // so an all-zero line here means the instrument is absent, not that the core is // idle. That distinction is exactly the kind this project has been bitten by, // so it is called out rather than left to be inferred. @@ -518,10 +662,11 @@ void firesim_sim::report_core_counters() { (kClassCore << 22) | ((c.csr - kMpmBase) << 16) | 0 /* core 0 */; uint32_t value = 0; if (this->dcr_read(kDcrMpmValue, tag, &value) != 0) { - line += " "; - line += c.name; - line += "=?"; - continue; + // A target that ignores one DCR read ignores them all, and each costs + // the full poll budget; the rest would only delay the report. + fprintf(stderr, "[firesim] core counters:%s %s=? (DCR path not " + "answering; remaining counters skipped)\n", line.c_str(), c.name); + return; } all_zero = all_zero && (value == 0); line += " "; @@ -562,9 +707,14 @@ void firesim_sim::report_core_counters() { firesim_sim::~firesim_sim() { delete impl_; } -int firesim_sim::init(const char *bitstream) { return impl_->init(bitstream); } +int firesim_sim::init(const std::vector &plusargs) { + return impl_->init(plusargs); +} int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { + if (!reporting_ && stopped()) { + return 0; + } return impl_->submit(label("dcr_write addr=0x%x data=0x%x", addr, value), [addr, value] { auto &pp = g_sim->peek_poke; pp.poke("ctrl_dcr_req_valid", 1, true); @@ -578,6 +728,10 @@ int firesim_sim::dcr_write(uint32_t addr, uint32_t value) { } int firesim_sim::dcr_read(uint32_t addr, uint32_t tag, uint32_t *value) { + if (!reporting_ && stopped()) { + *value = 0; + return -1; + } uint32_t sampled = 0; bool answered = false; int ret = impl_->submit(label("dcr_read addr=0x%x", addr), [addr, tag, &sampled, &answered] { @@ -606,8 +760,8 @@ int firesim_sim::dcr_read(uint32_t addr, uint32_t tag, uint32_t *value) { const uint32_t kPollRounds = dcr_poll_rounds(); for (uint32_t i = 0; i < kPollRounds; ++i) { g_sim->step(1); - if (pp.peek("ctrl_dcr_rsp_valid", true) != 0) { - sampled = pp.peek("ctrl_dcr_rsp_data", true); + if (g_sim->peek("ctrl_dcr_rsp_valid") != 0) { + sampled = g_sim->peek("ctrl_dcr_rsp_data"); answered = true; break; } @@ -655,7 +809,17 @@ static void prefill_region() { } int firesim_sim::start() { + if (stopped()) { + // Complete the launch without touching the target: busy once, then idle. + abort_phase_ = 1; + launch_started_s_ = 0; + return 0; + } impl_->submit("prefill", [] { prefill_region(); }); + awaiting_busy_ = true; + idle_cycles_ = 0; + abort_phase_ = 0; + launch_started_s_ = wall_now_s(); return impl_->submit("start", [] { auto &pp = g_sim->peek_poke; pp.poke("ctrl_start", 1, true); @@ -666,13 +830,42 @@ int firesim_sim::start() { } int firesim_sim::is_busy(bool *busy) { + // After the watchdog has given up the target is no longer advanced at all: + // report busy once, which is what a launch still waiting for its rising edge + // needs, then idle, so every launch state reaches its end. + if (abort_phase_ != 0) { + *busy = (abort_phase_ == 1); + abort_phase_ = 2; + return 0; + } + const bool timed_out = launch_timeout_s() > 0 && launch_started_s_ > 0 && + wall_now_s() - launch_started_s_ > launch_timeout_s(); + if (launch_started_s_ > 0 && (timed_out || stopped())) { + if (timed_out) { + fprintf(stderr, + "[firesim] launch exceeded VORTEX_FIRESIM_TIMEOUT_S=%.0f s " + "(busy %s); giving up on it\n", + launch_timeout_s(), awaiting_busy_ ? "never rose" : "never fell"); + } + report_progress(timed_out ? "launch watchdog" : "stopped"); + stopped_ = true; + int expected = 0; + g_stop_status.compare_exchange_strong(expected, timed_out ? 124 : 128 + SIGINT); + launch_started_s_ = 0; + abort_phase_ = awaiting_busy_ ? 1 : 2; + awaiting_busy_ = false; + *busy = (abort_phase_ == 1); + abort_phase_ = 2; + return 0; + } + uint32_t sampled = 0; // Advance before sampling: the target only moves while it is being asked to, // so a caller that merely polls without stepping would spin forever. const uint32_t batch = step_batch_; int ret = impl_->submit(label("step %u + busy", batch), [&sampled, batch] { g_sim->step(batch); - sampled = g_sim->peek_poke.peek("ctrl_busy", true); + sampled = g_sim->peek("ctrl_busy"); }); if (ret != 0) { return ret; @@ -683,6 +876,10 @@ int firesim_sim::is_busy(bool *busy) { // grow towards the ceiling and give a long kernel the throughput it needs. // Going idle drops back to the floor, ready for the next launch. if (*busy) { + if (awaiting_busy_) { + awaiting_busy_ = false; + stall_reported_ = false; + } const uint32_t ceiling = step_batch_ceiling(); step_batch_ = (batch >= ceiling / 4) ? ceiling : batch * 4; busy_cycles_ += batch; @@ -695,6 +892,15 @@ int firesim_sim::is_busy(bool *busy) { stall_reported_ = true; report_progress("target still busy"); } + } else if (awaiting_busy_) { + // Still waiting for the launch to take. A kernel is busy from its first + // fetch, so a start that has not raised busy within the stall budget never + // reached the core; say so once, with the counters, rather than poll on. + idle_cycles_ += batch; + if (!stall_reported_ && idle_cycles_ >= stall_report_cycles()) { + stall_reported_ = true; + report_progress("target never went busy after start"); + } } else { // Report on a healthy completion too, not only on a stall. Without this the // only counter dumps come from wedged runs, leaving nothing to compare a @@ -708,6 +914,7 @@ int firesim_sim::is_busy(bool *busy) { step_batch_ = std::min(kStepBatchFloor, step_batch_ceiling()); busy_cycles_ = 0; stall_reported_ = false; + launch_started_s_ = 0; } return 0; } @@ -730,7 +937,14 @@ int firesim_sim::ready_wait(int64_t timeout_ms) { } } +// Once a launch has been abandoned the target is left where it stopped, and +// its memory path may be the very thing that is stuck. Nothing more is sent to +// it, so the application runs to its own exit -- and reports its failure -- +// instead of blocking on a transfer that will never complete. int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { + if (!reporting_ && stopped()) { + return 0; + } dump_upload(addr, size, value); return impl_->submit( label("mem_write addr=0x%lx size=%lu", (unsigned long)addr, (unsigned long)size), @@ -738,6 +952,10 @@ int firesim_sim::mem_write(uint64_t addr, uint64_t size, const void *value) { } int firesim_sim::mem_read(uint64_t addr, uint64_t size, void *value) { + if (!reporting_ && stopped()) { + memset(value, 0, size); + return 0; + } return impl_->submit( label("mem_read addr=0x%lx size=%lu", (unsigned long)addr, (unsigned long)size), [addr, size, value] { access_mem(addr, size, value, false); }); diff --git a/sim/firesim/firesim_sim.h b/sim/firesim/firesim_sim.h index cd3bd1d54f..423ce92b0b 100644 --- a/sim/firesim/firesim_sim.h +++ b/sim/firesim/firesim_sim.h @@ -14,6 +14,8 @@ #pragma once #include +#include +#include namespace vortex { @@ -32,9 +34,10 @@ class firesim_sim { virtual ~firesim_sim(); // Starts the simulator thread and holds the target in reset until it is - // ready to accept requests. `bitstream` selects the FPGA image and is - // ignored by a metasimulation. - int init(const char* bitstream); + // ready to accept requests. `plusargs` go to the platform layer as given: + // the image (+binary_file=...) and whatever else the platform needs to find + // its card; a metasimulation takes none. + int init(const std::vector& plusargs); int mem_write(uint64_t addr, uint64_t size, const void* value); @@ -81,6 +84,29 @@ class firesim_sim { // been reported. Both reset when the target goes idle. uint64_t busy_cycles_; bool stall_reported_; + + // A launch waits for `busy` to rise and then to fall, and nothing bounds + // either wait. These track the launch in flight so a target that never + // starts is reported instead of polled forever in silence. + bool awaiting_busy_; + uint64_t idle_cycles_; + + // Set when the launch watchdog (VORTEX_FIRESIM_TIMEOUT_S) or a stop signal + // gives up on a launch. The launch is then completed -- busy once, then idle + // -- so the application finishes through its own teardown rather than being + // killed mid-run. + int abort_phase_; + double launch_started_s_; + + // Set once the run is being stopped -- by a signal or by the watchdog -- and + // never cleared: from then on nothing more is sent to the target. + bool stopped_; + bool stopped(); + + // True while a report reads counters, which must reach the target even as + // the run stops. + bool reporting_; + static void install_stop_handlers(); }; } diff --git a/sim/foldsim/Makefile b/sim/foldsim/Makefile new file mode 100644 index 0000000000..3572f0ebe7 --- /dev/null +++ b/sim/foldsim/Makefile @@ -0,0 +1,72 @@ +include ../common.mk + +# The foldsim kernel in Verilator, as a library the runtime's avedsim target +# links: the same simulator hw/syn/foldsim generates for the image, in +# Foldsim's shell and its in-process host (FOLDSIM_SHELL), so a run here is +# the image's run, cycle for cycle, but for device memory's timing. Vortex adds +# only the Xilinx floating-point cores' models its FPU may instantiate. + +DESTDIR ?= $(CURDIR) + +SRC_DIR := $(VORTEX_HOME)/sim/foldsim +FOLDSIM_RTL_DIR := $(FOLDSIM_SHELL)/rtl +WORK_DIR := $(DESTDIR)/foldsim_emul +GEN_SV := $(WORK_DIR)/gen/foldsim_sim.sv +# The kernel's memory ports, as the image's flow derived them. +KERNEL_VH := $(WORK_DIR)/gen/foldsim_kernel.vh + +CONFIG_FILE := $(DESTDIR)/foldsim_emul_config.stamp + +CXXFLAGS += -std=c++17 -fPIC -O2 -DNDEBUG -I$(FOLDSIM_SHELL)/include +LDFLAGS += -shared -pthread +# The Xilinx floating-point cores' simulation models compute with Vortex's DPI +# floating point. +CXXFLAGS += -I$(SW_COMMON_DIR) -I$(THIRD_PARTY_DIR)/softfloat/source/include +CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) +LDFLAGS += $(THIRD_PARTY_DIR)/softfloat/build/Linux-x86_64-GCC/softfloat.a + +VL_FLAGS += -O3 -Wno-fatal -Wno-lint -Wno-style -Wno-MULTIDRIVEN --x-initial 0 --x-assign 0 +VL_FLAGS += -I$(FOLDSIM_RTL_DIR) -I$(WORK_DIR)/gen -I$(DPI_DIR) +ifdef VCD +VL_FLAGS += --trace --trace-structs +endif +THREADS ?= $(shell python3 -c 'import multiprocessing as mp; print(min(32, mp.cpu_count()))') +VL_FLAGS += -j $(THREADS) + +TOP := foldsim_afu +# The simulation models of the Xilinx floating-point cores the simulator may +# instantiate (its FPU of type DSP): Verilator cannot run their netlists. +RTL_SRCS := $(SRC_DIR)/xilinx_fp_models.sv $(GEN_SV) $(FOLDSIM_RTL_DIR)/foldsim_token.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_mem_bmap.sv $(FOLDSIM_RTL_DIR)/foldsim_mem_bank.sv \ + $(FOLDSIM_RTL_DIR)/foldsim_mem.sv $(FOLDSIM_RTL_DIR)/foldsim_ctrl.sv $(FOLDSIM_RTL_DIR)/foldsim_afu.sv +SRCS := $(FOLDSIM_SHELL)/emul/foldsim_emul.cpp $(DPI_DIR)/dpi_float.cpp $(DPI_DIR)/dpi_util.cpp \ + $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp + +PROJECT := libfoldsim_emul.so + +.PHONY: all lib force clean + +all: lib +lib: $(DESTDIR)/$(PROJECT) + +# The simulator, generated for these CONFIGS by the image's own flow. +$(GEN_SV): force + $(MAKE) -C $(ROOT_DIR)/hw/syn/foldsim generate TARGET=hw FOLDSIM_WORK_DIR=$(WORK_DIR) CONFIGS='$(CONFIGS)' \ + $(if $(PERF),PERF=$(PERF)) $(if $(FOLDSIM_CLASSES),FOLDSIM_CLASSES='$(FOLDSIM_CLASSES)') \ + $(if $(FOLDSIM_LIFT),FOLDSIM_LIFT='$(FOLDSIM_LIFT)') $(if $(CIRCT_BIN),CIRCT_BIN=$(CIRCT_BIN)) \ + $(if $(FOLDSIM_FLAGS),FOLDSIM_FLAGS='$(FOLDSIM_FLAGS)') \ + $(if $(LIFT_PYTHON),LIFT_PYTHON=$(LIFT_PYTHON)) $(if $(FOLDSIM),FOLDSIM=$(FOLDSIM)) + +$(DESTDIR)/$(PROJECT): $(GEN_SV) $(SRCS) $(SRC_DIR)/xilinx_fp_models.sv $(wildcard $(FOLDSIM_RTL_DIR)/*) \ + $(FOLDSIM_SHELL)/include/foldsim_emul.h $(CONFIG_FILE) + $(VERILATOR_PATH)/bin/verilator --build --exe $(VL_FLAGS) --cc $(RTL_SRCS) --top-module $(TOP) \ + $(SRCS) -CFLAGS '$(CXXFLAGS) -DFOLDSIM_MEM_PORTS='$$(sed -n 's/^`define FOLDSIM_MEM_PORTS //p' $(KERNEL_VH)) \ + -LDFLAGS '$(LDFLAGS)' --Mdir $@.obj_dir -o $@ + +$(CONFIG_FILE): force + @mkdir -p $(@D) + @printf '%s\n' "$(VL_FLAGS)" "$(CXXFLAGS)" "$(CONFIGS)" > $@.tmp + @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi + +clean: + rm -rf $(DESTDIR)/$(PROJECT).obj_dir $(DESTDIR)/$(PROJECT) $(WORK_DIR) diff --git a/sim/foldsim/xilinx_fp_models.sv b/sim/foldsim/xilinx_fp_models.sv new file mode 100644 index 0000000000..096678222a --- /dev/null +++ b/sim/foldsim/xilinx_fp_models.sv @@ -0,0 +1,194 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Simulation models of the Xilinx floating-point cores (floating_point 7.1) +// that hw/scripts/xilinx_ip_gen.tcl generates, for Verilator, which cannot run +// their encrypted netlists. Simulation only: never synthesized, never read by +// the Foldsim compiler (which sees the black boxes of xilinx_ip_stubs.sv). +// +// Each has its core's ports, latency and clock enable: its result is computed +// from its inputs by Vortex's DPI floating point (softfloat) and delayed by +// the core's C_Latency cycles of aclk with aclken high, the pipeline its +// contract declares. The numerics follow what PG060 documents for these +// configurations: round to nearest even only; denormal inputs taken as zero +// and denormal results flushed to zero, keeping the sign, with underflow +// raised; the flags of each core on m_axis_result_tuser. What the documents +// leave open is softfloat's (RISC-V) choice here: a NaN result is the +// canonical quiet NaN 0x7fc00000, and invalid is raised for signaling NaN +// inputs as for the invalid operations. Bit-exactness to the cores can only be +// confirmed on silicon. + +`include "dpi_float.vh" + +package xilinx_fp_model_pkg; + // A denormal input, as the cores read it: zero of its sign. + function automatic logic [31:0] ftz_in(input logic [31:0] x); + return (x[30:23] == 8'h0) ? {x[31], 31'h0} : x; + endfunction + // A result as the cores give it: a denormal flushed to zero of its sign. + function automatic logic [31:0] ftz_out(input logic [31:0] x); + return (x[30:23] == 8'h0) ? {x[31], 31'h0} : x; + endfunction + function automatic logic is_denormal(input logic [31:0] x); + return (x[30:23] == 8'h0) && (x[22:0] != 23'h0); + endfunction + function automatic logic [63:0] box(input logic [31:0] x); + return {32'hffffffff, x}; + endfunction +endpackage + +// The cores' pipeline: `D` bits taken on aclk with aclken high, given LATENCY +// such cycles later. +module xilinx_fp_model_pipe #( + parameter LATENCY = 1, + parameter D = 1 +) ( + input wire aclk, + input wire aclken, + input wire [D-1:0] d, + output wire [D-1:0] q +); + logic [D-1:0] stage [LATENCY]; + always @(posedge aclk) begin + if (aclken) begin + stage[0] <= d; + for (int i = 1; i < LATENCY; ++i) + stage[i] <= stage[i - 1]; + end + end + assign q = stage[LATENCY - 1]; +endmodule + +module xil_fma ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_c_tvalid, + input wire [31:0] s_axis_c_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [2:0] m_axis_result_tuser // underflow, overflow, invalid +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fmadd(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), + box(ftz_in(s_axis_c_tdata)), 3'b000, r, f); + end + wire tiny = is_denormal(r[31:0]); + // tuser = {invalid, overflow, underflow}; fflags = {NV, DZ, OF, UF, NX}. + xilinx_fp_model_pipe #(.LATENCY(16), .D(1 + 32 + 3)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid & s_axis_c_tvalid, ftz_out(r[31:0]), f[4], f[2], f[1] | tiny}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fdiv ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [3:0] m_axis_result_tuser // underflow, overflow, invalid, divide by zero +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fdiv(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + wire tiny = is_denormal(r[31:0]); + // tuser = {divide by zero, invalid, overflow, underflow}. + xilinx_fp_model_pipe #(.LATENCY(28), .D(1 + 32 + 4)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid, ftz_out(r[31:0]), f[3], f[4], f[2], f[1] | tiny}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fsqrt ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata, + output wire [0:0] m_axis_result_tuser // invalid +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fsqrt(1'b1, 0, box(ftz_in(s_axis_a_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(28), .D(1 + 32 + 1)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid, ftz_out(r[31:0]), f[4]}), + .q({m_axis_result_tvalid, m_axis_result_tdata, m_axis_result_tuser})); +endmodule + +module xil_fmul ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + always @(*) begin + dpi_fmul(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(8), .D(1 + 32)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid, ftz_out(r[31:0])}), + .q({m_axis_result_tvalid, m_axis_result_tdata})); +endmodule + +module xil_fadd ( + input wire aclk, + input wire aclken, + input wire s_axis_a_tvalid, + input wire [31:0] s_axis_a_tdata, + input wire s_axis_b_tvalid, + input wire [31:0] s_axis_b_tdata, + input wire s_axis_operation_tvalid, + input wire [7:0] s_axis_operation_tdata, + output wire m_axis_result_tvalid, + output wire [31:0] m_axis_result_tdata +); + import xilinx_fp_model_pkg::*; + logic [63:0] r; + logic [4:0] f; + // Operation bit 0: subtract. + always @(*) begin + if (s_axis_operation_tdata[0]) + dpi_fsub(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + else + dpi_fadd(1'b1, 0, box(ftz_in(s_axis_a_tdata)), box(ftz_in(s_axis_b_tdata)), 3'b000, r, f); + end + xilinx_fp_model_pipe #(.LATENCY(11), .D(1 + 32)) pipe ( + .aclk(aclk), .aclken(aclken), + .d({s_axis_a_tvalid & s_axis_b_tvalid & s_axis_operation_tvalid, ftz_out(r[31:0])}), + .q({m_axis_result_tvalid, m_axis_result_tdata})); +endmodule diff --git a/sim/rtlsim/Makefile b/sim/rtlsim/Makefile index ed85464742..f3eaf2bb23 100644 --- a/sim/rtlsim/Makefile +++ b/sim/rtlsim/Makefile @@ -133,7 +133,20 @@ SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/soft SRCS += $(DPI_DIR)/dpi_util.cpp $(DPI_DIR)/dpi_float.cpp SRCS += $(SRC_DIR)/processor.cpp +# AXI_BUS=1 simulates Vortex behind its AXI4 adapter, as the FPGA platforms +# instantiate it, against a latency-bandwidth-pipe memory model, or with +# AXI_MEM_MODEL=bank the DRAM bank model (sim/common/dram_bank_model.h). +ifdef AXI_BUS +TOP = rtlsim_axi_shim +CXXFLAGS += -DAXI_BUS +ifeq ($(AXI_MEM_MODEL), bank) +CXXFLAGS += -DAXI_MEM_BANK +else ifneq ($(filter-out pipe,$(AXI_MEM_MODEL)),) +$(error AXI_MEM_MODEL must be pipe or bank (got '$(AXI_MEM_MODEL)')) +endif +else TOP = rtlsim_shim +endif VL_FLAGS = --exe VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic diff --git a/sim/rtlsim/main.cpp b/sim/rtlsim/main.cpp index ba92ba1a03..e6287b8488 100644 --- a/sim/rtlsim/main.cpp +++ b/sim/rtlsim/main.cpp @@ -139,6 +139,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sim/rtlsim/processor.cpp b/sim/rtlsim/processor.cpp index b383d556ff..70accd2a22 100644 --- a/sim/rtlsim/processor.cpp +++ b/sim/rtlsim/processor.cpp @@ -14,7 +14,15 @@ #include #include "processor.h" +#ifdef AXI_BUS +#include "Vrtlsim_axi_shim.h" +#include +#include +typedef Vrtlsim_axi_shim Device; +#else #include "Vrtlsim_shim.h" +typedef Vrtlsim_shim Device; +#endif #ifdef VCD_OUTPUT #include @@ -29,7 +37,9 @@ #endif +#include #include +#include #include #include #include @@ -39,8 +49,11 @@ #include #include +#include #include #include +#include +#include #include #include #include @@ -49,6 +62,25 @@ #include +// AXI_BUS memory timing: a latency-bandwidth pipe, by default with FASED's +// settings, or (AXI_MEM_BANK) a DRAM channel per port as the bank model +// times it, from the platform memory's configuration. +#ifndef AXI_MEM_READ_LATENCY +#define AXI_MEM_READ_LATENCY 30 +#endif + +#ifndef AXI_MEM_WRITE_LATENCY +#define AXI_MEM_WRITE_LATENCY 30 +#endif + +#ifndef AXI_MEM_MAX_READS +#define AXI_MEM_MAX_READS 16 +#endif + +#ifndef AXI_MEM_MAX_WRITES +#define AXI_MEM_MAX_WRITES 16 +#endif + #ifndef TRACE_START_TIME #define TRACE_START_TIME 0ull #endif @@ -72,9 +104,21 @@ typedef uint64_t Word; #define VL_WDATA_GETW(lwp, i, n, w) \ VL_SEL_IWII(0, n * w, 0, 0, lwp, i * w, w) +// Command-processor DMA port into the last-level cache: one cache word per +// request, and as many reads in flight as one 4 KB DMA burst holds. +#if VX_CFG_L3_ENABLED +#define CP_MEM_DATA_SIZE VX_CFG_L2_SECTOR_SIZE +#else +#define CP_MEM_DATA_SIZE VX_CFG_L1_LINE_SIZE +#endif +#define CP_MEM_RD_SLOTS (4096 / CP_MEM_DATA_SIZE) + using namespace vortex; +#ifndef AXI_BUS +static uint64_t g_mem_done_seq = 0; static uint32_t g_mem_bank_addr_width = (VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS)); +#endif static uint64_t timestamp = 0; @@ -98,7 +142,11 @@ bool sim_trace_enabled() { class Processor::Impl { public: +#ifdef AXI_BUS + Impl() { +#else Impl() : dram_sim_(DramSim::platform_config()) { +#endif // restart sim time so a re-created model registers at time zero timestamp = 0; @@ -120,7 +168,7 @@ class Processor::Impl { context_->assertOn(false); // create device instance - device_ = new Vrtlsim_shim(context_); + device_ = new Device(context_); #ifdef VCD_OUTPUT context_->traceEverOn(true); @@ -237,8 +285,8 @@ class Processor::Impl { device_->dcr_req_data = tag; this->tick(); device_->dcr_req_valid = 0; - this->tick(); - // READ response is returned when dcr_rsp_valid is high + // A core whose flush already finished answers on the next cycle, and the + // response is a one-cycle pulse, so start polling right away. while (!device_->dcr_rsp_valid) { this->tick(); } @@ -246,14 +294,80 @@ class Processor::Impl { return 0; } + void drain() { + while (device_->busy) { + this->tick(); + } + } + +#ifndef AXI_BUS + void cp_mem_write(uint64_t addr, const uint8_t* src, uint64_t size, bool noalloc) { + while (size != 0) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + cp_req_t req; + req.rw = true; + req.noalloc = noalloc; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = 0; + req.data.fill(0); + for (uint64_t i = 0; i < n; ++i) { + req.data[offset + i] = src[i]; + req.byteen |= uint64_t(1) << (offset + i); + } + req.tag = 0; + cp_reqs_.push_back(req); + addr += n; + src += n; + size -= n; + } + while (!cp_reqs_.empty()) { + this->tick(); + } + } + + void cp_mem_read(uint64_t addr, uint8_t* dst, uint64_t size) { + uint32_t inflight = 0; + while (size != 0 || inflight != 0) { + // two queued requests keep one read entering the port every cycle + while (size != 0 && cp_reqs_.size() < 2 && !cp_free_slots_.empty()) { + uint64_t offset = addr % CP_MEM_DATA_SIZE; + uint64_t n = std::min(size, CP_MEM_DATA_SIZE - offset); + uint32_t slot = cp_free_slots_.back(); + cp_free_slots_.pop_back(); + cp_rd_slots_[slot] = {dst, uint32_t(offset), uint32_t(n)}; + cp_req_t req; + req.rw = false; + req.noalloc = false; + req.addr = addr / CP_MEM_DATA_SIZE; + req.byteen = ~uint64_t(0); + req.data.fill(0); + req.tag = slot; + cp_reqs_.push_back(req); + ++inflight; + addr += n; + dst += n; + size -= n; + } + this->tick(); + inflight -= cp_rd_completed_; + cp_rd_completed_ = 0; + } + } +#endif + private: void reset() { this->mem_bus_reset(); this->dcr_bus_reset(); +#ifndef AXI_BUS + this->cp_mem_reset(); +#endif print_bufs_.clear(); +#ifndef AXI_BUS for (auto& reqs : pending_mem_reqs_) { reqs.clear(); } @@ -262,6 +376,7 @@ class Processor::Impl { std::queue empty; std::swap(dram_queue_[b], empty); } +#endif device_->start = 0; device_->reset = 1; @@ -284,21 +399,44 @@ class Processor::Impl { this->eval(); } + #ifdef AXI_BUS + this->axi_drive(); + #else for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { device_->mem_req_ready[b] = 1; } + #endif } +#ifdef AXI_BUS + void tick() { + device_->clk = 0; + this->eval(); + + this->axi_sample(); + + device_->clk = 1; + this->eval(); + + this->axi_step(); + + #ifndef NDEBUG + fflush(stdout); + #endif + } +#else void tick() { device_->clk = 0; this->eval(); this->mem_bus_eval(0); + this->cp_mem_eval(0); device_->clk = 1; this->eval(); this->mem_bus_eval(1); + this->cp_mem_eval(1); dram_sim_.tick(); @@ -309,6 +447,7 @@ class Processor::Impl { // mark completed request as ready auto orig_req = reinterpret_cast(arg); orig_req->ready = true; + orig_req->done_seq = g_mem_done_seq++; return true; }, mem_req); dram_queue_[b].pop(); @@ -319,6 +458,7 @@ class Processor::Impl { fflush(stdout); #endif } +#endif void eval() { device_->eval(); @@ -335,6 +475,214 @@ class Processor::Impl { ++timestamp; } +#if defined(AXI_BUS) && defined(AXI_MEM_BANK) + static constexpr uint32_t AXI_BEAT_SIZE = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + + // The read and write slots of foldsim's memory endpoint, whose timing this + // is cycle for cycle. + static DramBankAxi::Config axi_bank_config() { + auto dram = DramSim::platform_config(); + DramBankAxi::Config c; + // VORTEX_DRAM_REFRESH=off|bank|all, as the bank model's DramSim backend. + const char* refresh = getenv("VORTEX_DRAM_REFRESH"); + c.model = DramBankModel::Config::make(dram.type, dram.data_rate, dram.block_size, + DramBankModel::Config::parse_refresh(refresh ? refresh : "")); + c.core_clock = dram.core_clock; + c.data_rate = dram.data_rate; + c.latency = dram.latency; + c.queue = dram.queue_size; + c.read_slots = 32; + c.write_slots = 16; + return c; + } + + // The address within the port's channel, as dram_sim's banks see it. + static uint64_t axi_channel_addr(uint64_t addr) { + #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 + return (addr / AXI_BEAT_SIZE / VX_CFG_PLATFORM_MEMORY_NUM_BANKS) * AXI_BEAT_SIZE; + #else + constexpr uint32_t bank_addr_bits = VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH - log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + return addr & ((uint64_t(1) << (bank_addr_bits + log2ceil(AXI_BEAT_SIZE))) - 1); + #endif + } + + void mem_bus_reset() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + axi_mem_[b] = std::make_unique(axi_bank_config()); + axi_wr_addr_[b].clear(); + axi_rd_data_[b].assign(32, std::vector(AXI_BEAT_SIZE)); + axi_in_[b] = DramBankAxi::Inputs(); + } + this->axi_drive(); + } + + // Samples each port's request side before the clock edge and performs the + // accepted transactions on memory: writes first, then reads, each read's + // data kept in its slot. + void axi_sample() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = *axi_mem_[b]; + auto& in = axi_in_[b]; + in.ar_valid = device_->m_axi_arvalid[b]; + in.ar_id = device_->m_axi_arid[b]; + in.ar_addr = axi_channel_addr(device_->m_axi_araddr[b]); + in.ar_len = device_->m_axi_arlen[b]; + in.aw_valid = device_->m_axi_awvalid[b]; + in.aw_id = device_->m_axi_awid[b]; + in.aw_addr = axi_channel_addr(device_->m_axi_awaddr[b]); + in.aw_len = device_->m_axi_awlen[b]; + in.w_valid = device_->m_axi_wvalid[b]; + in.w_last = device_->m_axi_wlast[b]; + in.r_ready = device_->m_axi_rready[b]; + in.b_ready = device_->m_axi_bready[b]; + assert(!(in.ar_valid && in.ar_len != 0) && !(in.aw_valid && in.aw_len != 0)); + + if (in.aw_valid && mem.aw_ready()) { + axi_wr_addr_[b].push_back(uint64_t(device_->m_axi_awaddr[b])); + } + if (in.w_valid && mem.w_ready()) { + assert(!axi_wr_addr_[b].empty()); + uint64_t addr = axi_wr_addr_[b].front(); + auto data = VDataCast::get(device_->m_axi_wdata[b]); + uint64_t strb = device_->m_axi_wstrb[b]; + for (uint32_t i = 0; i < AXI_BEAT_SIZE; ++i) { + if ((strb >> i) & 0x1) { + (*ram_)[addr + i] = data[i]; + } + } + axi_wr_addr_[b].pop_front(); + } + if (in.ar_valid && mem.ar_ready()) { + ram_->read(axi_rd_data_[b].at(mem.ar_slot(in.r_ready)).data(), + uint64_t(device_->m_axi_araddr[b]), AXI_BEAT_SIZE); + } + } + } + + void axi_step() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + axi_mem_[b]->step(axi_in_[b]); + } + this->axi_drive(); + } + + void axi_drive() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = *axi_mem_[b]; + device_->m_axi_arready[b] = mem.ar_ready(); + device_->m_axi_awready[b] = mem.aw_ready(); + device_->m_axi_wready[b] = mem.w_ready(); + device_->m_axi_rvalid[b] = mem.r_valid(); + device_->m_axi_rresp[b] = 0; + if (mem.r_valid()) { + device_->m_axi_rid[b] = mem.r_id(); + device_->m_axi_rlast[b] = 1; + memcpy(VDataCast::get(device_->m_axi_rdata[b]), + axi_rd_data_[b].at(mem.r_slot()).data(), AXI_BEAT_SIZE); + } + device_->m_axi_bvalid[b] = mem.b_valid(); + device_->m_axi_bresp[b] = 0; + if (mem.b_valid()) { + device_->m_axi_bid[b] = mem.b_id(); + } + } + } +#elif defined(AXI_BUS) + static constexpr uint32_t AXI_BEAT_SIZE = VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + + void mem_bus_reset() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + AxiLatencyPipe::Config cfg; + cfg.read_latency = AXI_MEM_READ_LATENCY; + cfg.write_latency = AXI_MEM_WRITE_LATENCY; + cfg.max_reads = AXI_MEM_MAX_READS; + cfg.max_writes = AXI_MEM_MAX_WRITES; + axi_mem_[b] = AxiLatencyPipe(cfg); + axi_rd_data_[b].clear(); + axi_wr_addr_[b].clear(); + axi_in_[b] = AxiLatencyPipe::Inputs(); + } + this->axi_drive(); + } + + // Samples each port's request side before the clock edge and performs the + // accepted transactions on memory: writes first, then reads. + void axi_sample() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = axi_mem_[b]; + auto& in = axi_in_[b]; + in.ar_valid = device_->m_axi_arvalid[b]; + in.ar_id = device_->m_axi_arid[b]; + in.ar_len = device_->m_axi_arlen[b]; + in.aw_valid = device_->m_axi_awvalid[b]; + in.aw_id = device_->m_axi_awid[b]; + in.w_valid = device_->m_axi_wvalid[b]; + in.w_last = device_->m_axi_wlast[b]; + in.r_ready = device_->m_axi_rready[b]; + in.b_ready = device_->m_axi_bready[b]; + + if (in.aw_valid && mem.aw_ready()) { + axi_wr_addr_[b].push_back(uint64_t(device_->m_axi_awaddr[b])); + } + if (in.w_valid && mem.w_ready()) { + // a write beat never runs ahead of its address on this port + assert(!axi_wr_addr_[b].empty()); + uint64_t addr = axi_wr_addr_[b].front(); + auto data = VDataCast::get(device_->m_axi_wdata[b]); + uint64_t strb = device_->m_axi_wstrb[b]; + for (uint32_t i = 0; i < AXI_BEAT_SIZE; ++i) { + if ((strb >> i) & 0x1) { + (*ram_)[addr + i] = data[i]; + } + } + if (in.w_last) { + axi_wr_addr_[b].pop_front(); + } else { + axi_wr_addr_[b].front() += AXI_BEAT_SIZE; + } + } + if (in.ar_valid && mem.ar_ready()) { + uint64_t addr = uint64_t(device_->m_axi_araddr[b]); + std::vector data((in.ar_len + 1) * AXI_BEAT_SIZE); + ram_->read(data.data(), addr, data.size()); + axi_rd_data_[b].push_back(std::move(data)); + } + axi_rlast_fire_[b] = mem.r_valid() && in.r_ready && mem.r_last(); + } + } + + void axi_step() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + if (axi_rlast_fire_[b]) { + axi_rd_data_[b].pop_front(); + } + axi_mem_[b].step(axi_in_[b]); + } + this->axi_drive(); + } + + void axi_drive() { + for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + auto& mem = axi_mem_[b]; + device_->m_axi_arready[b] = mem.ar_ready(); + device_->m_axi_awready[b] = mem.aw_ready(); + device_->m_axi_wready[b] = mem.w_ready(); + device_->m_axi_rvalid[b] = mem.r_valid(); + device_->m_axi_rresp[b] = 0; + if (mem.r_valid()) { + device_->m_axi_rid[b] = mem.r_id(); + device_->m_axi_rlast[b] = mem.r_last(); + memcpy(VDataCast::get(device_->m_axi_rdata[b]), + axi_rd_data_[b].front().data() + mem.r_beat() * AXI_BEAT_SIZE, AXI_BEAT_SIZE); + } + device_->m_axi_bvalid[b] = mem.b_valid(); + device_->m_axi_bresp[b] = 0; + if (mem.b_valid()) { + device_->m_axi_bid[b] = mem.b_id(); + } + } + } +#else void mem_bus_reset() { for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { device_->mem_req_ready[b] = 0; @@ -355,22 +703,29 @@ class Processor::Impl { if (device_->mem_rsp_valid[b] && mem_rd_rsp_ready_[b]) { device_->mem_rsp_valid[b] = 0; } - if (device_->mem_rsp_valid[b] == 0) { - if (!pending_mem_reqs_[b].empty()) { - auto mem_rsp_it = pending_mem_reqs_[b].begin(); - auto mem_rsp = *mem_rsp_it; - if (mem_rsp->ready) { - if (!mem_rsp->write) { - // return read responses - device_->mem_rsp_valid[b] = 1; - memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - device_->mem_rsp_tag[b] = mem_rsp->tag; - } - // delete the request - pending_mem_reqs_[b].erase(mem_rsp_it); - delete mem_rsp; - } + // Reads carry distinct AXI IDs, so the memory returns them in the order + // they complete; writes are posted and never hold back a read. + auto& pending = pending_mem_reqs_[b]; + auto next_rd = pending.end(); + for (auto it = pending.begin(); it != pending.end();) { + auto mem_req = *it; + if (mem_req->ready && mem_req->write) { + it = pending.erase(it); + delete mem_req; + continue; + } + if (mem_req->ready && (next_rd == pending.end() || mem_req->done_seq < (*next_rd)->done_seq)) { + next_rd = it; } + ++it; + } + if (next_rd != pending.end() && !device_->mem_rsp_valid[b]) { + auto mem_rsp = *next_rd; + device_->mem_rsp_valid[b] = 1; + memcpy(VDataCast::get(device_->mem_rsp_data[b]), mem_rsp->data.data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + device_->mem_rsp_tag[b] = mem_rsp->tag; + pending.erase(next_rd); + delete mem_rsp; } // process memory requests @@ -426,12 +781,60 @@ class Processor::Impl { } } } +#endif void dcr_bus_reset() { device_->dcr_req_valid = 0; device_->dcr_req_rw = 0; } +#ifndef AXI_BUS + void cp_mem_reset() { + cp_reqs_.clear(); + cp_free_slots_.clear(); + for (uint32_t i = 0; i < CP_MEM_RD_SLOTS; ++i) { + cp_free_slots_.push_back(CP_MEM_RD_SLOTS - 1 - i); + } + cp_req_fire_ = false; + cp_rd_completed_ = 0; + device_->cp_mem_req_valid = 0; + device_->cp_mem_rsp_ready = 1; + } + + // Inputs change after the rising edge; the handshakes they take part in + // are sampled after the falling-edge evaluation, ahead of the next edge. + void cp_mem_eval(bool clk) { + if (!clk) { + cp_req_fire_ = device_->cp_mem_req_valid && device_->cp_mem_req_ready; + if (device_->cp_mem_rsp_valid) { + uint32_t slot = device_->cp_mem_rsp_tag; + auto& rd = cp_rd_slots_.at(slot); + auto data = VDataCast::get(device_->cp_mem_rsp_data); + std::memcpy(rd.dst, data + rd.offset, rd.size); + cp_free_slots_.push_back(slot); + ++cp_rd_completed_; + } + return; + } + if (cp_req_fire_) { + cp_reqs_.pop_front(); + cp_req_fire_ = false; + } + if (cp_reqs_.empty()) { + device_->cp_mem_req_valid = 0; + return; + } + auto& req = cp_reqs_.front(); + device_->cp_mem_req_valid = 1; + device_->cp_mem_req_rw = req.rw; + device_->cp_mem_req_noalloc = req.noalloc; + device_->cp_mem_req_addr = req.addr; + device_->cp_mem_req_byteen = req.byteen; + device_->cp_mem_req_tag = req.tag; + memcpy(VDataCast::get(device_->cp_mem_req_data), req.data.data(), CP_MEM_DATA_SIZE); + } +#endif + void wait(uint32_t cycles) { for (int i = 0; i < cycles; ++i) { this->tick(); @@ -440,17 +843,51 @@ class Processor::Impl { private: + std::unordered_map print_bufs_; + +#if defined(AXI_BUS) && defined(AXI_MEM_BANK) + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_mem_; + std::array axi_in_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_wr_addr_; + // Read data by slot. + std::array>, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_rd_data_; +#elif defined(AXI_BUS) + std::array axi_mem_; + std::array axi_in_; + std::array axi_rlast_fire_; + std::array>, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_rd_data_; + std::array, VX_CFG_PLATFORM_MEMORY_NUM_BANKS> axi_wr_addr_; +#else typedef struct { - Vrtlsim_shim* device; std::array data; uint64_t addr; uint64_t bank_addr; uint64_t tag; bool write; bool ready; + uint64_t done_seq; } mem_req_t; - std::unordered_map print_bufs_; + typedef struct { + std::array data; + uint64_t addr; + uint64_t byteen; + uint32_t tag; + bool rw; + bool noalloc; + } cp_req_t; + + typedef struct { + uint8_t* dst; + uint32_t offset; + uint32_t size; + } cp_rd_t; + + std::deque cp_reqs_; + std::array cp_rd_slots_; + std::vector cp_free_slots_; + uint32_t cp_rd_completed_; + bool cp_req_fire_; std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; @@ -459,9 +896,10 @@ class Processor::Impl { std::array mem_rd_rsp_ready_; DramSim dram_sim_; +#endif VerilatedContext* context_; - Vrtlsim_shim* device_; + Device* device_; RAM* ram_; @@ -497,4 +935,39 @@ int Processor::dcr_write(uint32_t addr, uint32_t value) { int Processor::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { return impl_->dcr_read(addr, tag, value); +} + +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { +#ifdef AXI_BUS + // Vortex_axi ties the CP DMA port off, so cp_llc_client() is false + __unused(addr, src, size, noalloc); + std::abort(); +#else + impl_->cp_mem_write(addr, static_cast(src), size, noalloc); +#endif +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { +#ifdef AXI_BUS + __unused(addr, dst, size); + std::abort(); +#else + impl_->cp_mem_read(addr, static_cast(dst), size); +#endif +} + +bool Processor::cp_llc_client() const { +#ifdef AXI_BUS + return false; +#else + return VX_CFG_CP_ON_LLC != 0; +#endif +} + +uint64_t Processor::cp_llc_size() const { + return VX_CFG_L3_ENABLED ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + +void Processor::drain() { + impl_->drain(); } \ No newline at end of file diff --git a/sim/rtlsim/processor.h b/sim/rtlsim/processor.h index 3293fe1823..05e0b4605c 100644 --- a/sim/rtlsim/processor.h +++ b/sim/rtlsim/processor.h @@ -40,6 +40,22 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Clock the device until busy drops, i.e. until memory has taken every + // write it issued. + void drain(); + + // Device-memory access through the command processor's port into the + // last-level cache. A write returns once every request entered the port; + // the device orders it ahead of the next launch. A read returns with data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: class Impl; diff --git a/sim/rtlsim/rtlsim_axi_shim.sv b/sim/rtlsim/rtlsim_axi_shim.sv new file mode 100644 index 0000000000..b4dac97b34 --- /dev/null +++ b/sim/rtlsim/rtlsim_axi_shim.sv @@ -0,0 +1,144 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +`include "VX_define.vh" + +`ifndef PLATFORM_MEMORY_ID_WIDTH +`define PLATFORM_MEMORY_ID_WIDTH 32 +`endif + +// Vortex behind its AXI4 memory adapter (Vortex_axi), as the FPGA platforms +// see it, with the AXI channels a memory model needs. The parameters are the +// ones the platform wrappers pass, so the adapter is the same logic they run. +module rtlsim_axi_shim import VX_gpu_pkg::*; #( + parameter AXI_DATA_WIDTH = `VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8, + parameter AXI_ADDR_WIDTH = `VX_CFG_MEM_ADDR_WIDTH, + parameter AXI_TID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, + parameter AXI_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS +) ( + `SCOPE_IO_DECL + + input wire clk, + input wire reset, + + output wire m_axi_awvalid [AXI_NUM_BANKS], + input wire m_axi_awready [AXI_NUM_BANKS], + output wire [AXI_ADDR_WIDTH-1:0] m_axi_awaddr [AXI_NUM_BANKS], + output wire [AXI_TID_WIDTH-1:0] m_axi_awid [AXI_NUM_BANKS], + output wire [7:0] m_axi_awlen [AXI_NUM_BANKS], + + output wire m_axi_wvalid [AXI_NUM_BANKS], + input wire m_axi_wready [AXI_NUM_BANKS], + output wire [AXI_DATA_WIDTH-1:0] m_axi_wdata [AXI_NUM_BANKS], + output wire [AXI_DATA_WIDTH/8-1:0] m_axi_wstrb [AXI_NUM_BANKS], + output wire m_axi_wlast [AXI_NUM_BANKS], + + input wire m_axi_bvalid [AXI_NUM_BANKS], + output wire m_axi_bready [AXI_NUM_BANKS], + input wire [AXI_TID_WIDTH-1:0] m_axi_bid [AXI_NUM_BANKS], + input wire [1:0] m_axi_bresp [AXI_NUM_BANKS], + + output wire m_axi_arvalid [AXI_NUM_BANKS], + input wire m_axi_arready [AXI_NUM_BANKS], + output wire [AXI_ADDR_WIDTH-1:0] m_axi_araddr [AXI_NUM_BANKS], + output wire [AXI_TID_WIDTH-1:0] m_axi_arid [AXI_NUM_BANKS], + output wire [7:0] m_axi_arlen [AXI_NUM_BANKS], + + input wire m_axi_rvalid [AXI_NUM_BANKS], + output wire m_axi_rready [AXI_NUM_BANKS], + input wire [AXI_DATA_WIDTH-1:0] m_axi_rdata [AXI_NUM_BANKS], + input wire m_axi_rlast [AXI_NUM_BANKS], + input wire [AXI_TID_WIDTH-1:0] m_axi_rid [AXI_NUM_BANKS], + input wire [1:0] m_axi_rresp [AXI_NUM_BANKS], + + input wire dcr_req_valid, + input wire dcr_req_rw, + input wire [VX_DCR_ADDR_WIDTH-1:0] dcr_req_addr, + input wire [VX_DCR_DATA_WIDTH-1:0] dcr_req_data, + + output wire dcr_rsp_valid, + output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + + input wire start, + output wire busy +); + `SCOPE_IO_SWITCH (1); + + Vortex_axi #( + .AXI_DATA_WIDTH (AXI_DATA_WIDTH), + .AXI_ADDR_WIDTH (AXI_ADDR_WIDTH), + .AXI_TID_WIDTH (AXI_TID_WIDTH), + .AXI_NUM_BANKS (AXI_NUM_BANKS) + ) vortex_axi ( + `SCOPE_IO_BIND (0) + + .clk (clk), + .reset (reset), + + .m_axi_awvalid (m_axi_awvalid), + .m_axi_awready (m_axi_awready), + .m_axi_awaddr (m_axi_awaddr), + .m_axi_awid (m_axi_awid), + .m_axi_awlen (m_axi_awlen), + `UNUSED_PIN (m_axi_awsize), + `UNUSED_PIN (m_axi_awburst), + `UNUSED_PIN (m_axi_awlock), + `UNUSED_PIN (m_axi_awcache), + `UNUSED_PIN (m_axi_awprot), + `UNUSED_PIN (m_axi_awqos), + `UNUSED_PIN (m_axi_awregion), + + .m_axi_wvalid (m_axi_wvalid), + .m_axi_wready (m_axi_wready), + .m_axi_wdata (m_axi_wdata), + .m_axi_wstrb (m_axi_wstrb), + .m_axi_wlast (m_axi_wlast), + + .m_axi_bvalid (m_axi_bvalid), + .m_axi_bready (m_axi_bready), + .m_axi_bid (m_axi_bid), + .m_axi_bresp (m_axi_bresp), + + .m_axi_arvalid (m_axi_arvalid), + .m_axi_arready (m_axi_arready), + .m_axi_araddr (m_axi_araddr), + .m_axi_arid (m_axi_arid), + .m_axi_arlen (m_axi_arlen), + `UNUSED_PIN (m_axi_arsize), + `UNUSED_PIN (m_axi_arburst), + `UNUSED_PIN (m_axi_arlock), + `UNUSED_PIN (m_axi_arcache), + `UNUSED_PIN (m_axi_arprot), + `UNUSED_PIN (m_axi_arqos), + `UNUSED_PIN (m_axi_arregion), + + .m_axi_rvalid (m_axi_rvalid), + .m_axi_rready (m_axi_rready), + .m_axi_rdata (m_axi_rdata), + .m_axi_rlast (m_axi_rlast), + .m_axi_rid (m_axi_rid), + .m_axi_rresp (m_axi_rresp), + + .dcr_req_valid (dcr_req_valid), + .dcr_req_rw (dcr_req_rw), + .dcr_req_addr (dcr_req_addr), + .dcr_req_data (dcr_req_data), + + .dcr_rsp_valid (dcr_rsp_valid), + .dcr_rsp_data (dcr_rsp_data), + + .start (start), + .busy (busy) + ); + +endmodule diff --git a/sim/rtlsim/rtlsim_shim.sv b/sim/rtlsim/rtlsim_shim.sv index f5159027b9..9e214e1c90 100644 --- a/sim/rtlsim/rtlsim_shim.sv +++ b/sim/rtlsim/rtlsim_shim.sv @@ -50,6 +50,21 @@ module rtlsim_shim import VX_gpu_pkg::*; #( output wire dcr_rsp_valid, output wire [VX_DCR_DATA_WIDTH-1:0] dcr_rsp_data, + // Command-processor DMA into the last-level cache + input wire cp_mem_req_valid, + input wire cp_mem_req_rw, + input wire cp_mem_req_noalloc, + input wire [CP_MEM_ADDR_WIDTH-1:0] cp_mem_req_addr, + input wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_req_data, + input wire [CP_MEM_DATA_SIZE-1:0] cp_mem_req_byteen, + input wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_req_tag, + output wire cp_mem_req_ready, + + output wire cp_mem_rsp_valid, + output wire [CP_MEM_DATA_SIZE*8-1:0] cp_mem_rsp_data, + output wire [CP_MEM_TAG_WIDTH-1:0] cp_mem_rsp_tag, + input wire cp_mem_rsp_ready, + // ctrl/status input wire start, output wire busy @@ -73,6 +88,8 @@ module rtlsim_shim import VX_gpu_pkg::*; #( wire [VX_MEM_TAG_WIDTH-1:0] vx_mem_rsp_tag [VX_MEM_PORTS]; wire vx_mem_rsp_ready [VX_MEM_PORTS]; + wire vx_busy; + `SCOPE_IO_SWITCH (1); Vortex vortex ( @@ -102,8 +119,21 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .dcr_rsp_valid (dcr_rsp_valid), .dcr_rsp_data (dcr_rsp_data), + .cp_mem_req_valid (cp_mem_req_valid), + .cp_mem_req_rw (cp_mem_req_rw), + .cp_mem_req_noalloc (cp_mem_req_noalloc), + .cp_mem_req_addr (cp_mem_req_addr), + .cp_mem_req_data (cp_mem_req_data), + .cp_mem_req_byteen (cp_mem_req_byteen), + .cp_mem_req_tag (cp_mem_req_tag), + .cp_mem_req_ready (cp_mem_req_ready), + .cp_mem_rsp_valid (cp_mem_rsp_valid), + .cp_mem_rsp_data (cp_mem_rsp_data), + .cp_mem_rsp_tag (cp_mem_rsp_tag), + .cp_mem_rsp_ready (cp_mem_rsp_ready), + .start (start), - .busy (busy) + .busy (vx_busy) ); wire mem_req_valid_a [VX_MEM_PORTS]; @@ -204,4 +234,28 @@ module rtlsim_shim import VX_gpu_pkg::*; #( .mem_rsp_ready_out (mem_rsp_ready) ); + // Memory applies a write once it takes it, so the device stays busy until + // every write Vortex issued has crossed the bank adapter. + wire [VX_MEM_PORTS-1:0] vx_wr_fire; + wire [MEM_NUM_BANKS-1:0] mem_wr_fire; + wire wr_empty; + for (genvar i = 0; i < VX_MEM_PORTS; ++i) begin : g_vx_wr_fire + assign vx_wr_fire[i] = mem_req_valid_a[i] && mem_req_ready_a[i] && mem_req_rw_a[i]; + end + for (genvar i = 0; i < MEM_NUM_BANKS; ++i) begin : g_mem_wr_fire + assign mem_wr_fire[i] = mem_req_valid[i] && mem_req_ready[i] && mem_req_rw[i]; + end + VX_mem_req_pending #( + .NUM_INPUTS (VX_MEM_PORTS), + .NUM_OUTPUTS (MEM_NUM_BANKS) + ) wr_pending ( + .clk (clk), + .reset (reset), + .in_fire (vx_wr_fire), + .out_fire (mem_wr_fire), + .empty (wr_empty) + ); + + assign busy = vx_busy || ~wr_empty; + endmodule diff --git a/sim/simx/Makefile b/sim/simx/Makefile index 4338c8ab6d..923a1a6485 100644 --- a/sim/simx/Makefile +++ b/sim/simx/Makefile @@ -33,7 +33,7 @@ XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME) # Source files definition SRCS = $(SW_COMMON_DIR)/util.cpp $(SIM_COMMON_DIR)/mem.cpp $(SW_COMMON_DIR)/softfloat_ext.cpp $(SW_COMMON_DIR)/rvfloats.cpp $(SIM_COMMON_DIR)/dram_sim.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp $(SIM_COMMON_DIR)/elf_loader.cpp $(SIM_COMMON_DIR)/host_monitor.cpp $(SIM_COMMON_DIR)/cout_drainer.cpp -SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp +SRCS += $(SRC_DIR)/processor.cpp $(SRC_DIR)/cluster.cpp $(SRC_DIR)/socket.cpp $(SRC_DIR)/core.cpp $(SRC_DIR)/scheduler.cpp $(SRC_DIR)/cta_dispatcher.cpp $(SRC_DIR)/barrier_unit.cpp $(SRC_DIR)/gbar_unit.cpp SRCS += $(SRC_DIR)/kmu/kmu.cpp SRCS += $(SRC_DIR)/decode.cpp $(SRC_DIR)/decompressor.cpp $(SRC_DIR)/scoreboard.cpp $(SRC_DIR)/sequencer.cpp $(SRC_DIR)/opc_unit.cpp $(SRC_DIR)/dispatcher.cpp SRCS += $(SRC_DIR)/alu_unit.cpp $(SRC_DIR)/fpu_unit.cpp $(SRC_DIR)/lsu_unit.cpp $(SRC_DIR)/sfu_unit.cpp $(SRC_DIR)/wctl_unit.cpp $(SRC_DIR)/csr_unit.cpp diff --git a/sim/simx/alu_unit.cpp b/sim/simx/alu_unit.cpp index a48fc3ebe5..e30768ac06 100644 --- a/sim/simx/alu_unit.cpp +++ b/sim/simx/alu_unit.cpp @@ -30,7 +30,16 @@ AluUnit::AluUnit(const SimContext& ctx, const char* name, Core* core) // multiplier keeps one result per cycle. : FuncUnit(ctx, name, core, kMulDivLatency + (kGather ? 1 : 0) + 2) , branch_ctl_out(this, VX_CFG_NUM_WARPS) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} // Cycles from this unit's execute to its output, beyond the commit-queue hop. // Execute stands for the integer ALU's registered result (the branch_ctl @@ -572,13 +581,16 @@ void AluUnit::on_tick() { auto trace = input.peek(); this->execute(trace); // A branch releases its warp as it resolves, not when its result - // retires. + // retires: the result register, then the branch register. if (std::get_if(&trace->op_type) && trace->eop && trace->resume_warp) { - branch_ctl_out.send(trace->wid, 1); + branch_ctl_out.send(trace->wid, kGather ? 1 : 2); trace->resume_warp = false; } output.send(trace, this->latency_of(trace) + (kGather ? 1 : 0)); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/alu_unit.h b/sim/simx/alu_unit.h index 6e2db38121..c7c11e83ad 100644 --- a/sim/simx/alu_unit.h +++ b/sim/simx/alu_unit.h @@ -30,6 +30,9 @@ class AluUnit : public FuncUnit { // Resolved branch of a stalled warp, registered toward the scheduler. SimChannel branch_ctl_out; + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; protected: void on_tick() override; diff --git a/sim/simx/barrier_unit.cpp b/sim/simx/barrier_unit.cpp index 92349d7add..968541ad16 100644 --- a/sim/simx/barrier_unit.cpp +++ b/sim/simx/barrier_unit.cpp @@ -21,8 +21,18 @@ using namespace vortex; +// Cycles from an SFU barrier op to its effect in the barrier store: the store +// is read a cycle before an arrival applies, and with several SFU blocks the +// request is also registered out of its block. A transaction-barrier +// completion crosses the completion queue and a register before it applies; a +// global-barrier release applies as it arrives. A release then reaches the +// warp scheduler registered. +static constexpr uint32_t kOpApplyLag = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 2 : 1; +static constexpr uint32_t kEventApplyLag = 2; + BarrierUnit::BarrierUnit(const SimContext& ctx, const char* name, Core* core, Scheduler* scheduler) : SimObject(ctx, name) + , unlock_out(this, VX_CFG_NUM_WARPS) , gbar_send_next_(0) , core_(core) , scheduler_(scheduler) @@ -34,13 +44,17 @@ BarrierUnit::~BarrierUnit() {} // One request leaves the queue per cycle, so a completion that lands while the // queue is still draining is delivered on the next free cycle rather than // immediately. Mirrors the RTL request FIFO's drain rate. -void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count) { +void BarrierUnit::gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag) { uint64_t now = SimPlatform::instance().cycles(); - uint64_t due = std::max(now + 1, gbar_send_next_); + uint64_t due = std::max(now + lag + 1, gbar_send_next_); gbar_send_next_ = due + 1; core_->gbar_arrive_out.send({bar_id, count, core_->id()}, due - now); } +void BarrierUnit::release(uint32_t wid, uint32_t lag) { + unlock_out.send(wid, lag + 1); +} + void BarrierUnit::on_reset() { gbar_send_next_ = 0; for (auto& barrier : barriers_) { @@ -74,7 +88,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ if (barrier.arrival_mask.count() == active_warps.count() && barrier.events == 0) { // wait_mask is preserved across the async hop (holds warps resumed by // global_resume) and cleared there. arrival_mask is local-only and reset here. - this->gbar_send(bar_id, count); + this->gbar_send(bar_id, count, kOpApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -90,7 +104,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kOpApplyLag); } } // reset barrier and advance phase @@ -106,7 +120,7 @@ void BarrierUnit::arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_ } } -bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { +void BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { auto bar_index = bar_id & 0x7fffffff; auto& barrier = barriers_.at(bar_index); @@ -118,9 +132,9 @@ bool BarrierUnit::wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { // add warp to wait list barrier.wait_mask.set(wid); DP(3, "*** Suspend core #" << core_->id() << ", warp #" << wid << " at barrier #" << bar_id); + } else { + this->release(wid, kOpApplyLag); } - - return wait; } void BarrierUnit::global_resume(uint32_t bar_id) { @@ -132,7 +146,7 @@ void BarrierUnit::global_resume(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, 0); } } barrier.wait_mask.reset(); @@ -156,7 +170,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { if (is_global) { if (barrier.arrival_mask.count() == active_warps.count()) { uint32_t num_cores = barrier.count; - this->gbar_send(bar_id, num_cores); + this->gbar_send(bar_id, num_cores, kEventApplyLag); barrier.arrival_mask.reset(); barrier.count = 0; barrier.events = 0; @@ -167,7 +181,7 @@ void BarrierUnit::event_release(uint32_t bar_id) { for (uint32_t i = 0; i < VX_CFG_NUM_WARPS; ++i) { if (barrier.wait_mask.test(i)) { DP(3, "*** Resume core #" << core_->id() << ", warp #" << i << " at barrier #" << bar_id); - scheduler_->resume(i); + this->release(i, kEventApplyLag); } } // reset barrier and advance phase diff --git a/sim/simx/barrier_unit.h b/sim/simx/barrier_unit.h index 253fb601c7..d75222612f 100644 --- a/sim/simx/barrier_unit.h +++ b/sim/simx/barrier_unit.h @@ -30,11 +30,14 @@ class BarrierUnit : public SimObject { uint32_t get_phase(uint32_t bar_id) const; void arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_resume(uint32_t bar_id); void event_attach(uint32_t bar_id, uint32_t count = 1); void event_release(uint32_t bar_id); + // warps a local barrier releases, to the core's warp scheduler + SimChannel unlock_out; + protected: void on_reset(); @@ -44,7 +47,9 @@ class BarrierUnit : public SimObject { // send on its own cycle so back-to-back completions serialize here too. uint64_t gbar_send_next_; - void gbar_send(uint32_t bar_id, uint32_t count); + // lag: cycles until the triggering request applies in the barrier store + void gbar_send(uint32_t bar_id, uint32_t count, uint32_t lag); + void release(uint32_t wid, uint32_t lag); Core* core_; Scheduler* scheduler_; diff --git a/sim/simx/cluster.cpp b/sim/simx/cluster.cpp index 937c2ae131..283324fc77 100644 --- a/sim/simx/cluster.cpp +++ b/sim/simx/cluster.cpp @@ -12,12 +12,16 @@ // limitations under the License. #include "cluster.h" +#include #include "socket.h" #include "core.h" #include "local_mem.h" #include "constants.h" #include "types.h" #include "debug.h" +#ifdef SIMX_CLUSTER_UNIT_MMU +#include "mem/mmu.h" +#endif #ifdef VX_CFG_EXT_OM_ENABLE #include "om_core.h" #include "om_unit.h" @@ -31,6 +35,11 @@ using namespace vortex; +// Output-buffer stages of the socket and cluster gbar arbiters (OUT_BUF 3, one +// cycle each), present when the arbiter has more than one input. +static constexpr uint32_t kSocketArbLag = (VX_CFG_SOCKET_SIZE > 1) ? 1 : 0; +static constexpr uint32_t kClusterArbLag = (NUM_SOCKETS > 1) ? 1 : 0; + class Cluster::Impl { public: Impl(Cluster* simobject) @@ -50,7 +59,9 @@ class Cluster::Impl { for (uint32_t i = 0; i < sockets_per_cluster; ++i) { uint32_t socket_id = cluster_id * sockets_per_cluster + i; snprintf(sname, 100, "%s-socket%d", name.c_str(), i); - sockets_.at(i) = Socket::Create(sname, socket_id, simobject_); + // A socket's unit MMU is a cluster-TLB client after every core's two. + uint32_t unit_tlb_client = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2 + i; + sockets_.at(i) = Socket::Create(sname, socket_id, simobject_, unit_tlb_client); } // Global-barrier event links: each core's arrive end fans into the @@ -67,24 +78,37 @@ class Cluster::Impl { snprintf(sname, 100, "%s-l2cache", name.c_str()); // L2 is the LLC iff L2 is enabled and L3 is not. - l2cache_ = Cache::Create(sname, Cache::Config{ - !VX_CFG_L2_ENABLED, - log2ceil(VX_CFG_L2_SIZE),// C - log2ceil(VX_CFG_L2_LINE_SIZE),// L - log2ceil(VX_CFG_L2_SECTOR_SIZE),// S - log2ceil(VX_CFG_L1_LINE_SIZE), // W - log2ceil(VX_CFG_L2_NUM_WAYS), // A - log2ceil(VX_CFG_L2_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - VX_CFG_L2_NUM_REQS, // request size - VX_CFG_L2_MEM_PORTS, // memory ports - VX_CFG_L2_WRITEBACK, // write-back - false, // write response - VX_CFG_L2_MSHR_SIZE, // mshr size - VX_CFG_L2_LATENCY, // pipeline latency - VX_CFG_L2_REPL_POLICY, // replacement policy - (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc - }); + { + // the CP writes into this cache when it is the last level: its state + // and a transfer's fills carry across the launch reset + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0); + l2cache_ = Cache::Create(sname, Cache::Config{ + !VX_CFG_L2_ENABLED, + log2ceil(VX_CFG_L2_SIZE),// C + log2ceil(VX_CFG_L2_LINE_SIZE),// L + log2ceil(VX_CFG_L2_SECTOR_SIZE),// S + log2ceil(VX_CFG_L1_LINE_SIZE), // W + log2ceil(VX_CFG_L2_NUM_WAYS), // A + log2ceil(VX_CFG_L2_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L2_CACHE_REQS, // request size + VX_CFG_L2_MEM_PORTS, // memory ports + VX_CFG_L2_WRITEBACK, // write-back + false, // write response + VX_CFG_L2_MSHR_SIZE, // mshr size + VX_CFG_L2_LATENCY, // pipeline latency + VX_CFG_L2_REPL_POLICY, // replacement policy + (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass + (VX_CFG_L2_BANK_HASH != 0) && (VX_CFG_L2_NUM_BANKS > 1), // bank_hash + L2_SHARED_LLC, // shared_llc + VX_CFG_L2_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + CP_ON_L2 != 0, // write_around + VX_CFG_L2_MRSQ_SIZE != 0, // memory-response queue + }); + } // connect l2cache memory interface @@ -114,6 +138,22 @@ class Cluster::Impl { l2arb->RspOut.at(kL2Rows * port + 0).bind(&sockets_.at(i)->mem_rsp_in.at(j)); } } + // A gfx cache's memory port reaches the L2 arbiter through the cluster + // unit MMU under VM. + auto bind_gfx_port = [&](uint32_t arb_idx, uint32_t cmmu_idx, + SimChannel& req_src, + SimChannel& rsp_dst) { + #ifdef SIMX_CLUSTER_UNIT_MMU + req_src.bind(&gfx_mmu_->ReqIn.at(cmmu_idx)); + gfx_mmu_->RspOut.at(cmmu_idx).bind(&rsp_dst); + gfx_mmu_->ReqOut.at(cmmu_idx).bind(&l2arb->ReqIn.at(arb_idx)); + l2arb->RspOut.at(arb_idx).bind(&gfx_mmu_->RspIn.at(cmmu_idx)); + #else + __unused(cmmu_idx); + req_src.bind(&l2arb->ReqIn.at(arb_idx)); + l2arb->RspOut.at(arb_idx).bind(&rsp_dst); + #endif + }; // L2 arb outputs → l2cache (after all rows are bound). for (uint32_t i = 0; i < VX_CFG_L2_DEMAND_REQS; ++i) { l2arb->ReqOut.at(i).bind(&l2cache_->core_req_in.at(i)); @@ -130,9 +170,21 @@ class Cluster::Impl { #endif // cluster-resident gfx caches #ifdef VX_CFG_VM_ENABLE - // Shared cluster L2 TLB. Every core's two L1 TLBs are clients; its - // misses export on the walker link the processor binds at the device. - uint32_t num_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; + // Shared cluster L2 TLB. Every core's two L1 TLBs are clients, then each + // socket's unit MMU; its misses export on the walker link the processor + // binds at the device. + uint32_t num_core_tlb_clients = sockets_per_cluster * VX_CFG_SOCKET_SIZE * 2; + #ifdef SIMX_SOCKET_UNIT_MMU + uint32_t num_unit_tlb_clients = sockets_per_cluster; + #else + uint32_t num_unit_tlb_clients = 0; + #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + uint32_t gfx_tlb_client = num_core_tlb_clients + num_unit_tlb_clients; + uint32_t num_tlb_clients = gfx_tlb_client + 1; + #else + uint32_t num_tlb_clients = num_core_tlb_clients + num_unit_tlb_clients; + #endif snprintf(sname, 100, "%s-l2tlb", name.c_str()); l2tlb_ = L2Tlb::Create(sname, num_tlb_clients); @@ -146,6 +198,25 @@ class Cluster::Impl { } } } + #ifdef SIMX_SOCKET_UNIT_MMU + for (uint32_t s = 0; s < sockets_per_cluster; ++s) { + auto& socket = sockets_.at(s); + socket->tlb_miss_out().bind(&l2tlb_->ReqIn.at(socket->unit_tlb_client())); + l2tlb_->RspOut.at(socket->unit_tlb_client()).bind(&socket->tlb_fill_in()); + } + #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + // Cluster unit MMU: between the rcache / ocache memory ports and the L2, + // so those caches are virtually tagged. Its TLB is the cluster TLB's last + // client. Like the socket unit MMU, each response leaves through a + // registered stage and a new miss is held a cycle before it allocates; + // the ocache writes, so a flush token is ordered behind its parks. + snprintf(sname, 100, "%s-gfx_mmu", name.c_str()); + gfx_mmu_ = Mmu::Create(sname, CMMU_NUM_REQS, VX_CFG_CTLB_SIZE, gfx_tlb_client, + false, true, VX_CFG_EXT_OM_ENABLED != 0, true); + gfx_mmu_->TlbMissOut.bind(&l2tlb_->ReqIn.at(gfx_tlb_client)); + l2tlb_->RspOut.at(gfx_tlb_client).bind(&gfx_mmu_->TlbFillIn); + #endif #endif @@ -190,8 +261,7 @@ class Cluster::Impl { // ocache memory side → l2arb. constexpr uint32_t kOmRow = 1; for (uint32_t i = 0; i < kOcacheMemPorts; ++i) { - ocache->mem_req_out.at(i).bind(&l2arb->ReqIn.at(kL2Rows * i + kOmRow)); - l2arb->RspOut.at(kL2Rows * i + kOmRow).bind(&ocache->mem_rsp_in.at(i)); + bind_gfx_port(kL2Rows * i + kOmRow, CMMU_OCACHE_IDX, ocache->mem_req_out.at(i), ocache->mem_rsp_in.at(i)); } // Per-core SFU.om_req_out (OmUnit decodes onto it) → OmCore::om_req_in[cid], @@ -254,8 +324,7 @@ class Cluster::Impl { // rcache memory side → l2arb. constexpr uint32_t kRasterRow = 1 + VX_CFG_EXT_OM_ENABLED; for (uint32_t i = 0; i < kRcacheMemPorts; ++i) { - rcache->mem_req_out.at(i).bind(&l2arb->ReqIn.at(kL2Rows * i + kRasterRow)); - l2arb->RspOut.at(kL2Rows * i + kRasterRow).bind(&rcache->mem_rsp_in.at(i)); + bind_gfx_port(kL2Rows * i + kRasterRow, CMMU_RCACHE_IDX, rcache->mem_req_out.at(i), rcache->mem_rsp_in.at(i)); } // Cluster-level RasterBus arbiter: NUM_CORES_PER_CLUSTER inputs (one per @@ -299,6 +368,7 @@ class Cluster::Impl { for (auto& gbar : gbarriers_) { gbar.reset(); } + gbar_fwd_next_ = 0; // Sockets are SimObjects; reset by SimPlatform. } @@ -313,6 +383,11 @@ class Cluster::Impl { if (l2tlb_->busy()) { return true; } + #endif + #ifdef SIMX_CLUSTER_UNIT_MMU + if (!gfx_mmu_->empty()) { + return true; + } #endif return false; } @@ -329,10 +404,7 @@ class Cluster::Impl { auto bar_index = bar_id % gbarriers_.size(); auto& gbar = gbarriers_.at(bar_index); - auto sockets_per_cluster = sockets_.size(); - auto cores_per_socket = cores_per_socket_; - - uint32_t cores_per_cluster = sockets_per_cluster * cores_per_socket; + uint32_t cores_per_cluster = sockets_.size() * cores_per_socket_; uint32_t local_core_id = core_id % cores_per_cluster; // set core arrival bit @@ -340,19 +412,45 @@ class Cluster::Impl { DT(4, "*** Global barrier arrive: cluster #" << simobject_->id() << ", core #" << core_id << " at barrier #" << bar_id << ", arrived=" << gbar.mask.count()); - if (gbar.mask.count() == (size_t)count) { - // resume all suspended cores - for (uint32_t s = 0; s < sockets_per_cluster; ++s) { - for (uint32_t c = 0; c < cores_per_socket; ++c) { - uint32_t i = s * cores_per_socket + c; - if (gbar.mask.test(i)) { - simobject_->gbar_resume_out.at(i).send({bar_id}); - } + // Up to the device unit: the socket arbiter's output buffer, then the + // cluster arbiter (one arrival per cycle) and its output buffer. A + // single-core device has no global-barrier path. + uint32_t lag = 0; + if constexpr (VX_CFG_NUM_CLUSTERS * VX_CFG_NUM_CORES > 1) { + uint64_t now = SimPlatform::instance().cycles(); + uint64_t out = std::max(now + kSocketArbLag, gbar_fwd_next_); + gbar_fwd_next_ = out + 1; + lag = out + kClusterArbLag - now; + } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + // the device arbiter's output buffer + simobject_->gbar_fwd_out.send({bar_id, count, core_id}, lag + 1); + } else if (gbar.mask.count() == (size_t)count) { + // With one cluster the device arbiter is a wire. The unit registers its + // release, which then crosses the cluster's and the socket's response + // forks. + this->global_barrier_resume(bar_id, lag + 1 + kClusterArbLag + kSocketArbLag); + } + } + + // The device unit's release crosses the cluster's and the socket's response + // forks on its way down. + void global_barrier_fwd_resume(uint32_t bar_id) { + this->global_barrier_resume(bar_id, 1 + kClusterArbLag + kSocketArbLag); + } + + // Resume the cores of this cluster that arrived, then clear the barrier. + void global_barrier_resume(uint32_t bar_id, uint32_t delay) { + auto& gbar = gbarriers_.at(bar_id % gbarriers_.size()); + for (uint32_t s = 0; s < sockets_.size(); ++s) { + for (uint32_t c = 0; c < cores_per_socket_; ++c) { + uint32_t i = s * cores_per_socket_ + c; + if (gbar.mask.test(i)) { + simobject_->gbar_resume_out.at(i).send({bar_id}, delay); } } - // reset mask and advance phase - gbar.mask.reset(); } + gbar.mask.reset(); } Cluster::PerfStats perf_stats() const { @@ -399,7 +497,22 @@ class Cluster::Impl { for (uint32_t c = 0; c < cores_per_socket_; ++c) { socket->core(c)->set_satp(value); } - } + #ifdef SIMX_SOCKET_UNIT_MMU + socket->set_satp(value); + #endif + } + #ifdef SIMX_CLUSTER_UNIT_MMU + // The rcache and ocache are virtually tagged: a satp commit flushes the + // cluster unit TLB and invalidates them. The ocache writes through, and a + // commit falls between launches, so it holds no write to drain first. + gfx_mmu_->set_satp(value); + #ifdef VX_CFG_EXT_RASTER_ENABLE + rcache_->invalidate(); + #endif + #ifdef VX_CFG_EXT_OM_ENABLE + ocache_->invalidate(); + #endif + #endif } SimChannel& ptw_req_out() { @@ -475,9 +588,9 @@ class Cluster::Impl { return 0; } - void dcache_flush_begin() { + void dcache_flush_begin(bool l1_only) { for (auto& socket : sockets_) { - socket->dcache_flush_begin(); + socket->dcache_flush_begin(l1_only); } } @@ -539,8 +652,20 @@ class Cluster::Impl { } #endif - void l2_flush_begin() { - l2cache_->flush_begin(); + void l2_flush_begin(bool l1_only) { + l2cache_->flush_begin(l1_only); + } + + void l2_keep_lines() { + l2cache_->keep_lines(); + } + + SimChannel& cp_mem_req_in() { + return l2cache_->core_req_in.at(L2_CP_IDX); + } + + SimChannel& cp_mem_rsp_out() { + return l2cache_->core_rsp_out.at(L2_CP_IDX); } bool l2_flush_done() const { @@ -563,10 +688,14 @@ class Cluster::Impl { Cluster* simobject_; std::vector sockets_; std::vector gbarriers_; + uint64_t gbar_fwd_next_ = 0; // next free cycle at the cluster's gbar arbiter Cache::Ptr l2cache_; uint32_t cores_per_socket_; #ifdef VX_CFG_VM_ENABLE L2Tlb::Ptr l2tlb_; +#ifdef SIMX_CLUSTER_UNIT_MMU + Mmu::Ptr gfx_mmu_; +#endif #endif #ifdef VX_CFG_EXT_OM_ENABLE OmCore::Ptr om_core_; @@ -593,11 +722,14 @@ Cluster::Cluster(const SimContext& ctx, , mem_rsp_in(VX_CFG_L2_MEM_PORTS, this) , gbar_arrive_in(this) , gbar_resume_out(NUM_SOCKETS * VX_CFG_SOCKET_SIZE, this) + , gbar_fwd_out(this) + , gbar_fwd_resume_in(this) , cluster_id_(cluster_id) , processor_(processor) , impl_(new Impl(this)) { gbar_arrive_in.bind(this, &Cluster::on_gbar_arrive); + gbar_fwd_resume_in.bind(this, &Cluster::on_gbar_fwd_resume); } Cluster::~Cluster() { @@ -620,6 +752,10 @@ void Cluster::on_gbar_arrive(const GbarArrive& msg) { impl_->global_barrier_arrive(msg.bar_id, msg.count, msg.core_id); } +void Cluster::on_gbar_fwd_resume(const GbarResume& msg) { + impl_->global_barrier_fwd_resume(msg.bar_id); +} + Cluster::PerfStats Cluster::perf_stats() const { return impl_->perf_stats(); } @@ -658,8 +794,8 @@ Core* Cluster::get_core(uint32_t idx) const { return impl_->get_core(idx); } -void Cluster::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Cluster::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Cluster::dcache_flush_done() const { @@ -694,8 +830,20 @@ void Cluster::rtcache_flush_begin() { impl_->rtcache_flush_begin(); } bool Cluster::rtcache_flush_done() const { return impl_->rtcache_flush_done(); } #endif -void Cluster::l2_flush_begin() { - impl_->l2_flush_begin(); +void Cluster::l2_flush_begin(bool l1_only) { + impl_->l2_flush_begin(l1_only); +} + +void Cluster::l2_keep_lines() { + impl_->l2_keep_lines(); +} + +SimChannel& Cluster::cp_mem_req_in() { + return impl_->cp_mem_req_in(); +} + +SimChannel& Cluster::cp_mem_rsp_out() { + return impl_->cp_mem_rsp_out(); } bool Cluster::l2_flush_done() const { diff --git a/sim/simx/cluster.h b/sim/simx/cluster.h index 2ff60836b4..e47453af1d 100644 --- a/sim/simx/cluster.h +++ b/sim/simx/cluster.h @@ -77,6 +77,11 @@ class Cluster : public SimObject { SimEventLink gbar_arrive_in; std::vector> gbar_resume_out; + // With more than one cluster the barrier spans clusters: arrivals go up to + // the device-level GbarUnit and its resume comes back down here. + SimEventLink gbar_fwd_out; + SimEventLink gbar_fwd_resume_in; + Cluster(const SimContext& ctx, const char* name, uint32_t cluster_id, @@ -117,7 +122,7 @@ class Cluster : public SimObject { // → L2 → L3) to avoid downstream evictions racing the next-level walk. // L1 fanout: dcache + icache + socket-local {tcache, rtcache} (forwarded // to the sockets) + cluster-local {rcache, ocache}. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; @@ -137,7 +142,14 @@ class Cluster : public SimObject { void rtcache_flush_begin(); bool rtcache_flush_done() const; #endif - void l2_flush_begin(); + void l2_flush_begin(bool l1_only); + // The L2, when it is the shared last-level cache, keeps its lines into the + // next launch. + void l2_keep_lines(); + // The command processor DMA's client slot on this cluster's L2, valid only + // under CP_ON_L2 (the L2 is the LLC). + SimChannel& cp_mem_req_in(); + SimChannel& cp_mem_rsp_out(); bool l2_flush_done() const; #ifdef VX_CFG_EXT_RASTER_ENABLE @@ -151,6 +163,7 @@ class Cluster : public SimObject { private: void on_gbar_arrive(const GbarArrive& msg); + void on_gbar_fwd_resume(const GbarResume& msg); uint32_t cluster_id_; ProcessorImpl* processor_; diff --git a/sim/simx/constants.h b/sim/simx/constants.h index 2ef860870a..ea6b196cfe 100644 --- a/sim/simx/constants.h +++ b/sim/simx/constants.h @@ -39,6 +39,32 @@ namespace vortex { inline constexpr uint32_t XLENB = (VX_CFG_XLEN / 8); +// A last-level cache that alone serves every memory client is the point of +// coherence between kernels. A dcache is one when it is the only dcache, +// shared by every core of the one socket. The texture, ray-tracing and DXA +// ports join below the dcache and the raster and OM caches beside it, so a +// dcache with any of them is not one. +inline constexpr bool DCACHE_SHARED_LLC = (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) + && (VX_CFG_NUM_CLUSTERS == 1) && (VX_CFG_NUM_CORES == VX_CFG_SOCKET_SIZE) + && (VX_CFG_NUM_DCACHES == 1) + && !VX_CFG_EXT_TEX_ENABLED && !VX_CFG_EXT_RTU_ENABLED + && !VX_CFG_EXT_RASTER_ENABLED && !VX_CFG_EXT_OM_ENABLED + && !VX_CFG_EXT_DXA_ENABLED; +inline constexpr bool L2_SHARED_LLC = (VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && (VX_CFG_NUM_CLUSTERS == 1); +inline constexpr bool L3_SHARED_LLC = (VX_CFG_L3_ENABLED != 0); +// An acquire must also invalidate the cache below the dcache unless that one +// is the point of coherence (an L3 always is; memory always is). +inline constexpr bool DCACHE_ACQ_FORWARD = (VX_CFG_L2_ENABLED != 0) && !L2_SHARED_LLC; + +// A write-back LLC must be the only instance: replicated ones never make their +// dirty lines visible to each other. +static_assert(!((VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_DCACHE_WRITEBACK) + || ((VX_CFG_NUM_CORES == 1) && (VX_CFG_NUM_CLUSTERS == 1)), + "a write-back dcache that is the last-level cache must be the only dcache"); +static_assert(!((VX_CFG_L2_ENABLED != 0) && (VX_CFG_L3_ENABLED == 0) && VX_CFG_L2_WRITEBACK) + || (VX_CFG_NUM_CLUSTERS == 1), + "a write-back L2 that is the last-level cache must be the only L2"); + inline constexpr uint32_t MAX_NUM_CORES = 1024; inline constexpr uint32_t MAX_NUM_WARPS = 64; inline constexpr uint32_t MAX_NUM_REGS = 32; @@ -60,6 +86,27 @@ inline constexpr uint32_t DCACHE_CHANNELS = __UP((VX_CFG_NUM_LSU_LANES * XLENB) inline constexpr uint32_t VX_CFG_DCACHE_NUM_REQS = (VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); inline constexpr uint32_t NUM_SOCKETS = __UP(VX_CFG_NUM_CORES / VX_CFG_SOCKET_SIZE); +static_assert((VX_CFG_NUM_CORES % VX_CFG_SOCKET_SIZE) == 0, + "NUM_CORES must be a multiple of SOCKET_SIZE"); + +// Under VM, a socket's unit MMU translates the memory ports of its +// socket-resident units (tcache and rtcache misses, DXA gmem reads), in socket +// arb order. +#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_TEX_ENABLE) || defined(VX_CFG_EXT_RTU_ENABLE) || defined(VX_CFG_EXT_DXA_ENABLE)) +#define SIMX_SOCKET_UNIT_MMU +#endif +inline constexpr uint32_t UMMU_TCACHE_IDX = 0; +inline constexpr uint32_t UMMU_RTCACHE_IDX = UMMU_TCACHE_IDX + VX_CFG_EXT_TEX_ENABLED; +inline constexpr uint32_t UMMU_DXA_IDX = UMMU_RTCACHE_IDX + VX_CFG_EXT_RTU_ENABLED; +inline constexpr uint32_t UMMU_NUM_REQS = VX_CFG_EXT_TEX_ENABLED + VX_CFG_EXT_RTU_ENABLED + VX_CFG_EXT_DXA_ENABLED; + +// Under VM, the cluster unit MMU translates the rcache and ocache memory ports. +#if defined(VX_CFG_VM_ENABLE) && (defined(VX_CFG_EXT_RASTER_ENABLE) || defined(VX_CFG_EXT_OM_ENABLE)) +#define SIMX_CLUSTER_UNIT_MMU +#endif +inline constexpr uint32_t CMMU_RCACHE_IDX = 0; +inline constexpr uint32_t CMMU_OCACHE_IDX = CMMU_RCACHE_IDX + VX_CFG_EXT_RASTER_ENABLED; +inline constexpr uint32_t CMMU_NUM_REQS = VX_CFG_EXT_RASTER_ENABLED + VX_CFG_EXT_OM_ENABLED; // Demand clients of the cluster L2 (sockets and gfx caches), excluding the // walker's PTE-fetch slot. @@ -81,7 +128,31 @@ inline constexpr uint32_t VX_CFG_L2_NUM_REQS = VX_CFG_L2_DEMAND_REQS; inline constexpr uint32_t VX_CFG_L3_NUM_REQS = VX_CFG_NUM_CLUSTERS * VX_CFG_L2_MEM_PORTS; #endif +// With VX_CFG_CP_ON_LLC the command processor's DMA is one more client of the +// shared last-level cache: the cluster L2 when it is the LLC of one cluster, +// the L3 otherwise. The slot does not size the banks. +inline constexpr uint32_t CP_ON_L2 = ((VX_CFG_CP_ON_LLC != 0) + && (VX_CFG_NUM_CLUSTERS == 1) + && (VX_CFG_L2_ENABLED != 0) + && (VX_CFG_L3_ENABLED == 0)) ? 1 : 0; +inline constexpr uint32_t CP_ON_L3 = ((VX_CFG_CP_ON_LLC != 0) && (VX_CFG_L3_ENABLED != 0)) ? 1 : 0; +inline constexpr uint32_t L2_CP_IDX = VX_CFG_L2_NUM_REQS; +inline constexpr uint32_t L3_CP_IDX = VX_CFG_L3_NUM_REQS; +inline constexpr uint32_t L2_CACHE_REQS = VX_CFG_L2_NUM_REQS + CP_ON_L2; +inline constexpr uint32_t L3_CACHE_REQS = VX_CFG_L3_NUM_REQS + CP_ON_L3; +// one last-level cache word per DMA request +inline constexpr uint32_t CP_MEM_DATA_SIZE = (CP_ON_L2 != 0) ? VX_CFG_L1_LINE_SIZE : VX_CFG_L2_SECTOR_SIZE; + inline constexpr uint32_t PER_ISSUE_WARPS = VX_CFG_NUM_WARPS / VX_CFG_ISSUE_WIDTH; inline constexpr uint32_t ISSUE_WIS_BITS = log2ceil(PER_ISSUE_WARPS); +// Frontend slice f owns the warps with wid % VX_CFG_FETCH_WIDTH == f; each has its +// own warp scheduler, fetch unit, decoder and icache request port. +inline constexpr uint32_t PER_FETCH_WARPS = VX_CFG_NUM_WARPS / VX_CFG_FETCH_WIDTH; +inline constexpr uint32_t ICACHE_NUM_REQS = VX_CFG_FETCH_WIDTH; +static_assert(VX_CFG_FETCH_WIDTH != 0 && (VX_CFG_FETCH_WIDTH & (VX_CFG_FETCH_WIDTH - 1)) == 0 + && VX_CFG_FETCH_WIDTH <= VX_CFG_NUM_WARPS + && (VX_CFG_ISSUE_WIDTH % VX_CFG_FETCH_WIDTH) == 0, + "FETCH_WIDTH must be a power of two that divides ISSUE_WIDTH and is at most NUM_WARPS"); + } // namespace vortex diff --git a/sim/simx/core.cpp b/sim/simx/core.cpp index ec729d83e3..d1d1baaf5c 100644 --- a/sim/simx/core.cpp +++ b/sim/simx/core.cpp @@ -54,6 +54,21 @@ using namespace vortex; +// Units whose output channel already holds every register ahead of the commit +// arbiter except the lane gather's. +static bool commit_single_hop(uint32_t fu) { + switch ((FUType)fu) { + case FUType::LSU: + case FUType::SFU: +#ifdef VX_CFG_EXT_TCU_ENABLE + case FUType::TCU: +#endif + return true; + default: + return false; + } +} + class Core::Impl { public: Impl(const SimContext& ctx, Core* simobject) @@ -66,21 +81,30 @@ class Core::Impl { , func_units_((uint32_t)FUType::Count) , lmem_switch_(VX_CFG_NUM_LSU_BLOCKS) , mem_coalescers_(VX_CFG_NUM_LSU_BLOCKS) - , fetch_latch_(ctx, "fetch_latch", 2, 2) - , decode_latch_(ctx, "decode_latch", 1, 2) - , pending_icache_(VX_CFG_NUM_WARPS) , ibuffer_arbs_(VX_CFG_ISSUE_WIDTH, {ArbiterType::GreedyRR, PER_ISSUE_WARPS}) , issue_lock_owner_(VX_CFG_ISSUE_WIDTH, kLockOpen) , fu_credits_(VX_CFG_ISSUE_WIDTH, std::vector((uint32_t)FUType::Count, 0)) , fu_full_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , fu_full_seen_(VX_CFG_ISSUE_WIDTH, BitVector<>((uint32_t)FUType::Count, 0)) , ibuf_inflight_(VX_CFG_NUM_WARPS, 0) + , ibuf_full_r_(VX_CFG_NUM_WARPS, false) , last_issue_(VX_CFG_NUM_WARPS, 0) , staged_since_(VX_CFG_NUM_WARPS, 0) + , head_staged_(VX_CFG_NUM_WARPS, false) { const std::string& name = simobject_->name(); char sname[100]; + // per frontend slice: scheduler-to-fetch and fetch-to-decode latches, and the + // icache tags of the slice's in-flight fetches + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "fetch_latch%d", f); + fetch_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); + snprintf(sname, 100, "decode_latch%d", f); + decode_latch_.emplace_back(std::make_unique>(ctx, sname, 1, 2)); + pending_icache_.emplace_back(PER_FETCH_WARPS); + } + // create scheduler (SimObject) — also creates the CTA dispatcher. snprintf(sname, 100, "%s-scheduler", name.c_str()); scheduler_ = SimPlatform::instance().create_object(sname, simobject_); @@ -92,8 +116,10 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE // create RVC fetch decompressor (SimObject) — owns per-warp half-word // buffer + refetch queue. Resets via SimObject::on_reset. - snprintf(sname, 100, "%s-decompressor", name.c_str()); - decompressor_ = SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS); + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + snprintf(sname, 100, "%s-decompressor%d", name.c_str(), f); + decompressors_.push_back(SimPlatform::instance().create_object(sname, VX_CFG_NUM_WARPS)); + } #endif // create scoreboard (SimObject) @@ -115,25 +141,44 @@ class Core::Impl { // create ibuffers for (uint32_t i = 0; i < ibuffers_.size(); ++i) { snprintf(sname, 100, "%s-ibuffer%d", name.c_str(), i); - ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE); + // IBUF_SIZE entries plus the head, which stands for the warp's staging + // register once it has moved there + ibuffers_.at(i) = TFifo::Create(sname, 1, VX_CFG_IBUF_SIZE + 1); } - // create the memory coalescer + // The LSU ticks ahead of the coalescer, the lane adapters and the lmem + // switch. The switch routes a request into its output buffers in the cycle + // the LSU hands it over, as the RTL's combinational split does. The + // switch's one-entry response buffer is drained by the LSU before the + // coalescer or an adapter refills it, its ready being combinational in the + // RTL. + snprintf(sname, 100, "%s-lsu", name.c_str()); + func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); + + // create the memory coalescer, or the lane adapter that replaces it. + const bool dcache_coalescing = (VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE); + std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { snprintf(sname, 100, "%s-coalescer%d", name.c_str(), b); - mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 1); + // a batch crosses the coalescer's output register, then the lane + // adapter's request buffer, on its way to the data cache + mem_coalescers_.at(b) = MemCoalescer::Create(sname, LSU_CHANNELS, DCACHE_CHANNELS, DCACHE_WORD_SIZE, LSU_QUEUE_OUT_SIZE, 2); + if (!dcache_coalescing) { + snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); + lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1, 1); + } } - // create local memory. + // create local memory: it takes each LSU block's warp requests whole. // The DXA writes and the TCU tile-buffer reads share the row-wide DMA - // port, DXA first. + // port, round-robin. snprintf(sname, 100, "%s-lmem", name.c_str()); local_mem_ = LocalMem::Create(sname, LocalMem::Config{ (1 << VX_CFG_LMEM_LOG_SIZE), LSU_WORD_SIZE, - LSU_NUM_REQS, + VX_CFG_NUM_LSU_BLOCKS, + LSU_CHANNELS, log2ceil(VX_CFG_LMEM_NUM_BANKS), - false, VX_CFG_EXT_DXA_ENABLED + VX_CFG_TCU_WGMMA_ENABLED }); @@ -143,62 +188,60 @@ class Core::Impl { lmem_switch_.at(b) = LocalMemSwitch::Create(sname, 1); } - // create per-block lmem adapters - std::vector lsu_lmem_adapter(VX_CFG_NUM_LSU_BLOCKS); - for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - snprintf(sname, 100, "%s-lsu_lmem_adapter%d", name.c_str(), b); - lsu_lmem_adapter.at(b) = LsuMemAdapter::Create(sname, LSU_CHANNELS, 1); - } - - // connect lmem switch to per-block adapters - for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - lmem_switch_.at(b)->ReqOutLmem.bind(&lsu_lmem_adapter.at(b)->ReqIn); - lsu_lmem_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInLmem); - } - - // connect per-block lmem adapters to local memory ports + // connect lmem switch to local memory for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - for (uint32_t c = 0; c < LSU_CHANNELS; ++c) { - lsu_lmem_adapter.at(b)->ReqOut.at(c).bind(&local_mem_->Inputs.at(b * LSU_CHANNELS + c)); - local_mem_->Outputs.at(b * LSU_CHANNELS + c).bind(&lsu_lmem_adapter.at(b)->RspIn.at(c)); - } + lmem_switch_.at(b)->ReqOutLmem.bind(&local_mem_->ReqIn.at(b)); + local_mem_->RspOut.at(b).bind(&lmem_switch_.at(b)->RspInLmem); } // Per-channel request/response endpoints facing the dcache ports. std::vector*> dc_req_out(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); std::vector*> dc_rsp_in(VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS); - // The first data-cache port also carries the cache-flush injection, which - // registers its requests once more. - if ((VX_CFG_NUM_LSU_LANES > 1) && (DCACHE_WORD_SIZE > LSU_WORD_SIZE)) { + if (dcache_coalescing) { // connect memory coalescer; its memory side drives the dcache // channels directly (combinational lane fan-out/fan-in). for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { lmem_switch_.at(b)->ReqOutDC.bind(&mem_coalescers_.at(b)->ReqIn); - mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // the coalescer's response reaches the switch's response buffer + // combinationally, ahead of a local-memory response + mem_coalescers_.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &mem_coalescers_.at(b)->RspIn.at(c); } } - mem_coalescers_.at(0)->set_port_delay(0, 1); } else { // bypass memory coalescer: per-block lane adapter (channel-fused // pass-through when DCACHE_CHANNELS == 1) - std::vector lsu_dcache_adapter(VX_CFG_NUM_LSU_BLOCKS); for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - snprintf(sname, 100, "%s-lsu_dcache_adapter%d", name.c_str(), b); - lsu_dcache_adapter.at(b) = LsuMemAdapter::Create(sname, DCACHE_CHANNELS, 1); lmem_switch_.at(b)->ReqOutDC.bind(&lsu_dcache_adapter.at(b)->ReqIn); - lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + // A multi-channel adapter packs its response combinationally into the + // switch's response buffer; a single-channel one passes the cache's + // own output buffer through, which the switch then drains. + if (DCACHE_CHANNELS > 1) { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspOut); + } else { + lsu_dcache_adapter.at(b)->RspOut.bind(&lmem_switch_.at(b)->RspInDC); + } for (uint32_t c = 0; c < DCACHE_CHANNELS; ++c) { dc_req_out.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->ReqOut.at(c); dc_rsp_in.at(b * DCACHE_CHANNELS + c) = &lsu_dcache_adapter.at(b)->RspIn.at(c); } } - lsu_dcache_adapter.at(0)->set_port_delay(0, 1); } + // The first data-cache port also carries the cache-flush injection, whose + // arbiter buffers and registers its requests once more. + snprintf(sname, 100, "%s-dcache_flush_buf", name.c_str()); + dcache_flush_buf_ = TxBuffer::Create(sname, 2); + dc_req_out.at(0)->bind(&dcache_flush_buf_->In); + dc_req_out.at(0) = &dcache_flush_buf_->Out; + + // The first icache port does the same for the icache. + snprintf(sname, 100, "%s-icache_flush_buf", name.c_str()); + icache_flush_buf_ = TxBuffer::Create(sname, 2); + #ifdef VX_CFG_VM_ENABLE // Per-core L1 TLB stages: hits translate in place, misses park in // the miss station and resolve through the shared cluster TLB/PTW @@ -209,11 +252,11 @@ class Core::Impl { dcache_mmu_ = Mmu::Create(sname, VX_CFG_DCACHE_NUM_REQS, VX_CFG_DTLB_SIZE, local_core * 2 + 0, false); - // Per-core icache MMU (1 port). Fetch reads/writes its upstream - // channels (ReqIn[0]/RspOut[0]) directly; the downstream side is - // bound to the Core's icache port below. + // Per-core icache MMU, one port per frontend slice. Fetch reads/writes its + // upstream channels directly; the downstream side is bound to the Core's + // icache ports below. snprintf(sname, 100, "%s-icache_mmu", name.c_str()); - icache_mmu_ = Mmu::Create(sname, 1, + icache_mmu_ = Mmu::Create(sname, ICACHE_NUM_REQS, VX_CFG_ITLB_SIZE, local_core * 2 + 1, true); // LSU memory side -> dcache MMU -> core's dcache port @@ -224,29 +267,44 @@ class Core::Impl { simobject_->dcache_rsp_in.at(p).bind(&dcache_mmu_->RspIn.at(p)); } - // icache MMU downstream side -> core's icache port. Upstream side - // (ReqIn[0]/RspOut[0]) is consumed directly by fetch() in on_tick. - icache_mmu_->ReqOut.at(0).bind(&simobject_->icache_req_out.at(0)); - simobject_->icache_rsp_in.at(0).bind(&icache_mmu_->RspIn.at(0)); + // icache MMU downstream side -> core's icache ports. Upstream side is + // consumed directly by fetch() in on_tick. + for (uint32_t f = 0; f < ICACHE_NUM_REQS; ++f) { + icache_mmu_->ReqOut.at(f).bind(&simobject_->icache_req_out.at(f)); + simobject_->icache_rsp_in.at(f).bind(&icache_mmu_->RspIn.at(f)); + } + icache_flush_buf_->Out.bind(&icache_mmu_->ReqIn.at(0)); #else // No-VM: direct passthrough. for (uint32_t p = 0; p < VX_CFG_NUM_LSU_BLOCKS * DCACHE_CHANNELS; ++p) { dc_req_out.at(p)->bind(&simobject_->dcache_req_out.at(p)); simobject_->dcache_rsp_in.at(p).bind(dc_rsp_in.at(p)); } + icache_flush_buf_->Out.bind(&simobject_->icache_req_out.at(0)); #endif - // initialize execute units - snprintf(sname, 100, "%s-alu", name.c_str()); - func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); - snprintf(sname, 100, "%s-fpu", name.c_str()); - func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); - snprintf(sname, 100, "%s-lsu", name.c_str()); - func_units_.at((int)FUType::LSU) = SimPlatform::instance().create_object(sname, simobject_); + // initialize execute units. A full-bandwidth ALU or FPU takes its op + // straight off the dispatch queue, in the cycle the queue releases it, so + // it is created (and ticks) after its dispatcher; the others take their + // input before the dispatcher fills it in the same cycle. + auto create_alu = [&]() { + snprintf(sname, 100, "%s-alu", name.c_str()); + func_units_.at((int)FUType::ALU) = SimPlatform::instance().create_object(sname, simobject_); + }; + auto create_fpu = [&]() { + snprintf(sname, 100, "%s-fpu", name.c_str()); + func_units_.at((int)FUType::FPU) = SimPlatform::instance().create_object(sname, simobject_); + }; + if (AluUnit::kGather) { + create_alu(); + } + if (FpuUnit::kGather) { + create_fpu(); + } snprintf(sname, 100, "%s-sfu", name.c_str()); func_units_.at((int)FUType::SFU) = SimPlatform::instance().create_object(sname, simobject_); - std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); std::static_pointer_cast(func_units_.at((int)FUType::SFU))->warp_ctl_out.bind(&simobject_->warp_ctl_in); + scheduler_->barrier_unit().unlock_out.bind(&simobject_->bar_unlock_in); #ifdef VX_CFG_EXT_TCU_ENABLE snprintf(sname, 100, "%s-tcu", name.c_str()); tcu_unit_ = SimPlatform::instance().create_object(sname, simobject_); @@ -254,7 +312,7 @@ class Core::Impl { #ifdef VX_CFG_TCU_WGMMA_ENABLE // Bind the TCU tile-buffer subsystem (TcuTbuf) to the LMEM DMA port, - // after the DXA. + // as the client after the DXA. { auto& tbuf = tcu_unit_->tbuf(); uint32_t base = VX_CFG_EXT_DXA_ENABLED * LocalMem::DMA_PORTS; @@ -265,21 +323,24 @@ class Core::Impl { } #endif #ifdef TCU_META_ENABLE - // Bind the TCU metadata AGU to the LSU block-0 client port. + // Bind the TCU metadata AGU ports to the LSU client ports. { auto lsu = std::static_pointer_cast(func_units_.at((int)FUType::LSU)); - tcu_unit_->agu_req_out.bind(&lsu->TcuReqIn); - lsu->TcuRspOut.bind(&tcu_unit_->agu_rsp_in); + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + tcu_unit_->agu_req_out.at(p).bind(&lsu->TcuReqIn.at(p)); + lsu->TcuRspOut.at(p).bind(&tcu_unit_->agu_rsp_in.at(p)); + } } #endif #endif - // initialize dispatchers after the units they feed, so a unit takes its - // input before the dispatcher fills it in the same cycle. + // initialize dispatchers after the buffered units they feed, so such a + // unit takes its input before the dispatcher fills it in the same cycle. { - // A reduced-width ALU or FPU registers the op once more on its way in. - uint32_t alu_dly = AluUnit::kGather ? 2 : 1; - uint32_t fpu_dly = FpuUnit::kGather ? 2 : 1; + // A reduced-width ALU or FPU takes the op through its lane-dispatch + // buffer; a full-bandwidth one takes it as the queue releases it. + uint32_t alu_dly = AluUnit::kGather ? 2 : 0; + uint32_t fpu_dly = FpuUnit::kGather ? 2 : 0; dispatchers_.at((int)FUType::ALU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_ALU_BLOCKS, VX_CFG_NUM_ALU_LANES, alu_dly); dispatchers_.at((int)FUType::FPU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_FPU_BLOCKS, VX_CFG_NUM_FPU_LANES, fpu_dly); dispatchers_.at((int)FUType::LSU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_LSU_BLOCKS, VX_CFG_NUM_LSU_LANES, 1); @@ -287,6 +348,19 @@ class Core::Impl { #ifdef VX_CFG_EXT_TCU_ENABLE dispatchers_.at((int)FUType::TCU) = SimPlatform::instance().create_object(name.c_str(), simobject_, VX_CFG_DISPATCH_QUEUE_SIZE, VX_CFG_NUM_TCU_BLOCKS, VX_CFG_NUM_TCU_LANES, 1); #endif + if (!AluUnit::kGather) { + create_alu(); + auto& dispatch = dispatchers_.at((int)FUType::ALU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + if (!FpuUnit::kGather) { + create_fpu(); + auto& dispatch = dispatchers_.at((int)FUType::FPU); + dispatch->set_release_on_accept(); + std::static_pointer_cast(func_units_.at((int)FUType::FPU))->ReleaseOut.bind(&dispatch->ReleaseOut); + } + std::static_pointer_cast(func_units_.at((int)FUType::ALU))->branch_ctl_out.bind(&simobject_->branch_ctl_in); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { auto& dispatch = dispatchers_.at(fu); auto& func_unit = func_units_.at(fu); @@ -304,9 +378,11 @@ class Core::Impl { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { auto& queues = commit_queues_.emplace_back(); for (uint32_t fu = 0; fu < (uint32_t)FUType::Count; ++fu) { - // Covers the results still crossing into the queue, so a unit can - // retire one result per cycle. - queues.emplace_back(std::make_unique>(simobject_, 3)); + // A unit's lane gather holds two results. The LSU's, the SFU's and the + // TCU's cross one register into it; the others cross two and take a + // third slot so that they still retire one result per cycle. + uint32_t capacity = commit_single_hop(fu) ? 2 : 3; + queues.emplace_back(std::make_unique>(simobject_, capacity)); } } @@ -327,14 +403,16 @@ class Core::Impl { std::fill(issue_lock_owner_.begin(), issue_lock_owner_.end(), kLockOpen); std::fill(last_issue_.begin(), last_issue_.end(), 0); std::fill(staged_since_.begin(), staged_since_.end(), 0); + std::fill(head_staged_.begin(), head_staged_.end(), false); pending_instrs_.clear(); pending_ifetches_ = 0; std::fill(ibuf_inflight_.begin(), ibuf_inflight_.end(), 0); + std::fill(ibuf_full_r_.begin(), ibuf_full_r_.end(), false); - // perf_stats_ deliberately survives: these are the MPM counters, which are - // free-running from device reset. A kernel launch re-enters reset() here but + // perf_stats_ deliberately survives: these are the MPM counters, which + // accumulate from device reset. A kernel launch re-enters reset() here but // does not reset the hardware, so clearing them would make each dump report // only the last launch -- an app that dumps between launches would then see // per-launch counts where the device reports a running total. @@ -349,7 +427,13 @@ class Core::Impl { this->fetch(); this->schedule(); - ++perf_stats_.cycles; + // The cycle counter advances only while the core has work -- a CTA + // launching, a warp active, or an instruction in flight -- so cycles it + // spends idle before its first CTA, after its last warp, or while the host + // moves data are not charged to it. + if (scheduler_->running() || !pending_instrs_.empty()) { + ++perf_stats_.cycles; + } DPN(2, std::flush); } @@ -360,61 +444,102 @@ class Core::Impl { perf_stats_.active_warps += scheduler_->active_warps().count(); perf_stats_.stalled_warps += scheduler_->stalled_warps().count(); - // stop when fetch_latch cannot accept - if (fetch_latch_.full()) - return; + scheduler_->schedule_begin(); - // eligible warps: ibuffer not yet saturated - WarpMask warp_mask; - for (uint32_t w = 0, nw = VX_CFG_NUM_WARPS; w < nw; ++w) { - if (ibuf_inflight_.at(w) < VX_CFG_IBUF_SIZE) { - warp_mask.set(w); - } + // Each frontend slice picks among its own warps whose ibuffer is not full; a + // buffer is full at exactly IBUF_SIZE entries, a flag registered at the end + // of the previous cycle. With an L1, once every warp's buffer is full any + // ready warp may fetch, its instruction waiting at decode. + bool all_full = (VX_CFG_ICACHE_ENABLED || VX_CFG_DCACHE_ENABLED); + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS && all_full; ++w) { + all_full = ibuf_full_r_.at(w); } + // the cycle is idle when no slice has a warp to fetch, its latch free or not + bool any_cand = false; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + auto& fetch_latch = fetch_latch_.at(f); + WarpMask warp_mask; + for (uint32_t w = f; w < VX_CFG_NUM_WARPS; w += VX_CFG_FETCH_WIDTH) { + if (all_full || !ibuf_full_r_.at(w)) { + warp_mask.set(w); + } + } + + if (fetch_latch->full()) { + any_cand |= scheduler_->has_ready(warp_mask); + continue; + } - auto trace = scheduler_->schedule(warp_mask); - if (trace == nullptr) { + auto trace = scheduler_->schedule(warp_mask); + if (trace == nullptr) { + continue; + } + any_cand = true; + + fetch_latch->push(trace); + DT(3, simobject_->name() << "-pipeline schedule: " << *trace); + pending_instrs_.push_back(trace); + ++ibuf_inflight_.at(trace->wid); + perf_stats_.issued_warps += 1; + perf_stats_.issued_threads += trace->tmask.count(); + } + if (!any_cand) { ++perf_stats_.sched_idle; - return; + } + for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { + ibuf_full_r_.at(w) = (ibuf_inflight_.at(w) == VX_CFG_IBUF_SIZE); } - fetch_latch_.push(trace); - DT(3, simobject_->name() << "-pipeline schedule: " << *trace); - pending_instrs_.push_back(trace); - ++ibuf_inflight_.at(trace->wid); - perf_stats_.issued_warps += 1; - perf_stats_.issued_threads += trace->tmask.count(); + scheduler_->schedule_end(); } void fetch() { perf_stats_.ifetch_latency += pending_ifetches_; + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->fetch_slice(f); + } + } + + // Frontend slice f: its icache response into its decode latch, then its next + // fetch request out of its fetch latch, on icache port f. + void fetch_slice(uint32_t f) { + auto& fetch_latch = *fetch_latch_.at(f); + auto& decode_latch = *decode_latch_.at(f); + auto& pending_icache = pending_icache_.at(f); + #ifdef VX_CFG_EXT_C_ENABLE + auto& decompressor = decompressors_.at(f); + #endif #ifdef VX_CFG_VM_ENABLE // With VM, fetch reads/writes the icache MMU's upstream channels. // The MMU translates and forwards to simobject_->icache_req_out // downstream (driven by SimChannel binding in the constructor). - auto& icache_rsp = icache_mmu_->RspOut.at(0); - auto& icache_req = icache_mmu_->ReqIn.at(0); + auto& icache_rsp = icache_mmu_->RspOut.at(f); + auto& icache_port = icache_mmu_->ReqIn.at(f); #else - auto& icache_rsp = simobject_->icache_rsp_in.at(0); - auto& icache_req = simobject_->icache_req_out.at(0); + auto& icache_rsp = simobject_->icache_rsp_in.at(f); + auto& icache_port = simobject_->icache_req_out.at(f); #endif + auto& icache_req = (f == 0) ? icache_flush_buf_->In : icache_port; // handle icache response — extract instruction word from the TLM line payload if (!icache_rsp.empty()){ auto& mem_rsp = icache_rsp.peek(); - auto trace = pending_icache_.at(mem_rsp.tag); + auto trace = pending_icache.at(mem_rsp.tag); assert(mem_rsp.data && "icache response must carry line payload"); #ifdef VX_CFG_EXT_C_ENABLE // Decompressor classifies the rsp: emit-ready (sets trace->code) or // queues a refetch internally for cross-word 32-bit completion. - bool emit = decompressor_->on_icache_rsp(trace, *mem_rsp.data); - bool progressed = !emit || decode_latch_.try_push(trace); + bool emit = decompressor->on_icache_rsp(trace, *mem_rsp.data); + bool progressed = !emit || decode_latch.try_push(trace); + if (emit && progressed) { + decompressor->on_emit(trace); + } #else bool emit = true; bool progressed = false; - if (decode_latch_.try_push(trace)) { + if (decode_latch.try_push(trace)) { uint32_t offset = trace->PC & (VX_CFG_MEM_BLOCK_SIZE - 1); std::memcpy(&trace->code, mem_rsp.data->data() + offset, sizeof(uint32_t)); progressed = true; @@ -427,7 +552,7 @@ class Core::Impl { << ", PC=0x" << std::hex << trace->PC << " (#" << std::dec << trace->uuid << ")"); } DT(3, simobject_->name() << " icache-rsp: addr=0x" << std::hex << trace->PC << ", tag=0x" << mem_rsp.tag << std::dec << ", " << *trace); - pending_icache_.release(mem_rsp.tag); + pending_icache.release(mem_rsp.tag); icache_rsp.pop(); --pending_ifetches_; } @@ -440,15 +565,15 @@ class Core::Impl { #ifdef VX_CFG_EXT_C_ENABLE bool from_refetch = false; { - auto pick = decompressor_->pick_request(fetch_latch_.empty() ? nullptr - : fetch_latch_.peek()); + auto pick = decompressor->pick_request(fetch_latch.empty() ? nullptr + : fetch_latch.peek()); trace = pick.trace; req_addr = pick.addr; from_refetch = pick.from_refetch; } #else - if (!fetch_latch_.empty()) { - trace = fetch_latch_.peek(); + if (!fetch_latch.empty()) { + trace = fetch_latch.peek(); req_addr = trace->PC; } #endif @@ -456,7 +581,7 @@ class Core::Impl { return; // Avoid leaking icache tags when the request port back-pressures. - if (pending_icache_.full()) { + if (pending_icache.full()) { ++perf_stats_.fetch_stalls; return; } @@ -466,31 +591,39 @@ class Core::Impl { // substitutes PA before forwarding; with VM off, this is just the PA. mem_req.addr = req_addr; // op defaults to MemOp::LD — icache request is a load. - uint32_t tag = pending_icache_.allocate(trace); + uint32_t tag = pending_icache.allocate(trace); mem_req.tag = tag; mem_req.hart_id = trace->cid; mem_req.uuid = trace->uuid; - if (icache_req.try_send(mem_req)) { + if (icache_req.try_send(mem_req, 1)) { DT(3, simobject_->name() << " icache-req: addr=0x" << std::hex << mem_req.addr << ", tag=0x" << mem_req.tag << std::dec << ", " << *trace); #ifdef VX_CFG_EXT_C_ENABLE - decompressor_->commit_request(from_refetch); - if (!from_refetch) fetch_latch_.pop(); + decompressor->commit_request(from_refetch); + if (!from_refetch) { fetch_latch.pop(); } #else - fetch_latch_.pop(); + fetch_latch.pop(); #endif ++perf_stats_.ifetches; ++pending_ifetches_; } else { - pending_icache_.release(tag); + pending_icache.release(tag); ++perf_stats_.fetch_stalls; } } void decode() { - if (decode_latch_.empty()) + for (uint32_t f = 0; f < VX_CFG_FETCH_WIDTH; ++f) { + this->decode_slice(*decode_latch_.at(f)); + } + } + + // One frontend slice's decoder: it feeds the issue slice of each decoded warp. + void decode_slice(TFifo& decode_latch) { + if (decode_latch.empty()) { return; + } - auto trace = decode_latch_.peek(); + auto trace = decode_latch.peek(); // check ibuffer capacity auto& ibuffer = ibuffers_.at(trace->wid); @@ -549,7 +682,7 @@ class Core::Impl { trace->ibuf_time = SimPlatform::instance().cycles(); ibuffer->push(trace); - decode_latch_.pop(); + decode_latch.pop(); } void issue() { @@ -588,9 +721,21 @@ class Core::Impl { // check scoreboard against sequencer micro-op (handles uop WAW/RAW) auto trace = ibuffer->peek(); + auto seq = sequencers_.at(wid); auto uop_trace = seq->get(trace); // returns cached uop or generates next + // The head moves into the warp's staging register, freeing its buffer + // slot, once it has arrived and the warp's previous instruction has + // issued; a macro-op leaves the buffer with its last micro-op. The + // scheduler sees the freed slot the cycle after. + if (!head_staged_.at(wid) + && (!trace->instr_ptr->is_macro_op() || seq->last()) + && SimPlatform::instance().cycles() > std::max(last_issue_.at(wid), trace->ibuf_time)) { + head_staged_.at(wid) = true; + this->ibuf_release(wid); + } + // The uop sequencer starts a macro-op once it reaches the front of the // buffer behind the staged instruction, and offers its first uop the // cycle after. A macro that arrives in an empty buffer, or behind an @@ -648,7 +793,8 @@ class Core::Impl { } } - if (ready_set.any()) { + // The arbiter advances only on an issue that fires. + if (ready_set.any() && !operands_.at(iw)->Input.full()) { // select one instruction from ready set auto w = ibuffer_arbs_.at(iw).grant(ready_set); uint32_t wid = w * VX_CFG_ISSUE_WIDTH + iw; @@ -703,19 +849,10 @@ class Core::Impl { trace_pool_.deallocate(trace, 1); } ibuffer->pop(); - // Release ibuffer backpressure slot (matches ++ in schedule()). - // Saturating decrement: flush_warp_pipeline only drains the - // ibuffer + resets ibuf_inflight to 0, but fetch_latch_ / - // decode-stage entries scheduled pre-trap can still arrive - // at issue() post-flush and decrement the (now-zeroed) - // counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, - // MISS) routinely hit this; a non-saturating decrement - // underflows to UINT32_MAX and stalls scheduling. Proper - // fix is to extend flush_warp_pipeline to also drain - // fetch_latch_ + pending icache fetches for the warp. - if (ibuf_inflight_.at(wid) > 0) { - --ibuf_inflight_.at(wid); + if (!head_staged_.at(wid)) { + this->ibuf_release(wid); } + head_staged_.at(wid) = false; } } } @@ -727,6 +864,21 @@ class Core::Impl { } } + // Release an ibuffer backpressure slot (matches ++ in schedule()). + // Saturating decrement: flush_warp_pipeline only drains the ibuffer and + // resets ibuf_inflight to 0, but fetch_latch_ / decode-stage entries + // scheduled pre-trap can still arrive at issue() post-flush and decrement + // the (now-zeroed) counter. Async-trap users (RTU CB_YIELD path: AHS, CHS, + // MISS) routinely hit this; a non-saturating decrement underflows to + // UINT32_MAX and stalls scheduling. Proper fix is to extend + // flush_warp_pipeline to also drain fetch_latch_ + pending icache fetches + // for the warp. + void ibuf_release(uint32_t wid) { + if (ibuf_inflight_.at(wid) > 0) { + --ibuf_inflight_.at(wid); + } + } + void update_queue_full() { for (uint32_t iw = 0; iw < VX_CFG_ISSUE_WIDTH; ++iw) { fu_full_seen_.at(iw) = fu_full_.at(iw); @@ -772,6 +924,11 @@ class Core::Impl { } warp_ctl.pop(); } + auto& bar_unlock = simobject_->bar_unlock_in; + while (!bar_unlock.empty()) { + scheduler_->resume(bar_unlock.peek()); + bar_unlock.pop(); + } // Fan-in: route per-block FU outputs to per-iw commit queues by trace->wid. // Each FU has NUM_*_BLOCKS outputs; the original iw was lost during @@ -789,8 +946,13 @@ class Core::Impl { auto trace = fu_out.peek(); uint32_t iw = trace->wid % VX_CFG_ISSUE_WIDTH; auto& arb_in = *commit_queues_.at(iw).at(fu); - // A unit registers its result once more before the commit arbiter. - if (arb_in.try_send(trace, 2)) { + // The unit's output registers ahead of the commit arbiter that its + // output channel does not already carry: the LSU's response buffer and + // result arbiter, the SFU's result buffer and response switch, and the + // TCU's result landing queue are in their channels, leaving each its + // lane gather. + uint32_t commit_delay = commit_single_hop(fu) ? 1 : 2; + if (arb_in.try_send(trace, commit_delay)) { // Release the warp as soon as its stalling instruction's result leaves // the functional unit — the branch target / fence / warp-control is // resolved at that point. The release lands in stalled_warps so the @@ -888,7 +1050,8 @@ class Core::Impl { // ----- Public-facing API (called from outer Core trampolines) ----- bool running() const { - if (scheduler_->running() || !pending_instrs_.empty()) { + // committed stores wait in the LSU request queue until sent to memory + if (scheduler_->running() || !pending_instrs_.empty() || !this->lsu_drained()) { return true; } #ifdef VX_CFG_VM_ENABLE @@ -931,6 +1094,7 @@ class Core::Impl { ibuffer->pop(); } ibuf_inflight_.at(wid) = 0; + head_staged_.at(wid) = false; uint32_t iw = wid % VX_CFG_ISSUE_WIDTH; if (issue_lock_owner_.at(iw) == wid / VX_CFG_ISSUE_WIDTH) { issue_lock_owner_.at(iw) = kLockOpen; @@ -1091,11 +1255,13 @@ class Core::Impl { LocalMem::Ptr local_mem_; std::vector lmem_switch_; std::vector mem_coalescers_; + TxBuffer::Ptr dcache_flush_buf_; + TxBuffer::Ptr icache_flush_buf_; - TFifo fetch_latch_; - TFifo decode_latch_; + std::vector>> fetch_latch_; + std::vector>> decode_latch_; - HashTable pending_icache_; + std::vector> pending_icache_; std::list> pending_instrs_; uint64_t pending_ifetches_; @@ -1115,15 +1281,17 @@ class Core::Impl { std::vector> fu_full_seen_; // [iw] that flag one cycle later, as issue sees it std::vector ibuf_inflight_; + std::vector ibuf_full_r_; std::vector last_issue_; // [wid] cycle of the warp's last issue std::vector staged_since_; // [wid] cycle the last issued candidate was staged + std::vector head_staged_; // [wid] the buffer head has left for staging PoolAllocator trace_pool_; #ifdef VX_CFG_EXT_C_ENABLE - // Per-core RVC fetch SimObject (per-warp halfword buffer + refetch + // RVC fetch SimObject per frontend slice (per-warp halfword buffer + refetch // queue + pick/commit_request helpers). See decompressor.h. - Decompressor::Ptr decompressor_; + std::vector decompressors_; #endif #ifdef VX_CFG_VM_ENABLE Mmu::Ptr dcache_mmu_; @@ -1139,12 +1307,13 @@ Core::Core(const SimContext& ctx, Socket* socket ) : SimObject(ctx, name) - , icache_req_out(1, this) - , icache_rsp_in(1, this) + , icache_req_out(ICACHE_NUM_REQS, this) + , icache_rsp_in(ICACHE_NUM_REQS, this) , dcache_req_out(VX_CFG_DCACHE_NUM_REQS, this) , dcache_rsp_in(VX_CFG_DCACHE_NUM_REQS, this) , branch_ctl_in(this, VX_CFG_NUM_WARPS) , warp_ctl_in(this, VX_CFG_NUM_WARPS) + , bar_unlock_in(this, VX_CFG_NUM_WARPS) , gbar_arrive_out(this) , gbar_resume_in(this) #ifdef VX_CFG_EXT_RASTER_ENABLE @@ -1197,8 +1366,8 @@ void Core::barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is impl_->scheduler()->barrier_unit().arrive(bar_id, count, wid, is_sync_bar); } -bool Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { - return impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); +void Core::barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid) { + impl_->scheduler()->barrier_unit().wait(bar_id, phase, wid); } void Core::global_barrier_resume(uint32_t bar_id) { diff --git a/sim/simx/core.h b/sim/simx/core.h index df1e117b7b..49462541bd 100644 --- a/sim/simx/core.h +++ b/sim/simx/core.h @@ -82,6 +82,7 @@ class Core : public SimObject { // A warp has at most one release in flight, so NUM_WARPS bounds each. SimChannel branch_ctl_in; SimChannel warp_ctl_in; + SimChannel bar_unlock_in; // Global-barrier event links, wired core <-> cluster at elaboration. SimEventLink gbar_arrive_out; @@ -110,7 +111,7 @@ class Core : public SimObject { void barrier_arrive(uint32_t bar_id, uint32_t count, uint32_t wid, bool is_sync_bar); - bool barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); + void barrier_wait(uint32_t bar_id, uint32_t phase, uint32_t wid); void global_barrier_resume(uint32_t bar_id); diff --git a/sim/simx/cp_llc_port.h b/sim/simx/cp_llc_port.h new file mode 100644 index 0000000000..ab90a85876 --- /dev/null +++ b/sim/simx/cp_llc_port.h @@ -0,0 +1,190 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include +#include "types.h" +#include "constants.h" +#include "mem_block_pool.h" +#include "debug.h" + +namespace vortex { + +// The command processor's DMA port into the last-level cache's client slot. +// Requests in the IO aperture carry the IO attribute and bypass the cache; a +// write flagged no-allocate writes around it on a miss. A launch that follows +// DMA writes first sends an L1-only flush token behind them, which the +// last-level cache answers once its inputs have drained. +class CpLlcPort : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel MemReqOut; + SimChannel MemRspIn; + + // one 4 KB DMA burst of reads in flight + static constexpr uint32_t kReadSlots = 4096 / CP_MEM_DATA_SIZE; + + CpLlcPort(const SimContext& ctx, const char* name) + : SimObject(ctx, name) + , MemReqOut(this) + , MemRspIn(this) + , slots_(kReadSlots) + {} + + // Queue one LLC word of a transfer; `src` is null for a read into `dst`. + void push(uint64_t addr, const uint8_t* src, uint8_t* dst, uint32_t offset, + uint32_t size, bool noalloc) { + MemReq req; + req.addr = addr; + req.op = src ? MemOp::ST : MemOp::LD; + req.flags.io = (get_addr_type(addr) == AddrType::IO); + req.flags.noalloc = src && noalloc; + // the word's bytes sit at its offset in the memory block + uint32_t boff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE) + offset; + if (src) { + req.data = make_mem_block(); + std::memset(req.data->data(), 0, req.data->size()); + std::memcpy(req.data->data() + boff, src, size); + req.byteen = ((size < 64) ? ((1ull << size) - 1) : ~0ull) << boff; + } else { + uint32_t woff = uint32_t(addr % VX_CFG_MEM_BLOCK_SIZE); + req.byteen = ((CP_MEM_DATA_SIZE < 64) ? ((1ull << CP_MEM_DATA_SIZE) - 1) : ~0ull) << woff; + } + queue_.push_back({req, dst, boff, size}); + this->tick_wake(); + } + + // Every queued request has entered the port, and every read has returned. + bool idle() const { + return queue_.empty() && reads_ == 0 && !fencing_; + } + + // DMA writes entered the port since the last fence. + bool writes_pending() const { + return wr_pending_; + } + + // Send the launch fence behind the pending writes. + void fence() { + fencing_ = true; + fence_sent_ = false; + this->tick_wake(); + } + +protected: + void on_reset() { + queue_.clear(); + free_slots_.clear(); + for (uint32_t i = 0; i < kReadSlots; ++i) { + free_slots_.push_back(kReadSlots - 1 - i); + } + reads_ = 0; + wr_pending_ = false; + fencing_ = false; + fence_sent_ = false; + } + + void on_tick() { + if (!MemRspIn.empty()) { + auto& rsp = MemRspIn.peek(); + DT(3, this->name() << " rsp: " << rsp); + if (rsp.tag == kFenceTag) { + fencing_ = false; + wr_pending_ = false; + } else { + uint32_t slot = rsp.tag >> 1; + auto& rd = slots_.at(slot); + std::memcpy(rd.dst, rsp.data->data() + rd.offset, rd.size); + free_slots_.push_back(slot); + --reads_; + } + MemRspIn.pop(); + } + if (fencing_) { + if (!fence_sent_) { + MemReq token; + token.flags.flush = 1; + token.flags.flush_l1 = 1; + token.tag = kFenceTag; + if (MemReqOut.try_send(token, 1)) { + DT(3, this->name() << " fence: " << token); + fence_sent_ = true; + } + } + return; + } + if (queue_.empty()) { + if (reads_ == 0 && MemRspIn.empty()) { + this->tick_sleep(); + } + return; + } + auto& entry = queue_.front(); + bool is_read = !entry.req.is_write(); + if (is_read && free_slots_.empty()) { + return; + } + MemReq req = entry.req; + uint32_t slot = 0; + if (is_read) { + slot = free_slots_.back(); + req.tag = slot << 1; + } else { + req.tag = 0; + } + if (!MemReqOut.try_send(req, 1)) { + return; + } + DT(3, this->name() << " req: " << req); + if (is_read) { + free_slots_.pop_back(); + slots_.at(slot) = {entry.dst, entry.offset, entry.size}; + ++reads_; + } else { + wr_pending_ = true; + } + queue_.pop_front(); + } + +private: + static constexpr uint32_t kFenceTag = 1; + + struct entry_t { + MemReq req; + uint8_t* dst; + uint32_t offset; + uint32_t size; + }; + + struct read_slot_t { + uint8_t* dst = nullptr; + uint32_t offset = 0; + uint32_t size = 0; + }; + + std::deque queue_; + std::vector slots_; + std::vector free_slots_; + uint32_t reads_ = 0; + bool wr_pending_ = false; + bool fencing_ = false; + bool fence_sent_ = false; + + friend class SimObject; +}; + +} diff --git a/sim/simx/cta_dispatcher.cpp b/sim/simx/cta_dispatcher.cpp index cb8c10f5e0..39502a56e5 100644 --- a/sim/simx/cta_dispatcher.cpp +++ b/sim/simx/cta_dispatcher.cpp @@ -34,16 +34,23 @@ CtaDispatcher::CtaDispatcher(const SimContext& ctx, const char* name, Core* core , wid_to_slot_(num_warps_, num_warps_) // num_warps_ = invalid/unassigned , tail_slot_(0) , has_cta_(false) + , cta_done_(false) , cur_slot_(0) , cta_size_(0) , rank_(0) , block_size_rem_(0) , lmem_addr_(0) + , lane_rr_(0) + , init_wait_(0) , cur_kernel_pc_(0) , cur_ctx_id_(0) , warp_init_lanes_(num_warps_) { thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; + for (uint32_t i = 0; i < 3; ++i) { + shape_step_[i] = 0; + shape_dim_[i] = 0; + } for (auto& lanes : warp_init_lanes_) { lanes.resize(num_threads_); } @@ -53,12 +60,17 @@ CtaDispatcher::~CtaDispatcher() {} void CtaDispatcher::on_reset() { has_cta_ = false; + cta_done_ = false; + cta_wids_.reset(); tail_slot_ = 0; + init_wait_ = 0; + retiring_.clear(); for (uint32_t i = 0; i < num_warps_; ++i) { slot_rem_warps_[i] = 0; wid_to_slot_[i] = num_warps_; // invalid } cur_kernel_pc_ = 0; + // lane_rr_ and the cached block shape are device state, like warp_init_lanes_. // warp_init_lanes_ and cur_ctx_id_ are excluded: which lanes of a slot have // already run the startup is keyed on the context, and a launch is not a // device reset. Clearing them here would make the context check dead and @@ -91,14 +103,35 @@ uint32_t CtaDispatcher::usable_slots(uint32_t stride) const { return fit; } -bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out) { +uint32_t CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs) { + // A retirement is registered, reads its slot's count from the table and + // writes it back: it updates the slot in its fourth cycle, and the write + // holds admission off for that cycle. + uint64_t now = SimPlatform::instance().cycles(); + bool retire_write = false; + for (size_t i = 0; i < retiring_.size();) { + if (retiring_[i].second > now) { + ++i; + continue; + } + assert(slot_rem_warps_[retiring_[i].first] > 0); + --slot_rem_warps_[retiring_[i].first]; + retire_write |= (retiring_[i].second == now); + retiring_.erase(retiring_.begin() + i); + } + + if (cta_done_) { + cta_done_ = false; + has_cta_ = false; + return 0; + } if (!has_cta_) { // The KMU pushes CTAs down a registered, credit-gated launch lane; the // endpoint holds the one CTA committed to this core. It is popped only on // admission — an unadmitted CTA keeps the lane's credit, so the KMU // cannot over-commit work to a busy core. - if (bus_in.empty()) - return false; + if (bus_in.empty() || retire_write) + return 0; const kmu_req_t& next = bus_in.peek(); // Fixed-stride admission. Round the per-CTA LMEM size up to a @@ -133,13 +166,13 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa k = max_slots; // cluster larger than co-residency: clamp (degenerate) for (uint32_t i = 0; i < k; ++i) { if (slot_rem_warps_[base + i] != 0) - return false; // window not free yet — wait + return 0; // window not free yet — wait } } else { // Standalone CTA, or a following cluster member (its slot was reserved by // the first-of-cluster window above and is guaranteed free). if (slot_rem_warps_[base] != 0) - return false; + return 0; } // A new launch is a new context: every slot owes the startup again. Keyed on @@ -156,11 +189,13 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa // the admission is reported back to the KMU's launch accounting. cta_ = next; bus_in.pop(); - credit_out.send(1, 1); + // the lane's buffer offers the freed entry to the KMU next cycle + credit_out.send(1, 0); cta_size_ = (cta_.block_size + num_threads_ - 1) / num_threads_; rank_ = 0; block_size_rem_ = cta_.block_size; + cta_wids_.reset(); thread_idx_[0] = thread_idx_[1] = thread_idx_[2] = 0; cur_slot_ = base; @@ -169,49 +204,78 @@ bool CtaDispatcher::step(const WarpMask& active_warps, uint32_t* wid_out, cta_wa tail_slot_ = (base + 1) % max_slots; has_cta_ = true; - } - // Find lowest-index free warp slot. - int free_wid = -1; - for (int wid = 0; wid < (int)num_warps_; ++wid) { - if (!active_warps.test(wid)) { - free_wid = wid; - break; - } - } - if (free_wid < 0) return false; + // The accept moves the dispatcher to its launch state. + init_wait_ = 1; - // Skip the startup only if every lane this launch activates has already run it. - uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; - auto& inited = warp_init_lanes_[free_wid]; - bool needs_init = false; - for (uint32_t t = 0; t < active; ++t) { - if (!inited.test(t)) { - needs_init = true; - break; + if (VX_CFG_FETCH_WIDTH > 1) { + bool same_shape = true; + for (uint32_t d = 0; d < 3; ++d) { + same_shape = same_shape && (shape_step_[d] == cta_.warp_step[d]) + && (shape_dim_[d] == cta_.block_dim[d]); + } + if (!same_shape) { + for (uint32_t d = 0; d < 3; ++d) { + shape_step_[d] = cta_.warp_step[d]; + shape_dim_[d] = cta_.block_dim[d]; + } + init_wait_ += VX_CFG_FETCH_WIDTH; + } } } - if (!next_warp(needs_init, rec_out)) - return false; - - for (uint32_t t = 0; t < active; ++t) { - inited.set(t); + if (init_wait_ != 0) { + --init_wait_; + return 0; } - wid_to_slot_[free_wid] = cur_slot_; - *wid_out = uint32_t(free_wid); - return true; + // Lanes with a free slot, round-robin from lane_rr_, up to the warps left; + // each launches its lowest free slot. + uint32_t count = 0; + for (uint32_t o = 0; o < VX_CFG_FETCH_WIDTH && !cta_done_; ++o) { + uint32_t lane = (lane_rr_ + o) % VX_CFG_FETCH_WIDTH; + int free_wid = -1; + for (uint32_t wid = lane; wid < num_warps_; wid += VX_CFG_FETCH_WIDTH) { + if (!active_warps.test(wid) && !cta_wids_.test(wid)) { + free_wid = int(wid); + break; + } + } + if (free_wid < 0) { + continue; + } + // Skip the startup only if every lane this launch activates has already run it. + uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; + auto& inited = warp_init_lanes_[free_wid]; + bool needs_init = false; + for (uint32_t t = 0; t < active; ++t) { + if (!inited.test(t)) { + needs_init = true; + break; + } + } + if (!next_warp(needs_init, &recs[count])) { + break; + } + for (uint32_t t = 0; t < active; ++t) { + inited.set(t); + } + wid_to_slot_[free_wid] = cur_slot_; + cta_wids_.set(free_wid); + wids[count] = uint32_t(free_wid); + ++count; + } + lane_rr_ = (lane_rr_ + count) % VX_CFG_FETCH_WIDTH; + return count; } void CtaDispatcher::warp_done(uint32_t wid) { uint32_t slot = wid_to_slot_[wid]; if (slot >= num_warps_) return; // not a CTA-dispatcher warp wid_to_slot_[wid] = num_warps_; // clear assignment - assert(slot_rem_warps_[slot] > 0); // When the last warp of a CTA exits, its slot (and the LMEM region at - // slot × stride) frees immediately for reuse — no in-order reclaim. - --slot_rem_warps_[slot]; + // slot × stride) frees for reuse as the retirement lands — no in-order reclaim. + retiring_.emplace_back(slot, SimPlatform::instance().cycles() + 3); } bool CtaDispatcher::slot_needs_init(uint32_t wid, const ThreadMask& tmask) const { @@ -232,7 +296,7 @@ void CtaDispatcher::mark_slot_inited(uint32_t wid, const ThreadMask& tmask) { } bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { - if (!has_cta_) return false; + if (!has_cta_ || cta_done_) return false; out->do_init = do_init; out->PC = Word(cta_.PC); @@ -263,7 +327,9 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { out->cluster_size = cta_.cluster_dim[0] * cta_.cluster_dim[1] * cta_.cluster_dim[2]; // Thread mask: full warp or partial last warp. + // The record is reused across launches, so clear the lanes of the last one. out->tmask.resize(num_threads_); + out->tmask.reset(); uint32_t active = (block_size_rem_ >= num_threads_) ? num_threads_ : block_size_rem_; for (uint32_t t = 0; t < active; ++t) { out->tmask.set(t); @@ -284,7 +350,7 @@ bool CtaDispatcher::next_warp(bool do_init, cta_warp_record_t* out) { ++rank_; if (rank_ >= cta_size_) { - has_cta_ = false; + cta_done_ = true; } return true; diff --git a/sim/simx/cta_dispatcher.h b/sim/simx/cta_dispatcher.h index 74d266c2e3..cf1d649b82 100644 --- a/sim/simx/cta_dispatcher.h +++ b/sim/simx/cta_dispatcher.h @@ -40,8 +40,9 @@ struct cta_warp_record_t { uint32_t cluster_size; }; -// CTA dispatcher: consumes CTAs pushed by the KMU and admits one warp rank -// per step() into a free warp slot. +// CTA dispatcher: consumes CTAs pushed by the KMU and, per step(), launches up +// to one warp of the CTA on each launch lane, into a free warp slot of that +// lane's frontend slice. class CtaDispatcher : public SimObject { public: // Launch-lane endpoint (capacity 1): the CTA the KMU has committed to this @@ -55,9 +56,10 @@ class CtaDispatcher : public SimObject { CtaDispatcher(const SimContext& ctx, const char* name, Core* core); ~CtaDispatcher(); - // Try to dispatch one warp rank into a free warp slot. - // Returns true on success and sets *wid_out / *rec_out. - bool step(const WarpMask& active_warps, uint32_t* wid_out, cta_warp_record_t* rec_out); + // Dispatch warps into free warp slots, at most one per launch lane: lane l + // owns the slots with wid % VX_CFG_FETCH_WIDTH == l. Fills wids[] / recs[] in + // launch order and returns how many warps were dispatched. + uint32_t step(const WarpMask& active_warps, uint32_t* wids, cta_warp_record_t* recs); // Notify that a warp has exited; frees its lmem slot when all warps of the CTA are done. void warp_done(uint32_t wid); @@ -104,14 +106,18 @@ class CtaDispatcher : public SimObject { // within a kernel. The occupancy bound floor(capacity/stride) (capped at // NUM_WARPS) replaces the byte ring's free_size accounting, so no wrap or // padding arithmetic is needed. Slots are allocated round-robin via a tail - // pointer and freed immediately when a CTA's last warp exits. + // pointer and freed when the retirement of a CTA's last warp lands. std::vector slot_rem_warps_; // remaining active warps; slot free when 0 + // Warp retirements on their way to the slot table: {slot, cycle it lands}. + std::vector> retiring_; // Reverse lookup: wid → slot index std::vector wid_to_slot_; uint32_t tail_slot_; // next slot to allocate (round-robin over usable slots) // Currently-in-flight CTA (being dispatched warp-rank by warp-rank) bool has_cta_; + // the last warp has launched; the dispatcher returns to accepting next cycle + bool cta_done_; uint32_t cur_slot_; // slot index assigned to the current in-flight CTA kmu_req_t cta_; uint32_t cta_size_; @@ -119,6 +125,17 @@ class CtaDispatcher : public SimObject { uint32_t block_size_rem_; uint32_t thread_idx_[3]; uint64_t lmem_addr_; + // Slots this CTA has launched into: a CTA never reuses one, even after the + // warp it launched there has exited. + WarpMask cta_wids_; + + // Launch lanes are taken round-robin from lane_rr_. Several lanes need the + // per-lane thread-index window rebuilt when the block shape changes, which + // holds the launch for one cycle per lane. + uint32_t lane_rr_; + uint32_t init_wait_; + uint32_t shape_step_[3]; + uint32_t shape_dim_[3]; Word cur_kernel_pc_; uint8_t cur_ctx_id_; diff --git a/sim/simx/decode.cpp b/sim/simx/decode.cpp index 630ae1bbd9..a9fccab17e 100644 --- a/sim/simx/decode.cpp +++ b/sim/simx/decode.cpp @@ -21,6 +21,7 @@ #include #include #include "debug.h" +#include "constants.h" #include "types.h" #include "decode.h" #include "decompressor.h" @@ -34,6 +35,27 @@ using namespace vortex; +#ifdef VX_CFG_EXT_TCU_ENABLE +// The scoreboard's two FCSR bits double as the TCU metadata slots, as in the +// RTL: a TCU_LD writes the sparse (fflags) or MX (frm) bit, and an MMA that +// consumes that metadata reads it. +static uint8_t tcu_meta_reads(uint32_t fmt_s, bool is_sparse) { + uint8_t mask = 0; + __unused(fmt_s, is_sparse); +#ifdef VX_CFG_TCU_SPARSE_ENABLE + if (is_sparse) { + mask |= Instr::FCSR_FFLAGS; + } +#endif +#ifdef VX_CFG_TCU_MX_ENABLE + if (fmt_s & 0x8) { + mask |= Instr::FCSR_FRM; + } +#endif + return mask; +} +#endif + static op_string_t op_string(const Instr &instr) { auto op_type = instr.get_op_type(); auto instrArgs = instr.get_args(); @@ -266,12 +288,18 @@ static op_string_t op_string(const Instr &instr) { case LsuType::LOAD: { auto lsuArgs = std::get(instrArgs); bool dst_float = (instr.get_dest_reg().type == RegType::Float); + if (lsuArgs.matrix) { + return {"VX_LDM", to_hex_str(lsuArgs.offset)}; + } if (instr.is_macro_op() && dst_float && lsuArgs.width == 0) { return {"PACKLB.F", ""}; } if (instr.is_macro_op() && dst_float && lsuArgs.width == 1) { return {"PACKLH.F", ""}; } + if (instr.is_macro_op()) { + return {(lsuArgs.width == 4) ? "VX_FLW.Q" : "VX_FLW.D", to_hex_str(lsuArgs.offset)}; + } if (dst_float) { switch (lsuArgs.width) { case 1: return {"FLH", to_hex_str(lsuArgs.offset)}; @@ -296,6 +324,12 @@ static op_string_t op_string(const Instr &instr) { } case LsuType::STORE: { auto lsuArgs = std::get(instrArgs); + if (lsuArgs.matrix) { + return {"VX_STM", to_hex_str(lsuArgs.offset)}; + } + if (instr.is_macro_op()) { + return {(lsuArgs.width == 4) ? "VX_FSW.Q" : "VX_FSW.D", to_hex_str(lsuArgs.offset)}; + } // For stores, the data source (rs2) carries the type info. bool src_float = (instr.get_src_reg(1).type == RegType::Float); if (src_float) { @@ -651,7 +685,24 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { case Opcode::FENCE: { instr->set_fu_type(FUType::LSU); instr->set_op_type(LsuType::FENCE); - instr->set_args(IntrLsuArgs{0, 0, 0}); + // pred = code[27:24], succ = code[23:20], IORW. A fence with no memory + // read or write in either set orders only device accesses. Otherwise a + // later read must see other harts' data (acquire) when the successor set + // reads, and earlier writes must be visible (release) when the + // predecessor set writes. A fence with neither releases; one with a + // device access in its predecessor set, or fence.i, does both. + uint32_t pred = (code >> 24) & 0xf; + uint32_t succ = (code >> 20) & 0xf; + FenceKind kind; + if (funct3 == 0 && (pred & 0x3) == 0 && (succ & 0x3) == 0) { + kind = FenceKind::IO; + } else { + bool full = (funct3 != 0) || (pred & 0xc) != 0; + bool aq = full || (succ & 0x2) != 0; + bool rl = full || (pred & 0x1) != 0 || (succ & 0x2) == 0; + kind = (aq && rl) ? FenceKind::FULL : (aq ? FenceKind::ACQUIRE : FenceKind::RELEASE); + } + instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); } break; case Opcode::AMO: { instr->set_fu_type(FUType::LSU); @@ -674,6 +725,11 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { std::abort(); } instr->set_args(IntrAmoArgs{funct3, aq, rl}); + // .aq/.rl order the atomic with fences around it (LsuUopGen), unless the + // dcache is itself the point of coherence + if ((aq || rl) && !DCACHE_SHARED_LLC) { + instr->set_macro_op(); + } instr->set_dest_reg(rd, RegType::Integer); instr->set_src_reg(0, rs1, RegType::Integer); instr->set_src_reg(1, rs2, RegType::Integer); @@ -978,6 +1034,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { bool is_sparse = (rs2 & 1) != 0; instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); + instr->set_fcsr_use(tcu_meta_reads(fmt_s, is_sparse), 0); instr->set_macro_op(); } break; #ifdef VX_CFG_TCU_WGMMA_ENABLE @@ -988,6 +1045,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { bool is_a_smem = (rs2 >> 3) & 1; instr->set_op_type(is_sparse ? TcuType::WGMMA_SP : TcuType::WGMMA); instr->set_args(IntrTcuArgs{is_a_smem ? 1u : 0u, cd_nregs, fmt_s, fmt_d, 0, 0, 0, 0, 0, 0}); + instr->set_fcsr_use(tcu_meta_reads(fmt_s, is_sparse), 0); instr->set_macro_op(); } break; #endif // VX_CFG_TCU_WGMMA_ENABLE @@ -999,6 +1057,7 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { instr->set_args(IntrTcuArgs{0, 0, fmt_s, slot, 0, 0, 0, 0, 0, 0}); // rs1 holds the warp-broadcast base address (real I-reg read). instr->set_src_reg(0, rs1, RegType::Integer); + instr->set_fcsr_use(0, (rd & 0x10) ? Instr::FCSR_FRM : Instr::FCSR_FFLAGS); } break; #endif // TCU_META_ENABLE default: @@ -1190,6 +1249,69 @@ Instr::Ptr Decoder::decode(uint32_t code, uint64_t uuid) { // known-violation handoff class, checked against its decoded destination. gfx_doctrine::check(*instr); } break; +#if defined(VX_CFG_EXT_F_ENABLE) && (defined(VX_CFG_EXT_TCU_ENABLE) || !defined(VX_CFG_FLEN_64)) + case Opcode::EXT4: { + #ifdef VX_CFG_EXT_TCU_ENABLE + // vx_ldm (funct3 0) and vx_stm (funct3 2): a macro-op of one flw/fsw per + // register of a TCU matrix fragment; imm[4:0] is the fragment shape. + if (funct3 == 0 || funct3 == 2) { + bool is_load = (funct3 == 0); + uint32_t imm12 = is_load ? (code >> shift_rs2) : ((funct7 << width_reg) | rd); + uint32_t shape = imm12 & 0x1f; + if (!TcuUopGen::ldm_valid(shape)) { + std::cout << "Error: " << (is_load ? "vx_ldm" : "vx_stm") << " shape " << shape + << " is not supported by this configuration" << std::endl; + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(is_load ? LsuType::LOAD : LsuType::STORE); + instr->set_src_reg(0, rs1, RegType::Integer); + if (is_load) { + instr->set_dest_reg(rd, RegType::Float); + } else { + instr->set_src_reg(1, rs2, RegType::Float); + } + IntrLsuArgs args{/*LW*/ 2, /*stride*/ 0, (int32_t)sext(imm12, width_i_imm)}; + args.matrix = true; + instr->set_args(args); + instr->set_macro_op(); + break; + } + #endif + #ifndef VX_CFG_FLEN_64 + // vx_flw.d/.q (funct3 3/4) and vx_fsw.d/.q (funct3 5/6): a macro-op of 2 + // or 4 flw/fsw uops over an aligned FP register group. The width is the + // access's log2 bytes, as for fld/flq. + bool is_load = (funct3 == 3 || funct3 == 4); + if (!is_load && funct3 != 5 && funct3 != 6) { + std::abort(); + } + uint32_t width = is_load ? funct3 : (funct3 - 2); + uint32_t words = 1u << (width - 2); + uint32_t reg = is_load ? rd : rs2; + uint32_t imm12 = is_load ? (code >> shift_rs2) : ((funct7 << width_reg) | rd); + auto offset = (int32_t)sext(imm12, width_i_imm); + if ((reg % words) != 0 || (offset % int32_t(4 * words)) != 0) { + std::cout << "Error: " << (is_load ? "vx_flw." : "vx_fsw.") << ((words == 4) ? "q" : "d") + << " needs f" << reg << " aligned to " << words << " registers and offset " + << offset << " to " << (4 * words) << " bytes" << std::endl; + std::abort(); + } + instr->set_fu_type(FUType::LSU); + instr->set_op_type(is_load ? LsuType::LOAD : LsuType::STORE); + instr->set_src_reg(0, rs1, RegType::Integer); + if (is_load) { + instr->set_dest_reg(rd, RegType::Float); + } else { + instr->set_src_reg(1, rs2, RegType::Float); + } + instr->set_args(IntrLsuArgs{width, /*stride*/ 0, offset}); + instr->set_macro_op(); + #else + std::abort(); + #endif + } break; +#endif #ifdef VX_CFG_EXT_MUFU_ENABLE case Opcode::EXT3: { if (funct7 != 0 || funct3 != 0 || rs2 > uint32_t(MufuType::TANH)) { diff --git a/sim/simx/decompressor.cpp b/sim/simx/decompressor.cpp index 0de1b35499..2c98736fc2 100644 --- a/sim/simx/decompressor.cpp +++ b/sim/simx/decompressor.cpp @@ -444,6 +444,10 @@ void Decompressor::commit_request(bool from_refetch) { } } +void Decompressor::on_emit(const instr_trace_t* trace) { + state_.at(trace->wid) = RvcSlot{}; +} + Word Decompressor::fetch_addr(const instr_trace_t* trace) const { const auto& rvc = state_.at(trace->wid); if (rvc.needs_second) { @@ -466,7 +470,6 @@ bool Decompressor::on_icache_rsp(instr_trace_t* trace, const mem_block_t& line) // Combine: low half from buffer, high half = low16 of new word. uint16_t high = word & 0xFFFFu; trace->code = (uint32_t(high) << 16) | rvc.low_half; - rvc = RvcSlot{}; return true; } diff --git a/sim/simx/decompressor.h b/sim/simx/decompressor.h index ea5ad0508d..2aed4892d1 100644 --- a/sim/simx/decompressor.h +++ b/sim/simx/decompressor.h @@ -72,8 +72,14 @@ class Decompressor : public SimObject { // 32-bit. The decoder later detects RVC from code[1:0] and // decompresses internally; // - queues `trace` for refetch internally and returns false. + // An emitted response may be refused by a full decode latch and offered + // again: the warp's buffered low half is kept until on_emit(). bool on_icache_rsp(instr_trace_t* trace, const mem_block_t& line); + // Notify that the instruction emitted for `trace` entered the decode + // latch, which frees the warp's half-word buffer. + void on_emit(const instr_trace_t* trace); + protected: void on_reset(); friend class SimObject; diff --git a/sim/simx/dispatcher.cpp b/sim/simx/dispatcher.cpp index f8a45998dd..3071ae1667 100644 --- a/sim/simx/dispatcher.cpp +++ b/sim/simx/dispatcher.cpp @@ -118,7 +118,9 @@ void Dispatcher::on_tick() { } else { block_pids_.at(b) = -1; // mark block as processed input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } ThreadMask tmask(VX_CFG_NUM_THREADS); @@ -132,7 +134,9 @@ void Dispatcher::on_tick() { } else { // issue the trace input.pop(); - ReleaseOut.send(trace, 0); + if (!release_on_accept_) { + ReleaseOut.send(trace, 0); + } ++block_sent; } DT(3, this->name() << "-pipeline dispatch: " << *new_trace); diff --git a/sim/simx/dispatcher.h b/sim/simx/dispatcher.h index 3fe1eb9468..3f3aad0cba 100644 --- a/sim/simx/dispatcher.h +++ b/sim/simx/dispatcher.h @@ -33,6 +33,10 @@ class Dispatcher : public SimObject { virtual ~Dispatcher(); + // The unit takes ops straight off the slot queue, so the credit returns as + // the unit accepts the op, from the unit itself. + void set_release_on_accept() { release_on_accept_ = true; } + protected: virtual void on_reset(); virtual void on_tick(); @@ -46,6 +50,7 @@ class Dispatcher : public SimObject { uint32_t batch_idx_; uint32_t out_delay_; std::vector block_pids_; + bool release_on_accept_ = false; friend class SimObject; }; diff --git a/sim/simx/dxa/dxa_core.cpp b/sim/simx/dxa/dxa_core.cpp index 1af9074776..bc908e448e 100644 --- a/sim/simx/dxa/dxa_core.cpp +++ b/sim/simx/dxa/dxa_core.cpp @@ -27,6 +27,10 @@ using namespace vortex; +// response tags index the in-flight slots by masking +static_assert(VX_CFG_DXA_MAX_INFLIGHT >= 2 && (VX_CFG_DXA_MAX_INFLIGHT & (VX_CFG_DXA_MAX_INFLIGHT - 1)) == 0, + "DXA_MAX_INFLIGHT must be a power of two of at least 2"); + namespace { // One GMEM port: the DXA joins its socket's L2-facing arb as a single peer. @@ -44,6 +48,8 @@ constexpr uint64_t kGmemLineMask = ~uint64_t(VX_CFG_L1_LINE_SIZE - 1); // Cores served by one (socket-resident) DXA engine. constexpr uint32_t kCoresPerSocket = VX_CFG_SOCKET_SIZE; +// A delivered line reaches its SMEM write through two pipeline registers. +constexpr uint32_t kSmemWrLatency = 2; } // namespace @@ -91,6 +97,10 @@ class DxaCore::Impl { struct InflightSlot { bool allocated = false; bool rsp_arrived = false; + bool oob_pending = false; // OOB line waiting for its turn into smem_wr + bool deferred = false; // last line parked until the others drain + uint64_t ready_cycle = 0; // first cycle the line can be written to SMEM + uint64_t free_cycle = 0; // first cycle the slot can carry a new read LineWork work; // captured at allocation std::shared_ptr rsp_data; }; @@ -108,12 +118,17 @@ class DxaCore::Impl { uint32_t smem_stride = 0; uint64_t issue_cycle = 0; + // setup engine: the next transfer is set up while this one drains + bool staged = false; + DxaReq staged_req; + uint64_t staged_ready = 0; + // addr_gen: pre-enumerated work list (one entry per GMEM CL + // multicast replays handled at smem_wr stage). std::vector work_list; uint32_t ag_idx = 0; // next entry to issue (gmem_req) - // gmem_req → rsp_buf → smem_wr inflight bookkeeping. + // gmem_req → smem_wr inflight bookkeeping. std::array inflight; std::deque issued_order; // outstanding tags in issue order uint32_t drain_slot = UINT32_MAX; // slot currently draining @@ -145,7 +160,8 @@ class DxaCore::Impl { w.cta_indices.clear(); w.issued_order.clear(); w.drain_slot = UINT32_MAX; - for (auto& s : w.inflight) { s.allocated = false; s.rsp_arrived = false; s.rsp_data.reset(); } + for (auto& s : w.inflight) { s = InflightSlot(); } + w.staged = false; w.ag_idx = 0; w.mc_cta_idx = 0; w.km_elem_idx = 0; @@ -203,40 +219,38 @@ class DxaCore::Impl { void tick() { ++cycle_; - // Drain in reverse-pipeline order (smem_wr → rsp_buf → gmem_req → - // addr_gen → setup → req-arb) so each stage's tick reads what its - // upstream produced earlier this cycle, mimicking forward stage flow. + // Drain in reverse-pipeline order (smem_wr → gmem_req → addr_gen → + // setup → req-arb) so each stage's tick reads what its upstream produced + // earlier this cycle, mimicking forward stage flow. - // 1) Drain GMEM responses → mark inflight slots arrived. - // gmem_arb_ already routes responses back to RspOut[worker_id] (its - // input index) and restores the user's tag (the per-worker slot id). + // 1) smem_wr drain, then 2) hand it one line per worker, then 3) issue + // the next GMEM read. A line leaving pend for the fill buffer frees pend + // for a capture in the same cycle. gmem_arb_ already routes responses + // back to RspOut[worker_id] (its input index) and restores the user's tag + // (the per-worker slot id). for (uint32_t worker_id = 0; worker_id < workers_.size(); ++worker_id) { - auto& ch = gmem_arb_->RspOut.at(worker_id); - // One response per port per cycle (each worker has its own RspOut port). - if (!ch.empty()) { - auto& rsp = ch.peek(); - uint32_t slot_id = uint32_t(rsp.tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); - auto& w = workers_[worker_id]; - if (slot_id < w.inflight.size() && w.inflight[slot_id].allocated) { - w.inflight[slot_id].rsp_arrived = true; - w.inflight[slot_id].rsp_data = rsp.data; - } - ch.pop(); - } - } - - // 2) Tick workers (smem_wr drain → gmem_req issue). - for (auto& w : workers_) { + auto& w = workers_[worker_id]; if (w.state == WState::RUNNING) { tick_worker_smem_wr(w); + } + capture_line(w, gmem_arb_->RspOut.at(worker_id)); + if (w.state == WState::RUNNING) { tick_worker_gmem_req(w); } } - // 3) Dispatch from queue to idle workers. + // 3) Start a set-up transfer on an idle worker; otherwise let the + // worker's setup engine take the next request. for (auto& w : workers_) { - if (w.state == WState::IDLE && !queue_.empty()) - start_worker(w, queue_.front()), queue_.pop_front(); + if (w.staged && w.state == WState::IDLE && cycle_ >= w.staged_ready) { + w.staged = false; + start_worker(w, w.staged_req); + } else if (!w.staged && !queue_.empty()) { + w.staged_req = queue_.front(); + w.staged_ready = cycle_ + this->setup_latency(w.staged_req); + w.staged = true; + queue_.pop_front(); + } } // 4) Drain DxaUnit channels → req queue (round-robin). @@ -319,6 +333,14 @@ class DxaCore::Impl { return (k_blk * n_steps + n_blk) * b_blk_elems + n_in * kw + r_in; } + // Parameter setup takes longer for higher-rank descriptors: each extra + // dimension adds a multiply phase. + uint32_t setup_latency(const DxaReq& req) const { + uint32_t rank = (req.desc_slot < VX_DCR_DXA_DESC_COUNT) + ? desc_rank(descriptors_.at(req.desc_slot).meta) : 1; + return 2 * std::max(rank, 2) + 2; + } + // ── Pull from per-core dxa_req_in[] into queue_ (round-robin) ──────── void drain_req_in() { auto& chs = simobject_->dxa_req_in; @@ -533,6 +555,52 @@ class DxaCore::Impl { } } + // ── smem_wr capture: one line per cycle into its line registers ─────── + // A GMEM response has priority over a pending OOB line. smem_wr holds one + // line being written, one captured behind it, and the transfer's last line + // while others are outstanding. A response it cannot take stays on the + // memory response path and holds the responses queued behind it. + void capture_line(Worker& w, SimChannel& ch) { + bool pend_busy = false; + bool defer_busy = false; + uint32_t oob_slot = UINT32_MAX; + for (uint32_t i = 0; i < w.inflight.size(); ++i) { + const auto& s = w.inflight[i]; + if (s.rsp_arrived && i != w.drain_slot) { + (s.deferred ? defer_busy : pend_busy) = true; + } + if (s.oob_pending && oob_slot == UINT32_MAX) { + oob_slot = i; + } + } + + uint32_t slot = oob_slot; + if (!ch.empty()) { + slot = uint32_t(ch.peek().tag) & (VX_CFG_DXA_MAX_INFLIGHT - 1); + if (!w.inflight[slot].allocated) { + ch.pop(); + return; + } + } + if (slot == UINT32_MAX) { + return; + } + + auto& s = w.inflight[slot]; + bool defer = s.work.last && (w.issued_order.size() > 1); + if (defer ? defer_busy : pend_busy) { + return; + } + if (!ch.empty()) { + s.rsp_data = ch.peek().data; + ch.pop(); + } + s.oob_pending = false; + s.rsp_arrived = true; + s.deferred = defer; + s.ready_cycle = cycle_ + kSmemWrLatency; + } + // ── gmem_req: issue MemReqs with available inflight slots ──────────── void tick_worker_gmem_req(Worker& w) { // One AGU step per cycle: advance ag_idx by at most one, issuing at most @@ -544,7 +612,7 @@ class DxaCore::Impl { // Find a free slot. uint32_t slot = UINT32_MAX; for (uint32_t s = 0; s < w.inflight.size(); ++s) { - if (!w.inflight[s].allocated) { + if (!w.inflight[s].allocated && cycle_ >= w.inflight[s].free_cycle) { slot = s; break; } @@ -561,11 +629,13 @@ class DxaCore::Impl { return; } - // OOB lines skip the GMEM request entirely; we synthesize an immediate - // arrival (rsp data left null — smem_wr will use cfill). + // OOB lines skip the GMEM request entirely; smem_wr takes them in a cycle + // with no GMEM response (rsp data left null — smem_wr will use cfill). if (lw.oob) { w.inflight[slot].allocated = true; - w.inflight[slot].rsp_arrived = true; + w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = true; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -587,6 +657,8 @@ class DxaCore::Impl { w.inflight[slot].allocated = true; w.inflight[slot].rsp_arrived = false; + w.inflight[slot].oob_pending = false; + w.inflight[slot].deferred = false; w.inflight[slot].rsp_data.reset(); w.inflight[slot].work = lw; w.issued_order.push_back(slot); @@ -610,7 +682,8 @@ class DxaCore::Impl { if (slot == UINT32_MAX) { for (uint32_t i = 0; i < w.inflight.size(); ++i) { const auto& candidate = w.inflight[i]; - if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last) { + if (candidate.allocated && candidate.rsp_arrived && !candidate.work.last + && cycle_ >= candidate.ready_cycle) { slot = i; break; } @@ -619,7 +692,7 @@ class DxaCore::Impl { if (slot == UINT32_MAX && w.issued_order.size() == 1) { uint32_t last_slot = w.issued_order.front(); const auto& candidate = w.inflight[last_slot]; - if (candidate.allocated && candidate.rsp_arrived) + if (candidate.allocated && candidate.rsp_arrived && cycle_ >= candidate.ready_cycle) slot = last_slot; } @@ -783,6 +856,8 @@ class DxaCore::Impl { // Slot done — release. s.allocated = false; s.rsp_arrived = false; + s.deferred = false; + s.free_cycle = cycle_ + 1; s.rsp_data.reset(); auto it = std::find(w.issued_order.begin(), w.issued_order.end(), slot); if (it != w.issued_order.end()) w.issued_order.erase(it); @@ -860,7 +935,16 @@ DxaCore::DxaCore(const SimContext& ctx, const char* name, Socket* socket) snprintf(sname, 100, "%s-gmem-arb", name); gmem_arb_ = MemArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_DXA_CORES, kDxaMemPorts); for (uint32_t i = 0; i < kDxaMemPorts; ++i) { + #ifdef SIMX_SOCKET_UNIT_MMU + // The unit MMU's lookup starts from a registered request: the DXA owns it. + snprintf(sname, 100, "%s-gmem-out%d", name, i); + auto out_reg = MemReqSlice::Create(sname, 1); + gmem_arb_->ReqOut.at(i).bind(&out_reg->In); + out_reg->Out.bind(&gmem_req_out.at(i)); + gmem_out_regs_.push_back(out_reg); + #else gmem_arb_->ReqOut.at(i).bind(&gmem_req_out.at(i)); + #endif gmem_rsp_in.at(i).bind(&gmem_arb_->RspIn.at(i)); } diff --git a/sim/simx/dxa/dxa_core.h b/sim/simx/dxa/dxa_core.h index 7f5c1219ea..cb49c85f29 100644 --- a/sim/simx/dxa/dxa_core.h +++ b/sim/simx/dxa/dxa_core.h @@ -58,6 +58,7 @@ class DxaCore : public SimObject { std::vector> gmem_req_out; std::vector> gmem_rsp_in; MemArbiter::Ptr gmem_arb_; + std::vector gmem_out_regs_; // LMEM row width the RTL drains per cycle: one DXA write beat covers the // full banked row (DXA_LMEM_WORD_SIZE = LMEM_NUM_BANKS * XLEN/8). The diff --git a/sim/simx/dxa/dxa_unit.cpp b/sim/simx/dxa/dxa_unit.cpp index 42db10595f..e398b84ffb 100644 --- a/sim/simx/dxa/dxa_unit.cpp +++ b/sim/simx/dxa/dxa_unit.cpp @@ -18,6 +18,10 @@ using namespace vortex; +static_assert(VX_CFG_NUM_ALU_LANES >= 4, + "the DXA descriptor is packed by WGATHER, which gathers within groups of 4 lanes, " + "so the ALU must hold a whole group in one packet"); + instr_trace_t* DxaUnit::process(instr_trace_t* trace) { if (req_out_.full()) { return nullptr; @@ -62,7 +66,7 @@ instr_trace_t* DxaUnit::process(instr_trace_t* trace) { // completion; pre-registration happens explicitly per-CTA so multicast // destinations correctly wait. - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(4, "dxa-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", slot=" << desc_slot << ", bar=" << raw_bar << ", cta_mask=0x" << std::hex << cta_mask << std::dec); diff --git a/sim/simx/dxa/dxa_unit.h b/sim/simx/dxa/dxa_unit.h index c076a14b53..04ab7c3435 100644 --- a/sim/simx/dxa/dxa_unit.h +++ b/sim/simx/dxa/dxa_unit.h @@ -43,8 +43,10 @@ struct DxaReq { class DxaUnit { public: // The outbound channel lives on SfuUnit (which is the SimObject). - DxaUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's DXA bus. + DxaUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Decode lanes 0..3 from the trace's source operands and try to push a // DxaReq on the SFU's outbound channel. Returns the trace on success @@ -55,6 +57,7 @@ class DxaUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/fpu_unit.cpp b/sim/simx/fpu_unit.cpp index 6316092c57..58b908a9f6 100644 --- a/sim/simx/fpu_unit.cpp +++ b/sim/simx/fpu_unit.cpp @@ -46,7 +46,16 @@ FpuUnit::FpuUnit(const SimContext& ctx, const char* name, Core* core) // until its result is taken off the response port, however deep the // pipelines are. : FuncUnit(ctx, name, core, VX_CFG_FPU_QUEUE_SIZE) -{} + , ReleaseOut(this, VX_CFG_ISSUE_WIDTH) +{ + if (!kGather) { + // No buffer between the dispatch queue and the unit: the input holds + // only the queue's head. + for (auto& input : Inputs) { + input = SimChannel(this, 1); + } + } +} uint32_t FpuUnit::latency_of(const instr_trace_t* trace) const { auto fpu_type = std::get(trace->op_type); @@ -409,6 +418,9 @@ void FpuUnit::on_tick() { uint32_t delay = this->latency_of(trace) + (kGather ? 0 : 1); output.send(trace, delay); input.pop(); + if (!kGather && trace->eop) { + ReleaseOut.send(trace, 0); + } } } idle &= (input.size() == 0); diff --git a/sim/simx/fpu_unit.h b/sim/simx/fpu_unit.h index 4de1f44f72..ef0dcd11a7 100644 --- a/sim/simx/fpu_unit.h +++ b/sim/simx/fpu_unit.h @@ -26,6 +26,10 @@ class FpuUnit : public FuncUnit { static constexpr bool kGather = (VX_CFG_NUM_FPU_BLOCKS != VX_CFG_ISSUE_WIDTH) || (VX_CFG_NUM_FPU_LANES != VX_CFG_SIMD_WIDTH); + // Full bandwidth: the dispatch queue feeds the unit directly, so an op + // returns its issue credit as the unit accepts it. + SimChannel ReleaseOut; + protected: void on_tick() override; diff --git a/sim/simx/gbar_unit.cpp b/sim/simx/gbar_unit.cpp new file mode 100644 index 0000000000..f331c6272c --- /dev/null +++ b/sim/simx/gbar_unit.cpp @@ -0,0 +1,50 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gbar_unit.h" +#include +#include "debug.h" + +using namespace vortex; + +GbarUnit::GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters) + : SimObject(ctx, name) + , arrive_in(this) + , resume_out(num_clusters, this) + , arrived_(VX_CFG_NUM_BARRIERS, 0) + , accept_next_(0) +{ + arrive_in.bind(this, &GbarUnit::on_arrive); +} + +void GbarUnit::on_reset() { + std::fill(arrived_.begin(), arrived_.end(), 0); + accept_next_ = 0; +} + +// A core arrives once per generation, so a count of arrivals tracks the +// participants without a mask over every core of the device. +void GbarUnit::on_arrive(const GbarArrive& msg) { + uint64_t now = SimPlatform::instance().cycles(); + uint64_t accept = std::max(now, accept_next_); + accept_next_ = accept + 1; + auto& arrived = arrived_.at(msg.bar_id % arrived_.size()); + ++arrived; + DT(4, "*** Device barrier arrive: core #" << msg.core_id << " at barrier #" << msg.bar_id << ", arrived=" << arrived); + if (arrived == msg.count) { + for (auto& link : resume_out) { + link.send({msg.bar_id}, accept - now + 1); + } + arrived = 0; + } +} diff --git a/sim/simx/gbar_unit.h b/sim/simx/gbar_unit.h new file mode 100644 index 0000000000..fc06fe8fe8 --- /dev/null +++ b/sim/simx/gbar_unit.h @@ -0,0 +1,45 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include "types.h" + +namespace vortex { + +// Device-level global barrier: with more than one cluster, a barrier's +// participants span clusters, so each cluster forwards its cores' arrivals +// here and every cluster is resumed once the count is met. Arrivals enter +// through the device arbiter's output buffer, one per cycle; the release is +// registered in the unit and again in the response fork to the clusters. +class GbarUnit : public SimObject { +public: + SimEventLink arrive_in; + std::vector> resume_out; + + GbarUnit(const SimContext& ctx, const char* name, uint32_t num_clusters); + +protected: + void on_reset(); + +private: + void on_arrive(const GbarArrive& msg); + + std::vector arrived_; + uint64_t accept_next_; // first cycle the unit can take another arrival + + friend class SimObject; +}; + +} // namespace vortex diff --git a/sim/simx/kmu/kmu.cpp b/sim/simx/kmu/kmu.cpp index ab20f74be8..731dcdf3ac 100644 --- a/sim/simx/kmu/kmu.cpp +++ b/sim/simx/kmu/kmu.cpp @@ -120,15 +120,12 @@ void Kmu::on_tick() { return; } // Steering: a locked cluster's remaining members stay with the owning core; - // otherwise the next cluster goes to the core the round-robin already points - // at. One CTA per cycle — the launch bus carries a single beat. A lane is - // ready when its transport has room and the core still owes fewer than - // LAUNCH_CREDITS admissions, so a busy core never accumulates work. - // - // The destination is fixed before the message is sent, so a busy core stalls - // the bus instead of yielding its turn to a neighbour. Scanning for any ready - // lane here would let a core that drains faster absorb its neighbour's share - // and leave the placement lopsided. + // otherwise the next cluster goes to the nearest ready lane from the + // round-robin pointer, so a core that drains its work sooner takes the next + // cluster instead of the launch waiting on a busy neighbour. One CTA per + // cycle — the launch bus carries a single beat. A lane is ready when its + // transport has room and the core holds fewer than LAUNCH_CREDITS CTAs it + // has not yet admitted. auto lane_ready = [&](uint32_t c) { return !bus_out.at(c).full() && inflight_[c] < LAUNCH_CREDITS; }; @@ -137,15 +134,22 @@ void Kmu::on_tick() { if (lane_ready(cluster_core_)) { target = cluster_core_; } - } else if (lane_ready(rr_ptr_)) { - target = rr_ptr_; + } else { + for (uint32_t k = 0; k < num_cores; ++k) { + uint32_t c = (rr_ptr_ + k) % num_cores; + if (lane_ready(c)) { + target = c; + break; + } + } } if (target == num_cores) return; // no ready lane this cycle kmu_req_t req{}; if (!this->next_cta(&req, target)) return; - bus_out.at(target).send(req, 1); + // the lane slice ticks after the KMU and registers the CTA into the core + bus_out.at(target).send(req, 0); ++inflight_[target]; if (!cluster_locked_) { // Message complete: advance past the granted core. diff --git a/sim/simx/kmu/kmu.h b/sim/simx/kmu/kmu.h index 537703601c..687bd40ec9 100644 --- a/sim/simx/kmu/kmu.h +++ b/sim/simx/kmu/kmu.h @@ -52,11 +52,10 @@ class Kmu : public SimObject { // rests on explicit slot accounting, not on transport back-pressure. std::vector> credit_in; - // In-flight bound per core: the boundary stage's endpoint (2) plus the - // dispatcher endpoint (1) plus one admission credit in return flight, so - // the lane's full() view stays the binding gate whenever the transport - // applies back-pressure. - static constexpr uint32_t LAUNCH_CREDITS = 4; + // Each core's launch buffer holds two CTAs it has not yet admitted. A + // deeper buffer would commit work to a busy core early and keep it from a + // core that frees a slot sooner. + static constexpr uint32_t LAUNCH_CREDITS = 2; Kmu(const SimContext& ctx, const char* name, uint32_t num_cores); diff --git a/sim/simx/lsu_unit.cpp b/sim/simx/lsu_unit.cpp index c891fc3f51..135d2f43d6 100644 --- a/sim/simx/lsu_unit.cpp +++ b/sim/simx/lsu_unit.cpp @@ -26,13 +26,30 @@ #include "stack_interleave.h" #include "VX_types.h" #include "VX_config.h" +#ifdef VX_CFG_EXT_TCU_ENABLE +#include "tcu_unit.h" +#endif using namespace vortex; uint32_t LsuUopGen::uop_count(const Instr& instr) { + // An atomic with .aq/.rl: [release fence] atomic [acquire fence] + if (auto amo = std::get_if(&instr.get_args())) { + return 1 + amo->aq + amo->rl; + } // PACKLB.F: width=0 (LB), 4 byte-elements // PACKLH.F: width=1 (LH), 2 halfword-elements + // VX_FLW/VX_FSW.D: width=3, 2 words; .Q: width=4, 4 words + // VX_LDM/VX_STM: one word per register of the fragment shape auto args = std::get(instr.get_args()); +#ifdef VX_CFG_EXT_TCU_ENABLE + if (args.matrix) { + return TcuUopGen::ldm_count(args.offset & 0x1f); + } +#endif + if (args.width >= 3) { + return 1u << (args.width - 2); + } return (args.width == 0) ? 4 : 2; } @@ -47,7 +64,66 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uint32_t steps_shift = (total > 1) ? (32 - log2ceil(total)) : 0; uint64_t uop_uuid = (static_cast(uuid_hi) << 32) | ((uop_index << steps_shift) | uuid_lo); + if (auto amo = std::get_if(¯o_instr.get_args())) { + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uint32_t amo_index = amo->rl; + if (uop_index == amo_index) { + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_args(*amo); + // an atomic into x0 writes nothing back + auto rd = macro_instr.get_dest_reg(); + if (!(rd.type == RegType::Integer && rd.idx == 0)) { + uop_instr->set_dest_reg(rd.idx, rd.type); + } + for (uint32_t i = 0; i < Instr::MAX_REG_SOURCES; ++i) { + auto rs = macro_instr.get_src_reg(i); + uop_instr->set_src_reg(i, rs.idx, rs.type); + } + } else { + auto kind = (uop_index < amo_index) ? FenceKind::RELEASE : FenceKind::ACQUIRE; + uop_instr->set_op_type(LsuType::FENCE); + uop_instr->set_args(IntrLsuArgs{static_cast(kind), 0, 0}); + } + return uop_instr; + } + auto args = std::get(macro_instr.get_args()); +#ifdef VX_CFG_EXT_TCU_ENABLE + if (args.matrix) { + // a fragment uop reads its base from rs1 | grp, ORs its column into the + // offset (whose low 5 bits held the shape) and moves register fd/fs2 + reg_off + bool is_load = (std::get(macro_instr.get_op_type()) == LsuType::LOAD); + auto u = TcuUopGen::ldm_uop(args.offset & 0x1f, uop_index); + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_src_reg(0, macro_instr.get_src_reg(0).idx | u.grp, RegType::Integer); + if (is_load) { + uop_instr->set_dest_reg((macro_instr.get_dest_reg().idx + u.reg_off) & 0x1f, RegType::Float); + } else { + uop_instr->set_src_reg(1, (macro_instr.get_src_reg(1).idx + u.reg_off) & 0x1f, RegType::Float); + } + uop_instr->set_args(IntrLsuArgs{/*LW*/ 2, /*stride*/ 0, int32_t((args.offset & ~0x1f) | u.col)}); + return uop_instr; + } +#endif + if (args.width >= 3) { + // a wide access's uop k moves word k: register fd/fs2 + k, offset + 4k + bool is_load = (std::get(macro_instr.get_op_type()) == LsuType::LOAD); + auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::LSU); + uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_op_type(macro_instr.get_op_type()); + uop_instr->set_src_reg(0, macro_instr.get_src_reg(0).idx, RegType::Integer); + if (is_load) { + uop_instr->set_dest_reg(macro_instr.get_dest_reg().idx + uop_index, RegType::Float); + } else { + uop_instr->set_src_reg(1, macro_instr.get_src_reg(1).idx + uop_index, RegType::Float); + } + uop_instr->set_args(IntrLsuArgs{/*LW*/ 2, /*stride*/ 0, args.offset + int32_t(4 * uop_index)}); + return uop_instr; + } + // Macro encodes elem-size as width: 0=byte (PACKLB), 1=halfword (PACKLH). // The uop is a regular unsigned load (LBU/LHU = width + 4) so the LSU // doesn't sign-extend the loaded byte/halfword before the bytesel write. @@ -74,31 +150,37 @@ Instr::Ptr LsuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { } LsuUnit::LsuUnit(const SimContext& ctx, const char* name, Core* core) - // Commit-side elasticity of 6 beats: the writeback path buffers a whole - // coalesced fragment burst, so a denied commit grant queues beats there - // instead of throttling the memory response stream. - : FuncUnit(ctx, name, core, 6) + // the response buffer and the result arbiter, two entries each + : FuncUnit(ctx, name, core, 4) #ifdef TCU_META_ENABLE - , TcuReqIn(this) - , TcuRspOut(this) + , TcuReqIn(TCU_AGU_PORTS, this) + , TcuRspOut(TCU_AGU_PORTS, this) #endif , pending_loads_(0) -{} +{ + // the lane-dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } +} LsuUnit::~LsuUnit() {} -// Returns true when all blocks' input queues are empty and no reads are outstanding. +// Returns true when no block holds an instruction with requests still to send +// or a read awaiting its response; ops still in the dispatch buffer do not hold +// a barrier. bool LsuUnit::drained() const { #ifdef TCU_META_ENABLE // TcuRspOut forwards to the TCU's endpoint, so query via size() (empty() // asserts on forwarding channels). - if (!TcuReqIn.empty() || TcuRspOut.size() != 0) { - return false; + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + if (!TcuReqIn.at(p).empty() || TcuRspOut.at(p).size() != 0) { + return false; + } } #endif for (uint32_t b = 0; b < VX_CFG_NUM_LSU_BLOCKS; ++b) { - if (!Inputs.at(b).empty()) return false; if (!states_.at(b).req_queue.empty()) return false; if (!states_.at(b).pending_reqs.empty()) return false; } @@ -177,7 +259,7 @@ void LsuUnit::compute_addrs(uint32_t b, instr_trace_t* trace) { // The datapath has no misalignment support: only a naturally aligned // access is guaranteed to sit inside one memory block, which is what // lets a lane's payload pack into a single block downstream. - // VX_lsu_slice.sv enforces the same contract in hardware, so modelling + // The hardware LSU enforces the same contract, so modelling // anything looser here would let a shader pass in simulation and fail // on the device. if ((e.addr % e.size) != 0) { @@ -221,11 +303,11 @@ void LsuUnit::process_response_step(uint32_t b) { // TCU metadata response: forward the fragment with the client's tag // restored; no register writeback, no trace retirement. if (entry.is_tcu) { - if (TcuRspOut.full()) + if (TcuRspOut.at(b).full()) return; // stall LsuRsp fwd = lsu_rsp; fwd.tag = entry.client_tag; - TcuRspOut.send(fwd); + TcuRspOut.at(b).send(fwd); DT(3, this->name() << " tcu-meta-rsp: " << fwd); entry.count -= lsu_rsp.mask.count(); if (entry.count == 0) { @@ -236,6 +318,16 @@ void LsuUnit::process_response_step(uint32_t b) { } #endif + if (entry.is_flush) { + DT(3, this->name() << " fence-token-rsp: " << lsu_rsp); + entry.count -= lsu_rsp.mask.count(); + if (entry.count == 0) { + state.pending_reqs.release(lsu_rsp.tag); + } + lsu_rsp_in.pop(); + return; + } + auto trace = entry.trace; auto& output = Outputs.at(b); // Every response fragment retires through its own writeback slot, one @@ -300,8 +392,7 @@ void LsuUnit::process_response_step(uint32_t b) { state.pending_reqs.release(lsu_rsp.tag); } if (is_final && entry.eop) { - // Load writeback crosses one more registered stage than the - // direct-commit path before reaching the commit arbiter. + // the response buffer, then the result arbiter's register output.send(trace, 2); } else { // A non-final fragment still spends a commit-port slot: it drains @@ -322,10 +413,27 @@ void LsuUnit::process_response_step(uint32_t b) { // At most one transaction/cycle: the trace is not dispatchable until the next // cycle, so req_queue adds a real cycle of latency. A while-loop here would // fabricate write bandwidth and collapse that latency to zero. -void LsuUnit::ingest_inputs(uint32_t b) { +void LsuUnit::ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held) { auto& state = states_.at(b); auto& input = Inputs.at(b); - if (input.empty() || state.req_queue.full()) + if (!queue_room) + return; +#ifdef TCU_META_ENABLE + // The TCU metadata client and the block's own requests share the request + // queue, one entry per cycle, granted round-robin. + bool meta_req = (b < TCU_AGU_PORTS) + && !TcuReqIn.at(b).empty() + && (reads_held < VX_CFG_LSU_PENDING_SIZE); + if (meta_req && (input.empty() || !state.meta_last)) { + state.meta_reqs.push_back(TcuReqIn.at(b).peek()); + TcuReqIn.at(b).pop(); + state.req_queue.push(nullptr); + ++state.staged_reads; + state.meta_last = true; + return; + } +#endif + if (input.empty()) return; auto trace = input.peek(); // Hold a fence at the input head until req_queue is empty so the @@ -336,13 +444,72 @@ void LsuUnit::ingest_inputs(uint32_t b) { return; const bool is_read = !lsu_type_tag || *lsu_type_tag == LsuType::LOAD; if (is_read) { - if (state.pending_reqs.size() + state.staged_reads >= VX_CFG_LSU_PENDING_SIZE) { + if (reads_held >= VX_CFG_LSU_PENDING_SIZE) { return; } ++state.staged_reads; } + if (lsu_type_tag && *lsu_type_tag == LsuType::STORE) { + // A store commits as the request queue accepts it, through the + // no-response path, and is sent to memory later. The commit stage + // retires the trace, so the queue keeps a copy of its own. + if (Outputs.at(b).full()) { + return; + } + auto copy = new (core_->trace_pool().allocate(1)) instr_trace_t(*trace); + // the no-response buffer, then the result arbiter's register + Outputs.at(b).send(trace, 2); + trace = copy; + } state.req_queue.push(trace); input.pop(); +#ifdef TCU_META_ENABLE + state.meta_last = false; +#endif +} + +// The fence travels to the data cache as a flush token on the lanes of its +// last packet, behind every request the block issued before it. The cache +// answers once it has drained, flushed and invalidated, so the fence holds +// the block until the token's response returns. +void LsuUnit::send_fence_token(uint32_t b) { + auto& state = states_.at(b); + // the token's lanes carry no address: it goes to the data cache + if (state.pending_reqs.full() || !core_->lmem_switch(b)->req_ready(true, false)) { + return; + } + auto trace = state.fence.trace(); + uint32_t base = ((VX_CFG_NUM_THREADS - 1) / VX_CFG_NUM_LSU_LANES) * VX_CFG_NUM_LSU_LANES; + LsuReq lsu_req(VX_CFG_NUM_LSU_LANES); + lsu_req.op = MemOp::LD; + lsu_req.flags.flush = 1; + // an IO fence's token walks nowhere; a memory fence's stops at the shared + // LLC, an acquire one owing no drain and a release one no invalidation + auto kind = lsu_fence_kind(std::get(trace->instr_ptr->get_args())); + lsu_req.flags.io = (kind == FenceKind::IO) || (kind == FenceKind::ACQUIRE); + lsu_req.flags.flush_l1 = (kind != FenceKind::IO); + lsu_req.flags.noalloc = (kind == FenceKind::RELEASE); + std::vector lane_entries(VX_CFG_NUM_LSU_LANES); + for (uint32_t t = base; t < VX_CFG_NUM_THREADS; ++t) { + if (trace->tmask.test(t)) { + lsu_req.mask.set(t - base); + lsu_req.tids.at(t - base) = t; + } + } + if (lsu_req.mask.none()) { + lsu_req.mask.set(0); + lsu_req.tids.at(0) = base; + } + uint32_t count = lsu_req.mask.count(); + pending_req_t entry{nullptr, count, true, std::move(lane_entries), IntrLsuArgs{}, false}; + entry.is_flush = true; + lsu_req.tag = state.pending_reqs.allocate(std::move(entry)); + lsu_req.cid = trace->cid; + lsu_req.wid = trace->wid; + lsu_req.uuid = trace->uuid; + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); + DT(3, this->name() << " fence-token: " << lsu_req); + state.fence_token = false; } // Per-cycle request handler. Dispatches at most one batch (one @@ -354,7 +521,10 @@ void LsuUnit::process_request_step(uint32_t b) { // Attempt fence release first so a steady stream of TCU client forwards // (which consume the dispatch slot and return) cannot starve it. const bool fence_locked = state.fence.locked(); - if (fence_locked) { + if (fence_locked && state.fence_token) { + this->send_fence_token(b); + } + if (fence_locked && !state.fence_token) { bool released = state.fence.try_release(Outputs.at(b), state.pending_reqs.empty()); if (released) { DT(3, this->name() << " fence-unlock: " << state.fence.trace()); @@ -362,15 +532,24 @@ void LsuUnit::process_request_step(uint32_t b) { } #ifdef TCU_META_ENABLE - // TCU metadata client (block 0 only): forward one pending request per - // cycle into lmem_switch, ahead of the block's own dispatch. The wire tag + // TCU metadata client (blocks below TCU_AGU_PORTS): its request leaves the + // shared request queue in order with the block's own. The wire tag // is re-allocated from pending_reqs so it stays unique on the switch; the // client's own tag is restored on the response. An independent client, so // not gated by the fence lock — its entries still hold fence release via // pending_reqs.empty(). - if (b == 0 && !TcuReqIn.empty()) { - if (!state.pending_reqs.full() && !core_->lmem_switch(0)->ReqIn.full()) { - LsuReq lsu_req = TcuReqIn.peek(); + if (!state.req_queue.empty() && state.req_queue.front() == nullptr) { + const LsuReq& meta_req = state.meta_reqs.front(); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + if (meta_req.mask.test(i)) { + bool local = (get_addr_type(meta_req.addrs.at(i)) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + } + if (!state.pending_reqs.full() && core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) { + LsuReq lsu_req = meta_req; uint32_t count = lsu_req.mask.count(); std::vector lane_entries(VX_CFG_NUM_LSU_LANES); for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { @@ -392,11 +571,13 @@ void LsuUnit::process_request_step(uint32_t b) { entry.client_tag = lsu_req.tag; uint32_t tag = state.pending_reqs.allocate(std::move(entry)); lsu_req.tag = tag; - core_->lmem_switch(0)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " tcu-meta-req: " << lsu_req); - TcuReqIn.pop(); - return; // one request per cycle into the switch + state.meta_reqs.pop_front(); + state.req_queue.pop(); + --state.staged_reads; } + return; // one request per cycle into the switch } #endif @@ -432,6 +613,7 @@ void LsuUnit::process_request_step(uint32_t b) { // Engage fence lock; the next call will try to release it once // the pending-reqs table drains. state.fence.engage(trace); + state.fence_token = true; DT(3, this->name() << " fence-lock: " << *trace); state.req_queue.pop(); return; @@ -457,9 +639,9 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = !is_write || is_amo; } - // AMO always returns to rd, so it is not direct-commit even though it - // carries a side-effect write. - bool direct_commit = (is_write || 0 == state.addr_list.size()) && !is_amo; + // A store committed when the queue accepted it. AMO always returns to rd, + // so it is not direct-commit even though it carries a side-effect write. + bool direct_commit = !is_write && 0 == state.addr_list.size() && !is_amo; // Skip beats whose tid group is entirely inactive — nothing to issue. while (state.remain_addrs != 0) { @@ -488,8 +670,19 @@ void LsuUnit::process_request_step(uint32_t b) { } if (state.remain_addrs != 0) { - // check lmem switch backpressure - if (core_->lmem_switch(b)->ReqIn.full()) + // check backpressure at the switch buffers this beat addresses + uint32_t beat_t0 = state.addr_list.size() - state.remain_addrs; + uint32_t beat_len = std::min(VX_CFG_NUM_LSU_LANES, state.remain_addrs); + bool to_dc = false, to_lmem = false; + for (uint32_t i = 0; i < beat_len; ++i) { + auto& entry = state.addr_list.at(beat_t0 + i); + if (entry.size == 0) + continue; + bool local = (get_addr_type(entry.addr) == AddrType::Shared); + to_lmem |= local; + to_dc |= !local; + } + if (!core_->lmem_switch(b)->req_ready(to_dc, to_lmem)) return; // stall // setup memory request @@ -566,7 +759,7 @@ void LsuUnit::process_request_step(uint32_t b) { lsu_req.uuid = trace->uuid; // send memory request - core_->lmem_switch(b)->ReqIn.send(lsu_req); + core_->lmem_switch(b)->ReqIn.send(lsu_req, 0); DT(3, this->name() << " mem-req: " << lsu_req); // update stats @@ -588,6 +781,10 @@ void LsuUnit::process_request_step(uint32_t b) { state.head_staged = false; } state.req_queue.pop(); + if (is_write) { + trace->~instr_trace_t(); + core_->trace_pool().deallocate(trace, 1); + } } } @@ -599,8 +796,13 @@ void LsuUnit::on_tick() { // as it stood at the start of this cycle, THEN ingest one new trace for // next cycle. Dispatching before ingest makes req_queue a 1-cycle stage // rather than a same-tick passthrough. + // The scheduler's request queue and its pending table register their + // full flags: a slot freed this cycle takes a new request next cycle. + auto& state = states_.at(b); + bool queue_room = !state.req_queue.full(); + uint32_t reads_held = state.pending_reqs.size() + state.staged_reads; this->process_response_step(b); this->process_request_step(b); - this->ingest_inputs(b); + this->ingest_inputs(b, queue_room, reads_held); } } diff --git a/sim/simx/lsu_unit.h b/sim/simx/lsu_unit.h index 02a693998a..e09feeb409 100644 --- a/sim/simx/lsu_unit.h +++ b/sim/simx/lsu_unit.h @@ -14,6 +14,7 @@ #pragma once #include +#include #include #include #include "func_unit.h" @@ -27,6 +28,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { // Micro-op generator for packed-load macro instructions (PACKLB.F / PACKLH.F). @@ -79,11 +87,12 @@ class LsuUnit : public FuncUnit { ~LsuUnit(); #ifdef TCU_META_ENABLE - // TCU metadata client port on block 0: TCU_LD load requests arbitrate - // with the LSU's own traffic into lmem_switch; response fragments return - // here with the client's own tag restored. - SimChannel TcuReqIn; - SimChannel TcuRspOut; + // TCU metadata client ports (TCU_AGU_PORTS, starting at block 0): TCU_LD + // load requests arbitrate with the block's own traffic into its + // lmem_switch; response fragments return here with the client's own tag + // restored. + std::vector> TcuReqIn; + std::vector> TcuRspOut; #endif // Returns true when all LSU blocks have no in-flight requests. @@ -100,9 +109,10 @@ class LsuUnit : public FuncUnit { // Per-cycle handlers. void process_response_step(uint32_t b); void process_request_step(uint32_t b); + void send_fence_token(uint32_t b); // Drain Inputs[b] into req_queue. - void ingest_inputs(uint32_t b); + void ingest_inputs(uint32_t b, bool queue_room, uint32_t reads_held); struct mem_addr_size_t { uint64_t addr; @@ -123,6 +133,8 @@ class LsuUnit : public FuncUnit { // TcuRspOut with client_tag restored instead of written to registers. bool is_tcu = false; uint32_t client_tag = 0; + // A fence's flush token: its response only releases the fence. + bool is_flush = false; }; // Per-block LSU state. Each member is a named hardware sub-block. @@ -138,8 +150,14 @@ class LsuUnit : public FuncUnit { // Reads waiting in req_queue hold a pending_reqs slot from the moment // they enter it, so staged and outstanding reads share the pool. uint32_t staged_reads = 0; + // TCU metadata requests granted a req_queue slot (a null entry there), + // and whether the TCU client took the last queue entry. + std::deque meta_reqs; + bool meta_last = false; bool head_staged = false; FenceController fence; + // the engaged fence's flush token is still to be sent + bool fence_token = false; std::vector addr_list; uint32_t remain_addrs = 0; @@ -147,8 +165,11 @@ class LsuUnit : public FuncUnit { this->req_queue.clear(); this->pending_reqs.clear(); this->staged_reads = 0; + this->meta_reqs.clear(); + this->meta_last = false; this->head_staged = false; this->fence.reset(); + this->fence_token = false; this->addr_list.clear(); this->remain_addrs = 0; } diff --git a/sim/simx/main.cpp b/sim/simx/main.cpp index ff1d406692..41f8bd473c 100644 --- a/sim/simx/main.cpp +++ b/sim/simx/main.cpp @@ -233,6 +233,7 @@ int main(int argc, char **argv) { // flush GPU caches before reading back results { uint32_t dummy; + processor.dcr_write(VX_DCR_BASE_CACHE_FLUSH, 0); for (uint32_t cid = 0; cid < VX_CFG_NUM_CORES * VX_CFG_NUM_CLUSTERS; ++cid) { processor.dcr_read(VX_DCR_BASE_CACHE_FLUSH, cid, &dummy); } diff --git a/sim/simx/mem/addr_hash.h b/sim/simx/mem/addr_hash.h new file mode 100644 index 0000000000..e9117edd33 --- /dev/null +++ b/sim/simx/mem/addr_hash.h @@ -0,0 +1,46 @@ +// Copyright © 2019-2026 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include + +namespace vortex { + +// IPOLY index hashing (Rau, "Pseudo-randomly interleaved memory", ISCA 1991), +// with the GF(2) equations GPGPU-Sim uses for its memory partitions and DRAM +// banks. Each returns the mask f(hi) that is XORed into the index; f depends +// only on the higher bits, so the same XOR restores the original index. + +inline uint32_t ipoly_bit(uint64_t v, int i) { return uint32_t(v >> i) & 1u; } + +// IPOLY(37), 32 sets. +inline uint32_t ipoly32(uint64_t a) { + uint32_t i0 = ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,14)^ipoly_bit(a,13)^ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,1); + uint32_t i2 = ipoly_bit(a,14)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,3)^ipoly_bit(a,2)^ipoly_bit(a,0); + uint32_t i3 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,1); + uint32_t i4 = ipoly_bit(a,12)^ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,5)^ipoly_bit(a,4)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3) | (i4 << 4); +} + +// IPOLY(5), 16 sets. +inline uint32_t ipoly16(uint64_t a) { + uint32_t i0 = ipoly_bit(a,11)^ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,3)^ipoly_bit(a,0); + uint32_t i1 = ipoly_bit(a,12)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,1)^ipoly_bit(a,0); + uint32_t i2 = ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,6)^ipoly_bit(a,4)^ipoly_bit(a,2)^ipoly_bit(a,1); + uint32_t i3 = ipoly_bit(a,10)^ipoly_bit(a,9)^ipoly_bit(a,8)^ipoly_bit(a,7)^ipoly_bit(a,5)^ipoly_bit(a,3)^ipoly_bit(a,2); + return i0 | (i1 << 1) | (i2 << 2) | (i3 << 3); +} + +} diff --git a/sim/simx/mem/cache.cpp b/sim/simx/mem/cache.cpp index df1a19d7ee..91b8d0b0ee 100644 --- a/sim/simx/mem/cache.cpp +++ b/sim/simx/mem/cache.cpp @@ -15,6 +15,7 @@ #include "mem_block_pool.h" #include "debug.h" #include "types.h" +#include "addr_hash.h" #if VX_CFG_EXT_A_ENABLED #include "amo_unit.h" #endif @@ -28,6 +29,39 @@ using namespace vortex; +namespace { + +constexpr uint32_t kMaxFanout = 8; + +// An arbiter over more than 1.5x the fan-out limit is split into a buffered +// tree: slices of up to the fan-out limit each grant into their own buffer, +// and those buffers are arbitrated the same way, one level per cycle, until +// few enough remain for a single join arbiter. Returns each buffered level's +// slice count, none for a single arbiter. +std::vector arb_tree_levels(uint32_t requesters) { + std::vector levels; + for (uint32_t n = requesters; n > kMaxFanout + kMaxFanout / 2;) { + n = (n + kMaxFanout - 1) / kMaxFanout; + levels.push_back(n); + } + return levels; +} + +// Slices of the tree's first level, 1 for a single arbiter. +uint32_t arb_tree_slices(uint32_t requesters) { + auto levels = arb_tree_levels(requesters); + return levels.empty() ? 1 : levels.front(); +} + +// Registered stages of a radix-2 omega network between a and b ports; up to +// two ports on each side it is a single crossbar. +uint32_t omega_stages(uint32_t a, uint32_t b) { + uint32_t n = std::max(a, b); + return (n <= 2) ? 1 : log2ceil(n); +} + +} + struct params_t { uint32_t sets_per_bank; uint32_t lines_per_set; @@ -50,7 +84,9 @@ struct params_t { int32_t tag_select_addr_start; int32_t tag_select_addr_end; - params_t(const Cache::Config &config) { + bool bank_hash; + + params_t(const Cache::Config &config) : bank_hash(config.bank_hash) { int32_t offset_bits = config.L - config.W; int32_t index_bits = config.C - (config.L + config.A + config.B); assert(offset_bits >= 0); @@ -85,13 +121,35 @@ struct params_t { // Tag select this->tag_select_addr_start = (1 + this->set_select_addr_end); this->tag_select_addr_end = (config.addr_width - 1); + assert(!bank_hash || (config.B != 0 && config.B <= 8)); + } + + // The bank is the raw bank bits XORed with a function of the bits above + // them, so power-of-two strides spread over the banks; applying the same + // XOR again restores the raw bits. IPOLY for 16/32 banks; other bank counts + // XOR-fold the higher bits (Frailong, Jalby, Lenfant, "XOR-schemes", ICPP 1985). + uint32_t bank_xor(uint64_t addr, uint32_t raw) const { + uint32_t bits = bank_select_addr_end - bank_select_addr_start + 1; + uint64_t hi = addr >> (bank_select_addr_end + 1); + if (bits == 4) { + return raw ^ ipoly16(hi); + } + if (bits == 5) { + return raw ^ ipoly32(hi); + } + uint32_t mask = (1u << bits) - 1, f = 0; + for (uint32_t i = 0; i < 15; i += bits) { + f ^= uint32_t(hi >> i) & mask; + } + return raw ^ f; } uint32_t addr_bank_id(uint64_t addr) const { - if (bank_select_addr_end >= bank_select_addr_start) - return (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); - else + if (bank_select_addr_end < bank_select_addr_start) { return 0; + } + uint32_t raw = (uint32_t)bit_getw(addr, bank_select_addr_start, bank_select_addr_end); + return bank_hash ? bank_xor(addr, raw) : raw; } uint32_t addr_set_id(uint64_t addr) const { @@ -123,6 +181,10 @@ struct params_t { addr = bit_setw(addr, set_select_addr_start, set_select_addr_end, set_id); if (tag_select_addr_end >= tag_select_addr_start) addr = bit_setw(addr, tag_select_addr_start, tag_select_addr_end, tag); + if (bank_hash && bank_select_addr_end >= bank_select_addr_start) { + uint32_t raw = bank_xor(addr, bank_id); + addr = bit_setw(addr, bank_select_addr_start, bank_select_addr_end, raw); + } return addr; } @@ -140,17 +202,30 @@ struct params_t { struct sector_t { bool valid; bool dirty; + bool partial; // allocated by a store: only dirty_mask bytes are known uint64_t dirty_mask; // per-byte dirty bits; the writeback byte-enable std::shared_ptr data; // sector bytes (= one mem_block) void reset() { valid = false; dirty = false; + partial = false; dirty_mask = 0; data.reset(); } }; +// A store may hit a partial sector; a load or AMO needs every byte. +static inline bool sector_hit(const sector_t& sec, bool needs_data) { + return sec.valid && !(needs_data && sec.partial); +} + +// Every byte of the sector has been stored: a refetch of it reads nothing. +static inline bool sector_written(const sector_t& sec) { + constexpr uint64_t full_mask = (VX_CFG_MEM_BLOCK_SIZE >= 64) ? ~0ull : ((1ull << VX_CFG_MEM_BLOCK_SIZE) - 1); + return sec.dirty_mask == full_mask; +} + struct line_t { uint64_t tag; uint32_t lru_ctr; @@ -340,12 +415,15 @@ struct bank_req_t { Fill = 1, Replay = 2, Core = 3, - AmoProbe = 4 // non-LLC AMO passthrough: probe-and-invalidate then forward + AmoProbe = 4, // non-LLC AMO passthrough: probe-and-invalidate then forward + Park = 5, // chain head parked on a partial sector: issues its refetch + AmoWb = 6 // an LLC atomic's result written back through the pipe }; uint64_t addr; uint32_t hart_id; uint64_t req_tag; + std::shared_ptr> merged_tags; // ride to the response uint64_t uuid; uint32_t mshr_id; ReqType type; @@ -357,6 +435,13 @@ struct bank_req_t { // emit another response — only run sector_merge. bool skip_core_rsp; + // Fill flavors under lazy fetch: a data-less allocation for a write-miss + // root, or a refetch of a resident partial sector (merges under its dirty + // bytes). + bool nodata; + bool refetch; + bool refetch_retry; // refetch re-issued after its sector was evicted + // TLM data: // For Core writes: incoming write data + byteen. // For Fill: captured fill data from below (mem_rsp.data). @@ -390,12 +475,16 @@ struct bank_req_t { addr = 0; hart_id = 0; req_tag = 0; + merged_tags.reset(); uuid = 0; mshr_id = 0; type = ReqType::None; write = false; op = MemOp::LD; skip_core_rsp = false; + nodata = false; + refetch = false; + refetch_retry = false; data.reset(); byteen = 0; amo_cmp = 0; @@ -422,6 +511,8 @@ struct mshr_entry_t { uint32_t line_id; uint64_t seq; // enqueue order; replays drain oldest-first to preserve // program order among coalesced same-line accesses. + bool refetch; // root of a fill that refetches a resident partial sector + bool refetch_retry; // ... re-issued after the previous refetch found it evicted mshr_entry_t() { this->reset(); @@ -434,6 +525,8 @@ struct mshr_entry_t { sector_id = 0; line_id = 0; seq = 0; + refetch = false; + refetch_retry = false; } }; @@ -520,7 +613,8 @@ class MSHR { } // Enqueue a new core request and return the allocated entry id. - int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id) { + int enqueue(const bank_req_t &bank_req, uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, + bool refetch = false) { assert(bank_req.type == bank_req_t::Core); for (uint32_t i = 0, n = entries_.size(); i < n; ++i) { auto &entry = entries_.at(i); @@ -531,6 +625,8 @@ class MSHR { entry.sector_id = sector_id; entry.line_id = 0; // victim is selected at Fill time entry.seq = bank_req.adm_seq; + entry.refetch = refetch; + entry.refetch_retry = false; ++size_; return i; } @@ -571,6 +667,34 @@ class MSHR { ++ready_reqs_; } + // Park the chain at its head: the oldest ready replay and the rest of its + // (set,tag,sector) chain return to waiting for a fill, which the head now + // roots as a refetch. Returns the head's id. + uint32_t park_next_replay(bool retry) { + int head = -1; + for (int i = 0, n = (int)entries_.size(); i < n; ++i) { + const auto &entry = entries_.at(i); + if (entry.bank_req.type != bank_req_t::Replay) { + continue; + } + if (head == -1 || entry.seq < entries_.at(head).seq) { + head = i; + } + } + assert(head != -1); + auto &root = entries_.at(head); + for (auto &entry : entries_) { + if (entry.bank_req.type == bank_req_t::Replay && entry.set_id == root.set_id + && entry.addr_tag == root.addr_tag && entry.sector_id == root.sector_id) { + entry.bank_req.type = bank_req_t::Core; + --ready_reqs_; + } + } + root.refetch = true; + root.refetch_retry = retry; + return head; + } + // Peek the entry dequeue() would return next (oldest ready replay), or // nullptr. Lets the fill-forward drain inspect the chain head without // consuming it. @@ -650,11 +774,14 @@ class CacheBank : public SimObject { const Cache::Config &config, const params_t ¶ms, uint32_t bank_id) - : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), rand_ctr_(0), adm_seq_ctr_(0) + : SimObject(ctx, name), core_req_in(this), core_rsp_out(this), mem_req_out(this), mem_rsp_in(this, config.mshr_size + AMO_PASSTHRU_CAP), config_(config), params_(params), bank_id_(bank_id), sets_(params.sets_per_bank, set_t(params.lines_per_set, params.sectors_per_line)), mshr_(config.mshr_size), pipe_req_(TFifo::Create("", config.latency, std::max(2u, config.latency))), lazy_q_(TFifo::Create("", 1, lazyq_size(config))), rand_ctr_(0), adm_seq_ctr_(0) #if VX_CFG_EXT_A_ENABLED , amo_unit_(__MAX(2u, (uint32_t)VX_CFG_AMO_RS_SIZE)) #endif { + // Lazy allocation leaves unfetched bytes in the line: only a write-back + // LLC can hold them until a read merges or a writeback masks them out. + assert(!config.lazy_fetch || (config.write_back && config.is_llc)); this->on_reset(); } @@ -667,14 +794,25 @@ class CacheBank : public SimObject { // a writeback request for every dirty line via mem_req_out (write-back only; // write-through caches have nothing to evict). flush_done() reports when the // walk finishes AND the cache is otherwise idle. - void flush_begin() { - if (!config_.write_back) { + // Drop every line, as an address-space change does to a read-only + // virtually tagged cache. Called between launches, with no line in flight. + void invalidate() { + for (auto &set : sets_) { + set.reset(); + } + } + + void flush_begin(bool keep_lines) { + keep_lines_ = keep_lines; + if (!config_.write_back || keep_lines) { flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; flush_sector_idx_ = 0; return; } + // the two walks share the sector cursor + __assert(!token_flushing_, "flush walk started during a flush-token walk"); flushing_ = true; flush_set_idx_ = 0; flush_way_idx_ = 0; @@ -686,31 +824,66 @@ class CacheBank : public SimObject { return !flushing_; } + // A flush token's walk: one entry per cycle, every set (every way of a + // write-back cache), writing back dirty data and invalidating the lines. + void token_flush_begin(bool skip) { + __assert(!flushing_, "flush-token walk started during a flush walk"); + token_flushing_ = true; + token_flush_skip_ = skip; + token_flush_idx_ = 0; + token_flush_settle_ = 0; + flush_sector_idx_ = 0; + this->tick_wake(); + } + + bool token_flush_done() const { + return !token_flushing_; + } + protected: void on_reset() { // perf_stats_ survives: the MPM counters run from device reset. pending_mshr_size_ = 0; + mshr_occupancy_ = 0; pending_amo_probes_ = 0; - pipe_fill_count_ = 0; pipe_core_lines_.clear(); pending_read_reqs_ = 0; pending_write_reqs_ = 0; pending_fill_reqs_ = 0; rand_ctr_ = 0; adm_seq_ctr_ = 0; - for (auto &set : sets_) { - set.reset(); + #ifdef DBG_MEM_BANK0_WDELAY + dbg_mem_req_free_ = 0; + #endif + if (!keep_lines_) { + for (auto &set : sets_) { + set.reset(); + } } + keep_lines_ = false; mshr_.reset(); fwd_active_ = false; fwd_set_ = 0; fwd_tag_ = 0; fwd_sector_ = 0; + fwd_data_.reset(); crsp_sent_ = false; + chain_partial_ = false; + chain_retry_ = false; + chain_set_ = 0; + chain_tag_ = 0; + chain_sector_ = 0; + refetch_hold_ = false; + refetch_hold_retry_ = false; + amo_busy_ = false; + amo_wb_pending_ = false; + amo_wb_tick_ = 0; flushing_ = false; flush_set_idx_ = 0; flush_way_idx_ = 0; flush_sector_idx_ = 0; + token_flushing_ = false; + token_flush_idx_ = 0; #if VX_CFG_EXT_A_ENABLED amo_unit_.reset(); for (auto &e : amo_passthru_) { @@ -724,6 +897,10 @@ class CacheBank : public SimObject { // priority and the forward drain yields (crsp_sent_). crsp_sent_ = false; + // MSHR occupancy is a registered count: slots a forward, replay or + // release frees this tick admit a new request only from the next one. + mshr_occupancy_ = mshr_.size() + pending_mshr_size_; + // Process next request at the head of the pipeline. if (!pipe_req_->empty()) { this->processRequests(); @@ -746,6 +923,9 @@ class CacheBank : public SimObject { if (flushing_) { this->processFlush(); } + if (token_flushing_) { + this->processTokenFlush(); + } // calculate memory latency perf_stats_.mem_latency += pending_fill_reqs_; @@ -755,8 +935,9 @@ class CacheBank : public SimObject { // the mem_latency accumulation exact. flush_begin() re-arms explicitly; // any packet reserved toward core_req_in/mem_rsp_in re-arms on arrival. if (this->core_req_in.size() == 0 && this->mem_rsp_in.size() == 0 - && pipe_req_->size() == 0 && !mshr_.has_ready_reqs() - && !fwd_active_ && !flushing_ && pending_fill_reqs_ == 0) { + && pipe_req_->size() == 0 && lazy_q_->size() == 0 && !mshr_.has_ready_reqs() + && !fwd_active_ && !flushing_ && !token_flushing_ && pending_fill_reqs_ == 0 + && !amo_wb_pending_) { this->tick_sleep(); } } @@ -775,10 +956,41 @@ class CacheBank : public SimObject { // // At most one input fires per tick. All inputs flow through pipe_req_. void processInputs(bool fwd_consumed) { + // A refetch fill learns whether its sector is still resident at its tag + // lookup, the tick after it enters: nothing enters behind it meanwhile, + // and its chain then replays, or parks again if the sector was evicted. + if (refetch_hold_) { + refetch_hold_ = false; + bool evicted = false; + if (!refetch_hold_retry_) { + const auto &set = sets_.at(chain_set_); + int resident_id = set.find_resident(chain_tag_); + evicted = (resident_id == -1) || !set.lines.at(resident_id).sectors.at(chain_sector_).valid; + } + chain_partial_ = evicted; + chain_retry_ = evicted; + return; + } + // 1) replay: one chain entry leaves per tick — when the fill-forward // drain completed the head this tick, this slot goes to new core // requests instead if (mshr_.has_ready_reqs() && !fwd_consumed) { + // A chain whose sector is partial cannot serve a load or AMO: park at + // that head and refetch the sector; the rest of the chain waits. + if (chain_partial_) { + const auto *head = mshr_.peek_next_replay(); + if (head != nullptr + && head->set_id == chain_set_ && head->addr_tag == chain_tag_ && head->sector_id == chain_sector_ + && !head->bank_req.write) { + bank_req_t park_req = head->bank_req; + park_req.mshr_id = mshr_.park_next_replay(chain_retry_); + park_req.type = bank_req_t::Park; + pipe_req_->push(park_req); + DT(3, this->name() << " mshr-park: " << park_req); + return; + } + } bank_req_t bank_req; mshr_.dequeue(&bank_req); pipe_req_->push(bank_req); @@ -787,11 +999,16 @@ class CacheBank : public SimObject { } // 2) fill only when no replay is pending and no forward drain is in - // progress (a new fill would re-arm the window mid-chain). A fill still - // riding the pipe also blocks the next one: its chain is only marked for - // replay at install, and the replay priority above must get a chance to - // drain it before another fill is accepted. - if (!this->mem_rsp_in.empty() && !fwd_active_ && pipe_fill_count_ == 0) { + // progress (a new fill would re-arm the window mid-chain). Each fill + // carries its own sector down the pipe, so fills may follow each other + // back to back. The fill wakes its chain on accept, so a same-line core + // request still in the pipe must reach its chain first (the hardware + // links it at S0): the fill waits, and so does any further request to + // that line, which keeps a stream of them from starving the fill. + bool fill_held = false; + uint32_t held_set_id = 0; + uint64_t held_addr_tag = 0; + if (!this->mem_rsp_in.empty() && !fwd_active_) { auto &mem_rsp = this->mem_rsp_in.peek(); #if VX_CFG_EXT_A_ENABLED // Non-LLC AMO passthrough response: forward straight to core @@ -819,25 +1036,114 @@ class CacheBank : public SimObject { #endif uint32_t mshr_id = mem_rsp.tag; const auto &root_peek = mshr_.peek(mshr_id); - bank_req_t bank_req; - bank_req.reset(); - bank_req.type = bank_req_t::Fill; - bank_req.addr = params_.mem_addr_sector(bank_id_, root_peek.set_id, root_peek.addr_tag, root_peek.sector_id); - bank_req.hart_id = root_peek.bank_req.hart_id; - bank_req.uuid = root_peek.bank_req.uuid; - bank_req.mshr_id = mshr_id; - bank_req.data = mem_rsp.data; - pipe_req_->push(bank_req); - ++pipe_fill_count_; - DT(3, this->name() << " fill-rsp: " << mem_rsp); - this->mem_rsp_in.pop(); - --pending_fill_reqs_; + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.data = mem_rsp.data; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; + pipe_req_->push(bank_req); + // The chain wakes at accept: its replays enter the pipe behind the fill, + // so they reach the array after the install. The forward window answers + // the chain's leading reads from the staged sector meanwhile; a refetch + // cannot, its staged sector lacks the dirty bytes the install merges in. + mshr_.replay(mshr_id); + this->setChain(set_id, addr_tag, sector_id, false); + if (bank_req.refetch) { + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; + } else { + fwd_active_ = true; + fwd_set_ = set_id; + fwd_tag_ = addr_tag; + fwd_sector_ = sector_id; + fwd_data_ = mem_rsp.data; + } + DT(3, this->name() << " fill-rsp: " << mem_rsp); + this->mem_rsp_in.pop(); + --pending_fill_reqs_; + return; + } + } else if (!lazy_q_->empty() && !fwd_active_) { + // A data-less fill through the same path, below memory fills in + // priority: the allocation of a write-miss root, or the refetch of a + // sector every store has covered. + uint32_t mshr_id = lazy_q_->peek(); + const auto &root_peek = mshr_.peek(mshr_id); + const uint32_t set_id = root_peek.set_id; + const uint64_t addr_tag = root_peek.addr_tag; + const uint32_t sector_id = root_peek.sector_id; + if (this->pipeLineBusy(set_id, addr_tag)) { + fill_held = true; + held_set_id = set_id; + held_addr_tag = addr_tag; + } else { + bank_req_t bank_req; + bank_req.reset(); + bank_req.type = bank_req_t::Fill; + bank_req.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + bank_req.hart_id = root_peek.bank_req.hart_id; + bank_req.uuid = root_peek.bank_req.uuid; + bank_req.mshr_id = mshr_id; + bank_req.nodata = true; + bank_req.refetch = root_peek.refetch; + bank_req.refetch_retry = root_peek.refetch_retry; + pipe_req_->push(bank_req); + mshr_.replay(mshr_id); + // A looped-back refetch completes a fully written sector: like a + // memory refetch, its chain waits for it to install. + this->setChain(set_id, addr_tag, sector_id, !bank_req.refetch); + if (bank_req.refetch) { + refetch_hold_ = true; + refetch_hold_retry_ = bank_req.refetch_retry; + } + DT(3, this->name() << " lazy-fill: " << bank_req); + lazy_q_->pop(); + return; + } + } + +#if VX_CFG_EXT_A_ENABLED + // An LLC atomic's result goes back into the line as a synthetic write + // through the core-request slot, one compute cycle after its commit. + if (amo_wb_pending_ && SimPlatform::instance().cycles() >= amo_wb_tick_) { + bank_req_t wb_req; + wb_req.reset(); + wb_req.type = bank_req_t::AmoWb; + pipe_req_->push(wb_req); + amo_wb_pending_ = false; + amo_busy_ = false; return; } +#endif // 3) core request if (!this->core_req_in.empty()) { +#if VX_CFG_EXT_A_ENABLED + // A store-bearing atomic holds new core requests off from its lookup + // until its result is written back. + if (amo_busy_) { + return; + } +#endif auto &core_req = this->core_req_in.peek(); + if (fill_held + && params_.addr_set_id(core_req.addr) == held_set_id + && params_.addr_tag(core_req.addr) == held_addr_tag) { + return; + } // Conservative MSHR occupancy check: any request that may miss must // reserve a slot. Counts both currently-allocated entries and // in-flight pipe requests that haven't reached MSHR allocation yet. @@ -875,11 +1181,9 @@ class CacheBank : public SimObject { // MSHR entry only at pipe exit; defer on those too, or the AMO // overtakes it and a younger same-address load can chain onto the // pre-AMO fill (and the fill installs a pre-AMO line). - for (const auto &kv : pipe_core_lines_) { - if (kv.first == amo_set_id && kv.second == amo_addr_tag) { - ++perf_stats_.mshr_stalls; - return; - } + if (this->pipeLineBusy(amo_set_id, amo_addr_tag)) { + ++perf_stats_.mshr_stalls; + return; } } #else @@ -892,10 +1196,15 @@ class CacheBank : public SimObject { // responses from draining — a deadlock. Reserve at admission. bool needs_mshr = !is_amo_passthru; (void)is_amo; - if (needs_mshr && (mshr_.size() + pending_mshr_size_) >= mshr_.capacity()) { + if (needs_mshr && mshr_occupancy_ >= mshr_.capacity()) { ++perf_stats_.mshr_stalls; return; } + // Any admitted request may become a write-miss root: the lazy queue + // keeps room for every request already in the pipe. + if (config_.lazy_fetch && !this->lazyRoom()) { + return; + } // Admission-time coalescing: a request whose (set,tag,sector) chain is // still awaiting its fill must join it here, not enter the pipe as an @@ -932,6 +1241,7 @@ class CacheBank : public SimObject { coalesced.hart_id = core_req.hart_id; coalesced.uuid = core_req.uuid; coalesced.req_tag = core_req.tag; + coalesced.merged_tags = core_req.merged_tags; coalesced.write = (core_req.op == MemOp::ST); coalesced.op = core_req.op; coalesced.data = core_req.data; @@ -970,6 +1280,7 @@ class CacheBank : public SimObject { bank_req.hart_id = core_req.hart_id; bank_req.uuid = core_req.uuid; bank_req.req_tag = core_req.tag; + bank_req.merged_tags = core_req.merged_tags; // bank_req.write means "route through the cache write path" — true only // for plain stores. AMOs (op == AMO_SC / AMO RMW) are write-bearing // semantically (memop_is_write returns true) but use the dedicated AMO @@ -988,11 +1299,14 @@ class CacheBank : public SimObject { // stays out of this counter. if (!is_amo_passthru) { ++pending_mshr_size_; - pipe_core_lines_.emplace_back(params_.addr_set_id(core_req.addr), - params_.addr_tag(core_req.addr)); + pipe_core_lines_.push_back({params_.addr_set_id(core_req.addr), + params_.addr_tag(core_req.addr)}); } #if VX_CFG_EXT_A_ENABLED if (is_amo_passthru) ++pending_amo_probes_; + if (is_amo && config_.is_llc && core_req.op != MemOp::AMO_LR) { + amo_busy_ = true; + } #endif if (core_req.is_write()) ++perf_stats_.writes; else ++perf_stats_.reads; @@ -1018,6 +1332,29 @@ class CacheBank : public SimObject { int resident_id = set.find_resident(addr_tag); const bool is_refill = (resident_id != -1); + // Only a refetch finds its sector resident (partial): it merges. A refetch + // whose sector was evicted meanwhile may carry data older than that + // eviction's writeback, so it installs nothing and its chain parks again; + // the retry then fills like any miss (an absent sector cannot be dirtied). + const bool sector_resident = is_refill && set.lines.at(resident_id).sectors.at(sector_id).valid; + const bool refetch_first = bank_req.refetch && !bank_req.refetch_retry; + assert(!sector_resident || refetch_first); + if (refetch_first) { + if (sector_resident) { + // Memory supplies the bytes no store wrote; the sector stays dirty + // in exactly the stored bytes, which are not in memory yet. + auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (bank_req.nodata) { + assert(sector_written(sec) && "looped-back refetch of an incompletely written sector"); + } else { + auto written = sec.data; + sec.data = bank_req.data; + sector_merge(sec, written, sec.dirty_mask); + } + sec.partial = false; + } + return true; + } int line_id; if (is_refill) { @@ -1025,6 +1362,10 @@ class CacheBank : public SimObject { } else { int32_t free_id = -1, repl_id = 0; line_id = set.select_victim(config_.repl_policy, rand_ctr_, &free_id, &repl_id); + // The FIFO pointer names the victim even when another way is free. + if (config_.repl_policy == Cache::FIFO) { + line_id = repl_id; + } auto &victim_line = set.lines.at(line_id); // Count the victim's dirty sectors; the egress queue must accept all @@ -1041,9 +1382,7 @@ class CacheBank : public SimObject { return false; // stall } - if (config_.repl_policy == Cache::FIFO) { - set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); - } else if (config_.repl_policy == Cache::RANDOM) { + if (config_.repl_policy == Cache::RANDOM) { ++rand_ctr_; } @@ -1060,7 +1399,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " writeback: " << wb); ++perf_stats_.evictions; } @@ -1069,24 +1408,54 @@ class CacheBank : public SimObject { victim_line.reset(); victim_line.tag = addr_tag; } + // Every installed fill advances the set's FIFO pointer, a sector refill + // into the resident way included. + if (config_.repl_policy == Cache::FIFO) { + set.fifo_ptr = (set.fifo_ptr + 1) % set.lines.size(); + } auto &line = set.lines.at(line_id); line.tag = addr_tag; line.lru_ctr = 0; auto &sec = line.sectors.at(sector_id); - sec.valid = true; - sec.dirty = false; - sec.dirty_mask = 0; - sec.data = bank_req.data; - mshr_.replay(bank_req.mshr_id); - // arm the fill-forward window for this chain - fwd_active_ = true; - fwd_set_ = set_id; - fwd_tag_ = addr_tag; - fwd_sector_ = sector_id; + if (bank_req.nodata) { + sec.valid = true; + sec.dirty = false; + sec.partial = true; + sec.dirty_mask = 0; + sec.data.reset(); + } else { + sec.valid = true; + sec.dirty = false; + sec.partial = false; + sec.dirty_mask = 0; + sec.data = bank_req.data; + } return true; } + // Track the partial state of the chain a fill just woke. + void setChain(uint32_t set_id, uint64_t addr_tag, uint32_t sector_id, bool partial) { + chain_partial_ = partial; + chain_retry_ = false; + chain_set_ = set_id; + chain_tag_ = addr_tag; + chain_sector_ = sector_id; + } + + // Lazy-fetch loopback depth: room for twice the pipe, so a few queued + // entries leave admission and the refetch loopback open. + static uint32_t lazyq_size(const Cache::Config &config) { + return 1u << log2ceil(2 * config.latency); + } + + // The loopback still fits every request in the pipe: a new request may be + // admitted, and a refetch with no data to read may loop back; one that + // cannot reads memory instead. + bool lazyRoom() const { + return lazy_q_->size() < lazyq_size(config_) - config_.latency; + } + // Fill-forward drain: complete the oldest ready replay directly when it is // a plain read on the armed chain — no pipeline traversal, and the input // slot stays free for new requests. The first write/AMO head closes the @@ -1098,6 +1467,7 @@ class CacheBank : public SimObject { bool processForward() { if (!mshr_.has_ready_reqs()) { fwd_active_ = false; // chain drained + fwd_data_.reset(); return false; } const auto *head = mshr_.peek_next_replay(); @@ -1110,15 +1480,7 @@ class CacheBank : public SimObject { || head->bank_req.write || memop_is_atomic(head->bank_req.op)) { fwd_active_ = false; - return false; - } - // The armed sector is normally resident (the fill just installed it); a - // concurrent invalidate falls back to the replay path, which re-fetches. - auto &set = sets_.at(head->set_id); - int present_id = set.find_resident(head->addr_tag); - uint32_t sector_id = head->sector_id; - if (present_id == -1 || !set.lines.at(present_id).sectors.at(sector_id).valid) { - fwd_active_ = false; + fwd_data_.reset(); return false; } if (crsp_sent_ || this->core_rsp_out.full()) { @@ -1127,7 +1489,8 @@ class CacheBank : public SimObject { bank_req_t req; mshr_.dequeue(&req); MemRsp rsp{req.req_tag, req.hart_id, req.uuid}; - rsp.data = set.lines.at(present_id).sectors.at(sector_id).data; + rsp.data = fwd_data_; + rsp.merged_tags = req.merged_tags; this->core_rsp_out.send(rsp); DT(3, this->name() << " fwd-rsp: " << rsp); return true; @@ -1210,13 +1573,22 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = store_block; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " amo-writethrough: " << w); } // Break other harts' reservations on this line. amo_unit_.invalidate(line_addr, /*except=*/hid); } + // A stored result is written back into the line through the pipe; without + // a store the commit frees the bank at once. + if (do_store) { + amo_wb_pending_ = true; + amo_wb_tick_ = SimPlatform::instance().cycles() + 2; + } else { + amo_busy_ = false; + } + // Send AMO response. MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; rsp.data = rsp_block; @@ -1229,6 +1601,25 @@ class CacheBank : public SimObject { // Pipeline tail: process the head of pipe_req_ — at most one bank_req_t per tick. + // Send a request down the bank's in-order memory queue. + // Debug: a write-through cache holds each write at the head of bank 0's + // queue for a fixed delay, the requests behind it included, so requests + // through the other banks overtake it unless something orders them. + void send_mem_req(const MemReq& req) { + #ifdef DBG_MEM_BANK0_WDELAY + if (bank_id_ == 0 && !config_.write_back) { + constexpr uint64_t DBG_WDELAY = 2048; + uint64_t now = SimPlatform::instance().cycles(); + uint64_t head = std::max(now, dbg_mem_req_free_); + uint64_t leave = head + (req.is_write() ? DBG_WDELAY : 0); + dbg_mem_req_free_ = leave + 1; + this->mem_req_out.send(req, MEM_REQ_DELAY + (leave - now)); + return; + } + #endif + this->mem_req_out.send(req, MEM_REQ_DELAY); + } + // Emit a write-through store downstream, with the optional core response. // All backpressure is checked before any send, so a false return stalls // the caller with no partial effects. @@ -1250,7 +1641,7 @@ class CacheBank : public SimObject { w.uuid = uuid; w.data = data; w.byteen = byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Committed write: break other harts' reservations. @@ -1280,6 +1671,7 @@ class CacheBank : public SimObject { switch (bank_req.type) { case bank_req_t::None: + case bank_req_t::AmoWb: pipe_req_->pop(); return; @@ -1329,7 +1721,7 @@ class CacheBank : public SimObject { wb.uuid = bank_req.uuid; wb.data = sec.data; wb.byteen = sec.dirty_mask; - this->mem_req_out.send(wb, MEM_REQ_DELAY); + this->send_mem_req(wb); DT(3, this->name() << " amo-probe-wb: " << wb); ++perf_stats_.evictions; } @@ -1355,7 +1747,7 @@ class CacheBank : public SimObject { amo_fwd.amo_cmp = bank_req.amo_cmp; amo_fwd.data = bank_req.data; // carries rhs (extracted at LLC) amo_fwd.flags = bank_req.flags; // carry amo_unsigned (signed vs unsigned MIN/MAX) - this->mem_req_out.send(amo_fwd, MEM_REQ_DELAY); + this->send_mem_req(amo_fwd); DT(3, this->name() << " amo-probe-fwd: " << amo_fwd); ++pending_fill_reqs_; // counts as an outstanding mem-roundtrip for perf --pending_amo_probes_; // reservation transferred to the table entry @@ -1368,8 +1760,36 @@ class CacheBank : public SimObject { if (!this->processFill(bank_req)) { return; // stall } - assert(pipe_fill_count_ != 0); - --pipe_fill_count_; + pipe_req_->pop(); + } break; + + case bank_req_t::Park: { + uint32_t set_id = params_.addr_set_id(bank_req.addr); + uint64_t addr_tag = params_.addr_tag(bank_req.addr); + uint32_t sector_id = params_.addr_sector_id(bank_req.addr); + auto &set = sets_.at(set_id); + // A parked head needs memory room whichever way its refetch goes. + if (this->mem_req_out.full()) { + return; // stall + } + int resident_id = set.find_resident(addr_tag); + if (resident_id != -1) { + const auto &sec = set.lines.at(resident_id).sectors.at(sector_id); + if (sec.valid && sec.partial && sector_written(sec) && this->lazyRoom()) { + lazy_q_->push(bank_req.mshr_id); + DT(3, this->name() << " refetch-loop: " << bank_req); + pipe_req_->pop(); + return; + } + } + MemReq fill; + fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); + fill.tag = bank_req.mshr_id; + fill.hart_id = bank_req.hart_id; + fill.uuid = bank_req.uuid; + this->send_mem_req(fill); + DT(3, this->name() << " refetch-req: " << fill); + ++pending_fill_reqs_; pipe_req_->pop(); } break; @@ -1380,8 +1800,10 @@ class CacheBank : public SimObject { auto &set = sets_.at(set_id); int32_t free_id = -1, repl_id = 0; int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); - // Sector hit: line resident AND the requested sector valid. - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // Sector hit: line resident AND the requested sector valid (and complete + // when the access reads it). + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; // The replayed sector is normally present — the fill that woke this // replay just installed it. It can still be gone when a concurrent @@ -1409,14 +1831,15 @@ class CacheBank : public SimObject { return; // stall: need a slot to re-track the miss bank_req_t refill = bank_req; refill.type = bank_req_t::Core; - int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id); + const bool refetch = (present_id != -1) && set.lines.at(present_id).sectors.at(sector_id).valid; + int mshr_id = mshr_.enqueue(refill, set_id, addr_tag, sector_id, refetch); if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); fill.tag = mshr_id; fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); ++pending_fill_reqs_; } pipe_req_->pop(); @@ -1465,6 +1888,7 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " replay-rsp: " << rsp); @@ -1487,7 +1911,11 @@ class CacheBank : public SimObject { int present_id = set.tag_match(addr_tag, config_.repl_policy, rand_ctr_, &free_id, &repl_id); // Sector hit: line resident AND the requested sector valid (a sector miss // on a resident line takes the miss path and refills just that sector). - int hit_id = (present_id != -1 && set.lines.at(present_id).sectors.at(sector_id).valid) ? present_id : -1; + // A load or AMO to a partial sector misses and refetches it. + const bool needs_data = !bank_req.write; + int hit_id = (present_id != -1 && sector_hit(set.lines.at(present_id).sectors.at(sector_id), needs_data)) ? present_id : -1; + const bool partial_resident = (present_id != -1) && (hit_id == -1) + && set.lines.at(present_id).sectors.at(sector_id).valid; if (hit_id != -1) { // Hit while a same-(set,tag,sector) chain still carries a write or @@ -1517,7 +1945,12 @@ class CacheBank : public SimObject { } else { assert(!mshr_.full()); int id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); - mshr_.defer_to_replay(id); + // A chain still awaiting its fill (a refetch of this partial + // sector) wakes this entry itself; only a draining chain needs it + // marked ready here. + if (!mshr_.has_pending_fill(set_id, addr_tag, sector_id, id)) { + mshr_.defer_to_replay(id); + } DT(3, this->name() << " mshr-defer: " << bank_req); } --pending_mshr_size_; @@ -1560,7 +1993,7 @@ class CacheBank : public SimObject { w.uuid = bank_req.uuid; w.data = bank_req.data; w.byteen = bank_req.byteen; - this->mem_req_out.send(w, MEM_REQ_DELAY); + this->send_mem_req(w); DT(3, this->name() << " writethrough: " << w); #if VX_CFG_EXT_A_ENABLED // Writethrough commit: break other harts' reservations. @@ -1582,13 +2015,24 @@ class CacheBank : public SimObject { MemRsp rsp{bank_req.req_tag, bank_req.hart_id, bank_req.uuid}; if (!bank_req.write) rsp.data = hit_sec.data; + rsp.merged_tags = bank_req.merged_tags; this->core_rsp_out.send(rsp); crsp_sent_ = true; DT(3, this->name() << " core-rsp: " << rsp); } } else { // Cache miss. - if (bank_req.write && !config_.write_back) { + if (config_.write_around && bank_req.write && bank_req.flags.noalloc + && !mshr_.lookup(set_id, addr_tag, sector_id)) { + // A no-allocate write miss with no fill pending on its line goes to + // memory as is: nothing is allocated or fetched. + if (!this->emitWritethrough(set_id, addr_tag, sector_id, + bank_req.addr, bank_req.hart_id, bank_req.uuid, + bank_req.data, bank_req.byteen, + need_core_rsp(bank_req), bank_req.req_tag)) { + return; + } + } else if (bank_req.write && !config_.write_back) { // Write-through write miss: forward store to memory and respond // immediately. No fill / no MSHR allocation (this store doesn't // wait for a line). If a fill for this line is already in flight @@ -1620,20 +2064,34 @@ class CacheBank : public SimObject { // entries never mark new waiters, so they must not suppress the // fill for this fresh miss. bool fill_pending = mshr_.has_pending_fill(set_id, addr_tag, sector_id, -1); - if (!fill_pending && this->mem_req_out.full()) + // Lazy fetch: a write-miss root allocates through the loopback + // instead of fetching the sector, and so does the refetch of a + // partial sector every store has already covered. + const bool lazy_alloc = config_.lazy_fetch && bank_req.write && !fill_pending; + const bool refetch_loop = partial_resident && !fill_pending + && sector_written(set.lines.at(present_id).sectors.at(sector_id)) + && this->lazyRoom(); + if ((lazy_alloc || refetch_loop) && lazy_q_->full()) { + return; + } + if (!fill_pending && !lazy_alloc && !refetch_loop && this->mem_req_out.full()) { return; + } assert(!mshr_.full()); - int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id); + int mshr_id = mshr_.enqueue(bank_req, set_id, addr_tag, sector_id, partial_resident); DT(3, this->name() << " mshr-enqueue: " << bank_req); - if (!fill_pending) { + if (lazy_alloc || refetch_loop) { + lazy_q_->push(mshr_id); + DT(3, this->name() << (lazy_alloc ? " lazy-alloc: " : " refetch-loop: ") << bank_req); + } else if (!fill_pending) { MemReq fill; fill.addr = params_.mem_addr_sector(bank_id_, set_id, addr_tag, sector_id); // op defaults to MemOp::LD — fill is a load. fill.tag = mshr_id; // routes the fill response back here fill.hart_id = bank_req.hart_id; fill.uuid = bank_req.uuid; - this->mem_req_out.send(fill, MEM_REQ_DELAY); + this->send_mem_req(fill); DT(3, this->name() << " fill-req: " << fill); ++pending_fill_reqs_; } @@ -1681,7 +2139,7 @@ class CacheBank : public SimObject { mem_req.op = MemOp::ST; mem_req.data = sec.data; mem_req.byteen = sec.dirty_mask; - this->mem_req_out.send(mem_req, MEM_REQ_DELAY); + this->send_mem_req(mem_req); DT(3, this->name() << " flush-wb: " << mem_req); ++perf_stats_.evictions; sec.dirty = false; @@ -1699,6 +2157,59 @@ class CacheBank : public SimObject { DT(3, this->name() << " flush-done"); } + void processTokenFlush() { + // the bank leaves its write-backs, then signals the end, a cycle each + if (token_flush_settle_ != 0) { + if (--token_flush_settle_ == 0) { + token_flushing_ = false; + DT(3, this->name() << (token_flush_skip_ ? " token-quiesce-done" : " token-flush-done")); + } + return; + } + if (pending_fill_reqs_ != 0 + || pipe_req_->size() != 0 + || !mshr_.empty()) { + return; + } + if (token_flush_skip_) { + token_flush_settle_ = TOKEN_FLUSH_SETTLE; + return; + } + uint32_t ways = config_.write_back ? params_.lines_per_set : 1; + uint32_t set_id = token_flush_idx_ % params_.sets_per_bank; + uint32_t way_id = token_flush_idx_ / params_.sets_per_bank; + auto &set = sets_.at(set_id); + if (config_.write_back) { + auto &line = set.lines.at(way_id); + while (flush_sector_idx_ < line.sectors.size()) { + auto &sec = line.sectors.at(flush_sector_idx_); + if (sec.valid && sec.dirty) { + if (this->mem_req_out.full()) { + return; + } + MemReq mem_req; + mem_req.addr = params_.mem_addr_sector(bank_id_, set_id, line.tag, flush_sector_idx_); + mem_req.op = MemOp::ST; + mem_req.data = sec.data; + mem_req.byteen = sec.dirty_mask; + this->send_mem_req(mem_req); + DT(3, this->name() << " flush-wb: " << mem_req); + ++perf_stats_.evictions; + } + ++flush_sector_idx_; + } + flush_sector_idx_ = 0; + line.reset(); + } else { + for (auto &line : set.lines) { + line.reset(); + } + } + if (++token_flush_idx_ == params_.sets_per_bank * ways) { + token_flush_settle_ = TOKEN_FLUSH_SETTLE; + } + } + Cache::Config config_; params_t params_; uint32_t bank_id_; @@ -1706,23 +2217,39 @@ class CacheBank : public SimObject { std::vector sets_; MSHR mshr_; uint32_t pending_mshr_size_; + uint32_t mshr_occupancy_; uint32_t pending_amo_probes_; // AmoProbe requests in the pipe, each holding a table reservation - uint32_t pipe_fill_count_; // fills riding the pipe; gates the next fill's admission - // (set, tag) of Core requests inside the pipe: their MSHR entries exist - // only from pipe exit, so ordering checks against in-flight lines (the - // AMO admission defer) must consult this mirror as well. - std::vector> pipe_core_lines_; + // {set, tag} of Core requests inside the pipe: their MSHR entries exist + // only from pipe exit, so ordering checks against in-flight lines (the AMO + // admission defer) must consult this mirror as well. + struct pipe_line_t { + uint32_t set_id; + uint64_t addr_tag; + }; + std::vector pipe_core_lines_; + + bool pipeLineBusy(uint32_t set_id, uint64_t addr_tag) const { + for (const auto &l : pipe_core_lines_) { + if (l.set_id == set_id && l.addr_tag == addr_tag) { + return true; + } + } + return false; + } + // The pipe is in order, so the oldest match is the request leaving it. void pipeLineErase(uint32_t set_id, uint64_t addr_tag) { for (auto it = pipe_core_lines_.begin(); it != pipe_core_lines_.end(); ++it) { - if (it->first == set_id && it->second == addr_tag) { + if (it->set_id == set_id && it->addr_tag == addr_tag) { pipe_core_lines_.erase(it); return; } } assert(false && "in-pipe line mirror underflow"); } + TFifo::Ptr pipe_req_; + TFifo::Ptr lazy_q_; // roots awaiting a data-less fill: write misses and written refetches Cache::PerfStats perf_stats_; @@ -1732,18 +2259,40 @@ class CacheBank : public SimObject { uint32_t rand_ctr_; uint64_t adm_seq_ctr_; // bank admission-order stamp (see bank_req_t::adm_seq) - // Fill-forward window: chain key armed by the last installed fill. + // Fill-forward window: chain key and sector staged by the last accepted fill. bool fwd_active_; uint32_t fwd_set_; uint64_t fwd_tag_; uint32_t fwd_sector_; + std::shared_ptr fwd_data_; bool crsp_sent_; // core-response port used this tick (pipeline priority) + // Partial state of the chain the last fill woke (lazy fetch). + bool chain_partial_; + bool chain_retry_; // the partial state came from an evicted refetch + uint32_t chain_set_; + uint64_t chain_tag_; + uint32_t chain_sector_; + bool refetch_hold_; // a refetch fill is at its tag lookup: its chain waits + bool refetch_hold_retry_; + bool amo_busy_; // a store-bearing LLC atomic holds core requests off + bool amo_wb_pending_; // its result still has to be written back + uint64_t amo_wb_tick_; // ... from this cycle on +#ifdef DBG_MEM_BANK0_WDELAY + uint64_t dbg_mem_req_free_ = 0; // the memory queue's next free departure cycle +#endif + // Flush walk state. bool flushing_; + bool keep_lines_ = false; // the next launch reset keeps the lines uint32_t flush_set_idx_; uint32_t flush_way_idx_; uint32_t flush_sector_idx_; + bool token_flushing_; + bool token_flush_skip_ = false; + static constexpr uint32_t TOKEN_FLUSH_SETTLE = 2; + uint32_t token_flush_settle_ = 0; + uint32_t token_flush_idx_; #if VX_CFG_EXT_A_ENABLED AmoUnit amo_unit_; @@ -1769,6 +2318,591 @@ class CacheBank : public SimObject { /////////////////////////////////////////////////////////////////////////////// +// The cache's core-port arbiter: a port's request is accepted only when its +// bank grants it, one per bank per cycle, round-robin over the ports, and only +// while the bank's input buffer has room; non-cacheable requests are split off +// for the bypass path. It ticks ahead of the banks, so a grant reaches its bank +// in the same cycle unless the crossbar registers its output. +// With read merging, plain reads of the same word with the same tag from ports +// of one group in the same cycle are served by one bank access: the lowest +// such port leads, the others stay out of arbitration and are accepted with +// it. The merged ports' mask rides below the tag through the bank. Line +// merging takes any plain read of the same line instead, whatever its word +// and tag. +class CachePortArb : public SimObject { +public: + std::vector> ReqIn; // core ports + std::vector> ReqOut; // banks + std::vector> IoOut; // non-cacheable requests, per port + + CachePortArb(const SimContext& ctx, const char* name, const params_t& params, + uint32_t num_inputs, uint32_t num_banks, uint32_t merge_group, + bool merge_line) + : SimObject(ctx, name) + , ReqIn(num_inputs, this) + , ReqOut(num_banks, this) + , IoOut(num_inputs, this) + , params_(params) + , lg2_inputs_(log2ceil(num_inputs)) + // the request crossbar registers its output when it has more than two inputs + , delay_((num_inputs > 2) ? 1 : 0) + , levels_(arb_tree_levels(num_inputs)) + , merge_group_(merge_group) + , merge_line_(merge_line) + { + __assert(merge_group_ <= 1 || (merge_group_ <= 64 && ispow2(merge_group_) && (num_inputs % merge_group_) == 0), + "a read-merge group must be a power of two of at most 64 dividing the ports"); + tree_stage_.reserve(levels_.size()); + for (auto slices : levels_) { + tree_stage_.emplace_back(num_banks * slices, this); + } + rr_.resize(levels_.size() + 1); + for (uint32_t s = 0; s <= levels_.size(); ++s) { + rr_.at(s).resize(num_banks * this->arb_slices(s)); + } + } + + // Without a registered tree level the hardware crossbar is combinational + // (no request buffer ahead of a bank): a request stays at its port until the + // bank takes it, where a later same-word read can still merge with it. The + // arbiter then offers each bank one request per cycle and dequeues the ports + // only when the bank pops the offer; an offer still pending next cycle is + // withdrawn and the ports are arbitrated afresh. + void attach_bank(uint32_t b, SimChannel* bank_in) { + if (!this->offers_enabled()) { + return; + } + offers_.resize(ReqOut.size()); + offers_.at(b).bank_in = bank_in; + bank_in->pop_callback([this, b]() { this->on_bank_pop(b); }); + } + + // Bits of the merged-port mask carried below the request tag. + uint32_t merge_bits() const { return (merge_group_ > 1) ? merge_group_ : 0; } + + uint64_t collisions() const { return collisions_; } + + bool is_token(uint32_t i) const { + return !ReqIn.at(i).empty() && ReqIn.at(i).peek().flags.flush; + } + + // While any port's head is a flush token, cacheable requests are held at + // every port. + bool any_token() const { + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (this->is_token(i)) { + return true; + } + } + return false; + } + + // Ports whose head is a flush token. + std::vector tokens() const { + std::vector mask(ReqIn.size(), false); + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + mask.at(i) = this->is_token(i); + } + return mask; + } + + // Tokens owing no walk: IO and release tokens, and L1-only or acquire ones + // unless they still need a walk here. + std::vector tokens(bool l1_needs_walk) const { + std::vector mask(ReqIn.size(), false); + for (uint32_t i = 0, n = ReqIn.size(); i < n; ++i) { + if (!this->is_token(i)) { + continue; + } + auto &flags = ReqIn.at(i).peek().flags; + bool io = flags.io && !flags.flush_l1; + bool rel = !flags.io && flags.flush_l1 && flags.noalloc; + mask.at(i) = io || rel || (flags.flush_l1 && !l1_needs_walk); + } + return mask; + } + + // No cacheable request is left between the ports and the banks. + bool drained() const { + for (auto &level : tree_stage_) { + for (auto &stage : level) { + if (stage.size() != 0) { + return false; + } + } + } + for (auto &out : ReqOut) { + if (out.size() != 0) { + return false; + } + } + return true; + } + + MemReq take_token(uint32_t i) { + auto req = ReqIn.at(i).peek(); + ReqIn.at(i).pop(); + return req; + } + + // The cache's line initialization holds the core ports for these cycles. + void set_init_cycles(uint32_t cycles) { init_cycles_ = cycles; } + +protected: + void on_reset() { + for (auto &rr : rr_) { + std::fill(rr.begin(), rr.end(), 0); + } + collisions_ = 0; + for (auto& offer : offers_) { + offer.valid = false; + offer.withdrawing = false; + } + } + + // Slices of the arbiter level s: level 0 grants the ports, each level above + // the buffers of the level below, and the last level is the join arbiter. + uint32_t arb_slices(uint32_t s) const { + return (s < levels_.size()) ? levels_.at(s) : 1; + } + + // Inputs of slice sl of the arbiter level s. + uint32_t arb_slice_size(uint32_t s, uint32_t sl) const { + uint32_t inputs = (s == 0) ? ReqIn.size() : levels_.at(s - 1); + if (s == levels_.size()) { + return inputs; + } + return std::min(kMaxFanout, inputs - sl * kMaxFanout); + } + + void on_tick() { + if (init_cycles_ != 0) { + --init_cycles_; + return; + } + uint32_t nb = ReqOut.size(); + uint32_t nl = levels_.size(); + for (auto& offer : offers_) { + if (offer.valid) { + offer.withdrawing = true; + offer.bank_in->pop(); + offer.withdrawing = false; + offer.valid = false; + } + } + // the tree's buffered levels, join first: each slice passes one buffer of + // the level below per cycle + for (uint32_t s = nl; s > 0; --s) { + uint32_t slices = this->arb_slices(s); + uint32_t inputs = levels_.at(s - 1); + for (uint32_t b = 0; b < nb; ++b) { + for (uint32_t sl = 0; sl < slices; ++sl) { + auto& out = (s < nl) ? tree_stage_.at(s).at(b * slices + sl) : ReqOut.at(b); + if (out.full()) { + continue; + } + uint32_t lo = (s < nl) ? sl * kMaxFanout : 0; + uint32_t size = this->arb_slice_size(s, sl); + auto& rr = rr_.at(s).at(b * slices + sl); + for (uint32_t k = 0; k < size; ++k) { + uint32_t j = (rr + k) % size; + auto& in = tree_stage_.at(s - 1).at(b * inputs + lo + j); + if (!in.empty()) { + out.send(in.peek(), (s < nl) ? 1 : delay_); + in.pop(); + rr = (j + 1) % size; + break; + } + } + } + } + } + uint32_t n = ReqIn.size(); + const bool hold = this->any_token(); + // merged ports riding each leader's bank access, by offset in its group + std::vector followers(n, false); + std::vector merged(n, 0); + if (merge_group_ > 1 && !hold) { + for (uint32_t i = 0; i < n; ++i) { + if (!this->is_plain_read(i)) { + continue; + } + auto& req_i = ReqIn.at(i).peek(); + uint32_t base = (i / merge_group_) * merge_group_; + for (uint32_t j = base; j < i; ++j) { + if (followers.at(j) || !this->is_plain_read(j)) { + continue; + } + auto& req_j = ReqIn.at(j).peek(); + bool same = merge_line_ + ? ((req_j.addr >> params_.bank_select_addr_start) == (req_i.addr >> params_.bank_select_addr_start)) + : ((req_j.addr >> params_.word_select_addr_start) == (req_i.addr >> params_.word_select_addr_start) + && req_j.tag == req_i.tag); + if (same) { + merged.at(j) |= 1ull << (i - base); + followers.at(i) = true; + break; + } + } + } + } + // requesting ports per bank and first-level slice; non-cacheable requests + // leave on their own port + uint32_t slices = this->arb_slices(0); + uint32_t slice_ports = (nl != 0) ? kMaxFanout : n; + std::vector want(nb * slices, 0); + std::vector wants(nb, 0); + for (uint32_t i = 0; i < n; ++i) { + auto& in = ReqIn.at(i); + if (in.empty() || followers.at(i)) { + continue; + } + auto& req = in.peek(); + if (req.flags.flush) { + continue; + } + if (req.flags.io) { + if (IoOut.at(i).try_send(req, 0)) { + in.pop(); + } + continue; + } + if (!hold) { + uint32_t b = params_.addr_bank_id(req.addr); + want.at(b * slices + i / slice_ports) |= 1u << (i % slice_ports); + ++wants.at(b); + } + } + // each bank's arbiter, or each slice of its tree, grants one port per cycle + for (uint32_t b = 0; b < nb; ++b) { + if (wants.at(b) == 0) { + continue; + } + bool granted = false; + for (uint32_t sl = 0; sl < slices; ++sl) { + uint32_t slice_want = want.at(b * slices + sl); + auto& out = (nl != 0) ? tree_stage_.at(0).at(b * slices + sl) : ReqOut.at(b); + if (slice_want == 0 || out.full()) { + continue; + } + uint32_t size = this->arb_slice_size(0, sl); + auto& rr = rr_.at(0).at(b * slices + sl); + uint32_t j = rr; + while (!(slice_want & (1u << j))) { + j = (j + 1) % size; + } + if (this->offers_enabled()) { + this->offer(b, sl * slice_ports + j, out, merged, wants.at(b) - 1, &rr, (j + 1) % size); + continue; + } + this->grant(sl * slice_ports + j, out, merged); + rr = (j + 1) % size; + granted = true; + } + if (granted) { + collisions_ += wants.at(b) - 1; + } + } + } + + // Offer port i's request, with the ports merged into it, to bank b; the + // ports are dequeued when the bank takes it (on_bank_pop). + void offer(uint32_t b, uint32_t i, SimChannel& out, const std::vector& merged, + uint32_t collisions, uint32_t* rr, uint32_t rr_next) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = (merge_group_ > 1) ? merged.at(i) : 0; + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + req.merged_tags = this->merged_tags(i, mask); + auto& offer = offers_.at(b); + offer.valid = true; + offer.port = i; + offer.mask = mask; + offer.collisions = collisions; + offer.rr = rr; + offer.rr_next = rr_next; + out.send(req, 0); + } + + void on_bank_pop(uint32_t b) { + auto& offer = offers_.at(b); + if (offer.withdrawing) { + return; + } + __assert(offer.valid, "bank took a request the arbiter did not offer"); + uint32_t i = offer.port; + if (offer.mask != 0) { + uint32_t base = (i / merge_group_) * merge_group_; + for (uint64_t f = offer.mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << ReqIn.at(port).peek()); + ReqIn.at(port).pop(); + } + } + ReqIn.at(i).pop(); + *offer.rr = offer.rr_next; + collisions_ += offer.collisions; + offer.valid = false; + } + + bool offers_enabled() const { return levels_.empty() && delay_ == 0; } + + // Under line merging, the merged ports' own tags, which ride the leader's + // bank access and its response. + std::shared_ptr> merged_tags(uint32_t i, uint64_t mask) const { + if (!merge_line_ || mask == 0) { + return nullptr; + } + uint32_t base = (i / merge_group_) * merge_group_; + auto tags = std::make_shared>(merge_group_, 0); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t k = __builtin_ctzll(f); + tags->at(k) = ReqIn.at(base + k).peek().tag; + } + return tags; + } + + // Send port i's request to its bank, with the ports merged into it. + void grant(uint32_t i, SimChannel& out, const std::vector& merged) { + auto req = ReqIn.at(i).peek(); + uint64_t mask = 0; + if (merge_group_ > 1) { + uint32_t base = (i / merge_group_) * merge_group_; + mask = merged.at(i); + req.merged_tags = this->merged_tags(i, mask); + for (uint64_t f = mask; f != 0; f &= f - 1) { + uint32_t port = base + __builtin_ctzll(f); + DT(3, this->name() << " read-merge: port=" << port << ", leader=" << i << ", " << req); + ReqIn.at(port).pop(); + } + } + uint64_t tag = (((uint64_t(req.tag) << this->merge_bits()) | mask) << lg2_inputs_) | i; + __assert((tag >> 32) == 0, "cache bank tag overflow"); + req.tag = uint32_t(tag); + out.send(req, levels_.empty() ? delay_ : 1); + ReqIn.at(i).pop(); + } + + // A plain read: no write, flush, IO, local or atomic attribute. + bool is_plain_read(uint32_t i) const { + if (ReqIn.at(i).empty()) { + return false; + } + auto& req = ReqIn.at(i).peek(); + return req.op == MemOp::LD && !req.flags.flush && !req.flags.io && !req.flags.local; + } + +private: + params_t params_; + uint32_t lg2_inputs_; + uint32_t delay_; + uint32_t init_cycles_ = 0; + std::vector levels_; // [level] slices per bank + std::vector>> tree_stage_; // [level][bank][slice] + std::vector> rr_; // [arbiter level][bank][slice] next to grant + uint32_t merge_group_; + bool merge_line_; + uint64_t collisions_ = 0; + struct offer_t { + SimChannel* bank_in = nullptr; + bool valid = false; + bool withdrawing = false; + uint32_t port = 0; + uint64_t mask = 0; + uint32_t collisions = 0; + uint32_t* rr = nullptr; + uint32_t rr_next = 0; + }; + std::vector offers_; // [bank], combinational crossbar only + + friend class SimObject; +}; + +// The core-response network under read merging: a flat crossbar of one +// round-robin arbiter per core port over the bank lanes. A merged read's +// response goes to the port that sent it and to every merged port; each port +// takes it when its arbiter grants that lane, and the lane is released once +// all of them have. Over many banks a port's arbiter is a tree: each slice of +// lanes grants into its own buffer, and a join picks among the slices. +class CacheRspMcast : public SimObject { +public: + using Ptr = std::shared_ptr; + + std::vector> RspIn; // banks + std::vector> RspOut; // core ports + + CacheRspMcast(const SimContext& ctx, const char* name, uint32_t num_banks, + uint32_t num_outputs, uint32_t merge_group, bool merge_line) + : SimObject(ctx, name) + , RspIn(num_banks, this) + , RspOut(num_outputs, this) + , lg2_outputs_(log2ceil(num_outputs)) + , merge_group_(merge_group) + , merge_line_(merge_line) + , slices_(arb_tree_slices(num_banks)) + , sent_(num_banks, 0) + , last_grant_(num_outputs * slices_, num_banks - 1) + , join_rr_(num_outputs, slices_ - 1) + , tree_stage_((slices_ > 1) ? num_outputs * slices_ : 0, this) + { + __assert(num_outputs <= 64 && merge_group < 64, "the read-merge response network takes at most 64 ports"); + __assert(arb_tree_levels(num_banks).size() <= 1, "the read-merge response network takes one tree level"); + } + +protected: + void on_reset() { + std::fill(sent_.begin(), sent_.end(), 0); + std::fill(last_grant_.begin(), last_grant_.end(), RspIn.size() - 1); + std::fill(join_rr_.begin(), join_rr_.end(), slices_ - 1); + } + + void on_tick() { + uint32_t nb = RspIn.size(); + uint32_t no = RspOut.size(); + bool idle = true; + // the tree's join level: one slice buffer per port per cycle + for (uint32_t o = 0; o < no && slices_ > 1; ++o) { + for (uint32_t k = 1; k <= slices_; ++k) { + uint32_t sl = (join_rr_.at(o) + k) % slices_; + auto& stage = tree_stage_.at(o * slices_ + sl); + if (stage.empty()) { + continue; + } + if (RspOut.at(o).try_send(stage.peek(), 0)) { + stage.pop(); + join_rr_.at(o) = sl; + } + break; + } + } + for (auto& stage : tree_stage_) { + idle &= (stage.size() == 0); + } + std::vector pending(nb, 0); + for (uint32_t b = 0; b < nb; ++b) { + if (RspIn.at(b).empty()) { + continue; + } + idle = false; + auto tag = RspIn.at(b).peek().tag; + uint32_t port = tag & ((1u << lg2_outputs_) - 1); + uint64_t mask = (tag >> lg2_outputs_) & ((1ull << merge_group_) - 1); + uint64_t targets = (1ull << port) | (mask << ((port / merge_group_) * merge_group_)); + pending.at(b) = targets & ~sent_.at(b); + } + if (idle) { + this->tick_sleep(); + return; + } + for (uint32_t o = 0; o < no; ++o) { + for (uint32_t sl = 0; sl < slices_; ++sl) { + uint32_t lo = (slices_ > 1) ? sl * kMaxFanout : 0; + uint32_t hi = (slices_ > 1) ? std::min(lo + kMaxFanout, nb) : nb; + auto& out = (slices_ > 1) ? tree_stage_.at(o * slices_ + sl) : RspOut.at(o); + auto& rr = last_grant_.at(o * slices_ + sl); + for (uint32_t k = 1; k <= nb; ++k) { + uint32_t b = (rr + k) % nb; + if (b < lo || b >= hi || !(pending.at(b) & (1ull << o))) { + continue; + } + auto rsp = RspIn.at(b).peek(); + uint32_t lane_tag = rsp.tag; + rsp.tag >>= (lg2_outputs_ + merge_group_); + if (merge_line_ && o != (lane_tag & ((1u << lg2_outputs_) - 1))) { + __assert(rsp.merged_tags, "a line-merged response lost its merged tags"); + rsp.tag = rsp.merged_tags->at(o % merge_group_); + } + if (out.try_send(rsp, (slices_ > 1) ? 1 : 0)) { + DT(4, this->name() << " rsp" << b << "_" << o << ": " << rsp); + sent_.at(b) |= 1ull << o; + rr = b; + } + break; + } + } + } + for (uint32_t b = 0; b < nb; ++b) { + if (pending.at(b) != 0 && (pending.at(b) & ~sent_.at(b)) == 0) { + RspIn.at(b).pop(); + sent_.at(b) = 0; + } + } + } + +private: + uint32_t lg2_outputs_; + uint32_t merge_group_; + bool merge_line_; + uint32_t slices_; + std::vector sent_; // [bank] ports that took the lane's response + std::vector last_grant_; // [port][slice] round-robin pointer + std::vector join_rr_; // [port] + std::vector> tree_stage_; // [port][slice] + + friend class SimObject; +}; + +// Sits on one memory port's cache-side response path, ahead of the response +// network. While the port's flush token is out the cache is quiescent, so the +// port's next response is the token's answer and stops here; every other +// response passes on, crossing the port's registered response stages. +class FlushTap : public SimObject { +public: + using Ptr = std::shared_ptr; + + SimChannel RspIn; + SimChannel RspOut; + + FlushTap(const SimContext& ctx, const char* name, uint32_t delay) + : SimObject(ctx, name) + , RspIn(this) + , RspOut(this) + , delay_(delay) + {} + + // the port's flush token has been sent + void arm() { + armed_ = true; + answered_ = false; + } + + // the token's answer has arrived; reading it disarms the tap + bool take_answer() { + bool answered = answered_; + answered_ = false; + return answered; + } + +protected: + void on_reset() { + armed_ = false; + answered_ = false; + } + + void on_tick() { + if (RspIn.empty()) { + this->tick_sleep(); + return; + } + if (armed_) { + DT(3, this->name() << " flush-fwd-rsp"); + armed_ = false; + answered_ = true; + RspIn.pop(); + } else if (RspOut.try_send(RspIn.peek(), delay_)) { + RspIn.pop(); + } + } + +private: + uint32_t delay_; + bool armed_ = false; + bool answered_ = false; + + friend class SimObject; +}; + class Cache::Impl { public: Impl(Cache *simobject, const Config &config) @@ -1797,28 +2931,92 @@ class Cache::Impl { nc_mem_arbs_.at(i) = MemArbiter::Create(sname, ArbiterType::Priority, 2, 1); } + // Bank memory arbiter delays. Each level of the memory-side request + // network beyond the first is a registered stage; the request path + // buffers the levels the arbiter's own register does not cover in + // mem_req_net_. + uint32_t mem_arb_delay = ((num_banks != config_.mem_ports) && (num_banks > 2 || config_.mem_ports > 2)) ? 1 : 0; + uint32_t mem_net_delay = omega_stages(num_banks, config_.mem_ports) - 1; + uint32_t mem_net_extra = (mem_net_delay > mem_arb_delay) ? (mem_net_delay - mem_arb_delay) : 0; + uint32_t mem_arb_rsp_delay = (num_banks != config_.mem_ports) ? (mem_arb_delay + mem_net_extra) : 0; + __assert(num_banks >= config_.mem_ports, "a cache needs a bank per memory port"); + + // A memory response reaches its bank through the port's response queue, + // when there is one, and a response network registered at every level. + // The taps carry what the bank arbiter's response delay does not. + uint32_t mem_rsp_regs = omega_stages(config_.mem_ports, num_banks) + (config_.mem_rsp_queue ? 1 : 0); + uint32_t mem_rsp_credit = mem_arb_rsp_delay + (config_.mem_rsp_slice ? 1 : 0); + __assert(mem_rsp_regs >= mem_rsp_credit, "memory response path shorter than its arbiter"); + uint32_t tap_delay = mem_rsp_regs - mem_rsp_credit; + + // flush-token taps, ticking after the arbiters that feed them and before + // the response network they feed + flush_taps_.resize(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + snprintf(sname, 100, "%s-flush_tap%d", simobject->name().c_str(), i); + flush_taps_.at(i) = FlushTap::Create(sname, tap_delay); + nc_mem_arbs_.at(i)->RspOut.at(0).bind(&flush_taps_.at(i)->RspIn); + } + // Connect non-cacheable arbiter output port 0 to outgoing memory ports for (uint32_t i = 0; i < config_.mem_ports; ++i) { nc_mem_arbs_.at(i)->ReqOut.at(0).bind(&simobject->mem_req_out.at(i)); simobject->mem_rsp_in.at(i).bind(&nc_mem_arbs_.at(i)->RspIn.at(0)); } - // Create bank's memory arbiter snprintf(sname, 100, "%s-mem_arb", simobject->name().c_str()); - auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports); + auto bank_mem_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_banks, config_.mem_ports, + mem_arb_delay + mem_net_extra, mem_arb_rsp_delay); // Connect bank's memory arbiter to non-cacheable arbiter's input port 0 + if (mem_net_extra != 0) { + mem_req_net_.reserve(config_.mem_ports); + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + mem_req_net_.emplace_back(simobject, mem_arb_delay + mem_net_extra + 1); + } + } for (uint32_t i = 0; i < config_.mem_ports; ++i) { - bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); - nc_mem_arbs_.at(i)->RspOut.at(0).bind(&bank_mem_arb->RspIn.at(i)); + if (mem_net_extra != 0) { + bank_mem_arb->ReqOut.at(i).bind(&mem_req_net_.at(i)); + } else { + bank_mem_arb->ReqOut.at(i).bind(&nc_mem_arbs_.at(i)->ReqIn.at(0)); + } + } + for (uint32_t i = 0; i < config_.mem_ports; ++i) { + flush_taps_.at(i)->RspOut.bind(&bank_mem_arb->RspIn.at(i)); + } + bank_arb_ = bank_mem_arb; + + // Create the core-port arbiter ahead of the banks it feeds + snprintf(sname, 100, "%s-port_arb", simobject->name().c_str()); + uint32_t merge_group = std::min(config_.read_merge, config_.num_inputs); + const bool merge_line = config_.merge_line && merge_group > 1; + port_arb_ = CachePortArb::Create(sname, params_, config_.num_inputs, num_banks, merge_group, merge_line); + for (uint32_t i = 0; i < config_.num_inputs; ++i) { + simobject->core_req_in.at(i).bind(&port_arb_->ReqIn.at(i)); } - // Create bank's core crossbar + // Create bank's core crossbar; it carries the bank responses back to the + // core ports. A registered core-response port is a network registered at + // every level, applied as the response leaves the cache. snprintf(sname, 100, "%s-core_xbar", simobject->name().c_str()); - bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, - [&](const MemCrossBar::ReqType &req) { - return params_.addr_bank_id(req.addr); - }); + auto bank_sel = [&](const MemCrossBar::ReqType &req) { + return params_.addr_bank_id(req.addr); + }; + if (port_arb_->merge_bits() != 0) { + // one buffered arbiter per core port over the bank lanes + __assert(config_.core_out_buf, "read merging needs a registered core-response port"); + rsp_mcast_ = CacheRspMcast::Create(sname, num_banks, config_.num_inputs, port_arb_->merge_bits(), merge_line); + core_rsp_delay_ = 1; + } else if (config_.core_out_buf) { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel, 0, 0); + core_rsp_delay_ = omega_stages(num_banks, config_.num_inputs); + } else { + bank_core_xbar_ = MemCrossBar::Create(sname, ArbiterType::RoundRobin, config_.num_inputs, num_banks, bank_sel); + } + if (config_.nc_enable) { + ++core_rsp_delay_; + } // Create cache banks for (uint32_t i = 0, n = num_banks; i < n; ++i) { @@ -1826,12 +3024,13 @@ class Cache::Impl { banks_.at(i) = CacheBank::Create(sname, config, params_, i); // bind core ports - bank_core_xbar_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); - banks_.at(i)->core_rsp_out.bind(&bank_core_xbar_->RspIn.at(i)); + port_arb_->ReqOut.at(i).bind(&banks_.at(i)->core_req_in); + port_arb_->attach_bank(i, &banks_.at(i)->core_req_in); + banks_.at(i)->core_rsp_out.bind(rsp_mcast_ ? &rsp_mcast_->RspIn.at(i) : &bank_core_xbar_->RspIn.at(i)); // bind memory ports - banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(i)); - bank_mem_arb->RspOut.at(i).bind(&banks_.at(i)->mem_rsp_in); + banks_.at(i)->mem_req_out.bind(&bank_mem_arb->ReqIn.at(this->bank_lane(i))); + bank_mem_arb->RspOut.at(this->bank_lane(i)).bind(&banks_.at(i)->mem_rsp_in); } } @@ -1839,7 +3038,9 @@ class Cache::Impl { if (config_.bypass) return; // calculate cache initialization cycles - init_cycles_ = params_.sets_per_bank; + init_cycles_ = keep_lines_ ? 0 : params_.sets_per_bank; + port_arb_->set_init_cycles(init_cycles_); + keep_lines_ = false; } void tick() { @@ -1865,34 +3066,218 @@ class Cache::Impl { } } + this->flush_step(); + + // bank memory requests through the memory-side network's deeper levels + for (uint32_t i = 0, n = mem_req_net_.size(); i < n; ++i) { + auto &net = mem_req_net_.at(i); + if (!net.empty() && nc_mem_arbs_.at(i)->ReqIn.at(0).try_send(net.peek(), 0)) { + net.pop(); + } + } + // schedule core responses for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &bank_rsp = bank_core_xbar_->RspOut.at(req_id); + auto &bank_rsp = rsp_mcast_ ? rsp_mcast_->RspOut.at(req_id) : bank_core_xbar_->RspOut.at(req_id); if (bank_rsp.empty()) continue; auto &core_rsp = bank_rsp.peek(); - if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, 0)) { + if (simobject_->core_rsp_out.at(req_id).try_send(core_rsp, core_rsp_delay_)) { DT(3, simobject_->name() << " core-rsp: " << core_rsp); bank_rsp.pop(); } } - // schedule core requests + // non-cacheable core requests split off by the port arbiter for (uint32_t req_id = 0, n = config_.num_inputs; req_id < n; ++req_id) { - auto &core_req_in = simobject_->core_req_in.at(req_id); - if (core_req_in.empty()) - continue; - auto &core_req = core_req_in.peek(); - if (core_req.flags.io) { - if (this->processBypassRequest(core_req, req_id)) { - core_req_in.pop(); + auto &io_req = port_arb_->IoOut.at(req_id); + if (!io_req.empty() && this->processBypassRequest(io_req.peek(), req_id)) { + io_req.pop(); + } + } + } + + // Flush tokens (fences) at any port start one walk: the ports hold their + // cacheable requests, in-flight ones drain, every bank writes back and + // invalidates its lines, a token goes down each memory port behind that + // port's write-backs, and once every port has answered, each token + // visible at the end of the walk is answered through the response path, + // one per cycle. + void flush_step() { + switch (flush_state_) { + case FlushState::Idle: + if (port_arb_->any_token()) { + flush_state_ = FlushState::Drain; + } + break; + case FlushState::Drain: { + if (!port_arb_->drained()) { + break; + } + for (auto &bank : banks_) { + if (bank->core_req_in.size() != 0) { + return; } + } + auto kinds = this->token_kinds(); + bool walk = kinds.walk(config_.shared_llc); + if (!walk && !kinds.drain(config_.shared_llc)) { + rsp_pending_ = port_arb_->tokens(); + flush_state_ = FlushState::Respond; + break; + } + // with no walk owed, an IO or release token still quiesces the banks + // and goes down + walk_skip_ = !walk; + for (auto &bank : banks_) { + bank->token_flush_begin(walk_skip_); + } + flush_state_ = FlushState::Walk; + break; + } + case FlushState::Walk: { + for (auto &bank : banks_) { + if (!bank->token_flush_done()) { + return; + } + } + auto kinds = this->token_kinds(); + // a quiesce without a walk serves only the tokens owing none, and + // forwards IO if it serves an IO token, else release + rsp_pending_ = walk_skip_ ? port_arb_->tokens(!config_.shared_llc) : port_arb_->tokens(); + // after a walk: full if a full token or an IO one asks for more than the + // point of coherence, L1-only if any token owes a drain to it, acquire + // if only acquire tokens were visible + bool fwd_acq = !walk_skip_ && kinds.any_acq && !kinds.any_full && !kinds.any_io + && !kinds.any_l1 && !kinds.any_rel; + bool fwd_l1 = !walk_skip_ && (kinds.any_l1 || kinds.any_rel) && !kinds.any_full && !kinds.any_io; + if (walk_skip_) { + this->forward_begin(kinds.any_io, !kinds.any_io, !kinds.any_io); } else { - if (bank_core_xbar_->ReqIn.at(req_id).try_send(core_req, 0)) { - core_req_in.pop(); + this->forward_begin(fwd_acq, fwd_l1 || fwd_acq, false); + } + // an acquire walk owes the next level nothing unless it too may hold + // stale lines + flush_state_ = (fwd_acq && !config_.acq_forward) ? FlushState::Respond : FlushState::Forward; + break; + } + case FlushState::Forward: { + bool mem_req_empty = true; + for (auto &lane : bank_arb_->ReqIn) { + mem_req_empty &= (lane.size() == 0); + } + for (auto &net : mem_req_net_) { + mem_req_empty &= (net.size() == 0); + } + // an IO token also waits for the non-cacheable path, which carries the + // IO writes it orders + if (fwd_io_ && !fwd_l1_ && mem_req_empty) { + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + mem_req_empty &= (port_arb_->IoOut.at(i).size() == 0); + } + for (auto &arb : nc_mem_arbs_) { + mem_req_empty &= (arb->ReqIn.at(0).size() == 0) && (arb->ReqIn.at(1).size() == 0); + } + } + fwd_go_ |= mem_req_empty; + for (uint32_t i = 0, n = config_.mem_ports; i < n && fwd_go_; ++i) { + if (fwd_sent_ & (1ull << i)) { + continue; + } + // the token for port i is a read of the port's first line, sent down + // bank i's lane (bank i maps to port i) behind its write-backs + MemReq token; + token.addr = uint64_t(i) << config_.L; + token.flags.flush = 1; + token.flags.io = fwd_io_; + token.flags.flush_l1 = fwd_l1_; + token.flags.noalloc = fwd_noalloc_; + token.tag = 0; + if (bank_arb_->ReqIn.at(this->bank_lane(i)).try_send(token, 1)) { + DT(3, simobject_->name() << " flush-fwd: " << token); + flush_taps_.at(i)->arm(); + fwd_sent_ |= 1ull << i; + } + } + for (uint32_t i = 0, n = config_.mem_ports; i < n; ++i) { + if (flush_taps_.at(i)->take_answer()) { + fwd_done_ |= 1ull << i; + } + } + uint64_t all = (config_.mem_ports < 64) ? ((1ull << config_.mem_ports) - 1) : ~0ull; + if (fwd_done_ == all) { + flush_state_ = FlushState::Respond; + } + break; + } + case FlushState::Respond: { + auto next = std::find(rsp_pending_.begin(), rsp_pending_.end(), true); + if (next == rsp_pending_.end()) { + flush_state_ = FlushState::Idle; + break; + } + uint32_t i = std::distance(rsp_pending_.begin(), next); + auto &token = port_arb_->ReqIn.at(i).peek(); + MemRsp rsp{token.tag, token.hart_id, token.uuid}; + if (simobject_->core_rsp_out.at(i).try_send(rsp, core_rsp_delay_)) { + DT(3, simobject_->name() << " flush-rsp: " << rsp); + port_arb_->take_token(i); + rsp_pending_.at(i) = false; + } + break; + } + } + } + + struct token_kinds_t { + bool any_full = false; + bool any_l1 = false; + bool any_io = false; + bool any_acq = false; + bool any_rel = false; + // a walk is owed to a full token, and to an L1-only or acquire one above + // the point of coherence + bool walk(bool shared_llc) const { return any_full || ((any_l1 || any_acq) && !shared_llc); } + // a drain with no walk is owed to an IO token, and to a release one above + // the point of coherence + bool drain(bool shared_llc) const { return any_io || (any_rel && !shared_llc); } + }; + + // Kinds of the visible tokens. Several kinds together combine to the + // weakest one that serves them all. + token_kinds_t token_kinds() const { + token_kinds_t kinds; + for (uint32_t i = 0, n = config_.num_inputs; i < n; ++i) { + if (!port_arb_->is_token(i)) { + continue; + } + auto &flags = port_arb_->ReqIn.at(i).peek().flags; + if (flags.io) { + if (flags.flush_l1) { + kinds.any_acq = true; + } else { + kinds.any_io = true; + } + } else if (flags.flush_l1) { + if (flags.noalloc) { + kinds.any_rel = true; + } else { + kinds.any_l1 = true; } + } else { + kinds.any_full = true; } } + return kinds; + } + + void forward_begin(bool io, bool l1_only, bool noalloc) { + fwd_io_ = io; + fwd_l1_ = l1_only; + fwd_noalloc_ = noalloc; + fwd_go_ = false; + fwd_sent_ = 0; + fwd_done_ = 0; } PerfStats perf_stats() const { @@ -1901,15 +3286,35 @@ class Cache::Impl { for (const auto &bank : banks_) { perf_stats += bank->perf_stats(); } - perf_stats.bank_stalls = bank_core_xbar_->collisions(); + perf_stats.bank_stalls = port_arb_->collisions(); } return perf_stats; } - void flush_begin() { + void invalidate() { + if (config_.bypass) { + return; + } + for (auto &bank : banks_) { + bank->invalidate(); + } + } + + void flush_begin(bool l1_only) { if (config_.bypass) return; + // The shared last-level cache stays the point of coherence across an + // L1-only flush and keeps its lines into the next launch. + keep_lines_ = l1_only && config_.shared_llc; for (auto &bank : banks_) { - bank->flush_begin(); + bank->flush_begin(keep_lines_); + } + } + + void keep_lines() { + if (config_.bypass || !config_.shared_llc) return; + keep_lines_ = true; + for (auto &bank : banks_) { + bank->flush_begin(true); } } @@ -1954,8 +3359,34 @@ class Cache::Impl { std::vector banks_; MemArbiter::Ptr bank_arb_; std::vector nc_mem_arbs_; + CachePortArb::Ptr port_arb_; MemCrossBar::Ptr bank_core_xbar_; + CacheRspMcast::Ptr rsp_mcast_; + // the core-response network's levels, then the non-cacheable bypass + // layer's response arbiter + uint32_t core_rsp_delay_ = 0; + enum class FlushState { Idle, Drain, Walk, Forward, Respond }; + FlushState flush_state_ = FlushState::Idle; + std::vector rsp_pending_; + uint64_t fwd_sent_ = 0; + uint64_t fwd_done_ = 0; + bool fwd_go_ = false; // nothing is left ahead of the forwarded tokens + bool fwd_io_ = false; // the forwarded tokens' kind + bool fwd_l1_ = false; + bool fwd_noalloc_ = false; + bool walk_skip_ = false; // the banks only quiesce: no walk is owed + std::vector> mem_req_net_; + std::vector flush_taps_; + + // Bank i drives memory port i mod mem_ports: its lane into the bank-to-port + // arbiter, which serves each port from a contiguous range of lanes. + uint32_t bank_lane(uint32_t bank_id) const { + uint32_t num_banks = 1 << config_.B; + uint32_t banks_per_port = num_banks / config_.mem_ports; + return (bank_id % config_.mem_ports) * banks_per_port + bank_id / config_.mem_ports; + } uint32_t init_cycles_; + bool keep_lines_ = false; }; /////////////////////////////////////////////////////////////////////////////// @@ -1979,10 +3410,18 @@ Cache::PerfStats Cache::perf_stats() const { return impl_->perf_stats(); } -void Cache::flush_begin() { - impl_->flush_begin(); +void Cache::flush_begin(bool l1_only) { + impl_->flush_begin(l1_only); } bool Cache::flush_done() const { return impl_->flush_done(); } + +void Cache::invalidate() { + impl_->invalidate(); +} + +void Cache::keep_lines() { + impl_->keep_lines(); +} diff --git a/sim/simx/mem/cache.h b/sim/simx/mem/cache.h index 70ad9e6e3f..9f5de60bac 100644 --- a/sim/simx/mem/cache.h +++ b/sim/simx/mem/cache.h @@ -43,6 +43,17 @@ class Cache : public SimObject { uint16_t latency; // pipeline latency uint8_t repl_policy; // replacement policy (ReplPolicy) bool is_llc; // is last-level cache + bool nc_enable = false; // non-cacheable bypass layer in front of the cache + bool bank_hash = false; // hashed bank select + bool shared_llc = false; // the LLC serves every memory client alone + bool lazy_fetch = false; // write miss allocates without a fill (write-back LLC only) + bool core_out_buf = false; // registered core-response port + uint8_t read_merge = 0; // merge same-word reads within groups of this many adjacent ports (0: off) + bool write_around = false; // a no-allocate write miss goes to memory (write-back only) + bool mem_rsp_queue = false; // memory responses enter through a response queue + bool mem_rsp_slice = false; // a registered boundary ahead of the memory-response port stands for that queue's register + bool acq_forward = false; // an acquire flush also invalidates the next level + bool merge_line = false; // read merging takes any read of the same line, each merged port keeping its own tag and word }; struct PerfStats { @@ -80,10 +91,19 @@ class Cache : public SimObject { // Flush API: arm a write-back walk that emits dirty lines via mem_req_out. // flush_done() returns true once the walk completes; the caller is expected - // to tick the simulator until then. Write-through caches no-op. - void flush_begin(); + // to tick the simulator until then. Write-through caches no-op. On an + // L1-only flush the write-back last-level cache skips the walk and keeps + // its lines across the next launch reset. + void flush_begin(bool l1_only = false); bool flush_done() const; + // A shared last-level cache keeps its lines into the next launch. + void keep_lines(); + + // Drop every line (a read-only, virtually tagged cache on an + // address-space change). Called between launches. + void invalidate(); + protected: void on_reset(); void on_tick(); diff --git a/sim/simx/mem/cache_cluster.cpp b/sim/simx/mem/cache_cluster.cpp index ff52d56a5f..5b8bddc2b8 100644 --- a/sim/simx/mem/cache_cluster.cpp +++ b/sim/simx/mem/cache_cluster.cpp @@ -35,11 +35,17 @@ CacheCluster::CacheCluster(const SimContext& ctx, char sname[100]; - // Arbitrate incoming core interfaces + // Arbitrate incoming core interfaces. Arbitrating cores onto fewer caches + // registers the request, and the response once more when the cache + // registers its core-response port. + const bool shared = (num_inputs != num_units); + const uint32_t arb_req_delay = shared ? 1 : 0; + const uint32_t arb_rsp_delay = (shared && cache_config.core_out_buf) ? 1 : 0; std::vector input_arbs(cache_config.num_inputs); for (uint32_t i = 0; i < cache_config.num_inputs; ++i) { snprintf(sname, 100, "%s-input-arb%d", name, i); - input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units); + input_arbs.at(i) = MemArbiter::Create(sname, ArbiterType::RoundRobin, num_inputs, num_units, + arb_req_delay, arb_rsp_delay); for (uint32_t j = 0; j < num_inputs; ++j) { this->core_req_in.at(j).at(i).bind(&input_arbs.at(i)->ReqIn.at(j)); input_arbs.at(i)->RspOut.at(j).bind(&this->core_rsp_out.at(j).at(i)); @@ -82,9 +88,9 @@ Cache::PerfStats CacheCluster::perf_stats() const { return perf; } -void CacheCluster::flush_begin() { +void CacheCluster::flush_begin(bool l1_only) { for (auto cache : caches_) { - cache->flush_begin(); + cache->flush_begin(l1_only); } } diff --git a/sim/simx/mem/cache_cluster.h b/sim/simx/mem/cache_cluster.h index edbe5958c4..174c355fcd 100644 --- a/sim/simx/mem/cache_cluster.h +++ b/sim/simx/mem/cache_cluster.h @@ -33,7 +33,7 @@ class CacheCluster : public SimObject { Cache::PerfStats perf_stats() const; - void flush_begin(); + void flush_begin(bool l1_only = false); bool flush_done() const; private: diff --git a/sim/simx/mem/local_mem.cpp b/sim/simx/mem/local_mem.cpp index 41b82e8575..337b574605 100644 --- a/sim/simx/mem/local_mem.cpp +++ b/sim/simx/mem/local_mem.cpp @@ -16,9 +16,8 @@ #include #include #include +#include #include -#include -#include "types.h" #if VX_CFG_EXT_A_ENABLED #include "amo_ops.h" #include "amo_unit.h" @@ -26,319 +25,577 @@ using namespace vortex; +// A request's timing, in cycles from the one its engine takes it in: +// 0 the engine takes the request into its input queue of two; it takes one +// only in a cycle that starts with at most one queued +// 1 the lanes' words are compared and the request moves on to be issued +// 2 its first pass is issued to the bank registers +// 3 the banks are accessed +// 4 each served lane takes its bank's word +// 5 the response reaches its block's response register +// 6 the response register goes to the LSU block +// With several LSU blocks, the block an engine takes from is chosen the cycle +// before (see arbitrate()), and a second, read-only engine works the banks' +// second port, so two warps' reads go through each cycle. Stores and atomics +// take the first engine alone: one enters it only once the read engine has +// drained, and the read engine takes no read while a store or atomic sits in +// the first engine, while one waited at an input the cycle before, or in a +// cycle the first engine may be taking one, so every write keeps its order +// against every read. +// The model does a pass's bank accesses when it is issued. Every access keeps +// its order relative to the others, so this is the order the banks see. class LocalMem::Impl { protected: - LocalMem* simobject_; - Config config_; - RAM ram_; - uint32_t addr_bits_; + // a pass issued at cycle t has its response at the response queue's head + // at t + 3, from where the block's response register takes it + static constexpr uint32_t RSP_LATENCY = 3; + // an engine's response slots, held from the pass until the block takes the + // response from its response register + static constexpr uint32_t RSP_QUEUE_SIZE = 8; #if VX_CFG_EXT_A_ENABLED - AmoUnit amo_unit_; - // Per-bank atomic occupancy. A bank has a single port and the value read out - // is registered before the update is computed, so an atomic owns its bank - // for three cycles -- read, capture, write back -- and it accepts nothing - // else meanwhile. - static constexpr uint32_t AMO_BANK_STALL_CYCLES = 2; - std::vector amo_busy_; - // A read issued the cycle after a write to the same word returns the word's - // pre-update value, so the bank holds such a read off for one cycle. The - // write-back is the third cycle of the occupancy, so this shadow covers the - // fourth -- the cycle the next contending atomic arrives, which is why - // back-to-back atomics on one word cost four cycles and not three. - std::vector amo_rdw_addr_; - std::vector amo_rdw_valid_; - // Whether the atomic now occupying the bank ends in a write at all. A - // load-reserved never writes and a failed store-conditional must not, so - // neither leaves a shadow behind. - std::vector amo_rdw_pending_; + // An atomic reads its word, registers it, then writes the new value back: + // the two cycles after its pass the banks take nothing else. + static constexpr uint32_t AMO_STALL_CYCLES = 2; #endif - MemCrossBar::Ptr mem_xbar_; - mutable PerfStats perf_stats_; - uint64_t to_local_addr(uint64_t addr) { - return bit_getw(addr, 0, addr_bits_-1); - } + struct Request { + LsuReq req; + uint32_t block; + Request() : req(0), block(0) {} + }; + struct RspEntry { + uint64_t ready; + uint32_t block; + LsuRsp rsp; + }; + + static constexpr uint32_t S0_SIZE = 2; + + struct Engine { + std::deque s0; + bool s1_valid = false; + Request s1; + BitVector<> pending; + // the next pass's bank winners are known: a pass that leaves lanes + // pending is followed by a cycle that finds the next ones + bool ready = false; + std::deque rsp_queue; + }; + + LocalMem* simobject_; + Config config_; + RAM ram_; + uint32_t addr_bits_; + uint32_t lg2_word_; #if VX_CFG_EXT_A_ENABLED - // The word a bank is addressed by. Two byte addresses falling in one word - // are the same access as far as the bank is concerned. - uint64_t bank_word(uint64_t addr) { - return to_local_addr(addr) >> log2ceil(config_.line_size); - } + AmoUnit amo_unit_; #endif + uint32_t stall_cycles_; + std::vector engines_; + struct RspReg { + bool valid = false; + uint32_t engine = 0; + LsuRsp rsp; + RspReg() : rsp(0) {} + }; + std::vector rsp_regs_; // per block + std::vector arb_last_; // per engine, its round-robin arbiter's last choice + struct Selection { + bool valid = false; + uint32_t block = 0; + bool read = false; // the chosen block's request is known to be a plain read + }; + Selection a_sel_; + Selection b_sel_; + bool write_waiting_r_; + bool dma_slot_; + uint32_t dma_last_grant_; + mutable PerfStats perf_stats_; -public: - Impl(LocalMem* simobject, const Config& config) - : simobject_(simobject) - , config_(config) - , ram_(config.capacity) - , addr_bits_(log2ceil(config.capacity)) + uint64_t to_local_addr(uint64_t addr) const { + return bit_getw(addr, 0, addr_bits_ - 1); + } + + // A bank serves one word per cycle; byte addresses within one word are the + // same access as far as the bank is concerned. + uint64_t bank_word(uint64_t addr) const { + return to_local_addr(addr) >> lg2_word_; + } + + uint32_t bank_of(uint64_t addr) const { + return (uint32_t)(bank_word(addr) & ((1u << config_.B) - 1)); + } + + static bool is_plain_read(const LsuReq& req) { + return !req.is_write() && !req.is_amo(); + } + + static bool holds_write(const Engine& e) { + for (auto& r : e.s0) { + if (!is_plain_read(r.req)) { + return true; + } + } + return e.s1_valid && !is_plain_read(e.s1.req); + } + + // The lanes one pass serves: each bank goes to its lowest pending lane, and + // a plain read also serves every pending lane reading the same word. A + // store or an atomic takes its bank alone, so lanes writing one word land + // in lane order and the highest lane's bytes are the ones left. + BitVector<> select_pass(const LsuReq& req, const BitVector<>& pending) const { + uint32_t num_banks = 1u << config_.B; + std::vector winner(num_banks, -1); + BitVector<> served(pending.size()); + bool join = is_plain_read(req); + for (uint32_t i = 0; i < pending.size(); ++i) { + if (!pending.test(i)) { + continue; + } + uint32_t b = this->bank_of(req.addrs.at(i)); + if (winner.at(b) == -1) { + winner.at(b) = i; + served.set(i); + } else if (join && this->bank_word(req.addrs.at(i)) == this->bank_word(req.addrs.at(winner.at(b)))) { + served.set(i); + } + } + return served; + } + + // Do one lane's bank access and return the word the lane gets back, if any. + std::shared_ptr access(const LsuReq& req, uint32_t i) { + uint64_t la = to_local_addr(req.addrs.at(i)); + uint64_t line_addr = la & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); #if VX_CFG_EXT_A_ENABLED - , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) - , amo_busy_(1 << config.B, 0) - , amo_rdw_addr_(1 << config.B, 0) - , amo_rdw_valid_(1 << config.B, false) - , amo_rdw_pending_(1 << config.B, false) + uint32_t hart_id = make_hart_id(req.cid, req.wid, req.tids.at(i)); + if (req.is_amo()) { + // Shared-memory atomics read-modify-write in place and return the old + // word; LR/SC reservations are kept per hart. + const uint32_t bmask = VX_CFG_MEM_BLOCK_SIZE - 1; + const uint32_t byte_off = (uint32_t)(la & bmask); + const uint64_t byteen = req.byteen.at(i); + const uint32_t wbytes = byteen ? (uint32_t)__builtin_popcountll(byteen) : 4u; + const uint8_t width = (wbytes >= 8) ? 3 : 2; + const bool is_lr = (req.op == MemOp::AMO_LR); + const bool is_sc = (req.op == MemOp::AMO_SC); + const bool sc_fail = is_sc && !amo_unit_.check(hart_id, la); + const bool do_store = !is_lr && !sc_fail; + + uint8_t obuf[8] = {0}; + ram_.read(obuf, la, wbytes); + const uint64_t old_word = amo_load_word(obuf, 0, width); + const uint64_t rhs = req.data.at(i) + ? amo_load_word(req.data.at(i)->data(), byte_off, width) : 0; + auto rmw = amo_compute(req.op, width, old_word, rhs, + req.flags.amo_unsigned, req.amo_cmp.at(i)); + + auto rsp_data = make_mem_block(); + std::memset(rsp_data->data(), 0, rsp_data->size()); + const uint64_t ret_word = is_sc ? (sc_fail ? 1ull : 0ull) : rmw.ret_word; + amo_store_word(rsp_data->data(), byte_off, width, ret_word); + + // An SC gives up its own reservation whether it succeeds or fails; an + // LR claims one. A committed store breaks every other hart's + // reservation on the word, or a second hart's stale reservation would + // survive the first hart's successful SC and one update would be lost. + if (is_lr) { + amo_unit_.reserve(hart_id, la); + } else if (is_sc) { + amo_unit_.clear(hart_id, la); + } + if (do_store) { + uint8_t sbuf[8]; + amo_store_word(sbuf, 0, width, rmw.new_word); + ram_.write(sbuf, la, wbytes); + amo_unit_.invalidate(la, hart_id); + } + // an atomic responds, so it counts as a read + ++perf_stats_.reads; + return rsp_data; + } #endif - { - char sname[100]; - snprintf(sname, 100, "%s-xbar", simobject->name().c_str()); - uint32_t lg2_line_size = log2ceil(config_.line_size); - uint32_t num_banks = 1 << config.B; - mem_xbar_ = MemCrossBar::Create(sname, ArbiterType::Priority, config.num_reqs, num_banks, - [lg2_line_size, num_banks](const MemCrossBar::ReqType& req) { - // Custom logic to calculate the output index using bank interleaving - return (uint32_t)((req.addr >> lg2_line_size) & (num_banks-1)); - }); - for (uint32_t i = 0; i < config.num_reqs; ++i) { - simobject->Inputs.at(i).bind(&mem_xbar_->ReqIn.at(i)); - mem_xbar_->RspOut.at(i).bind(&simobject->Outputs.at(i)); - } - } - - virtual ~Impl() {} - - void reset() { - // perf_stats_ survives: the MPM counters run from device reset. + if (req.is_write()) { + for (uint32_t b = 0; req.data.at(i) && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { + if (req.byteen.at(i) & (1ull << b)) { + uint8_t value = (*req.data.at(i))[b]; + ram_.write(&value, line_addr + b, 1); + } + } #if VX_CFG_EXT_A_ENABLED - amo_unit_.reset(); - std::fill(amo_busy_.begin(), amo_busy_.end(), 0); - std::fill(amo_rdw_valid_.begin(), amo_rdw_valid_.end(), false); - std::fill(amo_rdw_pending_.begin(), amo_rdw_pending_.end(), false); + // A plain store breaks a reservation on the word it writes, as an + // atomic's commit does, or a store between a hart's load-reserved and + // its store-conditional would go unnoticed. + amo_unit_.invalidate(la, hart_id); #endif - } - - // Serve one DMA client's row access this cycle and return the banks it - // occupies. Clients are served at fixed priority, and DMA wins every bank - // it touches over the LSU: a read touches all banks, a write those its - // byte enables cover. - uint64_t dma_tick() { - uint32_t num_banks = (1 << config_.B); - uint32_t lg2_line_size = log2ceil(config_.line_size); - for (uint32_t c = 0; c < config_.dma_clients; ++c) { - bool pending = false; - bool rsp_ready = true; - for (uint32_t p = 0; p < DMA_PORTS; ++p) { - auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); - if (in.empty()) { - continue; - } - pending = true; - if (!in.peek().is_write() && simobject_->DmaOutputs.at(c * DMA_PORTS + p).full()) { - rsp_ready = false; - } - } - if (!pending) { - continue; - } - if (!rsp_ready) { - return 0; - } - uint64_t banks = 0; - for (uint32_t p = 0; p < DMA_PORTS; ++p) { - auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); - if (in.empty()) { - continue; - } - auto& req = in.peek(); - uint64_t line_addr = to_local_addr(req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - if (req.is_write()) { - for (uint32_t b = 0; req.data && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { - if (req.byteen & (1ull << b)) { - uint8_t value = (*req.data)[b]; - ram_.write(&value, line_addr + b, 1); - banks |= 1ull << (((line_addr + b) >> lg2_line_size) & (num_banks - 1)); - } - } + ++perf_stats_.writes; + return nullptr; + } + auto rsp_data = make_mem_block(); + ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); + ++perf_stats_.reads; + return rsp_data; + } + + bool dma_pending() const { + for (auto& in : simobject_->DmaInputs) { + if (!in.empty()) { + return true; + } + } + return false; + } + + // Serve one DMA client's row access. Clients are served round-robin, + // starting after the last one served. + void dma_fire() { + for (uint32_t i = 0; i < config_.dma_clients; ++i) { + uint32_t c = (dma_last_grant_ + 1 + i) % config_.dma_clients; + bool pending = false; + bool rsp_ready = true; + for (uint32_t p = 0; p < DMA_PORTS; ++p) { + auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); + if (in.empty()) { + continue; + } + pending = true; + if (!in.peek().is_write() && simobject_->DmaOutputs.at(c * DMA_PORTS + p).full()) { + rsp_ready = false; + } + } + if (!pending) { + continue; + } + if (!rsp_ready) { + return; + } + for (uint32_t p = 0; p < DMA_PORTS; ++p) { + auto& in = simobject_->DmaInputs.at(c * DMA_PORTS + p); + if (in.empty()) { + continue; + } + auto& req = in.peek(); + uint64_t line_addr = to_local_addr(req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); + if (req.is_write()) { + for (uint32_t b = 0; req.data && b < VX_CFG_MEM_BLOCK_SIZE; ++b) { + if (req.byteen & (1ull << b)) { + uint8_t value = (*req.data)[b]; + ram_.write(&value, line_addr + b, 1); + } + } #if VX_CFG_EXT_A_ENABLED - amo_unit_.invalidate(to_local_addr(req.addr), req.hart_id); + amo_unit_.invalidate(to_local_addr(req.addr), req.hart_id); #endif - ++perf_stats_.writes; - } else { - MemRsp rsp{req.tag, req.hart_id, req.uuid}; - auto rsp_data = make_mem_block(); - ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); - rsp.data = rsp_data; - // The request arrived over a registered channel; the read - // data returns the cycle after it was issued. - simobject_->DmaOutputs.at(c * DMA_PORTS + p).send(rsp, 0); - banks = (num_banks >= 64) ? ~uint64_t(0) : ((uint64_t(1) << num_banks) - 1); - ++perf_stats_.reads; - } - DT(4, simobject_->name() << "-dma" << c << " req : " << req); - in.pop(); - } - return banks; - } - return 0; - } - - void tick() { - uint64_t dma_banks = this->dma_tick(); - - // process bank requets from xbar - uint32_t num_banks = (1 << config_.B); - for (uint32_t i = 0; i < num_banks; ++i) { + } else { + MemRsp rsp{req.tag, req.hart_id, req.uuid}; + auto rsp_data = make_mem_block(); + ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); + rsp.data = rsp_data; + // The request arrived over a registered channel; the read data + // returns the cycle after it was issued. + simobject_->DmaOutputs.at(c * DMA_PORTS + p).send(rsp, 0); + } + DT(4, simobject_->name() << "-dma" << c << " req : " << req); + in.pop(); + } + dma_last_grant_ = c; + return; + } + } + + // Issue an engine's next pass. Returns true when it was the request's last, + // which frees S1 for a new request in the same cycle. The response queue + // occupancy is the one from the start of the cycle. + bool issue_pass(Engine& e, uint32_t rsp_count) { + auto& req = e.s1.req; + bool has_rsp = !req.is_write() || req.is_amo(); + if (has_rsp && rsp_count >= RSP_QUEUE_SIZE) { + return false; + } + auto served = this->select_pass(req, e.pending); + LsuRsp rsp(config_.num_lanes); + rsp.tag = req.tag; + rsp.cid = req.cid; + rsp.uuid = req.uuid; + for (uint32_t i = 0; i < config_.num_lanes; ++i) { + if (!served.test(i)) { + continue; + } + auto data = this->access(req, i); + if (has_rsp) { + rsp.mask.set(i); + rsp.data.at(i) = data; + } + e.pending.reset(i); + } + DT(4, simobject_->name() << " pass: served=" << served << ", " << req); + if (has_rsp) { + e.rsp_queue.push_back({SimPlatform::instance().cycles() + RSP_LATENCY, e.s1.block, std::move(rsp)}); + } #if VX_CFG_EXT_A_ENABLED - // Cycles an in-flight atomic owns; the bank serves nothing else. - if (amo_busy_[i] != 0) { - if (0 == --amo_busy_[i]) { - amo_rdw_valid_[i] = amo_rdw_pending_[i]; // write-back just issued - } - continue; - } - // The shadow is one cycle wide: consume it here whether or not - // anything is waiting on the bank. - const bool rdw_shadow = amo_rdw_valid_[i]; - const uint64_t rdw_addr = amo_rdw_addr_[i]; - amo_rdw_valid_[i] = false; + if (req.is_amo()) { + stall_cycles_ = AMO_STALL_CYCLES; + } #endif - if ((dma_banks >> i) & 1) { - continue; - } - auto& xbar_req_out = mem_xbar_->ReqOut.at(i); - if (xbar_req_out.empty()) - continue; + if (e.pending.none()) { + e.s1_valid = false; + return true; + } + ++perf_stats_.bank_stalls; + e.ready = false; + return false; + } + + void take(Engine& e, uint32_t b) { + auto& in = simobject_->ReqIn.at(b); + Request r; + r.req = in.peek(); + r.block = b; + DT(4, simobject_->name() << "-req" << b << ": " << r.req); + e.s0.push_back(std::move(r)); + in.pop(); + } + + // The lowest eligible block after the last one chosen, round-robin. + int32_t pick(const std::vector& eligible, uint32_t& last) { + for (uint32_t i = 0; i < config_.num_blocks; ++i) { + uint32_t b = (last + 1 + i) % config_.num_blocks; + if (eligible.at(b)) { + last = b; + return b; + } + } + return -1; + } - auto& bank_req = xbar_req_out.peek(); + // Each engine takes the block chosen for it the cycle before, if its + // request is still there, then the choices for the next cycle are made from + // the requests waiting at the start of this one. A block served this cycle + // may present a new request of either kind next cycle, so a choice of it + // counts as a store: the first engine takes a store only while the read + // engine is idle, and the read engine takes nothing in that cycle. The read + // engine takes a read only while no store or atomic waited at an input the + // cycle before or sits in the first engine. + void arbitrate(const std::vector& s0_ready, bool a_write, bool b_busy) { + uint32_t n = config_.num_blocks; + std::vector in_valid(n); + std::vector in_read(n); + std::vector fired(n, false); + for (uint32_t b = 0; b < n; ++b) { + auto& in = simobject_->ReqIn.at(b); + in_valid.at(b) = !in.empty(); + in_read.at(b) = in_valid.at(b) && is_plain_read(in.peek()); + } + bool a_take = a_sel_.valid && s0_ready.at(0) && (a_sel_.read || !b_busy); + bool b_take = b_sel_.valid && s0_ready.at(1) && !a_write && !write_waiting_r_ + && !(a_take && !a_sel_.read); + bool b_fired = false; + if (a_take && in_valid.at(a_sel_.block)) { + this->take(engines_.at(0), a_sel_.block); + fired.at(a_sel_.block) = true; + } + if (b_take && in_valid.at(b_sel_.block)) { + this->take(engines_.at(1), b_sel_.block); + fired.at(b_sel_.block) = true; + b_fired = true; + } + std::vector known_read(n); + std::vector a_elig(n); + std::vector b_elig(n); + bool b_busy_n = b_busy || b_fired; + bool write_waiting = false; + for (uint32_t b = 0; b < n; ++b) { + known_read.at(b) = in_read.at(b) && !fired.at(b); + a_elig.at(b) = known_read.at(b) || (in_valid.at(b) && !b_busy_n); + write_waiting = write_waiting || (in_valid.at(b) && !in_read.at(b) && !fired.at(b)); + } + int32_t a_next = this->pick(a_elig, arb_last_.at(0)); + for (uint32_t b = 0; b < n; ++b) { + b_elig.at(b) = known_read.at(b) && ((int32_t)b != a_next); + } + int32_t b_next = this->pick(b_elig, arb_last_.at(1)); + a_sel_.valid = (a_next != -1); + a_sel_.block = (a_next != -1) ? a_next : 0; + a_sel_.read = (a_next != -1) && known_read.at(a_next); + b_sel_.valid = (b_next != -1); + b_sel_.block = (b_next != -1) ? b_next : 0; + write_waiting_r_ = write_waiting; + } + +public: + Impl(LocalMem* simobject, const Config& config) + : simobject_(simobject) + , config_(config) + , ram_(config.capacity) + , addr_bits_(log2ceil(config.capacity)) + , lg2_word_(log2ceil(config.word_size)) #if VX_CFG_EXT_A_ENABLED - // A read of the word the write-back just stored waits a cycle. An - // atomic reads the bank for its old value, so it counts as a reader - // here even though the request carries a write flag; that is what - // makes contending atomics on one word cost four cycles. - const bool reads_bank = !bank_req.is_write() || memop_is_atomic(bank_req.op); - if (rdw_shadow - && reads_bank - && this->bank_word(bank_req.addr) == rdw_addr) { - continue; - } - - // Shared-memory atomics: read-modify-write in place and ALWAYS - // return the old word. An AMO has a destination register, so the - // LSU reserved a response slot; a missing response hangs the warp - // (this was the prior behaviour — LMEM had no AMO path). LR/SC - // reservations are tracked per hart for correct contended - // compare-swap, mirroring the dcache AMO path used for global/SSBO. - if (memop_is_atomic(bank_req.op)) { - const uint32_t bmask = VX_CFG_MEM_BLOCK_SIZE - 1; - const uint64_t la = to_local_addr(bank_req.addr); - const uint32_t byte_off = (uint32_t)(la & bmask); - const uint32_t wbytes = bank_req.byteen - ? (uint32_t)__builtin_popcountll(bank_req.byteen) : 4u; - const uint8_t width = (wbytes >= 8) ? 3 : 2; - const MemOp op = bank_req.op; - const bool is_lr = (op == MemOp::AMO_LR); - const bool is_sc = (op == MemOp::AMO_SC); - const bool sc_fail = is_sc && !amo_unit_.check(bank_req.hart_id, la); - const bool do_store = !is_lr && !sc_fail; - - uint8_t obuf[8] = {0}; - ram_.read(obuf, la, wbytes); - const uint64_t old_word = amo_load_word(obuf, 0, width); - const uint64_t rhs = bank_req.data - ? amo_load_word(bank_req.data->data(), byte_off, width) : 0; - auto rmw = amo_compute(op, width, old_word, rhs, - bank_req.flags.amo_unsigned, - bank_req.amo_cmp); - - // Build the response before mutating any state so a full RspIn - // can stall-retry cleanly. - MemRsp bank_rsp{bank_req.tag, bank_req.hart_id, bank_req.uuid}; - auto rsp_data = make_mem_block(); - std::memset(rsp_data->data(), 0, rsp_data->size()); - const uint64_t ret_word = is_sc ? (sc_fail ? 1ull : 0ull) : rmw.ret_word; - amo_store_word(rsp_data->data(), byte_off, width, ret_word); - bank_rsp.data = rsp_data; - if (!mem_xbar_->RspIn.at(i).try_send(bank_rsp)) - continue; // stall; no state mutated yet - - // An SC gives up its own reservation whether it succeeds or - // fails; an LR claims one. Everything else is a plain RMW, - // whose commit breaks the other harts' reservations below. - if (is_lr) amo_unit_.reserve(bank_req.hart_id, la); - else if (is_sc) amo_unit_.clear(bank_req.hart_id, la); - if (do_store) { - uint8_t sbuf[8]; - amo_store_word(sbuf, 0, width, rmw.new_word); - ram_.write(sbuf, la, wbytes); - // Any committed store breaks every other hart's reservation - // on the word. Without this a second hart's stale - // reservation survives the first hart's successful SC, so - // both report success and one update is lost. - amo_unit_.invalidate(la, bank_req.hart_id); - } - perf_stats_.reads += !do_store; - perf_stats_.writes += do_store; - DT(4, simobject_->name() << "-bank" << i << " amo : " << bank_req); - amo_rdw_addr_[i] = this->bank_word(bank_req.addr); - amo_rdw_pending_[i] = do_store; - xbar_req_out.pop(); - amo_busy_[i] = AMO_BANK_STALL_CYCLES; // capture, then write back - continue; - } + , amo_unit_(VX_CFG_AMO_RS_SIZE < 2 ? 2u : (uint32_t)VX_CFG_AMO_RS_SIZE) #endif + , stall_cycles_(0) + , engines_((config.num_blocks > 1) ? 2 : 1) + , rsp_regs_(config.num_blocks) + , arb_last_(engines_.size(), 0) + , write_waiting_r_(false) + , dma_slot_(false) + , dma_last_grant_(0) + { + for (auto& e : engines_) { + e.pending = BitVector<>(config.num_lanes); + } + } + + virtual ~Impl() {} - // Apply byte-enabled writes from TLM payload to local RAM. - if (bank_req.is_write() && bank_req.data) { - uint64_t line_addr = to_local_addr(bank_req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - for (uint32_t b = 0; b < VX_CFG_MEM_BLOCK_SIZE; ++b) { - if (bank_req.byteen & (1ull << b)) { - uint8_t value = (*bank_req.data)[b]; - ram_.write(&value, line_addr + b, 1); - } - } + void reset() { + // perf_stats_ survives: the MPM counters run from device reset. #if VX_CFG_EXT_A_ENABLED - // A plain store breaks a reservation on the word it writes, the - // same as an atomic's commit does; otherwise a store between a - // hart's load-reserved and its store-conditional would go - // unnoticed and the conditional would wrongly succeed. - amo_unit_.invalidate(to_local_addr(bank_req.addr), bank_req.hart_id); + amo_unit_.reset(); #endif - } - - // Loads always respond. Stores respond when configured globally OR - // the request opts in via MEM_FLAG_STRSP. - if (!bank_req.is_write() || config_.write_reponse || bank_req.flags.strsp) { - // send xbar response — for reads, capture the line payload. - MemRsp bank_rsp{bank_req.tag, bank_req.hart_id, bank_req.uuid}; - if (!bank_req.is_write()) { - auto rsp_data = make_mem_block(); - uint64_t line_addr = to_local_addr(bank_req.addr) & ~uint64_t(VX_CFG_MEM_BLOCK_SIZE - 1); - ram_.read(rsp_data->data(), line_addr, VX_CFG_MEM_BLOCK_SIZE); - bank_rsp.data = rsp_data; - } - if (!mem_xbar_->RspIn.at(i).try_send(bank_rsp)) - continue; // stall - } - - DT(4, simobject_->name() << "-bank" << i << " req : " << bank_req); - - // update perf counters - perf_stats_.reads += !bank_req.is_write(); - perf_stats_.writes += bank_req.is_write(); - - // remove input - xbar_req_out.pop(); - } - } - - const PerfStats& perf_stats() const { - perf_stats_.bank_stalls = mem_xbar_->collisions(); - return perf_stats_; - } + stall_cycles_ = 0; + for (auto& e : engines_) { + e.s0.clear(); + e.s1_valid = false; + e.pending.reset(); + e.rsp_queue.clear(); + } + for (auto& r : rsp_regs_) { + r.valid = false; + } + std::fill(arb_last_.begin(), arb_last_.end(), config_.num_blocks - 1); + a_sel_ = Selection(); + b_sel_ = Selection(); + write_waiting_r_ = false; + dma_slot_ = false; + dma_last_grant_ = config_.dma_clients ? (config_.dma_clients - 1) : 0; + } + + void tick() { + uint32_t num_engines = engines_.size(); + uint64_t now = SimPlatform::instance().cycles(); + + // Responses. Each block's response register goes to the block; a free + // one then takes the first engine's queue head ahead of the read engine's. + // An engine's slots count the responses in its queue and in the registers. + std::vector rsp_count(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + rsp_count.at(k) = engines_.at(k).rsp_queue.size(); + } + for (auto& r : rsp_regs_) { + rsp_count.at(r.engine) += r.valid; + } + for (uint32_t b = 0; b < config_.num_blocks; ++b) { + auto& r = rsp_regs_.at(b); + if (r.valid && simobject_->RspOut.at(b).try_send(r.rsp, 0)) { + DT(4, simobject_->name() << "-rsp" << b << ": " << r.rsp); + r.valid = false; + } + } + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + if (e.rsp_queue.empty() || e.rsp_queue.front().ready > now) { + continue; + } + auto& r = rsp_regs_.at(e.rsp_queue.front().block); + if (r.valid) { + continue; + } + r.valid = true; + r.engine = k; + r.rsp = std::move(e.rsp_queue.front().rsp); + e.rsp_queue.pop_front(); + } + + // the arbiter's view of the engines, as registered at the cycle start + bool a_write = holds_write(engines_.at(0)); + bool b_busy = (num_engines > 1) && (!engines_.at(1).s0.empty() || engines_.at(1).s1_valid); + std::vector s0_ready(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + s0_ready.at(k) = (engines_.at(k).s0.size() < S0_SIZE); + } + + // S1: a DMA access, or the engines' passes. A DMA request seen in one + // cycle takes the banks the next; no pass is issued while one is seen. + // An engine whose last pass left lanes pending spends this cycle finding + // the next winners. + std::vector s1_ready(num_engines); + std::vector can_pass(num_engines); + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + s1_ready.at(k) = !e.s1_valid; + can_pass.at(k) = e.s1_valid && e.ready; + if (e.s1_valid && !e.ready) { + e.ready = true; + } + } + if (stall_cycles_ != 0) { + --stall_cycles_; + dma_slot_ = false; + } else { + bool dma_valid = this->dma_pending(); + if (dma_slot_) { + this->dma_fire(); + } + dma_slot_ = dma_valid; + if (!dma_valid) { + for (uint32_t k = 0; k < num_engines; ++k) { + if (can_pass.at(k)) { + s1_ready.at(k) = this->issue_pass(engines_.at(k), rsp_count.at(k)); + } + } + } + } + + // S0 -> S1, with the single-pass check + for (uint32_t k = 0; k < num_engines; ++k) { + auto& e = engines_.at(k); + if (s1_ready.at(k) && !e.s0.empty()) { + e.s1 = std::move(e.s0.front()); + e.s0.pop_front(); + e.s1_valid = true; + e.pending = e.s1.req.mask; + e.ready = true; + } + } + + if (num_engines == 1) { + // one block: its request goes straight to the engine + auto& in = simobject_->ReqIn.at(0); + if (s0_ready.at(0) && !in.empty()) { + this->take(engines_.at(0), 0); + } + } else { + this->arbitrate(s0_ready, a_write, b_busy); + } + + bool idle = (stall_cycles_ == 0) && !dma_slot_ && !a_sel_.valid && !b_sel_.valid; + for (auto& e : engines_) { + idle = idle && e.s0.empty() && !e.s1_valid && e.rsp_queue.empty(); + } + for (auto& r : rsp_regs_) { + idle = idle && !r.valid; + } + if (idle && !this->dma_pending()) { + simobject_->tick_sleep(); + } + } + const PerfStats& perf_stats() const { + return perf_stats_; + } }; /////////////////////////////////////////////////////////////////////////////// LocalMem::LocalMem(const SimContext& ctx, const char* name, const Config& config) - : SimObject(ctx, name) - , Inputs(config.num_reqs, this) - , Outputs(config.num_reqs, this) - , DmaInputs(config.dma_clients * DMA_PORTS, this) - , DmaOutputs(config.dma_clients * DMA_PORTS, this) - , impl_(new Impl(this, config)) + : SimObject(ctx, name) + , ReqIn(config.num_blocks, this) + , RspOut(config.num_blocks, this) + , DmaInputs(config.dma_clients * DMA_PORTS, this) + , DmaOutputs(config.dma_clients * DMA_PORTS, this) + , impl_(new Impl(this, config)) {} LocalMem::~LocalMem() { diff --git a/sim/simx/mem/local_mem.h b/sim/simx/mem/local_mem.h index 2a879974bd..b887b95798 100644 --- a/sim/simx/mem/local_mem.h +++ b/sim/simx/mem/local_mem.h @@ -17,6 +17,13 @@ namespace vortex { +// Warp-wide local memory. It takes whole warp requests from the LSU blocks, +// one per cycle per engine (two engines with several LSU blocks), and serves +// each in passes: each pass gives every bank to the lowest pending lane that +// maps to it, and every pending read lane of the same word rides along. Lanes +// left over replay in later passes, one every other cycle, and a request +// retires before the next one starts, so the banks see every lane in program +// order. class LocalMem : public SimObject { public: // A DMA access covers one full bank row. A row wider than a mem_block @@ -27,11 +34,11 @@ class LocalMem : public SimObject { struct Config { uint32_t capacity; - uint32_t line_size; - uint32_t num_reqs; - uint32_t B; // log2 number of banks - bool write_reponse; - uint32_t dma_clients; // row-wide DMA masters, in priority order + uint32_t word_size; + uint32_t num_blocks; // LSU blocks, one warp request port each + uint32_t num_lanes; // lanes of a warp request + uint32_t B; // log2 number of banks + uint32_t dma_clients; // row-wide DMA masters, served round-robin }; struct PerfStats { @@ -47,8 +54,8 @@ class LocalMem : public SimObject { } }; - std::vector> Inputs; - std::vector> Outputs; + std::vector> ReqIn; + std::vector> RspOut; // DMA port: client c uses channels [c * DMA_PORTS, (c + 1) * DMA_PORTS). std::vector> DmaInputs; diff --git a/sim/simx/mem/local_mem_switch.cpp b/sim/simx/mem/local_mem_switch.cpp index 0edd322f21..bdb36a8230 100644 --- a/sim/simx/mem/local_mem_switch.cpp +++ b/sim/simx/mem/local_mem_switch.cpp @@ -20,11 +20,11 @@ LocalMemSwitch::LocalMemSwitch( const char* name, uint32_t delay ) : SimObject(ctx, name) - // Single-entry ingress: the switch accepts a new request only as its - // current one drains, so upstream issue feels downstream cadence - // immediately instead of running ahead into queue slack. + // The split is combinational: a request handed over is routed into the + // output buffers in the same cycle, and ReqIn holds nothing across cycles. , ReqIn(this, 1) - , RspOut(this) + // the response arbiter's one-entry output buffer + , RspOut(this, 1) , ReqOutLmem(this) , RspInLmem(this) , ReqOutDC(this) diff --git a/sim/simx/mem/local_mem_switch.h b/sim/simx/mem/local_mem_switch.h index 6edbc3f1cb..73158544ac 100644 --- a/sim/simx/mem/local_mem_switch.h +++ b/sim/simx/mem/local_mem_switch.h @@ -36,6 +36,14 @@ class LocalMemSwitch : public SimObject { uint32_t delay ); + // The output buffers a request addresses can take it this cycle: to the data + // cache, to local memory, or both. + bool req_ready(bool to_dc, bool to_lmem) const { + return ReqIn.size() == 0 + && !(to_dc && ReqOutDC.full()) + && !(to_lmem && ReqOutLmem.full()); + } + protected: void on_tick(); diff --git a/sim/simx/mem/lsu_mem_adapter.cpp b/sim/simx/mem/lsu_mem_adapter.cpp index 7e4460c1c5..e9d5592a8e 100644 --- a/sim/simx/mem/lsu_mem_adapter.cpp +++ b/sim/simx/mem/lsu_mem_adapter.cpp @@ -19,14 +19,15 @@ LsuMemAdapter::LsuMemAdapter( const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay ) : SimObject(ctx, name) , ReqIn(this) , RspOut(this) , ReqOut(num_inputs, this) , RspIn(num_inputs, this) , delay_(delay) - , port_delay_(num_inputs, 0) + , rsp_delay_(rsp_delay) , pending_mask_(num_inputs) { assert(num_inputs > 0); @@ -45,7 +46,7 @@ LsuMemAdapter::LsuMemAdapter( mr.flags = req.flags; mr.amo_cmp = req.amo_cmp.at(0); auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); return mr; }); @@ -106,7 +107,7 @@ void LsuMemAdapter::on_tick() { } // send memory response - RspOut.send(out_rsp, 1); + RspOut.send(out_rsp, rsp_delay_); // remove input DT(4, this->name() << "-rsp" << i << ": " << rsp_in); @@ -140,11 +141,11 @@ void LsuMemAdapter::on_tick() { out_req.flags = in_req.flags; { auto t = get_addr_type(out_req.addr); - out_req.flags.io = (t == AddrType::IO); + out_req.flags.io = out_req.flags.flush ? out_req.flags.io : (t == AddrType::IO); out_req.flags.local = (t == AddrType::Shared); } - if (ReqOut.at(i).try_send(out_req, delay_ + port_delay_.at(i))) { + if (ReqOut.at(i).try_send(out_req, delay_)) { DT(4, this->name() << " req" << i << ": " << out_req); pending_mask_.reset(i); } diff --git a/sim/simx/mem/lsu_mem_adapter.h b/sim/simx/mem/lsu_mem_adapter.h index 0d0e05ebe6..1864926527 100644 --- a/sim/simx/mem/lsu_mem_adapter.h +++ b/sim/simx/mem/lsu_mem_adapter.h @@ -31,7 +31,8 @@ class LsuMemAdapter : public SimObject { const SimContext& ctx, const char* name, uint32_t num_inputs, - uint32_t delay + uint32_t delay, + uint32_t rsp_delay = 1 ); LsuMemAdapter( @@ -41,16 +42,13 @@ class LsuMemAdapter : public SimObject { ) : LsuMemAdapter(ctx, name, num_inputs, 0) {} - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); private: uint32_t delay_; - std::vector port_delay_; + uint32_t rsp_delay_; BitVector pending_mask_; friend class SimObject; diff --git a/sim/simx/mem/mem_coalescer.cpp b/sim/simx/mem/mem_coalescer.cpp index 533e236b13..11bd5af5e3 100644 --- a/sim/simx/mem/mem_coalescer.cpp +++ b/sim/simx/mem/mem_coalescer.cpp @@ -39,7 +39,6 @@ MemCoalescer::MemCoalescer( , sent_mask_(input_size) , line_size_(line_size) , delay_(delay) - , port_delay_(output_size, 0) {} void MemCoalescer::on_reset() { @@ -108,7 +107,7 @@ void MemCoalescer::on_tick() { } } - // send memory response + // packed combinationally into the lmem switch's response buffer RspOut.send(out_rsp, 1); DT(4, this->name() << " mem-rsp: " << out_rsp); @@ -256,7 +255,6 @@ void MemCoalescer::on_tick() { // build per-channel memory requests out_round_.valid = true; out_round_.lanes = out_mask; - out_round_.cur_mask = cur_mask; out_round_.reqs.assign(output_size_, MemReq{}); for (uint32_t o = 0; o < output_size_; ++o) { if (!out_mask.test(o)) { @@ -273,7 +271,7 @@ void MemCoalescer::on_tick() { mr.uuid = in_req.uuid; mr.flags = in_req.flags; auto t = get_addr_type(mr.addr); - mr.flags.io = (t == AddrType::IO); + mr.flags.io = mr.flags.flush ? mr.flags.io : (t == AddrType::IO); mr.flags.local = (t == AddrType::Shared); } @@ -285,7 +283,13 @@ void MemCoalescer::on_tick() { // The built round is presented to the downstream on the next tick, not // this one: the coalescer alternates a build tick with a drain tick, so a // batch issues at most every other cycle. Draining is handled at the top - // of on_tick when out_round_ is valid. + // of on_tick when out_round_ is valid. The input is released as its last + // batch is built, without waiting for that batch to be accepted. + sent_mask_ |= cur_mask; + if (sent_mask_ == in_req.mask) { + ReqIn.pop(); + sent_mask_.reset(); + } } // Issue the pending round's per-channel requests; commit the round once @@ -295,7 +299,7 @@ void MemCoalescer::flush_out_round() { if (!out_round_.lanes.test(o)) { continue; } - if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_ + port_delay_.at(o))) { + if (ReqOut.at(o).try_send(out_round_.reqs.at(o), delay_)) { out_round_.lanes.reset(o); } } @@ -304,12 +308,6 @@ void MemCoalescer::flush_out_round() { } out_round_.valid = false; - auto& in_req = ReqIn.peek(); - sent_mask_ |= out_round_.cur_mask; - if (sent_mask_ == in_req.mask) { - ReqIn.pop(); - sent_mask_.reset(); - } } const MemCoalescer::PerfStats& MemCoalescer::perf_stats() const { diff --git a/sim/simx/mem/mem_coalescer.h b/sim/simx/mem/mem_coalescer.h index e133c4be1a..cd9df21223 100644 --- a/sim/simx/mem/mem_coalescer.h +++ b/sim/simx/mem/mem_coalescer.h @@ -46,9 +46,6 @@ class MemCoalescer : public SimObject { const PerfStats& perf_stats() const; - // Extra request-side registers on one output port. - void set_port_delay(uint32_t port, uint32_t delay) { port_delay_.at(port) = delay; } - protected: void on_reset(); void on_tick(); @@ -67,7 +64,6 @@ class MemCoalescer : public SimObject { bool valid = false; BitVector<> lanes; // output lanes still to send std::vector reqs; // per output lane - BitVector<> cur_mask; // input lanes covered by this round }; uint32_t input_size_; @@ -79,7 +75,6 @@ class MemCoalescer : public SimObject { out_round_t out_round_; uint32_t line_size_; uint32_t delay_; - std::vector port_delay_; PerfStats perf_stats_; friend class SimObject; diff --git a/sim/simx/mem/mmu.cpp b/sim/simx/mem/mmu.cpp index f2c440a3fe..020d552641 100644 --- a/sim/simx/mem/mmu.cpp +++ b/sim/simx/mem/mmu.cpp @@ -19,7 +19,10 @@ Mmu::Mmu(const SimContext& ctx, uint32_t num_ports, uint32_t tlb_size, uint32_t client_id, - bool exec_side) + bool exec_side, + bool rsp_reg, + bool write_enable, + bool park_reg) : SimObject(ctx, name) , ReqIn(num_ports, this) , RspOut(num_ports, this) @@ -30,10 +33,14 @@ Mmu::Mmu(const SimContext& ctx, , num_ports_(num_ports) , client_id_(client_id) , exec_side_(exec_side) + , rsp_reg_(rsp_reg) + , write_enable_(write_enable) + , park_reg_(park_reg) , tlb_(tlb_size) , mshr_(VX_CFG_L1_TLB_MSHR_SIZE) , replay_(num_ports) , fault_rsp_(num_ports) + , pf_slot_(num_ports) {} Mmu::~Mmu() {} @@ -48,6 +55,10 @@ void Mmu::on_reset() { for (auto& q : fault_rsp_) { q = {}; } + for (auto& s : pf_slot_) { + s.reset(); + } + stage_.reset(); reports_ = {}; // Physical pages move under a virtual address between launches, so // cached translations cannot outlive one — the shared levels clear here @@ -63,6 +74,9 @@ void Mmu::set_satp(uint64_t satp) { } satp_ = std::make_unique(satp); tlb_.flush(); // sfence.vma + for (auto& e : mshr_) { + e.retired = false; + } } bool Mmu::empty() const { @@ -81,6 +95,14 @@ bool Mmu::empty() const { return false; } } + for (auto& s : pf_slot_) { + if (s) { + return false; + } + } + if (stage_) { + return false; + } return reports_.empty(); } @@ -91,11 +113,12 @@ bool Mmu::needs_translation(const MemReq& req) const { // needed. BARE mode (SATP unprogrammed) skips translation, covering // the few instruction fetches between reset and the kernel's csrw // satp. IO-flagged requests skip it too: the IO/OM apertures carry - // device registers or encoded coordinates, not virtual addresses. + // device registers or encoded coordinates, not virtual addresses. So do + // flush tokens, which carry no address. if (!satp_ || satp_->get_mode() == BARE) { return false; } - if (req.flags.io) { + if (req.flags.io || req.flags.flush) { return false; } return true; @@ -115,6 +138,8 @@ int Mmu::mshr_alloc(uint64_t vpn) { if (!mshr_[i].valid) { mshr_[i].valid = true; mshr_[i].issued = false; + mshr_[i].retired = false; + mshr_[i].rearmed = false; mshr_[i].vpn = vpn; mshr_[i].parked.clear(); return (int)i; @@ -138,14 +163,111 @@ void Mmu::kill_request(uint32_t port, const MemReq& req) { if (exec_side_) { return; } - // Same rule the caches apply: only a plain store retires without one. - if (req.op != MemOp::ST || req.flags.strsp) { + if (needs_kill_rsp(req)) { auto data = std::make_shared(); data->fill(0); fault_rsp_.at(port).push(MemRsp(req.tag, req.hart_id, req.uuid, data)); } } +bool Mmu::needs_kill_rsp(const MemReq& req) const { + // Same rule the caches apply: only a plain store retires without one. + return !exec_side_ && (req.op != MemOp::ST || req.flags.strsp); +} + +bool Mmu::port_parked(uint32_t port) const { + if (stage_ && stage_->port == port) { + return true; + } + for (auto& e : mshr_) { + if (!e.valid) { + continue; + } + for (auto& pr : e.parked) { + if (pr.first == port) { + return true; + } + } + } + return false; +} + +void Mmu::retire(int id, uint64_t ppn, uint32_t flags, uint32_t level) { + // A freed entry keeps its translation only behind a registered park, the + // one requester that can find its entry freed after probing it. + uint64_t vpn = mshr_[id].vpn; + mshr_[id] = MshrEntry(); + if (park_reg_) { + mshr_[id].vpn = vpn; + mshr_[id].retired = true; + mshr_[id].r_ppn = ppn; + mshr_[id].r_flags = flags; + mshr_[id].r_level = level; + } +} + +int Mmu::retired_find(uint64_t vpn) const { + for (size_t i = 0; i < mshr_.size(); ++i) { + if (!mshr_[i].valid && mshr_[i].retired && mshr_[i].vpn == vpn) { + return (int)i; + } + } + return -1; +} + +void Mmu::replay_or_kill(uint32_t port, const MemReq& req, uint64_t ppn, + uint32_t flags, uint32_t level) { + // Requests of differing intent chain onto one entry, so the walk's own + // check covers only the request that allocated it. Each parked request is + // re-checked against the flags the walk brought back. + if (!tlb_perm_ok(flags, access_of(req), memop_is_amo(req.op))) { + kill_request(port, req); + report_fault(req); + DT(3, this->name() << " perm-fault: vpn=0x" << std::hex + << (req.addr >> VX_VM_PAGE_LOG2_SIZE) << std::dec); + return; + } + uint64_t page_mask = (uint64_t(1) << VX_VM_PAGE_LOG2_SIZE) - 1; + uint64_t low_mask = (uint64_t(1) << (level * TLB_VPN_LEVEL_BITS)) - 1; + uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + MemReq translated = req; + translated.addr = ((((ppn & ~low_mask) | (vpn & low_mask))) << VX_VM_PAGE_LOG2_SIZE) + | (req.addr & page_mask); + replay_.at(port).push(translated); +} + +void Mmu::stage_drain() { + // The staged park allocates (or joins) a cycle after it was taken, against + // its own VPN; an entry that filled and freed meanwhile re-arms from its + // kept translation. + if (!stage_ || SimPlatform::instance().cycles() < stage_->cycle) { + return; + } + auto& st = *stage_; + int id = mshr_find(st.vpn); + if (id < 0) { + int r = retired_find(st.vpn); + if (r >= 0) { + // Re-armed: the entry is held again and drains as a filled one does, + // two cycles on, absorbing same-page joins meanwhile. + auto& e = mshr_[r]; + e.valid = true; + e.issued = true; + e.rearmed = true; + e.rearm_cycle = SimPlatform::instance().cycles() + 2; + e.parked.clear(); + e.parked.emplace_back(st.port, st.req); + stage_.reset(); + return; + } + id = mshr_alloc(st.vpn); + } + if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) { + mshr_[id].parked.emplace_back(st.port, st.req); + stage_.reset(); + } +} + void Mmu::report_fault(const MemReq& req) { TlbReq report; report.vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; @@ -160,21 +282,27 @@ void Mmu::on_tick() { // 1) Forward downstream responses upstream unchanged. Kill responses go // first: they belong to accesses that will never reach the cache, and the // warps waiting on them cannot drain until they land. + uint64_t rsp_delay = rsp_reg_ ? 1 : 0; for (uint32_t p = 0; p < num_ports_; ++p) { if (RspOut.at(p).full()) { continue; } if (!fault_rsp_.at(p).empty()) { - RspOut.at(p).send(fault_rsp_.at(p).front(), 0); + RspOut.at(p).send(fault_rsp_.at(p).front(), rsp_delay); fault_rsp_.at(p).pop(); continue; } + if (pf_slot_.at(p)) { + RspOut.at(p).send(*pf_slot_.at(p), rsp_delay); + pf_slot_.at(p).reset(); + continue; + } if (RspIn.at(p).empty()) { continue; } - // Responses pass straight through with no added latency: the translation - // stage's cost is charged on the request path, not the reply path. - RspOut.at(p).send(RspIn.at(p).peek(), 0); + // Responses pass straight through, through the port's response stage + // when registered: the translation is charged on the request path. + RspOut.at(p).send(RspIn.at(p).peek(), rsp_delay); RspIn.at(p).pop(); } @@ -190,35 +318,36 @@ void Mmu::on_tick() { // Dropping a fill would leave its parked requests waiting forever. __assert(id >= 0 && id < (int)mshr_.size() && mshr_[id].valid, "TLB fill does not match an outstanding miss"); - uint64_t page_mask = (uint64_t(1) << VX_VM_PAGE_LOG2_SIZE) - 1; - uint32_t shift = rsp.level * TLB_VPN_LEVEL_BITS; - uint64_t low_mask = (uint64_t(1) << shift) - 1; if (!rsp.fault) { tlb_.fill(mshr_[id].vpn, rsp.ppn, rsp.flags, rsp.level); } for (auto& [port, req] : mshr_[id].parked) { - // Requests of differing intent chain onto one entry, so the walk's - // own check covers only the request that allocated it. Each parked - // request is re-checked against the flags the walk brought back. if (rsp.fault) { kill_request(port, req); - } else if (!tlb_perm_ok(rsp.flags, access_of(req), memop_is_amo(req.op))) { - kill_request(port, req); - report_fault(req); - DT(3, this->name() << " perm-fault: vpn=0x" << std::hex - << (req.addr >> VX_VM_PAGE_LOG2_SIZE) << std::dec); } else { - MemReq translated = req; - uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; - uint64_t ppn = (rsp.ppn & ~low_mask) | (vpn & low_mask); - translated.addr = (ppn << VX_VM_PAGE_LOG2_SIZE) | (req.addr & page_mask); - replay_.at(port).push(translated); + replay_or_kill(port, req, rsp.ppn, rsp.flags, rsp.level); } } - mshr_[id] = MshrEntry(); + if (rsp.fault) { + mshr_[id] = MshrEntry(); + } else { + retire(id, rsp.ppn, rsp.flags, rsp.level); + } TlbFillIn.pop(); } + // 2b) Drain re-armed entries once their fill-equivalent cycle comes. + for (size_t i = 0; i < mshr_.size(); ++i) { + auto& e = mshr_[i]; + if (!e.valid || !e.rearmed || SimPlatform::instance().cycles() < e.rearm_cycle) { + continue; + } + for (auto& [port, req] : e.parked) { + replay_or_kill(port, req, e.r_ppn, e.r_flags, e.r_level); + } + retire((int)i, e.r_ppn, e.r_flags, e.r_level); + } + // 3) Issue pending miss requests (one per tick over the shared link). for (size_t i = 0; i < mshr_.size(); ++i) { auto& e = mshr_[i]; @@ -252,6 +381,16 @@ void Mmu::on_tick() { // while older requests are still parked, so only same-address order is // guaranteed — those share a VPN, hence one entry and its arrival-order // parked list. Replays drain ahead of new input on the same port. + // With a registered park, the stage's state at the start of the cycle + // decides both whether it takes a park and which page holds. + bool stage_free = !stage_.has_value(); + std::optional hold_vpn; + if (stage_) { + hold_vpn = stage_->vpn; + } + if (park_reg_) { + stage_drain(); + } for (uint32_t p = 0; p < num_ports_; ++p) { if (!replay_.at(p).empty()) { if (ReqOut.at(p).try_send(replay_.at(p).front())) { @@ -263,9 +402,14 @@ void Mmu::on_tick() { if (ReqIn.at(p).empty()) { continue; } - const MemReq& req = ReqIn.at(p).peek(); + MemReq req = ReqIn.at(p).peek(); if (!needs_translation(req)) { + // A flush token reaches the cache behind its port's parked requests + // (writing data-side clients only: an overtaken read is still correct). + if (req.flags.flush && write_enable_ && !exec_side_ && port_parked(p)) { + continue; + } if (ReqOut.at(p).try_send(req)) { ReqIn.at(p).pop(); } @@ -273,13 +417,27 @@ void Mmu::on_tick() { } uint64_t vpn = req.addr >> VX_VM_PAGE_LOG2_SIZE; + if (hold_vpn && *hold_vpn == vpn) { + continue; // its page is in the park stage + } auto res = tlb_.lookup(vpn); - if (res.hit) { + // A page with a live miss-station entry (a re-armed one still holds its + // requests) parks behind it, as the RTL's station probe does, so a hit + // cannot overtake an older same-page request. + if (res.hit && !(park_reg_ && mshr_find(vpn) >= 0)) { // A cached translation still has to satisfy the access: the entry // was installed for whichever intent first missed on this page. if (!tlb_perm_ok(res.flags, access_of(req), memop_is_amo(req.op))) { + bool needs_rsp = needs_kill_rsp(req); + if (needs_rsp && pf_slot_.at(p)) { + continue; // the port's fault slot is still owed out + } DT(3, this->name() << " perm-fault: vpn=0x" << std::hex << vpn << std::dec); - kill_request(p, req); + if (needs_rsp) { + auto data = std::make_shared(); + data->fill(0); + pf_slot_.at(p) = MemRsp(req.tag, req.hart_id, req.uuid, data); + } report_fault(req); ReqIn.at(p).pop(); continue; @@ -294,11 +452,19 @@ void Mmu::on_tick() { // Park the miss and keep the port flowing (hit-under-miss). // Same-VPN misses chain on one entry, preserving arrival order; // a full entry or a full station stalls this port only. + if (park_reg_) { + if (stage_free) { + stage_ = Staged{p, req, vpn, SimPlatform::instance().cycles() + 1}; + stage_free = false; + ReqIn.at(p).pop(); + } + continue; + } int id = mshr_find(vpn); if (id < 0) { id = mshr_alloc(vpn); } - if (id >= 0 && mshr_[id].parked.size() < 2) { + if (id >= 0 && mshr_[id].parked.size() < MAX_PARKED_PER_ENTRY) { mshr_[id].parked.emplace_back(p, req); ReqIn.at(p).pop(); } diff --git a/sim/simx/mem/mmu.h b/sim/simx/mem/mmu.h index e71edd5bc6..90ff0822b5 100644 --- a/sim/simx/mem/mmu.h +++ b/sim/simx/mem/mmu.h @@ -14,6 +14,7 @@ #include #include +#include #include #include #include @@ -54,7 +55,10 @@ class Mmu : public SimObject { uint32_t num_ports, uint32_t tlb_size, uint32_t client_id, - bool exec_side); + bool exec_side, + bool rsp_reg = false, + bool write_enable = true, + bool park_reg = false); ~Mmu(); @@ -82,12 +86,28 @@ class Mmu : public SimObject { bool valid = false; bool issued = false; uint64_t vpn = 0; + // Freed after a good fill: a registered park whose lane probe saw this + // entry re-arms it from the kept translation instead of walking again. + bool retired = false; + // Re-armed from its kept translation: drains, as if filled, from this cycle. + bool rearmed = false; + uint64_t rearm_cycle = 0; + uint64_t r_ppn = 0; + uint32_t r_flags = 0; + uint32_t r_level = 0; std::vector> parked; // (port, request) }; bool needs_translation(const MemReq& req) const; + bool port_parked(uint32_t port) const; + bool needs_kill_rsp(const MemReq& req) const; int mshr_find(uint64_t vpn) const; int mshr_alloc(uint64_t vpn); + int retired_find(uint64_t vpn) const; + void replay_or_kill(uint32_t port, const MemReq& req, uint64_t ppn, + uint32_t flags, uint32_t level); + void stage_drain(); + void retire(int id, uint64_t ppn, uint32_t flags, uint32_t level); TlbAccess access_of(const MemReq& req) const; void kill_request(uint32_t port, const MemReq& req); void report_fault(const MemReq& req); @@ -95,12 +115,31 @@ class Mmu : public SimObject { uint32_t num_ports_; uint32_t client_id_; bool exec_side_; + // Each response leaves a cycle late through its own stage. + bool rsp_reg_; + // Clients may write: a flush token is ordered behind the port's parks. + bool write_enable_; + // A registered park: a lane's miss (new or joining) is first held in a + // one-entry stage and reaches the miss station the next cycle at the + // earliest; the stage takes a request only while empty, and a lane whose + // page is staged holds meanwhile. + bool park_reg_; + struct Staged { + uint32_t port; + MemReq req; + uint64_t vpn; + uint64_t cycle; + }; + std::optional stage_; std::unique_ptr satp_; Tlb tlb_; std::vector mshr_; std::vector> replay_; // per-port, drains before new input std::vector> fault_rsp_;// per-port, kills faulted accesses + // Per-port single slot answering a permission fault on a hit; a lane whose + // slot is still full stalls its next such fault. + std::vector> pf_slot_; std::queue reports_; // permission faults awaiting the walker // Requests of one VPN that a single miss-station entry can absorb. diff --git a/sim/simx/opc_unit.cpp b/sim/simx/opc_unit.cpp index e5b1f8aac1..9585ba21fb 100644 --- a/sim/simx/opc_unit.cpp +++ b/sim/simx/opc_unit.cpp @@ -13,6 +13,7 @@ #include "opc_unit.h" #include "core.h" +#include #include #include #include "types.h" @@ -30,6 +31,9 @@ OpcUnit::OpcUnit(const SimContext &ctx, const char* name, , Input(this) , Output(this) , num_threads_(num_threads) { + // The scoreboard's output skid buffer: two entries, the one being collected + // among them, with a registered ready. + Input.set_registered_ready(true); regs_.reserve(num_warp_slots); for (uint32_t i = 0; i < num_warp_slots; ++i) { regs_.emplace_back(num_threads); @@ -43,29 +47,29 @@ void OpcUnit::on_reset() { total_stalls_ = 0; cur_trace_ = nullptr; release_cycle_ = 0; + consume_cycle_ = 0; + consumed_ = false; // The register files are excluded: they are storage, not pipeline state, and // a launch is not a device reset. A slot that skips its startup relies on the // stack pointer that startup left behind, so clearing them here would strand // it at zero. They are zero-initialized at construction. } +// Each bank serves one operand per cycle. static uint32_t compute_bank_conflicts(const instr_trace_t* trace) { - uint32_t stalls = 0; + uint32_t per_bank[VX_CFG_NUM_GPR_BANKS] = {}; + uint32_t most = 0; for (uint32_t i = 0; i < NUM_SRC_REGS; ++i) { - for (uint32_t j = i + 1; j < NUM_SRC_REGS; ++j) { - if ((trace->src_regs[i].type == RegType::None) - || (trace->src_regs[j].type == RegType::None)) - continue; - if ((trace->src_regs[i].type == RegType::Integer && trace->src_regs[i].id() == 0) - || (trace->src_regs[j].type == RegType::Integer && trace->src_regs[j].id() == 0)) - continue; // skip x0 - uint32_t bank_i = trace->src_regs[i].idx % VX_CFG_NUM_GPR_BANKS; - uint32_t bank_j = trace->src_regs[j].idx % VX_CFG_NUM_GPR_BANKS; - if (bank_i == bank_j) - ++stalls; + auto& reg = trace->src_regs[i]; + if (reg.type == RegType::None) { + continue; + } + if (reg.type == RegType::Integer && reg.id() == 0) { + continue; // skip x0 } + most = std::max(most, ++per_bank[reg.idx % VX_CFG_NUM_GPR_BANKS]); } - return stalls; + return most ? (most - 1) : 0; } void OpcUnit::on_tick() { @@ -79,14 +83,20 @@ void OpcUnit::on_tick() { cur_trace_ = nullptr; } - // accept next uop into the holding slot + // accept next uop into the holding slot; it stays in the input buffer + // until its last operand read if (cur_trace_ == nullptr && !Input.empty()) { auto trace = Input.peek(); uint32_t stalls = compute_bank_conflicts(trace); total_stalls_ += stalls; cur_trace_ = trace; + consume_cycle_ = cur_cycle + stalls; release_cycle_ = cur_cycle + 1 + stalls; + consumed_ = false; + } + if (cur_trace_ != nullptr && !consumed_ && cur_cycle >= consume_cycle_) { Input.pop(); + consumed_ = true; } } diff --git a/sim/simx/opc_unit.h b/sim/simx/opc_unit.h index af8f201468..ed15f26819 100644 --- a/sim/simx/opc_unit.h +++ b/sim/simx/opc_unit.h @@ -76,6 +76,8 @@ class OpcUnit : public SimObject { uint32_t total_stalls_ = 0; instr_trace_t* cur_trace_ = nullptr; uint64_t release_cycle_ = 0; + uint64_t consume_cycle_ = 0; + bool consumed_ = false; friend class SimObject; friend class Operands; // Operands::get_exit_code reads regs_ directly diff --git a/sim/simx/operands.cpp b/sim/simx/operands.cpp index aaaa018d71..56a9d971dd 100644 --- a/sim/simx/operands.cpp +++ b/sim/simx/operands.cpp @@ -108,12 +108,26 @@ Operands::Operands(const SimContext &ctx, const char* name, Core* core) } if (VX_CFG_NUM_OPCS >= 2) { + // the collectors' round-robin merge registers its output snprintf(sname, 100, "%s-rsp_arb", name); - rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1); + rsp_arb_ = TraceArbiter::Create(sname, ArbiterType::RoundRobin, VX_CFG_NUM_OPCS, 1, 1); for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { opc_units_.at(i)->Output.bind(&rsp_arb_->Inputs.at(i)); } rsp_arb_->Outputs.at(0).bind(&this->Output); + // The scoreboard's output skid buffer feeds the collectors through a + // demux: its head waits for its own collector, the entries behind it too. + // The head leaves the buffer at its last operand read, when its collector + // pops it, and the next head is offered from the following cycle. + Input.set_registered_ready(true); + Input.tx_callback([this](instr_trace_t* const& trace, uint64_t) { + if (!head_offered_ && this->Input.empty()) { + this->offer_head(trace, 0); + } + }); + for (uint32_t i = 0; i < VX_CFG_NUM_OPCS; ++i) { + opc_units_.at(i)->Input.pop_callback([this]() { this->on_head_consumed(); }); + } } else { // pass-thru this->Input.bind(&opc_units_.at(0)->Input); @@ -126,19 +140,23 @@ Operands::~Operands() { } void Operands::on_reset() { - //-- + head_offered_ = false; } void Operands::on_tick() { - if (VX_CFG_NUM_OPCS < 2) - return; // pass-thru - - // process requests - if (Input.empty()) - return; - auto trace = this->Input.peek(); - if (opc_units_.at(wid_to_opc_idx(trace->wid))->Input.try_send(trace)) { - Input.pop(); + this->tick_sleep(); // the demux runs on Input's delivery and pop events +} + +void Operands::offer_head(instr_trace_t* trace, uint64_t delay) { + opc_units_.at(wid_to_opc_idx(trace->wid))->Input.send(trace, delay); + head_offered_ = true; +} + +void Operands::on_head_consumed() { + this->Input.pop(); + head_offered_ = false; + if (!this->Input.empty()) { + this->offer_head(this->Input.peek(), 1); } } diff --git a/sim/simx/operands.h b/sim/simx/operands.h index c302f7abc3..077b905400 100644 --- a/sim/simx/operands.h +++ b/sim/simx/operands.h @@ -54,6 +54,10 @@ class Operands : public SimObject { Core* core_; std::vector opc_units_; TraceArbiter::Ptr rsp_arb_; + bool head_offered_ = false; + + void offer_head(instr_trace_t* trace, uint64_t delay); + void on_head_consumed(); friend class SimObject; }; diff --git a/sim/simx/processor.cpp b/sim/simx/processor.cpp index cc9116cfb3..2e1591f361 100644 --- a/sim/simx/processor.cpp +++ b/sim/simx/processor.cpp @@ -50,10 +50,14 @@ ProcessorImpl::ProcessorImpl() constexpr uint32_t total_cores = VX_CFG_NUM_CLUSTERS * NUM_SOCKETS * VX_CFG_SOCKET_SIZE; kmu_ = Kmu::Create("kmu", total_cores); - // create memory simulator + // create memory simulator; with the CP writing into the last-level cache, + // the memory side carries a transfer's fills across the launch reset auto dram_config = DramSim::platform_config(); dram_config.block_size = VX_CFG_MEM_BLOCK_SIZE; - memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + memsim_ = Memory::Create("dram", Memory::Config{VX_CFG_L3_MEM_PORTS, dram_config}); + } char sname[100]; @@ -63,6 +67,14 @@ ProcessorImpl::ProcessorImpl() clusters_.at(i) = Cluster::Create(sname, i, this); } + if constexpr (VX_CFG_NUM_CLUSTERS > 1) { + gbar_unit_ = GbarUnit::Create("gbar", VX_CFG_NUM_CLUSTERS); + for (uint32_t i = 0; i < VX_CFG_NUM_CLUSTERS; ++i) { + clusters_.at(i)->gbar_fwd_out.bind(&gbar_unit_->arrive_in); + gbar_unit_->resume_out.at(i).bind(&clusters_.at(i)->gbar_fwd_resume_in); + } + } + // Launch bus: a registered, credit-gated lane from the KMU to each core's // dispatcher; CTAs cross into the core's domain through the slice, and // admission credits return to the KMU through a slice of their own. @@ -87,25 +99,36 @@ ProcessorImpl::ProcessorImpl() // create L3 cache; when L3 is enabled it is the LLC, otherwise it is a // transparent bypass arbiter and the L2 (or L1) is the LLC. - l3cache_ = Cache::Create("l3cache", Cache::Config{ - !VX_CFG_L3_ENABLED, - log2ceil(VX_CFG_L3_SIZE), // C - log2ceil(VX_CFG_L3_LINE_SIZE), // L - log2ceil(VX_CFG_L3_SECTOR_SIZE), // S - log2ceil(VX_CFG_L2_SECTOR_SIZE), // W - log2ceil(VX_CFG_L3_NUM_WAYS), // A - log2ceil(VX_CFG_L3_NUM_BANKS), // B - VX_CFG_XLEN, // address bits - VX_CFG_L3_NUM_REQS, // request size - VX_CFG_L3_MEM_PORTS, // memory ports - VX_CFG_L3_WRITEBACK, // write-back - false, // write response - VX_CFG_L3_MSHR_SIZE, // mshr size - VX_CFG_L3_LATENCY, // pipeline latency - VX_CFG_L3_REPL_POLICY, // replacement policy - VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC - } - ); + { + SimPlatform::LaunchKeepScope keep(CP_ON_L2 != 0 || CP_ON_L3 != 0); + l3cache_ = Cache::Create("l3cache", Cache::Config{ + !VX_CFG_L3_ENABLED, + log2ceil(VX_CFG_L3_SIZE), // C + log2ceil(VX_CFG_L3_LINE_SIZE), // L + log2ceil(VX_CFG_L3_SECTOR_SIZE), // S + log2ceil(VX_CFG_L2_SECTOR_SIZE), // W + log2ceil(VX_CFG_L3_NUM_WAYS), // A + log2ceil(VX_CFG_L3_NUM_BANKS), // B + VX_CFG_XLEN, // address bits + L3_CACHE_REQS, // request size + VX_CFG_L3_MEM_PORTS, // memory ports + VX_CFG_L3_WRITEBACK, // write-back + false, // write response + VX_CFG_L3_MSHR_SIZE, // mshr size + VX_CFG_L3_LATENCY, // pipeline latency + VX_CFG_L3_REPL_POLICY, // replacement policy + VX_CFG_L3_ENABLED != 0, // is_llc when L3 is the LLC + true, // non-cacheable bypass + false, // bank_hash + L3_SHARED_LLC, // shared_llc + VX_CFG_L3_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register (behind the bypass layer) + 0, // read_merge + CP_ON_L3 != 0, // write_around + VX_CFG_L3_MRSQ_SIZE != 0, // memory-response queue + } + ); + } #if VX_CFG_EXT_A_ENABLED // Build-time invariant: every cache above the LLC must be write-through. @@ -156,6 +179,19 @@ ProcessorImpl::ProcessorImpl() } #endif + // The CP DMA's client slot on the shared last-level cache. + if constexpr (CP_ON_L2 != 0 || CP_ON_L3 != 0) { + SimPlatform::LaunchKeepScope keep(true); + cp_llc_port_ = CpLlcPort::Create("cp-llc-port"); + if constexpr (CP_ON_L2 != 0) { + cp_llc_port_->MemReqOut.bind(&clusters_.at(0)->cp_mem_req_in()); + clusters_.at(0)->cp_mem_rsp_out().bind(&cp_llc_port_->MemRspIn); + } else { + cp_llc_port_->MemReqOut.bind(&l3cache_->core_req_in.at(L3_CP_IDX)); + l3cache_->core_rsp_out.at(L3_CP_IDX).bind(&cp_llc_port_->MemRspIn); + } + } + // connect L3 memory interfaces for (uint32_t i = 0; i < VX_CFG_L3_MEM_PORTS; ++i) { l3cache_->mem_req_out.at(i).bind(&memsim_->mem_req_in.at(i)); @@ -204,7 +240,7 @@ void ProcessorImpl::attach_ram(RAM* ram) { #endif } -void ProcessorImpl::flush_caches() { +void ProcessorImpl::flush_caches(bool l1_only) { // Cache hierarchy is drained inside-out: issue all L1 flush_begin() calls // up-front so icache, dcache, and graphics caches flush in parallel, then // tick until all surfaces report flush_done(). @@ -212,7 +248,7 @@ void ProcessorImpl::flush_caches() { // L1 surfaces: dcache + icache + graphics caches. // Write-through surfaces early-exit in Cache::flush_begin(). for (auto& cluster : clusters_) { - cluster->dcache_flush_begin(); + cluster->dcache_flush_begin(l1_only); cluster->icache_flush_begin(); #ifdef VX_CFG_EXT_TEX_ENABLE cluster->tcache_flush_begin(); @@ -252,7 +288,7 @@ void ProcessorImpl::flush_caches() { // L2 caches. for (auto& cluster : clusters_) { - cluster->l2_flush_begin(); + cluster->l2_flush_begin(l1_only); } while (true) { bool all_done = true; @@ -265,14 +301,14 @@ void ProcessorImpl::flush_caches() { } // L3 cache (single instance at processor level). - l3cache_->flush_begin(); + l3cache_->flush_begin(l1_only); while (!l3cache_->flush_done() || !SimPlatform::instance().idle()) { SimPlatform::instance().tick(); } } int ProcessorImpl::run() { - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); @@ -311,6 +347,67 @@ int ProcessorImpl::run() { return exitcode; } +void ProcessorImpl::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + this->cp_mem_transfer(addr, static_cast(src), nullptr, size, noalloc); +} + +void ProcessorImpl::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + this->cp_mem_transfer(addr, nullptr, static_cast(dst), size, false); +} + +void ProcessorImpl::cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc) { + __assert(cp_llc_port_, "no CP port on the last-level cache"); + if (!platform_reset_) { + this->reset(); + } + while (size != 0) { + uint32_t offset = uint32_t(addr % CP_MEM_DATA_SIZE); + uint32_t n = uint32_t(std::min(size, CP_MEM_DATA_SIZE - offset)); + cp_llc_port_->push(addr - offset, src, dst, offset, n, noalloc); + addr += n; + size -= n; + if (src) { + src += n; + } + if (dst) { + dst += n; + } + } + while (!cp_llc_port_->idle()) { + SimPlatform::instance().tick(); + } + this->llc_keep_lines(); +} + +void ProcessorImpl::launch_reset() { + if (!cp_llc_port_) { + this->reset(); + return; + } + // The launch fence follows the CP's DMA writes, and the kernel starts once + // the last-level cache has taken them. A write that missed still has its + // line fill in flight, as on the hardware, so the memory side keeps its + // state across the reset and the kernel meets those fills. + if (cp_llc_port_->writes_pending()) { + cp_llc_port_->fence(); + } + while (!cp_llc_port_->idle()) { + SimPlatform::instance().tick(); + } + SimPlatform::instance().reset_launch(); + platform_reset_ = true; + // The perf_mem_* counters survive: the MPM counters run from device reset. + is_cycle_initialized_ = false; +} + +void ProcessorImpl::llc_keep_lines() { + if constexpr (CP_ON_L2 != 0) { + clusters_.at(0)->l2_keep_lines(); + } else { + l3cache_->keep_lines(); + } +} + bool ProcessorImpl::mmu_fault_pending() const { #ifdef VX_CFG_VM_ENABLE return dev_ptw_->fault_info().valid; @@ -331,6 +428,7 @@ void ProcessorImpl::forward_delegated_launch() { void ProcessorImpl::reset() { SimPlatform::instance().reset(); + platform_reset_ = true; // The perf_mem_* counters survive: the MPM counters run from device reset. perf_mem_pending_reads_ = 0; is_cycle_initialized_ = false; @@ -339,10 +437,10 @@ void ProcessorImpl::reset() { bool ProcessorImpl::cycle() { // Lazy first-call init mirrors run()'s top-of-loop sequence so the // external driver doesn't need to choreograph reset + kmu start - // separately. reset() clears is_cycle_initialized_ so a back-to-back + // separately. A reset clears is_cycle_initialized_ so a back-to-back // kernel launch re-dispatches. if (!is_cycle_initialized_) { - this->reset(); + this->launch_reset(); kmu_->start(); this->forward_delegated_launch(); is_cycle_initialized_ = true; @@ -359,6 +457,11 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { kmu_->dcr_write(addr, value); return 0; } + if (addr == VX_DCR_BASE_CACHE_FLUSH) { + // The flush itself runs on the per-core reads. + flush_l1_only_ = (value & VX_CACHE_FLUSH_L1_ONLY) != 0; + return 0; + } #ifdef VX_CFG_VM_ENABLE // Device SATP for the shared walker complex, assembled from two // 32-bit halves and fanned out to every cluster on the high write. @@ -391,10 +494,10 @@ int ProcessorImpl::dcr_write(uint32_t addr, uint32_t value) { int ProcessorImpl::dcr_read(uint32_t addr, uint32_t tag, uint32_t* value) { if (addr == VX_DCR_BASE_CACHE_FLUSH) { - // Drain dirty cache lines to DRAM before the host reads back results. - // After flush_caches() returns every dirty line has reached memsim_'s - // backing RAM. - this->flush_caches(); + // After flush_caches() returns every dirty line of the flushed levels has + // reached memsim_'s backing RAM; an L1-only flush leaves the shared + // last-level cache in place. + this->flush_caches(flush_l1_only_); *value = 0; return 0; } @@ -471,6 +574,22 @@ void Processor::reset() { impl_->reset(); } +void Processor::cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc) { + impl_->cp_mem_write(addr, src, size, noalloc); +} + +void Processor::cp_mem_read(uint64_t addr, void* dst, uint64_t size) { + impl_->cp_mem_read(addr, dst, size); +} + +bool Processor::cp_llc_client() const { + return (CP_ON_L2 != 0) || (CP_ON_L3 != 0); +} + +uint64_t Processor::cp_llc_size() const { + return (CP_ON_L3 != 0) ? VX_CFG_L3_SIZE : VX_CFG_L2_SIZE; +} + void Processor::start_kmu() { impl_->kmu().start(); } diff --git a/sim/simx/processor.h b/sim/simx/processor.h index 66081ebbd2..13bd99aa40 100644 --- a/sim/simx/processor.h +++ b/sim/simx/processor.h @@ -59,6 +59,19 @@ class Processor { int dcr_read(uint32_t addr, uint32_t tag, uint32_t* value); + // Device-memory access through the command processor's port into the + // last-level cache, when the configuration has one (VX_CFG_CP_ON_LLC). A + // write returns once every request entered the port; the next launch is + // ordered behind it. A read returns with its data. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); + + // The command processor's DMA is a client of the last-level cache, of + // this size in bytes. + bool cp_llc_client() const; + uint64_t cp_llc_size() const; + private: ProcessorImpl* impl_; }; diff --git a/sim/simx/processor_impl.h b/sim/simx/processor_impl.h index ebe59cf297..7a8d7c17e1 100644 --- a/sim/simx/processor_impl.h +++ b/sim/simx/processor_impl.h @@ -18,6 +18,8 @@ #include "constants.h" #include "cluster.h" #include "kmu.h" +#include "gbar_unit.h" +#include "cp_llc_port.h" namespace vortex { @@ -72,7 +74,13 @@ class ProcessorImpl { // Drain dirty data from caches (write-back path) all the way to DRAM. // Walks L1 dcaches → L2 → L3, ticking the simulator between phases so // each level's evictions reach the next before that level itself flushes. - void flush_caches(); + void flush_caches(bool l1_only); + + // Device-memory access through the command processor's port into the + // last-level cache (CP_ON_L2 / CP_ON_L3). A write returns once every + // request entered the port, a read once its data returned. + void cp_mem_write(uint64_t addr, const void* src, uint64_t size, bool noalloc); + void cp_mem_read(uint64_t addr, void* dst, uint64_t size); private: @@ -80,11 +88,22 @@ class ProcessorImpl { // the frame kick is forwarded to every cluster's raster engine instead. void forward_delegated_launch(); + // Queue a transfer on the CP port and tick until it has entered, or for a + // read returned; the last-level cache keeps what it took into the next launch. + void cp_mem_transfer(uint64_t addr, const uint8_t* src, uint8_t* dst, uint64_t size, bool noalloc); + + // Reset the model for a launch. After DMA writes it waits for the port's + // fence and keeps the memory side, whose line fills may still be in flight. + void launch_reset(); + + void llc_keep_lines(); + // True once the device walker has latched a page fault. bool mmu_fault_pending() const; Kmu::Ptr kmu_; std::vector clusters_; + GbarUnit::Ptr gbar_unit_; // device-level global barrier (multi-cluster) Memory::Ptr memsim_; #ifdef VX_CFG_VM_ENABLE uint64_t mmu_satp_ = 0; // assembled from the two DCR halves @@ -95,6 +114,8 @@ class ProcessorImpl { #endif RAM* ram_ = nullptr; // functional backing store (set by attach_ram) Cache::Ptr l3cache_; + CpLlcPort::Ptr cp_llc_port_; // CP DMA client of the last-level cache + bool platform_reset_ = false; uint64_t perf_mem_reads_ = 0; uint64_t perf_mem_writes_ = 0; uint64_t perf_mem_latency_ = 0; @@ -103,6 +124,7 @@ class ProcessorImpl { // kmu_->start()). reset() clears it so a back-to-back kernel launch // via cycle() re-dispatches the KMU. bool is_cycle_initialized_; + bool flush_l1_only_ = false; // kind of the flush the last start write opened }; } diff --git a/sim/simx/rtu/rtu_types.h b/sim/simx/rtu/rtu_types.h index cb72cd528a..90855c6857 100644 --- a/sim/simx/rtu/rtu_types.h +++ b/sim/simx/rtu/rtu_types.h @@ -259,6 +259,8 @@ static_assert(VX_CFG_RTU_MERGE_DEPTH == 0, // Cores served by one RTU, and the per-core slot partition. constexpr uint32_t kRtuCoresPerRtu = VX_CFG_SOCKET_SIZE / VX_CFG_NUM_RTU_CORES; +static_assert(!VX_CFG_EXT_RTU_ENABLED || (kRtuCoresPerRtu * VX_CFG_NUM_RTU_CORES == VX_CFG_SOCKET_SIZE), + "SOCKET_SIZE must be a multiple of NUM_RTU_CORES"); constexpr uint32_t kRtuSlotsPerCore = VX_CFG_RTU_NUM_SLOTS / kRtuCoresPerRtu; static_assert(kRtuSlotsPerCore >= 1, "a core with no slot of its own could never issue a trace"); diff --git a/sim/simx/rtu/rtu_unit.cpp b/sim/simx/rtu/rtu_unit.cpp index 2d8cef624f..4f8878c3ba 100644 --- a/sim/simx/rtu/rtu_unit.cpp +++ b/sim/simx/rtu/rtu_unit.cpp @@ -35,10 +35,11 @@ inline uint32_t float_to_bits(float f) { } } // namespace -RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window) +RtuUnit::RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay) : window_(window) , core_(core) , req_out_(req_out) + , bus_delay_(bus_delay) { trace_slot_.fill(-1); for (auto& s : trace_scene_) s.fill(0); @@ -252,7 +253,7 @@ instr_trace_t* RtuUnit::process_cb_ret(instr_trace_t* trace, uint32_t block_id) trace->dst_data[t].u = 0; // no writeback } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); DT(3, "rtu-cb_ret submit: core=" << core_->id() << ", wid=" << trace->wid << ", tmask=0x" << std::hex << bits << std::dec); return trace; @@ -527,7 +528,7 @@ instr_trace_t* RtuUnit::process_trace_uop(instr_trace_t* trace, uint32_t block_i req.cull_mask[t] = trace_cull_.at(wid); } req.tmask_bits = bits; - req_out_.send(req); + req_out_.send(req, 1 + bus_delay_); trace_slot_.at(wid) = -1; DT(3, "rtu-trace arm: core=" << core_->id() << ", wid=" << wid << ", slot=" << slot << ", tmask=0x" << std::hex << bits << std::dec); diff --git a/sim/simx/rtu/rtu_unit.h b/sim/simx/rtu/rtu_unit.h index 978e7ea511..a1ad4bdc4f 100644 --- a/sim/simx/rtu/rtu_unit.h +++ b/sim/simx/rtu/rtu_unit.h @@ -69,7 +69,9 @@ class RtuUopGen { // register file. Plain (non-SimObject) helper owned by SfuUnit. class RtuUnit { public: - RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window); + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's RTU bus. + RtuUnit(Core* core, SimChannel& req_out, RtuWindow& window, uint32_t bus_delay = 0); // Async ray pool. process_wait either: // - returns the trace with the per-lane status word written @@ -187,6 +189,7 @@ class RtuUnit { Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; // Async ray pool. Borrowed from the Socket via set_rtu_core(); // null until the Socket has wired it (TRACE/WAIT paths must NEVER // dereference rtu_core_ before that — but in practice the Socket diff --git a/sim/simx/scheduler.cpp b/sim/simx/scheduler.cpp index 551adf6976..793b1c4074 100644 --- a/sim/simx/scheduler.cpp +++ b/sim/simx/scheduler.cpp @@ -89,7 +89,7 @@ void Scheduler::on_reset() { stalled_warps_next_.reset(); active_warps_.reset(); cta_active_view_.reset(); - cta_fire_ = false; + cta_fire_cnt_ = 0; // Sequencers live on Core now; Core::on_reset() resets them. wspawn_.valid = false; @@ -155,26 +155,15 @@ void Scheduler::activate_warp(uint32_t wid, const cta_warp_record_t& rec) { << ", mscratch=0x" << std::hex << warp.mscratch << std::dec); } -instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { - int scheduled_warp = -1; - - // Dispatch one CTA warp. The dispatcher picks from the registered active - // set and its pick lands the next cycle; the warp it activates becomes - // schedulable the cycle after that. - if (cta_fire_) { - activate_warp(cta_fire_wid_, cta_fire_rec_); - stalled_warps_.set(cta_fire_wid_); - cta_fire_ = false; - } - { - uint32_t wid; - cta_warp_record_t rec; - if (cta_dispatcher_->step(cta_active_view_, &wid, &rec)) { - cta_fire_ = true; - cta_fire_wid_ = wid; - cta_fire_rec_ = rec; - } +void Scheduler::schedule_begin() { + // Dispatch CTA warps, one per launch lane. The dispatcher picks from the + // registered active set and its picks land the next cycle; a warp it + // activates becomes schedulable the cycle after that. + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + activate_warp(cta_fire_wids_[i], cta_fire_recs_[i]); + stalled_warps_.set(cta_fire_wids_[i]); } + cta_fire_cnt_ = cta_dispatcher_->step(cta_active_view_, cta_fire_wids_.data(), cta_fire_recs_.data()); #ifdef VX_CFG_EXT_RASTER_ENABLE // Inject ready fragment waves into free warp slots. @@ -200,6 +189,14 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { wspawn_.valid = false; this->resume(0); } +} + +bool Scheduler::has_ready(const WarpMask& warp_mask) const { + return (active_warps_ & ~stalled_warps_ & warp_mask).any(); +} + +instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { + int scheduled_warp = -1; // pick next ready warp for (size_t wid = 0, nw = VX_CFG_NUM_WARPS; wid < nw; ++wid) { @@ -244,20 +241,23 @@ instr_trace_t* Scheduler::schedule(const WarpMask& warp_mask) { this->suspend(scheduled_warp); } + return trace; +} + +void Scheduler::schedule_end() { // Clock the registered warp-stall state. The suspend above, or a resume from // decode/commit/FU earlier this cycle, drives stalled_warps_next_ and only // becomes visible to the pick loop next cycle — so a warp released as its // instruction resolves is never re-scheduled the same cycle. stalled_warps_ = stalled_warps_next_; cta_active_view_ = active_warps_; - if (cta_fire_) { - cta_active_view_.set(cta_fire_wid_); + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + cta_active_view_.set(cta_fire_wids_[i]); } - return trace; } bool Scheduler::running() const { - return active_warps_.any() || cta_fire_ || cta_dispatcher_->running() + return active_warps_.any() || (cta_fire_cnt_ != 0) || cta_dispatcher_->running() #ifdef VX_CFG_EXT_RASTER_ENABLE || fwd_armed_ #endif @@ -417,7 +417,11 @@ void Scheduler::fwd_try_inject() { // Find any free warp slot — there is no driver warp to skip in the push model. int wid = -1; for (uint32_t w = 0; w < VX_CFG_NUM_WARPS; ++w) { - if (!active_warps_.test(w) && !(cta_fire_ && w == cta_fire_wid_)) { wid = int(w); break; } + bool firing = false; + for (uint32_t i = 0; i < cta_fire_cnt_; ++i) { + firing = firing || (cta_fire_wids_[i] == w); + } + if (!active_warps_.test(w) && !firing) { wid = int(w); break; } } if (wid < 0) break; // no free slot this cycle diff --git a/sim/simx/scheduler.h b/sim/simx/scheduler.h index a397eb60ba..7a3fd6dee2 100644 --- a/sim/simx/scheduler.h +++ b/sim/simx/scheduler.h @@ -118,7 +118,18 @@ class Scheduler : public SimObject { ~Scheduler(); // ----- Warp lifecycle ----- + // Once per cycle, before the frontend slices pick: activate the warps the CTA + // dispatcher launched last cycle, launch the next ones, and apply a wspawn. + void schedule_begin(); + + // One frontend slice's pick: its lowest ready warp in warp_mask. instr_trace_t* schedule(const WarpMask& warp_mask); + + // Whether warp_mask holds a ready warp, without picking it. + bool has_ready(const WarpMask& warp_mask) const; + + // Once per cycle, after the picks: clock the registered warp state. + void schedule_end(); void suspend(uint32_t wid); void resume(uint32_t wid); // Advance the warp's PC by `inc` bytes (called at decode with 2 or 4 @@ -211,12 +222,12 @@ class Scheduler : public SimObject { WarpMask active_warps_; WarpMask stalled_warps_; // registered (current) state read by schedule() WarpMask stalled_warps_next_; // next-state written by suspend()/resume() - // CTA dispatch pipeline: the active set the dispatcher sees, and the warp - // it picked last cycle. + // the active set the CTA dispatcher sees WarpMask cta_active_view_; - bool cta_fire_ = false; - uint32_t cta_fire_wid_ = 0; - cta_warp_record_t cta_fire_rec_; + // Warps the CTA dispatcher launched this cycle, one per launch lane at most. + uint32_t cta_fire_cnt_ = 0; + std::array cta_fire_wids_; + std::array cta_fire_recs_; uint32_t ipdom_size_; wspawn_t wspawn_; uint32_t mpm_class_; diff --git a/sim/simx/scoreboard.h b/sim/simx/scoreboard.h index e05ade6479..d5514bc654 100644 --- a/sim/simx/scoreboard.h +++ b/sim/simx/scoreboard.h @@ -69,7 +69,9 @@ class Scoreboard : public SimObject { return (wid << RegOpd::ID_BITS) | reg.id(); } - static constexpr uint32_t kReleaseDelay = 1; + // A committed result is written back from the commit arbiter's output + // register, and the issue stage reads a registered readiness. + static constexpr uint32_t kReleaseDelay = 2; struct pending_release_t { uint32_t wid; diff --git a/sim/simx/sequencer.h b/sim/simx/sequencer.h index 9f7222f270..31f525ec21 100644 --- a/sim/simx/sequencer.h +++ b/sim/simx/sequencer.h @@ -51,6 +51,9 @@ class Sequencer : public SimObject { // A macro-op's first micro-op is the current one. bool starting() const { return state_.active && state_.uop_index == 0; } + // A macro-op's last micro-op is the current one. + bool last() const { return state_.active && (state_.uop_index + 1) == state_.uop_count; } + // Drop any cached uop / macro-op state for this sequencer. Used by // Core::flush_warp_pipeline at async-trap entry: when the ibuffer is // flushed, the trace pointer this sequencer cached in state_.current_uop diff --git a/sim/simx/sfu_unit.cpp b/sim/simx/sfu_unit.cpp index 2b680b1132..f1739893f3 100644 --- a/sim/simx/sfu_unit.cpp +++ b/sim/simx/sfu_unit.cpp @@ -30,6 +30,13 @@ using namespace vortex; +// With several SFU blocks, their TEX/RTU/DXA requests merge onto the core's bus +// through one registered arbiter stage. +static constexpr uint32_t kBusMergeDelay = (VX_CFG_NUM_SFU_BLOCKS > 1) ? 1 : 0; +// With several SFU blocks, warp-control requests are registered toward the +// core-wide warp-control unit; a sole block reaches it directly. +static constexpr bool kWctlBuffered = (VX_CFG_NUM_SFU_BLOCKS > 1); + SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 6) , warp_ctl_out(this, VX_CFG_NUM_WARPS) @@ -54,18 +61,22 @@ SfuUnit::SfuUnit(const SimContext& ctx, const char* name, Core* core) , wctl_unit_(new WctlUnit(core)) , csr_unit_(new CsrUnit(core)) #ifdef VX_CFG_EXT_DXA_ENABLE - , dxa_unit_(new DxaUnit(core, dxa_req_out)) + , dxa_unit_(new DxaUnit(core, dxa_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_TEX_ENABLE - , tex_unit_(new TexUnit(core, tex_req_out)) + , tex_unit_(new TexUnit(core, tex_req_out, kBusMergeDelay)) #endif #ifdef VX_CFG_EXT_OM_ENABLE , om_unit_(new OmUnit(core, om_req_out)) #endif #ifdef VX_CFG_EXT_RTU_ENABLE - , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_)) + , rtu_unit_(new RtuUnit(core, rtu_req_out, rtu_window_, kBusMergeDelay)) #endif { + // the dispatch skid buffer's ready is registered + for (auto& input : Inputs) { + input.set_registered_ready(true); + } } // Cycles from processing to the output: the result buffer and the PE @@ -85,6 +96,40 @@ bool SfuUnit::rtu_trace2_reserve_slot(uint32_t wid) { #endif +// Results sent toward commit with the unit that produced them, in the order +// they reach the output channel (arrival cycle, then send order), which is the +// order commit pops them in. The oldest PE_SWITCH_SIZE stand in the shared +// response switch; the rest still wait in their unit's own result buffer. +void SfuUnit::sync_out(uint32_t b) { + auto& q = out_pe_.at(b); + uint32_t n = Outputs.at(b).size(); + while (q.size() > n) { + q.pop_front(); + } +} + +bool SfuUnit::pe_buf_full(uint32_t b, PeClass pe) { + this->sync_out(b); + auto& q = out_pe_.at(b); + uint32_t held = 0; + for (size_t i = PE_SWITCH_SIZE; i < q.size(); ++i) { + held += (q[i].pe == pe); + } + return held >= PE_BUF_SIZE; +} + +void SfuUnit::send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe) { + this->sync_out(b); + Outputs.at(b).send(trace, delay); + auto& q = out_pe_.at(b); + uint64_t arrive = SimPlatform::instance().cycles() + delay; + auto it = q.end(); + while (it != q.begin() && std::prev(it)->arrive > arrive) { + --it; + } + q.insert(it, OutPe{arrive, pe}); +} + void SfuUnit::on_tick() { #ifdef VX_CFG_EXT_RTU_ENABLE // Drain RTU rsps. Two flavors, both completing the warp's parked WAIT @@ -114,7 +159,7 @@ void SfuUnit::on_tick() { auto wb = is_candidate ? rtu_unit_->on_candidate_rsp(rsp) : rtu_unit_->on_terminal_rsp(rsp); if (wb.trace) { - Outputs.at(wb.block_id).send(wb.trace, this->latency_of(wb.trace)); + this->send_out(wb.block_id, wb.trace, this->latency_of(wb.trace), PE_OTHER); DT(3, "rtu-rsp deliver: core=" << core_->id() << ", wid=" << wb.trace->wid << ", cand=" << is_candidate); } else { @@ -142,7 +187,7 @@ void SfuUnit::on_tick() { { // Unit latency is already modeled by the TEX pipeline; charge only // the gather/writeback hop. - output.send(trace, 2); + this->send_out(rsp.block_id, trace, 2, PE_OTHER); DT(3, "tex-rsp deliver: core=" << core_->id() << ", wid=" << trace->wid); } tex_rsp_in.pop(); @@ -256,12 +301,36 @@ void SfuUnit::on_tick() { } #endif + // Grant each shared resource to one requesting block this cycle. + SharedRes want[VX_CFG_NUM_SFU_BLOCKS]; + int winner[RES_COUNT]; + instr_trace_t* winner_head[RES_COUNT] = {}; + for (uint32_t r = 0; r < RES_COUNT; ++r) { + winner[r] = -1; + } + for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { + want[b] = this->shared_res(b); + } + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + for (uint32_t i = 0; i < VX_CFG_NUM_SFU_BLOCKS; ++i) { + uint32_t b = (res_rr_[r] + i) % VX_CFG_NUM_SFU_BLOCKS; + if (want[b] == r) { + winner[r] = b; + winner_head[r] = Inputs.at(b).peek(); + break; + } + } + } + // PE switch: peek input, route to the matching sub-unit (WCTL / CSR / // DXA / TEX / OM / RASTER) by op_type, gather to the single result port. for (uint32_t b = 0; b < VX_CFG_NUM_SFU_BLOCKS; ++b) { auto& input = Inputs.at(b); if (input.empty()) continue; + if (want[b] != RES_NONE && winner[want[b]] != int(b)) { + continue; // another block holds the shared resource this cycle + } auto& output = Outputs.at(b); auto trace = input.peek(); @@ -302,7 +371,7 @@ void SfuUnit::on_tick() { continue; // OM back-pressure — nothing was sent; retry } } - output.send(trace, 1); + this->send_out(b, trace, 1, PE_OTHER); input.pop(); continue; } @@ -334,7 +403,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_cb_ret(trace, b)) continue; // bus backpressure — nothing was sent - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -349,7 +418,7 @@ void SfuUnit::on_tick() { if (output.full()) continue; if (!rtu_unit_->process_trace_uop(trace, b, args.uop)) continue; - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -365,7 +434,7 @@ void SfuUnit::on_tick() { } instr_trace_t* wb = rtu_unit_->process_wait(trace, b); if (wb) { - output.send(wb, this->latency_of(wb)); + this->send_out(b, wb, this->latency_of(wb), PE_OTHER); } input.pop(); continue; @@ -379,7 +448,7 @@ void SfuUnit::on_tick() { auto args = std::get(trace->instr_ptr->get_args()); if (output.full()) continue; rtu_window_.process_getw_uop(trace, args.uop, *rtu_p == GfxwType::GETWF); - output.send(trace, this->latency_of(trace)); + this->send_out(b, trace, this->latency_of(trace), PE_OTHER); input.pop(); continue; } @@ -390,6 +459,42 @@ void SfuUnit::on_tick() { // The fragment work distributor (above + scheduler) launches fragment // warps directly from the autonomously-pulled covered-quad waves. + // A CSR op is held a cycle before it may fire: the CTA context RAMs are + // read and the CSR decode registered in it. The wait restarts per op, so + // the CSR path takes one op every two cycles. + const bool is_csr = std::get_if(&trace->op_type) != nullptr; + if (is_csr && csr_read_done_.at(b) != trace) { + csr_read_done_.at(b) = trace; + continue; + } + + // A barrier arrive is the one shared request with a response. With + // several SFU blocks its request is registered toward the shared unit, + // which answers it two cycles after it is sent; it is sent once the LSU + // has drained, and the arrival takes effect as it is sent, as the + // unit's one grant per cycle orders it among the other arrivals. A sole + // block is answered in the cycle it asks, like any other op. + bool bar_arrive = false; + if (auto wctl_p = std::get_if(&trace->op_type); + kWctlBuffered && wctl_p && *wctl_p == WctlType::BAR && trace->eop) { + auto args = std::get(trace->instr_ptr->get_args()); + bar_arrive = args.is_bar_arrive; + } + if (bar_arrive) { + auto& ba = bar_arrive_.at(b); + uint64_t now = SimPlatform::instance().cycles(); + if (ba.trace != trace) { + if (!core_->lsu_drained()) { + continue; // drain LSU before the barrier + } + ba = BarArrive{trace, now, false, false}; + ba.release_warp = wctl_unit_->process(trace, &ba.warp_exit); + continue; + } + if (now < ba.sent + 2 || output.full()) { + continue; + } + } #ifdef VX_CFG_EXT_MUFU_ENABLE if (std::get_if(&trace->op_type)) { if (output.full()) { @@ -398,8 +503,8 @@ void SfuUnit::on_tick() { if (!mufu_unit_.process(trace)) { continue; } - output.send(trace, this->latency_of(trace) + MufuUnit::kLatency); trace->resume_warp = trace->fetch_stall; + this->send_out(b, trace, this->latency_of(trace) + MufuUnit::kLatency, PE_OTHER); input.pop(); continue; } @@ -408,6 +513,13 @@ void SfuUnit::on_tick() { if (output.full()) continue; // stall — no side effects this tick + // The warp-control and CSR units each answer through their own + // two-entry result buffer ahead of the shared response switch. + const bool is_wctl = std::get_if(&trace->op_type) != nullptr; + const PeClass pe = is_csr ? PE_CSR : (is_wctl ? PE_WCTL : PE_OTHER); + if (pe != PE_OTHER && this->pe_buf_full(b, pe)) + continue; + // WSYNC has a structural gate: cannot complete until prior insts retire. // BAR (vx_barrier and vx_barrier_arrive) drains LSU before continuing — // implements CUDA __syncthreads / OpenCL barrier(CLK_LOCAL_MEM_FENCE) semantic. @@ -416,7 +528,7 @@ void SfuUnit::on_tick() { if (*wctl_p == WctlType::WSYNC) { if (core_->has_pending_instrs(trace->wid)) continue; // wait for the warp's prior instrs to retire - } else if (*wctl_p == WctlType::BAR) { + } else if (*wctl_p == WctlType::BAR && !bar_arrive) { if (!core_->lsu_drained()) continue; // drain LSU before the barrier } @@ -425,10 +537,15 @@ void SfuUnit::on_tick() { bool release_warp = trace->fetch_stall; bool warp_exit = false; - if (std::get_if(&trace->op_type)) { + if (bar_arrive) { + release_warp = bar_arrive_.at(b).release_warp; + warp_exit = bar_arrive_.at(b).warp_exit; + bar_arrive_.at(b).trace = nullptr; + } else if (std::get_if(&trace->op_type)) { release_warp = wctl_unit_->process(trace, &warp_exit); - } else if (std::get_if(&trace->op_type)) { + } else if (is_csr) { csr_unit_->process(trace); + csr_read_done_.at(b) = nullptr; #ifdef VX_CFG_EXT_DXA_ENABLE } else if (std::get_if(&trace->op_type)) { // process() returns nullptr on backpressure (idempotent retry next @@ -440,7 +557,7 @@ void SfuUnit::on_tick() { } uint32_t delay = this->latency_of(trace); - output.send(trace, delay); + this->send_out(b, trace, delay, pe); // Warp-control refines the default (fetch_stall) release decision: a // sync-barrier, a not-yet-last barrier arrival, a deferred wspawn, or a // warp that disabled itself (tmask=0) keeps the warp parked — it is @@ -456,4 +573,72 @@ void SfuUnit::on_tick() { input.pop(); } + + // A granted request that went through moves its resource's pointer past it. + for (uint32_t r = RES_WCTL; r < RES_COUNT; ++r) { + if (winner[r] < 0) { + continue; + } + auto& input = Inputs.at(winner[r]); + if (input.empty() || input.peek() != winner_head[r] + || bar_arrive_.at(winner[r]).trace == winner_head[r]) { + res_rr_[r] = (winner[r] + 1) % VX_CFG_NUM_SFU_BLOCKS; + } + } +} + +void SfuUnit::on_reset() { + std::fill(std::begin(res_rr_), std::end(res_rr_), 0); + csr_read_done_.fill(nullptr); + bar_arrive_.fill(BarArrive{}); + for (auto& q : out_pe_) { + q.clear(); + } +} + +SfuUnit::SharedRes SfuUnit::shared_res(uint32_t block) { + auto& input = Inputs.at(block); + if (input.empty()) { + return RES_NONE; + } + auto trace = input.peek(); +#ifdef VX_CFG_EXT_TEX_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_TEX; + } +#endif +#ifdef VX_CFG_EXT_RTU_ENABLE + if (auto rtu_p = std::get_if(&trace->op_type)) { + if (*rtu_p == GfxwType::TRACE || *rtu_p == GfxwType::CB_RET) { + return RES_RTU; + } + } +#endif +#ifdef VX_CFG_EXT_DXA_ENABLE + if (std::get_if(&trace->op_type)) { + return RES_DXA; + } +#endif + if (auto wctl_p = std::get_if(&trace->op_type)) { + // a barrier arrive whose request has left no longer needs the port + if (bar_arrive_.at(block).trace == trace) { + return RES_NONE; + } + // A block requests only when its op could fire this cycle. A barrier + // arrive's request does not wait on the output: it completes later. + bool bar_arrive = false; + if (*wctl_p == WctlType::BAR && trace->eop) { + bar_arrive = std::get(trace->instr_ptr->get_args()).is_bar_arrive; + } + if (!bar_arrive && Outputs.at(block).full()) { + return RES_NONE; + } + if (*wctl_p == WctlType::BAR && trace->eop && !core_->lsu_drained()) { + return RES_NONE; + } + if (wctl_unit_->is_shared(trace)) { + return RES_WCTL; + } + } + return RES_NONE; } diff --git a/sim/simx/sfu_unit.h b/sim/simx/sfu_unit.h index 9485f6bc19..12bf37b425 100644 --- a/sim/simx/sfu_unit.h +++ b/sim/simx/sfu_unit.h @@ -13,6 +13,8 @@ #pragma once +#include + #include #include #include "func_unit.h" @@ -121,13 +123,49 @@ class SfuUnit : public FuncUnit { #endif protected: + void on_reset() override; void on_tick() override; private: uint32_t latency_of(const instr_trace_t* trace) const; + // Core-wide resources the SFU blocks share: the warp-control resources + // (barrier unit, wspawn, warp retirement) and the TEX/RTU/DXA buses each + // accept one request per cycle, granted round-robin per resource. + enum SharedRes { RES_NONE = 0, RES_WCTL, RES_TEX, RES_RTU, RES_DXA, RES_COUNT }; + SharedRes shared_res(uint32_t block); + uint32_t res_rr_[RES_COUNT] = {}; + std::unique_ptr wctl_unit_; std::unique_ptr csr_unit_; + // The unit a result in the output channel came from: the warp-control and + // CSR units each hold PE_BUF_SIZE results ahead of the shared response + // switch, which holds PE_SWITCH_SIZE. Results reach the channel after + // different delays, so each is kept at its arrival cycle, in the channel's + // own order. + enum PeClass : uint8_t { PE_OTHER = 0, PE_WCTL, PE_CSR }; + static constexpr uint32_t PE_BUF_SIZE = 2; + static constexpr uint32_t PE_SWITCH_SIZE = 2; + struct OutPe { + uint64_t arrive; + PeClass pe; + }; + std::array, VX_CFG_NUM_SFU_BLOCKS> out_pe_; + void sync_out(uint32_t b); + bool pe_buf_full(uint32_t b, PeClass pe); + void send_out(uint32_t b, instr_trace_t* trace, uint32_t delay, PeClass pe); + + // Per block, the CSR op that has spent its one-cycle CTA-context read. + std::array csr_read_done_{}; + // Per block, the barrier arrive waiting for the shared barrier unit's + // response, which carries the phase it writes back. + struct BarArrive { + const instr_trace_t* trace = nullptr; + uint64_t sent = 0; + bool release_warp = false; + bool warp_exit = false; + }; + std::array bar_arrive_{}; #ifdef VX_CFG_EXT_RTU_ENABLE // Shared graphics register window (SETW/GETW/GETWF slot file) used by the RTU // ray/hit stream, TEX (vx_tex4) u,v payload + texel, and OM (vx_om4) payload. diff --git a/sim/simx/socket.cpp b/sim/simx/socket.cpp index 73f8bd0625..cb16ceaa86 100644 --- a/sim/simx/socket.cpp +++ b/sim/simx/socket.cpp @@ -17,6 +17,9 @@ #include "cluster.h" #include "constants.h" #include "types.h" +#ifdef SIMX_SOCKET_UNIT_MMU +#include "mem/mmu.h" +#endif #ifdef VX_CFG_EXT_DXA_ENABLE #include "dxa_core.h" #include "local_mem.h" @@ -60,9 +63,9 @@ class Socket::Impl { log2ceil(VX_CFG_L1_LINE_SIZE), // S (no sectoring) log2ceil(sizeof(uint32_t)), // W log2ceil(VX_CFG_ICACHE_NUM_WAYS),// A - log2ceil(1), // B + log2ceil(VX_CFG_ICACHE_NUM_BANKS), // B VX_CFG_XLEN, // address bits - 1, // number of inputs + ICACHE_NUM_REQS, // number of inputs (one per frontend slice) VX_CFG_ICACHE_MEM_PORTS, // memory ports false, // write-back false, // write response @@ -70,6 +73,17 @@ class Socket::Impl { VX_CFG_ICACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_ICACHE_REPL_POLICY, // replacement policy false, // is_llc (icache never carries AMO state) + false, // non-cacheable bypass + false, // bank_hash + false, // shared_llc + false, // lazy_fetch + true, // core-response register + (VX_CFG_ICACHE_READ_MERGE != 0) ? ICACHE_NUM_REQS : 0, // read_merge: the core's fetch slices + false, // write_around + VX_CFG_ICACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port + false, // acq_forward + true, // merge_line: slices fetch different words of a line }); snprintf(sname, 100, "%s-dcache", name.c_str()); @@ -91,6 +105,16 @@ class Socket::Impl { VX_CFG_DCACHE_LATENCY, // pipeline latency (capacity-scaled, matches hardware) VX_CFG_DCACHE_REPL_POLICY, // replacement policy (VX_CFG_DCACHE_ENABLED != 0) && (VX_CFG_L2_ENABLED == 0) && (VX_CFG_L3_ENABLED == 0), // is_llc + true, // non-cacheable bypass + (VX_CFG_DCACHE_BANK_HASH != 0) && (VX_CFG_DCACHE_NUM_BANKS > 1), // bank_hash + DCACHE_SHARED_LLC, // shared_llc + VX_CFG_DCACHE_LAZY_FETCH != 0, // lazy_fetch + true, // core-response register + (VX_CFG_DCACHE_READ_MERGE != 0) ? DCACHE_CHANNELS : 0, // read_merge + false, // write_around + VX_CFG_DCACHE_MRSQ_SIZE != 0, // memory-response queue + true, // the socket's registered memory port + DCACHE_ACQ_FORWARD, // acq_forward }); #ifdef VX_CFG_EXT_TEX_ENABLE @@ -103,7 +127,7 @@ class Socket::Impl { constexpr uint32_t kTcacheWordSize = 4; // = TCACHE_WORD_SIZE constexpr uint32_t kTcacheNumReqs = VX_CFG_TCACHE_NUM_BANKS; constexpr uint32_t kTcacheMemPorts = 1; // = TCACHE_MEM_PORTS - auto tcache = Cache::Create(sname, Cache::Config{ + Cache::Config tcache_config{ false, // bypass log2ceil(VX_CFG_TCACHE_SIZE), // C log2ceil(kTcacheLineSize), // L @@ -120,7 +144,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy (use L2 policy as default) false, // is_llc (TCACHE is auxiliary, not LLC) - }); + }; + tcache_config.mem_rsp_queue = (VX_CFG_TCACHE_MRSQ_SIZE != 0); + tcache_config.mem_rsp_slice = true; + auto tcache = Cache::Create(sname, tcache_config); tcache_ = tcache; // tex_core ↔ tcache (per-port). @@ -143,7 +170,7 @@ class Socket::Impl { // VX_CFG_RTCACHE_NUM_BANKS banks. constexpr uint32_t kRtcacheNumInputs = VX_CFG_NUM_RTU_BLOCKS; constexpr uint32_t kRtcacheMemPorts = 1; - auto rtcache = Cache::Create(sname, Cache::Config{ + Cache::Config rtcache_config{ false, // bypass log2ceil(VX_CFG_RTCACHE_SIZE), // C log2ceil(kRtcacheLineSize), // L @@ -160,7 +187,10 @@ class Socket::Impl { 2, // pipeline latency uint8_t(VX_CFG_L2_REPL_POLICY), // replacement policy false, // is_llc - }); + }; + rtcache_config.mem_rsp_queue = (VX_CFG_RTCACHE_MRSQ_SIZE != 0); + rtcache_config.mem_rsp_slice = true; + auto rtcache = Cache::Create(sname, rtcache_config); rtcache_ = rtcache; // RtuCore ↔ rtcache (per memory port). @@ -177,6 +207,17 @@ class Socket::Impl { dxa_core_ = DxaCore::Create(sname, simobject_); #endif +#ifdef SIMX_SOCKET_UNIT_MMU + // ── Socket unit MMU ───────────────────────────────────────────────── + // Translates the unit caches' and DXA's memory ports between them and the + // socket memory arbiter, so those caches are virtually tagged. Each + // response leaves through a registered stage, and a new miss is held a + // cycle in a park stage before it allocates; its clients only read. + snprintf(sname, 100, "%s-unit_mmu", name.c_str()); + unit_mmu_ = Mmu::Create(sname, UMMU_NUM_REQS, VX_CFG_UTLB_SIZE, + simobject_->unit_tlb_client(), false, true, false, true); +#endif + // find overlap uint32_t overlap = __MIN(VX_CFG_ICACHE_MEM_PORTS, VX_CFG_L1_MEM_PORTS); @@ -217,20 +258,32 @@ class Socket::Impl { sock_arb->RspOut.at(0).bind(&icaches_->mem_rsp_in.at(0)); dcaches_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(1)); sock_arb->RspOut.at(1).bind(&dcaches_->mem_rsp_in.at(0)); + // A unit port reaches the arbiter through the unit MMU under VM. + auto bind_unit_port = [&](uint32_t arb_idx, uint32_t ummu_idx, + SimChannel& req_src, + SimChannel& rsp_dst) { + #ifdef SIMX_SOCKET_UNIT_MMU + req_src.bind(&unit_mmu_->ReqIn.at(ummu_idx)); + unit_mmu_->RspOut.at(ummu_idx).bind(&rsp_dst); + unit_mmu_->ReqOut.at(ummu_idx).bind(&sock_arb->ReqIn.at(arb_idx)); + sock_arb->RspOut.at(arb_idx).bind(&unit_mmu_->RspIn.at(ummu_idx)); + #else + __unused(ummu_idx); + req_src.bind(&sock_arb->ReqIn.at(arb_idx)); + sock_arb->RspOut.at(arb_idx).bind(&rsp_dst); + #endif + }; #ifdef VX_CFG_EXT_TEX_ENABLE constexpr uint32_t kTexArbIdx = 2; - tcache_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(kTexArbIdx)); - sock_arb->RspOut.at(kTexArbIdx).bind(&tcache_->mem_rsp_in.at(0)); + bind_unit_port(kTexArbIdx, UMMU_TCACHE_IDX, tcache_->mem_req_out.at(0), tcache_->mem_rsp_in.at(0)); #endif #ifdef VX_CFG_EXT_RTU_ENABLE constexpr uint32_t kRtuArbIdx = 2 + VX_CFG_EXT_TEX_ENABLED; - rtcache_->mem_req_out.at(0).bind(&sock_arb->ReqIn.at(kRtuArbIdx)); - sock_arb->RspOut.at(kRtuArbIdx).bind(&rtcache_->mem_rsp_in.at(0)); + bind_unit_port(kRtuArbIdx, UMMU_RTCACHE_IDX, rtcache_->mem_req_out.at(0), rtcache_->mem_rsp_in.at(0)); #endif #ifdef VX_CFG_EXT_DXA_ENABLE constexpr uint32_t kDxaArbIdx = 2 + VX_CFG_EXT_TEX_ENABLED + VX_CFG_EXT_RTU_ENABLED; - dxa_core_->gmem_req_out.at(0).bind(&sock_arb->ReqIn.at(kDxaArbIdx)); - sock_arb->RspOut.at(kDxaArbIdx).bind(&dxa_core_->gmem_rsp_in.at(0)); + bind_unit_port(kDxaArbIdx, UMMU_DXA_IDX, dxa_core_->gmem_req_out.at(0), dxa_core_->gmem_rsp_in.at(0)); #endif bind_mem_port(0, sock_arb->ReqOut.at(0), sock_arb->RspIn.at(0)); @@ -242,7 +295,8 @@ class Socket::Impl { // connect l1 caches to outgoing memory interfaces for (uint32_t i = 0; i < VX_CFG_L1_MEM_PORTS; ++i) { snprintf(sname, 100, "%s-l1_arb%d", name.c_str(), i); - auto l1_arb = MemArbiter::Create(sname, ArbiterType::RoundRobin, 2 * overlap, overlap); + // icache first; the arbiter registers both directions + auto l1_arb = MemArbiter::Create(sname, ArbiterType::Priority, 2 * overlap, overlap, 1, 1); if (i < overlap) { icaches_->mem_req_out.at(i).bind(&l1_arb->ReqIn.at(i)); @@ -273,8 +327,10 @@ class Socket::Impl { // connect cores to caches for (uint32_t i = 0; i < cores_per_socket; ++i) { - cores_.at(i)->icache_req_out.at(0).bind(&icaches_->core_req_in.at(i).at(0)); - icaches_->core_rsp_out.at(i).at(0).bind(&cores_.at(i)->icache_rsp_in.at(0)); + for (uint32_t j = 0; j < ICACHE_NUM_REQS; ++j) { + cores_.at(i)->icache_req_out.at(j).bind(&icaches_->core_req_in.at(i).at(j)); + icaches_->core_rsp_out.at(i).at(j).bind(&cores_.at(i)->icache_rsp_in.at(j)); + } for (uint32_t j = 0; j < VX_CFG_DCACHE_NUM_REQS; ++j) { cores_.at(i)->dcache_req_out.at(j).bind(&dcaches_->core_req_in.at(i).at(j)); @@ -351,6 +407,12 @@ class Socket::Impl { if (core->running()) return true; } + #ifdef SIMX_SOCKET_UNIT_MMU + // Parked translations hold no channel packet. + if (!unit_mmu_->empty()) { + return true; + } + #endif return false; } @@ -416,7 +478,7 @@ class Socket::Impl { return cores_.at(idx); } - void dcache_flush_begin() { dcaches_->flush_begin(); } + void dcache_flush_begin(bool l1_only) { dcaches_->flush_begin(l1_only); } bool dcache_flush_done() const { return dcaches_->flush_done(); } void icache_flush_begin() { icaches_->flush_begin(); } bool icache_flush_done() const { return icaches_->flush_done(); } @@ -437,6 +499,22 @@ class Socket::Impl { RtuCore::Ptr& rtu_core() { return rtu_core_; } #endif +#ifdef SIMX_SOCKET_UNIT_MMU + SimChannel& tlb_miss_out() { return unit_mmu_->TlbMissOut; } + SimChannel& tlb_fill_in() { return unit_mmu_->TlbFillIn; } + // The unit caches are virtually tagged: a satp commit flushes the unit + // TLB and invalidates them. + void set_satp(uint64_t satp) { + unit_mmu_->set_satp(satp); + #ifdef VX_CFG_EXT_TEX_ENABLE + tcache_->invalidate(); + #endif + #ifdef VX_CFG_EXT_RTU_ENABLE + rtcache_->invalidate(); + #endif + } +#endif + private: Socket* simobject_; std::vector cores_; @@ -456,6 +534,9 @@ class Socket::Impl { Cache::Ptr rtcache_; RtuBusArbiter::Ptr rtu_bus_arb_; #endif +#ifdef SIMX_SOCKET_UNIT_MMU + Mmu::Ptr unit_mmu_; +#endif }; /////////////////////////////////////////////////////////////////////////////// @@ -463,12 +544,14 @@ class Socket::Impl { Socket::Socket(const SimContext& ctx, const char* name, uint32_t socket_id, - Cluster* cluster) + Cluster* cluster, + uint32_t unit_tlb_client) : SimObject(ctx, name) , mem_req_out(VX_CFG_L1_MEM_PORTS, this) , mem_rsp_in(VX_CFG_L1_MEM_PORTS, this) , socket_id_(socket_id) , cluster_(cluster) + , unit_tlb_client_(unit_tlb_client) , impl_(new Impl(this)) {} @@ -504,8 +587,8 @@ Core::Ptr& Socket::core(uint32_t idx) { return impl_->core(idx); } -void Socket::dcache_flush_begin() { - impl_->dcache_flush_begin(); +void Socket::dcache_flush_begin(bool l1_only) { + impl_->dcache_flush_begin(l1_only); } bool Socket::dcache_flush_done() const { @@ -541,3 +624,17 @@ RtuCore::Ptr& Socket::rtu_core() { return impl_->rtu_core(); } #endif + +#ifdef SIMX_SOCKET_UNIT_MMU +SimChannel& Socket::tlb_miss_out() { + return impl_->tlb_miss_out(); +} + +SimChannel& Socket::tlb_fill_in() { + return impl_->tlb_fill_in(); +} + +void Socket::set_satp(uint64_t satp) { + impl_->set_satp(satp); +} +#endif diff --git a/sim/simx/socket.h b/sim/simx/socket.h index f0d91a0605..70c8878ea5 100644 --- a/sim/simx/socket.h +++ b/sim/simx/socket.h @@ -14,7 +14,9 @@ #pragma once #include "types.h" +#include "constants.h" #include "cache.h" +#include "mem/tlb_types.h" #ifdef VX_CFG_EXT_DXA_ENABLE #include "dxa_core.h" #endif @@ -54,12 +56,16 @@ class Socket : public SimObject { Socket(const SimContext& ctx, const char* name, uint32_t socket_id, - Cluster* cluster); + Cluster* cluster, + uint32_t unit_tlb_client = 0); ~Socket(); uint32_t id() const { return socket_id_; } + // The unit MMU's client index at the cluster TLB. + uint32_t unit_tlb_client() const { return unit_tlb_client_; } + Cluster* cluster() const { return cluster_; } bool running() const; @@ -77,7 +83,7 @@ class Socket : public SimObject { // Forwarded cache flush (write-back eviction walk). The walk is a no-op // on write-through caches (`Cache::flush_begin` early-exits); forwarding // ensures a future write-back config exercises the same code path. - void dcache_flush_begin(); + void dcache_flush_begin(bool l1_only); bool dcache_flush_done() const; void icache_flush_begin(); bool icache_flush_done() const; @@ -98,12 +104,21 @@ class Socket : public SimObject { RtuCore::Ptr& rtu_core(); #endif +#ifdef SIMX_SOCKET_UNIT_MMU + // Unit MMU: its miss/fill link to the cluster TLB, and the device satp + // (Cluster::set_mmu_satp), which flushes its TLB on an actual change. + SimChannel& tlb_miss_out(); + SimChannel& tlb_fill_in(); + void set_satp(uint64_t satp); +#endif + protected: void on_reset(); private: uint32_t socket_id_; Cluster* cluster_; + uint32_t unit_tlb_client_; class Impl; Impl* impl_; diff --git a/sim/simx/stack_interleave.h b/sim/simx/stack_interleave.h index 39b4c6a4e2..4d37a9730b 100644 --- a/sim/simx/stack_interleave.h +++ b/sim/simx/stack_interleave.h @@ -38,12 +38,14 @@ class StackInterleave { static constexpr uint64_t NUM_THREADS = VX_CFG_NUM_THREADS; static constexpr uint64_t NUM_STACKS = uint64_t(VX_CFG_NUM_CLUSTERS) * NUM_SOCKETS * VX_CFG_SOCKET_SIZE * VX_CFG_NUM_WARPS * NUM_THREADS; - static constexpr uint64_t TOP = VX_MEM_STACK_BASE_ADDR; - static constexpr uint64_t SPAN = NUM_STACKS * STACK_SIZE; - static constexpr uint64_t BOTTOM = TOP - SPAN; + static constexpr uint64_t GROUP_SIZE = NUM_THREADS * STACK_SIZE; + static constexpr uint64_t TOP = VX_MEM_STACK_BASE_ADDR; + static constexpr uint64_t SPAN = NUM_STACKS * STACK_SIZE; + static constexpr uint64_t BOTTOM = TOP - SPAN; static_assert(ispow2(NUM_THREADS), "stack interleave needs a power-of-two thread count"); static_assert(SPAN <= TOP, "stack window underflows the address space"); + static_assert(!ENABLED || TOP % GROUP_SIZE == 0, "stack window top is not aligned to a group of stacks"); static bool contains(uint64_t addr) { return ENABLED && addr >= BOTTOM && addr < TOP; @@ -53,13 +55,12 @@ class StackInterleave { if (!contains(addr)) { return addr; } - uint64_t off = addr - BOTTOM; - uint64_t stack = off / STACK_SIZE; - uint64_t thread = stack % NUM_THREADS; - uint64_t group = stack / NUM_THREADS; + // The window is group-aligned, so the group index is the address's own. + uint64_t group = addr / GROUP_SIZE; + uint64_t thread = (addr / STACK_SIZE) % NUM_THREADS; uint64_t words = STACK_SIZE / WORD_SIZE; - uint64_t word = ((off % STACK_SIZE) / WORD_SIZE) ^ (group % words); - return BOTTOM + ((group * words + word) * NUM_THREADS + thread) * WORD_SIZE + (off % WORD_SIZE); + uint64_t word = ((addr % STACK_SIZE) / WORD_SIZE) ^ (group % words); + return group * GROUP_SIZE + (word * NUM_THREADS + thread) * WORD_SIZE + (addr % WORD_SIZE); } }; diff --git a/sim/simx/tcu/tcu_unit.cpp b/sim/simx/tcu/tcu_unit.cpp index 46d16b471b..7d958215ce 100644 --- a/sim/simx/tcu/tcu_unit.cpp +++ b/sim/simx/tcu/tcu_unit.cpp @@ -16,6 +16,7 @@ #include "tcu_unit.h" #include "tensor_cfg.h" #include +#include #include "core.h" #include "scheduler.h" #include "local_mem.h" @@ -75,8 +76,8 @@ static inline uint8_t unpack_u8(uint32_t word, uint32_t idx) { return (word >> (idx * 8)) & 0xffu; } -// FMA: fused multiply-add returning an Ot-typed accumulator (bit-packed in uint32). -// Widens narrow inputs/accumulator to fp32, performs mul+add, rounds once to Ot. +// FMA: one multiply-add step into an Ot-typed accumulator (bit-packed +// in uint32): fp32 for float inputs, int32 for integer inputs. template struct FMA { using itype = typename It::dtype; @@ -97,15 +98,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_htof_s(a, 0, nullptr); - auto fb = rv_htof_s(b, 0, nullptr); - auto fc = rv_htof_s(uint16_t(c), 0, nullptr); - return rv_ftoh_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf16 inputs -- template <> struct FMA { static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { @@ -114,15 +106,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint16_t a, uint16_t b, uint32_t c) { - auto fa = rv_btof_s(a, 0, nullptr); - auto fb = rv_btof_s(b, 0, nullptr); - auto fc = rv_btof_s(uint16_t(c), 0, nullptr); - return rv_ftob_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- fp8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -131,15 +114,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e4m3tof_s(a, 0, nullptr); - auto fb = rv_e4m3tof_s(b, 0, nullptr); - auto fc = rv_e4m3tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe4m3_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- bf8 inputs -- template <> struct FMA { static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { @@ -148,15 +122,6 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint8_t a, uint8_t b, uint32_t c) { - auto fa = rv_e5m2tof_s(a, 0, nullptr); - auto fb = rv_e5m2tof_s(b, 0, nullptr); - auto fc = rv_e5m2tof_s(uint8_t(c), 0, nullptr); - return rv_ftoe5m2_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - // -- tf32 inputs -- template <> struct FMA { static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { @@ -165,19 +130,21 @@ template <> struct FMA { return rv_fadd_s(rv_fmul_s(fa, fb, 0, nullptr), c, 0, nullptr); } }; -template <> struct FMA { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = rv_tf32tof_s(a, 0, nullptr); - auto fb = rv_tf32tof_s(b, 0, nullptr); - auto fc = rv_tf32tof_s(c, 0, nullptr); - return rv_ftotf32_s(rv_fmadd_s(fa, fb, fc, 0, nullptr), 0, nullptr); - } -}; - -// Generic FEDP: universal rule keyed on output width. -// * Wide Ot (fp32): accumulate Σ(a_k*b_k) in fp32, add c_val last. -// * Narrow Ot (fp16/bf16/fp8/bf8/…): chain FMA so the accumulator is -// rounded to Ot each step. +// Float source formats the native TFR datapath reduces (exponent, mantissa bits). +template struct TfrFormat { static constexpr bool valid = false; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 10; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 7; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 4, sb = 3; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 5, sb = 2; }; +template <> struct TfrFormat { static constexpr bool valid = true; static constexpr int eb = 8, sb = 10; }; + +// Generic FEDP, keyed on the output format. +// * fp32 Ot on the native TFR datapath: its fixed-point reduction — +// products aligned to the largest exponent in a 25-bit window, summed as +// integers, rounded once (the TfrFedp model the RTL is checked against). +// * fp32 Ot otherwise: accumulate Σ(a_k*b_k) in fp32 per register, +// the register sums in order, c_val last (the DPI datapath's order). +// * int32 Ot (integer inputs): exact integer accumulation. template struct FEDP { using itype = typename It::dtype; @@ -188,6 +155,22 @@ struct FEDP { static uint32_t eval_n(const reg_data_t *a_row, const reg_data_t *b_col, uint32_t c_val, uint32_t k_words) { constexpr uint32_t i_ratio = sizeof(uint32_t) / sizeof(itype); static_assert(i_ratio * sizeof(itype) == sizeof(uint32_t), "FEDP: tcK * i_ratio must be <= 32"); +#ifdef VX_CFG_TCU_TYPE_TFR + if constexpr (std::is_same_v && TfrFormat::valid) { + std::vector a(k_words), b(k_words); + for (uint32_t z = 0; z < k_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + TfrFedp fedp(TfrFormat::eb, TfrFormat::sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, k_words); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } else +#endif if constexpr (std::is_same_v) { uint32_t acc = 0; for (uint32_t z = 0; z < k_words; ++z) { @@ -201,6 +184,7 @@ struct FEDP { } return rv_fadd_s(c_val, acc, 0, nullptr); } else { + static_assert(std::is_same_v, "TCU output type must be fp32 (int32 for integer inputs)"); uint32_t acc = c_val; for (uint32_t z = 0; z < k_words; ++z) { auto a = reinterpret_cast(&a_row[z].u32); @@ -282,26 +266,6 @@ static PFN_FEDP_N select_FEDP_N(uint32_t IT, uint32_t OT) { break; } break; - case vt::fp16::id: - if (IT == vt::fp16::id) - return FEDP::eval_n; - break; - case vt::bf16::id: - if (IT == vt::bf16::id) - return FEDP::eval_n; - break; - case vt::fp8::id: - if (IT == vt::fp8::id) - return FEDP::eval_n; - break; - case vt::bf8::id: - if (IT == vt::bf8::id) - return FEDP::eval_n; - break; - case vt::tf32::id: - if (IT == vt::tf32::id) - return FEDP::eval_n; - break; case vt::int32::id: switch (IT) { case vt::int8::id: @@ -412,8 +376,8 @@ class TcuUnit::Impl { } #ifdef TCU_META_ENABLE - // Each TCU block tracks one metadata transaction. Requests share the LSU - // client port, while tags allow their memory latency to overlap. + // Each TCU block's AGU tracks one metadata load and writes each returned + // lane straight into the block's metadata SRAM. static constexpr uint32_t kAguBeats = (VX_CFG_NUM_THREADS + VX_CFG_NUM_LSU_LANES - 1) / VX_CFG_NUM_LSU_LANES; @@ -426,7 +390,6 @@ class TcuUnit::Impl { uint32_t next_beat = 0; uint32_t rsp_count = 0; std::array addrs{}; - std::array words{}; }; std::array agu_; uint32_t agu_issue_rr_ = 0; @@ -452,70 +415,108 @@ class TcuUnit::Impl { } } + LsuReq agu_make_req(uint32_t block, agu_state_t& agu) { + LsuReq req(VX_CFG_NUM_LSU_LANES); + req.op = MemOp::LD; + req.tag = block * kAguBeats + agu.next_beat; + req.wid = agu.wid; + req.cid = agu.trace->cid; + req.uuid = agu.trace->uuid; + for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { + uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; + if (idx >= VX_CFG_NUM_THREADS) { + break; + } + req.mask.set(i); + req.addrs.at(i) = agu.addrs.at(idx); + req.tids.at(i) = idx % VX_CFG_NUM_THREADS; + } + ++agu.next_beat; + return req; + } + + void agu_receive(const LsuRsp& rsp) { + uint32_t block = (uint32_t)rsp.tag / kAguBeats; + uint32_t beat = (uint32_t)rsp.tag % kAguBeats; + auto& agu = agu_.at(block); + for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { + if (!rsp.mask.test(lane)) { + continue; + } + uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; + assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); + uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); + uint32_t word = 0; + std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); + this->agu_write(agu, idx, word); + ++agu.rsp_count; + } + if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { + agu.complete = true; + } + } + void agu_step() { - if (!simobject_->agu_req_out.full()) { - for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { - uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; + if (TCU_AGU_PORTS == 1) { + // every block shares one port: round-robin one request per cycle + if (!simobject_->agu_req_out.at(0).full()) { + for (uint32_t offset = 0; offset < VX_CFG_NUM_TCU_BLOCKS; ++offset) { + uint32_t block = (agu_issue_rr_ + offset) % VX_CFG_NUM_TCU_BLOCKS; + auto& agu = agu_.at(block); + if (!agu.busy || agu.next_beat >= kAguBeats) { + continue; + } + simobject_->agu_req_out.at(0).send(this->agu_make_req(block, agu)); + agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; + break; + } + } + } else { + // each block issues on its own port + for (uint32_t block = 0; block < VX_CFG_NUM_TCU_BLOCKS; ++block) { auto& agu = agu_.at(block); - if (!agu.busy || agu.next_beat >= kAguBeats) { + if (!agu.busy || agu.next_beat >= kAguBeats + || simobject_->agu_req_out.at(block).full()) { continue; } - LsuReq req(VX_CFG_NUM_LSU_LANES); - req.op = MemOp::LD; - req.tag = block * kAguBeats + agu.next_beat; - req.wid = agu.wid; - req.cid = agu.trace->cid; - req.uuid = agu.trace->uuid; - for (uint32_t i = 0; i < VX_CFG_NUM_LSU_LANES; ++i) { - uint32_t idx = agu.next_beat * VX_CFG_NUM_LSU_LANES + i; - if (idx >= VX_CFG_NUM_THREADS) { - break; - } - req.mask.set(i); - req.addrs.at(i) = agu.addrs.at(idx); - req.tids.at(i) = idx % VX_CFG_NUM_THREADS; - } - simobject_->agu_req_out.send(req); - ++agu.next_beat; - agu_issue_rr_ = (block + 1) % VX_CFG_NUM_TCU_BLOCKS; - break; + simobject_->agu_req_out.at(block).send(this->agu_make_req(block, agu)); } } - if (!simobject_->agu_rsp_in.empty()) { - auto& rsp = simobject_->agu_rsp_in.peek(); - uint32_t block = (uint32_t)rsp.tag / kAguBeats; - uint32_t beat = (uint32_t)rsp.tag % kAguBeats; - auto& agu = agu_.at(block); - for (uint32_t lane = 0; lane < rsp.mask.size(); ++lane) { - if (!rsp.mask.test(lane)) { - continue; - } - uint32_t idx = beat * VX_CFG_NUM_LSU_LANES + lane; - assert(idx < VX_CFG_NUM_THREADS && rsp.data.at(lane)); - uint32_t off = agu.addrs.at(idx) & (VX_CFG_MEM_BLOCK_SIZE - 1); - uint32_t word = 0; - std::memcpy(&word, rsp.data.at(lane)->data() + off, 4); - agu.words.at(idx) = word; - ++agu.rsp_count; + for (uint32_t p = 0; p < TCU_AGU_PORTS; ++p) { + auto& rsp_in = simobject_->agu_rsp_in.at(p); + if (rsp_in.empty()) { + continue; } - simobject_->agu_rsp_in.pop(); - if (agu.busy && !agu.complete && agu.rsp_count == VX_CFG_NUM_THREADS) { - this->agu_deposit(agu); - agu.complete = true; + this->agu_receive(rsp_in.peek()); + rsp_in.pop(); + } + } + + // A TCU_LD retires once its words are deposited into the metadata SRAM; + // the AGU takes the next load the cycle after. + void agu_retire(uint64_t now) { + for (uint32_t b = 0; b < VX_CFG_NUM_TCU_BLOCKS; ++b) { + auto& agu = agu_.at(b); + if (!agu.complete) + continue; + // The result is offered in the cycle its last fragment lands, as the + // AGU's done flag; the lane gather adds the one register to commit. + if (simobject_->Outputs.at(b).try_send(agu.trace, 0)) { + result_due_.at(b).push_back(now); + DT(3, simobject_->name() << " execute: op=" << TcuType::TCU_LD << ", " << *agu.trace); + agu = agu_state_t{}; } } } - // Deposit the fetched words into the selected metadata SRAM. - void agu_deposit(const agu_state_t& agu) { + // Write lane T's fetched word into the selected metadata SRAM. + void agu_write(const agu_state_t& agu, uint32_t T, uint32_t word) { uint32_t wid = agu.wid; #ifdef VX_CFG_TCU_MX_ENABLE if (agu.selector & 0x10) { auto& dst = (agu.selector & 1) ? mx_meta_b_.at(wid) : mx_meta_a_.at(wid); - for (uint32_t lane = 0; lane < VX_CFG_NUM_THREADS; ++lane) { - dst.at(lane) = agu.words.at(lane); - } + dst.at(T) = word; return; } #endif @@ -530,26 +531,25 @@ class TcuUnit::Impl { constexpr uint32_t BPS = cfg::banks_per_store; constexpr uint32_t SPC = cfg::stores_per_col; constexpr uint32_t CPL = cfg::meta_cols_per_load; - for (uint32_t T = 0; T < VX_CFG_NUM_THREADS; ++T) { - uint32_t store_in_load = T / BPS; - uint32_t thread_in_store = T % BPS; - uint32_t flat_store = slot_idx * CPL + store_in_load; - uint32_t col = flat_store / SPC; - uint32_t store_in_col = flat_store % SPC; - uint32_t bank = store_in_col * BPS + thread_in_store; - if (bank >= PWD || col >= kMaxMetaCols) { - continue; - } - uint32_t word = agu.words.at(T); - sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; - // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). - if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { - if (p[0] == '1') { - fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", - wid, bank, col, (unsigned long)agu.addrs.at(T), word); - } + uint32_t store_in_load = T / BPS; + uint32_t thread_in_store = T % BPS; + uint32_t flat_store = slot_idx * CPL + store_in_load; + uint32_t col = flat_store / SPC; + uint32_t store_in_col = flat_store % SPC; + uint32_t bank = store_in_col * BPS + thread_in_store; + if (bank >= PWD || col >= kMaxMetaCols) { + return; + } + sparse_meta_.at(wid).at(bank * kMaxMetaCols + col) = word; + // Trace: META_SRAM write (CSV: wid,bank,col,addr,value). + if (const char* p = std::getenv("VORTEX_TCU_TRACE")) { + if (p[0] == '1') { + fprintf(stderr, "META_TRC,%u,%u,%u,0x%lx,0x%08x\n", + wid, bank, col, (unsigned long)agu.addrs.at(T), word); } } + #else + __unused(T, word); #endif } #endif // TCU_META_ENABLE @@ -575,13 +575,29 @@ class TcuUnit::Impl { auto& input = simobject_->Inputs.at(b); if (input.empty()) continue; - if (completing && backed_up) { - continue; - } auto trace = input.peek(); auto tcu_type = std::get(trace->op_type); auto tpuArgs = std::get(trace->instr_ptr->get_args()); + #ifdef TCU_META_ENABLE + // A TCU_LD leaves the input for the AGU once the AGU is free, so the ops + // behind it run while its metadata is in flight; it retires from the AGU. + if (tcu_type == TcuType::TCU_LD) { + if (!agu_.at(b).busy) { + // rs1 is a full-width address (.u64); use u64 to avoid truncation on XLEN=64. + auto& rs1_data = trace->src_data[0]; + uint64_t base_addr = rs1_data.empty() ? 0 : rs1_data.at(0).u64; + this->agu_start(b, trace->wid, tpuArgs.fmt_d, base_addr, trace); + input.pop(); + } + continue; + } + #endif + + if (completing && backed_up) { + continue; + } + #ifdef VX_CFG_TCU_WGMMA_ENABLE // A WGMMA compute uop fires once its tile-buffer operands are ready. if (tcu_is_wgmma(tcu_type) && !tpuArgs.is_setup_uop @@ -601,6 +617,12 @@ class TcuUnit::Impl { trace->dst_data.assign(num_threads, reg_data_t{}); auto& rd_data = trace->dst_data; + if (!vt::tcu_output_valid(tpuArgs.fmt_s, tpuArgs.fmt_d)) { + std::cerr << "*** TCU output format " << tpuArgs.fmt_d << " for input format " << tpuArgs.fmt_s + << " is not supported: the TCU returns fp32 (int32 for integer inputs)" << std::endl; + std::abort(); + } + switch (tcu_type) { case TcuType::WMMA: case TcuType::WMMA_SP: @@ -649,16 +671,6 @@ class TcuUnit::Impl { } } } break; - #endif - #ifdef TCU_META_ENABLE - case TcuType::TCU_LD: { - if (agu_.at(b).busy) { - continue; - } - // rs1 is a full-width address (.u64); use u64 to avoid truncation on XLEN=64. - uint64_t base_addr = rs1_data.empty() ? 0 : rs1_data.at(0).u64; - this->agu_start(b, wid, tpuArgs.fmt_d, base_addr, trace); - } break; #endif default: std::abort(); @@ -666,49 +678,18 @@ class TcuUnit::Impl { exec_done_.at(b) = true; } - uint32_t delay = 0; - switch (tcu_type) { - case TcuType::WMMA: - case TcuType::WMMA_SP: - case TcuType::WGMMA: - case TcuType::WGMMA_SP: - delay = kMmaLatency; - break; - #ifdef TCU_META_ENABLE - case TcuType::TCU_LD: - // The AGU round-trip models the memory latency; the remaining - // cycles cover the metadata deposit and retire. - delay = 3; - break; - #endif - default: - std::abort(); - } - #ifdef TCU_META_ENABLE - // TCU_LD retires only after its words are deposited into the - // metadata SRAM. - if (tcu_type == TcuType::TCU_LD) { - auto& agu = agu_.at(b); - assert(agu.busy && agu.trace == trace); - if (!agu.complete) { - continue; // metadata still in flight — hold the block - } - } - #endif - if (simobject_->Outputs.at(b).try_send(trace, delay)) { - due.push_back(now + delay); + if (simobject_->Outputs.at(b).try_send(trace, kMmaLatency)) { + due.push_back(now + kMmaLatency); exec_done_.at(b) = false; - #ifdef TCU_META_ENABLE - // TCU_LD retired: free the AGU for the next metadata load. - if (tcu_type == TcuType::TCU_LD) { - agu_.at(b) = agu_state_t{}; - } - #endif DT(3, simobject_->name() << " execute: op=" << tcu_type << ", " << *trace); input.pop(); } } + #ifdef TCU_META_ENABLE + this->agu_retire(now); + #endif + #ifdef VX_CFG_TCU_WGMMA_ENABLE // A setup uop drops its block's A stripe and the shared B row at the end // of the cycle; uops firing alongside it still read the old contents. @@ -1352,6 +1333,46 @@ class TcuUnit::Impl { }; if (fmt_d == vt::fp32::id) { +#ifdef VX_CFG_TCU_TYPE_TFR + // The native datapath: the TfrFedp model, with each lane's (16 bits of + // a register) scales taken at its first element. + { + int eb = 0, sb = 0; + bool e4m3_scale = false; + if (fmt_s == vt::mxfp8::id) { + eb = 4; sb = 3; + } else if (fmt_s == vt::mxbf8::id) { + eb = 5; sb = 2; + } else if (fmt_s == vt::mxfp4::id) { + eb = 2; sb = 1; + } else if (fmt_s == vt::nvfp4::id) { + eb = 2; sb = 1; e4m3_scale = true; + } else { + std::abort(); + } + uint32_t sparse_ratio = is_sparse ? 2 : 1; + std::vector a(fedp_words), b(fedp_words); + std::vector sf_a(2 * fedp_words), sf_b(2 * fedp_words); + for (uint32_t z = 0; z < fedp_words; ++z) { + a[z] = a_row[z].u32; + b[z] = b_col[z].u32; + } + for (uint32_t lane = 0; lane < 2 * fedp_words; ++lane) { + uint32_t z = lane / 2; + uint32_t e = (lane % 2) * (ratio / 2); + uint32_t elem_k = ((step_k * fedp_words + z) * ratio + e) * sparse_ratio; + sf_a[lane] = uint8_t(scale_a(elem_k)); + sf_b[lane] = uint8_t(scale_b(elem_k)); + } + TfrFedp fedp(eb, sb); + float c; + std::memcpy(&c, &c_val, sizeof(c)); + float d = fedp(a.data(), b.data(), c, fedp_words, sf_a.data(), sf_b.data(), e4m3_scale); + uint32_t d_bits; + std::memcpy(&d_bits, &d, sizeof(d_bits)); + return d_bits; + } +#endif uint32_t acc = c_val; for (uint32_t z = 0; z < fedp_words; ++z) { for (uint32_t e = 0; e < ratio; ++e) { @@ -1611,6 +1632,158 @@ op_string_t TcuUnit::op_string(TcuType tcu_type, IntrTcuArgs args) { /////////////////////////////////////////////////////////////////////////////// +namespace { + +constexpr uint32_t clog2(uint32_t x) { + return (x < 2) ? 0 : (1 + clog2(x / 2)); +} + +constexpr uint32_t kLgM = clog2(cfg::m_steps); +constexpr uint32_t kLgN = clog2(cfg::n_steps); +constexpr uint32_t kLgK = clog2(cfg::k_steps); +constexpr uint32_t kLgBSb = clog2(cfg::b_sub_blocks); + +constexpr uint32_t hi_bit(uint32_t x, uint32_t lg) { + return (x >> (lg ? (lg - 1) : 0)) & 1; +} + +// Register offsets of the WMMA operands of step (m, n, k), permuted so the +// three operands of a step never share a register bank. +// one B sub-block: A={m[0],~m[hi],k}, B={n^k,~k}, C={m[0],~m[hi],XNOR(m[hi],n)} +// two B sub-blocks: A={k[0],~m,m^k[hi]}, B={k[0],k[hi]^np,~np}, C={n[0],~m,n[hi]} +// asymmetric sparse: B={n[hi],~(n[0]^k),~k}, C={n[hi],m,~(m^n[0])} +uint32_t ra_off(uint32_t m, uint32_t k) { + if (kLgBSb == 0) { + return ((m & 1) << 2) | ((1 - hi_bit(m, kLgM)) << 1) | (k & 1); + } + return ((k & 1) << 2) | ((1 - (m & 1)) << 1) | ((m & 1) ^ hi_bit(k, kLgK)); +} + +uint32_t rb_off(uint32_t n, uint32_t k) { + if (kLgBSb == 0) { + return (((n ^ k) & 1) << 1) | (1 - (k & 1)); + } + uint32_t np = hi_bit(n, kLgN); + return ((k & 1) << 2) | ((hi_bit(k, kLgK) ^ np) << 1) | (1 - np); +} + +uint32_t rc_off(uint32_t m, uint32_t n) { + if (kLgBSb == 0) { + uint32_t m_hi = hi_bit(m, kLgM); + return ((m & 1) << 2) | ((1 - m_hi) << 1) | (1 - (m_hi ^ (n & 1))); + } + return ((n & 1) << 2) | ((1 - (m & 1)) << 1) | hi_bit(n, kLgN); +} + +uint32_t rb_off_sp(uint32_t n, uint32_t k) { + return (hi_bit(n, kLgN) << 2) | ((1 - ((n ^ k) & 1)) << 1) | (1 - (k & 1)); +} + +uint32_t rc_off_sp(uint32_t m, uint32_t n) { + return (hi_bit(n, kLgN) << 2) | ((m & 1) << 1) | (1 - ((m ^ n) & 1)); +} + +// vx_ldm/vx_stm shapes +constexpr uint32_t kLdmA = 0x00; // WMMA A, row-major +constexpr uint32_t kLdmASp = 0x01; // WMMA sparse A, row-major +constexpr uint32_t kLdmB = 0x06; // WMMA B, col-major +constexpr uint32_t kLdmBSp = 0x07; // WMMA symmetric sparse B, col-major +constexpr uint32_t kLdmC = 0x08; // WMMA C, row-major +constexpr uint32_t kLdmWgC = 0x18; // WGMMA C, row-major, one 8-register chunk + +// Byte strides between the words a lane holds along a fragment row +constexpr uint32_t kLdmLgCsK = clog2(cfg::tcK * 4); +constexpr uint32_t kLdmLgCsN = clog2(cfg::tcN * 4); +// Row groups of a col-major B fragment +constexpr uint32_t kLdmLgBRows = kLgN - kLgBSb; +constexpr uint32_t kLdmLgKSp = (kLgK > 0) ? (kLgK - 1) : 0; + +uint32_t ldm_lg_count(uint32_t shape) { + switch (shape) { + case kLdmA: return kLgM + kLgK; + case kLdmASp: return kLgM + kLdmLgKSp; + case kLdmB: + case kLdmBSp: return clog2(cfg::NRB); + default: return 3; // C: 8 registers + } +} + +} // namespace + +bool TcuUopGen::ldm_valid(uint32_t shape) { + switch (shape) { + case kLdmA: + case kLdmB: + case kLdmC: + return true; +#ifdef VX_CFG_TCU_SPARSE_ENABLE + case kLdmASp: + return true; + case kLdmBSp: + return cfg::sym_sparse; +#endif +#ifdef VX_CFG_TCU_WGMMA_ENABLE + case kLdmWgC: + return true; +#endif + default: + return false; + } +} + +uint32_t TcuUopGen::ldm_count(uint32_t shape) { + return 1u << ldm_lg_count(shape); +} + +// Element e of a fragment: its register offset, row group and column +static TcuUopGen::LdmUop ldm_elem(uint32_t shape, uint32_t e) { + TcuUopGen::LdmUop u{0, 0, 0}; + switch (shape) { + case kLdmA: // e = {m, k}: row group m, word column k + u.grp = e >> kLgK; + u.col = (e & ((1u << kLgK) - 1)) << kLdmLgCsK; + u.reg_off = ra_off(e >> kLgK, e); + break; + case kLdmASp: // e = {m, k} over half the K steps; registers in order + u.grp = e >> kLdmLgKSp; + u.col = (e & ((1u << kLdmLgKSp) - 1)) << kLdmLgCsK; + u.reg_off = e; + break; + case kLdmB: // e = {k, n}: a col-major B row is one of its columns (a pair with two sub-blocks) + u.grp = e & ((1u << kLdmLgBRows) - 1); + u.col = (e >> kLdmLgBRows) << kLdmLgCsK; + u.reg_off = rb_off(u.grp << kLgBSb, e >> kLdmLgBRows); + break; + case kLdmBSp: // symmetric sparse: one row per register + u.grp = e; + u.reg_off = e; + break; + case kLdmC: // e = {m, n} + u.grp = e >> kLgN; + u.col = (e & ((1u << kLgN) - 1)) << kLdmLgCsN; + u.reg_off = rc_off(e >> kLgN, e); + break; + default: // WGMMA C, e = {n, m}: n-major registers, two row groups + u.grp = e & 1; + u.col = (e >> 1) << kLdmLgCsN; + u.reg_off = e; + break; + } + return u; +} + +// Uop e moves register fd/fs2 + e, holding the element whose register offset is e +TcuUopGen::LdmUop TcuUopGen::ldm_uop(uint32_t shape, uint32_t e) { + uint32_t count = ldm_count(shape); + for (uint32_t r = 0; r < count; ++r) { + auto u = ldm_elem(shape, r); + if (u.reg_off == e) { + return u; + } + } + std::abort(); +} + uint32_t TcuUopGen::uop_count(const Instr& instr) { if (instr.get_fu_type() != FUType::TCU) return 1; @@ -1661,6 +1834,7 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { auto uop_instr = std::allocate_shared(pool_, uop_uuid, FUType::TCU); uop_instr->set_parent_uuid(parent_uuid); + uop_instr->set_fcsr_use(macro_instr.fcsr_reads(), macro_instr.fcsr_writes()); if (tcu_is_wmma(tcu_type)) { using wmma = vt::wmma_config_t; @@ -1705,51 +1879,22 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { uop_instr->set_tmask(ThreadMask(VX_CFG_NUM_THREADS, (mma_idx & 1) ? (all_lanes & ~sym_mask_lo) : sym_mask_lo)); } else { // Standard k-major triple loop (dense or non-sym sparse) - uint32_t b_sub = is_sparse ? wmma::b_sub_blocks_sp : wmma::b_sub_blocks; uint32_t mn = wmma::m_steps * wmma::n_steps; uint32_t k = mma_idx / mn; uint32_t rem = mma_idx % mn; uint32_t m = rem / wmma::n_steps; uint32_t n = rem % wmma::n_steps; - // Bank-conflict-free register offset formulas (0 stalls). - // Permutes A, B, C offsets so all three operands always land in - // different GPR banks for every uop. uint32_t reg_rs1, reg_rs2, reg_rs3; if (is_sparse) { - // Sparse non-sym: A=identity, B and C permuted for 0 stalls. - // B={n[hi], ~(n[0]^k), ~k}, C={n[hi], m, ~(m^n[0])} - uint32_t n_hi = n >> 1; - uint32_t n_lo = n & 1; + // Sparse non-sym: A=identity, B and C permuted. uint32_t a_off = (m / wmma::a_sub_blocks) * k_count + k; - uint32_t b_off = (n_hi << 2) | ((1 - (n_lo ^ k)) << 1) | (1 - k); - uint32_t c_off = (n_hi << 2) | (m << 1) | (1 - (m ^ n_lo)); reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; - } else if (b_sub == 1) { - // Dense Pattern A (NT=4,16,64): m_steps=4, k_steps=2, n_steps=2 - // A={m[0], ~m[hi], k}, B={n^k, ~k}, C={m[0], ~m[hi], XNOR(m[hi],n)} - uint32_t m_hi = m >> 1; - uint32_t m_lo = m & 1; - uint32_t a_off = (m_lo << 2) | ((1 - m_hi) << 1) | k; - uint32_t b_off = ((n ^ k) << 1) | (1 - k); - uint32_t c_off = (m_lo << 2) | ((1 - m_hi) << 1) | (1 - (m_hi ^ n)); - reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; + reg_rs2 = rb_base + rb_off_sp(n, k); + reg_rs3 = rc_base + rc_off_sp(m, n); } else { - // Dense Pattern B (NT=8,32): m_steps=2, k_steps=4, n_steps=4, b_sub=2 - // A={k[0], ~m, m^k[hi]}, B={k[0], k[hi]^np, ~np}, C={n[0], ~m, n[hi]} - uint32_t k_hi = k >> 1; - uint32_t k_lo = k & 1; - uint32_t n_pair = n >> 1; - uint32_t n_lo = n & 1; - uint32_t a_off = (k_lo << 2) | ((1 - m) << 1) | (m ^ k_hi); - uint32_t b_off = (k_lo << 2) | ((k_hi ^ n_pair) << 1) | (1 - n_pair); - uint32_t c_off = (n_lo << 2) | ((1 - m) << 1) | n_pair; - reg_rs1 = ra_base + a_off; - reg_rs2 = rb_base + b_off; - reg_rs3 = rc_base + c_off; + reg_rs1 = ra_base + ra_off(m, k); + reg_rs2 = rb_base + rb_off(n, k); + reg_rs3 = rc_base + rc_off(m, n); } uop_instr->set_op_type(is_sparse ? TcuType::WMMA_SP : TcuType::WMMA); uop_instr->set_args(IntrTcuArgs{0, 0, fmt_s, fmt_d, m, n, k, 0, 0, 0}); @@ -1834,8 +1979,8 @@ Instr::Ptr TcuUopGen::get(const Instr& macro_instr, uint32_t uop_index) { TcuUnit::TcuUnit(const SimContext &ctx, const char* name, Core* core) : FuncUnit(ctx, name, core, 1u << log2ceil(VX_CFG_TCU_LATENCY + 1)) // landing-queue depth bounds results awaiting the consumer #ifdef TCU_META_ENABLE - , agu_req_out(this) - , agu_rsp_in(this) + , agu_req_out(TCU_AGU_PORTS, this) + , agu_rsp_in(TCU_AGU_PORTS, this) #endif , impl_(new Impl(this, core)) { diff --git a/sim/simx/tcu/tcu_unit.h b/sim/simx/tcu/tcu_unit.h index 93328060f4..d36b5a181d 100644 --- a/sim/simx/tcu/tcu_unit.h +++ b/sim/simx/tcu/tcu_unit.h @@ -28,6 +28,13 @@ #define TCU_META_ENABLE #endif +// TCU_LD memory ports: one per TCU block, through the LSU block that serves +// the same warps, when the LSU has one block per TCU block; otherwise all +// TCU blocks share LSU block 0, as in the RTL (VX_tcu_unit). +#ifndef TCU_AGU_PORTS +#define TCU_AGU_PORTS (uint32_t((VX_CFG_NUM_LSU_BLOCKS == VX_CFG_NUM_TCU_BLOCKS) ? VX_CFG_NUM_TCU_BLOCKS : 1)) +#endif + namespace vortex { class Core; @@ -46,6 +53,19 @@ class TcuUopGen { // Generate micro-op Instr at uop_index for the given macro instruction. Instr::Ptr get(const Instr& macro_instr, uint32_t uop_index); + // Matrix fragment load/store (vx_ldm/vx_stm) shapes, {ctx, role[1:0], trans, + // sparse}. Uop e moves register fd/fs2 + e (reg_off = e) to or from + // x[rs1 | grp] + (offset | col), the address of the fragment element the + // WMMA/WGMMA expansion reads from that register. + struct LdmUop { + uint32_t reg_off; + uint32_t grp; + uint32_t col; + }; + static bool ldm_valid(uint32_t shape); + static uint32_t ldm_count(uint32_t shape); + static LdmUop ldm_uop(uint32_t shape, uint32_t e); + private: PoolAllocator& pool_; }; @@ -77,11 +97,11 @@ class TcuUnit : public FuncUnit { virtual ~TcuUnit(); #ifdef TCU_META_ENABLE - // Metadata AGU port: TCU_LD issues load requests through the LSU block-0 - // client port; response fragments return here and accumulate into the - // metadata SRAM. - SimChannel agu_req_out; - SimChannel agu_rsp_in; + // Metadata AGU ports (TCU_AGU_PORTS): TCU_LD issues load requests through + // the LSU client ports; response fragments return here and accumulate into + // the metadata SRAM. + std::vector> agu_req_out; + std::vector> agu_rsp_in; #endif void wmma(uint32_t wid, diff --git a/sim/simx/tex/tex_unit.cpp b/sim/simx/tex/tex_unit.cpp index fd88c0731b..462a8f924f 100644 --- a/sim/simx/tex/tex_unit.cpp +++ b/sim/simx/tex/tex_unit.cpp @@ -49,7 +49,7 @@ instr_trace_t* TexUnit::process(instr_trace_t* trace, uint32_t block_id) { // Combinational into the shared arbiter; the arbiter's forward is the // registered boundary. - req_out_.send(req, 0); + req_out_.send(req, bus_delay_); DT(3, "tex-unit submit: core=" << core_->id() << ", wid=" << trace->wid << ", stage=" << args.stage); return trace; diff --git a/sim/simx/tex/tex_unit.h b/sim/simx/tex/tex_unit.h index 710c011701..65b1319eb5 100644 --- a/sim/simx/tex/tex_unit.h +++ b/sim/simx/tex/tex_unit.h @@ -84,8 +84,10 @@ class TexCore; // backpressure (SfuUnit retries idempotently next cycle). class TexUnit { public: - TexUnit(Core* core, SimChannel& req_out) - : core_(core), req_out_(req_out) {} + // bus_delay: extra cycles a request spends merging with other SFU blocks' + // requests onto the core's texture bus. + TexUnit(Core* core, SimChannel& req_out, uint32_t bus_delay = 0) + : core_(core), req_out_(req_out), bus_delay_(bus_delay) {} // Submit one sample request: u/v/lod ride src_data[0..2], the texel returns in // rd. Returns the trace if accepted, or nullptr on full output channel. @@ -94,6 +96,7 @@ class TexUnit { private: Core* core_; SimChannel& req_out_; + uint32_t bus_delay_; }; } // namespace vortex diff --git a/sim/simx/types.h b/sim/simx/types.h index 31f7fbddbf..b307fa5963 100644 --- a/sim/simx/types.h +++ b/sim/simx/types.h @@ -384,8 +384,18 @@ struct IntrLsuArgs { uint32_t width : 3; uint32_t stride : 4; // per-uop multiplier: 0 = regular LD/ST; 0..N-1 = packLD uop_idx int32_t offset; // immediate addr offset (signed); 0 for packLD uops + bool matrix = false; // vx_ldm/vx_stm: offset[4:0] is the TCU fragment shape }; +// A fence carries no access width; its width field says what it orders. +enum class FenceKind : uint32_t { + FULL = 0, // earlier writes visible and later reads see others' writes + IO = 1, // only device (IO) accesses + ACQUIRE = 2, // only later reads see others' writes + RELEASE = 3 // only earlier writes visible +}; +inline FenceKind lsu_fence_kind(const IntrLsuArgs& args) { return static_cast(args.width); } + inline std::ostream &operator<<(std::ostream &os, const LsuType& type) { switch (type) { case LsuType::LOAD: os << "LOAD"; break; @@ -444,6 +454,9 @@ struct MemFlags { uint32_t io : 1; // bit 1: uncacheable I/O uint32_t local : 1; // bit 2: LMEM port uint32_t amo_unsigned : 1; // bit 3: MIN/MAX signedness (1=unsigned) + uint32_t flush : 1; // bit 4: flush token (a fence) + uint32_t flush_l1 : 1; // bit 5: L1-only flush token: stops at the shared LLC + uint32_t noalloc : 1; // bit 6: a write that misses a write-around cache goes to memory #ifdef VX_CFG_EXT_DXA_ENABLE // DXA completion sideband — populated only on DXA's LMEM-DMA writes. // notify_bar_id carries the RAW (kernel-encoded) bar handle: @@ -453,8 +466,8 @@ struct MemFlags { // be wide enough to hold (bar_no << 8) | (MAX_CTAS_per_core - 1). // For NUM_BARRIERS = 8 and NUM_WARPS ≤ 16, that's at most 11 bits; // 16 bits leaves headroom for larger configs. - uint32_t dxa_notify_done : 1; // bit 4 - uint32_t dxa_notify_bar_id : 16; // bits 5..20 + uint32_t dxa_notify_done : 1; // bit 7 + uint32_t dxa_notify_bar_id : 16; // bits 8..23 #endif }; }; @@ -1086,7 +1099,8 @@ class GreedyRRArbiter : public IArbiterImpl { class Arbiter { public: - Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) { + Arbiter(ArbiterType type = ArbiterType::Priority, uint32_t size = 0) + : type_(type), size_(size) { switch (type) { case ArbiterType::Priority: impl_ = std::make_shared(size); @@ -1105,6 +1119,20 @@ class Arbiter { } } + // A copy is a separate arbiter with its own grant state, so a vector of + // arbiters built from one prototype does not share a single state. + Arbiter(const Arbiter& other) : Arbiter(other.type_, other.size_) {} + + Arbiter& operator=(const Arbiter& other) { + if (this != &other) { + *this = Arbiter(other.type_, other.size_); + } + return *this; + } + + Arbiter(Arbiter&&) = default; + Arbiter& operator=(Arbiter&&) = default; + virtual ~Arbiter() {} uint32_t grant(const BitVector<>& requests) { @@ -1116,6 +1144,8 @@ class Arbiter { } private: + ArbiterType type_; + uint32_t size_; std::shared_ptr impl_; }; @@ -1250,6 +1280,10 @@ struct MemReq { uint32_t hart_id; uint64_t uuid; MemFlags flags; + // Under cache line merging: the tags of the ports merged into this read, by + // offset in the leader's merge group. They ride the bank access and its + // response, as in the hardware's bank tag. + std::shared_ptr> merged_tags; MemReq(MemOp _op = MemOp::LD, uint64_t _addr = 0, @@ -1289,6 +1323,7 @@ struct MemRsp { uint32_t hart_id; uint64_t uuid; std::shared_ptr data; + std::shared_ptr> merged_tags; // see MemReq MemRsp(uint64_t _tag = 0, uint32_t _hart_id = 0, @@ -1438,6 +1473,39 @@ class TFifo : public SimObject> { /////////////////////////////////////////////////////////////////////////////// +// A registered elastic buffer: In holds its entries, and the head moves on +// through Out one register later, one per cycle. +template +class TxBuffer : public SimObject> { +public: + using Ptr = std::shared_ptr>; + + SimChannel In; + SimChannel Out; + + TxBuffer(const SimContext& ctx, const char* name, uint32_t size) + : SimObject>(ctx, name) + , In(this, size) + , Out(this) + {} + +protected: + void on_reset() {} + + void on_tick() { + if (!In.empty() && Out.try_send(In.peek(), 1)) { + In.pop(); + } + if (In.size() == 0) { + this->tick_sleep(); + } + } + + friend class SimObject>; +}; + +/////////////////////////////////////////////////////////////////////////////// + template class TxArbiter : public SimObject> { public: @@ -1604,7 +1672,7 @@ class TxCrossBar : public SimObject> { , lg2_outputs_(log2ceil(num_outputs)) , output_sel_(output_sel) , collisions_(0) { - assert(num_inputs <= 64); + // per-output round-robin pointers only: no bound on the input count assert(num_outputs <= 64); assert(ispow2(num_outputs)); assert(output_sel != nullptr); diff --git a/sim/simx/wctl_unit.cpp b/sim/simx/wctl_unit.cpp index 4fbe8a531e..e31d612e72 100644 --- a/sim/simx/wctl_unit.cpp +++ b/sim/simx/wctl_unit.cpp @@ -20,6 +20,47 @@ using namespace vortex; +ThreadMask WctlUnit::next_tmask(const instr_trace_t* trace) const { + auto wctl_type = std::get(trace->op_type); + auto& warp = core_->scheduler().warp(trace->wid); + uint32_t num_threads = VX_CFG_NUM_THREADS; + int32_t thread_last = num_threads - 1; + for (; thread_last >= 0; --thread_last) { + if (warp.tmask.test(thread_last)) { break; } + } + auto& rs1_data = trace->src_data[0]; + auto& rs2_data = trace->src_data[1]; + if (wctl_type == WctlType::TMC) { + ThreadMask tmask(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); + } + return tmask; + } + auto wctlArgs = std::get(trace->instr_ptr->get_args()); + auto not_pred = wctlArgs.is_cond_neg; + ThreadMask pred(num_threads); + for (uint32_t t = 0; t < num_threads; ++t) { + auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; + pred[t] = warp.tmask.test(t) && cond; + } + return pred.any() ? (warp.tmask & pred) : ThreadMask(num_threads, rs2_data.at(thread_last).u); +} + +bool WctlUnit::is_shared(const instr_trace_t* trace) const { + if (!trace->eop) { + return false; + } + auto wctl_type = std::get(trace->op_type); + if (wctl_type == WctlType::BAR || wctl_type == WctlType::WSPAWN) { + return true; + } + if (wctl_type != WctlType::TMC && wctl_type != WctlType::PRED) { + return false; + } + return this->next_tmask(trace).none(); +} + bool WctlUnit::process(instr_trace_t* trace, bool* exit) { bool release_warp = trace->fetch_stall; *exit = false; @@ -43,16 +84,13 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { switch (wctl_type) { case WctlType::TMC: { - ThreadMask next_tmask(num_threads); - for (uint32_t t = 0; t < num_threads; ++t) { - next_tmask.set(t, rs1_data.at(thread_last).u & (1 << t)); - } if (trace->eop) { - if (next_tmask.none()) { + auto tmask = this->next_tmask(trace); + if (tmask.none()) { *exit = true; release_warp = false; } else { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, tmask); } } } break; @@ -143,25 +181,15 @@ bool WctlUnit::process(instr_trace_t* trace, bool* exit) { release_warp = false; } } else { - release_warp = !core_->barrier_wait(bar_id, arg2, trace->wid); + // the barrier unit releases the warp once its phase has passed + core_->barrier_wait(bar_id, arg2, trace->wid); + release_warp = false; } } } break; case WctlType::PRED: { - ThreadMask pred(num_threads); - auto not_pred = wctlArgs.is_cond_neg; - for (uint32_t t = 0; t < num_threads; ++t) { - auto cond = (rs1_data.at(t).i & 0x1) ^ not_pred; - pred[t] = warp.tmask.test(t) && cond; - } - ThreadMask next_tmask = warp.tmask; - if (pred.any()) { - next_tmask &= pred; - } else { - next_tmask = ThreadMask(num_threads, rs2_data.at(thread_last).u); - } if (trace->eop) { - release_warp = core_->setTmask(trace->wid, next_tmask); + release_warp = core_->setTmask(trace->wid, this->next_tmask(trace)); } } break; case WctlType::WSYNC: diff --git a/sim/simx/wctl_unit.h b/sim/simx/wctl_unit.h index 799c34647e..d1ab897a28 100644 --- a/sim/simx/wctl_unit.h +++ b/sim/simx/wctl_unit.h @@ -34,7 +34,15 @@ class WctlUnit { // the latency. bool process(instr_trace_t* trace, bool* exit); + // Whether `trace` needs a core-wide warp-control resource (the barrier unit, + // wspawn, or warp retirement), which takes one request per cycle across all + // SFU blocks. No side effects. + bool is_shared(const instr_trace_t* trace) const; + private: + // Thread mask a TMC or PRED installs. + ThreadMask next_tmask(const instr_trace_t* trace) const; + Core* core_; }; diff --git a/sim/xrtsim/Makefile b/sim/xrtsim/Makefile index df068a9785..45c25e05bb 100644 --- a/sim/xrtsim/Makefile +++ b/sim/xrtsim/Makefile @@ -143,6 +143,17 @@ RTL_PKGS += $(RTL_DIR)/VX_trace_pkg.sv # Verilator from the -I path above. RTL_PKGS += $(RTL_DIR)/cp/VX_cp_pkg.sv +# One AXI memory master per GPU memory port. The platform top must declare +# its ports literally, so resolve the count from the elaborated configuration. +AFU_MEM_PORTS := $(shell $(SCRIPT_DIR)/afu_mem_ports.sh -c $(DESTDIR)/.afu_mem_ports $(VERILATOR_PATH)/bin/verilator \ + -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) \ + $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) \ + $(RTL_INCLUDE) $(RTL_PKGS)) +ifeq ($(AFU_MEM_PORTS),) +$(error cannot resolve the AFU memory port count for this configuration) +endif +CXXFLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) + TOP = vortex_afu_shim VL_FLAGS += --language 1800-2012 --assert -Wall -Wpedantic @@ -157,6 +168,7 @@ VL_FLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) # VX_CFG_TCU_TYPE_DPI) unmatched. Add just the enum companions gen_config # resolves, so a CONFIGS enum override actually selects its RTL implementation. VL_FLAGS += $(CONFIGS) $(filter -DVX_CFG_TCU_TYPE_% -DVX_CFG_FPU_TYPE_%,$(XCONFIGS)) +VL_FLAGS += -DAFU_MEM_PORTS=$(AFU_MEM_PORTS) VL_FLAGS += $(RTL_INCLUDE) VL_FLAGS += $(RTL_PKGS) diff --git a/sim/xrtsim/vortex_afu_shim.sv b/sim/xrtsim/vortex_afu_shim.sv index 59ed67bba7..d6a453507d 100644 --- a/sim/xrtsim/vortex_afu_shim.sv +++ b/sim/xrtsim/vortex_afu_shim.sv @@ -20,14 +20,14 @@ module vortex_afu_shim #( parameter C_M_AXI_MEM_ID_WIDTH = `PLATFORM_MEMORY_ID_WIDTH, parameter C_M_AXI_MEM_DATA_WIDTH = (`VX_CFG_PLATFORM_MEMORY_DATA_SIZE * 8), parameter C_M_AXI_MEM_ADDR_WIDTH = 64, - parameter C_M_AXI_MEM_NUM_BANKS = `VX_CFG_PLATFORM_MEMORY_NUM_BANKS + parameter C_M_AXI_MEM_NUM_BANKS = `AFU_MEM_PORTS ) ( // System signals input wire ap_clk, input wire ap_rst_n, // AXI4 master interface - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, GEN_AXI_MEM, MP_COMMA), + `MP_REPEAT (`AFU_MEM_PORTS, GEN_AXI_MEM, MP_COMMA), // AXI4 host-memory master interface (CP command ring + host side of DMA) `GEN_AXI_HOST, @@ -54,6 +54,9 @@ module vortex_afu_shim #( output wire interrupt `IGNORE_WARNINGS_END ); + `AXI_MEM_BUSES (C_M_AXI_MEM_NUM_BANKS) + `MP_REPEAT (`AFU_MEM_PORTS, AXI_MEM_TO_BUS, MP_SEMI); + VX_afu_wrap #( .C_S_AXI_CTRL_ADDR_WIDTH (C_S_AXI_CTRL_ADDR_WIDTH), .C_S_AXI_CTRL_DATA_WIDTH (C_S_AXI_CTRL_DATA_WIDTH), @@ -65,7 +68,7 @@ module vortex_afu_shim #( .clk (ap_clk), .reset (~ap_rst_n), - `MP_REPEAT (`VX_CFG_PLATFORM_MEMORY_NUM_BANKS, AXI_MEM_ARGS, MP_COMMA), + `AXI_MEM_BUS_ARGS, `AXI_HOST_ARGS, diff --git a/sim/xrtsim/xrt_sim.cpp b/sim/xrtsim/xrt_sim.cpp index af948d92dc..d1d0e349fc 100644 --- a/sim/xrtsim/xrt_sim.cpp +++ b/sim/xrtsim/xrt_sim.cpp @@ -161,7 +161,7 @@ class xrt_sim::Impl { "returned ahead of an older ready one\n", rsp_reorder_seed(), (unsigned long)rsp_reordered_); } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { delete mem_alloc_[b]; } for (auto ram : ram_) { @@ -217,21 +217,21 @@ class xrt_sim::Impl { sfp_->open(saif_file ? saif_file : "trace.saif"); #endif - // calculate memory bank size - mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + // Each AXI port owns an equal, contiguous share of device memory. + mem_bank_size_ = (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / AFU_MEM_PORTS; // One memory per AXI port, as on the platform, where each kernel master // is wired to its own memory resource. A request issued on the wrong port // then misses its data instead of finding it in a shared image. - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { ram_[b] = new RAM(0, RAM_PAGE_SIZE); } // initialize AXI memory interfaces - MP_M_AXI_MEM(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + MP_M_AXI_MEM(AFU_MEM_PORTS); // initialize memory allocator - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { mem_alloc_[b] = new MemoryAllocator(0, mem_bank_size_, 4096, 64); } @@ -295,13 +295,13 @@ class xrt_sim::Impl { } int mem_alloc(uint64_t size, uint32_t bank_id, uint64_t* addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->allocate(size, addr); } int mem_free(uint32_t bank_id, uint64_t addr) { - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; return mem_alloc_[bank_id]->release(addr); } @@ -309,7 +309,7 @@ class xrt_sim::Impl { int mem_write(uint32_t bank_id, uint64_t addr, uint64_t size, const void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->write(data, base_addr, size); @@ -319,7 +319,7 @@ class xrt_sim::Impl { int mem_read(uint32_t bank_id, uint64_t addr, uint64_t size, void* data) { HostLock guard(*this); - if (bank_id >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if (bank_id >= AFU_MEM_PORTS) return -1; uint64_t base_addr = bank_id * mem_bank_size_ + addr; ram_[bank_id]->read(data, base_addr, size); @@ -328,7 +328,7 @@ class xrt_sim::Impl { int mem_copy(uint32_t bank_id_dest , uint32_t bank_id_src, uint64_t dest_addr, uint64_t src_addr, uint64_t size) { HostLock guard(*this); - if( bank_id_dest >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS || bank_id_src >= VX_CFG_PLATFORM_MEMORY_NUM_BANKS) + if( bank_id_dest >= AFU_MEM_PORTS || bank_id_src >= AFU_MEM_PORTS) return -1; uint64_t dest_base_addr = bank_id_dest * mem_bank_size_ + dest_addr; uint64_t src_base_addr = bank_id_src * mem_bank_size_ + src_addr; @@ -480,10 +480,12 @@ class xrt_sim::Impl { } for (auto& st : m_axi_states_) { + st.aw_pending = {}; st.w_staged.clear(); + st.w_bursts = {}; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { std::queue empty; std::swap(dram_queues_[b], empty); } @@ -500,7 +502,7 @@ class xrt_sim::Impl { device_->ap_rst_n = 1; // this AXI device is always ready to accept new requests - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { *m_axi_mem_[b].arready = 1; *m_axi_mem_[b].awready = 1; *m_axi_mem_[b].wready = 1; @@ -532,10 +534,11 @@ class xrt_sim::Impl { dram_sim_.tick(); - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { - if (!dram_queues_[b].empty() && dram_sim_.can_accept(b)) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { + if (!dram_queues_[b].empty() + && dram_sim_.can_accept(dram_channel(dram_queues_[b].front()->addr))) { auto mem_req = dram_queues_[b].front(); - dram_sim_.send_request(b, bank_local_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { + dram_sim_.send_request(dram_channel(mem_req->addr), dram_channel_addr(mem_req->addr), mem_req->write, [](void* arg)->bool { auto orig_req = reinterpret_cast(arg); if (orig_req->ready) { delete orig_req; @@ -553,17 +556,33 @@ class xrt_sim::Impl { #endif } - // Byte address within its bank (DRAM channel) of an AXI bank address: - // VX_mem_to_axi places the bank select above the data-size bits when the - // banks are interleaved. - static uint64_t bank_local_addr(uint64_t axi_addr) { - #if VX_CFG_PLATFORM_MEMORY_INTERLEAVE == 1 - constexpr uint32_t lg2_data_size = log2ceil(VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - constexpr uint32_t bank_sel_bits = log2ceil(VX_CFG_PLATFORM_MEMORY_NUM_BANKS); - return (axi_addr >> (bank_sel_bits + lg2_data_size)) << lg2_data_size; - #else - return axi_addr; - #endif + // Port addresses are physical: the AFU gives each port its own contiguous + // share of device memory, and the platform lays its memory channels out + // linearly across that space, so a port's traffic stays on its own group of + // channels. + static constexpr uint64_t DRAM_CHANNEL_SIZE = + (1ull << VX_CFG_PLATFORM_MEMORY_ADDR_WIDTH) / VX_CFG_PLATFORM_MEMORY_NUM_BANKS; + + static uint32_t dram_channel(uint64_t axi_addr) { + return uint32_t((axi_addr / DRAM_CHANNEL_SIZE) % VX_CFG_PLATFORM_MEMORY_NUM_BANKS); + } + + static uint64_t dram_channel_addr(uint64_t axi_addr) { + return axi_addr % DRAM_CHANNEL_SIZE; + } + + // A port bound to its own channel group cannot reach outside it on the + // platform; a request that would is a routing bug, not a slow access. + void check_port_range(int port, const char* what, uint64_t addr, uint32_t len) { + uint64_t last = addr + uint64_t(len) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; + uint64_t lo = uint64_t(port) * mem_bank_size_; + uint64_t hi = lo + mem_bank_size_; + if (addr < lo || last >= hi) { + fprintf(stderr, "[XRTSIM] Error: %s at 0x%lx on m_axi_mem_%d, outside its range " + "[0x%lx, 0x%lx)\n", what, (unsigned long)addr, port, + (unsigned long)lo, (unsigned long)hi); + std::abort(); + } } void eval() { @@ -603,7 +622,7 @@ class xrt_sim::Impl { } void axi_mem_bus_reset() { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // read request address *m_axi_mem_[b].arready = 0; @@ -619,24 +638,24 @@ class xrt_sim::Impl { // write response *m_axi_mem_[b].bvalid = 0; - // states - m_axi_states_[b].aw_active = false; } } void axi_host_bus_reset() { // Ready signals stay constant-high (like the m_axi_mem model): a slave // that toggles *ready races the master, which only samples it the cycle - // after it enters its issue state. Our masters keep one outstanding - // read/write at a time, so always-ready is correct. + // after it enters its issue state. Reads and both write channels queue + // what they accept, so always-ready is correct. *m_axi_host_.arready = 1; *m_axi_host_.awready = 1; *m_axi_host_.wready = 1; *m_axi_host_.rvalid = 0; *m_axi_host_.bvalid = 0; host_rd_active_ = false; - host_wr_active_ = false; - host_wr_b_pending_ = false; + host_aw_pending_ = {}; + host_w_staged_.clear(); + host_w_bursts_ = {}; + host_b_pending_ = {}; host_rd_rsp_ready_ = false; host_wr_rsp_ready_ = false; std::queue().swap(host_rd_pending_); @@ -675,7 +694,7 @@ class xrt_sim::Impl { // B channel — retire the write response. if (*m_axi_host_.bvalid && host_wr_rsp_ready_) { *m_axi_host_.bvalid = 0; - host_wr_b_pending_ = false; + host_b_pending_.pop(); } // Accept a read burst. arready stays high (so the master's handshake // always completes the cycle it issues — no ready-toggle race); a request @@ -706,49 +725,57 @@ class xrt_sim::Impl { *m_axi_host_.rresp = 0; *m_axi_host_.rlast = (host_rd_beat_ >= host_rd_len_); } - // Accept a write burst (one outstanding at a time). - if (*m_axi_host_.awvalid && *m_axi_host_.awready - && !host_wr_active_ && !host_wr_b_pending_) { - host_wr_active_ = true; - host_wr_addr_ = uint64_t(*m_axi_host_.awaddr); - host_wr_id_ = *m_axi_host_.awid; - host_wr_beat_ = 0; + // Write channels: the k-th AW pairs with the k-th W burst, and either may + // arrive first (awready/wready stay high), so both sides queue. A write + // lands in host memory once its pair is complete, in AW order. + if (*m_axi_host_.awvalid && *m_axi_host_.awready) { + host_aw_pending_.push({uint64_t(*m_axi_host_.awaddr), uint32_t(*m_axi_host_.awid)}); } - // Accept a write data beat. - if (host_wr_active_ && *m_axi_host_.wvalid && *m_axi_host_.wready) { - uint64_t a = host_wr_addr_ - + uint64_t(host_wr_beat_) * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - auto byteen = *m_axi_host_.wstrb; - auto data = (const uint8_t*)m_axi_host_.wdata->data(); - for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { - if ((byteen >> i) & 0x1) - reinterpret_cast(a)[i] = data[i]; - } + if (*m_axi_host_.wvalid && *m_axi_host_.wready) { + w_beat_t beat; + beat.addr = 0; + beat.byteen = uint64_t(*m_axi_host_.wstrb); + std::memcpy(beat.data.data(), m_axi_host_.wdata->data(), + VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + host_w_staged_.emplace_back(beat); if (*m_axi_host_.wlast) { - host_wr_active_ = false; - host_wr_b_pending_ = true; - } else { - ++host_wr_beat_; + host_w_bursts_.push(std::move(host_w_staged_)); + host_w_staged_.clear(); + } + } + while (!host_aw_pending_.empty() && !host_w_bursts_.empty()) { + auto aw = host_aw_pending_.front(); + auto& beats = host_w_bursts_.front(); + for (size_t k = 0; k < beats.size(); ++k) { + // Host memory is plain process memory (see the read path above). + auto dst = reinterpret_cast(aw.addr + k * VX_CFG_PLATFORM_MEMORY_DATA_SIZE); + for (int i = 0; i < VX_CFG_PLATFORM_MEMORY_DATA_SIZE; ++i) { + if ((beats[k].byteen >> i) & 0x1) + dst[i] = beats[k].data[i]; + } } + host_b_pending_.push(aw.tag); + host_aw_pending_.pop(); + host_w_bursts_.pop(); } // Present the write response. - if (host_wr_b_pending_ && !*m_axi_host_.bvalid) { + if (!host_b_pending_.empty() && !*m_axi_host_.bvalid) { *m_axi_host_.bvalid = 1; - *m_axi_host_.bid = host_wr_id_; + *m_axi_host_.bid = host_b_pending_.front(); *m_axi_host_.bresp = 0; } } void axi_mem_bus_eval(bool clk) { if (!clk) { - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { m_axi_states_[b].read_rsp_ready = *m_axi_mem_[b].rready; m_axi_states_[b].write_rsp_ready = *m_axi_mem_[b].bready; } return; } - for (int b = 0; b < VX_CFG_PLATFORM_MEMORY_NUM_BANKS; ++b) { + for (int b = 0; b < AFU_MEM_PORTS; ++b) { // handle read responses if (*m_axi_mem_[b].rvalid && m_axi_states_[b].read_rsp_ready) { *m_axi_mem_[b].rvalid = 0; @@ -810,6 +837,7 @@ class xrt_sim::Impl { if (*m_axi_mem_[b].arvalid && *m_axi_mem_[b].arready) { uint32_t len = *m_axi_mem_[b].arlen; // beats - 1 uint64_t base = uint64_t(*m_axi_mem_[b].araddr); + check_port_range(b, "read", base, len); for (uint32_t beat = 0; beat <= len; ++beat) { auto mem_req = new mem_req_t(); mem_req->tag = *m_axi_mem_[b].arid; @@ -823,48 +851,51 @@ class xrt_sim::Impl { } } - // handle write address — latch the burst. - if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready - && !m_axi_states_[b].aw_active) { - m_axi_states_[b].aw_active = true; - m_axi_states_[b].aw_addr = uint64_t(*m_axi_mem_[b].awaddr); - m_axi_states_[b].aw_tag = *m_axi_mem_[b].awid; - m_axi_states_[b].aw_beat = 0; + // Write channels. AXI4 lets AW run ahead of its data and data run ahead + // of its address; the k-th AW pairs with the k-th W burst, so both sides + // queue and a write issues once its pair is complete. + auto& st = m_axi_states_[b]; + if (*m_axi_mem_[b].awvalid && *m_axi_mem_[b].awready) { + check_port_range(b, "write", uint64_t(*m_axi_mem_[b].awaddr), *m_axi_mem_[b].awlen); + st.aw_pending.push({uint64_t(*m_axi_mem_[b].awaddr), uint32_t(*m_axi_mem_[b].awid)}); } - // handle write data beats — write one cache line per W beat; the - // last beat (WLAST) queues a single B response for the burst. - if (m_axi_states_[b].aw_active - && *m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { - uint64_t byte_addr = m_axi_states_[b].aw_addr - + uint64_t(m_axi_states_[b].aw_beat) - * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; - // The beat is accepted but NOT yet visible in memory: AXI makes a - // write observable only once its B response is returned, and a read to - // the same address issued before that may legally return either value. - // Holding the payload here is what lets a master's own store->fill - // ordering bug appear in simulation instead of only on hardware. + // The beat is accepted but NOT yet visible in memory: AXI makes a write + // observable only once its B response is returned, and a read to the + // same address issued before that may legally return either value. + // Holding the payload here is what lets a master's own store->fill + // ordering bug appear in simulation instead of only on hardware. + if (*m_axi_mem_[b].wvalid && *m_axi_mem_[b].wready) { w_beat_t beat; - beat.addr = byte_addr; + beat.addr = 0; beat.byteen = uint64_t(*m_axi_mem_[b].wstrb); std::memcpy(beat.data.data(), m_axi_mem_[b].wdata->data(), VX_CFG_PLATFORM_MEMORY_DATA_SIZE); - m_axi_states_[b].w_staged.emplace_back(beat); + st.w_staged.emplace_back(beat); if (*m_axi_mem_[b].wlast) { - auto mem_req = new mem_req_t(); - mem_req->tag = m_axi_states_[b].aw_tag; - mem_req->addr = m_axi_states_[b].aw_addr; - mem_req->write = true; - mem_req->ready = false; - mem_req->last = true; - mem_req->w_beats = std::move(m_axi_states_[b].w_staged); - m_axi_states_[b].w_staged.clear(); - pending_mem_reqs_[b].emplace_back(mem_req); - dram_queues_[b].push(mem_req); - m_axi_states_[b].aw_active = false; - } else { - ++m_axi_states_[b].aw_beat; + st.w_bursts.push(std::move(st.w_staged)); + st.w_staged.clear(); + } + } + + // One cache line per W beat (INCR addressing); the burst queues a single + // B response. + while (!st.aw_pending.empty() && !st.w_bursts.empty()) { + auto aw = st.aw_pending.front(); + auto mem_req = new mem_req_t(); + mem_req->tag = aw.tag; + mem_req->addr = aw.addr; + mem_req->write = true; + mem_req->ready = false; + mem_req->last = true; + mem_req->w_beats = std::move(st.w_bursts.front()); + for (size_t i = 0; i < mem_req->w_beats.size(); ++i) { + mem_req->w_beats[i].addr = aw.addr + i * VX_CFG_PLATFORM_MEMORY_DATA_SIZE; } + st.aw_pending.pop(); + st.w_bursts.pop(); + pending_mem_reqs_[b].emplace_back(mem_req); + dram_queues_[b].push(mem_req); } } } @@ -876,15 +907,17 @@ class xrt_sim::Impl { uint64_t byteen; } w_beat_t; + typedef struct { + uint64_t addr; + uint32_t tag; + } aw_req_t; + typedef struct { bool read_rsp_ready; bool write_rsp_ready; - // Write-burst state — latched on AW, advanced one cache line per W beat. - bool aw_active; - uint64_t aw_addr; - uint32_t aw_beat; - uint32_t aw_tag; - std::vector w_staged; // beats accepted so far in this burst + std::queue aw_pending; // accepted AWs awaiting their data + std::vector w_staged; // beats of the burst in progress + std::queue> w_bursts; // whole bursts awaiting their AW } m_axi_state_t; typedef struct { @@ -999,7 +1032,7 @@ class xrt_sim::Impl { } m_axi_mem_t; Vvortex_afu_shim* device_; - RAM* ram_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + RAM* ram_[AFU_MEM_PORTS]; RAM* host_ram_; MemoryAllocator* host_alloc_; DramSim dram_sim_; @@ -1028,17 +1061,17 @@ class xrt_sim::Impl { } }; - std::list pending_mem_reqs_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::list pending_mem_reqs_[AFU_MEM_PORTS]; uint32_t rsp_rng_ = rsp_reorder_seed(); uint64_t rsp_reordered_ = 0; - m_axi_mem_t m_axi_mem_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_mem_t m_axi_mem_[AFU_MEM_PORTS]; - MemoryAllocator* mem_alloc_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + MemoryAllocator* mem_alloc_[AFU_MEM_PORTS]; - m_axi_state_t m_axi_states_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + m_axi_state_t m_axi_states_[AFU_MEM_PORTS]; - std::queue dram_queues_[VX_CFG_PLATFORM_MEMORY_NUM_BANKS]; + std::queue dram_queues_[AFU_MEM_PORTS]; // m_axi_host AXI slave port + burst state. m_axi_mem_t m_axi_host_; @@ -1048,11 +1081,10 @@ class xrt_sim::Impl { uint32_t host_rd_beat_; uint32_t host_rd_id_; bool host_rd_rsp_ready_; - bool host_wr_active_; - uint64_t host_wr_addr_; - uint32_t host_wr_beat_; - uint32_t host_wr_id_; - bool host_wr_b_pending_; + std::queue host_aw_pending_; + std::vector host_w_staged_; + std::queue> host_w_bursts_; + std::queue host_b_pending_; bool host_wr_rsp_ready_; // Reads accepted while one is already in flight. Several CP masters diff --git a/sw/common/dram_timing.h b/sw/common/dram_timing.h new file mode 100644 index 0000000000..f409ec383d --- /dev/null +++ b/sw/common/dram_timing.h @@ -0,0 +1,237 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The DRAM devices the platform memory models (VX_CFG_PLATFORM_MEMORY_TYPE): +// each JEDEC standard's organization and timing as a data sheet gives it, and +// the conversion of its nanosecond timings to DRAM clocks at a data rate. +// sim/common/dram_sim.cpp hands these to Ramulator; the bank model +// (sim/common/dram_bank_model.h) and the Foldsim runtime convert them with the +// same rounding Ramulator applies, so all of them see the same clocks. + +#pragma once + +#include +#include +#include +#include +#include + +namespace vortex { +namespace dram_timing { + +enum class Standard { + HBM2, // HBM2/HBM2E: 128-bit channels of two 64-bit pseudo-channels, 32 B bursts + HBM3, // HBM3, a 128-bit channel per pair of 64-bit channels, 32 B bursts + GDDR6, // GDDR6 x16 channels, 32 B bursts, 2 KB pages +}; + +inline Standard parse_standard(const std::string& name) { + if (name == "HBM2") return Standard::HBM2; + if (name == "HBM3") return Standard::HBM3; + if (name == "GDDR6") return Standard::GDDR6; + throw std::invalid_argument("unknown DRAM standard " + name); +} + +// Bus timings are DRAM clocks, fixed by the burst; array timings are +// nanoseconds, converted at the configured data rate. Names are Ramulator's. +struct clock_timing_t { + const char* name; + int clocks; +}; + +struct ns_timing_t { + const char* name; + float ns; +}; + +// A 32 B burst is BL4 on a 64-bit pseudo-channel: two clocks, and the same +// spacing between column commands to another bank group (four within one). +// nRTW only completes the set Ramulator requires. +inline constexpr clock_timing_t hbm_bus_timing[] = { + {"nBL", 2}, {"nCCDS", 2}, {"nCCDL", 4}, {"nRTW", 3}, +}; + +// JEDEC HBM2 array timing as GPGPU-Sim's SM7_QV100 configuration lists it (its +// JEDEC line, at 1 GHz: RCD=RP=CL=14, RAS=33, RC=47, WR=12, RRD=4, RTPL=4); +// the rest from Ramulator's HBM2_2Gbps preset, whose 1 ns clock makes its +// cycles nanoseconds. The array does not speed up with the interface: HBM2E +// and HBM3 parts keep these timings (GPGPU-Sim's SM80_A100 and SM90_H100 HBM +// lines are the same 14/33/47 ns at their own clocks). Refresh: the per-bank +// tRFCSB and tREFI of that preset, and the all-bank tRFC of Ramulator's table +// for the 8 Gb channel dram_sim configures. +inline constexpr ns_timing_t hbm_array_timing[] = { + {"tCL", 14}, {"tRCDRD", 14}, {"tRCDWR", 14}, {"tRP", 14}, {"tRAS", 33}, {"tRC", 47}, + {"tCWL", 2}, {"tWR", 12}, {"tRTPS", 2}, {"tRTPL", 4}, {"tRRDS", 4}, {"tRRDL", 4}, + {"tWTRS", 3}, {"tWTRL", 4}, {"tFAW", 15}, {"tRFC", 350}, {"tRFCSB", 160}, {"tREFI", 3900}, + {"tRREFD", 8}, +}; + +// Ramulator's GDDR6 clock is half the data rate: a 32 B burst, 16 beats on a +// x16 channel, is eight clocks, and column commands are two (three within a +// bank group) command clocks apart, a command clock being eight data beats. +inline constexpr clock_timing_t gddr6_bus_timing[] = { + {"nBL", 8}, {"nCCDS", 8}, {"nCCDL", 12}, +}; + +// Ramulator's x16 GDDR6 timing at 2000 (its GDDR6_2000_1350mV_double preset +// and secondary tables) at its 570 ps command clock, in nanoseconds; refresh +// timings complete the set Ramulator requires. +inline constexpr ns_timing_t gddr6_array_timing[] = { + {"tCL", 13.7f}, {"tRCDRD", 14.8f}, {"tRCDWD", 9.1f}, {"tRP", 14.8f}, {"tRAS", 30.2f}, {"tRC", 45.0f}, + {"tWR", 14.8f}, {"tRTP", 2.3f}, {"tCWL", 3.4f}, {"tRRDS", 2.9f}, {"tRRDL", 3.4f}, + {"tWTRS", 5.1f}, {"tWTRL", 6.3f}, {"tFAW", 16.0f}, + {"tRFC", 120.0f}, {"tRFCpb", 60.0f}, {"tREFI", 1900.0f}, {"tRREFD", 8.0f}, +}; + +// One channel's organization and the address map the simulators lay over it. +struct geometry_t { + uint32_t log2_pcs; // pseudo-channels sharing the channel's commands + uint32_t log2_bgs; // bank groups per pseudo-channel + uint32_t log2_banks; // banks per bank group + uint32_t log2_row_bursts; // 32 B bursts per row + uint32_t log2_visit; // bursts per row visit before the next bank + bool two_clock_act; // an ACT holds the row-command bus for two clocks + uint32_t rtw_extra; // read-to-write turnaround beyond tCL + tBL - tCWL + uint32_t bank_bits() const { return log2_pcs + log2_bgs + log2_banks; } +}; + +// HBM: 2 pseudo-channels x 4 bank groups x 4 banks, 1 KB rows (32 bursts); +// GDDR6 x16: 4 x 4 banks, 2 KB rows (64 bursts). Either way a row visit is four +// bursts (minimalist open page, 128 B). Read-to-write is Ramulator's: HBM's +// assumes a one-clock write preamble, GDDR6's adds the bus turnaround. +inline geometry_t geometry(Standard standard) { + if (standard == Standard::GDDR6) { + return geometry_t{0, 2, 2, 6, 2, false, 4}; + } + return geometry_t{1, 2, 2, 5, 2, true, 2}; +} + +// Every device here holds 8 Gb per channel; addresses above alias. +inline constexpr uint64_t kChannelSize = uint64_t(1) << 30; +// Each standard moves 32 B per column command (prefetch x channel width). +inline constexpr uint32_t kBurstSize = 32; + +// Ramulator's DRAM clock period at a data rate (MT/s): half the data rate, in +// integer picoseconds. +inline int tck_ps(uint32_t data_rate) { + return int(1E6 / (data_rate / 2)); +} + +// Ramulator's JEDEC rounding of a nanosecond timing to clocks (base/utils.cpp), +// in the same float arithmetic. +inline uint32_t jedec_clocks(float t_ns, int tck) { + uint64_t t_ps = t_ns * 1000; + return uint32_t(((t_ps * 1000 / tck) + 974) / 1000); +} + +// The timings a scheduler of one channel needs, in DRAM clocks. +struct clocks_t { + uint32_t BL, CCDS, CCDL; + uint32_t CL, CWL; + uint32_t RCDRD, RCDWR, RP, RAS, RC; + uint32_t RTP, WR; + uint32_t RRDS, RRDL, FAW; + uint32_t WTRS, WTRL; +}; + +namespace detail { + +template +inline bool find_clock(const clock_timing_t (&table)[N], const char* name, uint32_t* out) { + for (auto& t : table) { + if (std::strcmp(t.name, name) == 0) { + *out = uint32_t(t.clocks); + return true; + } + } + return false; +} + +template +inline bool find_ns(const ns_timing_t (&table)[N], const char* name, int tck, uint32_t* out) { + for (auto& t : table) { + if (std::strcmp(t.name, name) == 0) { + *out = jedec_clocks(t.ns, tck); + return true; + } + } + return false; +} + +template +inline uint32_t get(const clock_timing_t (&bus)[B], const ns_timing_t (&array)[A], int tck, + const char* clock_name, const char* ns_name) { + uint32_t v = 0; + if (clock_name && find_clock(bus, clock_name, &v)) return v; + if (ns_name && find_ns(array, ns_name, tck, &v)) return v; + throw std::logic_error(std::string("dram_timing: no timing ") + (ns_name ? ns_name : clock_name)); +} + +} // namespace detail + +// The standard's timings at a data rate. A read may precharge its bank tRTP +// after its column command: HBM's bank-level read-to-precharge is the larger of +// its short and long forms. +inline clocks_t clocks(Standard standard, uint32_t data_rate) { + const int tck = tck_ps(data_rate); + clocks_t c{}; + if (standard == Standard::GDDR6) { + auto g = [&](const char* n, const char* t) { + return detail::get(gddr6_bus_timing, gddr6_array_timing, tck, n, t); + }; + c.BL = g("nBL", nullptr); c.CCDS = g("nCCDS", nullptr); c.CCDL = g("nCCDL", nullptr); + c.CL = g(nullptr, "tCL"); c.CWL = g(nullptr, "tCWL"); + c.RCDRD = g(nullptr, "tRCDRD"); c.RCDWR = g(nullptr, "tRCDWD"); + c.RP = g(nullptr, "tRP"); c.RAS = g(nullptr, "tRAS"); c.RC = g(nullptr, "tRC"); + c.RTP = g(nullptr, "tRTP"); c.WR = g(nullptr, "tWR"); + c.RRDS = g(nullptr, "tRRDS"); c.RRDL = g(nullptr, "tRRDL"); c.FAW = g(nullptr, "tFAW"); + c.WTRS = g(nullptr, "tWTRS"); c.WTRL = g(nullptr, "tWTRL"); + } else { + auto g = [&](const char* n, const char* t) { + return detail::get(hbm_bus_timing, hbm_array_timing, tck, n, t); + }; + c.BL = g("nBL", nullptr); c.CCDS = g("nCCDS", nullptr); c.CCDL = g("nCCDL", nullptr); + c.CL = g(nullptr, "tCL"); c.CWL = g(nullptr, "tCWL"); + c.RCDRD = g(nullptr, "tRCDRD"); c.RCDWR = g(nullptr, "tRCDWR"); + c.RP = g(nullptr, "tRP"); c.RAS = g(nullptr, "tRAS"); c.RC = g(nullptr, "tRC"); + uint32_t rtps = g(nullptr, "tRTPS"), rtpl = g(nullptr, "tRTPL"); + c.RTP = (rtps > rtpl) ? rtps : rtpl; + c.WR = g(nullptr, "tWR"); + c.RRDS = g(nullptr, "tRRDS"); c.RRDL = g(nullptr, "tRRDL"); c.FAW = g(nullptr, "tFAW"); + c.WTRS = g(nullptr, "tWTRS"); c.WTRL = g(nullptr, "tWTRL"); + } + return c; +} + +// Refresh, in DRAM clocks: the interval in which every bank is refreshed once +// (tREFI), and how long a refresh keeps its banks from activating, all of them +// (tRFC) or one (per-bank refresh: HBM's REFsb, GDDR6's REFpb). +struct refresh_t { + uint32_t REFI, RFC, RFCPB; +}; + +inline refresh_t refresh(Standard standard, uint32_t data_rate) { + const int tck = tck_ps(data_rate); + refresh_t r{}; + if (standard == Standard::GDDR6) { + auto g = [&](const char* t) { return detail::get(gddr6_bus_timing, gddr6_array_timing, tck, nullptr, t); }; + r.REFI = g("tREFI"); r.RFC = g("tRFC"); r.RFCPB = g("tRFCpb"); + } else { + auto g = [&](const char* t) { return detail::get(hbm_bus_timing, hbm_array_timing, tck, nullptr, t); }; + r.REFI = g("tREFI"); r.RFC = g("tRFC"); r.RFCPB = g("tRFCSB"); + } + return r; +} + +} // namespace dram_timing +} // namespace vortex diff --git a/hw/unittest/tcu_fedp/fedp.h b/sw/common/fedp.h similarity index 64% rename from hw/unittest/tcu_fedp/fedp.h rename to sw/common/fedp.h index d93beefb23..486128c3a0 100644 --- a/hw/unittest/tcu_fedp/fedp.h +++ b/sw/common/fedp.h @@ -25,9 +25,9 @@ #include #ifdef FEDP_TRACE -#define LOG(...) std::fprintf(stderr, __VA_ARGS__); +#define FEDP_LOG(...) std::fprintf(stderr, __VA_ARGS__); #else -#define LOG(...) +#define FEDP_LOG(...) #endif #ifdef FEDP_RTL_WATCH @@ -36,26 +36,42 @@ #define RTL_WATCH(...) #endif -class FEDP { +class TfrFedp { public: - explicit FEDP(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) + explicit TfrFedp(int exp_bits = 5, int sig_bits = 10, int frm = 0, int W = 25, bool renorm = false, bool no_window = false) : exp_bits_(exp_bits), sig_bits_(sig_bits), frm_(frm), W_(W), renorm_(renorm), no_window_(no_window) { - LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", + FEDP_LOG("[ctor] fmt=e%dm%d, frm=%d, W=%d, renorm_=%s, no_window_=%s\n", exp_bits_, sig_bits_, frm_, W_, (renorm_ ? "true" : "false"), (no_window_ ? "true" : "false")); assert(exp_bits_ > 0 && exp_bits_ <= 8); assert(sig_bits_ > 0 && sig_bits_ <= 10); assert(frm_ >= 0 && frm_ <= 4); } + // Block-scaled formats: one scale byte per lane for A and B, two lanes per + // 32-bit register (a lane is 16 bits of it: two fp8 or four fp4 elements). + // fp8/bf8 (MXFP8/MXBF8) and fp4 (MXFP4): E8M0 scales; + // fp4 with e4m3_scale (NVFP4): E4M3 scales. + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n, + const uint8_t *sf_a, const uint8_t *sf_b, bool e4m3_scale = false) { + sf_a_ = sf_a; + sf_b_ = sf_b; + e4m3_scale_ = e4m3_scale; + float d = (*this)(a, b, c, n); + sf_a_ = sf_b_ = nullptr; + e4m3_scale_ = false; + return d; + } + float operator()(const uint32_t *a, const uint32_t *b, float c, uint32_t n) { const auto c_enc = bitsFromF32(c); rtl_watch_s0(a, b, c_enc, n); - const auto terms = decode_inputs(a, b, n); const auto c_dec = decode_input(c_enc, 8, 23); const auto c_term = decodeC_to_common(c_dec); - const auto mul_res = multiply_to_common(terms, c_term); + const auto mul_res = (exp_bits_ == 2 && sig_bits_ == 1) + ? multiply_f4(a, b, n, c_term) + : multiply_to_common(decode_inputs(a, b, n), c_term); rtl_watch_s1(mul_res, c_term, c_enc); @@ -157,7 +173,7 @@ class FEDP { } // -------- [RTL-WATCH] helpers -------- - inline void rtl_watch_s0(const uint32_t* a, const uint32_t* b, uint32_t c_enc, uint32_t n) { + inline void rtl_watch_s0([[maybe_unused]] const uint32_t* a, [[maybe_unused]] const uint32_t* b, [[maybe_unused]] uint32_t c_enc, [[maybe_unused]] uint32_t n) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S0(%lu): a_row=", req_id_); RTL_WATCH("{"); @@ -174,7 +190,7 @@ class FEDP { #endif } - inline void rtl_watch_s1(const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s1([[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S1(%lu): max_exp=0x%x, shift_amt=", req_id_, res.L); RTL_WATCH("{"); @@ -202,7 +218,7 @@ class FEDP { #endif } - inline void rtl_watch_s2(const align_t& aln, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s2([[maybe_unused]] const align_t& aln, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S2(%lu): max_exp=0x%x, aln_sig=", req_id_, res.L); RTL_WATCH("{"); @@ -228,7 +244,7 @@ class FEDP { #endif } - inline void rtl_watch_s3(const acc_t& acc, const mul_res_t& res, const cterm_t& c_term, uint32_t c_enc) { + inline void rtl_watch_s3([[maybe_unused]] const acc_t& acc, [[maybe_unused]] const mul_res_t& res, [[maybe_unused]] const cterm_t& c_term, [[maybe_unused]] uint32_t c_enc) { #ifdef FEDP_RTL_WATCH uint32_t sigs_sign = 0; for (size_t i = 0; i < res.terms.size() - 1; ++i) { @@ -239,7 +255,7 @@ class FEDP { #endif } - inline void rtl_watch_norm(const nrm_t& nrm, int flags, int max_exp) { + inline void rtl_watch_norm([[maybe_unused]] const nrm_t& nrm, [[maybe_unused]] int flags, [[maybe_unused]] int max_exp) { #ifdef FEDP_RTL_WATCH int lsb = nrm.kept & 1; RTL_WATCH("[RTL_WATCH] FEDP-NORM(%lu): abs_sum=0x%x, L=%d, G=%d, R=%d, S=%d, Rup=%d\n", @@ -248,12 +264,19 @@ class FEDP { #endif } - inline void rtl_watch_s4(uint32_t result) { + inline void rtl_watch_s4([[maybe_unused]] uint32_t result) { #ifdef FEDP_RTL_WATCH RTL_WATCH("[RTL_WATCH] FEDP-S4(%lu): result=0x%08x\n", req_id_, result); #endif } + // Exponent of a zero term: it stands for the datapath's exponent 0, which + // add_c_term places WA - 1 below this model's exponent origin. + static constexpr int kZeroE = -(1 << 20); + + // The datapath's accumulator width over `lanes` product terms and C. + int acc_width(uint32_t lanes) const { return W_ + 2 + (32 - lzcN(lanes, 32)); } + // -------- logic -------- static inline std::pair csa(uint32_t a, uint32_t b, uint32_t c) { uint32_t s = a ^ b ^ c; @@ -303,10 +326,11 @@ class FEDP { const int sb = sig_bits_, eb = exp_bits_; std::vector out; std::vector eps; // per-term *window LSB exponent* (not raw Ep/Ec) + std::vector sel_eps; + std::vector nonzero; out.reserve(v.size() + 1); eps.reserve(v.size() + 1); - bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); uint8_t c_nan = (c_term.sp == 3) ? FL_NAN : 0; uint8_t c_pinf = (c_term.sp == 1) ? FL_PINF : 0; @@ -354,8 +378,14 @@ class FEDP { Ep -= lzc_prod; } + // the fp8 lane pairs its two products by their exponent fields even + // when one is zero (a zero field counts as 1, as for a subnormal) + sel_eps.push_back((std::max(a.exp, 1) - bias(eb)) + (std::max(b.exp, 1) - bias(eb)) + + F32_BIAS + EXP_POS_BIAS + 1 - W_); + nonzero.push_back(!p_is_zero); + if (p_is_zero) { - Ep = 0; + Ep = kZeroE; Mp = 0; } @@ -372,7 +402,22 @@ class FEDP { for (size_t j = 0; j < out.size(); j += 2) { if (j + 1 < out.size()) { - auto r = reduce_f8(out[j], eps[j], out[j + 1], eps[j + 1]); + auto r = reduce_f8(out[j], sel_eps[j], nonzero[j], out[j + 1], sel_eps[j + 1], nonzero[j + 1]); + // MX: the lane's E8M0 scales add to its exponent field, which is + // 10 bits wide and sits WA - 1 above this model's exponents + if (sf_a_) { + // an E8M0 scale of 0xFF is NaN, and so is every element it scales + if (sf_a_[j / 2] == 0xff || sf_b_[j / 2] == 0xff) { + flags |= FL_NAN; + } + // a lane scaled below the exponent range sits at exponent 0, + // where it only adds to the sticky bit + if (r.second != kZeroE) { + const int WA = acc_width(uint32_t(out.size() / 2)); + int lane_exp = r.second + (WA - 1) + int(sf_a_[j / 2]) + int(sf_b_[j / 2]) - 254; + r.second = std::max(lane_exp, 0) - (WA - 1); + } + } red_out.push_back(r.first); red_eps.push_back(r.second); } else { @@ -385,13 +430,23 @@ class FEDP { eps.swap(red_eps); } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // Append the C term and take every term's shift to the largest exponent. + mul_res_t add_c_term(std::vector out, std::vector eps, const cterm_t &c_term, uint8_t flags) { + bool c_is_zero = (c_term.cls == 0 && c_term.sp == 0); int Ec = c_term.Ec + F32_BIAS + EXP_POS_BIAS + 24 - W_; if (c_is_zero) { - Ec = 0; + Ec = kZeroE; } out.push_back({c_term.sign, c_term.Mc}); eps.push_back(Ec); + const int WA = acc_width(uint32_t(out.size() - 1)); + for (auto &e : eps) { + if (e == kZeroE) e = -(WA - 1); + } int L = *std::max_element(eps.begin(), eps.end()); @@ -404,12 +459,99 @@ class FEDP { return {out, L , shifts, flags}; } - std::pair reduce_f8(const pterm_t& t1, int e1, const pterm_t& t2, int e2) { + // Block-scaled fp4 (E2M1), one lane per 16 bits: its four products are + // aligned to their largest exponent in a 6-bit lane field, summed in 27-bit + // two's complement, and leave the datapath as one term. E2M1 has no + // infinity or NaN. MXFP4 adds the E8M0 scales to the lane exponent; NVFP4 + // folds the E4M3 scales' exponent fields into the 6-bit field (they fit) and + // multiplies each product by the scales' mantissa product. + mul_res_t multiply_f4(const uint32_t *a, const uint32_t *b, uint32_t n, const cterm_t &c_term) { + assert(sf_a_ && sf_b_ && "fp4 needs block scales"); + const int SIG_SHIFT = 11; + const int EXP_TERM_BIAS = 32; + const int exp_comp = e4m3_scale_ ? -(2 * 1 + 2 * (7 + 3)) : -(2 * 1 + 10); + const int exp_adj = (EXP_TERM_BIAS + exp_comp) & 63; + // the lane exponent the datapath emits is 10 bits wide; this + // model's lane exponents sit WA - 1 below it + const int WA = acc_width(2 * n); + const int bias_base = F32_BIAS + 2 * (23 - 22) - W_ + WA - 1 + EXP_POS_BIAS; + auto f4_man = [](uint32_t x) -> uint32_t { return ((x >> 1) & 3) == 0 ? 1u : (2u | (x & 1)); }; + auto f4_exp = [](uint32_t x) -> uint32_t { + uint32_t e2 = (x >> 2) & 1, e1 = (x >> 1) & 1; + return (e2 & ~e1 & 1) | ((e2 & e1) << 1); + }; + std::vector out; + std::vector eps; + uint8_t flags = 0; + if (c_term.sp == 3) flags |= FL_NAN; + if (c_term.sp == 1) flags |= FL_PINF; + if (c_term.sp == 2) flags |= FL_NINF; + for (uint32_t lane = 0; lane < 2 * n; ++lane) { + const uint32_t word = lane / 2, off = (lane % 2) * 16; + const uint32_t sfa = sf_a_[lane], sfb = sf_b_[lane]; + // NVFP4 UE4M3 scales (bit 7 is padding): 0x7F is NaN. MXFP4 E8M0 + // scales join the lane exponent; 0xFF is NaN. + uint32_t sf_man_prod = 1; + int sf_exp = 0; // into the 6-bit term exponent (NVFP4) + int sf_lane_exp = 0; // into the lane exponent (MXFP4) + if (e4m3_scale_) { + sf_man_prod = (8u | (sfa & 7)) * (8u | (sfb & 7)); + sf_exp = int((sfa >> 3) & 15) + int((sfb >> 3) & 15); + if ((sfa & 0x7f) == 0x7f || (sfb & 0x7f) == 0x7f) { + flags |= FL_NAN; + } + } else { + sf_lane_exp = int(sfa) + int(sfb) - 254; + if (sfa == 0xff || sfb == 0xff) { + flags |= FL_NAN; + } + } + uint32_t texp[4], tprod[4], tsign[4]; + bool tvalid[4]; + uint32_t max_exp = 0; + for (int j = 0; j < 4; ++j) { + uint32_t ra = (a[word] >> (off + 4 * j)) & 0xf; + uint32_t rb = (b[word] >> (off + 4 * j)) & 0xf; + tvalid[j] = (ra & 7) != 0 && (rb & 7) != 0; + tsign[j] = ((ra ^ rb) >> 3) & 1; + uint32_t prod = f4_man(ra) * f4_man(rb); + tprod[j] = e4m3_scale_ ? ((prod * sf_man_prod) & 0x7ff) : prod; + texp[j] = tvalid[j] ? (uint32_t(exp_adj + sf_exp + int(f4_exp(ra)) + int(f4_exp(rb))) & 63) : 0; + max_exp = std::max(max_exp, texp[j]); + } + uint32_t sum = 0; // 27-bit two's complement + for (int j = 0; j < 4; ++j) { + uint32_t shift = (max_exp - texp[j]) & 63; + uint32_t mag = tvalid[j] ? ((tprod[j] << SIG_SHIFT) & 0xffffff) : 0; + uint32_t aligned = (shift >= 24) ? 0 : (mag >> shift); + sum = (sum + (tsign[j] ? (0u - aligned) : aligned)) & 0x7ffffff; + } + uint32_t sum_sign = (sum >> 26) & 1; + uint32_t abs_sum = (sum_sign ? (0u - sum) : sum) & 0x3ffffff; + if (abs_sum == 0) { + out.push_back({0, 0}); + eps.push_back(kZeroE); + continue; + } + const int lane_exp = (int(max_exp) + bias_base + exp_comp + sf_lane_exp) & 1023; + out.push_back({int(sum_sign), int(abs_sum & 0xffffff)}); + eps.push_back(lane_exp - (WA - 1)); + } + return add_c_term(std::move(out), std::move(eps), c_term, flags); + } + + // One fp8 lane: its two products, aligned to the one the datapath picks as + // larger. A zero product still competes by its exponent fields when the + // other is nonzero and listed second, as the datapath compares them. + std::pair reduce_f8(const pterm_t& t1, int e1, bool v1, const pterm_t& t2, int e2, bool v2) { // Pick max exponent term int max_E, max_s, min_E, min_s; int64_t max_P, min_P; - if (e1 >= e2) { + if (!v1 && !v2) { + return {{0, 0}, kZeroE}; + } + if ((v1 && !v2) || (v2 && e1 >= e2)) { max_E = e1; max_P = t1.Mp; max_s = t1.sign; min_E = e2; min_P = t2.Mp; min_s = t2.sign; } else { @@ -435,7 +577,11 @@ class FEDP { int64_t val = val1 + val2; if (val == 0) { - return {{0, 0}, 0}; + // Only an exact cancellation of two equal-exponent products clears the + // lane exponent; a product aligned out of the lane keeps it, so the + // empty lane still sets the largest exponent. + bool cancel = (max_s != min_s) && (e1 == e2) && (t1.Mp == t2.Mp); + return {{0, 0}, cancel ? kZeroE : (max_E + 1)}; } int s = (val < 0) ? 1 : 0; @@ -446,7 +592,7 @@ class FEDP { P >>= 1; E += 1; - LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", + FEDP_LOG("[reduce_f8 (%ld)] t1=(%d, %d, %d), t2=(%d, %d, %d) | diff=%d max_P=0x%lx min_P=0x%lx aligned=0x%lx val=0x%lx -> s=%d P=0x%lx E=%d\n", req_id_, t1.sign, t1.Mp, e1, t2.sign, t2.Mp, e2, diff, max_P, min_P, aligned, val, s, P, E); return {{s, (int)P}, E}; @@ -569,6 +715,8 @@ class FEDP { uint32_t kept_rounded = r.kept + (round_up ? 1 : 0); int be = r.e; if (kept_rounded & (1u << 24)) { kept_rounded >>= 1; be += 1; } + // the datapath's result exponent is a signed 10-bit quantity + be = ((be & 1023) ^ 512) - 512; if (be >= 255) return s_bit | 0x7f800000u; if (be <= 0) return s_bit; return s_bit | ((be & 0xff) << 23) | (kept_rounded & 0x7FFFFFu); @@ -577,5 +725,8 @@ class FEDP { // members -------- int exp_bits_, sig_bits_, frm_, W_, HR_; bool renorm_, no_window_; + const uint8_t *sf_a_ = nullptr; // per-lane block scales, null when unscaled + const uint8_t *sf_b_ = nullptr; + bool e4m3_scale_ = false; uint64_t req_id_ = 0; }; \ No newline at end of file diff --git a/sw/common/tensor_cfg.h b/sw/common/tensor_cfg.h index 6ca343bb12..ecf3cef369 100644 --- a/sw/common/tensor_cfg.h +++ b/sw/common/tensor_cfg.h @@ -156,6 +156,12 @@ inline const char* fmt_string(uint32_t fmt) { } } +// The TCU accumulates and returns fp32 (int32 for integer inputs); it has no +// other output format. +inline constexpr bool tcu_output_valid(uint32_t it, uint32_t ot) { + return (it >= int32::id) ? (ot == int32::id) : (ot == fp32::id); +} + inline constexpr bool mx_scale_format(uint32_t fmt) { switch (fmt) { case mxfp8::id: @@ -281,7 +287,8 @@ struct wmma_config_t { static constexpr uint32_t b_sub_blocks = block_cap / b_block_size; // number of B micro-tiles per register static constexpr uint32_t b_sub_steps = n_steps / b_sub_blocks; // number of B sub-steps per register - // Symmetric sparse flag (NT=4, NT=16: block_em == block_en) + // Symmetric sparse flag (NT=4, NT=16: block_em == block_en). It selects the + // WMMA_SP layout only, so a WGMMA build keeps it: WGMMA_SP has its own. static constexpr bool sym_sparse = (block_em == block_en); // Symmetric: column-pair layout (block_cap lanes per block) diff --git a/sw/common/v80_pcie.h b/sw/common/v80_pcie.h new file mode 100644 index 0000000000..2fb7b7e79a --- /dev/null +++ b/sw/common/v80_pcie.h @@ -0,0 +1,54 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#pragma once + +#include +#include +#include + +namespace vortex { + +// Finds an AMD Alveo V80 the way XRT enumerates by index: scans sysfs for the +// SLASH control function (10ee:50c1) and returns its bus:device in the "01:00" +// form VRT takes. Empty when no card is on the bus. +inline std::string discover_v80_bdf() { + DIR* d = opendir("/sys/bus/pci/devices"); + if (d == nullptr) { + return ""; + } + std::string found; + while (struct dirent* e = readdir(d)) { + const std::string name = e->d_name; // "0000:01:00.1" + if (name.size() < 12 || name[0] == '.') { + continue; + } + auto read_hex = [&](const char* leaf) -> unsigned { + std::ifstream f("/sys/bus/pci/devices/" + name + "/" + leaf); + unsigned v = 0; + f >> std::hex >> v; + return v; + }; + if (read_hex("vendor") == 0x10ee && read_hex("device") == 0x50c1) { + found = name.substr(0, name.rfind('.')); // drop the function + if (found.rfind("0000:", 0) == 0) { + found = found.substr(5); // "01:00" form + } + break; + } + } + closedir(d); + return found; +} + +} // namespace vortex diff --git a/sw/kernel/include/vx_intrinsics.h b/sw/kernel/include/vx_intrinsics.h index 78468f7744..61ce362a14 100644 --- a/sw/kernel/include/vx_intrinsics.h +++ b/sw/kernel/include/vx_intrinsics.h @@ -204,7 +204,7 @@ inline __attribute__((const)) int vx_num_warps() { return (int)csr_read_nv(VX_CSR_NUM_WARPS); } -// Return the number of cores per cluster +// Return the number of cores across all clusters inline __attribute__((const)) int vx_num_cores() { return (int)csr_read_nv(VX_CSR_NUM_CORES); } @@ -636,6 +636,36 @@ inline float vx_packlh_f(const void* base, size_t stride) { return out; } +// +// Wide loads and stores +// + +// Two or four consecutive words, each 4-byte aligned. A compiler with the +// wide load/store extension emits one vx_flw.d/.q or vx_fsw.d/.q for each +// access; any other splits it into flw/fsw. +typedef float vx_float2 __attribute__((vector_size(8), aligned(4))); +typedef float vx_float4 __attribute__((vector_size(16), aligned(4))); + +__attribute__((always_inline)) +inline vx_float2 vx_flw_d(const void* addr) { + return *(const vx_float2*)addr; +} + +__attribute__((always_inline)) +inline vx_float4 vx_flw_q(const void* addr) { + return *(const vx_float4*)addr; +} + +__attribute__((always_inline)) +inline void vx_fsw_d(void* addr, vx_float2 value) { + *(vx_float2*)addr = value; +} + +__attribute__((always_inline)) +inline void vx_fsw_q(void* addr, vx_float4 value) { + *(vx_float4*)addr = value; +} + #ifdef __cplusplus } #endif diff --git a/sw/kernel/include/vx_tensor.h b/sw/kernel/include/vx_tensor.h index 618a95b030..21daf265f0 100644 --- a/sw/kernel/include/vx_tensor.h +++ b/sw/kernel/include/vx_tensor.h @@ -145,17 +145,143 @@ namespace detail { return *reinterpret_cast(&result_u); } }; + + // vx_ldm fd, imm(rs1) / vx_stm fs2, imm(rs1): imm[4:0] is the fragment + // shape, imm[11:5] a byte offset; rs1 is the first of J registers, a multiple + // of J, holding the lane's address in each of the fragment's row groups. + enum ldm_shape_t : int32_t { + LDM_NONE = -1, + LDM_A = 0x00, // WMMA A, row-major + LDM_A_SP = 0x01, // WMMA sparse A, row-major + LDM_B = 0x06, // WMMA B, col-major + LDM_B_SP = 0x07, // WMMA sparse B, col-major + LDM_C = 0x08, // WMMA C, row-major + LDM_WG_C = 0x18 // WGMMA C, row-major, one 8-register chunk + }; + +#if __riscv_xlen == 64 + using gpr_pair_t = unsigned __int128; +#else + using gpr_pair_t = uint64_t; +#endif + +#define VX_TENSOR_FREGS_0 "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7" +#define VX_TENSOR_FREGS_8 "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15" +#define VX_TENSOR_FREGS_10 "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17" +#define VX_TENSOR_FREGS_16 "f16", "f17", "f18", "f19", "f20", "f21", "f22", "f23" +#define VX_TENSOR_FREGS_24 "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31" +#define VX_TENSOR_FREGS4_10 "f10", "f11", "f12", "f13" +#define VX_TENSOR_FREGS4_28 "f28", "f29", "f30", "f31" + +// The register lists expand before the inner macros split them into names. +#define VX_TENSOR_LDM8(ival, regs, ...) VX_TENSOR_LDM8_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_LDM8_(ival, r0, r1, r2, r3, r4, r5, r6, r7, ...) { \ + register float x0 __asm__(r0); register float x1 __asm__(r1); \ + register float x2 __asm__(r2); register float x3 __asm__(r3); \ + register float x4 __asm__(r4); register float x5 __asm__(r5); \ + register float x6 __asm__(r6); register float x7 __asm__(r7); \ + __asm__ volatile (".insn i %[op], 0, %[x0], %[g], %[imm]" \ + : [x0]"=f"(x0), "=f"(x1), "=f"(x2), "=f"(x3), "=f"(x4), "=f"(x5), "=f"(x6), "=f"(x7)\ + : [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), __VA_ARGS__ : "memory"); \ + v[0] = x0; v[1] = x1; v[2] = x2; v[3] = x3; v[4] = x4; v[5] = x5; v[6] = x6; v[7] = x7; } +#define VX_TENSOR_LDM4(ival, regs, ...) VX_TENSOR_LDM4_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_LDM4_(ival, r0, r1, r2, r3, ...) { \ + register float x0 __asm__(r0); register float x1 __asm__(r1); \ + register float x2 __asm__(r2); register float x3 __asm__(r3); \ + __asm__ volatile (".insn i %[op], 0, %[x0], %[g], %[imm]" \ + : [x0]"=f"(x0), "=f"(x1), "=f"(x2), "=f"(x3) \ + : [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), __VA_ARGS__ : "memory"); \ + v[0] = x0; v[1] = x1; v[2] = x2; v[3] = x3; } +#define VX_TENSOR_STM8(ival, regs, ...) VX_TENSOR_STM8_(ival, regs, __VA_ARGS__) +#define VX_TENSOR_STM8_(ival, r0, r1, r2, r3, r4, r5, r6, r7, ...) { \ + register float x0 __asm__(r0) = v[0]; register float x1 __asm__(r1) = v[1]; \ + register float x2 __asm__(r2) = v[2]; register float x3 __asm__(r3) = v[3]; \ + register float x4 __asm__(r4) = v[4]; register float x5 __asm__(r5) = v[5]; \ + register float x6 __asm__(r6) = v[6]; register float x7 __asm__(r7) = v[7]; \ + __asm__ volatile (".insn s %[op], 2, %[x0], %[imm](%[g])" \ + :: [op]"i"(RISCV_CUSTOM3), [imm]"i"(ival), [x0]"f"(x0), "f"(x1), "f"(x2), "f"(x3), \ + "f"(x4), "f"(x5), "f"(x6), "f"(x7), __VA_ARGS__ : "memory"); } + + // A J=2 group rides in a register pair; a J=4 group is pinned to x28..x31 + // or x12..x15. +#define VX_TENSOR_LDM_GROUP(EMIT) \ + if constexpr (J == 2) { \ + gpr_pair_t p = gpr_pair_t(g[0]) | (gpr_pair_t(g[1]) << (8 * sizeof(uintptr_t))); \ + EMIT([g]"R"(p)); \ + } else if constexpr (Q == 28) { \ + register uintptr_t q0 __asm__("x28") = g[0]; register uintptr_t q1 __asm__("x29") = g[1]; \ + register uintptr_t q2 __asm__("x30") = g[2]; register uintptr_t q3 __asm__("x31") = g[3]; \ + EMIT([g]"r"(q0), "r"(q1), "r"(q2), "r"(q3)); \ + } else { \ + register uintptr_t q0 __asm__("x12") = g[0]; register uintptr_t q1 __asm__("x13") = g[1]; \ + register uintptr_t q2 __asm__("x14") = g[2]; register uintptr_t q3 __asm__("x15") = g[3]; \ + EMIT([g]"r"(q0), "r"(q1), "r"(q2), "r"(q3)); \ + } + + // vx_ldm of NR registers from fd = FD; Q picks a J=4 group's registers. + template + __attribute__((always_inline)) inline void vx_ldm(float (&v)[NR], const uintptr_t (&g)[J]) { + static_assert(J == 2 || J == 4, "a fragment has 2 or 4 row groups"); + #define VX_TENSOR_LDM_EMIT(...) \ + if constexpr (FD == 10 && NR == 8) VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_10, __VA_ARGS__) \ + else if constexpr (FD == 10) VX_TENSOR_LDM4(IMM, VX_TENSOR_FREGS4_10, __VA_ARGS__) \ + else if constexpr (FD == 24) VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_24, __VA_ARGS__) \ + else if constexpr (FD == 28) VX_TENSOR_LDM4(IMM, VX_TENSOR_FREGS4_28, __VA_ARGS__) \ + else { static_assert(FD == 0 && NR == 8, "unsupported fragment registers"); \ + VX_TENSOR_LDM8(IMM, VX_TENSOR_FREGS_0, __VA_ARGS__) } + VX_TENSOR_LDM_GROUP(VX_TENSOR_LDM_EMIT) + #undef VX_TENSOR_LDM_EMIT + } + + // vx_stm of 8 registers from fs2 = FD + template + __attribute__((always_inline)) inline void vx_stm(const float (&v)[8], const uintptr_t (&g)[J]) { + static_assert(J == 2 || J == 4, "a fragment has 2 or 4 row groups"); + #define VX_TENSOR_STM_EMIT(...) \ + if constexpr (FD == 0) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_0, __VA_ARGS__) \ + else if constexpr (FD == 8) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_8, __VA_ARGS__) \ + else if constexpr (FD == 16) VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_16, __VA_ARGS__) \ + else { static_assert(FD == 24, "unsupported fragment registers"); \ + VX_TENSOR_STM8(IMM, VX_TENSOR_FREGS_24, __VA_ARGS__) } + VX_TENSOR_LDM_GROUP(VX_TENSOR_STM_EMIT) + #undef VX_TENSOR_STM_EMIT + } + +#undef VX_TENSOR_FREGS_0 +#undef VX_TENSOR_FREGS_8 +#undef VX_TENSOR_FREGS_10 +#undef VX_TENSOR_FREGS_16 +#undef VX_TENSOR_FREGS_24 +#undef VX_TENSOR_FREGS4_10 +#undef VX_TENSOR_FREGS4_28 +#undef VX_TENSOR_LDM8 +#undef VX_TENSOR_LDM8_ +#undef VX_TENSOR_LDM4 +#undef VX_TENSOR_LDM4_ +#undef VX_TENSOR_STM8 +#undef VX_TENSOR_STM8_ +#undef VX_TENSOR_LDM_GROUP + + // The immediate of a fragment access at byte offset off from the group: off + // folds in when it is a multiple of 32 and of the column span (the OR with + // the uops' column offsets is then an add). + template + inline constexpr bool ldm_folds = (off % 32 == 0) && (off % int32_t(span) == 0) + && (off >= -2048) && (off <= 2016); } // Hoist detail::frag_use_t into the enclosing tensor namespace using detail::frag_use_t; +// Compile-time loop, e.g. over the K tiles of an unrolled K loop +using detail::unroll_for; template struct wmma_context { private: @@ -173,6 +299,9 @@ struct wmma_context { public: + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using input_t = typename It::dtype; using output_t = typename Ot::dtype; @@ -277,25 +406,257 @@ struct wmma_context { }); } + // Element offset of the calling lane's first fragment element in a tile with + // leading dimension ldm. A K loop computes it once and passes it to + // load_matrix_sync: an asm with side effects ahead of the load (a barrier, a + // metadata load) keeps the compiler from hoisting the lane id read. + template + static __attribute__((always_inline)) size_t lane_offset(size_t ldm) { + uint32_t row, col; + lane_coords(row, col); + return row * ldm + col; + } + template static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm) { + uint32_t row, col; + lane_coords(row, col); + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + ldm_load(dst, src, row * ldm + col, ldm); + } else { + load_fragment(dst, reinterpret_cast*>(src) + row * ldm + col, ldm); + } + } + + // k_off: a constant element offset from src, e.g. the next K tile of an + // unrolled loop, which a vx_ldm takes in its immediate. + template + static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm, size_t lane_off) { + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + ldm_load(dst, src, lane_off, ldm); + } else { + load_fragment(dst, reinterpret_cast*>(src) + lane_off + k_off, ldm); + } + } + +private: + + template + using frag_elem_t = std::conditional_t; + + // Row and column of the calling lane's first fragment element in its tile + template + static __attribute__((always_inline)) void lane_coords(uint32_t &row, uint32_t &col) { uint32_t lane = vx_thread_id(); if constexpr (Frag::Use == frag_use_t::matrix_a) { - // Load row-major matrix A uint32_t block_idx = (cfg::a_block_size == NT) ? 0 : (lane / cfg::a_block_size); uint32_t lane_in_blk = (cfg::a_block_size == NT) ? lane : (lane % cfg::a_block_size); - uint32_t block_row = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcM); - uint32_t block_col = (lane_in_blk % cfg::tcK) * i_ratio; + row = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcM); + col = (lane_in_blk % cfg::tcK) * i_ratio; + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + // Sparse B uses 2x tcK per block + constexpr uint32_t b_tcK = is_sparse ? (cfg::tcK * 2) : cfg::tcK; + constexpr uint32_t b_block_size = is_sparse ? cfg::b_block_size_sp : cfg::b_block_size; + uint32_t block_idx = (b_block_size == NT) ? 0 : (lane / b_block_size); + uint32_t lane_in_blk = (b_block_size == NT) ? lane : (lane % b_block_size); + row = (lane_in_blk % b_tcK) * i_ratio; + col = (lane_in_blk / b_tcK) + (block_idx * cfg::tcN); + } else { + row = lane / cfg::tcN; + col = lane % cfg::tcN; + } + if constexpr (layout == col_major) { + std::swap(row, col); + } + } + + // The TCU reads its operands in a bank-conflict-free register order, so a + // fragment must sit in registers in that order. These map a register offset + // back to the logical fragment index it holds. + static constexpr uint32_t b_sub_eff = is_sparse ? cfg::b_sub_blocks_sp : cfg::b_sub_blocks; + static constexpr bool bcfree_sp = is_sparse && !cfg::sym_sparse; // sparse non-sym + static constexpr bool bcfree_a = !is_sparse && (b_sub_eff == 1); // dense pattern A + static constexpr bool bcfree_b = !is_sparse && (b_sub_eff > 1); // dense pattern B + + // A inverse: physical offset → logical fragment index + static constexpr uint32_t ra_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: A is identity (off = m*k_count+k) + return off; + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward A={m[0],~m[hi],k} + uint32_t m = ((1 - ((off >> 1) & 1)) << 1) | (off >> 2); + uint32_t k = off & 1; + return m * cfg::k_steps + k; + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward A={k[0],~m,m^k[hi]} + uint32_t m = 1 - ((off >> 1) & 1); + uint32_t k_hi = m ^ (off & 1); + uint32_t k_lo = off >> 2; + return m * cfg::k_steps + (k_hi << 1 | k_lo); + } else { + return off; + } + } + + // B inverse: physical offset → logical fragment index + static constexpr uint32_t rb_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: forward B={n[hi], ~(n[0]^k), ~k} + // Inverse: k = 1-off[0], n[0] = (1-off[1])^k, n[hi] = off[2] + uint32_t k = 1 - (off & 1); + uint32_t n_lo = (1 - ((off >> 1) & 1)) ^ k; + uint32_t n_hi = off >> 2; + return k * cfg::n_steps + (n_hi << 1 | n_lo); + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward B={k[0],k[hi]^np,~np} + uint32_t n_pair = 1 - (off & 1); + uint32_t k_hi = ((off >> 1) & 1) ^ n_pair; + uint32_t k_lo = off >> 2; + return (k_hi << 1 | k_lo) * cfg::b_sub_blocks + n_pair; + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward B={n^k,~k} + uint32_t k = 1 - (off & 1); + uint32_t n = ((off >> 1) & 1) ^ k; + return k * cfg::n_steps + n; + } else { + return off; + } + } + + // C inverse: physical offset → logical fragment index + static constexpr uint32_t rc_idx(uint32_t off) { + if constexpr (bcfree_sp) { + // Sparse non-sym: forward C={n[hi], m, ~(m^n[0])} + // Inverse: m = off[1], n[0] = (1-off[0])^m, n[hi] = off[2] + uint32_t m = (off >> 1) & 1; + uint32_t n_lo = (1 - (off & 1)) ^ m; + uint32_t n_hi = off >> 2; + return m * cfg::n_steps + (n_hi << 1 | n_lo); + } else if constexpr (bcfree_a) { + // Dense Pattern A (b_sub==1): forward C={m[0],~m[hi],XNOR(m[hi],n)} + uint32_t m_hi = 1 - ((off >> 1) & 1); + uint32_t m = (m_hi << 1) | (off >> 2); + uint32_t n = 1 - (m_hi ^ (off & 1)); + return m * cfg::n_steps + n; + } else if constexpr (bcfree_b) { + // Dense Pattern B (b_sub==2): forward C={n[0],~m,n[hi]} + uint32_t m = 1 - ((off >> 1) & 1); + uint32_t n = ((off & 1) << 1) | (off >> 2); + return m * cfg::n_steps + n; + } else { + return off; + } + } + + // vx_ldm/vx_stm geometry of a fragment: NR registers from FD in J row groups + // row_step * ldm elements apart, span bytes of columns per row group, and Q + // the registers of a J=4 group. + template + struct ldm_geom { + static constexpr bool is_a = (Frag::Use == frag_use_t::matrix_a); + static constexpr bool is_b = (Frag::Use == frag_use_t::matrix_b); + static constexpr uint32_t NR = (is_a && is_sparse) ? sparse_regs : Frag::NR; + static constexpr uint32_t J = is_a ? cfg::m_steps + : is_b ? (is_sparse ? Frag::NR : cfg::b_sub_steps) + : cfg::m_steps; + static constexpr uint32_t row_step = is_a ? (cfg::a_sub_blocks * cfg::tcM) + : is_b ? (is_sparse ? (cfg::tcN / 2) : (cfg::b_sub_blocks * cfg::tcN)) + : cfg::tcM; + static constexpr uint32_t col_bytes = (is_b && is_sparse) ? 4 + : (is_a || is_b) ? (cfg::tcK * 4) : (cfg::tcN * 4); + static constexpr uint32_t span = (NR / J) * col_bytes; + static constexpr uint32_t FD = is_a ? 10 : is_b ? ((Frag::NR == 8) ? 24 : 28) : 0; + static constexpr uint32_t Q = is_b ? 12 : 28; + }; + + // The fragment's vx_ldm/vx_stm shape in a memory layout; LDM_NONE when a + // fragment register is not one word of the tile, the order is not covered, + // the context is not the 8-register C/D tile the shapes are defined on, or + // the macro-op does not pay: a sparse MX K tile is bound by its three + // serialized metadata loads, which the fragment macro-ops cannot shorten and + // their base-group setup delays; an asymmetric sparse B, one burst of eight + // uops, stalls the MMA on its operands longer than the scalar loads do (it + // has no shape). + template + static constexpr int32_t ldm_shape() { + if constexpr (NR_ != 8 || DK_ != 0 || (is_sparse && input_is_mx)) { + return detail::LDM_NONE; + } else if constexpr (Frag::Use == frag_use_t::matrix_a) { + return (layout == row_major) ? (is_sparse ? detail::LDM_A_SP : detail::LDM_A) : detail::LDM_NONE; + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + if constexpr (is_sparse && !cfg::sym_sparse) { + return detail::LDM_NONE; + } + return (layout == col_major) ? (is_sparse ? detail::LDM_B_SP : detail::LDM_B) : detail::LDM_NONE; + } else { + return (layout == row_major && !is_sparse) ? detail::LDM_C : detail::LDM_NONE; + } + } + + template + static constexpr uint32_t ldm_idx(uint32_t off) { + if constexpr (Frag::Use == frag_use_t::matrix_a) { + return ra_idx(off); + } else if constexpr (Frag::Use == frag_use_t::matrix_b) { + return rb_idx(off); + } else { + return rc_idx(off); + } + } + + // Lane addresses of the fragment's row groups; off_bytes is added unless it + // folds into the instruction's immediate. Each is the lane's tile address + // plus a loop-invariant row offset. + template + static __attribute__((always_inline)) void ldm_group(uintptr_t (&g)[ldm_geom::J], + const void *src, size_t lane_off, size_t ldm) { + using geom = ldm_geom; + constexpr bool fold = detail::ldm_folds; + size_t lane_bytes = lane_off * sizeof(frag_elem_t) + (fold ? 0 : off_bytes); + size_t row_bytes = geom::row_step * ldm * sizeof(frag_elem_t); + uintptr_t base = reinterpret_cast(src) + lane_bytes; + if constexpr (geom::J == 4) { + // opaque, so a K loop steps one induction variable and derives the other + // three registers by adds instead of stepping one per register; volatile, + // so those adds stay behind the tile's metadata loads instead of delaying them + __asm__ volatile("" : "+r"(base)); + } + detail::unroll_for([&](auto j) { + g[j] = base + j * row_bytes; + }); + } + + template + static constexpr int32_t ldm_imm() { + return (detail::ldm_folds::span> ? off_bytes : 0) | shape; + } + + template + static __attribute__((always_inline)) void ldm_load(Frag &dst, const void *src, size_t lane_off, size_t ldm) { + using geom = ldm_geom; + constexpr int32_t off_bytes = k_off * int32_t(sizeof(frag_elem_t)); + uintptr_t g[geom::J]; + ldm_group(g, src, lane_off, ldm); + float v[geom::NR]; + detail::vx_ldm()>()>(v, g); + detail::unroll_for([&](auto r) { + dst.data[ldm_idx(r)] = v[r]; + }); + } + + // base already points at the calling lane's first element + template + static __attribute__((always_inline)) void load_fragment(Frag &dst, const frag_elem_t *base, size_t ldm) { + if constexpr (Frag::Use == frag_use_t::matrix_a) { + // Load row-major matrix A uint32_t m_stride = cfg::a_sub_blocks * cfg::tcM; uint32_t k_stride = cfg::tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } if constexpr (is_sparse) { // Sparse A load: only load half the K-steps (compressed A) constexpr uint32_t sparse_k_steps = cfg::k_steps / 2; constexpr uint32_t sparse_regs = cfg::m_steps * sparse_k_steps; - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / sparse_k_steps; uint32_t block_k = r % sparse_k_steps; @@ -319,7 +680,6 @@ struct wmma_context { }); } else { // Dense A load: load all K-steps - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::k_steps; uint32_t block_k = r % cfg::k_steps; @@ -347,18 +707,10 @@ struct wmma_context { if constexpr (is_sparse) { // Sparse B load: uses 2x tcK for B block constexpr uint32_t b_tcK = cfg::tcK * 2; - uint32_t block_idx = (cfg::b_block_size_sp == NT) ? 0 : (lane / cfg::b_block_size_sp); - uint32_t lane_in_blk = (cfg::b_block_size_sp == NT) ? lane : (lane % cfg::b_block_size_sp); - uint32_t block_col = (lane_in_blk / b_tcK) + (block_idx * cfg::tcN); - uint32_t block_row = (lane_in_blk % b_tcK) * i_ratio; // NT=16 sparse: each register = 2 columns × full K (n_stride=2, no K iteration) // NT=8/32 sparse: standard interleaved layout uint32_t n_stride = cfg::sym_sparse ? (cfg::tcN / 2) : (cfg::b_sub_blocks_sp * cfg::tcN); uint32_t k_stride = b_tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; if constexpr (src_layout == row_major) { static_assert(input_is_subbyte == false, "row_major layout is not supported for sub-byte matrix_b"); @@ -410,16 +762,8 @@ struct wmma_context { } } else { // Dense B load - uint32_t block_idx = (cfg::b_block_size == NT) ? 0 : (lane / cfg::b_block_size); - uint32_t lane_in_blk = (cfg::b_block_size == NT) ? lane : (lane % cfg::b_block_size); - uint32_t block_col = (lane_in_blk / cfg::tcK) + (block_idx * cfg::tcN); - uint32_t block_row = (lane_in_blk % cfg::tcK) * i_ratio; uint32_t n_stride = cfg::b_sub_blocks * cfg::tcN; uint32_t k_stride = cfg::tcK * i_ratio; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_k = r / cfg::b_sub_steps; uint32_t block_n = r % cfg::b_sub_steps; @@ -445,14 +789,8 @@ struct wmma_context { } else { // Load accumulator matrix C - uint32_t block_row = lane / cfg::tcN; - uint32_t block_col = lane % cfg::tcN; uint32_t m_stride = cfg::tcM; uint32_t n_stride = cfg::tcN; - if constexpr (src_layout == col_major) { - std::swap(block_row, block_col); - } - auto base = reinterpret_cast(src) + block_row * ldm + block_col; detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::n_steps; uint32_t block_n = r % cfg::n_steps; @@ -473,18 +811,27 @@ struct wmma_context { } } +public: + template static __attribute__((always_inline)) void store_matrix_sync(void *dst, const Frag &src, size_t ldm) { static_assert(Frag::Use == frag_use_t::accumulator, "only accumulator fragment can be stored"); - uint32_t lane = vx_thread_id(); - uint32_t block_row = lane / cfg::tcN; - uint32_t block_col = lane % cfg::tcN; + uint32_t block_row, block_col; + lane_coords(block_row, block_col); uint32_t m_stride = cfg::tcM; uint32_t n_stride = cfg::tcN; - if constexpr (dst_layout == col_major) { - std::swap(block_row, block_col); - } auto base = reinterpret_cast(dst) + block_row * ldm + block_col; + if constexpr (ldst_matrix && ldm_shape() != detail::LDM_NONE) { + using geom = ldm_geom; + uintptr_t g[geom::J]; + ldm_group(g, dst, block_row * ldm + block_col, ldm); + float v[8]; + detail::unroll_for<8>([&](auto r) { + v[r] = src.data[rc_idx(r)]; + }); + detail::vx_stm(v, g); + return; + } detail::unroll_for([&](auto r) { uint32_t block_m = r / cfg::n_steps; uint32_t block_n = r % cfg::n_steps; @@ -511,87 +858,6 @@ struct wmma_context { static_assert(FragC::Use == frag_use_t::accumulator, "C must be accumulator"); static_assert(FragD::Use == frag_use_t::accumulator, "D must be accumulator"); - // Bank-conflict-free register offset permutations (0 stalls). - // SW must place fragment data into registers matching the HW's - // permuted offset order. These are the INVERSE of the forward - // formulas in VX_tcu_uops.sv, mapping physical register offset - // back to the logical fragment index. - constexpr uint32_t b_sub_eff = is_sparse ? cfg::b_sub_blocks_sp : cfg::b_sub_blocks; - constexpr bool bcfree_sp = is_sparse && !cfg::sym_sparse; // sparse non-sym - constexpr bool bcfree_a = !is_sparse && (b_sub_eff == 1); // dense pattern A - constexpr bool bcfree_b = !is_sparse && (b_sub_eff > 1); // dense pattern B - - // A inverse: physical offset → logical fragment index - constexpr auto ra_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: A is identity (off = m*k_count+k) - return off; - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward A={m[0],~m[hi],k} - uint32_t m = ((1 - ((off >> 1) & 1)) << 1) | (off >> 2); - uint32_t k = off & 1; - return m * cfg::k_steps + k; - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward A={k[0],~m,m^k[hi]} - uint32_t m = 1 - ((off >> 1) & 1); - uint32_t k_hi = m ^ (off & 1); - uint32_t k_lo = off >> 2; - return m * cfg::k_steps + (k_hi << 1 | k_lo); - } else { - return off; - } - }; - - // B inverse: physical offset → logical fragment index - constexpr auto rb_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: forward B={n[hi], ~(n[0]^k), ~k} - // Inverse: k = 1-off[0], n[0] = (1-off[1])^k, n[hi] = off[2] - uint32_t k = 1 - (off & 1); - uint32_t n_lo = (1 - ((off >> 1) & 1)) ^ k; - uint32_t n_hi = off >> 2; - return k * cfg::n_steps + (n_hi << 1 | n_lo); - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward B={k[0],k[hi]^np,~np} - uint32_t n_pair = 1 - (off & 1); - uint32_t k_hi = ((off >> 1) & 1) ^ n_pair; - uint32_t k_lo = off >> 2; - return (k_hi << 1 | k_lo) * cfg::b_sub_blocks + n_pair; - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward B={n^k,~k} - uint32_t k = 1 - (off & 1); - uint32_t n = ((off >> 1) & 1) ^ k; - return k * cfg::n_steps + n; - } else { - return off; - } - }; - - // C inverse: physical offset → logical fragment index - constexpr auto rc_idx = [](uint32_t off) constexpr -> uint32_t { - if constexpr (bcfree_sp) { - // Sparse non-sym: forward C={n[hi], m, ~(m^n[0])} - // Inverse: m = off[1], n[0] = (1-off[0])^m, n[hi] = off[2] - uint32_t m = (off >> 1) & 1; - uint32_t n_lo = (1 - (off & 1)) ^ m; - uint32_t n_hi = off >> 2; - return m * cfg::n_steps + (n_hi << 1 | n_lo); - } else if constexpr (bcfree_a) { - // Dense Pattern A (b_sub==1): forward C={m[0],~m[hi],XNOR(m[hi],n)} - uint32_t m_hi = 1 - ((off >> 1) & 1); - uint32_t m = (m_hi << 1) | (off >> 2); - uint32_t n = 1 - (m_hi ^ (off & 1)); - return m * cfg::n_steps + n; - } else if constexpr (bcfree_b) { - // Dense Pattern B (b_sub==2): forward C={n[0],~m,n[hi]} - uint32_t m = 1 - ((off >> 1) & 1); - uint32_t n = ((off & 1) << 1) | (off >> 2); - return m * cfg::n_steps + n; - } else { - return off; - } - }; - // frag_c initialized into accumulator registers (f0-f7) register float fd0 __asm__("f0") = frag_c.data[rc_idx(0)]; register float fd1 __asm__("f1") = frag_c.data[rc_idx(1)]; @@ -673,13 +939,17 @@ template + uint32_t NRC_ = 8, + bool ldst_matrix = false> // store the accumulator with vx_stm struct wgmma_context { + static_assert(tcu_output_valid(It::id, Ot::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + private: static constexpr uint32_t NRA = 4; - using ctx_c = wmma_context; - using ctx_a = wmma_context; + using ctx_c = wmma_context; + using ctx_a = wmma_context; // Direct geometry from NT (no wmma_config_t dependency) static constexpr uint32_t clog2(uint32_t x) { @@ -840,6 +1110,22 @@ struct wgmma_context { ctx_c::fill_fragment(dst, value); } + // Element offset of the calling lane's first A element (see wmma_context::lane_offset). + template + static __attribute__((always_inline)) size_t lane_offset(size_t ldm) { + static_assert(Frag::Use == frag_use_t::matrix_a, "only matrix_a fragment can be loaded from registers in wgmma context"); + static_assert(layout == row_major, "wgmma block-major load only accepts row_major caller hint"); + uint32_t lane = vx_thread_id(); + uint32_t i_in = lane / tcK; + uint32_t k_in_elem = (lane % tcK) * i_ratio; + if (ldm == 0) { + // Block-major: a single-K-step dense block holds fedpK elements per row + constexpr uint32_t row_elems = (k_steps == 1 && !is_sparse) ? (fedpK * i_ratio) : (tcK * i_ratio); + return i_in * row_elems + k_in_elem; + } + return i_in * ldm + k_in_elem; + } + // Load A fragment (NRA=4 config) from SMEM (RS path). // Layout selection via `ldm` (in elements): // ldm == 0 → block-major (cooperative-load layout): @@ -853,28 +1139,27 @@ struct wgmma_context { // ctx_a::sparse_regs slot is reserved for the metadata word. template static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm) { + load_matrix_sync(dst, src, ldm, lane_offset(ldm)); + } + + template + static __attribute__((always_inline)) void load_matrix_sync(Frag &dst, const void *src, size_t ldm, size_t lane_off) { static_assert(Frag::Use == frag_use_t::matrix_a, "only matrix_a fragment can be loaded from registers in wgmma context"); static_assert(src_layout == row_major, "wgmma block-major load only accepts row_major caller hint"); static_assert(tcM * tcK == NT, "wgmma block-major load assumes canonical config (TC_M*TC_K == NT)"); constexpr uint32_t k_row_elems = tcK * i_ratio; constexpr uint32_t sparse_blk_elems = tcM * k_row_elems; - uint32_t lane = vx_thread_id(); - uint32_t i_in = lane / tcK; - uint32_t k_in_elem = (lane % tcK) * i_ratio; auto load_reg = [&](uint32_t m_blk, uint32_t k_blk) { size_t elem_off; if (ldm == 0) { // Block-major SMEM: blocks contiguous, k-block outer. constexpr uint32_t block_elems = is_sparse ? sparse_blk_elems : a_blk_elems; - elem_off = (k_blk * m_steps + m_blk) * block_elems - + i_in * k_row_elems - + k_in_elem; + elem_off = (k_blk * m_steps + m_blk) * block_elems + lane_off; } else { // Row-major SMEM: standard row*ldm + col. - elem_off = (m_blk * tcM + i_in) * ldm - + (k_blk * k_row_elems + k_in_elem); + elem_off = (m_blk * tcM) * ldm + k_blk * k_row_elems + lane_off; } return *reinterpret_cast( reinterpret_cast(src) + elem_off); @@ -895,13 +1180,9 @@ struct wgmma_context { if constexpr (k_steps == 1) { size_t elem_off; if (ldm == 0) { - elem_off = m_blk * a_blk_elems - + i_in * (fedpK * i_ratio) - + k_blk * k_row_elems - + k_in_elem; + elem_off = m_blk * a_blk_elems + k_blk * k_row_elems + lane_off; } else { - elem_off = (m_blk * tcM + i_in) * ldm - + (k_blk * k_row_elems + k_in_elem); + elem_off = (m_blk * tcM) * ldm + k_blk * k_row_elems + lane_off; } dst.data[r] = *reinterpret_cast( reinterpret_cast(src) + elem_off); @@ -948,6 +1229,23 @@ struct wgmma_context { uint32_t base_row = lane / tcN; uint32_t base_col = lane % tcN; auto base = reinterpret_cast(dst) + base_row * ldm + base_col; + if constexpr (ldst_matrix && dst_layout == row_major) { + // one vx_stm per 8-register chunk; chunk q holds columns 4q..4q+3 + constexpr uint32_t chunk_bytes = 4 * tcN * sizeof(output_t); + static_assert((chunk_bytes % 32) == 0, "a chunk's column offset must fold into the immediate"); + uintptr_t g[2]; + g[0] = reinterpret_cast(base); + g[1] = g[0] + tcM * ldm * sizeof(output_t); + detail::unroll_for([&](auto q) { + constexpr uint32_t chunk = decltype(q)::value; + float v[8]; + detail::unroll_for<8>([&](auto r) { + v[r] = src.data[chunk * 8 + r]; + }); + detail::vx_stm(v, g); + }); + return; + } detail::unroll_for([&](auto r) { uint32_t block_m = r % m_steps; // n-major: m is inner uint32_t block_n = r / m_steps; // n-major: n is outer diff --git a/sw/runtime/aved/vortex.cpp b/sw/runtime/aved/vortex.cpp index 08d431c41a..977d48e8bd 100644 --- a/sw/runtime/aved/vortex.cpp +++ b/sw/runtime/aved/vortex.cpp @@ -58,6 +58,7 @@ #include #include #include +#include using namespace vortex; @@ -196,46 +197,13 @@ class vx_device { #endif } - // Auto-discover the card the way XRT enumerates by index: scan sysfs for - // the SLASH control function (10ee:50c1) and derive its BDF. Makes - // VRT_DEVICE_BDF optional -- the hardcoded default below is only the last - // resort when no card is on the bus (e.g. a sim run with no env set). - static std::string discover_bdf() { - DIR* d = opendir("/sys/bus/pci/devices"); - if (d == nullptr) { - return ""; - } - std::string found; - while (struct dirent* e = readdir(d)) { - const std::string name = e->d_name; // "0000:01:00.1" - if (name.size() < 12 || name[0] == '.') { - continue; - } - auto read_hex = [&](const char* leaf) -> unsigned { - std::ifstream f("/sys/bus/pci/devices/" + name + "/" + leaf); - unsigned v = 0; - f >> std::hex >> v; - return v; - }; - if (read_hex("vendor") == 0x10ee && read_hex("device") == 0x50c1) { - found = name.substr(0, name.rfind('.')); // drop the function - if (found.rfind("0000:", 0) == 0) { - found = found.substr(5); // "01:00" form - } - break; - } - } - closedir(d); - return found; - } - int init() { // An empty value is treated as unset: the test harness exports these // unconditionally, so they arrive empty rather than absent. const char* bdf = getenv("VRT_DEVICE_BDF"); std::string bdf_storage; if (bdf == nullptr || bdf[0] == '\0') { - bdf_storage = discover_bdf(); + bdf_storage = discover_v80_bdf(); if (!bdf_storage.empty()) { fprintf(stderr, "[VXDRV] auto-discovered V80 at %s (set " "VRT_DEVICE_BDF to override)\n", bdf_storage.c_str()); @@ -272,18 +240,35 @@ class vx_device { // previous image set, which can be far above or below this one's. if (!sim_mode_) { const uint64_t timed_hz = vrtDevice_.getMaxFrequency(); - vrtDevice_.setFrequency(timed_hz); - const uint64_t clk_hz = vrtDevice_.getFrequency(); - if (clk_hz == 0 || clk_hz > timed_hz + timed_hz / 200) { + // The wizard settles on the nearest rate its dividers reach, which can + // be a step above the request: lower the request below that step. + uint64_t want_hz = timed_hz, clk_hz = 0; + for (int i = 0; i < 8; ++i) { + vrtDevice_.setFrequency(want_hz); + clk_hz = vrtDevice_.getFrequency(); + if (clk_hz == 0 || clk_hz <= timed_hz) + break; + want_hz -= (clk_hz - want_hz) + timed_hz / 100; + } + if (clk_hz == 0 || clk_hz > timed_hz) { fprintf(stderr, "[VXDRV] Error: user clock reads %lu Hz, image is timed for %lu Hz\n", (unsigned long)clk_hz, (unsigned long)timed_hz); + park_clock(); return -1; } printf("[VXDRV] kernel clock %lu Hz (image timed for %lu Hz)\n", (unsigned long)clk_hz, (unsigned long)timed_hz); } vrtKernel_ = vrt::Kernel(vrtDevice_, KERNEL_NAME); - VRT_CATCH(-1) + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] VRT exception: %s\n", e.what()); + park_clock(); + return -1; + } catch (...) { + fprintf(stderr, "[VXDRV] VRT exception (unknown)\n"); + park_clock(); + return -1; + } // Must follow vrtKernel_: the connection map comes from the vbin's // system_map.xml, which the kernel handle parses. @@ -850,6 +835,20 @@ class vx_device { private: #ifdef CPP_API + // A design that fails to come up stays loaded: leave it clocked below any + // image's timing so it cannot disturb the shell before the next load. + void park_clock() noexcept { + constexpr uint64_t kSafeHz = 50000000; + if (sim_mode_) + return; + try { + vrtDevice_.setFrequency(kSafeHz); + fprintf(stderr, "[VXDRV] kernel clock parked at %lu Hz\n", + (unsigned long)vrtDevice_.getFrequency()); + } catch (...) { + } + } + // ----- Staged CP memory (m_axi_host wired to device memory) ----- // // WHY THIS EXISTS diff --git a/sw/runtime/common/caps.h b/sw/runtime/common/caps.h index ddb1def115..243fddc820 100644 --- a/sw/runtime/common/caps.h +++ b/sw/runtime/common/caps.h @@ -32,25 +32,34 @@ namespace vortex { // CP regfile offsets for the static capability words. // Pass to a backend's cp_mmio_read(), which applies any platform base offset. +static constexpr uint32_t CP_REG_GPU_DEV_CAPS_EXT = 0x00C; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_LO = 0x018; static constexpr uint32_t CP_REG_GPU_DEV_CAPS_HI = 0x01C; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_LO = 0x020; static constexpr uint32_t CP_REG_GPU_ISA_CAPS_HI = 0x024; -// Warps, sockets per cluster and clusters carry an exact count, so a machine -// that is not a power of two reports truly; the field is zero on devices -// built before it existed, which report only the 3-bit log2 field. -inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { - uint64_t exact = (dev_caps >> exact_lsb) & exact_mask; +// Warps, sockets per cluster, clusters and the socket size carry an exact +// count, so a machine that is not a power of two reports truly; the field is +// zero on devices built before it existed, which report only the 3-bit log2 +// field of GPU_DEV_CAPS. +inline uint64_t exact_or_log2(uint64_t exact, uint64_t dev_caps, uint32_t log2_lsb) { return exact ? exact : (1ull << ((dev_caps >> log2_lsb) & 0x7)); } +inline uint64_t exact_or_log2(uint64_t dev_caps, uint32_t exact_lsb, uint64_t exact_mask, uint32_t log2_lsb) { + return exact_or_log2((dev_caps >> exact_lsb) & exact_mask, dev_caps, log2_lsb); +} + // Decode a VX_CAPS_* id from the two CP capability words. Returns true // and writes *out when the id is encoded in the words; returns false // for ids the caller must resolve itself (CACHE_LINE_SIZE, // GLOBAL_MEM_SIZE, CLOCK_RATE, PEAK_MEM_BW). -inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, +inline bool decode_caps(uint64_t dev_caps, uint32_t dev_caps_ext, uint64_t isa_caps, uint32_t caps_id, uint64_t* out) { + // A CP without the extension word answers zero, a DECERR sentinel or all + // ones; any reserved bit set means the word is absent. + const uint32_t caps_ext = (dev_caps_ext >> 8) ? 0 : dev_caps_ext; + const uint64_t socket_size = exact_or_log2(caps_ext & 0xFF, dev_caps, 14); switch (caps_id) { case VX_CAPS_VERSION: *out = (dev_caps >> 0) & 0xff; @@ -62,14 +71,13 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, *out = exact_or_log2(dev_caps, 42, 0x7F, 11); return true; case VX_CAPS_NUM_CORES: { - uint64_t socket_size = 1ull << ((dev_caps >> 14) & 0x7); uint64_t cluster_size = exact_or_log2(dev_caps, 49, 0xFF, 17); uint64_t num_clusters = exact_or_log2(dev_caps, 57, 0x7F, 20); *out = num_clusters * cluster_size * socket_size; return true; } case VX_CAPS_SOCKET_SIZE: - *out = 1ull << ((dev_caps >> 14) & 0x7); + *out = socket_size; return true; case VX_CAPS_NUM_CLUSTERS: *out = exact_or_log2(dev_caps, 57, 0x7F, 20); @@ -94,14 +102,15 @@ inline bool decode_caps(uint64_t dev_caps, uint64_t isa_caps, } } -// Read GPU_DEV_CAPS / GPU_ISA_CAPS from the CP regfile through a -// backend's cp_mmio_read primitive. `Reader` is any callable matching -// int(uint32_t off, uint32_t* out) — returns non-zero on failure. +// Read GPU_DEV_CAPS / GPU_DEV_CAPS_EXT / GPU_ISA_CAPS from the CP regfile +// through a backend's cp_mmio_read primitive. `Reader` is any callable +// matching int(uint32_t off, uint32_t* out) — returns non-zero on failure. // Returns 0 on success. template inline int load_caps(Reader&& cp_mmio_read, - uint64_t* dev_caps, uint64_t* isa_caps) { + uint64_t* dev_caps, uint32_t* dev_caps_ext, uint64_t* isa_caps) { uint32_t lo = 0, hi = 0; + if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_EXT, dev_caps_ext)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_LO, &lo)) return -1; if (cp_mmio_read(CP_REG_GPU_DEV_CAPS_HI, &hi)) return -1; *dev_caps = uint64_t(lo) | (uint64_t(hi) << 32); diff --git a/sw/runtime/common/device.cpp b/sw/runtime/common/device.cpp index 0eee737e92..1ba76afe9e 100644 --- a/sw/runtime/common/device.cpp +++ b/sw/runtime/common/device.cpp @@ -84,6 +84,9 @@ constexpr uint32_t CP_STATUS_BUSY = 0x1; // is physical — the MMU-aware CP DMA skips translation. constexpr uint8_t CP_MEM_FLAG_PHYSICAL = 0x04; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; + constexpr uint32_t CP_RING_SIZE_LOG2 = 16; // 64 KiB constexpr uint32_t CP_RING_SIZE = 1u << CP_RING_SIZE_LOG2; constexpr uint8_t CP_OPCODE_MEM_WRITE = 0x01; @@ -154,8 +157,8 @@ Device::~Device() { // Drain the kernel-args scratch pool. { std::lock_guard g(args_pool_mu_); - for (uint64_t addr : args_pool_free_) - this->mem_free(addr); + for (const auto& slot : args_pool_free_) + this->mem_free(slot.addr); args_pool_free_.clear(); } // Park the CP before releasing the buffers it masters into. Ordering is @@ -179,10 +182,13 @@ Device::~Device() { // ============================================================================ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled) { + bool* out_pooled, bool* out_llc_clean) { if (!out_addr || !out_pooled) return VX_ERR_INVALID_VALUE; + bool llc_clean = false; + if (out_llc_clean) *out_llc_clean = false; if (size > ARGS_SLOT_SIZE) { - // Oversized args block — one-off allocation, not pooled. + // Oversized args block — one-off allocation, not pooled. The range may + // have been another buffer the cache still holds. *out_pooled = false; return this->mem_alloc(size, /*VX_MEM_READ*/ 0x1, out_addr); } @@ -190,8 +196,18 @@ vx_result_t Device::args_slot_acquire(uint64_t size, uint64_t* out_addr, { std::lock_guard g(args_pool_mu_); if (!args_pool_free_.empty()) { - *out_addr = args_pool_free_.back(); - args_pool_free_.pop_back(); + const uint64_t epoch = llc_epoch_.load(); + size_t pick = args_pool_free_.size() - 1; + for (size_t i = 0; i < args_pool_free_.size(); ++i) { + if (args_pool_free_[i].clean_epoch <= epoch) { + pick = i; + break; + } + } + *out_addr = args_pool_free_[pick].addr; + llc_clean = (args_pool_free_[pick].clean_epoch <= epoch); + args_pool_free_.erase(args_pool_free_.begin() + pick); + if (out_llc_clean) *out_llc_clean = llc_clean; return VX_SUCCESS; } } @@ -205,7 +221,7 @@ void Device::args_slot_release(uint64_t addr, bool pooled) { return; } std::lock_guard g(args_pool_mu_); - args_pool_free_.push_back(addr); + args_pool_free_.push_back({addr, llc_epoch_.load() + 1}); } vx_result_t Device::open(uint32_t index, Device** out) { @@ -331,16 +347,14 @@ vx_result_t Device::cp_init() { uint32_t dev_caps = 0; if (p->cp_reg_read(CP_DEV_CAPS, &dev_caps) != VX_SUCCESS) return VX_ERR_DEVICE_LOST; - // An all-ones read is never a valid capability word: bits [31:24] are + // An all-ones read is never a valid capability word: bits [31:29] are // hardwired zero by VX_cp_axil_regfile, so 0xFFFFFFFF means the read // did not reach the register (AXI DECERR / no response substitutes // all-ones on the way back). Decoding it would set every capability // bit -- and a spurious VM_ENABLED sends VMManager off to identity-map // 65,536 PTEs one CP round-trip at a time, which presents as a hang at // 100% CPU rather than as the bus error it actually is. - // Treat it as "no optional capabilities", which is also the correct - // answer for this CP: it has no SATP register, no OP_DRAW, and no - // CMD_LAUNCH_QMD (grep hw/rtl/cp -- all three are absent). + // Treat it as "no optional capabilities". if (dev_caps == 0xFFFFFFFFu) { printf("[VXDRV] Warning: CP_DEV_CAPS read returned all-ones; the " "AXI-Lite read did not reach the register. Assuming no " @@ -360,10 +374,13 @@ vx_result_t Device::cp_init() { // Reading them on a target that does not decode them stalls the DCR // bus waiting for a response that never arrives. mmu_fault_report_ = (dev_caps & (1u << 27)) != 0; + // DMA_LLC_CLIENT (bit 28): the CP DMA is a client of the last-level + // cache, which is then the point of coherence for host transfers. + cp_dma_llc_client_ = (dev_caps & (1u << 28)) != 0; } // Resolve the core count now, while nothing holds cp_mu_. Deferring it to - // the first cp_submit_cache_flush would reach query_caps from inside an + // the first cache flush would reach query_caps from inside an // open batch, which already owns cp_mu_ -- and it is not recursive. { auto rc = this->query_caps(VX_CAPS_NUM_CORES, &cp_num_cores_); @@ -469,6 +486,17 @@ vx_result_t Device::cp_init() { } } + // No kernel has run yet, so the last-level cache holds none of these. + { + std::lock_guard g(args_pool_mu_); + for (uint32_t i = 0; i < ARGS_POOL_CLEAN; ++i) { + uint64_t addr = 0; + auto r = this->mem_alloc(ARGS_SLOT_SIZE, /*VX_MEM_READ*/ 0x1, &addr); + if (r != VX_SUCCESS) return r; + args_pool_free_.push_back({addr, 0}); + } + } + return VX_SUCCESS; } @@ -513,7 +541,84 @@ const char* cp_opcode_name(uint8_t op) { } } // namespace -vx_result_t Device::cp_ring_append_(const void* cl) { +namespace { +// CMD_CACHE_FLUSH on-wire layout (cmd_size=12): +// bytes 0..3 header { opcode=0x0A, flags=CP_FLUSH_FLAG_L1_ONLY?, reserved=0 } +// bytes 4..11 arg0 number of cores to flush +// The CP sweeps a per-core flush DCR-read across [0, num_cores) and +// retires the command only when the last core's flush completes. +void build_cache_flush(uint8_t cl[CP_CL_BYTES], uint64_t num_cores, bool l1_only) { + std::memset(cl, 0, CP_CL_BYTES); + cl[0] = CP_OPCODE_CACHE_FLUSH; + cl[1] = l1_only ? CP_FLUSH_FLAG_L1_ONLY : 0; + std::memcpy(cl + 4, &num_cores, sizeof(num_cores)); +} + +// True when a CMD_MEM_* transfer reaches device memory outside the IO +// aperture. IO accesses bypass the caches, so they need no flush. +bool cp_dma_touches_device_mem(const uint8_t* cl) { + uint64_t dst = 0, src = 0, size = 0; + std::memcpy(&dst, cl + 4, sizeof(dst)); + std::memcpy(&src, cl + 12, sizeof(src)); + std::memcpy(&size, cl + 20, sizeof(size)); + auto outside_io = [size](uint64_t addr) { + return addr < VX_MEM_IO_BASE_ADDR || addr + size > VX_MEM_IO_END_ADDR; + }; + switch (cl[0]) { + case CP_OPCODE_MEM_WRITE: return outside_io(dst); + case CP_OPCODE_MEM_READ: return outside_io(src); + case CP_OPCODE_MEM_COPY: return outside_io(dst) || outside_io(src); + default: return false; + } +} +} // namespace + +vx_result_t Device::cp_ring_append_(const void* cl, bool llc_clean) { + // Caller holds cp_mu_. A kernel's results are flushed lazily, by the + // next command that needs them: the next launch gets an L1-only flush + // (the last-level cache is the point of coherence between kernels), and + // the next transfer to or from device memory a full one when the CP DMA + // reaches device memory beneath the last-level cache, an L1-only one when + // it is a client of that cache. Appending the flush under the same lock + // keeps any other command from slipping between. + const uint8_t* b = static_cast(cl); + // A write into memory the last-level cache holds no line of has nothing + // to wait for: the cache cannot serve stale data from it or write a + // dirty line back over it. + const bool dma = cp_dma_touches_device_mem(b) && !(llc_clean && b[0] == CP_OPCODE_MEM_WRITE); + const bool launch = (b[0] == CP_OPCODE_LAUNCH) + || (b[0] == CP_OPCODE_LAUNCH_QMD) + || (b[0] == CP_OPCODE_DRAW); + if ((dma && (l1_dirty_ || llc_dirty_)) || (launch && l1_dirty_)) { + uint8_t flush[CP_CL_BYTES]; + const bool full = dma && llc_dirty_; + build_cache_flush(flush, cp_num_cores_, /*l1_only=*/!full); + auto r = cp_ring_write_(flush); + if (r != VX_SUCCESS) return r; + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && !full; + if (full) { + ++llc_epoch_; + } + } + assert(!(dma && (l1_dirty_ || llc_dirty_))); + auto r = cp_ring_write_(cl); + if (r != VX_SUCCESS) return r; + if (launch) { + // a draw descriptor ends on its own L1-only flush step + l1_dirty_ = (b[0] != CP_OPCODE_DRAW); + llc_dirty_ = !cp_dma_llc_client_; + } else if (b[0] == CP_OPCODE_CACHE_FLUSH) { + l1_dirty_ = false; + llc_dirty_ = llc_dirty_ && (b[1] & CP_FLUSH_FLAG_L1_ONLY); + if (!(b[1] & CP_FLUSH_FLAG_L1_ONLY)) { + ++llc_epoch_; + } + } + return VX_SUCCESS; +} + +vx_result_t Device::cp_ring_write_(const void* cl) { // Caller holds cp_mu_. Write one CL into the ring at the current tail — // a plain memcpy through the ring's CP-visible host pointer — then bump // tail + reserve the seqnum slot. No doorbell, no poll. @@ -576,20 +681,19 @@ vx_result_t Device::cp_batch_end() { if (r != VX_SUCCESS) { return r; } - // The batch's trailing CMD_CACHE_FLUSH(es) have retired, so every kernel's - // writes are coherent: drain the console rings once for the whole batch - // (deferred from each in-batch cp_submit_launch). + // Drain the console rings once for the whole batch (deferred from each + // in-batch cp_submit_launch). return drain_cout(); } -vx_result_t Device::cp_submit_cl_(const void* cl) { +vx_result_t Device::cp_submit_cl_(const void* cl, bool llc_clean) { auto* p = platform(); // Batch mode: append only — cp_mu_ is already held for the batch, and // the single doorbell + poll happen in cp_batch_end. Only the thread that // owns the batch may take this path; anyone else must block on cp_mu_. if (tls_batch_owner == this) { - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r == VX_SUCCESS) cp_batch_target_ = cp_expected_seqnum_; return r; } @@ -602,7 +706,7 @@ vx_result_t Device::cp_submit_cl_(const void* cl) { std::lock_guard g(cp_mu_); // 1) Write the CL into the ring and reserve its seqnum. - auto r = cp_ring_append_(cl); + auto r = cp_ring_append_(cl, llc_clean); if (r != VX_SUCCESS) return r; target = cp_expected_seqnum_; @@ -726,37 +830,6 @@ vx_result_t Device::cp_poll_seqnum_(uint64_t target) { } vx_result_t Device::cp_submit_dcr_write(uint32_t addr, uint32_t value) { - // VM safety check: when VM is active and the pinned slab is configured, - // every HW-addr DCR must reference a buffer in the pinned slab — the HW - // master bypasses the per-core MMU and would otherwise dereference a stale VA. - // - // TEX / RASTER / OM address DCRs encode value as a cache-block index - // (pa = value << 6). Split BASE_LO/HI DCR pairs (e.g. DXA) are validated - // by the caller, not here. - if (vm_enabled_ && pinned_mem_) { - switch (addr) { - case VX_DCR_TEX_ADDR: - case VX_DCR_RASTER_TBUF_ADDR: - case VX_DCR_RASTER_PBUF_ADDR: - case VX_DCR_OM_CBUF_ADDR: - case VX_DCR_OM_ZBUF_ADDR: { - const uint64_t pa = uint64_t(value) << 6; - if (pa < pinned_base_ || - pa >= pinned_base_ + pinned_size_) { - std::cerr << "[VXDRV] dcr 0x" << std::hex << addr - << " value 0x" << value << " (pa 0x" << pa - << ") not in pinned slab [0x" << pinned_base_ - << ", 0x" << (pinned_base_ + pinned_size_) - << ") — buffer needs VX_MEM_PHYS" - << std::dec << std::endl; - return VX_ERR_INVALID_VALUE; - } - break; - } - default: - break; - } - } // CMD_DCR_WRITE on-wire layout (cmd_size=20): // bytes 0..3 header { opcode=0x04, flags=0, reserved=0 } // bytes 4..11 arg0 DCR addr @@ -811,15 +884,13 @@ vx_result_t Device::cp_submit_launch() { if (r != VX_SUCCESS) { return r; } - // Cache coherence: post an explicit cache flush right after the launch - // (ACQUIRE_MEM model) so the host observes coherent kernel results. - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; - // In a batch the flush has only been appended, not retired — defer the - // COUT drain to cp_batch_end (one drain for the whole sequence). + // The kernel-boundary cache flush (ACQUIRE_MEM model) is posted by the + // next command that needs it (see cp_ring_append_). In a batch the + // launch has only been appended, not retired — defer the COUT drain to + // cp_batch_end (one drain for the whole sequence). if (tls_batch_owner == this) return VX_SUCCESS; - // Final COUT drain: the flush has made the kernel's writes coherent, so - // the tail-end console output left in the rings is now safe to read. + // Final COUT drain: the console rings live in the uncached IO aperture, + // and a launch completes only once its stores have left the device. return drain_cout(); } @@ -860,7 +931,7 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { // The CP reads the in-memory KMU descriptor (a {count,(addr,value)...} // list the caller staged) and replays it before pulsing start — one ring // command in place of the ~18 CMD_DCR_WRITEs a plain launch costs. Same - // trailing CMD_CACHE_FLUSH / COUT-drain discipline as cp_submit_launch. + // COUT-drain discipline as cp_submit_launch. uint8_t cl[CP_CL_BYTES] = {0}; auto r = ensure_mmu_satp(); if (r != VX_SUCCESS) { @@ -874,8 +945,6 @@ vx_result_t Device::cp_submit_launch_qmd(uint64_t qmd_addr) { if (r != VX_SUCCESS) { return r; } - r = cp_submit_cache_flush(); - if (r != VX_SUCCESS) return r; if (tls_batch_owner == this) return VX_SUCCESS; return drain_cout(); } @@ -905,19 +974,6 @@ vx_result_t Device::cp_submit_draw(uint64_t desc_addr) { return drain_cout(); } -vx_result_t Device::cp_submit_cache_flush() { - // CMD_CACHE_FLUSH on-wire layout (cmd_size=12): - // bytes 0..3 header { opcode=0x0A, flags=0, reserved=0 } - // bytes 4..11 arg0 number of cores to flush - // The CP sweeps a per-core flush DCR-read across [0, num_cores) and - // retires the command only when the last core's flush completes. - // No-op on write-through cache configs (the Vortex default). - uint8_t cl[CP_CL_BYTES] = {0}; - cl[0] = CP_OPCODE_CACHE_FLUSH; - std::memcpy(cl + 4, &cp_num_cores_, sizeof(cp_num_cores_)); - return cp_submit_cl_(cl); -} - vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, uint32_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; @@ -953,7 +1009,7 @@ vx_result_t Device::cp_submit_dcr_read(uint32_t addr, uint32_t tag, vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, uint64_t arg2, - bool physical) { + bool physical, bool llc_clean) { // CMD_MEM_* on-wire layout (cmd_size=28): // bytes 0..3 header { opcode, flags, reserved=0 } // bytes 4..11 arg0 dst address @@ -965,7 +1021,7 @@ vx_result_t Device::cp_submit_mem_(uint8_t opcode, uint64_t arg0, std::memcpy(cl + 4, &arg0, sizeof(arg0)); std::memcpy(cl + 12, &arg1, sizeof(arg1)); std::memcpy(cl + 20, &arg2, sizeof(arg2)); - return cp_submit_cl_(cl); + return cp_submit_cl_(cl, llc_clean); } vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, @@ -975,7 +1031,7 @@ vx_result_t Device::cp_submit_mem_copy(uint64_t dst, uint64_t src, } vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical) { + uint64_t size, bool physical, bool llc_clean) { if (size == 0) return VX_SUCCESS; if (!host_src) return VX_ERR_INVALID_VALUE; // Stage through a bounded CP-visible host buffer, one chunk at a time: @@ -998,7 +1054,7 @@ vx_result_t Device::cp_submit_mem_write(uint64_t dev_dst, const void* host_src, r = platform()->host_mem_push(staging.cp_addr); if (r == VX_SUCCESS) r = cp_submit_mem_(CP_OPCODE_MEM_WRITE, dev_dst + off, - staging.cp_addr, n, physical); + staging.cp_addr, n, physical, llc_clean); } host_free(staging.cp_addr); return r; @@ -1181,7 +1237,7 @@ vx_result_t Device::memory_info(uint64_t* out_free, uint64_t* out_used) { vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { if (!out_value) return VX_ERR_INVALID_VALUE; - // The two static caps words are read once from the CP regfile caps + // The static caps words are read once from the CP regfile caps // window; serialize the load against concurrent cp_reg_* traffic. if (!caps_loaded_) { std::lock_guard g(cp_mu_); @@ -1189,12 +1245,12 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { auto rd = [this](uint32_t off, uint32_t* v) -> int { return platform_->cp_reg_read(off, v) == VX_SUCCESS ? 0 : -1; }; - if (vortex::load_caps(rd, &dev_caps_, &isa_caps_) != 0) + if (vortex::load_caps(rd, &dev_caps_, &dev_caps_ext_, &isa_caps_) != 0) return VX_ERR_DEVICE_LOST; caps_loaded_ = true; } } - if (vortex::decode_caps(dev_caps_, isa_caps_, caps_id, out_value)) + if (vortex::decode_caps(dev_caps_, dev_caps_ext_, isa_caps_, caps_id, out_value)) return VX_SUCCESS; // Caps not encoded in the CP words — platform/runtime constants. switch (caps_id) { @@ -1219,8 +1275,8 @@ vx_result_t Device::query_caps(uint32_t caps_id, uint64_t* out_value) { // ============================================================================ vx_result_t Device::dev_write(uint64_t dev_addr, const void* src, - uint64_t size) { - return cp_submit_mem_write(dev_addr, src, size); + uint64_t size, bool llc_clean) { + return cp_submit_mem_write(dev_addr, src, size, /*physical=*/false, llc_clean); } vx_result_t Device::dev_read(void* dst, uint64_t dev_addr, uint64_t size) { diff --git a/sw/runtime/common/graphics.cpp b/sw/runtime/common/graphics.cpp index 8b3d055dc1..7b44ead983 100644 --- a/sw/runtime/common/graphics.cpp +++ b/sw/runtime/common/graphics.cpp @@ -310,7 +310,7 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, return vx_buffer_address(b, out_addr); }; const uint32_t W = VX_MEM_WRITE; // host-write-intent scratch - const uint32_t RWP = VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS; // pinned FF-read outputs + const uint32_t RW = VX_MEM_READ | VX_MEM_WRITE; // FF-read outputs vx_result_t r; // No slot_prim/slot_bbox scratch: setup_k counts survivors and EMIT recomputes @@ -319,9 +319,9 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, // Two-heap residency split: the device-only shader scratch regions live // in ONE pooled slab (collapsing 12 separate allocations into a single resident - // buffer), while the FF-pinned-PA outputs the raster unit reads (prim, tilebuf) - // keep their own VX_MEM_PHYS buffers — both because they need the pinned heap and - // because callers read them back at offset 0 (prim_buffer()/tile_buffer()). + // buffer), while the outputs the raster unit reads (prim, tilebuf) keep their + // own buffers, because callers read them back at offset 0 + // (prim_buffer()/tile_buffer()). constexpr uint64_t SLAB_ALIGN = 64; // sub-region alignment (cache line) auto align_up = [](uint64_t v, uint64_t a) { return (v + a - 1) / a * a; }; @@ -353,9 +353,9 @@ vx_result_t FrontEndPool::init(vx_device_h dev, vx_kernel_h setup_k, } // FF-pinned-PA outputs (read by the raster unit; read back by the host at off 0). - if ((r = mk(NT * MS * PRIM_SZ, RWP, &impl_->prim)) != VX_SUCCESS) return r; + if ((r = mk(NT * MS * PRIM_SZ, RW, &impl_->prim)) != VX_SUCCESS) return r; impl_->prim_buf = impl_->bufs.back(); - if ((r = mk(B * HDR_SZ + (uint64_t)keys_cap * 4, RWP, &impl_->tilebuf)) != VX_SUCCESS) return r; + if ((r = mk(B * HDR_SZ + (uint64_t)keys_cap * 4, RW, &impl_->tilebuf)) != VX_SUCCESS) return r; impl_->tile_buf = impl_->bufs.back(); return VX_SUCCESS; } diff --git a/sw/runtime/common/queue.cpp b/sw/runtime/common/queue.cpp index df63f7a9dc..84d4c22f0d 100644 --- a/sw/runtime/common/queue.cpp +++ b/sw/runtime/common/queue.cpp @@ -413,16 +413,17 @@ vx_result_t Queue::enqueue_launch(const vx_launch_info_t* info, // Empty blob → caller pre-programmed the ARG DCRs (legacy path). uint64_t args_addr = 0; bool args_pooled = false; + bool args_clean = false; bool args_staged = !args_blob.empty(); if (args_staged) { auto r = device_->args_slot_acquire(args_blob.size(), - &args_addr, &args_pooled); + &args_addr, &args_pooled, &args_clean); if (r != VX_SUCCESS) { if (kernel) kernel->release(); *s = *e = now_ns(); return r; } r = device_->dev_write(args_addr, args_blob.data(), - args_blob.size()); + args_blob.size(), args_clean); if (r != VX_SUCCESS) { device_->args_slot_release(args_addr, args_pooled); if (kernel) kernel->release(); @@ -505,6 +506,8 @@ namespace { constexpr uint8_t CP_OP_DCR_WRITE = 0x04; constexpr uint8_t CP_OP_CACHE_FLUSH = 0x0A; constexpr uint8_t CP_OP_LAUNCH_QMD = 0x0B; +// CMD_CACHE_FLUSH header flag: flush only the levels above the last-level cache. +constexpr uint8_t CP_FLUSH_FLAG_L1_ONLY = 0x04; // Fixed per-step stride in a draw descriptor (28-byte cmd-record prefix). // Must equal CommandProcessor::DRAW_STEP_BYTES. constexpr int CP_DRAW_STEP_BYTES = 28; @@ -605,11 +608,12 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // 1) Args blob. if (!rec.args.empty()) { + bool clean = false; r = device_->args_slot_acquire(rec.args.size(), - &st.args_addr, &st.args_pooled); + &st.args_addr, &st.args_pooled, &clean); if (r != VX_SUCCESS) break; st.args_active = true; - r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size()); + r = device_->dev_write(st.args_addr, rec.args.data(), rec.args.size(), clean); if (r != VX_SUCCESS) break; } @@ -670,10 +674,11 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, st.qmd_words = std::move(qmd); if (device_->cp_supports_qmd()) { const uint64_t qmd_bytes = st.qmd_words.size() * sizeof(uint32_t); - r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled); + bool clean = false; + r = device_->args_slot_acquire(qmd_bytes, &st.qmd_addr, &st.qmd_pooled, &clean); if (r != VX_SUCCESS) break; st.qmd_active = true; - r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes); + r = device_->dev_write(st.qmd_addr, st.qmd_words.data(), qmd_bytes, clean); if (r != VX_SUCCESS) break; } } @@ -682,7 +687,7 @@ vx_result_t cmd_stage_qmds(Device* device_, const std::vector& recs, // Submit one staged launch: CMD_LAUNCH_QMD when the CP decodes it, else // replay the descriptor pairs as CMD_DCR_WRITEs followed by a plain -// CMD_LAUNCH (same trailing cache-flush discipline in cp_submit_launch). +// CMD_LAUNCH. vx_result_t cmd_submit_launch(Device* device_, const CmdStaged& st) { if (device_->cp_supports_qmd()) { return device_->cp_submit_launch_qmd(st.qmd_addr); @@ -820,9 +825,10 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, if (r == VX_SUCCESS && use_op_draw) { std::vector desc(4, 0); // header placeholder uint32_t num_steps = 0; - auto put_step = [&](uint8_t opcode, uint64_t a0, uint64_t a1) { + auto put_step = [&](uint8_t opcode, uint8_t flags, uint64_t a0, uint64_t a1) { uint8_t step[CP_DRAW_STEP_BYTES] = {0}; step[0] = opcode; + step[1] = flags; std::memcpy(step + 4, &a0, sizeof(a0)); std::memcpy(step + 12, &a1, sizeof(a1)); desc.insert(desc.end(), step, step + CP_DRAW_STEP_BYTES); @@ -831,17 +837,18 @@ vx_result_t Queue::enqueue_draw(const vx_command_t* commands, for (size_t i = 0; i < recs.size(); ++i) { const CmdRec& rec = recs[i]; if (rec.is_launch) { - put_step(CP_OP_LAUNCH_QMD, staged[i].qmd_addr, 0); - put_step(CP_OP_CACHE_FLUSH, num_cores, 0); + put_step(CP_OP_LAUNCH_QMD, 0, staged[i].qmd_addr, 0); + put_step(CP_OP_CACHE_FLUSH, CP_FLUSH_FLAG_L1_ONLY, num_cores, 0); } else { - put_step(CP_OP_DCR_WRITE, rec.dcr_addr, rec.dcr_value); + put_step(CP_OP_DCR_WRITE, 0, rec.dcr_addr, rec.dcr_value); } } std::memcpy(desc.data(), &num_steps, sizeof(num_steps)); - r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled); + bool desc_clean = false; + r = device_->args_slot_acquire(desc.size(), &desc_addr, &desc_pooled, &desc_clean); if (r == VX_SUCCESS) { desc_active = true; - r = device_->dev_write(desc_addr, desc.data(), desc.size()); + r = device_->dev_write(desc_addr, desc.data(), desc.size(), desc_clean); } } diff --git a/sw/runtime/common/vm.cpp b/sw/runtime/common/vm.cpp index 0922d8cfd1..051fed853f 100644 --- a/sw/runtime/common/vm.cpp +++ b/sw/runtime/common/vm.cpp @@ -34,6 +34,12 @@ uint32_t pte_flags_from_access(uint32_t access_flags) { return pte; } +// The OM aperture is decoded by address ahead of translation (the LSU steers +// it to the OM unit), so a randomized VA must never land in it. +bool overlaps_om_aperture(uint64_t va, uint64_t size) { + return va < uint64_t(VX_MEM_OM_END_ADDR) && (va + size) > uint64_t(VX_MEM_OM_BASE_ADDR); +} + // Ceil-log2 — used for the per-level VPN field width. constexpr unsigned vm_clog2(uint64_t n) { unsigned r = 0; @@ -175,7 +181,7 @@ uint64_t VMManager::map_p2v(uint64_t ppn, uint32_t flags) { uint64_t random_page_offset = dist(rng_); uint64_t candidate_va = va_range_start + (random_page_offset << VX_VM_PAGE_LOG2_SIZE); - bool in_use = false; + bool in_use = overlaps_om_aperture(candidate_va, VX_VM_PAGE_SIZE); for (const auto& mapping : addr_mapping) { if (mapping.second == (candidate_va >> VX_VM_PAGE_LOG2_SIZE)) { in_use = true; @@ -239,7 +245,8 @@ int VMManager::phy_to_virt_map(uint64_t size, uint64_t* dev_pAddr, uint32_t flag uint64_t random_page_offset = dist(rng_); uint64_t candidate_va = va_range_start + (random_page_offset << VX_VM_PAGE_LOG2_SIZE); - bool range_available = true; + bool range_available = + !overlaps_om_aperture(candidate_va, num_pages << VX_VM_PAGE_LOG2_SIZE); for (uint64_t i = 0; i < num_pages && range_available; ++i) { uint64_t test_vpn = (candidate_va >> VX_VM_PAGE_LOG2_SIZE) + i; for (const auto& mapping : addr_mapping) { @@ -358,7 +365,17 @@ int VMManager::install_identity_map(uint64_t addr, uint64_t size) { constexpr uint32_t IDENTITY_PTE_FLAGS = PTE_V | PTE_R | PTE_W | PTE_X | PTE_U | PTE_A | PTE_D; - (void)virtual_mem_->reserve(addr, size); + // A range inside one already mapped, as the kernel image is inside the + // page-table and stack region init() maps, holds its VAs already. + bool reserved = false; + for (auto [start, end] : identity_ranges_) { + if (addr >= start && addr + size <= end) { + reserved = true; + break; + } + } + if (!reserved && virtual_mem_->reserve(addr, size) == 0) + identity_ranges_.emplace_back(addr, addr + size); uint64_t cur = addr; uint64_t end = addr + size; diff --git a/sw/runtime/common/vm.h b/sw/runtime/common/vm.h index 581ecd43fe..58a0cbdf2c 100644 --- a/sw/runtime/common/vm.h +++ b/sw/runtime/common/vm.h @@ -19,6 +19,7 @@ #include #include #include +#include #include #include #include @@ -129,6 +130,9 @@ class VMManager { // Heap mapped once with superpages (VORTEX_VM_MEGAPAGE_HEAP). bool megapage_heap_; + + // Identity-mapped ranges [start, end) whose VAs are reserved. + std::vector> identity_ranges_; }; } // namespace vortex diff --git a/sw/runtime/common/vortex2_internal.h b/sw/runtime/common/vortex2_internal.h index ef803eefc7..f5a7e474ff 100644 --- a/sw/runtime/common/vortex2_internal.h +++ b/sw/runtime/common/vortex2_internal.h @@ -233,15 +233,15 @@ class Device : public RefCounted { vx_result_t cp_submit_dcr_write(uint32_t addr, uint32_t value); // Post one CMD_LAUNCH to the ring, commit Q_TAIL, and wait for - // Q_SEQNUM. Synchronous. Followed by an implicit CMD_CACHE_FLUSH so - // the host observes coherent results (see cp_submit_cache_flush). + // Q_SEQNUM. Synchronous. The kernel's results are flushed by the next + // command that needs them (see cp_ring_append_). vx_result_t cp_submit_launch(); // Post one CMD_LAUNCH_QMD: the CP reads the KMU descriptor (a // {count,(dcr_addr,value)...} list the caller staged to device memory at // `qmd_addr`) and replays it, then pulses start — one ring command in - // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Followed by an - // implicit CMD_CACHE_FLUSH, like cp_submit_launch. Batch-aware. + // place of a launch's ~18 CMD_DCR_WRITEs (NVIDIA QMD model). Flushed like + // cp_submit_launch. Batch-aware. vx_result_t cp_submit_launch_qmd(uint64_t qmd_addr); // Post one CMD_DRAW (OP_DRAW): arg0 = device address of a resident draw @@ -255,12 +255,6 @@ class Device : public RefCounted { bool cp_supports_draw() const { return cp_supports_draw_; } bool cp_supports_qmd() const { return cp_supports_qmd_; } - // Post one CMD_CACHE_FLUSH to the ring (AMD ACQUIRE_MEM model): the CP - // sweeps a per-core cache flush across all cores and retires the - // command only when the last core's flush completes. A no-op on - // write-through cache configs. Posted after every CMD_LAUNCH. - vx_result_t cp_submit_cache_flush(); - // Program the device MMU's SATP once, ahead of the first launch that // could translate. Called from every launch path. vx_result_t ensure_mmu_satp(); @@ -273,7 +267,7 @@ class Device : public RefCounted { // ----- Batched CP submission (one draw = one ring batch) ----- // cp_batch_begin holds the ring lock and switches cp_submit_* into // append-only mode: each subsequent cp_submit_dcr_write / cp_submit_launch - // / cp_submit_cache_flush writes its CL into the ring and reserves a + // writes its CL into the ring and reserves a // seqnum but does NOT ring the doorbell or poll. cp_batch_end commits // Q_TAIL once, polls Q_SEQNUM once for the last appended command, drains // COUT once, and releases the lock. The CP then retires the whole @@ -306,8 +300,11 @@ class Device : public RefCounted { // cp_submit_mem_read posts CMD_MEM_READ (device->host) into a // VX_MEM_HOST buffer and copies it back to `host_dst`. Synchronous. vx_result_t cp_submit_mem_copy (uint64_t dst, uint64_t src, uint64_t size); + // `llc_clean`: the last-level cache holds no line of the destination, so + // the write needs no cache flush ahead of it. vx_result_t cp_submit_mem_write(uint64_t dev_dst, const void* host_src, - uint64_t size, bool physical = false); + uint64_t size, bool physical = false, + bool llc_clean = false); vx_result_t cp_submit_mem_read (void* host_dst, uint64_t dev_src, uint64_t size, bool physical = false); @@ -318,7 +315,8 @@ class Device : public RefCounted { // engine (CMD_MEM_*) — identically on every backend. The runtime never // touches device memory directly: it only appends commands to the host // ring and the CP executes them. - vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size); + vx_result_t dev_write(uint64_t dev_addr, const void* src, uint64_t size, + bool llc_clean = false); vx_result_t dev_read (void* dst, uint64_t dev_addr, uint64_t size); vx_result_t dev_copy (uint64_t dst, uint64_t src, uint64_t size); @@ -335,8 +333,11 @@ class Device : public RefCounted { // args_slot_release frees correctly. Because vx_enqueue_launch holds // a slot only for the duration of one synchronous CP launch, the // free-list naturally settles at the count of concurrent launches. + // `*out_llc_clean` reports a slot no kernel has read since the last full + // cache flush: the last-level cache holds none of its lines, so staging + // into it needs no flush (see args_pool_free_). vx_result_t args_slot_acquire(uint64_t size, uint64_t* out_addr, - bool* out_pooled); + bool* out_pooled, bool* out_llc_clean = nullptr); void args_slot_release(uint64_t addr, bool pooled); // ----- Device-memory allocation (common-core allocator) ----- @@ -372,7 +373,7 @@ class Device : public RefCounted { // cp_submit_dcr_write / cp_submit_launch — they just build the CL. // While this thread has a batch open it appends only; the single // doorbell + poll are deferred to cp_batch_end. - vx_result_t cp_submit_cl_(const void* cl); + vx_result_t cp_submit_cl_(const void* cl, bool llc_clean = false); // Block until the CP has retired seqnum >= target. // @@ -391,13 +392,19 @@ class Device : public RefCounted { // Append one CL at the current tail and reserve its seqnum (bump // cp_tail_ + cp_expected_seqnum_). Caller must hold cp_mu_ (directly in // the non-batch path, or via cp_batch_begin). No doorbell, no poll. - vx_result_t cp_ring_append_(const void* cl); + // Inserts the lazy kernel-boundary CMD_CACHE_FLUSH (AMD ACQUIRE_MEM + // model) under the same lock: L1-only ahead of a launch while + // l1_dirty_, and ahead of a CMD_MEM_* touching device memory while + // either flag is set -- full while llc_dirty_. + vx_result_t cp_ring_append_(const void* cl, bool llc_clean = false); + vx_result_t cp_ring_write_(const void* cl); // Build + submit a CMD_MEM_* descriptor (opcode, dst, src, size). // `physical` sets the CMD_MEM header flag so the CP DMA skips VM // translation — used for page-table writes / the PT region. vx_result_t cp_submit_mem_(uint8_t opcode, uint64_t arg0, uint64_t arg1, - uint64_t arg2, bool physical = false); + uint64_t arg2, bool physical = false, + bool llc_clean = false); // ----- CP-visible host memory (command ring + DMA staging) ----- // host_alloc wraps platform()->host_mem_alloc and records the region @@ -436,8 +443,9 @@ class Device : public RefCounted { std::map host_mems_; // Lazily-loaded device/ISA caps words (read once from the CP regfile). - uint64_t dev_caps_ = 0; - uint64_t isa_caps_ = 0; + uint64_t dev_caps_ = 0; + uint32_t dev_caps_ext_ = 0; + uint64_t isa_caps_ = 0; bool caps_loaded_ = false; std::mutex mu_; @@ -457,6 +465,15 @@ class Device : public RefCounted { uint64_t cp_tail_ = 0; uint64_t cp_expected_seqnum_ = 0; uint64_t cp_num_cores_ = 0; // cached VX_CAPS_NUM_CORES, used for CMD_CACHE_FLUSH + // A kernel ran since the last flush, so the L1s may hold its writes or + // stale lines. + bool l1_dirty_ = false; + // A kernel ran since the last full cache flush, so the last-level cache + // may hold lines the CP DMA (when it reaches device memory beneath it) + // would miss or later overwrite. Both flags are guarded by cp_mu_. + bool llc_dirty_ = false; + // Full cache flushes appended to the ring so far. + std::atomic llc_epoch_{0}; std::mutex cp_mu_; // serialize ring writes // Seqnum the last command appended to the open batch will reach; // cp_batch_end polls for it once. Touched only by the batch owner, which @@ -486,6 +503,9 @@ class Device : public RefCounted { // the OP_DRAW mirror). Discovered at open. bool cp_supports_draw_ = false; bool cp_supports_qmd_ = false; + // The CP DMA reaches device memory through the last-level cache (CP + // DEV_CAPS bit 28), so transfers never need that cache flushed. + bool cp_dma_llc_client_ = false; class CpMemIO; std::unique_ptr vm_io_; std::unique_ptr vm_mgr_; @@ -502,9 +522,20 @@ class Device : public RefCounted { // Kernel-args scratch pool. Free-list of recycled fixed-size device slots; // ARGS_SLOT_SIZE covers typical kernel arg blocks. Drained in ~Device. + // Each free slot records the flush epoch from which the last-level cache + // holds none of its lines: a kernel that read the slot may have left them + // there until the next full flush. The pool starts with ARGS_POOL_CLEAN + // slots allocated at open, when no kernel has run, so a run of launches + // stages its arguments without flushing until it has used them all; the + // next transfer that needs a full flush makes them clean again. static constexpr uint64_t ARGS_SLOT_SIZE = 4096; + static constexpr uint32_t ARGS_POOL_CLEAN = 32; + struct ArgsSlot { + uint64_t addr; + uint64_t clean_epoch; + }; std::mutex args_pool_mu_; - std::vector args_pool_free_; + std::vector args_pool_free_; }; // ============================================================================ diff --git a/sw/runtime/firesim/Makefile b/sw/runtime/firesim/Makefile index e7dbbac418..d604055085 100644 --- a/sw/runtime/firesim/Makefile +++ b/sw/runtime/firesim/Makefile @@ -15,6 +15,21 @@ CXXFLAGS += -fPIC # Add external configuration CXXFLAGS += $(CONFIGS) +# The card, and its FireSim platform, which decides the variables that name +# the image and the card (those of the card's native driver). +include $(SRC_DIR)/card.mk +ifeq ($(FIRESIM_PLATFORM), aved) +CXXFLAGS += -DFIRESIM_PLATFORM_AVED +endif + +# TARGET=xrtsim or avedsim runs the metasimulator in-process in place of the +# card: the same FireSim-transformed target in Verilator, reached through the +# emulated host interface, with no image, no XRT and no VRT -- the counterpart +# of the native drivers' xrtsim and avedsim. +ifneq ($(filter $(TARGET),xrtsim avedsim),) +CXXFLAGS += -DFIRESIM_EMUL +endif + LDFLAGS += -shared -pthread # Find libfiresim.so siblings at runtime in the same dir libvortex-firesim.so lives in. LDFLAGS += -Wl,-rpath,'$$ORIGIN' @@ -39,14 +54,24 @@ all: $(DESTDIR)/$(PROJECT) driver: $(DESTDIR)/libfiresim.so $(DESTDIR)/libfiresim.so: force - DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim lib FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) \ + CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) $(DESTDIR)/$(PROJECT): $(SRCS) $(DESTDIR)/libfiresim.so $(CONFIG_FILE) $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ -$(CONFIG_FILE): - echo "$(CXXFLAGS)" > $@.tmp - @if ! cmp -s $@.tmp $@; then mv $@.tmp $@; else rm $@.tmp; fi +# Forced, as in every other runtime: without a prerequisite the stamp is written +# once and never compared again, so a changed CONFIGS keeps linking the runtime +# built for the previous one. +$(CONFIG_FILE): force + @mkdir -p $(@D) + @TMP=$@.tmp.$$$$ ; \ + printf '%s\n' "$(CXXFLAGS)" > $$TMP ; \ + if ! cmp -s $$TMP $@ 2>/dev/null; then \ + mv -f $$TMP $@ ; \ + else \ + rm -f $$TMP ; \ + fi clean-driver: DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/firesim clean diff --git a/sw/runtime/firesim/card.mk b/sw/runtime/firesim/card.mk new file mode 100644 index 0000000000..ffb1b3c34f --- /dev/null +++ b/sw/runtime/firesim/card.mk @@ -0,0 +1,39 @@ +# Which card a FireSim run is for, decided the same way wherever it is asked -- +# the runtime build, run-firesim, blackbox.sh's driver build -- from what the +# caller already has: TARGET and, for an image target, FPGA_BIN_DIR. +# +# TARGET=hw|hw_emu|sim the card hw/syn/firesim recorded beside the image +# ($(FPGA_BIN_DIR)/firesim.card) +# TARGET=xrtsim|avedsim no image; the U55C, or the V80 (FIRESIM_CARD overrides) +# TARGET empty xrtsim, or avedsim when FIRESIM_CARD names an aved card +# +# Sets FIRESIM_CARD and FIRESIM_PLATFORM (vitis or aved), and TARGET when empty. + +FIRESIM_GEN := python3 $(VORTEX_HOME)/hw/scripts/gen_platform.py +firesim_platform_of = $(shell $(FIRESIM_GEN) --card=$(1) --emit=firesim-mk 2>/dev/null | sed -n 's/^FIRESIM_PLATFORM := //p') + +ifeq ($(strip $(TARGET)),) +FIRESIM_CARD ?= u55c +override TARGET := $(if $(filter aved,$(call firesim_platform_of,$(FIRESIM_CARD))),avedsim,xrtsim) +else ifneq ($(filter $(TARGET),xrtsim avedsim),) +FIRESIM_CARD ?= $(if $(filter avedsim,$(TARGET)),v80,u55c) +else ifeq ($(filter $(TARGET),hw hw_emu sim),) +$(error TARGET=$(TARGET) is not a FireSim target: use xrtsim, hw_emu or hw on an XRT card, \ + avedsim, sim or hw on an aved card) +else +FIRESIM_CARD ?= $(shell cat $(FPGA_BIN_DIR)/firesim.card 2>/dev/null) +ifeq ($(FIRESIM_CARD),) +$(error TARGET=$(TARGET) runs an image: set FPGA_BIN_DIR to the bin directory hw/syn/firesim built \ + (it holds firesim.card), or use TARGET=xrtsim|avedsim) +endif +endif + +FIRESIM_PLATFORM := $(call firesim_platform_of,$(FIRESIM_CARD)) +ifeq ($(FIRESIM_PLATFORM),) +$(error cannot resolve the FireSim platform of card '$(FIRESIM_CARD)') +endif + +FIRESIM_TARGETS := $(if $(filter aved,$(FIRESIM_PLATFORM)),avedsim sim hw,xrtsim hw_emu hw) +ifeq ($(filter $(TARGET),$(FIRESIM_TARGETS)),) +$(error TARGET=$(TARGET) is not a FireSim target for card $(FIRESIM_CARD): use one of $(FIRESIM_TARGETS)) +endif diff --git a/sw/runtime/firesim/vortex.cpp b/sw/runtime/firesim/vortex.cpp index 973c79bc49..363710fda0 100644 --- a/sw/runtime/firesim/vortex.cpp +++ b/sw/runtime/firesim/vortex.cpp @@ -30,11 +30,15 @@ #include #include #include +#include #include #include #include #include +#include +#include +#include #include using namespace vortex; @@ -52,18 +56,54 @@ class vx_device { } int init() { - // The bitstream is selected by the environment so that the same runtime - // serves a metasimulation and an FPGA without a rebuild. XCLBIN_PATH is - // the same variable the xrt driver reads: both consume a Vitis image, and - // a caller switching between them should not have to switch variable names - // as well. - // - // Unset falls through as empty rather than to a default filename, unlike - // the xrt driver. An empty path tells the simulator to omit +binary_file, - // which is what metasimulation needs -- it has no image to load, and naming - // one it cannot open would turn a working configuration into a failure. - const char* bitstream = getenv("XCLBIN_PATH"); - return sim_.init(bitstream != nullptr ? bitstream : ""); + // The image and card are named by the same variables the card's native + // driver reads, so switching between a native and a FireSim run does not + // mean switching variable names as well. + std::vector plusargs; +#if defined(FIRESIM_EMUL) + // The metasimulator is the design itself: there is no image to load and no + // card to find. +#elif defined(FIRESIM_PLATFORM_AVED) + // As the aved driver: VRT_VBIN_PATH and VRT_DEVICE_BDF, with the card + // discovered on the bus when the BDF is unset. An image is required: the + // card and the platform's simulator both run one. + const char* vbin = getenv("VRT_VBIN_PATH"); + if (vbin == nullptr || vbin[0] == '\0') { + fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the FireSim .vbin\n"); + return -1; + } + const char* bdf_env = getenv("VRT_DEVICE_BDF"); + std::string bdf = (bdf_env != nullptr) ? bdf_env : ""; + if (bdf.empty()) { + bdf = discover_v80_bdf(); + if (bdf.empty()) { + fprintf(stderr, "[VXDRV] no V80 found on the bus; set VRT_DEVICE_BDF\n"); + return -1; + } + } + plusargs.push_back(std::string("+binary_file=") + vbin); + plusargs.push_back("+bdf=" + bdf); +#else + // As the xrt driver: XCLBIN_PATH. Unset falls through as no image rather + // than a default filename: an empty path is what metasimulation needs -- + // it has no image to load, and naming one it cannot open would turn a + // working configuration into a failure. + const char* xclbin = getenv("XCLBIN_PATH"); + if (xclbin != nullptr && xclbin[0] != '\0') { + plusargs.push_back(std::string("+binary_file=") + xclbin); + } +#endif + // Extra plusargs for FireSim's bridges, card and metasimulator alike -- + // chiefly FASED's runtime settings (+mm_readLatency_0=... with + // +mm_useHardwareDefaultRuntimeSettings_0 covering the rest), which are + // the only way to change the memory timing without a rebuild. + if (const char* extra = getenv("VORTEX_FIRESIM_ARGS")) { + std::istringstream in(extra); + for (std::string tok; in >> tok;) { + plusargs.push_back(tok); + } + } + return sim_.init(plusargs); } // ----- CP register channel ----- @@ -109,6 +149,16 @@ class vx_device { return 0; } + // Host regions are this process's memory, which the CP model reads and + // writes directly: coherent, so there is nothing to sync. + int host_mem_pull(uint64_t /*cp_addr*/) { + return 0; + } + + int host_mem_push(uint64_t /*cp_addr*/) { + return 0; + } + private: // If `addr` falls in a registered host region, return it as a host // pointer (cp_addr == the pointer); otherwise nullptr → device memory. @@ -158,6 +208,10 @@ class vx_device { return false; return busy; }; + h.vortex_drain = [this]() { + bool busy = true; + while (busy && sim_.is_busy(&busy) == 0) {} + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/sw/runtime/firesim/xrt.ini b/sw/runtime/firesim/xrt.ini new file mode 100644 index 0000000000..53a4f44ee4 --- /dev/null +++ b/sw/runtime/firesim/xrt.ini @@ -0,0 +1,7 @@ +# XRT settings for FireSim hardware emulation (TARGET=hw_emu). +# +# Without this XRT logs every signal of the emulated shell and target to a +# waveform database: 797 MB for a 22k-cycle demo, and the dominant cost of the +# run. Set debug_mode=batch to get the waveform back when it is wanted. +[Emulation] +debug_mode=off diff --git a/sw/runtime/foldsim/Makefile b/sw/runtime/foldsim/Makefile new file mode 100644 index 0000000000..72ac7e7a0d --- /dev/null +++ b/sw/runtime/foldsim/Makefile @@ -0,0 +1,71 @@ +include ../common.mk + +# TARGET=hw runs a foldsim image (hw/syn/foldsim) on the V80 through VRT; +# TARGET=avedsim simulates the same kernel in-process (sim/foldsim). +ifeq ($(strip $(TARGET)),) +override TARGET := avedsim +endif + +DESTDIR ?= $(CURDIR)/.. + +CONFIG_FILE := $(DESTDIR)/foldsim_rt_config.stamp +SRC_DIR := $(VORTEX_HOME)/sw/runtime/foldsim + +CXXFLAGS += -std=c++17 -Wall -Wextra -Wfatal-errors -Werror +CXXFLAGS += -I$(INC_DIR) -I$(ROOT_DIR)/sw -I$(ROOT_DIR)/hw -I$(SIM_COMMON_DIR) -I$(SW_COMMON_DIR) -I$(RT_COMMON_DIR) +# The shell's register map and in-process host, from the Foldsim installation. +CXXFLAGS += -I$(FOLDSIM_SHELL)/include +CXXFLAGS += -DVX_CFG_XLEN=$(XLEN) -DVX_CFG_XLEN_$(XLEN) +CXXFLAGS += -fPIC +CXXFLAGS += $(CONFIGS) + +LDFLAGS += -shared -pthread -Wl,-rpath,'$$ORIGIN' + +SRCS := $(SRC_DIR)/vortex.cpp $(RT_COMMON_DIR)/utils.cpp $(SIM_COMMON_DIR)/cmd_processor.cpp + +ifeq ($(TARGET), avedsim) +CXXFLAGS += -DFOLDSIM_EMUL +LDFLAGS += -L$(DESTDIR) -lfoldsim_emul +EMUL := $(DESTDIR)/libfoldsim_emul.so +else ifeq ($(TARGET), hw) +# VRT's public headers include and from +# versioned subdirectories. +CXXFLAGS += $(if $(VRT_HOME),-I$(VRT_HOME)/include) $(shell pkg-config --cflags jsoncpp libxml-2.0 2>/dev/null) +LDFLAGS += $(if $(VRT_HOME),-L$(VRT_HOME)/lib) -lvrt -lvrtdpp +else +$(error TARGET must be hw or avedsim (got '$(TARGET)')) +endif + +ifdef DEBUG + CXXFLAGS += -g -O0 +else + CXXFLAGS += -O2 -DNDEBUG +endif + +PROJECT := libvortex-foldsim.so + +.PHONY: all force driver clean-driver clean-runtime clean + +all: $(DESTDIR)/$(PROJECT) + +driver: $(EMUL) + +$(DESTDIR)/libfoldsim_emul.so: force + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/foldsim lib CONFIGS='$(CONFIGS)' $(if $(PERF),PERF=$(PERF)) + +$(DESTDIR)/$(PROJECT): $(SRCS) $(EMUL) $(CONFIG_FILE) + $(CXX) $(CXXFLAGS) $(SRCS) $(LDFLAGS) -o $@ + +$(CONFIG_FILE): force + @mkdir -p $(@D) + @TMP=$@.tmp.$$$$ ; \ + printf '%s\n' "$(CXXFLAGS)" "$(TARGET)" > $$TMP ; \ + if ! cmp -s $$TMP $@ 2>/dev/null; then mv -f $$TMP $@ ; else rm -f $$TMP ; fi + +clean-driver: + DESTDIR=$(DESTDIR) $(MAKE) -C $(ROOT_DIR)/sim/foldsim clean + +clean-runtime: + rm -f $(DESTDIR)/$(PROJECT) + +clean: clean-driver clean-runtime diff --git a/sw/runtime/foldsim/vortex.cpp b/sw/runtime/foldsim/vortex.cpp new file mode 100644 index 0000000000..eec4a8d5f2 --- /dev/null +++ b/sw/runtime/foldsim/vortex.cpp @@ -0,0 +1,815 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// ============================================================================ +// foldsim backend — a pure transport HAL (see callbacks.h): the target +// carries no Command Processor, so the regfile surface comes from the +// functional CommandProcessor model, and its hooks reach the foldsim +// simulator. Device memory is the simulator's, in device memory at +// FOLDSIM_REG_MEM_BASE; the target's DCR, start and busy are its command queue +// and status (Foldsim's shell, include/foldsim_regs.h). +// +// TARGET=hw runs the image on a V80 through VRT: registers over AXI4-Lite, +// device memory over a QDMA queue. TARGET=avedsim simulates the same kernel +// in-process (sim/foldsim). +// ============================================================================ + +#include +#include + +#include +#include +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef FOLDSIM_EMUL +#include +#else +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#endif + +using namespace vortex; + +namespace { + +// Where target address 0 is in device memory: the start of HBM, which the +// kernel's memory port reaches through the MEM tag (hw/syn/xilinx/platforms/ +// v80.toml). The target's 4 GB lie in HBM0..HBM7, which nothing else uses. +constexpr uint64_t kMemBase = 0x4000000000ull; + +#ifdef FOLDSIM_EMUL + +class transport_t { +public: + // The kernel advances while it is accessed. + static constexpr bool kRunsOnItsOwn = false; + int init() { return sim_.init(); } + int reg_write(uint32_t off, uint32_t v) { return sim_.reg_write(off, v); } + int reg_read(uint32_t off, uint32_t* v) { return sim_.reg_read(off, v); } + int mem_write(uint64_t addr, const void* src, uint64_t n) { + return sim_.mem_write(kMemBase + addr, src, n); + } + int mem_read(uint64_t addr, void* dst, uint64_t n) { + return sim_.mem_read(kMemBase + addr, dst, n); + } + +private: + foldsim::foldsim_emul sim_; +}; + +#else + +#define KERNEL_NAME "foldsim_afu_0" + +// VRT throws; nothing may cross the callbacks' C boundary. +#define VRT_TRY() try { +#define VRT_CATCH(_ret) \ + } \ + catch (const std::exception& _e) { \ + fprintf(stderr, "[VXDRV] VRT exception: %s\n", _e.what()); \ + return _ret; \ + } + +class transport_t { +public: + // The kernel runs on the card whether or not the host looks. + static constexpr bool kRunsOnItsOwn = true; + int init() { + const char* vbin = getenv("VRT_VBIN_PATH"); + if (vbin == nullptr || vbin[0] == '\0') { + fprintf(stderr, "[VXDRV] VRT_VBIN_PATH must name the foldsim .vbin\n"); + return -1; + } + const char* bdf_env = getenv("VRT_DEVICE_BDF"); + std::string bdf = (bdf_env != nullptr) ? bdf_env : ""; + if (bdf.empty()) { + bdf = discover_v80_bdf(); + if (bdf.empty()) { + fprintf(stderr, "[VXDRV] no V80 found on the bus; set VRT_DEVICE_BDF\n"); + return -1; + } + } + VRT_TRY() + device_ = std::make_unique(bdf, vbin, /*program=*/true); + // Clock the kernel at the rate the image was timed for, or lower if + // VORTEX_FOLDSIM_CLOCK_HZ locks it there. + const uint64_t timed_hz = device_->getMaxFrequency(); + uint64_t want_hz = timed_hz; + if (const char* s = getenv("VORTEX_FOLDSIM_CLOCK_HZ")) { + const uint64_t hz = strtoull(s, nullptr, 0); + if (hz == 0 || hz > timed_hz) { + fprintf(stderr, "[VXDRV] VORTEX_FOLDSIM_CLOCK_HZ=%s: must be in (0, %lu]\n", + s, (unsigned long)timed_hz); + park_clock(); + return -1; + } + want_hz = hz; + } + // The wizard settles on the nearest rate its dividers reach, which can + // be a step above the request: lower the request below that step. + const uint64_t limit_hz = want_hz; + uint64_t clk_hz = 0; + for (int i = 0; i < 8; ++i) { + device_->setFrequency(want_hz); + clk_hz = device_->getFrequency(); + if (clk_hz == 0 || clk_hz <= limit_hz) + break; + want_hz -= (clk_hz - want_hz) + limit_hz / 100; + } + if (clk_hz == 0 || clk_hz > limit_hz) { + fprintf(stderr, "[VXDRV] user clock reads %lu Hz, image is timed for %lu Hz\n", + (unsigned long)clk_hz, (unsigned long)timed_hz); + park_clock(); + return -1; + } + printf("[VXDRV] foldsim kernel clock %lu Hz\n", (unsigned long)clk_hz); + kernel_ = std::make_unique(*device_, KERNEL_NAME); + // A QDMA queue for device memory: the buffer's own memory is the + // smallest allocation, outside the target's 4 GB, and only its queue is + // used, at the target's addresses. + session_ = std::make_unique(); + auto vdev = session_->getDeviceByBdf(bdf); + dma_ = std::make_unique(vdev.openHbmBuffer(/*region=*/8, kDmaChunk)); + fd_ = dma_->getFd(); + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] VRT exception: %s\n", e.what()); + park_clock(); + return -1; + } + // The host side of the transfers: a kernel DMA buffer, mapped. + struct slash_qdma_buf_create create = {}; + create.size = sizeof(create); + create.flags = O_CLOEXEC; + create.length = kStaging; + staging_fd_ = ioctl(fd_, SLASH_QDMA_IOCTL_BUF_CREATE, &create); + if (staging_fd_ < 0) { + fprintf(stderr, "[VXDRV] cannot create a QDMA buffer: %s\n", strerror(errno)); + return -1; + } + void* p = mmap(nullptr, kStaging, PROT_READ | PROT_WRITE, MAP_SHARED, staging_fd_, 0); + if (p == MAP_FAILED) { + fprintf(stderr, "[VXDRV] cannot map the QDMA buffer: %s\n", strerror(errno)); + return -1; + } + staging_ = static_cast(p); + return 0; + } + + // A design that fails to come up stays loaded: leave it clocked below any + // image's timing so it cannot disturb the shell before the next load. + void park_clock() noexcept { + constexpr uint64_t kSafeHz = 50000000; + if (!device_) + return; + try { + device_->setFrequency(kSafeHz); + fprintf(stderr, "[VXDRV] kernel clock parked at %lu Hz\n", + (unsigned long)device_->getFrequency()); + } catch (...) { + } + } + + ~transport_t() { + if (staging_ != nullptr) + munmap(staging_, kStaging); + if (staging_fd_ >= 0) + close(staging_fd_); + } + + int reg_write(uint32_t off, uint32_t v) { + VRT_TRY() + kernel_->write(off, v); + VRT_CATCH(-1) + return 0; + } + + int reg_read(uint32_t off, uint32_t* v) { + VRT_TRY() + *v = kernel_->read(off); + VRT_CATCH(-1) + return 0; + } + + // QDMA moves any byte range: only the offset into the staging buffer must + // be page aligned, not the device address or the length. A write is the + // bytes it writes, with no read of the pages around them. + int mem_write(uint64_t addr, const void* src, uint64_t n) { + std::lock_guard lock(mutex_); + auto* bytes = static_cast(src); + while (n != 0) { + const uint64_t span = std::min(kDmaChunk, n); + std::memcpy(staging_, bytes, span); + if (dma(addr, staging_, span, true) != 0) + return -1; + addr += span; + bytes += span; + n -= span; + } + return 0; + } + + int mem_read(uint64_t addr, void* dst, uint64_t n) { + std::lock_guard lock(mutex_); + auto* bytes = static_cast(dst); + while (n != 0) { + const uint64_t span = std::min(kDmaChunk, n); + if (dma(addr, staging_, span, false) != 0) + return -1; + std::memcpy(bytes, staging_, span); + addr += span; + bytes += span; + n -= span; + } + return 0; + } + +private: + static constexpr uint64_t kDmaChunk = 64ull << 20; + static constexpr uint64_t kStaging = kDmaChunk; + + // `len` bytes at target address `addr`, from or to `host` in the staging + // buffer, page aligned. + int dma(uint64_t addr, uint8_t* host, uint64_t len, bool to_device) { + struct slash_qdma_transfer xfer = {}; + xfer.size = sizeof(xfer); + xfer.count = 1; + xfer.xfers[0].qpair_index = 0; + xfer.xfers[0].direction = to_device ? SLASH_QDMA_XFER_H2C : SLASH_QDMA_XFER_C2H; + xfer.xfers[0].buf_fd = staging_fd_; + xfer.xfers[0].buf_offset = static_cast(host - staging_); + xfer.xfers[0].dev_addr = kMemBase + addr; + xfer.xfers[0].length = len; + const int n = ioctl(fd_, SLASH_QDMA_QPAIR_IOCTL_TRANSFER, &xfer); + if (n < 0 || static_cast(n) != len) { + fprintf(stderr, "[VXDRV] QDMA %s of %lu bytes at 0x%lx failed: %s\n", + to_device ? "write" : "read", (unsigned long)len, + (unsigned long)(kMemBase + addr), n < 0 ? strerror(errno) : "short"); + return -1; + } + return 0; + } + + std::unique_ptr device_; + std::unique_ptr kernel_; + std::unique_ptr session_; + std::unique_ptr dma_; + int fd_ = -1; + int staging_fd_ = -1; + uint8_t* staging_ = nullptr; + std::mutex mutex_; +}; + +#endif + +// What the host spent on the transport: register accesses, and device-memory +// transfers, with their wall time. +struct transport_stats_t { + uint64_t reads = 0, writes = 0, dma_bytes = 0; + double reg_s = 0, dma_s = 0; +}; + +class timer_t { +public: + explicit timer_t(double& total) : total_(total), start_(std::chrono::steady_clock::now()) {} + ~timer_t() { + total_ += std::chrono::duration(std::chrono::steady_clock::now() - start_).count(); + } + +private: + double& total_; + std::chrono::steady_clock::time_point start_; +}; + +// The transport, counted. +class counted_transport_t : public transport_t { +public: + int reg_write(uint32_t off, uint32_t v) { + timer_t t(stats.reg_s); + ++stats.writes; + return transport_t::reg_write(off, v); + } + int reg_read(uint32_t off, uint32_t* v) { + timer_t t(stats.reg_s); + ++stats.reads; + return transport_t::reg_read(off, v); + } + int mem_write(uint64_t addr, const void* src, uint64_t n) { + timer_t t(stats.dma_s); + stats.dma_bytes += n; + return transport_t::mem_write(addr, src, n); + } + int mem_read(uint64_t addr, void* dst, uint64_t n) { + timer_t t(stats.dma_s); + stats.dma_bytes += n; + return transport_t::mem_read(addr, dst, n); + } + transport_stats_t stats; +}; + +} // namespace + +class vx_device { +public: + vx_device() : cp_(make_cp_hooks()) {} + + ~vx_device() { + if (open_) + report(); + for (auto& kv : host_regions_) + free(reinterpret_cast(kv.first)); + host_regions_.clear(); + } + + int init() { + if (transport_.init() != 0) + return -1; + uint32_t id = 0, version = 0; + if (transport_.reg_read(FOLDSIM_REG_ID, &id) != 0 || id != FOLDSIM_ID) { + fprintf(stderr, "[VXDRV] the kernel is not a foldsim simulator (id 0x%08x)\n", id); + return -1; + } + if (transport_.reg_read(FOLDSIM_REG_VERSION, &version) != 0 || version != FOLDSIM_VERSION) { + fprintf(stderr, "[VXDRV] the kernel's register map is version %u, this runtime's %u\n", + version, FOLDSIM_VERSION); + return -1; + } + // The memory model: the platform memory's DRAM, as the bank model times + // it. VORTEX_DRAM_REFRESH=off|bank|all: its refresh, as the + // simulators' bank model takes it. + const char* refresh = getenv("VORTEX_DRAM_REFRESH"); + const std::string refresh_name = (refresh != nullptr) ? refresh : ""; + if (refresh_name == "bank") { + refresh_ = FOLDSIM_MEM_REFRESH_BANK; + } else if (refresh_name == "all") { + refresh_ = FOLDSIM_MEM_REFRESH_ALL; + } else if (!refresh_name.empty() && refresh_name != "off") { + fprintf(stderr, "[VXDRV] VORTEX_DRAM_REFRESH must be off, bank or all (got '%s')\n", refresh); + return -1; + } + if (this->setup_bank_model() != 0) + return -1; + if (reg_write(FOLDSIM_REG_MEM_BASE_LO, uint32_t(kMemBase)) != 0 || + reg_write(FOLDSIM_REG_MEM_BASE_HI, uint32_t(kMemBase >> 32)) != 0 || + reg_write(FOLDSIM_REG_CTRL, 1) != 0) + return -1; + // A target left running by an earlier process is reset, as a card's + // native driver resets its device on open: Vortex counts VX_CFG_RESET_DELAY + // cycles of reset before it takes a request. The reset also clears a + // memory error that process left, so only the state after it counts, and + // starts the memory model with its settings and refresh. + uint32_t s = 0; + if (command(FOLDSIM_CMD_RESET, refresh_, 64) != 0 || wait_idle(false) != 0 || status(&s) != 0) + return -1; + if (mem_error(s)) + return -1; + open_ = true; + return 0; + } + + // ----- CP register channel ----- + // No hardware CP: the functional model answers, ticked around each access. + // The model's hooks return no status: a device error they hit fails every + // access from then on, so the command that hit it does not complete as a + // success. + int cp_reg_write(uint32_t off, uint32_t value) { + cp_.mmio_write(off, value); + for (int i = 0; i < 256 && cp_.busy(); ++i) + cp_.tick(); + return failed_ ? -1 : 0; + } + + int cp_reg_read(uint32_t off, uint32_t* value) { + for (int i = 0; i < 256 && cp_.busy(); ++i) + cp_.tick(); + *value = cp_.mmio_read(off); + return failed_ ? -1 : 0; + } + + // ----- CP-visible host memory ----- + // The CP model reads host memory directly; only device memory crosses the + // transport. + int host_mem_alloc(uint64_t size, void** host_ptr, uint64_t* cp_addr) { + uint64_t asize = aligned_size(size, CACHE_BLOCK_SIZE); + void* ptr = aligned_alloc(CACHE_BLOCK_SIZE, asize); + if (ptr == nullptr) + return -1; + std::lock_guard g(host_mu_); + host_regions_[reinterpret_cast(ptr)] = asize; + *host_ptr = ptr; + *cp_addr = reinterpret_cast(ptr); + return 0; + } + + int host_mem_free(uint64_t cp_addr) { + { + std::lock_guard g(host_mu_); + auto it = host_regions_.find(cp_addr); + if (it == host_regions_.end()) + return -1; + host_regions_.erase(it); + } + free(reinterpret_cast(cp_addr)); + return 0; + } + + int host_mem_pull(uint64_t /*cp_addr*/) { return 0; } + int host_mem_push(uint64_t /*cp_addr*/) { return 0; } + +private: + int reg_write(uint32_t off, uint32_t v) { return transport_.reg_write(off, v); } + +#define FOLDSIM_STR_(x) #x +#define FOLDSIM_STR(x) FOLDSIM_STR_(x) + + // The bank model's settings: the platform memory this runtime was built for + // (VX_CFG_PLATFORM_MEMORY_*), its data-sheet timings in DRAM clocks as + // dram_sim derives them, and dram_sim's controller queues and write + // watermarks; every port's channel takes the same. They take effect at the + // target reset that follows. The image fixes the structure: a channel per + // memory port of the target it was built for. + int setup_bank_model() { + uint32_t caps = 0; + if (transport_.reg_read(FOLDSIM_REG_MEM_CAPS, &caps) != 0) + return -1; + mem_ports_ = FOLDSIM_MEM_CAPS_PORTS(caps); + if (mem_ports_ == 0) { + fprintf(stderr, "[VXDRV] this foldsim image reports no memory ports\n"); + return -1; + } + dram_timing::Standard standard; + try { + standard = dram_timing::parse_standard(FOLDSIM_STR(VX_CFG_PLATFORM_MEMORY_TYPE)); + } catch (const std::exception& e) { + fprintf(stderr, "[VXDRV] %s\n", e.what()); + return -1; + } + const uint32_t rate = VX_CFG_PLATFORM_MEMORY_DATA_RATE; + const uint32_t clock = VX_CFG_PLATFORM_CLOCK_RATE; + const auto t = dram_timing::clocks(standard, rate); + const auto g = dram_timing::geometry(standard); + const uint32_t v[FOLDSIM_MEM_NUM_TIMINGS] = { + t.BL, t.CCDS, t.CCDL, t.CL, t.CWL, t.RCDRD, t.RCDWR, t.RP, t.RAS, t.RC, + t.RTP, t.WR, t.RRDS, t.RRDL, t.FAW, t.WTRS, t.WTRL, g.rtw_extra}; + const uint32_t time_max = (1u << FOLDSIM_MEM_CAPS_TIME_W(caps)) - 1; + for (uint32_t x : v) { + if (x > time_max) { + fprintf(stderr, "[VXDRV] a DRAM timing of %u clocks exceeds the image's %u\n", x, time_max); + return -1; + } + } + // The image places ACTs under tRRD and the per-bank timings; tFAW it + // leaves to tRRD, which covers it wherever four tRRD_S span a tFAW. + if (4 * t.RRDS < t.FAW) { + fprintf(stderr, "[VXDRV] the bank model of this image does not model tFAW, which %s at " + "%u MT/s needs (tRRD_S %u, tFAW %u clocks)\n", + FOLDSIM_STR(VX_CFG_PLATFORM_MEMORY_TYPE), rate, t.RRDS, t.FAW); + return -1; + } + if (rate > 0xffff || clock > 0xffff || VX_CFG_PLATFORM_MEMORY_LATENCY > 0xffff + || VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE > 0xffff) { + fprintf(stderr, "[VXDRV] the platform memory's settings exceed the bank model's registers\n"); + return -1; + } + const uint32_t geom = g.log2_pcs | (g.log2_bgs << 2) | (g.log2_banks << 4) + | (g.log2_row_bursts << 6) | (g.log2_visit << 10) + | (uint32_t(g.two_clock_act) << 12); + const uint32_t queue = VX_CFG_PLATFORM_MEMORY_QUEUE_SIZE | (32u << 16) | (32u << 24); + const uint32_t wmark = 26u | (7u << 8); + if (reg_write(FOLDSIM_REG_MEM_CLOCK, rate | (clock << 16)) != 0 || + reg_write(FOLDSIM_REG_MEM_LATENCY, VX_CFG_PLATFORM_MEMORY_LATENCY) != 0 || + reg_write(FOLDSIM_REG_MEM_QUEUE, queue) != 0 || + reg_write(FOLDSIM_REG_MEM_GEOM, geom) != 0 || + reg_write(FOLDSIM_REG_MEM_WMARK, wmark) != 0) + return -1; + for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_TIMINGS; ++i) { + if (reg_write(FOLDSIM_REG_MEM_TIMING(i), v[i]) != 0) + return -1; + } + // Refresh as the bank model takes it (DramBankModel::Config::set_refresh). + if (refresh_ != 0) { + const auto r = dram_timing::refresh(standard, rate); + const uint32_t banks = 1u << (g.log2_pcs + g.log2_bgs + g.log2_banks); + const bool all = (refresh_ == FOLDSIM_MEM_REFRESH_ALL); + const uint32_t refi = all ? r.REFI : r.REFI / banks; + const uint32_t rfc = all ? r.RFC : r.RFCPB; + if (refi > 0xffff || rfc > 0xffff) { + fprintf(stderr, "[VXDRV] the refresh timings exceed the bank model's registers\n"); + return -1; + } + if (reg_write(FOLDSIM_REG_MEM_REFI, refi) != 0 || reg_write(FOLDSIM_REG_MEM_RFC, rfc) != 0) + return -1; + // A bank refreshing beyond the reservations' reach holds every request + // behind one to it at the channel's ingress: refresh then costs more + // than it would with the reach to place them. + const uint32_t horizon = FOLDSIM_MEM_CAPS_HORIZON(caps) - 64; + if (rfc >= horizon) { + fprintf(stderr, "[VXDRV] warning: a refresh keeps its bank %u DRAM clocks, past this image's " + "%u-clock horizon: refresh costs more here than it would in a channel that could " + "place the requests behind it\n", rfc, horizon); + } + } + return 0; + } + + uint64_t read64(uint32_t lo) { + uint32_t l = 0, h = 0, h2 = 0; + do { + transport_.reg_read(lo + 4, &h); + transport_.reg_read(lo, &l); + transport_.reg_read(lo + 4, &h2); + } while (h != h2); + return (uint64_t(h) << 32) | l; + } + + // How fast the simulator ran since the kernel's reset: target cycles, the + // kernel clock's cycles, and those a target cycle waited for device memory. + void report() { + const uint64_t target = read64(FOLDSIM_REG_TCYCLES_LO); + const uint64_t host = read64(FOLDSIM_REG_HCYCLES_LO); + const uint64_t stall = read64(FOLDSIM_REG_MSTALL_LO); + fprintf(stderr, + "[foldsim] target cycles=%lu host cycles=%lu host/target=%.2f memory waits=%lu\n", + (unsigned long)target, (unsigned long)host, + target ? double(host) / double(target) : 0.0, (unsigned long)stall); + // Each class finishes each of its threads once a target cycle, so its + // finishes per host cycle is how busy its one physical copy is. + uint32_t classes = 0; + transport_.reg_read(FOLDSIM_REG_NUM_CLASSES, &classes); + std::string line; + for (uint32_t c = 0; c < classes && host != 0; ++c) { + const uint64_t finishes = read64(FOLDSIM_REG_FINISH(c)); + char buf[64]; + snprintf(buf, sizeof(buf), " %u:%.1f%%", c, 100.0 * double(finishes) / double(host)); + line += buf; + } + if (!line.empty()) + fprintf(stderr, "[foldsim] class busy:%s\n", line.c_str()); + // The bank model's counters, per channel and, with several, in all. + { + uint64_t total[FOLDSIM_MEM_NUM_STATS] = {}; + for (uint32_t p = 0; p <= mem_ports_; ++p) { + uint64_t st[FOLDSIM_MEM_NUM_STATS]; + const bool all = (p == mem_ports_); + if (all && mem_ports_ == 1) + break; + for (uint32_t i = 0; i < FOLDSIM_MEM_NUM_STATS; ++i) { + st[i] = all ? total[i] : read64(FOLDSIM_REG_MEM_STATS(p, i)); + total[i] += all ? 0 : st[i]; + } + char who[32] = "bank model"; + if (!all && mem_ports_ > 1) + snprintf(who, sizeof(who), "bank model, channel %u", p); + const uint64_t reads = st[FOLDSIM_MEM_STAT_READS]; + fprintf(stderr, + "[foldsim] %s: %lu reads, %lu writes; row hits %lu, misses %lu, conflicts %lu; " + "mean read latency %.1f cycles; %lu refreshes; %lu cycles full\n", + who, (unsigned long)reads, (unsigned long)st[FOLDSIM_MEM_STAT_WRITES], + (unsigned long)st[FOLDSIM_MEM_STAT_HITS], (unsigned long)st[FOLDSIM_MEM_STAT_MISSES], + (unsigned long)st[FOLDSIM_MEM_STAT_CONFLICTS], + reads ? double(st[FOLDSIM_MEM_STAT_LATENCY]) / double(reads) : 0.0, + (unsigned long)st[FOLDSIM_MEM_STAT_REFRESHES], (unsigned long)st[FOLDSIM_MEM_STAT_FULL]); + } + } + const auto& st = transport_.stats; + fprintf(stderr, + "[foldsim] host: %lu register reads, %lu writes in %.3f s; %lu bytes of device memory in %.3f s\n", + (unsigned long)st.reads, (unsigned long)st.writes, st.reg_s, + (unsigned long)st.dma_bytes, st.dma_s); + } + + // Latches a device error: reported once, and every later access fails. + int fail(const char* what) { + if (!failed_) { + fprintf(stderr, "[VXDRV] %s\n", what); + failed_ = true; + } + return -1; + } + + int status(uint32_t* value) { + if (transport_.reg_read(FOLDSIM_REG_STATUS, value) != 0) + return fail("cannot read the simulator's status"); + return 0; + } + + // Polls the status until `done` holds of it. A memory error fails the wait + // (unless `error_fails` is false), and so does a simulator whose target + // cycles stop advancing for the control timeout. Progress, not wall time, + // is the measure: in-process, a large target may legitimately take minutes + // of wall time to answer a DCR read that flushes its caches. + template + int poll(Done done, bool error_fails, const char* what) { + auto last_progress = std::chrono::steady_clock::now(); + uint32_t last_tcycles = 0; + if (transport_.reg_read(FOLDSIM_REG_TCYCLES_LO, &last_tcycles) != 0) + return fail("cannot read the simulator's target cycles"); + uint32_t s = 0; + for (uint64_t n = 0;; ++n) { + if (failed_ || status(&s) != 0 || (error_fails && mem_error(s))) + return -1; + if (done(s)) + break; + // Checked only every so often: in-process, a status read is the cheap + // part of a poll. + if ((n & 0xff) == 0xff) { + uint32_t tcycles = 0; + if (transport_.reg_read(FOLDSIM_REG_TCYCLES_LO, &tcycles) != 0) + return fail("cannot read the simulator's target cycles"); + if (tcycles != last_tcycles) { + last_tcycles = tcycles; + last_progress = std::chrono::steady_clock::now(); + } else if (elapsed_s(last_progress) > control_timeout_s_) { + char msg[192]; + snprintf(msg, sizeof(msg), "the target has not %s, and has not advanced a cycle in " + "%.0f s (status 0x%x); the simulation is stopped", what, control_timeout_s_, s); + return fail(msg); + } + } + } + return 0; + } + + // Queues a command once the queue has room. + int command(uint32_t op, uint32_t addr, uint32_t data) { + if (poll([](uint32_t s) { return !(s & FOLDSIM_STATUS_FULL); }, true, + "taken a command") != 0) + return -1; + if (reg_write(FOLDSIM_REG_CMD_ADDR, addr) != 0 || reg_write(FOLDSIM_REG_CMD_DATA, data) != 0 || + reg_write(FOLDSIM_REG_CMD, op) != 0) + return fail("cannot queue a command"); + return 0; + } + + int wait_idle(bool error_fails) { + return poll([](uint32_t s) { return !(s & FOLDSIM_STATUS_BUSY); }, error_fails, + "gone idle"); + } + + // The memory endpoint takes single-beat transactions; a target that makes a + // burst stops the simulation there, busy for good. + bool mem_error(uint32_t s) { + if (!(s & FOLDSIM_STATUS_MEM_ERR)) + return false; + fail("the target made a memory burst, which the memory model does not take; " + "the simulation is stopped"); + return true; + } + + static double elapsed_s(std::chrono::steady_clock::time_point since) { + return std::chrono::duration(std::chrono::steady_clock::now() - since).count(); + } + + // The target runs on its own: once it has been busy for a while, polling it + // back to back only spins a core, so the poll sleeps, longer the longer the + // target stays busy (up to kMaxPollSleepUs). + void poll_backoff(bool busy) { + if (!transport_t::kRunsOnItsOwn) + return; + if (!busy) { + busy_polls_ = 0; + return; + } + if (++busy_polls_ <= kSpinPolls) + return; + const uint64_t us = std::min(1ull << std::min(busy_polls_ - kSpinPolls, 7), + kMaxPollSleepUs); + std::this_thread::sleep_for(std::chrono::microseconds(us)); + } + + void* host_region_ptr(uint64_t addr) { + std::lock_guard g(host_mu_); + if (host_regions_.empty()) + return nullptr; + auto it = host_regions_.upper_bound(addr); + if (it == host_regions_.begin()) + return nullptr; + --it; + if (addr >= it->first && addr < it->first + it->second) + return reinterpret_cast(addr); + return nullptr; + } + + vortex::CommandProcessor::Hooks make_cp_hooks() { + vortex::CommandProcessor::Hooks h; + h.dram_read = [this](uint64_t addr, void* dst, std::size_t bytes) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(dst, hp, bytes); + return; + } + if (failed_ || transport_.mem_read(addr, dst, bytes) != 0) { + fail("cannot read device memory"); + std::memset(dst, 0, bytes); + } + }; + h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (failed_ || transport_.mem_write(addr, src, bytes) != 0) + fail("cannot write device memory"); + }; + h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + if (!failed_) + command(FOLDSIM_CMD_DCR_WRITE, addr, value); + }; + h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { + uint32_t value = 0; + if (failed_ || command(FOLDSIM_CMD_DCR_READ, addr, tag) != 0 || + poll([](uint32_t s) { return (s & FOLDSIM_STATUS_RSP) != 0; }, true, + "answered a DCR read") != 0) + return 0; + if (transport_.reg_read(FOLDSIM_REG_DCR_RSP, &value) != 0) { + fail("cannot read a DCR response"); + return 0; + } + return value; + }; + h.vortex_start = [this]() { + if (!failed_ && command(FOLDSIM_CMD_START, 0, 0) == 0) + launch_start_ = std::chrono::steady_clock::now(); + }; + // A failed device reads idle, so the launch ends; the access after it + // reports the failure. + h.vortex_busy = [this]() -> bool { + uint32_t s = 0; + if (failed_ || status(&s) != 0 || mem_error(s)) + return false; + const bool busy = (s & FOLDSIM_STATUS_BUSY) != 0; + if (busy && launch_timeout_s_ > 0 && elapsed_s(launch_start_) > launch_timeout_s_) { + char msg[128]; + snprintf(msg, sizeof(msg), "launch exceeded VORTEX_FOLDSIM_TIMEOUT_S=%.0f s; " + "the simulation is stopped", launch_timeout_s_); + fail(msg); + return false; + } + poll_backoff(busy); + return busy; + }; +#ifdef VX_CFG_VM_ENABLE + h.mmu_fault_report = true; +#endif + return h; + } + + // Polls back to back before the first sleep: about a millisecond at a PCIe + // round trip each. + static constexpr uint64_t kSpinPolls = 1024; + static constexpr uint64_t kMaxPollSleepUs = 100; + + // Wall-clock seconds a launch may run before the device is failed; 0, the + // default, never. A command, DCR read or reset the target leaves unanswered + // while it advances no cycle for that long fails it too, after 60 s when it + // is unset. + static double timeout_env() { + const char* s = getenv("VORTEX_FOLDSIM_TIMEOUT_S"); + return (s != nullptr && s[0] != '\0') ? atof(s) : 0.0; + } + static constexpr double kControlTimeoutS = 60; + + counted_transport_t transport_; + uint64_t busy_polls_ = 0; + const double launch_timeout_s_ = timeout_env(); + const double control_timeout_s_ = (launch_timeout_s_ > 0) ? launch_timeout_s_ : kControlTimeoutS; + std::chrono::steady_clock::time_point launch_start_; + bool failed_ = false; + bool open_ = false; + uint32_t refresh_ = 0; + uint32_t mem_ports_ = 1; + vortex::CommandProcessor cp_; + std::mutex host_mu_; + std::map host_regions_; +}; + +#include diff --git a/sw/runtime/include/dxa.h b/sw/runtime/include/dxa.h index 6f0a3f4262..ee77ba8a0d 100644 --- a/sw/runtime/include/dxa.h +++ b/sw/runtime/include/dxa.h @@ -93,9 +93,9 @@ enum class Layout : uint32_t { // 5D: 21 DCR writes // // `base_addr` is written to VX_DCR_DXA_DESC_BASE_{LO,HI} and is -// consumed by the DXA worker's AXI master, which bypasses the -// per-core MMU. The backing buffer must be allocated with -// VX_MEM_PHYS (see vortex2.h). +// consumed by the DXA worker's memory port, which the socket unit +// MMU translates under VM: it is a device virtual address like any +// buffer address (vx_buffer_address). // Program a 1D DXA descriptor (7 DCR writes). // size0: dimension size (elements) diff --git a/sw/runtime/rtlsim/vortex.cpp b/sw/runtime/rtlsim/vortex.cpp index e0677b66c8..7f32b98eb9 100644 --- a/sw/runtime/rtlsim/vortex.cpp +++ b/sw/runtime/rtlsim/vortex.cpp @@ -143,6 +143,13 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { @@ -150,9 +157,40 @@ class vx_device { std::memcpy(hp, src, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); + return; + } ram_.write(static_cast(src), addr, bytes); }; + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + } h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish so dcr_write + // does not race the Verilator state. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { @@ -170,6 +208,10 @@ class vx_device { if (!future_.valid()) return false; return future_.wait_for(std::chrono::seconds(0)) != std::future_status::ready; }; + h.vortex_drain = [this]() { + if (future_.valid()) future_.wait(); + processor_.drain(); + }; #ifdef VX_CFG_VM_ENABLE // The device MMU control surface answers the fault-report DCRs. h.mmu_fault_report = true; diff --git a/sw/runtime/simx/vortex.cpp b/sw/runtime/simx/vortex.cpp index 3d3088850f..b2120aaf0b 100644 --- a/sw/runtime/simx/vortex.cpp +++ b/sw/runtime/simx/vortex.cpp @@ -33,6 +33,7 @@ #include #include +#include #include #include #include @@ -150,6 +151,13 @@ class vx_device { std::memcpy(dst, hp, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_read(addr, dst, bytes); + return; + } ram_.read(static_cast(dst), addr, bytes); }; h.dram_write = [this](uint64_t addr, const void* src, std::size_t bytes) { @@ -157,9 +165,39 @@ class vx_device { std::memcpy(hp, src, bytes); return; } + if (processor_.cp_llc_client()) { + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, false); + return; + } ram_.write(static_cast(src), addr, bytes); }; + if (processor_.cp_llc_client()) { + // The CP DMA is a client of the last-level cache: transfers stay + // coherent with it, and those of at least half its size write around it. + h.dma_write = [this](uint64_t addr, const void* src, std::size_t bytes, bool no_alloc) { + if (void* hp = host_region_ptr(addr)) { + std::memcpy(hp, src, bytes); + return; + } + if (future_.valid()) { + future_.wait(); + } + processor_.cp_mem_write(addr, src, bytes, no_alloc); + }; + h.dma_llc_client = true; + h.dma_no_alloc_min = processor_.cp_llc_size() / 2; + if (const char* v = std::getenv("VORTEX_CP_NO_ALLOC_MIN")) { + h.dma_no_alloc_min = std::strtoull(v, nullptr, 0); + } + } h.vortex_dcr_write = [this](uint32_t addr, uint32_t value) { + // Wait for any background processor_.run() to finish before writing DCR. + if (future_.valid()) { + future_.wait(); + } processor_.dcr_write(addr, value); }; h.vortex_dcr_read = [this](uint32_t addr, uint32_t tag) -> uint32_t { diff --git a/tests/graphics/gfx_draw3d/main.cpp b/tests/graphics/gfx_draw3d/main.cpp index 3d0f2841ea..fd61f639c8 100644 --- a/tests/graphics/gfx_draw3d/main.cpp +++ b/tests/graphics/gfx_draw3d/main.cpp @@ -1,15 +1,14 @@ // 3D rendering regression test: full TEX + RASTER + OM pipeline // against a CGLTrace replay. // -// Pinned-buffer contract: -// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) → pinned -// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) → pinned -// tex_buffer → TEX HW (VX_DCR_TEX_ADDR) → pinned -// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) → pinned -// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) → pinned -// All five buffers are accessed by fixed-function HW whose AXI master -// bypasses the per-core MMU, so under VM they must be -// identity-mapped (allocated with VX_MEM_PHYS). +// Buffers read or written by fixed-function HW: +// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) +// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) +// tex_buffer → TEX HW (VX_DCR_TEX_ADDR) +// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) +// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) +// Under VM their caches translate through the socket / cluster unit MMUs, +// so all five are ordinary buffers at minted virtual addresses. #include #include @@ -326,10 +325,10 @@ int render(const CGLTrace& trace) { if (tile_buffer != nullptr) { vx_buffer_release(tile_buffer); tile_buffer = nullptr; } if (prim_buffer != nullptr) { vx_buffer_release(prim_buffer); prim_buffer = nullptr; } // With the FF unit present, tile_buffer / prim_buffer are bound to the raster - // unit (via VX_DCR_RASTER_T/PBUF_ADDR) which bypasses the per-core MMU. - // Software-routed, the shader walks the primitives through the LSU instead + // unit (via VX_DCR_RASTER_T/PBUF_ADDR), whose rcache translates through the + // cluster unit MMU under VM. Software-routed, the shader walks the primitives through the LSU instead // and the tile buffer goes unused (the SW walk is binning-independent). - uint32_t rast_flags = VX_MEM_READ | (ff_raster ? (uint32_t)VX_MEM_PHYS : 0u); + uint32_t rast_flags = VX_MEM_READ; RT_CHECK(vx_buffer_create(device, tilebuf.size(), rast_flags, &tile_buffer)); RT_CHECK(vx_buffer_address(tile_buffer, &tilebuf_addr)); RT_CHECK(vx_buffer_create(device, primbuf.size(), rast_flags, &prim_buffer)); @@ -474,9 +473,10 @@ int render(const CGLTrace& trace) { // allocate texture memory if (tex_buffer != nullptr) { vx_buffer_release(tex_buffer); tex_buffer = nullptr; } - // With the FF unit present, tex_buffer is bound to it (VX_DCR_TEX_ADDR) and - // bypasses the MMU; software-routed, the shader loads it through the LSU. - uint32_t tex_flags = VX_MEM_READ | (ff_tex ? (uint32_t)VX_MEM_PHYS : 0u); + // With the FF unit present, tex_buffer is bound to it (VX_DCR_TEX_ADDR), + // whose tcache translates through the socket unit MMU under VM; + // software-routed, the shader loads it through the LSU. + uint32_t tex_flags = VX_MEM_READ; RT_CHECK(vx_buffer_create(device, texbuf.size(), tex_flags, &tex_buffer)); RT_CHECK(vx_buffer_address(tex_buffer, &texbuf_addr)); std::cout << "tex_buffer=0x" << std::hex << texbuf_addr << std::dec << std::endl; @@ -753,9 +753,10 @@ int main(int argc, char *argv[]) { cbuf_size = dst_width * cbuf_pitch; // With the FF unit present, depth_buffer / color_buffer are bound to the OM - // unit (via VX_DCR_OM_Z/CBUF_ADDR) and bypass the MMU; software-routed, the - // shader read-modify-writes them through the LSU. - uint32_t om_flags = VX_MEM_READ_WRITE | (ff_om ? (uint32_t)VX_MEM_PHYS : 0u); + // unit (via VX_DCR_OM_Z/CBUF_ADDR), whose ocache translates through the + // cluster unit MMU under VM; software-routed, the shader read-modify-writes + // them through the LSU. + uint32_t om_flags = VX_MEM_READ_WRITE; RT_CHECK(vx_buffer_create(device, zbuf_size, om_flags, &depth_buffer)); RT_CHECK(vx_buffer_address(depth_buffer, &zbuf_addr)); RT_CHECK(vx_buffer_create(device, cbuf_size, om_flags, &color_buffer)); diff --git a/tests/graphics/gfx_om/main.cpp b/tests/graphics/gfx_om/main.cpp index 0d5bbae377..11cc1a2ffe 100644 --- a/tests/graphics/gfx_om/main.cpp +++ b/tests/graphics/gfx_om/main.cpp @@ -1,10 +1,10 @@ // Output-merger regression test. // -// Pinned-buffer contract: -// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) → VX_MEM_PHYS (identity-mapped) -// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) → VX_MEM_PHYS (identity-mapped) -// Both are read+written by the OM fixed-function block, which bypasses -// the per-core MMU; under VM the buffers must be identity-mapped. +// Buffers: +// depth_buffer → OM HW (VX_DCR_OM_ZBUF_ADDR) +// color_buffer → OM HW (VX_DCR_OM_CBUF_ADDR) +// Both are read+written by the OM fixed-function block through the ocache, +// whose misses translate through the cluster unit MMU under VM. #include #include @@ -173,12 +173,9 @@ int main(int argc, char *argv[]) { cbuf_pitch = dst_width * 4; cbuf_size = dst_height * cbuf_pitch; - // HW path: depth_buffer / color_buffer are bound to the OM unit (via - // VX_DCR_OM_ZBUF_ADDR / VX_DCR_OM_CBUF_ADDR), which reads/writes them through - // the OM/ocache path bypassing the per-core MMU — both need physical (pinned) - // addresses. SW path (-S): the kernel does the merge through the LSU/dcache, - // so allocate them as normal kernel-accessed buffers. - uint32_t om_flags = VX_MEM_READ_WRITE | (sw_path ? 0u : (uint32_t)VX_MEM_PHYS); + // HW path: the OM unit reads/writes them through the ocache (translated by + // the cluster unit MMU); SW path (-S): the kernel merges through the LSU. + uint32_t om_flags = VX_MEM_READ_WRITE; RT_CHECK(vx_buffer_create(device, zbuf_size, om_flags, &depth_buffer)); RT_CHECK(vx_buffer_address(depth_buffer, &zbuf_addr)); RT_CHECK(vx_buffer_create(device, cbuf_size, om_flags, &color_buffer)); diff --git a/tests/graphics/gfx_pipeline_om/main.cpp b/tests/graphics/gfx_pipeline_om/main.cpp index 6aa21eab7c..0300fe12e8 100644 --- a/tests/graphics/gfx_pipeline_om/main.cpp +++ b/tests/graphics/gfx_pipeline_om/main.cpp @@ -1,7 +1,7 @@ // gfx_v2 device front end -> RASTER, end to end. // // The fused setup+binning pipeline runs on the SIMT cores to produce RASTER's -// tilebuf + primbuf (pinned), which are bound to the RASTER unit and rasterised +// tilebuf + primbuf, which are bound to the RASTER unit and rasterised // by a trivial fragment kernel. Two validations: (1) the device buffers match // host Binning() (localises any front-end bug), and (2) the rendered image // matches the gfx-v1 reference PNG (proves the device front end drives the FF @@ -156,11 +156,11 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // OM writes the color buffer via its AXI master (bypasses the MMU) -> pinned. + // OM writes the color buffer through the ocache. uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h color_buffer = nullptr; uint64_t cbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &color_buffer)); + RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE, &color_buffer)); RT_CHECK(vx_buffer_address(color_buffer, &cbuf_addr)); { std::vector clr(cbuf_size / 4, 0xff000000); vx_event_h e = nullptr; RT_CHECK(vx_enqueue_write(q, color_buffer, 0, clr.data(), cbuf_size, 0, nullptr, &e)); @@ -195,7 +195,7 @@ int main(int argc, char** argv) { const size_t PRIM_SZ = sizeof(rast_prim_t), HDR_SZ = sizeof(rast_bin_header_t); const size_t TILEBUF_SZ = (size_t)B * HDR_SZ + (size_t)Kcap * sizeof(uint32_t); - // Buffers. prim/tilebuf are pinned (RASTER's AXI master reads them). + // Buffers. The raster unit reads prim/tilebuf through the rcache. vx_buffer_h verts_b, slot_prim_b, slot_bbox_b, keep_b, offset_b, tsum_b, prim_b, bbox_b, bcount_b, boffset_b, keys_b, btsum_b, thist_b, bincount_b, binbase_b, tilebuf_b, meta_b; RT_CHECK(vx_buffer_create(dev, 3 * ntri * sizeof(setup_vertex_t), VX_MEM_READ, &verts_b)); @@ -204,7 +204,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, ntri * sizeof(uint32_t), VX_MEM_WRITE, &keep_b)); RT_CHECK(vx_buffer_create(dev, (ntri + 1) * sizeof(uint32_t), VX_MEM_WRITE, &offset_b)); RT_CHECK(vx_buffer_create(dev, T * sizeof(uint32_t), VX_MEM_WRITE, &tsum_b)); - RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &prim_b)); + RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE, &prim_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(setup_bbox_t), VX_MEM_WRITE, &bbox_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(uint32_t), VX_MEM_WRITE, &bcount_b)); RT_CHECK(vx_buffer_create(dev, (ntri * MS + 1) * sizeof(uint32_t), VX_MEM_WRITE, &boffset_b)); @@ -213,7 +213,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, T * B * sizeof(uint32_t), VX_MEM_WRITE, &thist_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &bincount_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &binbase_b)); - RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &tilebuf_b)); + RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE, &tilebuf_b)); RT_CHECK(vx_buffer_create(dev, 3 * sizeof(uint32_t), VX_MEM_WRITE, &meta_b)); pipe_arg_t pa = {}; diff --git a/tests/graphics/gfx_pipeline_raster/main.cpp b/tests/graphics/gfx_pipeline_raster/main.cpp index 3138be19bb..f314dce328 100644 --- a/tests/graphics/gfx_pipeline_raster/main.cpp +++ b/tests/graphics/gfx_pipeline_raster/main.cpp @@ -1,7 +1,7 @@ // gfx_v2 device front end -> RASTER, end to end. // // The fused setup+binning pipeline runs on the SIMT cores to produce RASTER's -// tilebuf + primbuf (pinned), which are bound to the RASTER unit and rasterised +// tilebuf + primbuf, which are bound to the RASTER unit and rasterised // by a trivial fragment kernel. Two validations: (1) the device buffers match // host Binning() (localises any front-end bug), and (2) the rendered image // matches the gfx-v1 reference PNG (proves the device front end drives the FF @@ -157,7 +157,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // color buffer (kernel-written via LSU; not pinned) + // color buffer (kernel-written via LSU) uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h color_buffer = nullptr; uint64_t cbuf_addr = 0; @@ -196,7 +196,7 @@ int main(int argc, char** argv) { const size_t PRIM_SZ = sizeof(rast_prim_t), HDR_SZ = sizeof(rast_bin_header_t); const size_t TILEBUF_SZ = (size_t)B * HDR_SZ + (size_t)Kcap * sizeof(uint32_t); - // Buffers. prim/tilebuf are pinned (RASTER's AXI master reads them). + // Buffers. The raster unit reads prim/tilebuf through the rcache. vx_buffer_h verts_b, slot_prim_b, slot_bbox_b, keep_b, offset_b, tsum_b, prim_b, bbox_b, bcount_b, boffset_b, keys_b, btsum_b, thist_b, bincount_b, binbase_b, tilebuf_b, meta_b; RT_CHECK(vx_buffer_create(dev, 3 * ntri * sizeof(setup_vertex_t), VX_MEM_READ, &verts_b)); @@ -205,7 +205,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, ntri * sizeof(uint32_t), VX_MEM_WRITE, &keep_b)); RT_CHECK(vx_buffer_create(dev, (ntri + 1) * sizeof(uint32_t), VX_MEM_WRITE, &offset_b)); RT_CHECK(vx_buffer_create(dev, T * sizeof(uint32_t), VX_MEM_WRITE, &tsum_b)); - RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &prim_b)); + RT_CHECK(vx_buffer_create(dev, ntri * MS * PRIM_SZ, VX_MEM_READ | VX_MEM_WRITE, &prim_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(setup_bbox_t), VX_MEM_WRITE, &bbox_b)); RT_CHECK(vx_buffer_create(dev, ntri * MS * sizeof(uint32_t), VX_MEM_WRITE, &bcount_b)); RT_CHECK(vx_buffer_create(dev, (ntri * MS + 1) * sizeof(uint32_t), VX_MEM_WRITE, &boffset_b)); @@ -214,7 +214,7 @@ int main(int argc, char** argv) { RT_CHECK(vx_buffer_create(dev, T * B * sizeof(uint32_t), VX_MEM_WRITE, &thist_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &bincount_b)); RT_CHECK(vx_buffer_create(dev, B * sizeof(uint32_t), VX_MEM_WRITE, &binbase_b)); - RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &tilebuf_b)); + RT_CHECK(vx_buffer_create(dev, TILEBUF_SZ, VX_MEM_READ | VX_MEM_WRITE, &tilebuf_b)); RT_CHECK(vx_buffer_create(dev, 3 * sizeof(uint32_t), VX_MEM_WRITE, &meta_b)); pipe_arg_t pa = {}; diff --git a/tests/graphics/gfx_pipeline_tex/main.cpp b/tests/graphics/gfx_pipeline_tex/main.cpp index 2c3df6eaac..1aa25eeb61 100644 --- a/tests/graphics/gfx_pipeline_tex/main.cpp +++ b/tests/graphics/gfx_pipeline_tex/main.cpp @@ -164,19 +164,19 @@ int main(int argc, char** argv) { RT_CHECK(vx_module_get_kernel(mod, "binning_k", &k_binning)); RT_CHECK(vx_module_get_kernel(mod, "main", &k_frag)); - // OM color buffer (pinned: OM's AXI master writes it). + // OM color buffer (the OM writes it through the ocache). const uint32_t cbuf_stride = 4, cbuf_pitch = dst_width * cbuf_stride, cbuf_size = dst_height * cbuf_pitch; vx_buffer_h cbuf_b = nullptr; uint64_t cbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE | VX_MEM_PHYS, &cbuf_b)); + RT_CHECK(vx_buffer_create(dev, cbuf_size, VX_MEM_READ | VX_MEM_WRITE, &cbuf_b)); RT_CHECK(vx_buffer_address(cbuf_b, &cbuf_addr)); - // Texture: checkerboard -> mipmaps -> pinned tex buffer (TEX's AXI master). + // Texture: checkerboard -> mipmaps -> tex buffer (read through the tcache). const uint32_t tw = 64, th = 64; std::vector tex_pixels = gen_texture(tw, th); std::vector texbuf; std::vector mip_offsets; RT_CHECK(GenerateMipmaps(texbuf, mip_offsets, tex_pixels.data(), FORMAT_A8R8G8B8, tw, th, tw * 4)); vx_buffer_h tex_b = nullptr; uint64_t tex_addr = 0; - RT_CHECK(vx_buffer_create(dev, texbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &tex_b)); + RT_CHECK(vx_buffer_create(dev, texbuf.size(), VX_MEM_READ, &tex_b)); RT_CHECK(vx_buffer_address(tex_b, &tex_addr)); RT_CHECK(vx_enqueue_write(q, tex_b, 0, texbuf.data(), texbuf.size(), 0, nullptr, nullptr)); @@ -272,8 +272,8 @@ int main(int argc, char** argv) { // reads. std::vector a_binbuf = repack_bins(gold); vx_buffer_h a_tb = nullptr, a_pb = nullptr; uint64_t a_tb_addr = 0, a_pb_addr = 0; - RT_CHECK(vx_buffer_create(dev, a_binbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &a_tb)); - RT_CHECK(vx_buffer_create(dev, a_primbuf.size(), VX_MEM_READ | VX_MEM_PHYS, &a_pb)); + RT_CHECK(vx_buffer_create(dev, a_binbuf.size(), VX_MEM_READ, &a_tb)); + RT_CHECK(vx_buffer_create(dev, a_primbuf.size(), VX_MEM_READ, &a_pb)); RT_CHECK(vx_buffer_address(a_tb, &a_tb_addr)); RT_CHECK(vx_buffer_address(a_pb, &a_pb_addr)); RT_CHECK(vx_enqueue_write(q, a_tb, 0, a_binbuf.data(), a_binbuf.size(), 0, nullptr, nullptr)); @@ -408,7 +408,7 @@ int main(int argc, char** argv) { const uint32_t zbuf_pitch = dst_width * 4, zbuf_size = dst_height * zbuf_pitch; vx_buffer_h zbuf_b = nullptr; uint64_t zbuf_addr = 0; - RT_CHECK(vx_buffer_create(dev, zbuf_size, VX_MEM_READ_WRITE | VX_MEM_PHYS, &zbuf_b)); + RT_CHECK(vx_buffer_create(dev, zbuf_size, VX_MEM_READ_WRITE, &zbuf_b)); RT_CHECK(vx_buffer_address(zbuf_b, &zbuf_addr)); frag_arg_t fa_d = fa; fa_d.prim_addr = prim_addr; fa_d.depth_enabled = 1; diff --git a/tests/graphics/gfx_raster/main.cpp b/tests/graphics/gfx_raster/main.cpp index 5ca49d0a68..eaaa27e6c1 100644 --- a/tests/graphics/gfx_raster/main.cpp +++ b/tests/graphics/gfx_raster/main.cpp @@ -1,12 +1,11 @@ // Vortex rasterizer regression test. // -// Pinned-buffer layout: -// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) → pinned -// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) → pinned -// color_buffer → kernel-only (LSU stores from shaded quads) → not pinned -// The raster fixed-function block reads tile_buffer + prim_buffer via its -// own AXI master (no per-core MMU). The kernel writes color_buffer via -// the LSU, which goes through the MMU — no pinning needed. +// Buffers: +// tile_buffer → RASTER HW (VX_DCR_RASTER_TBUF_ADDR) +// prim_buffer → RASTER HW (VX_DCR_RASTER_PBUF_ADDR) +// color_buffer → kernel-only (LSU stores from shaded quads) +// The raster fixed-function block reads tile_buffer + prim_buffer through the +// rcache, whose misses translate through the cluster unit MMU under VM. #include #include @@ -212,15 +211,12 @@ int render(const CGLTrace& trace) { // allocate tile memory if (tile_buffer != nullptr) { vx_buffer_release(tile_buffer); tile_buffer = nullptr; } if (prim_buffer != nullptr) { vx_buffer_release(prim_buffer); prim_buffer = nullptr; } - // HW path: tile_buffer / prim_buffer are bound to the raster unit (via - // VX_DCR_RASTER_T/PBUF_ADDR), which reads them through its own AXI master - // bypassing the per-core MMU — both need physical (pinned) addresses. SW path - // (-z): the kernel reads prim_buffer through the LSU, so allocate it as a - // normal kernel-read buffer (tile_buffer is unused by the SW walk). - uint32_t prim_flags = VX_MEM_READ | (use_sw ? 0u : (uint32_t)VX_MEM_PHYS); - RT_CHECK(vx_buffer_create(device, tilebuf.size(), VX_MEM_READ | VX_MEM_PHYS, &tile_buffer)); + // HW path: the raster unit reads them through the rcache (translated by + // the cluster unit MMU); SW path (-z): the kernel reads prim_buffer through + // the LSU (tile_buffer is unused by the SW walk). + RT_CHECK(vx_buffer_create(device, tilebuf.size(), VX_MEM_READ, &tile_buffer)); RT_CHECK(vx_buffer_address(tile_buffer, &tilebuf_addr)); - RT_CHECK(vx_buffer_create(device, primbuf.size(), prim_flags, &prim_buffer)); + RT_CHECK(vx_buffer_create(device, primbuf.size(), VX_MEM_READ, &prim_buffer)); RT_CHECK(vx_buffer_address(prim_buffer, &primbuf_addr)); std::cout << "tile_buffer=0x" << std::hex << tilebuf_addr << std::dec << std::endl; std::cout << "prim_buffer=0x" << std::hex << primbuf_addr << std::dec << std::endl; diff --git a/tests/graphics/gfx_tex/main.cpp b/tests/graphics/gfx_tex/main.cpp index 4fe29506c7..5dc512ee95 100644 --- a/tests/graphics/gfx_tex/main.cpp +++ b/tests/graphics/gfx_tex/main.cpp @@ -1,11 +1,10 @@ // Texture regression test using the KMU launch API. // -// Pinned-buffer contract: -// src_buffer → TEX HW (VX_DCR_TEX_ADDR) → VX_MEM_PHYS (identity-mapped) -// dst_buffer → kernel-only (LSU stores) → not pinned -// Only src_buffer is read by the TEX fixed-function block. dst_buffer is -// written by kernel code through the per-core MMU, so it gets a regular -// minted VA under VM. +// Buffers: +// src_buffer → TEX HW (VX_DCR_TEX_ADDR) +// dst_buffer → kernel-only (LSU stores) +// Both get a regular minted VA under VM: the TEX unit's tcache misses +// translate through the socket unit MMU like the kernel's loads and stores. // // Host loads a PNG, generates a mipmap chain, configures TEX stage 0 // DCRs, then launches a 2D grid where each thread samples one output @@ -209,9 +208,9 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_get_kernel(module_, border_mode ? "kernel_main_border" : "main", &kernel)); // ---- allocate device buffers ----------------------------------------- - // src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR) which - // bypasses the per-core MMU — needs a physical address. - RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ | VX_MEM_PHYS, &src_buffer)); + // src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR); under VM its + // reads translate through the socket unit MMU like any kernel access. + RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ, &src_buffer)); RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); // dst_buffer is kernel-written via the LSU only — no HW-block reader, // no need to pin (the per-core MMU translates the VA for the kernel). diff --git a/tests/graphics/gfx_tex4/main.cpp b/tests/graphics/gfx_tex4/main.cpp index 5f4702572e..a2c171e600 100644 --- a/tests/graphics/gfx_tex4/main.cpp +++ b/tests/graphics/gfx_tex4/main.cpp @@ -1,11 +1,10 @@ // Texture regression test using the KMU launch API. // -// Pinned-buffer contract: -// src_buffer → TEX HW (VX_DCR_TEX_ADDR) → VX_MEM_PHYS (identity-mapped) -// dst_buffer → kernel-only (LSU stores) → not pinned -// Only src_buffer is read by the TEX fixed-function block. dst_buffer is -// written by kernel code through the per-core MMU, so it gets a regular -// minted VA under VM. +// Buffers: +// src_buffer → TEX HW (VX_DCR_TEX_ADDR) +// dst_buffer → kernel-only (LSU stores) +// Both get a regular minted VA under VM: the TEX unit's tcache misses +// translate through the socket unit MMU like the kernel's loads and stores. // // Host loads a PNG, generates a mipmap chain, configures TEX stage 0 // DCRs, then launches a 2D grid where each thread samples one output @@ -197,14 +196,11 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); // ---- allocate device buffers ----------------------------------------- - // HW path: src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR), which reads - // it through the TEX/tcache path bypassing the per-core MMU — needs a physical - // (pinned) address. SW path (-S): the kernel reads the texture through the - // LSU/dcache like any other resident buffer, so allocate it as a normal - // kernel-read buffer (a PHYS buffer read via the LSU is not dcache-coherent - // with the host upload). - uint32_t src_flags = VX_MEM_READ | (sw_path ? 0u : (uint32_t)VX_MEM_PHYS); - RT_CHECK(vx_buffer_create(device, src_bufsize, src_flags, &src_buffer)); + // HW path: src_buffer is bound to the TEX unit (VX_DCR_TEX_ADDR), whose + // tcache translates through the socket unit MMU. SW path (-S): the kernel + // reads the texture through the LSU/dcache. Either way it is a normal + // kernel-read buffer. + RT_CHECK(vx_buffer_create(device, src_bufsize, VX_MEM_READ, &src_buffer)); RT_CHECK(vx_buffer_address(src_buffer, &src_addr)); // dst_buffer is kernel-written via the LSU only — no HW-block reader, // no need to pin (the per-core MMU translates the VA for the kernel). diff --git a/tests/hip/atomicreduce/main.cpp b/tests/hip/atomicreduce/main.cpp index 9d863b4ecb..9a9b0fd83e 100644 --- a/tests/hip/atomicreduce/main.cpp +++ b/tests/hip/atomicreduce/main.cpp @@ -108,5 +108,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/common.mk b/tests/hip/common.mk index 77c80baa6e..28101e6df1 100644 --- a/tests/hip/common.mk +++ b/tests/hip/common.mk @@ -169,9 +169,46 @@ else $(HIP_OCL_ENV) SCOPE_JSON_PATH=$(VORTEX_RT_LIB)/scope.json LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) VORTEX_DRIVER=xrt ./$(PROJECT) $(OPTS) endif +# FireSim takes the same targets as the card's native driver; the card and its +# variables are decided in sw/runtime/firesim/card.mk (convention established +# in tests/regression/common.mk). +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk +endif + +run-firesim: $(PROJECT) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) +ifneq ($(filter $(TARGET),xrtsim avedsim),) + $(HIP_OCL_ENV) LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh +endif + $(HIP_OCL_ENV) VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) + $(HIP_OCL_ENV) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else + $(HIP_OCL_ENV) XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif + +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; +# any other TARGET, avedsim, simulates the same kernel in-process (convention +# established in tests/regression/common.mk). +run-foldsim: $(PROJECT) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + $(HIP_OCL_ENV) VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +else + $(HIP_OCL_ENV) LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(CHIPSTAR_PATH)/lib:$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +endif + clean: rm -f $(PROJECT) *.o *.vxbin *.dump *.ll *.log *.spv common.h -.PHONY: all run-simx run-rtlsim run-opae run-xrt clean +.PHONY: all run-simx run-rtlsim run-opae run-xrt run-firesim run-foldsim clean FORCE: diff --git a/tests/hip/histogram/main.cpp b/tests/hip/histogram/main.cpp index 3155a05760..9aaf7494f1 100644 --- a/tests/hip/histogram/main.cpp +++ b/tests/hip/histogram/main.cpp @@ -111,5 +111,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/sgemm/main.cpp b/tests/hip/sgemm/main.cpp index 99b63c0da8..383f9748e9 100644 --- a/tests/hip/sgemm/main.cpp +++ b/tests/hip/sgemm/main.cpp @@ -151,5 +151,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/hip/vecadd/main.cpp b/tests/hip/vecadd/main.cpp index 5e8e8b7e4b..7af6a5c871 100644 --- a/tests/hip/vecadd/main.cpp +++ b/tests/hip/vecadd/main.cpp @@ -131,5 +131,5 @@ int main(int argc, char** argv) { } else { printf("FAILED! - %d errors\n", errors); } - return errors; + return errors != 0; } diff --git a/tests/kernel/conform/main.cpp b/tests/kernel/conform/main.cpp index b34997ad6d..4790a0de34 100644 --- a/tests/kernel/conform/main.cpp +++ b/tests/kernel/conform/main.cpp @@ -41,5 +41,5 @@ int main() { PRINTF("Failed!\n"); } - return errors; + return errors != 0; } diff --git a/tests/kernel/fibonacci/main.cpp b/tests/kernel/fibonacci/main.cpp index c6fc036aa9..b083db6d1b 100644 --- a/tests/kernel/fibonacci/main.cpp +++ b/tests/kernel/fibonacci/main.cpp @@ -24,7 +24,7 @@ int main() { errors = 1; } - return errors; + return errors != 0; } diff --git a/tests/opencl/atomicreduce/main.cc b/tests/opencl/atomicreduce/main.cc index 7ff0f74fab..cce4c9f7ce 100644 --- a/tests/opencl/atomicreduce/main.cc +++ b/tests/opencl/atomicreduce/main.cc @@ -170,5 +170,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/backprop/main.cc b/tests/opencl/backprop/main.cc index 4823ecea7f..47dd70fc9f 100644 --- a/tests/opencl/backprop/main.cc +++ b/tests/opencl/backprop/main.cc @@ -302,7 +302,7 @@ int main(int argc, char** argv) { if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/bfs/main.cc b/tests/opencl/bfs/main.cc index c676a40c53..325bafb204 100755 --- a/tests/opencl/bfs/main.cc +++ b/tests/opencl/bfs/main.cc @@ -75,8 +75,9 @@ void run_bfs_gpu(int no_of_nodes, Node *h_graph_nodes, int edge_list_size, // int number_elements = height*width; char h_over; - cl_mem d_graph_nodes, d_graph_edges, d_graph_mask, d_updating_graph_mask, - d_graph_visited, d_cost, d_over; + cl_mem d_graph_nodes = NULL, d_graph_edges = NULL, d_graph_mask = NULL, + d_updating_graph_mask = NULL, d_graph_visited = NULL, d_cost = NULL, + d_over = NULL; try { //--1 transfer data from host to device @@ -306,7 +307,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { printf("Failed!\n"); - return errors; + return errors != 0; } printf("Passed!\n"); diff --git a/tests/opencl/blackscholes/oclUtils.cpp b/tests/opencl/blackscholes/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/blackscholes/oclUtils.cpp +++ b/tests/opencl/blackscholes/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); diff --git a/tests/opencl/btree/main.cc b/tests/opencl/btree/main.cc index a4a2d1e940..d72fdf86fe 100644 --- a/tests/opencl/btree/main.cc +++ b/tests/opencl/btree/main.cc @@ -504,7 +504,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/cfd/main.cc b/tests/opencl/cfd/main.cc index 1d8d11c741..ca00e4f18d 100644 --- a/tests/opencl/cfd/main.cc +++ b/tests/opencl/cfd/main.cc @@ -519,7 +519,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/common.mk b/tests/opencl/common.mk index 84774d3e2d..facdae4d56 100644 --- a/tests/opencl/common.mk +++ b/tests/opencl/common.mk @@ -152,6 +152,43 @@ else SCOPE_JSON_PATH=$(VORTEX_RT_LIB)/scope.json LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=xrt ./$(PROJECT) $(OPTS) endif +# FireSim takes the same targets as the card's native driver; the card and its +# variables are decided in sw/runtime/firesim/card.mk (convention established +# in tests/regression/common.mk). +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk +endif + +run-firesim: $(PROJECT) $(KERNEL_SRCS) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) +ifneq ($(filter $(TARGET),xrtsim avedsim),) + LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh +endif + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) + XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else + XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif + +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; +# any other TARGET, avedsim, simulates the same kernel in-process (convention +# established in tests/regression/common.mk). +run-foldsim: $(PROJECT) $(KERNEL_SRCS) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +else + LD_LIBRARY_PATH=$(OCL_ICD_LIB_DIR):$(POCL_PATH)/lib:$(VORTEX_RT_LIB):$(LLVM_PATH)/lib:$(LD_LIBRARY_PATH) $(POCL_CC_FLAGS) OCL_ICD_VENDORS=$(OCL_ICD_VENDORS) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +endif + .depend: $(SRCS) $(CXX) $(CXXFLAGS) -MM $^ > .depend; diff --git a/tests/opencl/conv3/main.cc b/tests/opencl/conv3/main.cc index 44bfe64886..34b083837e 100644 --- a/tests/opencl/conv3/main.cc +++ b/tests/opencl/conv3/main.cc @@ -247,5 +247,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/dotproduct/oclUtils.cpp b/tests/opencl/dotproduct/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/dotproduct/oclUtils.cpp +++ b/tests/opencl/dotproduct/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); diff --git a/tests/opencl/dwt2d/main.cc b/tests/opencl/dwt2d/main.cc index 92e223b744..edf8e4f5df 100644 --- a/tests/opencl/dwt2d/main.cc +++ b/tests/opencl/dwt2d/main.cc @@ -222,7 +222,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/guassian/main.cc b/tests/opencl/guassian/main.cc index a06ebbbdcd..b2ee50de55 100755 --- a/tests/opencl/guassian/main.cc +++ b/tests/opencl/guassian/main.cc @@ -123,7 +123,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/heartwall/main.cc b/tests/opencl/heartwall/main.cc index 0770c4e43c..11d922fcdd 100644 --- a/tests/opencl/heartwall/main.cc +++ b/tests/opencl/heartwall/main.cc @@ -357,7 +357,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/histogram/main.cc b/tests/opencl/histogram/main.cc index d6b1d3b879..b8131f4417 100644 --- a/tests/opencl/histogram/main.cc +++ b/tests/opencl/histogram/main.cc @@ -174,5 +174,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/hotspot/main.cc b/tests/opencl/hotspot/main.cc index a399f93616..7e519d39b7 100644 --- a/tests/opencl/hotspot/main.cc +++ b/tests/opencl/hotspot/main.cc @@ -279,7 +279,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/hotspot3D/main.cc b/tests/opencl/hotspot3D/main.cc index af729c56b7..c2a30a0d17 100644 --- a/tests/opencl/hotspot3D/main.cc +++ b/tests/opencl/hotspot3D/main.cc @@ -273,7 +273,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/hybridsort/main.cc b/tests/opencl/hybridsort/main.cc index 60b2349c5d..0b891363ee 100644 --- a/tests/opencl/hybridsort/main.cc +++ b/tests/opencl/hybridsort/main.cc @@ -310,7 +310,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/lavaMD/main.cc b/tests/opencl/lavaMD/main.cc index 4da28826f2..a6d713b531 100644 --- a/tests/opencl/lavaMD/main.cc +++ b/tests/opencl/lavaMD/main.cc @@ -355,7 +355,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/lbm/main.cc b/tests/opencl/lbm/main.cc index 2da7435108..c8a917242b 100644 --- a/tests/opencl/lbm/main.cc +++ b/tests/opencl/lbm/main.cc @@ -150,7 +150,7 @@ int main(int nArgs, char *arg[]) { pb_SwitchToTimer(&timers, pb_TimerID_NONE); pb_PrintTimerSet(&timers); pb_FreeParameters(params); - return errors; + return errors != 0; } /*############################################################################*/ diff --git a/tests/opencl/lud/main.cc b/tests/opencl/lud/main.cc index c1f00fa916..c40e63f95f 100644 --- a/tests/opencl/lud/main.cc +++ b/tests/opencl/lud/main.cc @@ -265,7 +265,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/myocyte/main.cc b/tests/opencl/myocyte/main.cc index e359e8c033..117ea15f37 100644 --- a/tests/opencl/myocyte/main.cc +++ b/tests/opencl/myocyte/main.cc @@ -474,7 +474,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/nearn/main.cc b/tests/opencl/nearn/main.cc index ea8238e167..fa73fbfbef 100755 --- a/tests/opencl/nearn/main.cc +++ b/tests/opencl/nearn/main.cc @@ -68,7 +68,7 @@ int main(int argc, char *argv[]) { printf("Passed!\n"); } - return errors; + return errors != 0; } float *OpenClFindNearestNeighbors(cl_context context, int numRecords, diff --git a/tests/opencl/nw/main.cc b/tests/opencl/nw/main.cc index 38375c3485..bfbff5cdec 100644 --- a/tests/opencl/nw/main.cc +++ b/tests/opencl/nw/main.cc @@ -279,7 +279,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/pathfinder/main.cc b/tests/opencl/pathfinder/main.cc index a2b9d67696..69c868643d 100644 --- a/tests/opencl/pathfinder/main.cc +++ b/tests/opencl/pathfinder/main.cc @@ -223,7 +223,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/psort/main.cc b/tests/opencl/psort/main.cc index 0e72599ece..d7ca45dc7a 100644 --- a/tests/opencl/psort/main.cc +++ b/tests/opencl/psort/main.cc @@ -229,5 +229,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/psum/main.cc b/tests/opencl/psum/main.cc index 5606de8c5b..4e37212cae 100644 --- a/tests/opencl/psum/main.cc +++ b/tests/opencl/psum/main.cc @@ -225,5 +225,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/saxpy/main.cc b/tests/opencl/saxpy/main.cc index 2cb4b71ba8..b208755051 100644 --- a/tests/opencl/saxpy/main.cc +++ b/tests/opencl/saxpy/main.cc @@ -257,5 +257,5 @@ int main(int argc, char **argv) { Cleanup(device_id, context, queue, program, kernel, memObjects); - return errors; + return errors != 0; } diff --git a/tests/opencl/sfilter/main.cc b/tests/opencl/sfilter/main.cc index c56d26ffdc..7cf4f6e126 100644 --- a/tests/opencl/sfilter/main.cc +++ b/tests/opencl/sfilter/main.cc @@ -298,5 +298,5 @@ int main(int argc, char **argv) { Cleanup(kernel_bin, device_id, context, queue, program, kernel, memObjects); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm/main.cc b/tests/opencl/sgemm/main.cc index f8a7350b6a..95a4d707cd 100644 --- a/tests/opencl/sgemm/main.cc +++ b/tests/opencl/sgemm/main.cc @@ -262,5 +262,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm2/main.cc b/tests/opencl/sgemm2/main.cc index 43c016514b..3cc02fddb6 100644 --- a/tests/opencl/sgemm2/main.cc +++ b/tests/opencl/sgemm2/main.cc @@ -267,5 +267,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/sgemm3/main.cc b/tests/opencl/sgemm3/main.cc index 21bec0c71e..f567d4795f 100644 --- a/tests/opencl/sgemm3/main.cc +++ b/tests/opencl/sgemm3/main.cc @@ -273,5 +273,5 @@ int main (int argc, char **argv) { // Clean up cleanup(); - return errors; + return errors != 0; } diff --git a/tests/opencl/spmv/main.cc b/tests/opencl/spmv/main.cc index fada19c156..0e72c6c70b 100644 --- a/tests/opencl/spmv/main.cc +++ b/tests/opencl/spmv/main.cc @@ -379,5 +379,5 @@ int main(int argc, char **argv) { pb_PrintTimerSet(&timers); pb_FreeParameters(parameters); - return errors; + return errors != 0; } diff --git a/tests/opencl/srad/main.cc b/tests/opencl/srad/main.cc index c15d1deb8f..516ee78e1c 100644 --- a/tests/opencl/srad/main.cc +++ b/tests/opencl/srad/main.cc @@ -361,7 +361,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/stencil/main.cc b/tests/opencl/stencil/main.cc index 30dee81bd0..a1994acca6 100644 --- a/tests/opencl/stencil/main.cc +++ b/tests/opencl/stencil/main.cc @@ -354,5 +354,5 @@ int main(int argc, char** argv) { pb_PrintTimerSet(&timers); pb_FreeParameters(parameters); - return errors; + return errors != 0; } diff --git a/tests/opencl/streamcluster/main.cc b/tests/opencl/streamcluster/main.cc index 97039e1c5b..0f549a1983 100644 --- a/tests/opencl/streamcluster/main.cc +++ b/tests/opencl/streamcluster/main.cc @@ -298,7 +298,7 @@ int main(int argc, char** argv) { cleanup(); if (errors != 0) { printf("FAILED! - %d errors\n", errors); - return errors; + return errors != 0; } printf("PASSED!\n"); return 0; diff --git a/tests/opencl/transpose/oclUtils.cpp b/tests/opencl/transpose/oclUtils.cpp index 7a238b556d..8f82cf0d12 100644 --- a/tests/opencl/transpose/oclUtils.cpp +++ b/tests/opencl/transpose/oclUtils.cpp @@ -259,7 +259,7 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) // CL_DEVICE_EXTENSIONS: get device extensions, and if any then parse & log the string onto separate lines clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(device_string), &device_string, NULL); - if (device_string != 0) + if (device_string[0] != '\0') { shrLogEx(iLogMode, 0, "\n CL_DEVICE_EXTENSIONS:"); std::string stdDevString; @@ -341,13 +341,13 @@ void oclPrintDevInfo(int iLogMode, cl_device_id device) ////////////////////////////////////////////////////////////////////////////// int oclGetDevCap(cl_device_id device) { - char cDevString[1024]; + char cDevString[1024] = ""; bool bDevAttributeQuery = false; int iDevArch = -1; // Get device extensions, and if any then search for cl_nv_device_attribute_query clGetDeviceInfo(device, CL_DEVICE_EXTENSIONS, sizeof(cDevString), &cDevString, NULL); - if (cDevString != 0) + if (cDevString[0] != '\0') { std::string stdDevString; stdDevString = std::string(cDevString); diff --git a/tests/opencl/vecadd/main.cc b/tests/opencl/vecadd/main.cc index 2334e0b4e9..d66d486d20 100644 --- a/tests/opencl/vecadd/main.cc +++ b/tests/opencl/vecadd/main.cc @@ -239,5 +239,5 @@ int main (int argc, char **argv) { cleanup(); - return errors; + return errors != 0; } diff --git a/tests/raytracing/rt_smoke/Makefile b/tests/raytracing/rt_smoke/Makefile index 0874691901..b72f9c9f11 100644 --- a/tests/raytracing/rt_smoke/Makefile +++ b/tests/raytracing/rt_smoke/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke diff --git a/tests/raytracing/rt_smoke_ahs/Makefile b/tests/raytracing/rt_smoke_ahs/Makefile index f6052b4538..7991aa54f7 100644 --- a/tests/raytracing/rt_smoke_ahs/Makefile +++ b/tests/raytracing/rt_smoke_ahs/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_ahs diff --git a/tests/raytracing/rt_smoke_ahs_mixed/Makefile b/tests/raytracing/rt_smoke_ahs_mixed/Makefile index 400ebe2a57..0cbb329586 100644 --- a/tests/raytracing/rt_smoke_ahs_mixed/Makefile +++ b/tests/raytracing/rt_smoke_ahs_mixed/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_ahs_mixed diff --git a/tests/raytracing/rt_smoke_async_batch/Makefile b/tests/raytracing/rt_smoke_async_batch/Makefile index 18c4957554..65cac992c1 100644 --- a/tests/raytracing/rt_smoke_async_batch/Makefile +++ b/tests/raytracing/rt_smoke_async_batch/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # The whole point of the test: four traces issued back to back before any wait. # Four traces in flight is four slots, and the default pool holds one per core. CONFIGS += -DVX_CFG_RTU_NUM_SLOTS=4 diff --git a/tests/raytracing/rt_smoke_chs/Makefile b/tests/raytracing/rt_smoke_chs/Makefile index e12fc4293a..51336c5dd8 100644 --- a/tests/raytracing/rt_smoke_chs/Makefile +++ b/tests/raytracing/rt_smoke_chs/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_chs diff --git a/tests/raytracing/rt_smoke_cull_back/Makefile b/tests/raytracing/rt_smoke_cull_back/Makefile index dbd4074892..1f65b72d2a 100644 --- a/tests/raytracing/rt_smoke_cull_back/Makefile +++ b/tests/raytracing/rt_smoke_cull_back/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_cull_back diff --git a/tests/raytracing/rt_smoke_is/Makefile b/tests/raytracing/rt_smoke_is/Makefile index 46b5149f15..065923ac22 100644 --- a/tests/raytracing/rt_smoke_is/Makefile +++ b/tests/raytracing/rt_smoke_is/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_is diff --git a/tests/raytracing/rt_smoke_miss/Makefile b/tests/raytracing/rt_smoke_miss/Makefile index 6b9b4c4bcb..3c96b4ada8 100644 --- a/tests/raytracing/rt_smoke_miss/Makefile +++ b/tests/raytracing/rt_smoke_miss/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_miss diff --git a/tests/raytracing/rt_smoke_multitri/Makefile b/tests/raytracing/rt_smoke_multitri/Makefile index e8b594dc18..209b953dc5 100644 --- a/tests/raytracing/rt_smoke_multitri/Makefile +++ b/tests/raytracing/rt_smoke_multitri/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_multitri diff --git a/tests/raytracing/rt_smoke_numctx/Makefile b/tests/raytracing/rt_smoke_numctx/Makefile index 4c240bd8c4..680cfe6b74 100644 --- a/tests/raytracing/rt_smoke_numctx/Makefile +++ b/tests/raytracing/rt_smoke_numctx/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # Two resident traces: RTU_NUM_SLOTS = 2 DERIVES RTU_NUM_CTX = 8 (a slot owns one # context per thread of the warp it traces). Exercises the multi-slot RTU — two warps # traversing concurrently, per-slot completion barriers, and the window-write arbiter diff --git a/tests/raytracing/rt_smoke_recursive/Makefile b/tests/raytracing/rt_smoke_recursive/Makefile index 7dc5dc0eb2..f80ce73dd8 100644 --- a/tests/raytracing/rt_smoke_recursive/Makefile +++ b/tests/raytracing/rt_smoke_recursive/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # The sub-ray is traced from inside the closest-hit callback, so the warp holds # its parent's slot while it asks for a second one. Two traces in flight per warp # is two slots; the default pool (one per core) cannot serve it and the warp diff --git a/tests/raytracing/rt_smoke_reform/Makefile b/tests/raytracing/rt_smoke_reform/Makefile index ea6ab993c8..635b932a4e 100644 --- a/tests/raytracing/rt_smoke_reform/Makefile +++ b/tests/raytracing/rt_smoke_reform/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_reform diff --git a/tests/raytracing/rt_smoke_reform_mw/Makefile b/tests/raytracing/rt_smoke_reform_mw/Makefile index c3fea042d2..5b7d917f99 100644 --- a/tests/raytracing/rt_smoke_reform_mw/Makefile +++ b/tests/raytracing/rt_smoke_reform_mw/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE # Reformation batches candidates from several warps into one callback dispatch, # so several warps have to be tracing AT THE SAME TIME. One slot per core would # serialise them and the test would pass without ever exercising what it is for. diff --git a/tests/raytracing/rt_smoke_reform_sbt/Makefile b/tests/raytracing/rt_smoke_reform_sbt/Makefile index e0e479021a..bb04bb17ac 100644 --- a/tests/raytracing/rt_smoke_reform_sbt/Makefile +++ b/tests/raytracing/rt_smoke_reform_sbt/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_reform_sbt diff --git a/tests/raytracing/rt_smoke_sbt/Makefile b/tests/raytracing/rt_smoke_sbt/Makefile index 6f8f29081a..bfca028c5e 100644 --- a/tests/raytracing/rt_smoke_sbt/Makefile +++ b/tests/raytracing/rt_smoke_sbt/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_sbt diff --git a/tests/raytracing/rt_smoke_shadow/Makefile b/tests/raytracing/rt_smoke_shadow/Makefile index 6662e36517..86d5090d45 100644 --- a/tests/raytracing/rt_smoke_shadow/Makefile +++ b/tests/raytracing/rt_smoke_shadow/Makefile @@ -2,8 +2,9 @@ ROOT_DIR := $(realpath ../../..) include $(ROOT_DIR)/config.mk CONFIGS := $(if $(findstring -DVX_CFG_EXT_RTU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_RTU_ENABLE) -# Flat triangle-list scene -> build the RTU as a flat walker (true-HW model). -CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 +# Flat triangle-list scene -> build the RTU as a flat walker without TLAS, which +# would expect a TLAS scene (true-HW model; overrides a profile that enables it). +CONFIGS += -DVX_CFG_RTU_BVH_WIDTH=0 -DVX_CFG_RTU_TLAS_DISABLE PROJECT := rt_smoke_shadow diff --git a/tests/regression/Makefile b/tests/regression/Makefile index e5be8fed12..6d3221dea4 100644 --- a/tests/regression/Makefile +++ b/tests/regression/Makefile @@ -5,16 +5,17 @@ include $(ROOT_DIR)/config.mk TESTS := \ minimal \ basic demo dogfood dropout dotproduct dotproduct2 mstress io_addr \ - arith printf diverge diverge_large sort fence ldst_order \ + arith printf diverge diverge_large sort fence fence_mp ldst_order \ vecadd vecadd_v1 sgemm sgemmx sgemm_v1 conv3 relu sgemv \ sgemm2 sgemm2_v1 madmax stencil3d raycast bfs jacobi pathfinder \ - softmax mufu zfhmin \ + softmax uniform_ld lmem_access ldst_wide ldst_matrix mufu zfhmin \ sgemm_tcu sgemm_tcu_mx sgemm_tcu_sp_mx sgemm_tcu_wg sgemm_tcu_wg_mx sgemm_tcu_sp sgemm_tcu_wg_sp \ sgemm2_dxa sgemm2_tcu sgemm_tcu_wg_dxa sgemm_tcu_wg_sp_dxa \ sgemm2_dxa_mcast sgemm_tcu_wg_dxa_mcast \ dxa_copy dxa_copy_mcast dxa_kmajor_check \ async_barrier async_gbarrier gbar_phase bar_slot_phase packld wgather wsync quad_lod \ occupancy multikernel reopen module_reload single_cta \ + flush_coherence lazy_fetch sector_merge raw_order \ vm_test vm_fault vm_stress # --- common exclude list --------------------------------------------- diff --git a/tests/regression/amo/common.h b/tests/regression/amo/common.h index e9828a2d2f..f9017a823c 100644 --- a/tests/regression/amo/common.h +++ b/tests/regression/amo/common.h @@ -13,6 +13,9 @@ #define CAS_LMEM_SLOTS 16 #define CAS_LMEM_GROUP_MARK 0x10000u +// Initial word of the local-memory swap chain in the AMO op sweep. +#define AMO_OPS_SWAP_INIT 0x5EED5EEDu + // Args shared between host (main.cpp) and the GPU kernel (kernel.cpp). // Layout matches dogfood's, with `iters` added for the contention loops. typedef struct { diff --git a/tests/regression/amo/kernel.cpp b/tests/regression/amo/kernel.cpp index 4e9733015f..56ba507fed 100644 --- a/tests/regression/amo/kernel.cpp +++ b/tests/regression/amo/kernel.cpp @@ -336,8 +336,87 @@ void kernel_lrsc_lmem(kernel_arg_t* __UNIFORM__ arg) { } } +// 15) LMEM AMO op sweep. +// Every hart of a group applies swap, and, or, xor, min, max, minu and maxu +// to one local-memory word per operation, all harts on the same words, so +// the LMEM banks serialize every operation. The leader then rebuilds each +// final word from the group's contributions, checks that the swapped-out +// values and the final word sum to the initial word plus every value swapped +// in, and publishes its mismatch count with the group marker. +#define AMO_LMEM(op, addr, val) ({ \ + uint32_t _old; \ + asm volatile (op " %0, %2, (%1)" \ + : "=r"(_old) : "r"(addr), "r"(val) : "memory"); \ + _old; }) + +static inline uint32_t amo_ops_value(uint32_t t) { + return t * 0x9E3779B1u + 0x01234567u; +} + +void kernel_amo_ops_lmem(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = (uint32_t*)__local_mem(); + auto per_hart = (uint32_t*)arg->per_hart_addr; + const uint32_t hid = hart_id(); + const uint32_t t = threadIdx.x; + const uint32_t n = blockDim.x; + // 0 swap, 1 and, 2 or, 3 xor, 4 min, 5 max, 6 minu, 7 maxu, then each + // hart's swapped-out value + auto olds = &lmem[8]; + + if (t == 0) { + lmem[0] = AMO_OPS_SWAP_INIT; + lmem[1] = ~0u; + lmem[2] = 0; + lmem[3] = 0; + lmem[4] = 0x7fffffffu; + lmem[5] = 0x80000000u; + lmem[6] = ~0u; + lmem[7] = 0; + } + __syncthreads(); + + uint32_t v = amo_ops_value(t); + uint32_t bit = 1u << (t % 32); + olds[t] = AMO_LMEM("amoswap.w", &lmem[0], v); + AMO_LMEM("amoand.w", &lmem[1], ~bit); + AMO_LMEM("amoor.w", &lmem[2], bit); + AMO_LMEM("amoxor.w", &lmem[3], v); + AMO_LMEM("amomin.w", &lmem[4], v); + AMO_LMEM("amomax.w", &lmem[5], v); + AMO_LMEM("amominu.w", &lmem[6], v); + AMO_LMEM("amomaxu.w", &lmem[7], v); + __syncthreads(); + + uint32_t mismatches = 0; + if (t == 0) { + uint32_t and_e = ~0u, or_e = 0, xor_e = 0, minu_e = ~0u, maxu_e = 0, sum_in = 0, sum_out = 0; + int32_t min_e = 0x7fffffff, max_e = (int32_t)0x80000000u; + for (uint32_t i = 0; i < n; ++i) { + uint32_t vi = amo_ops_value(i); + and_e &= ~(1u << (i % 32)); + or_e |= 1u << (i % 32); + xor_e ^= vi; + min_e = ((int32_t)vi < min_e) ? (int32_t)vi : min_e; + max_e = ((int32_t)vi > max_e) ? (int32_t)vi : max_e; + minu_e = (vi < minu_e) ? vi : minu_e; + maxu_e = (vi > maxu_e) ? vi : maxu_e; + sum_in += vi; + sum_out += olds[i]; + } + mismatches += (sum_out + lmem[0]) != (AMO_OPS_SWAP_INIT + sum_in); + mismatches += lmem[1] != and_e; + mismatches += lmem[2] != or_e; + mismatches += lmem[3] != xor_e; + mismatches += (int32_t)lmem[4] != min_e; + mismatches += (int32_t)lmem[5] != max_e; + mismatches += lmem[6] != minu_e; + mismatches += lmem[7] != maxu_e; + } + per_hart[hid] = mismatches + ((t == 0) ? CAS_LMEM_GROUP_MARK : 0); +} + #if VX_CFG_EXT_ZACAS_ENABLED -// 15) Native compare-and-swap contention. +// 16) Native compare-and-swap contention. // Every hart races to claim each slot of a ladder with its own id, so for // slot i exactly one hart sees the empty value and swaps, and the others see // that winner's id. Each hart records how many slots it won. @@ -367,7 +446,7 @@ void kernel_cas_ladder(kernel_arg_t* __UNIFORM__ arg) { per_hart[hid] = wins; } -// 16) LMEM compare-and-swap ladder. +// 17) LMEM compare-and-swap ladder. // The cas_ladder race moved onto local memory, so the swap and the old word // it returns come from the LMEM banks. Each group races for its own ladder // and exactly one of its harts wins each slot, so the win counts sum to @@ -417,9 +496,10 @@ static const PFN_Kernel sc_tests[] = { kernel_self_consistency, // 12 issuer self-consistency (load->AMO->load) kernel_amoadd_lmem, // 13 AMOADD on local memory (LMEM banks) kernel_lrsc_lmem, // 14 LR/SC on local memory (LMEM banks) + kernel_amo_ops_lmem, // 15 every other AMO op on local memory (LMEM banks) #if VX_CFG_EXT_ZACAS_ENABLED - kernel_cas_ladder, // 15 native compare-and-swap (Zacas amocas) - kernel_cas_lmem, // 16 compare-and-swap on local memory (LMEM banks) + kernel_cas_ladder, // 16 native compare-and-swap (Zacas amocas) + kernel_cas_lmem, // 17 compare-and-swap on local memory (LMEM banks) #endif }; diff --git a/tests/regression/amo/main.cpp b/tests/regression/amo/main.cpp index 74bf4265c9..7b7d49a306 100644 --- a/tests/regression/amo/main.cpp +++ b/tests/regression/amo/main.cpp @@ -176,5 +176,5 @@ int main(int argc, char *argv[]) { } cleanup(); - return errors; + return errors != 0; } diff --git a/tests/regression/amo/testcases.h b/tests/regression/amo/testcases.h index ff69860e5e..bc3bdb4e7e 100644 --- a/tests/regression/amo/testcases.h +++ b/tests/regression/amo/testcases.h @@ -404,7 +404,37 @@ class Test_LRSC_LMEM : public ITestCase { } }; -// 15) Native compare-and-swap ladder: every hart races for each slot, so each +// 15) LMEM AMO op sweep: swap, and, or, xor, min, max, minu and maxu on local +// memory. Each group's leader checks its group's final words in the kernel +// and publishes the mismatch count with the group marker. +class Test_AMO_OPS_LMEM : public ITestCase { +public: + Test_AMO_OPS_LMEM(TestSuite* s) : ITestCase(s, "amo_ops_lmem") {} + void setup(uint32_t n, uint32_t* /*shared*/, uint32_t* per_hart) override { + for (uint32_t h = 0; h < n; ++h) per_hart[h] = 0; + } + int verify(uint32_t n, uint32_t /*iters*/, + const uint32_t* /*shared*/, const uint32_t* per_hart) override { + uint32_t groups = 0, mismatches = 0; + for (uint32_t h = 0; h < n; ++h) { + uint32_t v = per_hart[h]; + if (v >= CAS_LMEM_GROUP_MARK) { ++groups; v -= CAS_LMEM_GROUP_MARK; } + mismatches += v; + } + if (groups == 0) { + std::cout << " no group published -- the kernel did not run" << std::endl; + return 1; + } + if (mismatches != 0) { + std::cout << " " << mismatches << " wrong final word(s) across " + << groups << " group(s)" << std::endl; + return 1; + } + return 0; + } +}; + +// 16) Native compare-and-swap ladder: every hart races for each slot, so each // slot has exactly one winner. Summing the per-hart win counts checks the // per-lane return value, which a compare-and-swap that returned the wrong // old word would get wrong even while leaving the ladder itself plausible. @@ -438,7 +468,7 @@ class Test_CAS_LADDER : public ITestCase { } }; -// 16) LMEM compare-and-swap ladder: the cas_ladder race on local memory, so +// 17) LMEM compare-and-swap ladder: the cas_ladder race on local memory, so // the LMEM banks resolve the swap and the returned old word. Each group // has its own ladder with exactly one winner per slot; the leaders' publish // marker carries the group count, so the win counts must sum to @@ -495,6 +525,7 @@ inline TestSuite::TestSuite(vx_device_h device) : device_(device) { add_test(new Test_SELF_CONSISTENCY(this)); add_test(new Test_AMOADD_LMEM(this)); add_test(new Test_LRSC_LMEM(this)); + add_test(new Test_AMO_OPS_LMEM(this)); #if VX_CFG_EXT_ZACAS_ENABLED // Native compare-and-swap only exists when Zacas is built in. Without it a // compare-exchange lowers to an LR/SC retry loop, which is the construct diff --git a/tests/regression/async_barrier/main.cpp b/tests/regression/async_barrier/main.cpp index 9f62c03e8e..e775e1040f 100644 --- a/tests/regression/async_barrier/main.cpp +++ b/tests/regression/async_barrier/main.cpp @@ -263,7 +263,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/async_gbarrier/main.cpp b/tests/regression/async_gbarrier/main.cpp index 845212dc83..6bb42eb3b5 100644 --- a/tests/regression/async_gbarrier/main.cpp +++ b/tests/regression/async_gbarrier/main.cpp @@ -171,7 +171,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/basic/main.cpp b/tests/regression/basic/main.cpp index 0967e27ad4..c628d2d1ae 100755 --- a/tests/regression/basic/main.cpp +++ b/tests/regression/basic/main.cpp @@ -266,7 +266,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/common.mk b/tests/regression/common.mk index 25ffaf4462..ac503bee3b 100644 --- a/tests/regression/common.mk +++ b/tests/regression/common.mk @@ -21,6 +21,12 @@ ifneq ($(findstring -DVX_CFG_TCU_WGMMA_ENABLE,$(CONFIGS)),) CONFIGS := $(if $(findstring -DWGMMA_RS,$(CONFIGS))$(findstring -DWGMMA_SS,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DWGMMA_RS) endif +# config.mk folded the profile in before this app appended its own flags; fold +# again so an app flag replaces the profile's setting of the same knob. +ifneq ($(strip $(PROFILE)),) +CONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --profile=$(PROFILE) --xlen=$(XLEN) --merge-configs --cflags='$(CONFIGS)') +endif + XCONFIGS := $(shell python3 $(ROOT_DIR)/ci/gen_config.py --config=$(VORTEX_HOME)/VX_config.toml --cflags='$(CONFIGS) -DVX_CFG_XLEN=$(XLEN)') ifneq (,$(filter -DVX_CFG_EXT_C_ENABLE, $(XCONFIGS))) @@ -242,12 +248,42 @@ run-rtlsim: $(PROJECT) kernel.vxbin $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/rtlsim DESTDIR=$(VORTEX_RT_LIB) LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=rtlsim ./$(PROJECT) $(OPTS) +# FireSim takes the same targets as the card's native driver: xrtsim, hw_emu +# and hw on an XRT card; avedsim, sim and hw on an aved card. An image target +# runs FPGA_BIN_DIR's firesim.xclbin or firesim.vbin; the card is decided in +# sw/runtime/firesim/card.mk, and named to the run with its native driver's +# variables. With no TARGET given it runs the in-process metasimulator. +ifneq ($(filter run-firesim,$(MAKECMDGOALS)),) +ifeq ($(origin TARGET),file) +override TARGET := +endif +include $(VORTEX_HOME)/sw/runtime/firesim/card.mk +endif + run-firesim: $(PROJECT) kernel.vxbin - $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/firesim DESTDIR=$(VORTEX_RT_LIB) FIRESIM_CARD=$(FIRESIM_CARD) TARGET=$(TARGET) +ifneq ($(filter $(TARGET),xrtsim avedsim),) + LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(FIRESIM_PLATFORM), aved) ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh +endif + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/firesim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +else ifeq ($(TARGET), hw) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) else - XCL_EMULATION_MODE=$(TARGET) EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) + XCL_EMULATION_MODE=$(TARGET) XRT_INI_PATH=$(VORTEX_RT_SRC)/firesim/xrt.ini EMCONFIG_PATH=$(FPGA_BIN_DIR) XCLBIN_PATH=$(FPGA_BIN_DIR)/firesim.xclbin LD_LIBRARY_PATH=$(XILINX_XRT)/lib:$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=firesim ./$(PROJECT) $(OPTS) +endif + +# foldsim: TARGET=hw runs FPGA_BIN_DIR's foldsim.vbin (hw/syn/foldsim) on the V80; +# TARGET=avedsim, the default, simulates the same kernel in-process. +run-foldsim: $(PROJECT) kernel.vxbin + $(RUNTIME_ARGS) $(MAKE) -C $(VORTEX_RT_SRC)/foldsim DESTDIR=$(VORTEX_RT_LIB) TARGET=$(if $(filter hw,$(TARGET)),hw,avedsim) +ifeq ($(TARGET), hw) + $(VORTEX_HOME)/ci/v80_preflight.sh + VRT_DEVICE_BDF=$(VRT_DEVICE_BDF) VRT_VBIN_PATH=$(FPGA_BIN_DIR)/foldsim.vbin LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) +else + LD_LIBRARY_PATH=$(VORTEX_RT_LIB):$(LD_LIBRARY_PATH) VORTEX_DRIVER=foldsim ./$(PROJECT) $(OPTS) endif run-opae: $(PROJECT) kernel.vxbin diff --git a/tests/regression/conv3/main.cpp b/tests/regression/conv3/main.cpp index 93e67b278b..4a3f0b8f6d 100644 --- a/tests/regression/conv3/main.cpp +++ b/tests/regression/conv3/main.cpp @@ -291,7 +291,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/cta_cluster/main.cpp b/tests/regression/cta_cluster/main.cpp index 5c97e6bb3c..8cbdab5326 100644 --- a/tests/regression/cta_cluster/main.cpp +++ b/tests/regression/cta_cluster/main.cpp @@ -217,7 +217,7 @@ int main(int argc, char* argv[]) { } cleanup(); - if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors; } + if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors != 0; } std::cout << "PASSED\n"; return 0; } diff --git a/tests/regression/demo/main.cpp b/tests/regression/demo/main.cpp index 73b0387f42..b59869a27e 100644 --- a/tests/regression/demo/main.cpp +++ b/tests/regression/demo/main.cpp @@ -254,7 +254,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/diverge/main.cpp b/tests/regression/diverge/main.cpp index 41f77fab4d..ea092746a4 100644 --- a/tests/regression/diverge/main.cpp +++ b/tests/regression/diverge/main.cpp @@ -277,7 +277,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!\nFAILED!\n"; - return errors; + return errors != 0; } std::cout << "PASSED!\n"; diff --git a/tests/regression/dogfood/main.cpp b/tests/regression/dogfood/main.cpp index e2f6bda765..7c52206262 100644 --- a/tests/regression/dogfood/main.cpp +++ b/tests/regression/dogfood/main.cpp @@ -237,5 +237,5 @@ int main(int argc, char *argv[]) { std::cout << "cleanup" << std::endl; cleanup(); - return errors; + return errors != 0; } \ No newline at end of file diff --git a/tests/regression/dxa_copy_mcast/main.cpp b/tests/regression/dxa_copy_mcast/main.cpp index 16134186e1..0316d6c51f 100644 --- a/tests/regression/dxa_copy_mcast/main.cpp +++ b/tests/regression/dxa_copy_mcast/main.cpp @@ -195,7 +195,7 @@ int main(int argc, char* argv[]) { cleanup(); if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; - return errors; + return errors != 0; } std::cout << "PASSED\n"; return 0; diff --git a/tests/regression/fence/main.cpp b/tests/regression/fence/main.cpp index 30ce2da511..98c701057b 100644 --- a/tests/regression/fence/main.cpp +++ b/tests/regression/fence/main.cpp @@ -184,7 +184,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/fence_mp/Makefile b/tests/regression/fence_mp/Makefile new file mode 100644 index 0000000000..84b07d28fe --- /dev/null +++ b/tests/regression/fence_mp/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := fence_mp + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/fence_mp/common.h b/tests/regression/fence_mp/common.h new file mode 100644 index 0000000000..1b04613e07 --- /dev/null +++ b/tests/regression/fence_mp/common.h @@ -0,0 +1,39 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define FENCE_MP_MAGIC 0x5a5a0000u + +// The data lives in the even 64-byte lines of its buffer and the flag in an +// odd one, so on a two-bank dcache they leave through different banks: a test +// can delay one bank's writes and check that only a release orders the flag +// behind them. +#define MP_LINE_WORDS 16 + +static inline uint32_t mp_data_index(uint32_t k) { + return (k / MP_LINE_WORDS) * (2 * MP_LINE_WORDS) + (k % MP_LINE_WORDS); +} + +static inline uint32_t mp_data_words(uint32_t count) { + return ((count + MP_LINE_WORDS - 1) / MP_LINE_WORDS) * (2 * MP_LINE_WORDS); +} + +// How the producer publishes the flag and the consumer observes it. +enum { + MP_FENCE = 0, // full fences on both sides + MP_ACQ_REL = 1, // a release fence before the flag store, an acquire fence after each flag load + MP_AMO = 2 // the flag swapped in by an amoswap.rl, read by an amoor.aq +}; + +typedef struct { + uint32_t num_tasks; // threads launched + uint32_t count; // data words passed from producer to consumer + uint32_t spin_limit; // consumer polls before giving up + uint32_t mode; // MP_* + uint64_t data_addr; // count words, zero on entry + uint64_t flag_addr; // one word, zero on entry + uint64_t out_addr; // [0] mismatches, [1] 1 if the consumer timed out, [2] pre-read sum +} kernel_arg_t; + +#endif diff --git a/tests/regression/fence_mp/kernel.cpp b/tests/regression/fence_mp/kernel.cpp new file mode 100644 index 0000000000..b552343e02 --- /dev/null +++ b/tests/regression/fence_mp/kernel.cpp @@ -0,0 +1,78 @@ +#include +#include +#include "common.h" + +// Message passing across cores through plain memory: the producer (the first +// thread) writes data, then raises a flag; the consumer (the last thread, on +// the last core) reads the data once beforehand so stale copies sit in its +// caches, polls the flag, and checks the data. The release must make the data +// visible across every cache the two cores do not share before the flag, and +// the acquire must drop the consumer's stale copies, whether they are full +// fences, acquire/release fences, or the .aq/.rl bits of atomics. +static void publish(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: + __atomic_thread_fence(__ATOMIC_RELEASE); + *flag = 1; + break; + case MP_AMO: + __atomic_exchange_n(const_cast(flag), 1u, __ATOMIC_RELEASE); + break; + default: + vx_fence(); + *flag = 1; + vx_fence(); + break; + } +} + +static bool observe(volatile uint32_t* flag, uint32_t mode) { + switch (mode) { + case MP_ACQ_REL: { + uint32_t f = *flag; + __atomic_thread_fence(__ATOMIC_ACQUIRE); + return f != 0; + } + case MP_AMO: + return __atomic_fetch_or(const_cast(flag), 0u, __ATOMIC_ACQUIRE) != 0; + default: + vx_fence(); + return *flag != 0; + } +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + auto data = reinterpret_cast(arg->data_addr); + auto flag = reinterpret_cast(arg->flag_addr); + auto out = reinterpret_cast(arg->out_addr); + if (gid == 0) { + for (uint32_t k = 0; k < arg->count; ++k) { + data[mp_data_index(k)] = FENCE_MP_MAGIC + k; + } + publish(flag, arg->mode); + } else if (gid == arg->num_tasks - 1) { + uint32_t sum = 0; + for (uint32_t k = 0; k < arg->count; ++k) { + sum += data[mp_data_index(k)]; + } + out[2] = sum; + uint32_t timeout = 1; + for (uint32_t s = 0; s < arg->spin_limit; ++s) { + if (observe(flag, arg->mode)) { + timeout = 0; + break; + } + } + if (arg->mode == MP_FENCE) { + vx_fence(); + } + // newest first: the words written last are the ones still in flight + uint32_t errors = 0; + for (uint32_t k = arg->count; k-- > 0;) { + errors += (data[mp_data_index(k)] != FENCE_MP_MAGIC + k); + } + out[0] = errors; + out[1] = timeout; + } +} diff --git a/tests/regression/fence_mp/main.cpp b/tests/regression/fence_mp/main.cpp new file mode 100644 index 0000000000..78f32f09e3 --- /dev/null +++ b/tests/regression/fence_mp/main.cpp @@ -0,0 +1,140 @@ +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 64; +uint32_t spin_limit = 4096; +uint32_t mode = MP_FENCE; + +vx_device_h device = nullptr; +vx_buffer_h data_buffer = nullptr; +vx_buffer_h flag_buffer = nullptr; +vx_buffer_h out_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n words] [-s spin limit] [-m mode: 0 fences, 1 acquire/release fences, 2 .aq/.rl atomics] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:s:m:k:h")) != -1) { + switch (c) { + case 'n': count = atoi(optarg); break; + case 's': spin_limit = atoi(optarg); break; + case 'm': mode = atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 'h': show_usage(); exit(0); break; + default: show_usage(); exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (data_buffer) vx_buffer_release(data_buffer); + if (flag_buffer) vx_buffer_release(flag_buffer); + if (out_buffer) vx_buffer_release(out_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + RT_CHECK(vx_device_open(0, &device)); + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + // one thread per hardware slot, so producer and consumer are resident together + uint32_t total_threads = num_cores * num_warps * num_threads; + + kernel_arg.num_tasks = total_threads; + kernel_arg.count = count; + kernel_arg.spin_limit = spin_limit; + kernel_arg.mode = mode; + + std::vector zeros(mp_data_words(count), 0); + std::vector out(4, 0); + uint32_t data_size = mp_data_words(count) * sizeof(uint32_t); + uint32_t out_size = out.size() * sizeof(uint32_t); + RT_CHECK(vx_buffer_create(device, data_size, VX_MEM_READ_WRITE, &data_buffer)); + RT_CHECK(vx_buffer_address(data_buffer, &kernel_arg.data_addr)); + // the flag sits in the buffer's second (odd) line + RT_CHECK(vx_buffer_create(device, 2 * MP_LINE_WORDS * sizeof(uint32_t), VX_MEM_READ_WRITE, &flag_buffer)); + RT_CHECK(vx_buffer_address(flag_buffer, &kernel_arg.flag_addr)); + kernel_arg.flag_addr += MP_LINE_WORDS * sizeof(uint32_t); + RT_CHECK(vx_buffer_create(device, out_size, VX_MEM_READ_WRITE, &out_buffer)); + RT_CHECK(vx_buffer_address(out_buffer, &kernel_arg.out_addr)); + RT_CHECK(vx_enqueue_write(queue, data_buffer, 0, zeros.data(), data_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, flag_buffer, MP_LINE_WORDS * sizeof(uint32_t), zeros.data(), sizeof(uint32_t), 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, out_buffer, 0, out.data(), out_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = total_threads; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + RT_CHECK(vx_enqueue_read(queue, out.data(), out_buffer, 0, out_size, 1, &launch_ev, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + int errors = 0; + if (out[1] != 0) { + std::cout << "consumer timed out waiting for the flag" << std::endl; + ++errors; + } + if (out[0] != 0) { + std::cout << "consumer saw " << std::dec << out[0] << " stale data words" << std::endl; + errors += out[0]; + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/flush_coherence/Makefile b/tests/regression/flush_coherence/Makefile new file mode 100644 index 0000000000..5ceced321a --- /dev/null +++ b/tests/regression/flush_coherence/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := flush_coherence + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 -r16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/flush_coherence/common.h b/tests/regression/flush_coherence/common.h new file mode 100644 index 0000000000..6b04d30d2d --- /dev/null +++ b/tests/regression/flush_coherence/common.h @@ -0,0 +1,22 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +typedef struct { + uint32_t count; + uint32_t round; + uint32_t rotate; + uint32_t burst; + uint32_t dxa_slot; // 1 + DXA descriptor slot of src, 0 to load with the LSU + uint64_t src_addr; + uint64_t dst_addr; + uint64_t acc_addr; + uint64_t tail_addr; +} kernel_arg_t; + +inline uint32_t tail_value(uint32_t i, uint32_t k, uint32_t round) { + return (i * 0x01000193u) ^ (k << 20) ^ (round * 0x2545F491u); +} + +#endif diff --git a/tests/regression/flush_coherence/kernel.cpp b/tests/regression/flush_coherence/kernel.cpp new file mode 100644 index 0000000000..1c3e14ed9f --- /dev/null +++ b/tests/regression/flush_coherence/kernel.cpp @@ -0,0 +1,50 @@ +#include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#include +#endif +#include "common.h" + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + if (tid >= arg->count) { + return; + } + // The rotation moves each element, and so each line, to another thread + // and core from one launch to the next. + uint32_t i = tid + arg->rotate; + if (i >= arg->count) { + i -= arg->count; + } + auto src = reinterpret_cast(arg->src_addr); + auto dst = reinterpret_cast(arg->dst_addr); + auto acc = reinterpret_cast(arg->acc_addr); + uint32_t v; +#ifdef VX_CFG_EXT_DXA_ENABLE + if (arg->dxa_slot != 0) { + // The tile copy reads memory beside the dcache, not through it, so it + // sees the previous launch's stores only once a flush has written them + // back. The host sets no rotation, so this block's tile holds element i. + auto shmem = reinterpret_cast(__local_mem()); + vortex::barrier bar(0); + if (get_sub_group_id() == 0) { + bar.expect_tx(1); + vx_dxa_issue_1d_wg(arg->dxa_slot - 1, bar.id(), shmem, blockIdx.x * blockDim.x); + } + bar.arrive_and_wait(); + v = shmem[threadIdx.x]; + } else +#endif + { + v = src[i]; + } + acc[i] += v ^ arg->round; + // A store burst saturates the shared levels, so the final stores are + // still queued in front of them when the flushes start. + auto tail = reinterpret_cast(arg->tail_addr); + for (uint32_t k = 0; k < arg->burst; ++k) { + tail[k * arg->count + i] = tail_value(i, k, arg->round); + } + // Ending on a store leaves the kernel's last writes in flight at exit. + dst[i] = v * 3u + arg->round; +} diff --git a/tests/regression/flush_coherence/main.cpp b/tests/regression/flush_coherence/main.cpp new file mode 100644 index 0000000000..9f45ff84ba --- /dev/null +++ b/tests/regression/flush_coherence/main.cpp @@ -0,0 +1,298 @@ +// Kernel-boundary flush coherence under back-to-back launches. +// +// Every round the host rewrites src, then launches a kernel that reads src, +// accumulates into acc and writes dst. The element-to-thread mapping is +// rotated on three rounds out of four, so a line usually moves to another +// thread and often to another core. The post-launch cache flush must: +// - publish each launch's writes before the host reads dst or acc; +// - invalidate every core's (and every shared level's) copy of src before +// the next launch, or a core re-reading a line it cached would see the +// previous round's value; +// - make one launch's acc writes visible to the next launch's reads on +// other cores, also when no host transfer separates them (odd rounds do +// not read dst back). +// With -b, each thread ends on a burst of stores to a tail buffer, so the +// last stores may still be queued in front of the shared levels when the +// flushes start, and every round is checked. +// With -x, each launch loads src with a DXA tile copy, which reaches memory +// beside the dcache, instead of with the LSU. +// With -c, the rounds form one command batch with no host transfer between +// kernels: each round reads the buffer the previous round wrote (ping-pong), +// and the host only reads back after the last round. Kernel-boundary flushes +// alone must then make each round's writes visible to the next, including to +// a core whose cache still holds the lines from two rounds earlier. + +#include +#ifdef VX_CFG_EXT_DXA_ENABLE +#include +#endif +#include "common.h" + +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ + } while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t count = 2048; +uint32_t rounds = 16; +uint32_t burst = 0; +bool chain = false; +bool use_dxa = false; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:r:b:k:cxh")) != -1) { + switch (c) { + case 'n': count = (uint32_t)std::atoi(optarg); break; + case 'r': rounds = (uint32_t)std::atoi(optarg); break; + case 'b': burst = (uint32_t)std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 'c': chain = true; break; + case 'x': use_dxa = true; break; + default: + std::printf("Usage: [-k kernel] [-n words] [-r rounds] [-b burst] [-c] [-x] [-h]\n"); + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t src_value(uint32_t i, uint32_t round) { + uint32_t x = i * 0x9E3779B1u + round * 0x85EBCA77u + 1; + x ^= x >> 15; + return x * 0x2C1B3C6Du; +} + +#ifdef VX_CFG_EXT_DXA_ENABLE +void program_dxa(vx_device_h dev, uint32_t slot, uint64_t addr, uint32_t tile) { + if (vortex::dxa::program_1d(dev, slot, addr, count, tile, sizeof(uint32_t)) != 0) { + std::fprintf(stderr, "FAIL: DXA descriptor %u\n", slot); + std::exit(1); + } +} +#endif + +uint32_t rotation(uint32_t round) { + return (round % 4 == 0) ? 0 : (round * 0x9E3779B1u) % count; +} + +int check(const char* what, uint32_t round, const std::vector& got, + const std::vector& ref) { + int errors = 0; + for (size_t i = 0; i < ref.size(); ++i) { + if (got[i] != ref[i]) { + if (errors < 8) { + std::printf("*** round %u %s[%zu]: expected=0x%08x actual=0x%08x\n", + round, what, i, ref[i], got[i]); + } + ++errors; + } + } + return errors; +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + const uint64_t bytes = (uint64_t)count * sizeof(uint32_t); + const uint64_t tail_bytes = bytes * burst; + std::printf("flush_coherence: n=%u rounds=%u burst=%u chain=%d\n", count, rounds, burst, chain); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr, dst_buf = nullptr, acc_buf = nullptr, tail_buf = nullptr; + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &acc_buf)); + if (burst) { + CHECK(vx_buffer_create(dev, tail_bytes, VX_MEM_WRITE, &tail_buf)); + } + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t arg{}; + arg.count = count; + CHECK(vx_buffer_address(src_buf, &arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &arg.dst_addr)); + CHECK(vx_buffer_address(acc_buf, &arg.acc_addr)); + if (burst) { + CHECK(vx_buffer_address(tail_buf, &arg.tail_addr)); + } + arg.burst = burst; + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &count, grid, block)); + + // DXA descriptor slot 0 covers src, 1 dst and 2 the chain's middle buffer. + uint32_t lmem_size = 0; + if (use_dxa) { +#ifdef VX_CFG_EXT_DXA_ENABLE + if (count % block[0] != 0) { + std::printf("Error: -x needs a word count that is a multiple of the block size (%u)\n", block[0]); + return -1; + } + lmem_size = block[0] * sizeof(uint32_t); + program_dxa(dev, 0, arg.src_addr, block[0]); + program_dxa(dev, 1, arg.dst_addr, block[0]); +#else + std::printf("Error: -x needs a build with VX_CFG_EXT_DXA_ENABLE\n"); + return -1; +#endif + } + + // Enqueued writes and launches read their host data asynchronously, so + // each round gets its own source buffer and argument block. + std::vector> h_src(rounds, std::vector(count)); + std::vector args(rounds, arg); + std::vector h_dst(count), h_acc(count, 0); + std::vector ref_dst(count), ref_acc(count, 0); + std::vector h_tail(count * burst), ref_tail(count * burst); + CHECK(vx_enqueue_write(q, acc_buf, 0, h_acc.data(), bytes, 0, nullptr, nullptr)); + + int errors = 0; + vx_buffer_h mid_buf = nullptr; + if (chain) { + // round r reads the buffer round r-1 wrote: src, then dst and mid in turn + CHECK(vx_buffer_create(dev, bytes, VX_MEM_READ_WRITE, &mid_buf)); + uint64_t pp_addr[2]; + CHECK(vx_buffer_address(dst_buf, &pp_addr[0])); + CHECK(vx_buffer_address(mid_buf, &pp_addr[1])); +#ifdef VX_CFG_EXT_DXA_ENABLE + if (use_dxa) { + program_dxa(dev, 2, pp_addr[1], block[0]); + } +#endif + auto& src = h_src[0]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, 0); + ref_dst[i] = src[i]; + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + std::vector lis(rounds); + std::vector cmds(rounds); + for (uint32_t r = 0; r < rounds; ++r) { + for (uint32_t i = 0; i < count; ++i) { + ref_acc[i] += ref_dst[i] ^ r; + ref_dst[i] = ref_dst[i] * 3u + r; + } + args[r].round = r; + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].src_addr = (r == 0) ? arg.src_addr : pp_addr[(r - 1) % 2]; + args[r].dxa_slot = !use_dxa ? 0 : ((r == 0) ? 1 : 2 + (r - 1) % 2); + args[r].dst_addr = pp_addr[r % 2]; + vx_launch_info_t& li = lis[r]; + li = vx_launch_info_t{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; + cmds[r].type = VX_COMMAND_LAUNCH; + cmds[r].data.launch = &li; + } + for (uint32_t i = 0; i < count; ++i) { + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, rounds - 1); + } + } + CHECK(vx_enqueue_commands(q, cmds.data(), rounds, 0, nullptr, nullptr)); + + vx_buffer_h last_buf = ((rounds - 1) % 2 == 0) ? dst_buf : mid_buf; + CHECK(vx_enqueue_read(q, h_dst.data(), last_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", rounds - 1, h_dst, ref_dst); + if (burst) { + errors += check("tail", rounds - 1, h_tail, ref_tail); + } + } + for (uint32_t r = 0; !chain && r < rounds; ++r) { + auto& src = h_src[r]; + for (uint32_t i = 0; i < count; ++i) { + src[i] = src_value(i, r); + ref_dst[i] = src[i] * 3u + r; + ref_acc[i] += src[i] ^ r; + for (uint32_t k = 0; k < burst; ++k) { + ref_tail[k * count + i] = tail_value(i, k, r); + } + } + CHECK(vx_enqueue_write(q, src_buf, 0, src.data(), bytes, 0, nullptr, nullptr)); + + args[r].round = r; + args[r].rotate = use_dxa ? 0 : rotation(r); + args[r].dxa_slot = use_dxa ? 1 : 0; + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &args[r]; + li.args_size = sizeof(args[r]); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0] = block[0]; + li.lmem_size = lmem_size; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, nullptr)); + + if (burst || r % 2 == 0 || r + 1 == rounds) { + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, bytes, 0, nullptr, nullptr)); + if (burst) { + CHECK(vx_enqueue_read(q, h_tail.data(), tail_buf, 0, tail_bytes, 0, nullptr, nullptr)); + } + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("dst", r, h_dst, ref_dst); + if (burst) { + errors += check("tail", r, h_tail, ref_tail); + } + } + } + + CHECK(vx_enqueue_read(q, h_acc.data(), acc_buf, 0, bytes, 0, nullptr, nullptr)); + CHECK(vx_queue_finish(q, VX_TIMEOUT_INFINITE)); + errors += check("acc", rounds, h_acc, ref_acc); + + if (mid_buf) { + vx_buffer_release(mid_buf); + } + if (tail_buf) { + vx_buffer_release(tail_buf); + } + vx_buffer_release(acc_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::printf("Found %d errors!\nFAILED!\n", errors); + return 1; + } + std::printf("PASSED!\n"); + return 0; +} diff --git a/tests/regression/gbar_phase/main.cpp b/tests/regression/gbar_phase/main.cpp index 9a0a77adbc..7c947a2de3 100644 --- a/tests/regression/gbar_phase/main.cpp +++ b/tests/regression/gbar_phase/main.cpp @@ -168,7 +168,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/io_addr/main.cpp b/tests/regression/io_addr/main.cpp index c3334ef377..1907980435 100644 --- a/tests/regression/io_addr/main.cpp +++ b/tests/regression/io_addr/main.cpp @@ -226,7 +226,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/lazy_fetch/Makefile b/tests/regression/lazy_fetch/Makefile new file mode 100644 index 0000000000..7a39f6c48c --- /dev/null +++ b/tests/regression/lazy_fetch/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lazy_fetch + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lazy_fetch/common.h b/tests/regression/lazy_fetch/common.h new file mode 100644 index 0000000000..5b6758319b --- /dev/null +++ b/tests/regression/lazy_fetch/common.h @@ -0,0 +1,181 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Each thread owns NUM_CASES regions per round, one cache line (REGION_WORDS +// words) each, laid out round-major so neighbouring lines belong to different +// threads and the buffer outgrows the cache across rounds. +#define REGION_WORDS 16 +#define NUM_CASES 6 +#define LANE_WORDS 8 +// Same-set lines: one per thread and round, a page apart, so they compete for +// one cache set and a line being refetched is often evicted before its fill. +#define SET_STRIDE_WORDS 1024 +// A whole cache line of the largest configured line size: two 64-byte +// sectors, stored before any of it is loaded. +#define FULL_LINE_WORDS 32 +#define SECTOR_WORDS 16 +// Lines a flood launch reads one word of, each a memory block apart. +#define FLOOD_LINE_BYTES 64 + +// How a phase-2 launch stores and loads its lines. +enum { + FULL_SEPARATE = 0, // each thread stores its lines, then loads them all back + FULL_CHAIN = 1, // a block's threads store a sector together, load it at once, store, load + FULL_MIXED = 2 // as FULL_SEPARATE, with every odd line left partial +}; + +enum { + CASE_HALF = 0, // store half the line, then load all of it (same thread) + CASE_BYTES = 1, // byte/halfword stores, then load the words holding them + CASE_AMO = 2, // stores, then AMO add and LR/SC on written and unwritten words + CASE_STORE = 3, // byte stores only: checked after the kernel + CASE_WARP = 4, // one warp stores half the line, the next warp loads all of it + CASE_LANE = 5 // a warp's lanes store into one line, then load its unwritten half +}; + +typedef struct { + uint64_t buf_addr; + uint64_t set_addr; + uint64_t err_addr; + uint32_t rounds; + uint32_t num_threads; // threads in the grid + uint32_t warp_size; + uint32_t cross_warp; // blocks hold an even number of warps + uint32_t amo; // the device implements the A extension + uint32_t phase; // 0: exercise the cases, 1: re-check every region, 2: full lines, 3: flood + uint32_t rotate; // phase 1: check the regions of thread (gid + rotate) + uint64_t full_addr; + uint32_t full_lines; // phase 2: lines per thread (per warp in FULL_CHAIN); 3: lines it reads + uint32_t full_mode; // phase 2: FULL_* + uint32_t full_skip; // phase 2: leave each sector's last word unwritten + uint32_t num_blocks; +} kernel_arg_t; + +// Pre-pattern the host writes over the whole buffer. +static inline uint8_t pattern_byte(uint32_t i) { + return (uint8_t)(i * 37u + (i >> 9) + 0x5Au); +} + +static inline uint32_t pattern_word(uint32_t word_idx) { + uint32_t b = word_idx * 4; + return (uint32_t)pattern_byte(b) + | ((uint32_t)pattern_byte(b + 1) << 8) + | ((uint32_t)pattern_byte(b + 2) << 16) + | ((uint32_t)pattern_byte(b + 3) << 24); +} + +static inline uint32_t store_value(uint32_t region, uint32_t w) { + return 0xC0000000u | (region << 4) | w; +} + +// Word w of a line that a phase-2 launch leaves unwritten. +static inline bool full_skipped(uint32_t line, uint32_t w, const kernel_arg_t* arg) { + bool partial = (arg->full_mode == FULL_MIXED) ? ((line & 1) != 0) : (arg->full_skip != 0); + return partial && (w % SECTOR_WORDS) == (SECTOR_WORDS - 1); +} + +static inline uint32_t full_value(uint32_t line, uint32_t w) { + return 0x50000000u | (line << 5) | w; +} + +// The second store a FULL_CHAIN thread makes to its word of each sector. +static inline uint32_t full_value2(uint32_t line, uint32_t w) { + return 0x60000000u | (line << 5) | w; +} + +// Word `word_idx` of a phase-2 buffer once its launch has run; the host +// pre-patterns the buffer. +static inline uint32_t full_expected(uint32_t word_idx, const kernel_arg_t* arg) { + uint32_t line = word_idx / FULL_LINE_WORDS; + uint32_t w = word_idx % FULL_LINE_WORDS; + if (full_skipped(line, w, arg)) { + return pattern_word(word_idx); + } + if (arg->full_mode == FULL_CHAIN) { + return full_value2(line, w); + } + return full_value(line, w); +} + +static inline uint32_t region_index(uint32_t round, uint32_t num_threads, uint32_t gid, uint32_t c) { + return (round * num_threads + gid) * NUM_CASES + c; +} + +static inline uint32_t set_line(uint32_t round, uint32_t num_threads, uint32_t gid) { + return round * num_threads + gid; +} + +// Expected word `word_idx` of the same-set buffer once the kernel has run: +// each line's word 0 is stored, the rest keeps the pre-pattern. +static inline uint32_t expected_set_word(uint32_t word_idx) { + if ((word_idx % SET_STRIDE_WORDS) == 0) { + return 0xA0000000u | ((word_idx / SET_STRIDE_WORDS) << 4); + } + return pattern_word(word_idx); +} + +// Owner thread of a CASE_WARP region writes it when it sits in an even warp +// of its block; the thread one warp up reads it. +static inline bool warp_producer(uint32_t gid, uint32_t warp_size) { + return ((gid / warp_size) & 1) == 0; +} + +// Expected content of word w of region `region` once the kernel has run. +static inline uint32_t expected_word(uint32_t region, uint32_t w, const kernel_arg_t* arg) { + uint32_t p = pattern_word(region * REGION_WORDS + w); + uint32_t v = store_value(region, w); + switch (region % NUM_CASES) { + case CASE_HALF: + return (w < REGION_WORDS / 2) ? v : p; + case CASE_BYTES: + if (w == 1) { + return (p & ~0x0000ff00u) | 0x0000ee00u; + } + if (w == 10) { + return (p & ~0x0000ffffu) | 0x0000beefu; + } + return p; + case CASE_AMO: + if (w < 4 && !arg->amo) { + return v; + } + if (!arg->amo) { + return p; + } + if (w == 1) { + return v + 3; + } + if (w < 4) { + return v; + } + if (w == 8) { + return p + 5; + } + if (w == 12) { + return p + 1; + } + return p; + case CASE_STORE: + return ((w & 1) == 0) ? ((p & ~0xffu) | (0x30u + w)) : p; + case CASE_WARP: { + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + if (arg->cross_warp && warp_producer(gid, arg->warp_size) && w >= REGION_WORDS / 2) { + return v; + } + return p; + } + default: { + // the region of each warp's lane 0 is shared by the warp's first lanes + uint32_t gid = (region / NUM_CASES) % arg->num_threads; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if ((gid % arg->warp_size) == 0 && w < lanes) { + return v; + } + return p; + } + } +} + +#endif diff --git a/tests/regression/lazy_fetch/kernel.cpp b/tests/regression/lazy_fetch/kernel.cpp new file mode 100644 index 0000000000..a704245ec5 --- /dev/null +++ b/tests/regression/lazy_fetch/kernel.cpp @@ -0,0 +1,239 @@ +#include +#include +#include "common.h" + +// Every access goes through volatile pointers: the point is to send each load +// to the cache right behind the stores it must observe, not to let the +// compiler forward the stored value. +static inline volatile uint32_t* region_ptr(const kernel_arg_t* arg, uint32_t region) { + return reinterpret_cast(arg->buf_addr) + region * REGION_WORDS; +} + +static inline uint32_t check_word(volatile uint32_t* r, uint32_t region, uint32_t w, + const kernel_arg_t* arg) { + uint32_t ref = expected_word(region, w, arg); + return (r[w] != ref) ? 1 : 0; +} + +static uint32_t run_half(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < REGION_WORDS / 2; ++w) { + r[w] = store_value(region, w); + } + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +static uint32_t run_bytes(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + reinterpret_cast(r)[5] = 0xee; + uint32_t errors = check_word(r, region, 1, arg); + reinterpret_cast(r)[20] = 0xbeef; + errors += check_word(r, region, 10, arg); + errors += check_word(r, region, 0, arg); + errors += check_word(r, region, 15, arg); + return errors; +} + +static uint32_t run_amo(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + for (uint32_t w = 0; w < 4; ++w) { + r[w] = store_value(region, w); + } + if (!arg->amo) { + return check_word(r, region, 8, arg) + check_word(r, region, 1, arg); + } + uint32_t errors = 0; + uint32_t* word8 = const_cast(&r[8]); + uint32_t* word1 = const_cast(&r[1]); + uint32_t* word12 = const_cast(&r[12]); + uint32_t old8 = __atomic_fetch_add(word8, 5u, __ATOMIC_SEQ_CST); + errors += (old8 != pattern_word(region * REGION_WORDS + 8)) ? 1 : 0; + uint32_t old1 = __atomic_fetch_add(word1, 3u, __ATOMIC_SEQ_CST); + errors += (old1 != store_value(region, 1)) ? 1 : 0; + // Only this thread touches the word, so the SC fails only spuriously. + uint32_t first = 0; + bool done = false; + for (uint32_t attempt = 0; attempt < 64 && !done; ++attempt) { + uint32_t old, sc_fail; + asm volatile ("lr.w %0, (%1)" : "=r"(old) : "r"(word12) : "memory"); + if (attempt == 0) { + first = old; + } + uint32_t newv = old + 1; + asm volatile ("sc.w %0, %2, (%1)" : "=r"(sc_fail) : "r"(word12), "r"(newv) : "memory"); + done = (sc_fail == 0); + } + errors += (!done || first != pattern_word(region * REGION_WORDS + 12)) ? 1 : 0; + return errors; +} + +static void run_store(const kernel_arg_t* arg, uint32_t region) { + auto b = reinterpret_cast(region_ptr(arg, region)); + for (uint32_t w = 0; w < REGION_WORDS; w += 2) { + b[w * 4] = (uint8_t)(0x30 + w); + } +} + +// Each lane's load follows the warp's coalesced store into the same line +// right away, so it reaches the cache while the store's allocation is still +// pending and must wait for the unwritten half to be fetched. +static uint32_t run_lane(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t lane = gid % arg->warp_size; + uint32_t lanes = (arg->warp_size < LANE_WORDS) ? arg->warp_size : LANE_WORDS; + if (lane >= lanes) { + return 0; + } + uint32_t region = region_index(round, arg->num_threads, gid - lane, CASE_LANE); + auto r = region_ptr(arg, region); + uint32_t value = store_value(region, lane); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, %3(%1)" + : "=&r"(loaded) + : "r"(&r[lane]), "r"(value), "i"(LANE_WORDS * 4) + : "memory"); + uint32_t errors = (loaded != expected_word(region, lane + LANE_WORDS, arg)) ? 1 : 0; + errors += check_word(r, region, (lane + 1) % lanes, arg); + return errors; +} + +// Store word 0 of this thread's same-set line, then load word 8 right away. +static uint32_t run_set(const kernel_arg_t* arg, uint32_t round, uint32_t gid) { + uint32_t line = set_line(round, arg->num_threads, gid); + auto r = reinterpret_cast(arg->set_addr) + line * SET_STRIDE_WORDS; + uint32_t value = expected_set_word(line * SET_STRIDE_WORDS); + uint32_t loaded; + asm volatile ("sw %2, 0(%1)\n\tlw %0, 32(%1)" + : "=&r"(loaded) + : "r"(r), "r"(value) + : "memory"); + uint32_t errors = (loaded != expected_set_word(line * SET_STRIDE_WORDS + 8)) ? 1 : 0; + errors += (r[0] != value) ? 1 : 0; + return errors; +} + +static uint32_t check_region(const kernel_arg_t* arg, uint32_t region) { + auto r = region_ptr(arg, region); + uint32_t errors = 0; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += check_word(r, region, w, arg); + } + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t errors = 0; + auto err = reinterpret_cast(arg->err_addr); + + if (arg->phase == 3) { + // Read fresh lines from memory, one word each. + auto base = reinterpret_cast(arg->full_addr); + uint32_t sum = 0; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + sum += base[line * (FLOOD_LINE_BYTES / sizeof(uint32_t))]; + } + asm volatile ("" :: "r"(sum)); + err[gid] = 0; + return; + } + + if (arg->phase == 2 && arg->full_mode == FULL_CHAIN) { + // The first SECTOR_WORDS threads of a block store one word each of a + // sector, every warp at once after a barrier, and load it right away: the + // loads reach the cache while the stores' allocation is still pending. + // The second store lands while that chain is being refetched. + auto base = reinterpret_cast(arg->full_addr); + uint32_t t = threadIdx.x; + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_blocks + blockIdx.x; + for (uint32_t s = 0; s < FULL_LINE_WORDS; s += SECTOR_WORDS) { + uint32_t idx = line * FULL_LINE_WORDS + s + t; + bool skip = full_skipped(line, s + t, arg); + // a skipped word is stored to a scratch word: no branch delays the load + volatile uint32_t* dst = skip ? &err[arg->num_threads + gid] : &base[idx]; + __syncthreads(); + if (t < SECTOR_WORDS) { + *dst = full_value(line, s + t); + uint32_t first = base[idx]; + *dst = full_value2(line, s + t); + uint32_t second = base[idx]; + errors += (first != (skip ? pattern_word(idx) : full_value(line, s + t))) ? 1 : 0; + errors += (second != full_expected(idx, arg)) ? 1 : 0; + } + } + } + err[gid] = errors; + return; + } + + if (arg->phase == 2) { + // Store this thread's lines, all of each sector or all but one word, + // then load them all back. + auto base = reinterpret_cast(arg->full_addr); + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + if (!full_skipped(line, w, arg)) { + base[line * FULL_LINE_WORDS + w] = full_value(line, w); + } + } + } + for (uint32_t i = 0; i < arg->full_lines; ++i) { + uint32_t line = i * arg->num_threads + gid; + for (uint32_t w = 0; w < FULL_LINE_WORDS; ++w) { + uint32_t idx = line * FULL_LINE_WORDS + w; + errors += (base[idx] != full_expected(idx, arg)) ? 1 : 0; + } + } + err[gid] = errors; + return; + } + + if (arg->phase != 0) { + // Every region again, from another thread (another core when there are + // several), after the launch boundary flushed the partial lines. + uint32_t owner = (gid + arg->rotate) % arg->num_threads; + for (uint32_t round = 0; round < arg->rounds; ++round) { + for (uint32_t c = 0; c < NUM_CASES; ++c) { + errors += check_region(arg, region_index(round, arg->num_threads, owner, c)); + } + auto r = reinterpret_cast(arg->set_addr); + uint32_t base = set_line(round, arg->num_threads, owner) * SET_STRIDE_WORDS; + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + errors += (r[base + w] != expected_set_word(base + w)) ? 1 : 0; + } + } + err[arg->num_threads + gid] = errors; + return; + } + + for (uint32_t round = 0; round < arg->rounds; ++round) { + errors += run_half(arg, region_index(round, arg->num_threads, gid, CASE_HALF)); + errors += run_bytes(arg, region_index(round, arg->num_threads, gid, CASE_BYTES)); + errors += run_amo(arg, region_index(round, arg->num_threads, gid, CASE_AMO)); + run_store(arg, region_index(round, arg->num_threads, gid, CASE_STORE)); + errors += run_lane(arg, round, gid); + errors += run_set(arg, round, gid); + if (arg->cross_warp) { + bool producer = warp_producer(gid, arg->warp_size); + if (producer) { + uint32_t region = region_index(round, arg->num_threads, gid, CASE_WARP); + auto r = region_ptr(arg, region); + for (uint32_t w = REGION_WORDS / 2; w < REGION_WORDS; ++w) { + r[w] = store_value(region, w); + } + } + vx_fence(); + __syncthreads(); + if (!producer) { + errors += check_region(arg, region_index(round, arg->num_threads, gid - arg->warp_size, CASE_WARP)); + } + } + } + err[gid] = errors; +} diff --git a/tests/regression/lazy_fetch/main.cpp b/tests/regression/lazy_fetch/main.cpp new file mode 100644 index 0000000000..413e17bf48 --- /dev/null +++ b/tests/regression/lazy_fetch/main.cpp @@ -0,0 +1,373 @@ +#include +#include +#include +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t rounds = 16; +bool full_mode = false; + +vx_device_h device = nullptr; +vx_buffer_h buf_buffer = nullptr; +vx_buffer_h err_buffer = nullptr; +vx_buffer_h set_buffer = nullptr; +std::vector full_buffers; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n rounds] [-f: fully written lines read no memory] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:k:fh")) != -1) { + switch (c) { + case 'n': + rounds = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'f': + full_mode = true; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (buf_buffer) vx_buffer_release(buf_buffer); + if (err_buffer) vx_buffer_release(err_buffer); + if (set_buffer) vx_buffer_release(set_buffer); + for (auto b : full_buffers) { + vx_buffer_release(b); + } + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +static int launch(const kernel_arg_t& arg, uint32_t grid, uint32_t block) { + vx_event_h ev = nullptr; + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &arg; + li.args_size = sizeof(arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0] = block; + int ret = vx_enqueue_launch(queue, &li, 0, nullptr, &ev); + if (ret != 0) { + return ret; + } + ret = vx_event_wait_value(ev, 1, VX_TIMEOUT_INFINITE); + vx_event_release(ev); + return ret; +} + +static int mem_reads(uint64_t* value) { + return vx_device_mpm_query(device, VX_DCR_MPM_CLASS_MEM, VX_CSR_MPM_MEM_READS, 0, value); +} + +static vx_buffer_h new_buffer(uint32_t size, uint64_t* addr) { + vx_buffer_h buffer = nullptr; + RT_CHECK(vx_buffer_create(device, size, VX_MEM_READ_WRITE, &buffer)); + full_buffers.push_back(buffer); + RT_CHECK(vx_buffer_address(buffer, addr)); + return buffer; +} + +static void read_buffer(void* dst, vx_buffer_h buffer, uint32_t size) { + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, dst, buffer, 0, size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); +} + +static int check_errors(uint32_t num_threads) { + std::vector h_err(num_threads); + read_buffer(h_err.data(), err_buffer, num_threads * sizeof(uint32_t)); + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << t << " read back 0x" << std::hex << h_err[t] << std::dec + << " wrong words of its lines" << std::endl; + } + ++errors; + } + } + return errors; +} + +static const char* full_name(const kernel_arg_t& arg) { + switch (arg.full_mode) { + case FULL_CHAIN: return arg.full_skip ? "chained, partially written" : "chained, fully written"; + case FULL_MIXED: return "mixed"; + default: return arg.full_skip ? "partially written" : "fully written"; + } +} + +// One phase-2 launch over a fresh pre-patterned buffer, after a launch that +// reads far more. Drivers differ on whether the memory read counter restarts +// at each launch; after the flood, a counter that restarted reads below its +// previous value. Returns the launch's memory reads. +static int64_t run_lines(kernel_arg_t arg, uint32_t grid, uint32_t block, uint32_t lines, int* errors) { + const uint32_t flood_lines = 4096; + uint32_t line_threads = (arg.full_mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + uint32_t words = lines * line_threads * FULL_LINE_WORDS; + vx_buffer_h buffer = new_buffer(words * sizeof(uint32_t), &arg.full_addr); + std::vector h_buf(words); + for (uint32_t i = 0; i < words; ++i) { + h_buf[i] = pattern_word(i); + } + RT_CHECK(vx_enqueue_write(queue, buffer, 0, h_buf.data(), words * sizeof(uint32_t), 0, nullptr, nullptr)); + + kernel_arg_t flood = arg; + flood.phase = 3; + flood.full_lines = flood_lines / arg.num_threads; + new_buffer(flood_lines * FLOOD_LINE_BYTES, &flood.full_addr); + RT_CHECK(launch(flood, grid, block)); + *errors += check_errors(arg.num_threads); + + arg.phase = 2; + arg.full_lines = lines; + uint64_t before = 0, after = 0; + RT_CHECK(mem_reads(&before)); + RT_CHECK(launch(arg, grid, block)); + RT_CHECK(mem_reads(&after)); + int64_t reads = (after < before) ? (int64_t)after : (int64_t)(after - before); + *errors += check_errors(arg.num_threads); + + read_buffer(h_buf.data(), buffer, words * sizeof(uint32_t)); + int bad = 0; + for (uint32_t i = 0; i < words; ++i) { + uint32_t ref = full_expected(i, &arg); + if (h_buf[i] != ref) { + if (bad < 16) { + std::cout << "error: " << full_name(arg) << " word " << i << std::hex << ": actual 0x" << h_buf[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++bad; + } + } + *errors += bad; + std::cout << full_name(arg) << " lines, memory reads: " << reads << std::endl; + return reads; +} + +// Lines stored whole and then loaded must not be read from memory. Each pair +// of launches stores and loads the same lines, one leaving a word of each +// sector unwritten: every partial sector is read from memory once, a whole +// one never, and the rest of the two launches' traffic is alike. +static int run_full(kernel_arg_t arg, uint32_t grid, uint32_t block) { + // few enough lines that a 16 KB cache keeps them resident + const uint32_t lines = 2; + int errors = 0; + for (uint32_t mode : {FULL_SEPARATE, FULL_CHAIN}) { + // a chained sector takes a whole block, which needs a word per thread + if (mode == FULL_CHAIN && block < SECTOR_WORDS) { + std::cout << "chained lines skipped: " << block << " threads per block" << std::endl; + continue; + } + arg.full_mode = mode; + uint32_t line_threads = (mode == FULL_CHAIN) ? arg.num_blocks : arg.num_threads; + // a chained sector belongs to a block: give the blocks more lines + uint32_t mode_lines = (mode == FULL_CHAIN) ? lines * 16 : lines; + arg.full_skip = 1; + int64_t partial = run_lines(arg, grid, block, mode_lines, &errors); + arg.full_skip = 0; + int64_t full = run_lines(arg, grid, block, mode_lines, &errors); + if (partial == 0) { + std::cout << "error: memory read counter inactive (run with --perf)" << std::endl; + return errors + 1; + } + int64_t sectors = (int64_t)mode_lines * line_threads * (FULL_LINE_WORDS / SECTOR_WORDS); + int64_t saved = partial - full; + // A sector evicted before its load is read back from memory either way. + int64_t slack = std::max(2, sectors / 16); + std::cout << full_name(arg) << " sectors: " << sectors << ", memory reads saved: " << saved << std::endl; + if (saved < sectors - slack || saved > sectors + slack) { + std::cout << "error: fully written sectors were read from memory" << std::endl; + ++errors; + } + } + // Loads of whole sectors loop back while partial ones read memory. + arg.full_mode = FULL_MIXED; + run_lines(arg, grid, block, lines, &errors); + return errors; +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + if (rounds == 0) { + rounds = 1; + } + + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads, isa_flags; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + RT_CHECK(vx_device_query(device, VX_CAPS_ISA_FLAGS, &isa_flags)); + + // One block per core, holding all of its warps. + uint32_t block = num_warps * num_threads; + uint32_t grid = num_cores; + uint32_t total_threads = grid * block; + uint32_t num_regions = rounds * total_threads * NUM_CASES; + uint32_t num_words = num_regions * REGION_WORDS; + uint32_t buf_size = num_words * sizeof(uint32_t); + uint32_t err_size = 2 * total_threads * sizeof(uint32_t); + uint32_t set_words = rounds * total_threads * SET_STRIDE_WORDS; + uint32_t set_size = set_words * sizeof(uint32_t); + + kernel_arg_t arg = {}; + arg.rounds = rounds; + arg.num_threads = total_threads; + arg.warp_size = num_threads; + arg.cross_warp = (num_warps % 2) == 0; + arg.amo = (isa_flags & VX_ISA_STD_A) != 0; + arg.num_blocks = grid; + + std::cout << "threads: " << total_threads << ", amo: " << arg.amo << ", regions: " << num_regions + << ", buffer size: " << buf_size << " bytes" << std::endl; + + RT_CHECK(vx_buffer_create(device, buf_size, VX_MEM_READ_WRITE, &buf_buffer)); + RT_CHECK(vx_buffer_address(buf_buffer, &arg.buf_addr)); + RT_CHECK(vx_buffer_create(device, err_size, VX_MEM_READ_WRITE, &err_buffer)); + RT_CHECK(vx_buffer_address(err_buffer, &arg.err_addr)); + RT_CHECK(vx_buffer_create(device, set_size, VX_MEM_READ_WRITE, &set_buffer)); + RT_CHECK(vx_buffer_address(set_buffer, &arg.set_addr)); + + std::vector h_buf(buf_size); + for (uint32_t i = 0; i < buf_size; ++i) { + h_buf[i] = pattern_byte(i); + } + std::vector h_err(2 * total_threads, 0xdeadbeef); + std::vector h_set(set_words); + for (uint32_t i = 0; i < set_words; ++i) { + h_set[i] = pattern_word(i); + } + + std::cout << "upload buffers" << std::endl; + RT_CHECK(vx_enqueue_write(queue, buf_buffer, 0, h_buf.data(), buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, err_buffer, 0, h_err.data(), err_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_write(queue, set_buffer, 0, h_set.data(), set_size, 0, nullptr, nullptr)); + + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + if (full_mode) { + int errors = run_full(arg, grid, block); + cleanup(); + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; + } + + std::cout << "launch kernel (exercise)" << std::endl; + arg.phase = 0; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "launch kernel (re-check)" << std::endl; + arg.phase = 1; + arg.rotate = (num_cores > 1) ? block : num_threads; + RT_CHECK(launch(arg, grid, block)); + + std::cout << "download buffers" << std::endl; + vx_event_h read_ev = nullptr; + RT_CHECK(vx_enqueue_read(queue, h_buf.data(), buf_buffer, 0, buf_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_set.data(), set_buffer, 0, set_size, 0, nullptr, nullptr)); + RT_CHECK(vx_enqueue_read(queue, h_err.data(), err_buffer, 0, err_size, 0, nullptr, &read_ev)); + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + + std::cout << "verify result" << std::endl; + int errors = 0; + for (uint32_t t = 0; t < 2 * total_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::cout << "error: thread " << (t % total_threads) << (t < total_threads ? " (exercise)" : " (re-check)") + << " reported 0x" << std::hex << h_err[t] << std::dec << " mismatches" << std::endl; + } + ++errors; + } + } + auto words = reinterpret_cast(h_buf.data()); + for (uint32_t region = 0; region < num_regions; ++region) { + for (uint32_t w = 0; w < REGION_WORDS; ++w) { + uint32_t ref = expected_word(region, w, &arg); + uint32_t cur = words[region * REGION_WORDS + w]; + if (cur != ref) { + if (errors < 16) { + std::cout << "error: region " << region << " (case " << (region % NUM_CASES) << ") word " << w + << std::hex << ": actual 0x" << cur << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + } + + for (uint32_t i = 0; i < set_words; ++i) { + uint32_t ref = expected_set_word(i); + if (h_set[i] != ref) { + if (errors < 16) { + std::cout << "error: same-set word " << i << std::hex << ": actual 0x" << h_set[i] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + cleanup(); + + if (errors != 0) { + std::cout << "Found " << std::dec << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/ldst_matrix/Makefile b/tests/regression/ldst_matrix/Makefile new file mode 100644 index 0000000000..03dd696dd3 --- /dev/null +++ b/tests/regression/ldst_matrix/Makefile @@ -0,0 +1,18 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +CONFIGS := $(if $(findstring -DVX_CFG_EXT_TCU_ENABLE,$(CONFIGS)),$(CONFIGS),$(CONFIGS) -DVX_CFG_EXT_TCU_ENABLE) + +PROJECT := ldst_matrix + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/ldst_matrix/common.h b/tests/regression/ldst_matrix/common.h new file mode 100644 index 0000000000..c9de0c0003 --- /dev/null +++ b/tests/regression/ldst_matrix/common.h @@ -0,0 +1,26 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Each thread owns a source and a destination region in global memory and in +// local memory; a fragment's row groups sit inside them. +#define GLOBAL_REGION_WORDS 512 +#define LOCAL_REGION_WORDS 128 + +// Source word i of a thread's region; local and global words differ. +static inline uint32_t src_word(uint32_t tid, uint32_t i, uint32_t salt) { + return ((tid << 12) | i | 0x40000000u) ^ salt; +} + +#define GLOBAL_SALT 0x00800000u +#define LOCAL_SALT 0x01000000u + +typedef struct { + uint64_t src_addr; // global source regions, filled by the host + uint64_t dst_addr; // global destination regions, zeroed by the host + uint64_t err_addr; // out: per-thread error count + uint64_t shape_addr; // out: bit s set when shape s ran +} kernel_arg_t; + +#endif diff --git a/tests/regression/ldst_matrix/kernel.cpp b/tests/regression/ldst_matrix/kernel.cpp new file mode 100644 index 0000000000..12aa3c3b2c --- /dev/null +++ b/tests/regression/ldst_matrix/kernel.cpp @@ -0,0 +1,258 @@ +#include +#include +#include "common.h" + +namespace vt = vortex::tensor; +using cfg = vt::wmma_config_t; + +// vx_ldm fd, imm(rs1) and vx_stm fs2, imm(rs1) (custom-3 funct3 0/2, I- and +// S-type), written as .insn so the encodings are tested independently of the +// compiler. imm[4:0] is the fragment shape, imm[11:5] a byte offset; rs1 is +// the first of J registers, a multiple of J, holding one address per row group. +enum : uint32_t { + SHAPE_A = 0x00, + SHAPE_A_SP = 0x01, + SHAPE_B = 0x06, + SHAPE_B_SP = 0x07, // symmetric sparse only + SHAPE_C = 0x08, + SHAPE_WG_C = 0x18 +}; + +// A fragment of a shape: J row groups of C words, CS bytes apart in a row. +struct geom_t { + uint32_t J, C, CS; +}; + +constexpr geom_t geom(uint32_t shape) { + switch (shape) { + case SHAPE_A: return {cfg::m_steps, cfg::k_steps, cfg::tcK * 4}; + case SHAPE_A_SP: return {cfg::m_steps, cfg::k_steps / 2, cfg::tcK * 4}; + case SHAPE_B: return {cfg::b_sub_steps, cfg::k_steps, cfg::tcK * 4}; + case SHAPE_B_SP: return {cfg::NRB, 1, 4}; + case SHAPE_C: return {cfg::m_steps, cfg::n_steps, cfg::tcN * 4}; + default: return {2, 4, cfg::tcN * 4}; + } +} + +constexpr uint32_t lg2(uint32_t x) { + return (x < 2) ? 0 : 1 + lg2(x / 2); +} + +constexpr uint32_t bit(uint32_t x, uint32_t i) { + return (x >> i) & 1; +} + +// the top bit of a step index of lg bits +constexpr uint32_t hi(uint32_t x, uint32_t lg) { + return bit(x, (lg > 0) ? (lg - 1) : 0); +} + +// The register the WMMA/WGMMA expansion reads element c of row group j of a +// fragment from: the TCU's bank-conflict-free operand permutations, written +// out independently of the header, SimX and the RTL. +constexpr uint32_t frag_reg(uint32_t shape, uint32_t j, uint32_t c) { + constexpr uint32_t lgM = lg2(cfg::m_steps), lgN = lg2(cfg::n_steps), lgK = lg2(cfg::k_steps); + constexpr bool two_sb = (cfg::b_sub_blocks > 1); + switch (shape) { + case SHAPE_A: // A={m[0],~m[hi],k} or {k[0],~m,m^k[hi]} + return two_sb ? ((bit(c, 0) << 2) | ((1 - bit(j, 0)) << 1) | (bit(j, 0) ^ hi(c, lgK))) + : ((bit(j, 0) << 2) | ((1 - hi(j, lgM)) << 1) | bit(c, 0)); + case SHAPE_B: { // B={n^k,~k} or {k[0],k[hi]^n[hi],~n[hi]}, n the row group's first step + uint32_t n = j * cfg::b_sub_blocks; + return two_sb ? ((bit(c, 0) << 2) | ((hi(c, lgK) ^ hi(n, lgN)) << 1) | (1 - hi(n, lgN))) + : (((bit(n, 0) ^ bit(c, 0)) << 1) | (1 - bit(c, 0))); + } + case SHAPE_C: // C={m[0],~m[hi],~(m[hi]^n)} or {n[0],~m,n[hi]} + return two_sb ? ((bit(c, 0) << 2) | ((1 - bit(j, 0)) << 1) | hi(c, lgN)) + : ((bit(j, 0) << 2) | ((1 - hi(j, lgM)) << 1) | (1 - (hi(j, lgM) ^ bit(c, 0)))); + case SHAPE_WG_C: // n-major + return (c << 1) | j; + default: // sparse A and B: element order + return j * geom(shape).C + c; + } +} + +template +constexpr bool frag_reg_is_permutation() { + constexpr geom_t g = geom(SHAPE); + uint32_t seen = 0; + for (uint32_t j = 0; j < g.J; ++j) { + for (uint32_t c = 0; c < g.C; ++c) { + uint32_t r = frag_reg(SHAPE, j, c); + if (r >= g.J * g.C || (seen & (1u << r))) { + return false; + } + seen |= 1u << r; + } + } + return true; +} + +constexpr uint32_t pow2ceil(uint32_t x) { + uint32_t p = 1; + while (p < x) { + p <<= 1; + } + return p; +} + +// One asm block: vx_ldm from the source group, read every register of the +// fragment, vx_stm the same registers to the destination group. +#define FRAG_MOVE8(imm, r0, r1, r2, r3, r4, r5, r6, r7) \ + asm volatile (".insn i 0x7b, 0, " r0 ", %[gs], %[im]\n\t" \ + "fmv.x.w %0, " r0 "\n\t" "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 "\n\t" "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %4, " r4 "\n\t" "fmv.x.w %5, " r5 "\n\t" \ + "fmv.x.w %6, " r6 "\n\t" "fmv.x.w %7, " r7 "\n\t" \ + ".insn s 0x7b, 2, " r0 ", %[im](%[gd])" \ + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), \ + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) \ + : [gs]"r"(s0), "r"(s1), "r"(s2), "r"(s3), \ + [gd]"r"(d0), "r"(d1), "r"(d2), "r"(d3), [im]"i"(imm) \ + : r0, r1, r2, r3, r4, r5, r6, r7, "memory") + +#define FRAG_MOVE4(imm, r0, r1, r2, r3) \ + asm volatile (".insn i 0x7b, 0, " r0 ", %[gs], %[im]\n\t" \ + "fmv.x.w %0, " r0 "\n\t" "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 "\n\t" "fmv.x.w %3, " r3 "\n\t" \ + ".insn s 0x7b, 2, " r0 ", %[im](%[gd])" \ + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) \ + : [gs]"r"(s0), "r"(s1), "r"(s2), "r"(s3), \ + [gd]"r"(d0), "r"(d1), "r"(d2), "r"(d3), [im]"i"(imm) \ + : r0, r1, r2, r3, "memory") + +// Moves the fragment of SHAPE at byte offset OFF, with fd = FD, between the +// thread's source and destination regions, whose row groups are row bytes +// apart from byte S on; returns the number of wrong words. +template +uint32_t check(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + constexpr geom_t g = geom(SHAPE); + constexpr uint32_t NR = g.J * g.C; + constexpr uint32_t S = pow2ceil((g.C * g.CS > 32) ? (g.C * g.CS) : 32); + static_assert((g.J == 2 || g.J == 4) && (NR == 4 || NR == 8), "unexpected fragment geometry"); + static_assert(frag_reg_is_permutation(), "fragment register map is not a permutation"); + + uintptr_t gs[4] = {0, 0, 0, 0}; + uintptr_t gd[4] = {0, 0, 0, 0}; + for (uint32_t j = 0; j < g.J; ++j) { + gs[j] = reinterpret_cast(src) + S + j * row; + gd[j] = reinterpret_cast(dst) + S + j * row; + } + register uintptr_t s0 __asm__("x28") = gs[0]; + register uintptr_t s1 __asm__("x29") = gs[1]; + register uintptr_t s2 __asm__("x30") = gs[2]; + register uintptr_t s3 __asm__("x31") = gs[3]; + register uintptr_t d0 __asm__("x12") = gd[0]; + register uintptr_t d1 __asm__("x13") = gd[1]; + register uintptr_t d2 __asm__("x14") = gd[2]; + register uintptr_t d3 __asm__("x15") = gd[3]; + + constexpr int32_t imm = OFF | int32_t(SHAPE); + uint32_t w[8] = {0, 0, 0, 0, 0, 0, 0, 0}; + if constexpr (NR == 8 && FD == 0) { + FRAG_MOVE8(imm, "f0", "f1", "f2", "f3", "f4", "f5", "f6", "f7"); + } else if constexpr (NR == 8 && FD == 10) { + FRAG_MOVE8(imm, "f10", "f11", "f12", "f13", "f14", "f15", "f16", "f17"); + } else if constexpr (NR == 8 && FD == 13) { + FRAG_MOVE8(imm, "f13", "f14", "f15", "f16", "f17", "f18", "f19", "f20"); + } else if constexpr (NR == 8 && FD == 24) { + FRAG_MOVE8(imm, "f24", "f25", "f26", "f27", "f28", "f29", "f30", "f31"); + } else if constexpr (NR == 4 && FD == 10) { + FRAG_MOVE4(imm, "f10", "f11", "f12", "f13"); + } else { + static_assert(NR == 4 && FD == 28, "unsupported fragment registers"); + FRAG_MOVE4(imm, "f28", "f29", "f30", "f31"); + } + + // the footprint, as word indices into the regions + uint32_t foot[8]; + uint32_t errors = 0; + for (uint32_t j = 0, e = 0; j < g.J; ++j) { + for (uint32_t c = 0; c < g.C; ++c, ++e) { + foot[e] = (S + j * row + OFF + c * g.CS) / 4; + // each register holds the element the TCU reads from it + errors += (w[frag_reg(SHAPE, j, c)] != src[foot[e]]); + } + } + // the store wrote the footprint; clearing it lets the final scans of the + // destination regions catch a word written anywhere else + for (uint32_t e = 0; e < NR; ++e) { + errors += (foot[e] >= region_words) || (dst[foot[e]] != src[foot[e]]); + } + for (uint32_t e = 0; e < NR; ++e) { + dst[foot[e]] = 0; + } + return errors; +} + +// Every offset form on one shape and register base: zero, positive, negative. +template +uint32_t check_offsets(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + constexpr geom_t g = geom(SHAPE); + constexpr int32_t S = int32_t(pow2ceil((g.C * g.CS > 32) ? (g.C * g.CS) : 32)); + uint32_t errors = 0; + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + errors += check(src, dst, region_words, row); + return errors; +} + +constexpr uint32_t kFdB = (cfg::NRB == 8) ? 24 : 28; + +uint32_t check_shapes(volatile uint32_t* src, volatile uint32_t* dst, uint32_t region_words, uint32_t row) { + uint32_t errors = 0; + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); +#ifdef VX_CFG_TCU_SPARSE_ENABLE + errors += check_offsets(src, dst, region_words, row); + if constexpr (cfg::sym_sparse) { + errors += check_offsets(src, dst, region_words, row); + } +#endif +#ifdef VX_CFG_TCU_WGMMA_ENABLE + errors += check_offsets(src, dst, region_words, row); + errors += check_offsets(src, dst, region_words, row); +#endif + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + const uint32_t lid = threadIdx.x; + const uint32_t tid = blockIdx.x * blockDim.x + lid; + + if (tid == 0) { + uint32_t shapes = (1u << SHAPE_A) | (1u << SHAPE_B) | (1u << SHAPE_C); + #ifdef VX_CFG_TCU_SPARSE_ENABLE + shapes |= (1u << SHAPE_A_SP) | (cfg::sym_sparse ? (1u << SHAPE_B_SP) : 0); + #endif + #ifdef VX_CFG_TCU_WGMMA_ENABLE + shapes |= (1u << SHAPE_WG_C); + #endif + *reinterpret_cast(arg->shape_addr) = shapes; + } + + // local regions: this thread's source, then its destination + auto lsrc = lmem + lid * 2 * LOCAL_REGION_WORDS; + auto ldst = lsrc + LOCAL_REGION_WORDS; + for (uint32_t i = 0; i < LOCAL_REGION_WORDS; ++i) { + lsrc[i] = src_word(tid, i, LOCAL_SALT); + ldst[i] = 0; + } + auto gsrc = reinterpret_cast(arg->src_addr) + tid * GLOBAL_REGION_WORDS; + auto gdst = reinterpret_cast(arg->dst_addr) + tid * GLOBAL_REGION_WORDS; + + // row strides that are not powers of two; the global one exceeds 256 bytes + uint32_t errors = 0; + errors += check_shapes(lsrc, ldst, LOCAL_REGION_WORDS, 68); + errors += check_shapes(gsrc, gdst, GLOBAL_REGION_WORDS, 68); + errors += check_shapes(gsrc, gdst, GLOBAL_REGION_WORDS, 260); + // the host scans the global destination regions + for (uint32_t i = 0; i < LOCAL_REGION_WORDS; ++i) { + errors += (ldst[i] != 0); + } + reinterpret_cast(arg->err_addr)[tid] = errors; +} diff --git a/tests/regression/ldst_matrix/main.cpp b/tests/regression/ldst_matrix/main.cpp new file mode 100644 index 0000000000..444025711c --- /dev/null +++ b/tests/regression/ldst_matrix/main.cpp @@ -0,0 +1,163 @@ +// ldst_matrix — the matrix fragment loads and stores (vx_ldm, vx_stm) move the +// words of each fragment shape the configuration has, from and to local and +// global memory, with row strides that are not powers of two, zero, positive +// and negative offsets, and aligned and unaligned register bases. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "k:h")) != -1) { + switch (c) { + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + uint64_t nt = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; + uint64_t isa = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + if (!(isa & VX_ISA_EXT_TCU)) { + std::cout << "TCU extension not supported!\nFAILED!" << std::endl; + return 1; + } + + // a block is one warp, as far as each thread's two local regions fit + const uint32_t lmem_per_thread = 2 * LOCAL_REGION_WORDS * sizeof(uint32_t); + const uint32_t block = static_cast(std::min(nt, lmem_size / lmem_per_thread)); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + const uint32_t global_words = num_threads * GLOBAL_REGION_WORDS; + std::cout << "ldst_matrix: block=" << block << " grid=" << grid << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr; + vx_buffer_h dst_buf = nullptr; + vx_buffer_h err_buf = nullptr; + vx_buffer_h shape_buf = nullptr; + CHECK(vx_buffer_create(dev, global_words * sizeof(uint32_t), VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, global_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, sizeof(uint32_t), VX_MEM_WRITE, &shape_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + CHECK(vx_buffer_address(src_buf, &kernel_arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &kernel_arg.dst_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(shape_buf, &kernel_arg.shape_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = block * lmem_per_thread; + + std::vector h_src(global_words); + for (uint32_t t = 0; t < num_threads; ++t) { + for (uint32_t i = 0; i < GLOBAL_REGION_WORDS; ++i) { + h_src[t * GLOBAL_REGION_WORDS + i] = src_word(t, i, GLOBAL_SALT); + } + } + std::vector h_dst(global_words, 0); + std::vector h_err(num_threads); + uint32_t h_shapes = 0; + vx_event_h write_ev[2] = {nullptr, nullptr}; + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; + CHECK(vx_enqueue_write(q, src_buf, 0, h_src.data(), h_src.size() * sizeof(uint32_t), 0, nullptr, &write_ev[0])); + CHECK(vx_enqueue_write(q, dst_buf, 0, h_dst.data(), h_dst.size() * sizeof(uint32_t), 0, nullptr, &write_ev[1])); + CHECK(vx_enqueue_launch(q, &li, 2, write_ev, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, &h_shapes, shape_buf, 0, sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, h_dst.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + std::cout << "shapes=0x" << std::hex << h_shapes << std::dec << std::endl; + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::printf("*** thread %u: %u wrong words\n", t, h_err[t]); + } + errors += h_err[t]; + } + } + // each store's footprint is cleared after its check + for (uint32_t i = 0; i < global_words; ++i) { + if (h_dst[i] != 0) { + if (errors < 16) { + std::printf("*** global destination word %u = 0x%x\n", i, h_dst[i]); + } + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_event_release(write_ev[1]); + vx_event_release(write_ev[0]); + vx_buffer_release(shape_buf); + vx_buffer_release(err_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/ldst_wide/Makefile b/tests/regression/ldst_wide/Makefile new file mode 100644 index 0000000000..63ce26deef --- /dev/null +++ b/tests/regression/ldst_wide/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := ldst_wide + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/ldst_wide/common.h b/tests/regression/ldst_wide/common.h new file mode 100644 index 0000000000..b66d79a3f7 --- /dev/null +++ b/tests/regression/ldst_wide/common.h @@ -0,0 +1,47 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +// Read-only words the wide loads read, in local memory and in a global buffer, +// followed in local memory by a SLOT_WORDS-word slot per thread that the wide +// stores fill; each thread has a slot in a global buffer too. +#define RO_WORDS 1024 +#define SLOT_WORDS 16 + +// The global read-only words differ from the local ones. +#define GLOBAL_SALT 0x0F0F1234u + +// Read-only word i: words differ from each other and about half of them have +// the sign bit set. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} + +static inline uint32_t wide_word(uint32_t tid, uint32_t k) { + return 0x6C000000u | (tid << 4) | k; +} + +static inline uint32_t alt_word(uint32_t tid, uint32_t k) { + return 0x3A000000u | (tid << 4) | k; +} + +// A slot's final word k: odd lanes overwrite words 0..3 with a second wide +// store, and a scalar store overwrites word 7 after the wide store of 5..8. +static inline uint32_t slot_word(uint32_t tid, uint32_t lid, uint32_t k) { + if ((k < 4 && (lid & 1)) || k == 7) { + return alt_word(tid, k); + } + return wide_word(tid, k); +} + +typedef struct { + uint32_t lmem_words; // local-memory words per block + uint64_t src_addr; // global read-only words + uint64_t dst_addr; // global slots + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory + uint64_t wide_addr; // out: 1 if the kernel uses the wide forms +} kernel_arg_t; + +#endif diff --git a/tests/regression/ldst_wide/kernel.cpp b/tests/regression/ldst_wide/kernel.cpp new file mode 100644 index 0000000000..b98a43c176 --- /dev/null +++ b/tests/regression/ldst_wide/kernel.cpp @@ -0,0 +1,298 @@ +#include +#include "common.h" + +// vx_flw.d/.q (custom-3 funct3 3/4, I-type like fld/flq) and vx_fsw.d/.q +// (funct3 5/6, S-type like fsd/fsq), written as .insn so the encodings are +// tested independently of the compiler. fd/fs2 is an aligned pair or quad and +// imm a multiple of 8 or 16; each word needs only 4-byte alignment. +#if defined(__riscv_f) && (__riscv_flen == 32) +#define HAS_WIDE 1 +#endif + +#ifdef HAS_WIDE +// A wide load and the reads of its registers sit in one asm block, so the +// compiler cannot reuse them in between. The last uop's register is read first. +#define VX_FLW_Q(r0, r1, r2, r3, off, p, w) \ + asm volatile (".insn i 0x7b, 4, " r0 ", %4, %5\n\t" \ + "fmv.x.w %3, " r3 "\n\t" \ + "fmv.x.w %0, " r0 "\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %2, " r2 \ + : "=&r"((w)[0]), "=&r"((w)[1]), "=&r"((w)[2]), "=&r"((w)[3]) \ + : "r"(p), "i"(off) \ + : r0, r1, r2, r3, "memory") + +#define VX_FLW_D(r0, r1, off, p, w) \ + asm volatile (".insn i 0x7b, 3, " r0 ", %2, %3\n\t" \ + "fmv.x.w %1, " r1 "\n\t" \ + "fmv.x.w %0, " r0 \ + : "=&r"((w)[0]), "=&r"((w)[1]) \ + : "r"(p), "i"(off) \ + : r0, r1, "memory") + +#define VX_FSW_Q(r0, r1, r2, r3, off, p, w) \ + asm volatile ("fmv.w.x " r0 ", %0\n\t" \ + "fmv.w.x " r1 ", %1\n\t" \ + "fmv.w.x " r2 ", %2\n\t" \ + "fmv.w.x " r3 ", %3\n\t" \ + ".insn s 0x7b, 6, " r0 ", %5(%4)" \ + : \ + : "r"((w)[0]), "r"((w)[1]), "r"((w)[2]), "r"((w)[3]), "r"(p), "i"(off) \ + : r0, r1, r2, r3, "memory") + +#define VX_FSW_D(r0, r1, off, p, w) \ + asm volatile ("fmv.w.x " r0 ", %0\n\t" \ + "fmv.w.x " r1 ", %1\n\t" \ + ".insn s 0x7b, 5, " r0 ", %3(%2)" \ + : \ + : "r"((w)[0]), "r"((w)[1]), "r"(p), "i"(off) \ + : r0, r1, "memory") +#else +// Without the wide forms, generic accesses move the same words. +#define VX_GENERIC_LD(n, off, p, w) \ + do { \ + auto _a = reinterpret_cast((p) + (off)); \ + for (uint32_t _k = 0; _k < (n); ++_k) { \ + (w)[_k] = _a[_k]; \ + } \ + } while (0) +#define VX_GENERIC_ST(n, off, p, w) \ + do { \ + auto _a = reinterpret_cast((p) + (off)); \ + for (uint32_t _k = 0; _k < (n); ++_k) { \ + _a[_k] = (w)[_k]; \ + } \ + } while (0) +#define VX_FLW_Q(r0, r1, r2, r3, off, p, w) VX_GENERIC_LD(4, off, p, w) +#define VX_FLW_D(r0, r1, off, p, w) VX_GENERIC_LD(2, off, p, w) +#define VX_FSW_Q(r0, r1, r2, r3, off, p, w) VX_GENERIC_ST(4, off, p, w) +#define VX_FSW_D(r0, r1, off, p, w) VX_GENERIC_ST(2, off, p, w) +#endif + +static inline uint32_t ro_misses(const uint32_t* w, uint32_t n, uint32_t first, uint32_t salt) { + uint32_t misses = 0; + for (uint32_t k = 0; k < n; ++k) { + if (w[k] != (ro_word(first + k) ^ salt)) { + ++misses; + } + } + return misses; +} + +// Wide loads of the read-only words at ro (local or global), checked against +// the words the host defines: +// - a warp-uniform group (broadcast), a 16-byte lane stride (every lane on +// the same banks), a lane permutation, half the lanes under a divergent +// branch, and addresses 4- but not 8- or 16-byte aligned +// - fd = f0 and f28; two wide loads back to back +// - scalar writes to fd + 2 just before and just after a wide load +// - a wide load, an FP op on fd + 2 that waits for it, and a wide load of +// the same group again +// - the offset limits, -2048 and 2032 (.q) or 2040 (.d) +static inline uint32_t check_loads(uintptr_t ro, uint32_t salt, uint32_t lid) { + // gs + 8, gp + 5 and gq + 3 stay in the read-only words for any block + // size, and gq * 4 exceeds the largest offset + const uint32_t gu = 384; + const uint32_t gs = 256 + (lid % 128) * 4; + const uint32_t gp = 512 + ((lid * 7 + 3) % 64) * 4; + const uint32_t go = 512; + const uint32_t gq = 512 + (lid % 128) * 4; + uint32_t errors = 0; + uint32_t w[8]; + + VX_FLW_Q("f8", "f9", "f10", "f11", 0, ro + gu * 4, w); + errors += ro_misses(w, 4, gu, salt); + VX_FLW_D("f8", "f9", 8, ro + gu * 4, w); + errors += ro_misses(w, 2, gu + 2, salt); + + VX_FLW_Q("f12", "f13", "f14", "f15", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f14", "f15", 0, ro + gs * 4, w); + errors += ro_misses(w, 2, gs, salt); + VX_FLW_Q("f12", "f13", "f14", "f15", 16, ro + gs * 4, w); + errors += ro_misses(w, 4, gs + 4, salt); + + VX_FLW_Q("f16", "f17", "f18", "f19", 0, ro + gp * 4, w); + errors += ro_misses(w, 4, gp, salt); + VX_FLW_D("f16", "f17", 8, ro + gp * 4, w); + errors += ro_misses(w, 2, gp + 2, salt); + + if (lid & 1) { + VX_FLW_Q("f20", "f21", "f22", "f23", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f22", "f23", 8, ro + gp * 4, w); + errors += ro_misses(w, 2, gp + 2, salt); + } + + VX_FLW_Q("f20", "f21", "f22", "f23", 16, ro + (gs + 1) * 4, w); + errors += ro_misses(w, 4, gs + 5, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", 0, ro + (gp + 2) * 4, w); + errors += ro_misses(w, 4, gp + 2, salt); + VX_FLW_D("f26", "f27", 8, ro + (gq - 1) * 4, w); + errors += ro_misses(w, 2, gq + 1, salt); + + VX_FLW_Q("f0", "f1", "f2", "f3", 0, ro + gp * 4, w); + errors += ro_misses(w, 4, gp, salt); + VX_FLW_Q("f28", "f29", "f30", "f31", 0, ro + gs * 4, w); + errors += ro_misses(w, 4, gs, salt); + VX_FLW_D("f30", "f31", 0, ro + gu * 4, w); + errors += ro_misses(w, 2, gu, salt); + + VX_FLW_Q("f4", "f5", "f6", "f7", -2048, ro + go * 4 + 2048, w); + errors += ro_misses(w, 4, go, salt); + VX_FLW_Q("f4", "f5", "f6", "f7", 2032, ro + go * 4 - 2032, w); + errors += ro_misses(w, 4, go, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", -2048, ro + (gs + 1) * 4 + 2048, w); + errors += ro_misses(w, 4, gs + 1, salt); + VX_FLW_Q("f24", "f25", "f26", "f27", 2032, ro + gq * 4 - 2032, w); + errors += ro_misses(w, 4, gq, salt); + VX_FLW_D("f26", "f27", -2048, ro + gp * 4 + 2048, w); + errors += ro_misses(w, 2, gp, salt); + VX_FLW_D("f26", "f27", 2040, ro + (gq + 1) * 4 - 2040, w); + errors += ro_misses(w, 2, gq + 1, salt); + +#ifdef HAS_WIDE + asm volatile (".insn i 0x7b, 4, f8, %8, 0\n\t" + ".insn i 0x7b, 4, f12, %8, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11\n\t" + "fmv.x.w %4, f12\n\t" + "fmv.x.w %5, f13\n\t" + "fmv.x.w %6, f14\n\t" + "fmv.x.w %7, f15" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]), + "=&r"(w[4]), "=&r"(w[5]), "=&r"(w[6]), "=&r"(w[7]) + : "r"(ro + gs * 4) + : "f8", "f9", "f10", "f11", "f12", "f13", "f14", "f15", "memory"); + errors += ro_misses(w, 8, gs, salt); + + const uint32_t marker = 0x7F00D000u | lid; + asm volatile ("fmv.w.x f10, %1\n\t" + ".insn i 0x7b, 4, f8, %2, 0\n\t" + "fmv.x.w %0, f10" + : "=&r"(w[2]) + : "r"(marker), "r"(ro + gp * 4) + : "f8", "f9", "f10", "f11", "memory"); + errors += (w[2] != (ro_word(gp + 2) ^ salt)); + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fmv.w.x f10, %5\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(ro + gp * 4), "r"(marker) + : "f8", "f9", "f10", "f11", "memory"); + errors += (w[0] != (ro_word(gp) ^ salt)) + (w[1] != (ro_word(gp + 1) ^ salt)) + + (w[2] != marker) + (w[3] != (ro_word(gp + 3) ^ salt)); + asm volatile (".insn i 0x7b, 4, f8, %4, 0\n\t" + "fadd.s f10, f10, f10\n\t" + ".insn i 0x7b, 4, f8, %4, 16\n\t" + "fmv.x.w %0, f8\n\t" + "fmv.x.w %1, f9\n\t" + "fmv.x.w %2, f10\n\t" + "fmv.x.w %3, f11" + : "=&r"(w[0]), "=&r"(w[1]), "=&r"(w[2]), "=&r"(w[3]) + : "r"(ro + gs * 4) + : "f8", "f9", "f10", "f11", "memory"); + errors += ro_misses(w, 4, gs + 4, salt); +#endif + return errors; +} + +// Wide stores into the thread's slot (local or global), then wide loads of +// the same words: +// - .q at the slot, then on odd lanes (divergent) .q again over it at the +// -2048 offset limit +// - .q at a 4- but not 16-byte aligned word, then a scalar store into it +// - .d at the 2040 offset limit and at a 4- but not 8-byte aligned word +// - scalar stores between the wide ones +static inline uint32_t check_stores(uintptr_t slot, uint32_t tid, uint32_t lid) { + auto s = reinterpret_cast(slot); + uint32_t v[16]; + for (uint32_t k = 0; k < 16; ++k) { + v[k] = wide_word(tid, k); + } + uint32_t a[4]; + for (uint32_t k = 0; k < 4; ++k) { + a[k] = alt_word(tid, k); + } + + VX_FSW_Q("f8", "f9", "f10", "f11", 0, slot, v); + if (lid & 1) { + VX_FSW_Q("f12", "f13", "f14", "f15", -2048, slot + 2048, a); + } + s[4] = v[4]; + VX_FSW_Q("f16", "f17", "f18", "f19", 16, slot + 4, v + 5); + s[7] = alt_word(tid, 7); + s[9] = v[9]; + VX_FSW_D("f20", "f21", 2040, slot + 40 - 2040, v + 10); + s[12] = v[12]; + VX_FSW_D("f22", "f23", 0, slot + 52, v + 13); + s[15] = v[15]; + + uint32_t errors = 0; + uint32_t w[4]; + VX_FLW_Q("f24", "f25", "f26", "f27", 0, slot, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, k)); + } + VX_FLW_Q("f0", "f1", "f2", "f3", 0, slot + 20, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, 5 + k)); + } + VX_FLW_D("f4", "f5", 8, slot + 32, w); + for (uint32_t k = 0; k < 2; ++k) { + errors += (w[k] != slot_word(tid, lid, 10 + k)); + } + VX_FLW_D("f6", "f7", 0, slot + 52, w); + for (uint32_t k = 0; k < 2; ++k) { + errors += (w[k] != slot_word(tid, lid, 13 + k)); + } + VX_FLW_Q("f28", "f29", "f30", "f31", 32, slot + 16, w); + for (uint32_t k = 0; k < 4; ++k) { + errors += (w[k] != slot_word(tid, lid, 12 + k)); + } + return errors; +} + +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t words = arg->lmem_words; + + if (tid == 0) { + #ifdef HAS_WIDE + *reinterpret_cast(arg->wide_addr) = 1; + #else + *reinterpret_cast(arg->wide_addr) = 0; + #endif + } + + for (uint32_t i = lid; i < RO_WORDS; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + const uintptr_t lmem_ro = reinterpret_cast(lmem); + const uintptr_t lmem_slot = lmem_ro + (RO_WORDS + lid * SLOT_WORDS) * 4; + const uintptr_t glob_slot = static_cast(arg->dst_addr) + tid * SLOT_WORDS * 4; + + uint32_t errors = 0; + errors += check_loads(lmem_ro, 0, lid); + errors += check_loads(static_cast(arg->src_addr), GLOBAL_SALT, lid); + errors += check_stores(lmem_slot, tid, lid); + errors += check_stores(glob_slot, tid, lid); + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/ldst_wide/main.cpp b/tests/regression/ldst_wide/main.cpp new file mode 100644 index 0000000000..36281332ef --- /dev/null +++ b/tests/regression/ldst_wide/main.cpp @@ -0,0 +1,195 @@ +// ldst_wide — the wide loads and stores (vx_flw.d/.q, vx_fsw.d/.q) move what +// the scalar accesses would, in local and global memory, across lane patterns, +// register groups, word alignments and the limits of their offset. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "k:h")) != -1) { + switch (c) { + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // One block per core holding all its warps, two waves of blocks. + uint64_t nt = 0; + uint64_t nw = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; + uint64_t isa = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + CHECK(vx_device_query(dev, VX_CAPS_ISA_FLAGS, &isa)); + // the wide forms exist with F and FLEN 32; elsewhere the kernel moves the + // same words with generic accesses + const bool wide = (isa & VX_ISA_STD_F) && !(isa & VX_ISA_STD_D); + const uint32_t block = static_cast(nt * nw); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + const uint32_t words = RO_WORDS + block * SLOT_WORDS; + std::cout << "ldst_wide: block=" << block << " grid=" << grid << " words=" << words + << " wide=" << wide << std::endl; + if (words * sizeof(uint32_t) > lmem_size) { + std::cout << "local memory too small\nFAILED!" << std::endl; + return 1; + } + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h src_buf = nullptr; + vx_buffer_h dst_buf = nullptr; + vx_buffer_h err_buf = nullptr; + vx_buffer_h dump_buf = nullptr; + vx_buffer_h wide_buf = nullptr; + CHECK(vx_buffer_create(dev, RO_WORDS * sizeof(uint32_t), VX_MEM_READ, &src_buf)); + CHECK(vx_buffer_create(dev, num_threads * SLOT_WORDS * sizeof(uint32_t), VX_MEM_READ_WRITE, &dst_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); + CHECK(vx_buffer_create(dev, sizeof(uint32_t), VX_MEM_WRITE, &wide_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.lmem_words = words; + CHECK(vx_buffer_address(src_buf, &kernel_arg.src_addr)); + CHECK(vx_buffer_address(dst_buf, &kernel_arg.dst_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); + CHECK(vx_buffer_address(wide_buf, &kernel_arg.wide_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = words * sizeof(uint32_t); + + std::vector h_src(RO_WORDS); + for (uint32_t i = 0; i < RO_WORDS; ++i) { + h_src[i] = ro_word(i) ^ GLOBAL_SALT; + } + std::vector h_dst(num_threads * SLOT_WORDS, 0); + std::vector h_err(num_threads); + std::vector h_dump(grid * words); + uint32_t h_wide = ~0u; + vx_event_h write_ev[2] = {nullptr, nullptr}; + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; + vx_event_h dst_ev = nullptr; + CHECK(vx_enqueue_write(q, src_buf, 0, h_src.data(), h_src.size() * sizeof(uint32_t), 0, nullptr, &write_ev[0])); + CHECK(vx_enqueue_write(q, dst_buf, 0, h_dst.data(), h_dst.size() * sizeof(uint32_t), 0, nullptr, &write_ev[1])); + CHECK(vx_enqueue_launch(q, &li, 2, write_ev, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, &h_wide, wide_buf, 0, sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, h_dump.size() * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_enqueue_read(q, h_dst.data(), dst_buf, 0, h_dst.size() * sizeof(uint32_t), 1, &launch_ev, &dst_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + CHECK(vx_event_wait_value(dst_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + // the kernel must take the wide path exactly where the ISA has it + if (h_wide != (wide ? 1u : 0u)) { + std::printf("*** kernel wide path=%u, expected %u\n", h_wide, wide ? 1u : 0u); + ++errors; + } + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] != 0) { + if (errors < 16) { + std::printf("*** thread %u: %u wrong values\n", t, h_err[t]); + } + errors += h_err[t]; + } + } + for (uint32_t b = 0; b < grid; ++b) { + for (uint32_t i = 0; i < words; ++i) { + const uint32_t lid = (i - RO_WORDS) / SLOT_WORDS; + uint32_t expected = (i < RO_WORDS) ? ro_word(i) + : slot_word(b * block + lid, lid, (i - RO_WORDS) % SLOT_WORDS); + uint32_t actual = h_dump[b * words + i]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** block %u lmem[%u] expected=0x%x actual=0x%x\n", b, i, expected, actual); + } + ++errors; + } + } + } + + for (uint32_t t = 0; t < num_threads; ++t) { + for (uint32_t k = 0; k < SLOT_WORDS; ++k) { + uint32_t expected = slot_word(t, t % block, k); + uint32_t actual = h_dst[t * SLOT_WORDS + k]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** thread %u global slot[%u] expected=0x%x actual=0x%x\n", t, k, expected, actual); + } + ++errors; + } + } + } + + vx_event_release(dst_ev); + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_event_release(write_ev[1]); + vx_event_release(write_ev[0]); + vx_buffer_release(wide_buf); + vx_buffer_release(dump_buf); + vx_buffer_release(err_buf); + vx_buffer_release(dst_buf); + vx_buffer_release(src_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/lmem_access/Makefile b/tests/regression/lmem_access/Makefile new file mode 100644 index 0000000000..c62a891e30 --- /dev/null +++ b/tests/regression/lmem_access/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := lmem_access + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n16 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/lmem_access/common.h b/tests/regression/lmem_access/common.h new file mode 100644 index 0000000000..02db3dd69b --- /dev/null +++ b/tests/regression/lmem_access/common.h @@ -0,0 +1,60 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define LMEM_MARK 0x80000000u + +// Local-memory layout per block, in words: read-only words first, then one +// RAW, one WAR and one AMO word per warp and iteration, then a read-only +// bank-conflict region of warp_size^2 words, then per warp and iteration +// SUB_WORDS(warp_size) words of sub-word stores and one same-word store word, +// then one cross-warp word per iteration, then the atomic counter and the +// LR/SC counter that every thread of the block updates. +#define RO_WORDS 128 + +// retry bound of the LR/SC increment +#define LRSC_MAX_RETRIES 4096 + +// words holding one byte store of each lane, four lanes to a word +#define SUB_WORDS(w) (((w) + 3) / 4) + +// The byte a lane stores into its sub-word store word, and the word a lane +// stores into the warp's same-word store word. +static inline uint8_t sub_byte(uint32_t slot, uint32_t lane) { + return (uint8_t)(0x40 + slot * 7 + lane); +} +static inline uint32_t same_word(uint32_t slot, uint32_t lane) { + return 0x80000000u | (slot << 8) | lane; +} + +// The value one warp stores into iteration k's cross-warp word. +static inline uint32_t xwarp_word(uint32_t k) { + return 0xC0000000u | (k << 12) | 0x5A5u; +} + +// Initial value of word i: every byte differs and the high bytes are set, so +// sub-word loads see distinct, sign-extended values. +static inline uint32_t ro_word(uint32_t i) { + return (i * 0x9E3779B1u) ^ 0xA5C30000u; +} + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t warp_size; // threads per warp + uint32_t lmem_words; // local-memory words per block + uint32_t raw_base; // first RAW word + uint32_t war_base; // first WAR word + uint32_t amo_base; // first AMO word + uint32_t cf_base; // first bank-conflict word + uint32_t sub_base; // first sub-word store word + uint32_t same_base; // first same-word store word + uint32_t xwarp_base; // first cross-warp word + uint32_t count_word; // atomic counter + uint32_t lrsc_word; // LR/SC counter + uint32_t lrsc; // run the LR/SC phase + uint64_t err_addr; // out: per-thread error count + uint64_t dump_addr; // out: each block's final local memory +} kernel_arg_t; + +#endif diff --git a/tests/regression/lmem_access/kernel.cpp b/tests/regression/lmem_access/kernel.cpp new file mode 100644 index 0000000000..fa0d26f26c --- /dev/null +++ b/tests/regression/lmem_access/kernel.cpp @@ -0,0 +1,220 @@ +#include +#include "common.h" + +// Local-memory reads that share words, around single-lane stores and +// atomics. Per iteration k, lane s = k % W takes a turn: +// +// shared words - lanes in groups of 1, 2, 4, ... W read one word; then 2 +// and 4 distinct words per warp, in one bank (32 words +// apart) and in different banks: every lane must get its +// own word; a third of the lanes read one word (partial +// thread mask); every warp reads the same word (lanes of +// different LSU blocks) +// conflicts - every lane reads its own word of one bank, so the request +// replays over W passes +// sub-words - every lane loads a byte (signed and unsigned) and a +// half-word of one shared word, at its own offset +// RAW - lane s stores its word, then the whole warp reads it: +// lane s must see its own store +// WAR - the whole warp reads a word, then lane s stores it: lane +// s's read must not see its own later store +// AMO - the whole warp reads a word, every lane adds 1 to it +// atomically, then the whole warp reads it again: each lane's +// first read must not see its own add, and its last must +// sub-word - every lane stores one byte, four lanes to a word, then +// stores reads its word back: it must hold all four lanes' bytes +// same-word - every lane stores its own value to one word, then the +// stores whole warp reads it: the highest lane's store is the last +// cross-warp - one lane of warp k % NW stores iteration k's word while +// the other warps, on other LSU blocks, read it: each read +// returns the old word or the stored one +// counter - every thread adds 1 to one word of the block atomically +// each iteration: no add is lost +// After the iterations, every thread increments one more word of the block +// through a load-reserved/store-conditional loop: no increment is lost. The +// host can leave this phase out (its retries make the instruction count +// depend on timing). +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto lmem = reinterpret_cast(__local_mem()); + auto err = reinterpret_cast(arg->err_addr); + auto dump = reinterpret_cast(arg->dump_addr); + const uint32_t W = arg->warp_size; + const uint32_t lid = threadIdx.x; + const uint32_t bsize = blockDim.x; + const uint32_t tid = blockIdx.x * bsize + lid; + const uint32_t lane = lid % W; + const uint32_t warp = lid / W; + const uint32_t NW = bsize / W; + const uint32_t words = arg->lmem_words; + + for (uint32_t i = lid; i < words; i += bsize) { + lmem[i] = ro_word(i); + } + __syncthreads(); + + uint32_t errors = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + uint32_t base = (slot * 7) % (RO_WORDS / 2); + uint32_t shared_idx[5] = { + base + (lane >> (k % 6)) % (RO_WORDS / 2), + base + (lane & 1) * 32, + base + (lane & 3) * 16, + base + (lane & 1), + base + (lane & 3), + }; + for (uint32_t p = 0; p < 5; ++p) { + uint32_t idx = shared_idx[p] % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + if (lane % 3 == k % 3) { + uint32_t idx = (base + 5) % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + { + uint32_t idx = (k * 5) % RO_WORDS; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + + { + uint32_t idx = arg->cf_base + lane * W + s; + if (lmem[idx] != ro_word(idx)) { + ++errors; + } + } + + { + uint32_t idx = (base + 9) % RO_WORDS; + uint32_t v = ro_word(idx); + uint32_t bo = lane & 3; + uint32_t ho = lane & 1; + auto lmem_u8 = reinterpret_cast(lmem); + auto lmem_s8 = reinterpret_cast(lmem); + auto lmem_u16 = reinterpret_cast(lmem); + if (lmem_u8[idx * 4 + bo] != ((v >> (8 * bo)) & 0xff)) { + ++errors; + } + if (lmem_s8[idx * 4 + bo] != (int8_t)(v >> (8 * bo))) { + ++errors; + } + if (lmem_u16[idx * 2 + ho] != ((v >> (16 * ho)) & 0xffff)) { + ++errors; + } + } + + uint32_t ri = arg->raw_base + slot; + if (lane == s) { + lmem[ri] = LMEM_MARK + ri; + } + uint32_t rv = lmem[ri]; + if (lane == s && rv != LMEM_MARK + ri) { + ++errors; + } + + uint32_t wi = arg->war_base + slot; + uint32_t wv = lmem[wi]; + if (lane == s) { + lmem[wi] = LMEM_MARK + wi; + } + if (lane == s && wv != ro_word(wi)) { + ++errors; + } + + #if VX_CFG_EXT_A_ENABLED + uint32_t ai = arg->amo_base + slot; + uint32_t a0 = lmem[ai]; + uint32_t old = __atomic_fetch_add(const_cast(&lmem[ai]), 1u, __ATOMIC_RELAXED); + uint32_t a1 = lmem[ai]; + // Other lanes' adds may land on either side of this lane's reads; this + // lane's own add must not reach its earlier read and must reach its later + // one, and returns a value no other add has passed over. + if (old - ro_word(ai) >= W || (old - a0) >= W || (a1 - old - 1) >= W) { + ++errors; + } + #endif + + { + uint32_t si = arg->sub_base + slot * SUB_WORDS(W) + lane / 4; + auto lmem_u8 = reinterpret_cast(lmem); + lmem_u8[si * 4 + (lane & 3)] = sub_byte(slot, lane); + uint32_t v = lmem[si]; + uint32_t expected = ro_word(si); + for (uint32_t b = 0; b < 4; ++b) { + uint32_t l = (lane & ~3u) + b; + if (l < W) { + expected = (expected & ~(0xffu << (8 * b))) | ((uint32_t)sub_byte(slot, l) << (8 * b)); + } + } + if (v != expected) { + ++errors; + } + } + + { + uint32_t si = arg->same_base + slot; + lmem[si] = same_word(slot, lane); + if (lmem[si] != same_word(slot, W - 1)) { + ++errors; + } + } + + { + uint32_t xi = arg->xwarp_base + k; + if (warp == k % NW) { + if (lane == s) { + lmem[xi] = xwarp_word(k); + } + } else { + uint32_t v = lmem[xi]; + if (v != ro_word(xi) && v != xwarp_word(k)) { + ++errors; + } + } + } + + #if VX_CFG_EXT_A_ENABLED + { + uint32_t ci = arg->count_word; + uint32_t old = __atomic_fetch_add(const_cast(&lmem[ci]), 1u, __ATOMIC_RELAXED); + if (old - ro_word(ci) >= bsize * arg->num_iters) { + ++errors; + } + } + #endif + } + +#if VX_CFG_EXT_A_ENABLED + if (arg->lrsc) { + // The retry is written out, not left to the compiler's compare-and-swap + // expansion, whose inner retry branch is not reconverged per lane. It is + // bounded, so a store-conditional that never succeeds leaves the count + // short instead of hanging the kernel. + auto word = const_cast(&lmem[arg->lrsc_word]); + uint32_t old; + uint32_t sc_fail; + uint32_t guard = 0; + do { + asm volatile ("lr.w %0, (%1)" : "=r"(old) : "r"(word) : "memory"); + asm volatile ("sc.w %0, %2, (%1)" : "=r"(sc_fail) : "r"(word), "r"(old + 1) : "memory"); + ++guard; + } while (sc_fail && guard < LRSC_MAX_RETRIES); + if (sc_fail || old - ro_word(arg->lrsc_word) >= bsize) { + ++errors; + } + } +#endif + err[tid] = errors; + + __syncthreads(); + for (uint32_t i = lid; i < words; i += bsize) { + dump[blockIdx.x * words + i] = lmem[i]; + } +} diff --git a/tests/regression/lmem_access/main.cpp b/tests/regression/lmem_access/main.cpp new file mode 100644 index 0000000000..3b60946c55 --- /dev/null +++ b/tests/regression/lmem_access/main.cpp @@ -0,0 +1,197 @@ +// lmem_access — local-memory reads that share words must each get their own +// word, stores of several lanes to one word must all land, and every thread +// keeps its order with its own stores. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_iters = 16; +// The LR/SC phase retries a contended store-conditional, so its instruction +// count depends on timing; -s leaves it out where instruction counts must +// match, as in the model-parity runs. +bool skip_lrsc = false; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:k:sh")) != -1) { + switch (c) { + case 'n': num_iters = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + case 's': skip_lrsc = true; break; + default: + std::cout << "Usage: [-n iterations] [-k kernel] [-s (no LR/SC phase)] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (num_iters == 0) { + num_iters = 1; + } + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // One block per core holding all its warps, two waves of blocks. + uint64_t nt = 0; + uint64_t nw = 0; + uint64_t nc = 0; + uint64_t lmem_size = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + CHECK(vx_device_query(dev, VX_CAPS_LOCAL_MEM_SIZE, &lmem_size)); + const uint32_t W = static_cast(nt); + const uint32_t block = static_cast(nt * nw); + const uint32_t grid = static_cast(nc * 2); + const uint32_t num_threads = block * grid; + // shrink the iteration count until a block's per-iteration words fit + const uint32_t cf_words = W * W; + const uint32_t slot_words = 4 + SUB_WORDS(W); + while (num_iters > 1 && (RO_WORDS + slot_words * nw * num_iters + cf_words + num_iters + 2) * sizeof(uint32_t) > lmem_size) { + num_iters /= 2; + } + const uint32_t slots = static_cast(nw) * num_iters; + const uint32_t words = RO_WORDS + slot_words * slots + cf_words + num_iters + 2; + std::cout << "lmem_access: iters=" << num_iters << " block=" << block << " grid=" << grid + << " words=" << words << std::endl; + if (words * sizeof(uint32_t) > lmem_size) { + std::cout << "local memory too small\nFAILED!" << std::endl; + return 1; + } + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + vx_buffer_h err_buf = nullptr; + vx_buffer_h dump_buf = nullptr; + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, grid * words * sizeof(uint32_t), VX_MEM_WRITE, &dump_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_iters = num_iters; + kernel_arg.warp_size = W; + kernel_arg.lmem_words = words; + kernel_arg.raw_base = RO_WORDS; + kernel_arg.war_base = RO_WORDS + slots; + kernel_arg.amo_base = RO_WORDS + 2 * slots; + kernel_arg.cf_base = RO_WORDS + 3 * slots; + kernel_arg.sub_base = kernel_arg.cf_base + cf_words; + kernel_arg.same_base = kernel_arg.sub_base + SUB_WORDS(W) * slots; + kernel_arg.xwarp_base = kernel_arg.same_base + slots; + kernel_arg.count_word = kernel_arg.xwarp_base + num_iters; + kernel_arg.lrsc_word = kernel_arg.count_word + 1; + kernel_arg.lrsc = skip_lrsc ? 0 : 1; + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(dump_buf, &kernel_arg.dump_addr)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid; + li.block_dim[0]= block; + li.lmem_size = words * sizeof(uint32_t); + + std::vector h_err(num_threads); + std::vector h_dump(grid * words); + vx_event_h launch_ev = nullptr; + vx_event_h read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_dump.data(), dump_buf, 0, grid * words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] == 0) { + continue; + } + if (errors < 16) { + std::printf("*** thread %u: %u wrong reads\n", t, h_err[t]); + } + errors += h_err[t]; + } + // every RAW and WAR word was stored by its lane, every sub-word store word + // holds its lanes' bytes, every same-word store word the highest lane's + // store, every cross-warp word its warp's store, and the counters every + // thread's increments; the read-only words kept their value + for (uint32_t b = 0; b < grid; ++b) { + for (uint32_t i = 0; i < words; ++i) { + uint32_t expected = ro_word(i); + if (i >= kernel_arg.raw_base && i < kernel_arg.amo_base) { + expected = LMEM_MARK + i; + } else if (i >= kernel_arg.amo_base && i < kernel_arg.cf_base && VX_CFG_EXT_A_ENABLED) { + expected += W; + } else if (i >= kernel_arg.sub_base && i < kernel_arg.same_base) { + uint32_t slot = (i - kernel_arg.sub_base) / SUB_WORDS(W); + uint32_t lane0 = ((i - kernel_arg.sub_base) % SUB_WORDS(W)) * 4; + for (uint32_t k = 0; k < 4 && lane0 + k < W; ++k) { + expected = (expected & ~(0xffu << (8 * k))) | ((uint32_t)sub_byte(slot, lane0 + k) << (8 * k)); + } + } else if (i >= kernel_arg.same_base && i < kernel_arg.xwarp_base) { + expected = same_word(i - kernel_arg.same_base, W - 1); + } else if (i >= kernel_arg.xwarp_base && i < kernel_arg.count_word) { + expected = xwarp_word(i - kernel_arg.xwarp_base); + } else if (i == kernel_arg.count_word && VX_CFG_EXT_A_ENABLED) { + expected += block * num_iters; + } else if (i == kernel_arg.lrsc_word && VX_CFG_EXT_A_ENABLED && kernel_arg.lrsc) { + expected += block; + } + uint32_t actual = h_dump[b * words + i]; + if (actual != expected) { + if (errors < 16) { + std::printf("*** block %u lmem[%u] expected=0x%x actual=0x%x\n", b, i, expected, actual); + } + ++errors; + } + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(dump_buf); + vx_buffer_release(err_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/madmax/main.cpp b/tests/regression/madmax/main.cpp index 2af9a86614..c2b7c45dd9 100755 --- a/tests/regression/madmax/main.cpp +++ b/tests/regression/madmax/main.cpp @@ -162,7 +162,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/module_reload/main.cpp b/tests/regression/module_reload/main.cpp index 6378d093e4..7ec08adc10 100644 --- a/tests/regression/module_reload/main.cpp +++ b/tests/regression/module_reload/main.cpp @@ -233,7 +233,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/mstress/kernel.cpp b/tests/regression/mstress/kernel.cpp index 8a66f2047c..2ae7f15ac3 100644 --- a/tests/regression/mstress/kernel.cpp +++ b/tests/regression/mstress/kernel.cpp @@ -4,18 +4,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t stride = arg->stride; uint32_t* addr_ptr = (uint32_t*)arg->src0_addr; - float* src_ptr = (float*)arg->src1_addr; - float* dst_ptr = (float*)arg->dst_addr; + uint32_t* src_ptr = (uint32_t*)arg->src1_addr; + uint32_t* dst_ptr = (uint32_t*)arg->dst_addr; size_t gid = blockIdx.x * blockDim.x + threadIdx.x; size_t offset = gid * stride; for (size_t i = 0; i < stride; ++i) { - float value = 0.0f; + uint32_t value = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { size_t addr = offset + i + j; uint32_t index = addr_ptr[addr]; - value *= src_ptr[index]; + value = value * 31 + src_ptr[index]; } dst_ptr[offset+i] = value; } diff --git a/tests/regression/mstress/main.cpp b/tests/regression/mstress/main.cpp index 9754c79856..b5a6d63bd7 100644 --- a/tests/regression/mstress/main.cpp +++ b/tests/regression/mstress/main.cpp @@ -4,8 +4,6 @@ #include #include "common.h" #include -#include -#include #include #define RT_CHECK(_expr) \ @@ -20,51 +18,6 @@ /////////////////////////////////////////////////////////////////////////////// -union Float_t { - float f; - int i; - struct { - uint32_t man : 23; - uint32_t exp : 8; - uint32_t sign : 1; - } parts; -}; - -inline float fround(float x, int32_t precision = 8) { - auto power_of_10 = std::pow(10, precision); - return std::round(x * power_of_10) / power_of_10; -} - -inline bool almost_equal_eps(float a, float b, int ulp = 128) { - auto eps = std::numeric_limits::epsilon() * (std::max(fabs(a), fabs(b)) * ulp); - auto d = fabs(a - b); - if (d > eps) { - std::cout << "*** almost_equal_eps: d=" << d << ", eps=" << eps << std::endl; - return false; - } - return true; -} - -inline bool almost_equal_ulp(float a, float b, int32_t ulp = 6) { - Float_t fa{a}, fb{b}; - auto d = std::abs(fa.i - fb.i); - if (d > ulp) { - std::cout << "*** almost_equal_ulp: a=" << a << ", b=" << b << ", ulp=" << d << ", ia=" << std::hex << fa.i << ", ib=" << fb.i << std::endl; - return false; - } - return true; -} - -inline bool almost_equal(float a, float b) { - if (a == b) - return true; - /*if (almost_equal_eps(a, b)) - return true;*/ - return almost_equal_ulp(a, b); -} - -/////////////////////////////////////////////////////////////////////////////// - const char* kernel_file = "kernel.vxbin"; uint32_t count = 0; @@ -116,7 +69,7 @@ void cleanup() { } } -void gen_src_data(std::vector& test_data, +void gen_src_data(std::vector& test_data, std::vector& addr_table, uint32_t num_points, uint32_t num_addrs) { @@ -124,8 +77,7 @@ void gen_src_data(std::vector& test_data, addr_table.resize(num_addrs); for (uint32_t i = 0; i < num_points; ++i) { - float r = static_cast(std::rand()) / RAND_MAX; - test_data[i] = r; + test_data[i] = static_cast(std::rand()); } for (uint32_t i = 0; i < num_addrs; ++i) { @@ -190,8 +142,8 @@ int main(int argc, char *argv[]) { // allocate host buffers std::cout << "allocate host buffers" << std::endl; std::vector h_addr; - std::vector h_src; - std::vector h_dst(num_points); + std::vector h_src; + std::vector h_dst(num_points); gen_src_data(h_src, h_addr, num_points, num_addrs); // upload source buffer0 @@ -239,17 +191,17 @@ int main(int argc, char *argv[]) { std::cout << "verify result" << std::endl; int errors = 0; for (uint32_t i = 0; i < num_points; ++i) { - float ref = 0.0f; + uint32_t ref = 0; for (uint32_t j = 0; j < NUM_LOADS; ++j) { uint32_t addr = i + j; uint32_t index = h_addr[addr]; - float value = h_src[index]; + uint32_t value = h_src[index]; //printf("*** [%d] addr=%d, index=%d, value=%f\n", i, addr, index, value); - ref *= value; + ref = ref * 31 + value; } - float cur = h_dst[i]; - if (!almost_equal(cur, ref)) { + uint32_t cur = h_dst[i]; + if (cur != ref) { std::cout << "error at result #" << std::dec << i << ": actual " << cur << ", expected " << ref << std::endl; ++errors; diff --git a/tests/regression/raw_order/Makefile b/tests/regression/raw_order/Makefile new file mode 100644 index 0000000000..48e56c03b6 --- /dev/null +++ b/tests/regression/raw_order/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := raw_order + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n2048 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/raw_order/common.h b/tests/regression/raw_order/common.h new file mode 100644 index 0000000000..dde800c43d --- /dev/null +++ b/tests/regression/raw_order/common.h @@ -0,0 +1,19 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define RAW_MARK 0x80000000u + +typedef struct { + uint32_t num_words; // words in the data buffer + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t delay_sweep; // iteration k delays the checked load (k % delay_sweep) * delay_step steps + uint32_t delay_step; + uint64_t data_addr; // in: initial words; out: regions 0, 1 and 3 overwritten with RAW_MARK + index + uint64_t new_addr; // out: the value each region-0 load returned + uint64_t sink_addr; // out: per-thread sum of the region-2 loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/raw_order/kernel.cpp b/tests/regression/raw_order/kernel.cpp new file mode 100644 index 0000000000..f704172ba3 --- /dev/null +++ b/tests/regression/raw_order/kernel.cpp @@ -0,0 +1,61 @@ +#include +#include "common.h" + +// Each warp owns a group of four regions (one word per lane each, one cache +// line per group) and, per iteration, issues in program order: +// +// load r2 - misses: heads the group's miss chain (answered from the fill) +// store r3 - chained behind it +// store r1 - chained behind it +// store r0 - chained behind it +// delay - warp-uniform countdown, swept per iteration +// load r0 - the checked load: must return the store to r0 above +// +// On a write-through cache the stores leave for memory at once and update +// the array when they replay behind the fill. The sweep lands the checked +// load at varying points of the chain's drain, including after the head has +// replayed and before the store to r0 has. The sequence is inline assembly +// so no compiler scheduling moves the requests relative to each other. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto data = reinterpret_cast(arg->data_addr); + auto nval = reinterpret_cast(arg->new_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + const uint32_t num_warps = arg->num_threads / W; + const uint32_t group_words = 4 * W; + uint32_t acc = 0; + uint32_t k = 0; + for (uint32_t g = warp; (g + 1) * group_words <= arg->num_words; g += num_warps, ++k) { + uint32_t* r0 = data + g * group_words + lane; + uint32_t* r1 = r0 + W; + uint32_t* r2 = r0 + 2 * W; + uint32_t* r3 = r0 + 3 * W; + uint32_t i0 = g * group_words + lane; + uint32_t mark0 = RAW_MARK + i0; + uint32_t mark1 = RAW_MARK + i0 + W; + uint32_t mark3 = RAW_MARK + i0 + 3 * W; + uint32_t delay = (k % arg->delay_sweep) * arg->delay_step; + uint32_t a, v; + __asm__ volatile( + "lw %[a], 0(%[r2])\n\t" + "sw %[m3], 0(%[r3])\n\t" + "sw %[m1], 0(%[r1])\n\t" + "sw %[m0], 0(%[r0])\n\t" + "beqz %[n], 2f\n\t" + "1:\n\t" + "addi %[n], %[n], -1\n\t" + "bnez %[n], 1b\n\t" + "2:\n\t" + "lw %[v], 0(%[r0])\n\t" + : [a] "=&r"(a), [v] "=&r"(v), [n] "+r"(delay) + : [r0] "r"(r0), [r1] "r"(r1), [r2] "r"(r2), [r3] "r"(r3), + [m0] "r"(mark0), [m1] "r"(mark1), [m3] "r"(mark3) + : "memory"); + acc += a; + nval[i0] = v; + } + sink[tid] = acc; +} diff --git a/tests/regression/raw_order/main.cpp b/tests/regression/raw_order/main.cpp new file mode 100644 index 0000000000..24a9c44404 --- /dev/null +++ b/tests/regression/raw_order/main.cpp @@ -0,0 +1,158 @@ +// raw_order — a thread's load must observe its own earlier store to the same +// word, even while the line is still draining its miss chain. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_words = 1024; +uint32_t delay_sweep = 4; +uint32_t delay_step = 1; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:d:k:h")) != -1) { + switch (c) { + case 'n': num_words = std::atoi(optarg); break; + case 's': delay_sweep = std::atoi(optarg); break; + case 'd': delay_step = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n words] [-s delay sweep] [-d delay step] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} + +uint32_t init_word(uint32_t i) { return i * 7 + 1; } +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (delay_sweep == 0) delay_sweep = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t num_threads = static_cast(nt * nw * nc); + std::cout << "raw_order: words=" << num_words << " threads=" << num_threads + << " delay=" << delay_sweep << "x" << delay_step << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint64_t data_size = uint64_t(num_words) * sizeof(uint32_t); + const uint64_t sink_size = uint64_t(num_threads) * sizeof(uint32_t); + vx_buffer_h data_buf = nullptr, new_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, data_size, VX_MEM_READ_WRITE, &data_buf)); + CHECK(vx_buffer_create(dev, data_size, VX_MEM_WRITE, &new_buf)); + CHECK(vx_buffer_create(dev, sink_size, VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_words = num_words; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = static_cast(nt); + kernel_arg.delay_sweep = delay_sweep; + kernel_arg.delay_step = delay_step; + CHECK(vx_buffer_address(data_buf, &kernel_arg.data_addr)); + CHECK(vx_buffer_address(new_buf, &kernel_arg.new_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_data(num_words), h_new(num_words); + for (uint32_t i = 0; i < num_words; ++i) h_data[i] = init_word(i); + CHECK(vx_enqueue_write(q, data_buf, 0, h_data.data(), data_size, 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_new.data(), new_buf, 0, data_size, 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_data.data(), data_buf, 0, data_size, 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + // Each warp owns groups of four regions of warp_size words: regions 0, 1 + // and 3 are overwritten, region 0 is then loaded back, region 2 is only + // read. + const uint32_t W = static_cast(nt); + const uint32_t group_words = 4 * W; + const uint32_t groups = num_words / group_words; + if (groups < num_threads / W) { + std::cout << "words (" << num_words << ") must cover one group per warp (" + << (num_threads / W) * group_words << ")" << std::endl; + return 1; + } + int errors = 0; + auto expect = [&](const char* what, uint32_t i, uint32_t actual, uint32_t ref) { + if (actual == ref) return; + if (errors < 16) + std::printf("*** %s [%u] expected=0x%x actual=0x%x%s\n", what, i, ref, actual, + actual == init_word(i) ? " (before its own earlier store)" : ""); + ++errors; + }; + for (uint32_t g = 0; g < groups; ++g) { + for (uint32_t l = 0; l < W; ++l) { + uint32_t i0 = g * group_words + l; + expect("load r0", i0, h_new[i0], RAW_MARK + i0); + expect("data r0", i0, h_data[i0], RAW_MARK + i0); + expect("data r1", i0 + W, h_data[i0 + W], RAW_MARK + i0 + W); + expect("data r2", i0 + 2 * W, h_data[i0 + 2 * W], init_word(i0 + 2 * W)); + expect("data r3", i0 + 3 * W, h_data[i0 + 3 * W], RAW_MARK + i0 + 3 * W); + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(new_buf); + vx_buffer_release(data_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/raycast/Makefile b/tests/regression/raycast/Makefile index d736258fb2..b1dedb71a2 100644 --- a/tests/regression/raycast/Makefile +++ b/tests/regression/raycast/Makefile @@ -11,8 +11,8 @@ VX_SRCS := $(SRC_DIR)/kernel.cpp CXXFLAGS += -DASSETS_PATHS='"$(SRC_DIR)"' -# Default workload is small for fast CI sweeps; override OPTS for full-resolution runs. -OPTS ?= -w 16 -h 12 +# Default workload is a 64x64 render target; override OPTS for other resolutions. +OPTS ?= -w 64 -h 64 KERNEL_LIB := vortex2 diff --git a/tests/regression/reopen/main.cpp b/tests/regression/reopen/main.cpp index abd691f4b0..f873a5dcaf 100644 --- a/tests/regression/reopen/main.cpp +++ b/tests/regression/reopen/main.cpp @@ -187,7 +187,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "Test PASSED" << std::endl; diff --git a/tests/regression/sector_merge/Makefile b/tests/regression/sector_merge/Makefile new file mode 100644 index 0000000000..62863e5aee --- /dev/null +++ b/tests/regression/sector_merge/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := sector_merge + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n256 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/sector_merge/common.h b/tests/regression/sector_merge/common.h new file mode 100644 index 0000000000..7b3a54079a --- /dev/null +++ b/tests/regression/sector_merge/common.h @@ -0,0 +1,13 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +typedef struct { + uint32_t num_lines; + uint32_t line_words; + uint32_t group; + uint32_t reserved; + uint64_t src_addr; + uint64_t dst_addr; +} kernel_arg_t; + +#endif diff --git a/tests/regression/sector_merge/kernel.cpp b/tests/regression/sector_merge/kernel.cpp new file mode 100644 index 0000000000..389b7cbb55 --- /dev/null +++ b/tests/regression/sector_merge/kernel.cpp @@ -0,0 +1,23 @@ +#include +#include "common.h" + +// Consecutive thread groups read opposite halves of the same lines, so the +// misses for both sectors of a sectored line are in flight together. +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + const uint32_t* src = (const uint32_t*)arg->src_addr; + uint32_t* dst = (uint32_t*)arg->dst_addr; + uint32_t num_lines = arg->num_lines; + uint32_t line_words = arg->line_words; + uint32_t half_words = line_words / 2; + + uint32_t gid = blockIdx.x * blockDim.x + threadIdx.x; + uint32_t half = (gid / arg->group) & 1; + uint32_t word = half * half_words + (gid % arg->group) % half_words; + + uint32_t sum = 0; + #pragma unroll 4 + for (uint32_t k = 0; k < num_lines; ++k) { + sum += src[k * line_words + word] * (k + 1); + } + dst[gid] = sum; +} diff --git a/tests/regression/sector_merge/main.cpp b/tests/regression/sector_merge/main.cpp new file mode 100644 index 0000000000..54e9134872 --- /dev/null +++ b/tests/regression/sector_merge/main.cpp @@ -0,0 +1,180 @@ +#include +#include +#include +#include +#include "common.h" + +#define RT_CHECK(_expr) \ + do { \ + int _ret = _expr; \ + if (0 == _ret) \ + break; \ + printf("Error: '%s' returned %d!\n", #_expr, (int)_ret); \ + cleanup(); \ + exit(-1); \ + } while (false) + +/////////////////////////////////////////////////////////////////////////////// + +const char* kernel_file = "kernel.vxbin"; +uint32_t num_lines = 256; +uint32_t line_bytes = 128; + +vx_device_h device = nullptr; +vx_buffer_h src_buffer = nullptr; +vx_buffer_h dst_buffer = nullptr; +vx_queue_h queue = nullptr; +vx_module_h module_ = nullptr; +vx_kernel_h kernel = nullptr; +kernel_arg_t kernel_arg = {}; + +static void show_usage() { + std::cout << "Vortex Test." << std::endl; + std::cout << "Usage: [-k: kernel] [-n lines] [-l line bytes] [-h: help]" << std::endl; +} + +static void parse_args(int argc, char **argv) { + int c; + while ((c = getopt(argc, argv, "n:l:k:h")) != -1) { + switch (c) { + case 'n': + num_lines = atoi(optarg); + break; + case 'l': + line_bytes = atoi(optarg); + break; + case 'k': + kernel_file = optarg; + break; + case 'h': + show_usage(); + exit(0); + break; + default: + show_usage(); + exit(-1); + } + } +} + +void cleanup() { + if (device) { + if (src_buffer) vx_buffer_release(src_buffer); + if (dst_buffer) vx_buffer_release(dst_buffer); + if (kernel) vx_kernel_release(kernel); + if (module_) vx_module_release(module_); + if (queue) vx_queue_release(queue); + vx_device_dump_perf(device, stdout); + vx_device_release(device); + } +} + +int main(int argc, char *argv[]) { + parse_args(argc, argv); + + // open device connection + std::cout << "open device connection" << std::endl; + RT_CHECK(vx_device_open(0, &device)); + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + RT_CHECK(vx_queue_create(device, &qi, &queue)); + + uint64_t num_cores, num_warps, num_threads; + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_CORES, &num_cores)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_WARPS, &num_warps)); + RT_CHECK(vx_device_query(device, VX_CAPS_NUM_THREADS, &num_threads)); + + uint32_t line_words = line_bytes / sizeof(uint32_t); + uint32_t num_tasks = num_cores * num_warps * num_threads; + uint32_t num_words = num_lines * line_words; + + kernel_arg.num_lines = num_lines; + kernel_arg.line_words = line_words; + kernel_arg.group = num_threads; + + std::cout << "number of tasks: " << num_tasks << ", lines: " << num_lines + << ", line bytes: " << line_bytes << std::endl; + + // allocate device memory + std::cout << "allocate device memory" << std::endl; + RT_CHECK(vx_buffer_create(device, num_words * sizeof(uint32_t), VX_MEM_READ, &src_buffer)); + RT_CHECK(vx_buffer_address(src_buffer, &kernel_arg.src_addr)); + RT_CHECK(vx_buffer_create(device, num_tasks * sizeof(uint32_t), VX_MEM_WRITE, &dst_buffer)); + RT_CHECK(vx_buffer_address(dst_buffer, &kernel_arg.dst_addr)); + + // upload source buffer + std::cout << "upload source buffer" << std::endl; + std::vector h_src(num_words); + for (uint32_t i = 0; i < num_words; ++i) { + h_src[i] = i * 2654435761u + 12345u; + } + RT_CHECK(vx_enqueue_write(queue, src_buffer, 0, h_src.data(), num_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + // load kernel module + std::cout << "load kernel module" << std::endl; + RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); + RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); + + // start device + std::cout << "start device" << std::endl; + vx_event_h launch_ev = nullptr, read_ev = nullptr; + { + uint32_t n = num_tasks; + uint32_t grid_dim[1], block_dim[1]; + RT_CHECK(vx_device_max_occupancy_grid(device, 1, &n, grid_dim, block_dim)); + vx_launch_info_t li = {}; + li.struct_size = sizeof(li); + li.kernel = kernel; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid_dim[0]; + li.block_dim[0] = block_dim[0]; + RT_CHECK(vx_enqueue_launch(queue, &li, 0, nullptr, &launch_ev)); + } + + // download destination buffer + std::cout << "download destination buffer" << std::endl; + std::vector h_dst(num_tasks); + RT_CHECK(vx_enqueue_read(queue, h_dst.data(), dst_buffer, 0, num_tasks * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + + // wait for completion + std::cout << "wait for completion" << std::endl; + RT_CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + vx_event_release(read_ev); + vx_event_release(launch_ev); + + // verify result + std::cout << "verify result" << std::endl; + uint32_t half_words = line_words / 2; + int errors = 0; + for (uint32_t gid = 0; gid < num_tasks; ++gid) { + uint32_t half = (gid / num_threads) & 1; + uint32_t word = half * half_words + (gid % num_threads) % half_words; + uint32_t ref = 0; + for (uint32_t k = 0; k < num_lines; ++k) { + ref += h_src[k * line_words + word] * (k + 1); + } + if (h_dst[gid] != ref) { + if (errors < 100) { + std::cout << "error at result #" << gid << ": actual 0x" << std::hex << h_dst[gid] + << ", expected 0x" << ref << std::dec << std::endl; + } + ++errors; + } + } + + // cleanup + std::cout << "cleanup" << std::endl; + cleanup(); + + if (errors != 0) { + std::cout << "Found " << errors << " errors!" << std::endl; + std::cout << "FAILED!" << std::endl; + return 1; + } + + std::cout << "PASSED!" << std::endl; + + return 0; +} diff --git a/tests/regression/sgemm/common.h b/tests/regression/sgemm/common.h index d3c6362d7d..64d1a6926d 100644 --- a/tests/regression/sgemm/common.h +++ b/tests/regression/sgemm/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; uint64_t A_addr; diff --git a/tests/regression/sgemm/kernel.cpp b/tests/regression/sgemm/kernel.cpp index 7b79bbfb11..2a0278aed7 100644 --- a/tests/regression/sgemm/kernel.cpp +++ b/tests/regression/sgemm/kernel.cpp @@ -11,10 +11,35 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { int row = blockIdx.y * blockDim.y + threadIdx.y; TYPE sum(0); +#if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + const TYPE* a_row = A + row * size; + const TYPE* b_col = B + col; + size_t e = 0; + for (; e + 8 <= size; e += 8) { + vx_float4 a0 = vx_flw_q(a_row + e); + vx_float4 a1 = vx_flw_q(a_row + e + 4); + #pragma unroll + for (int k = 0; k < 4; ++k) { + sum += a0[k] * *b_col; + b_col += size; + } + #pragma unroll + for (int k = 0; k < 4; ++k) { + sum += a1[k] * *b_col; + b_col += size; + } + } + for (; e < size; ++e) { + sum += a_row[e] * *b_col; + b_col += size; + } +#else #pragma unroll 8 for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } +#endif C[row * size + col] = sum; } diff --git a/tests/regression/sgemm/main.cpp b/tests/regression/sgemm/main.cpp index 585de0d841..0f6cf0900d 100644 --- a/tests/regression/sgemm/main.cpp +++ b/tests/regression/sgemm/main.cpp @@ -158,7 +158,7 @@ int main(int argc, char** argv) { if (errors) { std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/sgemm2/common.h b/tests/regression/sgemm2/common.h index 59a521c347..6957c0737b 100644 --- a/tests/regression/sgemm2/common.h +++ b/tests/regression/sgemm2/common.h @@ -5,11 +5,15 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; - uint32_t tile_size; - uint32_t chunk_k; - uint32_t pad_; + uint32_t block_size; uint64_t A_addr; uint64_t B_addr; uint64_t C_addr; diff --git a/tests/regression/sgemm2/kernel.cpp b/tests/regression/sgemm2/kernel.cpp index afe6c336e8..396cc01150 100644 --- a/tests/regression/sgemm2/kernel.cpp +++ b/tests/regression/sgemm2/kernel.cpp @@ -1,49 +1,56 @@ #include #include "common.h" -__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { - auto A_ptr = reinterpret_cast(arg->A_addr); - auto B_ptr = reinterpret_cast(arg->B_addr); - auto C_ptr = reinterpret_cast(arg->C_addr); - - size_t size = arg->size; - size_t tile_size = arg->tile_size; - size_t chunk_k = arg->chunk_k; - - // Global and local thread coordinates. - size_t g_row = blockIdx.x * blockDim.x + threadIdx.x; - size_t g_col = blockIdx.y * blockDim.y + threadIdx.y; - size_t l_row = threadIdx.x; - size_t l_col = threadIdx.y; - - // Shared memory layout: - // local_A: [tile_size × chunk_k], row-major (row = l_row, col = k_local) - // local_B: [chunk_k × tile_size], row-major (row = k_local, col = l_col) - auto tile_elems_a = tile_size * chunk_k; - auto local_ptr = reinterpret_cast(__local_mem()); - auto local_A = local_ptr; - auto local_B = local_A + tile_elems_a; - - TYPE sum(0); +// Shared-memory tiled matrix multiply, as in the CUDA samples' matrixMul: each +// thread computes one element of C, and the block steps BLOCK_SIZE x +// BLOCK_SIZE tiles of A and B through local memory. +template +static void matrix_mul(const TYPE* A, const TYPE* B, TYPE* C, uint32_t size) { + auto As = reinterpret_cast(__local_mem()); // [BLOCK_SIZE][BLOCK_SIZE] + auto Bs = As + BLOCK_SIZE * BLOCK_SIZE; // [BLOCK_SIZE][BLOCK_SIZE] + + uint32_t tx = threadIdx.x; + uint32_t ty = threadIdx.y; + uint32_t row = blockIdx.y * BLOCK_SIZE + ty; + uint32_t col = blockIdx.x * BLOCK_SIZE + tx; + + TYPE sum = 0; + for (uint32_t t = 0; t < size; t += BLOCK_SIZE) { + As[ty * BLOCK_SIZE + tx] = A[row * size + t + tx]; + Bs[ty * BLOCK_SIZE + tx] = B[(t + ty) * size + col]; + __syncthreads(); - for (size_t k = 0; k < size; k += chunk_k) { - // Cooperative load: each thread loads chunk_k/tile_size elements per tile. - // Thread (l_row, l_col) loads columns l_col, l_col+tile_size, ... of its row. - for (size_t kk = l_col; kk < chunk_k; kk += tile_size) { - local_A[l_row * chunk_k + kk] = A_ptr[g_row * size + (k + kk)]; + #if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + static_assert((BLOCK_SIZE % 4) == 0, "LDST_WIDE reads A in groups of 4"); + #pragma unroll + for (uint32_t k = 0; k < BLOCK_SIZE; k += 4) { + vx_float4 a = vx_flw_q(&As[ty * BLOCK_SIZE + k]); + sum += a[0] * Bs[k * BLOCK_SIZE + tx]; + sum += a[1] * Bs[(k + 1) * BLOCK_SIZE + tx]; + sum += a[2] * Bs[(k + 2) * BLOCK_SIZE + tx]; + sum += a[3] * Bs[(k + 3) * BLOCK_SIZE + tx]; } - for (size_t kk = l_row; kk < chunk_k; kk += tile_size) { - local_B[kk * tile_size + l_col] = B_ptr[(k + kk) * size + g_col]; + #else + #pragma unroll + for (uint32_t k = 0; k < BLOCK_SIZE; ++k) { + sum += As[ty * BLOCK_SIZE + k] * Bs[k * BLOCK_SIZE + tx]; } - + #endif __syncthreads(); + } - for (size_t j = 0; j < chunk_k; ++j) { - sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; - } + C[row * size + col] = sum; +} - __syncthreads(); +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto A = reinterpret_cast(arg->A_addr); + auto B = reinterpret_cast(arg->B_addr); + auto C = reinterpret_cast(arg->C_addr); + switch (arg->block_size) { + case 4: matrix_mul<4>(A, B, C, arg->size); break; + case 8: matrix_mul<8>(A, B, C, arg->size); break; + case 16: matrix_mul<16>(A, B, C, arg->size); break; + case 32: matrix_mul<32>(A, B, C, arg->size); break; } - - C_ptr[g_row * size + g_col] = sum; } diff --git a/tests/regression/sgemm2/main.cpp b/tests/regression/sgemm2/main.cpp index 04e7b5376e..667a85f4fe 100644 --- a/tests/regression/sgemm2/main.cpp +++ b/tests/regression/sgemm2/main.cpp @@ -86,8 +86,7 @@ static void matmul_cpu(TYPE* out, const TYPE* A, const TYPE* B, uint32_t width, const char* kernel_file = "kernel.vxbin"; uint32_t size = 64; -uint32_t tile_size = 4; -uint32_t chunk_k = 0; +uint32_t block_size = 4; vx_device_h device = nullptr; vx_buffer_h A_buffer = nullptr; @@ -100,22 +99,19 @@ kernel_arg_t kernel_arg = {}; static void show_usage() { std::cout << "Vortex Test." << std::endl; - std::cout << "Usage: [-k kernel] [-n matrix_size] [-t tile_size] [-c chunk_k] [-h help]" << std::endl; - std::cout << " chunk_k=0 means chunk_k=tile_size (original behavior)" << std::endl; + std::cout << "Usage: [-k kernel] [-n matrix_size] [-t block_size] [-h help]" << std::endl; + std::cout << " block_size: 4, 8, 16 or 32; a block of block_size x block_size threads computes a tile of C" << std::endl; } static void parse_args(int argc, char **argv) { int c; - while ((c = getopt(argc, argv, "n:t:c:k:h")) != -1) { + while ((c = getopt(argc, argv, "n:t:k:h")) != -1) { switch (c) { case 'n': size = atoi(optarg); break; case 't': - tile_size = atoi(optarg); - break; - case 'c': - chunk_k = atoi(optarg); + block_size = atoi(optarg); break; case 'k': kernel_file = optarg; @@ -148,16 +144,13 @@ int main(int argc, char *argv[]) { // parse command arguments parse_args(argc, argv); - if ((size / tile_size) * tile_size != size) { - printf("Error: matrix size %d must be a multiple of tile size %d\n", size, tile_size); + if (block_size != 4 && block_size != 8 && block_size != 16 && block_size != 32) { + printf("Error: unsupported block size %d\n", block_size); + show_usage(); return -1; } - - // Default chunk_k to tile_size. - if (chunk_k == 0) - chunk_k = tile_size; - if ((size % chunk_k) != 0) { - printf("Error: matrix size %d must be a multiple of chunk_k %d\n", size, chunk_k); + if ((size % block_size) != 0) { + printf("Error: matrix size %d must be a multiple of block size %d\n", size, block_size); return -1; } @@ -172,14 +165,12 @@ int main(int argc, char *argv[]) { uint32_t size_sq = size * size; uint32_t buf_size = size_sq * sizeof(TYPE); - uint32_t group_size = tile_size * tile_size; - uint32_t tile_elems_a = tile_size * chunk_k; - uint32_t tile_elems_b = chunk_k * tile_size; - uint32_t local_mem = (tile_elems_a + tile_elems_b) * sizeof(TYPE); + uint32_t group_size = block_size * block_size; + uint32_t local_mem = 2 * group_size * sizeof(TYPE); std::cout << "data type: " << Comparator::type_str() << std::endl; std::cout << "matrix size: " << size << "x" << size << std::endl; - std::cout << "tile size: " << tile_size << ", chunk_k: " << chunk_k << std::endl; + std::cout << "block size: " << block_size << std::endl; std::cout << "local memory: " << local_mem << " bytes" << std::endl; // check work group occupancy @@ -223,12 +214,11 @@ int main(int argc, char *argv[]) { RT_CHECK(vx_module_load_file(device, kernel_file, &module_)); RT_CHECK(vx_module_get_kernel(module_, "main", &kernel)); - kernel_arg.size = size; - kernel_arg.tile_size = tile_size; - kernel_arg.chunk_k = chunk_k; + kernel_arg.size = size; + kernel_arg.block_size = block_size; - uint32_t grid_dim[2] = {size / tile_size, size / tile_size}; - uint32_t block_dim[2] = {tile_size, tile_size}; + uint32_t grid_dim[2] = {size / block_size, size / block_size}; + uint32_t block_dim[2] = {block_size, block_size}; // launch kernel — args passed as a host blob (UVA), no args device buffer std::cout << "launch kernel" << std::endl; @@ -280,7 +270,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return 1; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm2_dxa/common.h b/tests/regression/sgemm2_dxa/common.h index c646688ddd..730db98ebe 100644 --- a/tests/regression/sgemm2_dxa/common.h +++ b/tests/regression/sgemm2_dxa/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t size; uint32_t tile_size; diff --git a/tests/regression/sgemm2_dxa/kernel.cpp b/tests/regression/sgemm2_dxa/kernel.cpp index 7693034844..53fe18fe58 100644 --- a/tests/regression/sgemm2_dxa/kernel.cpp +++ b/tests/regression/sgemm2_dxa/kernel.cpp @@ -13,9 +13,24 @@ constexpr uint32_t kDescB = 1; // shB layout: [chunk_k × tile_size], row-major (row = k, col = l_col) static inline void gemm_accumulate(TYPE& sum, const TYPE* shA, const TYPE* shB, size_t tile_size, size_t chunk_k, size_t l_row, size_t l_col) { +#if LDST_WIDE + size_t k = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; k + 4 <= chunk_k; k += 4) { + vx_float4 a = vx_flw_q(&shA[l_row * chunk_k + k]); + sum += a[0] * shB[k * tile_size + l_col]; + sum += a[1] * shB[(k + 1) * tile_size + l_col]; + sum += a[2] * shB[(k + 2) * tile_size + l_col]; + sum += a[3] * shB[(k + 3) * tile_size + l_col]; + } + for (; k < chunk_k; ++k) { + sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; + } +#else for (size_t k = 0; k < chunk_k; ++k) { sum += shA[l_row * chunk_k + k] * shB[k * tile_size + l_col]; } +#endif } __kernel void kernel_main(kernel_arg_t* arg) { diff --git a/tests/regression/sgemm2_dxa/main.cpp b/tests/regression/sgemm2_dxa/main.cpp index 3f5849209d..27837c320c 100644 --- a/tests/regression/sgemm2_dxa/main.cpp +++ b/tests/regression/sgemm2_dxa/main.cpp @@ -304,7 +304,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "Found " << errors << " errors\n"; std::cout << "FAILED\n"; - return errors; + return errors != 0; } std::cout << "PASSED\n"; diff --git a/tests/regression/sgemm2_dxa_mcast/common.h b/tests/regression/sgemm2_dxa_mcast/common.h index 40de5d8401..627d1bb294 100644 --- a/tests/regression/sgemm2_dxa_mcast/common.h +++ b/tests/regression/sgemm2_dxa_mcast/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + // Intra-core multicast (mw = "multicast warp-group", multiple CTAs on one core). // `mc_group_size` CTAs co-resident on one core share the same B column block. // Each CTA computes one row of the output; A is per-CTA, B is multicast. diff --git a/tests/regression/sgemm2_dxa_mcast/kernel.cpp b/tests/regression/sgemm2_dxa_mcast/kernel.cpp index 7552459835..34321ae0e1 100644 --- a/tests/regression/sgemm2_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm2_dxa_mcast/kernel.cpp @@ -70,9 +70,24 @@ __kernel void kernel_main(kernel_arg_t* arg) { // ── Compute: 1 row × tile_size cols ─────────────────────────────────────── const size_t l_col = threadIdx.x; // 0..tile_size-1 (single warp) TYPE sum(0); +#if LDST_WIDE + size_t k = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; k + 4 <= chunk_k; k += 4) { + vx_float4 a = vx_flw_q(&shA[k]); + sum += a[0] * shB[k * tile_size + l_col]; + sum += a[1] * shB[(k + 1) * tile_size + l_col]; + sum += a[2] * shB[(k + 2) * tile_size + l_col]; + sum += a[3] * shB[(k + 3) * tile_size + l_col]; + } + for (; k < chunk_k; ++k) { + sum += shA[k] * shB[k * tile_size + l_col]; + } +#else for (size_t k = 0; k < chunk_k; ++k) { sum += shA[k] * shB[k * tile_size + l_col]; } +#endif // Store C[row, col_blk*tile_size + l_col] C[row * size + col_blk * tile_size + l_col] = sum; diff --git a/tests/regression/sgemm2_dxa_mcast/main.cpp b/tests/regression/sgemm2_dxa_mcast/main.cpp index e238e943b8..c3c11ce607 100644 --- a/tests/regression/sgemm2_dxa_mcast/main.cpp +++ b/tests/regression/sgemm2_dxa_mcast/main.cpp @@ -223,7 +223,7 @@ int main(int argc, char* argv[]) { } cleanup(); - if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors; } + if (errors) { std::cout << "FAILED (" << errors << " errors)\n"; return errors != 0; } std::cout << "PASSED\n"; return 0; } diff --git a/tests/regression/sgemm2_tcu/common.h b/tests/regression/sgemm2_tcu/common.h index fde4cc31de..453bc225b1 100644 --- a/tests/regression/sgemm2_tcu/common.h +++ b/tests/regression/sgemm2_tcu/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint32_t cta_M; diff --git a/tests/regression/sgemm2_tcu/kernel.cpp b/tests/regression/sgemm2_tcu/kernel.cpp index 3cc808bf0d..5a69f64a09 100644 --- a/tests/regression/sgemm2_tcu/kernel.cpp +++ b/tests/regression/sgemm2_tcu/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -22,7 +22,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { size_t tile_row = blockIdx.y * cta_M; size_t tile_col = blockIdx.x * ctx::tileN; - // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] row-major + // Shared memory: A [cta_M x tileK] row-major, B [tileK x tileN] col-major auto smem = reinterpret_cast(__local_mem()); auto A_smem = smem; auto B_smem = smem + cta_M * ctx::tileK; @@ -32,6 +32,12 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(ctx::tileK); + size_t b_lane_off = ctx::lane_offset(ctx::tileK); + +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A tile [cta_M x tileK] from DRAM (row-major). uint32_t a_size = cta_M * ctx::tileK; @@ -43,21 +49,22 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } // Cooperatively load B tile [tileK x tileN] from DRAM (col-major source) - // and store row-major in shared memory. + // and keep it col-major in shared memory: K-contiguous, each B fragment + // register is one word rather than a gather across K rows. uint32_t b_size = ctx::tileK * ctx::tileN; for (size_t i = 0; i < b_size; i += num_threads) { size_t idx = i + tid; - size_t r = idx / ctx::tileN; - size_t c = idx % ctx::tileN; - B_smem[r * ctx::tileN + c] = pB[(tile_col + c) * K + (k + r)]; + size_t c = idx / ctx::tileK; + size_t r = idx % ctx::tileK; + B_smem[c * ctx::tileK + r] = pB[(tile_col + c) * K + (k + r)]; } __syncthreads(); // Per-warp WMMA from shared memory. auto A_warp = A_smem + warp_rank * ctx::tileM * ctx::tileK; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); - ctx::load_matrix_sync(fragB, B_smem, ctx::tileN); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); + ctx::load_matrix_sync(fragB, B_smem, ctx::tileK, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); __syncthreads(); diff --git a/tests/regression/sgemm2_tcu/main.cpp b/tests/regression/sgemm2_tcu/main.cpp index 400cc84793..505efb20e1 100644 --- a/tests/regression/sgemm2_tcu/main.cpp +++ b/tests/regression/sgemm2_tcu/main.cpp @@ -255,17 +255,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -275,17 +264,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -330,6 +308,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { @@ -346,8 +327,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { auto a = data_accessor_t::read(A, m * K + k); @@ -608,7 +588,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm2_v1/common.h b/tests/regression/sgemm2_v1/common.h index fcc070f6d2..b5e7e49db3 100644 --- a/tests/regression/sgemm2_v1/common.h +++ b/tests/regression/sgemm2_v1/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t grid_dim[2]; uint32_t block_dim[2]; diff --git a/tests/regression/sgemm2_v1/kernel.cpp b/tests/regression/sgemm2_v1/kernel.cpp index 9f32b87f3d..5dd1060349 100644 --- a/tests/regression/sgemm2_v1/kernel.cpp +++ b/tests/regression/sgemm2_v1/kernel.cpp @@ -39,9 +39,24 @@ void kernel_body(kernel_arg_t *arg) { __syncthreads(); +#if LDST_WIDE + size_t j = 0; + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + for (; j + 4 <= chunk_k; j += 4) { + vx_float4 a = vx_flw_q(&local_A[l_row * chunk_k + j]); + sum += a[0] * local_B[j * tile_size + l_col]; + sum += a[1] * local_B[(j + 1) * tile_size + l_col]; + sum += a[2] * local_B[(j + 2) * tile_size + l_col]; + sum += a[3] * local_B[(j + 3) * tile_size + l_col]; + } + for (; j < chunk_k; ++j) { + sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; + } +#else for (size_t j = 0; j < chunk_k; ++j) { sum += local_A[l_row * chunk_k + j] * local_B[j * tile_size + l_col]; } +#endif __syncthreads(); } diff --git a/tests/regression/sgemm2_v1/main.cpp b/tests/regression/sgemm2_v1/main.cpp index cf70a37949..e01d37ba26 100644 --- a/tests/regression/sgemm2_v1/main.cpp +++ b/tests/regression/sgemm2_v1/main.cpp @@ -282,7 +282,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu/common.h b/tests/regression/sgemm_tcu/common.h index 5e79c82a4a..8c3c3f8941 100644 --- a/tests/regression/sgemm_tcu/common.h +++ b/tests/regression/sgemm_tcu/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu/kernel.cpp b/tests/regression/sgemm_tcu/kernel.cpp index b3f8c8d2d9..b5f68054e0 100644 --- a/tests/regression/sgemm_tcu/kernel.cpp +++ b/tests/regression/sgemm_tcu/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -29,6 +29,36 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t cycles = 0; #endif +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + size_t a_lane_off = ctx::lane_offset(K); + size_t b_lane_off = ctx::lane_offset(K); + auto k_tiles = [&](size_t i, auto count) { + auto pTileA = pA + tile_row * K + i; + auto pTileB = pB + tile_col * K + i; +#ifdef PROFILE_ENABLE + __rdcycle_time t0 = vx_rdcycle_sync_begin(); +#endif + vt::unroll_for([&](auto u) { + constexpr int32_t k_off = decltype(u)::value * ctx::tileK; + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); +#ifdef PROFILE_ENABLE + __rdcycle_time t1 = vx_rdcycle_sync_end(); + cycles += vx_rdcycle_sync_diff(t0, t1); +#endif + }; + size_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (size_t i = 0; i < K; i += ctx::tileK) { auto pTileA = pA + tile_row * K + i; auto pTileB = pB + tile_col * K + i; @@ -44,6 +74,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { cycles += vx_rdcycle_sync_diff(t0, t1); #endif } +#endif // Store the computed C tile auto pTileC = pC + tile_row * N + tile_col; diff --git a/tests/regression/sgemm_tcu/main.cpp b/tests/regression/sgemm_tcu/main.cpp index 1815228882..565f39530d 100644 --- a/tests/regression/sgemm_tcu/main.cpp +++ b/tests/regression/sgemm_tcu/main.cpp @@ -368,17 +368,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -388,17 +377,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -408,17 +386,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -428,17 +395,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -448,17 +404,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -495,6 +440,9 @@ inline typename T::dtype generate_B_value() { /////////////////////////////////////////////////////////////////////////////// +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; @@ -507,10 +455,10 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { if constexpr (std::is_same::value) { - // fp32 output: the tensor core accumulates the K products in a wide - // accumulator and rounds to fp32 once; a per-step-rounded reference - // drifts by several ULP over K. Each product is exact in fp32, so a - // double accumulation reproduces the single-rounding dot product. + // The tensor core accumulates the K products in a wide accumulator + // and rounds to fp32 once; a per-step-rounded reference drifts by + // several ULP over K. Each product is exact in fp32, so a double + // accumulation reproduces the single-rounding dot product. double acc = 0.0; for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -519,8 +467,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -817,7 +764,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_mx/common.h b/tests/regression/sgemm_tcu_mx/common.h index f490685e4d..673ec252d5 100644 --- a/tests/regression/sgemm_tcu_mx/common.h +++ b/tests/regression/sgemm_tcu_mx/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_mx/kernel.cpp b/tests/regression/sgemm_tcu_mx/kernel.cpp index f091d62569..45252cc78f 100644 --- a/tests/regression/sgemm_tcu_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_mx/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -30,6 +30,36 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(K); + size_t b_lane_off = ctx::lane_offset(K); + +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + auto k_tiles = [&](uint32_t i, auto count) { + uint32_t k_tile = i / ctx::tileK; + auto pTileA = pA + tile_row * K + i; + auto pTileB = pB + tile_col * K + i; + auto pTileMxA = pMxA + (blockIdx.y * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMxB = pMxB + (blockIdx.x * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + constexpr int32_t k_off = t * ctx::tileK; + ctx::load_mx_metadata(fragA, pTileMxA + t * VX_CFG_NUM_THREADS); + ctx::load_mx_metadata(fragB, pTileMxB + t * VX_CFG_NUM_THREADS); + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; @@ -40,10 +70,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_mx_metadata(fragA, pTileMxA); ctx::load_mx_metadata(fragB, pTileMxB); - ctx::load_matrix_sync(fragA, pTileA, K); - ctx::load_matrix_sync(fragB, pTileB, K); + ctx::load_matrix_sync(fragA, pTileA, K, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } +#endif #ifdef TCU_MX_TLS if constexpr (std::is_same::value) { diff --git a/tests/regression/sgemm_tcu_mx/main.cpp b/tests/regression/sgemm_tcu_mx/main.cpp index a0cff63253..83a566e5b3 100644 --- a/tests/regression/sgemm_tcu_mx/main.cpp +++ b/tests/regression/sgemm_tcu_mx/main.cpp @@ -33,6 +33,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; @@ -421,7 +424,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_sp/common.h b/tests/regression/sgemm_tcu_sp/common.h index 1d4f38e744..9f82306c12 100644 --- a/tests/regression/sgemm_tcu_sp/common.h +++ b/tests/regression/sgemm_tcu_sp/common.h @@ -11,6 +11,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_sp/kernel.cpp b/tests/regression/sgemm_tcu_sp/kernel.cpp index a71f5a298e..38544d128b 100644 --- a/tests/regression/sgemm_tcu_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_sp/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; // is_sparse=true +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -45,6 +45,39 @@ __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { constexpr uint32_t a_k_stride = ctx::tileK / 2; auto pTileB = pB + tile_col * K; +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + size_t a_lane_off = ctx::lane_offset(stride_A); + size_t b_lane_off = ctx::lane_offset(K); + auto k_tiles = [&](auto count) { + constexpr uint32_t n = decltype(count)::value; +#ifdef PROFILE_ENABLE + __rdcycle_time t0 = vx_rdcycle_sync_begin(); +#endif + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + ctx::load_matrix_sync(fragA, pTileA, stride_A, a_lane_off); + ctx::load_sp_metadata(fragA, pMetaSp + t * per_k_tile_words); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); +#ifdef PROFILE_ENABLE + __rdcycle_time t1 = vx_rdcycle_sync_end(); + cycles += vx_rdcycle_sync_diff(t0, t1); +#endif + pMetaSp += n * per_k_tile_words; + pTileA += n * a_k_stride; + pTileB += n * ctx::tileK; + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(std::integral_constant{}); + } +#else for (int i = 0; i < (int)K; i += (int)ctx::tileK) { #ifdef PROFILE_ENABLE __rdcycle_time t0 = vx_rdcycle_sync_begin(); @@ -61,6 +94,7 @@ __kernel void kernel_main(kernel_arg_t *__UNIFORM__ arg) { pTileA += a_k_stride; pTileB += ctx::tileK; } +#endif auto pTileC = pC + tile_row * N + tile_col; ctx::store_matrix_sync(pTileC, fragC, N); diff --git a/tests/regression/sgemm_tcu_sp/main.cpp b/tests/regression/sgemm_tcu_sp/main.cpp index c3237eea4b..3fa3a674f8 100644 --- a/tests/regression/sgemm_tcu_sp/main.cpp +++ b/tests/regression/sgemm_tcu_sp/main.cpp @@ -12,9 +12,6 @@ #include #include -#ifndef FLOAT_ULP -#define FLOAT_ULP 10 -#endif #define MAX_ERRORS 100 #define RT_CHECK(_expr) \ @@ -30,6 +27,9 @@ using namespace vortex; namespace vt = tensor; +// fp32 result tolerance in ULPs; it grows with K (see fp32_ulp_bound). +static uint32_t fp32_max_ulp = 0; + /////////////////////////////////////////////////////////////////////////////// static void convert_row_to_col_major_4bit(uint8_t *dst, uint32_t width, uint32_t height, const uint8_t *src) { @@ -330,7 +330,7 @@ class Comparator { fa.f = a; fb.f = b; auto d = std::abs(fa.i - fb.i); - if (d > FLOAT_ULP) { + if (uint32_t(d) > fp32_max_ulp) { if (errors < MAX_ERRORS) { printf("*** error: [%d] expected=%f, actual=%f\n", index, fb.f, fa.f); } @@ -361,17 +361,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -381,17 +370,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -401,17 +379,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -421,17 +388,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -441,17 +397,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -493,6 +438,21 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + +// Error bound of the device's fp32 result against the exact dot product. Each +// output accumulates through a chain of uops; a uop folds its products into +// the fp32 accumulator, rounding or truncating each of those terms and the sum +// once, every time by less than half an ulp of the uop's result. The inputs are +// non-negative, so no partial sum exceeds the final one: a uop of P products +// costs at most (P + 1) / 2 ulp of the result, and the chain adds them up. The +// reference's own rounding adds one more. +static uint32_t fp32_ulp_bound(uint32_t K) { + constexpr uint32_t products = cfg::tcK * cfg::i_ratio; + uint32_t uops = (K / cfg::tileK) * (cfg::k_steps / 2); + return (uops * (products + 1) + 1) / 2 + 1; +} // Dense CPU reference matmul (pruned A has zeros at masked positions) static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { @@ -501,10 +461,9 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t for (uint32_t m = 0; m < M; ++m) { for (uint32_t n = 0; n < N; ++n) { if constexpr (std::is_same::value) { - // fp32 output: the tensor core accumulates the K products in a wide - // accumulator and rounds to fp32 once; a per-step-rounded reference - // drifts by several ULP over K. Each product is exact in fp32, so a - // double accumulation reproduces the single-rounding dot product. + // fp32 output: the exact dot product, rounded once. Each product is + // exact in fp32 and their double sum is exact to far below an fp32 + // ulp; fp32_ulp_bound covers the device's per-uop rounding. double acc = 0.0; for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -513,8 +472,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -728,6 +686,8 @@ int main(int argc, char *argv[]) { return -1; } + fp32_max_ulp = fp32_ulp_bound(K); + size_t sizeA_full = M * K; size_t sizeA = (M * K) / 2; size_t sizeB = K * N; @@ -1230,7 +1190,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return 1; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_sp_mx/common.h b/tests/regression/sgemm_tcu_sp_mx/common.h index 0c9f792d30..7b941e5d38 100644 --- a/tests/regression/sgemm_tcu_sp_mx/common.h +++ b/tests/regression/sgemm_tcu_sp_mx/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp index f75f2e2be4..1ca07382f8 100644 --- a/tests/regression/sgemm_tcu_sp_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/kernel.cpp @@ -4,7 +4,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wmma_context; +using ctx = vt::wmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -31,6 +31,37 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fill_fragment(fragC, 0); + size_t a_lane_off = ctx::lane_offset(K / 2); + size_t b_lane_off = ctx::lane_offset(K); + +#if KUNROLL > 1 + // KUNROLL tiles per iteration: each tile's load takes its K offset as a + // constant, which a vx_ldm folds into its immediate. + auto k_tiles = [&](uint32_t i, auto count) { + uint32_t k_tile = i / ctx::tileK; + auto pTileA = pA + tile_row * (K / 2) + i / 2; + auto pTileB = pB + tile_col * K + i; + auto pTileMxA = pMxA + (blockIdx.y * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMxB = pMxB + (blockIdx.x * num_k_tiles + k_tile) * VX_CFG_NUM_THREADS; + auto pTileMetaSp = pMetaSp + (blockIdx.y * num_k_tiles + k_tile) * ctx::meta_stride; + vt::unroll_for([&](auto u) { + constexpr uint32_t t = decltype(u)::value; + ctx::load_sp_metadata(fragA, pTileMetaSp + t * ctx::meta_stride); + ctx::load_mx_metadata(fragA, pTileMxA + t * VX_CFG_NUM_THREADS); + ctx::load_mx_metadata(fragB, pTileMxB + t * VX_CFG_NUM_THREADS); + ctx::load_matrix_sync(fragA, pTileA, K / 2, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); + ctx::mma_sync(fragC, fragA, fragB, fragC); + }); + }; + uint32_t i = 0; + for (; i + KUNROLL * ctx::tileK <= K; i += KUNROLL * ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } + for (; i < K; i += ctx::tileK) { + k_tiles(i, std::integral_constant{}); + } +#else for (uint32_t i = 0; i < K; i += ctx::tileK) { uint32_t k_tile = i / ctx::tileK; auto pTileA = pA + tile_row * (K / 2) + i / 2; @@ -42,10 +73,11 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::load_sp_metadata(fragA, pTileMetaSp); ctx::load_mx_metadata(fragA, pTileMxA); ctx::load_mx_metadata(fragB, pTileMxB); - ctx::load_matrix_sync(fragA, pTileA, K / 2); - ctx::load_matrix_sync(fragB, pTileB, K); + ctx::load_matrix_sync(fragA, pTileA, K / 2, a_lane_off); + ctx::load_matrix_sync(fragB, pTileB, K, b_lane_off); ctx::mma_sync(fragC, fragA, fragB, fragC); } +#endif #ifdef TCU_MX_TLS if constexpr (std::is_same::value) { diff --git a/tests/regression/sgemm_tcu_sp_mx/main.cpp b/tests/regression/sgemm_tcu_sp_mx/main.cpp index 8dabef023d..1033b7df89 100644 --- a/tests/regression/sgemm_tcu_sp_mx/main.cpp +++ b/tests/regression/sgemm_tcu_sp_mx/main.cpp @@ -34,6 +34,9 @@ using cfg = vt::wmma_config_t; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; @@ -501,7 +504,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg/common.h b/tests/regression/sgemm_tcu_wg/common.h index 7671ea6464..f0072eab37 100644 --- a/tests/regression/sgemm_tcu_wg/common.h +++ b/tests/regression/sgemm_tcu_wg/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_wg/kernel.cpp b/tests/regression/sgemm_tcu_wg/kernel.cpp index 5d75739669..235fcebe00 100644 --- a/tests/regression/sgemm_tcu_wg/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto pA = reinterpret_cast(arg->A_addr); @@ -33,6 +33,18 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); +#ifdef WGMMA_RMAJOR_A + constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements +#else + constexpr uint32_t a_ldm = 0; // block-major A +#endif +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(a_ldm); +#endif + +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { // Cooperatively load A [cta_M × tileK] into smem. // Default (block-major): A_smem[a_blockmajor_idx(r, c)]. @@ -74,15 +86,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #endif auto desc_b = vt::vx_make_smem_desc(B_smem, 0); // block-major: stride field unused - #ifdef WGMMA_RMAJOR_A - constexpr uint32_t a_ldm = ctx::tileK; // row-major A: ldm = tileK elements - #else - constexpr uint32_t a_ldm = 0; // block-major A - #endif #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only). ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, a_ldm); + ctx::load_matrix_sync(fragA, A_warp, a_ldm, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both from smem diff --git a/tests/regression/sgemm_tcu_wg/main.cpp b/tests/regression/sgemm_tcu_wg/main.cpp index 18b07262a1..9fe0860e2b 100644 --- a/tests/regression/sgemm_tcu_wg/main.cpp +++ b/tests/regression/sgemm_tcu_wg/main.cpp @@ -366,17 +366,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_htof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_htof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_htof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoh_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint16_t a, uint16_t b, float c) { @@ -386,17 +375,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint16_t eval(uint16_t a, uint16_t b, uint16_t c) { - auto fa = bit_cast(rv_btof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_btof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_btof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftob_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -406,17 +384,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e4m3tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e4m3tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e4m3tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe4m3_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint8_t a, uint8_t b, float c) { @@ -426,17 +393,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint8_t eval(uint8_t a, uint8_t b, uint8_t c) { - auto fa = bit_cast(rv_e5m2tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_e5m2tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_e5m2tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftoe5m2_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static float eval(uint32_t a, uint32_t b, float c) { @@ -446,17 +402,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - template <> struct muladd_t { static int32_t eval(uint8_t a, uint8_t b, int32_t c) { @@ -501,6 +446,9 @@ static constexpr uint32_t per_warp_N = wg_cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t M, uint32_t N, uint32_t K) { uint32_t subbytes = 8 / vt::ITYPE::bits; @@ -520,8 +468,7 @@ static void matmul_cpu(otype_t *C, const itype_t *A, const itype_t *B, uint32_t } data_accessor_t::write(C, m * N + n, static_cast(acc)); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < KS; ++k) { auto a = data_accessor_t::read(A, m * KS + k); @@ -815,7 +762,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_dxa/common.h b/tests/regression/sgemm_tcu_wg_dxa/common.h index c2eee71301..6b23f832cb 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/common.h +++ b/tests/regression/sgemm_tcu_wg_dxa/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; diff --git a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp index 7b2596dabb..01a7d58e68 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/kernel.cpp @@ -6,7 +6,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // DXA descriptor slots (programmed by host in main.cpp). [[maybe_unused]] constexpr uint32_t kDescA = 0; @@ -119,6 +119,10 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { (void)A_smem; (void)B_smem; (void)bar; (void)k; }; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + // One WGMMA accumulation step out of the given buffer pair. auto compute_tile = [&](ctx::input_t* A_smem, ctx::input_t* B_smem) { auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; @@ -127,7 +131,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only) ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both from smem @@ -146,6 +150,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // the buffer may be re-staged. Loop-carried state is {A_cur, B_cur, // bar_cur} mutated by XOR — no arrays, no unrolling, no object reloads. load_tile(A_cur, B_cur, bar_cur, 0); +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t i = 0; i < num_k_tiles; ++i) { if (i + 1 < num_k_tiles) { // Prefetch tile i+1 into the other buffer while tile i is computed. @@ -171,6 +178,9 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #else // Serial single-buffer schedule: stage and compute alternately in buf0. load_tile(A_cur, B_cur, bar_cur, 0); +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t i = 0; i < num_k_tiles; ++i) { if (i != 0) { load_tile(A_cur, B_cur, bar_cur, i * ctx::tileK); diff --git a/tests/regression/sgemm_tcu_wg_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_dxa/main.cpp index a651a7bda6..e29f62f7c9 100644 --- a/tests/regression/sgemm_tcu_wg_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); @@ -490,7 +481,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h b/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h index b69e7ef5e8..5083f575e6 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/common.h @@ -17,6 +17,17 @@ // Intra-core multicast variant of sgemm_tcu_wg_dxa. `mc_group_size` CTAs // co-resident on one core share the same B tile via DXA multicast. +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint32_t mc_group_size; diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp index 569b999df7..43d4f615fe 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/kernel.cpp @@ -15,7 +15,7 @@ #include namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; constexpr uint32_t kDescA = 0; constexpr uint32_t kDescB = 1; @@ -59,6 +59,13 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { vortex::group_barrier group_B(2, mc_group_size); const bool is_loader_warp = (get_sub_group_id() == 0); +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { if (is_loader_warp) { local_A.expect_tx(1); // A: per-CTA, manual @@ -85,7 +92,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) ctx::fragment_a fragA; - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else auto desc_a = vt::vx_make_smem_desc(A_warp, ctx::tileK * sizeof(ctx::input_t)); diff --git a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp index f32fbd3da4..c72114c769 100644 --- a/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp +++ b/tests/regression/sgemm_tcu_wg_dxa_mcast/main.cpp @@ -174,17 +174,6 @@ struct muladd_t { } }; -template <> -struct muladd_t { - static uint32_t eval(uint32_t a, uint32_t b, uint32_t c) { - auto fa = bit_cast(rv_tf32tof_s(a, 0, nullptr)); - auto fb = bit_cast(rv_tf32tof_s(b, 0, nullptr)); - auto fc = bit_cast(rv_tf32tof_s(c, 0, nullptr)); - auto fd = fa * fb + fc; - return rv_ftotf32_s(bit_cast(fd), 0, nullptr); - } -}; - /////////////////////////////////////////////////////////////////////////////// using cfg = vt::wgmma_config_t; @@ -192,6 +181,9 @@ using cfg = vt::wgmma_config_t(acc); } else { - // Narrow outputs re-round the accumulator to the output type every - // step, matching the hardware chain exactly. + // int32 output (integer inputs): exact accumulation. otype_t sum(0); for (uint32_t k = 0; k < K; ++k) { sum = muladd_t::eval(A[m * K + k], B[k * N + n], sum); @@ -516,7 +507,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_mx/common.h b/tests/regression/sgemm_tcu_wg_mx/common.h index f6cc3275d0..c8fe174ac4 100644 --- a/tests/regression/sgemm_tcu_wg_mx/common.h +++ b/tests/regression/sgemm_tcu_wg_mx/common.h @@ -13,6 +13,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr, B_addr, C_addr; diff --git a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp index 34bb9e5c9c..4a507b86d2 100644 --- a/tests/regression/sgemm_tcu_wg_mx/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; using ctx = vt::wgmma_context; + false, WGMMA_NRC, LDST_MATRIX>; __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { static_assert(sizeof(ctx::input_t) == 1, "MX WGMMA inputs use byte storage"); @@ -37,6 +37,13 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { ctx::fragment_acc fragC; ctx::fill_fragment(fragC, 0); +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK); +#endif + +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; uint32_t a_bytes = cta_m * ctx::tileK; @@ -62,7 +69,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { auto A_warp = A_smem + warp_rank * ctx::xtileM * ctx::tileK; auto desc_b = vt::vx_make_smem_desc(B_smem, ctx::tileK); #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else auto desc_a = vt::vx_make_smem_desc(A_warp, ctx::tileK); diff --git a/tests/regression/sgemm_tcu_wg_mx/main.cpp b/tests/regression/sgemm_tcu_wg_mx/main.cpp index f149b1dffc..a54056ddc2 100644 --- a/tests/regression/sgemm_tcu_wg_mx/main.cpp +++ b/tests/regression/sgemm_tcu_wg_mx/main.cpp @@ -35,6 +35,9 @@ static constexpr uint32_t kTileN = cfg::xtileN; using itype_t = typename vt::ITYPE::dtype; using otype_t = typename vt::OTYPE::dtype; +static_assert(vt::tcu_output_valid(vt::ITYPE::id, vt::OTYPE::id), + "TCU output type must be fp32 (int32 for integer inputs)"); + static constexpr uint32_t kElemsPerByte = (vt::ITYPE::bits < 8) ? (8 / vt::ITYPE::bits) : 1; static constexpr uint32_t kScalePack = 4; @@ -427,7 +430,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_sp/common.h b/tests/regression/sgemm_tcu_wg_sp/common.h index 7a2301fc08..6c62a5a380 100644 --- a/tests/regression/sgemm_tcu_wg_sp/common.h +++ b/tests/regression/sgemm_tcu_wg_sp/common.h @@ -19,6 +19,17 @@ #define CTA_SIZE VX_CFG_NUM_THREADS #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; // compressed A in global memory (M × K/2 elements) diff --git a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp index f1872cbed6..ea3efe2ec9 100644 --- a/tests/regression/sgemm_tcu_wg_sp/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/kernel.cpp @@ -5,7 +5,7 @@ namespace vt = vortex::tensor; -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // Per-warp smem layout: [A_compressed][reserved metadata row], then shared B. static constexpr uint32_t smem_a_elems = ctx::xtileM * (ctx::tileK / 2); @@ -55,6 +55,13 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK / 2); +#endif + +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (size_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; @@ -94,7 +101,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only). - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both A and B from smem descriptors diff --git a/tests/regression/sgemm_tcu_wg_sp/main.cpp b/tests/regression/sgemm_tcu_wg_sp/main.cpp index b96e99740e..d1d388f265 100644 --- a/tests/regression/sgemm_tcu_wg_sp/main.cpp +++ b/tests/regression/sgemm_tcu_wg_sp/main.cpp @@ -482,7 +482,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/common.h b/tests/regression/sgemm_tcu_wg_sp_dxa/common.h index b556c4fc6c..add5468cb7 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/common.h +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/common.h @@ -15,6 +15,17 @@ #define OTYPE fp32 #endif +// LDST_MATRIX=1 moves each tensor fragment with one vx_ldm/vx_stm where its +// shape has one; 0 keeps one flw/fsw per register. +#ifndef LDST_MATRIX +#define LDST_MATRIX 0 +#endif + +// K tiles per iteration of the K loop +#ifndef KUNROLL +#define KUNROLL 1 +#endif + typedef struct { uint32_t M, N, K; uint64_t A_addr; // compressed A in global memory (M x K/2 elements) diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp index b02b4ea17c..982cd35319 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/kernel.cpp @@ -7,7 +7,7 @@ namespace vt = vortex::tensor; // WGMMA accumulator; is_sparse=true (A is 2:4 compressed in smem) -using ctx = vt::wgmma_context; +using ctx = vt::wgmma_context; // Per-warp smem layout: [A_compressed][reserved metadata row], then shared B. static constexpr uint32_t smem_a_elems = ctx::xtileM * (ctx::tileK / 2); @@ -55,7 +55,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { uint32_t num_k_tiles = K / ctx::tileK; uint32_t meta_words_per_tile = ctx::wg_meta_total_bytes / 4; +#if defined(WGMMA_RS) && (WGMMA_NRC <= 16) + size_t a_lane_off = ctx::lane_offset(ctx::tileK / 2); +#endif + // Loop over K tiles +#if KUNROLL > 1 +#pragma unroll KUNROLL +#endif for (uint32_t k = 0; k < K; k += ctx::tileK) { uint32_t k_tile = k / ctx::tileK; @@ -88,7 +95,7 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { #if defined(WGMMA_RS) && (WGMMA_NRC <= 16) // RS: A from registers, B from smem (NRC <= 16 only) - ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2); + ctx::load_matrix_sync(fragA, A_warp, ctx::tileK / 2, a_lane_off); ctx::wgmma_sync(fragC, fragA, desc_b, fragC); #else // SS: both A and B from smem descriptors diff --git a/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp b/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp index 95c3b1277f..58ef247e40 100644 --- a/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp +++ b/tests/regression/sgemm_tcu_wg_sp_dxa/main.cpp @@ -473,7 +473,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << errors << " / " << sizeC << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemm_v1/common.h b/tests/regression/sgemm_v1/common.h index b58ec5f5e1..f244066363 100644 --- a/tests/regression/sgemm_v1/common.h +++ b/tests/regression/sgemm_v1/common.h @@ -5,6 +5,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads each thread's consecutive words with the wide loads +// (vx_flw_q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + typedef struct { uint32_t grid_dim[2]; uint32_t size; diff --git a/tests/regression/sgemm_v1/kernel.cpp b/tests/regression/sgemm_v1/kernel.cpp index 001651da5f..94ba29b4cd 100644 --- a/tests/regression/sgemm_v1/kernel.cpp +++ b/tests/regression/sgemm_v1/kernel.cpp @@ -11,9 +11,25 @@ void kernel_body(kernel_arg_t* __UNIFORM__ arg) { int row = blockIdx.y; TYPE sum(0); +#if LDST_WIDE + static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + const TYPE* a_row = A + row * size; + size_t e = 0; + for (; e + 4 <= size; e += 4) { + vx_float4 a = vx_flw_q(a_row + e); + sum += a[0] * B[e * size + col]; + sum += a[1] * B[(e + 1) * size + col]; + sum += a[2] * B[(e + 2) * size + col]; + sum += a[3] * B[(e + 3) * size + col]; + } + for (; e < size; ++e) { + sum += a_row[e] * B[e * size + col]; + } +#else for (size_t e = 0; e < size; ++e) { sum += A[row * size + e] * B[e * size + col]; } +#endif C[row * size + col] = sum; } diff --git a/tests/regression/sgemm_v1/main.cpp b/tests/regression/sgemm_v1/main.cpp index 88326e9b27..d47a686a11 100644 --- a/tests/regression/sgemm_v1/main.cpp +++ b/tests/regression/sgemm_v1/main.cpp @@ -231,7 +231,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sgemmx/common.h b/tests/regression/sgemmx/common.h index b490d09f17..942fbc1e59 100644 --- a/tests/regression/sgemmx/common.h +++ b/tests/regression/sgemmx/common.h @@ -7,6 +7,12 @@ #define TYPE float #endif +// LDST_WIDE=1 reads and writes each thread's consecutive words with the wide +// loads and stores (vx_flw_d/q, vx_fsw_d/q); 0 keeps scalar accesses. +#ifndef LDST_WIDE +#define LDST_WIDE 0 +#endif + // Per-thread output tile #ifndef THREAD_SIZE_Y #define THREAD_SIZE_Y 4 diff --git a/tests/regression/sgemmx/kernel.cpp b/tests/regression/sgemmx/kernel.cpp index 0f0b1698c3..eae70317ad 100644 --- a/tests/regression/sgemmx/kernel.cpp +++ b/tests/regression/sgemmx/kernel.cpp @@ -5,6 +5,51 @@ #define BLOCK_SIZE_M (BLOCK_DIM_Y * THREAD_SIZE_Y) #define BLOCK_SIZE_N (BLOCK_DIM_X * THREAD_SIZE_X) +#if LDST_WIDE +static_assert(__is_same(TYPE, float), "LDST_WIDE needs float"); + +// N consecutive words, in groups of 4, then 2, then 1. +template +static inline void wide_load(TYPE* dst, const TYPE* src) { + int j = 0; + #pragma unroll + for (; j + 4 <= N; j += 4) { + vx_float4 v = vx_flw_q(src + j); + dst[j] = v[0]; + dst[j + 1] = v[1]; + dst[j + 2] = v[2]; + dst[j + 3] = v[3]; + } + #pragma unroll + for (; j + 2 <= N; j += 2) { + vx_float2 v = vx_flw_d(src + j); + dst[j] = v[0]; + dst[j + 1] = v[1]; + } + #pragma unroll + for (; j < N; ++j) { + dst[j] = src[j]; + } +} + +template +static inline void wide_store(TYPE* dst, const TYPE* src) { + int j = 0; + #pragma unroll + for (; j + 4 <= N; j += 4) { + vx_fsw_q(dst + j, vx_float4{src[j], src[j + 1], src[j + 2], src[j + 3]}); + } + #pragma unroll + for (; j + 2 <= N; j += 2) { + vx_fsw_d(dst + j, vx_float2{src[j], src[j + 1]}); + } + #pragma unroll + for (; j < N; ++j) { + dst[j] = src[j]; + } +} +#endif + __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { const TYPE* A = reinterpret_cast(arg->A_addr); const TYPE* B = reinterpret_cast(arg->B_addr); @@ -44,12 +89,21 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { } // Load B into shared memory + #if LDST_WIDE + { + TYPE b_row_data[THREAD_SIZE_X]; + int b_row = k_tile * BLOCK_SIZE_K + ty; + wide_load(b_row_data, &B[b_row * N + bx * BLOCK_SIZE_N + tx * THREAD_SIZE_X]); + wide_store(&Bs[ty * BLOCK_SIZE_N + tx * THREAD_SIZE_X], b_row_data); + } + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { int b_row = k_tile * BLOCK_SIZE_K + ty; int b_col = bx * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j; Bs[ty * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j] = B[b_row * N + b_col]; } + #endif __syncthreads(); @@ -60,10 +114,14 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { for (int i = 0; i < THREAD_SIZE_Y; ++i) { a_frag[i] = As[(ty * THREAD_SIZE_Y + i) * BLOCK_SIZE_K + k]; } + #if LDST_WIDE + wide_load(b_frag, &Bs[k * BLOCK_SIZE_N + tx * THREAD_SIZE_X]); + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { b_frag[j] = Bs[k * BLOCK_SIZE_N + tx * THREAD_SIZE_X + j]; } + #endif #pragma unroll for (int i = 0; i < THREAD_SIZE_Y; ++i) { @@ -80,11 +138,22 @@ __kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { // --- Write Back to Global Memory --- #pragma unroll for (int i = 0; i < THREAD_SIZE_Y; ++i) { + #if LDST_WIDE + TYPE c_data[THREAD_SIZE_X]; + TYPE* c_ptr = &C[(row + i) * N + col]; + wide_load(c_data, c_ptr); + #pragma unroll + for (int j = 0; j < THREAD_SIZE_X; ++j) { + c_data[j] = alpha * accum[i][j] + beta * c_data[j]; + } + wide_store(c_ptr, c_data); + #else #pragma unroll for (int j = 0; j < THREAD_SIZE_X; ++j) { int c_row = row + i; int c_col = col + j; C[c_row * N + c_col] = alpha * accum[i][j] + beta * C[c_row * N + c_col]; } + #endif } } diff --git a/tests/regression/sgemmx/main.cpp b/tests/regression/sgemmx/main.cpp index 105d4c62da..528022d23b 100644 --- a/tests/regression/sgemmx/main.cpp +++ b/tests/regression/sgemmx/main.cpp @@ -252,7 +252,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/sort/main.cpp b/tests/regression/sort/main.cpp index bfe56aa562..8a69a6bf18 100644 --- a/tests/regression/sort/main.cpp +++ b/tests/regression/sort/main.cpp @@ -197,7 +197,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/stencil3d/main.cpp b/tests/regression/stencil3d/main.cpp index e333438615..6a8d481f57 100644 --- a/tests/regression/stencil3d/main.cpp +++ b/tests/regression/stencil3d/main.cpp @@ -321,7 +321,7 @@ int main(int argc, char *argv[]) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/uniform_ld/Makefile b/tests/regression/uniform_ld/Makefile new file mode 100644 index 0000000000..fb9ed28371 --- /dev/null +++ b/tests/regression/uniform_ld/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := uniform_ld + +SRC_DIR := $(VORTEX_HOME)/tests/regression/$(PROJECT) + +SRCS := $(SRC_DIR)/main.cpp + +VX_SRCS := $(SRC_DIR)/kernel.cpp + +OPTS ?= -n64 + +KERNEL_LIB := vortex2 + +include ../common.mk diff --git a/tests/regression/uniform_ld/common.h b/tests/regression/uniform_ld/common.h new file mode 100644 index 0000000000..010b9dc91b --- /dev/null +++ b/tests/regression/uniform_ld/common.h @@ -0,0 +1,27 @@ +#ifndef _COMMON_H_ +#define _COMMON_H_ + +#include + +#define UNIFORM_MARK 0x80000000u + +// Per warp and iteration: one RAW word and one WAR word (four words apart, +// so four iterations share a line), and one 16-word line of read-only words. +#define RAW_STRIDE 4 +#define WAR_STRIDE 4 +#define RO_LINE 16 + +typedef struct { + uint32_t num_iters; // iterations per warp + uint32_t num_threads; // total threads launched + uint32_t warp_size; // threads per warp + uint32_t scatter_words; + uint64_t raw_addr; // in: initial words; out: RAW words overwritten with UNIFORM_MARK + index + uint64_t war_addr; // in: initial words; out: WAR words overwritten with UNIFORM_MARK + index + uint64_t ro_addr; // in: read-only words + uint64_t scatter_addr;// in: read-only words scattered across lines + uint64_t err_addr; // out: per-thread error count + uint64_t sink_addr; // out: per-thread sum of the scattered loads (keeps them alive) +} kernel_arg_t; + +#endif diff --git a/tests/regression/uniform_ld/kernel.cpp b/tests/regression/uniform_ld/kernel.cpp new file mode 100644 index 0000000000..7673a99508 --- /dev/null +++ b/tests/regression/uniform_ld/kernel.cpp @@ -0,0 +1,60 @@ +#include +#include "common.h" + +// Warp-uniform loads around single-lane stores. Per iteration k, lane +// s = k % W (so every lane, and with it every dcache channel, takes a turn): +// +// scattered loads - every lane a different line, so the channels drain +// unevenly before the accesses below +// RAW: lane s stores its word, then the whole warp loads it (one address): +// lane s must see its own store +// WAR: the whole warp loads a word, then lane s stores it: lane s's load +// must not see its own later store +// read-only loads - lanes in groups of 1, 2, 4, 8 or 16 share a word of +// one line: every lane must get its own word +__kernel void kernel_main(kernel_arg_t* __UNIFORM__ arg) { + auto raw = reinterpret_cast(arg->raw_addr); + auto war = reinterpret_cast(arg->war_addr); + auto ro = reinterpret_cast(arg->ro_addr); + auto scatter = reinterpret_cast(arg->scatter_addr); + auto err = reinterpret_cast(arg->err_addr); + auto sink = reinterpret_cast(arg->sink_addr); + const uint32_t W = arg->warp_size; + const uint32_t tid = blockIdx.x * blockDim.x + threadIdx.x; + const uint32_t lane = tid % W; + const uint32_t warp = tid / W; + uint32_t errors = 0; + uint32_t acc = 0; + for (uint32_t k = 0; k < arg->num_iters; ++k) { + uint32_t s = k % W; + uint32_t slot = warp * arg->num_iters + k; + + acc += scatter[(lane * RO_LINE + slot * 5) % arg->scatter_words]; + + uint32_t ri = slot * RAW_STRIDE; + if (lane == s) { + raw[ri] = UNIFORM_MARK + ri; + } + uint32_t rv = raw[ri]; + if (lane == s && rv != UNIFORM_MARK + ri) { + ++errors; + } + + uint32_t wi = slot * WAR_STRIDE; + uint32_t wv = war[wi]; + if (lane == s) { + war[wi] = UNIFORM_MARK + wi; + } + if (lane == s && wv != wi * 7 + 1) { + ++errors; + } + + uint32_t shift = k % 5; + uint32_t oi = slot * RO_LINE + ((lane >> shift) & (RO_LINE - 1)); + if (ro[oi] != oi * 3 + 2) { + ++errors; + } + } + err[tid] = errors; + sink[tid] = acc; +} diff --git a/tests/regression/uniform_ld/main.cpp b/tests/regression/uniform_ld/main.cpp new file mode 100644 index 0000000000..40d0943689 --- /dev/null +++ b/tests/regression/uniform_ld/main.cpp @@ -0,0 +1,164 @@ +// uniform_ld — warp-uniform loads must keep each thread's order with its own +// stores, and every lane must receive its own word. + +#include +#include "common.h" + +#include +#include +#include +#include +#include +#include + +#define CHECK(expr) do { \ + vx_result_t _r = (expr); \ + if (_r != VX_SUCCESS) { \ + std::fprintf(stderr, "FAIL %s:%d: '%s' returned %s\n", \ + __FILE__, __LINE__, #expr, vx_result_string(_r)); \ + std::exit(1); \ + } \ +} while (0) + +namespace { +const char* kernel_file = "kernel.vxbin"; +uint32_t num_iters = 64; +uint32_t scatter_words = 65536; + +void parse_args(int argc, char** argv) { + int c; + while ((c = getopt(argc, argv, "n:s:k:h")) != -1) { + switch (c) { + case 'n': num_iters = std::atoi(optarg); break; + case 's': scatter_words = std::atoi(optarg); break; + case 'k': kernel_file = optarg; break; + default: + std::cout << "Usage: [-n iterations] [-s scattered words] [-k kernel] [-h]" << std::endl; + std::exit(c == 'h' ? 0 : -1); + } + } +} +} // namespace + +int main(int argc, char** argv) { + parse_args(argc, argv); + if (num_iters == 0) num_iters = 1; + if (scatter_words == 0) scatter_words = 1; + + vx_device_h dev = nullptr; + CHECK(vx_device_open(0, &dev)); + + // Fill every warp of every core. + uint64_t nt = 0, nw = 0, nc = 0; + CHECK(vx_device_query(dev, VX_CAPS_NUM_THREADS, &nt)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_WARPS, &nw)); + CHECK(vx_device_query(dev, VX_CAPS_NUM_CORES, &nc)); + const uint32_t W = static_cast(nt); + const uint32_t num_threads = static_cast(nt * nw * nc); + const uint32_t num_slots = (num_threads / W) * num_iters; + std::cout << "uniform_ld: iters=" << num_iters << " threads=" << num_threads + << " scatter=" << scatter_words << std::endl; + + vx_queue_info_t qi = { sizeof(qi), nullptr, VX_QUEUE_PRIORITY_NORMAL, 0 }; + vx_queue_h q = nullptr; + CHECK(vx_queue_create(dev, &qi, &q)); + + const uint32_t raw_words = num_slots * RAW_STRIDE; + const uint32_t war_words = num_slots * WAR_STRIDE; + const uint32_t ro_words = num_slots * RO_LINE; + vx_buffer_h raw_buf = nullptr, war_buf = nullptr, ro_buf = nullptr; + vx_buffer_h scatter_buf = nullptr, err_buf = nullptr, sink_buf = nullptr; + CHECK(vx_buffer_create(dev, raw_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &raw_buf)); + CHECK(vx_buffer_create(dev, war_words * sizeof(uint32_t), VX_MEM_READ_WRITE, &war_buf)); + CHECK(vx_buffer_create(dev, ro_words * sizeof(uint32_t), VX_MEM_READ, &ro_buf)); + CHECK(vx_buffer_create(dev, scatter_words * sizeof(uint32_t), VX_MEM_READ, &scatter_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &err_buf)); + CHECK(vx_buffer_create(dev, num_threads * sizeof(uint32_t), VX_MEM_WRITE, &sink_buf)); + + vx_module_h mod = nullptr; + vx_kernel_h kern = nullptr; + CHECK(vx_module_load_file(dev, kernel_file, &mod)); + CHECK(vx_module_get_kernel(mod, "main", &kern)); + + kernel_arg_t kernel_arg{}; + kernel_arg.num_iters = num_iters; + kernel_arg.num_threads = num_threads; + kernel_arg.warp_size = W; + kernel_arg.scatter_words = scatter_words; + CHECK(vx_buffer_address(raw_buf, &kernel_arg.raw_addr)); + CHECK(vx_buffer_address(war_buf, &kernel_arg.war_addr)); + CHECK(vx_buffer_address(ro_buf, &kernel_arg.ro_addr)); + CHECK(vx_buffer_address(scatter_buf, &kernel_arg.scatter_addr)); + CHECK(vx_buffer_address(err_buf, &kernel_arg.err_addr)); + CHECK(vx_buffer_address(sink_buf, &kernel_arg.sink_addr)); + + std::vector h_raw(raw_words), h_war(war_words), h_ro(ro_words), h_scatter(scatter_words); + for (uint32_t i = 0; i < raw_words; ++i) h_raw[i] = i * 7 + 1; + for (uint32_t i = 0; i < war_words; ++i) h_war[i] = i * 7 + 1; + for (uint32_t i = 0; i < ro_words; ++i) h_ro[i] = i * 3 + 2; + for (uint32_t i = 0; i < scatter_words; ++i) h_scatter[i] = i; + CHECK(vx_enqueue_write(q, raw_buf, 0, h_raw.data(), raw_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, war_buf, 0, h_war.data(), war_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, ro_buf, 0, h_ro.data(), ro_words * sizeof(uint32_t), 0, nullptr, nullptr)); + CHECK(vx_enqueue_write(q, scatter_buf, 0, h_scatter.data(), scatter_words * sizeof(uint32_t), 0, nullptr, nullptr)); + + uint32_t grid[1], block[1]; + CHECK(vx_device_max_occupancy_grid(dev, 1, &num_threads, grid, block)); + + vx_launch_info_t li{}; + li.struct_size = sizeof(li); + li.kernel = kern; + li.args_host = &kernel_arg; + li.args_size = sizeof(kernel_arg); + li.ndim = 1; + li.grid_dim[0] = grid[0]; + li.block_dim[0]= block[0]; + + std::vector h_err(num_threads); + vx_event_h launch_ev = nullptr, read_ev = nullptr; + CHECK(vx_enqueue_launch(q, &li, 0, nullptr, &launch_ev)); + CHECK(vx_enqueue_read(q, h_err.data(), err_buf, 0, num_threads * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_raw.data(), raw_buf, 0, raw_words * sizeof(uint32_t), 1, &launch_ev, nullptr)); + CHECK(vx_enqueue_read(q, h_war.data(), war_buf, 0, war_words * sizeof(uint32_t), 1, &launch_ev, &read_ev)); + CHECK(vx_event_wait_value(read_ev, 1, VX_TIMEOUT_INFINITE)); + + int errors = 0; + for (uint32_t t = 0; t < num_threads; ++t) { + if (h_err[t] == 0) continue; + if (errors < 16) + std::printf("*** thread %u: %u wrong loads\n", t, h_err[t]); + errors += h_err[t]; + } + for (uint32_t slot = 0; slot < num_slots; ++slot) { + uint32_t ri = slot * RAW_STRIDE, wi = slot * WAR_STRIDE; + if (h_raw[ri] != UNIFORM_MARK + ri) { + if (errors < 16) std::printf("*** raw[%u] expected=0x%x actual=0x%x\n", ri, UNIFORM_MARK + ri, h_raw[ri]); + ++errors; + } + if (h_war[wi] != UNIFORM_MARK + wi) { + if (errors < 16) std::printf("*** war[%u] expected=0x%x actual=0x%x\n", wi, UNIFORM_MARK + wi, h_war[wi]); + ++errors; + } + } + + vx_event_release(read_ev); + vx_event_release(launch_ev); + vx_buffer_release(sink_buf); + vx_buffer_release(err_buf); + vx_buffer_release(scatter_buf); + vx_buffer_release(ro_buf); + vx_buffer_release(war_buf); + vx_buffer_release(raw_buf); + vx_kernel_release(kern); + vx_module_release(mod); + vx_queue_release(q); + vx_device_dump_perf(dev, stdout); + vx_device_release(dev); + + if (errors) { + std::cout << "Found " << errors << " errors!\nFAILED!" << std::endl; + return 1; + } + std::cout << "PASSED!" << std::endl; + return 0; +} diff --git a/tests/regression/vm_stress/main.cpp b/tests/regression/vm_stress/main.cpp index cbb3a98939..9efa622346 100644 --- a/tests/regression/vm_stress/main.cpp +++ b/tests/regression/vm_stress/main.cpp @@ -177,7 +177,7 @@ int main(int argc, char *argv[]) { if (errors != 0) { std::cout << "Found " << std::dec << errors << " errors!" << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/regression/vm_test/main.cpp b/tests/regression/vm_test/main.cpp index 20f9b4b123..89bd2c400e 100644 --- a/tests/regression/vm_test/main.cpp +++ b/tests/regression/vm_test/main.cpp @@ -236,7 +236,7 @@ int main(int argc, char* argv[]) { if (errors != 0) { std::cout << "FAILED! - " << errors << " errors" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; return 0; diff --git a/tests/regression/wsync/main.cpp b/tests/regression/wsync/main.cpp index c1433659a6..c17611968b 100644 --- a/tests/regression/wsync/main.cpp +++ b/tests/regression/wsync/main.cpp @@ -157,7 +157,7 @@ int main(int argc, char* argv[]) { if (0 != errors) { std::cout << "WSYNC timing mismatches: " << errors << std::endl; std::cout << "FAILED!" << std::endl; - return errors; + return errors != 0; } std::cout << "PASSED!" << std::endl; diff --git a/tests/unittest/Makefile b/tests/unittest/Makefile index 33554509cd..66626ae757 100644 --- a/tests/unittest/Makefile +++ b/tests/unittest/Makefile @@ -1,7 +1,7 @@ ROOT_DIR := $(realpath ../..) include $(ROOT_DIR)/config.mk -PROJECTS := vx_malloc linked_list dram_sim dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt +PROJECTS := vx_malloc linked_list dram_sim dram_bank_model dxa_ooo rvc_decompress gfx_binsort gfx_tex_sw gfx_rast_sw gfx_msaa gfx_om_mrt all: $(foreach p,$(PROJECTS),$(MAKE) -C $(p) &&) true diff --git a/tests/unittest/dram_bank_model/Makefile b/tests/unittest/dram_bank_model/Makefile new file mode 100644 index 0000000000..243cb3366b --- /dev/null +++ b/tests/unittest/dram_bank_model/Makefile @@ -0,0 +1,16 @@ +ROOT_DIR := $(realpath ../../..) +include $(ROOT_DIR)/config.mk + +PROJECT := dram_bank_model + +SRC_DIR := $(VORTEX_HOME)/tests/unittest/$(PROJECT) +RAMULATOR_DIR := $(THIRD_PARTY_DIR)/ramulator + +SRCS := $(SRC_DIR)/main.cpp $(SIM_COMMON_DIR)/dram_sim.cpp + +CXXFLAGS += -I$(SIM_COMMON_DIR) +CXXFLAGS += -isystem $(RAMULATOR_DIR)/ext/spdlog/include -isystem $(RAMULATOR_DIR)/ext/yaml-cpp/include -isystem $(RAMULATOR_DIR)/src +LDFLAGS += -Wl,-rpath,$(RAMULATOR_DIR) -L$(RAMULATOR_DIR) -lramulator +LDFLAGS += -Wl,-rpath,$(THIRD_PARTY_DIR)/ramulator_ext -L$(THIRD_PARTY_DIR)/ramulator_ext -Wl,--push-state,--no-as-needed -lramulator-vortex -Wl,--pop-state + +include ../common.mk diff --git a/tests/unittest/dram_bank_model/main.cpp b/tests/unittest/dram_bank_model/main.cpp new file mode 100644 index 0000000000..26458cdace --- /dev/null +++ b/tests/unittest/dram_bank_model/main.cpp @@ -0,0 +1,280 @@ +// Copyright © 2019-2023 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The bank model (sim/common/dram_bank_model.h) against Ramulator (DramSim) on +// the same closed-loop traffic: per channel, `window` requests outstanding, +// reads until their response, writes until acknowledged. Both take the +// traffic through DramSim's interface, so the clock loop, ingress, fixed +// latency and response drains are the same and only the scheduling differs. +// +// The gate: an idle channel's reads (one outstanding) take the same latency in +// both, read for read, on every configuration -- row hits, misses and +// conflicts. Loaded traffic is reported: bandwidth, mean / p50 / p99 read +// latency, the Kolmogorov-Smirnov distance between the latency distributions, +// and the bank model's row-hit rate. +// +// ./dram_bank_model the gate, on short runs +// ./dram_bank_model full [cfg] the full table (20,000 requests per cell) +// ./dram_bank_model refresh the bandwidth refresh costs the bank model, +// per bank and all banks, at 64 outstanding +// (dram_sim models no refresh). The gate: one +// refresh per interval, to the operation; and a +// stream, which moves data every clock it may, +// loses at least tRFC / tREFI to all-bank +// refresh, the time no bank is open + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include + +using namespace vortex; + +namespace { + +struct Machine { + const char* name; + DramSim::Config cfg; +}; + +// The default machine and the profiles' channels (profiles/*.json), one +// channel each unless named otherwise. +const Machine kMachines[] = { + {"default", {DramSim::Type::HBM2, 1, 64, 400, 800, 0, 0}}, + {"default4", {DramSim::Type::HBM2, 4, 64, 400, 800, 0, 0}}, + {"v100", {DramSim::Type::HBM2, 1, 64, 1530, 1752, 0, 0}}, + {"a100", {DramSim::Type::HBM2, 1, 64, 1410, 3038, 190, 257}}, + {"h100", {DramSim::Type::HBM3, 1, 64, 1980, 6545, 0, 0}}, + {"gtx1080", {DramSim::Type::GDDR6, 1, 64, 1607, 10000, 0, 0}}, + {"rtx5080", {DramSim::Type::GDDR6, 1, 64, 2295, 15000, 339, 0}}, +}; + +struct Rec { + uint64_t sent, done; + bool w; +}; + +uint64_t g_cycle = 0; + +bool on_rsp(void* arg) { + static_cast(arg)->done = g_cycle; + return true; +} + +struct Pattern { + const char* name; + std::function block; + double write_frac; +}; + +struct Result { + double bw = 0, mean = 0, p50 = 0, p99 = 0; + std::vector lat; +}; + +template +Result run(M& m, const DramSim::Config& cfg, const Pattern& p, uint32_t window, uint32_t n) { + std::mt19937_64 rng(12345), rng_w(777); + std::vector recs(uint64_t(n) * cfg.num_channels); + std::vector sent(cfg.num_channels, 0); + std::vector> live(cfg.num_channels); + const uint64_t total = recs.size(); + uint64_t done = 0; + g_cycle = 0; + std::uniform_real_distribution u(0, 1); + while (done < total) { + for (uint32_t ch = 0; ch < cfg.num_channels; ++ch) { + auto& l = live[ch]; + for (size_t i = 0; i < l.size();) { + if (l[i]->done != ~0ull) { + l[i] = l.back(); + l.pop_back(); + ++done; + } else { + ++i; + } + } + if (sent[ch] < n && l.size() < window && m.can_accept(ch)) { + Rec* r = &recs[uint64_t(ch) * n + sent[ch]]; + r->sent = g_cycle; + r->done = ~0ull; + r->w = u(rng_w) < p.write_frac; + m.send_request(ch, p.block(sent[ch], rng) * cfg.block_size, r->w, on_rsp, r); + l.push_back(r); + ++sent[ch]; + } + } + m.tick(); + ++g_cycle; + } + Result res; + res.bw = double(total) * cfg.block_size / double(g_cycle); + for (auto& r : recs) { + if (!r.w) res.lat.push_back(r.done - r.sent); + } + std::sort(res.lat.begin(), res.lat.end()); + double s = 0; + for (auto v : res.lat) s += double(v); + if (!res.lat.empty()) { + res.mean = s / double(res.lat.size()); + res.p50 = double(res.lat[res.lat.size() / 2]); + res.p99 = double(res.lat[std::min(res.lat.size() - 1, size_t(0.99 * double(res.lat.size())))]); + } + return res; +} + +double ks_distance(const std::vector& a, const std::vector& b) { + if (a.empty() || b.empty()) return 0; + double ks = 0; + size_t i = 0, j = 0; + while (i < a.size() && j < b.size()) { + uint64_t v = std::min(a[i], b[j]); + while (i < a.size() && a[i] <= v) ++i; + while (j < b.size() && b[j] <= v) ++j; + ks = std::max(ks, std::fabs(double(i) / double(a.size()) - double(j) / double(b.size()))); + } + return ks; +} + +DramBankSim::Config bank_config(const DramSim::Config& c, + DramBankModel::Refresh refresh = DramBankModel::Refresh::Off) { + return DramBankSim::Config{c.type, c.num_channels, c.block_size, c.core_clock, c.data_rate, + c.latency, c.queue_size, refresh}; +} + +std::vector patterns() { + const uint64_t blocks = (256ull << 20) / 64; + return { + {"seq", [](uint64_t i, std::mt19937_64&) { return i; }, 0.0}, + {"rand", [blocks](uint64_t, std::mt19937_64& r) { return r() % blocks; }, 0.0}, + // Four sequential streams 1 MB apart, interleaved. + {"strided4", [](uint64_t i, std::mt19937_64&) { return (i % 4) * (1ull << 14) + i / 4; }, 0.0}, + // Blocks in random rows among 64 that all map to one bank. + {"conflict", [](uint64_t i, std::mt19937_64& r) { + uint64_t row = r() % 64; + return (row * 33 << 15) / 64 + (i % 2); + }, 0.0}, + {"rand_w30", [blocks](uint64_t, std::mt19937_64& r) { return r() % blocks; }, 0.3}, + {"seq_w50", [](uint64_t i, std::mt19937_64&) { return i; }, 0.5}, + }; +} + +} // namespace + +// Bandwidth with refresh off, per bank and all banks. +int refresh_table() { + using R = DramBankModel::Refresh; + int failures = 0; + const uint32_t n = 60000, w = 64; + printf("%-9s %-9s | %7s | %7s %6s %6s | %7s %6s %6s %6s\n", "machine", "pattern", "off B/c", + "bank", "loss%", "refs", "all", "loss%", "refs", "tRFC/tREFI%"); + for (const auto& mach : kMachines) { + if (mach.cfg.type == DramSim::Type::GDDR6) continue; + const auto std = (mach.cfg.type == DramSim::Type::HBM3) ? dram_timing::Standard::HBM3 + : dram_timing::Standard::HBM2; + const auto r = dram_timing::refresh(std, mach.cfg.data_rate); + const double analytic = 100.0 * double(r.RFC) / double(r.REFI); + for (const auto& p : patterns()) { + if (p.write_frac != 0 || !std::strcmp(p.name, "conflict")) continue; + double bw[3]; + uint64_t refs[3]; + const R modes[3] = {R::Off, R::PerBank, R::AllBank}; + for (int k = 0; k < 3; ++k) { + DramBankSim mod(bank_config(mach.cfg, modes[k])); + bw[k] = run(mod, mach.cfg, p, w, n).bw; + refs[k] = 0; + for (uint32_t ch = 0; ch < mach.cfg.num_channels; ++ch) { + const auto& m = mod.channel(ch); + refs[k] += m.stats().refreshes; + // The refreshes due by the last clock, but one in progress. + const uint32_t refi = m.config().refi; + const uint64_t due = refi ? m.clock() / refi : 0; + if (m.stats().refreshes + 1 < due || m.stats().refreshes > due) { + printf("FAIL %s %s: %lu refreshes in %lu clocks of %u\n", mach.name, p.name, + (unsigned long)m.stats().refreshes, (unsigned long)m.clock(), refi); + ++failures; + } + } + } + const double lb = 100.0 * (1 - bw[1] / bw[0]), la = 100.0 * (1 - bw[2] / bw[0]); + printf("%-9s %-9s | %7.2f | %7.2f %6.2f %6lu | %7.2f %6.2f %6lu %6.2f\n", mach.name, p.name, + bw[0], bw[1], lb, (unsigned long)refs[1], bw[2], la, (unsigned long)refs[2], analytic); + if (!std::strcmp(p.name, "seq") && la < analytic) { + printf("FAIL %s %s: all-bank refresh costs %.2f%%, less than tRFC/tREFI (%.2f%%)\n", mach.name, + p.name, la, analytic); + ++failures; + } + } + } + if (failures) { + printf("FAILED: %d\n", failures); + return 1; + } + printf("PASSED!\n"); + return 0; +} + +int main(int argc, char** argv) { + if (argc > 1 && !std::strcmp(argv[1], "refresh")) + return refresh_table(); + const bool full = argc > 1 && !std::strcmp(argv[1], "full"); + const char* only = (full && argc > 2) ? argv[2] : nullptr; + const uint32_t n = full ? 20000 : 2000; + const std::vector windows = full ? std::vector{1, 4, 16, 64} + : std::vector{1, 16}; + int failures = 0; + printf("%-9s %-9s %4s | %7s %7s %6s %6s | %7s %7s %6s %6s | %6s %6s %5s %5s\n", "machine", + "pattern", "win", "ref B/c", "mean", "p50", "p99", "mod B/c", "mean", "p50", "p99", "dBW%", + "dMean%", "KS", "hit%"); + for (const auto& mach : kMachines) { + if (only && std::strcmp(only, mach.name)) continue; + for (const auto& p : patterns()) { + for (uint32_t w : windows) { + DramSim ref(mach.cfg); + ref.reset(); + Result a = run(ref, mach.cfg, p, w, n); + DramBankSim mod(bank_config(mach.cfg)); + Result b = run(mod, mach.cfg, p, w, n); + uint64_t hits = 0, ops = 0; + for (uint32_t ch = 0; ch < mach.cfg.num_channels; ++ch) { + const auto& st = mod.channel(ch).stats(); + hits += st.hits; + ops += st.hits + st.misses + st.conflicts; + } + const double ks = ks_distance(a.lat, b.lat); + printf("%-9s %-9s %4u | %7.2f %7.1f %6.0f %6.0f | %7.2f %7.1f %6.0f %6.0f | %6.1f %6.1f %5.3f %5.1f\n", + mach.name, p.name, w, a.bw, a.mean, a.p50, a.p99, b.bw, b.mean, b.p50, b.p99, + 100.0 * (b.bw - a.bw) / a.bw, a.mean ? 100.0 * (b.mean - a.mean) / a.mean : 0.0, ks, + ops ? 100.0 * double(hits) / double(ops) : 0.0); + // One read outstanding on an idle channel: every latency the same. + if (w == 1 && p.write_frac == 0 && a.lat != b.lat) { + printf("FAIL %s %s: idle read latencies differ\n", mach.name, p.name); + ++failures; + } + } + } + } + if (failures) { + printf("FAILED: %d\n", failures); + return 1; + } + printf("PASSED!\n"); + return 0; +}