diff --git a/docs/8852c-quirks.md b/docs/8852c-quirks.md index 8714fc11..fa9a2f1c 100644 --- a/docs/8852c-quirks.md +++ b/docs/8852c-quirks.md @@ -92,3 +92,56 @@ it: the value jumps ~24 dB the same window bit 30 clears, ~1.5 s after tune, while the LNA/RXBB gain registers stay constant. The shipped `tests/kestrel_nhm_2g_onair.sh` validates the current band gate (2.4 GHz emits a plausible floor; 5 GHz is null → passive fallback). + +## An undrained USB bulk-IN during bring-up halts the firmware + +### What the chip does + +`intf_init` arms the USB RX path partway through the bring-up. If the host +does not read the bulk-IN from then on, the 8852C flags its RX DMA stuck +during the H2C-heavy BB/RF table load: `HAXI_IDCT` bit 1 +(`B_AX_RXMDA_STUCK_IDCT`) -> `DMAC_ERR_ISR` `B_AX_HAXIDMA_ERR_FLAG` (0x4000) +-> L1 pre-error `0x999` (`L1_PREERR_DMAC`) in `HALT_C2H`, and with no host L1 +handshake the firmware asserts (`0x30000024`). Every bring-up step still +reports success. The radio then transmits for a while, until its TX queues +stop draining for good (bulk-OUT timeouts on every send). The 8852B tolerates +the same undrained window. Same failure family as the MT7612U's undrained +bulk-IN (`mt7612u.md`). + +### What devourer does about it + +`BringUpMonitorOnce` starts the WP/bulk-IN drain right after +`PowerOnFwAndTrx`; `StartRxLoop` takes the endpoint over from it as before. +`BringUpCheckingFwHalt` then reads `HALT_C2H_CTRL` after every monitor and TX +bring-up. On a halt it logs the DMAC error registers (plus the HAXI block, on +this die only, as the vendor's `dump_err_status_haxidma_ax` does) and, on +USB, redoes the whole bring-up from `power_on` (3 attempts). On PCIe the redo +is unverified, so a halt fails the bring-up there. + +Measured on a TX50UH, back-to-back txdemo bring-ups (200 frames, ch149, log +level warn), with the same halt check in both arms and only the early +`start_wp_drain()` call removed in the "undrained" one: + +| session | undrained: runs that halted | of which failed all 3 attempts | drained | +|---|---|---|---| +| 2026-10-18, 16:46-17:00 | 17 of 30 | 3 | 0 of 30 | +| 2026-10-20, 07:56-08:07 | 7 of 66 | 0 | 0 of 50 | + +Every halt read `HAXI_IDCT=0x2` and `DMAC_ERR_ISR=0x4000`. The rate varies a +lot between sessions; it is timing-sensitive (debug logging hides it) and the +traffic the radio hears during the bring-up may matter (not isolated). The +8852B (TX20U Nano) halted 0 of 20 both ways on 2026-10-18, with the same +bring-up time. + +### Residual cost + +None measured: bring-up time is unchanged on both dies. Not covered: the +8852CE (PCIe, no drain thread: its release reports arrive on the RPQ ring) +and Linux hosts. + +### Reproducer + +`tests/kestrel_bringup_halt.sh` runs N bring-ups and counts halted attempts +(exit 1 on any). Run it against a build with the early `start_wp_drain()` in +`BringUpMonitorOnce` removed to see the halt, and against the stock build for +the drained arm. diff --git a/src/kestrel/CLAUDE.md b/src/kestrel/CLAUDE.md index f9fa61aa..01e5462a 100644 --- a/src/kestrel/CLAUDE.md +++ b/src/kestrel/CLAUDE.md @@ -48,6 +48,10 @@ MAC off from any retained state, while a USB reset on running firmware drops the chip to ROM — a stale-handle re-enumeration that can land in the dead ZeroCD DISK id (`0bda:1a2b`). +Bring-up drains the USB bulk-IN from `intf_init` on and checks for a halted +firmware at its end; do not move the drain later (8852C quirk, +`docs/8852c-quirks.md`). + ## Validated capability surface On-air-validated: monitor RX (both dies, 2.4/5 GHz; the RX front-end needs diff --git a/src/kestrel/HalKestrel.cpp b/src/kestrel/HalKestrel.cpp index 9439bb6f..6c146c1a 100644 --- a/src/kestrel/HalKestrel.cpp +++ b/src/kestrel/HalKestrel.cpp @@ -3,6 +3,7 @@ #include #include #include +#include #include #include @@ -816,11 +817,37 @@ uint32_t HalKestrel::fw_err_state(const char *where) { const uint32_t ctrl = _device.rtw_read32(r::R_AX_HALT_C2H_CTRL); if (ctrl == 0) return 0; - const uint32_t err = _device.rtw_read32(r::R_AX_HALT_C2H); - _logger->error("Kestrel FW-SER at [{}]: HALT_C2H_CTRL=0x{:08x} err=0x{:08x} " - "(0x2010=L2_AH_HCI 0x1000=L1_DMAC 0x2000=L2_AH_DMA)", - where, ctrl, err); - return err; + /* The halt is established by now: everything below is diagnostics, and a + * failed read here (the USB transport throws) must not hide it from the + * caller, so it is logged and the halt still returned. */ + uint32_t err = 0; + try { + err = _device.rtw_read32(r::R_AX_HALT_C2H); + _logger->error("Kestrel FW-SER at [{}]: HALT_C2H_CTRL=0x{:08x} err=0x{:08x} " + "(0x2010=L2_AH_HCI 0x1000=L1_DMAC 0x2000=L2_AH_DMA)", + where, ctrl, err); + /* Which DMAC error raised it (dbgpkg.c dump_err_status_dmac), then on the + * 8852C the HAXI block its USB DMA runs through + * (dump_err_status_haxidma_ax, which the vendor builds for the AXI dies + * only). */ + auto rd = [&](uint16_t a) { return _device.rtw_read32(a); }; + _logger->error("Kestrel FW-SER dmac_isr=0x{:08x} ser_dbg=0x{:08x}", + rd(r::R_AX_DMAC_ERR_ISR), rd(r::R_AX_SER_DBG_INFO)); + if (_variant == ChipVariant::C8852C) + _logger->error("Kestrel FW-SER haxi_idct=0x{:08x}/msk 0x{:08x} " + "dbg_err_v1=0x{:08x} haxi cfg1=0x{:08x} stop1=0x{:08x} " + "busy1=0x{:08x} busy3=0x{:08x} hdp_dbg2=0x{:08x}", + rd(r::R_AX_HAXI_IDCT), rd(r::R_AX_HAXI_IDCT_MSK), + rd(r::R_AX_DBG_ERR_FLAG_V1), rd(r::R_AX_HAXI_INIT_CFG1), + rd(r::R_AX_HAXI_DMA_STOP1), rd(r::R_AX_HAXI_DMA_BUSY1), + rd(r::R_AX_HAXI_DMA_BUSY3), rd(r::R_AX_HDP_DBG_INFO_2_V1)); + } catch (const std::exception &e) { + _logger->error("Kestrel FW-SER at [{}]: HALT_C2H_CTRL=0x{:08x}, error " + "registers unreadable ({})", + where, ctrl, e.what()); + } + /* A posted halt with no (readable) error code still counts as a halt. */ + return err != 0 ? err : ctrl; } void HalKestrel::fw_err_settle(const char *where, uint32_t ms) { @@ -2349,6 +2376,21 @@ HalKestrel::~HalKestrel() { delete static_cast(_halbb_bridge); } +void HalKestrel::reset_for_rebringup() { + if (_halrf_ctx) + kestrel_halrf_destroy(_halrf_ctx); + if (_halbb_ctx) + kestrel_halbb_destroy(_halbb_ctx); + delete static_cast(_halbb_bridge); + _halrf_ctx = nullptr; + _halbb_ctx = nullptr; + _halbb_bridge = nullptr; + _halrf_rfk_inited = false; + _kfr_primed = false; + _last_gain_bucket = -1; + _fw.reset_h2c_seq(); +} + namespace { constexpr uint32_t BB_WIN = 0x10000; /* bb0_cr_offset (wIndex=1) */ } /* namespace */ diff --git a/src/kestrel/HalKestrel.h b/src/kestrel/HalKestrel.h index 1b917316..ac9366a7 100644 --- a/src/kestrel/HalKestrel.h +++ b/src/kestrel/HalKestrel.h @@ -555,10 +555,18 @@ class HalKestrel { } /* Firmware SER probe: returns the latched mac_ax_err_info code if the fw has - * posted a halt error (HALT_C2H_CTRL set), else 0. Logs a labeled line so the - * bring-up can pinpoint which init step crashes the running fw. */ + * posted a halt error (HALT_C2H_CTRL set; HALT_C2H_CTRL itself when that + * code reads 0 or cannot be read), else 0. Logs a labeled line so the + * bring-up can pinpoint which init step crashes the running fw. Only the + * HALT_C2H_CTRL read can throw; the diagnostics after it never do. */ uint32_t fw_err_state(const char *where); + /* Forget the host-side state one bring-up leaves behind (the vendored + * halbb/halrf contexts and their one-shot inits, the fast-retune caches, the + * H2C sequence), so the next power_on -> ... -> set_channel runs as it would + * on a freshly constructed device. power_on itself forces the MAC off. */ + void reset_for_rebringup(); + /* Poll HALT_C2H for `ms` with no other bus traffic — detects an async fw * self-crash after boot (vs a write-triggered one). */ void fw_err_settle(const char *where, uint32_t ms); diff --git a/src/kestrel/KestrelFw.h b/src/kestrel/KestrelFw.h index aa90aa12..73b067c6 100644 --- a/src/kestrel/KestrelFw.h +++ b/src/kestrel/KestrelFw.h @@ -24,6 +24,11 @@ namespace kestrel { class KestrelFw { public: KestrelFw(RtlAdapter device, Logger_t logger, ChipVariant variant); + /* A re-downloaded fw starts its H2C sequence afresh, as a new device would. */ + void reset_h2c_seq() { + std::lock_guard lk(_h2c_mu); + _h2c_seq = 0; + } /* mac_hal_init pre-FWDL half: hci_func_en + dmac_pre_init (DLE/HFC for * DLFW). The caller then runs the intf_pre_init step (usb_pre_init / diff --git a/src/kestrel/MacRegAx.h b/src/kestrel/MacRegAx.h index 5cfcc688..f634803d 100644 --- a/src/kestrel/MacRegAx.h +++ b/src/kestrel/MacRegAx.h @@ -471,6 +471,10 @@ constexpr uint16_t R_AX_HALT_H2C_CTRL = 0x0160; constexpr uint16_t R_AX_HALT_C2H_CTRL = 0x0164; constexpr uint16_t R_AX_HALT_H2C = 0x0168; constexpr uint16_t R_AX_HALT_C2H = 0x016C; +/* SER debug dump (dbgpkg.c dump_err_status_dmac): which DMAC error raised a + * halt. Both dies. */ +constexpr uint16_t R_AX_DMAC_ERR_ISR = 0x8524; +constexpr uint16_t R_AX_SER_DBG_INFO = 0x8424; constexpr uint16_t R_AX_HISR0 = 0x01A4; constexpr uint16_t R_AX_WCPU_FW_CTRL = 0x01E0; /* RISC-V (8852C) bootrom debug: [31:16]=boot_status (fwdl-step code), @@ -1755,6 +1759,12 @@ constexpr uint16_t R_AX_HAXI_DMA_BUSY2 = 0x11C8; /* [1:0] CH10/CH11 */ constexpr uint32_t HAXI_DMA_BUSY2_CHANS = 0x3u; constexpr uint16_t R_AX_HAXI_DMA_BUSY3 = 0x1208; /* [1:0] RXQ/RPQ */ constexpr uint32_t HAXI_DMA_BUSY3_RX = 0x3u; +/* HAXI error dump (dbgpkg.c dump_err_status_haxidma_ax, 8852C/8192XB/8852D + * only). HAXI_IDCT bit 1 = B_AX_RXMDA_STUCK_IDCT. */ +constexpr uint16_t R_AX_HAXI_IDCT_MSK = 0x10B8; +constexpr uint16_t R_AX_HAXI_IDCT = 0x10BC; +constexpr uint16_t R_AX_DBG_ERR_FLAG_V1 = 0x1104; +constexpr uint16_t R_AX_HDP_DBG_INFO_2_V1 = 0x8888; constexpr uint16_t R_AX_TXBD_RWPTR_CLR1 = 0x1014; /* [10:0] ACH0-7,CH8,CH9,CH12 */ constexpr uint32_t TXBD_RWPTR_CLR1_ALL = 0x7FFu; constexpr uint16_t R_AX_TXBD_RWPTR_CLR2_V1 = 0x11C4; /* [1:0] CH10/CH11 */ diff --git a/src/kestrel/RtlKestrelDevice.cpp b/src/kestrel/RtlKestrelDevice.cpp index 83c8cf85..02365437 100644 --- a/src/kestrel/RtlKestrelDevice.cpp +++ b/src/kestrel/RtlKestrelDevice.cpp @@ -192,13 +192,71 @@ bool RtlKestrelDevice::PowerOnTrxAndPhy(kestrel::EfuseInfo &out) { } +bool RtlKestrelDevice::BringUpCheckingFwHalt( + const char *what, const std::function &once) { + /* The redo is measured on USB only (the 8852CE ring re-init under it is + * not), so on PCIe a halt fails the bring-up instead. */ + const int attempts = _device.is_usb() ? kFwHaltAttempts : 1; + for (int attempt = 1; attempt <= attempts; ++attempt) { + if (attempt > 1) + _hal.reset_for_rebringup(); + /* A failed or throwing pass (or halt probe) stops the drain + * BringUpMonitorOnce started. */ + bool up = false; + uint32_t err = 0; + try { + up = once(); + if (up) + err = _hal.fw_err_state(what); + } catch (...) { + stop_wp_drain(); + throw; + } + if (!up) { + stop_wp_drain(); + return false; + } + if (err == 0) { + if (attempt > 1) + _logger->info("Kestrel: {} clean on attempt {}", what, attempt); + return true; + } + /* Seen on the 8852C (TX50UH) without the early bulk-IN drain: + * DMAC_ERR_ISR B_AX_HAXIDMA_ERR_FLAG during the BB/RF table load, err + * 0x999 (L1_PREERR_DMAC), then a fw assert (0x3xxxxxxx). The redo saved + * most such runs but not all, so it is a safety net, not the fix; it has + * not fired with the drain (docs/8852c-quirks.md). */ + _logger->warn("Kestrel: firmware halted during {} (attempt {}/{}, " + "HALT_C2H 0x{:08x}){}", + what, attempt, attempts, err, + attempt < attempts ? "; redoing it from power_on" : ""); + _brought_up = false; + _tx_up = false; + stop_wp_drain(); + } + _logger->error("Kestrel: firmware halted on all {} attempt(s) of {}", + attempts, what); + return false; +} + bool RtlKestrelDevice::BringUpMonitor(SelectedChannel channel) { + return BringUpCheckingFwHalt("monitor bring-up", + [&] { return BringUpMonitorOnce(channel); }); +} + +bool RtlKestrelDevice::BringUpMonitorOnce(SelectedChannel channel) { _channel = channel; /* hal_start_8852b order (rtl8852b_halinit.c:508): mac_hal_init [pwr -> * FWDL -> enable_bb_rf -> sys_init -> trx_init -> intf_init] -> efuse -> * BB/RF tables -> channel. */ if (!PowerOnFwAndTrx(_efuse)) return false; + /* Drain the bulk-IN from here on: once intf_init has armed the USB RX + * path, nothing in the bring-up reads it, and the 8852C flags its RX DMA + * stuck (HAXI_IDCT B_AX_RXMDA_STUCK_IDCT -> DMAC_ERR_ISR HAXIDMA_ERR -> + * L1 0x999 -> fw assert 0x30000024) during the H2C-heavy BB/RF table load. */ + if (_device.is_usb()) + start_wp_drain(); if (!_hal.phy_bb_rf_init(_efuse.rfe_type, _hal.read_cut())) return false; /* halrf RFK: DAC/ADC DC-offset calibration (vendored halrf DACK + RX-DCK) @@ -271,10 +329,20 @@ void RtlKestrelDevice::InitWrite(SelectedChannel channel) { /* A refused bring-up is an exception here as on every other generation * (the IRadio contract the demos rely on): a caller that went on to * send_packet would otherwise learn of it one refused frame at a time. */ - if (!BringUpMonitor(channel)) { + /* The firmware-halt check runs after the TX half too, so a halt anywhere + * in the whole bring-up redoes all of it. */ + if (!BringUpCheckingFwHalt("TX bring-up", [&] { + if (!BringUpMonitorOnce(channel)) + return false; + InitWriteTail(channel); + return true; + })) { _logger->error("Kestrel: TX bring-up failed"); throw std::runtime_error("Kestrel TX bring-up failed (BringUpMonitor)"); } +} + +void RtlKestrelDevice::InitWriteTail(SelectedChannel channel) { /* TX-only: enable the CMAC port + scheduler contention queues. Kept out * of BringUpMonitor so the pure-monitor RX path keeps CCA on and can * actually hear frames. diff --git a/src/kestrel/RtlKestrelDevice.h b/src/kestrel/RtlKestrelDevice.h index e0bfefb2..5db790b7 100644 --- a/src/kestrel/RtlKestrelDevice.h +++ b/src/kestrel/RtlKestrelDevice.h @@ -4,6 +4,7 @@ #include #include #include +#include #include #include #include @@ -274,6 +275,20 @@ class RtlKestrelDevice : public IRtlRadio { void EnableTxScheduler(); private: + /* One pass of BringUpMonitor, no firmware-halt check. */ + bool BringUpMonitorOnce(SelectedChannel channel); + /* The TX half of InitWrite, run after BringUpMonitorOnce. */ + void InitWriteTail(SelectedChannel channel); + /* Run `once` (a full bring-up from power_on), then read HALT_C2H_CTRL. A + * bring-up can end with the firmware halted (L1 pre-error 0x999, then an + * assertion) while every step reported success; the radio then works for a + * while and its TX queues stop draining for good. Redo the bring-up, up to + * kFwHaltAttempts in all on USB (PCIe: no redo). False, with the bulk-IN + * drain stopped, when `once` fails or every attempt halts. */ + bool BringUpCheckingFwHalt(const char *what, + const std::function &once); + static constexpr int kFwHaltAttempts = 3; + /* Route a C2H firmware message (RPKT_TYPE_C2H payload). Currently decodes the * USR_TX_RPT_INFO report and emits its freerun TX-egress timestamp. */ void handle_c2h(const uint8_t *payload, uint32_t len); diff --git a/tests/kestrel_bringup_halt.sh b/tests/kestrel_bringup_halt.sh new file mode 100644 index 00000000..f0aa0df0 --- /dev/null +++ b/tests/kestrel_bringup_halt.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +# Kestrel bring-up firmware-halt count: N back-to-back txdemo bring-ups, each +# graded by the halt check at the end of bring-up (BringUpCheckingFwHalt logs +# "firmware halted during ..." per halted attempt). The halt it guards is the +# 8852C RX-DMA-stuck one (HAXI_IDCT 0x2 -> L1 0x999 -> fw assert 0x30000024) +# that an undrained bulk-IN during bring-up raises; see src/kestrel/CLAUDE.md. +# +# It is timing-sensitive: debug logging slows the H2C stream enough to hide +# it, so the bring-ups run at log level warn. A/B: build once as is and once +# with the start_wp_drain() call in BringUpMonitorOnce removed (BUILD=...). +# +# sudo DEVOURER_PID=0x0101 bash tests/kestrel_bringup_halt.sh # 8852C +# sudo DEVOURER_PID=0x0108 N=20 bash tests/kestrel_bringup_halt.sh # 8852B +# +# Exit 0 when every run completed (txdemo exit 0) and no attempt halted, 1 +# otherwise. A run that exits non-zero for any other reason (adapter absent, +# open or bring-up error, timeout) tested nothing and counts as an error. +set -u +ROOT="$(cd "$(dirname "$0")/.." && pwd)" +BUILD=${BUILD:-$ROOT/build} +OUT=${OUT:-/tmp/kestrel-bringup-halt} +N=${N:-30} +CH=${CH:-149} +VID=${DEVOURER_VID:-0x35bc}; PID=${DEVOURER_PID:-0x0101} +SUDO=${SUDO-sudo} +TXDEMO="$BUILD/txdemo"; [ -x "$TXDEMO" ] || TXDEMO="$BUILD/txdemo.exe" +mkdir -p "$OUT" + +runs_halted=0 attempts_halted=0 runs_failed=0 runs_errored=0 attempts=0 +for i in $(seq 1 "$N"); do + log="$OUT/run$i.log" + $SUDO env DEVOURER_VID=$VID DEVOURER_PID=$PID DEVOURER_CHANNEL=$CH \ + DEVOURER_TX_FRAMES=200 DEVOURER_LOG_LEVEL=warn DEVOURER_EVENTS=off \ + timeout 120 "$TXDEMO" >"$log" 2>&1 + rc=$? + h=$(grep -c "firmware halted during" "$log" || true) + f=$(grep -c "firmware halted on all" "$log" || true) + attempts_halted=$((attempts_halted + h)) + [ "$h" -gt 0 ] && runs_halted=$((runs_halted + 1)) + note="" + if [ "$f" -gt 0 ]; then + attempts=$((attempts + h)); runs_failed=$((runs_failed + 1)) + note=" (all attempts halted)" + elif [ "$rc" -ne 0 ]; then + attempts=$((attempts + h)); runs_errored=$((runs_errored + 1)) + note=" (ERROR: txdemo failed without a halt; see $log)" + else + attempts=$((attempts + h + 1)) + fi + echo "run $i: rc=$rc halted_attempts=$h$note" + grep -h "FW-SER" "$log" | sed 's/^/ /' | sort -u +done +echo "=== $VID:$PID ch$CH: $runs_halted of $N runs halted at least once," \ + "$attempts_halted of $attempts attempts, $runs_failed runs failed," "$runs_errored runs errored" +[ "$attempts_halted" -eq 0 ] && [ "$runs_errored" -eq 0 ]