From 46b78474a13c6a2cbf59db6c783e74b820291cad Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:00:00 -0700 Subject: [PATCH 01/22] ppmem: give each CPU a cached view of memory A CPU's cache, its decode cache and its compiled code reach RAM through ppmem's window, and every CPU and Physical's direct paths share that one window. The R10000-family PROMs use the caches as working memory before memory is set up: lines made valid with Index Store Tag and Index Store Data hold a stack or code while uncached accesses, which bypass the caches (NEC VR10000 user's manual p. 75), see memory. The Tezro's processors hold lines for the same addresses with different contents at once. One shared window has no room for any of that. PpMemSpace::add_cached_view gives a CPU a window of its own. Every mapping laid in the shared window is laid in each view as well, and a view publishes its bitmap to that CPU's inline fields, so the CPU reads its view through the same base pointer and bitmap test it used before. hold_line_page turns a page of a view into private memory, starting as a copy of memory, and release_line_page lays memory back when the last line goes, dropping what the page held. A remap saves and restores the held pages around it. A page with no memory behind it is refused. One in a region the window maps only in part takes the whole region in that view, reading zeros where nothing is mapped, because the region bitmap is all a CPU tests: the O2's POST holds lines for page 0 before CRIME maps the rest of its region. Nothing holds a page yet. The views are not coherent with each other or with DMA. The generic machine, the Octane, the O2 and the IP27 machine give each CPU a view now. Physical and the devices that read the window directly keep the shared one. Nothing on a load, store or fetch changes. The CPU no longer takes over the shared window's bitmap, so set_bitmap_sink and add_bitmap_sink lose their last callers and go. --- src/cpu/mips_exec.rs | 4 +- src/ip27_machine.rs | 20 +- src/ip30_machine.rs | 35 ++-- src/ip32_machine.rs | 26 ++- src/machine.rs | 36 ++-- src/ppmem/ppmem.rs | 459 +++++++++++++++++++++++++++++++++++-------- 6 files changed, 421 insertions(+), 159 deletions(-) diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 928a20735..c4bb09ae1 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -10566,7 +10566,7 @@ impl MipsCpu { } /// ppmem: pointer to the executor's inline `ppmem_bitmap` field, for - /// `PpMemSpace::set_bitmap_sink`. Fixed for the life of this `MipsCpu` — + /// `PpMemSpace::add_cached_view`. Fixed for the life of this `MipsCpu` — /// taken once in `new()` after the executor reached its final address. pub fn ppmem_bitmap_ptr(&self) -> *mut u64 { self.ppmem_bitmap_ptr @@ -11298,7 +11298,7 @@ pub trait CpuDevice: Device + Resettable + Saveable + Send + Sync { fn cycles_ptr(&self) -> crate::cpu::mips_core::CyclesPtr; fn interrupts_ptr(&self) -> *const AtomicU64; /// ppmem: the executor's inline `ppmem_bitmap` field, for - /// `PpMemSpace::set_bitmap_sink`. + /// `PpMemSpace::add_cached_view`. fn ppmem_bitmap_ptr(&self) -> *mut u64; /// tcache: hand the cache ppmem's window base + mapped-region bitmap. /// diff --git a/src/ip27_machine.rs b/src/ip27_machine.rs index c2674e6a0..b8bce13d8 100644 --- a/src/ip27_machine.rs +++ b/src/ip27_machine.rs @@ -497,21 +497,21 @@ impl Ip27Machine { } else { crate::with_cpu!(module.part, true, false, C => Self::cpu::(module, bus, phys, memory, &node.timers, count_hz, cpu_id, smp)) }; - // The shadow cache and the JIT read RAM straight from the ppmem window. + // The shadow cache and the JIT read RAM through a cached view of the + // ppmem window that is the CPU's own. if let Some(sp) = node.memory.physical().ppmem_space() { - use crate::ppmem::MappedMemory; + let v = sp + .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()]) + .map_err(|e| format!("ppmem: a CPU's cached view: {e}"))?; unsafe { - sp.set_bitmap_sink(cpu.ppmem_bitmap_ptr()); - cpu.set_tcache_window(sp.window_base()); + cpu.set_tcache_window(sp.view_base(v)); cpu.set_tcache_shift(crate::ppmem::window_shift(sp.window_size())); - let bitmap = cpu.tcache_bitmap_ptr(); - if !bitmap.is_null() { - sp.set_bitmap_sink2(bitmap); - } #[cfg(feature = "jitv2")] - cpu.set_tcache_gen_window(sp.gen_window_base()); + { + use crate::ppmem::MappedMemory; + cpu.set_tcache_gen_window(sp.gen_window_base()); + } } - node.memory.physical().resync_ppmem_bitmap(); } #[cfg(feature = "jitv2")] { diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index e3847b932..3ce7ae15c 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -448,33 +448,24 @@ impl OctaneMachine { iris_hostcall::set_display(Arc::new(crate::dev::mgras::ImpactScreen(g.mgras().clone()))); } - // The shadow cache and the JIT read RAM straight from the ppmem - // window; descriptor writes lay it out again on the CPU thread. + // The shadow cache and the JIT read RAM through a cached view of the + // ppmem window, one per CPU; descriptor writes lay the window and + // every view out again on the CPU thread. if let Some(sp) = phys.phys.ppmem_space() { - use crate::ppmem::MappedMemory; - unsafe { - sp.set_bitmap_sink(cpu.ppmem_bitmap_ptr()); - cpu.set_tcache_window(sp.window_base()); - cpu.set_tcache_shift(crate::ppmem::window_shift(sp.window_size())); - let bitmap = cpu.tcache_bitmap_ptr(); - if !bitmap.is_null() { - sp.set_bitmap_sink2(bitmap); - } - #[cfg(feature = "jitv2")] - cpu.set_tcache_gen_window(sp.gen_window_base()); - for c in cpus.iter().skip(1) { - sp.add_bitmap_sink(c.ppmem_bitmap_ptr()); - c.set_tcache_window(sp.window_base()); + for c in cpus.iter() { + let v = sp + .add_cached_view(&[c.ppmem_bitmap_ptr(), c.tcache_bitmap_ptr()]) + .map_err(|e| format!("ppmem: a CPU's cached view: {e}"))?; + unsafe { + c.set_tcache_window(sp.view_base(v)); c.set_tcache_shift(crate::ppmem::window_shift(sp.window_size())); - let bitmap = c.tcache_bitmap_ptr(); - if !bitmap.is_null() { - sp.add_bitmap_sink(bitmap); - } #[cfg(feature = "jitv2")] - c.set_tcache_gen_window(sp.gen_window_base()); + { + use crate::ppmem::MappedMemory; + c.set_tcache_gen_window(sp.gen_window_base()); + } } } - phys.phys.resync_ppmem_bitmap(); } // Publish the descriptors as they stand into the sinks just set. ram.set_memcfg(heart.memcfg()); diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index 4715a1050..53c269181 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -248,25 +248,23 @@ impl O2Machine { let cpu: Arc = crate::with_cpu!(part, true, true, C => Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, count_hz, boot.clone(), bus.ram.clone(), part.prid())); - // ppmem: the CPU's cache and the JIT reach RAM through the memory's - // window, publishing the mapped bitmap into the CPU's own fields, as - // the Indigo2 does. Remaps happen on the CPU thread (a CRIME bank - // control write), which is the window's contract. + // ppmem: the CPU's cache and the JIT reach RAM through a cached view + // of the memory's window, publishing its bitmap into the CPU's own + // fields, as the Indigo2 does. Remaps happen on the CPU thread (a + // CRIME bank control write), which is the window's contract. { - use crate::ppmem::MappedMemory; let sp = bus.phys.ppmem_space().ok_or("ppmem: the O2 needs its memory window")?; + let v = sp + .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()]) + .map_err(|e| format!("ppmem: the CPU's cached view: {e}"))?; unsafe { - sp.set_bitmap_sink(cpu.ppmem_bitmap_ptr()); - cpu.set_tcache_window(sp.window_base()); - let bitmap = cpu.tcache_bitmap_ptr(); - if !bitmap.is_null() { - sp.set_bitmap_sink2(bitmap); - } + cpu.set_tcache_window(sp.view_base(v)); #[cfg(feature = "jitv2")] - cpu.set_tcache_gen_window(sp.gen_window_base()); + { + use crate::ppmem::MappedMemory; + cpu.set_tcache_gen_window(sp.gen_window_base()); + } } - // The bus's direct paths read the same bitmap. - bus.phys.resync_ppmem_bitmap(); } // After the windows above: each compile worker takes the cache's diff --git a/src/machine.rs b/src/machine.rs index 9efbfe6e6..d5ed400b7 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -800,33 +800,21 @@ impl Machine { mc.set_cpu(Arc::downgrade(&cpu_device)); ioc.set_interrupts(cpu.interrupts_ptr()); - // ppmem: hand the CPU's inline bitmap field to the memory space, so a - // MEMCFG remap publishes straight into the object the CPU executes out - // of. Safe per `set_bitmap_sink`: the executor is at its final address - // by now and the only writer is this same CPU thread's MEMCFG path. + // ppmem: the CPU's cache and the JIT read RAM through a cached view of + // the window that is the CPU's own, so what the cache holds stays out + // of every other master's sight (`PpMemSpace::add_cached_view`). Its + // bitmap is published straight into the CPU's inline fields, the + // executor's and the cache's, which are at their final addresses by + // now; the only writer is this CPU thread's MEMCFG path. A cache with + // no bitmap field (the full cache without tcache) returns null, which + // the view skips. Physical's direct paths keep the shared window. { use crate::ppmem::MappedMemory; if let Some(sp) = phys.ppmem_space() { - unsafe { sp.set_bitmap_sink(cpu.ppmem_bitmap_ptr()) }; - } - // Point Physical's cached bitmap at the CPU's field too, or its - // direct-access fast path would keep reading the sink the space - // just abandoned. - phys.resync_ppmem_bitmap(); - // The cache reads RAM straight out of the same window: the full - // cache under tcache, the shadow cache always. A cache that does - // not (the full cache without tcache) has no bitmap field and - // returns null. Must come after set_bitmap_sink, so it captures - // the CPU's inline field rather than the sink the space just - // abandoned. - if let Some(sp) = phys.ppmem_space() { - unsafe { cpu.set_tcache_window(sp.window_base()) }; - // The cache keeps the bitmap inline; ppmem publishes into it - // directly on every remap, so the hot path needs no indirection. - let bitmap = cpu.tcache_bitmap_ptr(); - if !bitmap.is_null() { - unsafe { sp.set_bitmap_sink2(bitmap) }; - } + let v = sp + .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()]) + .expect("ppmem: the CPU's cached view"); + unsafe { cpu.set_tcache_window(sp.view_base(v)) }; // Window writes bypass BusDevice, so the cache needs the gen // window to bump page counters itself — otherwise jitv2 keeps // running compiled code for pages the guest has modified. diff --git a/src/ppmem/ppmem.rs b/src/ppmem/ppmem.rs index a30c0adb5..51b847a93 100644 --- a/src/ppmem/ppmem.rs +++ b/src/ppmem/ppmem.rs @@ -151,32 +151,14 @@ pub trait MappedMemory { /// a finer, multi-word bitmap. fn mapped_bitmap(&self) -> u64; - /// Redirect bitmap publication into the CPU's inline `ppmem_bitmap` field. - /// - /// Pass `MipsExecutor::ppmem_bitmap_ptr()`, queried once the executor is at - /// its final address. From then on every mapping change writes straight - /// through this pointer, so the CPU's RAM-vs-bus test is a load from the - /// object it is already executing out of — no indirection through - /// `PpMemSpace`, no atomic, no lock. - /// - /// Optional: until this is called the space publishes into a `u64` it owns - /// itself, so the sink is always valid and publication never branches. - /// - /// # Safety - /// - /// `ptr` must point to a live `u64` that outlives this `PpMemSpace`, and - /// must only ever be written from the CPU thread. Both hold for the CPU's - /// inline field: the executor never moves once inside its `Arc`, and the - /// only writer is the MEMCFG path on that same thread. - unsafe fn set_bitmap_sink(&self, ptr: *mut u64); - - /// A second publication target — see [`MappedMemory::set_bitmap_sink`]. + /// A second publication target, the cache's inline copy. /// /// Exists so the cache can hold the bitmap *inline* instead of chasing a /// pointer on every access. Both sinks receive every update. /// /// # Safety - /// Same contract as `set_bitmap_sink`. + /// `ptr` must point to a live `u64` that outlives this `PpMemSpace` and is + /// written only from the CPU thread. unsafe fn set_bitmap_sink2(&self, ptr: *mut u64); } @@ -656,11 +638,17 @@ impl BusDevice for Arc { // --------------------------------------------------------------------------- /// One live mapping inside the window, tracked so `clear_mappings` can undo it -/// and `mapped_bitmap` can be recomputed. +/// and `mapped_bitmap` can be recomputed. It also says where its bytes come +/// from, so a cached view can lay part of it again when a held page leaves: +/// `bank`'s object from `src`, or repeating every `period` bytes from `src` +/// into the period (0: no repeat). #[derive(Clone, Copy, Debug)] struct Mapping { at: u64, len: u64, + bank: usize, + src: u64, + period: u64, } /// The 4GB window that banks are mapped into, plus the banks themselves. @@ -692,7 +680,7 @@ pub struct PpMemSpace { /// `mapped_bitmap()`; written only via `state.sink`. own_bitmap: OwnBitmap, /// No lock: every path that touches this — `map_bank`, `map_alias`, - /// `clear_mappings`, `set_bitmap_sink` — runs on the CPU thread, reached + /// `clear_mappings`, `set_bitmap_sink2` — runs on the CPU thread, reached /// as CPU store → MC register write → `remap_banks`. No device thread ever /// enters this type. (`unsafe impl Sync` below records that contract.) state: UnsafeCell, @@ -711,8 +699,23 @@ struct SpaceState { /// path is one load instead of two — hence one sink per consumer rather /// than a shared indirection. Points at `own_bitmap` until claimed. sink2: *mut u64, - /// Further CPUs' inline copies (`PpMemSpace::add_bitmap_sink`). - more_sinks: Vec<*mut u64>, + /// The bitmap last published to the sinks above. + bits: u64, + /// Each CPU's cached view of memory: see [`PpMemSpace::add_cached_view`]. + views: Vec, +} + +/// One CPU's cached view of memory: a window of its own, with every mapping +/// the shared window has, plus the pages its cache holds lines for, which +/// nobody else sees. +struct View { + space: AddrSpace, + /// Where this view's region bitmap is published: the CPU's inline copies. + sinks: Vec<*mut u64>, + /// The pages this view holds, with how many of the CPU's valid lines each + /// holds. Only its CPU changes them, at CACHE instructions, under the + /// contract below. + held: std::collections::BTreeMap, } /// The bitmap `PpMemSpace` publishes into before a CPU claims it. @@ -727,12 +730,15 @@ struct OwnBitmap(Box); // formality: // // * Every mutating entry point — `map_bank`, `map_alias`, `clear_mappings`, -// `set_bitmap_sink` — is reached only from `Physical::remap_banks`, which +// `set_bitmap_sink2` — is reached only from `Physical::remap_banks`, which // the MC invokes synchronously inside the CPU's store to MEMCFG. That is // always a CPU thread, and with several CPUs the others are parked for it // (`smp::World::stop_others`, wired where HEART's descriptors reach memory). // * Device threads (MC-DMA, seeq, VINO) reach RAM through `BusDevice` on the // individual `PpMemory` banks. They never touch `PpMemSpace`. +// * `hold_line_page`/`release_line_page` run on a CPU thread at a CACHE +// instruction and change only that CPU's own view. They read the mapping +// list and the bitmap, which only a remap changes, with the others parked. // * `mapped_bitmap()` reads the sink and is likewise CPU-thread-only; the // hot-path reader is the CPU's own inline `MipsCore::ppmem_bitmap` field, // which is a plain `u64` and never goes through here. @@ -769,7 +775,8 @@ impl PpMemSpace { mappings: Vec::new(), sink, sink2, - more_sinks: Vec::new(), + bits: 0, + views: Vec::new(), }), }) } @@ -792,8 +799,8 @@ impl PpMemSpace { /// The `u64` the mapped-region bitmap is published into. /// - /// Never null: points at this space's own `u64` until a CPU claims it via - /// [`MappedMemory::set_bitmap_sink`], and at the CPU's inline field after. + /// Never null: this space's own `u64`. A CPU reads its cached view's + /// copy instead (`add_cached_view`). pub fn bitmap_ptr(&self) -> *const u64 { let st = unsafe { &*self.state.get() }; st.sink as *const u64 @@ -814,6 +821,7 @@ impl PpMemSpace { unsafe { self.space.map(offset as usize, size as usize, b, src, Prot::ReadWrite)?; } + self.map_views(offset, size, b, src)?; #[cfg(feature = "jitv2")] { let (gen_off, gen_len, gen_src) = (offset / GEN_RATIO, size / GEN_RATIO, src / GEN_RATIO); @@ -827,7 +835,7 @@ impl PpMemSpace { } } let st = unsafe { &mut *self.state.get() }; - Self::record(st, offset, size); + Self::record(st, Mapping { at: offset, len: size, bank, src, period: 0 }); Ok(()) } @@ -867,37 +875,234 @@ impl PpMemSpace { unsafe { *state.sink = bits; *state.sink2 = bits; - for &s in &state.more_sinks { - *s = bits; - } + } + state.bits = bits; + for v in 0..state.views.len() { + Self::publish_view(state, v); } } - fn record(state: &mut SpaceState, at: u64, len: u64) { + /// Publish view `v`'s bitmap: the shared one, plus every region the view + /// holds a page in, which `hold_line_page` made safe to touch whole. + fn publish_view(state: &SpaceState, v: usize) { + let shift = window_shift(state.window); + let view = &state.views[v]; + let bits = view.held.keys().fold(state.bits, |b, &page| b | 1u64 << (page >> shift)); + for &s in &view.sinks { + unsafe { *s = bits }; + } + } + + fn record(state: &mut SpaceState, m: Mapping) { // Drop anything this mapping fully replaces, then record it. + let (at, len) = (m.at, m.len); state .mappings .retain(|m| !(m.at >= at && m.at + m.len <= at + len)); - state.mappings.push(Mapping { at, len }); + state.mappings.push(m); Self::recompute_bitmap(state); } } +/// Cached views. +/// +/// A CPU's cache and its compiled code read memory through ppmem's window, so +/// the window is that CPU's cached view of memory. Giving each CPU a window of +/// its own makes room for what the cache holds: a valid cache line's page can +/// become private memory in that CPU's view while the shared window, the +/// decode table and every other CPU go on seeing memory. Uncached accesses +/// bypass the caches (NEC VR10000 user's manual p. 75), and each CPU's cache +/// is its own. The views are not coherent: on a coherent system another +/// master's read would be answered from a dirty line, and here it reads +/// memory. +/// +/// Nothing on a load, store or fetch changes: the CPU reads its view through +/// the same base pointer and inline bitmap it read the shared window through. +/// The page is the host's mapping granule (4 KB on Linux and x86 macOS, +/// 16 KB on Apple silicon, 64 KB on Windows). impl PpMemSpace { - /// Publish the bitmap to one more `u64` as well: a second CPU's inline - /// copies. Same contract as [`MappedMemory::set_bitmap_sink`]. The value - /// arrives with the next remap. + /// Give a CPU a cached view of its own: a window with the same mappings + /// as this one, from now on, whose region bitmap is published to `sinks` + /// (the CPU's inline copies; null entries are skipped) instead of this + /// window's. Returns the view's number. Set up before the CPU runs. + pub fn add_cached_view(&self, sinks: &[*mut u64]) -> io::Result { + let st = unsafe { &mut *self.state.get() }; + let space = AddrSpace::reserve(st.window as usize)?; + let sinks = sinks.iter().copied().filter(|p| !p.is_null()).collect(); + st.views.push(View { space, sinks, held: Default::default() }); + let v = st.views.len() - 1; + for m in st.mappings.clone() { + self.lay(st, v, &m, m.at, m.at + m.len)?; + } + Self::publish_view(st, v); + Ok(v) + } + + /// The base of view `v`'s window. + pub fn view_base(&self, v: usize) -> *mut u8 { + unsafe { (&(*self.state.get()).views)[v].space.base() } + } + + /// The pages view `v` holds, in address order. + pub fn held_line_pages(&self, v: usize) -> Vec { + unsafe { (&(*self.state.get()).views)[v].held.keys().copied().collect() } + } + + /// A line of view `v`'s CPU at `pa` became valid: from now on the view + /// shows the page holding it as private memory, starting as a copy of + /// memory. Each call is one line, and [`Self::release_line_page`] gives + /// it back. Returns whether the page is newly held, or `None` when no + /// memory is mapped behind it. /// - /// # Safety - /// `ptr` must stay valid for as long as this space exists. - pub unsafe fn add_bitmap_sink(&self, ptr: *mut u64) { - unsafe { (*self.state.get()).more_sinks.push(ptr) }; + /// The region bitmap is all a CPU tests, so the page's whole region is + /// the view's while it stands, reading zeros where nothing is mapped. The + /// O2's POST holds lines for page 0 before CRIME maps the rest of its + /// region. + pub fn hold_line_page(&self, v: usize, pa: u64) -> Option { + let st = unsafe { &mut *self.state.get() }; + let gran = granularity() as u64; + let page = pa & !(gran - 1); + if !st.mappings.iter().any(|m| m.at <= page && m.at + m.len >= page + gran) { + return None; + } + let shift = window_shift(st.window); + if let Some(n) = st.views[v].held.get_mut(&page) { + *n += 1; + return Some(false); + } + if !bitmap_maps_at(st.bits, shift, page) && !st.views[v].held.keys().any(|&p| p >> shift == page >> shift) { + self.scrub_region_gaps(st, v, page >> shift << shift, 1u64 << shift); + } + let view = &st.views[v]; + let at = unsafe { view.space.base().add(page as usize) }; + let mut bytes = vec![0u8; gran as usize]; + unsafe { + std::ptr::copy_nonoverlapping(at, bytes.as_mut_ptr(), bytes.len()); + let _ = view.space.scrub(page as usize, gran as usize); + std::ptr::copy_nonoverlapping(bytes.as_ptr(), at, bytes.len()); + } + st.views[v].held.insert(page, 1); + Self::publish_view(st, v); + Some(true) + } + + /// One line of [`Self::hold_line_page`]'s page is no longer valid. + /// Returns true when it was the page's last: the view shows memory there + /// again, and what the page held is dropped, as a line invalidated + /// without writeback is. + pub fn release_line_page(&self, v: usize, pa: u64) -> bool { + let st = unsafe { &mut *self.state.get() }; + let gran = granularity() as u64; + let page = pa & !(gran - 1); + let Some(n) = st.views[v].held.get_mut(&page) else { return false }; + *n -= 1; + if *n > 0 { + return false; + } + st.views[v].held.remove(&page); + let _ = unsafe { st.views[v].space.scrub(page as usize, gran as usize) }; + for m in st.mappings.iter().filter(|m| m.at < page + gran && m.at + m.len > page) { + let _ = self.lay(st, v, m, page.max(m.at), (page + gran).min(m.at + m.len)); + } + Self::publish_view(st, v); + true + } + + /// Make the parts of `[r, r + len)` that nothing maps private zeros in + /// view `v`, and give them generation counters where none are mapped, so + /// the region can be marked whole in the view's bitmap: compiled stores + /// there bump a counter. + fn scrub_region_gaps(&self, st: &SpaceState, v: usize, r: u64, len: u64) { + let end = r + len; + let mut at = r; + while at < end { + if let Some(m) = st.mappings.iter().find(|m| m.at <= at && m.at + m.len > at) { + at = m.at + m.len; + continue; + } + let next = st.mappings.iter().filter(|m| m.at > at && m.at < end).map(|m| m.at).min().unwrap_or(end); + let _ = unsafe { st.views[v].space.scrub(at as usize, (next - at) as usize) }; + #[cfg(feature = "jitv2")] + { + let g = granularity() as u64; + let (gs, ge) = ((at / GEN_RATIO) & !(g - 1), (next / GEN_RATIO).div_ceil(g) * g); + let touches = |m: &Mapping| m.at < ge * GEN_RATIO && m.at + m.len > gs * GEN_RATIO; + if !st.mappings.iter().any(touches) { + let _ = unsafe { self.gen_space.scrub(gs as usize, (ge - gs) as usize) }; + } + } + at = next; + } + } + + /// A mapping just laid in the shared window goes into every view too, + /// under the pages each holds. + fn map_views(&self, at: u64, len: u64, b: &SharedMem, src: u64) -> io::Result<()> { + let st = unsafe { &*self.state.get() }; + for view in &st.views { + let saved = Self::save_held(view, at, len); + unsafe { view.space.map(at as usize, len as usize, b, src, Prot::ReadWrite)? }; + Self::put_held(view, saved); + } + Ok(()) + } + + /// Lay `[from, to)` of mapping `m` in view `v`. Data only: the views + /// share the window's generation counters. + fn lay(&self, st: &SpaceState, v: usize, m: &Mapping, from: u64, to: u64) -> io::Result<()> { + let b = &self.banks[m.bank]; + let mut at = from; + while at < to { + let rel = at - m.at; + let (src, run) = if m.period == 0 { + (m.src + rel, to - at) + } else { + let o = (m.src + rel) % m.period; + (o, (m.period - o).min(to - at)) + }; + unsafe { st.views[v].space.map(at as usize, run as usize, b, src, Prot::ReadWrite)? }; + at += run; + } + Ok(()) + } + + /// The bytes of each page `view` holds inside `[at, at + len)`, before + /// a mapping change replaces them. + fn save_held(view: &View, at: u64, len: u64) -> Vec<(u64, Vec)> { + let gran = granularity(); + let base = view.space.base(); + view.held.keys() + .filter(|&&p| p < at + len && p + gran as u64 > at) + .map(|&p| { + let mut b = vec![0u8; gran]; + unsafe { std::ptr::copy_nonoverlapping(base.add(p as usize), b.as_mut_ptr(), gran) }; + (p, b) + }) + .collect() + } + + /// Put [`Self::save_held`]'s pages back, private again. + fn put_held(view: &View, saved: Vec<(u64, Vec)>) { + let base = view.space.base(); + for (page, bytes) in saved { + let _ = unsafe { view.space.scrub(page as usize, bytes.len()) }; + unsafe { std::ptr::copy_nonoverlapping(bytes.as_ptr(), base.add(page as usize), bytes.len()) }; + } } } impl MappedMemory for PpMemSpace { fn clear_mappings(&self) { let st = unsafe { &mut *self.state.get() }; + // Views keep the pages they hold. Scrubbed, not left mapped, so a + // view never shows memory the shared window no longer maps. + for view in &st.views { + let saved = Self::save_held(view, 0, st.window); + for m in &st.mappings { + let _ = unsafe { view.space.scrub(m.at as usize, m.len as usize) }; + } + Self::put_held(view, saved); + } for m in std::mem::take(&mut st.mappings) { // Best-effort: a failure here leaves the range mapped, which is // still safe — it just isn't reverted. @@ -915,13 +1120,7 @@ impl MappedMemory for PpMemSpace { } } } - unsafe { - *st.sink = 0; - *st.sink2 = 0; - for &s in &st.more_sinks { - *s = 0; - } - } + Self::recompute_bitmap(st); } fn map_bank(&self, bank: usize, offset: u64, size: u64, period: u64) -> io::Result<()> { @@ -950,6 +1149,7 @@ impl MappedMemory for PpMemSpace { self.space .map(at as usize, chunk as usize, b, 0, Prot::ReadWrite)?; } + self.map_views(at, chunk, b, 0)?; // Lockstep: the counters for this mirror land at the same offset // scaled by GEN_RATIO, from the same bank's gen object. Every // mirror therefore shares one physical counter page with the @@ -975,7 +1175,7 @@ impl MappedMemory for PpMemSpace { } let st = unsafe { &mut *self.state.get() }; - Self::record(st, offset, size); + Self::record(st, Mapping { at: offset, len: size, bank, src: 0, period }); Ok(()) } @@ -994,6 +1194,7 @@ impl MappedMemory for PpMemSpace { self.space .map(offset as usize, size as usize, b, 0, Prot::ReadWrite)?; } + self.map_views(offset, size, b, 0)?; // Alias the counters too — this is NOT optional. // // An earlier version skipped the gen mapping whenever the alias's gen @@ -1045,7 +1246,7 @@ impl MappedMemory for PpMemSpace { } } let st = unsafe { &mut *self.state.get() }; - Self::record(st, offset, size); + Self::record(st, Mapping { at: offset, len: size, bank, src: 0, period: 0 }); Ok(()) } @@ -1058,15 +1259,6 @@ impl MappedMemory for PpMemSpace { self.gen_space.base() as *mut AtomicU64 } - unsafe fn set_bitmap_sink(&self, ptr: *mut u64) { - let st = unsafe { &mut *self.state.get() }; - // Carry the current value across: the boot-time remap runs before the - // CPU thread starts, so banks are typically already mapped by now. - let current = unsafe { *st.sink }; - st.sink = ptr; - unsafe { *ptr = current }; - } - unsafe fn set_bitmap_sink2(&self, ptr: *mut u64) { let st = unsafe { &mut *self.state.get() }; let current = unsafe { *st.sink }; @@ -1354,33 +1546,6 @@ mod tests { assert_eq!(space.mapped_bitmap() & 1, 0, "partial region must read unmapped"); } - /// The bitmap must land in whatever `u64` the CPU handed over, so the hot - /// path reads it as a plain inline field. - #[test] - fn bitmap_publishes_into_the_cpu_field() { - let (space, _banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); - - // Stand in for MipsCore.ppmem_bitmap. Boxed so its address is stable. - let cpu_field = Box::new(0u64); - let ptr = &*cpu_field as *const u64 as *mut u64; - - // Map before registering: the boot-time remap runs before the CPU - // thread exists, so the current value must carry across. - space.map_bank(0, 0x0800_0000, 64 * MB as u64, 64 * MB as u64).unwrap(); - let expected = space.mapped_bitmap(); - assert_ne!(expected, 0); - - unsafe { space.set_bitmap_sink(ptr) }; - assert_eq!(*cpu_field, expected, "existing bitmap not carried to the CPU field"); - - // Subsequent changes must publish straight through. - space.map_bank(0, 0x1000_0000, 64 * MB as u64, 64 * MB as u64).unwrap(); - assert_eq!(*cpu_field, space.mapped_bitmap(), "later remap did not publish"); - - space.clear_mappings(); - assert_eq!(*cpu_field, 0, "clear_mappings did not publish"); - } - /// The bit the CPU tests must be the one covering that physical address. #[test] fn cpu_bitmap_test_selects_the_right_region() { @@ -1696,4 +1861,124 @@ mod tests { let gen_bytes = min_bank / GEN_RATIO as usize; assert_eq!(gen_bytes, 16 * 1024, "8MB bank => 16KB of counters"); } + + // ---- cached views ---- + + /// A CPU's view of `space`, publishing into a boxed stand-in for its + /// inline bitmap field. + fn view_of(space: &PpMemSpace) -> (usize, Box) { + let field = Box::new(0u64); + let v = space.add_cached_view(&[&*field as *const u64 as *mut u64]).unwrap(); + (v, field) + } + + fn word(base: *mut u8, at: u64) -> u32 { + unsafe { *(base.add(at as usize) as *const u32) } + } + + fn set_word(base: *mut u8, at: u64, w: u32) { + unsafe { *(base.add(at as usize) as *mut u32) = w }; + } + + /// A view shows every mapping the window has, laid before or after it + /// was made, and publishes the same bitmap to its own sink. + #[test] + fn a_view_follows_the_windows_mappings() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[64, 64]).unwrap(); + space.map_bank(0, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + let (v, field) = view_of(&space); + space.map_alias(1, 0x0800_0000, 64 * MB as u64).unwrap(); + banks[0].write32(0x40, 0x1111_1111); + banks[1].write32(0x80, 0x2222_2222); + assert_eq!(word(space.view_base(v), 0x40), 0x1111_1111); + assert_eq!(word(space.view_base(v), 0x0800_0080), 0x2222_2222); + assert_eq!(*field, space.mapped_bitmap()); + set_word(space.view_base(v), 0x44, 0x3333_3333); + assert_eq!(banks[0].read32(0x44).data, 0x3333_3333, "an unheld page is memory itself"); + space.clear_mappings(); + assert_eq!(*field, 0); + } + + /// A held page is the view's own: what is written there stays there, the + /// window and memory keep their bytes, and releasing it drops the page. + #[test] + fn a_held_page_is_private_to_its_view_until_released() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + space.map_bank(0, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + let (v, _field) = view_of(&space); + banks[0].write32(0x2040, 0x1234_5678); + assert_eq!(space.hold_line_page(v, 0x2040), Some(true)); + assert_eq!(space.hold_line_page(v, 0x2080), Some(false), "a second line of the page"); + let base = space.view_base(v); + assert_eq!(word(base, 0x2040), 0x1234_5678, "it starts as memory"); + set_word(base, 0x2040, 0xaaaa_aaaa); + banks[0].write32(0x2044, 0x5555_5555); + assert_eq!(banks[0].read32(0x2040).data, 0x1234_5678); + assert_eq!(word(space.window_base(), 0x2040), 0x1234_5678); + assert_eq!(word(base, 0x2044), 0, "the page does not see later memory writes"); + assert!(!space.release_line_page(v, 0x2040), "one line left"); + assert_eq!(space.held_line_pages(v), vec![0x2000]); + assert!(space.release_line_page(v, 0x2080)); + assert!(space.held_line_pages(v).is_empty()); + assert_eq!(word(base, 0x2040), 0x1234_5678, "memory again"); + assert_eq!(word(base, 0x2044), 0x5555_5555); + } + + /// Two CPUs holding one page each see their own bytes. + #[test] + fn two_views_hold_one_page_apart() { + let (space, _banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + space.map_bank(0, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + let (a, _fa) = view_of(&space); + let (b, _fb) = view_of(&space); + space.hold_line_page(a, 0x10_0000); + space.hold_line_page(b, 0x10_0000); + set_word(space.view_base(a), 0x10_0040, 0xaaaa_aaaa); + set_word(space.view_base(b), 0x10_0040, 0xbbbb_bbbb); + assert_eq!(word(space.view_base(a), 0x10_0040), 0xaaaa_aaaa); + assert_eq!(word(space.view_base(b), 0x10_0040), 0xbbbb_bbbb); + assert_eq!(word(space.window_base(), 0x10_0040), 0); + } + + /// A remap lays memory out again under every view and must not take a + /// held page with it. + #[test] + fn a_remap_keeps_held_pages() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + space.map_bank(0, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + let (v, field) = view_of(&space); + space.hold_line_page(v, 0x3000); + set_word(space.view_base(v), 0x3010, 0x7777_7777); + space.clear_mappings(); + assert_eq!(*field, 1, "the region the page stands in stays the view's"); + space.map_bank(0, 0, 64 * MB as u64, 32 * MB as u64).unwrap(); + assert_eq!(word(space.view_base(v), 0x3010), 0x7777_7777); + assert_eq!(banks[0].read32(0x3010).data, 0); + space.release_line_page(v, 0x3000); + banks[0].write32(0x3010, 0x9999_9999); + assert_eq!(word(space.view_base(v), 0x3010), 0x9999_9999); + assert_eq!(word(space.view_base(v), 0x200_3010), 0x9999_9999, "laid again with the bank's period"); + } + + /// A page with no memory behind it cannot be held. One in a region the + /// window maps only in part makes the region the view's, reading zeros + /// where nothing is mapped, until the page goes. + #[test] + fn a_page_needs_memory_and_takes_its_region() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[32]).unwrap(); + let (v, field) = view_of(&space); + assert_eq!(space.hold_line_page(v, 0x10_0000), None); + assert_eq!(space.hold_line_page(v, space.window_size()), None, "outside the window"); + space.map_bank(0, 0, 32 * MB as u64, 32 * MB as u64).unwrap(); + assert_eq!((space.mapped_bitmap(), *field), (0, 0), "a 32 MB bank leaves its 64 MB region to the bus"); + assert_eq!(space.hold_line_page(v, 0x40 * MB as u64), None, "nothing behind it"); + banks[0].write32(0x10_0040, 0x600d); + assert_eq!(space.hold_line_page(v, 0x10_0040), Some(true)); + assert_eq!(*field, 1); + assert_eq!(space.mapped_bitmap(), 0, "the shared window is unchanged"); + assert_eq!(word(space.view_base(v), 0x10_0040), 0x600d, "the page starts as memory"); + assert_eq!(word(space.view_base(v), 0x3ff_fffc), 0, "the rest of the region is safe to touch"); + space.release_line_page(v, 0x10_0040); + assert_eq!(*field, 0); + } } From 45182750de5240531a9019e322de0cbfb9e9d06d Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:05:00 -0700 Subject: [PATCH 02/22] jitv2: compile no page a CPU's cache holds lines in The Fuel's and the Tezro's PROMs run code from primary instruction lines they make valid with Index Store Tag and Index Store Data, and on iris those words differ from memory once memory is set up there (measured on the model, not on hardware). Compiles read memory and compiled code is shared by every CPU, so code compiled from memory would run in place of the lines. Upstream iris has no machine that runs code from a held page yet: the O2 keeps only a stack in lines and runs POST uncached, and the Octane's lines name no memory. This is for the Fuel and the Tezro. The first CPU to hold a page now puts GEN_HELD on the page's generation counter, which retires what was compiled there, and the last release takes it off with an ordinary bump. A compile that finds the mark publishes no code for that generation and denies every offset, so arrivals interpret the page from the CPU's own view without asking again. The first compile after the release allows every offset once more. The release lowers the counter, so publish treats any change of generation as new bytes, not only a rise. ppmem owns the mark, so it is one mark however many CPUs hold the page, and a remap moves it to the counter that backs the page afterwards. All of it is at compile time. Dispatch and compiled code are unchanged, and every other page stays compiled. Without the denial, four Tezro processors asking for the same held PROM page took 65-77 s to the BASEIO monitor against 38-42 s with it (an earlier version of this change, jitv2 loop-profile builds). --- CHANGELOG.md | 3 ++ src/cpu/jitv2/comp.rs | 44 +++++++++++++++++++++++++- src/cpu/jitv2/jitv2.rs | 20 ++++++++++-- src/ppmem/ppmem.rs | 70 ++++++++++++++++++++++++++++++++++++++++++ 4 files changed, 133 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0369d28b4..2b7533993 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -58,6 +58,9 @@ hashes are given where a change is easiest to understand by reading the commit. behind a PCI-to-PCI bridge (`bridge = true`). One namespace backed by a raw or CHD image, optionally copy-on-write; pin interrupts; saved in snapshots with the other disks ([docs/nvme.md](docs/nvme.md)). +### R10000-family cache lines as memory + +- jitv2 does not compile a page any CPU's cache holds lines in: that CPU runs it from its own view in the interpreter, and every other page stays compiled. ### R10000 family fixes, R12000 and R14000 diff --git a/src/cpu/jitv2/comp.rs b/src/cpu/jitv2/comp.rs index d5d009ec8..595750f31 100644 --- a/src/cpu/jitv2/comp.rs +++ b/src/cpu/jitv2/comp.rs @@ -116,6 +116,8 @@ pub(crate) enum Snapshot { Taken(u64), /// A word could not be read (not RAM right now). Unreadable, + /// A CPU's cache holds lines in the page, under this generation. + Held(u64), /// The generation moved under every attempt. Busy, } @@ -129,6 +131,10 @@ pub(crate) fn snapshot_page(page: &PhysicalCodePage, bus: &dyn BusDevice, phys_b const MAX_SEQLOCK_RETRIES: u32 = 8; for _ in 0..MAX_SEQLOCK_RETRIES { let gen0 = page.current_gen(); + // A CPU's cache holds lines in the page (`GEN_HELD`). + if gen0 >= crate::cpu::jitv2::jitv2::GEN_HELD { + return Snapshot::Held(gen0); + } // A seqlock reader: the words are read after `gen0` and before // `gen1` (acquire), and every writer bumps the generation after its // bytes, with release. Then a generation that held means these are @@ -199,11 +205,27 @@ fn prepare_multi_entry_compile( // §13.3 step 2: seqlock-snapshot the page bytes. let mut words = [0u32; ENTRIES_PER_PAGE]; let gen_snap = match snapshot_page(page, &**bus, phys_base, &mut words) { - Snapshot::Taken(gen) => gen, + Snapshot::Taken(gen) => { + // The lines are gone: lift the held publish's denials (below). + if page.entry_gen() >= crate::cpu::jitv2::jitv2::GEN_HELD { + page.allow_all(); + } + gen + } Snapshot::Unreadable => { page.mark_prepare_bounced(); return PrepareOutcome::Done(false); // page not readable right now; retry later } + Snapshot::Held(gen) => { + // Publish no code for this generation and deny every offset, so + // arrivals interpret the page without asking again until the + // release moves the generation (`step_jit`'s denylist gate). + if page.publish(&[0; crate::cpu::jitv2::BITMAP_WORDS], std::ptr::null(), gen, 0, 0, req.compiled_for_fr1) { + page.denylist_all(); + } + page.mark_prepare_bounced(); + return PrepareOutcome::Done(false); + } Snapshot::Busy => { // A sufficiently hot self-modifying/shared-DMA page can bounce here // EVERY single attempt (gen keeps changing faster than @@ -976,6 +998,26 @@ mod tests { assert!(!page.is_denylisted(4)); } + /// A page whose counter carries `GEN_HELD` compiles nothing and denies + /// every offset, and compiles once the mark comes off. + #[test] + fn a_held_page_compiles_once_its_lines_go() { + let _stable = crate::cpu::jitv2::opcode_support::table_stable(); + let bus: Arc = Arc::new(AddiuDevice); + let counter = AtomicU64::new(crate::cpu::jitv2::jitv2::GEN_HELD); + let mut page = PhysicalCodePage::new(0, &counter as *const AtomicU64); + let mut compile = |page: &mut PhysicalCodePage| { + page.mark_requested(4); + let req = CompileRequest { page: page as *mut PhysicalCodePage, compiled_for_fr1: true }; + handle_request_for_test(&req, &bus, &mut Analyzer::new(), &mut Codegen::new()); + }; + compile(&mut page); + assert!(!page.is_runnable(4) && page.is_denylisted(4), "held: interpreted, not asked again"); + counter.fetch_add(1u64.wrapping_sub(crate::cpu::jitv2::jitv2::GEN_HELD), std::sync::atomic::Ordering::Release); + compile(&mut page); + assert!(page.is_runnable(4) && !page.is_denylisted(4), "released: compiled"); + } + #[test] fn a_second_incremental_compile_does_not_drop_the_first_entry() { // §13's one-function-per-page model fully replaces `func` on every diff --git a/src/cpu/jitv2/jitv2.rs b/src/cpu/jitv2/jitv2.rs index 6442262fa..01455afab 100644 --- a/src/cpu/jitv2/jitv2.rs +++ b/src/cpu/jitv2/jitv2.rs @@ -228,6 +228,13 @@ pub fn push_hot_request(hot: &crossbeam_queue::ArrayQueue, req: /// for this constant. pub const PAGE_SIZE: u32 = 4096; +/// Carried by a page's generation counter while some CPU's cache holds +/// lines there (`PpMemSpace::hold_line_page`): that CPU runs the page from +/// its own view, so it is not compiled (`comp::snapshot_page`). It comes off +/// with an ordinary bump, below the held page's last publish, so `publish` +/// takes any change of generation as new bytes. +pub const GEN_HELD: u64 = 1 << 48; + /// Number of possible entry offsets per page: one per 4-byte-aligned word /// (MIPS instructions are always word-aligned), i.e. `PAGE_SIZE / 4` (§2.4: /// `entry_bits` 16×u64 = 1024 bits, `entry_table` 1024 entries). @@ -1620,6 +1627,11 @@ impl PhysicalCodePage { for word in self.denied.iter() { word.store(0, Ordering::Relaxed); } } + /// Undo [`Self::denylist_all`]. + pub fn allow_all(&self) { + for word in self.denied.iter() { word.store(u64::MAX, Ordering::Relaxed); } + } + /// Clear `offset_word`'s `compiled` bit only — NOT `denied`. Two /// distinct callers: /// @@ -1854,8 +1866,9 @@ impl PhysicalCodePage { /// a reader that observes a `compiled` bit set (Acquire-paired via /// `is_runnable`'s `entry_gen` load) never observes a stale/null /// `func` for it. `entry_gen` is written unconditionally on every actual - /// publish, but only **numerically advances** when `snap_gen > - /// entry_gen` (a real page-bump-triggered recompile, §13.6/§13.3's + /// publish, but only **changes** when `snap_gen != entry_gen` (a real + /// page-bump-triggered recompile, or the first after `GEN_HELD` comes + /// off, §13.6/§13.3's /// discussion) — a pure entry-coverage publish against unchanged bytes /// (`snap_gen == entry_gen` already) rewrites the same value. Readers /// never need to distinguish the two: `entry_gen == current_gen()` is @@ -1909,7 +1922,8 @@ impl PhysicalCodePage { } let prev_entry_gen = self.entry_gen.load(Ordering::Relaxed); - let is_real_invalidation = snap_gen > prev_entry_gen; + // Not `>`: a page's generation drops when `GEN_HELD` comes off. + let is_real_invalidation = snap_gen != prev_entry_gen; // Code for the other FR mode replaces the old function outright, so // the old function's entries must go with it: its dispatch switch is // gone, and the new one only has cases for `new_entries`. diff --git a/src/ppmem/ppmem.rs b/src/ppmem/ppmem.rs index 51b847a93..e6da8ba9e 100644 --- a/src/ppmem/ppmem.rs +++ b/src/ppmem/ppmem.rs @@ -818,6 +818,7 @@ impl PpMemSpace { let b = &self.banks[bank]; assert!(src + size <= b.len() as u64, "ppmem: range [{src:#x}, +{size:#x}) escapes bank {bank}"); assert!(offset + size <= self.window_size(), "ppmem: [{offset:#x}, +{size:#x}) escapes the window"); + self.marks(false); unsafe { self.space.map(offset as usize, size as usize, b, src, Prot::ReadWrite)?; } @@ -836,6 +837,7 @@ impl PpMemSpace { } let st = unsafe { &mut *self.state.get() }; Self::record(st, Mapping { at: offset, len: size, bank, src, period: 0 }); + self.marks(true); Ok(()) } @@ -973,6 +975,9 @@ impl PpMemSpace { if !bitmap_maps_at(st.bits, shift, page) && !st.views[v].held.keys().any(|&p| p >> shift == page >> shift) { self.scrub_region_gaps(st, v, page >> shift << shift, 1u64 << shift); } + if !Self::held_anywhere(st, page) { + self.mark(st, page, true); + } let view = &st.views[v]; let at = unsafe { view.space.base().add(page as usize) }; let mut bytes = vec![0u8; gran as usize]; @@ -1000,6 +1005,9 @@ impl PpMemSpace { return false; } st.views[v].held.remove(&page); + if !Self::held_anywhere(st, page) { + self.mark(st, page, false); + } let _ = unsafe { st.views[v].space.scrub(page as usize, gran as usize) }; for m in st.mappings.iter().filter(|m| m.at < page + gran && m.at + m.len > page) { let _ = self.lay(st, v, m, page.max(m.at), (page + gran).min(m.at + m.len)); @@ -1008,6 +1016,38 @@ impl PpMemSpace { true } + fn held_anywhere(st: &SpaceState, page: u64) -> bool { + st.views.iter().any(|view| view.held.contains_key(&page)) + } + + /// A page some view holds carries one `GEN_HELD` in its generation + /// counters while memory backs it, so jitv2 does not compile it. + #[cfg_attr(not(feature = "jitv2"), allow(unused_variables))] + fn mark(&self, st: &SpaceState, page: u64, on: bool) { + #[cfg(feature = "jitv2")] + { + use crate::cpu::jitv2::jitv2::{GEN_HELD, PAGE_SIZE}; + let end = page + granularity() as u64; + if !st.mappings.iter().any(|m| m.at <= page && m.at + m.len >= end) { + return; + } + let d = if on { GEN_HELD } else { 1u64.wrapping_sub(GEN_HELD) }; + let gens = self.gen_space.base() as *const AtomicU64; + for a in (page..end).step_by(PAGE_SIZE as usize) { + unsafe { (*gens.add((a / PAGE_SIZE as u64) as usize)).fetch_add(d, Ordering::Release) }; + } + } + } + + /// Every held page's mark, off before a mapping change and on after. + fn marks(&self, on: bool) { + let st = unsafe { &*self.state.get() }; + let pages: std::collections::BTreeSet = st.views.iter().flat_map(|v| v.held.keys().copied()).collect(); + for page in pages { + self.mark(st, page, on); + } + } + /// Make the parts of `[r, r + len)` that nothing maps private zeros in /// view `v`, and give them generation counters where none are mapped, so /// the region can be marked whole in the view's bitmap: compiled stores @@ -1093,6 +1133,7 @@ impl PpMemSpace { impl MappedMemory for PpMemSpace { fn clear_mappings(&self) { + self.marks(false); let st = unsafe { &mut *self.state.get() }; // Views keep the pages they hold. Scrubbed, not left mapped, so a // view never shows memory the shared window no longer maps. @@ -1124,6 +1165,7 @@ impl MappedMemory for PpMemSpace { } fn map_bank(&self, bank: usize, offset: u64, size: u64, period: u64) -> io::Result<()> { + self.marks(false); let b = &self.banks[bank]; let bank_size = b.len() as u64; assert!(size > 0, "ppmem: map_bank with zero size"); @@ -1176,10 +1218,12 @@ impl MappedMemory for PpMemSpace { let st = unsafe { &mut *self.state.get() }; Self::record(st, Mapping { at: offset, len: size, bank, src: 0, period }); + self.marks(true); Ok(()) } fn map_alias(&self, bank: usize, offset: u64, size: u64) -> io::Result<()> { + self.marks(false); let b = &self.banks[bank]; assert!( size <= b.len() as u64, @@ -1247,6 +1291,7 @@ impl MappedMemory for PpMemSpace { } let st = unsafe { &mut *self.state.get() }; Self::record(st, Mapping { at: offset, len: size, bank, src: 0, period: 0 }); + self.marks(true); Ok(()) } @@ -1981,4 +2026,29 @@ mod tests { space.release_line_page(v, 0x10_0040); assert_eq!(*field, 0); } + + /// A held page's counter carries one `GEN_HELD` however many views hold + /// it, and the mark follows the page to the counter that backs it after + /// a remap, leaving none on the old one. + #[cfg(feature = "jitv2")] + #[test] + fn the_held_mark_follows_the_page() { + use crate::cpu::jitv2::jitv2::GEN_HELD; + let (space, banks) = PpMemSpace::with_bank_sizes(&[64, 64]).unwrap(); + space.map_bank(0, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + let ((a, _fa), (b, _fb)) = (view_of(&space), view_of(&space)); + let gen = |g: *const AtomicU64| unsafe { (*g).load(Ordering::Relaxed) }; + let at = || unsafe { space.gen_window_base().add(0x2000 >> 12) as *const AtomicU64 }; + space.hold_line_page(a, 0x2000); + space.hold_line_page(b, 0x2040); + assert_eq!(gen(at()) / GEN_HELD, 1, "one mark for two CPUs"); + space.clear_mappings(); + space.map_bank(1, 0, 64 * MB as u64, 64 * MB as u64).unwrap(); + assert_eq!(gen(at()) / GEN_HELD, 1, "on the counter that backs the page now"); + assert_eq!(gen(banks[0].gen_ptr(0x2000)) / GEN_HELD, 0, "and off the old one"); + space.release_line_page(a, 0x2000); + assert_eq!(gen(at()) / GEN_HELD, 1); + space.release_line_page(b, 0x2040); + assert_eq!(gen(at()) / GEN_HELD, 0); + } } From d2cb3cfc3ba223dc157b686fa1a1e1350cd5144f Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:10:00 -0700 Subject: [PATCH 03/22] cpu: keep valid primary data and secondary lines in the CPU's cached view The R10000-family PROMs keep stacks and data in cache lines they make valid with Index Store Tag and Index Store Data. The O2's POST tests page 0 uncached while its stack lives in secondary and primary data lines for the same page, the Fuel's and Tezro's stacks sit in primary data lines for 0x100000-0x107fff, and on the Tezro every processor holds those lines at once with its own contents. The shadow cache kept the tags and data arrays for diagnostics only, so a cached access read memory. A tag change that makes a primary data or secondary line valid now holds its page in the CPU's cached view and writes the line's data array into it, since Index Store Tag writes the tag array alone (NEC VR5000/VR10000 instruction manual pp. 24-25). Loads, stores, fetches, the decode cache and compiled code read the view as before and change not at all. Uncached accesses bypass the caches (NEC VR10000 user's manual p. 75) and keep seeing memory, as do other masters, since the views are not coherent. A tag that names another line, or none, ends the line, and the page goes with its last line. Index Store Data into a held line writes through. The tags stay the only line state, plus a held bit. The O2's R5000 POST keeps its stack the same way, in primary data lines for page 0 made valid by Index Store Tag (D) with TagLo 0xc0, and tests page 0 uncached meanwhile. An R5000 primary data tag has the R10000's PTag and PState fields (NEC VR5000 user's manual U11761EJ6V0UM p. 133), and a store hit on a write-back line makes no bus request (p. 243), so a valid one holds its page too. The R5000 has no Index Store Data, so the page starts as memory. Its instruction lines hold nothing: with no Index Store Data to fill one, it could hold only what memory holds. Every line the shipped PROMs stage ends by Index Store Tag, none is written back, and none is read back with Index Load Data while it holds a page, so the writeback and invalidate operations stay as they were. The R12000 zeroing a primary data tag at Index Writeback Invalidate ends a line like any other tag change. A secondary doubleword puts TagLo at the lower address. The manuals fix only that TagHi is the doubleword's high word and TagLo its low word (MIPS R10000 user's manual p. 201, NEC VR10000 user's manual p. 289); which address each pairs with is inferred, not established, from shipped PROMs. PROM 6.210 on the Fuel and the Tezro stages code with the word at A in TagLo and the word at A + 4 in TagHi, 990 pairs in a Tezro boot, and its staged code equals its image only this way. The IP35 boot-flash PROM's staging pairs them the same way: simulated with the other order, 378 of its 562 staged fetches break, every trampoline fetch among them. With the other order the Tezro's PROM falls back to running its cache test uncached, which took twice as long to reach the BASEIO monitor. Limits: a page, not a line, is private, so the rest of a held page shows memory as it was when the page was taken, and nothing is coherent with other CPUs or DMA. A snapshot taken while lines hold pages is refused at restore. Lines that name no memory, like the Octane's tag test patterns, hold nothing. `l2 probe 0` lists the pages held. --- CHANGELOG.md | 1 + clanker/INDEX.md | 1 + .../cpu/cache-lines-live-in-the-cpus-view.md | 88 ++++ src/cpu/mips_cache_shadow.rs | 483 +++++++++++++++++- src/cpu/mips_cache_v2.rs | 9 + src/cpu/mips_exec.rs | 24 + src/ip30_machine.rs | 1 + src/ip32_machine.rs | 1 + src/machine.rs | 1 + 9 files changed, 604 insertions(+), 5 deletions(-) create mode 100644 clanker/cpu/cache-lines-live-in-the-cpus-view.md diff --git a/CHANGELOG.md b/CHANGELOG.md index 2b7533993..f0b52600f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -60,6 +60,7 @@ hashes are given where a change is easiest to understand by reading the commit. with the other disks ([docs/nvme.md](docs/nvme.md)). ### R10000-family cache lines as memory +- A valid primary data or secondary cache line now holds its page in that CPU's own copy of ppmem's window, so a PROM that keeps its stack in cache lines reads and writes the lines with cached accesses while the uncached path, DMA and the other CPUs see memory. Index Store Data writes through to such a line. `l2 probe 0` lists the pages held. Nothing changes on a load, store or fetch. A valid R5000 primary data line does the same, for the O2's POST stack. - jitv2 does not compile a page any CPU's cache holds lines in: that CPU runs it from its own view in the interpreter, and every other page stays compiled. ### R10000 family fixes, R12000 and R14000 diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 833a3845e..0c760c3cb 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -24,6 +24,7 @@ CPU architecture: CP0, TLB and its fast paths, caches, FPU, LL/SC, CPU models. B | Note | Kind | Title | |---|---|---| | [cached-flash-is-a-cache-with-two-cpus.md](cpu/cached-flash-is-a-cache-with-two-cpus.md) | rule | With two CPUs, cached flash reads must hold their lines: a fetch through to a chip in status mode executes `0x80808080` | +| [cache-lines-live-in-the-cpus-view.md](cpu/cache-lines-live-in-the-cpus-view.md) | design | A valid R10000-family cache line, or R5000 primary data line, makes its page private in that CPU's own copy of ppmem's window; the uncached path and other masters keep memory. Limits and how to see what is held | | [cache-attributes-and-fetch-alignment.md](cpu/cache-attributes-and-fetch-alignment.md) | rule | Cache attributes and fetch alignment: three hardcodings, all wrong | | [cache-tag-index-and-check-bits.md](cpu/cache-tag-index-and-check-bits.md) | rule | Cache tags: index ops index by address; check bits are generated, not stored | | [cop0-requires-kernel-or-cu0.md](cpu/cop0-requires-kernel-or-cu0.md) | rule | CP0 instructions must be gated on Kernel-or-CU0 | diff --git a/clanker/cpu/cache-lines-live-in-the-cpus-view.md b/clanker/cpu/cache-lines-live-in-the-cpus-view.md new file mode 100644 index 000000000..1b883675d --- /dev/null +++ b/clanker/cpu/cache-lines-live-in-the-cpus-view.md @@ -0,0 +1,88 @@ +# Cache lines live in the CPU's own view of memory + +The R10000-family PROMs use the caches as memory before memory works. They make +lines valid with Index Store Tag and Index Store Data and then load, store and +fetch through them, while uncached accesses, which bypass the caches (NEC +VR10000 user's manual p. 75), and every other master see memory. What the shipped PROMs do on iris: + +| Machine | Lines | Memory behind them | +|---|---|---| +| O2 | secondary and primary data lines for page 0 and 0x1000 hold POST's stack for about 50-65 ms, while POST tests page 0 uncached. On the R5000, primary data lines for page 0 (TagLo 0xc0) | yes | +| Octane | a tag and data RAM test on secondary set 0, read back only with Index Load Tag/Data | no: the patterns name no memory | +| Fuel | primary data lines for 0x100000-0x107fff hold the stack; primary instruction lines hold PROM code | no, until memory is configured mid-phase | +| Tezro | the same as the Fuel, on every processor at once, with different stack contents at the same addresses | the same | + +Every line ends by Index Store Tag. None is written back, and none is read back +with Index Load Data while it holds a page. + +## How + +- Each CPU reads memory through a window of its own (`PpMemSpace::add_cached_view`). + Every mapping laid in the shared window is laid in each view too, and the view's + region bitmap goes to that CPU's inline fields. The shared window, which + `Physical`'s direct paths, CRIME's rendering engine and HEART read, never holds a + line. +- When Index Store Tag makes a line valid, its page becomes private memory in that + view, a copy of memory, and the line's data array is written into it + (`ShadowCache::line_enters`). An invalid or different tag ends the line, and the + page goes with its last line. The shadow's tags stay the only line state, with a + held bit beside each. Index Store Data into a held line writes through. +- The R5000 does the same for a valid primary data line, whose tag has the + R10000's PTag and PState fields (NEC VR5000 user's manual U11761EJ6V0UM p. 133). + It has no Index Store Data, so the page starts as memory. Its instruction lines + hold nothing, since one could hold only what memory holds. +- A page with no memory behind it holds nothing: the line stays in the arrays, as + a tag test's patterns do. The region bitmap is all a CPU tests, so a held page's + whole region is the view's while it stands, reading zeros wherever nothing is + mapped. The O2's POST holds lines for page 0 before CRIME maps the rest of that + region. +- A remap saves and restores held pages, so memory configuration mid-phase keeps + the stack. +- Loads, stores, fetches, the decode cache and compiled code are untouched: they + read the view through the base pointer and bitmap they always had. The decode + cache compares every word it serves, so it sees a page change hands. + +## Limits + +Each is an approximation, stated where it is made. + +- Page, not line: the rest of a held page shows memory as it was when the page was + taken, uncached writes to it meanwhile are not seen by cached reads, cached stores + to it are dropped at release, and a line that ends leaves its bytes until the + page's last line goes. The page is the host granule: 4 KB on Linux and x86 macOS, + 16 KB on Apple silicon, 64 KB on Windows. +- Lines end only by a tag change the shadow already models: Index Store Tag, and + Index Writeback Invalidate (D) zeroing the tag from the R12000 on. Nothing is + written back. The hit operations and the other index invalidates leave a held + line as they leave every shadow tag, untouched. +- Index Load Data reads the data array, not the page, so it does not see cached + stores. +- A region holding a page reads zeros in its unmapped gaps for that CPU's cached + accesses, where the bus would answer. +- A secondary doubleword puts TagLo at the lower address. The manuals say only + that TagHi is the high word and TagLo the low word (MIPS R10000 UM p. 201, NEC + VR10000 UM p. 289); which address each pairs with is inferred. PROM 6.210 (Fuel, + Tezro) stages code with the word at A in TagLo and the word at A + 4 in TagHi, + 990 times in a Tezro boot, and its staged code equals its image only this way. + The IP35 boot-flash PROM's staging pairs the words the same way: simulated with + the other order, 378 of its 562 staged fetches break, every trampoline fetch + among them. With the other order the Tezro's PROM ran its long cache test + uncached, taking twice as long. +- No replacement, parity or ECC, and no coherence with other CPUs or DMA. +- A snapshot taken while lines hold pages is refused at restore. + +## Compiled code + +A page some CPU holds is not compiled: compiles read memory, and compiled code is +shared. ppmem puts `GEN_HELD` on the page's generation counter at the first hold +and takes it off at the last release, moving it with the page on a remap +(`PpMemSpace::mark`). The compile that finds it publishes nothing and denies every +offset until the release. ppmem's counters are the bus's for memory; a machine whose bus +maps a device over memory, as the IP35 boards map their reset PROM, must mark that +device's counter as well. + +## Seeing it + +`l2 probe 0` in the monitor prints how many lines hold pages and which. Every +machine should show none at the IRIX login. `log l2c mask op` logs each page held +and released. diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 7f056a863..53486644d 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -30,6 +30,18 @@ //! The shadow is deliberately not consulted by `read`/`write`/`fetch`. If it //! ever needs to be, this type is the wrong shape and should say so loudly //! rather than growing a slow path. +//! +//! The R10000-family PROMs use the caches as memory before memory works: they +//! make lines valid with Index Store Tag and Index Store Data and keep a stack +//! or run code in them, while the uncached path sees memory (NEC VR10000 +//! user's manual p. 75). Such a line lives in ppmem's window rather than +//! here: the window this cache reads is the CPU's own cached view +//! (`PpMemSpace::add_cached_view`), and a valid line makes its page private +//! memory in that view, holding the line's bytes +//! ([`ShadowCache::line_enters`]). Loads, stores and fetches read the view as +//! they always did. The CACHE instruction is the only thing that moves a page +//! in or out, and the tags stay the only record of which lines are valid. +//! `clanker/cpu/cache-lines-live-in-the-cpus-view.md` has the limits. use std::cell::UnsafeCell; use std::sync::Arc; @@ -38,6 +50,7 @@ use crate::cpu::mips_cache_v2::{ cache_op_name, model, r4k_load_taglo, r4k_store_taglo, CpuModel, FetchInstrResult, MipsCache, C_ILT, C_IST, C_IWBINV, C_R10K_CBARRIER, C_R10K_ILD, C_R10K_ISD, CACH_PD, CACH_PI, CACH_SD, CACH_SI, }; +use crate::ppmem::PpMemSpace; use crate::traits::{BusDevice, BusRead64, BUS_OK}; use crate::devlog::{LogModule, CACHE_LOG_HIT, CACHE_LOG_OP, devlog_is_active, devlog_mask}; @@ -63,11 +76,15 @@ struct Shadow { /// — see `MRU_BIT`. Hardware state rather than tag storage, which is why /// it is kept here and re-applied on read instead of living in `tags`. mru: Box<[bool]>, + /// The line holds its page in the CPU's cached view + /// (`ShadowCache::line_enters`). + held: Box<[bool]>, } impl Shadow { fn new(lines: usize, data_words: usize) -> Self { Self { + held: vec![false; lines.max(1)].into_boxed_slice(), tags: vec![0u64; lines.max(1)].into_boxed_slice(), data: vec![0u64; data_words].into_boxed_slice(), ecc: vec![0u32; data_words].into_boxed_slice(), @@ -87,6 +104,13 @@ impl Shadow { /// nothing, because the way that is read is never the way that was written. const MRU_BIT: u64 = 1 << 63; +/// R10000-family tag states (NEC U10278EJ3V0UM pp. 285-287): the primary data +/// state in TagLo[7:6] and the secondary state in TagLo[11:10], 0 Invalid. +const PD_STATE_SHIFT: u32 = 6; +const SD_STATE_SHIFT: u32 = 10; + + + /// R10000 only: ways per set, as CACHE index operations address them. Bit 0 /// of the index selects the way; the set starts above the line offset. The /// R4400 and R5000 index differently — see `tag_slot`. @@ -156,6 +180,10 @@ pub struct ShadowCache< decode: UnsafeCell, /// See [`MipsCache::set_pd_tag_store_hook`]. Consulted by `cache_op` only. pd_tag_hook: UnsafeCell>>, + /// The cached view `tc_base` is, and its number in `PpMemSpace`: where + /// valid lines hold their pages. Null without one. + view_space: UnsafeCell<*const PpMemSpace>, + view: UnsafeCell, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -213,6 +241,8 @@ impl< tc_gen: UnsafeCell::new(std::ptr::null_mut()), decode: UnsafeCell::new(crate::cpu::decode_cache::DecodeCache::new()), pd_tag_hook: UnsafeCell::new(None), + view_space: UnsafeCell::new(std::ptr::null()), + view: UnsafeCell::new(0), } } @@ -340,6 +370,155 @@ impl< crate::dlog!(LogModule::L2c, "shadow: -> ISD slot={slot} data={phys_addr:#018x} ecc={:#x}", s.ecc[slot]); } + if s.held.get(self.tag_slot(sel, virt_addr)) == Some(&true) { + self.store_into_held_line(sel, virt_addr, phys_addr); + } + } + + // ---- valid lines hold pages in the CPU's cached view ---- + + fn view_space(&self) -> Option<&PpMemSpace> { + unsafe { (*self.view_space.get()).as_ref() } + } + + fn view_no(&self) -> usize { + unsafe { *self.view.get() } + } + + /// The physical address and length of the line `tag` names, if it is + /// valid. `va` is an index operand naming the line's slot: the primary + /// caches take PA[11:5] or PA[11:6] from it, the secondary PA[17:7] (NEC + /// U10278EJ3V0UM pp. 285-287; the tag holds the rest). The R5000's + /// primary data tag has the same PTag and PState fields (NEC + /// U11761EJ6V0UM p. 133). + fn line_of(&self, sel: u32, tag: u64, va: u64) -> Option<(u64, usize)> { + let pa_hi = ((tag >> 32) & 0xf) << 36; + match sel { + CACH_PD if DC_LINE != 0 && (tag >> PD_STATE_SHIFT) & 3 != 0 => { + Some((pa_hi | ((tag >> 8) & 0xff_ffff) << 12 | (va & 0xfff & !(DC_LINE as u64 - 1)), DC_LINE)) + } + CACH_SD if L2_LINE != 0 && (tag >> SD_STATE_SHIFT) & 3 != 0 => { + Some((pa_hi | ((tag >> 14) & 0x3_ffff) << 18 | (va & 0x3_ffff & !(L2_LINE as u64 - 1)), L2_LINE)) + } + _ => None, + } + } + + /// An index operand that names slot `i` of `sel`: `tag_slot`'s inverse. + fn slot_va(&self, sel: u32, i: usize) -> u64 { + let line = match sel { CACH_PI => IC_LINE, CACH_PD => DC_LINE, _ => L2_LINE }; + if Self::R5K && matches!(sel, CACH_PI | CACH_PD) { + let sets = if sel == CACH_PI { Self::IC_LINES } else { Self::DC_LINES } / 2; + return ((i / sets) << 14 | (i % sets) * line) as u64; + } + ((i / WAYS) * line + (i % WAYS)) as u64 + } + + /// A CACHE instruction changed slot `idx` of `sel`'s tag from `old` to + /// `new`; `va` is the operand that named it. A held line that stops being + /// valid, or moves, gives its page back; one that becomes valid takes one. + /// The tags stay the only record of which lines are valid. + #[cold] + #[inline(never)] + fn tag_changes(&self, sel: u32, idx: usize, va: u64, old: u64, new: u64) { + let Some(sp) = self.view_space() else { return }; + let now = self.line_of(sel, new, va); + if self.shadow(sel).held[idx] { + let was = self.line_of(sel, old, va).expect("a held line is valid"); + if now == Some(was) { + return; // the state changed, not the line + } + self.line_leaves(sp, sel, idx, was); + } + if let Some(line) = now { + self.line_enters(sp, sel, idx, va, line); + } + } + + /// Slot `idx` of `sel` holds a valid line for `[pa, pa + len)`: its page + /// becomes private in the CPU's view and the line's bytes are written + /// into it from the data array, since Index Store Tag writes the tag + /// alone (NEC VR5000/VR10000 instruction manual pp. 24-25). A line with + /// no memory behind it holds no page and stays in the arrays, as every + /// line did before. + fn line_enters(&self, sp: &PpMemSpace, sel: u32, idx: usize, va: u64, (pa, len): (u64, usize)) { + let Some(new) = sp.hold_line_page(self.view_no(), pa) else { return }; + let base = unsafe { *self.tc_base.get() }; + if l2_op_log(CACHE_LOG_OP) { + crate::dlog!(LogModule::L2c, "shadow: -> line {pa:#x} holds its page (new: {new})"); + } + let s = self.shadow(sel); + s.held[idx] = true; + if s.data.is_empty() { + return; // the R5000 has no Index Store Data: the page starts as memory + } + let (way, first) = (va & 1, va & !(len as u64 - 1)); + let unit = if sel == CACH_SD { 8 } else { 4 }; + for k in 0..len / unit { + let word = s.data[self.data_slot(sel, s.data.len(), first + (unit * k) as u64 + way)]; + let at = pa + (unit * k) as u64; + match sel { + CACH_PD => unsafe { *(base.add(at as usize) as *mut u32) = word as u32 }, + // TagHi is the doubleword's high word and TagLo its low word + // (MIPS R10000 UM p. 201; NEC VR10000 UM p. 289, Fig. 13-33). + // Neither manual says which address each word belongs to. + // TagLo at the lower address is inferred from shipped PROMs: + // PROM 6.210 stages code pairs with the word at A in TagLo + // and the one at A + 4 in TagHi, and its staged code equals + // its image only this way, as does the IP35 boot-flash + // PROM's staging. The window keeps a doubleword's words in + // address order, low half first, so this is the data as is. + _ => unsafe { *(base.add(at as usize) as *mut u64) = word }, + } + } + } + + /// The line in slot `idx` of `sel` leaves, unwritten, as Index Store Tag + /// leaves it. With the page's last line the view shows memory there + /// again. Until then the line's bytes stay in the page. + fn line_leaves(&self, sp: &PpMemSpace, sel: u32, idx: usize, (pa, len): (u64, usize)) { + let s = self.shadow(sel); + s.held[idx] = false; + let gone = sp.release_line_page(self.view_no(), pa); + if l2_op_log(CACHE_LOG_OP) { + crate::dlog!(LogModule::L2c, "shadow: -> line {pa:#x} leaves (page released: {gone})"); + } + } + + /// Index Store Data into a held line writes through to its page, so the + /// next cached access sees it. Same quadword rule as the data array. + fn store_into_held_line(&self, sel: u32, va: u64, value: u64) { + let idx = self.tag_slot(sel, va); + let Some((pa, len)) = self.line_of(sel, self.shadow(sel).tags[idx], va) else { return }; + let unit = if sel == CACH_SD { 8 } else { 4 }; + let at = pa + (va & (len as u64 - 1) & !(unit - 1)); + let base = unsafe { *self.tc_base.get() }; + match sel { + CACH_PD => unsafe { *(base.add(at as usize) as *mut u32) = value as u32 }, + _ => unsafe { + *(base.add(at as usize) as *mut u64) = value; + *(base.add((at ^ 8) as usize) as *mut u64) = 0; + }, + } + } + + /// How many lines hold pages. + fn held_count(&self) -> usize { + [CACH_PD, CACH_SD].iter().map(|&sel| self.shadow(sel).held.iter().filter(|&&h| h).count()).sum() + } + + /// Every held line goes, with no writeback: power-on, or a snapshot + /// restore. + fn drop_held_lines(&self) { + let Some(sp) = self.view_space() else { return }; + for sel in [CACH_PD, CACH_SD] { + for i in 0..self.shadow(sel).held.len() { + if self.shadow(sel).held[i] { + let line = self.line_of(sel, self.shadow(sel).tags[i], self.slot_va(sel, i)).expect("a held line is valid"); + self.line_leaves(sp, sel, i, line); + } + } + } } } @@ -421,6 +600,21 @@ impl< unsafe { *self.pd_tag_hook.get() = Some(hook) }; } + unsafe fn set_cached_view(&self, sp: *const PpMemSpace, view: usize) { + unsafe { + *self.view_space.get() = sp; + *self.view.get() = view; + } + } + + /// `l2 probe`: the lines holding pages in the CPU's cached view. + fn debug_probe(&self, _cache_name: &str, _virt_addr: u64, _phys_addr: u64) -> String { + let pages = self.view_space().map(|sp| sp.held_line_pages(self.view_no())).unwrap_or_default(); + let list: Vec = pages.iter().map(|p| format!("{p:#x}")).collect(); + format!("shadow: {} valid lines hold {} pages in the cached view [{}]", + self.held_count(), pages.len(), list.join(" ")) + } + /// Tags only: a fetch reads memory through the decode cache and changes /// nothing a guest can see. fn fetch_is_pure(&self) -> bool { true } @@ -589,7 +783,13 @@ impl< let idx = self.tag_slot(sel, virt_addr); let s = self.shadow(sel); if op == C_IST { - s.tags[idx] = r4k_store_taglo(Self::R5K, sel, phys_addr as u32) as u64; + let old = std::mem::replace(&mut s.tags[idx], r4k_store_taglo(Self::R5K, sel, phys_addr as u32) as u64); + // The O2's POST keeps its stack in R5000 primary data lines + // as it does in R10000 ones. An instruction line, with no + // Index Store Data to fill it, could hold only memory. + if Self::R5K && sel == CACH_PD && old != s.tags[idx] { + self.tag_changes(sel, idx, virt_addr, old, s.tags[idx]); + } return 0; } return r4k_load_taglo(Self::R5K, sel, s.tags[idx] as u32) as u64; @@ -619,7 +819,11 @@ impl< // here destroyed real tag bits. let set = (idx / WAYS).min(s.mru.len() - 1); s.mru[set] = phys_addr & MRU_BIT != 0; + let old = s.tags[idx]; s.tags[idx] = phys_addr & mask & !MRU_BIT; + if Self::R10K && matches!(sel, CACH_PD | CACH_SD) && old != s.tags[idx] { + self.tag_changes(sel, idx, virt_addr, old, s.tags[idx]); + } } 0 } @@ -666,7 +870,8 @@ impl< let idx = self.tag_slot(sel, virt_addr); let s = self.shadow(sel); if idx < s.tags.len() { - s.tags[idx] = 0; + let old = std::mem::take(&mut s.tags[idx]); + self.tag_changes(sel, idx, virt_addr, old, 0); } 0 } @@ -680,6 +885,7 @@ impl< /// Power-on: tags read back invalid, and nothing decoded survives — RAM /// may be reloaded without a write counter moving. fn power_on(&self) { + self.drop_held_lines(); for sel in [CACH_PI, CACH_PD, CACH_SD] { let s = self.shadow(sel); s.tags.fill(0); @@ -688,10 +894,27 @@ impl< unsafe { (*self.decode.get()).flush() }; } - /// A restore replaces RAM wholesale; drop every decoded instruction. The - /// shadow arrays are not snapshotted (nothing reads them but diagnostics). - fn load_cache_state(&self, _v: &toml::Value) -> Result<(), String> { + /// The shadow arrays are not snapshotted (nothing reads them but + /// diagnostics). Lines holding pages are, as a count: a PROM's cache + /// phase is the only time any are held, and a snapshot of it would + /// restore without them, so it is refused. + fn save_cache_state(&self) -> toml::Value { + let mut t = toml::map::Map::new(); + let n = self.held_count(); + if n > 0 { + t.insert("held_lines".into(), toml::Value::Integer(n as i64)); + } + toml::Value::Table(t) + } + + /// A restore replaces RAM wholesale; drop every decoded instruction and + /// every held line. + fn load_cache_state(&self, v: &toml::Value) -> Result<(), String> { + if let Some(n) = crate::snapshot::get_field(v, "held_lines") { + return Err(format!("cache: the snapshot was taken with {n} cache lines holding memory, which it does not carry")); + } unsafe { (*self.decode.get()).flush() }; + self.drop_held_lines(); Ok(()) } @@ -1144,4 +1367,254 @@ mod tests { assert!(g.supported && g.tagless && !g.has_l2, "tagless, and no L2 decode slots for a store to invalidate: {g:?}"); } + + // ---- valid lines hold pages in the CPU's cached view ---- + + use crate::ppmem::{MappedMemory, PpMemory}; + + /// A cache reading memory through a cached view of a fresh window, with + /// a 64 MB bank mapped at 0. Its bus is the bank itself: memory as the + /// uncached path sees it. Boxed: the view publishes into the cache's + /// inline bitmap field. + fn windowed() -> (Box, Box, Arc) { + let (space, mut banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let space = Box::new(space); + let bank = Arc::new(banks.remove(0)); + let c = Box::new(R10000ShadowCache::from(bank.clone() as Arc)); + attach(&c, &space); + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + (c, space, bank) + } + + fn attach(c: &R10000ShadowCache, space: &PpMemSpace) -> usize { + let v = space.add_cached_view(&[c.tcache_bitmap_ptr()]).unwrap(); + unsafe { + c.set_tcache_window(space.view_base(v)); + c.set_cached_view(space, v); + } + v + } + + /// TagLo of a DirtyExclusive primary data line for `pa`'s page. + fn pd_dirty(pa: u64) -> u64 { (pa >> 12) << 8 | 3 << PD_STATE_SHIFT } + + /// TagLo of a secondary line for `pa` in `state`. + fn sd_tag(pa: u64, state: u64) -> u64 { (pa >> 18) << 14 | state << SD_STATE_SHIFT } + + /// The O2's POST: a valid tag for page 0, the page tested uncached while + /// the cached side keeps its stack, and an invalid tag that drops the + /// line without a writeback. + #[test] + fn a_valid_line_gives_the_cached_side_its_own_page() { + let (c, space, bank) = windowed(); + bank.write32(0x40, 0x1111_1111); + c.cache_op(C_R10K_ISD | CACH_PD, 0x8000_0040, 0x1111_1111); + c.cache_op(C_IST | CACH_PD, 0x8000_0040, pd_dirty(0)); + assert_eq!(space.held_line_pages(0), vec![0]); + assert_eq!(c.read::<4>(0, 0x40).data, 0x1111_1111); + c.write::<4>(0, 0x40, 0x2222_2222); + bank.write32(0x40, 0x3333_3333); + assert_eq!(c.read::<4>(0, 0x40).data, 0x2222_2222, "cached: the line"); + assert_eq!(bank.read32(0x40).data, 0x3333_3333, "uncached: memory"); + c.cache_op(C_IST | CACH_PD, 0x8000_0040, 0); + assert!(space.held_line_pages(0).is_empty()); + assert_eq!(c.read::<4>(0, 0x40).data, 0x3333_3333, "dropped, not written back"); + } + + /// Index Store Tag writes the tag alone, so the line holds what Index + /// Store Data put in the data array before, and Index Store Data after + /// writes through. Index Load Data reads the array. + #[test] + fn a_line_holds_its_data_array_and_takes_later_stores() { + let (c, _space, bank) = windowed(); + bank.write32(0x2004, 0x5555_5555); + c.cache_op(C_R10K_ISD | CACH_PD, 0x8000_0004, 0xdead_beef); + c.cache_op(C_IST | CACH_PD, 0x8000_0000, pd_dirty(0x2000)); + assert_eq!(c.read::<4>(0, 0x2004).data, 0xdead_beef); + c.cache_op(C_R10K_ISD | CACH_PD, 0x8000_000c, 0x0123_4567); + assert_eq!(c.read::<4>(0, 0x200c).data, 0x0123_4567); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PD, 0x8000_000c, 0), 0x0123_4567); + assert_eq!(bank.read32(0x2004).data, 0x5555_5555, "memory never saw the line"); + } + + /// A secondary line takes a doubleword at a time with TagLo at the lower + /// address. That is how the shipped PROMs stage code: the instruction at + /// A in TagLo and the one at A + 4 in TagHi, then Index Store Data (S), + /// and cached accesses see them in that order. Index Store Data (S) also + /// zeroes the other doubleword of its quadword (R10000 UM p. 202). + #[test] + fn a_secondary_line_runs_code_staged_the_way_the_proms_stage_it() { + let (c, space, _bank) = windowed(); + let (at_a, at_a4) = (0x2404_0002u64, 0x03e0_0008u64); // addiu $4,$0,2; jr $31 + c.cache_op(C_R10K_ISD | CACH_SD, 0x8000_3008, at_a4 << 32 | at_a); + c.cache_op(C_IST | CACH_SD, 0x8000_3000, sd_tag(0x3000, 2)); + assert_eq!(space.held_line_pages(0), vec![0x3000]); + assert_eq!(c.read::<4>(0, 0x3008).data, at_a, "the word at A"); + assert_eq!(c.read::<4>(0, 0x300c).data, at_a4, "the word at A + 4"); + c.cache_op(C_R10K_ISD | CACH_SD, 0x8000_3000, 0x9999); + assert_eq!(c.read::<8>(0, 0x3000).data, 0x9999 << 32); + assert_eq!(c.read::<8>(0, 0x3008).data, 0, "the partner doubleword is zeroed"); + } + + /// A tag test's patterns name no memory: they hold nothing, and stay + /// in the arrays as before. + #[test] + fn a_line_with_no_memory_behind_holds_no_page() { + let (c, space, _bank) = windowed(); + c.cache_op(C_IST | CACH_SD, 0x8000_0000, sd_tag(0x55_5554_0000, 3) | 0x5); + c.cache_op(C_IST | CACH_PD, 0x8000_0000, pd_dirty(0x8000_0000)); + assert!(space.held_line_pages(0).is_empty()); + assert_eq!(c.cache_op(C_ILT | CACH_SD, 0x8000_0000, 0) & 0xffff_ffff, (sd_tag(0x55_5554_0000, 3) | 0x5) & 0xffff_ffff); + } + + /// The R4400 PROMs keep nothing in their caches: a valid tag there holds + /// no page, and none of this runs for them. + #[test] + fn an_r4400_tag_holds_nothing() { + let (space, mut banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let bank = Arc::new(banks.remove(0)); + let c = Box::new(R4400ShadowCache::from(bank as Arc)); + let v = space.add_cached_view(&[c.tcache_bitmap_ptr()]).unwrap(); + unsafe { + c.set_tcache_window(space.view_base(v)); + c.set_cached_view(&space, v); + } + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + c.cache_op(C_IST | CACH_PD, 0x8000_2000, 0x2 << 8 | 3 << 6); + c.cache_op(C_IST | CACH_SD, 0x8000_2000, 0x1 << 13 | 5 << 10); + assert!(space.held_line_pages(v).is_empty()); + } + + /// An R5000 cache on a cached view of a fresh window with a 64 MB bank + /// at 0, as `windowed` builds an R10000 one. + fn r5000_windowed() -> (Box, Box, Arc) { + let (space, mut banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let space = Box::new(space); + let bank = Arc::new(banks.remove(0)); + let c = Box::new(R5000ShadowCache::from(bank.clone() as Arc)); + let v = space.add_cached_view(&[c.tcache_bitmap_ptr()]).unwrap(); + unsafe { + c.set_tcache_window(space.view_base(v)); + c.set_cached_view(&*space, v); + } + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + (c, space, bank) + } + + /// The O2's POST on the R5000, as the shipped PROM runs it on iris: + /// Index Store Tag (D) with TagLo 0xc0 (page 0, Valid) at every index of + /// 0x8000_0000..=0x8000_1000, the stack stored cached, page 0 tested + /// uncached, then an invalid tag at each index. The cached side keeps + /// the stack until the last line goes, which drops it unwritten. The + /// R5000 has no Index Store Data, so the page starts as memory. + #[test] + fn an_r5000_primary_data_line_gives_the_cached_side_its_own_page() { + let (c, space, bank) = r5000_windowed(); + bank.write32(0xff4, 0x1111_1111); + for va in (0x8000_0000..=0x8000_1000u64).step_by(32) { + c.cache_op(C_IST | CACH_PD, va, 0xc0); + } + assert_eq!(space.held_line_pages(0), vec![0]); + assert_eq!(c.read::<4>(0, 0xff4).data, 0x1111_1111, "the page starts as memory"); + c.write::<4>(0, 0xff4, 0xbfc0_4400); + bank.write32(0xff4, 0xa000_0ff0); + assert_eq!(c.read::<4>(0, 0xff4).data, 0xbfc0_4400, "cached: the stack"); + assert_eq!(bank.read32(0xff4).data, 0xa000_0ff0, "uncached: the memory test"); + assert_eq!(c.cache_op(C_ILT | CACH_PD, 0x8000_0fe0, 0) & 0xc0, 0xc0, "the tag reads back"); + for va in (0x8000_0020..=0x8000_1000u64).step_by(32) { + c.cache_op(C_IST | CACH_PD, va, 0); + } + assert_eq!(space.held_line_pages(0), vec![0], "until its last line goes"); + c.cache_op(C_IST | CACH_PD, 0x8000_0000, 0); + assert!(space.held_line_pages(0).is_empty()); + assert_eq!(c.read::<4>(0, 0xff4).data, 0xa000_0ff0, "dropped, not written back"); + } + + /// The R5000's second way, VA[14], holds a page like the first, and + /// power-on drops it. Instruction lines hold nothing: with no Index + /// Store Data, one could hold only what memory holds. There is no + /// secondary. + #[test] + fn an_r5000_line_in_either_way_holds_and_power_on_drops_it() { + let (c, space, _bank) = r5000_windowed(); + c.cache_op(C_IST | CACH_PI, 0x8000_0000, 0x3 << 8 | 0x80); + c.cache_op(C_IST | CACH_SD, 0x8000_0000, 0x1 << 13 | 5 << 10); + assert!(space.held_line_pages(0).is_empty()); + c.cache_op(C_IST | CACH_PD, 0x8000_4fe0, 0x5 << 8 | 0xc0); + assert_eq!(space.held_line_pages(0), vec![0x5000]); + assert_eq!(c.cache_op(C_ILT | CACH_PD, 0x8000_0fe0, 0) & 0xc0, 0, "way 0 is untouched"); + c.power_on(); + assert!(space.held_line_pages(0).is_empty()); + assert!(c.debug_probe("l2", 0, 0).contains("0 valid lines hold 0 pages")); + } + + /// The Tezro: two CPUs keep their stacks in lines for the same physical + /// page, with different contents, and neither sees the other's. + #[test] + fn two_cpus_hold_the_same_page_apart() { + let (space, mut banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let space = Box::new(space); + let bank = Arc::new(banks.remove(0)); + let a = Box::new(R10000ShadowCache::from(bank.clone() as Arc)); + let b = Box::new(R10000ShadowCache::from(bank.clone() as Arc)); + attach(&a, &space); + attach(&b, &space); + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + bank.write32(0x10_0040, 0x1234_5678); + for c in [&a, &b] { + c.cache_op(C_R10K_ISD | CACH_PD, 0x8000_0040, 0x1234_5678); + c.cache_op(C_IST | CACH_PD, 0x8000_0040, pd_dirty(0x10_0000)); + } + a.write::<4>(0, 0x10_0040, 0xaaaa_aaaa); + b.write::<4>(0, 0x10_0040, 0xbbbb_bbbb); + assert_eq!(a.read::<4>(0, 0x10_0040).data, 0xaaaa_aaaa); + assert_eq!(b.read::<4>(0, 0x10_0040).data, 0xbbbb_bbbb); + assert_eq!(bank.read32(0x10_0040).data, 0x1234_5678, "memory is neither"); + assert_eq!(unsafe { *(space.window_base().add(0x10_0040) as *const u32) }, 0x1234_5678, + "the shared window, which other masters read, is memory"); + a.cache_op(C_IST | CACH_PD, 0x8000_0040, 0); + assert_eq!(a.read::<4>(0, 0x10_0040).data, 0x1234_5678); + assert_eq!(b.read::<4>(0, 0x10_0040).data, 0xbbbb_bbbb); + } + + /// Two lines of one page: the page stays until the last goes. The R12000 + /// zeroing a primary data tag at Index Writeback Invalidate ends a line + /// like an invalid tag store. + #[test] + fn a_page_goes_with_its_last_line() { + let (space, mut banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let space = Box::new(space); + let bank = Arc::new(banks.remove(0)); + let c = Box::new(R12000ShadowCache::from(bank.clone() as Arc)); + let v = space.add_cached_view(&[c.tcache_bitmap_ptr()]).unwrap(); + unsafe { + c.set_tcache_window(space.view_base(v)); + c.set_cached_view(&*space, v); + } + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + bank.write32(0x6000, 0x600d); + c.cache_op(C_IST | CACH_PD, 0x8000_6000, pd_dirty(0x6000)); + c.cache_op(C_IST | CACH_PD, 0x8000_6040, pd_dirty(0x6000)); + c.write::<4>(0, 0x6000, 0xbad); + c.cache_op(C_IST | CACH_PD, 0x8000_6000, 0); + assert_eq!(space.held_line_pages(v), vec![0x6000]); + assert_eq!(c.read::<4>(0, 0x6000).data, 0xbad, "the page stays the view's"); + c.cache_op(C_IWBINV | CACH_PD, 0x8000_6040, 0); + assert!(space.held_line_pages(v).is_empty()); + assert_eq!(c.read::<4>(0, 0x6000).data, 0x600d); + } + + /// Power-on drops every held page. A snapshot taken while lines hold + /// pages is refused at restore. + #[test] + fn power_on_drops_held_pages_and_a_snapshot_of_them_is_refused() { + let (c, space, _bank) = windowed(); + c.cache_op(C_IST | CACH_PD, 0x8000_0000, pd_dirty(0x7000)); + let saved = c.save_cache_state(); + assert!(c.load_cache_state(&saved).is_err()); + c.power_on(); + assert!(space.held_line_pages(0).is_empty()); + assert!(c.debug_probe("l2", 0, 0).contains("0 valid lines hold 0 pages")); + assert!(c.load_cache_state(&c.save_cache_state()).is_ok()); + } } diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 2b92dc849..55b2752e7 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -684,6 +684,15 @@ pub trait MipsCache: Send + Sync { /// `base` must be ppmem's window base and `bitmap` a live `u64` that /// outlives this cache. unsafe fn set_tcache_window(&self, _base: *mut u8) {} + + /// The cached view this cache reads memory through + /// (`PpMemSpace::add_cached_view`), as `sp` and its number, so valid + /// lines can hold their pages there. Set with the window, before the CPU + /// runs. No-op where the cache holds no lines. + /// + /// # Safety + /// `sp` must outlive the cache. + unsafe fn set_cached_view(&self, _sp: *const crate::ppmem::PpMemSpace, _view: usize) {} /// The ppmem window's `window_shift`, when it is not the 4GB window's. fn set_tcache_shift(&self, _shift: u32) {} diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index c4bb09ae1..6e316bed7 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -2927,6 +2927,14 @@ impl MipsExecutor { unsafe { self.cache.set_tcache_window(base) }; } + /// See `MipsCache::set_cached_view`. + /// + /// # Safety + /// `sp` must outlive the executor. + pub unsafe fn set_cached_view(&self, sp: *const crate::ppmem::PpMemSpace, view: usize) { + unsafe { self.cache.set_cached_view(sp, view) }; + } + /// tcache: the window's `ppmem::window_shift`, when it is not 4GB's. pub fn set_tcache_shift(&self, shift: u32) { self.cache.set_tcache_shift(shift); @@ -10580,6 +10588,14 @@ impl MipsCpu { unsafe { self.executor.lock().set_tcache_window(base) }; } + /// See `MipsExecutor::set_cached_view`. + /// + /// # Safety + /// `sp` must outlive the CPU. + pub unsafe fn set_cached_view(&self, sp: *const crate::ppmem::PpMemSpace, view: usize) { + unsafe { self.executor.lock().set_cached_view(sp, view) }; + } + /// tcache: see `MipsExecutor::set_tcache_shift`. pub fn set_tcache_shift(&self, shift: u32) { self.executor.lock().set_tcache_shift(shift); @@ -11305,6 +11321,11 @@ pub trait CpuDevice: Device + Resettable + Saveable + Send + Sync { /// # Safety /// Same contract as `CpuCache::set_tcache_window`. unsafe fn set_tcache_window(&self, base: *mut u8); + /// See `MipsCache::set_cached_view`. + /// + /// # Safety + /// `sp` must outlive the CPU. + unsafe fn set_cached_view(&self, sp: *const crate::ppmem::PpMemSpace, view: usize); /// tcache: see `MipsExecutor::set_tcache_shift`. fn set_tcache_shift(&self, shift: u32); /// tcache: pointer to the cache's inline mapped-region bitmap. @@ -11372,6 +11393,9 @@ impl CpuDevice for MipsCp unsafe fn set_tcache_window(&self, base: *mut u8) { unsafe { MipsCpu::set_tcache_window(self, base) } } + unsafe fn set_cached_view(&self, sp: *const crate::ppmem::PpMemSpace, view: usize) { + unsafe { MipsCpu::set_cached_view(self, sp, view) } + } fn tcache_bitmap_ptr(&self) -> *mut u64 { MipsCpu::tcache_bitmap_ptr(self) } #[cfg(feature = "jitv2")] unsafe fn set_tcache_gen_window(&self, gen_base: *mut AtomicU64) { diff --git a/src/ip30_machine.rs b/src/ip30_machine.rs index 3ce7ae15c..68d093c04 100644 --- a/src/ip30_machine.rs +++ b/src/ip30_machine.rs @@ -458,6 +458,7 @@ impl OctaneMachine { .map_err(|e| format!("ppmem: a CPU's cached view: {e}"))?; unsafe { c.set_tcache_window(sp.view_base(v)); + c.set_cached_view(sp, v); c.set_tcache_shift(crate::ppmem::window_shift(sp.window_size())); #[cfg(feature = "jitv2")] { diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index 53c269181..8386531e7 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -259,6 +259,7 @@ impl O2Machine { .map_err(|e| format!("ppmem: the CPU's cached view: {e}"))?; unsafe { cpu.set_tcache_window(sp.view_base(v)); + cpu.set_cached_view(sp, v); #[cfg(feature = "jitv2")] { use crate::ppmem::MappedMemory; diff --git a/src/machine.rs b/src/machine.rs index d5ed400b7..62566f107 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -815,6 +815,7 @@ impl Machine { .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()]) .expect("ppmem: the CPU's cached view"); unsafe { cpu.set_tcache_window(sp.view_base(v)) }; + unsafe { cpu.set_cached_view(sp, v) }; // Window writes bypass BusDevice, so the cache needs the gen // window to bump page counters itself — otherwise jitv2 keeps // running compiled code for pages the guest has modified. From 74bdac72b4a86c5e604b6a2260c3cd42f217daf2 Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:15:00 -0700 Subject: [PATCH 04/22] cpu: keep valid primary instruction lines in the CPU's cached view Upstream iris has no machine that needs this yet. The Fuel's and the Tezro's PROM 6.210 copies itself to memory from code it stages in primary instruction lines with Index Store Tag and Index Store Data (I) and runs through cached xkphys, and once memory is configured, memory at those addresses holds something else. Without the lines the PROM hangs on iris at "Copying PROM code to memory ..." (measured on the model). A valid primary instruction line now holds its page in the CPU's view like a data line, with each word written there as the instruction it predecodes. A primary instruction word is 36 bits of predecoded instruction (NEC VR10000 user's manual p. 288, established). Neither manual gives the layout, so it is inferred from the words PROM 6.210 stores, and only its eleven sampled classes decode. A word of any other class leaves memory's word in the page, where hardware would run it. Index Store Data (I) into a held line writes through. The PROM gets the predecoded form of other code from the hardware: it runs a secondary line it staged, which refills a primary line, and reads the words back with Index Load Data (I) to stage them itself (measured on the model). The model does not refill on a fetch, so Index Load Data (I) in a set a held secondary line maps to now answers with that line's words predecoded, as if it had been refilled, until Index Store Data (I) writes the set. A guest's own pattern in the set reads back as written (MIPS R10000 user's manual pp. 201-202). An instruction outside the eleven classes comes back in a form only this model reads, bits 35:32 set over the instruction, which takes predecode classes 0x78-0x7f. Nothing on a fetch, load or store, and nothing in the decode cache, changes. A page held only for code also answers cached loads with the code's bytes, where hardware would read memory. --- CHANGELOG.md | 1 + .../cpu/cache-lines-live-in-the-cpus-view.md | 16 ++ src/cpu/mips_cache_shadow.rs | 220 +++++++++++++++++- 3 files changed, 234 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f0b52600f..ae3b51fc0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -61,6 +61,7 @@ hashes are given where a change is easiest to understand by reading the commit. ### R10000-family cache lines as memory - A valid primary data or secondary cache line now holds its page in that CPU's own copy of ppmem's window, so a PROM that keeps its stack in cache lines reads and writes the lines with cached accesses while the uncached path, DMA and the other CPUs see memory. Index Store Data writes through to such a line. `l2 probe 0` lists the pages held. Nothing changes on a load, store or fetch. A valid R5000 primary data line does the same, for the O2's POST stack. +- Primary instruction lines hold their pages too, each word written there as the instruction it predecodes, for the eleven predecode classes the model knows. Index Load Data (I) where a held secondary line would refill the set returns that line's words, predecoded, until Index Store Data (I) writes the set. Only the Fuel's and Tezro's PROMs use them. - jitv2 does not compile a page any CPU's cache holds lines in: that CPU runs it from its own view in the interpreter, and every other page stays compiled. ### R10000 family fixes, R12000 and R14000 diff --git a/clanker/cpu/cache-lines-live-in-the-cpus-view.md b/clanker/cpu/cache-lines-live-in-the-cpus-view.md index 1b883675d..d4a157db0 100644 --- a/clanker/cpu/cache-lines-live-in-the-cpus-view.md +++ b/clanker/cpu/cache-lines-live-in-the-cpus-view.md @@ -81,6 +81,22 @@ offset until the release. ppmem's counters are the bus's for memory; a machine w maps a device over memory, as the IP35 boards map their reset PROM, must mark that device's counter as well. +## Primary instruction lines + +Only the Fuel's and the Tezro's PROMs use these. A valid primary instruction line +writes each 36-bit predecoded word (p. 288) into its page as the instruction it +predecodes, through an eleven-class table inferred from PROM 6.210's words +(`l1i_decode`). A word of another class leaves memory's word, where hardware would +run it. A page held only for code also answers cached loads with code. + +The model never refills on a fetch. The Tezro's PROM runs a staged secondary line +and reads the primary words back with Index Load Data (I) to stage them itself, so +Index Load Data (I) in a set a held secondary line maps to answers with that line's +words predecoded (`refilled_word`), until Index Store Data (I) writes the set. When +two held lines map one set, the later wins. An instruction no class holds comes back +in the model's own form (bits 35:32 set, which takes classes 0x78-0x7f), which +hardware would not produce. Index Load Tag (I) reads the stored tag. + ## Seeing it `l2 probe 0` in the monitor prints how many lines hold pages and which. Every diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 53486644d..9b8e242f9 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -109,7 +109,65 @@ const MRU_BIT: u64 = 1 << 63; const PD_STATE_SHIFT: u32 = 6; const SD_STATE_SHIFT: u32 = 10; +/// The primary instruction state, TagLo[6] (NEC U10278EJ3V0UM p. 285). +const PI_VALID: u64 = 1 << 6; + +/// A primary instruction word is the instruction predecoded, 36 bits (NEC +/// U10278EJ3V0UM p. 288). Neither manual gives the layout. This one is +/// inferred from the words PROM 6.210 stores, `p` the word and `f` the +/// instruction: p[25:20] = f[5:0], p[19:15] = f[10:6], p[14:10] = f[25:21], +/// p[28:26] = f[28:26], p[35:29] a class that also gives f[31:29], and p[4:0] +/// and p[9:5] the rt and rd fields in the order the class gives. Only the +/// sampled classes decode, and every row is inferred. `addiu $4,$0,2` +/// (0x24040002) is 0x3_4420_0004. +/// +/// Rows: class, f[31:29], whether p[4:0] is rt, the f[28:26] values as a bit +/// mask, and the function fields of the SPECIAL members. +const L1I_CLASSES: &[(u8, u32, bool, u8, &[u8])] = &[ + (0x00, 0, false, 1 << 0, &[0x00]), // nop + (0x08, 0, false, 1 << 0, &[0x38, 0x3c]), // dsll, dsll32 + (0x18, 0, false, 1 << 0, &[0x25]), // or + (0x1a, 1, true, 0b1010_0011, &[]), // addi, addiu, ori, lui + (0x1e, 3, true, 1 << 1, &[]), // daddiu + (0x35, 5, false, 1 << 7, &[]), // cache + (0x36, 6, true, 1 << 7, &[]), // ld + (0x3f, 2, false, 1 << 0, &[]), // mtc0 + (0x40, 0, false, 1 << 3, &[]), // jal + (0x41, 0, false, 0b101, &[0x08]), // jr, j + (0x71, 0, false, 1 << 1, &[]), // REGIMM +]; + +/// The model's own form, bits 35:32 set over an instruction no class holds. +/// Hardware would never produce it. It takes classes 0x78-0x7f, so a real +/// word of those classes would be read as raw. +const L1I_RAW: u64 = 0xf << 32; + +/// The instruction a predecoded word encodes, or the class that is refused. +fn l1i_decode(p: u64) -> Result { + if p >> 32 == L1I_RAW >> 32 { + return Ok(p as u32); + } + let (class, op, funct) = ((p >> 29) as u8 & 0x7f, (p >> 26) as u32 & 7, (p >> 20) as u8 & 0x3f); + let &(_, hi3, rt_low, ops, functs) = L1I_CLASSES.iter().find(|c| c.0 == class).ok_or(class)?; + if ops >> op & 1 == 0 || (op == 0 && !functs.is_empty() && !functs.contains(&funct)) || (class == 0 && p != 0) { + return Err(class); + } + let (low, high) = ((p & 0x1f) as u32, (p >> 5) as u32 & 0x1f); + let (rt, rd) = if rt_low { (low, high) } else { (high, low) }; + Ok(hi3 << 29 | op << 26 | ((p >> 10) as u32 & 0x1f) << 21 | rt << 16 | rd << 11 | ((p >> 15) as u32 & 0x1f) << 6 | funct as u32) +} +/// The predecoded word for `f`, or the raw form where no class holds it. +fn l1i_encode(f: u32) -> u64 { + let w = f as u64; + let (rt, rd) = (w >> 16 & 0x1f, w >> 11 & 0x1f); + L1I_CLASSES.iter().find_map(|&(class, _, rt_low, _, _)| { + let (low, high) = if rt_low { (rt, rd) } else { (rd, rt) }; + let p = (class as u64) << 29 | (w >> 26 & 7) << 26 | (w & 0x3f) << 20 | (w >> 6 & 0x1f) << 15 + | (w >> 21 & 0x1f) << 10 | high << 5 | low; + (l1i_decode(p) == Ok(f)).then_some(p) + }).unwrap_or(L1I_RAW | w) +} /// R10000 only: ways per set, as CACHE index operations address them. Bit 0 /// of the index selects the way; the set starts above the line offset. The @@ -184,6 +242,10 @@ pub struct ShadowCache< /// valid lines hold their pages. Null without one. view_space: UnsafeCell<*const PpMemSpace>, view: UnsafeCell, + /// For each primary instruction set, the 64-byte half of a held + /// secondary line that a refill there would load (`refilled_word`), or + /// `u64::MAX`. + refill: UnsafeCell>, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -243,6 +305,7 @@ impl< pd_tag_hook: UnsafeCell::new(None), view_space: UnsafeCell::new(std::ptr::null()), view: UnsafeCell::new(0), + refill: UnsafeCell::new(vec![u64::MAX; Self::PI_SETS].into_boxed_slice()), } } @@ -373,10 +436,19 @@ impl< if s.held.get(self.tag_slot(sel, virt_addr)) == Some(&true) { self.store_into_held_line(sel, virt_addr, phys_addr); } + if sel == CACH_PI { + // The set now holds what the guest wrote, not a refill. + if let Some(r) = unsafe { &mut *self.refill.get() }.get_mut((virt_addr as usize / IC_LINE.max(1)) % Self::PI_SETS.max(1)) { + *r = u64::MAX; + } + } } // ---- valid lines hold pages in the CPU's cached view ---- + /// Primary instruction sets, as Index Load Data (I) addresses them. + const PI_SETS: usize = if IC_LINE == 0 { 0 } else { IC_SIZE / WAYS / IC_LINE }; + fn view_space(&self) -> Option<&PpMemSpace> { unsafe { (*self.view_space.get()).as_ref() } } @@ -397,6 +469,9 @@ impl< CACH_PD if DC_LINE != 0 && (tag >> PD_STATE_SHIFT) & 3 != 0 => { Some((pa_hi | ((tag >> 8) & 0xff_ffff) << 12 | (va & 0xfff & !(DC_LINE as u64 - 1)), DC_LINE)) } + CACH_PI if IC_LINE != 0 && tag & PI_VALID != 0 => { + Some((pa_hi | ((tag >> 8) & 0xff_ffff) << 12 | (va & 0xfff & !(IC_LINE as u64 - 1)), IC_LINE)) + } CACH_SD if L2_LINE != 0 && (tag >> SD_STATE_SHIFT) & 3 != 0 => { Some((pa_hi | ((tag >> 14) & 0x3_ffff) << 18 | (va & 0x3_ffff & !(L2_LINE as u64 - 1)), L2_LINE)) } @@ -458,6 +533,7 @@ impl< let word = s.data[self.data_slot(sel, s.data.len(), first + (unit * k) as u64 + way)]; let at = pa + (unit * k) as u64; match sel { + CACH_PI => self.put_instruction(at, word), CACH_PD => unsafe { *(base.add(at as usize) as *mut u32) = word as u32 }, // TagHi is the doubleword's high word and TagLo its low word // (MIPS R10000 UM p. 201; NEC VR10000 UM p. 289, Fig. 13-33). @@ -471,6 +547,9 @@ impl< _ => unsafe { *(base.add(at as usize) as *mut u64) = word }, } } + if sel == CACH_SD { + self.set_refill(s.tags[idx], pa, len, true); + } } /// The line in slot `idx` of `sel` leaves, unwritten, as Index Store Tag @@ -479,6 +558,9 @@ impl< fn line_leaves(&self, sp: &PpMemSpace, sel: u32, idx: usize, (pa, len): (u64, usize)) { let s = self.shadow(sel); s.held[idx] = false; + if sel == CACH_SD { + self.set_refill(s.tags[idx], pa, len, false); + } let gone = sp.release_line_page(self.view_no(), pa); if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> line {pa:#x} leaves (page released: {gone})"); @@ -494,24 +576,78 @@ impl< let at = pa + (va & (len as u64 - 1) & !(unit - 1)); let base = unsafe { *self.tc_base.get() }; match sel { + CACH_PI => self.put_instruction(at, value & 0xf_ffff_ffff), CACH_PD => unsafe { *(base.add(at as usize) as *mut u32) = value as u32 }, _ => unsafe { *(base.add(at as usize) as *mut u64) = value; *(base.add((at ^ 8) as usize) as *mut u64) = 0; }, } + if sel == CACH_SD { + self.set_refill(self.shadow(sel).tags[idx], pa, len, true); + } + } + + /// A primary instruction word goes into its line's page as the + /// instruction it predecodes, so a fetch runs it. A word of a class the + /// model does not know leaves the page as it was, where hardware would + /// run it. + fn put_instruction(&self, at: u64, word: u64) { + match l1i_decode(word) { + Ok(f) => unsafe { *((*self.tc_base.get()).add(at as usize) as *mut u32) = f }, + Err(class) => crate::dlog!(LogModule::L2c, + "shadow: primary instruction word {word:#011x} at {at:#x}: predecode class {class:#04x} unknown, memory's word runs"), + } + } + + /// A held secondary line `[pa, pa + len)` with tag `tag` is what a primary + /// instruction refill in each set it maps to would load, or stops being. + /// The set is the line's virtual index, TagLo[8:7] (VA[13:12], NEC + /// U10278EJ3V0UM p. 287), over PA[11:6]. Two lines for one set: the + /// later wins. + fn set_refill(&self, tag: u64, pa: u64, len: usize, held: bool) { + let refill = unsafe { &mut *self.refill.get() }; + if !held { + // By address: the tag may already name another line. + refill.iter_mut().filter(|r| (pa..pa + len as u64).contains(*r)).for_each(|r| *r = u64::MAX); + return; + } + for half in (pa..pa + len as u64).step_by(IC_LINE.max(1)) { + if let Some(r) = refill.get_mut(((tag >> 7 & 3) << 12 | (half & 0xfff)) as usize / IC_LINE) { + *r = half; + } + } + } + + /// Index Load Data (I) in a set a held secondary line maps to, which + /// Index Store Data (I) has not written since and where no held primary + /// line is: the word a refill from that + /// line would have left, predecoded. The model never refills on a fetch, + /// so this assumes the line was refilled, whichever way the operand + /// names. The Tezro's PROM runs a staged secondary line and reads the + /// primary words back to stage them itself. Index Load Tag (I) still + /// reads the stored tag. + #[cold] + #[inline(never)] + fn refilled_word(&self, va: u64) -> Option { + let half = *unsafe { &*self.refill.get() }.get((va as usize / IC_LINE.max(1)) % Self::PI_SETS.max(1))?; + if half == u64::MAX || self.shadow(CACH_PI).held[self.tag_slot(CACH_PI, va)] { + return None; + } + let at = half + (va & (IC_LINE as u64 - 1) & !3); + Some(l1i_encode(unsafe { *((*self.tc_base.get()).add(at as usize) as *const u32) })) } /// How many lines hold pages. fn held_count(&self) -> usize { - [CACH_PD, CACH_SD].iter().map(|&sel| self.shadow(sel).held.iter().filter(|&&h| h).count()).sum() + [CACH_PI, CACH_PD, CACH_SD].iter().map(|&sel| self.shadow(sel).held.iter().filter(|&&h| h).count()).sum() } /// Every held line goes, with no writeback: power-on, or a snapshot /// restore. fn drop_held_lines(&self) { let Some(sp) = self.view_space() else { return }; - for sel in [CACH_PD, CACH_SD] { + for sel in [CACH_PI, CACH_PD, CACH_SD] { for i in 0..self.shadow(sel).held.len() { if self.shadow(sel).held[i] { let line = self.line_of(sel, self.shadow(sel).tags[i], self.slot_va(sel, i)).expect("a held line is valid"); @@ -821,7 +957,7 @@ impl< s.mru[set] = phys_addr & MRU_BIT != 0; let old = s.tags[idx]; s.tags[idx] = phys_addr & mask & !MRU_BIT; - if Self::R10K && matches!(sel, CACH_PD | CACH_SD) && old != s.tags[idx] { + if Self::R10K && matches!(sel, CACH_PI | CACH_PD | CACH_SD) && old != s.tags[idx] { self.tag_changes(sel, idx, virt_addr, old, s.tags[idx]); } } @@ -858,6 +994,11 @@ impl< crate::dlog!(LogModule::L2c, "shadow: -> ILD slot={slot} data={:#018x} ecc={:#x}", s.data[slot], s.ecc[slot]); } + if sel == CACH_PI { + if let Some(p) = self.refilled_word(virt_addr) { + return p; + } + } s.data[slot] } } @@ -1617,4 +1758,77 @@ mod tests { assert!(c.debug_probe("l2", 0, 0).contains("0 valid lines hold 0 pages")); assert!(c.load_cache_state(&c.save_cache_state()).is_ok()); } + + // ---- primary instruction lines ---- + + /// Every class of the table, encoded and decoded back, one field moved + /// at a time for the example word, and what the table refuses. + #[test] + fn predecoded_words_round_trip() { + let p = 0x3_4420_0004; // addiu $4,$0,2 + for (q, f) in [(p, 0x2404_0002), (p | 0x1f << 5, 0x2404_f802), (p | 0x1f << 10, 0x27e4_0002), (p | 0x1f << 15, 0x2404_07c2)] { + assert_eq!(l1i_decode(q), Ok(f), "{q:#x}"); + } + for (class, f) in [(0x08, 0x0009_20f8), (0x08, 0x0009_20fc), (0x18, 0x0109_5025), (0x1a, 0x2084_0010), + (0x1a, 0x3484_0010), (0x1a, 0x3c04_8000), (0x1e, 0x6484_0010), (0x35, 0xbd04_0040), + (0x36, 0xdd04_0008), (0x3f, 0x4084_6000), (0x40, 0x0c00_1234), (0x41, 0x03e0_0008), + (0x41, 0x0800_1234), (0x71, 0x0481_0004), (0, 0)] { + assert_eq!((l1i_encode(f) >> 29, l1i_decode(l1i_encode(f))), (class, Ok(f)), "{f:#x}"); + } + for f in [0x0009_20c0, 0x2884_0010, 0x4422_3000] { // sll, slti, mtc1: no class + assert_eq!((l1i_encode(f), l1i_decode(L1I_RAW | f as u64)), (L1I_RAW | f as u64, Ok(f))); + } + assert_eq!((l1i_decode(1), l1i_decode(0x2a << 29)), (Err(0), Err(0x2a))); + } + + /// TagLo of a valid primary instruction line for `pa`'s page. + fn pi_valid(pa: u64) -> u64 { (pa >> 12) << 8 | PI_VALID } + + /// A valid primary instruction line puts the instructions its words + /// predecode into the page, leaving memory's word for one it cannot + /// decode, and Index Store Data (I) writes through. + #[test] + fn a_primary_instruction_line_puts_its_instructions_in_the_page() { + let (c, space, bank) = windowed(); + bank.write32(0x9008, 0x1111_1111); + let va = 0x8000_9001u64; // way 1 + c.cache_op(C_R10K_ISD | CACH_PI, va, 0x3_4420_0004); + c.cache_op(C_R10K_ISD | CACH_PI, va + 8, 0x2a << 29); + c.cache_op(C_IST | CACH_PI, va, pi_valid(0x9000)); + assert_eq!(space.held_line_pages(0), vec![0x9000]); + assert_eq!([0x9000, 0x9008].map(|a| c.read::<4>(0, a).data), [0x2404_0002, 0x1111_1111]); + c.cache_op(C_R10K_ISD | CACH_PI, va + 4, l1i_encode(0x03e0_0008)); + assert_eq!(c.read::<4>(0, 0x9004).data, 0x03e0_0008); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, va, 0), 0x3_4420_0004, "Index Load Data (I) reads the array"); + assert_eq!(bank.read32(0x9000).data, 0, "memory never saw it"); + c.cache_op(C_IST | CACH_PI, va, 0); + assert!(space.held_line_pages(0).is_empty()); + } + + /// A held secondary line answers Index Load Data (I) in the primary sets + /// it maps to, as the predecoded words a refill would leave, until Index + /// Store Data (I) writes the set or the line goes. A held primary line + /// answers for itself. + #[test] + fn index_load_data_i_reads_a_refill_until_the_guest_writes_the_set() { + let (c, _space, _bank) = windowed(); + c.cache_op(C_IST | CACH_SD, 0x8000_4f00, sd_tag(0x4f00, 2) | 1 << 7); // VA[13:12] = 1 + c.cache_op(C_R10K_ISD | CACH_SD, 0x8000_4f10, 0x0140_0008); // jr $10 at 0x4f10 + let set = |off: u64| 0x8000_1f00 | off; + for va in [set(0x10), set(0x10) | 1] { + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, va, 0), l1i_encode(0x0140_0008), "either way"); + } + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, 0x8000_0f10, 0), 0, "another set: the array"); + c.cache_op(C_R10K_ISD | CACH_PI, set(0x10), 0x5_5555_5555); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, set(0x10), 0), 0x5_5555_5555, "the guest's pattern"); + c.cache_op(C_R10K_ISD | CACH_SD, 0x8000_4f10, 0x0140_0008); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, set(0x10), 0), l1i_encode(0x0140_0008), "restaged: a refill again"); + c.cache_op(C_R10K_ISD | CACH_PI, set(0x10), 0x3_4420_0004); + c.cache_op(C_IST | CACH_PI, set(0x10), pi_valid(0x1_4000)); + c.cache_op(C_R10K_ISD | CACH_SD, 0x8000_4f10, 0x0140_0008); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, set(0x10), 0), 0x3_4420_0004, "a held primary line answers for itself"); + c.cache_op(C_IST | CACH_PI, set(0x10), 0); + c.cache_op(C_IST | CACH_SD, 0x8000_4f00, 0); + assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, set(0x10), 0), 0x3_4420_0004, "gone with the line: the array"); + } } From fd4e8f950a3fe3441162dba3590e7b5c2b08e43b Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:20:00 -0700 Subject: [PATCH 05/22] ip32: retire the POST stack page The O2 kept POST's stack in bank 0 and, while a primary data tag for page 0 stood, pointed the decode table at a page of its own for the uncached path, switched by a hook on every Index Store Tag (D). The CPU's cached view holds those lines now, with the bank, the decode table and the rendering engine on memory meanwhile, so the side page, its hook in the shadow cache, the slot-1 mapping it needed at reset and MipsCache::set_pd_tag_store_hook go. A snapshot taken during POST by an older iris carries the side page but no cache lines to rebuild the stack from, and is refused. --- CHANGELOG.md | 1 + clanker/INDEX.md | 2 +- ...2-prom-stack-is-a-cached-page-at-phys-0.md | 11 +- src/cpu/mips_cache_shadow.rs | 27 --- src/cpu/mips_cache_v2.rs | 6 - src/dev/crime_mem.rs | 208 +----------------- src/ip32.rs | 15 +- src/ip32/tests.rs | 12 + src/ip32_machine.rs | 5 +- 9 files changed, 33 insertions(+), 254 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ae3b51fc0..22ca48c68 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -63,6 +63,7 @@ hashes are given where a change is easiest to understand by reading the commit. - A valid primary data or secondary cache line now holds its page in that CPU's own copy of ppmem's window, so a PROM that keeps its stack in cache lines reads and writes the lines with cached accesses while the uncached path, DMA and the other CPUs see memory. Index Store Data writes through to such a line. `l2 probe 0` lists the pages held. Nothing changes on a load, store or fetch. A valid R5000 primary data line does the same, for the O2's POST stack. - Primary instruction lines hold their pages too, each word written there as the instruction it predecodes, for the eleven predecode classes the model knows. Index Load Data (I) where a held secondary line would refill the set returns that line's words, predecoded, until Index Store Data (I) writes the set. Only the Fuel's and Tezro's PROMs use them. - jitv2 does not compile a page any CPU's cache holds lines in: that CPU runs it from its own view in the interpreter, and every other page stays compiled. +- The O2 keeps POST's stack in those lines. CRIME's side page for the uncached path and the tag-store hook are gone, and an O2 snapshot an older iris took during POST is refused. ### R10000 family fixes, R12000 and R14000 diff --git a/clanker/INDEX.md b/clanker/INDEX.md index 0c760c3cb..f719a0931 100644 --- a/clanker/INDEX.md +++ b/clanker/INDEX.md @@ -129,7 +129,7 @@ Non-graphics devices: MC/MEMCFG and RAM banks, HPC3, VDMA, EEPROMs and NVRAM, PS | [aic7880-patchsim.py](devices/aic7880-patchsim.py) | howto | Reproduces the Linux aic7xxx firmware patch selection and critical sections for a 7880 | | [o2-no-ecc-alias-is-memory.md](devices/o2-no-ecc-alias-is-memory.md) | rule | O2: physical 0x00_8000_0000 is CRIME's no-ECC memory alias, folded onto RAM by the CPU (netstat/Netscape hang, /dev/mem all ones) | | [o2-cache-handover-full-to-shadow.md](devices/o2-cache-handover-full-to-shadow.md) | history | O2 cache as it was until 2026-10-30: the full model through POST, the shadow model after, switched by PC range and a tag-store trigger | -| [o2-prom-stack-is-a-cached-page-at-phys-0.md](devices/o2-prom-stack-is-a-cached-page-at-phys-0.md) | design | O2 PROM stack: one tag-installed page at physical 0 that the memory test overwrites through KSEG1; a 4 KB page in the bus decode answers for it uncached, so the shadow cache runs from reset | +| [o2-prom-stack-is-a-cached-page-at-phys-0.md](devices/o2-prom-stack-is-a-cached-page-at-phys-0.md) | design | O2 PROM stack: one tag-installed page at physical 0 that the memory test overwrites through KSEG1. Retired: the CPU's cached view holds it now | | [memcfg-limit-is-not-the-bank-size.md](devices/memcfg-limit-is-not-the-bank-size.md) | rule | MEMCFG: `limit` is the slot, `addr_mask+1` is the mirror period — they differ | | [nvram-persistence.md](devices/nvram-persistence.md) | design | Battery-backed state on Stop | | [ps2-aux-mux-probe-and-status-bit.md](devices/ps2-aux-mux-probe-and-status-bit.md) | rule | PS/2 mouse not detected under Linux 3.2 guests (and keyboard sometimes flaky too) | diff --git a/clanker/devices/o2-prom-stack-is-a-cached-page-at-phys-0.md b/clanker/devices/o2-prom-stack-is-a-cached-page-at-phys-0.md index 5356cce38..39f6e935b 100644 --- a/clanker/devices/o2-prom-stack-is-a-cached-page-at-phys-0.md +++ b/clanker/devices/o2-prom-stack-is-a-cached-page-at-phys-0.md @@ -1,10 +1,11 @@ # O2 PROM stack is one cached page at physical 0, and RAM behind it is a different thing -Status: landed 2026-10-30. Both O2 CPUs run the shadow cache from reset; the handover -([o2-cache-handover-full-to-shadow.md](o2-cache-handover-full-to-shadow.md)) is gone. Code: -`O2Memory::stack_tag_store` in `src/dev/crime_mem.rs`, the tag-store hook in -`src/cpu/mips_cache_shadow.rs`. Sections below are the measurements that chose the design; the -prototype results used earlier binaries. +Status: retired 2026-11-08. The stack now lives in the CPU's own cached view of ppmem's window: +a valid primary data or secondary line holds its page there (`ShadowCache::line_enters` in +`src/cpu/mips_cache_shadow.rs`, `PpMemSpace::add_cached_view`), and the uncached path, the decode +table and the rendering engine keep seeing memory. `O2Memory::stack_tag_store`, its side page and +the tag-store hook are gone. The sections below are the measurements that chose the old design; +what they say about the PROM still holds. ## Where the stack is diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 9b8e242f9..c49045c89 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -236,8 +236,6 @@ pub struct ShadowCache< /// Decoded instructions, by physical page. Never touches the shadow /// arrays above — see `decode_cache`. decode: UnsafeCell, - /// See [`MipsCache::set_pd_tag_store_hook`]. Consulted by `cache_op` only. - pd_tag_hook: UnsafeCell>>, /// The cached view `tc_base` is, and its number in `PpMemSpace`: where /// valid lines hold their pages. Null without one. view_space: UnsafeCell<*const PpMemSpace>, @@ -302,7 +300,6 @@ impl< #[cfg(feature = "jitv2")] tc_gen: UnsafeCell::new(std::ptr::null_mut()), decode: UnsafeCell::new(crate::cpu::decode_cache::DecodeCache::new()), - pd_tag_hook: UnsafeCell::new(None), view_space: UnsafeCell::new(std::ptr::null()), view: UnsafeCell::new(0), refill: UnsafeCell::new(vec![u64::MAX; Self::PI_SETS].into_boxed_slice()), @@ -732,10 +729,6 @@ impl< fn tcache_base_ptr(&self) -> *mut u8 { unsafe { *self.tc_base.get() } } - fn set_pd_tag_store_hook(&self, hook: Box) { - unsafe { *self.pd_tag_hook.get() = Some(hook) }; - } - unsafe fn set_cached_view(&self, sp: *const PpMemSpace, view: usize) { unsafe { *self.view_space.get() = sp; @@ -895,12 +888,6 @@ impl< let sel = cache_op & 3; let op = cache_op & 0x1C; - if op == C_IST && sel == CACH_PD { - if let Some(hook) = unsafe { &*self.pd_tag_hook.get() } { - hook(phys_addr as u32); - } - } - // Tag operations under `op`, the whole data-array walk only under // `hit` as well — see `l2_op_log`. if l2_op_log(if matches!(op, C_IST | C_ILT) { CACHE_LOG_OP } else { CACHE_LOG_HIT }) { @@ -1448,20 +1435,6 @@ mod tests { (::PRID, ::FIR, ::MIPS4, ::TLB_ENTRIES, ::VA_BITS, ::R10K_CACHE_OPS)); } - /// The tag store hook sees primary data tag stores and nothing else. - #[test] - fn the_tag_store_hook_sees_only_primary_data_tag_stores() { - let c = cache(); - let seen = Arc::new(std::sync::Mutex::new(Vec::new())); - let s = seen.clone(); - c.set_pd_tag_store_hook(Box::new(move |tag| s.lock().unwrap().push(tag))); - c.cache_op(C_IST | CACH_PI, 0x8000_0000, 0x11); - c.cache_op(C_IST | CACH_SD, 0x8000_0000, 0x22); - c.cache_op(C_ILT | CACH_PD, 0x8000_0000, 0x33); - c.cache_op(C_IST | CACH_PD, 0x8000_0000, 0xc8); - assert_eq!(*seen.lock().unwrap(), vec![0xc8]); - } - /// Index_Store_Tag then Index_Load_Tag returns exactly what the full model /// returns, for random TagLo words at random indexes: the shadow must not /// be observably different to a PROM, IDE or IRIX reading tags. Both diff --git a/src/cpu/mips_cache_v2.rs b/src/cpu/mips_cache_v2.rs index 55b2752e7..5619a9cb3 100644 --- a/src/cpu/mips_cache_v2.rs +++ b/src/cpu/mips_cache_v2.rs @@ -669,12 +669,6 @@ pub trait MipsCache: Send + Sync { /// Share the L1-I hit/fetch counters with the status display. No-op where absent. fn set_l1i_counters(&mut self, _hit: Arc, _fetch: Arc) {} - /// Have `hook` called with TagLo on every Index Store Tag to the primary - /// data cache: a machine whose firmware builds memory out of cache tags - /// (the O2's POST stack, see `O2Memory::stack_tag_store`). No-op where - /// absent. Set once, before the CPU runs. - fn set_pd_tag_store_hook(&self, _hook: Box) {} - /// tcache: point the cache at ppmem's window and mapped-region bitmap, so /// cacheable RAM accesses read/write RAM directly instead of copying line /// data (docs/tcache-design.md). Default no-op — a cache with no diff --git a/src/dev/crime_mem.rs b/src/dev/crime_mem.rs index 4b362456f..f8501c5cd 100644 --- a/src/dev/crime_mem.rs +++ b/src/dev/crime_mem.rs @@ -49,11 +49,10 @@ //! ([`RamWindow`]). Every other master reaches memory through the bus. //! //! POST keeps its stack in cache lines tagged for physical page 0 while it -//! tests the same page uncached, so on the machine the two are different bytes. -//! The CPU's cached path and the window always mean the bank, so that is where -//! the stack lives here too, and for as long as those tags stand the decode -//! table points that page at a page of its own for the uncached path. See -//! [`O2Memory::stack_tag_store`]. +//! tests the same page uncached, so on the machine the two are different +//! bytes. The lines live in the CPU's cached view of the window +//! (`PpMemSpace::add_cached_view`), and the bank, the decode table and every +//! other master go on seeing memory. //! //! `O2Memory` is CRIME's bank logic: the sizes fitted, the controls, the //! decode they make, and the remap. Its decode is atomics: the firmware @@ -84,13 +83,6 @@ const SLOTS: usize = (RAM_WINDOW_SIZE / SLOT) as usize; /// A 32 MB slot in the decode table's 4 KB entries. const TABLE_PER_SLOT: u64 = (SLOT >> SLOT_SHIFT) as u64; -/// The one page of physical memory POST keeps its stack in. -const STACK_PAGE: u32 = 4096; -/// TagLo as R10000 and R5000 share it: the primary state, 0 for invalid. -const TAG_STATE: u32 = 0xc0; -/// TagLo's physical page number. -const TAG_PAGE_SHIFT: u32 = 8; - /// The kits SGI sold, in MB per pair. pub const KITS_MB: [u32; 4] = [32, 64, 128, 256]; @@ -179,12 +171,6 @@ pub struct O2Memory { /// What [`RamWindow`]s need to know of the decode, published as it /// changes. shared: Arc, - /// Page 0 as the uncached path sees it while POST's stack is in the bank. - /// It wraps at 4 KB, so the same device answers at every alias. Boxed: the - /// decode table points at it and `O2Memory` is built by value. - stack_page: Box, - /// Whether the decode table is pointing at [`Self::stack_page`]. - stack_live: AtomicBool, } impl O2Memory { @@ -241,8 +227,6 @@ impl O2Memory { } let phys = Physical::new(banks); phys.set_default(Arc::new(NoMemory), "no memory"); - let stack_page = Box::new(RamBank::new(1)); - stack_page.set_addr_mask(STACK_PAGE - 1); let m = Self { phys, size, @@ -253,8 +237,6 @@ impl O2Memory { decode_gen: AtomicU64::new(0), rebuilding: std::sync::Mutex::new(()), shared: Default::default(), - stack_page, - stack_live: AtomicBool::new(false), }; // At reset every control is zero: all eight banks at the bottom of // the window, 32 MB each, so bank 0 answers there. @@ -356,106 +338,10 @@ impl O2Memory { let _ = sp.map_range(i, src, at, SLOT as u64); } } - if self.stack_live.load(Ordering::Relaxed) { - self.place_stack_page(slots, mapped); - } self.shared.decode_gen.store(self.decode_gen.load(Ordering::Acquire), Release); self.shared.mapped.store(mapped, Release); } - /// The decode table's entry for bank 0's offset 0 goes to the stack page, - /// at every place that offset answers: a 32 MB bank under a 128 MB setting - /// repeats, and sizing relies on it. - fn place_stack_page(&self, slots: &[Slot; SLOTS], mapped: u32) { - let phys = &*self.phys; - let page: *const RamBank = &*self.stack_page; - let page = page as crate::ppmem::devmap::Entry; - let size = self.size[0] as u64; - for (s, slot) in slots.iter().enumerate() { - let Slot::Ram { bank: 0, chunk } = *slot else { continue }; - if (chunk as u64 * SLOT as u64) & (size - 1) != 0 { - continue; - } - let at = s as u64 * SLOT as u64; - phys.fill_slots((RAM_BASE as u64 + at) >> SLOT_SHIFT, 1, page); - if at < LOW_ALIAS_SIZE as u64 { - phys.fill_slots(at >> SLOT_SHIFT, 1, page); - } - } - // ppmem's bitmap wants a whole 64 MB region mapped. At reset only the - // first 32 MB slot is, so the cached path would take the bus and find - // the stack page, then find the bank once sizing filled the region. - // Show bank 0 again in the next slot, for the CPU only. - if mapped & 2 == 0 { - if let (Some(sp), Some(i)) = (phys.ppmem_space(), self.index[0]) { - let _ = sp.map_range(i, 0, SLOT as u64, SLOT as u64); - } - } - } - - /// The guest stored `tag` in a primary data cache tag. - /// - /// POST sweeps a page's worth of tags to dirty for physical page 0 and - /// fills them, with nothing read from memory, then writes the same page - /// uncached to test it, and sweeps every tag invalid without a writeback - /// once it has copied what it needs and moved its stack to KSEG1. So a - /// valid tag for page 0 starts the stack and the next invalid tag ends it. - /// IRIX reboots by jumping into the PROM, so this catches a reboot as it - /// does a power-on. Kernels and `sash` store invalid tags only, which is - /// nothing to do here. - pub fn stack_tag_store(&self, tag: u32) { - if tag & TAG_STATE == 0 { - if self.stack_live.load(Ordering::Relaxed) { - self.set_stack_page(false); - } - } else if tag >> TAG_PAGE_SHIFT == 0 && !self.stack_live.load(Ordering::Relaxed) { - self.set_stack_page(true); - } - } - - /// Start the stack (the uncached path gets a copy of the page, the bank - /// goes on holding what the cached path sees) or end it (what was written - /// uncached is what memory holds, as it is when the cache lines go). - pub fn set_stack_page(&self, on: bool) { - let _one = self.rebuilding.lock().unwrap(); - if self.stack_live.load(Ordering::Relaxed) == on { - return; - } - let bank = &self.phys.banks()[self.index[0].expect("bank 0 is fitted")]; - for o in (0..STACK_PAGE as u64).step_by(8) { - if on { - self.stack_page.write64(o, bank.read64(o).data); - } else { - bank.write64(o, self.stack_page.read64(o).data); - } - } - self.stack_live.store(on, Ordering::Relaxed); - self.rebuild_locked(); - } - - /// The stack page's dwords while the stack is live, for a snapshot: the - /// bank is saved with memory and this is what the uncached path sees. - pub fn stack_page_words(&self) -> Option> { - self.stack_live.load(Ordering::Relaxed).then(|| { - (0..STACK_PAGE as u64).step_by(8).map(|o| self.stack_page.read64(o).data).collect() - }) - } - - /// Put back what [`Self::stack_page_words`] gave. The stack is not live - /// beforehand (a machine being restored has been powered on). - pub fn restore_stack_page(&self, words: &[u64]) -> Result<(), String> { - if words.len() * 8 != STACK_PAGE as usize { - return Err(format!("saved stack page is {} dwords", words.len())); - } - let _one = self.rebuilding.lock().unwrap(); - for (k, &w) in words.iter().enumerate() { - self.stack_page.write64(k as u64 * 8, w); - } - self.stack_live.store(true, Ordering::Relaxed); - self.rebuild_locked(); - Ok(()) - } - /// A view of memory through the window, for a master on its own thread /// (the rendering engine). pub fn window(&self) -> RamWindow { @@ -1118,92 +1004,6 @@ mod tests { assert_eq!(w, 0xbeef, "the window agrees"); } - const STACK_TAG: u32 = 0xc8; // dirty, page 0: what POST stores on an R10000 - - /// What each path sees of page 0 once the stack is live, and that memory - /// ends up holding what was written uncached. - #[test] - fn the_stack_and_the_uncached_page_are_different_bytes() { - let m = O2Memory::with_total(64).unwrap(); - m.set_ctrl(0, 0); - let bus = m.phys(); - let base = bus.ppmem_space().unwrap().window_base(); - let cached = |o: usize, base: *mut u8| unsafe { (*(base.add(o) as *const u64)).rotate_left(32) }; - let store = |o: usize, v: u64, base: *mut u8| unsafe { *(base.add(o) as *mut u64) = v.rotate_left(32) }; - assert_eq!(bus.write64(0xff0, 0x1111), BUS_OK); - m.stack_tag_store(0x0); - m.stack_tag_store(STACK_TAG); - assert_eq!(bus.read64(0xff0).data, 0x1111, "the uncached page starts as a copy"); - store(0xff0, 0x2222, base); - assert_eq!(bus.write64(0xff8, 0x3333), BUS_OK); - assert_eq!(cached(0xff0, base), 0x2222, "the stack, cached"); - assert_eq!(cached(0xff8, base), 0, "the test's write does not reach the stack"); - assert_eq!(bus.read64(0xff0).data, 0x1111, "and the stack does not reach the test"); - assert_eq!(bus.read64(0xff8).data, 0x3333); - assert_eq!(bus.read64(RAM_BASE as u64 + 0xff8).data, 0x3333, "through the window's address too"); - assert_eq!(bus.write64(0x1000, 0x4444), BUS_OK); - assert_eq!(cached(0x1000, base), 0x4444, "the next page is memory"); - m.stack_tag_store(0x0); - assert_eq!(cached(0xff8, base), 0x3333, "memory holds what was written uncached"); - assert_eq!(bus.read64(0xff0).data, 0x1111); - store(0xff8, 0x5555, base); - assert_eq!(bus.read64(0xff8).data, 0x5555, "one memory again"); - } - - /// Sizing reads back through a 32 MB bank repeating under a 128 MB - /// setting, so page 0 has to answer the same way at every repeat. - #[test] - fn the_uncached_page_follows_a_small_bank_repeating() { - let m = O2Memory::with_total(32).unwrap(); - m.stack_tag_store(STACK_TAG); - m.set_ctrl(0, CTRL_128M); - put(&m, RAM_BASE + 8, 0xfeed); - assert_eq!(get(&m, RAM_BASE + 3 * SLOT + 8), Some(0xfeed)); - assert_eq!(get(&m, 3 * SLOT + 8), Some(0xfeed), "the low alias"); - m.stack_tag_store(0); - assert_eq!(m.phys().banks()[0].read64(8).data, 0xfeed); - } - - /// At reset one 32 MB slot is mapped, which ppmem's bitmap does not count; - /// the cached path must still reach the bank, not the bus, or the stack - /// moves out from under POST when sizing completes the region. - #[test] - fn the_cached_path_is_on_the_bank_from_reset() { - let m = O2Memory::with_total(32).unwrap(); - let sp = m.phys().ppmem_space().unwrap(); - let region = 1u64 << 0; - assert_eq!(sp.mapped_bitmap() & region, 0, "a hole at reset"); - m.stack_tag_store(STACK_TAG); - assert_ne!(sp.mapped_bitmap() & region, 0, "filled while the stack is live"); - m.stack_tag_store(0); - assert_eq!(sp.mapped_bitmap() & region, 0, "and only then"); - } - - /// Only a valid tag for page 0 starts the stack; other firmware's tags - /// and a kernel's invalid ones leave the decode alone. - #[test] - fn other_tags_leave_the_decode_alone() { - let m = O2Memory::with_total(32).unwrap(); - for tag in [0u32, 0x1_00c8, 0x08] { - m.stack_tag_store(tag); - assert!(m.stack_page_words().is_none(), "{tag:#x}"); - } - } - - /// A snapshot taken during POST comes back with both views. - #[test] - fn a_snapshot_carries_the_uncached_page() { - let m = O2Memory::with_total(32).unwrap(); - m.stack_tag_store(STACK_TAG); - put(&m, 0x20, 0xabcd); - let words = m.stack_page_words().unwrap(); - let n = O2Memory::with_total(32).unwrap(); - n.restore_stack_page(&words).unwrap(); - assert_eq!(get(&n, 0x20), Some(0xabcd)); - n.stack_tag_store(0); - assert_eq!(n.phys().banks()[0].read64(0x20).data, 0xabcd); - } - /// An empty bank's slots read zero; with no control matching, the bus's /// default answers. #[test] diff --git a/src/ip32.rs b/src/ip32.rs index 6b78f83a2..5ba6fc837 100644 --- a/src/ip32.rs +++ b/src/ip32.rs @@ -701,14 +701,18 @@ impl crate::traits::Saveable for Ip32Bus { t.insert("mace".into(), self.mace.save_state()); let scsi = self.scsi.lock().unwrap().iter().map(|c| c.save_state()).collect(); t.insert("scsi".into(), toml::Value::Array(scsi)); - if let Some(words) = self.ram.stack_page_words() { - t.insert("stack_page".into(), toml::Value::Array(words.into_iter().map(crate::snapshot::hex_u64).collect())); - } toml::Value::Table(t) } fn load_state(&self, v: &toml::Value) -> Result<(), String> { use crate::snapshot::get_field; + // Older snapshots taken while POST's stack stood kept the page the + // uncached path saw. The stack lives in the CPU's cached view now, + // and they hold no cache lines to rebuild it from. + if get_field(v, "stack_page").is_some() { + return Err("this snapshot was taken during POST by an older iris, which kept the \ + stack page outside the cache; take it again".into()); + } if let Some(m) = get_field(v, "mace") { self.mace.load_state(m)?; } @@ -730,10 +734,6 @@ impl crate::traits::Saveable for Ip32Bus { c.load_state(s)?; } } - if let Some(toml::Value::Array(a)) = get_field(v, "stack_page") { - let words: Option> = a.iter().map(crate::snapshot::toml_u64).collect(); - self.ram.restore_stack_page(&words.ok_or("stack_page: not a list of dwords")?)?; - } // MACE loaded before CRIME and the controllers: tell CRIME the pin // levels now that it can take them. self.mace.pci.bus().redrive(); @@ -747,7 +747,6 @@ impl Ip32Bus { /// on the hardware. The PROM sets up the rest again as it starts. pub fn power_on(&self) { use crate::traits::Resettable; - self.ram.set_stack_page(false); self.crime.regs.power_on(); let m = &self.mace; let devices: [&dyn Resettable; 8] = diff --git a/src/ip32/tests.rs b/src/ip32/tests.rs index 4c2c9c503..8f93271f0 100644 --- a/src/ip32/tests.rs +++ b/src/ip32/tests.rs @@ -214,6 +214,18 @@ fn a_bus_saves_and_loads_whole() { std::fs::remove_dir_all(&dir).ok(); } +/// A snapshot an older iris took during POST kept the stack page outside +/// the cache, which nothing can put back now: it is refused. +#[test] +fn a_snapshot_with_the_old_stack_page_is_refused() { + use crate::traits::Saveable; + let bus = Ip32Bus::new(16 << 20, vec![0u8; PROM_SIZE as usize]); + let mut v = bus.save_state(); + v.as_table_mut().unwrap().insert("stack_page".into(), toml::Value::Array(vec![])); + let e = bus.load_state(&v).unwrap_err(); + assert!(e.contains("take it again"), "{e}"); +} + /// Setting CONTROL's cold-reset bit asks the machine for a reset once, and /// the bit reads back clear, as the hardware clears it. #[test] diff --git a/src/ip32_machine.rs b/src/ip32_machine.rs index 8386531e7..d7b120cc6 100644 --- a/src/ip32_machine.rs +++ b/src/ip32_machine.rs @@ -246,7 +246,7 @@ impl O2Machine { let boot = Arc::new(BootState::new()); let memory = Arc::new(crate::ip32::memory_map()); let cpu: Arc = crate::with_cpu!(part, true, true, C => - Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, count_hz, boot.clone(), bus.ram.clone(), part.prid())); + Self::build_cpu::(memory.clone(), sysad, &timer_manager, arcs_fpu, count_hz, boot.clone(), part.prid())); // ppmem: the CPU's cache and the JIT reach RAM through a cached view // of the memory's window, publishing its bitmap into the CPU's own @@ -371,7 +371,7 @@ impl O2Machine { } fn build_cpu(memory: Arc, sysad: Arc, tm: &Arc, arcs_fpu: bool, count_hz: u64, boot: Arc, - ram: Arc, prid: u32) -> Arc + prid: u32) -> Arc where C: crate::cpu::mips_cache_v2::CpuModel + MipsCache + From> + 'static, MipsCpu: CpuDevice, @@ -379,7 +379,6 @@ impl O2Machine { let cfg = MipsCpuConfig::for_model::(memory).with_prid(prid); let tlb = MipsTlb::new(cfg.tlb_entries); let mut exec: MipsExecutor = MipsExecutor::new(sysad, tlb, &cfg); - exec.cache.set_pd_tag_store_hook(Box::new(move |tag| ram.stack_tag_store(tag))); exec.core.set_timer_manager(tm.clone()); // Count runs at half the pipeline clock on both CPUs. exec.core.set_count_hz(count_hz.max(1)); From 2d7ba0b55ccd501efdefe010b315fbfd7a4d7c50 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:36 +0000 Subject: [PATCH 06/22] isp1040: add the dual-channel ISP12160 part Fuel uses two independent SCSI buses behind one ISP12160 function. Reuse the ISP1040 command engine and let the part trait own the firmware, register and channel differences. A bus reset reaches every attached target as unit attention rather than a media change. Keep the reset attention on the shared target interface so a controller can notify each device through the same bus. The ISP12160 is its only caller: the WD33C93A, AIC-7880 and ISP1040 models do not call it, so their targets behave as before. A bus reset restores each disk's write cache bit per channel: a reset of one channel leaves the other's disks alone, and a chip or soft reset restores both. The engine changes reach the Octane's ISP1040: mailbox commands now run with dispatch held, and the target interface gains the bus reset. The Octane's IRIX 6.5.30 SCSI disk (Octane2, R10000, 256 MiB, V12, jitv2, release build, headless, NAT) boots to the login prompt and runs commands on this branch's tip and on main c6dea8c4: login after 126 s on the branch and 109 s on main in one run each (host load makes the times indicative only), the same `hinv` listing of both integral controllers and the disk, and the same command output. --- HACKING.md | 6 + src/dev/isp1040.rs | 395 ++++++++++++++++++++++++------- src/dev/isp1040/tests.rs | 488 ++++++++++++++++++++++++++++++++++++++- src/dev/isp12160.rs | 337 +++++++++++++++++++++++++++ src/dev/mod.rs | 1 + src/scsi.rs | 52 ++++- src/scsi/bus.rs | 11 +- 7 files changed, 1196 insertions(+), 94 deletions(-) create mode 100644 src/dev/isp12160.rs diff --git a/HACKING.md b/HACKING.md index e770f6a0e..28795b501 100644 --- a/HACKING.md +++ b/HACKING.md @@ -101,6 +101,12 @@ the machine supplies only a route from controller number to bus. Flash command engines share `FlashBacking` file storage while retaining their own command state. +`ScsiTarget::bus_reset` defaults to no action. Disk targets record reset as +unit attention. Only the ISP12160 calls it; the other controllers keep +their earlier behaviour of not reporting resets to targets. +The ISP12160 releases its controller state lock before walking either bus. +Target callbacks run under that bus lock and must not reacquire it. + ## 2. Data path and endianness **Word-transparent architecture.** Host `u32`/`u64` values are bit diff --git a/src/dev/isp1040.rs b/src/dev/isp1040.rs index aeb90dd41..dab58d466 100644 --- a/src/dev/isp1040.rs +++ b/src/dev/isp1040.rs @@ -50,8 +50,9 @@ //! again when the hold ends: an approximation, a chip that had fetched it //! would plausibly move on. A parked entry the guest rewrites meanwhile is //! answered incomplete. A snapshot carries each bus's remaining hold and -//! the parked positions. The ISP12160 would widen `BUSES` and decode the -//! bus from mailbox 2 and its request entries; the per-bus state is there. +//! the parked positions. Only the ISP1040's bus reset starts a hold-off: +//! the ISP12160 (two buses, per-bus state is there) keeps its own bus-reset +//! handling, and its hold-off is not modelled. //! * Only logical unit 0 of a present target exists. A command for another //! unit is passed to the target with that unit number, which answers as //! SCSI says (INQUIRY: no device at this unit; anything else: check @@ -81,7 +82,7 @@ use parking_lot::{Condvar, Mutex, MutexGuard}; use crate::devlog::LogModule; use crate::irq::IrqLine; -use crate::pci::{Bar, PciConfig, PciDma, PciFunction, PciFunctionState, PCI_INTA}; +use crate::pci::{Bar, PciConfig, PciConfigPolicy, PciDma, PciFunction, PciFunctionState, PciRegisterPolicy, PCI_INTA}; use crate::scsi::{scsi_cmd, ScsiBus, ScsiDataLength, ScsiIdentities, ScsiRequest, CDROM_IDENTITY, DISK_IDENTITY}; use crate::snapshot::{get_field, hex_u16, hex_u32, load_u16_slice, toml_u16, toml_u32, toml_u8, u16_slice_to_toml, u8_slice_to_toml}; @@ -115,7 +116,7 @@ pub mod reg { pub const WINDOW_SIZE: u32 = 0x100; } -const MAILBOX_COUNT: usize = 8; +pub(crate) const MAILBOX_COUNT: usize = 8; const MAILBOX_END: u32 = reg::MAILBOX_BASE + MAILBOX_COUNT as u32 * 2; const BANK_END: u32 = 0xc0; const BANK_WORDS: usize = ((BANK_END - reg::BANK_BASE) / 2) as usize; @@ -163,6 +164,8 @@ pub mod mbox { pub const INIT_REQ_QUEUE: u16 = 0x0010; pub const INIT_RSP_QUEUE: u16 = 0x0011; pub const ABORT_COMMAND: u16 = 0x0015; + pub const ABORT_DEVICE: u16 = 0x0016; + pub const ABORT_TARGET: u16 = 0x0017; pub const BUS_RESET: u16 = 0x0018; pub const GET_FIRMWARE_STATUS: u16 = 0x001f; pub const GET_TARGET_PARAMETERS: u16 = 0x0028; @@ -187,13 +190,14 @@ pub mod mbox { pub const INVALID: u16 = 0x4001; pub const HOST_INTERFACE_ERROR: u16 = 0x4002; pub const TEST_FAILED: u16 = 0x4003; + pub const COMMAND_ERROR: u16 = 0x4005; pub const PARAM_ERROR: u16 = 0x4006; } const RAM_WORDS: usize = 0x10000; /// The command budget the adapter announces in its firmware status. pub const MAX_REQUESTS: u16 = 512; -const FW_VERSION: [u16; 3] = [1, 0, 0]; +pub(crate) const FW_VERSION: [u16; 3] = [1, 0, 0]; pub const HOST_CMD_SHIFT: u32 = 12; pub const RISC_FLAG_HOST_INT: u16 = 0x0080; @@ -229,6 +233,7 @@ pub const SCSI_STATUS_SENSE_VALID: u16 = 0x0200; pub const IOCB_COMPLETE: u16 = 0x0000; pub const IOCB_INCOMPLETE: u16 = 0x0001; pub const IOCB_RESET: u16 = 0x0004; +pub const IOCB_ABORTED: u16 = 0x0005; pub const IOCB_DATA_UNDERRUN: u16 = 0x0015; pub const STATE_GOT_BUS: u16 = 0x0100; @@ -256,13 +261,14 @@ const DEFAULT_SELECTION_TIMEOUT: u16 = 250; /// source for a device, so it runs on the adapter's clock, host wall time /// less the time the machine was stopped, like the selection timeout. const BUS_RESET_DELAY_UNIT: Duration = Duration::from_secs(1); -/// The SCSI buses this part drives. The hold-off, the parked commands and the -/// selection are all kept per bus, so a two-bus part widens this and decodes -/// the bus from its own request entries and from mailbox 2. -const BUSES: usize = 1; +/// The most SCSI buses any part drives (the ISP12160's two; the ISP1040 has +/// one, `IspPart::channels`). The hold-off, the parked commands and the +/// selection are all kept per bus. Only the ISP1040's bus reset starts a +/// hold-off: the ISP12160 keeps its own bus-reset handling. +const BUSES: usize = 2; const CHECK_CONDITION: u8 = 0x02; -const MAX_TARGET: u8 = 15; +pub(crate) const MAX_TARGET: u8 = 15; const MAX_LUN: u8 = 7; /// A completion held for response-ring room. @@ -374,6 +380,49 @@ enum Prepared { Run(Job), } +/// Part-specific register and firmware behavior; the ring engine owns the state. +/// Defaults preserve the ISP1040 contract, including wrapping firmware RAM. +pub(crate) trait IspPart: Send + Sync { + fn name(&self) -> &'static str; + fn channels(&self) -> usize { 1 } + fn pci_config(&self, bars: [Bar; 6]) -> PciConfig { + PciConfig::new(PCI_VENDOR_QLOGIC, PCI_DEVICE_ISP1040, PCI_CLASS_SCSI, PCI_REVISION_ISP1040, bars, PCI_INTA) + } + fn firmware_version(&self) -> [u16; 3] { FW_VERSION } + fn active_bank(&self, cfg1: u16) -> Bank { + if cfg1 & CFG1_BANK_SELECT != 0 { Bank::Sxp } else { Bank::Risc } + } + fn checks_ram_bounds(&self) -> bool { false } + fn has_abort_commands(&self) -> bool { false } + fn target_on(&self, _channel: u8, target: u8) -> u8 { target } + fn decode_target(&self, target: u8) -> (usize, u8) { (0, target) } + fn id_high(&self) -> u16 { PCI_DEVICE_ISP1040 } + /// The ISP1040 register revision runs one ahead of its PCI revision. + fn bus_cfg0(&self) -> u16 { PCI_REVISION_ISP1040 as u16 + 1 } + fn cfg1_write_mask(&self) -> u16 { CFG1_WRITE_MASK } + fn unmodelled_dma_register(&self, _off: u32) -> bool { false } + fn completion_phases(&self, _moved: u64, _total: u64) -> u16 { STATE_ALL_PHASES & !STATE_GOT_SENSE } + fn mailbox_ready(&self, _semaphore: u16) -> bool { true } + /// Extra target-parameter results, beyond the common asynchronous/narrow reply. + fn clear_target_parameters(&self, _mailbox: &mut [u16; MAILBOX_COUNT]) {} + fn set_reset_delay(&self, st: &mut State) { st.reset_delay = st.mailbox[1]; } + /// Queue target notifications for the worker; the CPU never locks a bus. + fn reset_targets(&self, _st: &mut State) -> bool { false } + /// Handle extra mailbox commands, leaving result signalling to the engine. + fn execute_mailbox(&self, _core: &Core, _st: &mut MutexGuard<'_, State>) -> bool { false } + fn save_extra(&self, _st: &State, _t: &mut toml::map::Map) {} + fn load_extra(&self, _st: &mut State, _v: &toml::Value) {} +} + +struct Isp1040Part; + +impl IspPart for Isp1040Part { + fn name(&self) -> &'static str { "ISP1040" } +} + +/// The register window selected by BUS_CFG1, independent of its encoding. +pub(crate) enum Bank { Risc, Sxp, Sxp1, Dma } + /// A command waiting out its bus's post-reset hold-off. Only the ring /// position is kept: the entry, its data segments included, is read again /// from the guest's ring when the hold ends. @@ -392,7 +441,15 @@ struct Parked { started: bool, } -struct State { +pub(crate) struct State { + pub(crate) reset_targets: bool, + pub(crate) pending_target_resets: VecDeque<(u8, Option)>, + pub(crate) active: Option<(u8, u8, u32)>, + cancelled: bool, + pub(crate) outstanding: Vec<(u8, u8, u32)>, + pub(crate) sxp1_bank: [u16; BANK_WORDS], + pub(crate) dma_bank: [u16; BANK_WORDS], + pub(crate) reset_delay1: u16, ctl: Ctl, bus_cfg1: u16, bus_ctrl: u16, @@ -402,11 +459,11 @@ struct State { risc_flags: u16, dma_cmd_cfg: u16, dma_data_cfg: u16, - mailbox: [u16; MAILBOX_COUNT], + pub(crate) mailbox: [u16; MAILBOX_COUNT], risc_bank: [u16; BANK_WORDS], sxp_bank: [u16; BANK_WORDS], risc_ram: Vec, - reset_delay: u16, + pub(crate) reset_delay: u16, /// Mailbox command 0x0031's value, in [`SELECTION_TIMEOUT_UNIT`]s. selection_timeout: u16, rqst_base: u64, @@ -434,6 +491,14 @@ struct State { impl State { fn new() -> Self { let mut s = State { + reset_targets: false, + pending_target_resets: VecDeque::new(), + active: None, + cancelled: false, + outstanding: Vec::new(), + sxp1_bank: [0; BANK_WORDS], + dma_bank: [0; BANK_WORDS], + reset_delay1: 0, ctl: Ctl::default(), bus_cfg1: 0, bus_ctrl: 0, @@ -485,6 +550,12 @@ impl State { self.mailbox = [0, SIG1, SIG2, SIG3, SIG4, 0, 0, 0]; self.risc_bank = [0; BANK_WORDS]; self.sxp_bank = [0; BANK_WORDS]; + self.sxp1_bank = [0; BANK_WORDS]; + self.dma_bank = [0; BANK_WORDS]; + self.reset_delay1 = 0; + self.active = None; + self.cancelled = false; + self.outstanding.clear(); // The driver re-downloads firmware after every reset. self.risc_ram.iter_mut().for_each(|w| *w = 0); self.reset_delay = 0; @@ -499,6 +570,8 @@ impl State { self.resp_seen = 0; self.resp_cons = 0; self.pending.clear(); + self.reset_targets = false; + self.pending_target_resets.clear(); self.quiet_until = [None; BUSES]; self.parked.clear(); } @@ -524,9 +597,15 @@ impl State { self.resp_len != 0 && (u32::from(self.resp_prod) + 1) % u32::from(self.resp_len) != u32::from(self.resp_cons) } - fn active_bank(&mut self) -> &mut [u16; BANK_WORDS] { - if self.bus_cfg1 & CFG1_BANK_SELECT != 0 { &mut self.sxp_bank } else { &mut self.risc_bank } + fn active_bank(&mut self, bank: Bank) -> &mut [u16; BANK_WORDS] { + match bank { + Bank::Risc => &mut self.risc_bank, + Bank::Sxp => &mut self.sxp_bank, + Bank::Sxp1 => &mut self.sxp1_bank, + Bank::Dma => &mut self.dma_bank, + } } + } /// The adapter. Its state lives in a [`Core`] shared with the thread that @@ -543,8 +622,8 @@ impl std::ops::Deref for Isp1040 { /// Lock order: `line_order`, then `st`. Nothing else is taken with `st` /// held except the leaf locks (`notes`, `dma_misses`, `odd_accesses`): DMA, -/// the targets and `fence` are reached with it released. `fence` is held -/// across one guest-memory store at a time, and `targets` across one +/// either bus and `fence` are reached with it released. `fence` is held +/// across one guest-memory store at a time, and either bus across one /// target command; neither is held with the other. pub struct Core { st: Mutex, @@ -560,13 +639,17 @@ pub struct Core { wake: Condvar, /// Serialises driving the interrupt line; see [`Core::sync_irq`]. line_order: Mutex<()>, - /// The targets on the bus, by SCSI id. - bus: ScsiBus, - /// A bus reset the targets have not yet seen. `st` and `bus` are never - /// held together, so the reset (taken under `st`) is passed on by the - /// next command to take `bus`, before it runs: no command can see the - /// targets as they were. See [`Core::scsi`]. - targets_reset: AtomicBool, + /// Generation for commands cancelled by aborts without resetting the chip. + cancel: AtomicU64, + /// Immutable part policy, shared by register, worker and snapshot paths. + part: &'static dyn IspPart, + /// The controller's one bus, or both channels of the ISP12160. + buses: [ScsiBus; 2], + /// A bus reset the targets on that bus have not yet seen. `st` and the + /// buses are never held together, so the reset (taken under `st`) is + /// passed on by the next command to take the bus, before it runs: no + /// command can see the targets as they were. See [`Core::scsi`]. + targets_reset: [AtomicBool; 2], /// Which of the Octane's two controllers this is (0 internal, 1 /// external): the monitor's `scsi`/`cow` address the first, /// `scsi1`/`cow1` the second. @@ -601,7 +684,11 @@ impl Isp1040 { /// The controller at `index` (0 internal, 1 external) on `bus`. pub fn new_at(index: u8, bus: ScsiBus) -> Self { - Self { core: Arc::new(Core::new_at(index, bus)), worker: Mutex::new(None) } + Self { core: Arc::new(Core::new_at(index, &Isp1040Part, [bus, Self::new_bus()])), worker: Mutex::new(None) } + } + + pub(crate) fn new_12160(bus0: ScsiBus, bus1: ScsiBus) -> Self { + Self { core: Arc::new(Core::new_at(0, &crate::dev::isp12160::Isp12160Part, [bus0, bus1])), worker: Mutex::new(None) } } /// Start the ring thread (idempotent) and let it take whatever was @@ -615,7 +702,7 @@ impl Isp1040 { let c = self.core.clone(); *worker = Some( std::thread::Builder::new() - .name(format!("ISP1040-{}", self.index)) + .name(format!("{}-{}", self.core.model_name(), self.index)) .spawn(move || c.run_ring()) .expect("spawn isp1040 thread"), ); @@ -639,13 +726,17 @@ impl Isp1040 { /// Name the adapter's long-lived locks in the lock registry, which /// holds only a weak reference to them. pub fn register_locks(&self) { - for (what, probe) in [ + let mut locks: Vec<(&str, fn(&Core) -> bool)> = vec![ ("st", (|c: &Core| c.st.is_locked()) as fn(&Core) -> bool), ("fence", |c| c.fence.is_locked()), - ("targets", |c| c.bus.is_locked()), - ] { + ("targets", |c| c.buses[0].is_locked()), + ]; + if self.core.part.channels() > 1 { + locks.push(("targets1", |c| c.buses[1].is_locked())); + } + for (what, probe) in locks { let core = Arc::downgrade(&self.core); - crate::locks::register_lock(format!("isp1040-{}::{what}", self.index), move || { + crate::locks::register_lock(format!("{}-{}::{what}", self.core.model_name().to_ascii_lowercase(), self.index), move || { core.upgrade().is_some_and(|c| probe(&c)) }); } @@ -660,25 +751,25 @@ impl Drop for Isp1040 { } impl Core { - fn new_at(index: u8, bus: ScsiBus) -> Self { + fn new_at(index: u8, part: &'static dyn IspPart, buses: [ScsiBus; 2]) -> Self { let st = State::new(); let gen = st.ctl.gen; + let bars = [Bar::Io(reg::WINDOW_SIZE), Bar::Mem(reg::WINDOW_SIZE), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused]; + let pci = part.pci_config(bars); Self { + part, + cancel: AtomicU64::new(0), st: Mutex::new(st), gen: AtomicU64::new(gen), fence: Mutex::new(()), wake: Condvar::new(), line_order: Mutex::new(()), - bus, - targets_reset: AtomicBool::new(false), + buses, + targets_reset: [AtomicBool::new(false), AtomicBool::new(false)], index, // BAR 0 is the I/O window and BAR 1 its memory-space alias, one // register file. - pci: PciFunctionState::new(PciConfig::new( - PCI_VENDOR_QLOGIC, PCI_DEVICE_ISP1040, PCI_CLASS_SCSI, PCI_REVISION_ISP1040, - [Bar::Io(reg::WINDOW_SIZE), Bar::Mem(reg::WINDOW_SIZE), Bar::Unused, Bar::Unused, Bar::Unused, Bar::Unused], - PCI_INTA, - )), + pci: PciFunctionState::new(pci), notes: Mutex::new(Vec::new()), dma_misses: Mutex::new(std::collections::BTreeSet::new()), odd_accesses: Mutex::new(0), @@ -688,6 +779,8 @@ impl Core { } } + fn model_name(&self) -> &'static str { self.part.name() } + pub fn index(&self) -> u8 { self.index } /// A machine reset: the chip as at power-on (no firmware, no queues), @@ -698,6 +791,7 @@ impl Core { { let mut st = self.st.lock(); st.reset(); + if self.part.reset_targets(&mut st) { self.kick(&mut st); } self.publish_gen(&st); self.wake.notify_all(); } @@ -919,7 +1013,7 @@ impl Core { } /// Tell the ring thread the ring may have moved. - fn kick(&self, st: &mut State) { + pub(crate) fn kick(&self, st: &mut State) { st.ctl.work = true; self.wake.notify_all(); } @@ -947,11 +1041,16 @@ impl Core { /// The SCSI bus: attach targets to it and ask what is on it. pub fn bus(&self) -> &ScsiBus { - &self.bus + &self.buses[0] + } + + pub(crate) fn bus_for_channel(&self, channel: u8) -> Option<&ScsiBus> { + ((channel as usize) < self.part.channels()).then(|| &self.buses[channel as usize]) } - fn has_target(&self, id: u8) -> bool { - self.bus.has_target(id) + fn bus_for_target(&self, target: u8) -> (&ScsiBus, u8) { + let (channel, id) = self.part.decode_target(target); + (&self.buses[channel], id) } // ---- DMA --------------------------------------------------------- @@ -1035,12 +1134,18 @@ impl Core { /// A RISC RAM load or dump of up to 128 KiB, with the state unlocked /// for the DMA. The whole transfer is read before a word of it is kept, /// so a refused DMA leaves the array as it was. Words travel - /// little-endian. This part has no edge to its array: a transfer past - /// the end folds to the start. - fn ram_transfer(&self, st: &mut MutexGuard<'_, State>, load: bool) -> u16 { + /// little-endian. The ISP1040 wraps past the end; the ISP12160 rejects + /// transfers that would cross it. + pub(crate) fn ram_transfer(&self, st: &mut MutexGuard<'_, State>, load: bool) -> u16 { let base = st.mailbox[1]; let count = st.mailbox[4] as usize; - let host = Self::mbox_host_addr(&st.mailbox); + if self.part.checks_ram_bounds() && base as usize + count > RAM_WORDS { + return mbox::PARAM_ERROR; + } + let mut host = Self::mbox_host_addr(&st.mailbox); + if st.mailbox[0] == mbox::LOAD_RAM_A64 { + host |= ((st.mailbox[7] as u64) << 32) | ((st.mailbox[6] as u64) << 48); + } if count == 0 { return mbox::COMPLETE; } @@ -1067,6 +1172,20 @@ impl Core { /// Synchronous: the protocol has no way to leave a command pending. fn mailbox_command(&self, st: &mut MutexGuard<'_, State>) { let cmd = st.mailbox[0]; + if self.part.has_abort_commands() && matches!(cmd, mbox::ABORT_COMMAND | mbox::ABORT_DEVICE | mbox::ABORT_TARGET | mbox::BUS_RESET) { + self.with_dispatch_held(st, |st| self.execute_mailbox_command(st)); + } else { + self.execute_mailbox_command(st); + } + } + + fn execute_mailbox_command(&self, st: &mut MutexGuard<'_, State>) { + let cmd = st.mailbox[0]; + if self.part.execute_mailbox(self, st) { + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } match cmd { mbox::NOP => st.mailbox[0] = mbox::COMPLETE, mbox::WRITE_RAM_WORD => { @@ -1088,7 +1207,19 @@ impl Core { // zero. mbox::VERIFY_CHECKSUM => { let start = st.mailbox[1]; + if self.part.checks_ram_bounds() && start as usize + 4 > RAM_WORDS { + st.mailbox[0] = mbox::PARAM_ERROR; + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } let len = st.risc_ram[start.wrapping_add(3) as usize]; + if self.part.checks_ram_bounds() && start as usize + len as usize > RAM_WORDS { + st.mailbox[0] = mbox::PARAM_ERROR; + st.semaphore = SEMAPHORE_LOCK; + st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; + return; + } let mut sum = 0u16; for i in 0..len { sum = sum.wrapping_add(st.risc_ram[start.wrapping_add(i) as usize]); @@ -1101,9 +1232,7 @@ impl Core { // Accepted, never executed. mbox::EXEC_FIRMWARE => st.mailbox[0] = mbox::COMPLETE, mbox::ABOUT_FIRMWARE => { - st.mailbox[1] = FW_VERSION[0]; - st.mailbox[2] = FW_VERSION[1]; - st.mailbox[3] = FW_VERSION[2]; + st.mailbox[1..4].copy_from_slice(&self.part.firmware_version()); st.mailbox[6] = 0; st.mailbox[0] = mbox::COMPLETE; } @@ -1118,6 +1247,7 @@ impl Core { mbox::GET_TARGET_PARAMETERS => { st.mailbox[2] = 0; st.mailbox[3] = 0; + self.part.clear_target_parameters(&mut st.mailbox); st.mailbox[0] = mbox::COMPLETE; } // Setters for state nothing reads back; answering invalid is a @@ -1139,7 +1269,7 @@ impl Core { st.mailbox[0] = mbox::COMPLETE; } mbox::SET_RESET_DELAY_PARAMS => { - st.reset_delay = st.mailbox[1]; + self.part.set_reset_delay(st); st.mailbox[0] = mbox::COMPLETE; } mbox::INIT_REQ_QUEUE | mbox::INIT_REQ_QUEUE_A64 | mbox::INIT_RSP_QUEUE | mbox::INIT_RSP_QUEUE_A64 => { @@ -1194,7 +1324,7 @@ impl Core { mbox::BUS_RESET => { let now = self.now_ns(st); let bus = usize::from(st.mailbox[2]); - if bus >= BUSES { + if bus >= self.part.channels() { // A bus this part does not have: nothing to quiet or // retire, and the datasheet's per-bus hold leaves the // real bus alone. Noted so a driver that leaves mailbox 2 @@ -1218,7 +1348,7 @@ impl Core { st.pending.push_back(Pending { handle: p.handle, completion: IOCB_RESET, ..Default::default() }); } } - self.targets_reset.store(true, Ordering::Release); + self.targets_reset[bus].store(true, Ordering::Release); let delay = BUS_RESET_DELAY_UNIT * u32::from(st.mailbox[1]); st.quiet_until[bus] = (!delay.is_zero()).then(|| now.saturating_add(delay.as_nanos() as i64)); self.kick(st); @@ -1235,6 +1365,67 @@ impl Core { st.bus_status |= STATUS_INT_REQ | STATUS_RISC_INT_REQ; } + /// Prevent the worker from taking another entry while a mailbox drops `st`. + fn with_dispatch_held(&self, st: &mut MutexGuard<'_, State>, f: impl FnOnce(&mut MutexGuard<'_, State>) -> R) -> R { + struct Hold<'a, 'b> { + core: &'a Core, + st: &'a mut MutexGuard<'b, State>, + } + impl Drop for Hold<'_, '_> { + fn drop(&mut self) { + self.st.ctl.hold -= 1; + self.core.kick(self.st); + } + } + st.ctl.hold += 1; + let held = Hold { core: self, st }; + f(held.st) + } + + /// A reset of `channel`'s bus: each disk on it puts its mode parameters back before the next + /// command, see [`ScsiTarget::reset_mode_parameters`]. A channel the part lacks is ignored. + pub(crate) fn note_bus_reset(&self, channel: usize) { + if let Some(flag) = self.targets_reset.get(channel) { flag.store(true, Ordering::Release); } + } + + /// Notify targets with `st` released; a blocked target must not stall register access. + fn reset_targets(&self, st: &mut MutexGuard<'_, State>, channel: u8, target: Option) { + let Some(bus) = self.bus_for_channel(channel) else { return }; + self.with_dispatch_held(st, |st| MutexGuard::unlocked(st, || bus.for_each(|id, dev| { + if target.is_none_or(|target| target as usize == id) { + dev.bus_reset(); + } + }))); + } + + pub(crate) fn retire_outstanding(&self, st: &mut State, matches: impl Fn((u8, u8, u32)) -> bool, completion: u16) -> usize { + let mut retired = 0; + let mut keep = Vec::new(); + for item in st.outstanding.drain(..) { + if matches(item) { + st.pending.push_back(Pending { handle: item.2, completion, ..Pending::default() }); + retired += 1; + } else { + keep.push(item); + } + } + st.outstanding = keep; + if retired != 0 { self.kick(st); } + retired + } + + /// Queue one cancellation reply, then fence its remaining guest-memory stores. + pub(crate) fn cancel_active(&self, st: &mut MutexGuard<'_, State>, completion: u16) { + if st.cancelled { return; } + if let Some((_, _, handle)) = st.active { + self.cancel.fetch_add(1, Ordering::AcqRel); + st.cancelled = true; + st.pending.push_back(Pending { handle, completion, ..Pending::default() }); + self.kick(st); + MutexGuard::unlocked(st, || self.retire_stores()); + } + } + fn host_command(&self, st: &mut MutexGuard<'_, State>, val: u16) { match val >> HOST_CMD_SHIFT { 0x0 => {} @@ -1245,6 +1436,10 @@ impl Core { 0x3 => st.risc_flags &= !(RISC_FLAG_PAUSED | RISC_FLAG_RESET), // "A mailbox command is ready": the trigger to process one. 0x5 => { + if !self.part.mailbox_ready(st.semaphore) { + self.note("mailbox submitted with an unacknowledged result".into()); + return; + } st.risc_flags |= RISC_FLAG_HOST_INT; self.mailbox_command(st); } @@ -1430,17 +1625,19 @@ impl Core { /// Run a CDB against the target; `data` is what a write sends it. None /// when a reset cancelled the command while it waited for the target. - fn scsi(&self, gen: u64, target: u8, cdb: &[u8], data: Option>) -> Option<(Vec, u8)> { - let mut targets = self.bus.lock(); - if self.targets_reset.swap(false, Ordering::AcqRel) { + fn scsi(&self, gen: u64, cancel: u64, target: u8, cdb: &[u8], data: Option>) -> Option<(Vec, u8)> { + let (channel, _) = self.part.decode_target(target); + let (bus, id) = self.bus_for_target(target); + let mut targets = bus.lock(); + if self.targets_reset[channel].swap(false, Ordering::AcqRel) { // The bus reset puts each target's mode parameters back; see // [`ScsiTarget::reset_mode_parameters`]. targets.iter_mut().flatten().for_each(|t| t.reset_mode_parameters()); } - if self.gen.load(Ordering::Acquire) != gen { + if self.gen.load(Ordering::Acquire) != gen || self.cancel.load(Ordering::Acquire) != cancel { return None; } - let Some(dev) = targets.get_mut(target as usize).and_then(|t| t.as_mut()) else { + let Some(dev) = targets.get_mut(id as usize).and_then(|t| t.as_mut()) else { return Some((Vec::new(), CHECK_CONDITION)); }; let req = ScsiRequest { cdb: cdb.to_vec(), data_len: ScsiDataLength::Unlimited, data_in: data }; @@ -1471,14 +1668,15 @@ impl Core { let lun = (device & 0xff) as u8; // A bus this part does not have. - if usize::from(channel) >= BUSES { + if channel as usize >= self.part.channels() { return Prepared::Answered(Self::incomplete(handle, 0, 0)); } // GOT_BUS alone is Linux's bad-target; the timeout bit beside it is // what lets the empty slot be treated as benign. - if target > MAX_TARGET || !self.has_target(target) { + if target > MAX_TARGET || !self.buses[channel as usize].has_target(target) { return Prepared::Absent(channel as u8, Self::incomplete(handle, STATE_GOT_BUS, STATUS_FLAG_TIMEOUT)); } + let id = target; if lun > MAX_LUN { return Prepared::Answered(Self::incomplete(handle, STATE_GOT_BUS, STATUS_FLAG_TIMEOUT)); } @@ -1487,9 +1685,9 @@ impl Core { cdb[1] = (cdb[1] & 0x1f) | (lun << 5); let cdb = &cdb[..(cdb_len as usize).clamp(1, CDB_FIELD_LEN)]; let block = self - .bus + .buses[channel as usize] .lock() - .get(target as usize) + .get(id as usize) .and_then(|t| t.as_ref()) .and_then(|t| t.block_size()) .map(|b| b as usize) @@ -1497,6 +1695,7 @@ impl Core { .unwrap_or(BLOCK_SIZE); let (xfer, to_dev) = Self::cdb_transfer(cdb, block); let (segs, described) = self.read_segments(ring, addr, entry_count, seg_count, xfer as u64, a64); + let target = self.part.target_on(channel as u8, target); Prepared::Run(Job { handle, target, cdb: cdb.to_vec(), xfer, to_dev, segs, described }) } @@ -1512,7 +1711,7 @@ impl Core { /// None too when a reset cancelled the command (`gen` is the generation /// it was read under): what it had not yet stored in host memory is not /// stored, and no completion is built. - fn run_job(&self, job: Job, gen: u64) -> Option { + fn run_job(&self, job: Job, gen: u64, cancel: u64) -> Option { let Job { handle, target, cdb, xfer, to_dev, segs, described } = job; // The bar has lights for SCSI ids 0-5. if let (Some(hb), 0..=5) = (self.heartbeat.get(), target) { @@ -1530,9 +1729,10 @@ impl Core { buf.extend_from_slice(&chunk); } moved = buf.len() as u64; - status = self.scsi(gen, target, &cdb, Some(buf))?.1; + if self.cancel.load(Ordering::Acquire) != cancel { return None; } + status = self.scsi(gen, cancel, target, &cdb, Some(buf))?.1; } else { - let (data, s) = self.scsi(gen, target, &cdb, None)?; + let (data, s) = self.scsi(gen, cancel, target, &cdb, None)?; status = s; // Offer the target's bytes to the descriptors in order. let take = (data.len() as u64).min(described).min(xfer as u64) as usize; @@ -1540,7 +1740,7 @@ impl Core { for &(a, l) in &segs { if off >= take { break; } let n = (l as usize).min(take - off); - if !self.fenced(gen, || self.dma_write(a, &data[off..off + n]))? { + if !self.fenced(gen, || self.cancel.load(Ordering::Acquire) == cancel && self.dma_write(a, &data[off..off + n]))? { self.note(format!("payload DMA for handle {handle:#x} refused by the fabric; command stopped")); return None; } @@ -1551,12 +1751,13 @@ impl Core { // Auto-sense runs only after a CHECK CONDITION. let mut sense = Vec::new(); + if self.cancel.load(Ordering::Acquire) != cancel { return None; } if status == CHECK_CONDITION { - sense = self.scsi(gen, target, &[scsi_cmd::REQUEST_SENSE, 0, 0, 0, SENSE_MAX as u8, 0], None)?.0; + sense = self.scsi(gen, cancel, target, &[scsi_cmd::REQUEST_SENSE, 0, 0, 0, SENSE_MAX as u8, 0], None)?.0; sense.truncate(SENSE_MAX); } let mut scsi_status = status as u16; - let mut state_flags = STATE_ALL_PHASES & !STATE_GOT_SENSE; + let mut state_flags = self.part.completion_phases(moved, described.max(xfer as u64)); if !sense.is_empty() { scsi_status |= SCSI_STATUS_SENSE_VALID; state_flags |= STATE_GOT_SENSE; @@ -1653,6 +1854,17 @@ impl Core { st.ctl.work = true; return st; } + if std::mem::take(&mut st.reset_targets) { + for channel in 0..self.part.channels() { + self.note_bus_reset(channel); + self.reset_targets(&mut st, channel as u8, None); + } + continue; + } + if let Some((channel, target)) = st.pending_target_resets.pop_front() { + self.reset_targets(&mut st, channel, target); + continue; + } // Completions held for room go first, in order, as the driver // frees it; while any is held the request ring waits. if !st.pending.is_empty() { @@ -1794,8 +2006,16 @@ impl Core { } Prepared::Run(job) => { st.total_cmds += 1; + st.active = Some((job.target, job.cdb.get(1).copied().unwrap_or(0) >> 5, job.handle)); + st.cancelled = false; st.ctl.busy = true; - let done = MutexGuard::unlocked(&mut st, || self.run_job(job, ring.gen)); + let cancel = self.cancel.load(Ordering::Acquire); + let mut done = MutexGuard::unlocked(&mut st, || self.run_job(job, ring.gen, cancel)); + if st.cancelled { done = None; } + else if self.part.has_abort_commands() && done.is_none() && st.ctl.gen == ring.gen { + if let Some(active) = st.active { st.outstanding.push(active); } + } + st.active = None; st.ctl.busy = false; self.wake.notify_all(); // A reset cancelled the command. @@ -1828,13 +2048,16 @@ impl Core { return st.mailbox[((off - reg::MAILBOX_BASE) / 2) as usize]; } if (reg::BANK_BASE..BANK_END).contains(&off) { - return st.active_bank()[((off - reg::BANK_BASE) / 2) as usize]; + let bank = self.part.active_bank(st.bus_cfg1); + return st.active_bank(bank)[((off - reg::BANK_BASE) / 2) as usize]; + } + if self.part.unmodelled_dma_register(off) { + return 0; } match off { reg::ID_LOW => PCI_VENDOR_QLOGIC, - reg::ID_HIGH => PCI_DEVICE_ISP1040, - // The revision field runs one ahead of the PCI byte. - reg::BUS_CFG0 => PCI_REVISION_ISP1040 as u16 + 1, + reg::ID_HIGH => self.part.id_high(), + reg::BUS_CFG0 => self.part.bus_cfg0(), reg::BUS_CFG1 => st.bus_cfg1, reg::BUS_CTRL => st.bus_ctrl, reg::BUS_STATUS => st.bus_status, @@ -1852,6 +2075,10 @@ impl Core { } fn reg_write(&self, off: u32, val: u16) { + if self.part.unmodelled_dma_register(off) { + self.note(format!("write to ISP12160 DMA register {off:#04x} is not modelled")); + return; + } let mut st = self.st.lock(); if off == reg::RESPONSE_QUEUE { // The consumer acknowledgement: recorded, never over the @@ -1874,15 +2101,17 @@ impl Core { return; } if (reg::BANK_BASE..BANK_END).contains(&off) { - st.active_bank()[((off - reg::BANK_BASE) / 2) as usize] = val; + let bank = self.part.active_bank(st.bus_cfg1); + st.active_bank(bank)[((off - reg::BANK_BASE) / 2) as usize] = val; return; } match off { reg::BUS_CFG1 => { - if val & !CFG1_WRITE_MASK != 0 { - self.note(format!("bus configuration 1 bits {:#06x} are not modelled", val & !CFG1_WRITE_MASK)); + let mask = self.part.cfg1_write_mask(); + if val & !mask != 0 { + self.note(format!("bus configuration 1 bits {:#06x} are not modelled", val & !mask)); } - st.bus_cfg1 = val & CFG1_WRITE_MASK; + st.bus_cfg1 = val & mask; } // Applied after the enables are stored, so the reset wins. reg::BUS_CTRL => { @@ -1890,6 +2119,7 @@ impl Core { // The command in flight is cancelled, not waited for. if val & CTRL_SOFT_RESET != 0 { st.reset(); + if self.part.reset_targets(&mut st) { self.kick(&mut st); } self.publish_gen(&st); self.wake.notify_all(); drop(st); @@ -1995,7 +2225,7 @@ impl crate::traits::Device for Isp1040 { impl crate::scsi_monitor::ScsiTargets for Isp1040 { fn scsi_bus(&self) -> &ScsiBus { - &self.bus + &self.buses[0] } } @@ -2032,14 +2262,15 @@ impl crate::traits::Saveable for Isp1040 { // A hold-off already over but not acted on still crosses: the // commands parked behind it start when the load re-arms it at 0. let quiet_left = st.quiet_until.map(|until| until.map(|u| u - now)); - Self::save_locked(st, left, quiet_left) + self.save_locked(st, left, quiet_left) }) } fn load_state(&self, v: &toml::Value) -> Result<(), String> { self.quiesced(|st| { let now = self.now_ns(st); - Self::load_locked(st, v, now); + self.load_locked(st, v, now); + self.cancel.store(0, Ordering::Release); self.publish_gen(st); }); self.sync_irq(); @@ -2048,7 +2279,7 @@ impl crate::traits::Saveable for Isp1040 { } impl Isp1040 { - fn save_locked(st: &State, selecting_left: Option, quiet_left: [Option; BUSES]) -> toml::Value { + fn save_locked(&self, st: &State, selecting_left: Option, quiet_left: [Option; BUSES]) -> toml::Value { let mut t = toml::map::Map::new(); for (k, v) in [ ("bus_cfg1", st.bus_cfg1), ("bus_ctrl", st.bus_ctrl), ("bus_status", st.bus_status), @@ -2121,16 +2352,18 @@ impl Isp1040 { m.insert("sense".into(), u8_slice_to_toml(&sel.done.sense)); t.insert("selecting".into(), toml::Value::Table(m)); } + self.part.save_extra(st, &mut t); toml::Value::Table(t) } /// Unprocessed request entries survive: the ring thread is kicked and /// runs them after the load. - fn load_locked(st: &mut State, v: &toml::Value, now: i64) { + fn load_locked(&self, st: &mut State, v: &toml::Value, now: i64) { { let ctl = std::mem::take(&mut st.ctl); *st = State::new(); st.ctl = ctl; + self.part.load_extra(st, v); st.ctl.gen += 1; st.ctl.rqst_epoch += 1; st.ctl.resp_epoch += 1; diff --git a/src/dev/isp1040/tests.rs b/src/dev/isp1040/tests.rs index 4c17086c2..d4f701970 100644 --- a/src/dev/isp1040/tests.rs +++ b/src/dev/isp1040/tests.rs @@ -2,7 +2,8 @@ //! and ring contract. Host memory is a flat byte array behind a fake `PciDma`; the disk //! is a small raw image. //! -//! Ported from tests/qtest/qlogic-isp-test.c in jamesbraid/qemu-sgi-origami. +//! Ported from tests/qtest/qlogic-isp-test.c and tests/qtest/sn0-isp12160-test.c +//! in jamesbraid/qemu-sgi-origami. use super::*; use crate::irq::LevelProbe; @@ -59,15 +60,16 @@ fn image(name: &str, blocks: usize) -> std::path::PathBuf { } impl Rig { - /// A chip on fake memory, no disks. - fn bare() -> Rig { + fn on(isp: Isp1040) -> Rig { let mem = FakeMem::new(); - let isp = Isp1040::new(); Device::start(&isp); isp.state().connect_dma(mem.clone()); Rig { isp, mem } } + /// A chip on fake memory, no disks. + fn bare() -> Rig { Rig::on(Isp1040::new()) } + /// A chip with a disk of `blocks` blocks at target `id`. fn with_disk(name: &str, id: u8, blocks: usize) -> Rig { let r = Rig::bare(); @@ -1646,6 +1648,403 @@ fn a_controller_built_on_a_filled_bus_answers_for_its_target() { let _ = std::fs::remove_file(img); } + +struct MarkerTarget(u8); + +impl crate::scsi::ScsiTarget for MarkerTarget { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: vec![self.0] }) + } +} + +/// Target 0 on channel 1 must stay distinct from target 0 on channel 0. +#[test] +fn the_isp12160_routes_ring_requests_to_separate_buses() { + let bus0 = Isp1040::new_bus(); + let bus1 = Isp1040::new_bus(); + bus0.attach(0, Box::new(MarkerTarget(0x10))).unwrap(); + bus1.attach(0, Box::new(MarkerTarget(0x20))).unwrap(); + let r = Rig::on(Isp1040::new_12160(bus0, bus1)); + r.init_rings(4, 4); + + r.put_entry(0, &request(0x1234, 0x80, &[0x12, 0, 0, 0, 36, 0], &[(BUF, 36)])); + r.doorbell(1); + assert_eq!(r.mem.get(BUF, 1), vec![0x20], "channel 1 reaches its target 0"); + + r.put_entry(1, &request(0x1235, 0, &[0x12, 0, 0, 0, 36, 0], &[(BUF + 64, 36)])); + r.doorbell(2); + assert_eq!(r.mem.get(BUF + 64, 1), vec![0x10], "channel 0 keeps its own target 0"); +} + +struct ResetProbe(Arc); + +impl crate::scsi::ScsiTarget for ResetProbe { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: Vec::new() }) + } + + fn bus_reset(&mut self) { + self.0.fetch_add(1, Ordering::Relaxed); + } +} + +/// A snapshot retains both the chip-wide reset and the selected target reset, +/// and the resumed worker delivers both through the original channel buses. +#[test] +fn the_isp12160_preserves_queued_bus_and_target_resets_across_save_load() { + let src = Isp1040::new_12160(Isp1040::new_bus(), Isp1040::new_bus()); + { + let mut state = src.st.lock(); + state.reset_targets = true; + state.pending_target_resets.push_back((1, Some(2))); + } + let saved = src.save_state(); + + let resets0 = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let resets1 = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let selected_reset = Arc::new(std::sync::atomic::AtomicUsize::new(0)); + let bus0 = Isp1040::new_bus(); + let bus1 = Isp1040::new_bus(); + bus0.attach(1, Box::new(ResetProbe(resets0.clone()))).unwrap(); + bus1.attach(1, Box::new(ResetProbe(resets1.clone()))).unwrap(); + bus1.attach(2, Box::new(ResetProbe(selected_reset.clone()))).unwrap(); + let dst = Isp1040::new_12160(bus0, bus1); + dst.load_state(&saved).unwrap(); + Device::start(&dst); + dst.wait_idle(); + Device::stop(&dst); + + assert_eq!(resets0.load(Ordering::Relaxed), 1); + assert_eq!(resets1.load(Ordering::Relaxed), 1); + assert_eq!(selected_reset.load(Ordering::Relaxed), 2, "the full reset and queued target reset both run"); + assert_eq!(saved.get("reset_targets").and_then(toml::Value::as_bool), Some(true)); + assert_eq!(saved.get("target_resets").and_then(toml::Value::as_array).map(Vec::len), Some(1)); + let after = dst.save_state(); + assert_eq!(after.get("reset_targets").and_then(toml::Value::as_bool), Some(false)); + assert_eq!(after.get("target_resets").and_then(toml::Value::as_array).map(Vec::len), Some(0)); +} + +// ---- ISP12160 second channel and recovery commands ----------------------- + +fn rig12160() -> Rig { Rig::on(Isp1040::new_12160(Isp1040::new_bus(), Isp1040::new_bus())) } + +/// A 64-block image whose first block is `fill`, so a read says which disk answered. +fn filled_image(name: &str, fill: u8) -> std::path::PathBuf { + let p = image(name, 64); + let mut data = vec![0u8; 64 * BLOCK_SIZE]; + data[..BLOCK_SIZE].fill(fill); + std::fs::write(&p, data).expect("test image"); + p +} + +/// A 12160 with a different disk at target 1 on each channel. +fn disks_on_both_channels(name: &str) -> Rig { + let r = rig12160(); + for channel in 0..2u8 { + let img = filled_image(&format!("{name}{channel}"), 0xa0 + channel); + r.isp.bus_for_channel(channel).unwrap().attach_image(1, &img, false).expect("attach"); + } + r +} + +/// Issues entries one at a time on a 16-entry ring and returns each completion. +struct Seq<'a> { + r: &'a Rig, + n: u16, +} + +impl<'a> Seq<'a> { + fn new(r: &'a Rig) -> Self { + r.init_rings(16, 16); + Seq { r, n: 0 } + } + fn run(&mut self, handle: u32, target: u8, cdb: &[u8], segs: &[(u32, u32)]) -> [u8; ENTRY_SIZE] { + let slot = self.n; + self.r.put_entry(slot as u32, &request(handle, target, cdb, segs)); + self.n += 1; + self.r.doorbell(self.n); + self.r.status(slot as u32) + } +} + +const TUR: [u8; 6] = [0; 6]; + +/// Counts bus resets and answers any command with a full block pair, so a read +/// whose second buffer segment is unreachable stays outstanding. +struct CountingDisk(Arc); + +impl crate::scsi::ScsiTarget for CountingDisk { + fn request(&mut self, _req: &crate::scsi::ScsiRequest) -> std::io::Result { + Ok(crate::scsi::ScsiResponse { status: 0, data: vec![0; 2 * BLOCK_SIZE] }) + } + fn bus_reset(&mut self) { self.0.fetch_add(1, Ordering::Relaxed); } +} + +struct Held { + r: Rig, + /// Bus resets seen by the counting disks at target 1 on channel 0, target 1 on channel 1, + /// and target 2 on channel 1. + ch0: Arc, + ch1: Arc, + ch1_other: Arc, +} + +impl Held { + fn new() -> Held { + let counters: Vec<_> = (0..3).map(|_| Arc::new(std::sync::atomic::AtomicUsize::new(0))).collect(); + let (bus0, bus1) = (Isp1040::new_bus(), Isp1040::new_bus()); + bus0.attach(1, Box::new(CountingDisk(counters[0].clone()))).unwrap(); + bus1.attach(1, Box::new(CountingDisk(counters[1].clone()))).unwrap(); + bus1.attach(2, Box::new(CountingDisk(counters[2].clone()))).unwrap(); + let r = Rig::on(Isp1040::new_12160(bus0, bus1)); + r.init_rings(16, 16); + Held { r, ch0: counters[0].clone(), ch1: counters[1].clone(), ch1_other: counters[2].clone() } + } + /// A read whose payload store the fabric refuses: the adapter keeps it outstanding, which + /// is what the abort and reset commands act on. + fn hold(&self, slot: u32, handle: u32, target: u8) { + self.r.put_entry(slot, &request(handle, target, &cdb10(scsi_cmd::READ_10, 0, 2), + &[(BUF, BLOCK_SIZE as u32), (MEM as u32, BLOCK_SIZE as u32)])); + self.r.doorbell(slot as u16 + 1); + assert_eq!(g32(&self.r.status(slot), 4), 0, "handle {handle:#x} is held, not completed"); + } + fn command(&self, regs: &[u16]) -> u16 { + let status = self.r.cmd(regs); + self.r.isp.wait_idle(); + status + } + /// The completion posted in response slot `slot`: (handle, completion code). + fn answer(&self, slot: u32) -> (u32, u16) { + let s = self.r.status(slot); + (g32(&s, 4), g16(&s, 10)) + } +} + +const CH1: u16 = 0x8000; + +/// A driver sizing its queues and recognising the part reads these before any ring exists. +#[test] +fn the_isp12160_reports_its_own_identity_firmware_and_unmapped_registers() { + let r = rig12160(); + assert_eq!(r.r(reg::ID_HIGH), 0x1216); + assert_eq!(r.r(reg::BUS_CFG0), 0, "no revision field on this part"); + r.w(reg::DMA_CMD_CFG, 0xffff); + r.w(reg::DMA_DATA_CFG, 0xffff); + assert_eq!((r.r(reg::DMA_CMD_CFG), r.r(reg::DMA_DATA_CFG)), (0, 0), "the flat ISP1040 DMA blocks are not here"); + + // Mailbox 6 is seeded because an untouched output would also read zero. + r.w(r.mb(6), 0x1234); + r.w(r.mb(0), mbox::ABOUT_FIRMWARE); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!([r.r(r.mb(1)), r.r(r.mb(2))], [10, 4]); + assert_eq!(r.r(r.mb(6)), 0, "the attribute word is cleared, not echoed"); + r.ack(); + r.w(r.mb(0), mbox::GET_FIRMWARE_STATUS); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!([r.r(r.mb(1)), r.r(r.mb(2))], [0, MAX_REQUESTS]); + r.ack(); +} + +/// The driver switches register banks through bus configuration 1 bits 9:8; each of the +/// four selections must show its own words, and no other bit may stick. +#[test] +fn the_isp12160_selects_four_distinct_banks_and_keeps_only_the_bank_field() { + let r = rig12160(); + let selections = [0x0000, 0x0100, 0x0200, 0x0300]; + for sel in selections { + r.w(reg::BUS_CFG1, sel); + r.w(reg::BANK_BASE, 0x1000 + sel); + r.w(reg::BANK_BASE + 0x20, 0x2000 + sel); + } + for sel in selections { + r.w(reg::BUS_CFG1, sel); + assert_eq!(r.r(reg::BANK_BASE), 0x1000 + sel, "bank {sel:#x}"); + assert_eq!(r.r(reg::BANK_BASE + 0x20), 0x2000 + sel, "bank {sel:#x}"); + } + r.w(reg::BUS_CFG1, 0xb2c5); + assert_eq!(r.r(reg::BUS_CFG1), 0x0200); + + // A soft reset hands a re-initialising driver empty banks, not the previous run's words. + r.w(reg::BUS_CTRL, CTRL_SOFT_RESET); + assert_eq!(r.r(reg::BUS_CFG1), 0); + for sel in selections { + r.w(reg::BUS_CFG1, sel); + assert_eq!(r.r(reg::BANK_BASE), 0, "bank {sel:#x} after reset"); + } +} + +/// A second command submitted before the driver acknowledged the first result must not +/// run, or it would overwrite the answer the driver has not read. +#[test] +fn the_isp12160_ignores_a_mailbox_submission_while_the_result_is_unacknowledged() { + let r = rig12160(); + r.w(r.mb(0), mbox::NOP); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!(r.r(reg::SEMAPHORE) & SEMAPHORE_LOCK, SEMAPHORE_LOCK); + + r.w(r.mb(1), 0x9999); + r.w(r.mb(0), mbox::ABOUT_FIRMWARE); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::ABOUT_FIRMWARE, "the second submission did not run"); + assert_eq!(r.r(r.mb(1)), 0x9999); + + r.ack(); + r.w(reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT); + assert_eq!(r.r(r.mb(0)), mbox::COMPLETE); + assert_eq!(r.r(r.mb(1)), 10, "acknowledged, the same submission runs"); + r.ack(); +} + +/// An empty slot on either channel times out on its own bus; a driver probing channel 1 +/// must see an absent target, not a hang or the channel 0 answer. +#[test] +fn a_missing_target_times_out_on_each_channel() { + let r = rig12160(); + r.init_rings(16, 16); + r.clock(0); + // An absent target answers once its selection timeout has run out on + // the adapter's clock. + let mut now = 0; + for (slot, (handle, target)) in [(0x31, 5), (0x32, 0x80 | 5)].into_iter().enumerate() { + r.put_entry(slot as u32, &request(handle, target, &TUR, &[])); + r.doorbell(slot as u16 + 1); + assert_ne!(g32(&r.status(slot as u32), 4), handle, "nothing before the selection timeout"); + now += DEFAULT_SELECTION_TIMEOUT as i64 * MS; + r.clock(now); + let s = r.status(slot as u32); + assert_eq!(g32(&s, 4), handle); + assert_eq!(g16(&s, 10), IOCB_INCOMPLETE); + assert_eq!(g16(&s, 12), STATE_GOT_BUS); + assert_eq!(g16(&s, 14), STATUS_FLAG_TIMEOUT); + } +} + +/// A command refused on channel 1 must not disturb its neighbour on channel 0, and +/// a write must land on the disk of the channel it named. +#[test] +fn a_refused_command_on_channel_1_leaves_channel_0_and_the_media_alone() { + let r = disks_on_both_channels("refuse"); + let mut seq = Seq::new(&r); + // Collect each disk's power-on unit attention so the reads below see real answers. + seq.run(0xa0, 1, &TUR, &[]); + seq.run(0xa1, 0x81, &TUR, &[]); + + r.mem.put(BUF, &[0x5c; BLOCK_SIZE]); + let beyond_disk = seq.run(0xb1, 0x81, &cdb10(scsi_cmd::READ_10, 1000, 1), &[(BUF, BLOCK_SIZE as u32)]); + assert_eq!(g16(&beyond_disk, 8) & 0xff, CHECK_CONDITION as u16); + assert_eq!(r.mem.get(BUF, BLOCK_SIZE), vec![0x5c; BLOCK_SIZE], "the refused read stored nothing"); + + let good = seq.run(0xb0, 1, &cdb10(scsi_cmd::READ_10, 0, 1), &[(BUF + 0x1000, BLOCK_SIZE as u32)]); + assert_eq!(g16(&good, 10), IOCB_COMPLETE); + assert_eq!(r.mem.get(BUF + 0x1000, BLOCK_SIZE), vec![0xa0; BLOCK_SIZE], "channel 0 still reads its own disk"); + + r.mem.put(BUF + 0x2000, &[0x77; BLOCK_SIZE]); + seq.run(0xc0, 0x81, &cdb10(scsi_cmd::WRITE_10, 3, 1), &[(BUF + 0x2000, BLOCK_SIZE as u32)]); + seq.run(0xc1, 0x81, &cdb10(scsi_cmd::READ_10, 3, 1), &[(BUF + 0x3000, BLOCK_SIZE as u32)]); + seq.run(0xc2, 1, &cdb10(scsi_cmd::READ_10, 3, 1), &[(BUF + 0x4000, BLOCK_SIZE as u32)]); + assert_eq!(r.mem.get(BUF + 0x3000, BLOCK_SIZE), vec![0x77; BLOCK_SIZE], "channel 1 holds what was written"); + assert_eq!(r.mem.get(BUF + 0x4000, BLOCK_SIZE), vec![0; BLOCK_SIZE], "channel 0's block 3 is untouched"); +} + +/// Abort device names a channel in the top bit of its selector; the same target number +/// held on the other bus must stay outstanding. +#[test] +fn abort_device_retires_only_the_named_channels_commands() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::ABORT_DEVICE, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_ABORTED)); + assert_eq!(h.answer(1), (0, 0), "channel 0's command is still held"); + // The survivor is still the adapter's to abort. + assert_eq!(h.command(&[mbox::ABORT_DEVICE, 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(1), (0x41, IOCB_ABORTED)); +} + +/// Abort command names one handle: two commands to the same channel, target and unit differ +/// only by it, and a selector that matches a held handle on the wrong channel is an error. +#[test] +fn abort_command_matches_the_handle_and_the_whole_selector() { + let h = Held::new(); + h.hold(0, 0x51, 0x81); + h.hold(1, 0x52, 0x81); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x52]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x52, IOCB_ABORTED)); + assert_eq!(h.answer(1), (0, 0), "its neighbour on the same target is untouched"); + + assert_eq!(h.command(&[mbox::ABORT_COMMAND, 0x0100, 0, 0x51]), mbox::COMMAND_ERROR, + "a held handle named on the wrong channel is not a match"); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x51]), mbox::COMPLETE); + assert_eq!(h.answer(1), (0x51, IOCB_ABORTED)); +} + +/// A driver clearing a device it already drained must read success, while naming a handle +/// nothing is using must read as an error rather than claim a retirement. +#[test] +fn aborting_nothing_is_complete_but_aborting_an_unknown_handle_is_refused() { + let h = Held::new(); + assert_eq!(h.command(&[mbox::ABORT_DEVICE, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.command(&[mbox::ABORT_COMMAND, CH1 | 0x0100, 0, 0x77]), mbox::COMMAND_ERROR); + assert_eq!(h.answer(0), (0, 0), "neither posted a completion"); +} + +/// Target reset reaches one target on one channel; the same number on the other channel and +/// the neighbour on the same channel must not see it, and the reset target keeps serving. +#[test] +fn target_reset_is_scoped_to_one_target_on_one_channel() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::ABORT_TARGET, CH1 | 0x0100, 0, 0]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_RESET)); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!([h.ch0.load(Ordering::Relaxed), h.ch1.load(Ordering::Relaxed), h.ch1_other.load(Ordering::Relaxed)], + [0, 1, 0]); + + h.r.put_entry(2, &request(0x43, 0x81, &TUR, &[])); + h.r.doorbell(3); + assert_eq!(h.answer(1), (0x43, IOCB_COMPLETE)); +} + +/// Bus reset takes its channel from mailbox 2; mailbox 1 is the reset delay and here holds a value +/// that is not even a channel. The whole named bus resets, the other one does not. +#[test] +fn bus_reset_takes_its_channel_from_mailbox_two_and_resets_every_target_on_it() { + let h = Held::new(); + h.hold(0, 0x41, 1); + h.hold(1, 0x42, 0x81); + assert_eq!(h.command(&[mbox::BUS_RESET, 3, 1]), mbox::COMPLETE); + assert_eq!(h.answer(0), (0x42, IOCB_RESET)); + assert_eq!(h.answer(1), (0, 0)); + assert_eq!([h.ch0.load(Ordering::Relaxed), h.ch1.load(Ordering::Relaxed), h.ch1_other.load(Ordering::Relaxed)], + [0, 1, 1]); + assert_eq!(h.command(&[mbox::BUS_RESET, 0, 2]), mbox::PARAM_ERROR, "there is no third bus"); +} + +/// A bus reset leaves a unit attention on the disks of that channel only; the driver's +/// first command afterwards is expected to collect it. +#[test] +fn a_bus_reset_leaves_a_unit_attention_on_the_reset_channel_only() { + let r = disks_on_both_channels("uareset"); + let mut seq = Seq::new(&r); + seq.run(0x10, 1, &TUR, &[]); + seq.run(0x11, 0x81, &TUR, &[]); + assert_eq!(g16(&seq.run(0x12, 0x81, &TUR, &[]), 8) & 0xff, 0, "attention was collected"); + + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 1]), mbox::COMPLETE); + r.isp.wait_idle(); + let after = seq.run(0x13, 0x81, &TUR, &[]); + assert_eq!(g16(&after, 8) & 0xff, CHECK_CONDITION as u16); + assert_eq!(after[34] & 0x0f, 0x06, "unit attention"); + let other = seq.run(0x14, 1, &TUR, &[]); + assert_eq!(g16(&other, 8) & 0xff, 0, "channel 0 was not reset"); + assert_eq!(g16(&seq.run(0x15, 0x81, &TUR, &[]), 8) & 0xff, 0, "reported once"); +} + // ---- the hold-off after a bus reset --------------------------------------- // // Datasheet, "Bus-reset hold-off (0x0018)": after the reset the chip starts no @@ -1681,6 +2080,87 @@ fn a_bus_reset_puts_a_disks_write_cache_back() { assert_eq!(r.mem.get(BUF + 6, 1)[0] & 0x04, 0x04, "WCE is back"); } +/// A 12160 with a cached disk at target 1 on each channel whose guests cleared WCE on both. +fn cleared_write_cache_on_both_channels(name: &str) -> Rig { + let r = rig12160(); + for channel in 0..2u8 { + let bus = r.isp.bus_for_channel(channel).unwrap(); + bus.attach_cached_image(1, &image(&format!("{name}{channel}"), 64)).expect("attach"); + bus.select_write_cache(1, false); + assert!(!bus.write_cache_reported(1), "channel {channel}: the guest cleared it"); + } + r +} + +/// The write cache bit MODE SENSE(6) of the caching page reports for target 1 on `channel`. A TUR +/// goes first: it collects the reset's unit attention, and consumes the reset flag as a guest's +/// first command does, so the mode sense that follows returns its page. +fn wce_after_next_command(seq: &mut Seq, channel: u8, handle: u32) -> bool { + let target = channel << 7 | 1; + seq.run(handle, target, &TUR, &[]); + let done = seq.run(handle + 0x100, target, &[0x1a, 0x08, 0x08, 0, 24, 0], &[(BUF, 24)]); + assert_eq!(g16(&done, 10), IOCB_COMPLETE); + assert_eq!(seq.r.mem.get(BUF + 4, 1)[0], 0x08, "caching page"); + seq.r.mem.get(BUF + 6, 1)[0] & 0x04 != 0 +} + +/// Each channel's disks put their write cache back on that channel's reset and not on the +/// other's, so the flag has to be set for the channel the mailbox names. +#[test] +fn a_12160_bus_reset_of_channel_1_restores_write_cache_on_channel_1_only() { + let r = cleared_write_cache_on_both_channels("wce12160-ch1"); + let mut seq = Seq::new(&r); + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 1]), mbox::COMPLETE); + r.isp.wait_idle(); + assert!(!wce_after_next_command(&mut seq, 0, 0x31), "channel 0 was not reset"); + assert!(wce_after_next_command(&mut seq, 1, 0x32), "channel 1 was reset"); +} + +#[test] +fn a_12160_bus_reset_of_channel_0_restores_write_cache_on_channel_0_only() { + let r = cleared_write_cache_on_both_channels("wce12160-ch0"); + let mut seq = Seq::new(&r); + assert_eq!(r.cmd(&[mbox::BUS_RESET, 0, 0]), mbox::COMPLETE); + r.isp.wait_idle(); + assert!(!wce_after_next_command(&mut seq, 1, 0x33), "channel 1 was not reset"); + assert!(wce_after_next_command(&mut seq, 0, 0x34), "channel 0 was reset"); +} + +/// Both reset the chip's SCSI buses, which the disks see as a bus reset on each channel. +#[test] +fn a_12160_chip_reset_restores_write_cache_on_both_channels() { + let r = cleared_write_cache_on_both_channels("wce12160-chip"); + r.isp.reset(); + r.isp.wait_idle(); + let mut seq = Seq::new(&r); + assert!(wce_after_next_command(&mut seq, 0, 0x35), "channel 0"); + assert!(wce_after_next_command(&mut seq, 1, 0x36), "channel 1"); +} + +#[test] +fn a_12160_soft_reset_restores_write_cache_on_both_channels() { + let r = cleared_write_cache_on_both_channels("wce12160-soft"); + r.w(reg::BUS_CTRL, CTRL_SOFT_RESET); + r.isp.wait_idle(); + let mut seq = Seq::new(&r); + assert!(wce_after_next_command(&mut seq, 0, 0x37), "channel 0"); + assert!(wce_after_next_command(&mut seq, 1, 0x38), "channel 1"); +} + +/// The single-bus part has no bus 1: a mailbox that names it is ignored, and the one bus it has +/// keeps its disks' write cache as the guest set it. +#[test] +fn an_isp1040_bus_reset_naming_bus_1_leaves_the_reset_flag_unset() { + let r = Rig::bare(); + r.isp.bus().attach_cached_image(1, &image("wce-bus1", 64)).expect("attach"); + r.init_rings(4, 4); + r.isp.bus().select_write_cache(1, false); + bus_reset(&r, 0, 1); + r.isp.wait_idle(); + assert!(!r.isp.targets_reset[0].load(Ordering::Acquire)); + assert!(!r.isp.targets_reset[1].load(Ordering::Acquire)); +} + /// A TUR for the disk at target 1. fn tur1(handle: u32) -> [u8; ENTRY_SIZE] { request(handle, 1, &[0, 0, 0, 0, 0, 0], &[]) diff --git a/src/dev/isp12160.rs b/src/dev/isp12160.rs new file mode 100644 index 000000000..e0cf902de --- /dev/null +++ b/src/dev/isp12160.rs @@ -0,0 +1,337 @@ +//! The QLogic ISP12160: two SCSI buses behind one PCI function. +//! +//! The ISP12160 shares the ISP1040 ring, DMA, and mailbox mechanism, while +//! selecting targets from either channel. Each channel remains an explicit +//! [`ScsiBus`], including for monitor and run-time disc operations. +//! +//! Ported from hw/scsi/qlogic-isp.c and include/hw/scsi/qlogic-isp.h +//! in jamesbraid/qemu-sgi-origami. + +use std::collections::VecDeque; +use std::ops::Deref; + +use parking_lot::MutexGuard; + +use crate::dev::isp1040::{ + mbox, reg, Bank, Core, Isp1040, IspPart, State, CFG1_BANK_SELECT, FW_VERSION, IOCB_ABORTED, IOCB_RESET, MAILBOX_COUNT, + MAX_REQUESTS, MAX_TARGET, PCI_CLASS_SCSI, PCI_VENDOR_QLOGIC, SEMAPHORE_LOCK, STATE_ALL_PHASES, +}; +use crate::irq::IrqLine; +use crate::pci::{Bar, PciConfig, PciConfigPolicy, PciFunction, PciFunctionState, PciRegisterPolicy, PCI_INTA}; +use crate::scsi::{ScsiBus, ScsiIdentities, ScsiIdentity}; +use crate::snapshot::{get_field, hex_u16, hex_u32, load_u16_slice, toml_u16, toml_u32, toml_u8, u16_slice_to_toml}; +use crate::traits::{Device, Resettable, Saveable}; +use crate::workstation::DiscChanger; + +const ISP_DISK: ScsiIdentity = ScsiIdentity::new(b"SGI ", b"IRIS EMULATED ", b"1.0 "); +const ISP_CDROM: ScsiIdentity = ISP_DISK; + +pub(crate) struct Isp12160Part; + +impl IspPart for Isp12160Part { + fn name(&self) -> &'static str { "ISP12160" } + fn channels(&self) -> usize { 2 } + fn pci_config(&self, bars: [Bar; 6]) -> PciConfig { + let registers = [ + PciRegisterPolicy { reg_dword: 13, initial: 0x44, writable_mask: 0 }, + PciRegisterPolicy { reg_dword: 17, initial: 0x0001_0001, writable_mask: 0 }, + PciRegisterPolicy { reg_dword: 18, initial: 0, writable_mask: 0x100 }, + ]; + PciConfig::new_with_policy( + PCI_VENDOR_QLOGIC, + 0x1216, + PCI_CLASS_SCSI, + 6, + bars, + PCI_INTA, + PciConfigPolicy { + status: 0x02b0, + status_w1c_mask: 0, + command_mask: 0x0156, + latency_mask: 0xff, + cache_line_mask: 0xff, + registers: ®isters, + }, + ) + } + fn firmware_version(&self) -> [u16; 3] { [10, 4, FW_VERSION[2]] } + fn active_bank(&self, cfg1: u16) -> Bank { + match (cfg1 >> 8) & 3 { + 1 => Bank::Sxp, + 2 => Bank::Sxp1, + 3 => Bank::Dma, + _ => Bank::Risc, + } + } + fn checks_ram_bounds(&self) -> bool { true } + fn has_abort_commands(&self) -> bool { true } + fn target_on(&self, channel: u8, target: u8) -> u8 { target | (channel << 4) } + fn decode_target(&self, target: u8) -> (usize, u8) { ((target >> 4) as usize, target & 0x0f) } + fn id_high(&self) -> u16 { 0x1216 } + fn bus_cfg0(&self) -> u16 { 0 } + fn cfg1_write_mask(&self) -> u16 { 0x0300 } + fn unmodelled_dma_register(&self, off: u32) -> bool { + matches!(off, reg::DMA_CMD_CFG | reg::DMA_CMD_CTRL | reg::DMA_DATA_CFG | reg::DMA_DATA_CTRL) + } + fn completion_phases(&self, moved: u64, total: u64) -> u16 { + let mut flags = 0x1700; + if moved != 0 { + flags |= 0x0800; + if moved == total { flags |= 0x4000; } + } + flags + } + fn mailbox_ready(&self, semaphore: u16) -> bool { semaphore & SEMAPHORE_LOCK == 0 } + fn clear_target_parameters(&self, mailbox: &mut [u16; MAILBOX_COUNT]) { mailbox[6] = 0; } + fn set_reset_delay(&self, st: &mut State) { + st.reset_delay = st.mailbox[1]; + st.reset_delay1 = st.mailbox[2]; + } + fn reset_targets(&self, st: &mut State) -> bool { + st.reset_targets = true; + true + } + fn execute_mailbox(&self, core: &Core, st: &mut MutexGuard<'_, State>) -> bool { + let cmd = st.mailbox[0]; + match cmd { + mbox::LOAD_RAM_A64 => st.mailbox[0] = core.ram_transfer(st, true), + mbox::ABORT_COMMAND | mbox::ABORT_DEVICE | mbox::ABORT_TARGET => { + let device = st.mailbox[1]; + let valid_target = (device >> 8) & 0x7f <= MAX_TARGET as u16; + let target = ((device >> 8) & 0x7f) as u8 | (((device >> 15) as u8) << 4); + let lun = device as u8; + let handle = ((st.mailbox[2] as u32) << 16) | st.mailbox[3] as u32; + let hit = valid_target && st.active.is_some_and(|(t, l, h)| t == target + && (cmd == mbox::ABORT_TARGET || l == lun) && (cmd != mbox::ABORT_COMMAND || h == handle)); + let completion = if cmd == mbox::ABORT_TARGET { IOCB_RESET } else { IOCB_ABORTED }; + if hit { core.cancel_active(st, completion); } + let retired = core.retire_outstanding(st, |(t, l, h)| valid_target && t == target + && (cmd == mbox::ABORT_TARGET || l == lun) && (cmd != mbox::ABORT_COMMAND || h == handle), completion); + if cmd == mbox::ABORT_TARGET && valid_target { + st.pending_target_resets.push_back((target >> 4, Some(target & 15))); + } + st.mailbox[0] = if cmd == mbox::ABORT_COMMAND && !hit && retired == 0 { mbox::COMMAND_ERROR } else { mbox::COMPLETE }; + } + // Mailbox 1 is the reset delay; mailbox 2 selects the bus. + mbox::BUS_RESET => { + let channel = st.mailbox[2]; + if channel > 1 { st.mailbox[0] = mbox::PARAM_ERROR; } + else { + if st.active.is_some_and(|(target, _, _)| target >> 4 == channel as u8) { + core.cancel_active(st, IOCB_RESET); + } + core.retire_outstanding(st, |(target, _, _)| target >> 4 == channel as u8, IOCB_RESET); + st.pending_target_resets.push_back((channel as u8, None)); + core.note_bus_reset(usize::from(channel)); + core.kick(st); + st.mailbox[0] = mbox::COMPLETE; + } + } + _ => return false, + } + true + } + fn save_extra(&self, st: &State, t: &mut toml::map::Map) { + t.insert("reset_targets".into(), toml::Value::Boolean(st.reset_targets)); + t.insert("target_resets".into(), toml::Value::Array(st.pending_target_resets.iter().map(|(channel, target)| { + toml::Value::Array(vec![hex_u16(*channel as u16), hex_u16(target.unwrap_or(0xff) as u16)]) + }).collect())); + t.insert("sxp1_bank".into(), u16_slice_to_toml(&st.sxp1_bank)); + t.insert("dma_bank".into(), u16_slice_to_toml(&st.dma_bank)); + t.insert("reset_delay1".into(), hex_u16(st.reset_delay1)); + t.insert("outstanding".into(), toml::Value::Array(st.outstanding.iter().map(|&(target, lun, handle)| { + toml::Value::Array(vec![hex_u16(target as u16), hex_u16(lun as u16), hex_u32(handle)]) + }).collect())); + } + fn load_extra(&self, st: &mut State, v: &toml::Value) { + st.reset_targets = get_field(v, "reset_targets").and_then(|x| x.as_bool()).unwrap_or(false); + if let Some(toml::Value::Array(items)) = get_field(v, "target_resets") { + for item in items.iter().take(64) { + if let toml::Value::Array(fields) = item { + if fields.len() == 2 { + if let (Some(channel), Some(target)) = (toml_u8(&fields[0]), toml_u8(&fields[1])) { + if channel < 2 && (target < 16 || target == 0xff) { + st.pending_target_resets.push_back((channel, (target < 16).then_some(target))); + } + } + } + } + } + } + let u16f = |k: &str| get_field(v, k).and_then(toml_u16).unwrap_or(0); + st.reset_delay1 = u16f("reset_delay1"); + if let Some(m) = get_field(v, "sxp1_bank") { load_u16_slice(m, &mut st.sxp1_bank); } + if let Some(m) = get_field(v, "dma_bank") { load_u16_slice(m, &mut st.dma_bank); } + if let Some(toml::Value::Array(items)) = get_field(v, "outstanding") { + for item in items.iter().take(MAX_REQUESTS as usize) { + if let toml::Value::Array(fields) = item { + if fields.len() == 3 { + if let (Some(target), Some(lun), Some(handle)) = + (toml_u8(&fields[0]), toml_u8(&fields[1]), toml_u32(&fields[2])) { + if target < 32 && lun < 8 { st.outstanding.push((target, lun, handle)); } + } + } + } + } + } + } +} + +/// The two-channel controller. The shared ISP engine owns both buses. +pub struct Isp12160(Isp1040); + +impl Default for Isp12160 { + fn default() -> Self { Self::new(Self::new_bus(), Self::new_bus()) } +} + +impl Isp12160 { + /// Light the status bar's SCSI activity bits in `heartbeat`. + pub fn set_heartbeat(&self, heartbeat: std::sync::Arc) { + self.0.set_heartbeat(heartbeat); + } + + /// A 16-id bus with the ISP controller's disk and CD-ROM identities. + pub fn new_bus() -> ScsiBus { + ScsiBus::new(16, ScsiIdentities { disk: ISP_DISK, cdrom: ISP_CDROM }) + } + + /// The controller with its two channel buses already populated. + pub fn new(bus0: ScsiBus, bus1: ScsiBus) -> Self { + Self(Isp1040::new_12160(bus0, bus1)) + } + + /// Borrow one channel for the shared monitor commands. + pub fn channel(&self, channel: u8) -> Option> { + (channel < 2).then_some(Channel { adapter: self, channel }) + } + + pub fn register_locks(&self) { self.0.register_locks(); } +} + +impl Deref for Isp12160 { + type Target = Core; + fn deref(&self) -> &Core { &self.0 } +} + +/// One channel, with SCSI ids 0–15 as the monitor presents them. +pub struct Channel<'a> { + adapter: &'a Isp12160, + channel: u8, +} + +impl crate::scsi_monitor::ScsiTargets for Channel<'_> { + fn scsi_bus(&self) -> &ScsiBus { + self.adapter.bus_for_channel(self.channel).expect("valid ISP12160 channel") + } +} + +/// The machine API flattens the ISP12160's channels to ids 0–31. +/// Monitor commands use a `Channel` instead, so their ids remain 0–15. +impl DiscChanger for Isp12160 { + fn disc_status(&self) -> Vec<(usize, String, Vec, u64, u64)> { + (0..2).flat_map(|channel| { + crate::scsi_monitor::disc_status(&self.channel(channel).expect("valid ISP12160 channel")) + .into_iter() + .map(move |(id, current, discs, physical, logical)| (id + channel as usize * 16, current, discs, physical, logical)) + }).collect() + } + + fn load_disc(&self, id: usize, path: String) -> Result { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::load_disc(&self.channel(channel).unwrap(), target, path) + } + + fn eject_disc(&self, id: usize) -> Result { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::eject_disc(&self.channel(channel).unwrap(), target) + } + + fn eject_to_empty(&self, id: usize) -> Result<(), String> { + let (channel, target) = Self::split_id(id)?; + crate::scsi_monitor::eject_to_empty(&self.channel(channel).unwrap(), target) + } +} + +impl Isp12160 { + fn split_id(id: usize) -> Result<(u8, usize), String> { + if id >= 32 { return Err(format!("SCSI ID {id} is out of range")); } + Ok(((id / 16) as u8, id % 16)) + } +} + +impl Device for Isp12160 { + fn step(&self, cycles: u64) { Device::step(&self.0, cycles) } + fn start(&self) { Device::start(&self.0) } + fn stop(&self) { Device::stop(&self.0) } + fn is_running(&self) -> bool { Device::is_running(&self.0) } + fn get_clock(&self) -> u64 { Device::get_clock(&self.0) } + + fn register_commands(&self) -> Vec<(String, String)> { + let usage = crate::scsi_monitor::SCSI_USAGE; + vec![ + ("scsi".into(), format!("SCSI (QLogic ISP12160, channel 0): {usage} | scsi notes")), + ("scsi0".into(), "Alias for `scsi` (ISP12160 channel 0)".into()), + ("cow".into(), "COW overlays on ISP12160 channel 0: cow status | cow commit [id] | cow reset [id]".into()), + ("scsi1".into(), format!("SCSI (QLogic ISP12160, channel 1): {usage}")), + ("cow1".into(), "COW overlays on ISP12160 channel 1: same subcommands as `cow`".into()), + ] + } + + fn execute_command(&self, cmd: &str, args: &[&str], mut w: Box) -> Result<(), String> { + use std::io::Write as _; + let (channel, cow) = match cmd { + "scsi" | "scsi0" | "cow" => (0, cmd.starts_with("cow")), + "scsi1" | "cow1" => (1, cmd.starts_with("cow")), + _ => return Err("Command not found".into()), + }; + let ctl = self.channel(channel).expect("valid ISP12160 channel"); + if cow { + return crate::scsi_monitor::cow_command(&ctl, args, &mut *w); + } + if let Some(done) = crate::scsi_monitor::scsi_subcommand(&ctl, args, &mut *w) { + return done; + } + if channel == 0 && args.first().copied() == Some("notes") { + let _ = writeln!(w, "QLogic ISP12160: {} commands", self.total_commands()); + for note in self.notes() { let _ = writeln!(w, " note: {note}"); } + return Ok(()); + } + Err(format!("Usage: {}", crate::scsi_monitor::SCSI_USAGE)) + } +} + +impl Resettable for Isp12160 { + fn power_on(&self) { + self.0.reset(); + } +} + +impl Saveable for Isp12160 { + fn save_state(&self) -> toml::Value { Saveable::save_state(&self.0) } + fn load_state(&self, v: &toml::Value) -> Result<(), String> { Saveable::load_state(&self.0, v) } +} + +impl PciFunction for Isp12160 { + fn state(&self) -> &PciFunctionState { PciFunction::state(&self.0) } + fn bar_read(&self, bar: usize, off: u32, width: usize) -> u32 { PciFunction::bar_read(&self.0, bar, off, width) } + fn bar_write(&self, bar: usize, off: u32, val: u32, width: usize) { PciFunction::bar_write(&self.0, bar, off, val, width) } + fn attached(&self) { PciFunction::attached(&self.0) } + fn reset(&self) { PciFunction::reset(&self.0) } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The machine disc API uses target 17 for channel 1, target 1. + #[test] + fn disc_status_flattens_the_second_channels_target_ids() { + let bus0 = Isp12160::new_bus(); + let bus1 = Isp12160::new_bus(); + bus1.attach_device(1, "", true, Vec::new(), false, None, false).unwrap(); + let isp = Isp12160::new(bus0, bus1); + + let status = DiscChanger::disc_status(&isp); + assert_eq!(status.iter().map(|drive| drive.0).collect::>(), vec![17]); + } +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index ef508a00b..51c0eff5d 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -78,3 +78,4 @@ pub mod dp83843; pub mod ics1890; pub mod timekeeper; pub mod prom_image; +pub mod isp12160; diff --git a/src/scsi.rs b/src/scsi.rs index bfb2ffda8..eb82e3347 100644 --- a/src/scsi.rs +++ b/src/scsi.rs @@ -162,6 +162,7 @@ pub struct ScsiDevice { kind: DeviceKind, buffer: Vec, // Internal buffer for WRITE_BUFFER/READ_BUFFER commands pending_sense: [u8; 18], // Stored sense data, served on REQUEST_SENSE + pending_attention: Option, /// Physical sector size: always 2048 for CD-ROM (Yellow Book), 512 for HDD. /// Used for TOC lead-out LBA (CD spec requires TOC in physical sectors). phys_block_size: u64, @@ -222,6 +223,7 @@ impl ScsiDevice { kind: if is_cdrom { DeviceKind::Cdrom } else { DeviceKind::Disk }, buffer: vec![0u8; SCSI_BUFFER_SIZE], pending_sense: [0u8; 18], + pending_attention: None, phys_block_size: if is_cdrom { CDROM_SECTOR } else { 512 }, // CD-ROM drives default to 2048-byte logical blocks (Sony CDU-76S behaviour). // HDD defaults to 512. dksc switches CD-ROM to 512 for EFS, back to 2048 for ISO. @@ -242,6 +244,7 @@ impl ScsiDevice { kind: DeviceKind::Cdrom, buffer: vec![0u8; SCSI_BUFFER_SIZE], pending_sense: [0u8; 18], + pending_attention: None, phys_block_size: CDROM_SECTOR, logical_block_size: 2048, identity, @@ -326,6 +329,10 @@ impl ScsiDevice { ScsiResponse { status: 0x02, data: vec![] } } + fn take_attention(&mut self) -> Option { + self.pending_attention.take().or_else(|| self.media.take_changed().then_some(0x28)) + } + fn exec_test_unit_ready(&mut self, _cdb: &[u8]) -> Result { if self.media.take_changed() { // Sense key 0x06 UNIT_ATTENTION, ASC 0x28 "Not Ready to Ready Transition / Medium Changed" @@ -1118,11 +1125,10 @@ impl ScsiTarget for ScsiDevice { } // Unit attention fires before any other command (except REQUEST_SENSE/INQUIRY). - if req.cdb[0] != scsi_cmd::REQUEST_SENSE - && req.cdb[0] != scsi_cmd::INQUIRY - && self.media.take_changed() - { - return Ok(self.check_condition(0x06, 0x28, 0x00)); + if req.cdb[0] != scsi_cmd::REQUEST_SENSE && req.cdb[0] != scsi_cmd::INQUIRY { + if let Some(asc) = self.take_attention() { + return Ok(self.check_condition(0x06, asc, 0x00)); + } } let mut response = match req.cdb[0] { @@ -1268,6 +1274,12 @@ impl ScsiTarget for ScsiDevice { fn block_size(&self) -> Option { Some(self.logical_block_size) } + fn bus_reset(&mut self) { + self.pending_sense = [0; 18]; + self.pending_sense[0] = 0x70; + self.pending_attention = Some(0x29); + } + fn reset_block_size(&mut self, cdrom_512: bool) { if cdrom_512 && self.is_cdrom() { self.logical_block_size = 512; @@ -1867,12 +1879,38 @@ mod tests { cd.changer().unwrap().load(&chd.display().to_string()).unwrap(); assert!(cd.has_media()); assert_eq!(cd.size(), 16 * 2048, "the CHD's size, not the file's"); - // The medium change is reported once, then the disc reads. + // A reset is reported before a media change that was already pending. + crate::scsi::ScsiTarget::bus_reset(&mut cd); + let reset = cd.request(&request(&[0, 0, 0, 0, 0, 0], None)).unwrap(); + assert_eq!(reset.status, 0x02, "unit attention after the reset"); + let reset_sense = cd.request(&request(&[0x03, 0, 0, 0, 18, 0], None)).unwrap(); + assert_eq!(reset_sense.data[12], 0x29, "reset attention comes first"); + let tur = cd.request(&request(&[0, 0, 0, 0, 0, 0], None)).unwrap(); - assert_eq!(tur.status, 0x02, "unit attention after the change"); + assert_eq!(tur.status, 0x02, "the pending media change follows"); + let sense = cd.request(&request(&[0x03, 0, 0, 0, 18, 0], None)).unwrap(); + assert_eq!(sense.data[12], 0x28, "media change keeps its ASC"); let read = cd.request(&request(&[0x28, 0, 0, 0, 0, 3, 0, 0, 1, 0], None)).unwrap(); assert_eq!(read.status, 0x00); assert_eq!(read.data, vec![0x42u8; 2048]); let _ = std::fs::remove_dir_all(dir); } + #[test] + fn bus_reset_reports_unit_attention_once_and_keeps_block_size() { + let mut dev = cdrom(); + dev.logical_block_size = 512; + dev.set_sense(0x05, 0x20, 0x00); + crate::scsi::ScsiTarget::bus_reset(&mut dev); + + let old_sense = run(&mut dev, &[0x03, 0, 0, 0, 18, 0]); + assert_eq!((old_sense.data[2], old_sense.data[12]), (0, 0), "reset clears previous sense"); + let response = run(&mut dev, &[0x00, 0, 0, 0, 0, 0]); + assert_eq!(response.status, 0x02, "a bus reset arms unit attention"); + let sense = run(&mut dev, &[0x03, 0, 0, 0, 18, 0]); + assert_eq!(sense.data[12], 0x29, "the reset attention has the reset ASC"); + + let response = run(&mut dev, &[0x00, 0, 0, 0, 0, 0]); + assert_eq!(response.status, 0x00, "the attention is reported once"); + assert_eq!(dev.block_sizes().1, 512, "bus reset leaves the selected block size alone"); + } } diff --git a/src/scsi/bus.rs b/src/scsi/bus.rs index 4283d763f..cb978d19a 100644 --- a/src/scsi/bus.rs +++ b/src/scsi/bus.rs @@ -51,8 +51,15 @@ pub trait ScsiTarget: Send { /// The logical block size, for a target that has one. fn block_size(&self) -> Option { None } - /// A controller's bus reset: block size back to what the controller - /// leaves, with a CD-ROM at 512 bytes if `cdrom_512`. + /// A SCSI bus reset reached this target, which owes the initiator a + /// unit attention on its next command. The ISP12160 calls this when the + /// guest resets either of its buses; the WD33C93A, AIC-7880 and ISP1040 + /// models never told their targets about a reset and still do not. + fn bus_reset(&mut self) {} + + /// The AIC-7880's bus reset: block size back to what that controller + /// leaves, with a CD-ROM at 512 bytes if `cdrom_512`. Only the AIC-7880 + /// calls this. fn reset_block_size(&mut self, _cdrom_512: bool) {} /// Put the mode parameters a guest can change back to what the target From ee81e7c6ec4ca0950e2cb7bc9c1172ef67fa5b23 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 05:16:10 -0700 Subject: [PATCH 07/22] cpu: carry the R10000 family's uncached attribute through mapped accesses The R10000 drives a two-bit uncached attribute on the system bus. For an unmapped xkphys access it is address bits 58:57; for a mapped reference it comes from the TLB's uncached-attribute field, which EntryLo0/1 bits 63:62 load (R10000 User's Manual 007-2490-001). EntryLo already keeps those bits on the family, and a TLB entry holds them, but the entry's physical base dropped them, so a mapped uncached page always reached attribute zero. Carry them to physical bits 58:57 only for pages the model's C table decodes as uncached, so cached mappings are unchanged. The TLB already holds that table for its shadow entries; reading it there, and recomputing each entry's base when a table is installed, avoids a second decode of C. An entry with no attribute bits set, the usual case, skips the C table. The attribute reaches the bus only on a board that decodes physical bits 58:57. The Octane, Indigo2 and O2 mask the physical address to 40, 32 and 32 bits before the bus, which folds it away, and a test shows that at the executor. The R4400 and R5000 never store the bits. Tests cover the attribute and the board masks, and the C table (a value the model decodes as cached carries nothing; a table installed after the entry was written decides). The uncached-attribute comments cite NEC U10278EJ4V0UM, the fourth edition of the VR10000/VR12000 user's manual, section 11.3, by document number; the manual itself is not added to the repository. --- src/cpu/mips_exec_test.rs | 56 +++++++++++++++++++++++++++++++++++++++ src/cpu/mips_tlb.rs | 43 ++++++++++++++++++++++++++---- src/cpu/mips_tlb_test.rs | 49 ++++++++++++++++++++++++++++++++++ 3 files changed, 143 insertions(+), 5 deletions(-) diff --git a/src/cpu/mips_exec_test.rs b/src/cpu/mips_exec_test.rs index 9596a3c5d..f1ac7902d 100644 --- a/src/cpu/mips_exec_test.rs +++ b/src/cpu/mips_exec_test.rs @@ -4370,6 +4370,62 @@ mod tests { check::(0xF_FFFF_FFFF); } + /// A mapped uncached page's attribute (EntryLo 63:62, carried to physical + /// bits 58:57) reaches the bus on a board that decodes them, and is folded + /// away on the Octane (40 bits), Indigo2 and O2 (32), which is what keeps + /// those machines unchanged. + #[test] + fn mapped_uncached_attribute_is_folded_by_the_boards_physical_mask() { + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + use crate::cpu::mips_tlb::MipsTlb; + const VA: u64 = 0x4000_0000; + + fn translate>>( + phys_mask: u64, alias: Option<(u64, u64, u64)>, pfn: u64, + ) -> u64 { + let bus: Arc = Arc::new(MockMemory::new()); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip22_bus::memory_map(false))); + let mut exec = Box::new(MipsExecutor::::new( + bus, MipsTlb::default(), &cfg)); + disable_jitv2(&mut exec); + exec.install_status_cb(); + let dmtc0 = |exec: &mut Box>, rd: u32, value: u64| { + exec.core.write_gpr(1, value); + let op = (OP_COP0 << 26) | (RS_DMTC0 << 21) | (1 << 16) | (rd << 11); + assert_eq!(exec.exec(op), EXEC_COMPLETE); + }; + let status = (exec.core.cp0_status & !crate::cpu::mips_core::STATUS_ERL) | STATUS_KX; + dmtc0(&mut exec, 12, status as u64); + dmtc0(&mut exec, 10, VA); + dmtc0(&mut exec, 0, 0); + dmtc0(&mut exec, 5, 0); + // C=2 (uncached), valid, dirty, attribute 1 in bits 63:62. + dmtc0(&mut exec, 2, (1 << 62) | (pfn << 6) | (2 << 3) | (1 << 2) | (1 << 1)); + dmtc0(&mut exec, 3, 0); + assert_eq!(exec.exec((OP_COP0 << 26) | (RS_TLB << 21) | FUNCT_TLBWI), EXEC_COMPLETE); + exec.phys_mask = phys_mask; + exec.phys_alias = alias; + exec.debug_translate(VA).phys + } + + type R10K = crate::cpu::mips_cache_shadow::R10000ShadowCache; + type R14K = crate::cpu::mips_cache_shadow::R12000ShadowCache; + const PA: u64 = 0x0d10_0000; + assert_eq!(translate::(u64::MAX, None, PA >> 12), (1 << 57) | PA, + "no mask: the attribute reaches the bus"); + for (name, mask) in [("Octane", (1u64 << 40) - 1), ("Indigo2/O2", u32::MAX as u64)] { + assert_eq!(translate::(mask, None, PA >> 12), PA, "{name} folds it away"); + } + assert_eq!(translate::(u64::MAX, None, PA >> 12), PA, + "a model without the attribute never stores it"); + }) + .unwrap() + .join() + .unwrap(); + } + /// The Indy, Indigo2 and O2 decode 32 physical bits, and IRIX relies on /// it: on the IP28 the kernel stack's wired entry carries PTE software /// bits as PA[34:33]. `phys_mask` must fold them away before the bus. diff --git a/src/cpu/mips_tlb.rs b/src/cpu/mips_tlb.rs index 3eaed8e8c..ff504582f 100644 --- a/src/cpu/mips_tlb.rs +++ b/src/cpu/mips_tlb.rs @@ -147,6 +147,23 @@ pub enum AccessType { Debug = 3, } +/// The physical base a TLB entry's EntryLo maps. A model that keeps the +/// R10000 family's uncached attribute (EntryLo bits 63:62, see +/// `CpuModel::R10K_FAMILY`) has it carried to physical bits 58:57 +/// for an uncached page, where an unmapped xkphys access carries it; other +/// models never store those bits. For a mapped reference the R10000 takes the +/// two-bit uncached attribute from the TLB's uncached-attribute field, loaded +/// from EntryLo0/1 bits 63:62 (NEC U10278EJ4V0UM, VR10000/VR12000 user's +/// manual, 4th edition, section 11.3; R10000 User's Manual 007-2490-001). +/// +/// Whether the page is uncached is the caller's decode of C through the model's +/// table (`MipsTlb::c_attr`), not another decode here. +#[inline] +fn pfn_base(lo: u64, pfn_mask: u64, uncached: bool) -> u64 { + let attribute = (((lo >> 62) & 3) << 57) & 0u64.wrapping_sub(uncached as u64); + ((lo << 6) & pfn_mask) | attribute +} + /// TLB Entry structure matching R4000 TLB format #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub struct TlbEntry { @@ -1008,6 +1025,21 @@ impl Default for MipsTlb { } } +impl MipsTlb { + /// `pfn_base` for both halves of an entry, deciding "uncached" with the + /// model's C table. + #[inline] + fn pfn_bases(&self, lo: &[u64; 2], pfn_mask: u64) -> [u64; 2] { + // No attribute bits set, the usual case: nothing to carry, and the C + // table need not be read. + if (lo[0] | lo[1]) >> 62 == 0 { + return [(lo[0] << 6) & pfn_mask, (lo[1] << 6) & pfn_mask]; + } + let uncached = |lo: u64| self.c_attr[((lo >> 3) & 7) as usize] == CacheAttr::Uncached; + [pfn_base(lo[0], pfn_mask, uncached(lo[0])), pfn_base(lo[1], pfn_mask, uncached(lo[1]))] + } +} + impl Tlb for MipsTlb { #[inline] fn translate(&mut self, virt_addr: u64, asid: u8, access_type: AccessType) -> TlbResult { @@ -1177,10 +1209,7 @@ impl Tlb for MipsTlb { entry.vpn_hi64 = entry.entry_hi & entry.vcmp64; entry.offset_mask = (1u64 << entry.selector_bit_shift) - 1; let pfn_mask = 0xFF_FFFF_FFFF_000 & !entry.offset_mask; - entry.pfn_base = [ - (entry.entry_lo[0] << 6) & pfn_mask, - (entry.entry_lo[1] << 6) & pfn_mask, - ]; + entry.pfn_base = self.pfn_bases(&entry.entry_lo, pfn_mask); self.irregular_mask |= mask & (mask + 1) != 0; if self.invalidate_on_write { self.invalidate_overlapped(index, &entry); @@ -1322,6 +1351,10 @@ impl Tlb for MipsTlb { fn set_cache_attrs(&mut self, table: [CacheAttr; 8]) { self.c_attr = table; for i in 0..self.num_entries { + // pfn_base carries the attribute for pages the new table decodes + // as uncached, so it follows the table. + let pfn_mask = 0xFF_FFFF_FFFF_000 & !self.entries[i].offset_mask; + self.entries[i].pfn_base = self.pfn_bases(&self.entries[i].entry_lo, pfn_mask); self.shadow[i] = ShadowEntry::from_entry(&self.entries[i], &self.c_attr); } } @@ -1380,7 +1413,7 @@ impl Tlb for MipsTlb { vcmp64, vpn_hi64: entry_hi & vcmp64, offset_mask, - pfn_base: [(lo[0] << 6) & pfn_mask, (lo[1] << 6) & pfn_mask], + pfn_base: self.pfn_bases(&lo, pfn_mask), }; } } diff --git a/src/cpu/mips_tlb_test.rs b/src/cpu/mips_tlb_test.rs index be121f291..65ca06067 100644 --- a/src/cpu/mips_tlb_test.rs +++ b/src/cpu/mips_tlb_test.rs @@ -535,3 +535,52 @@ fn tlb_write_inside_a_larger_entry_count_is_kept() { assert_eq!(r10000.read(63).entry_hi, entry.entry_hi, "...and its last slot"); } } + +/// A mapped uncached page carries EntryLo's uncached attribute (bits 63:62) +/// to the bus as physical bits 58:57; a cached page never does, whatever those bits +/// hold, and no attribute leaves the PFN alone. "Uncached" is the model's C +/// table, so a model that decodes a C value differently, or a table installed +/// after the entry was written, changes the answer. +#[test] +fn only_uncached_pages_carry_the_uncached_attribute() { + use super::{MipsTlb, Tlb, TlbEntry}; + use crate::cpu::mips_exec::CacheAttr; + // MipsTlb is large; give the one instance room, as its neighbours do. + std::thread::Builder::new() + .stack_size(16 * 1024 * 1024) + .spawn(|| { + let pfn = 0x0d100u64 << 6; + let base = |tlb: &MipsTlb| tlb.read(0).pfn_base; + let write = |tlb: &mut MipsTlb, lo0: u64, lo1: u64| { + let mut entry = TlbEntry::new(); + entry.entry_hi = 0xC000_0000_0004_0000; + entry.entry_lo = [lo0, lo1]; + tlb.write(0, entry); + }; + let mut tlb = Box::new(MipsTlb::new(64)); + // The default table: C=2 and the reserved values are uncached. + for (cca, expect) in [(2u64, 1u64 << 57), (6, 1 << 57), (3, 0), (5, 0)] { + let lo = (1 << 62) | pfn | (cca << 3) | 0x6; + write(&mut tlb, lo, lo); + assert_eq!(base(&tlb), [0x0d10_0000 | expect; 2], "cca {cca}"); + } + write(&mut tlb, pfn | (2 << 3), 0); + assert_eq!(base(&tlb)[0], 0x0d10_0000, "no attribute leaves the PFN alone"); + + // The R10000 table reads C=7 (uncached accelerated) as uncached and + // 5 as cacheable; a model that read 7 as cacheable carries nothing. + let mut table = [CacheAttr::Uncached; 8]; + table[3] = CacheAttr::Cacheable; + table[5] = CacheAttr::CacheableCoherent; + tlb.set_cache_attrs(table); + let lo = (1 << 62) | pfn | (7 << 3) | 0x6; + write(&mut tlb, lo, lo); + assert_eq!(base(&tlb), [0x0d10_0000 | 1 << 57; 2], "C=7 uncached under this table"); + table[7] = CacheAttr::Cacheable; + tlb.set_cache_attrs(table); + assert_eq!(base(&tlb), [0x0d10_0000; 2], "the table, installed later, decides"); + }) + .unwrap() + .join() + .unwrap(); +} From 68aa36879ef21515fcddade345546210606f17f9 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 08/22] bedrock: add the chip and its processor interfaces The Fuel's node controller is one chip: two processor interfaces, the memory controller, the I/O, local-bus, network and crossbar blocks. As seven separate bus devices joined by closures, an interrupt took about 27 locks to reach the CPU: a PI mask store took a reentrant dispatch lock, the state lock, the hooks lock and two locks per output line to recompute all ten outputs. The PI comparators only advanced when the CPU's batch hook polled both interfaces every 1000 instructions. Model Bedrock as one device with the PIs first; the other blocks join it as plain fields. Each PI's interrupt inputs (pending banks, masks, requests, RT and profile status and enables, error summary and causes) are atomics. A store changes one and publishes the slot's levels into Cause bits 10-14 of its processor's interrupt word, as HEART does on the Octane, with no lock: a writer that published a stale level reads the inputs again and corrects it, so the last write matches the registers. A mask store, a pending-modify store and a counter read take no lock. The counter's origin is read without a lock through a sequence word. The comparators fire on host timers, as the Hub PI's do, so they reach the CPU when due rather than at the CPU's next batch. The board names its fitted processor slots (Population): one bit per slot, CPU n in the nth slot of the order PI0/A, PI1/A, PI0/B, PI1/B, so two processors sit one per interface. Every fitted slot reads present and starts with its CPU-enable bit set; the CPU-number register answers the reading processor's slice. The Fuel fits PI0/A. A store that arms a slot with no processor (a nonzero mask or error cause, a timer enable, a CPU enable) or any NMI is recorded where it is made, so the publish never visits an empty slot. The local XIO register form answers the requester's own PI when processors sit on both. The graphics flow-control registers reuse the VPro's PiGfx block, which the Hub PI already embeds; IRIX's VPro driver writes them and reads the credit counter back. They store what is written and count nothing. Tests cover reset survivors, the counter's wrap and pause, every output reaching its Cause bit and no other, timer interrupts with no register polling, slot presence, the CPU number per processor, the refusal records, racing stores ending with the word matching the registers, and the hot paths finishing while another thread holds every chip lock. --- src/dev/bedrock/mod.rs | 360 +++++++++++++++ src/dev/bedrock/pi.rs | 899 ++++++++++++++++++++++++++++++++++++ src/dev/bedrock/pi/tests.rs | 283 ++++++++++++ src/dev/bedrock/tests.rs | 94 ++++ src/dev/mod.rs | 1 + 5 files changed, 1637 insertions(+) create mode 100644 src/dev/bedrock/mod.rs create mode 100644 src/dev/bedrock/pi.rs create mode 100644 src/dev/bedrock/pi/tests.rs create mode 100644 src/dev/bedrock/tests.rs diff --git a/src/dev/bedrock/mod.rs b/src/dev/bedrock/mod.rs new file mode 100644 index 000000000..387f453b7 --- /dev/null +++ b/src/dev/bedrock/mod.rs @@ -0,0 +1,360 @@ +//! Bedrock, the node controller of the IP35 boards, as one device: two +//! processor interfaces (PI), the memory and directory controller (MD) with +//! its memory, the I/O block (II) that faces the crossbar widgets, the local +//! block (LB), the network interface (NI) and the crossbar block (XB). +//! +//! A board gives the chip its processors, banks and clock in a +//! [`BedrockConfig`], maps the faces it hands out into its processor bus, +//! and gives each processor's interrupt word to [`Bedrock::set_cpu_interrupts`]. +//! The units are plain fields; each locks only its own state. +//! +//! # Locks +//! +//! - PI: the interrupt inputs (pending banks, masks, requests, timer and +//! error status) are atomics, so a mask store, a pending-modify store and +//! a counter read take no lock. `state` holds the stored words and the +//! graphics registers; `timer` the comparators and the host timer. +//! +//! Ported from hw/misc/sgi-bedrock*.c in jamesbraid/qemu-sgi-origami. + +mod pi; +#[cfg(test)] +mod tests; + +pub use pi::{Population, Refused}; + +use crate::dev::heart::Clock; +use crate::hptimer::{TimerId, TimerManager, TimerReturn}; +use crate::irq::{InterruptController, IrqLine}; +use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Device, Resettable, BUS_ERR, BUS_OK}; +use pi::{Levels, Pi}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, OnceLock, Weak}; +use std::time::Duration; + +/// Unit bases inside the chip's 8 MiB register window, the same in both XIO +/// register forms, and the registers a board or test names. +pub mod reg { + pub const PI0_BASE: u64 = 0x00_0000; + pub const PI1_BASE: u64 = 0x20_0000; + pub const IO_BASE: u64 = 0x40_0000; + pub const LB_BASE: u64 = 0x60_0000; + pub const NI_BASE: u64 = 0x68_0000; + pub const XB_BASE: u64 = 0x70_0000; + pub const MD_BASE: u64 = 0x78_0000; + pub const WINDOW: u64 = 0x80_0000; + /// The public Linux Hub I/O protection-override key literal. + /// + /// Linux `arch/mips/include/asm/sn/sn0/hubio.h` at + /// `db2ddb87143519e20a95aa36c60b36107b736a58` defines + /// `IIO_PROTECT_OVRRD_KEY` as this value (lines 31, 47). This establishes + /// the literal's provenance only; it does not establish a register mapping. + pub const IIO_PROTECT_OVRRD_KEY: u64 = 0x53474972756c6573; + pub use super::pi::reg as pi; +} + +/// What a board gives its Bedrock. Fixed for the chip's life. +pub struct BedrockConfig { + /// Host nanoseconds: the PIs' counter and timers run on it, and the + /// other units' clocked work. + pub clock: Clock, + /// The processor slots the board fits. CPU n sits in + /// `processors.slot_of(n)`; a fitted slot starts with its enable bit set. + pub processors: Population, +} + +/// The two XIO register forms. `Local` answers the requester's own PI at +/// offset 0 when the board fits processors on both PIs; otherwise it is the +/// node form. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Form { + Node, + Local, +} + +/// A PI's host timer: the manager it runs on and the chip it fires into. +struct HostTimer<'a> { + tm: &'a TimerManager, + chip: &'a Weak, +} +impl HostTimer<'_> { + /// A one-shot `delay` nanoseconds out that latches PI `interface`'s due + /// comparators and arms the next. + fn arm(&self, interface: u8, delay: u64) -> TimerId { + self.tm.add_one_shot(Duration::from_nanos(delay), (self.chip.clone(), interface), |(chip, interface)| { + if let Some(chip) = chip.upgrade() { + chip.expire(usize::from(*interface)); + } + TimerReturn::Delete + }) + } +} + +pub struct Bedrock { + clock: Clock, + processors: Population, + pi: [Pi; 2], + levels: Levels, + timers: OnceLock>, + weak: Weak, +} + +impl Bedrock { + pub fn new(config: BedrockConfig) -> Result, String> { + let BedrockConfig { clock, processors } = config; + let now = clock(); + Ok(Arc::new_cyclic(|weak: &Weak| Self { + pi: [Pi::new(0, processors, now), Pi::new(1, processors, now)], + levels: Levels::default(), + timers: OnceLock::new(), + clock, + processors, + weak: weak.clone(), + })) + } + + pub fn register_locks(self: &Arc) { + use crate::locks::register_mutex_fn; + register_mutex_fn("bedrock-pi0::state", self, |c| c.pi[0].state_locked()); + register_mutex_fn("bedrock-pi1::state", self, |c| c.pi[1].state_locked()); + register_mutex_fn("bedrock-pi0::timer", self, |c| c.pi[0].timer_locked()); + register_mutex_fn("bedrock-pi1::timer", self, |c| c.pi[1].timer_locked()); + } + + // The faces a board maps into its processor bus. Each is its own type, + // so no access tests which face it came through. + + /// The 8 MiB register window in one of its two XIO forms. + pub fn registers(self: &Arc, form: Form) -> Arc { + if form == Form::Local && (0..2).any(|slot| self.processors.fitted(1, slot)) { + Arc::new(Registers::(self.clone())) + } else { + Arc::new(Registers::(self.clone())) + } + } + + // Processors. + + /// Where CPU `cpu`'s IP2..IP6 go: Cause bits 10-14 of its interrupt word + /// (`MipsCpu::interrupts_ptr`). The chip writes no other bit, and no + /// other device may write these. The word must outlive the chip. + pub fn set_cpu_interrupts(&self, cpu: usize, word: *const AtomicU64) { + if let Some((interface, slot)) = self.processors.slot_of(cpu) { + let pi = &self.pi[usize::from(interface)]; + pi.set_word(slot, word); + pi.publish(&self.levels); + } + } + /// The PIs' comparators and profile timers fire on `tm`. Held weakly. + pub fn set_timer_manager(&self, tm: &Arc) { + let _ = self.timers.set(Arc::downgrade(tm)); + let now = (self.clock)(); + let host = self.host_timer(tm); + for pi in &self.pi { + pi.reschedule(Some(&host), now); + } + } + /// The L1 UART's line: bank 0 bit 4 of both PIs. + pub fn uart_irq(self: &Arc) -> IrqLine { + IrqLine::new(Arc::new(Uart(self.weak.clone())), 0) + } + fn host_timer<'a>(&'a self, tm: &'a TimerManager) -> HostTimer<'a> { + HostTimer { tm, chip: &self.weak } + } + fn timer_manager(&self) -> Option> { + self.timers.get().and_then(Weak::upgrade) + } + /// PI `interface`'s host timer fired. + fn expire(&self, interface: usize) { + let Some(tm) = self.timer_manager() else { return }; + if self.pi[interface].expire(&self.host_timer(&tm), (self.clock)()) { + self.pi[interface].publish(&self.levels); + } + } + fn publish_all(&self) { + for pi in &self.pi { + pi.publish(&self.levels); + } + } + + /// The PI a local-form access at offset 0 reaches: the requester's own. + /// A reader that is no processor reaches PI0. + fn own_pi(&self) -> usize { + crate::cpu::smp::on_cpu_thread() + .and_then(|cpu| self.processors.slot_of(cpu)) + .map_or(0, |(interface, _)| usize::from(interface)) + } + + /// An access of `size` bytes at `offset` in the register window, a write + /// when `value` is given; PI0's range reaches PI `pi0`. What no unit + /// answers, the padding after a unit's registers included, is a bus error. + fn access(&self, pi0: usize, offset: u64, size: u8, value: Option) -> BusRead64 { + match offset { + reg::PI0_BASE..reg::PI1_BASE => self.pi_access(pi0, offset, size, value), + reg::PI1_BASE..reg::IO_BASE => self.pi_access(1, offset - reg::PI1_BASE, size, value), + _ => BusRead64::err(), + } + } + fn pi_access(&self, index: usize, a: u64, size: u8, value: Option) -> BusRead64 { + let pi = &self.pi[index]; + let now = (self.clock)(); + match (size, value) { + (8, None) if a < pi::SIZE && a & 7 == 0 => { + let (data, latched) = pi.read(a, now, &self.levels); + if latched { + pi.publish(&self.levels); + } + data.map_or(BusRead64::err(), BusRead64::ok) + } + (8, Some(v)) if a < pi::SIZE && a & 7 == 0 => { + if pi.write(a, v, now) { + let tm = self.timer_manager(); + pi.reschedule(tm.as_deref().map(|tm| self.host_timer(tm)).as_ref(), now); + } + pi.publish(&self.levels); + BusRead64 { status: BUS_OK, data: 0 } + } + // The counter's two halves, high word first. + (4, None) if a & !4 == pi::reg::COUNTER => { + BusRead64::ok(pi.count(now) >> (32 - (a & 4) * 8) & 0xffff_ffff) + } + _ => { + pi.note(a, value.is_some()); + BusRead64::err() + } + } + } +} + +impl Device for Bedrock { + fn step(&self, _cycles: u64) {} + /// The counters run and the PIs' timers arm. + fn start(&self) { + let now = (self.clock)(); + let tm = self.timer_manager(); + let host = tm.as_deref().map(|tm| self.host_timer(tm)); + for pi in &self.pi { + pi.start(host.as_ref(), now); + } + self.publish_all(); + } + /// The counters pause where they are and the PIs' timers stop. + fn stop(&self) { + let now = (self.clock)(); + let tm = self.timer_manager(); + let host = tm.as_deref().map(|tm| self.host_timer(tm)); + for pi in &self.pi { + pi.stop(host.as_ref(), now); + } + self.publish_all(); + } + fn is_running(&self) -> bool { + self.pi[0].is_running() + } + fn get_clock(&self) -> u64 { + self.pi[0].count((self.clock)()) + } +} + +impl Resettable for Bedrock { + /// Every unit's reset. Memory keeps its contents. + fn power_on(&self) { + let now = (self.clock)(); + let tm = self.timer_manager(); + let host = tm.as_deref().map(|tm| self.host_timer(tm)); + for pi in &self.pi { + pi.power_on(now); + pi.reschedule(host.as_ref(), now); + } + self.publish_all(); + } +} + +/// The register window. With `LOCAL`, PI0's range answers the requester's +/// own PI. +struct Registers(Arc); +impl Registers { + #[inline] + fn access(&self, offset: u64, size: u8, value: Option) -> BusRead64 { + let pi0 = if LOCAL { self.0.own_pi() } else { 0 }; + self.0.access(pi0, offset, size, value) + } +} +macro_rules! face_access { + ($r:ident, $w:ident, $ty:ty, $result:ident, $size:expr) => { + fn $r(&self, a: u64) -> $result { + let r = self.access(a, $size, None); + $result { data: r.data as $ty, status: r.status } + } + fn $w(&self, a: u64, v: $ty) -> u32 { + self.access(a, $size, Some(v as u64)).status + } + }; +} +impl BusDevice for Registers { + face_access!(read8, write8, u8, BusRead8, 1); + face_access!(read16, write16, u16, BusRead16, 2); + face_access!(read32, write32, u32, BusRead32, 4); + face_access!(read64, write64, u64, BusRead64, 8); +} + +/// The L1 UART's interrupt line. +struct Uart(Weak); +impl InterruptController for Uart { + fn set_line(&self, _: u32, level: bool) { + if let Some(chip) = self.0.upgrade() { + if chip.levels.set(Levels::UART, level) { + chip.publish_all(); + } + } + } +} + +/// A test chip's configuration, on `clock`. +#[cfg(test)] +fn test_config(clock: Clock, processors: Population) -> BedrockConfig { + BedrockConfig { clock, processors } +} + +/// A chip on a hand-driven clock, for the unit tests: its register window +/// and CPU 0's interrupt word. +#[cfg(test)] +struct Rig { + chip: Arc, + now: Arc, + word: Arc, + regs: Arc, +} +#[cfg(test)] +impl Rig { + /// The Fuel's processor. + fn new() -> Self { + Self::with(Population::SINGLE) + } + fn with(processors: Population) -> Self { + let now = Arc::new(AtomicU64::new(0)); + let clock = now.clone(); + let config = test_config(Arc::new(move || clock.load(Ordering::SeqCst)), processors); + let chip = Bedrock::new(config).unwrap(); + let word = Arc::new(AtomicU64::new(0)); + chip.set_cpu_interrupts(0, &*word); + chip.start(); + let regs = chip.registers(Form::Node); + Self { chip, now, word, regs } + } + fn r(&self, at: u64) -> u64 { + let r = self.regs.read64(at); + assert_eq!(r.status, BUS_OK, "read at {at:#x}"); + r.data + } + fn w(&self, at: u64, value: u64) { + assert_eq!(self.regs.write64(at, value), BUS_OK, "write at {at:#x}"); + } + fn set_time(&self, ns: u64) { + self.now.store(ns, Ordering::SeqCst); + } + /// CPU 0's IP2..IP6, bit 0 being IP2. + fn cause(&self) -> u64 { + (self.word.load(Ordering::SeqCst) >> 10) & 0x1f + } +} diff --git a/src/dev/bedrock/pi.rs b/src/dev/bedrock/pi.rs new file mode 100644 index 000000000..89bef640b --- /dev/null +++ b/src/dev/bedrock/pi.rs @@ -0,0 +1,899 @@ +//! A Bedrock processor interface: processor-slot control, pending +//! interrupts, the real-time counter and its comparators, the profile timer, +//! error records and the graphics flow-control registers. A node has two +//! PIs with two processor slots each, A and B; the board names which of the +//! four slots hold a processor ([`Population`]). The Fuel fits PI0 slot A. +//! +//! Each slot drives IP2..IP6 of its processor: bank 0 under the slot's bank +//! 0 mask, bank 1 under its bank 1 mask, its RT comparator, the profile +//! timer and the error summary under its error cause. The inputs of those +//! five levels are atomics. A register store or an inbound packet changes +//! one and then publishes the slot's levels into Cause bits 10-14 of its +//! processor's interrupt word, without a lock; see [`Pi::publish`]. +//! +//! # What is approximated here +//! +//! Every fitted slot runs its processor from reset, with its CPU-enable bit +//! set, whatever the register later holds: no cold-reset value for the bit +//! is known and the register is not known to stop a processor. A write that +//! clears a running slot's enable is stored and recorded, not carried out. +//! +//! For a slot with no processor nothing is delivered. A store that arms +//! such a slot is recorded the first time it happens ([`Refused`]) and +//! logged under the `bedrock` module instead of being dropped silently: a +//! CPU enable, an NMI, a nonzero interrupt mask (an interprocessor interrupt +//! needs one), a timer enable and an error cause. The register itself is +//! stored, as hardware holds it. No processor model has an NMI entry, so +//! every NMI is recorded. +//! +//! The CPU-number register answers the reading processor's slice: its slot +//! in bit 0 and the interface in bit 1. A read from a thread that runs no +//! processor (a device, the monitor, a test) answers slot A; hardware always +//! has a requester. +//! +//! The model accepts Linux's public `SGIrules` key at offset 0x8 for +//! compatibility. Linux documents the literal for Hub I/O, but the available +//! evidence does not establish that this PI offset maps to the same register. +//! +//! The graphics flow-control registers store what software writes and count +//! nothing: no uncached store into the graphics page takes a credit and no +//! board returns one. +//! +//! Ported from hw/misc/sgi-bedrock-pi.c in jamesbraid/qemu-sgi-origami. + +use super::reg::IIO_PROTECT_OVRRD_KEY; +use crate::dev::odyssey::{PiGfx, PI_GFX_REGS}; +use super::HostTimer; +use crate::hptimer::TimerId; +use parking_lot::Mutex; +use std::sync::atomic::{fence, AtomicPtr, AtomicU64, Ordering::{Acquire, Relaxed, Release, SeqCst}}; + +pub(super) const RT_MASK: u64 = (1 << 55) - 1; +pub(super) const ERROR_MASK: u64 = (1 << 35) - 1; +pub(super) const MEMORY_ERROR: u64 = 1 << 24; +/// Bank 1 bits that reflect another block's level; a pending-modify write +/// cannot change them. +pub(super) const REFLECT: u64 = (1 << 54) | (1 << 55) | (1 << 62) | (1 << 63); +/// Nanoseconds per real-time counter tick: the counter runs at 1.25 MHz. +pub(super) const TICK: u64 = 800; +/// Length of a PI's register aperture. +pub(super) const SIZE: u64 = 0x30108; +/// IP2 is Cause bit 10. +const CAUSE_IP2: u32 = 10; +/// The request region the requests and the protection key name. +const REQUEST_REGION: u64 = 0; + +pub mod reg { + pub const NODE_REGION: u64 = 0x0; + pub const PROTECTION_KEY: u64 = 0x8; + pub const OFFSET_0010: u64 = 0x10; + pub const OFFSET_0018: u64 = 0x18; + pub const INTERFACE_ID: u64 = 0x20; + pub const CACHED_ALIAS: u64 = 0x28; + pub const OFFSET_0030: u64 = 0x30; + pub const OFFSET_0038: u64 = 0x38; + pub const INTERFACE_SLOT_A: u64 = 0x40; + pub const INTERFACE_SLOT_B: u64 = 0x48; + pub const CPU_A_ENABLE: u64 = 0x50; + pub const CPU_B_ENABLE: u64 = 0x58; + pub const NMI_SLOT_A: u64 = 0x70; + pub const NMI_SLOT_B: u64 = 0x78; + pub const PENDING_MODIFY: u64 = 0x90; + pub const PENDING_STATUS_A: u64 = 0x98; + pub const PENDING_STATUS_B: u64 = 0xa0; + pub const OFFSET_00A8: u64 = 0xa8; + pub const OFFSET_00B0: u64 = 0xb0; + pub const OFFSET_00B8: u64 = 0xb8; + pub const OFFSET_00C0: u64 = 0xc0; + pub const REQUEST_A: u64 = 0xc8; + pub const REQUEST_B: u64 = 0xd0; + pub const PENDING_A_CLEAR: u64 = 0xd8; + pub const PENDING_B_CLEAR: u64 = 0xe0; + pub const OFFSET_00E8: u64 = 0xe8; + pub const OFFSET_00F0: u64 = 0xf0; + pub const RT_COMPARE_A: u64 = 0x108; + pub const RT_COMPARE_B: u64 = 0x110; + pub const PROFILE_COMPARE: u64 = 0x118; + pub const RT_STATUS_A: u64 = 0x120; + pub const RT_STATUS_B: u64 = 0x128; + pub const PROFILE_STATUS_A: u64 = 0x130; + pub const PROFILE_STATUS_B: u64 = 0x138; + pub const RT_ENABLE_A: u64 = 0x140; + pub const RT_ENABLE_B: u64 = 0x148; + pub const PROFILE_ENABLE_A: u64 = 0x150; + pub const PROFILE_ENABLE_B: u64 = 0x158; + pub const PENDING_MODIFY_B: u64 = 0x180; + pub const ERROR_SUMMARY_ALIAS: u64 = 0x3f8; + pub const ERROR_SUMMARY: u64 = 0x400; + pub const ERROR_CAUSE_A: u64 = 0x408; + pub const ERROR_CAUSE_B: u64 = 0x410; + pub const ERROR_ADDRESS_A: u64 = 0x418; + pub const ERROR_ADDRESS_B: u64 = 0x420; + pub const ERROR_CONTROL: u64 = 0x428; + pub const ERROR_CAPTURE_A: u64 = 0x438; + pub const ERROR_CAPTURE_B: u64 = 0x448; + pub const ERROR_CAPTURE_C: u64 = 0x458; + pub const ERROR_CAPTURE_D: u64 = 0x468; + pub const ERROR_SOURCE_A: u64 = 0x490; + pub const ERROR_SOURCE_B: u64 = 0x4c0; + pub const ERROR_DETAIL_A: u64 = 0x4a8; + pub const ERROR_DETAIL_B: u64 = 0x4b0; + pub const ERROR_DETAIL_C: u64 = 0x4b8; + pub const OFFSET_0430: u64 = 0x430; + pub const OFFSET_0440: u64 = 0x440; + pub const OFFSET_0450: u64 = 0x450; + pub const OFFSET_0460: u64 = 0x460; + pub const OFFSET_04C8: u64 = 0x4c8; + pub const OFFSET_04D0: u64 = 0x4d0; + pub const CREDIT_MODE: u64 = 0x20068; + pub const COUNTER: u64 = 0x30100; + pub const GRAPHICS_BASE: u64 = 0x300; + pub const GRAPHICS_END: u64 = 0x350; +} + +/// The processor slots a board fits, one bit per slot: bit `interface * 2 + +/// slot`, slot 0 being A. CPU 0 sits in PI0 slot A, so every population has +/// it; a board fills the others in the order PI0/A, PI1/A, PI0/B, PI1/B, so +/// two processors sit one per interface. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct Population(u8); +impl Default for Population { + fn default() -> Self { + Self::SINGLE + } +} +impl Population { + /// One processor, in PI0 slot A. + pub const SINGLE: Self = Self(0b0001); + /// Slots named by `mask`. CPU 0 runs in PI0/A, so it must be fitted, and + /// only the four slots exist. + pub fn new(mask: u8) -> Result { + if mask & !0b1111 != 0 { + return Err(format!("processor mask {mask:#x} names a slot outside the node's four")); + } + if mask & 1 == 0 { + return Err(format!("processor mask {mask:#x} leaves PI0 slot A empty, where CPU 0 runs")); + } + Ok(Self(mask)) + } + /// The slot (interface, 0 = A or 1 = B) CPU `cpu` sits in: the board puts + /// CPU n in the nth slot of this set in its order. + pub fn slot_of(self, cpu: usize) -> Option<(u8, usize)> { + ORDER.iter().filter(|&&bit| self.0 & (1 << bit) != 0).nth(cpu).map(|&bit| (bit / 2, usize::from(bit % 2))) + } + pub fn mask(self) -> u8 { + self.0 + } + pub fn count(self) -> usize { + self.0.count_ones() as usize + } + /// Whether a processor is fitted in `slot` (0 = A, 1 = B) of `interface`. + pub fn fitted(self, interface: u8, slot: usize) -> bool { + self.0 & (1 << (usize::from(interface) * 2 + slot)) != 0 + } +} +/// The board's population order, as slot numbers `interface * 2 + slot`: +/// PI0/A, PI1/A, PI0/B, PI1/B. +const ORDER: [u8; 4] = [0, 2, 1, 3]; + +/// Something software armed for a processor slot that the model does not +/// carry out. See the module notes. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Refused { + /// A write set the CPU-enable bit of a slot with no processor. + Start { interface: u8, slot: usize }, + /// A write cleared the CPU-enable bit of a slot whose processor runs; it + /// keeps running. + Stop { interface: u8, slot: usize }, + /// A non-maskable interrupt: no processor model takes one. + Nmi { interface: u8, slot: usize }, + /// A nonzero interrupt mask or error cause for a slot with no processor. + Interrupt { interface: u8, slot: usize }, + /// A timer enable for a slot with no processor. + Timer { interface: u8, slot: usize }, +} + +/// Bank 0 bit 4 and bank 1 bits 54, 55 and 62: levels other blocks of the +/// chip drive, which both PIs reflect. Each writer sets its own bit under +/// its own lock, so two writers' levels land in register order. +#[derive(Default)] +pub(super) struct Levels([AtomicU64; 2]); +impl Levels { + pub(super) const UART: (usize, u32) = (0, 4); + /// Drive `(bank, bit)` to `level`. Whether the bit changed. + pub(super) fn set(&self, (bank, bit): (usize, u32), level: bool) -> bool { + let mask = 1u64 << bit; + let before = if level { + self.0[bank].fetch_or(mask, SeqCst) + } else { + self.0[bank].fetch_and(!mask, SeqCst) + }; + (before & mask != 0) != level + } + fn bank(&self, bank: usize) -> u64 { + self.0[bank].load(SeqCst) + } +} + +/// The real-time counter's origin, read without a lock. The counter is +/// `base + (time - epoch) / TICK`, `time` being the pause instant while the +/// machine is stopped. A writer holds the PI's timer lock and makes `seq` +/// odd while it changes the three words; a reader retries while `seq` is +/// odd or moved. +struct Counter { + seq: AtomicU64, + base: AtomicU64, + epoch: AtomicU64, + /// The pause instant, or `RUNNING`. + paused: AtomicU64, +} +const RUNNING: u64 = u64::MAX; +#[derive(Clone, Copy)] +struct Origin { + base: u64, + epoch: u64, + paused: u64, +} +impl Origin { + fn time(&self, now: u64) -> u64 { + if self.paused == RUNNING { now } else { self.paused } + } + fn count(&self, now: u64) -> u64 { + self.base.wrapping_add(self.time(now).saturating_sub(self.epoch) / TICK) & RT_MASK + } + /// The start of the tick `time` falls in. + fn tick_start(&self, time: u64) -> u64 { + time - time.saturating_sub(self.epoch) % TICK + } +} +impl Counter { + fn new(now: u64) -> Self { + Self { seq: AtomicU64::new(0), base: AtomicU64::new(0), epoch: AtomicU64::new(now), paused: AtomicU64::new(now) } + } + fn read(&self) -> Origin { + loop { + let seq = self.seq.load(Acquire); + let origin = Origin { base: self.base.load(Relaxed), epoch: self.epoch.load(Relaxed), paused: self.paused.load(Relaxed) }; + fence(Acquire); + if seq & 1 == 0 && self.seq.load(Relaxed) == seq { + return origin; + } + std::hint::spin_loop(); + } + } + /// Publish `origin`. The caller holds the timer lock, so writers are one at a time. + fn write(&self, origin: Origin) { + let seq = self.seq.load(Relaxed); + self.seq.store(seq + 1, Relaxed); + fence(Release); + self.base.store(origin.base, Relaxed); + self.epoch.store(origin.epoch, Relaxed); + self.paused.store(origin.paused, Relaxed); + self.seq.store(seq + 2, Release); + } +} + +/// The comparators and their deadlines, in host nanoseconds, which change +/// together and with the counter's origin. +struct Timer { + rt_compare: [u64; 2], + profile_compare: u64, + rt_due: [u128; 2], + profile_due: u128, + /// The host timer that fires at the earliest deadline. + id: Option, +} + +/// The registers that are not interrupt inputs: stored words, the graphics +/// registers and the record of refused stores. +struct State { + w: [u64; 0x4d8 / 8], + gfx: PiGfx, + credit_mode: u64, + refused: Vec<(Refused, u64)>, +} +impl State { + fn new() -> Self { + let mut w = [0; 0x4d8 / 8]; + w[reg::NODE_REGION as usize / 8] = !0; + w[reg::OFFSET_0018 as usize / 8] = !0; + Self { w, gfx: PiGfx::default(), credit_mode: 0, refused: Vec::new() } + } +} + +pub(super) struct Pi { + interface: u8, + processors: Population, + pending: [AtomicU64; 2], + /// Each slot's bank 0 and bank 1 masks: 0xa8 and 0xb0 for A, 0xb8 and + /// 0xc0 for B. + mask: [[AtomicU64; 2]; 2], + /// Requests A and B and their enable, which make bank 0 bits 5 and 6. + request: [AtomicU64; 2], + request_enable: AtomicU64, + /// The regions an inbound interrupt packet may come from (0x10). + region: AtomicU64, + rt_status: [AtomicU64; 2], + rt_enable: [AtomicU64; 2], + profile_status: [AtomicU64; 2], + profile_enable: [AtomicU64; 2], + error_summary: AtomicU64, + error_cause: [AtomicU64; 2], + /// Each slot's processor's interrupt word, null for an empty slot. + words: [AtomicPtr; 2], + counter: Counter, + /// The earliest deadline in host nanoseconds; `u64::MAX` while stopped. + next_due: AtomicU64, + timer: Mutex, + state: Mutex, +} + +impl Pi { + pub(super) fn new(interface: u8, processors: Population, now: u64) -> Self { + let pi = Self { + interface, + processors, + pending: Default::default(), + mask: Default::default(), + request: Default::default(), + request_enable: AtomicU64::new(0), + region: AtomicU64::new(!0), + rt_status: Default::default(), + rt_enable: Default::default(), + profile_status: Default::default(), + profile_enable: Default::default(), + error_summary: AtomicU64::new(0), + error_cause: Default::default(), + words: Default::default(), + counter: Counter::new(now), + next_due: AtomicU64::new(RUNNING), + timer: Mutex::new(Timer { rt_compare: [0; 2], profile_compare: 0, rt_due: [0; 2], profile_due: 0, id: None }), + state: Mutex::new(State::new()), + }; + let mut s = pi.state.lock(); + for slot in 0..2 { + s.w[reg::CPU_A_ENABLE as usize / 8 + slot] = u64::from(processors.fitted(interface, slot)); + } + drop(s); + pi.arm_all(&mut pi.timer.lock(), now); + pi + } + + pub(super) fn state_locked(&self) -> bool { + self.state.is_locked() + } + pub(super) fn timer_locked(&self) -> bool { + self.timer.is_locked() + } + /// Hold both locks, for tests of the paths that must not take them. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + (self.state.lock(), self.timer.lock()) + } + + /// Where slot `slot`'s IP2..IP6 go. + pub(super) fn set_word(&self, slot: usize, word: *const AtomicU64) { + self.words[slot].store(word as *mut AtomicU64, SeqCst); + } + + /// Whether a processor sits in `slot` of this interface. + fn started(&self, slot: usize) -> bool { + self.processors.fitted(self.interface, slot) + } + + /// Report an access to a register the model does not implement. + pub(super) fn note(&self, off: u64, write: bool) { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock PI{} unsupported {} at {off:#x}", self.interface, + if write { "write" } else { "read" }); + } + + /// Record a store this interface does not carry out. + fn refuse(&self, what: Refused) { + let mut s = self.state.lock(); + if let Some(entry) = s.refused.iter_mut().find(|(seen, _)| *seen == what) { + entry.1 += 1; + } else if s.refused.len() < 64 { + s.refused.push((what, 1)); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock PI{}: not carried out: {what:?}", self.interface); + } + } + + /// What has been refused, each with how often. + #[cfg(test)] + pub(super) fn refused(&self) -> Vec<(Refused, u64)> { + self.state.lock().refused.clone() + } + + fn pending0(&self, levels: &Levels) -> u64 { + let enable = self.request_enable.load(SeqCst); + self.pending[0].load(SeqCst) + | levels.bank(0) + | u64::from(self.request[0].load(SeqCst) & enable != 0) << 5 + | u64::from(self.request[1].load(SeqCst) & enable != 0) << 6 + } + fn pending1(&self, levels: &Levels) -> u64 { + (self.pending[1].load(SeqCst) & !REFLECT) | levels.bank(1) + } + + /// Slot `slot`'s IP2..IP6, bit 0 being IP2. + fn outputs(&self, slot: usize, levels: &Levels) -> u64 { + u64::from(self.pending0(levels) & self.mask[slot][0].load(SeqCst) != 0) + | u64::from(self.pending1(levels) & self.mask[slot][1].load(SeqCst) != 0) << 1 + | (self.rt_status[slot].load(SeqCst) & self.rt_enable[slot].load(SeqCst)) << 2 + | (self.profile_status[slot].load(SeqCst) & self.profile_enable[slot].load(SeqCst)) << 3 + | u64::from(self.error_summary.load(SeqCst) & self.error_cause[slot].load(SeqCst) != 0) << 4 + } + + /// Bring each processor's IP2..IP6 to what the registers say. Every + /// input change is a sequentially consistent store or read-modify-write + /// followed by a publish, so no lock is needed: take the last write to a + /// word. Its writer read the inputs again after it, and would have + /// written again had any change before that read left the word stale. A + /// change after that read is followed by its own publish, which would + /// make a later write. So once writers stop, the word matches the + /// registers. A publish that changes nothing reads the inputs and the + /// word and stores nothing. Only the PI writes Cause bits 10-14. + pub(super) fn publish(&self, levels: &Levels) { + for slot in 0..2 { + let p = self.words[slot].load(Acquire); + if p.is_null() { + continue; + } + // SAFETY: `Bedrock::set_cpu_interrupts`' contract: the word + // outlives the chip. + let word = unsafe { &*p }; + loop { + let want = self.outputs(slot, levels); + let have = (word.load(SeqCst) >> CAUSE_IP2) & 0x1f; + if want == have { + break; + } + let (set, clear) = (want & !have, have & !want); + if set != 0 { + word.fetch_or(set << CAUSE_IP2, SeqCst); + #[cfg(feature = "idle-pause")] + crate::cpu::idle_park::wake(); + } + if clear != 0 { + word.fetch_and(!(clear << CAUSE_IP2), SeqCst); + } + } + } + } + + /// Set or clear one pending bit, as a pending-modify store does: bit 8 + /// sets, bit 6 picks the bank and bits 5:0 the bit. The reflected bits + /// of bank 1 are left alone. + fn modify(&self, value: u64) { + let bank = ((value >> 6) & 1) as usize; + let mask = 1u64 << (value & 63); + if bank == 1 && mask & REFLECT != 0 { + return; + } + if value & 0x100 != 0 { + self.pending[bank].fetch_or(mask, SeqCst); + } else { + self.pending[bank].fetch_and(!mask, SeqCst); + } + } + + + + /// The real-time counter at host time `now`. Lock-free. + pub(super) fn count(&self, now: u64) -> u64 { + self.counter.read().count(now) + } + + fn arm_rt(&self, t: &mut Timer, o: &Origin, slot: usize, now: u64) { + let time = o.time(now); + let ticks = (t.rt_compare[slot].wrapping_sub(o.count(now)).wrapping_sub(1) & RT_MASK) + 1; + t.rt_due[slot] = u128::from(o.tick_start(time)) + u128::from(ticks) * u128::from(TICK); + } + fn arm_profile(&self, t: &mut Timer, o: &Origin, now: u64) { + let time = o.time(now); + let ticks = (t.profile_compare as u32).wrapping_sub(o.count(now) as u32); + let ticks = if ticks == 0 { 1u64 << 32 } else { u64::from(ticks) }; + t.profile_due = u128::from(o.tick_start(time)) + u128::from(ticks) * u128::from(TICK); + } + fn arm_all(&self, t: &mut Timer, now: u64) { + let o = self.counter.read(); + self.arm_rt(t, &o, 0, now); + self.arm_rt(t, &o, 1, now); + self.arm_profile(t, &o, now); + self.set_next_due(t, &o); + } + fn set_next_due(&self, t: &Timer, o: &Origin) { + let due = if o.paused != RUNNING { + RUNNING + } else { + t.rt_due.iter().copied().chain([t.profile_due]).min().unwrap().min(u128::from(RUNNING - 1)) as u64 + }; + self.next_due.store(due, SeqCst); + } + + /// Latch the comparators that are due at `now`, under the timer lock. + /// Whether one latched. + fn latch(&self, t: &mut Timer, now: u64) -> bool { + let o = self.counter.read(); + let time = o.time(now); + let mut latched = false; + for slot in 0..2 { + if u128::from(time) >= t.rt_due[slot] { + self.rt_status[slot].store(1, SeqCst); + self.arm_rt(t, &o, slot, now); + latched = true; + } + } + if u128::from(time) >= t.profile_due { + self.profile_status[0].store(1, SeqCst); + self.profile_status[1].store(1, SeqCst); + self.arm_profile(t, &o, now); + latched = true; + } + self.set_next_due(t, &o); + latched + } + + /// Latch what is due at `now`: one load when nothing is. Whether a + /// comparator latched, so the caller publishes. + pub(super) fn sync(&self, now: u64) -> bool { + if now < self.next_due.load(SeqCst) { + return false; + } + self.latch(&mut self.timer.lock(), now) + } + + /// The host timer fired: latch what is due and arm the next deadline. + pub(super) fn expire(&self, host: &HostTimer, now: u64) -> bool { + let mut t = self.timer.lock(); + let latched = self.latch(&mut t, now); + self.arm_host(&mut t, host, now); + latched + } + + /// (Re)arm the host timer for the earliest deadline, under the timer + /// lock. A deadline more than a second out is armed for a second. + fn arm_host(&self, t: &mut Timer, host: &HostTimer, now: u64) { + if let Some(id) = t.id.take() { + host.tm.remove(id); + } + let due = self.next_due.load(SeqCst); + if due == RUNNING { + return; + } + let delay = due.saturating_sub(now).clamp(1, 1_000_000_000); + t.id = Some(host.arm(self.interface, delay)); + } + + /// Re-arm after the comparators or the counter moved. + pub(super) fn reschedule(&self, host: Option<&HostTimer>, now: u64) { + if let Some(host) = host { + self.arm_host(&mut self.timer.lock(), host, now); + } + } + + /// The machine started at `now`: the counter resumes from where it paused. + pub(super) fn start(&self, host: Option<&HostTimer>, now: u64) { + let mut t = self.timer.lock(); + let mut o = self.counter.read(); + if o.paused == RUNNING { + return; + } + let delta = now.saturating_sub(o.paused); + o.epoch = o.epoch.saturating_add(delta); + o.paused = RUNNING; + self.counter.write(o); + for due in &mut t.rt_due { + *due += u128::from(delta); + } + t.profile_due += u128::from(delta); + self.set_next_due(&t, &o); + if let Some(host) = host { + self.arm_host(&mut t, host, now); + } + } + + /// The machine stopped at `now`: latch what is due, then freeze the counter. + pub(super) fn stop(&self, host: Option<&HostTimer>, now: u64) -> bool { + let mut t = self.timer.lock(); + let latched = self.latch(&mut t, now); + let mut o = self.counter.read(); + if o.paused == RUNNING { + o.paused = now; + self.counter.write(o); + } + self.next_due.store(RUNNING, SeqCst); + if let (Some(host), Some(id)) = (host, t.id.take()) { + host.tm.remove(id); + } + latched + } + + pub(super) fn is_running(&self) -> bool { + self.counter.read().paused == RUNNING + } + + /// The CPU-number register: the reading processor's slot in bit 0 and + /// this interface in bit 1; slot A for a reader that is no processor + /// here. + fn cpu_number(&self) -> u64 { + let slot = crate::cpu::smp::on_cpu_thread() + .and_then(|cpu| self.processors.slot_of(cpu)) + .filter(|&(interface, _)| interface == self.interface) + .map_or(0, |(_, slot)| slot as u64); + slot | u64::from(self.interface) << 1 + } + + /// A doubleword read at `off`, and whether a comparator latched on the + /// way, so the caller publishes. `None` for a register the model does + /// not have. + pub(super) fn read(&self, off: u64, now: u64, levels: &Levels) -> (Option, bool) { + let mut latched = false; + let value = match off { + reg::INTERFACE_ID => self.cpu_number(), + reg::INTERFACE_SLOT_A => u64::from(self.started(0)), + reg::INTERFACE_SLOT_B => u64::from(self.started(1)), + reg::OFFSET_0010 => self.region.load(SeqCst), + reg::PENDING_STATUS_A => self.pending0(levels), + reg::PENDING_STATUS_B => self.pending1(levels), + reg::OFFSET_00A8 | reg::OFFSET_00B0 | reg::OFFSET_00B8 | reg::OFFSET_00C0 => { + let n = (off - reg::OFFSET_00A8) as usize / 8; + self.mask[n / 2][n % 2].load(SeqCst) + } + reg::REQUEST_A | reg::PENDING_A_CLEAR => self.request[0].load(SeqCst), + reg::REQUEST_B | reg::PENDING_B_CLEAR => self.request[1].load(SeqCst), + reg::OFFSET_00E8 => self.request_enable.load(SeqCst), + reg::RT_COMPARE_A | reg::RT_COMPARE_B => self.timer.lock().rt_compare[((off - reg::RT_COMPARE_A) / 8) as usize], + reg::PROFILE_COMPARE => self.timer.lock().profile_compare, + reg::RT_STATUS_A | reg::RT_STATUS_B => { + latched = self.sync(now); + self.rt_status[((off - reg::RT_STATUS_A) / 8) as usize].load(SeqCst) + } + reg::PROFILE_STATUS_A | reg::PROFILE_STATUS_B => { + latched = self.sync(now); + self.profile_status[((off - reg::PROFILE_STATUS_A) / 8) as usize].load(SeqCst) + } + reg::RT_ENABLE_A | reg::RT_ENABLE_B => self.rt_enable[((off - reg::RT_ENABLE_A) / 8) as usize].load(SeqCst), + reg::PROFILE_ENABLE_A | reg::PROFILE_ENABLE_B => self.profile_enable[((off - reg::PROFILE_ENABLE_A) / 8) as usize].load(SeqCst), + reg::ERROR_SUMMARY | reg::ERROR_SUMMARY_ALIAS => self.error_summary.load(SeqCst), + reg::ERROR_CAUSE_A | reg::ERROR_CAUSE_B => self.error_cause[((off - reg::ERROR_CAUSE_A) / 8) as usize].load(SeqCst), + reg::COUNTER => self.count(now), + reg::GRAPHICS_BASE..reg::GRAPHICS_END if off % 8 == 0 => { + let n = ((off - reg::GRAPHICS_BASE) / 8) as u32; + debug_assert!(n < PI_GFX_REGS); + self.state.lock().gfx.read(n) + } + reg::CREDIT_MODE => self.state.lock().credit_mode, + reg::ERROR_CAPTURE_A | reg::ERROR_CAPTURE_B | reg::ERROR_CAPTURE_C | reg::ERROR_CAPTURE_D => { + self.state.lock().w[(off - reg::PROTECTION_KEY) as usize / 8] + } + _ if stored(off) => self.state.lock().w[off as usize / 8], + _ => { + self.note(off, false); + return (None, false); + } + }; + (Some(value), latched) + } + + /// A doubleword store at `off`. Whether the comparators or the counter + /// moved, so the caller re-arms the host timer. The caller publishes. + pub(super) fn write(&self, off: u64, value: u64, now: u64) -> bool { + let index = |base: u64| ((off - base) / 8) as usize; + match off { + reg::OFFSET_0010 => self.region.store(value, SeqCst), + reg::OFFSET_00A8 | reg::OFFSET_00B0 | reg::OFFSET_00B8 | reg::OFFSET_00C0 => { + let n = index(reg::OFFSET_00A8); + self.mask[n / 2][n % 2].store(value, SeqCst); + if value != 0 && !self.started(n / 2) { + self.refuse(Refused::Interrupt { interface: self.interface, slot: n / 2 }); + } + } + reg::PENDING_MODIFY | reg::PENDING_MODIFY_B => self.modify(value), + reg::REQUEST_A | reg::REQUEST_B => { + self.request[index(reg::REQUEST_A)].fetch_or(1 << REQUEST_REGION, SeqCst); + } + reg::PENDING_A_CLEAR | reg::PENDING_B_CLEAR => { + self.request[index(reg::PENDING_A_CLEAR)].fetch_and(!value, SeqCst); + } + reg::OFFSET_00E8 => self.request_enable.store(value, SeqCst), + reg::RT_STATUS_A | reg::RT_STATUS_B => { + // Latch a deadline that has passed first, so the clear takes it. + self.sync(now); + self.rt_status[index(reg::RT_STATUS_A)].store(0, SeqCst); + } + reg::PROFILE_STATUS_A | reg::PROFILE_STATUS_B => { + self.sync(now); + self.profile_status[index(reg::PROFILE_STATUS_A)].store(0, SeqCst); + } + reg::RT_ENABLE_A | reg::RT_ENABLE_B | reg::PROFILE_ENABLE_A | reg::PROFILE_ENABLE_B => { + let rt = matches!(off, reg::RT_ENABLE_A | reg::RT_ENABLE_B); + let slot = index(if rt { reg::RT_ENABLE_A } else { reg::PROFILE_ENABLE_A }); + if rt { &self.rt_enable[slot] } else { &self.profile_enable[slot] }.store(value & 1, SeqCst); + if value & 1 != 0 && !self.started(slot) { + self.refuse(Refused::Timer { interface: self.interface, slot }); + } + } + reg::ERROR_SUMMARY_ALIAS => { + // Software sets every bit but the memory error, which only MD sets. + let mut summary = self.error_summary.load(SeqCst); + while let Err(now) = self.error_summary.compare_exchange_weak( + summary, (summary & MEMORY_ERROR) | (value & ERROR_MASK & !MEMORY_ERROR), SeqCst, SeqCst, + ) { + summary = now; + } + } + reg::ERROR_SUMMARY => { + self.error_summary.fetch_and(!(value & ERROR_MASK), SeqCst); + } + reg::ERROR_CAUSE_A | reg::ERROR_CAUSE_B => { + let slot = index(reg::ERROR_CAUSE_A); + self.error_cause[slot].store(value & ERROR_MASK, SeqCst); + if value & ERROR_MASK != 0 && !self.started(slot) { + self.refuse(Refused::Interrupt { interface: self.interface, slot }); + } + } + reg::RT_COMPARE_A | reg::RT_COMPARE_B | reg::PROFILE_COMPARE | reg::COUNTER => { + let mut t = self.timer.lock(); + self.latch(&mut t, now); + let mut o = self.counter.read(); + match off { + reg::RT_COMPARE_A | reg::RT_COMPARE_B => { + let slot = index(reg::RT_COMPARE_A); + t.rt_compare[slot] = value & RT_MASK; + self.arm_rt(&mut t, &o, slot, now); + } + reg::PROFILE_COMPARE => { + t.profile_compare = value & 0xffff_ffff; + self.arm_profile(&mut t, &o, now); + } + _ => { + o.base = value & RT_MASK; + o.epoch = o.time(now); + self.counter.write(o); + self.arm_rt(&mut t, &o, 0, now); + self.arm_rt(&mut t, &o, 1, now); + self.arm_profile(&mut t, &o, now); + } + } + self.set_next_due(&t, &o); + return true; + } + reg::GRAPHICS_BASE..reg::GRAPHICS_END if off % 8 == 0 => { + let n = ((off - reg::GRAPHICS_BASE) / 8) as u32; + debug_assert!(n < PI_GFX_REGS); + self.state.lock().gfx.write(n, value); + } + reg::CREDIT_MODE => self.state.lock().credit_mode = value & 1, + _ => self.write_stored(off, value), + } + false + } + + /// The stores that only change a stored word. + fn write_stored(&self, off: u64, value: u64) { + let mut s = self.state.lock(); + let w = |s: &mut State, at: u64, v: u64| s.w[at as usize / 8] = v; + match off { + reg::NODE_REGION | reg::OFFSET_0018 | reg::OFFSET_04C8 | reg::OFFSET_04D0 => w(&mut s, off, value), + reg::PROTECTION_KEY => { + if value == IIO_PROTECT_OVRRD_KEY { + s.w[0] |= 1 << REQUEST_REGION; + } + } + reg::CACHED_ALIAS => w(&mut s, off, value & 15), + reg::OFFSET_0030 => w(&mut s, off, value & 255), + reg::OFFSET_0038 => w(&mut s, off, value & 0xffffff), + reg::CPU_A_ENABLE | reg::CPU_B_ENABLE => { + let slot = ((off - reg::CPU_A_ENABLE) / 8) as usize; + w(&mut s, off, value & 1); + drop(s); + if value & 1 != 0 && !self.started(slot) { + self.refuse(Refused::Start { interface: self.interface, slot }); + } else if value & 1 == 0 && self.started(slot) { + self.refuse(Refused::Stop { interface: self.interface, slot }); + } + } + reg::NMI_SLOT_A | reg::NMI_SLOT_B => { + drop(s); + if value & 1 != 0 { + self.refuse(Refused::Nmi { interface: self.interface, slot: ((off - reg::NMI_SLOT_A) / 8) as usize }); + } else { + self.note(off, true); + } + } + reg::OFFSET_00F0 => { + w(&mut s, off, value & 31); + drop(s); + if value & 31 != 0 { + self.note(off, true); + } + } + reg::ERROR_ADDRESS_A | reg::ERROR_ADDRESS_B => w(&mut s, off, value & 0x1fffffff8), + reg::ERROR_CONTROL => { + w(&mut s, off, value & 15); + drop(s); + if value & 15 != 0 { + self.note(off, true); + } + } + reg::ERROR_CAPTURE_A | reg::ERROR_CAPTURE_B | reg::ERROR_CAPTURE_C | reg::ERROR_CAPTURE_D => { + let slot = usize::from(off >= 0x450); + s.w[0x430 / 8 + slot * 4] = 0; + s.w[0x440 / 8 + slot * 4] = 0; + } + reg::ERROR_SOURCE_A | reg::ERROR_SOURCE_B => w(&mut s, off, value & 63), + reg::ERROR_DETAIL_A | reg::ERROR_DETAIL_B => w(&mut s, off, value & 0x1fffff), + reg::ERROR_DETAIL_C => w(&mut s, off, value & 0xfffff), + _ => { + drop(s); + self.note(off, true); + } + } + } + + /// The PI block's local reset at `now`. The controls and captures + /// hardware keeps across it survive; the interrupt inputs, timers, + /// counter and graphics registers start over. + pub(super) fn power_on(&self, now: u64) { + let mut s = self.state.lock(); + let old = s.w; + let refused = std::mem::take(&mut s.refused); + *s = State::new(); + for off in [0, 0x18, 0x28, 0x50, 0x58, 0x418, 0x420, 0x428, 0x430, 0x440, 0x450, 0x460, 0x4c8, 0x4d0] { + s.w[off / 8] = old[off / 8]; + } + s.refused = refused; + drop(s); + for bank in &self.pending { + bank.store(0, SeqCst); + } + for a in self.mask.iter().flatten().chain(&self.request).chain([&self.request_enable]) + .chain(&self.rt_status).chain(&self.rt_enable).chain(&self.profile_status) + .chain(&self.profile_enable).chain(&self.error_cause) + { + a.store(0, SeqCst); + } + let mut t = self.timer.lock(); + t.rt_compare = [0; 2]; + t.profile_compare = 0; + let o = self.counter.read(); + let time = o.time(now); + self.counter.write(Origin { base: 0, epoch: time, paused: o.paused }); + self.arm_all(&mut t, now); + } +} + +/// The doubleword registers that only hold what was stored. +fn stored(off: u64) -> bool { + matches!( + off, + reg::NODE_REGION + | reg::OFFSET_0018 + | reg::CACHED_ALIAS + | reg::OFFSET_0030 + | reg::OFFSET_0038 + | reg::CPU_A_ENABLE + | reg::CPU_B_ENABLE + | reg::OFFSET_00F0 + | reg::ERROR_ADDRESS_A + | reg::ERROR_ADDRESS_B + | reg::ERROR_CONTROL + | reg::OFFSET_0430 + | reg::OFFSET_0440 + | reg::OFFSET_0450 + | reg::OFFSET_0460 + | reg::ERROR_SOURCE_A + | reg::ERROR_DETAIL_A + | reg::ERROR_DETAIL_B + | reg::ERROR_DETAIL_C + | reg::ERROR_SOURCE_B + | reg::OFFSET_04C8 + | reg::OFFSET_04D0 + ) +} + +#[cfg(test)] +mod tests; diff --git a/src/dev/bedrock/pi/tests.rs b/src/dev/bedrock/pi/tests.rs new file mode 100644 index 000000000..9f77249df --- /dev/null +++ b/src/dev/bedrock/pi/tests.rs @@ -0,0 +1,283 @@ +// Cases follow tests/qtest/sn1-bedrock-pi-test.c in jamesbraid/qemu-sgi-origami: +// power-on-values, local-reset, pending-reflection, real-time-comparators, +// profile-counter-wrap, cpu-number-fallback, cpu-enable-lines and nmi, and +// sn1-cpu-identity-test's four-processors-know-their-slots. + +use super::super::reg::PI1_BASE; +use super::super::Rig; +use super::*; +use crate::traits::{BusDevice, Device, BUS_ERR, BUS_OK}; +use std::sync::atomic::Ordering; + +/// Drive the reflected bank 1 bit 55, as LB's error level does. +fn reflect(rig: &Rig, level: bool) { + if rig.chip.levels.set((1, 55), level) { + rig.chip.publish_all(); + } +} +use std::sync::Arc; +use std::time::Duration; + +/// The PROM reads PI slot presence and relies on the documented reset survivors. +#[test] +fn the_pi_reports_interface_state_and_retains_registers_across_reset() { + { + let rig = Rig::new(); + assert_eq!(rig.r(reg::INTERFACE_ID), 0); + assert_eq!(rig.r(reg::INTERFACE_SLOT_A), 1); + assert_eq!(rig.r(reg::INTERFACE_SLOT_B), 0); + rig.w(reg::NODE_REGION, 0); + rig.w(reg::PROTECTION_KEY, 0x53474972756c6573); + assert_eq!(rig.r(reg::NODE_REGION), 1); + assert_eq!(rig.r(PI1_BASE + reg::INTERFACE_ID), 2); + assert_eq!(rig.r(PI1_BASE + reg::INTERFACE_SLOT_A), 0); + assert_eq!(rig.regs.read32(reg::INTERFACE_ID).status, BUS_ERR); + assert_eq!(rig.regs.write32(reg::COUNTER, 0), BUS_ERR); + } + { + let rig = Rig::new(); + rig.w(reg::PENDING_MODIFY, 0x10c); + assert_eq!(rig.r(reg::PENDING_STATUS_A), 1 << 12); + rig.w(reg::PENDING_MODIFY_B, 12); + assert_eq!(rig.r(reg::PENDING_STATUS_A), 0); + reflect(&rig, true); + rig.w(reg::PENDING_MODIFY, 64 + 55); + assert_eq!(rig.r(reg::PENDING_STATUS_B), 1 << 55); + rig.w(reg::PENDING_MODIFY, 0x100 | 64 | 63); + assert_eq!(rig.r(reg::PENDING_STATUS_B), 1 << 55); + rig.w(reg::REQUEST_A, 0); + rig.w(reg::OFFSET_00E8, 1); + assert_eq!(rig.r(reg::PENDING_STATUS_A), 1 << 5); + rig.w(reg::PENDING_A_CLEAR, 1); + assert_eq!(rig.r(reg::PENDING_STATUS_A), 0); + } + { + let rig = Rig::new(); + rig.w(reg::CACHED_ALIAS, 9); + rig.w(reg::CPU_A_ENABLE, 1); + rig.w(reg::ERROR_ADDRESS_A, !0); + rig.chip.pi[0].error_summary.fetch_or(MEMORY_ERROR, Ordering::SeqCst); + rig.w(reg::ERROR_SUMMARY_ALIAS, 3); + rig.w(reg::RT_COMPARE_A, 0x24); + rig.chip.pi[0].power_on(0); + assert_eq!(rig.r(reg::ERROR_SUMMARY), MEMORY_ERROR | 3); + assert_eq!(rig.r(reg::CACHED_ALIAS), 9); + assert_eq!(rig.r(reg::CPU_A_ENABLE), 1); + assert_eq!(rig.r(reg::ERROR_ADDRESS_A), 0x1fffffff8); + assert_eq!(rig.r(reg::RT_COMPARE_A), 0); + assert_eq!(rig.r(reg::COUNTER), 0); + rig.w(reg::ERROR_SUMMARY_ALIAS, 0); + assert_eq!(rig.r(reg::ERROR_SUMMARY), MEMORY_ERROR); + rig.w(reg::ERROR_SUMMARY, MEMORY_ERROR); + assert_eq!(rig.r(reg::ERROR_SUMMARY), 0); + } +} + +/// The counter wraps, the comparators latch at their tick, and the counter +/// pauses while the machine is stopped. +#[test] +fn the_pi_timer_wraps_and_pauses() { + { + let rig = Rig::new(); + rig.w(reg::COUNTER, 0x1_0000_0000); + rig.w(reg::RT_COMPARE_A, 0x1_0000_0002); + rig.set_time(1599); + assert_eq!(rig.r(reg::RT_STATUS_A), 0); + rig.set_time(1600); + assert_eq!(rig.r(reg::RT_STATUS_A), 1); + assert_eq!(rig.regs.read32(reg::COUNTER).data, 1); + assert_eq!(rig.regs.read32(reg::COUNTER + 4).data, 2); + rig.w(reg::RT_STATUS_A, !0); + assert_eq!(rig.r(reg::RT_STATUS_A), 0); + rig.w(reg::COUNTER, RT_MASK); + rig.w(reg::PROFILE_COMPARE, 0); + rig.set_time(2400); + assert_eq!(rig.r(reg::COUNTER), 0); + assert_eq!(rig.r(reg::PROFILE_STATUS_A), 1); + assert_eq!(rig.r(reg::PROFILE_STATUS_B), 1); + rig.w(reg::PROFILE_STATUS_A, 0); + assert_eq!(rig.r(reg::PROFILE_STATUS_A), 0); + assert_eq!(rig.r(reg::PROFILE_STATUS_B), 1); + } + { + let rig = Rig::new(); + rig.set_time(1_000_000); + rig.w(reg::COUNTER, 0); + rig.w(reg::RT_COMPARE_A, 2); + rig.set_time(1_000_400); + rig.chip.stop(); + rig.set_time(2_000_000); + assert_eq!(rig.r(reg::COUNTER), 0); + assert_eq!(rig.r(reg::RT_STATUS_A), 0, "no time passes while stopped"); + rig.chip.start(); + rig.set_time(2_001_199); + assert_eq!(rig.r(reg::RT_STATUS_A), 0); + rig.set_time(2_001_200); + assert_eq!(rig.r(reg::RT_STATUS_A), 1); + } +} + +/// Each of a slot's five outputs follows its own inputs into Cause bits +/// 10-14 of the processor's word, and nothing else in the word moves. +#[test] +fn pi_interrupt_outputs_follow_masks_into_the_cpus_word() { + let rig = Rig::new(); + rig.word.store(0xffff_0000_0000_83ff & !(0x1f << 10), Ordering::SeqCst); + rig.w(reg::OFFSET_00A8, 1 << 12); + rig.w(reg::OFFSET_00B0, 1 << 55); + rig.w(reg::PENDING_MODIFY, 0x10c); + assert_eq!(rig.cause(), 1); + reflect(&rig, true); + assert_eq!(rig.cause(), 3); + rig.w(reg::RT_COMPARE_A, 2); + rig.w(reg::PROFILE_COMPARE, 2); + rig.w(reg::RT_ENABLE_A, 1); + rig.w(reg::PROFILE_ENABLE_A, 1); + rig.w(reg::ERROR_CAUSE_A, 1); + rig.w(reg::ERROR_SUMMARY_ALIAS, 1); + assert_eq!(rig.cause(), 0b10011); + rig.set_time(1600); + assert_eq!(rig.r(reg::RT_STATUS_A), 1); + assert_eq!(rig.cause(), 31); + rig.w(reg::RT_STATUS_A, 0); + assert_eq!(rig.cause(), 27); + rig.w(reg::PENDING_MODIFY, 12); + reflect(&rig, false); + rig.w(reg::PROFILE_STATUS_A, 0); + rig.w(reg::ERROR_SUMMARY, 1); + assert_eq!(rig.cause(), 0); + assert_eq!(rig.word.load(Ordering::SeqCst), 0xffff_0000_0000_83ff & !(0x1f << 10), "only bits 10-14 are the PI's"); +} + +/// A guest can wait for a timer interrupt without touching a register: the +/// host timer latches the comparator and raises IP4. +#[test] +fn the_timer_manager_delivers_expirations_without_mmio_polling() { + let config = super::super::test_config(crate::dev::heart::host_clock(), Population::SINGLE); + let chip = super::super::Bedrock::new(config).unwrap(); + let word = Arc::new(std::sync::atomic::AtomicU64::new(0)); + chip.set_cpu_interrupts(0, &*word); + let tm = Arc::new(crate::hptimer::TimerManager::new()); + chip.set_timer_manager(&tm); + let regs = chip.registers(crate::dev::bedrock::Form::Node); + assert_eq!(regs.write64(reg::RT_COMPARE_A, 12500), BUS_OK); + assert_eq!(regs.write64(reg::RT_ENABLE_A, 1), BUS_OK); + chip.start(); + let deadline = std::time::Instant::now() + Duration::from_secs(5); + while word.load(Ordering::SeqCst) & (1 << 12) == 0 { + assert!(std::time::Instant::now() < deadline, "IP4 never rose"); + std::thread::sleep(Duration::from_millis(1)); + } + chip.stop(); + assert!(!chip.is_running()); +} + +/// The PROM counts processors from the present bits. A board fills PI0/A, +/// then PI1/A, then PI0/B and PI1/B, so two processors sit one per interface. +#[test] +fn every_fitted_slot_reads_present_and_starts_enabled() { + for (mask, present) in [(0b0001, [1, 0, 0, 0]), (0b0101, [1, 0, 1, 0]), (0b0111, [1, 1, 1, 0]), (0b1111, [1, 1, 1, 1])] { + let rig = Rig::with(Population::new(mask).unwrap()); + let read = |interface: u64, off| rig.r(interface * PI1_BASE + off); + assert_eq!([read(0, reg::INTERFACE_SLOT_A), read(0, reg::INTERFACE_SLOT_B), + read(1, reg::INTERFACE_SLOT_A), read(1, reg::INTERFACE_SLOT_B)], present, "{mask:#06b}"); + assert_eq!([read(0, reg::CPU_A_ENABLE), read(0, reg::CPU_B_ENABLE), + read(1, reg::CPU_A_ENABLE), read(1, reg::CPU_B_ENABLE)], present, "{mask:#06b}: enables"); + } +} + +/// A population that cannot hold CPU 0, or names a fifth slot, is refused +/// rather than built; the board's order puts CPU n in the nth fitted slot. +#[test] +fn a_population_without_cpu_0_or_beyond_four_slots_is_refused() { + for mask in [0, 0b0010, 0b1100, 0b1_0001, 0xff] { + assert!(Population::new(mask).is_err(), "{mask:#x}"); + } + let four = Population::new(0b1111).unwrap(); + assert_eq!((0..4).map(|cpu| four.slot_of(cpu).unwrap()).collect::>(), [(0, 0), (1, 0), (0, 1), (1, 1)]); + assert_eq!(four.slot_of(4), None); + assert_eq!(Population::new(0b0101).unwrap().slot_of(1), Some((1, 0))); + assert_eq!(Population::SINGLE.count(), 1); +} + +/// Each processor reads its own slice from the CPU-number register of its +/// interface, and a reader that is no processor reads slot A. The local +/// register form answers the requester's own PI at offset 0. +#[test] +fn four_processors_know_their_slots_and_their_own_pi() { + let rig = Arc::new(Rig::with(Population::new(0b1111).unwrap())); + assert_eq!((rig.r(reg::INTERFACE_ID), rig.r(PI1_BASE + reg::INTERFACE_ID)), (0, 2), "no issuing processor"); + let local = rig.chip.registers(crate::dev::bedrock::Form::Local); + let slices: Vec<(u64, u64)> = (0..4u8) + .map(|cpu| { + let (rig, local) = (rig.clone(), local.clone()); + std::thread::spawn(move || { + crate::cpu::smp::set_current_cpu(cpu); + let (interface, _) = Population::new(0b1111).unwrap().slot_of(cpu as usize).unwrap(); + (rig.r(u64::from(interface) * PI1_BASE + reg::INTERFACE_ID), local.read64(reg::INTERFACE_ID).data) + }) + .join() + .unwrap() + }) + .collect(); + assert_eq!(slices, [(0, 0), (2, 2), (1, 1), (3, 3)], "slices A, C, B, D"); + // With processors on PI0 alone, the local form is the node form. + let fuel = Rig::new(); + std::thread::spawn(move || { + crate::cpu::smp::set_current_cpu(0); + assert_eq!(fuel.chip.registers(crate::dev::bedrock::Form::Local).read64(reg::INTERFACE_ID).data, 0); + }) + .join() + .unwrap(); +} + +/// Enabling a processor that is not fitted is stored, as the register holds +/// it, and recorded; clearing a fitted one's bit is recorded too and stops +/// nothing. An NMI is always recorded: no processor takes one. +#[test] +fn enables_and_nmis_the_model_does_not_carry_out_are_recorded() { + let rig = Rig::new(); + let pi = |n: usize| &rig.chip.pi[n]; + rig.w(reg::CPU_A_ENABLE, 1); + assert!(pi(0).refused().is_empty(), "CPU 0 runs"); + rig.w(reg::CPU_B_ENABLE, 1); + rig.w(PI1_BASE + reg::CPU_A_ENABLE, 3); + rig.w(PI1_BASE + reg::CPU_A_ENABLE, 1); + rig.w(PI1_BASE + reg::CPU_B_ENABLE, 0); + assert_eq!(rig.r(reg::CPU_B_ENABLE), 1); + assert_eq!(pi(0).refused(), [(Refused::Start { interface: 0, slot: 1 }, 1)]); + assert_eq!(pi(1).refused(), [(Refused::Start { interface: 1, slot: 0 }, 2)], "counted, and a clear bit is not an attempt"); + rig.w(reg::CPU_A_ENABLE, 0); + assert_eq!(pi(0).refused()[1], (Refused::Stop { interface: 0, slot: 0 }, 1)); + rig.w(reg::NMI_SLOT_A, 1); + rig.w(PI1_BASE + reg::NMI_SLOT_B, u64::MAX); + assert_eq!(pi(0).refused()[2], (Refused::Nmi { interface: 0, slot: 0 }, 1)); + assert_eq!(pi(1).refused()[1], (Refused::Nmi { interface: 1, slot: 1 }, 1)); +} + +/// A nonzero mask, timer enable or error cause for a slot with no processor +/// is recorded at the store that arms it, so the publish never visits an +/// empty slot. Zero stores and CPU 0's own slot are not records. +#[test] +fn arming_a_slot_with_no_processor_is_recorded_at_the_store() { + let rig = Rig::new(); + let pi = |n: usize| &rig.chip.pi[n]; + rig.w(reg::OFFSET_00A8, 1 << 7); + rig.w(reg::RT_ENABLE_A, 1); + rig.w(reg::PROFILE_ENABLE_A, 1); + rig.w(reg::ERROR_CAUSE_A, 1); + assert!(pi(0).refused().is_empty(), "PI0 slot A is CPU 0's"); + rig.w(reg::OFFSET_00B8, 0); + rig.w(reg::OFFSET_00C0, 1 << 3); + rig.w(PI1_BASE + reg::OFFSET_00A8, 1 << 7); + rig.w(reg::RT_ENABLE_B, 1); + rig.w(reg::PROFILE_ENABLE_B, 1); + rig.w(PI1_BASE + reg::ERROR_CAUSE_B, MEMORY_ERROR); + assert_eq!(pi(0).refused(), [(Refused::Interrupt { interface: 0, slot: 1 }, 1), (Refused::Timer { interface: 0, slot: 1 }, 2)]); + assert_eq!(pi(1).refused(), [(Refused::Interrupt { interface: 1, slot: 0 }, 1), (Refused::Interrupt { interface: 1, slot: 1 }, 1)]); + // An interprocessor interrupt to PI1/A lands in its pending bank and + // reaches no processor. + rig.w(PI1_BASE + reg::PENDING_MODIFY, 0x107); + assert_eq!(rig.r(PI1_BASE + reg::PENDING_STATUS_A) & (1 << 7), 1 << 7); + assert_eq!(rig.cause(), 0); +} diff --git a/src/dev/bedrock/tests.rs b/src/dev/bedrock/tests.rs new file mode 100644 index 000000000..09a5f5c8f --- /dev/null +++ b/src/dev/bedrock/tests.rs @@ -0,0 +1,94 @@ +//! Tests of the chip as a whole: what one unit's access does to another, +//! the lock-free interrupt publish, and the paths that take no lock. + +use super::reg::{pi, PI1_BASE}; +use super::*; +use crate::traits::{BUS_ERR, BUS_OK}; + +/// The L1 UART's line is bank 0 bit 4 of both PIs. +#[test] +fn the_uart_line_reaches_both_interfaces() { + let rig = Rig::new(); + rig.w(pi::OFFSET_00A8, 1 << 4); + let line = rig.chip.uart_irq(); + line.set(true); + assert_eq!(rig.r(pi::PENDING_STATUS_A), 1 << 4); + assert_eq!(rig.r(PI1_BASE + pi::PENDING_STATUS_A), 1 << 4); + assert_eq!(rig.cause(), 1); + line.set(false); + assert_eq!(rig.cause(), 0); +} + +/// Pending-modify stores, mask stores and a reflected level race on four +/// threads. Each publish is lock-free; once every writer has stopped, the +/// processor's word matches what the registers say, whatever the +/// interleaving. +#[test] +fn racing_stores_leave_the_word_matching_the_registers() { + const ROUNDS: u64 = 20_000; + let rig = Arc::new(Rig::new()); + let writers: Vec<_> = (0..4u64) + .map(|n| { + let rig = rig.clone(); + let uart = rig.chip.uart_irq(); + std::thread::spawn(move || { + for i in 0..ROUNDS { + match n { + // Two writers set and clear bank 0 bits 1 and 2. + 0 | 1 => rig.w(pi::PENDING_MODIFY, if (i + n) % 2 == 0 { 0x100 } else { 0 } | (n + 1)), + // One flips bank 0's mask between them. + 2 => rig.w(pi::OFFSET_00A8, if i % 3 == 0 { 0b010 } else { 0b110 }), + // One drives the UART's level, bank 0 bit 4. + _ => uart.set(i % 2 == 0), + } + } + }) + }) + .collect(); + for writer in writers { + writer.join().unwrap(); + } + let pending = rig.r(pi::PENDING_STATUS_A); + let mask = rig.r(pi::OFFSET_00A8); + assert_eq!(rig.cause(), u64::from(pending & mask != 0), "pending {pending:#x} mask {mask:#x}"); + rig.w(pi::OFFSET_00A8, 1 << 4); + rig.chip.uart_irq().set(true); + assert_eq!(rig.cause(), 1); +} + +/// Run `work` on another thread while this one holds every lock of the +/// chip; it must finish anyway. +fn without_locks(rig: &Arc, work: impl FnOnce(&Rig) + Send + 'static) { + let chip = &rig.chip; + let held = ( + chip.pi[0].hold(), + chip.pi[1].hold(), + ); + let (done, finished) = std::sync::mpsc::channel(); + let other = rig.clone(); + let worker = std::thread::spawn(move || { + work(&other); + done.send(()).unwrap(); + }); + let result = finished.recv_timeout(std::time::Duration::from_secs(10)); + drop(held); + worker.join().unwrap(); + assert!(result.is_ok(), "the path waited for a lock"); +} + +/// The PI paths a running guest takes most take no lock: a mask store, a +/// pending-modify store and a counter read, each publishing as it goes. +#[test] +fn the_pi_hot_paths_take_no_lock() { + let rig = Arc::new(Rig::new()); + without_locks(&rig, |rig| { + rig.w(pi::OFFSET_00A8, 1 << 9); + rig.w(pi::PENDING_MODIFY, 0x109); + assert_eq!(rig.cause(), 1); + rig.w(pi::PENDING_MODIFY, 0x009); + assert_eq!(rig.cause(), 0); + rig.set_time(8000); + assert_eq!(rig.r(pi::COUNTER), 10); + assert_eq!(rig.regs.read32(pi::COUNTER + 4).data, 10); + }); +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 51c0eff5d..30cebaf7b 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -74,6 +74,7 @@ pub mod hub_md; pub mod hub_ni; pub mod hub_pi; pub mod dp83840; +pub mod bedrock; pub mod dp83843; pub mod ics1890; pub mod timekeeper; From ba2aecd6d8e447502b37b27184eaf816b0b6c667 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 09/22] bedrock: add the memory controller Bedrock's memory controller decodes the node's banks by its configuration register, keeps ECC check bytes, directory entries and protection words, runs the memory self-test and answers the fetch-op, directory, ECC and scrub views. Ordinary CPU and DMA memory stays on Physical's bank views and takes none of the controller's locks. The board passes its bank sizes, locational bank 0 upward, and they are checked: at most eight banks, each a whole number of direct-window regions up to the 1 GiB the largest size code names. IP35 boards fit different banks, and the Fuel's two 256 MiB ones are only one case. Decode, self-test and the sparse ECC, directory and protection state read the size of the bank they touch and key it by where the bank starts, so banks of different sizes never share a key. RamExtent says where RAM can answer under any bank-select setting, for the processor bus and its cached view. A bank above 4 GiB answers through Physical's decode table, with no direct fast path, and its code pages keep logical generation counters like the rest. The PROM's memory test marks and lifts a few hundred ECC doublewords and directory lines thousands of times. Rebuilding the whole memory map for each made that slow, so a mark now takes one decode-table entry, and the first mark in a 64 MiB region withdraws the region from the direct bitmap without unmapping its host pages: a processor that read the old bitmap an instant earlier still reaches the bank's bytes. Each CPU's cached view follows the withdrawal, and a page it holds there is laid from the bank again when released, so a held line may stand on withdrawn memory. A lifted mark changes nothing at once. Lifted pages return to their banks together, when 1024 have piled up, after a second of controller time, on a zero self-test, or when the board asks. A test pins the cost: one table entry and two counters per marked page, none for a lift. The cell register's bit 0 gates the chip's register window and the board's Hub-special windows, so it is published as an atomic and the gate is one load. A memory error an access finds is returned with the controller's locks released, and the chip reports it to both PIs: bank 1 bit 61 for a correctable one, error-summary bit 24 for an uncorrectable one. The ECC view takes an uncached SH's check word from the whole source register, at the position the Fuel's PROM 6.210 memory test uses. Every other view stores the architectural low bits. --- src/dev/bedrock/md.rs | 2194 ++++++++++++++++++++++++++++++++++++++ src/dev/bedrock/mem.rs | 531 +++++++++ src/dev/bedrock/mod.rs | 130 ++- src/dev/bedrock/pi.rs | 8 + src/dev/bedrock/tests.rs | 19 +- src/ppmem/ppmem.rs | 190 +++- 6 files changed, 3033 insertions(+), 39 deletions(-) create mode 100644 src/dev/bedrock/md.rs create mode 100644 src/dev/bedrock/mem.rs diff --git a/src/dev/bedrock/md.rs b/src/dev/bedrock/md.rs new file mode 100644 index 000000000..2428e0037 --- /dev/null +++ b/src/dev/bedrock/md.rs @@ -0,0 +1,2194 @@ +//! Bedrock's memory and directory controller: bank configuration, memory +//! self-test, ECC, scrubbing and the directory and fetch-operation views. +//! +//! Memory is held by [`Memory`]. CPU coherence requests +//! and hardware protection enforcement are absent; protection registers +//! retain their words. Known unsupported writes are counted and ignored, +//! and unmapped accesses return a bus error. +//! +//! Plain register words, directory entries and reference counters, +//! protection entries and injected ECC check bytes live in sparse maps. The +//! guest touches a few of the millions of possible entries, and an entry it +//! never wrote reads as its default, so a dense table would only cost memory. +//! +//! Ported from hw/misc/sgi-bedrock-md.c in jamesbraid/qemu-sgi-origami. + +use super::mem::{Memory, RamExtent, BANK_SPAN}; +use super::Bedrock; +use crate::traits::*; +use parking_lot::Mutex; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::{Arc, Weak}; + + +pub mod reg { + pub const CELL: u64 = 0x0; + pub const CONFIG: u64 = 0x8; + pub const ARBITRATION: u64 = 0x10; + pub const MIGRATION_CONFIG: u64 = 0x18; + /// Outgoing reply queue sizes: read-only, so a write changes nothing. + pub const REPLY_QUEUE_SIZE: u64 = 0x60; + pub const PERF_SELECT0: u64 = 0x10000; + pub const PERF_SELECT1: u64 = 0x10008; + pub const PERF_COUNTER0: u64 = 0x10010; + pub const PERF_COUNTER1: u64 = 0x10018; + pub const REFRESH: u64 = 0x20000; + /// Flash, UART and Synergy timing on the local "junk" bus. + pub const JUNK_BUS_TIMING: u64 = 0x20008; + pub const SELFTEST_COMMAND: u64 = 0x20030; + pub const SELFTEST_DATA: u64 = 0x20038; + pub const SELFTEST_ERROR: u64 = 0x20040; + pub const SELFTEST_STATUS: u64 = 0x20048; + pub const ISSUE_DEBUG: u64 = 0x20060; + pub const DIRECTORY_CONFIG: u64 = 0x40000; + pub const DIRECTORY_ERROR: u64 = 0x40010; + /// Reads the directory error record and clears it. + pub const DIRECTORY_ERROR_ALIAS: u64 = 0x40018; + pub const MIGRATION_DIFFERENCE: u64 = 0x40040; + pub const MIGRATION_VALUE: u64 = 0x40048; + /// Outgoing request queue sizes; the model keeps them fixed. + pub const REQUEST_QUEUE_SIZE: u64 = 0x40050; + pub const DIRECTORY_DEBUG: u64 = 0x40060; + pub const ECC_CONFIG: u64 = 0x60000; + pub const MEMORY_ERROR: u64 = 0x60010; + /// Reads the memory error record and clears it. + pub const MEMORY_ERROR_ALIAS: u64 = 0x60018; + pub const MEMORY_DEBUG: u64 = 0x60040; + + /// Reset sizes: 12 entries in each header field and 24 in each data field. + pub const REPLY_QUEUE_SIZES: u64 = 0x0018_1818_0c0c_0c00; + /// Reset sizes: four requests for each of the three queues. + pub const REQUEST_QUEUE_SIZES: u64 = 0x0404_0400; +} + +const CONFIG_MASK: u64 = 0xfbfbfbfb | (3 << 32) | (15 << 44) | (63 << 48) | (7 << 56) | (3 << 60); +const ECC_COLUMNS: [u8; 72] = [ + 0x13, 0x23, 0x43, 0x83, 0x2f, 0xf1, 0x0d, 0x07, 0xd0, 0x70, 0x4f, 0xf8, 0x61, 0x62, 0x64, 0x68, + 0x1c, 0x2c, 0x4c, 0x8c, 0x15, 0x25, 0x45, 0x85, 0x19, 0x29, 0x49, 0x89, 0x1a, 0x2a, 0x4a, 0x8a, + 0x51, 0x52, 0x54, 0x58, 0x91, 0x92, 0x94, 0x98, 0xa1, 0xa2, 0xa4, 0xa8, 0x31, 0x32, 0x34, 0x38, + 0x16, 0x26, 0x46, 0x86, 0x1f, 0xf2, 0x0b, 0x0e, 0xb0, 0xe0, 0x8f, 0xf4, 0xc1, 0xc2, 0xc4, 0xc8, + 1, 2, 4, 8, 16, 32, 64, 128, +]; +pub fn ecc_compute(mut value: u64) -> u8 { + let mut code = 0; + while value != 0 { + code ^= ECC_COLUMNS[value.trailing_zeros() as usize]; + value &= value - 1; + } + code +} + +/// A memory error an access found, which the chip reports to both +/// processor interfaces once the access has released MD's lock. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub(super) enum MemoryError { + Correctable, + Uncorrectable, +} + +struct State { + cell: u64, + config: u64, + refresh: u64, + directory_config: u64, + ecc_config: u64, + storage: HashMap, + directory: HashMap, + counters: HashMap, + protection: HashMap, + compat: HashMap, + ecc: HashMap, + admitted: [u16; 2], + memory_error: u64, + directory_error: u64, + marked_directory: HashSet, + /// Marks in each 4 KiB page of the dense walk: injected check bytes and + /// marked directory lines. A page is checked while it has any. + marked_pages: HashMap, + /// Pages that gained their first mark since the CPU side last placed + /// them, and how many have, so an access can tell whether it added one. + newly_marked: Vec, + marks_added: u64, + /// A self-test cleared whole banks of marks. + retire_now: bool, + clock_ns: u64, + clock_anchor: Option, + selftest_deadline: Option, + selftest_result: bool, + selftest_status: u64, + selftest_error: u64, +} +impl Default for State { + fn default() -> Self { + Self { + cell: u64::MAX, + config: 0, + refresh: 0, + directory_config: 0, + ecc_config: 0, + storage: HashMap::new(), + directory: HashMap::new(), + counters: HashMap::new(), + protection: HashMap::new(), + compat: HashMap::new(), + ecc: HashMap::new(), + admitted: [0; 2], + memory_error: 0, + directory_error: 0, + marked_directory: HashSet::new(), + marked_pages: HashMap::new(), + newly_marked: Vec::new(), + marks_added: 0, + retire_now: false, + clock_ns: 0, + clock_anchor: None, + selftest_deadline: None, + selftest_result: false, + selftest_status: 0, + selftest_error: 0, + } + } +} + +impl State { + fn page_marked(&mut self, page: u64) { + let marks = self.marked_pages.entry(page).or_insert(0); + *marks += 1; + if *marks == 1 { + self.newly_marked.push(page); + self.marks_added += 1; + } + } + fn page_cleared(&mut self, page: u64) { + if let Some(marks) = self.marked_pages.get_mut(&page) { + *marks -= 1; + if *marks == 0 { + self.marked_pages.remove(&page); + } + } + } + /// Inject check byte `code` over the doubleword `key` of the dense walk. + fn ecc_mark(&mut self, key: u64, code: u8) { + if self.ecc.insert(key, code).is_none() { + self.page_marked(key / (4096 / 8)); + } + } + fn ecc_lift(&mut self, key: u64) { + if self.ecc.remove(&key).is_some() { + self.page_cleared(key / (4096 / 8)); + } + } + /// Mark directory line `line` of the dense walk as holding a bad entry. + fn directory_mark(&mut self, line: u64) { + if self.marked_directory.insert(line) { + self.page_marked(line / (4096 / 128)); + } + } + fn directory_lift(&mut self, line: u64) { + if self.marked_directory.remove(&line) { + self.page_cleared(line / (4096 / 128)); + } + } + /// Drop every mark, as a reset does. + fn lift_all(&mut self) { + self.ecc.clear(); + self.marked_directory.clear(); + self.marked_pages.clear(); + self.newly_marked.clear(); + } +} + +pub(super) struct Md { + memory: Arc, + clock: crate::dev::heart::Clock, + // Ordinary RAM uses Physical's bank views. Diagnostic metadata, repair + // and fetch operations retain the state lock; direct bank traffic has + // the same concurrency contract as the other machines' RAM. + state: Mutex, + // CPU-side metadata writes and resets serialize snapshot/publication. + // RAM/DMA reads and writes may remove metadata without taking this lock. + remapping: Mutex<()>, + /// The cell register, published for the checks every register access + /// and the board's Hub-special windows make. Written under `state`. + cell: AtomicU64, + unsupported: AtomicU64, +} +#[derive(Clone, Copy)] +pub(super) enum View { + Registers, + Memory, + Uncached, + Scrub, + Directory, + Fetchop, + Ecc, +} +/// One of MD's views as a bus device. An access that finds a memory error +/// reports it through the chip, so the view holds the chip, weakly: Physical +/// owns the memory view, and the chip owns Physical. +pub(super) struct MdBus { + chip: Weak, + view: View, + // External endpoints may outlive MD. Keep their generation pointers valid. + // The endpoint owned by Physical itself needs no extra reference. + _memory: Option>, + #[cfg(feature = "jitv2")] + _generations: Arc, +} + +impl Md { + /// A controller over `memory`, with `clock` timing its self-test, in + /// `chip`. Checked pages answer through MD's memory view. + pub(super) fn new(memory: Memory, clock: crate::dev::heart::Clock, chip: Weak) -> Self { + let md = Self { + memory: Arc::new(memory), + clock, + state: Mutex::new(State::default()), + remapping: Mutex::new(()), + cell: AtomicU64::new(State::default().cell), + unsupported: AtomicU64::new(0), + }; + md.memory.set_slow_view(md.view(chip, View::Memory)); + md + } + /// Where fitted RAM can answer, for the processor bus's cached view. + pub(super) fn ram_extent(&self) -> RamExtent { + self.memory.extent() + } + pub(super) fn state_locked(&self) -> bool { + self.state.is_locked() + } + pub(super) fn remapping_locked(&self) -> bool { + self.remapping.is_locked() + } + /// Hold both locks, for tests of the paths that must not take them. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + (self.remapping.lock(), self.state.lock()) + } + pub(super) fn physical(&self) -> &Arc { + self.memory.physical() + } + pub(super) fn view(&self, chip: Weak, view: View) -> Arc { + Arc::new(MdBus { + chip, + view, + #[cfg(feature = "jitv2")] + _generations: self.memory.generations(), + _memory: if matches!(view, View::Uncached) { + Some(self.physical().clone()) + } else { + None + }, + }) + } + /// DMA uses the same Physical entries as the CPU, without accessing + /// ppmem's CPU-owned bitmap. Hardware protection remains unsupported. + pub(super) fn dma_view(&self) -> Arc { + Arc::new(DmaView(self.memory.clone())) + } + + fn mapping(s: &State) -> (u64, BTreeSet) { + (s.config, s.marked_pages.keys().copied().collect()) + } + + /// Reinstall RAM after the board sets Physical's default. Call on the + /// CPU thread or with the CPU stopped, before installing board overlays. + /// Ordinary CPU config/diagnostic writes rebuild their changed mappings + /// automatically. This also releases pages whose metadata DMA consumed. + pub(super) fn rebuild_memory(&self) { + let _remapping = self.remapping.lock(); + let (config, pages) = Self::mapping(&self.state.lock()); + self.memory.remap(config, pages, true); + } + /// Zero every bank, as a power-on does. Physical's own `reset_memory` + /// advances only the banks' counters; code pages follow the logical ones, + /// so a board resets the banks through here. + #[cfg(test)] + pub(super) fn reset_memory(&self) { + self.memory.physical().reset_memory(); + self.memory.changed_decode(); + } + /// Load bank `bank` from a file of `Physical::save_bank`'s format, and + /// advance the logical counters the bytes changed under. + #[cfg(test)] + pub(super) fn load_bank(&self, bank: usize, path: impl AsRef) -> std::io::Result<()> { + let loaded = self.memory.physical().load_bank(bank, path); + self.memory.changed_decode(); + loaded + } + /// Restore bank `bank` from words `Physical::snapshot_bank_inmem` took, + /// and advance the logical counters the bytes changed under. + #[cfg(test)] + pub(super) fn restore_bank(&self, bank: usize, words: &[u32]) { + self.memory.physical().restore_bank_inmem(bank, words); + self.memory.changed_decode(); + } + /// The stable logical-physical counters used by both compiled stores + /// and SysAd page claims. Their storage outlives the CPU via its bus. + #[cfg(feature = "jitv2")] + pub(super) fn generation_base(&self) -> *mut AtomicU64 { + self.memory.generations().base() + } + /// Explicit elapsed time, for tests that drive the self-test by hand. + #[cfg(test)] + pub(super) fn advance_ns(&self, elapsed: u64) { + let mut s = self.state.lock(); + Self::advance(&mut s, elapsed); + } + fn advance(s: &mut State, elapsed: u64) { + s.clock_ns = s.clock_ns.saturating_add(elapsed); + if s.selftest_deadline.is_some_and(|end| s.clock_ns >= end) { + s.selftest_deadline = None; + s.selftest_status = if s.selftest_result { 3 } else { 2 }; + s.selftest_error = if s.selftest_result { 0 } else { 1 << 48 }; + } + } + fn catch_up(s: &mut State, now: u64) { + if let Some(anchor) = s.clock_anchor { + s.clock_anchor = Some(anchor.max(now)); + Self::advance(s, now.saturating_sub(anchor)); + } + } + pub(super) fn start_clock(&self) { + let now = (self.clock)(); + self.state.lock().clock_anchor.get_or_insert(now); + } + pub(super) fn stop_clock(&self) { + let now = (self.clock)(); + let mut s = self.state.lock(); + Self::catch_up(&mut s, now); + s.clock_anchor = None; + } + /// Cell register bit 0, which gates the register window and the board's + /// Hub-special windows. One atomic load. + #[inline] + pub(super) fn cell_permits(&self) -> bool { + self.cell.load(Ordering::Relaxed) & 1 != 0 + } + #[cfg(test)] + pub(super) fn unsupported(&self) -> u64 { + self.unsupported.load(Ordering::Relaxed) + } + /// Count and log an access the model does not implement; the caller + /// answers a bus error. + fn refuse(&self, what: &str, at: u64) -> Option { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD refused {what} access at {at:#x}"); + None + } + /// Called on the CPU thread or with the CPU stopped. + #[cfg(test)] + pub(super) fn soft_reset(&self) { + let _remapping = self.remapping.lock(); + let mut s = self.state.lock(); + s.cell = u64::MAX; + self.cell.store(s.cell, Ordering::Relaxed); + s.storage.clear(); + s.ecc_config = 0; + s.memory_error = 0; + s.directory_error = 0; + s.lift_all(); + s.selftest_deadline = None; + s.selftest_status = 0; + s.selftest_error = 0; + s.refresh &= (1 << 63) | 0xfff; + let (config, pages) = Self::mapping(&s); + drop(s); + self.memory.remap(config, pages, false); + } + /// An access through `view`, and the memory errors it found. The errors + /// are returned with every MD lock released, for the chip to report. + pub(super) fn access(&self, view: View, addr: u64, bytes: u64, write: Option) -> (Option, Vec) { + let mut event = Vec::new(); + if ![1, 2, 4, 8].contains(&bytes) || addr % bytes != 0 { + return (self.refuse("access", addr), event); + } + // Only CPU register/configuration and diagnostic writes can add + // metadata. Keep their remap off DMA threads: ordinary RAM, scrub + // and fetch operations only remove entries, leaving safe slow pages + // behind until a CPU-side retirement. These write views, like HEART's + // MEMCFG, are CPU-thread-only while the machine runs. + let reconfigure = write.is_some() && matches!(view, View::Registers) && addr == reg::CONFIG; + let marking = write.is_some() && matches!(view, View::Directory | View::Ecc | View::Registers); + let remapping = reconfigure.then(|| self.remapping.lock()); + // Sample outside state: even an injected clock must not call out under + // the memory lock. Only register observations need the self-test timer. + let now = matches!(view, View::Registers).then(|| (self.clock)()); + let (result, snapshot, placing) = { + let mut s = self.state.lock(); + if let Some(now) = now { + Self::catch_up(&mut s, now); + } + let marks_added = s.marks_added; + let result = match view { + View::Registers => self.register(&mut s, addr, bytes, write), + View::Directory => self.directory(&mut s, addr, bytes, write), + View::Ecc => self.ecc(&mut s, addr, bytes, write), + View::Scrub => self.scrub(&mut s, addr, bytes, write, &mut event), + View::Uncached => { + let result = self.ram(&mut s, addr, bytes, write, &mut event); + if write.is_none() && event.contains(&MemoryError::Uncorrectable) { + None + } else { + result + } + } + View::Fetchop => self.fetchop(&mut s, addr, bytes, write, &mut event), + View::Memory => self.ram(&mut s, addr, bytes, write, &mut event), + }; + self.cell.store(s.cell, Ordering::Relaxed); + let placing = marking && (s.marks_added != marks_added || s.retire_now); + (result, reconfigure.then(|| Self::mapping(&s)), placing) + }; + if let Some((config, pages)) = snapshot { + self.memory.reconfigure(config, || pages); + } + drop(remapping); + if placing { + self.place_marks(); + } + (result, event) + } + /// Route the pages that gained a mark through the MD endpoint, and + /// retire the pages whose marks were lifted when enough have piled up or + /// have idled long enough. Only a mark can make a page checked, so this + /// runs after the writes that add one, on the CPU thread. + fn place_marks(&self) { + let _remapping = self.remapping.lock(); + let now = (self.clock)(); + let (added, active, retire_now) = { + let mut s = self.state.lock(); + let added = std::mem::take(&mut s.newly_marked); + (added, s.marked_pages.len(), std::mem::take(&mut s.retire_now)) + }; + self.memory.place(&added); + if retire_now || self.memory.retirement_due(active, Some(now)) { + self.retire_lifted(); + } + } + /// Return the pages whose marks are lifted to their banks, in one change. + /// Call on the CPU thread, holding `remapping`. + fn retire_lifted(&self) { + let lifted = { + let s = self.state.lock(); + self.memory.lifted(|page| s.marked_pages.contains_key(&page)) + }; + self.memory.retire(&lifted); + } + /// Return the pages whose ECC or directory marks were lifted to the + /// direct window now, without waiting for the batch to fill or idle. For + /// the board to call from the CPU thread, or with the CPU stopped, when + /// it has a quiet moment. + pub(super) fn retire_lifted_marks(&self) { + let _remapping = self.remapping.lock(); + self.retire_lifted(); + } + fn register(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if bytes != 8 { + return self.refuse("register", at); + } + let old = match at { + reg::CELL => s.cell, + reg::CONFIG => s.config, + reg::REPLY_QUEUE_SIZE => reg::REPLY_QUEUE_SIZES, + reg::REFRESH => s.refresh, + reg::SELFTEST_ERROR => s.selftest_error, + reg::SELFTEST_STATUS => s.selftest_status, + reg::DIRECTORY_CONFIG => s.directory_config, + reg::ECC_CONFIG => s.ecc_config, + reg::REQUEST_QUEUE_SIZE => reg::REQUEST_QUEUE_SIZES, + reg::MEMORY_ERROR | reg::MEMORY_ERROR_ALIAS => s.memory_error, + reg::DIRECTORY_ERROR | reg::DIRECTORY_ERROR_ALIAS => s.directory_error, + _ if storage_mask(at).is_some() => *s.storage.get(&at).unwrap_or(&0), + _ if idle(at) => 0, + _ => return self.refuse("register", at), + }; + if let Some(value) = write { + match at { + reg::SELFTEST_COMMAND => return self.selftest(s, value), + reg::CELL => s.cell = value, + reg::CONFIG => { + s.config = value & CONFIG_MASK; + } + reg::REFRESH => s.refresh = value & ((1 << 63) | 0xfff), + reg::DIRECTORY_CONFIG => s.directory_config = value & 3, + reg::ECC_CONFIG => s.ecc_config = value & 1, + _ if storage_mask(at).is_some() => { + s.storage.insert(at, value & storage_mask(at).unwrap()); + } + _ => { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD ignored register write at {at:#x}: {value:#x}"); + } + } + } else if at == reg::MEMORY_ERROR_ALIAS { + s.memory_error = 0; + } else if at == reg::DIRECTORY_ERROR_ALIAS { + s.directory_error = 0; + } + Some(old) + } + fn directory(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if bytes != 8 || at >= 1 << 30 { + return self.refuse("directory", at); + } + let software = at / (BANK_SPAN / 8); + let bank = software ^ ((s.config >> 32) & 3); + if at & 0x7c8 == 0x408 { + let fitted = self.memory.fitted(bank); + if fitted == 0 { + return Some(0); + } + let within = + ((((at % (BANK_SPAN / 8)) >> 11) << 14) | (((at >> 4) & 3) << 7)) % fitted; + let key = ((self.memory.dense_base(bank) + within) >> 14) * 4 + ((within >> 7) & 3); + let old = *s.protection.get(&key).unwrap_or(&0); + if let Some(v) = write { + s.protection.insert(key, v); + } + return Some(old); + } + let within = ((at % (BANK_SPAN / 8)) & !15) * 8; + let field = s.config >> (bank / 2 * 8); + let code = (field >> 4) & 15; + if field & (1 << (bank & 1)) == 0 + || !(1..=5).contains(&code) + || within >= (32 << 20) << code + { + return self.refuse("directory", at); + } + let mask = if at & 8 != 0 || s.directory_config & 1 != 0 { + u64::MAX + } else { + u32::MAX as u64 + }; + let table; + let key; + let fitted = self.memory.fitted(bank); + if within >= fitted { + if !s.compat.contains_key(&at) && s.compat.len() >= 74 { + return self.refuse("directory", at); + } + s.compat.entry(at).or_insert(0); + table = &mut s.compat; + key = at; + } else { + key = (self.memory.dense_base(bank) + within) / 128; + table = if at & 8 != 0 { + &mut s.counters + } else { + &mut s.directory + }; + } + let old = *table.get(&key).unwrap_or(&0) & mask; + if let Some(value) = write { + let value = if at & 8 == 0 && s.directory_config & 2 == 0 { + directory_encode(value, s.directory_config & 1 != 0) + } else { + value + }; + table.insert(key, value & mask); + if within < fitted && at & 8 == 0 { + if directory_syndrome(value, s.directory_config & 1 != 0) != 0 { + s.directory_mark(key); + } else { + s.directory_lift(key); + } + } + } + Some(old) + } + fn checked_word( + &self, + s: &mut State, + bank: usize, + at: u64, + address: u64, + event: &mut Vec, + ) -> u64 { + let mut data = self.memory.raw_read(bank, at); + if s.ecc_config & 1 != 0 || s.ecc.is_empty() { + return data; + } + let key = (self.memory.dense_base(bank as u64) + at) / 8; + if let Some(&check) = s.ecc.get(&key) { + let syndrome = check ^ ecc_compute(data); + if syndrome != 0 { + let bit = ECC_COLUMNS.iter().position(|&v| v == syndrome); + let valid = if bit.is_some() { 1 << 52 } else { 1 << 60 }; + if bit.is_none() + && s.memory_error & (1 << 60) != 0 + && (s.memory_error ^ address) & (((1u64 << 30) - 1) << 3) & !127 != 0 + { + s.memory_error |= 1 << 63; + } + if s.memory_error & (valid | (1 << 60)) != 0 { + s.memory_error |= valid << 1; + } else { + s.memory_error = (s.memory_error + & !((0xff << 40) | (((1u64 << 30) - 1) << 3))) + | ((syndrome as u64) << 40) + | (address & (((1u64 << 30) - 1) << 3)); + } + s.memory_error |= valid; + if let Some(bit) = bit { + if bit < 64 { + data ^= 1 << bit; + self.memory.raw_write(bank, at, data); + } + s.ecc_lift(key); + event.push(MemoryError::Correctable); + } else { + event.push(MemoryError::Uncorrectable); + } + } + } + data + } + fn ram( + &self, + s: &mut State, + address: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + let Some((bank, offset)) = self.memory.decode(s.config, address, bytes) else { + return self.refuse("ram", address); + }; + if !s.marked_directory.is_empty() { + let line = (self.memory.dense_base(bank as u64) + offset) / 128; + if s.marked_directory.contains(&line) { + let premium = s.directory_config & 1 != 0; + let word = *s.directory.get(&line).unwrap_or(&0); + if write.is_some() { + s.directory.insert(line, directory_encode(word, premium)); + s.directory_lift(line); + } else if s.directory_config & 2 == 0 { + let syndrome = directory_syndrome(word, premium); + if syndrome != 0 { + let bit = directory_columns(premium) + .iter() + .position(|&v| v == syndrome); + let valid = if bit.is_some() { 1 << 61 } else { 1 << 63 }; + if s.directory_error & ((1 << 63) | valid) != 0 { + s.directory_error |= if bit.is_some() { 1 << 57 } else { 1 << 59 }; + } else { + let carrier = 0x1c0000000 + + (address / BANK_SPAN) * (BANK_SPAN / 8) + + ((offset & !127) / 8); + s.directory_error = (s.directory_error + & !((0x7f << 40) | (((1u64 << 30) - 1) << 3))) + | ((syndrome as u64) << 40) + | (carrier & (((1u64 << 30) - 1) << 3)); + } + s.directory_error |= valid; + if let Some(bit) = bit { + s.directory.insert(line, word ^ (1 << bit)); + s.directory_lift(line); + event.push(MemoryError::Correctable); + } else { + event.push(MemoryError::Uncorrectable); + return None; + } + } + } + } + } + let at = offset & !7; + let shift = (8 - (offset & 7) - bytes) * 8; + let mask = u64::MAX >> ((8 - bytes) * 8); + if let Some(value) = write { + let old = self.memory.raw_read(bank, at); + self.memory.raw_write( + bank, + at, + (old & !(mask << shift)) | ((value & mask) << shift), + ); + if !s.ecc.is_empty() { + s.ecc_lift((self.memory.dense_base(bank as u64) + at) / 8); + } + Some(0) + } else { + Some((self.checked_word(s, bank, at, address, event) >> shift) & mask) + } + } + fn fetchop( + &self, + s: &mut State, + at: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + if bytes != 4 && bytes != 8 { + return self.refuse("fetchop", at); + } + let op = at & 0x38; + if op > if write.is_some() { 32 } else { 24 } { + return self.refuse("fetchop", at); + } + let address = (at & !63) | (at & 4); + let old = self.ram(s, address, bytes, None, event)?; + let next = match (write, op) { + (None, 0) => return Some(old), + (Some(v), 0) => v, + (_, 8) => old.wrapping_add(1), + (_, 16) => old.wrapping_sub(1), + (None, 24) => 0, + (Some(v), 24) => old & v, + (Some(v), 32) => old | v, + _ => return self.refuse("fetchop", at), + }; + self.ram(s, address, bytes, Some(next), event)?; + Some(old) + } + // Approximation: report whether the selected fitting covers the + // programmed bank, after 1000 guest ns. Only the zero write phase mutates + // memory; other patterns have no published transformation here. + fn selftest(&self, s: &mut State, command: u64) -> Option { + if command == 2 || command == 4 { + s.selftest_deadline = None; + s.selftest_status = 0; + s.selftest_error = 0; + return Some(0); + } + if command & 7 != 1 || command & !0x71 != 0 { + self.unsupported.fetch_add(1, Ordering::Relaxed); + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock MD ignored self-test command {command:#x}"); + return Some(0); + } + let selected = (command >> 4) & 7; + let configured = selected ^ ((s.config >> 32) & 3); + let field = s.config >> ((configured / 2) * 8); + let code = (field >> 4) & 15; + let size = if (1..=5).contains(&code) { + (32 << 20) << code + } else { + 0 + }; + let fits = field & (1 << (configured & 1)) != 0 + && size != 0 + && size <= self.memory.fitted(selected); + s.selftest_result = fits; + s.selftest_status = 0; + s.selftest_error = 0; + s.selftest_deadline = Some(s.clock_ns.saturating_add(1000)); + if fits && s.storage.get(&0x20038).copied().unwrap_or(0) == 0x80 { + self.memory.zero(selected as usize, size); + let start = self.memory.dense_base(selected); + let words: Vec = s + .ecc + .keys() + .copied() + .filter(|&key| key >= start / 8 && key < (start + size) / 8) + .collect(); + for key in words { + s.ecc_lift(key); + } + // The pages those marks held can go back to the window now. + s.retire_now = true; + let lines: Vec = s + .marked_directory + .iter() + .copied() + .filter(|&line| line >= start / 128 && line < (start + size) / 128) + .collect(); + for line in lines { + let word = *s.directory.get(&line).unwrap_or(&0); + s.directory + .insert(line, directory_encode(word, s.directory_config & 1 != 0)); + s.directory_lift(line); + } + } + Some(0) + } + fn scrub( + &self, + s: &mut State, + at: u64, + bytes: u64, + write: Option, + event: &mut Vec, + ) -> Option { + if write.is_none() { + return self.ecc(s, at, bytes, None); + } + if bytes != 8 || at >= 1 << 30 { + return self.refuse("scrub", at); + } + let software = at / (BANK_SPAN / 8); + let within = at % (BANK_SPAN / 8); + let address = software * BANK_SPAN + within * 8; + let Some((bank, offset)) = self.memory.decode(s.config, address, 8) else { + return self.refuse("scrub", at); + }; + for i in 0..8 { + if self.memory.decode(s.config, address + i * 8, 8).is_some() { + self.checked_word(s, bank, offset + i * 8, address + i * 8, event); + } + } + Some(0) + } + fn ecc(&self, s: &mut State, at: u64, bytes: u64, write: Option) -> Option { + if at >= 1 << 30 || (bytes != 1 && bytes != 2) { + return self.refuse("ecc", at); + } + let software = at / (BANK_SPAN / 8); + let within = (at % (BANK_SPAN / 8)) ^ if bytes == 1 { 7 } else { 6 }; + let address = software * BANK_SPAN + within * 8; + let Some((bank, offset)) = self.memory.decode(s.config, address, bytes * 8) else { + if bytes == 2 && [0x04000006, 0x08000006].contains(&at) { + let i = usize::from(at == 0x08000006); + let old = s.admitted[i]; + if let Some(value) = write { + s.admitted[i] = value as u16; + } + return Some(old as u64); + } + return self.refuse("ecc", at); + }; + let mut old = 0; + for i in 0..bytes { + let raw = offset + i * 8; + let key = (self.memory.dense_base(bank as u64) + raw) / 8; + let computed = ecc_compute(self.memory.raw_read(bank, raw)); + old |= (s.ecc.get(&key).copied().unwrap_or(computed) as u64) << (i * 8); + if let Some(value) = write { + let code = (value >> (i * 8)) as u8; + if code == computed { + s.ecc_lift(key); + } else { + s.ecc_mark(key, code); + } + } + } + Some(old) + } +} +impl Resettable for Md { + fn power_on(&self) { + // Reset is called on the CPU thread or with it stopped. + let _remapping = self.remapping.lock(); + let mut s = self.state.lock(); + *s = State::default(); + self.cell.store(s.cell, Ordering::Relaxed); + drop(s); + self.memory.remap(0, BTreeSet::new(), false); + } +} +/// Resolve each DMA access through the atomic Physical table, as HEART +/// does. Do not ask Physical for mem_ptr or gen_ptr: those read the inline +/// CPU bitmap. Default block methods split accesses and respect slow pages +/// even if a caller supplies a block crossing a 4 KiB page boundary. +struct DmaView(Arc); +impl DmaView { + fn at(&self, address: u64, bytes: u64) -> Option<&dyn BusDevice> { + self.0.decode(self.0.config(), address, bytes)?; + let phys = self.0.physical(); + let entry = phys.entry_at(address); + if crate::ppmem::devmap::entry_eq(entry, phys.default_entry()) { + return None; + } + // SAFETY: the memory this view holds retains Physical, which keeps every entry + // alive across remaps, including the weak MD slow endpoint. + Some(unsafe { &*entry }) + } +} +macro_rules! dma_rw { + ($read:ident, $write:ident, $result:ident, $ty:ty) => { + fn $read(&self, address: u64) -> $result { + self.at(address, std::mem::size_of::<$ty>() as u64) + .map_or($result::err(), |device| device.$read(address)) + } + fn $write(&self, address: u64, value: $ty) -> u32 { + self.at(address, std::mem::size_of::<$ty>() as u64) + .map_or(BUS_ERR, |device| { + let result = device.$write(address, value); + if result == BUS_OK { self.0.written(address); } + result + }) + } + }; +} +impl BusDevice for DmaView { + dma_rw!(read8, write8, BusRead8, u8); + dma_rw!(read16, write16, BusRead16, u16); + dma_rw!(read32, write32, BusRead32, u32); + dma_rw!(read64, write64, BusRead64, u64); + fn cas32(&self, address: u64, old: u32, new: u32) -> Option { + let swapped = self.at(address, 4)?.cas32(address, old, new)?; + if swapped { self.0.written(address); } + Some(swapped) + } + fn cas64(&self, address: u64, old: u64, new: u64) -> Option { + let swapped = self.at(address, 8)?.cas64(address, old, new)?; + if swapped { self.0.written(address); } + Some(swapped) + } + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + self.0.gen_ptr(address) + } +} + +impl BusDevice for MdBus { + /// Bedrock's ECC view is the one device that sees the whole source register + /// of an uncached SH (`BusDevice::write_narrow_from`); every other view + /// takes the architectural low bits. + fn write_narrow_from(&self, addr: u64, size: usize, gpr: u64) -> u32 { + if size == 2 && matches!(self.view, View::Ecc) { + // PROM 6.210's memory test stores each check word with `sh` from a + // register whose low 16 bits are zero: the word sits elsewhere in + // the register, by the address of the memory it checks. Observed + // (memory address mod 0x40, carrier offset mod 8, register bits): + // 0x00 6 15:0 + // 0x10 4 63:48 + // 0x20 2 47:32 + // 0x30 0 31:16 + // The shift below gives exactly these four. Which lanes the R14000 + // drives for a narrow store, and how Bedrock wires them to the + // check bits, is unknown; this matches the PROM, not a hardware + // description. + let memory_address = (addr ^ 6).wrapping_mul(8); + return self.write16(addr, (gpr >> (memory_address.wrapping_neg() & 63)) as u16); + } + if size == 1 { self.write8(addr, gpr as u8) } else { self.write16(addr, gpr as u16) } + } + + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, address: u64) -> *const std::sync::atomic::AtomicU64 { + let Some(chip) = self.chip.upgrade() else { + return std::ptr::null(); + }; + if !matches!(self.view, View::Memory | View::Uncached) { + return std::ptr::null(); + } + chip.md.memory.gen_ptr(address) + } + + fn read8(&self, a: u64) -> BusRead8 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 1, None)) + .map_or(BusRead8::err(), |v| BusRead8::ok(v as u8)) + } + fn read16(&self, a: u64) -> BusRead16 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 2, None)) + .map_or(BusRead16::err(), |v| BusRead16::ok(v as u16)) + } + fn read32(&self, a: u64) -> BusRead32 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 4, None)) + .map_or(BusRead32::err(), |v| BusRead32::ok(v as u32)) + } + fn read64(&self, a: u64) -> BusRead64 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 8, None)) + .map_or(BusRead64::err(), BusRead64::ok) + } + fn write8(&self, a: u64, v: u8) -> u32 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 1, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write16(&self, a: u64, v: u16) -> u32 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 2, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write32(&self, a: u64, v: u32) -> u32 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 4, Some(v as u64))) + .map_or(BUS_ERR, |_| BUS_OK) + } + fn write64(&self, a: u64, v: u64) -> u32 { + self.chip + .upgrade() + .and_then(|chip| chip.md_access(self.view, a, 8, Some(v))) + .map_or(BUS_ERR, |_| BUS_OK) + } +} +fn storage_mask(at: u64) -> Option { + Some(match at { + reg::ARBITRATION => 0xff, + reg::MIGRATION_CONFIG => 0x3ff | (0xff << 16) | (1 << 32), + reg::PERF_SELECT0 | reg::PERF_SELECT1 => { + 3 | (15 << 4) | (127 << 8) | (15 << 16) | (0xffffffffff << 20) | (1 << 63) + } + reg::PERF_COUNTER0 | reg::PERF_COUNTER1 => (1 << 41) - 1, + reg::JUNK_BUS_TIMING => 0xffffffffffff, + reg::SELFTEST_DATA => 0xff, + reg::ISSUE_DEBUG => 3, + reg::MIGRATION_DIFFERENCE => 0x7fff | (7 << 32) | (7 << 35) | (1 << 63), + reg::MIGRATION_VALUE => 0x7fff | (7 << 32) | (7 << 35) | (3 << 62), + reg::DIRECTORY_DEBUG => 3, + reg::MEMORY_DEBUG => 1, + _ => return None, + }) +} +fn idle(at: u64) -> bool { + matches!( + at, + 0x20 | 0x28 + | 0x40 + | 0x48 + | 0x58 + | 0x20010 + | 0x20018 + | 0x20020 + | 0x20028 + | 0x20030 + | 0x20040 + | 0x20048 + | 0x40020 + | 0x40028 + | 0x40030 + | 0x40038 + | 0x40058 + | 0x60020 + | 0x60028 + | 0x60030 + | 0x60038 + ) +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-md-test.c in + // jamesbraid/qemu-sgi-origami: memory-banks, reset, queue-sizes, + // writable-masks, selftest-phases, fetchop-operations, ecc-carrier and + // directory-storage. + use super::*; + use super::super::mem::{RETIRED_IDLE_NS, RETIRED_LIMIT}; + use super::super::{BedrockConfig, Population}; + + /// The controller inside a chip of its own, with the views MD hands out. + struct TestMd(Arc); + impl std::ops::Deref for TestMd { + type Target = Md; + fn deref(&self) -> &Md { + &self.0.md + } + } + impl TestMd { + fn new(banks: &[u32]) -> Result, String> { + Self::with_clock(banks, Arc::new(|| 0)) + } + fn with_clock(banks: &[u32], clock: crate::dev::heart::Clock) -> Result, String> { + let chip = Bedrock::new(BedrockConfig { banks_mib: banks, ..super::super::test_config(clock, Population::SINGLE) })?; + Ok(Arc::new(Self(chip))) + } + fn md_bus(&self) -> Arc { + self.0.md.view(Arc::downgrade(&self.0), View::Registers) + } + fn memory_view(&self) -> Arc { + self.0.md.view(Arc::downgrade(&self.0), View::Memory) + } + fn uncached_view(&self) -> Arc { + self.0.uncached_view() + } + fn directory_bus(&self) -> Arc { + self.0.directory_bus() + } + fn ecc_bus(&self) -> Arc { + self.0.ecc_bus() + } + fn ecc_scrub_bus(&self) -> Arc { + self.0.ecc_scrub_bus() + } + fn fetchop_bus(&self) -> Arc { + self.0.fetchop_bus() + } + fn dma_view(&self) -> Arc { + self.0.dma_view() + } + /// Whether each PI has a correctable (bank 1 bit 61) and an + /// uncorrectable (error-summary bit 24) memory error reported. + fn reported(&self) -> [(bool, bool); 2] { + let regs = self.0.registers(super::super::Form::Node); + std::array::from_fn(|n| { + let base = n as u64 * super::super::reg::PI1_BASE; + let pending = regs.read64(base + super::super::reg::pi::PENDING_STATUS_B).data; + let summary = regs.read64(base + super::super::reg::pi::ERROR_SUMMARY).data; + (pending & (1 << 61) != 0, summary & (1 << 24) != 0) + }) + } + } + const FUEL_BANKS: [u32; 2] = [256, 256]; + const FITTED_BANK: u64 = 256 << 20; + fn fuel() -> Arc { + TestMd::new(&FUEL_BANKS).unwrap() + } + fn configured() -> Arc { + let md = fuel(); + assert_eq!(md.md_bus().write64(8, 0x33), BUS_OK); + md + } + + /// Ordinary CPU and DMA RAM accesses remain available while MD state is locked. + #[test] + fn ordinary_cpu_and_dma_memory_do_not_take_the_md_lock() { + let md = configured(); + let dma = md.dma_view(); + let _state = md.state.lock(); + assert_eq!(md.physical().write64(0x1000, 0x1122334455667788), BUS_OK); + assert_eq!(md.physical().read64(0x1000).data, 0x1122334455667788); + assert_eq!(dma.read64(0x1000).data, 0x1122334455667788); + assert_eq!(dma.write32(0x1004, 0xabcdef01), BUS_OK); + assert_eq!(md.physical().read64(0x1000).data, 0x11223344abcdef01); + assert_eq!(dma.read64(FITTED_BANK).status, BUS_ERR); + } + + /// An ECC injection removes its direct mapping; DMA repair keeps the page slow. + #[test] + fn injected_checks_disable_the_window_and_dma_repair_keeps_the_slow_page() { + use crate::ppmem::MappedMemory; + let md = configured(); + let phys = md.physical(); + let space = phys.ppmem_space().expect("a window"); + assert_eq!(space.mapped_bitmap() & 3, 3); + assert!(phys.mem_ptr(0).is_some()); + assert_eq!(md.ecc_bus().write8(7, 0x13), BUS_OK); + assert_eq!(space.mapped_bitmap() & 3, 2, "the full 64 MiB region loses its direct bit"); + assert!(phys.mem_ptr(0).is_none()); + let slow = phys.entry_at(0); + assert!(!crate::ppmem::devmap::entry_eq(slow, phys.entry_at(0x1000))); + let dma = md.dma_view(); + let repaired = std::thread::spawn(move || dma.read64(0)).join().unwrap(); + assert_eq!(repaired.status, BUS_OK); + assert_eq!(repaired.data, 1); + assert!(md.state.lock().ecc.is_empty()); + assert_eq!(space.mapped_bitmap() & 3, 2, "a DMA thread never republishes the CPU bitmap"); + assert!(crate::ppmem::devmap::entry_eq(slow, phys.entry_at(0))); + // Adjacent pages still use the ordinary bank entry, even though + // the region no longer has a direct window mapping. + { + let _state = md.state.lock(); + assert_eq!(phys.write64(0x1000, 9), BUS_OK); + assert_eq!(phys.read64(0x1000).data, 9); + } + md.soft_reset(); + assert_eq!(space.mapped_bitmap() & 3, 3); + assert!(phys.mem_ptr(0).is_some()); + assert_eq!(phys.read64(0).data, 1); + } + + /// Directory-marked pages follow their physical bank after a configuration swap. + #[test] + fn marked_directory_pages_follow_their_bank_when_swizzled() { + use crate::ppmem::MappedMemory; + let md = configured(); + let phys = md.physical(); + let space = phys.ppmem_space().expect("a window"); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 2); + md.directory_bus().write64(0, 0x001e0001); + assert_eq!(space.mapped_bitmap() & 1, 0); + assert!(phys.mem_ptr(0).is_none()); + md.md_bus().write64(reg::CONFIG, 0x33 | (1 << 32)); + assert_ne!(space.mapped_bitmap() & 1, 0, "the other bank now answers at zero"); + assert_eq!(space.mapped_bitmap() & (1 << 16), 0); + assert!(phys.mem_ptr(BANK_SPAN).is_none()); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 0); + assert_eq!(phys.read64(BANK_SPAN).status, BUS_OK); + assert_ne!(md.md_bus().read64(reg::DIRECTORY_ERROR).data & (1 << 61), 0); + md.rebuild_memory(); + assert!(phys.mem_ptr(BANK_SPAN).is_some()); + md.power_on(); + assert_eq!(space.mapped_bitmap(), 0); + assert_eq!(phys.read64(BANK_SPAN).status, BUS_ERR); + } + + /// Guest code must not keep a compiled page after its backing bank changes. + /// page_for deliberately retains its generation pointer. A remap must + /// invalidate that page and subsequent writes to its new bank must keep + /// invalidating the same pointer, including after swapping back. + #[cfg(feature = "jitv2")] + #[test] + fn a_claimed_jit_page_tracks_physical_ownership_across_bank_swaps() { + use crate::cpu::jitv2::jitv2::Jitv2; + let md = configured(); + let dma = md.dma_view(); + let mut jit = Jitv2::new(4); + let slot = jit.page_for(0, 0, dma.as_ref(), false).unwrap(); + let page = jit.page_ptr(slot); + let generation = || unsafe { (*page).current_gen() }; + let pointer = dma.gen_ptr(0); + let mut entries = [0u64; crate::cpu::jitv2::jitv2::BITMAP_WORDS]; + entries[0] = 1; + // Exercise the JIT publication/invalidation protocol without + // executing a synthetic function body. + let code = std::ptr::NonNull::<()>::dangling().as_ptr() as *const (); + assert!(unsafe { (*page).publish(&entries, code, generation(), 1, 0, false) }); + assert!(unsafe { (*page).is_runnable(0) }); + let first = generation(); + md.md_bus().write64(reg::CONFIG, 0x33 | (1 << 32)); + assert_eq!(jit.page_for(0, 0, dma.as_ref(), false), Some(slot)); + assert_eq!(dma.gen_ptr(0), pointer); + assert!(generation() > first); + assert!(!unsafe { (*page).is_runnable(0) }); + let swapped = generation(); + assert!(unsafe { (*page).publish(&entries, code, swapped, 1, 0, false) }); + assert!(unsafe { (*page).is_runnable(0) }); + assert_eq!(dma.write64(0, 0x11223344), BUS_OK); + assert!(generation() > swapped); + assert!(!unsafe { (*page).is_runnable(0) }, "code published after a swap sees new-bank writes"); + assert_eq!(md.physical().banks()[1].read64(0).data, 0x11223344); + let written = generation(); + md.md_bus().write64(reg::CONFIG, 0x33); + assert!(generation() > written, "swapping back cannot lower the logical generation"); + let restored = generation(); + md.uncached_view().write64(0, 0x55667788); + assert!(generation() > restored); + let before_fetchop = generation(); + md.fetchop_bus().write64(0, 0); + assert!(generation() > before_fetchop); + md.ecc_bus().write8(7, 0x13); + let before_repair = generation(); + assert_eq!(dma.read64(0).data, 1); + assert!(generation() > before_repair, "ECC repair invalidates logical code pages"); + md.md_bus().write64(reg::CONFIG, 0x13); + md.md_bus().write64(0x20038, 0x80); + let before_zero = generation(); + md.md_bus().write64(reg::SELFTEST_COMMAND, 1); + assert!(generation() > before_zero, "zero self-test invalidates logical code pages"); + let before_disable = generation(); + md.power_on(); + assert!(generation() > before_disable); + assert_eq!(dma.gen_ptr(0), pointer, "a disabled bank keeps its logical counter alive"); + } + + /// The Tezro's banks, in MiB from locational bank 0. + const TEZRO_BANKS: [u32; 6] = [1024, 1024, 512, 512, 512, 512]; + /// Banks 0 and 1 at size code 5 (1 GiB), the other four at code 4 (512 MiB). + const TEZRO_CONFIG: u64 = 0x53 | (0x43 << 8) | (0x43 << 16); + fn tezro() -> Arc { + let md = TestMd::new(&TEZRO_BANKS).unwrap(); + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG), BUS_OK); + md + } + + /// Each Tezro bank answers in its own 1 GiB slot at its own size: a + /// 1 GiB bank fills its slot, a 512 MiB bank stops halfway, and the + /// slots above bank 5 answer nothing. Banks 4 and 5 sit above 4 GiB, + /// where the CPU's direct window does not reach, and still answer + /// through the bus. Follows the qtest memory-banks and + /// memory-size-boundaries cases in tests/qtest/sn1-bedrock-md-test.c. + #[test] + fn every_tezro_bank_answers_at_its_own_size_including_above_4_gib() { + use crate::ppmem::MappedMemory; + let md = tezro(); + let bus = md.physical(); + let sizes = [1024u64, 1024, 512, 512, 512, 512]; + for (slot, mib) in sizes.iter().enumerate() { + let base = slot as u64 * BANK_SPAN; + let size = mib << 20; + assert_eq!(bus.write64(base, 0x1111 * (slot as u64 + 1)), BUS_OK, "slot {slot}"); + assert_eq!(bus.write64(base + size - 8, 0xe0 + slot as u64), BUS_OK, "slot {slot} top"); + if size < BANK_SPAN { + assert_eq!(bus.read64(base + size).status, BUS_ERR, "slot {slot} past its bank"); + } + } + for slot in 0..6u64 { + assert_eq!(bus.read64(slot * BANK_SPAN).data, 0x1111 * (slot + 1), "slot {slot}"); + let top = (sizes[slot as usize] << 20) - 8; + assert_eq!(bus.read64(slot * BANK_SPAN + top).data, 0xe0 + slot, "slot {slot} top"); + } + for slot in [6, 7] { + assert_eq!(bus.read64(slot * BANK_SPAN).status, BUS_ERR, "slot {slot} holds no bank"); + } + assert_eq!(bus.banks()[4].read64(0).data, 0x1111 * 5, "bank 4 holds what slot 4 wrote"); + assert_eq!(bus.banks()[5].read64(0).data, 0x1111 * 6); + + // The window covers the first 4 GiB: 1 GiB, 1 GiB, 512 MiB, 512 MiB. + let space = bus.ppmem_space().expect("a window"); + assert_eq!(space.mapped_bitmap(), 0xffff | (0xffff << 16) | (0xff << 32) | (0xff << 48)); + assert!(!crate::ppmem::bitmap_maps(space.mapped_bitmap(), 4 * BANK_SPAN)); + assert!(!crate::ppmem::bitmap_maps(space.mapped_bitmap(), 5 * BANK_SPAN)); + assert_eq!(md.dma_view().read64(5 * BANK_SPAN).data, 0x1111 * 6, "DMA reaches it too"); + } + + /// A processor's SC to a bank above 4 GiB, outside its direct window, + /// swaps through the DMA view in one step and leaves a failed compare + /// alone; a successful one advances the page's logical counter. + #[test] + fn a_compare_and_swap_above_4_gib_is_one_step() { + let md = tezro(); + let view = md.dma_view(); + let at = 5 * BANK_SPAN + 0x40; + assert_eq!(view.write64(at, 7), BUS_OK); + #[cfg(feature = "jitv2")] + let before = unsafe { (*view.gen_ptr(at)).load(Ordering::Relaxed) }; + assert_eq!(view.cas64(at, 6, 9), Some(false)); + assert_eq!(view.read64(at).data, 7); + assert_eq!(view.cas64(at, 7, 9), Some(true)); + assert_eq!(view.read64(at).data, 9); + assert_eq!(view.cas32(at + 4, 9, 3), Some(true)); + assert_eq!(view.read64(at).data, 3); + #[cfg(feature = "jitv2")] + assert!(unsafe { (*view.gen_ptr(at)).load(Ordering::Relaxed) } > before); + assert_eq!(view.cas64(7 * BANK_SPAN, 0, 1), None, "no bank there"); + } + + /// Selector 1 swaps the two banks of each pair, so the 512 MiB banks 4 and 5 + /// trade places above 4 GiB. Follows memory-selector-high-bank. + #[test] + fn the_bank_select_swaps_banks_above_4_gib() { + let md = tezro(); + let bus = md.physical(); + assert_eq!(bus.write64(4 * BANK_SPAN + 0x28, 0xfedc_ba98_7654_3210), BUS_OK); + assert_eq!(bus.read64(5 * BANK_SPAN + 0x28).data, 0); + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG | (1 << 32)), BUS_OK); + assert_eq!(bus.read64(5 * BANK_SPAN + 0x28).data, 0xfedc_ba98_7654_3210); + assert_eq!(bus.read64(4 * BANK_SPAN + 0x28).data, 0); + // Banks 0 and 1 swap too: the 1 GiB banks have different sizes only + // if the board says so, so give bank 1 a distinct value. + assert_eq!(bus.write64(0, 0xaa), BUS_OK); + assert_eq!(bus.write64(BANK_SPAN, 0xbb), BUS_OK); + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG), BUS_OK); + assert_eq!(bus.read64(0).data, 0xbb); + assert_eq!(bus.read64(BANK_SPAN).data, 0xaa); + } + + /// A 1 GiB population is wholly inside ppmem's direct window: all 16 of its + /// 64 MiB regions are mapped, with none left to the decode table. The 4 GiB + /// default maps only what is below 4 GiB, and its banks 4 and 5 answer + /// through the table. + #[test] + fn a_1_gib_population_is_directly_mapped_and_the_default_4_gib_one_is_not_wholly() { + use crate::ppmem::MappedMemory; + let md = TestMd::new(&[512, 512]).unwrap(); + md.md_bus().write64(reg::CONFIG, 0x43 | (0x43 << 8)); + let bus = md.physical(); + let space = bus.ppmem_space().expect("a window"); + assert_eq!(space.mapped_bitmap(), 0xff | (0xff << 16)); + assert_eq!(bus.write64(BANK_SPAN + (512 << 20) - 8, 7), BUS_OK); + assert_eq!(bus.read64(BANK_SPAN + (512 << 20) - 8).data, 7); + assert_eq!(bus.read64(512 << 20).status, BUS_ERR, "bank 0 ends at 512 MiB"); + + let md = TestMd::new(&TEZRO_BANKS).unwrap(); + md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG); + let space = md.physical().ppmem_space().expect("a window"); + assert!(!crate::ppmem::bitmap_maps(space.mapped_bitmap(), 4 * BANK_SPAN)); + assert_eq!(md.physical().write64(4 * BANK_SPAN, 9), BUS_OK, "still reachable"); + } + + /// Directory and ECC state is keyed per bank, so banks of different sizes + /// never share an entry. Follows directory-bank-translation. + #[test] + fn directory_entries_of_different_sized_banks_stay_apart() { + let md = tezro(); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 2); + let dir = md.directory_bus(); + // Software bank n's carriers start at n * 128 MiB. + let at = |bank: u64, offset: u64| bank * (BANK_SPAN / 8) + offset; + for bank in 0..6 { + assert_eq!(dir.write64(at(bank, 0x20), 0x1000 + bank), BUS_OK, "bank {bank}"); + } + for bank in 0..6 { + assert_eq!(dir.read64(at(bank, 0x20)).data, 0x1000 + bank, "bank {bank}"); + } + // A 512 MiB bank's directory ends at its size code's reach. + assert_eq!(dir.read64(at(2, 0x20) + (BANK_SPAN / 16)).status, BUS_ERR); + assert_eq!(dir.read64(at(6, 0x20)).status, BUS_ERR, "no bank 6"); + } + + /// A fitting that covers the bank under test zero-fills it, whichever slot + /// the bank is in; a bank that is not fitted fails. Follows the + /// selftest-bank-remap and origin300-selftest-fill cases. + #[test] + fn selftest_fills_a_tezro_bank_above_4_gib_and_fails_a_missing_bank() { + let md = TestMd::new(&TEZRO_BANKS).unwrap(); + // Every pair at size code 1 (64 MiB): small enough to fill in a test. + let regs = md.md_bus(); + regs.write64(reg::CONFIG, 0x13 | (0x13 << 8) | (0x13 << 16) | (0x13 << 24)); + let bus = md.physical(); + bus.write64(4 * BANK_SPAN + 0x40, 0x5555); + regs.write64(0x20038, 0x80); + regs.write64(0x20030, (4 << 4) | 1); + md.advance_ns(1000); + assert_eq!(regs.read64(0x20048).data, 3, "bank 4 is fitted"); + assert_eq!(bus.read64(4 * BANK_SPAN + 0x40).data, 0, "and zeroed"); + regs.write64(0x20030, (6 << 4) | 1); + md.advance_ns(1000); + assert_eq!(regs.read64(0x20048).data, 2, "bank 6 is not fitted"); + } + + /// The board names what it fits; nothing else is accepted. A bank is a + /// whole number of direct-window regions and at most the largest size + /// code names. + #[test] + fn a_board_may_only_fit_banks_the_controller_can_decode() { + assert!(TestMd::new(&FUEL_BANKS).is_ok()); + assert!(TestMd::new(&TEZRO_BANKS).is_ok()); + for banks in [&[][..], &[0, 256], &[2048], &[100], &[64; 9]] { + assert!(TestMd::new(banks).is_err(), "{banks:?}"); + } + } + + /// Where RAM can be, for a board's cached view: the swizzle permutes slots + /// within groups of four, so a slot reaches as far as the largest bank of + /// its group. + #[test] + fn the_ram_extent_reaches_as_far_as_the_largest_bank_of_a_slot_group() { + let fuel = RamExtent::new(&FUEL_BANKS); + for slot in 0..4 { + assert!(fuel.contains(slot * BANK_SPAN + FITTED_BANK - 8), "slot {slot}"); + assert!(!fuel.contains(slot * BANK_SPAN + FITTED_BANK), "slot {slot}"); + } + assert!(!fuel.contains(4 * BANK_SPAN)); + let tezro = RamExtent::new(&TEZRO_BANKS); + assert!(tezro.contains(3 * BANK_SPAN + BANK_SPAN - 8), "slot 3 may hold a 1 GiB bank"); + assert!(tezro.contains(5 * BANK_SPAN + (512 << 20) - 8)); + assert!(!tezro.contains(5 * BANK_SPAN + (512 << 20))); + assert!(!tezro.contains(8 * BANK_SPAN)); + assert_eq!(tezro, TestMd::new(&TEZRO_BANKS).unwrap().ram_extent()); + } + + /// Code pages above 4 GiB have stable logical generation counters like + /// the rest, and a write bumps them. + #[cfg(feature = "jitv2")] + #[test] + fn code_pages_above_4_gib_keep_a_logical_generation() { + let md = tezro(); + let bus = md.physical(); + let above = 4 * BANK_SPAN + 0x2000; + let pointer = md.dma_view().gen_ptr(above); + assert!(!pointer.is_null()); + assert_ne!(pointer, md.dma_view().gen_ptr(5 * BANK_SPAN + 0x2000)); + assert!(md.dma_view().gen_ptr(8 * BANK_SPAN).is_null(), "past the node's 8 GiB"); + assert!(md.dma_view().gen_ptr(4 * BANK_SPAN + (512 << 20)).is_null(), "past the largest bank there"); + let before = unsafe { (*pointer).load(Ordering::Relaxed) }; + assert_eq!(md.dma_view().write64(above, 1), BUS_OK); + assert!(unsafe { (*pointer).load(Ordering::Relaxed) } > before); + md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG | (1 << 32)); + assert_eq!(md.dma_view().gen_ptr(above), pointer, "a bank swap keeps the counter"); + let _ = bus; + } + + /// The ECC carrier byte for the doubleword at `address`, as the guest + /// addresses it. + fn ecc_carrier(address: u64) -> u64 { + address / BANK_SPAN * (BANK_SPAN / 8) + ((address % BANK_SPAN) / 8 ^ 7) + } + + /// The cost counters now: host mappings, decode-table rewrites and + /// generation counters. + fn counted(md: &Arc) -> [u64; 3] { + md.memory.cost.snapshot() + } + /// The counters one bumped 4 KiB page, and one whole 64 MiB region, cost: + /// the logical counter and the bank's, which only a jitv2 build keeps. + const PAGE_BUMPS: u64 = if cfg!(feature = "jitv2") { 2 } else { 0 }; + const REGION_BUMPS: u64 = PAGE_BUMPS * (64 << 20) / 4096; + fn mark(md: &Arc, address: u64) { + assert_eq!(md.ecc_bus().write8(ecc_carrier(address), 0x13), BUS_OK); + } + /// A doubleword of zero has a check byte of zero, so writing that lifts + /// a mark, and writing 0x13 puts one there. + fn lift(md: &Arc, address: u64) { + assert_eq!(md.ecc_bus().write8(ecc_carrier(address), 0), BUS_OK); + } + fn spent(md: &Arc, work: impl FnOnce()) -> [u64; 3] { + let before = counted(md); + work(); + let after = counted(md); + [after[0] - before[0], after[1] - before[1], after[2] - before[2]] + } + + /// What marking and lifting checked pages costs in host mappings, + /// decode-table rewrites and generation counters. Taking a region off the + /// direct path and putting it back maps nothing: the pages stay mapped + /// and only the bitmap changes. The PROM's memory test + /// marks and lifts the same few hundred doublewords thousands of times + /// (the QEMU model counted 6630 of each for 310 doublewords, and + /// rebuilding the map each time took the test from ten seconds to + /// minutes), so what one costs is what the test costs. Counts, not + /// seconds, so the test can pin them. Before this was incremental each of + /// these was a full rebuild: 15 mappings, 3 or 2 table rewrites and + /// 393,216 counters on the Fuel; 95, 7 or 6 and 2,621,440 on the Tezro. + /// + /// Follows the qtest ecc-lifted-mark-keeps-overlay case in + /// tests/qtest/sn1-bedrock-md-test.c (QEMU commit 4f0f6072dc0b), which + /// pins the same thing in the flat view: a lifted mark's overlay stays, + /// a second mark there is not a map change, and the idle ones go + /// together. + #[test] + fn a_checked_page_costs_a_page_not_a_rebuild() { + let (fuel, tezro) = (configured(), tezro()); + for (md, base, windowed) in [(&fuel, 0, true), (&tezro, 0, true), (&tezro, 4 * BANK_SPAN, false)] { + let (first, second) = (base + 0x5000, base + 0x6000); + // The first page in a region takes the region off the direct + // window and drops code compiled from any of it; the next page + // in that region is one more table entry. Above the window there + // is no region to take. + let cost = spent(md, || mark(md, first)); + assert_eq!(cost, if windowed { [0, 1, REGION_BUMPS] } else { [0, 1, PAGE_BUMPS] }, "first page at {base:#x}"); + assert_eq!(spent(md, || mark(md, second)), [0, 1, PAGE_BUMPS], "second page at {base:#x}"); + // Lifting costs nothing: both pages stay on the MD endpoint. + assert_eq!(spent(md, || { lift(md, first); lift(md, second) }), [0, 0, 0], "lifts at {base:#x}"); + // Marking a lifted page again is not a map change either. + assert_eq!(spent(md, || mark(md, first)), [0, 0, 0], "re-mark at {base:#x}"); + // The retirement returns what is lifted, in one batch, and + // leaves the marked page where it is. + assert_eq!(spent(md, || md.retire_lifted_marks()), [0, 1, PAGE_BUMPS], "retire at {base:#x}"); + lift(md, first); + // The region comes back to the direct path with no host mapping: + // withdrawing it left the pages mapped. + assert_eq!(spent(md, || md.retire_lifted_marks()), [0, 1, PAGE_BUMPS], "last retire at {base:#x}"); + } + } + + fn counter(md: &Arc, address: u64) -> u64 { + #[cfg(feature = "jitv2")] + return unsafe { (*md.memory.gen_ptr(address)).load(Ordering::Relaxed) }; + #[cfg(not(feature = "jitv2"))] + { let _ = (md, address); 0 } + } + + /// With the controller's clock under the test's hand. + fn clocked() -> (Arc, Arc) { + let now = Arc::new(AtomicU64::new(1)); + let hand = now.clone(); + let md = TestMd::with_clock(&FUEL_BANKS, Arc::new(move || hand.load(Ordering::SeqCst))).unwrap(); + assert_eq!(md.md_bus().write64(reg::CONFIG, 0x33), BUS_OK); + (md, now) + } + + /// Lifting a mark keeps its page on the MD endpoint, which answers as + /// the bank would: the lifted doubleword reads and writes plainly, a + /// second mark there is checked again, and the page goes back to the bank + /// together with the other idle ones, a second after the first was seen + /// idle, or at once on reset. Follows the qtest + /// ecc-lifted-mark-keeps-overlay case in + /// tests/qtest/sn1-bedrock-md-test.c (QEMU commit 4f0f6072dc0b). + #[test] + fn a_lifted_mark_keeps_its_page_on_the_md_endpoint_until_it_idles() { + use crate::ppmem::MappedMemory; + let (md, now) = clocked(); + let (phys, bus) = (md.physical(), md.ecc_bus()); + let space = phys.ppmem_space().expect("a window"); + let (address, data) = (0x5000, 0x0123_4567_89ab_cdef); + let on_bank = |md: &Arc, address| { + crate::ppmem::devmap::entry_eq(md.physical().entry_at(address), md.physical().entry_at(0x20_0000)) + }; + assert_eq!(phys.write64(address, data), BUS_OK); + assert!(on_bank(&md, address)); + assert_eq!(bus.write8(ecc_carrier(address), ecc_compute(data) ^ ECC_COLUMNS[5]), BUS_OK); + assert!(!on_bank(&md, address) && space.mapped_bitmap() & 1 == 0); + + // A write lifts the mark; the page stays on the endpoint and reads as memory. + assert_eq!(phys.write64(address, data), BUS_OK); + assert!(!on_bank(&md, address) && space.mapped_bitmap() & 1 == 0); + assert_eq!(phys.read64(address).data, data); + assert_eq!(phys.read32(address + 4).data, data as u32); + assert_eq!(phys.write32(address, 0x7654_3210), BUS_OK); + assert_eq!(phys.read64(address).data, 0x7654_3210_89ab_cdef); + assert_eq!(phys.write64(address, data), BUS_OK); + assert_eq!(bus.read8(ecc_carrier(address)).data, ecc_compute(data)); + assert_eq!(md.md_bus().read64(reg::MEMORY_ERROR).data, 0); + + // Marked again, the same doubleword is checked again. + assert_eq!(bus.write8(ecc_carrier(address), ecc_compute(data) ^ ECC_COLUMNS[6]), BUS_OK); + assert_eq!(phys.read64(address).data, data ^ (1 << 6)); + assert_ne!(md.md_bus().read64(reg::MEMORY_ERROR).data & (1 << 52), 0); + assert!(md.state.lock().ecc.is_empty(), "the correcting read lifted it"); + + // Idling a second retires it, on the next publication after that. + let other = |n: u64| 0x1_0000 + n * 0x1000; + assert_eq!(phys.write64(other(0), 0), BUS_OK); + mark(&md, other(0)); + now.store(1 + RETIRED_IDLE_NS - 1, Ordering::SeqCst); + mark(&md, other(1)); + assert!(!on_bank(&md, address), "not yet a second"); + now.store(1 + RETIRED_IDLE_NS, Ordering::SeqCst); + mark(&md, other(2)); + assert!(on_bank(&md, address), "a second"); + assert_eq!(phys.read64(address).data, data ^ (1 << 6)); + assert!(!on_bank(&md, other(0)) && !on_bank(&md, other(2)), "the marked pages stay checked"); + + // Reset takes a kept page back without waiting. + lift(&md, other(0)); + assert_eq!(phys.write64(other(0), 0), BUS_OK); + assert!(!on_bank(&md, other(0))); + md.soft_reset(); + assert!(on_bank(&md, other(0)) && on_bank(&md, other(2))); + assert_eq!(space.mapped_bitmap() & 1, 1); + } + + /// A page kept after its mark is lifted still checks and rewrites a + /// marked directory line in it, as the line's own placement would. + /// Follows the qtest ecc-lifted-mark-over-marked-line case in + /// tests/qtest/sn1-bedrock-md-test.c (QEMU commit 4f0f6072dc0b). + #[test] + fn a_kept_page_checks_a_directory_line_marked_after_the_lift() { + let md = configured(); + let phys = md.physical(); + mark(&md, 0x48); + lift(&md, 0x48); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 2); + let cost = spent(&md, || { md.directory_bus().write64(0, 0x001e0001); }); + assert_eq!(cost, [0, 0, 0], "the page is already on the endpoint"); + md.md_bus().write64(reg::DIRECTORY_CONFIG, 0); + assert_eq!(phys.read64(0x48).status, BUS_OK); + assert_ne!(md.md_bus().read64(reg::DIRECTORY_ERROR).data & (1 << 61), 0, "the read checked the line"); + // Marked again, a write through the kept page rewrites the entry. + md.md_bus().write64(reg::DIRECTORY_CONFIG, 2); + md.directory_bus().write64(0, 0x001e0001); + assert!(!md.state.lock().marked_directory.is_empty()); + assert_eq!(phys.write64(0x48, 0x22), BUS_OK); + assert!(md.state.lock().marked_directory.is_empty()); + assert_eq!(phys.read64(0x48).data, 0x22); + } + + /// The PROM's memory test marks and lifts the same few hundred + /// doublewords over and over; each page is placed once however often. + /// The QEMU model counted 6630 placements for 310 doublewords before it + /// kept lifted overlays. + #[test] + fn marking_the_same_pages_over_and_over_places_each_once() { + let md = configured(); + let pages: Vec = (0..300).map(|i| i * 0x9_0000 + 0x1000).collect(); + let regions: HashSet = pages.iter().map(|page| page >> 26).collect(); + let cost = spent(&md, || { + for _ in 0..20 { + pages.iter().for_each(|&page| mark(&md, page)); + pages.iter().for_each(|&page| lift(&md, page)); + } + }); + let regions = regions.len() as u64; + assert_eq!(cost, [0, 300, regions * REGION_BUMPS + (300 - regions) * PAGE_BUMPS]); + } + + /// Lifted pages pile up only so far: the first mark after + /// `RETIRED_LIMIT` have been lifted returns them all in one batch. + #[test] + fn many_lifted_pages_are_retired_in_one_batch() { + let md = configured(); + let pages: Vec = (0..RETIRED_LIMIT as u64).map(|i| 0x1000 * i).collect(); + pages.iter().for_each(|&page| mark(&md, page)); + pages.iter().for_each(|&page| lift(&md, page)); + // One more page, in another region, makes the 1024 lifted pages due. + let cost = spent(&md, || mark(&md, 0x400_0000 + 0x1000)); + let retired = RETIRED_LIMIT as u64; + assert_eq!(cost, [0, 1 + retired, REGION_BUMPS + PAGE_BUMPS * retired]); + assert!(crate::ppmem::devmap::entry_eq(md.physical().entry_at(0x5000), md.physical().entry_at(0x20_0000))); + } + + /// A zero self-test of a bank clears its marks, and so returns the pages + /// they held. + #[test] + fn a_zero_self_test_returns_the_banks_lifted_pages() { + use crate::ppmem::MappedMemory; + let md = configured(); + let space = md.physical().ppmem_space().expect("a window"); + mark(&md, 0x5000); + assert_eq!(space.mapped_bitmap() & 1, 0); + md.md_bus().write64(0x20038, 0x80); + md.md_bus().write64(reg::SELFTEST_COMMAND, 1); + assert_eq!(space.mapped_bitmap() & 1, 1); + assert!(crate::ppmem::devmap::entry_eq(md.physical().entry_at(0x5000), md.physical().entry_at(0x20_0000))); + } + + /// A page that is marked, or its region's direct path, drops the code + /// compiled from it; a page elsewhere keeps its counter, and lifting + /// changes nothing until the page is returned. + #[test] + fn marking_a_page_advances_the_counters_of_what_left_the_direct_path() { + let md = configured(); + let (page, neighbour, elsewhere) = (0x5000, 0x9000, 0x400_0000 + 0x5000); + let seen = |md: &Arc| [page, neighbour, elsewhere].map(|a| counter(md, a)); + let start = seen(&md); + #[cfg(feature = "jitv2")] + let bank = |offset: u64| unsafe { (*md.physical().banks()[0].gen_ptr(offset)).load(Ordering::Relaxed) }; + #[cfg(feature = "jitv2")] + let banks = [page, neighbour, elsewhere].map(bank); + mark(&md, page); + let marked = seen(&md); + let step = |a: [u64; 3], b: [u64; 3]| [b[0] - a[0], b[1] - a[1], b[2] - a[2]]; + let jit = u64::from(cfg!(feature = "jitv2")); + assert_eq!(step(start, marked), [jit, jit, 0], "the page and its region"); + #[cfg(feature = "jitv2")] + { + // The bank's own counters follow, as a full remap would bump them. + let now = [page, neighbour, elsewhere].map(bank); + assert!(now[0] > banks[0] && now[1] > banks[1] && now[2] == banks[2]); + } + mark(&md, neighbour); + assert_eq!(step(marked, seen(&md)), [0, jit, 0], "only the second page"); + lift(&md, page); + lift(&md, neighbour); + assert_eq!(step(marked, seen(&md)), [0, jit, 0], "lifting changes nothing"); + md.retire_lifted_marks(); + assert_eq!(step(marked, seen(&md)), [jit, 2 * jit, 0], "returning a page advances it"); + } + + /// Asserts a code page's counter moved, in builds that keep counters. + fn assert_advanced(what: &str, before: u64, after: u64) { + if cfg!(feature = "jitv2") { + assert!(after > before, "{what}: {before} -> {after}"); + } + } + + /// Physical's bulk writers, on a bank directly, advance only that bank's + /// counters, which nothing that follows code pages reads on this board. + /// The controller's own routes advance the logical ones as well. + #[test] + fn bulk_memory_writes_through_the_controller_advance_the_logical_counters() { + let md = TestMd::new(&[64]).unwrap(); + assert_eq!(md.md_bus().write64(reg::CONFIG, 0x11), BUS_OK); + let phys = md.physical(); + let at = 0x5000; + let logical = counter(&md, at); + phys.reset_memory(); + phys.restore_bank_inmem(0, &[1, 2]); + assert_eq!(counter(&md, at), logical, "Physical alone leaves the logical counter"); + md.reset_memory(); + let reset = counter(&md, at); + assert_advanced("reset", logical, reset); + md.restore_bank(0, &[0x0102_0304, 0x0506_0708]); + let restored = counter(&md, at); + assert_advanced("restore", reset, restored); + assert_eq!(phys.read64(0).data, 0x0102_0304_0506_0708); + let path = std::env::temp_dir().join(format!("iris-bedrock-load-{}", std::process::id())); + std::fs::write(&path, [0, 0, 0, 9, 0, 0, 0, 8]).unwrap(); + let loaded = md.load_bank(0, &path); + std::fs::remove_file(&path).unwrap(); + loaded.unwrap(); + assert_advanced("load", restored, counter(&md, at)); + assert_eq!(phys.read64(0).data, 0x0000_0009_0000_0008); + } + + /// Guest density settings remap fitted RAM, and reset restores the default map. + #[test] + fn bank_configuration_remaps_memory_and_reset_restores_defaults() { + let md = configured(); + let bus = md.physical(); + assert_eq!(bus.write64(0, 0x1122334455667788), BUS_OK); + assert_eq!(bus.write64(BANK_SPAN, 0xabcdef), BUS_OK); + assert_eq!(bus.read8(0).data, 0x11); + assert_eq!(bus.read32(4).data, 0x55667788); + assert_eq!(bus.read64(FITTED_BANK).status, BUS_ERR); + assert_eq!(bus.read64(2 * BANK_SPAN).status, BUS_ERR); + assert_eq!(md.md_bus().write64(8, 0x33 | 1 << 32), BUS_OK); + assert_eq!(bus.read64(0).data, 0xabcdef); + assert_eq!(bus.read64(BANK_SPAN).data, 0x1122334455667788); + assert!(bus.mem_ptr(0).is_some()); + md.soft_reset(); + assert_eq!(bus.read64(0).data, 0xabcdef); + md.power_on(); + assert_eq!(bus.read64(0).status, BUS_ERR); + bus.map_unmapped(0x1fc00000, 0x1fc01000); + for (code, size) in [(1, 64 << 20), (2, 128 << 20), (3, FITTED_BANK), + (4, FITTED_BANK), (5, FITTED_BANK)] { + md.md_bus().write64(reg::CONFIG, 1 | (code << 4)); + assert_eq!(bus.read64(size - 8).status, BUS_OK); + assert_eq!(bus.read64(size).status, BUS_ERR); + assert_eq!(bus.read64(0x1fc00000).status, BUS_OK, "a non-RAM mapping survives"); + } + md.md_bus().write64(reg::CONFIG, 0x11); + assert_eq!(bus.read64(64 << 20).status, BUS_ERR, "the old prefix is removed on shrink"); + md.md_bus().write64(reg::CONFIG, 0x61); + assert_eq!(bus.read64(0).status, BUS_ERR, "unsupported density does not map RAM"); + #[cfg(feature = "jitv2")] + { + let md = configured(); + let ram = md.physical(); + let first = ram.gen_ptr(0); + let second = ram.gen_ptr(BANK_SPAN); + assert!(!first.is_null() && !second.is_null()); + assert_ne!(first, second); + let before = unsafe { (*first).load(Ordering::Relaxed) }; + ram.write64(0, 0x55); + assert!(unsafe { (*first).load(Ordering::Relaxed) } > before); + md.md_bus().write64(8, 0x33 | (1 << 32)); + // The window address stays fixed while its backing bank changes. + assert_eq!(ram.gen_ptr(0), first); + assert_eq!(ram.gen_ptr(BANK_SPAN), second); + let bank0 = ram.banks()[0].gen_ptr(0); + let bank1 = ram.banks()[1].gen_ptr(0); + let before0 = unsafe { (*bank0).load(Ordering::Relaxed) }; + let before1 = unsafe { (*bank1).load(Ordering::Relaxed) }; + assert_eq!(ram.write64(0, 0xaa), BUS_OK); + assert_eq!(unsafe { (*bank0).load(Ordering::Relaxed) }, before0); + assert_eq!(unsafe { (*bank1).load(Ordering::Relaxed) }, before1 + 1); + assert_eq!(unsafe { (*first).load(Ordering::Relaxed) }, before1 + 1); + let view = md.uncached_view(); + let logical = view.gen_ptr(0); + assert_eq!(logical, md.memory.gen_ptr(0)); + let logical_before = unsafe { (*logical).load(Ordering::Relaxed) }; + drop(md); + assert_eq!(unsafe { (*logical).load(Ordering::Relaxed) }, logical_before, + "an external endpoint keeps its logical generation pointers alive"); + // The endpoint keeps the backing memory alive after MD disappears. + assert!(unsafe { (*bank1).load(Ordering::Relaxed) } > 0); + assert_eq!(view.read64(0).status, BUS_ERR); + } + + } + /// The guest reads ECC configuration and memory or directory errors through status registers. + #[test] + fn guest_registers_expose_ecc_configuration_and_error_latches() { + { + let md = configured(); + let r = md.md_bus(); + // These carrier offsets are idle; they do not select storage. + for at in [0x20, 0x20010, 0x40020, 0x60020] { + let before = md.unsupported(); + assert_eq!(r.write64(at, u64::MAX), BUS_OK, "{at:#x}"); + assert_eq!(r.read64(at).data, 0, "{at:#x}"); + assert_eq!(md.unsupported(), before + 1); + } + for (at, value) in [(0x60, 0x001818180c0c0c00), (0x40050, 0x04040400)] { + let before = md.unsupported(); + assert_eq!(r.write64(at, 0), BUS_OK); + assert_eq!(r.read64(at).data, value); + assert_eq!(md.unsupported(), before + 1); + } + // Error-clear writes are ignored too: only a read clears the record. + md.ecc_bus().write8(7, 3); + md.uncached_view().read64(0); + r.write64(0x40000, 2); + md.directory_bus().write64(0, 3 << 16); + r.write64(0x40000, 0); + md.physical().read64(0); + let memory_error = r.read64(0x60010).data; + let directory_error = r.read64(0x40010).data; + assert_ne!(memory_error & (1 << 60), 0); + assert_ne!(directory_error & (1 << 63), 0); + for at in [0x40010, 0x40018, 0x60010, 0x60018] { + let before = md.unsupported(); + assert_eq!(r.write64(at, u64::MAX), BUS_OK); + assert_eq!(r.read64(0x60010).data, memory_error); + assert_eq!(r.read64(0x40010).data, directory_error); + assert_eq!(md.unsupported(), before + 1); + } + // An unsupported self-test command leaves an in-flight command alone. + r.write64(0x20030, 1); + let before = md.unsupported(); + assert_eq!(r.write64(0x20030, 3), BUS_OK); + assert_eq!(md.unsupported(), before + 1); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 3); + assert_eq!(r.write64(0x70000, 1), BUS_ERR); + assert_eq!(r.read64(0x70000).status, BUS_ERR); + assert_eq!(r.write32(0x60, 0), BUS_ERR); + assert_eq!(r.write64(0x61, 0), BUS_ERR); + } + { + let md = configured(); + let regs = md.md_bus(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.write64(0x60000, u64::MAX), BUS_OK); + assert_eq!(regs.read64(0x60000).data, 1); + assert_eq!(regs.write64(0x60000, !1), BUS_OK); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.write8(0x60000, 1), BUS_ERR); + assert_eq!(regs.write64(0x60001, 1), BUS_ERR); + regs.write64(0x60000, 1); + md.soft_reset(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.read64(8).data, 0x33); + regs.write64(0x60000, 1); + md.power_on(); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(regs.read64(8).data, 0); + } + { + for (code, kind, status, valid) in [ + (1, MemoryError::Correctable, BUS_OK, 1 << 52), + (3, MemoryError::Uncorrectable, BUS_ERR, 1 << 60), + ] { + let md = configured(); + let regs = md.md_bus(); + let ecc = md.ecc_bus(); + let ram = md.uncached_view(); + assert_eq!(ram.read64(0).status, BUS_OK); + ecc.write8(7, code); + regs.write64(0x60000, u64::MAX); + assert_eq!(ram.read64(0).status, BUS_OK); + assert_eq!(ecc.read8(7).data, code); + assert_eq!(regs.read64(0x60010).data, 0); + assert_eq!(md.reported(), [(false, false); 2]); + regs.write64(0x60000, !1); + assert_eq!(regs.read64(0x60000).data, 0); + assert_eq!(ram.read64(0).status, status); + assert_ne!(regs.read64(0x60010).data & valid, 0); + let correctable = kind == MemoryError::Correctable; + assert_eq!(md.reported(), [(correctable, !correctable); 2], "both interfaces"); + if correctable { + assert_eq!(ecc.read8(7).data, 0); + assert_eq!(ram.read64(0).status, BUS_OK); + } + } + } + { + let md = configured(); + let regs = md.md_bus(); + let ecc = md.ecc_bus(); + let ram = md.uncached_view(); + ecc.write8(7, 3); + regs.write64(0x60000, 1); + assert_eq!(ram.write8(7, 0x5a), BUS_OK); + assert_eq!(ecc.read8(7).data, 0x82); + regs.write64(0x60000, 0); + assert_eq!(ram.read64(0).status, BUS_OK); + assert_eq!(ram.read64(0).data, 0x5a); + assert_eq!(regs.read64(0x60010).data, 0); + } + } + /// Self-test completion is visible only after its configured clock interval. + #[test] + fn selftest_waits_for_fitted_clock_cycles_and_zero_fills_ram() { + { + let now = Arc::new(AtomicU64::new(100)); + let time = now.clone(); + let md = TestMd::with_clock(&FUEL_BANKS, Arc::new(move || time.load(Ordering::Relaxed))).unwrap(); + let regs = md.md_bus(); + regs.write64(8, 0x33); + md.start_clock(); + regs.write64(0x20030, 1); + now.store(1099, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(1100, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + now.store(5000, Ordering::Relaxed); + regs.write64(0x20030, 1); + now.store(5500, Ordering::Relaxed); + md.stop_clock(); + now.store(15000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + md.start_clock(); + now.store(15499, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(15500, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + regs.write64(0x20030, 1); + now.store(15400, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(16499, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(16500, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + regs.write64(0x20030, 1); + regs.write64(0x20030, 2); + now.store(20000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + md.power_on(); + regs.write64(8, 0x33); + md.start_clock(); + regs.write64(0x20030, 1); + now.store(20999, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 0); + now.store(21000, Ordering::Relaxed); + assert_eq!(regs.read64(0x20048).data, 3); + } + { + let md = configured(); + let r = md.md_bus(); + r.write64(0x20030, 0x21); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 2); + assert_eq!(r.read64(0x20040).data, 1 << 48); + r.write64(0x20030, 1); + r.write64(0x20030, 2); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 0); + assert_eq!(r.write64(0x20030, 3), BUS_OK); + } + { + let md = configured(); + let r = md.md_bus(); + r.write64(8, 0x13); + md.physical().write64(0, 0x1234); + md.ecc_bus().write8(7, 1); + r.write64(0x20038, 0x80); + r.write64(0x20030, 1); + md.advance_ns(1000); + assert_eq!(r.read64(0x20048).data, 3); + assert_eq!(md.physical().read64(0).data, 0); + assert_eq!(md.ecc_bus().read8(7).data, 0); + assert_eq!(r.read64(0x60010).data, 0); + } + } + /// Fetch-and-op registers preserve the big-endian lanes used by the processor. + #[test] + fn fetchop_preserves_big_endian_word_lanes() { + let md = configured(); + let ram = md.physical(); + let f = md.fetchop_bus(); + ram.write64(0, 0xffffffff00000009); + assert_eq!(f.read32(8).data, u32::MAX); + assert_eq!(ram.read64(0).data, 9); + assert_eq!(f.read32(12).data, 9); + assert_eq!(ram.read64(0).data, 10); + assert_eq!(f.write32(28, 3), BUS_OK); + assert_eq!(ram.read64(0).data, 2); + assert_eq!(f.write64(32, 8), BUS_OK); + assert_eq!(ram.read64(0).data, 10); + assert_eq!(f.read64(24).data, 10); + assert_eq!(ram.read64(0).data, 0); + assert_eq!(f.read64(32).status, BUS_ERR); + assert_eq!(f.write64(40, 1), BUS_ERR); + assert_eq!(f.read16(0).status, BUS_ERR); + } + /// The guest reads ECC bytes and scrubbing repairs correctable memory. + #[test] + fn ecc_carriers_expose_check_bytes_and_scrub_corrections() { + { + let md = configured(); + let ecc = md.ecc_bus(); + for carrier in 0..8 { + let source = 0x11223344556677a5u64; + assert_eq!(ecc.write8(carrier, source as u8), BUS_OK); + assert_eq!(ecc.read8(carrier).data, source as u8, "carrier {carrier}"); + } + assert_eq!(ecc.write16(4, 0x1234), BUS_OK); + assert_eq!(ecc.read16(4).data, 0x1234); + assert_eq!(ecc.write16(6, 0x2313), BUS_OK); + assert_eq!(ecc.read16(6).data, 0x2313); + assert_eq!(ecc.write16(0x04000006, 0xface), BUS_OK); + assert_eq!(ecc.read16(0x04000006).data, 0xface); + + } + { + let md = configured(); + assert_eq!(md.ecc_bus().write8(7, 3), BUS_OK); + assert_eq!(md.ecc_bus().write8(6, 0x13), BUS_OK); + assert_eq!(md.ecc_bus().write8(5, 0x23), BUS_OK); + assert_eq!(md.ecc_scrub_bus().write64(0, 0), BUS_OK); + assert_eq!(md.reported(), [(true, true); 2], "the scrub's errors reach both interfaces"); + let record = md.md_bus().read64(0x60010).data; + assert_ne!(record & (1 << 60), 0); + assert_ne!(record & (1 << 52), 0); + assert_ne!(record & (1 << 53), 0); + assert_eq!((record >> 40) & 0xff, 3); + assert_eq!(record & (((1u64 << 30) - 1) << 3), 0); + assert_eq!(md.ecc_bus().read8(6).data, 0x13); + assert_eq!(md.ecc_bus().read8(5).data, 0x23); + } + } + /// The ECC view takes an uncached SH's check word from wherever the PROM + /// put it in the source register; other views, and bytes, take the low bits. + #[test] + fn ecc_view_reads_the_check_word_from_the_whole_source_register() { + let md = configured(); + let ecc = md.ecc_bus(); + // (carrier offset, register bits holding the check word) + for (carrier, shift) in [(6u64, 0u32), (4, 48), (2, 32), (0, 16)] { + let source = 0x1111_2222_3333_4444u64 ^ (0xa55a_u64 << shift); + let word = (source >> shift) as u16; + assert_eq!(ecc.write_narrow_from(carrier, 2, source), BUS_OK); + assert_eq!(ecc.read16(carrier).data, word, "carrier {carrier}"); + } + assert_eq!(ecc.write_narrow_from(5, 1, 0xfeed_0000_0000_0023), BUS_OK); + assert_eq!(ecc.read8(5).data, 0x23, "a byte keeps the architectural low bits"); + // The memory view is not the ECC view: its halfword is the low bits. + let view = md.uncached_view(); + assert_eq!(view.write_narrow_from(0x40, 2, 0xdead_beef_1234_5678), BUS_OK); + assert_eq!(md.dma_view().read64(0x40).data >> 48, 0x5678); + } + /// Uncached CPU and DMA views report uncorrectable errors with bus status. + #[test] + fn uncached_and_dma_views_report_errors_to_the_guest() { + let md = configured(); + let r = md.md_bus(); + let e = md.ecc_bus(); + e.write8(7, 3); + assert_eq!(md.uncached_view().read64(0).status, BUS_ERR); + assert_eq!(md.dma_view().read64(0).status, BUS_OK); + assert_ne!(r.read64(0x60018).data & (1 << 60), 0); + e.write8(7, 0x13); + assert_eq!(md.ecc_scrub_bus().write64(0, 0), BUS_OK); + assert_eq!(md.physical().read64(0).data, 1); + assert_eq!(e.read8(7).data, 0x13); + assert_eq!(md.ecc_scrub_bus().write32(0, 0), BUS_ERR); + } + /// Directory reads report invalid metadata and protection faults to the guest. + #[test] + fn directory_reads_check_metadata_and_report_protection_errors() { + { + let md = configured(); + let r = md.md_bus(); + md.directory_bus().write64(0, 1); + assert_eq!(md.directory_bus().read64(0).data, 0x001f0001); + r.write64(0x40000, 2); + md.directory_bus().write64(0, 0x001e0001); + r.write64(0x40000, 0); + md.ecc_bus().write8(7, 3); + assert_eq!(md.uncached_view().read64(0).status, BUS_ERR); + assert_eq!(md.directory_bus().read64(0).data, 0x001f0001); + assert_eq!(md.reported(), [(true, true); 2]); + assert_ne!(r.read64(0x40010).data & (1 << 61), 0); + assert_ne!(r.read64(0x60010).data & (1 << 60), 0); + } + { + let md = configured(); + let d = md.directory_bus(); + assert_eq!(d.write64(0x408, 0x1234), BUS_OK); + assert_eq!(d.read64(0x2000408).data, 0x1234); + assert_eq!(d.write64(0x418, 0x5678), BUS_OK); + assert_eq!(d.read64(0x408).data, 0x1234); + assert_eq!(d.read64(0x418).data, 0x5678); + assert_eq!(d.write64(8, u64::MAX), BUS_OK); + assert_eq!(d.read64(8).data, u64::MAX); + assert_eq!(d.read32(0).status, BUS_ERR); + assert_eq!(d.read64(1 << 30).status, BUS_ERR); + } + } +} + +const DIRECTORY_PREMIUM: [u8; 64] = [ + 0x1f, 0x2f, 0x37, 0x3b, 0x3d, 0x3e, 0x07, 0x0b, 0x0d, 0x0e, 0x13, 0x15, 0x16, 0x19, 0x1a, 0x1c, + 0x7f, 0x4f, 0x57, 0x5b, 0x5d, 0x5e, 0x67, 0x6b, 0x6d, 0x6e, 0x73, 0x75, 0x76, 0x79, 0x7a, 0x7c, + 1, 2, 4, 8, 16, 32, 0x23, 0x25, 0x26, 0x29, 0x2a, 0x2c, 0x31, 0x32, 0x34, 0x38, 0x40, 0x43, + 0x45, 0x46, 0x49, 0x4a, 0x4c, 0x51, 0x52, 0x54, 0x58, 0x61, 0x62, 0x64, 0x68, 0x70, +]; +const DIRECTORY_STANDARD: [u8; 32] = [ + 0x1f, 0x2f, 0x37, 0x3b, 0x3d, 0x3e, 0x07, 0x0b, 0x0d, 0x0e, 0x13, 0x15, 0x16, 0x19, 0x1a, 0x1c, + 1, 2, 4, 8, 16, 32, 0x23, 0x25, 0x26, 0x29, 0x2a, 0x2c, 0x31, 0x32, 0x34, 0x38, +]; +fn directory_columns(premium: bool) -> &'static [u8] { + if premium { + &DIRECTORY_PREMIUM + } else { + &DIRECTORY_STANDARD + } +} +fn directory_syndrome(mut word: u64, premium: bool) -> u8 { + if !premium { + word &= u32::MAX as u64; + } + let mut check = 0; + while word != 0 { + check ^= directory_columns(premium)[word.trailing_zeros() as usize]; + word &= word - 1; + } + check +} +pub fn directory_encode(word: u64, premium: bool) -> u64 { + let word = word + & !if premium { + (63 << 32) | (1 << 48) + } else { + 63 << 16 + }; + let check = directory_syndrome(word, premium) as u64; + word | if premium { + (check & 63) << 32 | (check >> 6) << 48 + } else { + check << 16 + } +} diff --git a/src/dev/bedrock/mem.rs b/src/dev/bedrock/mem.rs new file mode 100644 index 000000000..517ecf504 --- /dev/null +++ b/src/dev/bedrock/mem.rs @@ -0,0 +1,531 @@ +//! A Bedrock node's fitted memory: banks owned by `Physical`, with the bank +//! decode selected by Bedrock's memory configuration register. +//! +//! The board names the fitted banks, locational bank 0 upward, one size in +//! MiB each: two 256 MiB banks on the Fuel, six on the Tezro. Each bank +//! answers in its own 1 GiB software slot, so a Tezro's banks 4 and 5 sit +//! at 4 to 6 GiB, above the 4 GiB window ppmem maps for the CPU's direct +//! access. Those two answer through Physical's decode table only, and +//! their code pages keep the logical generation counters like the rest. +//! +//! These populations follow the QEMU models; they are not measured +//! topologies. Ordinary pages answer through Physical's bank views, without +//! the MD state lock. Pages with injected ECC or marked directory lines use +//! the MD endpoint. Their entire 64 MiB region is absent from the direct +//! window, so cache fills cannot bypass those checks. +//! +//! Remapping runs on the CPU thread, or with the CPU stopped, as HEART's +//! memory descriptors do. A bank configuration change rebuilds the whole +//! map. Marking a page adds only that page: it takes its entry in the decode +//! table, and the first in a 64 MiB region takes the region out of the direct +//! window. Lifting a mark removes nothing. The page stays on the MD endpoint, +//! which answers as the bank would, until a batch retires every lifted page +//! at once: the PROM's memory test marks and lifts the same few hundred pages +//! thousands of times, and a map change for each made it take minutes in the +//! QEMU model. DMA may remove diagnostic metadata but retains the slow +//! mapping until a later retirement. Physical publishes complete +//! decode-table pages atomically; a concurrent bus master sees an old or new +//! entry. This does not promise a transaction across a remap. +//! +//! Ported from hw/misc/sgi-bedrock-md.c in jamesbraid/qemu-sgi-origami. + +use crate::physical::{Physical, RamBank}; +use crate::traits::*; +use crate::ppmem::devmap::SLOT_SHIFT; +use crate::ppmem::{MappedMemory, BITMAP_SHIFT, WINDOW_SIZE}; +use crate::cpu::smp::{current_cpu, MAX_CPUS}; +use parking_lot::Mutex; +use std::collections::{BTreeSet, HashMap, HashSet}; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::{Arc, OnceLock}; + +pub const BANK_SPAN: u64 = 1 << 30; +/// Software slots of [`BANK_SPAN`] in the node's 8 GiB. +pub const SLOTS: usize = 8; +/// The most memory a bank can hold: what the largest size code names. +const MAX_BANK_MIB: u32 = 1024; +/// Banks are mapped into the direct window in whole regions of this size. +const REGION_MIB: u32 = 1 << (BITMAP_SHIFT - 20); + +/// Where fitted RAM can answer: for each software slot, the most any bank the +/// bank-select swizzle can place there holds. The swizzle permutes slots +/// within groups of four, so a slot may hold any bank of its group. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct RamExtent([u64; SLOTS]); +impl RamExtent { + /// `banks` are the fitted sizes in MiB from locational bank 0. + pub fn new(banks: &[u32]) -> Self { + let fitted = |bank: usize| banks.get(bank).map_or(0, |&mib| u64::from(mib) << 20); + Self(std::array::from_fn(|slot| { + (0..4).map(|group_slot| fitted(slot / 4 * 4 + group_slot)).max().unwrap_or(0) + })) + } + /// Whether `address` can be fitted RAM under some bank-select setting. + pub fn contains(&self, address: u64) -> bool { + let slot = address / BANK_SPAN; + slot < SLOTS as u64 && address % BANK_SPAN < self.0[slot as usize] + } + /// Slots from 0 through the last that can hold RAM. + #[cfg(feature = "jitv2")] + fn slots(&self) -> usize { + self.0.iter().rposition(|&size| size != 0).map_or(0, |last| last + 1) + } + /// The most RAM software slot `slot` can hold, in bytes. + pub fn prefix(&self, slot: usize) -> u64 { self.0[slot] } +} + +#[derive(Clone, Copy, PartialEq, Eq)] +struct Run { + bank: usize, + base: u64, + size: u64, +} + +/// Lifted pages kept on the MD endpoint before a batch retires them. +pub(crate) const RETIRED_LIMIT: usize = 1024; +/// How long a lifted page may idle before the next publication retires it: +/// a second of the controller's clock. +pub(crate) const RETIRED_IDLE_NS: u64 = 1_000_000_000; + +#[derive(Default)] +struct Mapping { + runs: Vec, + /// The pages that answer through the MD endpoint: those with a mark and + /// those whose mark was lifted since the last retirement. Bank-relative + /// 4 KiB page numbers, bank 1 immediately after bank 0. + placed: BTreeSet, + /// How many placed pages each 64 MiB region of the node's address space + /// holds, for the regions inside a run. + held: HashMap, + /// The regions mapped into the direct window now. + windowed: HashSet, + /// When the first lifted page was seen idle, on the controller's clock. + idle_since: Option, + slow: Option>, +} + +struct NoMemory; +impl BusDevice for NoMemory {} + +/// JIT pages belong to physical addresses, not to the bank currently +/// answering there. Keep these counters separate from the banks' counters: +/// a compiled page pins its pointer across later bank reconfiguration. +#[cfg(feature = "jitv2")] +pub(crate) struct Generations(Box<[AtomicU64]>); +#[cfg(feature = "jitv2")] +impl Generations { + /// One counter per 4 KiB page, up to the last slot that can hold RAM. + fn new(extent: &RamExtent) -> Self { + let pages = extent.slots() << (BANK_SPAN.trailing_zeros() - 12); + Self((0..pages).map(|_| AtomicU64::new(0)).collect::>().into_boxed_slice()) + } + pub(crate) fn base(&self) -> *mut AtomicU64 { + self.0.as_ptr() as *mut AtomicU64 + } + pub(crate) fn pages(&self) -> usize { + self.0.len() + } + fn bump(&self, address: u64) { + if let Some(counter) = self.0.get((address >> 12) as usize) { + counter.fetch_add(1, Ordering::Relaxed); + } + } + fn bump_all(&self, extent: &RamExtent) { + for slot in 0..extent.slots() { + let first = slot * (BANK_SPAN >> 12) as usize; + let pages = (extent.prefix(slot) >> 12) as usize; + for counter in &self.0[first..first + pages] { + counter.fetch_add(1, Ordering::Relaxed); + } + } + } +} + +/// What a remap costs, counted where the work happens so a test can state +/// it without timing anything. +#[cfg(test)] +#[derive(Default)] +pub(crate) struct Cost { + /// Host mappings made or scrubbed in ppmem's direct window. + pub window_ops: AtomicU64, + /// Calls that rewrite decode-table slots. + pub table_ops: AtomicU64, + /// Generation counters incremented, logical and bank. + pub bumps: AtomicU64, +} +#[cfg(test)] +impl Cost { + pub fn add(counter: &AtomicU64, n: usize) { + counter.fetch_add(n as u64, Ordering::Relaxed); + } + pub fn snapshot(&self) -> [u64; 3] { + [&self.window_ops, &self.table_ops, &self.bumps].map(|c| c.load(Ordering::Relaxed)) + } +} + +pub struct Memory { + physical: Arc, + /// Fitted bytes in each locational bank, bank 0 first. Physical's banks + /// are these, in the same order. + fitted: Vec, + config: AtomicU64, + mapping: Mutex, + /// Each processor's window while its PROM cache lines stand. + extent: RamExtent, + #[cfg(feature = "jitv2")] + generations: Arc, + #[cfg(test)] + pub(crate) cost: Cost, +} +impl Memory { + /// Memory with `banks` fitted, locational bank 0 upward, each a size in + /// MiB: at most [`SLOTS`] banks, none empty or over 1 GiB (the largest + /// size code), each a whole number of direct-window regions. + pub fn new(banks: &[u32]) -> Result { + if banks.is_empty() || banks.len() > SLOTS { + return Err(format!("memory: {} banks; fit 1 to {SLOTS}", banks.len())); + } + if let Some((bank, mib)) = banks.iter().enumerate() + .find(|(_, &mib)| mib == 0 || mib > MAX_BANK_MIB || mib % REGION_MIB != 0) + { + return Err(format!( + "memory: bank {bank} is {mib} MiB; a bank is a multiple of {REGION_MIB} MiB up to {MAX_BANK_MIB} MiB")); + } + let physical = Physical::new(banks.iter().map(|&mib| RamBank::new(mib as usize)).collect()); + physical.set_default(Arc::new(NoMemory), "Bedrock unconfigured memory"); + Ok(Self { physical, fitted: banks.iter().map(|&mib| u64::from(mib) << 20).collect(), + config: AtomicU64::new(0), mapping: Mutex::new(Mapping::default()), + #[cfg(feature = "jitv2")] + generations: Arc::new(Generations::new(&RamExtent::new(banks))), + extent: RamExtent::new(banks), + #[cfg(test)] + cost: Cost::default(), + }) + } + /// Fitted bytes in locational bank `bank`; zero where nothing is fitted. + pub(crate) fn fitted(&self, bank: u64) -> u64 { + self.fitted.get(bank as usize).copied().unwrap_or(0) + } + /// Where the fitted banks start in a dense walk of all of them, bank 0 + /// first. Sparse tables key ECC, directory and protection state by it, so + /// two banks never share a key. + pub(crate) fn dense_base(&self, bank: u64) -> u64 { + self.fitted.iter().take(bank as usize).sum() + } + pub fn extent(&self) -> RamExtent { self.extent } + pub fn physical(&self) -> &Arc { + &self.physical + } + pub(crate) fn config(&self) -> u64 { + self.config.load(Ordering::Acquire) + } + + #[cfg(feature = "jitv2")] + pub(crate) fn generations(&self) -> Arc { self.generations.clone() } + #[cfg(feature = "jitv2")] + pub(crate) fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + // Disabled RAM must still have a counter: a JIT page may already + // have claimed it before the guest enables or moves a bank. + if self.extent.contains(address) { + unsafe { self.generations.base().add((address >> 12) as usize) } + } else { std::ptr::null() } + } + pub(crate) fn written(&self, _address: u64) { + #[cfg(feature = "jitv2")] + self.generations.bump(_address); + } + + /// The endpoint is weak back to MD: Physical owns it without a cycle. + pub(crate) fn set_slow_view(&self, slow: Arc) { + self.mapping.lock().slow = Some(slow); + } + + fn runs(&self, config: u64) -> Vec { + (0..SLOTS as u64).filter_map(|slot| { + let base = slot * BANK_SPAN; + let (bank, _) = self.decode(config, base, 1)?; + let code = (config >> (bank / 2 * 8 + 4)) & 15; + Some(Run { bank, base, size: self.fitted(bank as u64).min((32 << 20) << code) }) + }).collect() + } + + /// Publish a state snapshot after releasing MD's lock. Only the CPU + /// thread, or a caller with the CPU stopped, may change ppmem's bitmap. + /// A forced rebuild follows the machine setting Physical's default. + /// This is the whole-map rebuild: `pages` replaces every placed page, so + /// it retires the lifted ones too. Without `force` it does nothing when + /// neither the runs nor the placed pages would change. + pub(crate) fn remap(&self, config: u64, pages: BTreeSet, force: bool) { + let runs = self.runs(config); + let mut old = self.mapping.lock(); + if !force && old.runs == runs && old.placed == pages { + self.config.store(config, Ordering::Release); + return; + } + self.rebuild(&mut old, config, runs, pages); + } + + /// The bank configuration register changed to `config`. Rebuild, with + /// the marked `pages`, only if it changes which banks answer where. + pub(crate) fn reconfigure(&self, config: u64, pages: impl FnOnce() -> BTreeSet) { + let runs = self.runs(config); + let mut old = self.mapping.lock(); + if old.runs == runs { + self.config.store(config, Ordering::Release); + return; + } + self.rebuild(&mut old, config, runs, pages()); + } + + fn rebuild(&self, old: &mut Mapping, config: u64, runs: Vec, pages: BTreeSet) { + let phys = &self.physical; + let space = phys.ppmem_space(); + // Withdraw the direct window before publishing any checked page. + if let Some(space) = space { + #[cfg(test)] + Cost::add(&self.cost.window_ops, space.mapping_count()); + space.clear_mappings(); + } + // Restore only ranges previously owned by RAM. In particular, the + // unused part of a 1 GiB slot may contain flash or other devices. + for run in &old.runs { + let retained = runs.iter().find(|next| next.base == run.base) + .map_or(0, |next| next.size.min(run.size)); + if retained < run.size { + #[cfg(test)] + Cost::add(&self.cost.table_ops, 1); + phys.fill_slots((run.base + retained) >> SLOT_SHIFT, + (run.size - retained) >> SLOT_SHIFT, phys.default_entry()); + } + } + old.held.clear(); + old.windowed.clear(); + old.idle_since = None; + for run in &runs { + #[cfg(test)] + Cost::add(&self.cost.table_ops, 1); + phys.fill_slots(run.base >> SLOT_SHIFT, run.size >> SLOT_SHIFT, phys.bank_view(run.bank, run.base)); + let bank_page = self.dense_base(run.bank as u64) >> SLOT_SHIFT; + let run_pages = run.size >> SLOT_SHIFT; + for &page in pages.range(bank_page..bank_page + run_pages) { + let address = run.base + ((page - bank_page) << SLOT_SHIFT); + *old.held.entry(address >> BITMAP_SHIFT).or_default() += 1; + #[cfg(test)] + Cost::add(&self.cost.table_ops, 1); + phys.map(address, address + (1 << SLOT_SHIFT), + old.slow.as_ref().expect("Bedrock MD slow endpoint").clone(), "Bedrock checked RAM"); + } + if let Some(space) = space { + let region_size = 1 << BITMAP_SHIFT; + for offset in (0..run.size).step_by(region_size as usize) { + let region = (run.base + offset) >> BITMAP_SHIFT; + // Above the window the bank answers through the table alone. + let in_window = run.base + offset + region_size <= WINDOW_SIZE; + if in_window && !old.held.contains_key(®ion) { + #[cfg(test)] + Cost::add(&self.cost.window_ops, 1); + match space.map_range(run.bank, offset, run.base + offset, region_size) { + Ok(()) => { old.windowed.insert(region); } + Err(error) => crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock RAM window at {:#x}: {error}", run.base + offset), + } + } + } + } + } + self.config.store(config, Ordering::Release); + // Publish changed ownership before advancing the stable counters, + // so an in-flight compile cannot accept bytes from the old decode. + self.changed_decode(); + old.runs = runs; + old.placed = pages; + } + + /// Where bank-relative `page` lies in the run holding it: the run, and the + /// page's byte offset in the bank. + fn locate(&self, runs: &[Run], page: u64) -> Option<(Run, u64)> { + runs.iter().find_map(|run| { + let first = self.dense_base(run.bank as u64) >> SLOT_SHIFT; + (first..first + (run.size >> SLOT_SHIFT)).contains(&page) + .then(|| (*run, (page - first) << SLOT_SHIFT)) + }) + } + + /// Advance the counters of `pages` 4 KiB pages starting at `address`, the + /// bank's offset `offset` for the same bytes. + fn bump_pages(&self, _address: u64, _bank: usize, _offset: u64, _pages: u64) { + #[cfg(feature = "jitv2")] + { + for page in 0.._pages { + self.generations.bump(_address + (page << SLOT_SHIFT)); + self.physical.banks()[_bank].bump_gen_off((_offset + (page << SLOT_SHIFT)) as usize); + } + #[cfg(test)] + Cost::add(&self.cost.bumps, 2 * _pages as usize); + } + } + + /// Route the pages in `marked` through the MD endpoint, adding to what is + /// placed and changing nothing else. A page already placed, as a lifted + /// one marked again is, costs no map change. The first page in a region + /// takes the region out of the direct window, and a page whose mapping + /// changes advances its counters, so code compiled from it, or from the + /// region's direct path, is dropped. CPU thread only, as `remap`. + pub(crate) fn place(&self, marked: &[u64]) { + let mut m = self.mapping.lock(); + let phys = &self.physical; + let space = phys.ppmem_space(); + let region_size = 1u64 << BITMAP_SHIFT; + for &page in marked { + if !m.placed.insert(page) { + continue; + } + let Some((run, offset)) = self.locate(&m.runs, page) else { continue }; + let address = run.base + offset; + let region = address >> BITMAP_SHIFT; + *m.held.entry(region).or_default() += 1; + // Withdraw the direct window before publishing the checked page. + if m.windowed.remove(®ion) { + if let Some(space) = space { + // Off the direct path, host pages left mapped: another + // processor that read the old bitmap still reaches the bank. + space.withdraw_range(region << BITMAP_SHIFT, region_size); + } + self.bump_pages(region << BITMAP_SHIFT, run.bank, + offset & !(region_size - 1), region_size >> SLOT_SHIFT); + } else { + self.bump_pages(address, run.bank, offset, 1); + } + #[cfg(test)] + Cost::add(&self.cost.table_ops, 1); + phys.map(address, address + (1 << SLOT_SHIFT), + m.slow.as_ref().expect("Bedrock MD slow endpoint").clone(), "Bedrock checked RAM"); + } + } + + /// Whether the lifted pages are due to be retired: `active` pages of the + /// placed ones still hold a mark, and `now` is the controller's clock if + /// it has one. Due when too many lifted pages have piled up, or when the + /// first has idled for a second. Notes the time it first sees one. + pub(crate) fn retirement_due(&self, active: usize, now: Option) -> bool { + let mut m = self.mapping.lock(); + if m.placed.len() <= active { + m.idle_since = None; + return false; + } + if m.placed.len() - active >= RETIRED_LIMIT { + return true; + } + match (now, m.idle_since) { + (Some(now), Some(since)) => now.saturating_sub(since) >= RETIRED_IDLE_NS, + (Some(now), None) => { m.idle_since = Some(now); false } + _ => false, + } + } + + /// The placed pages for which `marked` is false, in order. + pub(crate) fn lifted(&self, marked: impl Fn(u64) -> bool) -> Vec { + self.mapping.lock().placed.iter().copied().filter(|&page| !marked(page)).collect() + } + + /// Return every page of `lifted` to its bank in one change: its table + /// entry, then the direct window for each region left with no placed + /// page. The pages' counters advance as for any change of mapping. CPU + /// thread only, as `remap`. + pub(crate) fn retire(&self, lifted: &[u64]) { + let mut m = self.mapping.lock(); + let phys = &self.physical; + let space = phys.ppmem_space(); + let region_size = 1u64 << BITMAP_SHIFT; + let mut emptied = Vec::new(); + for &page in lifted { + if !m.placed.remove(&page) { + continue; + } + let Some((run, offset)) = self.locate(&m.runs, page) else { continue }; + let address = run.base + offset; + let region = address >> BITMAP_SHIFT; + #[cfg(test)] + Cost::add(&self.cost.table_ops, 1); + phys.fill_slots(address >> SLOT_SHIFT, 1, phys.bank_view(run.bank, run.base)); + self.bump_pages(address, run.bank, offset, 1); + let held = m.held.get_mut(®ion).expect("a placed page's region is held"); + *held -= 1; + if *held == 0 { + m.held.remove(®ion); + emptied.push((region, run, offset & !(region_size - 1))); + } + } + // Return a region to the window only once none of its pages is placed. + for (region, run, offset) in emptied { + let in_window = (region + 1) << BITMAP_SHIFT <= WINDOW_SIZE; + if let (Some(space), true) = (space, in_window) { + let base = region << BITMAP_SHIFT; + // A region withdrawn from the path is still mapped; one the + // last rebuild left out because it held a page was never. + if space.restore_range(base, region_size) { + m.windowed.insert(region); + continue; + } + #[cfg(test)] + Cost::add(&self.cost.window_ops, 1); + match space.map_range(run.bank, offset, base, region_size) { + Ok(()) => { m.windowed.insert(region); } + Err(error) => crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock RAM window at {:#x}: {error}", base), + } + } + } + m.idle_since = None; + } + + pub(crate) fn decode(&self, config: u64, addr: u64, bytes: u64) -> Option<(usize, u64)> { + let software = addr / BANK_SPAN; + if software >= SLOTS as u64 { + return None; + } + let bank = software ^ ((config >> 32) & 3); + let field = config >> (bank / 2 * 8); + let code = (field >> 4) & 15; + if self.fitted(bank) == 0 || field & (1 << (bank & 1)) == 0 || !(1..=5).contains(&code) { + return None; + } + let limit = self.fitted(bank).min((32 << 20) << code); + let offset = addr % BANK_SPAN; + (offset < limit && bytes <= limit - offset).then_some((bank as usize, offset)) + } + pub(crate) fn raw_read(&self, bank: usize, at: u64) -> u64 { + self.physical.banks()[bank].read64(at).data + } + pub(crate) fn raw_write(&self, bank: usize, at: u64, value: u64) -> u32 { + let result = self.physical.banks()[bank].write64(at, value); + if result == BUS_OK { + let slot = bank as u64 ^ ((self.config() >> 32) & 3); + self.written(slot * BANK_SPAN + at); + } + result + } + pub(crate) fn zero(&self, bank: usize, size: u64) { + assert!(size <= self.fitted(bank as u64)); + // The bank owns this contiguous mapping; zero has the same representation + // in host word order. Self-test does not synchronize concurrent bank accesses. + unsafe { + std::ptr::write_bytes(self.physical.banks()[bank].host_ptr(), 0, size as usize); + } + #[cfg(feature = "jitv2")] + self.physical.banks()[bank].bump_gen_all(); + #[cfg(feature = "jitv2")] + self.generations.bump_all(&self.extent); + } + pub(crate) fn changed_decode(&self) { + #[cfg(feature = "jitv2")] + self.generations.bump_all(&self.extent); + #[cfg(feature = "jitv2")] + for bank in self.physical.banks() { + bank.bump_gen_all(); + } + #[cfg(all(test, feature = "jitv2"))] + Cost::add(&self.cost.bumps, (0..self.extent.slots()).map(|slot| (self.extent.prefix(slot) >> 12) as usize).sum::() + + self.physical.banks().iter().map(|bank| bank.size() >> 12).sum::()); + } +} diff --git a/src/dev/bedrock/mod.rs b/src/dev/bedrock/mod.rs index 387f453b7..b83f4c41c 100644 --- a/src/dev/bedrock/mod.rs +++ b/src/dev/bedrock/mod.rs @@ -14,19 +14,25 @@ //! error status) are atomics, so a mask store, a pending-modify store and //! a counter read take no lock. `state` holds the stored words and the //! graphics registers; `timer` the comparators and the host timer. +//! - MD: the cell register is an atomic and ordinary memory takes no lock; +//! `state` and `remapping` as the controller's notes say. //! //! Ported from hw/misc/sgi-bedrock*.c in jamesbraid/qemu-sgi-origami. +mod md; +mod mem; mod pi; #[cfg(test)] mod tests; +pub use mem::{RamExtent, BANK_SPAN, SLOTS}; pub use pi::{Population, Refused}; use crate::dev::heart::Clock; use crate::hptimer::{TimerId, TimerManager, TimerReturn}; use crate::irq::{InterruptController, IrqLine}; use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Device, Resettable, BUS_ERR, BUS_OK}; +use md::{Md, MemoryError, View}; use pi::{Levels, Pi}; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, OnceLock, Weak}; @@ -50,17 +56,20 @@ pub mod reg { /// `IIO_PROTECT_OVRRD_KEY` as this value (lines 31, 47). This establishes /// the literal's provenance only; it does not establish a register mapping. pub const IIO_PROTECT_OVRRD_KEY: u64 = 0x53474972756c6573; + pub use super::md::reg as md; pub use super::pi::reg as pi; } /// What a board gives its Bedrock. Fixed for the chip's life. -pub struct BedrockConfig { +pub struct BedrockConfig<'a> { /// Host nanoseconds: the PIs' counter and timers run on it, and the /// other units' clocked work. pub clock: Clock, /// The processor slots the board fits. CPU n sits in /// `processors.slot_of(n)`; a fitted slot starts with its enable bit set. pub processors: Population, + /// The memory banks in MiB, locational bank 0 upward. + pub banks_mib: &'a [u32], } /// The two XIO register forms. `Local` answers the requester's own PI at @@ -94,23 +103,34 @@ pub struct Bedrock { clock: Clock, processors: Population, pi: [Pi; 2], + md: Md, levels: Levels, + /// MD's DMA view: the BTEs and inbound memory requests use it. + dma: Arc, timers: OnceLock>, weak: Weak, } impl Bedrock { - pub fn new(config: BedrockConfig) -> Result, String> { - let BedrockConfig { clock, processors } = config; + pub fn new(config: BedrockConfig<'_>) -> Result, String> { + let BedrockConfig { clock, processors, banks_mib } = config; + let memory = mem::Memory::new(banks_mib)?; let now = clock(); - Ok(Arc::new_cyclic(|weak: &Weak| Self { - pi: [Pi::new(0, processors, now), Pi::new(1, processors, now)], - levels: Levels::default(), - timers: OnceLock::new(), - clock, - processors, - weak: weak.clone(), - })) + let chip = Arc::new_cyclic(|weak: &Weak| { + let md = Md::new(memory, clock.clone(), weak.clone()); + Self { + pi: [Pi::new(0, processors, now), Pi::new(1, processors, now)], + dma: md.dma_view(), + md, + levels: Levels::default(), + timers: OnceLock::new(), + clock, + processors, + weak: weak.clone(), + } + }); + chip.md.rebuild_memory(); + Ok(chip) } pub fn register_locks(self: &Arc) { @@ -119,6 +139,8 @@ impl Bedrock { register_mutex_fn("bedrock-pi1::state", self, |c| c.pi[1].state_locked()); register_mutex_fn("bedrock-pi0::timer", self, |c| c.pi[0].timer_locked()); register_mutex_fn("bedrock-pi1::timer", self, |c| c.pi[1].timer_locked()); + register_mutex_fn("bedrock-md::state", self, |c| c.md.state_locked()); + register_mutex_fn("bedrock-md::remapping", self, |c| c.md.remapping_locked()); } // The faces a board maps into its processor bus. Each is its own type, @@ -132,6 +154,54 @@ impl Bedrock { Arc::new(Registers::(self.clone())) } } + /// Memory without the checks MD's views make, for the processor's + /// cached view and DMA: Physical's entries. + pub fn dma_view(&self) -> Arc { + self.dma.clone() + } + /// Uncached memory: RAM with MD's checks. + pub fn uncached_view(self: &Arc) -> Arc { + self.md.view(self.weak.clone(), View::Uncached) + } + pub fn directory_bus(self: &Arc) -> Arc { + self.md.view(self.weak.clone(), View::Directory) + } + pub fn ecc_bus(self: &Arc) -> Arc { + self.md.view(self.weak.clone(), View::Ecc) + } + pub fn ecc_scrub_bus(self: &Arc) -> Arc { + self.md.view(self.weak.clone(), View::Scrub) + } + pub fn fetchop_bus(self: &Arc) -> Arc { + self.md.view(self.weak.clone(), View::Fetchop) + } + pub fn physical(&self) -> &Arc { + self.md.physical() + } + /// Where fitted RAM can answer under any bank-select setting. + pub fn ram_extent(&self) -> RamExtent { + self.md.ram_extent() + } + /// Reinstall RAM after the board sets Physical's default; on the CPU + /// thread or with the CPU stopped. + pub fn rebuild_memory(&self) { + self.md.rebuild_memory(); + } + /// Return pages whose ECC or directory marks were lifted to the direct + /// window now; on the CPU thread or with the CPU stopped. + pub fn retire_lifted_marks(&self) { + self.md.retire_lifted_marks(); + } + #[cfg(feature = "jitv2")] + pub(crate) fn generation_base(&self) -> *mut AtomicU64 { + self.md.generation_base() + } + /// MD's cell register bit 0, which gates the register window and the + /// board's Hub-special windows. One atomic load. + #[inline] + pub fn cell_permits(&self) -> bool { + self.md.cell_permits() + } // Processors. @@ -176,6 +246,28 @@ impl Bedrock { pi.publish(&self.levels); } } + /// Report what an access through one of MD's views found: a correctable + /// error sets bank 1 bit 61 and an uncorrectable one error-summary bit + /// 24, on both PIs. + fn memory_errors(&self, errors: &[MemoryError]) { + for &error in errors { + for pi in &self.pi { + match error { + MemoryError::Correctable => pi.memory_correctable(), + MemoryError::Uncorrectable => pi.memory_uncorrectable(), + } + } + } + self.publish_all(); + } + /// An access through one of MD's views. + fn md_access(&self, view: View, address: u64, bytes: u64, write: Option) -> Option { + let (result, errors) = self.md.access(view, address, bytes, write); + if !errors.is_empty() { + self.memory_errors(&errors); + } + result + } /// The PI a local-form access at offset 0 reaches: the requester's own. /// A reader that is no processor reaches PI0. @@ -189,9 +281,16 @@ impl Bedrock { /// when `value` is given; PI0's range reaches PI `pi0`. What no unit /// answers, the padding after a unit's registers included, is a bus error. fn access(&self, pi0: usize, offset: u64, size: u8, value: Option) -> BusRead64 { + if !self.md.cell_permits() { + return BusRead64::err(); + } match offset { reg::PI0_BASE..reg::PI1_BASE => self.pi_access(pi0, offset, size, value), reg::PI1_BASE..reg::IO_BASE => self.pi_access(1, offset - reg::PI1_BASE, size, value), + reg::MD_BASE..reg::WINDOW => { + let result = self.md_access(View::Registers, offset - reg::MD_BASE, u64::from(size), value); + result.map_or(BusRead64::err(), BusRead64::ok) + } _ => BusRead64::err(), } } @@ -231,6 +330,7 @@ impl Device for Bedrock { /// The counters run and the PIs' timers arm. fn start(&self) { let now = (self.clock)(); + self.md.start_clock(); let tm = self.timer_manager(); let host = tm.as_deref().map(|tm| self.host_timer(tm)); for pi in &self.pi { @@ -246,6 +346,7 @@ impl Device for Bedrock { for pi in &self.pi { pi.stop(host.as_ref(), now); } + self.md.stop_clock(); self.publish_all(); } fn is_running(&self) -> bool { @@ -266,6 +367,7 @@ impl Resettable for Bedrock { pi.power_on(now); pi.reschedule(host.as_ref(), now); } + self.md.power_on(); self.publish_all(); } } @@ -310,10 +412,10 @@ impl InterruptController for Uart { } } -/// A test chip's configuration, on `clock`. +/// A test chip's configuration: one 64 MiB bank, on `clock`. #[cfg(test)] -fn test_config(clock: Clock, processors: Population) -> BedrockConfig { - BedrockConfig { clock, processors } +fn test_config(clock: Clock, processors: Population) -> BedrockConfig<'static> { + BedrockConfig { clock, processors, banks_mib: &[64] } } /// A chip on a hand-driven clock, for the unit tests: its register window diff --git a/src/dev/bedrock/pi.rs b/src/dev/bedrock/pi.rs index 89bef640b..87611121a 100644 --- a/src/dev/bedrock/pi.rs +++ b/src/dev/bedrock/pi.rs @@ -480,6 +480,14 @@ impl Pi { } + /// Bank 1 bit 61 for a correctable memory error. + pub(super) fn memory_correctable(&self) { + self.pending[1].fetch_or(1 << 61, SeqCst); + } + /// Error-summary bit 24 for an uncorrectable memory error. + pub(super) fn memory_uncorrectable(&self) { + self.error_summary.fetch_or(MEMORY_ERROR, SeqCst); + } /// The real-time counter at host time `now`. Lock-free. pub(super) fn count(&self, now: u64) -> u64 { diff --git a/src/dev/bedrock/tests.rs b/src/dev/bedrock/tests.rs index 09a5f5c8f..8580e6a2c 100644 --- a/src/dev/bedrock/tests.rs +++ b/src/dev/bedrock/tests.rs @@ -1,7 +1,7 @@ //! Tests of the chip as a whole: what one unit's access does to another, //! the lock-free interrupt publish, and the paths that take no lock. -use super::reg::{pi, PI1_BASE}; +use super::reg::{md, pi, MD_BASE, PI1_BASE}; use super::*; use crate::traits::{BUS_ERR, BUS_OK}; @@ -63,6 +63,7 @@ fn without_locks(rig: &Arc, work: impl FnOnce(&Rig) + Send + 'static) { let held = ( chip.pi[0].hold(), chip.pi[1].hold(), + chip.md.hold(), ); let (done, finished) = std::sync::mpsc::channel(); let other = rig.clone(); @@ -92,3 +93,19 @@ fn the_pi_hot_paths_take_no_lock() { assert_eq!(rig.regs.read32(pi::COUNTER + 4).data, 10); }); } + +/// With MD's cell register bit 0 clear, the register window refuses every +/// access, MD's own registers included. The padding after a unit's +/// registers, and anything past the window, answers a bus error. +#[test] +fn the_cell_bit_gates_the_window_and_padding_answers_an_error() { + let rig = Rig::new(); + assert!(rig.chip.cell_permits()); + assert_eq!(rig.regs.read64(super::pi::SIZE).status, BUS_ERR, "past PI0's registers"); + assert_eq!(rig.regs.read64(MD_BASE - 8).status, BUS_ERR); + assert_eq!(rig.regs.read64(super::reg::WINDOW).status, BUS_ERR); + rig.w(MD_BASE + md::CELL, 0); + assert!(!rig.chip.cell_permits()); + assert_eq!(rig.regs.read64(pi::INTERFACE_ID).status, BUS_ERR); + assert_eq!(rig.regs.read64(MD_BASE + md::CELL).status, BUS_ERR); +} diff --git a/src/ppmem/ppmem.rs b/src/ppmem/ppmem.rs index e6da8ba9e..35c94b02b 100644 --- a/src/ppmem/ppmem.rs +++ b/src/ppmem/ppmem.rs @@ -651,6 +651,13 @@ struct Mapping { period: u64, } +impl Mapping { + /// The part of this mapping in `[from, to)`, laid from the same bytes. + fn part(&self, from: u64, to: u64) -> Mapping { + Mapping { at: from, len: to - from, src: self.src + (from - self.at), ..*self } + } +} + /// The 4GB window that banks are mapped into, plus the banks themselves. /// /// Holding the banks here is what lets `map_bank` take an index — matching the @@ -690,6 +697,9 @@ struct SpaceState { /// The window's size in bytes. Never changes. window: u64, mappings: Vec, + /// Ranges still mapped on the host but off the direct path; see + /// [`PpMemSpace::withdraw_range`]. + withdrawn: Vec, /// Where the bitmap is published. Points at `own_bitmap` until a consumer /// hands over its own inline field, so it is never null and publication is /// an unconditional store. @@ -773,6 +783,7 @@ impl PpMemSpace { state: UnsafeCell::new(SpaceState { window, mappings: Vec::new(), + withdrawn: Vec::new(), sink, sink2, bits: 0, @@ -810,6 +821,12 @@ impl PpMemSpace { self.banks.len() } + /// How many ranges are mapped now; a remap scrubs each one. + #[cfg(test)] + pub(crate) fn mapping_count(&self) -> usize { + unsafe { &*self.state.get() }.mappings.len() + } + /// Map `size` bytes of `bank`, from object offset `src`, at window /// `offset`, without repeating, with the generation counters in lockstep. /// The O2 maps its memory this way: one 32 MB slot of the window at a @@ -841,6 +858,72 @@ impl PpMemSpace { Ok(()) } + /// Return `at..at + len` of the window, and its counters, to unmapped. + fn scrub_range(&self, at: u64, len: u64) { + // Best-effort: a failure here leaves the range mapped, which is + // still safe — it just isn't reverted. + // `scrub`, not `unmap`: see `AddrSpace::scrub`. + let _ = unsafe { self.space.scrub(at as usize, len as usize) }; + #[cfg(feature = "jitv2")] + { + let gen_off = at / GEN_RATIO; + let gen_len = len / GEN_RATIO; + let gran = super::map::granularity() as u64; + if gen_len >= gran && gen_off % gran == 0 && gen_len % gran == 0 { + let _ = unsafe { self.gen_space.scrub(gen_off as usize, gen_len as usize) }; + } + } + } + + /// Take `offset..offset + size` off the direct path without unmapping + /// it. The bitmap is republished to every sink with the regions this + /// leaves partly covered clear; the host mapping and its counters stay, + /// so a processor that read the old bitmap an instant ago still reaches + /// the bank's bytes rather than a scrubbed page, and needs no other + /// processor stopped. A mapping that only partly overlaps keeps the rest + /// on the path, but a bit is set only for a region still wholly mapped, + /// so withdrawing part of a region takes the whole region off. + /// [`Self::restore_range`] puts it back. CPU thread only, as the rest. + pub fn withdraw_range(&self, offset: u64, size: u64) { + assert!(offset + size <= self.window_size(), "ppmem: [{offset:#x}, +{size:#x}) escapes the window"); + let st = unsafe { &mut *self.state.get() }; + let end = offset + size; + let mut kept = Vec::with_capacity(st.mappings.len() + 1); + for m in std::mem::take(&mut st.mappings) { + if m.at >= end || m.at + m.len <= offset { + kept.push(m); + continue; + } + let (from, to) = (m.at.max(offset), (m.at + m.len).min(end)); + st.withdrawn.push(m.part(from, to)); + if m.at < offset { + kept.push(m.part(m.at, offset)); + } + if m.at + m.len > end { + kept.push(m.part(end, m.at + m.len)); + } + } + st.mappings = kept; + Self::recompute_bitmap(st); + } + + /// Put what [`Self::withdraw_range`] took off the direct path in + /// `offset..offset + size` back on it, republishing the bitmap. Returns + /// whether anything was withdrawn there; if not, nothing was mapped + /// and the caller maps the range with [`Self::map_range`]. + pub fn restore_range(&self, offset: u64, size: u64) -> bool { + let st = unsafe { &mut *self.state.get() }; + let end = offset + size; + let (back, still): (Vec<_>, Vec<_>) = std::mem::take(&mut st.withdrawn) + .into_iter() + .partition(|m| m.at >= offset && m.at + m.len <= end); + st.withdrawn = still; + let any = !back.is_empty(); + st.mappings.extend(back); + Self::recompute_bitmap(st); + any + } + /// Recompute the bitmap from the live mapping list. /// /// A bit is set only if the whole region (64MB in a 4GB window) is covered, so a region that @@ -901,11 +984,22 @@ impl PpMemSpace { state .mappings .retain(|m| !(m.at >= at && m.at + m.len <= at + len)); + state + .withdrawn + .retain(|m| !(m.at >= at && m.at + m.len <= at + len)); state.mappings.push(m); Self::recompute_bitmap(state); } } +impl SpaceState { + /// Every range with memory behind it: the mappings, and the ranges taken + /// off the direct path but still mapped (`PpMemSpace::withdraw_range`). + fn present(&self) -> impl Iterator { + self.mappings.iter().chain(&self.withdrawn) + } +} + /// Cached views. /// /// A CPU's cache and its compiled code read memory through ppmem's window, so @@ -933,7 +1027,7 @@ impl PpMemSpace { let sinks = sinks.iter().copied().filter(|p| !p.is_null()).collect(); st.views.push(View { space, sinks, held: Default::default() }); let v = st.views.len() - 1; - for m in st.mappings.clone() { + for m in st.present().copied().collect::>() { self.lay(st, v, &m, m.at, m.at + m.len)?; } Self::publish_view(st, v); @@ -964,7 +1058,7 @@ impl PpMemSpace { let st = unsafe { &mut *self.state.get() }; let gran = granularity() as u64; let page = pa & !(gran - 1); - if !st.mappings.iter().any(|m| m.at <= page && m.at + m.len >= page + gran) { + if !st.present().any(|m| m.at <= page && m.at + m.len >= page + gran) { return None; } let shift = window_shift(st.window); @@ -1009,7 +1103,7 @@ impl PpMemSpace { self.mark(st, page, false); } let _ = unsafe { st.views[v].space.scrub(page as usize, gran as usize) }; - for m in st.mappings.iter().filter(|m| m.at < page + gran && m.at + m.len > page) { + for m in st.present().filter(|m| m.at < page + gran && m.at + m.len > page) { let _ = self.lay(st, v, m, page.max(m.at), (page + gran).min(m.at + m.len)); } Self::publish_view(st, v); @@ -1028,7 +1122,7 @@ impl PpMemSpace { { use crate::cpu::jitv2::jitv2::{GEN_HELD, PAGE_SIZE}; let end = page + granularity() as u64; - if !st.mappings.iter().any(|m| m.at <= page && m.at + m.len >= end) { + if !st.present().any(|m| m.at <= page && m.at + m.len >= end) { return; } let d = if on { GEN_HELD } else { 1u64.wrapping_sub(GEN_HELD) }; @@ -1056,18 +1150,18 @@ impl PpMemSpace { let end = r + len; let mut at = r; while at < end { - if let Some(m) = st.mappings.iter().find(|m| m.at <= at && m.at + m.len > at) { + if let Some(m) = st.present().find(|m| m.at <= at && m.at + m.len > at) { at = m.at + m.len; continue; } - let next = st.mappings.iter().filter(|m| m.at > at && m.at < end).map(|m| m.at).min().unwrap_or(end); + let next = st.present().filter(|m| m.at > at && m.at < end).map(|m| m.at).min().unwrap_or(end); let _ = unsafe { st.views[v].space.scrub(at as usize, (next - at) as usize) }; #[cfg(feature = "jitv2")] { let g = granularity() as u64; let (gs, ge) = ((at / GEN_RATIO) & !(g - 1), (next / GEN_RATIO).div_ceil(g) * g); let touches = |m: &Mapping| m.at < ge * GEN_RATIO && m.at + m.len > gs * GEN_RATIO; - if !st.mappings.iter().any(touches) { + if !st.present().any(touches) { let _ = unsafe { self.gen_space.scrub(gs as usize, (ge - gs) as usize) }; } } @@ -1139,27 +1233,13 @@ impl MappedMemory for PpMemSpace { // view never shows memory the shared window no longer maps. for view in &st.views { let saved = Self::save_held(view, 0, st.window); - for m in &st.mappings { + for m in st.mappings.iter().chain(&st.withdrawn) { let _ = unsafe { view.space.scrub(m.at as usize, m.len as usize) }; } Self::put_held(view, saved); } - for m in std::mem::take(&mut st.mappings) { - // Best-effort: a failure here leaves the range mapped, which is - // still safe — it just isn't reverted. - // `scrub`, not `unmap`: see `AddrSpace::scrub`. - let _ = unsafe { self.space.scrub(m.at as usize, m.len as usize) }; - #[cfg(feature = "jitv2")] - { - let gen_off = m.at / GEN_RATIO; - let gen_len = m.len / GEN_RATIO; - let gran = super::map::granularity() as u64; - if gen_len >= gran && gen_off % gran == 0 && gen_len % gran == 0 { - let _ = unsafe { - self.gen_space.scrub(gen_off as usize, gen_len as usize) - }; - } - } + for m in std::mem::take(&mut st.mappings).into_iter().chain(std::mem::take(&mut st.withdrawn)) { + self.scrub_range(m.at, m.len); } Self::recompute_bitmap(st); } @@ -1703,6 +1783,39 @@ mod tests { assert_eq!(space.mapped_bitmap(), 0, "bitmap not cleared"); } + /// Withdrawing one region clears only its bit in every sink and leaves + /// the host mapping, so the bank's bytes stay reachable; restoring it sets + /// the bit again without mapping anything. Clearing the mappings scrubs + /// withdrawn ranges too. + #[test] + fn withdraw_range_clears_one_region_in_every_sink_and_keeps_the_mapping() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[256]).unwrap(); + let mut third = 0u64; + unsafe { + space.set_bitmap_sink2(&mut third); + } + let region = 64 * MB as u64; + for i in 0..3 { + space.map_range(0, i * region, i * region, region).unwrap(); + } + assert_eq!((space.mapped_bitmap(), third), (0b111, 0b111)); + space.withdraw_range(region, region); + assert_eq!((space.mapped_bitmap(), third), (0b101, 0b101)); + assert_eq!(space.mapping_count(), 2); + // The host mapping is still there: a store through the window lands in the bank. + unsafe { *(space.window_base().add(region as usize + 0x100) as *mut u32) = 0x0BAD_C0DE }; + assert_eq!(banks[0].read32(region + 0x100).data, 0x0BAD_C0DE); + assert!(space.restore_range(region, region)); + assert_eq!((space.mapped_bitmap(), third), (0b111, 0b111)); + assert!(!space.restore_range(region, region), "nothing left to restore"); + // Part of a region leaves the whole region off the direct path. + space.withdraw_range(region + 0x1000, 0x1000); + assert_eq!((space.mapped_bitmap(), third), (0b101, 0b101)); + space.clear_mappings(); + assert!(!space.restore_range(0, 3 * region), "a clear scrubs what was withdrawn"); + assert_eq!((space.mapped_bitmap(), third), (0, 0)); + } + /// The low-512KB alias: bank 0's first 512KB visible at physical 0, the /// same pages as at LOMEM_BASE. Replaces `physical.rs`'s AliasBus. #[test] @@ -2005,6 +2118,35 @@ mod tests { assert_eq!(word(space.view_base(v), 0x200_3010), 0x9999_9999, "laid again with the bank's period"); } + /// A region withdrawn from the direct path leaves a view's bytes where + /// they are: a page the view holds in it, released while it is off the + /// path, shows the bank again, and the region returns to every sink. + #[test] + fn a_withdrawn_region_keeps_its_bytes_in_every_view() { + let (space, banks) = PpMemSpace::with_bank_sizes(&[256]).unwrap(); + let region = 64 * MB as u64; + for i in 0..4 { + space.map_range(0, i * region, i * region, region).unwrap(); + } + let (v, field) = view_of(&space); + banks[0].write32(region + 0x2040, 0x1234_5678); + assert_eq!(*field, 0b1111); + space.withdraw_range(region, region); + assert_eq!(*field, 0b1101, "the view's bitmap follows the withdrawal"); + assert_eq!(word(space.view_base(v), region + 0x2040), 0x1234_5678, "its bytes stay mapped"); + // A page held and released while the region is off the path lays the bank again. + assert_eq!(space.hold_line_page(v, region + 0x2000), Some(true)); + assert_eq!(*field, 0b1111, "a held page makes its region the view's"); + set_word(space.view_base(v), region + 0x2040, 0x7777_7777); + assert_eq!(banks[0].read32(region + 0x2040).data, 0x1234_5678); + assert!(space.release_line_page(v, region + 0x2000)); + assert_eq!(*field, 0b1101); + assert_eq!(word(space.view_base(v), region + 0x2040), 0x1234_5678); + assert!(space.restore_range(region, region)); + assert_eq!(*field, 0b1111); + assert_eq!(word(space.view_base(v), region + 0x2040), 0x1234_5678); + } + /// A page with no memory behind it cannot be held. One in a region the /// window maps only in part makes the region the view's, reading zeros /// where nothing is mapped, until the page goes. From bada675025769f9c1b0ce0bf342d82d902ccba54 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 10/22] bedrock: add the I/O, local-bus, network and crossbar blocks The Fuel's crossbar widgets reach memory and the processors through Bedrock's I/O block (the II); the local block (LB) holds the node and region registers, scratch words and the MicroLAN master; the network interface (NI) and crossbar block (XB) keep routing and error records. The guest programs all four during setup, and their error levels reach both PIs as bank 1 bits 55, 62 and 54. They join the chip as plain fields with no reference to each other and no closure. A block drives its error level under its own lock, so two stores' levels land in register order. What a store changes beyond its block comes back as an effect the chip acts on with the block's lock released: publish the reflected levels, run a block-transfer engine, or reset the node. A node reset, LB's soft-reset bit or NI's local-reset bit, leaves as a HardReset on the board's event channel; the chip calls nothing in the board. A widget's request enters through inbound_read and inbound_write: memory below address bit 47, and above it this node's window, where only a doubleword store to either PI's pending-modify register (0x90, 0x200090) is taken. A packet the PI's region mask refuses answers a bus error, so a sender that reissues a level knows it was never set. The II's widget and device permission words, PIO enables and translation entries are atomics stored under its lock, so PIO, inbound memory and an interrupt packet take no lock; an interrupt packet used to take about 28 between the II's register map, a reentrancy set, its hub handle and the PI. Widget 0, the crossbar itself, is admitted by its widget bit alone. The BTEs run synchronously when a start is written, against MD's DMA view, and report completion by a notification write or an interrupt packet to the PI the engine names, on this node. NI port status reads 1 on an uncabled one-node port: link up (bit 1) clear and bit 0 set. Reading 0 made the IRIX 6.5.30 bus-error handler print "Detected a link reset due to other partition reset" on every faulting widget probe. The value is inferred, not measured: bit 1 follows a real Tezro PROM reporting its link down, and bit 0 a real one-node Fuel taking a PIO bus error with no link message. --- src/dev/bedrock/io.rs | 626 +++++++++++++++++++++++++++++++++++++++ src/dev/bedrock/lb.rs | 333 +++++++++++++++++++++ src/dev/bedrock/mod.rs | 234 ++++++++++++++- src/dev/bedrock/ni.rs | 302 +++++++++++++++++++ src/dev/bedrock/pi.rs | 13 + src/dev/bedrock/tests.rs | 147 ++++++++- src/dev/bedrock/xb.rs | 253 ++++++++++++++++ 7 files changed, 1901 insertions(+), 7 deletions(-) create mode 100644 src/dev/bedrock/io.rs create mode 100644 src/dev/bedrock/lb.rs create mode 100644 src/dev/bedrock/ni.rs create mode 100644 src/dev/bedrock/xb.rs diff --git a/src/dev/bedrock/io.rs b/src/dev/bedrock/io.rs new file mode 100644 index 000000000..0f715fb07 --- /dev/null +++ b/src/dev/bedrock/io.rs @@ -0,0 +1,626 @@ +//! Bedrock's I/O block (the II) routes programmed PIO windows to crossbar +//! widgets, applies inbound permission checks to requests from widgets and +//! runs two block-transfer engines. Guests configure the widget masks before +//! touching devices; a BTE moves 128-byte lines and reports completion by a +//! memory write or an interrupt packet. +//! +//! The permission words and the translation entries every PIO and inbound +//! access checks are atomics, stored under the block's lock and read without +//! it. +//! +//! # What is approximated here +//! +//! BTE transfers and their completion run synchronously when a start is +//! written. Timing and unpublished register behavior are not modeled. +//! +//! The other register words sit in a sparse map keyed by offset, since the +//! 0x30010-byte block holds only a few dozen of them. +//! +//! Ported from hw/misc/sgi-bedrock-io.c in jamesbraid/qemu-sgi-origami. + +use super::reg::IIO_PROTECT_OVRRD_KEY; +use super::Effect; +use crate::traits::{BusDevice, BusRead64, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU64, Ordering::Relaxed}; +use std::sync::{Arc, OnceLock}; + +/// Length of the block-relative register aperture. +pub(super) const SIZE: u64 = 0x30010; + +pub mod reg { + pub const DEVICE_ID: u64 = 0x0; + pub const PROTECTION_KEY: u64 = 0x108; + pub const PIO_PERMISSION: u64 = 0x100; + pub const INBOUND_PERMISSION: u64 = 0x110; + pub const INBOUND_DEVICE_PERMISSION: u64 = 0x120; + pub const XIO_STATUS: u64 = 0x128; + pub const LINK_LOG: u64 = 0x130; + pub const LINK_STATUS: u64 = 0x138; + pub const BTE1_BASE: u64 = 0x10000; + pub const BTE_REGISTER_SIZE: u64 = 0x30; + pub const READ_ONLY: u64 = 0x8; + pub const BTE2_BASE: u64 = 0x20000; + pub const WIDGET_STATUS: u64 = 0x20; + pub const WIDGET_PERMISSION: u64 = 0x118; + pub const REGISTER_0150: u64 = 0x150; + pub const REGISTER_0158: u64 = 0x158; + pub const ITTE_BASE: u64 = 0x160; + pub const ITTE_LAST: u64 = 0x190; + pub const REGISTER_0198_BASE: u64 = 0x198; + pub const REGISTER_01D8_LAST: u64 = 0x1d8; + pub const READ_ONLY_ALIAS: u64 = 0x1f8; + pub const REGISTER_01E0: u64 = 0x1e0; + pub const REGISTER_01E8: u64 = 0x1e8; + pub const REGISTER_01F0: u64 = 0x1f0; + pub const REGISTER_0200: u64 = 0x200; + pub const REGISTER_03A8: u64 = 0x3a8; + pub const REGISTER_03B0: u64 = 0x3b0; + pub const REGISTER_03B8: u64 = 0x3b8; + pub const REGISTER_03C0: u64 = 0x3c0; + pub const RETAINED_WINDOW_BASE: u64 = 0x308; + pub const RETAINED_WINDOW_LAST: u64 = 0x348; + pub const RETAINED_CONFIG_BASE: u64 = 0x400; + pub const RETAINED_CONFIG_LAST: u64 = 0x5d8; + pub const GFX_MAP_BASE: u64 = 0x140; +} +pub mod bte_reg { + pub const STATUS: u64 = 0x0; + pub const SOURCE: u64 = 0x8; + pub const DESTINATION: u64 = 0x10; + pub const CONTROL: u64 = 0x18; + pub const NOTIFICATION: u64 = 0x20; + pub const INTERRUPT: u64 = 0x28; +} + +const ID: u64 = (2 << 28) | (0xc110 << 12) | (0x36 << 1); +const ADDR_MASK: u64 = 0xffffffff80; +pub(super) const BUSY: u64 = 1 << 20; +pub(super) const ERROR: u64 = 1 << 16; +/// XIO status bit 13: a crossbar peer answers the link. +const PEER: u64 = 1 << 13; +const ITTES: usize = 7; + +#[derive(Clone, Copy, Default)] +pub(super) struct Bte { + status: u64, + source: u64, + destination: u64, + control: u64, + notification: u64, + interrupt: u64, +} +struct Regs { + words: BTreeMap, + bte: [Bte; 2], +} +impl Default for Regs { + fn default() -> Self { + let mut words = BTreeMap::new(); + for (a, v) in [(0x20, 0xa), (0x128, 0x3fc0fff6403 & !PEER), (0x1e8, 0x111)] { + words.insert(a, v); + } + Self { words, bte: [Bte::default(); 2] } + } +} + +/// Where a finished BTE's interrupt packet goes: the PI it names, and the +/// pending-modify value. +pub(super) struct Completion { + pub pi: usize, + pub value: u64, +} + +pub(super) struct Io { + regs: Mutex, + /// PIO permission (0x100): bit 0 lets PIO through. + pio_permission: AtomicU64, + /// The widgets PIO may reach (0x110). + pio_widgets: AtomicU64, + /// The widgets whose requests are admitted (0x118). + widget_permission: AtomicU64, + /// Per-device admission for widgets 8..15, eight bits each (0x120). + device_permission: AtomicU64, + /// The large-window translation entries (0x160..0x190). + itte: [AtomicU64; ITTES], + widgets: [OnceLock>; 16], +} + +impl Io { + pub(super) fn new() -> Self { + let io = Self { + regs: Mutex::new(Regs::default()), + pio_permission: AtomicU64::new(0), + pio_widgets: AtomicU64::new(0), + widget_permission: AtomicU64::new(0), + device_permission: AtomicU64::new(0), + itte: Default::default(), + widgets: Default::default(), + }; + io.reset_permissions(); + io + } + fn reset_permissions(&self) { + for word in [&self.pio_permission, &self.pio_widgets, &self.widget_permission, &self.device_permission] { + word.store(!0, Relaxed); + } + for entry in &self.itte { + entry.store(0, Relaxed); + } + } + pub(super) fn locked(&self) -> bool { + self.regs.is_locked() + } + /// Hold the block's lock, for tests of the paths that must not take it. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + self.regs.lock() + } + /// Connect crossbar widget `widget`. Once per widget, before the machine runs. + pub(super) fn connect_widget(&self, widget: u8, dev: Arc) { + assert!(self.widgets[usize::from(widget)].set(dev).is_ok(), "Bedrock widget {widget} connected twice"); + *self.regs.lock().words.entry(reg::XIO_STATUS).or_default() |= PEER; + } + fn peer(&self) -> bool { + self.widgets.iter().any(|w| w.get().is_some()) + } + #[cfg(test)] + pub(super) fn disable_device(&self, widget: u8, device: u8) { + if (8..16).contains(&widget) && device < 8 { + let _r = self.regs.lock(); + self.device_permission.fetch_and(!(1 << ((widget - 8) * 8 + device)), Relaxed); + } + } + fn permitted(&self, widget: u8, device: u8) -> bool { + widget < 16 + && device < 8 + && self.widget_permission.load(Relaxed) & (1 << widget) != 0 + && (widget < 8 || self.device_permission.load(Relaxed) & (1 << ((widget - 8) * 8 + device)) != 0) + } + /// Whether a request from `widget`'s device `device` is admitted; + /// `None` asks for every master on the widget. Widget 0, the crossbar + /// itself, is admitted by its widget bit alone. + pub(super) fn admits(&self, widget: u8, device: Option) -> bool { + let allowed = match device { + Some(device) => self.permitted(widget, device), + None => (0..8).all(|d| self.permitted(widget, d)), + }; + if !allowed { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "Bedrock IO inbound access refused from widget {widget} device {device:?}"); + } + allowed + } + fn retained(a: u64) -> bool { + matches!( + a, + 0x208 | 0x210 | 0x218 | 0x220 | 0x228 | 0x300 | reg::GFX_MAP_BASE | 0x148 + | 0x388 | 0x390 | 0x398 | 0x3a0 | 0x600 | 0x608 | 0x610 | 0x30000 | 0x30008 + ) || (reg::RETAINED_WINDOW_BASE..reg::RETAINED_WINDOW_LAST).contains(&a) + || (reg::RETAINED_CONFIG_BASE..=reg::RETAINED_CONFIG_LAST).contains(&a) + } + fn mask(a: u64) -> Option { + Some(match a { + reg::WIDGET_STATUS => 0x7fe01f, + reg::PIO_PERMISSION | reg::INBOUND_PERMISSION | reg::WIDGET_PERMISSION | reg::INBOUND_DEVICE_PERMISSION | reg::REGISTER_0150 | reg::REGISTER_0158 => !0, + reg::XIO_STATUS => 0x3ff0fff7d3f, + reg::LINK_STATUS => 0x101ff7f, + reg::ITTE_BASE..=reg::ITTE_LAST => 0x1f1f, + reg::REGISTER_0198_BASE..=reg::REGISTER_01D8_LAST => !0, + reg::REGISTER_01E0 => 0x3ffffff, + reg::REGISTER_01E8 => 0x1111, + reg::REGISTER_01F0 => 0x0fffffff03ffffff, + reg::REGISTER_0200 => 0x10f, + reg::REGISTER_03A8 => 0x13ffffffff30f3, + reg::REGISTER_03B0 => 0x1ff0f, + reg::REGISTER_03B8 => 0xff, + reg::REGISTER_03C0 => 0xffffff, + _ if Self::retained(a) => !0, + _ => return None, + }) + } + /// The atomic that holds register `a`, for the words PIO and inbound + /// accesses check. + fn atomic(&self, a: u64) -> Option<&AtomicU64> { + match a { + reg::PIO_PERMISSION => Some(&self.pio_permission), + reg::INBOUND_PERMISSION => Some(&self.pio_widgets), + reg::WIDGET_PERMISSION => Some(&self.widget_permission), + reg::INBOUND_DEVICE_PERMISSION => Some(&self.device_permission), + reg::ITTE_BASE..=reg::ITTE_LAST if a % 8 == 0 => Some(&self.itte[((a - reg::ITTE_BASE) / 8) as usize]), + _ => None, + } + } + fn bte_at(a: u64) -> Option<(usize, u64)> { + if (reg::BTE1_BASE..reg::BTE1_BASE + reg::BTE_REGISTER_SIZE).contains(&a) { + Some((0, a - reg::BTE1_BASE)) + } else if (reg::BTE2_BASE..reg::BTE2_BASE + reg::BTE_REGISTER_SIZE).contains(&a) { + Some((1, a - reg::BTE2_BASE)) + } else { + None + } + } + /// The widget and widget address a PIO offset below 4 GiB reaches: the + /// small windows directly, the large ones through their translation entry. + fn target(&self, offset: u64) -> Option<(u8, u64)> { + if offset < 0x1000_0000 { + Some(((offset >> 24) as u8, offset & 0xff_ffff)) + } else if offset >= 0x2000_0000 { + let entry = ((offset >> 29) - 1) as usize; + let itte = self.itte.get(entry)?.load(Relaxed); + Some(( + ((itte >> 8) & 15) as u8, + ((itte & 31) << 29) | (offset & 0x1fff_ffff) | if itte & (1 << 12) != 0 { 1 << 47 } else { 0 }, + )) + } else { + None + } + } + /// A processor's uncached access to the I/O view below 4 GiB. + pub(super) fn pio(&self, offset: u64, size: u8, value: Option) -> BusRead64 { + let Some((widget, target)) = self.target(offset) else { + return BusRead64::err(); + }; + if self.pio_permission.load(Relaxed) & 1 == 0 || self.pio_widgets.load(Relaxed) & (1 << widget) == 0 { + return BusRead64::err(); + } + if !matches!(size, 1 | 2 | 4 | 8) { + return BusRead64::err(); + } + if widget == 1 && target == 0 { + return BusRead64::ok(ID); + } + match self.widgets[usize::from(widget)].get() { + Some(dev) => access(dev.as_ref(), target, size, value), + None => BusRead64::err(), + } + } + pub(super) fn read(&self, a: u64) -> u64 { + if let Some(word) = self.atomic(a) { + return word.load(Relaxed); + } + let mut r = self.regs.lock(); + if let Some((n, a)) = Self::bte_at(a) { + let b = &mut r.bte[n]; + return match a { + bte_reg::STATUS => b.status, + bte_reg::SOURCE => b.source, + bte_reg::DESTINATION => b.destination, + bte_reg::CONTROL => { + b.status &= !BUSY; + b.control + } + bte_reg::NOTIFICATION => b.notification, + bte_reg::INTERRUPT => b.interrupt, + _ => 0, + }; + } + match a { + reg::DEVICE_ID => ID, + reg::READ_ONLY | reg::READ_ONLY_ALIAS => 0, + reg::LINK_STATUS => r.words.get(&a).copied().unwrap_or(0) & !(1 << 28), + reg::LINK_LOG => r.words.get(&a).copied().unwrap_or(0), + _ if Self::mask(a).is_some() => r.words.get(&a).copied().unwrap_or(0), + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO unsupported read {a:#x}"); + 0 + } + } + } + /// A doubleword store at `a`: [`Effect::Bte`] when it starts an engine. + pub(super) fn write(&self, a: u64, v: u64) -> Effect { + let mut r = self.regs.lock(); + if let Some((n, offset)) = Self::bte_at(a) { + let b = &mut r.bte[n]; + match offset { + bte_reg::STATUS => b.status = v & (0xffff | BUSY), + bte_reg::SOURCE => b.source = v & ADDR_MASK, + bte_reg::DESTINATION => b.destination = v & ADDR_MASK, + bte_reg::CONTROL => { + b.control = v & 0x111; + if b.control & 0x100 != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE poison unsupported"); + } else if b.status & BUSY != 0 { + if b.status & 0xffff == 0 { + b.status = 0; + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO zero-count BTE start"); + } else { + return Effect::Bte(n); + } + } + } + bte_reg::NOTIFICATION => b.notification = v & ADDR_MASK, + bte_reg::INTERRUPT => b.interrupt = v & 0x7f01ff, + _ => {} + } + return Effect::None; + } + match a { + reg::PROTECTION_KEY => { + if v == IIO_PROTECT_OVRRD_KEY { + self.pio_permission.fetch_or(1, Relaxed); + } + } + reg::INBOUND_DEVICE_PERMISSION => { + self.device_permission.fetch_or(v, Relaxed); + } + _ if self.atomic(a).is_some() => self.atomic(a).unwrap().store(v & Self::mask(a).unwrap(), Relaxed), + reg::REGISTER_01E8 => { + *r.words.entry(a).or_default() |= v & Self::mask(a).unwrap(); + } + reg::XIO_STATUS => { + let v = v & Self::mask(a).unwrap() & if self.peer() { !0 } else { !PEER }; + r.words.insert(a, v); + } + reg::LINK_LOG => { + if v == 0 { + r.words.insert(a, 0); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO nonzero link-log write ignored"); + } + } + reg::READ_ONLY_ALIAS | reg::DEVICE_ID | reg::READ_ONLY => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO read-only or unsupported clear write {a:#x}"); + } + _ => { + if let Some(mask) = Self::mask(a) { + r.words.insert(a, v & mask); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO unsupported write {a:#x} = {v:#x}"); + } + } + } + Effect::None + } + /// Run engine `n` against `memory`, the node's DMA view, on a node + /// numbered `node`. The completion interrupt, if the engine sends one to + /// this node, is returned for the chip to deliver. + pub(super) fn run_bte(&self, n: usize, memory: &dyn BusDevice, node: u8) -> Option { + let mut b = self.regs.lock().bte[n]; + let count = b.status & 0xffff; + let bytes = count * 128; + let overflow = (b.control & 1 == 0 && b.source > (1 << 40) - bytes) || b.destination > (1 << 40) - bytes; + let mut remaining = count; + if overflow { + b.status = remaining | ERROR; + } else { + while remaining != 0 { + let mut line = [0u64; 16]; + let mut status = BUS_OK; + if b.control & 1 == 0 { + for (i, v) in line.iter_mut().enumerate() { + let r = memory.read64(b.source + i as u64 * 8); + status = r.status; + if status != BUS_OK { + break; + } + *v = r.data; + } + } + if status == BUS_OK { + for (i, v) in line.into_iter().enumerate() { + status = memory.write64(b.destination + i as u64 * 8, v); + if status != BUS_OK { + break; + } + } + } + if status != BUS_OK { + break; + } + remaining -= 1; + b.source = (b.source + 128) & ADDR_MASK; + b.destination = (b.destination + 128) & ADDR_MASK; + } + b.status = if remaining == 0 { 0 } else { remaining | ERROR }; + } + self.regs.lock().bte[n] = b; + if b.control & 0x10 != 0 { + if memory.write64(b.notification, b.status) != BUS_OK { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE notification write failed"); + } + return None; + } + if ((b.interrupt >> 1) & 255) as u8 != node { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE interrupt reaches no node"); + return None; + } + Some(Completion { pi: (b.interrupt & 1) as usize, value: 0x100 | ((b.interrupt >> 16) & 127) }) + } + pub(super) fn power_on(&self) { + let mut r = self.regs.lock(); + *r = Regs::default(); + if self.peer() { + *r.words.entry(reg::XIO_STATUS).or_default() |= PEER; + } + self.reset_permissions(); + } +} + +/// An access of `size` bytes to `dev`: a write when `value` is given. +pub(super) fn access(dev: &dyn BusDevice, a: u64, size: u8, value: Option) -> BusRead64 { + if let Some(v) = value { + let status = match size { + 1 => dev.write8(a, v as u8), + 2 => dev.write16(a, v as u16), + 4 => dev.write32(a, v as u32), + 8 => dev.write64(a, v), + _ => BUS_ERR, + }; + return BusRead64 { status, data: 0 }; + } + match size { + 1 => { + let r = dev.read8(a); + BusRead64 { status: r.status, data: r.data as u64 } + } + 2 => { + let r = dev.read16(a); + BusRead64 { status: r.status, data: r.data as u64 } + } + 4 => { + let r = dev.read32(a); + BusRead64 { status: r.status, data: r.data as u64 } + } + 8 => dev.read64(a), + _ => BusRead64::err(), + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-io-test.c in + // jamesbraid/qemu-sgi-origami: inbound-protection, bte and + // bte-error-termination. + use super::super::reg::{md, pi, IO_BASE, MD_BASE, PI1_BASE}; + use super::super::Rig; + use super::*; + use crate::traits::{BusRead64, BUS_BUSY}; + + /// 64 MiB at bank 0, enabled. + const BANK: u64 = 64 << 20; + fn rig() -> Rig { + let rig = Rig::new(); + rig.w(MD_BASE + md::CONFIG, 0x11); + rig + } + #[derive(Default)] + struct Memory(Mutex>); + impl BusDevice for Memory { + fn read64(&self, a: u64) -> BusRead64 { + BusRead64::ok(self.0.lock().get(&a).copied().unwrap_or(0)) + } + fn write64(&self, a: u64, v: u64) -> u32 { + self.0.lock().insert(a, v); + BUS_OK + } + } + struct Busy; + impl BusDevice for Busy { + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::busy() + } + fn write64(&self, _: u64, _: u64) -> u32 { + BUS_BUSY + } + } + fn launch(rig: &Rig, base: u64, count: u64, src: u64, dst: u64, notify: u64, control: u64) { + rig.w(IO_BASE + base, count | BUSY); + rig.w(IO_BASE + base + 8, src); + rig.w(IO_BASE + base + 0x10, dst); + rig.w(IO_BASE + base + 0x20, notify); + rig.w(IO_BASE + base + 0x18, control); + } + + /// The guest must enable widget and requester masks before PIO or inbound DMA succeeds. + #[test] + fn the_io_block_checks_pio_and_inbound_requester_permissions() { + { + let rig = rig(); + let pio = rig.chip.pio(); + let m = Arc::new(Memory::default()); + rig.chip.connect_widget(8, m.clone()); + rig.w(IO_BASE + reg::ITTE_BASE, 0x802); + assert_eq!(pio.write64(0x20000080, 0xabcd), BUS_OK); + assert_eq!(m.read64(0x40000080).data, 0xabcd); + assert_eq!(pio.read64(0x20000080).data, 0xabcd); + rig.w(IO_BASE + reg::INBOUND_PERMISSION, 0); + assert_eq!(pio.read64(0x20000080).status, BUS_ERR); + rig.w(IO_BASE + reg::INBOUND_PERMISSION, !0); + rig.w(IO_BASE + reg::PIO_PERMISSION, 0); + assert_eq!(pio.read64(0x20000080).status, BUS_ERR); + rig.w(IO_BASE + reg::PROTECTION_KEY, IIO_PROTECT_OVRRD_KEY); + assert_eq!(pio.read64(0x20000080).status, BUS_OK); + assert_eq!(pio.read64(0x10000000).status, BUS_ERR); + assert_ne!(rig.r(IO_BASE + reg::XIO_STATUS) & PEER, 0, "a widget answers the link"); + rig.chip.io.power_on(); + assert_eq!(rig.r(IO_BASE + reg::PIO_PERMISSION), !0); + assert_eq!(rig.r(IO_BASE + reg::REGISTER_01E8), 0x111); + assert_ne!(rig.r(IO_BASE + reg::XIO_STATUS) & PEER, 0, "the peer survives reset"); + } + { + let rig = rig(); + let chip = &rig.chip; + assert_eq!(chip.inbound_write(14, Some(2), 0x80, 8, 42), BUS_OK); + assert_eq!(chip.dma_view().read64(0x80).data, 42); + chip.disable_device(14, 2); + assert_eq!(chip.inbound_read(14, Some(2), 0x80, 8).status, BUS_ERR); + assert_eq!(chip.inbound_read(14, None, 0x80, 8).status, BUS_ERR); + assert_eq!(chip.inbound_read(14, Some(1), 0x80, 8).status, BUS_OK); + // An interrupt packet to PI0's pending-modify register sets bank 0 bit 35. + let a = (1 << 47) | 0x1800090; + assert_eq!(chip.inbound_write(15, Some(0), a, 8, 0x123), BUS_OK); + assert_eq!(rig.r(pi::PENDING_STATUS_A), 1 << 35); + assert_eq!(chip.inbound_write(0, None, a + PI1_BASE, 8, 0x101), BUS_OK, "widget 0 is admitted"); + assert_eq!(rig.r(PI1_BASE + pi::PENDING_STATUS_A), 1 << 1); + assert_eq!(chip.inbound_read(15, Some(0), a, 8).status, BUS_ERR, "only stores reach a PI"); + assert_eq!(chip.inbound_write(15, Some(0), a, 4, 0x123), BUS_ERR, "only doublewords"); + assert_eq!(chip.inbound_read(15, Some(0), a | (1 << 39), 8).status, BUS_ERR, "another node"); + assert_eq!(chip.inbound_read(15, Some(0), 1 << 48, 8).status, BUS_ERR); + assert_eq!(chip.inbound_read(15, Some(0), a + 0x800000, 8).status, BUS_ERR); + assert_eq!(chip.inbound_read(15, Some(0), BANK, 8).status, BUS_ERR, "beyond the bank"); + } + } + /// The guest observes BTE line copies and zero-fill through its completion registers. + #[test] + fn the_bte_copies_and_zeroes_lines_before_completion() { + let rig = rig(); + let m = rig.chip.dma_view(); + for i in 0..32 { + m.write64(0x1000 + i * 8, 0xfeed0000 | i); + } + m.write64(0x3000, 0xdead); + launch(&rig, reg::BTE1_BASE, 2, 0x1000, 0x2000, 0x3000, 0x10); + for i in 0..32 { + assert_eq!(m.read64(0x2000 + i * 8).data, 0xfeed0000 | i); + } + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 0); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::SOURCE), 0x1100); + assert_eq!(m.read64(0x3000).data, 0, "the notification holds the final status"); + rig.w(IO_BASE + reg::BTE2_BASE + bte_reg::INTERRUPT, 53 << 16); + launch(&rig, reg::BTE2_BASE, 1, 0, 0x2000, 0, 1); + for i in 0..16 { + assert_eq!(m.read64(0x2000 + i * 8).data, 0); + } + assert_eq!(rig.r(pi::PENDING_STATUS_A), 1 << 53, "the completion interrupt"); + } + /// A failed BTE line stops the transfer and reports the failing source and destination. + #[test] + fn the_bte_reports_transfer_errors_and_stops_after_a_failed_line() { + // The second line runs off the end of the bank, on the read or the write side. + for (src, dst) in [(BANK - 0x80, 0x2000), (0x1000, BANK - 0x80)] { + let rig = rig(); + launch(&rig, reg::BTE1_BASE, 3, src, dst, 0x3000, 0x10); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 2 | ERROR); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::SOURCE), src + 0x80); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::DESTINATION), dst + 0x80); + assert_eq!(rig.chip.dma_view().read64(0x3000).data, 2 | ERROR); + } + { + let rig = rig(); + launch(&rig, reg::BTE1_BASE, 1, 0x1000, 0x2000, BANK, 0x10); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 0, "a failed notification still completes"); + launch(&rig, reg::BTE1_BASE, 2, 0xffffffff80, 0x2000, 0x3000, 0x10); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 2 | ERROR); + } + { + let rig = rig(); + launch(&rig, reg::BTE1_BASE, 1, 0, 0, 0, 0x100); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 1 | BUSY); + rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::CONTROL); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 1); + launch(&rig, reg::BTE1_BASE, 0, 0, 0, 0, 0x10); + assert_eq!(rig.r(IO_BASE + reg::BTE1_BASE + bte_reg::STATUS), 0); + } + } + /// A widget that is busy answers busy through PIO, so the CPU retries. + #[test] + fn a_busy_widget_answers_busy_through_pio() { + let rig = rig(); + rig.chip.connect_widget(8, Arc::new(Busy)); + let pio = rig.chip.pio(); + assert_eq!(pio.read64(0x8000000).status, BUS_BUSY); + assert_eq!(pio.write64(0x8000000, 42), BUS_BUSY); + } +} diff --git a/src/dev/bedrock/lb.rs b/src/dev/bedrock/lb.rs new file mode 100644 index 000000000..39d7dcee0 --- /dev/null +++ b/src/dev/bedrock/lb.rs @@ -0,0 +1,333 @@ +//! Bedrock's local block (LB) provides the node and region registers, atomic +//! scratch aliases, error and vector status, and the MicroLAN master used by +//! board identity parts. The guest polls the MicroLAN completion bit after +//! issuing a slot. +//! +//! # What is approximated here +//! +//! The slot is decoded immediately and completes after one microsecond of +//! machine time; no 1-wire part answers, so a reset pulse and a short write +//! slot read back as presence-free and as a one. Permission registers retain +//! guest writes, but the register path carries no requester identity, so it +//! cannot enforce those permissions. +//! +//! The plain register words sit in a sparse map keyed by offset, since the +//! 0xc040-byte block holds only a few dozen of them. +//! +//! Ported from hw/misc/sgi-bedrock-lb.c in jamesbraid/qemu-sgi-origami. + +use super::pi::Levels; +use super::reg::IIO_PROTECT_OVRRD_KEY; +use super::{Effect, NodeReset}; +use crate::dev::w1; +use parking_lot::Mutex; +use std::collections::BTreeMap; +use std::sync::atomic::{AtomicU8, Ordering}; + +/// Length of the block-relative register aperture. +pub(super) const SIZE: u64 = 0xc040; + +pub mod reg { + pub const NODE_REGION: u64 = 0x0; + pub const PIO_PERMISSION: u64 = 0x4000; + pub const PIO_PERMISSION_KEY: u64 = 0x4008; + pub const INBOUND_PERMISSION: u64 = 0x4010; + pub const SOFT_RESET: u64 = 0x4018; + pub const MICRO_LAN: u64 = 0x4030; + pub const ERROR_STATUS: u64 = 0x4040; + pub const ERROR_CLEAR: u64 = 0x4048; + pub const PIO_REGION_MASK: u64 = 0x4028; + pub const NODE_REGION_CONFIG: u64 = 0x4050; + pub const NODE_REGION_MASK: u64 = 0x4058; + pub const PIO_CONFIG: u64 = 0x4100; + pub const INBOUND_CONFIG: u64 = 0x4200; + pub const INBOUND_MASK: u64 = 0x4208; + pub const VECTOR_REQUEST: u64 = 0xc000; + pub const VECTOR_STATUS: u64 = 0xc020; + pub const VECTOR_STATUS_ALIAS: u64 = 0xc038; + pub const PIO_WINDOW: u64 = 0x8000; + pub const PIO_WINDOW_END: u64 = 0x8020; + pub const ATOMIC_SCRATCH: u64 = 0x8040; + pub const ATOMIC_SCRATCH_END: u64 = 0x8050; + pub const ATOMIC_READ0: u64 = 0x8058; + pub const ATOMIC_READ1: u64 = 0x8060; + pub const REGISTER_4020: u64 = 0x4020; + pub const REGISTER_4060: u64 = 0x4060; + pub const REGISTER_C008: u64 = 0xc008; + pub const REGISTER_C010: u64 = 0xc010; + pub const REGISTER_C028: u64 = 0xc028; + pub const REGISTER_C030: u64 = 0xc030; +} + +const ID: u64 = (0x36 << 1) | (0x3012 << 12) | (2 << 28); +const VECTOR_MASK: u64 = 3 | (0x1fffff << 3) | (0xff << 32) | (0x7ff << 40); +struct Regs { + words: BTreeMap, + scratch: [u64; 5], + node: u8, + region: u8, + deadline: Option, + sample: bool, +} +impl Default for Regs { + fn default() -> Self { + let mut words = BTreeMap::new(); + words.insert(reg::PIO_PERMISSION, !0); + words.insert(reg::INBOUND_PERMISSION, !0); + Self { words, scratch: [0; 5], node: 0, region: 0, deadline: None, sample: true } + } +} +impl Regs { + fn error_level(&self) -> bool { + self.words.get(®::ERROR_STATUS).copied().unwrap_or(0) != 0 + } +} + +pub(super) struct Lb { + regs: Mutex, +} + +impl Lb { + pub(super) fn new() -> Self { + Self { regs: Mutex::new(Regs::default()) } + } + pub(super) fn locked(&self) -> bool { + self.regs.is_locked() + } + /// Hold the block's lock, for tests of the paths that must not take it. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + self.regs.lock() + } + fn complete(r: &mut Regs, now: u64) { + if r.deadline.is_some_and(|d| now >= d) { + *r.words.entry(reg::MICRO_LAN).or_default() |= 2 | u64::from(r.sample); + r.deadline = None; + } + } + fn mask(a: u64) -> Option { + Some(match a { + reg::PIO_PERMISSION | reg::INBOUND_PERMISSION | reg::REGISTER_4020 | reg::REGISTER_4060 | reg::REGISTER_C008 | reg::REGISTER_C010 | reg::REGISTER_C028 | reg::REGISTER_C030 => !0, + reg::PIO_REGION_MASK => 0x80ff80ff80ff80ff, + reg::ERROR_STATUS => 0x7ff, + reg::NODE_REGION_CONFIG => 0x7f | (0x7ff << 8) | (0x7ff << 20) | (0xf << 32) | (1 << 40) | (1 << 44), + reg::NODE_REGION_MASK => (1 << 38) - 1, + reg::PIO_CONFIG => 0x3f, + reg::INBOUND_CONFIG => 1 | (0x3ff << 4) | (0x3ff << 16) | (1 << 28) | (1 << 32), + reg::INBOUND_MASK => { + 0x1f | (0xfff << 8) | (1 << 20) | (0x3ff << 24) | (0x3ff << 36) | (0x3ff << 48) + } + reg::VECTOR_REQUEST => VECTOR_MASK, + _ => return None, + }) + } + /// The block's reset, under its lock: the scratch words 1 and 2 and the + /// error status survive it. The node number returns to 0. + fn reset(r: &mut Regs, node: &AtomicU8) { + let survivors = [r.scratch[1], r.scratch[2], r.words.get(®::ERROR_STATUS).copied().unwrap_or(0)]; + *r = Regs::default(); + r.scratch[1] = survivors[0]; + r.scratch[2] = survivors[1]; + r.words.insert(reg::ERROR_STATUS, survivors[2]); + node.store(0, Ordering::Release); + } + pub(super) fn power_on(&self, node: &AtomicU8) { + Self::reset(&mut self.regs.lock(), node); + } + pub(super) fn read(&self, a: u64, now: u64) -> u64 { + let mut r = self.regs.lock(); + Self::complete(&mut r, now); + match a { + reg::NODE_REGION => ID | ((r.node as u64) << 32) | ((r.region as u64) << 46), + reg::PIO_WINDOW..=reg::PIO_WINDOW_END => r.scratch[(a - reg::PIO_WINDOW) as usize / 8], + reg::ATOMIC_READ0 | reg::ATOMIC_READ1 => { + let n = 3 + (a - reg::ATOMIC_READ0) as usize / 8; + let v = r.scratch[n]; + r.scratch[n] = 1; + v + } + reg::VECTOR_STATUS_ALIAS => r.words.remove(®::VECTOR_STATUS).unwrap_or(0), + reg::VECTOR_STATUS | reg::MICRO_LAN => r.words.get(&a).copied().unwrap_or(0), + _ if Self::mask(a).is_some() => r.words.get(&a).copied().unwrap_or(0), + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB unsupported read {a:#x}"); + 0 + } + } + } + /// A doubleword store at `a`. The error level reaches `levels` and the + /// node number `node` under the block's lock, so concurrent stores + /// publish in register order. A soft reset resets the block and asks + /// for the node's. + pub(super) fn write(&self, a: u64, v: u64, now: u64, levels: &Levels, node: &AtomicU8) -> Effect { + let mut r = self.regs.lock(); + Self::complete(&mut r, now); + match a { + reg::NODE_REGION => { + let region = ((v >> 46) & 3) as u8; + if region == 3 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB invalid region size"); + } else { + r.node = (v >> 32) as u8; + r.region = region; + node.store(r.node, Ordering::Release); + } + } + reg::PIO_WINDOW..=reg::PIO_WINDOW_END => { + let n = (a - reg::PIO_WINDOW) as usize / 8; + r.scratch[n] = if n < 3 { v } else { v & 1 }; + } + reg::ATOMIC_SCRATCH..=reg::ATOMIC_SCRATCH_END => { + let n = (a - reg::ATOMIC_SCRATCH) as usize / 8; + if r.scratch[n] == 0 { + r.scratch[n] = v; + } + } + reg::PIO_PERMISSION_KEY => { + if v == IIO_PROTECT_OVRRD_KEY { + *r.words.entry(reg::PIO_PERMISSION).or_default() |= 1; + } + } + reg::SOFT_RESET => { + if v & 1 != 0 { + Self::reset(&mut r, node); + return Effect::Reset(NodeReset::LocalBlock); + } + } + reg::MICRO_LAN => { + if r.deadline.is_some() { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB MicroLAN write while slot pending discarded"); + } else { + r.words.insert(a, v & ((1 << 34) - 4)); + let (pulse, _) = w1::slot::fields(v); + r.sample = pulse >= w1::slot::RESET_PULSE || pulse < w1::slot::ONE_PULSE; + r.deadline = Some(now.saturating_add(1000)); + } + } + reg::ERROR_STATUS => { + *r.words.entry(a).or_default() |= v & 0x7ff; + } + reg::ERROR_CLEAR => { + *r.words.entry(reg::ERROR_STATUS).or_default() &= !(v & 0x7ff); + } + reg::VECTOR_REQUEST => { + r.words.insert(a, v & VECTOR_MASK); + if r.words.get(®::VECTOR_STATUS).copied().unwrap_or(0) & (1 << 63) != 0 { + *r.words.entry(reg::VECTOR_STATUS).or_default() |= 1 << 62; + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB vector launch reaches no peer; no reply latched"); + } + } + reg::VECTOR_STATUS | reg::VECTOR_STATUS_ALIAS => { + if v == 0 { + r.words.insert(reg::VECTOR_STATUS, 0); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB nonzero vector status write ignored"); + } + } + _ => { + if let Some(mask) = Self::mask(a) { + r.words.insert(a, v & mask); + if (a == reg::PIO_PERMISSION || a == reg::INBOUND_PERMISSION) && v != !0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB permission enforcement requires requester context"); + } + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock LB unsupported write {a:#x} = {v:#x}"); + } + } + } + if levels.set(Levels::LB_ERROR, r.error_level()) { Effect::Reflect } else { Effect::None } + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-lb-test.c in + // jamesbraid/qemu-sgi-origami: identity-permissions, scratch-reset and + // microlan-slot. + use super::super::reg::{pi, LB_BASE, PI1_BASE}; + use super::super::Rig; + use super::*; + use crate::traits::{BusDevice, BUS_ERR}; + + fn error_reflected(rig: &Rig) -> [bool; 2] { + [0, PI1_BASE].map(|base| rig.r(base + pi::PENDING_STATUS_B) & (1 << 55) != 0) + } + + /// The guest configures node permissions and expects selected scratch words to survive reset. + #[test] + fn permission_writes_latch_errors_and_reset_retains_scratch() { + { + let rig = Rig::new(); + let node = || rig.chip.node.load(Ordering::SeqCst); + rig.w(LB_BASE + reg::NODE_REGION, (0x5a << 32) | (2 << 46)); + assert_eq!(node(), 0x5a); + rig.w(LB_BASE + reg::NODE_REGION, 3 << 46); + assert_eq!(node(), 0x5a); + rig.w(LB_BASE + reg::PIO_PERMISSION, 0); + rig.w(LB_BASE + reg::PIO_PERMISSION_KEY, 0); + assert_eq!(rig.r(LB_BASE + reg::PIO_PERMISSION), 0); + rig.w(LB_BASE + reg::PIO_PERMISSION_KEY, IIO_PROTECT_OVRRD_KEY); + assert_eq!(rig.r(LB_BASE + reg::PIO_PERMISSION), 1); + rig.w(LB_BASE + reg::PIO_WINDOW + 8, 42); + rig.w(LB_BASE + reg::ERROR_STATUS, !0); + assert_eq!(rig.r(LB_BASE + reg::ERROR_STATUS), 0x7ff); + assert_eq!(error_reflected(&rig), [true; 2], "bank 1 bit 55 on both interfaces"); + rig.chip.lb.power_on(&rig.chip.node); + assert_eq!(rig.r(LB_BASE + reg::PIO_WINDOW + 8), 42); + assert_eq!(error_reflected(&rig), [true; 2], "the error status survives reset"); + rig.w(LB_BASE + reg::ERROR_CLEAR, 0x7ff); + assert_eq!(error_reflected(&rig), [false; 2]); + assert_eq!(node(), 0); + } + { + let rig = Rig::new(); + rig.w(LB_BASE + reg::ATOMIC_SCRATCH, 42); + rig.w(LB_BASE + reg::ATOMIC_SCRATCH, 43); + assert_eq!(rig.r(LB_BASE + reg::PIO_WINDOW), 42); + rig.w(LB_BASE + reg::REGISTER_C030, 0xdead); + rig.w(LB_BASE + reg::VECTOR_REQUEST, !0); + assert_eq!(rig.r(LB_BASE + reg::VECTOR_STATUS), 0); + assert_eq!(rig.r(LB_BASE + reg::REGISTER_C030), 0xdead); + assert_eq!(rig.regs.read32(LB_BASE + reg::PIO_WINDOW).status, BUS_ERR); + assert_eq!(rig.regs.read64(LB_BASE + 1).status, BUS_ERR); + } + } + /// The guest polls MicroLAN completion and cannot replace a slot that is still pending. + #[test] + fn microlan_slots_finish_after_one_microsecond_and_reject_overlap() { + let rig = Rig::new(); + let word = (0x66 << 27) | (60 << 10) | (0x41 << 2) | 3; + rig.w(LB_BASE + reg::MICRO_LAN, word); + assert_eq!(rig.r(LB_BASE + reg::MICRO_LAN), word & !3); + rig.w(LB_BASE + reg::MICRO_LAN, 8 << 10); + assert_eq!(rig.r(LB_BASE + reg::MICRO_LAN), word & !3); + rig.set_time(1000); + assert_eq!(rig.r(LB_BASE + reg::MICRO_LAN), (word & !3) | 2); + rig.w(LB_BASE + reg::MICRO_LAN, 8 << 10); + rig.set_time(2000); + assert_eq!(rig.r(LB_BASE + reg::MICRO_LAN), (8 << 10) | 3); + rig.w(LB_BASE + reg::MICRO_LAN, 60 << 10); + rig.chip.lb.power_on(&rig.chip.node); + rig.set_time(3000); + assert_eq!(rig.r(LB_BASE + reg::MICRO_LAN), 0); + } + /// The soft-reset bit resets the block, as a power-on does, and asks the + /// board for a node reset once; the other bits do neither. + #[test] + fn the_soft_reset_bit_resets_the_block_and_asks_for_a_node_reset() { + let rig = Rig::new(); + rig.w(LB_BASE + reg::NODE_REGION, 5 << 32); + rig.w(LB_BASE + reg::PIO_WINDOW + 8, 42); + rig.w(LB_BASE + reg::REGISTER_C030, 0xdead); + rig.w(LB_BASE + reg::SOFT_RESET, 2); + assert!(rig.events.lock().try_recv().is_err()); + assert_eq!(rig.r(LB_BASE + reg::REGISTER_C030), 0xdead); + rig.w(LB_BASE + reg::SOFT_RESET, 1); + assert!(matches!(rig.events.lock().try_recv(), Ok(crate::traits::MachineEvent::HardReset))); + assert!(rig.events.lock().try_recv().is_err(), "one request"); + assert_eq!(rig.r(LB_BASE + reg::REGISTER_C030), 0); + assert_eq!(rig.r(LB_BASE + reg::PIO_WINDOW + 8), 42); + assert_eq!(rig.chip.node.load(Ordering::SeqCst), 0); + } +} diff --git a/src/dev/bedrock/mod.rs b/src/dev/bedrock/mod.rs index b83f4c41c..7f6deafb1 100644 --- a/src/dev/bedrock/mod.rs +++ b/src/dev/bedrock/mod.rs @@ -8,6 +8,13 @@ //! and gives each processor's interrupt word to [`Bedrock::set_cpu_interrupts`]. //! The units are plain fields; each locks only its own state. //! +//! A unit holds no reference to another and no closure. What a unit's +//! access changes beyond the unit comes back as an `Effect`, which the chip +//! acts on once the unit's lock is released. The error levels LB, NI and XB +//! drive are PI level bits that each writer sets under its own lock. The +//! chip calls nothing in the board: a node reset leaves as a +//! [`MachineEvent::HardReset`] on the board's event channel. +//! //! # Locks //! //! - PI: the interrupt inputs (pending banks, masks, requests, timer and @@ -16,12 +23,20 @@ //! graphics registers; `timer` the comparators and the host timer. //! - MD: the cell register is an atomic and ordinary memory takes no lock; //! `state` and `remapping` as the controller's notes say. +//! - II: the permission words and translation entries are atomics, so PIO, +//! inbound memory and an inbound interrupt packet take no lock; `regs` +//! holds the rest. +//! - LB, NI and XB: `regs` each, under which a block drives its error level. //! //! Ported from hw/misc/sgi-bedrock*.c in jamesbraid/qemu-sgi-origami. +mod io; +mod lb; mod md; mod mem; +mod ni; mod pi; +mod xb; #[cfg(test)] mod tests; @@ -31,10 +46,12 @@ pub use pi::{Population, Refused}; use crate::dev::heart::Clock; use crate::hptimer::{TimerId, TimerManager, TimerReturn}; use crate::irq::{InterruptController, IrqLine}; -use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Device, Resettable, BUS_ERR, BUS_OK}; +use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Device, MachineEvent, Resettable, BUS_ERR, BUS_OK}; +use io::Io; use md::{Md, MemoryError, View}; use pi::{Levels, Pi}; -use std::sync::atomic::{AtomicU64, Ordering}; +use std::sync::atomic::{AtomicU64, AtomicU8, Ordering}; +use std::sync::mpsc::Sender; use std::sync::{Arc, OnceLock, Weak}; use std::time::Duration; @@ -56,12 +73,18 @@ pub mod reg { /// `IIO_PROTECT_OVRRD_KEY` as this value (lines 31, 47). This establishes /// the literal's provenance only; it does not establish a register mapping. pub const IIO_PROTECT_OVRRD_KEY: u64 = 0x53474972756c6573; + pub use super::io::{bte_reg as bte, reg as io}; + pub use super::lb::reg as lb; pub use super::md::reg as md; + pub use super::ni::reg as ni; pub use super::pi::reg as pi; + pub use super::xb::reg as xb; } /// What a board gives its Bedrock. Fixed for the chip's life. pub struct BedrockConfig<'a> { + /// The board's name in log lines. + pub name: &'static str, /// Host nanoseconds: the PIs' counter and timers run on it, and the /// other units' clocked work. pub clock: Clock, @@ -70,6 +93,9 @@ pub struct BedrockConfig<'a> { pub processors: Population, /// The memory banks in MiB, locational bank 0 upward. pub banks_mib: &'a [u32], + /// Where a node reset goes: LB's soft reset and NI's local reset each + /// send [`MachineEvent::HardReset`]. + pub events: Option>, } /// The two XIO register forms. `Local` answers the requester's own PI at @@ -81,6 +107,28 @@ pub enum Form { Local, } +/// What asked for a node reset. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +enum NodeReset { + /// LB's soft-reset bit. + LocalBlock, + /// NI's local-reset bit. + Network, +} + +/// What a unit's access changed outside the unit, returned with the unit's +/// lock released for [`Bedrock::apply`]. +#[must_use] +enum Effect { + None, + /// A bit of the chip's levels changed: both PIs publish. + Reflect, + /// Run block-transfer engine `n` now. + Bte(usize), + /// The unit has reset itself; the node resets. + Reset(NodeReset), +} + /// A PI's host timer: the manager it runs on and the chip it fires into. struct HostTimer<'a> { tm: &'a TimerManager, @@ -100,30 +148,46 @@ impl HostTimer<'_> { } pub struct Bedrock { + name: &'static str, clock: Clock, processors: Population, pi: [Pi; 2], md: Md, + io: Io, + lb: lb::Lb, + ni: ni::Ni, + xb: xb::Xb, levels: Levels, + /// The node number LB's region register names; inbound packets and the + /// BTEs' interrupts check it. Set under LB's lock. + node: AtomicU8, /// MD's DMA view: the BTEs and inbound memory requests use it. dma: Arc, timers: OnceLock>, + events: Option>, weak: Weak, } impl Bedrock { pub fn new(config: BedrockConfig<'_>) -> Result, String> { - let BedrockConfig { clock, processors, banks_mib } = config; + let BedrockConfig { name, clock, processors, banks_mib, events } = config; let memory = mem::Memory::new(banks_mib)?; let now = clock(); let chip = Arc::new_cyclic(|weak: &Weak| { let md = Md::new(memory, clock.clone(), weak.clone()); Self { + name, pi: [Pi::new(0, processors, now), Pi::new(1, processors, now)], dma: md.dma_view(), md, + io: Io::new(), + lb: lb::Lb::new(), + ni: ni::Ni::new(), + xb: xb::Xb::new(), levels: Levels::default(), + node: AtomicU8::new(0), timers: OnceLock::new(), + events, clock, processors, weak: weak.clone(), @@ -141,6 +205,10 @@ impl Bedrock { register_mutex_fn("bedrock-pi1::timer", self, |c| c.pi[1].timer_locked()); register_mutex_fn("bedrock-md::state", self, |c| c.md.state_locked()); register_mutex_fn("bedrock-md::remapping", self, |c| c.md.remapping_locked()); + register_mutex_fn("bedrock-io", self, |c| c.io.locked()); + register_mutex_fn("bedrock-lb", self, |c| c.lb.locked()); + register_mutex_fn("bedrock-ni", self, |c| c.ni.locked()); + register_mutex_fn("bedrock-xb", self, |c| c.xb.locked()); } // The faces a board maps into its processor bus. Each is its own type, @@ -154,6 +222,11 @@ impl Bedrock { Arc::new(Registers::(self.clone())) } } + /// The II's PIO face: the processor's uncached I/O view below 4 GiB, + /// decoded to a widget. + pub fn pio(self: &Arc) -> Arc { + Arc::new(Pio(self.clone())) + } /// Memory without the checks MD's views make, for the processor's /// cached view and DMA: Physical's entries. pub fn dma_view(&self) -> Arc { @@ -203,6 +276,62 @@ impl Bedrock { self.md.cell_permits() } + // Widgets. + + /// Connect crossbar widget `widget`, once, before the machine runs. + pub fn connect_widget(&self, widget: u8, dev: Arc) { + self.io.connect_widget(widget, dev); + } + /// A read arriving from widget `source`'s device `device` (`None`: every + /// master on the widget must be admitted): memory below bit 47. + pub fn inbound_read(&self, source: u8, device: Option, address: u64, size: u8) -> BusRead64 { + self.inbound(source, device, address, size, None) + } + /// A write arriving from widget `source`: memory below bit 47, or an + /// interrupt packet to a PI's pending-modify register in this node's + /// window. A packet the PI refuses answers a bus error. + pub fn inbound_write(&self, source: u8, device: Option, address: u64, size: u8, value: u64) -> u32 { + self.inbound(source, device, address, size, Some(value)).status + } + fn inbound(&self, widget: u8, device: Option, address: u64, size: u8, value: Option) -> BusRead64 { + if address >= (1 << 48) || u64::from(size) > (1 << 48) - address || !self.io.admits(widget, device) { + return BusRead64::err(); + } + if address & (1 << 47) == 0 { + return io::access(self.dma.as_ref(), address, size, value); + } + let node = ((address >> 39) & 255) as u8; + let window = address & ((1 << 39) - 1); + if node != self.node.load(Ordering::Acquire) || window & !0x7f_ffff != 0x180_0000 { + return BusRead64::err(); + } + // XIO register packets reach only the two PIs' pending-modify registers. + match (window & 0x7f_ffff, size, value) { + (offset @ (0x90 | 0x20_0090), 8, Some(value)) if self.interrupt(usize::from(offset >= reg::PI1_BASE), value) => { + BusRead64 { status: BUS_OK, data: 0 } + } + _ => BusRead64::err(), + } + } + /// An interrupt packet to PI `pi`'s pending-modify register; whether the + /// PI took it. + fn interrupt(&self, pi: usize, value: u64) -> bool { + let taken = self.pi[pi].io_modify(value, 0); + if taken { + self.pi[pi].publish(&self.levels); + } + taken + } + /// Widget `widget`'s DMA as a bus device: crosstalk addresses, with the + /// widget's inbound permissions applied. + pub fn widget_dma(self: &Arc, widget: u8) -> Arc { + Arc::new(WidgetDma { chip: self.weak.clone(), widget }) + } + #[cfg(test)] + pub(crate) fn disable_device(&self, widget: u8, device: u8) { + self.io.disable_device(widget, device); + } + // Processors. /// Where CPU `cpu`'s IP2..IP6 go: Cause bits 10-14 of its interrupt word @@ -268,6 +397,27 @@ impl Bedrock { } result } + fn apply(&self, effect: Effect) { + match effect { + Effect::None => {} + Effect::Reflect => self.publish_all(), + Effect::Bte(n) => { + let node = self.node.load(Ordering::Acquire); + if let Some(done) = self.io.run_bte(n, self.dma.as_ref(), node) { + if !self.interrupt(done.pi, done.value) { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock IO BTE completion interrupt failed"); + } + } + } + Effect::Reset(cause) => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "{}: {cause:?} reset from CPU {:?}; restarting the node", + self.name, crate::cpu::smp::on_cpu_thread()); + if let Some(events) = &self.events { + let _ = events.send(MachineEvent::HardReset); + } + } + } + } /// The PI a local-form access at offset 0 reaches: the requester's own. /// A reader that is no processor reaches PI0. @@ -287,6 +437,7 @@ impl Bedrock { match offset { reg::PI0_BASE..reg::PI1_BASE => self.pi_access(pi0, offset, size, value), reg::PI1_BASE..reg::IO_BASE => self.pi_access(1, offset - reg::PI1_BASE, size, value), + reg::IO_BASE..reg::MD_BASE => self.block_access(offset, size, value), reg::MD_BASE..reg::WINDOW => { let result = self.md_access(View::Registers, offset - reg::MD_BASE, u64::from(size), value); result.map_or(BusRead64::err(), BusRead64::ok) @@ -323,6 +474,33 @@ impl Bedrock { } } } + /// An access to the II, LB, NI or XB at `offset` in the window: aligned + /// doublewords only. + fn block_access(&self, offset: u64, size: u8, value: Option) -> BusRead64 { + let (base, end) = match offset { + reg::IO_BASE..reg::LB_BASE => (reg::IO_BASE, io::SIZE), + reg::LB_BASE..reg::NI_BASE => (reg::LB_BASE, lb::SIZE), + reg::NI_BASE..reg::XB_BASE => (reg::NI_BASE, ni::SIZE), + _ => (reg::XB_BASE, xb::SIZE), + }; + let a = offset - base; + if size != 8 || a & 7 != 0 || a >= end { + return BusRead64::err(); + } + let now = (self.clock)(); + let (data, effect) = match (base, value) { + (reg::IO_BASE, None) => (self.io.read(a), Effect::None), + (reg::IO_BASE, Some(v)) => (0, self.io.write(a, v)), + (reg::LB_BASE, None) => (self.lb.read(a, now), Effect::None), + (reg::LB_BASE, Some(v)) => (0, self.lb.write(a, v, now, &self.levels, &self.node)), + (reg::NI_BASE, None) => self.ni.read(a, &self.levels), + (reg::NI_BASE, Some(v)) => (0, self.ni.write(a, v, &self.levels)), + (_, None) => (self.xb.read(a), Effect::None), + (_, Some(v)) => (0, self.xb.write(a, v, &self.levels)), + }; + self.apply(effect); + BusRead64::ok(data) + } } impl Device for Bedrock { @@ -368,6 +546,10 @@ impl Resettable for Bedrock { pi.reschedule(host.as_ref(), now); } self.md.power_on(); + self.lb.power_on(&self.node); + self.io.power_on(); + let _ = self.ni.power_on(&self.levels); + let _ = self.xb.power_on(&self.levels); self.publish_all(); } } @@ -400,6 +582,43 @@ impl BusDevice for Registers { face_access!(read64, write64, u64, BusRead64, 8); } +/// The II's PIO face. +struct Pio(Arc); +impl Pio { + #[inline] + fn access(&self, a: u64, size: u8, value: Option) -> BusRead64 { + self.0.io.pio(a, size, value) + } +} +impl BusDevice for Pio { + face_access!(read8, write8, u8, BusRead8, 1); + face_access!(read16, write16, u16, BusRead16, 2); + face_access!(read32, write32, u32, BusRead32, 4); + face_access!(read64, write64, u64, BusRead64, 8); +} + +/// A widget's DMA as inbound traffic. Weak: the widget's device may hold +/// it, and the chip holds the widget. +struct WidgetDma { + chip: Weak, + widget: u8, +} +impl WidgetDma { + #[inline] + fn access(&self, a: u64, size: u8, value: Option) -> BusRead64 { + match self.chip.upgrade() { + Some(chip) => chip.inbound(self.widget, None, a, size, value), + None => BusRead64::err(), + } + } +} +impl BusDevice for WidgetDma { + face_access!(read8, write8, u8, BusRead8, 1); + face_access!(read16, write16, u16, BusRead16, 2); + face_access!(read32, write32, u32, BusRead32, 4); + face_access!(read64, write64, u64, BusRead64, 8); +} + /// The L1 UART's interrupt line. struct Uart(Weak); impl InterruptController for Uart { @@ -415,7 +634,7 @@ impl InterruptController for Uart { /// A test chip's configuration: one 64 MiB bank, on `clock`. #[cfg(test)] fn test_config(clock: Clock, processors: Population) -> BedrockConfig<'static> { - BedrockConfig { clock, processors, banks_mib: &[64] } + BedrockConfig { name: "test", clock, processors, banks_mib: &[64], events: None } } /// A chip on a hand-driven clock, for the unit tests: its register window @@ -426,6 +645,8 @@ struct Rig { now: Arc, word: Arc, regs: Arc, + /// The node's events. + events: parking_lot::Mutex>, } #[cfg(test)] impl Rig { @@ -437,12 +658,13 @@ impl Rig { let now = Arc::new(AtomicU64::new(0)); let clock = now.clone(); let config = test_config(Arc::new(move || clock.load(Ordering::SeqCst)), processors); - let chip = Bedrock::new(config).unwrap(); + let (events, receiver) = std::sync::mpsc::channel(); + let chip = Bedrock::new(BedrockConfig { events: Some(events), ..config }).unwrap(); let word = Arc::new(AtomicU64::new(0)); chip.set_cpu_interrupts(0, &*word); chip.start(); let regs = chip.registers(Form::Node); - Self { chip, now, word, regs } + Self { chip, now, word, regs, events: parking_lot::Mutex::new(receiver) } } fn r(&self, at: u64) -> u64 { let r = self.regs.read64(at); diff --git a/src/dev/bedrock/ni.rs b/src/dev/bedrock/ni.rs new file mode 100644 index 000000000..bd0480253 --- /dev/null +++ b/src/dev/bedrock/ni.rs @@ -0,0 +1,302 @@ +//! Bedrock's network interface (NI) exposes node routing, sideband and error +//! registers. The Fuel runs as one node without a NUMAlink peer, but the +//! guest still reads and programs these local control words during machine +//! setup. +//! +//! # What is approximated here +//! +//! Routing and sideband writes are retained; the network fabric and packet +//! delivery are not simulated. Unspecified power-on fields start at zero, and +//! the sideband word survives reset as the current compatibility behavior. +//! +//! Ported from hw/misc/sgi-bedrock-ni.c and hw/misc/sgi-bedrock-ni-regs.c +//! in jamesbraid/qemu-sgi-origami. + +use super::pi::Levels; +use super::{Effect, NodeReset}; +use parking_lot::Mutex; + +/// Length of the block-relative register aperture. +pub(super) const SIZE: u64 = 0x2008; + +/// Port status bits 1:0 of a node with no cable: bit 1 (link up) clear, bit 0 +/// set. Inferred, not measured. Bit 1 follows a real Tezro PROM reporting its +/// NUMAlink as down; bit 0 follows a real one-node Fuel taking a PIO bus +/// error with no link message (a 2003 installation kernel). The IRIX 6.5.30 +/// bus-error handler prints a link-reset notice when bits 1:0 read 0 with +/// reset enable 0. A PROM-monitor read of this register on an uncabled node +/// would settle it. +const PORT_STATUS_UNCABLED: u64 = 1; + +pub mod reg { + /// Port status, hub-relative `0x680000` (the first word of the block). + pub const PORT_STATUS: u64 = 0x0; + pub const LOCAL_RESET: u64 = 0x8; + pub const CONTROL_0: u64 = 0x10; + pub const CONTROL_1: u64 = 0x18; + pub const CONTROL_2: u64 = 0x20; + pub const CONTROL_3: u64 = 0x30; + pub const CONTROL_4: u64 = 0x38; + pub const ERROR_STATUS: u64 = 0x100; + pub const ERROR_STATUS_UPPER: u64 = 0x108; + pub const ERROR_STATUS_LOWER: u64 = 0x110; + pub const SIDE_BAND: u64 = 0x118; + pub const ERROR_CLEAR: u64 = 0x120; + pub const ROUTES: u64 = 0x1000; + pub const ROUTES_END: u64 = 0x1400; + pub const GLOBAL_CONTROL: u64 = 0x2000; +} + +#[derive(Clone)] +struct Regs { + control: [u64; 8], + routes: [u64; 128], + global: u64, + errors: u64, + sideband: u64, +} +impl Default for Regs { + fn default() -> Self { + Self { control: [0; 8], routes: [0; 128], global: 0, errors: 0, sideband: 0 } + } +} +impl Regs { + fn error_level(&self) -> bool { + self.errors & 0x1fffffff000000 != 0 + } +} + +pub(super) struct Ni { + regs: Mutex, +} + +impl Ni { + pub(super) fn new() -> Self { + Self { regs: Mutex::new(Regs::default()) } + } + pub(super) fn locked(&self) -> bool { + self.regs.is_locked() + } + /// Hold the block's lock, for tests of the paths that must not take it. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + self.regs.lock() + } + fn mask(a: u64) -> u64 { + match a { + reg::CONTROL_0 => 1, + reg::CONTROL_1 => 7, + reg::CONTROL_2 => 0x1ff, + reg::CONTROL_3 => 0x7efffffff, + reg::CONTROL_4 => 0xffff, + _ => 0, + } + } + fn reflect(r: &Regs, levels: &Levels) -> Effect { + if levels.set(Levels::NI_ERROR, r.error_level()) { Effect::Reflect } else { Effect::None } + } + pub(super) fn power_on(&self, levels: &Levels) -> Effect { + let mut r = self.regs.lock(); + let sideband = r.sideband; + *r = Regs::default(); + r.sideband = sideband; + Self::reflect(&r, levels) + } + /// A doubleword read. Reading the error-clear word clears the errors, so + /// the error level reaches `levels` under the block's lock. + pub(super) fn read(&self, a: u64, levels: &Levels) -> (u64, Effect) { + let mut r = self.regs.lock(); + let v = if Self::mask(a) != 0 { + r.control[a as usize / 8] + } else if (reg::ROUTES..reg::ROUTES_END).contains(&a) { + r.routes[(a - reg::ROUTES) as usize / 8] + } else { + match a { + reg::PORT_STATUS => PORT_STATUS_UNCABLED, + reg::ERROR_STATUS => r.errors, + reg::ERROR_STATUS_UPPER | reg::ERROR_STATUS_LOWER => 0, + reg::SIDE_BAND => r.sideband, + reg::ERROR_CLEAR => { + let v = r.errors; + r.errors = 0; + return (v, Self::reflect(&r, levels)); + } + reg::GLOBAL_CONTROL => r.global, + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock NI unsupported read {a:#x}"); + 0 + } + } + }; + (v, Effect::None) + } + /// A doubleword store. The local-reset bit (bit 2 of the reset word) + /// asks for the node's reset; the block's own registers are left to it. + pub(super) fn write(&self, a: u64, v: u64, levels: &Levels) -> Effect { + let mut r = self.regs.lock(); + if Self::mask(a) != 0 { + r.control[a as usize / 8] = v & Self::mask(a); + } else if (reg::ROUTES..reg::ROUTES_END).contains(&a) { + r.routes[(a - reg::ROUTES) as usize / 8] = v & 31; + } else { + match a { + reg::LOCAL_RESET => { + if v & 4 != 0 { + return Effect::Reset(NodeReset::Network); + } + } + reg::ERROR_STATUS => r.errors = v & ((1 << 53) - 1), + reg::SIDE_BAND => r.sideband = v & 0xfffff, + reg::GLOBAL_CONTROL => r.global = v & 31, + _ => crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock NI unsupported or read-only write {a:#x} = {v:#x}"), + } + } + Self::reflect(&r, levels) + } +} + +#[cfg(test)] +mod tests { + // Cases ported from tests/qtest/sn1-bedrock-test.c in jamesbraid/qemu-sgi-origami. + use super::super::reg::{pi, NI_BASE, PI1_BASE}; + use super::super::Rig; + use super::*; + use crate::traits::{BusDevice, BUS_ERR}; + + fn error_level(rig: &Rig) -> bool { + let levels = [0, PI1_BASE].map(|base| rig.r(base + pi::PENDING_STATUS_B) & (1 << 62) != 0); + assert_eq!(levels[0], levels[1], "both interfaces reflect bit 62"); + levels[0] + } + fn resets(rig: &Rig) -> usize { + rig.events.lock().try_iter().count() + } + + /// The guest programs route fields and reads the error register to acknowledge faults. + #[test] + fn route_registers_store_fields_and_error_reads_clear_pending_errors() { + let rig = Rig::new(); + for (a, mask) in [ + (reg::CONTROL_0, 1), + (reg::CONTROL_1, 7), + (reg::CONTROL_2, 0x1ff), + (reg::CONTROL_3, 0x7efffffff), + (reg::CONTROL_4, 0xffff), + ] { + rig.w(NI_BASE + a, !0); + assert_eq!(rig.r(NI_BASE + a), mask); + } + for a in [reg::ROUTES, reg::ROUTES_END - 8] { + rig.w(NI_BASE + a, !0); + assert_eq!(rig.r(NI_BASE + a), 31); + } + rig.w(NI_BASE + reg::GLOBAL_CONTROL, !0); + assert_eq!(rig.r(NI_BASE + reg::GLOBAL_CONTROL), 31); + rig.w(NI_BASE + reg::CONTROL_0, 0); + rig.w(NI_BASE + reg::LOCAL_RESET, 4); + assert_eq!(resets(&rig), 1); + rig.w(NI_BASE + reg::ERROR_STATUS, 0xffffff); + assert!(!error_level(&rig)); + rig.w(NI_BASE + reg::ERROR_STATUS, 1 << 24); + assert!(error_level(&rig)); + rig.r(NI_BASE + reg::ERROR_CLEAR); + assert!(!error_level(&rig)); + rig.w(NI_BASE + reg::SIDE_BAND, !0); + let _ = rig.chip.ni.power_on(&rig.chip.levels); + assert_eq!(rig.r(NI_BASE + reg::SIDE_BAND), 0xfffff); + assert_eq!(rig.r(NI_BASE + reg::CONTROL_3), 0); + } + + /// The PROM's one power-on pass writes a sign-extended port-parameter constant and zero + /// to the error and sideband words; each must read back through its defined bits only. + #[test] + fn power_on_programming_pass_reads_back_through_the_defined_bits() { + let rig = Rig::new(); + rig.w(NI_BASE + reg::CONTROL_0, 1); + rig.w(NI_BASE + reg::CONTROL_1, 0); + rig.w(NI_BASE + reg::CONTROL_2, 0x166); + rig.w(NI_BASE + reg::CONTROL_3, 0xffff_ffff_a3ff_1990); + rig.w(NI_BASE + reg::CONTROL_4, 0xc800); + rig.w(NI_BASE + reg::ERROR_STATUS, 0); + rig.r(NI_BASE + reg::ERROR_CLEAR); + rig.w(NI_BASE + reg::SIDE_BAND, 0); + for (a, want) in [ + (reg::CONTROL_0, 1), + (reg::CONTROL_1, 0), + (reg::CONTROL_2, 0x166), + (reg::CONTROL_3, 0x7_a3ff_1990), + (reg::CONTROL_4, 0xc800), + (reg::ERROR_STATUS, 0), + (reg::SIDE_BAND, 0), + (reg::ERROR_CLEAR, 0), + ] { + assert_eq!(rig.r(NI_BASE + a), want, "register {a:#x}"); + } + } + + /// Only the local-reset bit of the reset word reaches the node; the other bits must + /// not restart it, and the interface's own registers are left to the board's reset. + #[test] + fn only_the_local_reset_bit_requests_a_node_reset() { + let rig = Rig::new(); + for enable in 0..2 { + rig.w(NI_BASE + reg::CONTROL_0, enable); + for command in 0..8u64 { + rig.w(NI_BASE + reg::LOCAL_RESET, command); + assert_eq!(resets(&rig), usize::from(command & 4 != 0), "command {command:#x}"); + } + } + // The request does not clear this block. + assert_eq!(rig.r(NI_BASE + reg::CONTROL_0), 1); + } + + /// A latched port error raises the interrupt line through its interrupting bits only; + /// drivers rely on the line dropping when the status is rewritten or cleared. + #[test] + fn stored_error_status_drives_the_error_line_through_the_interrupting_bits() { + let rig = Rig::new(); + for (value, high) in [(1u64 << 30, true), (0xff, false), (1 << 52, true), (0, false)] { + rig.w(NI_BASE + reg::ERROR_STATUS, value); + assert_eq!(error_level(&rig), high, "status {value:#x}"); + } + // Bit 53 is beyond the register and must not be retained. + rig.w(NI_BASE + reg::ERROR_STATUS, 1 << 53); + assert_eq!(rig.r(NI_BASE + reg::ERROR_STATUS), 0); + } + + /// Reset values of the programmable words are unknown on hardware, so the model clears them; + /// a guest that re-runs its setup after a reset must see zeroes, not stale routes. + #[test] + fn reset_clears_every_programmable_register_and_misaligned_accesses_fault() { + let rig = Rig::new(); + let offsets = [ + reg::CONTROL_0, reg::CONTROL_1, reg::CONTROL_2, reg::CONTROL_3, reg::CONTROL_4, + reg::ROUTES, reg::ROUTES_END - 8, reg::GLOBAL_CONTROL, + ]; + for a in offsets { + rig.w(NI_BASE + a, !0); + } + // A misaligned write must not land in the neighbouring register. + assert_eq!(rig.regs.write64(NI_BASE + reg::CONTROL_4 + 1, 0), BUS_ERR); + assert_eq!(rig.regs.read64(NI_BASE + reg::CONTROL_4 + 1).status, BUS_ERR); + assert_eq!(rig.r(NI_BASE + reg::CONTROL_4), 0xffff); + let _ = rig.chip.ni.power_on(&rig.chip.levels); + for a in offsets { + assert_eq!(rig.r(NI_BASE + a), 0, "register {a:#x}"); + } + } + + /// An uncabled one-node port reads status bits 1:0 = 1: link down, bit 0 + /// set. Reading 0 there makes the guest's bus-error handler print a + /// link-reset notice. Bit 18 stays clear (a one-node machine) and a + /// write does not change the value. + #[test] + fn uncabled_port_status_reads_one_and_ignores_writes() { + let rig = Rig::new(); + assert_eq!(rig.r(NI_BASE + reg::PORT_STATUS), 1); + rig.w(NI_BASE + reg::PORT_STATUS, !0); + assert_eq!(rig.r(NI_BASE + reg::PORT_STATUS), 1); + let _ = rig.chip.ni.power_on(&rig.chip.levels); + assert_eq!(rig.r(NI_BASE + reg::PORT_STATUS), 1); + } +} diff --git a/src/dev/bedrock/pi.rs b/src/dev/bedrock/pi.rs index 87611121a..d417f34b0 100644 --- a/src/dev/bedrock/pi.rs +++ b/src/dev/bedrock/pi.rs @@ -200,6 +200,9 @@ pub enum Refused { pub(super) struct Levels([AtomicU64; 2]); impl Levels { pub(super) const UART: (usize, u32) = (0, 4); + pub(super) const XB_ERROR: (usize, u32) = (1, 54); + pub(super) const LB_ERROR: (usize, u32) = (1, 55); + pub(super) const NI_ERROR: (usize, u32) = (1, 62); /// Drive `(bank, bit)` to `level`. Whether the bit changed. pub(super) fn set(&self, (bank, bit): (usize, u32), level: bool) -> bool { let mask = 1u64 << bit; @@ -479,6 +482,16 @@ impl Pi { } } + /// An interrupt packet from region `region` to the pending-modify + /// register. Refused, and false, unless the region may send one. + pub(super) fn io_modify(&self, value: u64, region: u8) -> bool { + if region >= 64 || self.region.load(SeqCst) & (1 << region) == 0 { + self.note(reg::PENDING_MODIFY, true); + return false; + } + self.modify(value); + true + } /// Bank 1 bit 61 for a correctable memory error. pub(super) fn memory_correctable(&self) { diff --git a/src/dev/bedrock/tests.rs b/src/dev/bedrock/tests.rs index 8580e6a2c..ffb497974 100644 --- a/src/dev/bedrock/tests.rs +++ b/src/dev/bedrock/tests.rs @@ -1,7 +1,7 @@ //! Tests of the chip as a whole: what one unit's access does to another, //! the lock-free interrupt publish, and the paths that take no lock. -use super::reg::{md, pi, MD_BASE, PI1_BASE}; +use super::reg::{io, lb, md, pi, LB_BASE, MD_BASE, PI1_BASE}; use super::*; use crate::traits::{BUS_ERR, BUS_OK}; @@ -64,6 +64,10 @@ fn without_locks(rig: &Arc, work: impl FnOnce(&Rig) + Send + 'static) { chip.pi[0].hold(), chip.pi[1].hold(), chip.md.hold(), + chip.io.hold(), + chip.lb.hold(), + chip.ni.hold(), + chip.xb.hold(), ); let (done, finished) = std::sync::mpsc::channel(); let other = rig.clone(); @@ -109,3 +113,144 @@ fn the_cell_bit_gates_the_window_and_padding_answers_an_error() { assert_eq!(rig.regs.read64(pi::INTERFACE_ID).status, BUS_ERR); assert_eq!(rig.regs.read64(MD_BASE + md::CELL).status, BUS_ERR); } + +/// An inbound interrupt packet's address for PI `pi` of node `node`. +fn packet_address(node: u64, pi: u64) -> u64 { + (1 << 47) | (node << 39) | 0x180_0000 | (pi * PI1_BASE) | 0x90 +} + +/// The PI region mask gates interrupt packets: a refused packet answers a +/// bus error, so the sender knows the level was never set, and sets nothing. +#[test] +fn a_refused_interrupt_packet_answers_an_error_from_inbound_write() { + let rig = Rig::new(); + let chip = &rig.chip; + rig.w(pi::OFFSET_00A8, 1 << 9); + rig.w(pi::OFFSET_0010, 0); + assert_eq!(chip.inbound_write(15, Some(0), packet_address(0, 0), 8, 0x109), BUS_ERR); + assert_eq!(rig.r(pi::PENDING_STATUS_A), 0); + assert_eq!(rig.cause(), 0); + rig.w(pi::OFFSET_0010, 1); + assert_eq!(chip.inbound_write(15, Some(0), packet_address(0, 0), 8, 0x109), BUS_OK); + assert_eq!(rig.r(pi::PENDING_STATUS_A), 1 << 9); + assert_eq!(rig.cause(), 1, "IP2"); + assert_eq!(chip.inbound_write(15, Some(0), packet_address(0, 0), 8, 0x009), BUS_OK); + assert_eq!(rig.cause(), 0); +} + +/// LB's region register names the node; inbound packets and the BTEs' +/// interrupts answer only to that node from then on. +#[test] +fn the_local_blocks_node_number_moves_the_hub_window() { + let rig = Rig::new(); + let chip = &rig.chip; + rig.w(LB_BASE + lb::NODE_REGION, 3 << 32); + assert_eq!(chip.inbound_write(15, Some(0), packet_address(0, 0), 8, 0x101), BUS_ERR); + assert_eq!(chip.inbound_write(15, Some(0), packet_address(3, 0), 8, 0x101), BUS_OK); + assert_eq!(rig.r(pi::PENDING_STATUS_A), 1 << 1); + // A BTE completion interrupt names its node in bits 8:1. + rig.w(MD_BASE + md::CONFIG, 0x11); + rig.w(super::reg::IO_BASE + io::BTE1_BASE + super::reg::bte::INTERRUPT, (5 << 16) | (3 << 1)); + rig.w(super::reg::IO_BASE + io::BTE1_BASE, 1 | super::io::BUSY); + rig.w(super::reg::IO_BASE + io::BTE1_BASE + super::reg::bte::CONTROL, 0); + assert_eq!(rig.r(pi::PENDING_STATUS_A), (1 << 1) | (1 << 5)); +} + +/// A node reset leaves as one event on the board's channel, for either +/// cause, and the chip calls nothing else in the board. +#[test] +fn a_node_reset_is_one_event_on_the_boards_channel() { + let rig = Rig::new(); + rig.w(LB_BASE + lb::SOFT_RESET, 1); + rig.w(super::reg::NI_BASE + super::reg::ni::LOCAL_RESET, 4); + let events: Vec<_> = rig.events.lock().try_iter().collect(); + assert!(matches!(events[..], [MachineEvent::HardReset, MachineEvent::HardReset]), "{events:?}"); +} + +/// Inbound set and clear packets race mask stores and LB's reflected error +/// level on other threads; once every writer has stopped, the processor's +/// word matches what the registers say. +#[test] +fn racing_packets_and_reflected_levels_leave_the_word_matching_the_registers() { + const ROUNDS: u64 = 20_000; + let rig = Arc::new(Rig::new()); + rig.w(pi::OFFSET_00B0, 1 << 55); + let writers: Vec<_> = (0..4u64) + .map(|n| { + let rig = rig.clone(); + std::thread::spawn(move || { + for i in 0..ROUNDS { + match n { + // Two packet sources set and clear bits 1 and 2. + 0 | 1 => { + let set = if (i + n) % 2 == 0 { 0x100 } else { 0 }; + let status = rig.chip.inbound_write(15, Some(0), packet_address(0, 0), 8, set | (n + 1)); + assert_eq!(status, BUS_OK); + } + // One store flips bank 0's mask between the two bits. + 2 => rig.w(pi::OFFSET_00A8, if i % 3 == 0 { 0b010 } else { 0b110 }), + // One drives LB's error level, bank 1 bit 55. + _ => rig.w(LB_BASE + if i % 2 == 0 { lb::ERROR_STATUS } else { lb::ERROR_CLEAR }, 1), + } + } + }) + }) + .collect(); + for writer in writers { + writer.join().unwrap(); + } + let pending = rig.r(pi::PENDING_STATUS_A); + let mask = rig.r(pi::OFFSET_00A8); + let reflected = rig.r(pi::PENDING_STATUS_B) & rig.r(pi::OFFSET_00B0); + let want = u64::from(pending & mask != 0) | u64::from(reflected != 0) << 1; + assert_eq!(rig.cause(), want, "pending {pending:#x} mask {mask:#x} reflected {reflected:#x}"); +} + +/// Two stores to LB's error registers race; each sets the reflected bit +/// under LB's lock, so the level both PIs see is the one the register +/// holds when the stores are done. +#[test] +fn a_reflected_level_follows_its_register_under_racing_stores() { + let rig = Arc::new(Rig::new()); + let writers: Vec<_> = [lb::ERROR_STATUS, lb::ERROR_CLEAR] + .into_iter() + .map(|at| { + let rig = rig.clone(); + std::thread::spawn(move || { + for _ in 0..20_000 { + rig.w(LB_BASE + at, 1); + } + }) + }) + .collect(); + for writer in writers { + writer.join().unwrap(); + } + let level = rig.r(LB_BASE + lb::ERROR_STATUS) != 0; + for base in [0, PI1_BASE] { + assert_eq!(rig.r(base + pi::PENDING_STATUS_B) & (1 << 55) != 0, level); + } +} + +/// The crossbar's paths take no lock either: an inbound interrupt packet, +/// an 8-byte inbound memory write and read, and PIO to a widget. +#[test] +fn the_inbound_and_pio_paths_take_no_lock() { + struct Stub; + impl BusDevice for Stub { + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::ok(0x5a) + } + } + let rig = Arc::new(Rig::new()); + rig.w(MD_BASE + md::CONFIG, 0x11); + rig.chip.connect_widget(8, Arc::new(Stub)); + rig.w(pi::OFFSET_00A8, 1 << 9); + without_locks(&rig, |rig| { + assert_eq!(rig.chip.inbound_write(15, Some(0), packet_address(0, 0), 8, 0x109), BUS_OK); + assert_eq!(rig.cause(), 1); + assert_eq!(rig.chip.inbound_write(14, None, 0x1000, 8, 0x1234), BUS_OK); + assert_eq!(rig.chip.inbound_read(14, None, 0x1000, 8).data, 0x1234); + assert_eq!(rig.chip.pio().read64(0x800_0000).data, 0x5a); + }); +} diff --git a/src/dev/bedrock/xb.rs b/src/dev/bedrock/xb.rs new file mode 100644 index 000000000..b8e35dc14 --- /dev/null +++ b/src/dev/bedrock/xb.rs @@ -0,0 +1,253 @@ +//! Bedrock's crossbar block (XB) records the fabric error queues and lets +//! software service queued errors one at a time; its error level reaches both +//! processor interfaces. +//! +//! # What is approximated here +//! +//! Arbitration timing and diagnostic outputs are not modeled. Unsupported +//! queue reads return zero, while writes are logged and ignored. +//! +//! Ported from hw/misc/sgi-bedrock-xb.c in jamesbraid/qemu-sgi-origami. + +use super::pi::Levels; +use super::Effect; +use parking_lot::Mutex; + +/// Length of the block-relative register aperture. +pub(super) const SIZE: u64 = 0x198; + +pub mod reg { + pub const CONTROL_BASE: u64 = 0x0; + pub const CONTROL_END: u64 = 0x18; + pub const QUEUE_BASE: u64 = 0x20; + pub const QUEUE_END: u64 = 0x88; + pub const FIRST_ERROR: u64 = 0x90; + pub const DEBUG_BASE: u64 = 0x98; + pub const DEBUG_END: u64 = 0xa0; + pub const CLEAR_QUEUE_BASE: u64 = 0x120; + pub const CLEAR_QUEUE_END: u64 = 0x188; + pub const CLEAR_FIRST_ERROR: u64 = 0x190; +} + +const MASKS: [u64; 14] = [ + 0x3333, 0x33, 0, 0, 0x3330, 0, 0, 0xf0, 0xfff0, 0, 0, 0, 0x3303, 0x11, +]; +struct Regs { + control: [u64; 4], + queues: [u64; 14], + first: u64, + debug: [u64; 2], + unserviced: u16, +} +impl Default for Regs { + fn default() -> Self { + Self { + control: [0x0008000080000021, 0, 0, 0], + queues: [0; 14], + first: 0, + debug: [0; 2], + unserviced: 0, + } + } +} +impl Regs { + fn error_level(&self) -> bool { + self.first & 0x1000 != 0 + } +} + +pub(super) struct Xb { + regs: Mutex, +} + +impl Xb { + pub(super) fn new() -> Self { + Self { regs: Mutex::new(Regs::default()) } + } + pub(super) fn locked(&self) -> bool { + self.regs.is_locked() + } + /// Hold the block's lock, for tests of the paths that must not take it. + #[cfg(test)] + pub(super) fn hold(&self) -> impl Sized + '_ { + self.regs.lock() + } + fn reflect(r: &Regs, levels: &Levels) -> Effect { + if levels.set(Levels::XB_ERROR, r.error_level()) { Effect::Reflect } else { Effect::None } + } + pub(super) fn power_on(&self, levels: &Levels) -> Effect { + let mut r = self.regs.lock(); + *r = Regs::default(); + Self::reflect(&r, levels) + } + pub(super) fn read(&self, a: u64) -> u64 { + let r = self.regs.lock(); + match a { + reg::CONTROL_BASE..=reg::CONTROL_END => r.control[a as usize / 8], + reg::QUEUE_BASE..=reg::QUEUE_END => { + let n = (a - reg::QUEUE_BASE) as usize / 8; + if MASKS[n] == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported queue {n}"); + } + r.queues[n] + } + reg::FIRST_ERROR => r.first, + reg::DEBUG_BASE | reg::DEBUG_END => r.debug[(a - reg::DEBUG_BASE) as usize / 8], + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported read {a:#x}"); + 0 + } + } + } + /// A doubleword store. The error level reaches `levels` under the + /// block's lock. + pub(super) fn write(&self, a: u64, v: u64, levels: &Levels) -> Effect { + let mut r = self.regs.lock(); + match a { + reg::CONTROL_BASE..=reg::CONTROL_END => { + let n = a as usize / 8; + r.control[n] = v & [0x000ffffffffffff1, 0xffffffff, 0x7ff, 0xf][n]; + if a == 0x10 && r.control[n] != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB traffic observation unsupported"); + } + } + reg::QUEUE_BASE..=reg::QUEUE_END => { + let n = (a - reg::QUEUE_BASE) as usize / 8; + let v = v & MASKS[n]; + if MASKS[n] == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported queue {n}"); + } + if v != 0 { + r.queues[n] |= v; + r.unserviced |= 1 << n; + if r.first & 0x1000 == 0 { + r.first = 0x1000 | ((15 - n) as u64) << 8; + } + } + } + reg::CLEAR_QUEUE_BASE..=reg::CLEAR_QUEUE_END => { + let n = (a - reg::CLEAR_QUEUE_BASE) as usize / 8; + r.queues[n] = 0; + r.unserviced &= !(1 << n); + } + reg::CLEAR_FIRST_ERROR => { + if r.first & 0x1000 != 0 { + let n = 15 - ((r.first >> 8) & 15) as usize; + if r.unserviced & (1 << n) != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB first error cleared before queue"); + } else { + r.first = 0; + for n in 0..14 { + if r.queues[n] != 0 { + r.first = 0x1000 | ((15 - n) as u64) << 8; + break; + } + } + } + } + } + reg::DEBUG_BASE | reg::DEBUG_END => { + let n = (a - reg::DEBUG_BASE) as usize / 8; + r.debug[n] = v & [0xffff, 7][n]; + if r.debug[n] != 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB debug output unsupported"); + } + } + _ => crate::dlog!(crate::devlog::LogModule::Bedrock, "Bedrock XB unsupported or read-only write {a:#x} = {v:#x}"), + } + Self::reflect(&r, levels) + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-xb-test.c in + // jamesbraid/qemu-sgi-origami: control-storage, service-order and + // cold-reset. + use super::super::reg::{pi, PI1_BASE, XB_BASE}; + use super::super::Rig; + use super::*; + use crate::traits::{BusDevice, BUS_ERR}; + + fn error_level(rig: &Rig) -> bool { + let levels = [0, PI1_BASE].map(|base| rig.r(base + pi::PENDING_STATUS_B) & (1 << 54) != 0); + assert_eq!(levels[0], levels[1], "both interfaces reflect bit 54"); + levels[0] + } + + /// The guest sees masked crossbar control words and cleared errors after reset. + #[test] + fn crossbar_control_masks_apply_and_reset_clears_errors() { + let rig = Rig::new(); + for (a, mask) in [ + (reg::CONTROL_BASE, 0x000ffffffffffff1), + (reg::CONTROL_BASE + 8, 0xffffffff), + (reg::CONTROL_BASE + 0x10, 0x7ff), + (reg::CONTROL_END, 15), + (reg::DEBUG_BASE, 0xffff), + (reg::DEBUG_END, 7), + ] { + rig.w(XB_BASE + a, !0); + assert_eq!(rig.r(XB_BASE + a), mask); + } + // Keep expected register values independent of the decoder's mask table. + for (address, expected) in [ + (reg::QUEUE_BASE, 0x3333), + (reg::QUEUE_BASE + 8, 0x0033), + (reg::QUEUE_BASE + 0x10, 0), + (reg::QUEUE_BASE + 0x18, 0), + (reg::QUEUE_BASE + 0x20, 0x3330), + (reg::QUEUE_BASE + 0x28, 0), + (reg::QUEUE_BASE + 0x30, 0), + (reg::QUEUE_BASE + 0x38, 0x00f0), + (reg::QUEUE_BASE + 0x40, 0xfff0), + (reg::QUEUE_BASE + 0x48, 0), + (reg::QUEUE_BASE + 0x50, 0), + (reg::QUEUE_BASE + 0x58, 0), + (reg::QUEUE_BASE + 0x60, 0x3303), + (reg::QUEUE_END, 0x0011), + ] { + rig.w(XB_BASE + address, !0); + assert_eq!(rig.r(XB_BASE + address), expected, "register {address:#x}"); + } + assert!(error_level(&rig)); + let _ = rig.chip.xb.power_on(&rig.chip.levels); + assert_eq!(rig.r(XB_BASE + reg::CONTROL_BASE), 0x0008000080000021); + assert!(!error_level(&rig)); + assert_eq!(rig.regs.read32(XB_BASE + reg::CONTROL_BASE).status, BUS_ERR); + } + /// The guest services queued crossbar faults in order, one error at a time. + #[test] + fn servicing_one_crossbar_error_exposes_the_next_pending_error() { + { + let rig = Rig::new(); + rig.w(XB_BASE + reg::QUEUE_BASE, 1); + assert!(error_level(&rig)); + assert_eq!(rig.r(XB_BASE + reg::FIRST_ERROR), 0x1f00); + rig.w(XB_BASE + reg::CLEAR_FIRST_ERROR, !0); + assert_eq!(rig.r(XB_BASE + reg::FIRST_ERROR), 0x1f00); + rig.w(XB_BASE + reg::CLEAR_QUEUE_END, !0); + rig.w(XB_BASE + reg::CLEAR_QUEUE_BASE, 0); + rig.w(XB_BASE + reg::CLEAR_FIRST_ERROR, 0); + assert!(!error_level(&rig)); + let _ = rig.chip.xb.power_on(&rig.chip.levels); + assert!(!error_level(&rig)); + } + { + let rig = Rig::new(); + rig.w(XB_BASE + reg::QUEUE_END, 0x10); + rig.w(XB_BASE + reg::QUEUE_BASE, 1); + assert_eq!(rig.r(XB_BASE + reg::FIRST_ERROR), 0x1200); + rig.w(XB_BASE + reg::CLEAR_QUEUE_END, 0); + rig.w(XB_BASE + reg::CLEAR_FIRST_ERROR, 0); + assert_eq!(rig.r(XB_BASE + reg::FIRST_ERROR), 0x1f00); + rig.w(XB_BASE + reg::CLEAR_QUEUE_BASE, 0); + rig.w(XB_BASE + reg::QUEUE_BASE, 0x10); + rig.w(XB_BASE + reg::CLEAR_FIRST_ERROR, 0); + assert_eq!(rig.r(XB_BASE + reg::FIRST_ERROR), 0x1f00); + rig.w(XB_BASE + reg::CLEAR_QUEUE_BASE, 0); + rig.w(XB_BASE + reg::CLEAR_FIRST_ERROR, 0); + assert!(!error_level(&rig)); + } + } +} From 48605dddc7fa949d7cce97c210b752d6784666b3 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 11/22] bedrock_flash: add the Fuel boot flash windows Fuel reaches its AMD flash through executable PROM aliases and programming windows. Share the array bytes and persistent store while preserving the command and erase state of each flash part. The PROM reads its configuration block from flash before it touches the processor, and a boot-mode word that disagrees with the CPU's reset mode sends it to its error routine before it sets up GP. So the board passes a FirmwareRecord, the one source for both: the PROM's boot-mode word, whose bits 24:0 are the mode inputs the CPU latches into Config, the processor clock, the machine type word and a header policy. The Fuel's record is the 500 MHz R14000 with a 2 MiB secondary cache, mode word 0x549aa985 and machine type 1, the R14000 as the IP35 PROM's own configuration editor lists it (0 for the R10000 and R12000, 2 for the R16000); its CPU resets with Config 0x6c9aa985. Tests pin both. The Fuel's header is refreshed: its fields are written over the payload's own block whenever a store is formatted or opened, with the compensation word set so the block keeps its byte sum. The other policy, authored, writes a whole block once when a store is formatted, with the compensation byte the PROM's editor would choose so the image still sums to zero, and leaves an opened store's block alone, so a block the PROM rewrote survives the next start. --- src/dev/bedrock_flash.rs | 893 +++++++++++++++++++++++++++++++++++++++ src/dev/mod.rs | 1 + 2 files changed, 894 insertions(+) create mode 100644 src/dev/bedrock_flash.rs diff --git a/src/dev/bedrock_flash.rs b/src/dev/bedrock_flash.rs new file mode 100644 index 000000000..1c4a14b32 --- /dev/null +++ b/src/dev/bedrock_flash.rs @@ -0,0 +1,893 @@ +//! Bedrock's boot flash: two independent 8 MiB arrays, selected 32 MiB +//! apart, with the AMD command set and CFI query data. +//! +//! Boot aliases and executable views share the same array bytes. Each array +//! has its own command state; instruction fetches read stored bytes even +//! while register reads return command status. +//! +//! Ported from hw/misc/sgi-bedrock-flash.c and hw/mips/sgi/initial-flash.c +//! in jamesbraid/qemu-sgi-origami. + + + +use super::sflash::amd::{AmdCore, Effect, SECTOR_SIZE}; +use super::sflash::storage::FlashBacking; +use crate::traits::{ + BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Resettable, BUS_ERR, BUS_OK, +}; +use parking_lot::Mutex; +use std::io; +use std::path::Path; +use std::sync::{ + atomic::{AtomicU64, Ordering}, + Arc, +}; + +pub const PART_SIZE: usize = 8 << 20; +pub const FLASH_SIZE: usize = 2 * PART_SIZE; +pub const SECOND_POSITION: u64 = 32 << 20; + +fn sum(bytes: &[u8]) -> u8 { + bytes.iter().fold(0u8, |s, b| s.wrapping_add(*b)) +} + +/// The fields of the PROM's configuration block that depend on the board. +/// +/// The PROM reads them from flash before it touches the processor, so they +/// must agree with the CPU the machine builds: a mismatch in the boot-mode +/// word sends it to its error routine before it initialises GP. The values +/// are not a measured flash dump; each record's doc says where they come +/// from. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct FirmwareRecord { + /// The CPU boot-mode word the PROM compares with the processor's mode. + /// Its bits 24:0 are the mode inputs the processor latches into CP0 + /// Config at reset (the R10000 family's reset-mode inputs). + pub cpu_mode_word: u32, + /// The processor clock the PROM reports and times against, in Hz. + pub cpu_clock_hz: u32, + /// The record's word at +0x30, which the PROM's start-up check compares + /// with a machine class of its own. When the node board's L1 record + /// carries a CPU configuration, the PROM derives that class from the + /// board word's top nibble (0x4 gives 2, 0x8 gives 1, any other 0; read + /// from the stock PROM 6.210's code, established for that PROM). The + /// Fuel's record is 1, the R14000 as the PROM's configuration editor + /// lists it; its L1 carries no CPU configuration and the PROM's own table + /// row for its board word is what it is checked against. + pub machine_type: u32, + /// How the record reaches the configuration block. + pub header: Header, +} + +/// How a board's record is put into the configuration block at 0x60. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum Header { + /// Rewrite the board's fields over the payload's own block, keeping its + /// byte sum, whenever a store is formatted or opened. The Fuel's flash is + /// configured this way. + Refreshed, + /// Replace the whole block with the one authored for the board's flash + /// seed, once, when a store is formatted, with the compensation byte the + /// PROM's editor would choose (the block's byte sum is the replaced + /// block's, so the image still sums to zero). An opened store keeps the + /// block it holds, so a flash update the PROM wrote survives. + Authored, +} + +impl FirmwareRecord { + /// The Fuel's 500 MHz R14000 with a 2 MiB secondary cache. + pub const FUEL: Self = Self { + cpu_mode_word: 0x549a_a985, + cpu_clock_hz: 500_000_000, + machine_type: 1, + header: Header::Refreshed, + }; + /// CP0 Config at reset: the fixed 32 KiB primary cache sizes (fields 3 + /// in bits 31:29 and 28:26) with the board's mode bits. + pub const fn reset_config0(&self) -> u32 { + (3 << 29) | (3 << 26) | self.reset_mode() + } + /// The mode bits the processor latches into Config at reset: the mode + /// word's bits 24:0. + pub const fn reset_mode(&self) -> u32 { + self.cpu_mode_word & 0x01ff_ffff + } + + /// The configuration block an authored record writes: the format word, + /// the board's mode word, the `ip27conf` magic, the processor and + /// 200 MHz controller clocks as 64-bit words, the real-time clock's + /// 1,250,000 Hz, the machine type, a zero compensation byte (the caller sets + /// it), an edit counter of one, and the PROM's 6.210 version fields. + /// The layout is the PROM editor's. + fn authored_block(&self) -> [u8; CONFIG_LEN] { + let mut block = [0u8; CONFIG_LEN]; + let mut put = |offset: usize, bytes: &[u8]| block[offset..offset + bytes.len()].copy_from_slice(bytes); + put(0x00, &21u32.to_be_bytes()); + put(0x04, &self.cpu_mode_word.to_be_bytes()); + put(0x08, b"ip27conf"); + put(0x10, &u64::from(self.cpu_clock_hz).to_be_bytes()); + put(0x18, &200_000_000u64.to_be_bytes()); + put(0x20, &1_250_000u64.to_be_bytes()); + for (offset, value) in [ + (0x28, 1), + (0x2c, 16), + (0x30, self.machine_type), + (0x34, 0), + (0x38, 1), + (0x3c, 16), + (0x40, 6), + (0x44, 210), + (0x48, 0), + ] { + put(offset, &u32::to_be_bytes(value)); + } + block + } +} + +/// Where the PROM's configuration block sits in the payload, and its size. +const CONFIG_BASE: usize = 0x60; +const CONFIG_LEN: usize = 0x50; + +/// Put the board's record into a store being formatted. +fn format_header(data: &mut [u8], record: &FirmwareRecord) { + match record.header { + Header::Refreshed => configure(data, record), + Header::Authored => { + // The compensation byte is the PROM editor's rule: the new block + // sums, byte for byte, to what the block it replaces did, so the + // image still sums to zero (the check the PROM runs on itself). + let old = sum(&data[CONFIG_BASE..CONFIG_BASE + CONFIG_LEN]); + let mut block = record.authored_block(); + block[0x37] = old.wrapping_sub(sum(&block)); + data[CONFIG_BASE..CONFIG_BASE + CONFIG_LEN].copy_from_slice(&block); + } + } +} + +/// Put the board's record into a store that was opened rather than formatted. +fn reopen_header(data: &mut [u8], record: &FirmwareRecord) { + match record.header { + Header::Refreshed => configure(data, record), + Header::Authored => {} + } +} + +/// Apply the board's firmware record while retaining the configuration sum. +fn configure(data: &mut [u8], record: &FirmwareRecord) { + let config = &mut data[CONFIG_BASE..CONFIG_BASE + CONFIG_LEN]; + let old = sum(config); + config[0x34..0x38].fill(0); + config[4..8].copy_from_slice(&record.cpu_mode_word.to_be_bytes()); + config[0x14..0x18].copy_from_slice(&record.cpu_clock_hz.to_be_bytes()); + config[0x30..0x34].copy_from_slice(&record.machine_type.to_be_bytes()); + config[0x38..0x3c].copy_from_slice(&1u32.to_be_bytes()); + let adjustment = old.wrapping_sub(sum(config)); + config[0x34..0x38].copy_from_slice(&(adjustment as u32).to_be_bytes()); +} + +struct State { + data: Vec, + parts: [AmdCore; 2], + backing: Option, +} + +impl State { + fn new(data: Vec) -> Self { + Self { + data, + parts: std::array::from_fn(|_| AmdCore::default()), + backing: None, + } + } + // Keep the in-memory array unchanged until each backing write succeeds. + // If a later range fails, restore the old bytes for all ranges already tried. + fn persist(&mut self, ranges: &[(usize, &[u8])]) -> io::Result<()> { + let Some(backing) = &mut self.backing else { + return Ok(()); + }; + for (index, &(offset, bytes)) in ranges.iter().enumerate() { + if let Err(error) = backing.write_range(offset, bytes) { + for &(restore_offset, restore_bytes) in &ranges[..=index] { + let old = &self.data[restore_offset..restore_offset + restore_bytes.len()]; + if let Err(rollback) = backing.write_range(restore_offset, old) { + log::error!( + "bedrock-flash: backing rollback at {restore_offset:#x} failed: {rollback}" + ); + } + } + return Err(error); + } + } + Ok(()) + } + fn replace(&mut self, offset: usize, bytes: &[u8]) -> io::Result { + if self.data[offset..offset + bytes.len()] == *bytes { + return Ok(false); + } + self.persist(&[(offset, bytes)])?; + self.data[offset..offset + bytes.len()].copy_from_slice(bytes); + Ok(true) + } + fn erase(&mut self, part: usize, sectors: &[usize]) -> io::Result { + let erased = vec![0xff; SECTOR_SIZE]; + let ranges: Vec<_> = sectors + .iter() + .map(|sector| (part * PART_SIZE + sector * SECTOR_SIZE, erased.as_slice())) + .collect(); + self.persist(&ranges)?; + let mut changed = false; + for (offset, bytes) in ranges { + let data = &mut self.data[offset..offset + bytes.len()]; + changed |= data.iter().any(|byte| *byte != 0xff); + data.fill(0xff); + } + Ok(changed) + } +} + +pub struct Flash { + st: Mutex, + gen: AtomicU64, +} + +impl Flash { + /// Format a fresh two-position store from a decoded payload, with the + /// board's firmware record. + pub fn new(payload: &[u8], record: FirmwareRecord) -> io::Result { + if payload.len() > PART_SIZE { + return Err(io::Error::new( + io::ErrorKind::InvalidData, + "PROM payload exceeds 8 MiB", + )); + } + let mut data = vec![0xff; FLASH_SIZE]; + data[..payload.len()].copy_from_slice(payload); + let label = PART_SIZE + 0x1e0000; + data[label + 0x10..label + 0x14].copy_from_slice(&0x504c4f49u32.to_be_bytes()); + data[label + 0x14..label + 0x18].copy_from_slice(&1u32.to_be_bytes()); + data[label + 0x18..label + 0x1c].copy_from_slice(&1u32.to_be_bytes()); + format_header(&mut data, &record); + Ok(Self { + st: Mutex::new(State::new(data)), + gen: AtomicU64::new(0), + }) + } + + /// Load an exact 16 MiB store, or seed a missing/empty one. Wrong sizes are + /// refused without writes. Reopening reapplies a refreshed firmware record + /// and leaves an authored one as the store holds it. + /// Completed guest programming and erasure write through to this file. + pub fn with_file(payload: &[u8], path: impl AsRef, record: FirmwareRecord) -> io::Result { + let mut data = Self::new(payload, record)?.st.into_inner().data; + let mut backing = FlashBacking::open(path, &mut data).map_err(|error| { + if error.kind() == io::ErrorKind::InvalidData { + io::Error::new( + io::ErrorKind::InvalidData, + "flash store must be exactly 16 MiB (or empty)", + ) + } else { + error + } + })?; + reopen_header(&mut data, &record); + // Opening an existing store loads it before a refreshed record's + // fields are applied; an empty store was seeded from this same data. + backing.write_range(0, &data)?; + let flash = Self { + st: Mutex::new(State { + data, + parts: std::array::from_fn(|_| AmdCore::default()), + backing: Some(backing), + }), + gen: AtomicU64::new(0), + }; + Ok(flash) + } + + pub fn contents(&self) -> Vec { + self.st.lock().data.clone() + } + pub fn generation(&self) -> u64 { + self.gen.load(Ordering::Acquire) + } + pub fn reset(&self) { + let mut st = self.st.lock(); + for part in &mut st.parts { + part.reset(); + } + self.gen.fetch_add(1, Ordering::Release); + } + /// Advance the model clock independently of flash reads. + pub fn advance_ns(&self, ns: u64) { + let mut st = self.st.lock(); + for part in 0..2 { + let selecting = st.parts[part].selecting(); + let complete = st.parts[part].advance_ns(ns); + if selecting && !st.parts[part].selecting() { + self.gen.fetch_add(1, Ordering::Release); + } + if !complete { + continue; + } + let sectors: Vec<_> = st.parts[part].selected_sectors().collect(); + let success = match st.erase(part, §ors) { + Ok(_) => true, + Err(e) => { + log::error!("bedrock-flash: position {part} erase backing write failed: {e}"); + false + } + }; + st.parts[part].finish_erase(success); + self.gen.fetch_add(1, Ordering::Release); + } + } + pub fn register_locks(self: &Arc) { + let me = self.clone(); + crate::locks::register_lock_fn("bedrock_flash::data", move || me.st.is_locked()); + } + /// Use the same shared flash for local and remote boot spaces. + pub fn boot_view(self: &Arc) -> FlashView { + FlashView { + flash: self.clone(), + range: None, + } + } + /// A contiguous executable alias. Offsets refer to the 16 MiB backing. + pub fn array_view(self: &Arc, offset: usize, len: usize) -> io::Result { + if offset.checked_add(len).is_none_or(|end| end > FLASH_SIZE) { + return Err(io::Error::new( + io::ErrorKind::InvalidInput, + "Fuel flash alias exceeds backing", + )); + } + Ok(FlashView { + flash: self.clone(), + range: Some((offset, len)), + }) + } +} +impl Resettable for Flash { + fn power_on(&self) { + self.reset(); + } +} + +pub struct FlashView { + flash: Arc, + range: Option<(usize, usize)>, +} +impl FlashView { + fn offset(&self, addr: u64, width: usize) -> Option { + if let Some((base, len)) = self.range { + let off = usize::try_from(addr).ok()?; + return off + .checked_add(width) + .filter(|end| *end <= len) + .map(|_| base + off); + } + let (base, within) = if addr < PART_SIZE as u64 { + (0, addr) + } else if addr >= SECOND_POSITION { + (PART_SIZE, addr - SECOND_POSITION) + } else { + return None; + }; + let within = usize::try_from(within).ok()?; + within + .checked_add(width) + .filter(|end| *end <= PART_SIZE) + .map(|_| base + within) + } + fn read(&self, addr: u64, width: usize) -> Option { + let off = self.offset(addr, width)?; + let mut st = self.flash.st.lock(); + if self.range.is_none() { + let part = off / PART_SIZE; + if !st.parts[part].array_reads() && (width != 2 || off & 1 != 0) { + return None; + } + if width == 2 && off & 1 == 0 { + if let Some(value) = st.parts[part].read16(off % PART_SIZE) { + return Some(value as u64); + } + } + } + Some( + st.data[off..off + width] + .iter() + .fold(0, |v, b| (v << 8) | *b as u64), + ) + } + fn refuse(&self, addr: u64, width: usize, value: u64) -> u32 { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "bedrock-flash: unsupported {width}-byte store {value:#x} at {addr:#x}; invalid bus cycle or executable view" + ); + BUS_ERR + } +} +impl BusDevice for FlashView { + fn read8(&self, a: u64) -> BusRead8 { + self.read(a, 1) + .map_or_else(BusRead8::err, |v| BusRead8::ok(v as u8)) + } + fn read16(&self, a: u64) -> BusRead16 { + self.read(a, 2) + .map_or_else(BusRead16::err, |v| BusRead16::ok(v as u16)) + } + fn read32(&self, a: u64) -> BusRead32 { + self.read(a, 4) + .map_or_else(BusRead32::err, |v| BusRead32::ok(v as u32)) + } + fn read64(&self, a: u64) -> BusRead64 { + self.read(a, 8).map_or_else(BusRead64::err, BusRead64::ok) + } + fn write8(&self, a: u64, v: u8) -> u32 { + self.refuse(a, 1, v as u64) + } + fn write16(&self, a: u64, v: u16) -> u32 { + self.refuse(a, 2, v as u64) + } + fn write32(&self, a: u64, v: u32) -> u32 { + self.refuse(a, 4, v as u64) + } + fn write64(&self, a: u64, v: u64) -> u32 { + if self.range.is_some() || a & 7 != 0 || v > u16::MAX as u64 { + return self.refuse(a, 8, v); + } + let (part, within) = if a < SECOND_POSITION { + (0, a) + } else { + (1, a - SECOND_POSITION) + }; + let Some(end) = within.checked_add(8) else { + return self.refuse(a, 8, v); + }; + if end > SECOND_POSITION { + return self.refuse(a, 8, v); + } + let offset = (within / 4) as usize; + let mut st = self.flash.st.lock(); + let effect = match st.parts[part].write16(offset, v as u16) { + Ok(effect) => effect, + Err(()) => { + self.flash.gen.fetch_add(1, Ordering::Release); + return BUS_ERR; + } + }; + if let Effect::Program { offset, value } = effect { + let backing = part * PART_SIZE + offset; + let old = u16::from_be_bytes([st.data[backing], st.data[backing + 1]]); + match st.replace(backing, &(old & value).to_be_bytes()) { + Ok(_) => {} + Err(e) => { + log::error!("bedrock-flash: program backing write at {backing:#x} failed: {e}"); + self.flash.gen.fetch_add(1, Ordering::Release); + return BUS_ERR; + } + } + } + self.flash.gen.fetch_add(1, Ordering::Release); + BUS_OK + } + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { + &self.flash.gen + } +} + +#[cfg(test)] +mod tests { + // Cases follow tests/qtest/sn1-bedrock-flash-test.c in + // jamesbraid/qemu-sgi-origami: program, sector-erase, erase-polling, + // reset-states and backing-file. + use super::*; + fn unlock(v: &FlashView, part: u64) { + assert_eq!(v.write64(part + 8 * 0x555, 0xaa), crate::traits::BUS_OK); + assert_eq!(v.write64(part + 8 * 0x2aa, 0x55), crate::traits::BUS_OK); + } + fn program(v: &FlashView, part: u64, off: u64, value: u16) { + unlock(v, part); + assert_eq!(v.write64(part + 8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!( + v.write64(part + 4 * off, value as u64), + crate::traits::BUS_OK + ); + } + fn erase_select(v: &FlashView, part: u64, offset: u64) { + unlock(v, part); + assert_eq!(v.write64(part + 8 * 0x555, 0x80), BUS_OK); + unlock(v, part); + assert_eq!(v.write64(part + 4 * offset, 0x30), BUS_OK); + } + + /// The PROM reads CPU configuration and the volume label from its boot flash. + #[test] + fn the_seed_exposes_fuel_configuration_and_volume_label() { + let seed = vec![0x21; 0xb0]; + let f = Flash::new(&seed, FirmwareRecord::FUEL).unwrap(); + let bytes = f.contents(); + assert_eq!(sum(&bytes[0x60..0xb0]), sum(&seed[0x60..0xb0])); + assert_eq!(&bytes[0x64..0x68], &0x549aa985u32.to_be_bytes()); + assert_eq!(&bytes[0x74..0x78], &500_000_000u32.to_be_bytes()); + assert_eq!(&bytes[0x90..0x94], &1u32.to_be_bytes(), "CPU type: R14000"); + let label = PART_SIZE + 0x1e0000; + assert_eq!(&bytes[label..label + 16], &[0xff; 16]); + assert_eq!( + &bytes[label + 16..label + 28], + &[0x50, 0x4c, 0x4f, 0x49, 0, 0, 0, 1, 0, 0, 0, 1] + ); + assert!(bytes[label + 28..].iter().all(|b| *b == 0xff)); + assert!(Flash::new(&vec![0; PART_SIZE + 1], FirmwareRecord::FUEL).is_err()); + } + /// The stock image's factory block: the 400 MHz mode word, machine type 0 + /// and a zero check adjustment, before any board configures it. + fn factory_block() -> Vec { + let mut image = vec![0x21; 0xb0]; + image[0x60..0xb0].fill(0); + image[0x64..0x68].copy_from_slice(&0x5219a785u32.to_be_bytes()); + image[0x70..0x78].copy_from_slice(&400_000_000u64.to_be_bytes()); + image + } + + /// A refreshed record leaves the block's byte sum unchanged: the check + /// adjustment absorbs the difference. + #[test] + fn a_refreshed_record_configures_the_block_and_keeps_its_byte_sum() { + let record = FirmwareRecord::FUEL; + let image = factory_block(); + let bytes = Flash::new(&image, record).unwrap().contents(); + let block = &bytes[0x60..0xb0]; + assert_eq!(&block[4..8], &record.cpu_mode_word.to_be_bytes()); + assert_eq!(&block[0x14..0x18], &record.cpu_clock_hz.to_be_bytes()); + assert_eq!(&block[0x30..0x34], &record.machine_type.to_be_bytes()); + assert_eq!(&block[0x38..0x3c], &1u32.to_be_bytes(), "flash program count"); + assert_eq!(&block[0x34..0x38], &[0, 0, 0, 0x9c]); + assert_eq!(sum(block), sum(&image[0x60..0xb0])); + } + + /// The Fuel's record with an authored header: the policy a board whose + /// flash keeps the block the PROM writes uses. + const AUTHORED: FirmwareRecord = FirmwareRecord { machine_type: 0, header: Header::Authored, ..FirmwareRecord::FUEL }; + + /// The stock block the PROM ships: the 400 MHz R10000 and R12000 row, + /// machine type 0, no compensation, edit counter 0. Built field by field, not + /// copied; its byte sum is 0xac, as the PROM's own checksum rule sees it. + fn stock_block() -> [u8; CONFIG_LEN] { + let mut block = FirmwareRecord { cpu_mode_word: 0x5219_a785, cpu_clock_hz: 400_000_000, ..AUTHORED } + .authored_block(); + block[0x38..0x3c].fill(0); + assert_eq!(sum(&block), 0xac, "the stock block's byte sum"); + block + } + + /// An image whose configuration block is the stock one, as the PROM's own + /// image has it. + fn stock_image() -> Vec { + let mut image = vec![0x21; 0x200]; + image[0x60..0xb0].copy_from_slice(&stock_block()); + image + } + + /// An authored block follows the PROM editor's layout and rule: the mode + /// word, the clock, the Bedrock and real-time clocks, the machine type + /// and an edit counter of one, and a compensation byte that makes the + /// block sum to what the stock block did, 0xac, so the image keeps + /// summing to zero. + #[test] + fn an_authored_record_writes_the_block_the_proms_editor_would() { + let image = stock_image(); + let bytes = Flash::new(&image, AUTHORED).unwrap().contents(); + let block = &bytes[0x60..0xb0]; + assert_eq!(&block[0..4], &21u32.to_be_bytes()); + assert_eq!(&block[4..8], &AUTHORED.cpu_mode_word.to_be_bytes()); + assert_eq!(&block[8..16], b"ip27conf"); + assert_eq!(&block[0x10..0x18], &u64::from(AUTHORED.cpu_clock_hz).to_be_bytes()); + assert_eq!(&block[0x18..0x20], &200_000_000u64.to_be_bytes(), "Bedrock clock"); + assert_eq!(&block[0x20..0x28], &1_250_000u64.to_be_bytes(), "real-time clock"); + assert_eq!(&block[0x30..0x34], &0u32.to_be_bytes(), "machine type"); + assert_eq!(&block[0x38..0x3c], &1u32.to_be_bytes(), "one edit"); + assert_eq!(sum(block), 0xac, "the stock block's sum"); + assert_eq!(&bytes[..0x60], &image[..0x60], "the payload around the block is untouched"); + assert_eq!(&bytes[0xb0..image.len()], &image[0xb0..]); + } + + /// The compensation follows whatever block the payload carried (the whole + /// image keeps its sum), and changing any one byte of a seeded block off + /// its compensated value breaks the sum. + #[test] + fn the_compensation_byte_keeps_the_images_sum_and_any_change_breaks_it() { + // An image that sums to zero, as the PROM's own does, still sums to + // zero once a record's block is seeded into it. + let mut zero = stock_image(); + zero[0x1ff] = 0u8.wrapping_sub(sum(&zero[..0x1ff])); + assert_eq!(sum(&zero), 0); + for record in [FirmwareRecord::FUEL, AUTHORED] { + assert_eq!(sum(&Flash::new(&zero, record).unwrap().contents()[..0x200]), 0, "{record:?}: the seeded image sums to zero"); + } + for payload in [stock_image(), vec![0x5a; 0x200], factory_block()] { + let before = sum(&payload[..0x200.min(payload.len())]); + for record in [FirmwareRecord::FUEL, AUTHORED] { + let bytes = Flash::new(&payload, record).unwrap().contents(); + let end = 0x200.min(payload.len()); + assert_eq!(sum(&bytes[..end]), before, "{record:?}: the image's byte sum"); + // A record that differs in one field is compensated to the same sum. + let other = FirmwareRecord { cpu_mode_word: record.cpu_mode_word ^ 0x0100_0000, ..record }; + assert_eq!(sum(&Flash::new(&payload, other).unwrap().contents()[..end]), before, "{record:?}: another mode word"); + for at in 0x60..0xb0 { + let mut broken = bytes[..end].to_vec(); + broken[at] ^= 1; + assert_ne!(sum(&broken), before, "{record:?}: byte {at:#x}"); + } + } + } + } + + /// The Config the CPU resets to carries the mode word's low 25 bits under + /// the fixed primary cache sizes, so the PROM's check of one against the + /// other passes: the Fuel's 500 MHz record gives 0x6c9aa985. + #[test] + fn the_reset_config_carries_the_mode_words_low_bits() { + assert_eq!(FirmwareRecord::FUEL.reset_mode(), 0x009a_a985); + assert_eq!(FirmwareRecord::FUEL.reset_config0(), 0x6c9a_a985); + let other = FirmwareRecord { cpu_mode_word: 0xfe00_0000 | 0x0012_3456, ..FirmwareRecord::FUEL }; + assert_eq!(other.reset_mode(), 0x0012_3456); + } + + /// Reopening a store reapplies a refreshed record, so a Fuel follows the + /// machine that opens it, and leaves an authored block as stored, so a + /// block the PROM rewrote survives the next start. + #[test] + fn reopening_refreshes_a_refreshed_record_and_keeps_an_authored_block() { + let dir = std::env::temp_dir().join(format!("bedrock-flash-record-{}", std::process::id())); + std::fs::create_dir_all(&dir).unwrap(); + let fuel_path = dir.join("fuel.bin"); + drop(Flash::with_file(&factory_block(), &fuel_path, AUTHORED).unwrap()); + let fuel = Flash::with_file(&[], &fuel_path, FirmwareRecord::FUEL).unwrap().contents(); + assert_eq!(&fuel[0x64..0x68], &0x549aa985u32.to_be_bytes()); + assert_eq!(&fuel[0x90..0x94], &1u32.to_be_bytes(), "CPU type: R14000"); + + let authored_path = dir.join("authored.bin"); + let seeded = Flash::with_file(&factory_block(), &authored_path, AUTHORED).unwrap().contents(); + assert_eq!(std::fs::read(&authored_path).unwrap(), seeded, "the seed is what the store holds"); + let mut stored = seeded.clone(); + stored[0x64..0x68].copy_from_slice(&0x1234_5678u32.to_be_bytes()); + std::fs::write(&authored_path, &stored).unwrap(); + let reopened = Flash::with_file(&factory_block(), &authored_path, AUTHORED).unwrap().contents(); + assert_eq!(reopened, stored); + std::fs::remove_dir_all(&dir).unwrap(); + } + /// The PROM programs through one view while executable aliases must + /// keep reading array bytes. + #[test] + fn programming_changes_shared_bytes_without_exposing_status_to_instruction_fetches() { + { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + let local = f.boot_view(); + let remote = f.boot_view(); + let execute = f.array_view(0, PART_SIZE).unwrap(); + assert_eq!(local.write64(SECOND_POSITION, 0xf0), crate::traits::BUS_OK); + for part in [0, SECOND_POSITION] { + assert_eq!(local.write64(part + 8 * 0x55, 0x98), crate::traits::BUS_OK); + assert_eq!(remote.read16(part + 0x20).data, b'Q' as u16); + assert_eq!(remote.read16((SECOND_POSITION - part) + 0x20).data, 0xffff); + assert_eq!(execute.read16(0x20).data, 0xffff); + assert_eq!(remote.write64(part, 0xf0), crate::traits::BUS_OK); + assert_eq!(local.write64(part + 8 * 0x555, 0xaa), crate::traits::BUS_OK); + assert_eq!( + remote.write64(part + 8 * 0x2aa, 0x55), + crate::traits::BUS_OK + ); + assert_eq!(local.write64(part + 8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!( + remote.write64(part + 4 * 0x200, 0x1234), + crate::traits::BUS_OK + ); + assert_eq!(local.read16(part + 0x200).data, 0x1234); + } + assert_eq!(execute.read16(0x200).data, 0x1234); + assert!(!local.read8(PART_SIZE as u64).is_ok()); + } + { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0x1234); + program(&v, 0, 0x200, 0xf0ff); + assert_eq!(v.read16(0x200).data, 0x1034); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), crate::traits::BUS_OK); + let gen = f.generation(); + for (addr, value) in [ + (0x800, 0x10000), + (0x804, 0), + (64 << 20, 0), + ] { + assert_eq!(v.write64(addr, value), BUS_ERR); + } + assert_eq!(v.write16(0x800, 0), BUS_ERR); + assert_eq!(f.generation(), gen); + assert_eq!(v.write64(0x800, 0x00ff), crate::traits::BUS_OK); + assert_eq!(v.read16(0x200).data, 0x0034); + let execute = f.array_view(0, PART_SIZE).unwrap(); + unlock(&v, 0); + assert_eq!(execute.write64(8 * 0x555, 0xa0), BUS_ERR); + assert_eq!(v.write64(8 * 0x555, 0xa0), crate::traits::BUS_OK); + assert_eq!(v.write64(0x808, 0x5555), crate::traits::BUS_OK); + assert_eq!(execute.read16(0x202).data, 0x5555); + } + { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + assert_eq!(v.write64(0x550, 0x98), BUS_ERR); + assert_eq!(v.read16(0x20).data, 0xffff); + assert_eq!(v.write64(8 * 0x55, 0x98), BUS_OK); + let expected = [ + 0x51, 0x52, 0x59, 2, 0, 0, 0, 0, 0, 0, 0, 0x27, 0x36, 0, 0, 3, 0, 9, 0, 4, 0, 4, 0, 23, + 1, 0, 0, 0, 1, 0x7f, 0, 0, 1, + ]; + for (index, value) in expected.into_iter().enumerate() { + assert_eq!(v.read16(((index + 0x10) * 2) as u64).data, value); + } + assert_eq!(v.read16(0x100).data, 0); + assert!(!v.read8(0x20).is_ok()); + assert!(!v.read16(0x21).is_ok()); + f.reset(); + assert_eq!(v.read16(0x20).data, 0xffff); + } + } + /// The PROM polls erase completion and can reset the chip before + /// selected sectors change. + #[test] + fn erase_completes_only_for_selected_sectors_and_reset_cancels_it() { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0); + program(&v, 0, 0x10000, 0); + erase_select(&v, 0, 0x200); + assert!(!v.read8(0x200).is_ok()); + let a = v.read16(0x200).data; + let b = v.read16(0x200).data; + assert_eq!((a ^ b) & 0x44, 0x44); + let a = v.read16(0x10000).data; + let b = v.read16(0x10000).data; + assert_eq!((a ^ b) & 0x44, 0x40); + f.advance_ns(40_000); + assert_eq!(v.write64(4 * 0x10000, 0x30), BUS_OK); + f.advance_ns(40_000); + assert_eq!(v.read16(0x10000).data & 8, 0); + assert_eq!(v.write64(SECOND_POSITION + 8 * 0x55, 0x98), BUS_OK); + assert_eq!(v.read16(SECOND_POSITION + 0x20).data, b'Q' as u16); + f.advance_ns(10_000); + assert_eq!(v.read16(0x200).data & 8, 8); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + assert_eq!(v.read16(0x200).data & 0x80, 0); + f.reset(); + f.advance_ns(u64::MAX); + assert_eq!(v.read16(0x200).data, 0); + assert_eq!(v.read16(0x10000).data, 0); + assert_eq!(v.read16(SECOND_POSITION + 0x20).data, 0xffff); + erase_select(&v, 0, 0x200); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + f.advance_ns(u64::MAX); + assert_eq!(v.read16(0x200).data, 0); + erase_select(&v, 0, 0x200); + assert_eq!(v.write64(4 * 0x10000, 0x30), BUS_OK); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data, 0xffff); + assert_eq!(v.read16(0x10000).data, 0xffff); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), BUS_OK); + f.reset(); + assert_eq!(v.write64(4 * 0x200, 0x1234), BUS_ERR); + assert_eq!(v.read16(0x200).data, 0xffff); + program(&v, 0, 0x200, 0); + program(&v, 0, 0x10000, 0x3456); + erase_select(&v, 0, 0x200); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data, 0xffff); + assert_eq!(v.read16(0x10000).data, 0x3456); + } + /// Firmware updates must persist across emulator restarts without + /// overwriting a supplied image. + #[test] + fn completed_programming_and_erases_survive_reopening_the_backing_file() { + { + let path = std::env::temp_dir().join(format!("iris-bedrock-store-{}", std::process::id())); + let mut bytes = vec![0x5a; FLASH_SIZE]; + bytes[PART_SIZE] = 0xc7; + std::fs::write(&path, &bytes).unwrap(); + let f = Flash::with_file(&[0x11], &path, FirmwareRecord::FUEL).unwrap(); + assert_eq!(f.contents()[0], 0x5a); + assert_eq!(f.contents()[PART_SIZE], 0xc7); + assert_eq!(std::fs::read(&path).unwrap(), f.contents()); + drop(f); + std::fs::write(&path, [1, 2, 3]).unwrap(); + assert!(Flash::with_file(&[], &path, FirmwareRecord::FUEL).is_err()); + assert_eq!(std::fs::read(&path).unwrap(), [1, 2, 3]); + std::fs::write(&path, []).unwrap(); + let g = Flash::with_file(&[0x34], &path, FirmwareRecord::FUEL).unwrap(); + assert_eq!(g.contents()[0], 0x34); + assert_eq!(std::fs::metadata(&path).unwrap().len(), FLASH_SIZE as u64); + drop(g); + std::fs::remove_file(path).unwrap(); + } + { + let path = std::env::temp_dir().join(format!("iris-amd-persist-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + let f = Arc::new(Flash::with_file(&[], &path, FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0x1234); + program(&v, SECOND_POSITION, (PART_SIZE - 2) as u64, 0x5678); + drop(v); + drop(f); + let f = Arc::new(Flash::with_file(&[], &path, FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + assert_eq!(v.read16(0x200).data, 0x1234); + assert_eq!( + v.read16(SECOND_POSITION + PART_SIZE as u64 - 2).data, + 0x5678 + ); + unlock(&v, SECOND_POSITION); + assert_eq!(v.write64(SECOND_POSITION + 8 * 0x555, 0x80), BUS_OK); + unlock(&v, SECOND_POSITION); + assert_eq!( + v.write64(SECOND_POSITION + 4 * (PART_SIZE as u64 - 2), 0x30), + BUS_OK + ); + f.advance_ns(50_000 + 512_000_000); + drop(v); + drop(f); + let f = Arc::new(Flash::with_file(&[], &path, FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + assert_eq!(v.read16(0x200).data, 0x1234); + assert_eq!( + v.read16(SECOND_POSITION + PART_SIZE as u64 - 2).data, + 0xffff + ); + std::fs::remove_file(path).unwrap(); + } + } + + /// A host write failure must preserve flash contents and remain visible + /// to the guest. + #[test] + fn failed_backing_writes_leave_array_bytes_unchanged() { + { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + // A read-only descriptor makes writes fail without a special backend. + let path = std::env::temp_dir().join(format!("iris-amd-read-only-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + f.st.lock().backing = Some(FlashBacking::from_file( + std::fs::File::open(&path).unwrap(), + FLASH_SIZE, + )); + let v = f.boot_view(); + unlock(&v, 0); + assert_eq!(v.write64(8 * 0x555, 0xa0), BUS_OK); + assert_eq!(v.write64(4 * 0x200, 0x1234), BUS_ERR); + assert_eq!(v.read16(0x200).data, 0xffff); + std::fs::remove_file(path).unwrap(); + } + { + let f = Arc::new(Flash::new(&[], FirmwareRecord::FUEL).unwrap()); + let v = f.boot_view(); + program(&v, 0, 0x200, 0); + let path = + std::env::temp_dir().join(format!("iris-amd-erase-failure-{}", std::process::id())); + std::fs::write(&path, []).unwrap(); + f.st.lock().backing = Some(FlashBacking::from_file( + std::fs::File::open(&path).unwrap(), + FLASH_SIZE, + )); + erase_select(&v, 0, 0x200); + f.advance_ns(50_000 + 512_000_000); + assert_eq!(v.read16(0x200).data & 0x20, 0x20); + assert_eq!(f.contents()[0x200], 0); + assert_eq!(v.write64(0, 0xf0), BUS_OK); + assert_eq!(v.read16(0x200).data, 0); + std::fs::remove_file(path).unwrap(); + } + } + +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 30cebaf7b..22233858b 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -75,6 +75,7 @@ pub mod hub_ni; pub mod hub_pi; pub mod dp83840; pub mod bedrock; +pub mod bedrock_flash; pub mod dp83843; pub mod ics1890; pub mod timekeeper; From 77c4a97e53c4e55f184c484ea7ca9c9340b55ff0 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 12/22] xbridge: add the two-head Fuel PCI host bridge Fuel routes PCI configuration, DMA and interrupt packets through two XBridge heads. Each head owns a PCI bus, as the Octane's Bridge does: the bus holds the head's functions, decodes their BARs and combines their interrupt pins, and the head supplies only what is its own. That is how a processor access becomes configuration cycles and byte order (from a device window's register, or from Control for an alias), the translation table and DMA, and where an interrupt pin goes. Whole-page BAR runs of the device and alias windows are served from the head's own table, so the bridge shares the machine memory map without allocating more Physical tables. A board attached to one of the crossbar's external ports raises its interrupts as crosstalk packets, the same way the PCI heads do. Route them through the same sender, so the Fuel's graphics board reaches the Bedrock's processor interface with no path of its own, and say whether the destination took the packet. The crossbar answers with the Crossbow's widget 0 register map, so it takes its offsets from the Octane's Crossbow model instead of restating them. What stays separate is behavior that differs between the two: the XBridge crossbar's status register carries the requesting port, wide and malformed accesses set the access-error status and answer zero, its interrupts go out as crosstalk packets to any destination, and it resets the heads and ports its fences select. --- src/dev/mod.rs | 1 + src/dev/xbow.rs | 66 +- src/dev/xbridge.rs | 1390 ++++++++++++++++++++++++++++++++++++++ src/dev/xbridge/tests.rs | 756 +++++++++++++++++++++ 4 files changed, 2180 insertions(+), 33 deletions(-) create mode 100644 src/dev/xbridge.rs create mode 100644 src/dev/xbridge/tests.rs diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 22233858b..cf217a2a3 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -81,3 +81,4 @@ pub mod ics1890; pub mod timekeeper; pub mod prom_image; pub mod isp12160; +pub mod xbridge; diff --git a/src/dev/xbow.rs b/src/dev/xbow.rs index 3f3352254..95a4b0276 100644 --- a/src/dev/xbow.rs +++ b/src/dev/xbow.rs @@ -40,49 +40,49 @@ pub const fn widget_id(rev: u32, part: u32, mfgr: u32) -> u32 { (rev << 28) | (part << 12) | (mfgr << 1) | 1 } -const FIRST_WIDGET: u8 = 8; -const LINKS: usize = 8; +pub(crate) const FIRST_WIDGET: u8 = 8; +pub(crate) const LINKS: usize = 8; /// The register block's span in widget 0's window. pub const REG_SPAN: u32 = 0x0100_0000; -const IDENTIFICATION: u32 = 0x00; -const STATUS: u32 = 0x08; -const ERR_UPPER: u32 = 0x10; -const ERR_LOWER: u32 = 0x18; -const CONTROL: u32 = 0x20; -const PACKET_TIMEOUT: u32 = 0x28; -const INT_UPPER: u32 = 0x30; -const INT_LOWER: u32 = 0x38; -const ERR_CMD: u32 = 0x40; -const LLP_CONTROL: u32 = 0x48; -const STATUS_CLEAR: u32 = 0x50; -const ARB_RELOAD: u32 = 0x58; -const PERF_A: u32 = 0x60; -const PERF_B: u32 = 0x68; -const NIC: u32 = 0x70; +pub(crate) const IDENTIFICATION: u32 = 0x00; +pub(crate) const STATUS: u32 = 0x08; +pub(crate) const ERR_UPPER: u32 = 0x10; +pub(crate) const ERR_LOWER: u32 = 0x18; +pub(crate) const CONTROL: u32 = 0x20; +pub(crate) const PACKET_TIMEOUT: u32 = 0x28; +pub(crate) const INT_UPPER: u32 = 0x30; +pub(crate) const INT_LOWER: u32 = 0x38; +pub(crate) const ERR_CMD: u32 = 0x40; +pub(crate) const LLP_CONTROL: u32 = 0x48; +pub(crate) const STATUS_CLEAR: u32 = 0x50; +pub(crate) const ARB_RELOAD: u32 = 0x58; +pub(crate) const PERF_A: u32 = 0x60; +pub(crate) const PERF_B: u32 = 0x68; +pub(crate) const NIC: u32 = 0x70; /// Revision code 5 and later: widget 0's reset fence, then links 8 to 15's /// at a stride of 8, then the resource lock and its clear. -const FENCE_BASE: u32 = 0x78; -const FENCES: usize = 1 + LINKS; -const FENCE_END: u32 = FENCE_BASE + 8 * FENCES as u32; -const LOCK: u32 = 0xc0; -const LOCK_CLEAR: u32 = 0xc8; +pub(crate) const FENCE_BASE: u32 = 0x78; +pub(crate) const FENCES: usize = 1 + LINKS; +pub(crate) const FENCE_END: u32 = FENCE_BASE + 8 * FENCES as u32; +pub(crate) const LOCK: u32 = 0xc0; +pub(crate) const LOCK_CLEAR: u32 = 0xc8; const FENCE_MASK: u32 = 0xff; /// The first revision code with fences and the lock (Crossbow 2.0 in the /// software tables; an Octane2's code-5 part shows as 1.4 in its inventory). const REVISION_FENCES: u32 = 5; -const LINK_BASE: u32 = 0x100; -const LINK_STRIDE: u32 = 0x40; -const LINK_END: u32 = LINK_BASE + LINK_STRIDE * LINKS as u32; -const LINK_FLUSH: u32 = 0x00; -const LINK_CONTROL: u32 = 0x08; -const LINK_STATUS: u32 = 0x10; -const LINK_ARB_UPPER: u32 = 0x18; -const LINK_ARB_LOWER: u32 = 0x20; -const LINK_STATUS_CLEAR: u32 = 0x28; -const LINK_RESET: u32 = 0x30; -const LINK_AUX_STATUS: u32 = 0x38; +pub(crate) const LINK_BASE: u32 = 0x100; +pub(crate) const LINK_STRIDE: u32 = 0x40; +pub(crate) const LINK_END: u32 = LINK_BASE + LINK_STRIDE * LINKS as u32; +pub(crate) const LINK_FLUSH: u32 = 0x00; +pub(crate) const LINK_CONTROL: u32 = 0x08; +pub(crate) const LINK_STATUS: u32 = 0x10; +pub(crate) const LINK_ARB_UPPER: u32 = 0x18; +pub(crate) const LINK_ARB_LOWER: u32 = 0x20; +pub(crate) const LINK_STATUS_CLEAR: u32 = 0x28; +pub(crate) const LINK_RESET: u32 = 0x30; +pub(crate) const LINK_AUX_STATUS: u32 = 0x38; const STATUS_REQUEST: u32 = 1 << 23; const STATUS_LINK_REQUESTS: u32 = 0xff00_0000; diff --git a/src/dev/xbridge.rs b/src/dev/xbridge.rs new file mode 100644 index 000000000..306261f4b --- /dev/null +++ b/src/dev/xbridge.rs @@ -0,0 +1,1390 @@ +//! Fuel's XBridge revision 2 contains two independent PCI heads and their +//! shared crossbar. Each head has its own 1024-entry ATE, device windows and +//! interrupt state. Revision 2 places aliases differently from the Octane's +//! single-head Bridge, uses different PCI slot data-phase widths, and sends +//! host DMA through widget 0 with source-aware packets. +//! +//! This remains separate from `bridge.rs` because its two-head topology, +//! alias map and host packet path are externally visible parts of this chip. +//! Each head owns a shared `PciBus` for its functions, BAR decode and +//! interrupt pins, and supplies only what is its own: how a processor access +//! becomes configuration cycles and byte order, the ATE and DMA, and where an +//! interrupt pin goes. Whole-page BAR mappings of the device and alias windows +//! decode PCI PIO from the head's own table, without a `Physical` table; the +//! machine's RAM still uses one. +//! +//! # What is approximated here +//! +//! Slot widths, immediate link initialization, synchronous flushes and +//! revision-2 alias placement follow the project's BSD QEMU model, not a +//! silicon measurement. Timing, external SRAM, flash and diagnostic engines +//! are absent. Undefined accesses latch an error and remain observable. +//! +//! Ported from hw/pci-host/sgi-xbridge.c and hw/misc/sgi-xbridge-xbow.c +//! in jamesbraid/qemu-sgi-origami, revision +//! fe128635b6dfe2511f1bd451684cb30007686df1. + +use crate::dev::w1; +use crate::pci::{BarDevice, BarMapping, ByteOrder, CpuWindow, DevFn, IntxRouting, PciBus, PciDma, PciDmaStatus, PciFunction, PciHost, Space}; +use crate::physical::{read_bytes, write_bytes}; +use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, Resettable, BUS_OK}; +use parking_lot::{Mutex, RwLock}; +use std::sync::{Arc, Weak}; + +pub const HOST_WIDGET: u8 = 10; +pub const HEAD_E: u8 = 14; +pub const HEAD_F: u8 = 15; +pub const ATE_ENTRIES: usize = 1024; +pub const ATE_MAPPED: usize = 512; + +/// Offsets of the PCI head register words, at eight-byte strides. +pub mod reg { + pub const IDENTIFICATION: usize = 0x0; + pub const STATUS: usize = 0x8; + pub const ERR_UPPER: usize = 0x10; + pub const ERR_LOWER: usize = 0x18; + pub const CONTROL: usize = 0x20; + pub const REQUEST_TIMEOUT: usize = 0x28; + pub const INT_DEST_UPPER: usize = 0x30; + pub const INT_DEST_LOWER: usize = 0x38; + pub const LINK_CONFIG: usize = 0x48; + pub const TEST_PIN_CONTROL: usize = 0x70; + pub const DIRECT_MAP: usize = 0x80; + pub const SSRAM_PARITY_ERR: usize = 0x90; + pub const ARBITRATION: usize = 0xa0; + pub const NIC: usize = 0xb0; + pub const BUS_TIMEOUT: usize = 0xc0; + pub const PCI_TYPE1_CONFIG: usize = 0xc8; + pub const PCI_ERR_LOWER: usize = 0xd8; + pub const INTERRUPT_STATUS: usize = 0x100; + pub const INTERRUPT_ENABLE: usize = 0x108; + pub const INTERRUPT_RESET: usize = 0x110; + pub const INTERRUPT_MODE: usize = 0x118; + pub const INTERRUPT_DEVICE: usize = 0x120; + pub const INTERRUPT_ERROR_HOST: usize = 0x128; + pub const INTERRUPT_HOST_BASE: usize = 0x130; + pub const INTERRUPT_HOST_LAST: usize = 0x168; + pub const INTERRUPT_MULTIPLE: usize = 0x170; + pub const FORCE_INTERRUPT_BASE: usize = 0x180; + pub const FORCE_PIN_BASE: usize = 0x1c0; + pub const FORCE_PIN_LAST: usize = 0x1f8; + pub const DEVICE_BASE: usize = 0x200; + pub const DEVICE_LAST: usize = 0x238; + pub const RRB_EVEN: usize = 0x280; + pub const RRB_ODD: usize = 0x288; + pub const RRB_CLEAR: usize = 0x298; + pub const ATE_UPPER_BASE: u64 = 0x10000; + pub const ATE_LOWER_BASE: u64 = 0x12000; + pub const ATE_END: u64 = 0x14000; + +} + +/// The crossbar has the Crossbow's widget 0 register map, which `xbow.rs` +/// owns; the register offsets here are its constants. The access rules and +/// reset values below are this chip's own (the Octane's Crossbow model differs +/// in status, link state, wide accesses and where its interrupts go). +use crate::dev::xbow; + +/// Fences run from widget 0's to link 15's. +const FENCE_LAST: u32 = xbow::FENCE_END - 8; +/// Read-only registers between the resource lock and the first link. +const READ_ONLY_BASE: u32 = xbow::LOCK_CLEAR + 8; +const READ_ONLY_LAST: u32 = xbow::LINK_BASE - 8; +fn ix(off: u32) -> usize { + (off / 8) as usize +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct InterruptPacket { + pub source_widget: u8, + pub widget: u8, + pub address: u64, + pub vector: u8, + pub level: bool, +} +pub type InterruptSender = Arc bool + Send + Sync>; + +/// Source identity and placement hints retained for a host's inbound packet path. +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub struct DmaPacket { + pub source_widget: u8, + pub slot: u8, + pub widget: u8, + pub address: u64, + pub write: bool, + pub half_write_requested: bool, + pub half_read_requested: bool, +} +/// Bytes are in XIO address order after bridge placement. A false return +/// refuses that access. The host applies the requested exchange of doubleword halves. +pub type DmaSender = Arc bool + Send + Sync>; + +struct Routes { + targets: RwLock<[Option>; 16]>, + interrupts: RwLock>, + dma: RwLock>, +} +impl Routes { + fn interrupt(&self, p: InterruptPacket) -> bool { + let sender = self.interrupts.read().clone(); + let accepted = if let Some(send) = sender { + send(p) + } else { + let target = self.targets.read()[p.widget as usize].clone(); + target.is_some_and(|t| { + t.write64( + p.address, + p.vector as u64 | if p.level { 1 << 8 } else { 0 }, + ) == BUS_OK + }) + }; + if !accepted { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: refused interrupt {p:?}"); + } + accepted + } +} + +/// One chip owns both heads and widget 0; views share routing and reset state. +pub struct XBridge { + routes: Arc, + heads: [Arc; 2], + xbow: Arc, +} +impl XBridge { + pub fn new(memory: Arc) -> Arc { + let routes = Arc::new(Routes { + targets: RwLock::new(std::array::from_fn(|i| { + if i == HOST_WIDGET as usize { + Some(memory.clone()) + } else { + None + } + })), + interrupts: RwLock::new(None), + dma: RwLock::new(None), + }); + let heads = [ + XBridgeHead::new(HEAD_E, 1, routes.clone()), + XBridgeHead::new(HEAD_F, 0, routes.clone()), + ]; + let xbow = Arc::new(Crossbar { + st: Mutex::new(XbowState::new()), + order: Mutex::new(()), + routes: routes.clone(), + heads: heads.each_ref().map(Arc::downgrade), + reset_output: RwLock::new(None), + }); + Arc::new(Self { + routes, + heads, + xbow, + }) + } + pub fn register_locks(self: &Arc) { + use crate::locks::register_lock_fn; + for head in &self.heads { + for (what, probe) in [ + ("state", (|h: &XBridgeHead| h.st.is_locked()) as fn(&XBridgeHead) -> bool), + ("irq_order", |h| h.order.is_locked()), + ("microlan", |h| h.microlan.is_locked()), + ] { + let h = Arc::downgrade(head); + let name = format!("xbridge::head{:x}::{what}", head.widget); + register_lock_fn(name, move || h.upgrade().is_some_and(|h| probe(&h))); + } + } + for (what, probe) in [ + ("state", (|x: &Crossbar| x.st.is_locked()) as fn(&Crossbar) -> bool), + ("irq_order", |x| x.order.is_locked()), + ] { + let x = Arc::downgrade(&self.xbow); + register_lock_fn(format!("xbridge::crossbar::{what}"), move || { + x.upgrade().is_some_and(|x| probe(&x)) + }); + } + } + fn pci_head(&self, widget: u8) -> &Arc { + assert!((HEAD_E..=HEAD_F).contains(&widget)); + &self.heads[(widget - HEAD_E) as usize] + } + pub fn head(&self, widget: u8) -> Arc { + self.pci_head(widget).clone() + } + pub fn crossbar(&self) -> Arc { + self.xbow.clone() + } + /// A reader-specific widget-0 view; the host reads port 10 by default. + pub fn crossbar_for_port(&self, port: u8) -> Arc { + assert!((8..=15).contains(&port)); + Arc::new(CrossbarView { + xbow: self.xbow.clone(), + port, + }) + } + /// Host-port processor projection uses the XBridge widget-0 alias. + pub fn widget(&self, widget: u8) -> Option> { + match widget { + 0 | HOST_WIDGET => Some(self.crossbar()), + HEAD_E | HEAD_F => Some(self.head(widget)), + _ => self + .routes + .targets + .read() + .get(widget as usize) + .and_then(Clone::clone), + } + } + /// The PCI bus behind head `widget`: functions are fitted with + /// [`PciBus::attach`] and the SuperIO interrupt is `bus.board_irq(BoardIrq(0))`. + pub fn bus(&self, widget: u8) -> &PciBus { + &self.pci_head(widget).bus + } + pub fn dma(&self, widget: u8, slot: u8) -> Arc { + assert!(slot < 8); + self.pci_head(widget).dma_for(slot) + } + pub fn set_interrupt_sender(&self, send: InterruptSender) { + *self.routes.interrupts.write() = Some(send); + } + pub fn set_dma_sender(&self, send: DmaSender) { + *self.routes.dma.write() = Some(send); + } + pub fn set_dma_64bit(&self, widget: u8, slot: u8, wide: bool) { + assert!(slot < 8); + let head = self.pci_head(widget); + { + let mut st = head.st.lock(); + st.dma_64bit = (st.dma_64bit & !(1 << slot)) | ((wide as u8) << slot); + } + // A 64-bit slot changes the byte order of the 64-bit alias. + head.bus.set_host_windows(&|| head.windows()); + } + pub fn set_wide_slot_mask(&self, widget: u8, mask: u8) { + self.pci_head(widget).st.lock().wide_slots = mask; + } + pub fn set_microlan(&self, widget: u8, bus: w1::Bus) { + *self.pci_head(widget).microlan.lock() = Some(bus); + } + /// An interrupt packet from the widget attached at `p.source_widget`, + /// routed as the heads' packets are. False when nothing takes it. + pub fn send_interrupt(&self, p: InterruptPacket) -> bool { + assert!((8..14).contains(&p.source_widget) && p.source_widget != HOST_WIDGET); + self.routes.interrupt(p) + } + pub fn attach_widget(&self, widget: u8, target: Arc) { + assert!((8..14).contains(&widget) && widget != HOST_WIDGET); + self.routes.targets.write()[widget as usize] = Some(target); + self.set_link_present(widget, true); + } + pub fn set_link_present(&self, widget: u8, present: bool) { + assert!((8..=15).contains(&widget)); + let mut st = self.xbow.st.lock(); + st.present = (st.present & !(1 << (widget - 8))) | ((present as u8) << (widget - 8)); + } + pub fn set_link_8bit(&self, widget: u8, enabled: bool) { + assert!((8..=15).contains(&widget)); + let mut st = self.xbow.st.lock(); + let bit = 1 << (widget - 8); + st.eight_bit = (st.eight_bit & !bit) | if enabled { bit } else { 0 }; + } + /// A pulse for each reset port, outside chip locks. A head's registers, + /// routed PCI lines and function headers are already reset. The board + /// must reset that port's attached engines through this callback. + pub fn set_reset_output(&self, output: Arc) { + *self.xbow.reset_output.write() = Some(output); + } + /// Time is supplied by the machine's virtual clock, in nanoseconds. + pub fn warm_reset(&self, source_port: u8, now_ns: u64) { + self.xbow.warm_reset(source_port, now_ns); + } + pub fn reset(&self) { + for h in &self.heads { + h.reset(); + } + let mut st = self.xbow.st.lock(); + st.reset(); + st.quiet_until = 0; + } +} +impl Resettable for XBridge { + fn power_on(&self) { + self.reset(); + } +} + +struct HeadState { + reg: [u32; 0x300 / 8], + ate: [u64; ATE_ENTRIES], + pins: u8, + dma_64bit: u8, + wide_slots: u8, +} +impl HeadState { + fn r(&self, off: usize) -> u32 { + self.reg[off / 8] + } + fn set(&mut self, off: usize, value: u32) { + self.reg[off / 8] = value; + } + fn reset(&mut self, speed: u8) { + self.reg.fill(0); + self.ate.fill(0); + self.pins = 0; + for (off, value) in [ + (0, 0x2d002049), + (8, 0x20), + (0x20, 0x0f00320f | ((speed as u32) << 4)), + (0x28, 0xfffff), + (reg::LINK_CONFIG, 0x03ff1810), + (0x80, 0x20000), + (0xa0, 0x20008), + (0xc0, 0x10000), + ] { + self.set(off, value); + } + for slot in 0..8 { + self.set( + 0x200 + slot * 8, + 0x18001000 + | if slot < 2 { + slot as u32 * 2 + } else { + slot as u32 + 2 + }, + ); + } + } + fn packet(&self, pin: Option, level: bool) -> InterruptPacket { + let upper = self.r(reg::INT_DEST_UPPER); + let mut address = ((upper as u64 & 0xffff) << 32) | self.r(reg::INT_DEST_LOWER) as u64; + let vector = if let Some(pin) = pin { + let host = self.r(reg::INTERRUPT_HOST_BASE + pin * 8); + address = (address & !(0x3ff << 38)) | (((host >> 8) as u64 & 0x3ff) << 38); + host as u8 + } else { + self.r(reg::INTERRUPT_ERROR_HOST) as u8 + }; + InterruptPacket { + source_widget: (self.r(reg::CONTROL) & 15) as u8, + widget: ((upper >> 16) & 15) as u8, + address, + vector, + level, + } + } +} +// Lock order: order, then microlan (NIC only), then st. +struct XBridgeHead { + widget: u8, + speed: u8, + st: Mutex, + order: Mutex<()>, + routes: Arc, + microlan: Mutex>, + /// The head's PCI bus: its functions, BAR windows and interrupt pins. + bus: PciBus, + /// Whole-page BAR mappings of the device and alias windows, which the + /// processor reaches without a search of the bus. + pio: PioDecode, + /// This head, for the DMA a function is given: it must not keep the head alive. + me: Weak, +} + +/// A run of the head's address space that one BAR covers whole, and the bus +/// device that answers it. +struct PioRun { + start: u64, + end: u64, + dev: Arc, +} + +/// Whole BAR-page runs are few and carry no RAM or native fault-handler state. +/// Readers clone one endpoint and release the table lock before calling a device. +#[derive(Default)] +struct PioDecode { + runs: RwLock>, +} +impl PioDecode { + fn publish(&self, mut runs: Vec) -> Vec { + runs.sort_unstable_by_key(|run| run.start); + debug_assert!(runs.windows(2).all(|pair| pair[0].end <= pair[1].start)); + std::mem::replace(&mut *self.runs.write(), runs) + } + fn target(&self, address: u64, size: usize) -> Option> { + let runs = self.runs.read(); + let i = runs.partition_point(|run| run.start <= address).checked_sub(1)?; + let run = &runs[i]; + (address.checked_add(size as u64)? <= run.end).then(|| run.dev.clone()) + } +} + +/// The head's PCI interrupt routing. Device `d`'s INTA reaches interrupt pin `d`; the +/// pins are shared with the board's own IRQs (the IOC3's SuperIO interrupt +/// is `BoardIrq(0)`). The Fuel uses devices 1, 4 and 5; the others follow the +/// same rule, and INTB to INTD are not wired. +pub static PCI_ROUTING: IntxRouting = IntxRouting { + devices: 8, + irqs: 8, + intx: &[ + (0, [Some(0), None, None, None]), + (1, [Some(1), None, None, None]), + (2, [Some(2), None, None, None]), + (3, [Some(3), None, None, None]), + (4, [Some(4), None, None, None]), + (5, [Some(5), None, None, None]), + (6, [Some(6), None, None, None]), + (7, [Some(7), None, None, None]), + ], + board_irqs: &[0, 1, 2, 3, 4, 5, 6, 7], +}; + +/// The head's CPU windows, by [`CpuWindow`] index: the eight device windows, +/// then the 32-bit and 64-bit memory aliases and the I/O alias. +const DEVICE_WINDOWS: usize = 8; +const ALIAS_MEM32: usize = 8; +const ALIAS_MEM64: usize = 9; +const ALIAS_IO: usize = 10; +const WINDOWS: usize = 11; + +struct SlotDma { + head: Weak, + slot: u8, +} +impl PciDma for SlotDma { + fn dma_read(&self, address: u64, bytes: &mut [u8]) -> PciDmaStatus { + if let Some(h) = self.head.upgrade() { + h.dma(self.slot, address, bytes, false) + } else { + bytes.fill(0); + PciDmaStatus::MasterAbort + } + } + fn dma_write(&self, address: u64, bytes: &[u8]) -> PciDmaStatus { + self.head.upgrade().map_or(PciDmaStatus::MasterAbort, |h| { + h.dma(self.slot, address, &mut bytes.to_vec(), true) + }) + } +} +/// What the head's bus asks of it: the DMA of a requester, and an interrupt +/// pin's level, as the OR of every device line and board IRQ routed to it. +impl PciHost for XBridgeHead { + fn dma_for(&self, requester: u8) -> Arc { + Arc::new(SlotDma { head: self.me.clone(), slot: requester }) + } + fn set_irq(&self, pin: u8, level: bool) { + self.pin(pin as usize, level); + } +} +impl XBridgeHead { + fn new(widget: u8, speed: u8, routes: Arc) -> Arc { + let mut st = HeadState { + reg: [0; 96], + ate: [0; ATE_ENTRIES], + pins: 0, + dma_64bit: 0, + wide_slots: 0, + }; + st.reset(speed); + let this = Arc::new_cyclic(|me: &Weak| Self { + widget, + speed, + st: Mutex::new(st), + order: Mutex::new(()), + routes, + microlan: Mutex::new(None), + // iris's bus numbers count per bus tree; the heads are separate trees. + bus: PciBus::new((widget - HEAD_E) * 16, &PCI_ROUTING, me.clone()), + pio: PioDecode::default(), + me: me.clone(), + }); + this.bus.set_host_windows(&|| this.windows()); + let head = Arc::downgrade(&this); + this.bus.on_bar_mappings(Box::new(move |runs| { + if let Some(head) = head.upgrade() { + head.publish_mappings(runs); + } + })); + this + } + fn pin(&self, pin: usize, level: bool) { + if pin >= 8 { + return; + } + let _order = self.order.lock(); + let packet = { + let mut st = self.st.lock(); + let bit = 1 << pin; + let old = st.pins & bit != 0; + st.pins = (st.pins & !bit) | if level { bit } else { 0 }; + let pins = st.pins as u32; + let status = (st.r(reg::INTERRUPT_STATUS) & !255) | pins; + st.set(reg::INTERRUPT_STATUS, status); + (old != level + && st.r(reg::INTERRUPT_ENABLE) & bit as u32 != 0 + && (level || st.r(reg::INTERRUPT_MODE) & bit as u32 != 0)) + .then(|| st.packet(Some(pin), level)) + }; + if let Some(packet) = packet { + self.routes.interrupt(packet); + } + } + fn reset(&self) { + { + let _order = self.order.lock(); + self.st.lock().reset(self.speed); + } + // RST#: headers, then functions, which drop their own lines. The + // Device(x) registers are back at reset, so the windows move too. + self.bus.reset(); + self.bus.set_host_windows(&|| self.windows()); + } + fn invalid(&self, address: u64) { + let _order = self.order.lock(); + let packet = { + let mut st = self.st.lock(); + let first = st.r(reg::INTERRUPT_STATUS) & (1 << 24) == 0; + let status = st.r(reg::INTERRUPT_STATUS) | (1 << 24); + st.set(reg::INTERRUPT_STATUS, status); + let multi = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 24); + st.set(reg::INTERRUPT_MULTIPLE, multi); + if first { + st.set(reg::ERR_UPPER, (address >> 32) as u32); + st.set(reg::ERR_LOWER, address as u32); + } + (first && st.r(reg::INTERRUPT_ENABLE) & (1 << 24) != 0).then(|| st.packet(None, true)) + }; + if let Some(p) = packet { + self.routes.interrupt(p); + } + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} invalid access at {address:#x}", + self.widget + ); + } + fn defined(off: usize) -> bool { + matches!(off, reg::IDENTIFICATION..=reg::TEST_PIN_CONTROL | reg::DIRECT_MAP | reg::SSRAM_PARITY_ERR | reg::ARBITRATION | reg::NIC | reg::BUS_TIMEOUT..=reg::PCI_ERR_LOWER | reg::INTERRUPT_STATUS..=reg::FORCE_PIN_LAST | reg::DEVICE_BASE..=reg::RRB_CLEAR) + } + fn deferred(address: u64) -> bool { + (0x300..0x600).contains(&address) + || (0x30000..0x30008).contains(&address) + || (0xc00000..0x1000000).contains(&address) + } + fn config(&self, address: u64, size: usize, value: Option) -> u64 { + if size == 8 { + let hi = self.config(address, 4, value.map(|v| v >> 32)); + return (hi << 32) | self.config(address + 4, 4, value.map(|v| v & 0xffffffff)); + } + let reg = ((address & 255) as usize) ^ (4 - size); + let function = ((address >> 8) & 7) as u8; + // The type 0 aperture selects a device by address; the type 1 + // aperture takes it, and the bus, from the PCI Type 1 Configuration + // register. Bus 0 there is this head's bus. + let (bus, device) = if address < 0x28000 { + (Some(self.bus.clone()), ((address - 0x20000) >> 12) as u8) + } else { + let sel = self.st.lock().r(reg::PCI_TYPE1_CONFIG); + (self.bus.find_bus(((sel >> 16) & 255) as u8), ((sel >> 11) & 31) as u8) + }; + let at = DevFn { device, function }; + let mask = u32::MAX >> (32 - size * 8); + if let Some(value) = value { + if let Some(bus) = bus { + bus.config_write(at, reg, size, value as u32 & mask); + } + return 0; + } + match bus.and_then(|b| b.config_read(at, reg, size)) { + Some(v) => v as u64, + None => { + let mut st = self.st.lock(); + let multi = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 11); + st.set(reg::INTERRUPT_MULTIPLE, multi); + mask as u64 + } + } + } + fn reg_write(&self, address: u64, value: u32) { + let off = address as usize & !7; + let _order = self.order.lock(); + let mut packets = Vec::new(); + if off == 0xb0 { + let pulse = (value >> 10) & 0x3ff; + let sample = (value >> 2) & 255; + let mut bus = self.microlan.lock(); + if let Some(bus) = bus.as_mut() { + let level = w1::slot::slot(bus, pulse, sample); + self.st.lock().set(reg::NIC, 2 | level as u32); + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: head {} NIC has no board", self.widget); + } + return; + } + { + let mut st = self.st.lock(); + let masked = match off { + reg::CONTROL => (value & 0x0fe1f20f) | ((self.speed as u32) << 4), + reg::REQUEST_TIMEOUT | reg::INT_DEST_UPPER => value & 0xfffff, + reg::INT_DEST_LOWER | reg::RRB_EVEN | reg::RRB_ODD => value, + reg::LINK_CONFIG => value & 0x03ffffff, + reg::TEST_PIN_CONTROL => value & 0xffffff, + reg::DIRECT_MAP => value & 0xf3ffff, + reg::ARBITRATION => value & 0x3ff7f, + reg::BUS_TIMEOUT => value & 0x1f03ff, + reg::PCI_TYPE1_CONFIG => value & 0xfff800, + reg::INTERRUPT_DEVICE => value & 0xffffff, + reg::INTERRUPT_ERROR_HOST | reg::INTERRUPT_MODE => value & 255, + reg::INTERRUPT_ENABLE => { + let value = value & 0x7fffffff; + let newly = value & !st.r(reg::INTERRUPT_ENABLE) & st.pins as u32; + st.set(off, value); + for pin in 0..8 { + if newly & (1 << pin) != 0 { + packets.push(st.packet(Some(pin), true)); + } + } + value + } + reg::INTERRUPT_RESET => { + let clear = (if value & 2 != 0 { 1 << 30 } else { 0 }) + | (if value & 8 != 0 { 1 << 24 } else { 0 }); + let status = st.r(reg::INTERRUPT_STATUS) & !clear; + st.set(reg::INTERRUPT_STATUS, status); + let multi = if value & 64 != 0 { + 0 + } else { + st.r(reg::INTERRUPT_MULTIPLE) & !clear + }; + st.set(reg::INTERRUPT_MULTIPLE, multi); + return; + } + reg::FORCE_INTERRUPT_BASE..=reg::FORCE_PIN_LAST => { + let pin = (off - reg::FORCE_INTERRUPT_BASE) / 8 & 7; + if st.r(reg::INTERRUPT_ENABLE) & (1 << pin) == 0 { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: masked force semantics unresolved"); + } else if off < reg::FORCE_PIN_BASE || st.pins & (1 << pin) != 0 { + packets.push(st.packet(Some(pin), true)); + } + 0 + } + reg::INTERRUPT_HOST_BASE..=reg::INTERRUPT_HOST_LAST => value & 0x3ffff, + reg::DEVICE_BASE..=reg::DEVICE_LAST => value & 0x1eefffff, + _ => { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} ignored register write {off:#x}", + self.widget + ); + return; + } + }; + if !(reg::FORCE_INTERRUPT_BASE..=reg::FORCE_PIN_LAST).contains(&off) { + st.set(off, masked); + } + } + for p in packets { + self.routes.interrupt(p); + } + drop(_order); + if off == reg::CONTROL || (reg::DEVICE_BASE..=reg::DEVICE_LAST).contains(&off) { + self.bus.set_host_windows(&|| self.windows()); + } + } + + /// The CPU windows, from the Device(x) registers; see [`WINDOWS`]. + fn windows(&self) -> Vec { + let st = self.st.lock(); + (0..WINDOWS) + .map(|kind| { + if kind < DEVICE_WINDOWS { + let (space, base, _) = Self::device(st.r(reg::DEVICE_BASE + kind * 8), kind); + let (cpu, len) = if kind < 2 { + (0x200000 + kind as u64 * 0x200000, 0x200000) + } else { + (0x600000 + (kind as u64 - 2) * 0x100000, 0x100000) + }; + CpuWindow { space, cpu, pci: base << 20, len, mask: u64::MAX } + } else { + let (cpu, len, space) = match kind { + ALIAS_MEM32 => (0x40000000, 0x40000000, Space::Mem), + ALIAS_MEM64 => (0x80000000, 0x40000000, Space::Mem), + _ => (0x100000000, 0x100000000, Space::Io), + }; + CpuWindow { space, cpu, pci: 0, len, mask: u64::MAX } + } + }) + .collect() + } + + /// Device(x) register `dev` of window `slot`: the space it opens onto, + /// its PCI base in megabytes, and whether it is a 64-bit device. + fn device(dev: u32, slot: usize) -> (Space, u32, bool) { + let memory = dev & (1 << 12) != 0; + let mut base = dev & if memory { 0x3ff } else { 0xfff }; + if slot < 2 { + base &= !1; + } + (if memory { Space::Mem } else { Space::Io }, base, dev & (1 << 22) != 0) + } + + /// The byte order of a CPU access to window `kind` reaching function `f`. A device + /// window follows its Device(x) register: the swap bit gives a byte + /// stream, otherwise the device's width gives word values. An alias + /// follows Control's memory or I/O swap bit; the 64-bit memory alias is + /// 64 bits wide for the slots set 64-bit. + fn order(&self, kind: usize, f: &dyn PciFunction) -> ByteOrder { + let st = self.st.lock(); + if kind < DEVICE_WINDOWS { + let dev = st.r(reg::DEVICE_BASE + kind * 8); + return if dev & (1 << 13) != 0 { ByteOrder::Stream } else { ByteOrder::values(if dev & (1 << 22) != 0 { 8 } else { 4 }) }; + } + if st.r(reg::CONTROL) & (1 << if kind == ALIAS_IO { 23 } else { 22 }) != 0 { + return ByteOrder::Stream; + } + let wide = kind == ALIAS_MEM64 && self.slot_of(f).is_some_and(|slot| st.dma_64bit & (1 << slot) != 0); + ByteOrder::values(if wide { 8 } else { 4 }) + } + + /// The device number `f` sits at. + fn slot_of(&self, f: &dyn PciFunction) -> Option { + let same = |g: &Arc| std::ptr::addr_eq(Arc::as_ptr(g), f as *const dyn PciFunction); + (0..DEVICE_WINDOWS as u8).find(|&device| (0..8).any(|function| self.bus.function(DevFn { device, function }).is_some_and(same))) + } + + /// The bus reports the BAR mappings the head's windows hold: serve them + /// from the head's own table. + fn publish_mappings(&self, runs: Vec) { + let runs = runs + .iter() + .map(|r| { + let order = self.order(r.window as usize, &*r.function); + let base = r.cpu.wrapping_sub(r.off as u64); + PioRun { start: r.cpu, end: r.cpu + r.len, dev: Arc::new(BarDevice::new(r.function.clone(), r.bar, base, u64::MAX, order)) } + }) + .collect(); + // Replaced runs may run shutdown callbacks from their destructor. + drop(self.pio.publish(runs)); + } + + /// The window kind, address space and PCI address that head address + /// `address` reaches, for the slow path. + fn pio_target(&self, address: u64) -> Option<(usize, Space, u32)> { + if (0x200000..0xc00000).contains(&address) { + let (slot, off) = if address < 0x600000 { + (((address - 0x200000) >> 21) as usize, address & 0x1fffff) + } else { + (2 + ((address - 0x600000) >> 20) as usize, address & 0xfffff) + }; + let (space, base, _) = Self::device(self.st.lock().r(reg::DEVICE_BASE + slot * 8), slot); + return Some((slot, space, (base << 20).wrapping_add(off as u32))); + } + match address { + 0x40000000..=0x7fffffff => Some((ALIAS_MEM32, Space::Mem, (address - 0x40000000) as u32)), + 0x80000000..=0xbfffffff => Some((ALIAS_MEM64, Space::Mem, (address - 0x80000000) as u32)), + 0x100000000..=0x1ffffffff => Some((ALIAS_IO, Space::Io, (address - 0x100000000) as u32)), + _ => None, + } + } + + /// A processor access that no whole-page run answers: the bus finds the + /// BAR and the window's byte order carries the bytes. Nothing claiming the + /// address reads as ones and drops a write. + fn slow_pio(&self, address: u64, size: usize, value: Option) -> u64 { + let Some((kind, space, pci)) = self.pio_target(address) else { return 0 }; + let Some((f, bar, off)) = self.bus.decode(space, pci) else { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} no PCI BAR for {space:?} {pci:#x}", + self.widget + ); + return if value.is_none() { u64::MAX >> (64 - size * 8) } else { 0 }; + }; + let order = self.order(kind, f); + match value { + None => order.read(f, bar, off, size), + Some(v) => { + order.write(f, bar, off, size, v); + 0 + } + } + } + fn access(&self, address: u64, size: usize, value: Option) -> u64 { + let write = value.is_some(); + if (0x20000..0x29000).contains(&address) { + return self.config(address, size, value); + } + if (0x200000..0xc00000).contains(&address) + || (0x40000000..0xc0000000).contains(&address) + || (0x100000000..0x200000000).contains(&address) + { + // An unaligned access can straddle a BAR's final mapped page. + if address & 0xfff > 0x1000 - size as u64 { + return self.slow_pio(address, size, value); + } + let Some(endpoint) = self.pio.target(address, size) else { + return self.slow_pio(address, size, value); + }; + return if let Some(v) = value { + match size { + 1 => { endpoint.write8(address, v as u8); } + 2 => { endpoint.write16(address, v as u16); } + 4 => { endpoint.write32(address, v as u32); } + _ => { endpoint.write64(address, v); } + } + 0 + } else { + match size { + 1 => endpoint.read8(address).data as u64, + 2 => endpoint.read16(address).data as u64, + 4 => endpoint.read32(address).data as u64, + _ => endpoint.read64(address).data, + } + }; + } + if size == 8 && address & 7 == 0 && (reg::ATE_UPPER_BASE..reg::ATE_LOWER_BASE).contains(&address) && write { + self.st.lock().ate[((address - reg::ATE_UPPER_BASE) / 8) as usize] = + value.unwrap() & 0x0000ffffffffffff; + return 0; + } + if size == 4 && address & 7 == 4 { + if !write { + if (reg::ATE_UPPER_BASE..reg::ATE_LOWER_BASE).contains(&address) { + return self.st.lock().ate[((address - reg::ATE_UPPER_BASE) / 8) as usize] >> 32; + } + if (reg::ATE_LOWER_BASE..reg::ATE_END).contains(&address) { + return self.st.lock().ate[((address - reg::ATE_LOWER_BASE) / 8) as usize] as u32 as u64; + } + } + if address < 0x300 && Self::defined(address as usize & !7) { + if let Some(value) = value { + self.reg_write(address, value as u32); + return 0; + } + return self.st.lock().r(address as usize & !7) as u64; + } + } + if Self::deferred(address) { + crate::dlog!(crate::devlog::LogModule::Bedrock, + "xbridge: head {} deferred access at {address:#x}", + self.widget + ); + } else { + self.invalid(address); + } + 0 + } + fn translate( + &self, + slot: u8, + address: u64, + write: bool, + packets: bool, + ) -> Option<(u8, u64, u64, bool)> { + let mut st = self.st.lock(); + let dev = st.r(reg::DEVICE_BASE + slot as usize * 8); + let wide = st.dma_64bit & (1 << slot) != 0; + let device_swap = dev & (1 << 19) != 0; + let (widget, target, swap, direct, page, wide64) = if (0x40000000..0x40800000) + .contains(&address) + || (0x60000000..0x60800000).contains(&address) + { + let shift = if st.r(reg::CONTROL) & (1 << 21) != 0 { 14 } else { 12 }; + let index = ((address & 0x1fffffff) >> shift) as usize; + if index >= ATE_MAPPED || st.ate[index] & 1 == 0 { + let v = st.r(reg::INTERRUPT_STATUS) | (1 << 30); + st.set(reg::INTERRUPT_STATUS, v); + let v = st.r(reg::INTERRUPT_MULTIPLE) | (1 << 30); + st.set(reg::INTERRUPT_MULTIPLE, v); + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge: invalid page DMA {address:#x}, entry {index}" + ); + return None; + } + let ate = st.ate[index]; + let mask = (1 << shift) - 1; + ( + ((ate >> 8) & 15) as u8, + (ate & 0x0000fffffffff000 & !mask) | (address & mask), + ate & 32 != 0 || address & (1 << 29) != 0, + false, + true, + false, + ) + } else if address >= 0x1000000000000000 { + ( + (address >> 60) as u8, + address & 0x0000ffffffffffff, + address & (1 << 55) != 0, + false, + false, + true, + ) + } else if (0x80000000..=0xffffffff).contains(&address) { + let direct = st.r(reg::DIRECT_MAP); + let mut target = ((direct as u64 & 0x1ffff) << 31) | (address & 0x7fffffff); + if direct & 0x1ffff == 0 && direct & (1 << 17) != 0 { + target += 0x20000000; + } + ( + ((direct >> 20) & 15) as u8, + target, + device_swap, + true, + false, + false, + ) + } else { + crate::dlog!(crate::devlog::LogModule::Bedrock, "xbridge: unsupported DMA {address:#x}"); + return None; + }; + let half = packets + && swap + && st.wide_slots & (1 << slot) != 0 + && if write { + direct || page || (wide64 && device_swap) + } else { + direct || (wide64 && wide && device_swap) || (page && address & (1 << 29) != 0) + }; + let mask = if swap || half { + 0 + } else if wide { + 7 + } else { + 3 + }; + Some((widget, target, mask, half)) + } + // Retain the existing memory path's probe before issuing a chunk's writes. + // The neutral helper alone can commit bytes before reporting a refusal. + fn write_memory(mem: &dyn BusDevice, address: u64, data: &[u8]) -> bool { + if data.is_empty() { + return true; + } + let first = address & !7; + let Some(end) = address + .checked_add(data.len() as u64) + .and_then(|a| a.checked_add(7)) + else { + return false; + }; + let mut probe = vec![0; ((end & !7) - first) as usize]; + read_bytes(mem, first, &mut probe) == BUS_OK && write_bytes(mem, address, data) == BUS_OK + } + fn dma(&self, slot: u8, address: u64, bytes: &mut [u8], write: bool) -> PciDmaStatus { + let sender = self.routes.dma.read().clone(); + let mut good = true; + let mut done = 0; + while done < bytes.len() { + let Some(at) = address.checked_add(done as u64) else { + if !write { + bytes[done..].fill(0); + } + return PciDmaStatus::MasterAbort; + }; + let n = (0x1000 - (at as usize & 0xfff)).min(bytes.len() - done); + if (0x3fff0000..0x40000000).contains(&at) { + if !write { + bytes[done..done + n].fill(0); + } + done += n; + continue; + } + if !write { + bytes[done..done + n].fill(0); + } + let translated = self.translate(slot, at, write, sender.is_some()); + let accepted = if let Some((widget, target, mask, half)) = translated { + if let Some(send) = &sender { + // Packet legs use naturally aligned accesses, retaining source identity. + let mut offset = 0; + let mut ok = true; + while offset < n { + let p = target + offset as u64; + let mut size = 8usize; + while size > n - offset || p & (size as u64 - 1) != 0 { + size /= 2; + } + let mut data = [0u8; 8]; + if write { + for i in 0..size { + data[i] = bytes[done + + offset + + if half { + i + } else { + i ^ (mask as usize & (size - 1)) + }]; + } + } + let placed = if half { + p + } else { + p ^ (mask & !(size as u64 - 1)) + }; + let packet = DmaPacket { + source_widget: (self.st.lock().r(reg::CONTROL) & 15) as u8, + slot, + widget, + address: placed, + write, + half_write_requested: half && write && size == 8, + half_read_requested: half && !write && size == 8, + }; + let part = send(packet, &mut data[..size]); + ok &= part; + if !write { + for i in 0..size { + bytes[done + offset + i] = if part { + data[if half { + i + } else { + i ^ (mask as usize & (size - 1)) + }] + } else { + 0 + }; + } + } + offset += size; + } + ok + } else { + let target_device = self.routes.targets.read()[widget as usize].clone(); + target_device.is_some_and(|mem| { + let low = target & !mask; + let high = (target + n as u64 + mask) & !mask; + let mut data = vec![0; (high - low) as usize]; + if write { + for i in 0..n { + data[(((target + i as u64) ^ mask) - low) as usize] = + bytes[done + i]; + } + if low == target && high == target + n as u64 { + Self::write_memory(&*mem, low, &data) + } else { + let mut ok = true; + for i in 0..n { + let p = (target + i as u64) ^ mask; + ok &= Self::write_memory( + &*mem, + p, + &data[(p - low) as usize..(p - low) as usize + 1], + ); + } + ok + } + } else { + // Preserve the old helper's covering-doubleword probe + // and zero the whole failed physical chunk, even if + // the neutral helper returned part of its contents. + let first = low & !7; + let mut fetched = vec![0; (((high + 7) & !7) - first) as usize]; + let ok = read_bytes(&*mem, first, &mut fetched) == BUS_OK; + for i in 0..n { + bytes[done + i] = if ok { + fetched[(((target + i as u64) ^ mask) - first) as usize] + } else { + 0 + }; + } + ok + } + }) + } + } else { + false + }; + if !accepted { + good = false; + } + done += n; + } + if good { + PciDmaStatus::Ok + } else { + PciDmaStatus::MasterAbort + } + } +} + +struct XbowState { + reg: [u32; 96], + status: u32, + present: u8, + eight_bit: u8, + quiet_until: u64, +} +impl XbowState { + fn new() -> Self { + let mut st = Self { + reg: [0; 96], + status: 0, + present: (1 << 2) | (1 << 6) | (1 << 7), + eight_bit: 0, + quiet_until: 0, + }; + st.reset(); + st + } + fn reset(&mut self) { + self.reg.fill(0); + self.status = 0; + for (off, value) in [ + (xbow::IDENTIFICATION, 0x2d000001), + (xbow::CONTROL, 0x80), + (xbow::PACKET_TIMEOUT, 0xfffff), + (xbow::INT_UPPER, 0x80000), + (xbow::LLP_CONTROL, 0x03ff1810), + (xbow::ARB_RELOAD, 2), + (xbow::NIC, 2), + (xbow::FENCE_BASE, 255), + ] { + self.reg[ix(off)] = value; + } + for i in 0..8 { + self.reg[ix(xbow::FENCE_BASE + 8 + i * 8)] = 255; + self.reg[ix(xbow::LINK_BASE + xbow::LINK_CONTROL + i * xbow::LINK_STRIDE)] = 0x80000; + } + } + fn packet(&self, level: bool) -> InterruptPacket { + let upper = self.reg[ix(xbow::INT_UPPER)]; + InterruptPacket { + source_widget: 0, + widget: ((upper >> 16) & 15) as u8, + address: ((upper as u64 & 0xffff) << 32) | self.reg[ix(xbow::INT_LOWER)] as u64, + vector: (upper >> 24) as u8, + level, + } + } +} +// Lock order: order, then st. Reset outputs run with both released. +struct Crossbar { + st: Mutex, + order: Mutex<()>, + routes: Arc, + heads: [Weak; 2], + reset_output: RwLock>>, +} +impl Crossbar { + fn reset_port(&self, port: u8) { + if (14..=15).contains(&port) { + if let Some(h) = self.heads[(port - 14) as usize].upgrade() { + h.reset(); + } + } + let out = self.reset_output.read().clone(); + if let Some(out) = out { + out(port); + } + } + fn warm_reset(&self, source: u8, now: u64) { + assert!((8..=15).contains(&source)); + let ports = { + let mut st = self.st.lock(); + if now < st.quiet_until { + return; + } + st.quiet_until = now.saturating_add(10240); + if st.reg[ix(xbow::FENCE_BASE)] & (1 << (source - 8)) == 0 { + return; + } + let ports: Vec<_> = (0..8) + .filter(|&i| st.reg[ix(xbow::FENCE_BASE + 8 + i * 8)] & (1 << (source - 8)) != 0) + .map(|i| i as u8 + 8) + .collect(); + st.reset(); + ports + }; + for port in ports { + self.reset_port(port); + } + } + fn access(&self, port: u8, address: u64, size: usize, value: Option) -> u64 { + let _order = self.order.lock(); + let mut packet = None; + let mut reset = None; + let mut access_error = false; + let result = { + let mut st = self.st.lock(); + let off = address as u32 & !7; + if size != 4 || address & 7 != 4 || address >= xbow::LINK_END as u64 || (READ_ONLY_BASE..=READ_ONLY_LAST).contains(&off) { + st.status |= 32; + access_error = true; + if st.reg[ix(xbow::CONTROL)] & 32 != 0 && st.status & (1 << 23) == 0 { + st.status |= 1 << 23; + packet = Some(st.packet(true)); + } + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: invalid access {address:#x} width {size}" + ); + 0 + } else if let Some(v) = value { + let v = v as u32; + if off >= xbow::LINK_BASE { + match (off - xbow::LINK_BASE) % xbow::LINK_STRIDE { + xbow::LINK_CONTROL => st.reg[ix(off)] = v & 0xbf7f01ff, + xbow::LINK_ARB_UPPER | xbow::LINK_ARB_LOWER => st.reg[ix(off)] = v, + xbow::LINK_RESET => reset = Some((8 + (off - xbow::LINK_BASE) / xbow::LINK_STRIDE) as u8), + _ => { + st.status |= 32; + access_error = true; + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: read-only link register offset {off:#x}" + ); + } + } + } else { + match off { + xbow::CONTROL => st.reg[ix(off)] = v & 0xfe, + xbow::PACKET_TIMEOUT => st.reg[ix(off)] = v & 0xfffff, + xbow::INT_UPPER => st.reg[ix(off)] = (v & 0xff07ffff) | 0x80000, + xbow::INT_LOWER => st.reg[ix(off)] = v, + xbow::ERR_CMD => { + st.reg[ix(xbow::ERR_CMD)] = 0; + st.reg[ix(xbow::ERR_UPPER)] = 0; + st.reg[ix(xbow::ERR_LOWER)] = 0; + } + xbow::LLP_CONTROL => st.reg[ix(off)] = v & 0x3ffffff, + xbow::ARB_RELOAD => st.reg[ix(off)] = v & 0x3f, + xbow::PERF_A | xbow::PERF_B => st.reg[ix(off)] = v & 0x700000, + xbow::NIC => { + st.reg[ix(off)] = (v & 0xffffc) | 2; + if v & 0xffc00 != 0 { + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: one-wire pulse unimplemented at register offset 0x70" + ); + } + } + xbow::FENCE_BASE..=FENCE_LAST => st.reg[ix(off)] = v & 255, + xbow::LOCK => { + if st.reg[ix(off)] == 0 { + st.reg[ix(off)] = v & 255; + } + } + xbow::LOCK_CLEAR => st.reg[ix(xbow::LOCK)] = 0, + _ => { + st.status |= 32; + access_error = true; + crate::dlog!( + crate::devlog::LogModule::Bedrock, + "xbridge-xbow: read-only register offset {off:#x}" + ); + } + } + } + 0 + } else if off >= xbow::LINK_BASE { + let link = (off - xbow::LINK_BASE) / xbow::LINK_STRIDE; + let present = st.present & (1 << link) != 0; + match (off - xbow::LINK_BASE) % xbow::LINK_STRIDE { + xbow::LINK_FLUSH => 0, + xbow::LINK_STATUS | xbow::LINK_STATUS_CLEAR => { + if present { + 1 << 31 + } else { + 1 << 5 + } + } + xbow::LINK_AUX_STATUS => { + (if present { 1 << 5 } else { 1 << 6 }) + | (if st.eight_bit & (1 << link) != 0 { + 1 << 4 + } else { + 0 + }) + } + xbow::LINK_RESET => { + st.status |= 32; + access_error = true; + 0 + } + _ => st.reg[ix(off)] as u64, + } + } else { + match off { + xbow::STATUS => (st.status | ((port as u32) << 6)) as u64, + xbow::STATUS_CLEAR => { + let v = st.status | ((port as u32) << 6); + if st.status & (1 << 23) != 0 { + packet = Some(st.packet(false)); + } + st.status &= !((1 << 23) | 0x3f); + v as u64 + } + xbow::LOCK_CLEAR => st.reg[ix(xbow::LOCK)] as u64, + _ => st.reg[ix(off)] as u64, + } + } + }; + if access_error && packet.is_none() { + let mut st = self.st.lock(); + if st.reg[ix(xbow::CONTROL)] & 32 != 0 && st.status & (1 << 23) == 0 { + st.status |= 1 << 23; + packet = Some(st.packet(true)); + } + } + if let Some(p) = packet { + self.routes.interrupt(p); + } + // Reset outputs may call back into the chip, so release ordering too. + drop(_order); + if let Some(port) = reset { + self.reset_port(port); + } + result + } +} +struct CrossbarView { + xbow: Arc, + port: u8, +} + +macro_rules! bus_view { + ($ty:ty, $read:expr) => { + impl BusDevice for $ty { + fn read8(&self, a: u64) -> BusRead8 { + BusRead8::ok(($read)(self, a, 1, None) as u8) + } + fn read16(&self, a: u64) -> BusRead16 { + BusRead16::ok(($read)(self, a, 2, None) as u16) + } + fn read32(&self, a: u64) -> BusRead32 { + BusRead32::ok(($read)(self, a, 4, None) as u32) + } + fn read64(&self, a: u64) -> BusRead64 { + BusRead64::ok(($read)(self, a, 8, None)) + } + fn write8(&self, a: u64, v: u8) -> u32 { + ($read)(self, a, 1, Some(v as u64)); + BUS_OK + } + fn write16(&self, a: u64, v: u16) -> u32 { + ($read)(self, a, 2, Some(v as u64)); + BUS_OK + } + fn write32(&self, a: u64, v: u32) -> u32 { + ($read)(self, a, 4, Some(v as u64)); + BUS_OK + } + fn write64(&self, a: u64, v: u64) -> u32 { + ($read)(self, a, 8, Some(v)); + BUS_OK + } + } + }; +} +bus_view!(XBridgeHead, |s: &XBridgeHead, a, n, v| s.access(a, n, v)); +bus_view!(Crossbar, |s: &Crossbar, a, n, v| s.access( + HOST_WIDGET, + a, + n, + v +)); +bus_view!(CrossbarView, |s: &CrossbarView, a, n, v| s + .xbow + .access(s.port, a, n, v)); + +#[cfg(test)] +mod tests; diff --git a/src/dev/xbridge/tests.rs b/src/dev/xbridge/tests.rs new file mode 100644 index 000000000..78ec74298 --- /dev/null +++ b/src/dev/xbridge/tests.rs @@ -0,0 +1,756 @@ +// Cases follow tests/qtest/sn1-xbridge-test.c (page-dma, interrupt-packets, +// access-errors), sn1-xbridge-chip-test.c (head-dma-reaches-the-host, +// warm-reset-reaches-the-heads) and sn1-xbridge-xbow-test.c (reset-values, +// lock, fences, link-reset-one-shot) in jamesbraid/qemu-sgi-origami. +use super::*; +use crate::pci::BoardIrq; +use crate::traits::BUS_ERR; + +fn at(device: u8) -> DevFn { + DevFn { device, function: 0 } +} +struct Memory(Mutex>); +impl BusDevice for Memory { + fn read32(&self, a: u64) -> BusRead32 { + let mem = self.0.lock(); + if a as usize + 4 > mem.len() { + return BusRead32::err(); + } + BusRead32::ok( + mem[a as usize..a as usize + 4] + .iter() + .fold(0, |v, &b| (v << 8) | b as u32), + ) + } + fn write32(&self, a: u64, v: u32) -> u32 { + let mut mem = self.0.lock(); + if a as usize + 4 > mem.len() { + return BUS_ERR; + } + for i in 0..4 { + mem[a as usize + i] = (v >> (24 - i * 8)) as u8; + } + BUS_OK + } + + fn read64(&self, a: u64) -> BusRead64 { + let mem = self.0.lock(); + if a as usize + 8 > mem.len() { + return BusRead64::err(); + } + BusRead64::ok( + mem[a as usize..a as usize + 8] + .iter() + .fold(0, |v, &b| (v << 8) | b as u64), + ) + } + fn write64(&self, a: u64, v: u64) -> u32 { + let mut mem = self.0.lock(); + if a as usize + 8 > mem.len() { + return BUS_ERR; + } + for i in 0..8 { + mem[a as usize + i] = (v >> (56 - i * 8)) as u8; + } + BUS_OK + } + fn write64_masked(&self, a: u64, v: u64, mask: u64) -> u32 { + let old = self.read64(a); + if old.status != BUS_OK { + return BUS_ERR; + } + self.write64(a, (old.data & !mask) | (v & mask)) + } + + fn read8(&self, a: u64) -> BusRead8 { + match self.0.lock().get(a as usize) { + Some(&value) => BusRead8 { + data: value, + status: BUS_OK, + }, + None => BusRead8 { + data: 0, + status: BUS_ERR, + }, + } + } + fn write8(&self, a: u64, v: u8) -> u32 { + match self.0.lock().get_mut(a as usize) { + Some(b) => { + *b = v; + BUS_OK + } + None => BUS_ERR, + } + } +} +fn chip() -> (Arc, Arc) { + let mem = Arc::new(Memory(Mutex::new(vec![0; 0x8000]))); + (XBridge::new(mem.clone()), mem) +} + +/// The machine lock registry must not extend the lifetime of a chip or attached function. +#[test] +fn lock_registry_does_not_keep_the_chip_or_its_devices_alive() { + let memory = Arc::new(Memory(Mutex::new(vec![0; 0x8000]))); + let memory_ref = Arc::downgrade(&memory); + let chip = XBridge::new(memory); + let chip_ref = Arc::downgrade(&chip); + let heads = chip.heads.each_ref().map(Arc::downgrade); + let crossbar = Arc::downgrade(&chip.xbow); + let card = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + let card_ref = Arc::downgrade(&card); + chip.bus(15).attach(at(1), card.clone()); + chip.set_reset_output(Arc::new(move |_| card.state.config().reset())); + chip.head(15).write32(0x21010, 0x02000000); + chip.head(15).write32(0x21004, 2); + assert!(!chip.pci_head(15).pio.runs.read().is_empty()); + chip.register_locks(); + + assert!(heads.iter().all(|head| head.upgrade().is_some())); + assert!(crossbar.upgrade().is_some()); + assert!(card_ref.upgrade().is_some()); + assert!(memory_ref.upgrade().is_some()); + drop(chip); + assert!(chip_ref.upgrade().is_none()); + assert!(heads.iter().all(|head| head.upgrade().is_none())); + assert!(crossbar.upgrade().is_none()); + assert!(card_ref.upgrade().is_none()); + assert!(memory_ref.upgrade().is_none()); +} +/// A DMA packet reports failure without losing the valid prefix already transferred. +#[test] +fn page_dma_keeps_valid_prefix_and_refuses_invalid_suffix() { + physical_chunk_refusal_does_not_publish_a_prefix(); + let (c, mem) = chip(); + let f = c.head(15); + f.write64(reg::ATE_UPPER_BASE as u64, 0x2000 | (10 << 8) | 0x21); + let dma = c.dma(15, 2); + let bytes = [1, 2, 3, 4, 5, 6, 7, 8]; + assert_eq!(dma.dma_write(0x40000ffc, &bytes), PciDmaStatus::MasterAbort); + assert_eq!(&mem.0.lock()[0x2ffc..0x3000], &[1, 2, 3, 4]); + assert_eq!(&mem.0.lock()[0x3000..0x3004], &[0; 4]); + let mut read = [0xff; 8]; + assert_eq!( + dma.dma_read(0x40000ffc, &mut read), + PciDmaStatus::MasterAbort + ); + assert_eq!(read, [1, 2, 3, 4, 0, 0, 0, 0]); + assert_ne!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 30), 0); + f.write32((reg::INTERRUPT_RESET + 4) as u64, 2); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 30), 0); + f.write64((reg::ATE_UPPER_BASE + 800 * 8) as u64, 0xffff123456789abc); + assert_eq!(f.read32((reg::ATE_UPPER_BASE + 4 + 800 * 8) as u64).data, 0x1234); + assert_eq!(f.read32((reg::ATE_LOWER_BASE + 4 + 800 * 8) as u64).data, 0x56789abc); + f.write64((reg::ATE_UPPER_BASE + 512 * 8) as u64, 0x2a21); + f.write32((reg::DIRECT_MAP + 4) as u64, 9 << 20); + for (address, status) in [(0x80002000, PciDmaStatus::MasterAbort), + (0x3fff8000, PciDmaStatus::Ok), (0x40800000, PciDmaStatus::MasterAbort), + (0x40200000, PciDmaStatus::MasterAbort)] { + let mut bytes = [0xff; 4]; + assert_eq!(dma.dma_read(address, &mut bytes), status); + assert_eq!(bytes, [0; 4]); + } +} +// A mapped memory range whose middle word refuses accesses. +struct FaultMemory(Arc); +impl BusDevice for FaultMemory { + fn read8(&self, a: u64) -> BusRead8 { + if (0x2004..0x2008).contains(&a) { + BusRead8::err() + } else { + self.0.read8(a) + } + } + fn write8(&self, a: u64, v: u8) -> u32 { + if (0x2004..0x2008).contains(&a) { + BUS_ERR + } else { + self.0.write8(a, v) + } + } + fn read32(&self, a: u64) -> BusRead32 { + if (0x2004..0x2008).contains(&a) { + BusRead32::err() + } else { + self.0.read32(a) + } + } + fn write32(&self, a: u64, v: u32) -> u32 { + if (0x2004..0x2008).contains(&a) { + BUS_ERR + } else { + self.0.write32(a, v) + } + } +} +fn physical_chunk_refusal_does_not_publish_a_prefix() { + let mem = Arc::new(Memory(Mutex::new(vec![0xaa; 0x8000]))); + let c = XBridge::new(Arc::new(FaultMemory(mem.clone()))); + let f = c.head(15); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write32((reg::DEVICE_BASE + 4) as u64, 1 << 19); + let dma = c.dma(15, 0); + let mut bytes = [0xff; 8]; + assert_eq!( + dma.dma_read(0x80002002, &mut bytes), + PciDmaStatus::MasterAbort + ); + assert_eq!( + bytes, [0; 8], + "a failed physical chunk zeroes its readable prefix and suffix" + ); + assert_eq!( + dma.dma_write(0x80002002, &[1; 8]), + PciDmaStatus::MasterAbort + ); + assert_eq!( + &mem.0.lock()[0x2000..0x2010], + &[0xaa; 16], + "the probe prevents writes before or after the refused word" + ); +} +/// Enabling an asserted guest interrupt pin delivers its programmed destination. +#[test] +fn interrupt_keeps_destination_and_enabling_a_live_pin_asserts() { + let (c, _) = chip(); + let f = c.head(15); + let packets = Arc::new(Mutex::new(Vec::new())); + let capture = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + capture.lock().push(p); + true + })); + f.write32((reg::INT_DEST_UPPER + 4) as u64, 0xa0001); + f.write32((reg::INT_DEST_LOWER + 4) as u64, 0x23450080); + f.write32((reg::INTERRUPT_HOST_BASE + 4) as u64, (0x155 << 8) | 0x42); + let wire = c.bus(15).board_irq(BoardIrq(0)); + wire.set(true); + assert!(packets.lock().is_empty()); + f.write32((reg::INTERRUPT_ENABLE + 4) as u64, 1); + f.write32((reg::INTERRUPT_MODE + 4) as u64, 1); + wire.set(false); + let p = packets.lock(); + assert_eq!(p.len(), 2); + assert_eq!(p[0].source_widget, 15); + assert_eq!(p[0].widget, 10); + assert_eq!(p[0].vector, 0x42); + assert_eq!(p[0].address, (0x123450080 & !(0x3ff << 38)) | (0x155 << 38)); + assert!(p[0].level); + assert!(!p[1].level); + drop(p); + // A device's INTA reaches the head's pin of the same number. + let card = Card::new(); + c.bus(15).attach(at(2), card.clone()); + card.state.irq().unwrap().set(true); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & 255, 1 << 2); + // RST# reaches the function, which drops its own line. + c.reset(); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & 255, 0); +} + +/// Each head has a bus of its own: the same device number on the two heads +/// is two functions, configuration reaches only the head addressed, and a +/// function's INTA raises its own head's pin of that number alone. +#[test] +fn each_head_has_its_own_configuration_space_and_interrupt_pins() { + let (c, _) = chip(); + let (first, second) = (Card::new(), Card::new()); + c.bus(HEAD_E).attach(at(3), first.clone()); + c.bus(HEAD_F).attach(at(3), second.clone()); + let (e, f) = (c.head(HEAD_E), c.head(HEAD_F)); + for head in [&e, &f] { + assert_eq!(head.read32(0x23000).data, 0x56781234); + } + assert_eq!(e.write32(0x2303c, 0x5a), BUS_OK); + assert_eq!(first.state.config().read(15) & 0xff, 0x5a); + assert_eq!(second.state.config().read(15) & 0xff, 0, "the other head's function is untouched"); + let pins = |head: &Arc| head.read32((reg::INTERRUPT_STATUS + 4) as u64).data & 255; + second.state.irq().unwrap().set(true); + assert_eq!((pins(&e), pins(&f)), (0, 1 << 3)); + first.state.irq().unwrap().set(true); + assert_eq!((pins(&e), pins(&f)), (1 << 3, 1 << 3)); + second.state.irq().unwrap().set(false); + assert_eq!((pins(&e), pins(&f)), (1 << 3, 0)); + // An unclaimed slot reads as ones and notes the failed cycle. + assert_eq!(f.read32(0x24000).data, u32::MAX); + assert_ne!(f.read32((reg::INTERRUPT_MULTIPLE + 4) as u64).data & (1 << 11), 0); + assert_eq!(e.read32((reg::INTERRUPT_MULTIPLE + 4) as u64).data & (1 << 11), 0); +} + +/// The guest receives host DMA packets with their original widget and address placement. +#[test] +fn host_dma_places_packets_and_rejects_invalid_addresses() { + let (c, mem) = chip(); + let f = c.head(15); + let d = c.dma(15, 0); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write64(reg::ATE_UPPER_BASE as u64, 0x3000 | (10 << 8) | 1); + for (wide, address, target, expected) in [ + (false, 0x80002000, 0x2000, [3, 2, 1, 0, 7, 6, 5, 4]), + (true, 0x80002000, 0x2000, [7, 6, 5, 4, 3, 2, 1, 0]), + (true, 0x60000000, 0x3000, [0, 1, 2, 3, 4, 5, 6, 7]), + (true, (10 << 60) | (1 << 55) | 0x2000, 0x2000, [0, 1, 2, 3, 4, 5, 6, 7]), + ] { + c.set_dma_64bit(15, 0, wide); + assert_eq!(d.dma_write(address, &[0, 1, 2, 3, 4, 5, 6, 7]), PciDmaStatus::Ok); + assert_eq!(&mem.0.lock()[target..target + 8], &expected); + } + let records = Arc::new(Mutex::new(Vec::new())); + let out = records.clone(); + c.set_dma_sender(Arc::new(move |p, b| { + out.lock().push((p, b.to_vec())); + true + })); + c.set_wide_slot_mask(15, 1 << 3); + c.set_dma_64bit(15, 3, true); + f.write32((reg::CONTROL + 4) as u64, 8); + f.write32((reg::DIRECT_MAP + 4) as u64, 10 << 20); + f.write32((reg::DEVICE_BASE + 3 * 8 + 4) as u64, 1 << 19); + assert_eq!( + c.dma(15, 3) + .dma_write(0x80002000, &[0, 1, 2, 3, 4, 5, 6, 7]), + PciDmaStatus::Ok + ); + let r = records.lock(); + assert_eq!(r.len(), 1); + assert_eq!(r[0].0.source_widget, 8); + assert_eq!(r[0].0.slot, 3); + assert_eq!(r[0].0.widget, 10); + assert_eq!(r[0].0.address, 0x2000); + assert!(r[0].0.half_write_requested); + assert_eq!(r[0].1, [0, 1, 2, 3, 4, 5, 6, 7]); + drop(r); + c.set_dma_sender(Arc::new(|p, b| { + if p.address >= 0x2008 { + return false; + } + b.fill(0x42); + true + })); + let mut bytes = [0xff; 16]; + assert_eq!( + c.dma(15, 0).dma_read(0x80002000, &mut bytes), + PciDmaStatus::MasterAbort + ); + assert_eq!( + bytes, + [0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0x42, 0, 0, 0, 0, 0, 0, 0, 0] + ); +} +/// A PCI function may inspect XBridge from its link-reset callback. +#[test] +fn link_reset_callback_resets_a_function_and_can_reenter_the_chip() { + use crate::dev::isp1040::{mbox, reg as isp_reg, CTRL_INT_ENABLE, CTRL_RISC_INT_ENABLE, HOST_CMD_SHIFT}; + use crate::dev::isp12160::Isp12160; + + let (c, _) = chip(); + let isp = Arc::new(Isp12160::new(Isp12160::new_bus(), Isp12160::new_bus())); + c.bus(15).attach(at(1), isp.clone()); + let pin1 = |c: &XBridge| c.head(15).read32((reg::INTERRUPT_STATUS + 4) as u64).data & 2 != 0; + isp.state().config().write(1, 6); + isp.state().config().write(5, 0x02000000); + isp.bar_write(1, isp_reg::BUS_CTRL, u32::from(CTRL_INT_ENABLE | CTRL_RISC_INT_ENABLE), 2); + isp.bar_write(1, isp_reg::MAILBOX_BASE, u32::from(mbox::NOP), 2); + isp.bar_write(1, isp_reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT, 2); + assert!(pin1(&c)); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE, 2), u32::from(mbox::COMPLETE)); + + let weak = Arc::downgrade(&c); + let reset_isp = isp.clone(); + c.set_reset_output(Arc::new(move |port| { + assert_eq!(port, 15); + let chip = weak.upgrade().unwrap(); + // Callbacks own attached engines and may inspect or rewire the chip. + assert_eq!(chip.head(port).read32((reg::DIRECT_MAP + 4) as u64).data, 0x20000); + assert_eq!(chip.crossbar().read32((xbow::IDENTIFICATION + 4) as u64).data, 0x2d000001); + assert_eq!(reset_isp.state().config().command(), 0); + reset_isp.power_on(); + chip.set_reset_output(Arc::new(|_| {})); + })); + c.head(15).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.crossbar().write32((xbow::LINK_BASE + 7 * xbow::LINK_STRIDE + xbow::LINK_RESET + 4) as u64, 1); + assert!(!pin1(&c)); + assert_eq!(isp.state().config().bar_base(1), 0); + assert_eq!(isp.bar_read(1, isp_reg::BUS_CTRL, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::BUS_STATUS, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE, 2), 0); + assert_eq!(isp.bar_read(1, isp_reg::MAILBOX_BASE + 2, 2), 0x4953); +} + +/// Only admitted guest ports receive warm-reset notifications after both heads reset. +#[test] +fn warm_reset_notifies_only_admitted_ports_after_head_state_is_reset() { + let (c, _) = chip(); + let ports = Arc::new(Mutex::new(Vec::new())); + let output = ports.clone(); + let weak = Arc::downgrade(&c); + c.set_reset_output(Arc::new(move |port| { + if port >= 14 { + assert_eq!(weak.upgrade().unwrap().head(port).read32((reg::DIRECT_MAP + 4) as u64).data, 0x20000); + } + output.lock().push(port); + })); + let x = c.crossbar(); + x.write32((xbow::FENCE_BASE + 8 + 6 * 8 + 4) as u64, 0xff & !(1 << 2)); + c.head(14).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.head(15).write32((reg::DIRECT_MAP + 4) as u64, 0xa00000); + c.warm_reset(10, 0); + assert_eq!(*ports.lock(), vec![8, 9, 10, 11, 12, 13, 15]); + assert_eq!(c.head(14).read32((reg::DIRECT_MAP + 4) as u64).data, 0xa00000); + ports.lock().clear(); + c.warm_reset(10, 10239); + assert!(ports.lock().is_empty()); + c.warm_reset(10, 10240); + assert_eq!(*ports.lock(), vec![8, 9, 10, 11, 12, 13, 14, 15]); +} +/// The crossbar reports the first error once and retains its programmed destination. +#[test] +fn crossbar_errors_deliver_once_then_clear_the_original_destination() { + let (c, _) = chip(); + let x = c.crossbar(); + let packets = Arc::new(Mutex::new(Vec::new())); + let out = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + out.lock().push(p); + true + })); + x.write32((xbow::INT_UPPER + 4) as u64, 0x550a0000); + x.write32((xbow::INT_LOWER + 4) as u64, 0x80); + x.write32((xbow::CONTROL + 4) as u64, 0x20); + x.read64(xbow::IDENTIFICATION as u64); + x.write32((xbow::IDENTIFICATION + 4) as u64, 1); + assert_eq!(packets.lock().len(), 1); + assert_eq!(x.read32((xbow::STATUS_CLEAR + 4) as u64).data & ((1 << 23) | 32), (1 << 23) | 32); + let p = packets.lock(); + assert_eq!(p.len(), 2); + assert_eq!(p[0].vector, 0x55); + assert_eq!(p[0].widget, 10); + assert_eq!(p[0].address, 0x80); + assert!(p[0].level); + assert!(!p[1].level); +} + +struct Card { + state: crate::pci::PciFunctionState, + bytes: Mutex>, +} +impl Card { + fn new() -> Arc { + Self::with_status_events(0) + } + fn with_status_events(mask: u16) -> Arc { + Self::with_bar(crate::pci::Bar::Mem(32), mask) + } + fn with_bar(bar: crate::pci::Bar, mask: u16) -> Arc { + use crate::pci::{Bar, PciConfig, PciFunctionState}; + let size = match bar { Bar::Mem(size) | Bar::Io(size) | Bar::Mem64(size) => size as usize, Bar::Unused => 0 }; + Arc::new(Self { + state: PciFunctionState::new(PciConfig::new_with_policy( + 0x1234, + 0x5678, + 0, + 0, + [ + bar, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + ], + 1, + crate::pci::PciConfigPolicy { status_w1c_mask: mask, ..Default::default() }, + )), + bytes: Mutex::new((0..size).map(|i| i as u8).collect()), + }) + } +} +impl PciFunction for Card { + fn state(&self) -> &crate::pci::PciFunctionState { + &self.state + } + fn bar_read(&self, _: usize, off: u32, width: usize) -> u32 { + let b = self.bytes.lock(); + (0..width).fold(0, |v, i| v | ((b[off as usize + i] as u32) << (i * 8))) + } + fn bar_write(&self, _: usize, off: u32, value: u32, width: usize) { + let mut b = self.bytes.lock(); + for i in 0..width { + b[off as usize + i] = (value >> (i * 8)) as u8; + } + } + fn reset(&self) { + if let Some(line) = self.state.irq() { + line.set(false); + } + } +} +/// The guest can identify the first invalid head access and its error destination. +#[test] +fn invalid_register_captures_first_address_and_reports_error_destination() { + let (c, _) = chip(); + let f = c.head(15); + let packets = Arc::new(Mutex::new(Vec::new())); + let out = packets.clone(); + c.set_interrupt_sender(Arc::new(move |p| { + out.lock().push(p); + true + })); + f.write32((reg::INT_DEST_UPPER + 4) as u64, 0xa0000); + f.write32((reg::INT_DEST_LOWER + 4) as u64, 0x80); + f.write32((reg::INTERRUPT_ERROR_HOST + 4) as u64, 73); + f.write32((reg::INTERRUPT_ENABLE + 4) as u64, 1 << 24); + f.read32(0x1000004); + f.read32(0x2000004); + assert_eq!(f.read32((reg::ERR_LOWER + 4) as u64).data, 0x1000004); + assert_eq!(packets.lock().len(), 1); + assert_eq!(packets.lock()[0].vector, 73); + f.write32((reg::INTERRUPT_RESET + 4) as u64, 8); + assert_eq!(f.read32((reg::INTERRUPT_STATUS + 4) as u64).data & (1 << 24), 0); +} +/// Narrow guest configuration writes preserve status bits outside the written bytes. +#[test] +fn partial_configuration_writes_preserve_unwritten_status_events() { + let (c, _) = chip(); + let f = c.head(15); + let card = Card::with_status_events(0xc000); + c.bus(15).attach(at(0), card.clone()); + for (bus, expected) in [(0, 0x56781234), (1, u32::MAX)] { + f.write32((reg::PCI_TYPE1_CONFIG + 4) as u64, bus << 16); + assert_eq!(f.read32(0x28000).data, expected); + } + card.state.config().set_status_bits(0xc000); + assert_eq!(f.write16(0x20006, 0x0102), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0102); + assert_eq!(f.write8(0x20007, 3), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0103); + assert_eq!(f.write8(0x20006, 2), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0203); + assert_eq!(f.write8(0x20005, 0xff), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0xc200_0203); + assert_eq!(f.write8(0x20004, 0x80), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x4200_0203); + assert_eq!(f.write16(0x20004, 0x4000), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x0200_0203); + card.state.config().set_status_bits(0xc000); + assert_eq!(f.write32(0x20004, 0x8000_0002), BUS_OK); + assert_eq!(f.read32(0x20004).data, 0x4200_0002); +} + +fn assert_pio_matches_slow(head: &XBridgeHead, card: &Card, address: u64, mask: usize) { + for size in [1, 2, 4, 8] { + for offset in (0..32).step_by(size) { + let at = address + offset as u64; + let before = card.bytes.lock().clone(); + let expected = (0..size).fold(0u64, |v, i| (v << 8) | before[(offset + i) ^ mask] as u64); + assert_eq!(head.access(at, size, None), expected, "read {at:#x} width {size}"); + assert_eq!(head.slow_pio(at, size, None), expected); + head.access(at, size, Some(0x0123456789abcdef)); + let fast = card.bytes.lock().clone(); + *card.bytes.lock() = before.clone(); + head.slow_pio(at, size, Some(0x0123456789abcdef)); + assert_eq!(*card.bytes.lock(), fast, "write {at:#x} width {size}"); + *card.bytes.lock() = before; + } + } +} + +/// Native BAR dispatch preserves the bridge byte order at every supported access width. +#[test] +fn native_bar_pages_preserve_the_byte_order_at_every_width() { + for memory in [false, true] { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let bar = if memory { crate::pci::Bar::Mem(0x2000) } else { crate::pci::Bar::Io(0x2000) }; + let card = Card::with_bar(bar, 0); + // Device window 0 reaches a function owned by slot 3. + chip.bus(15).attach(at(3), card.clone()); + head.write32(0x23010, 0x02000000 | u32::from(!memory)); + head.write32(0x23004, if memory { 2 } else { 1 }); + for wide in [false, true] { + chip.set_dma_64bit(15, 3, wide); + for stream in [false, true] { + head.write32((reg::DEVICE_BASE + 4) as u64, 0x20 | if memory { 1 << 12 } else { 0 } + | if wide { 1 << 22 } else { 0 } | if stream { 1 << 13 } else { 0 }); + assert!(runs(head).contains(&(0x200000, 0x202000))); + assert_pio_matches_slow(head, &card, 0x200000, if stream { 0 } else if wide { 7 } else { 3 }); + for swap in [false, true] { + head.write32((reg::CONTROL + 4) as u64, if swap { 1 << if memory { 22 } else { 23 } } else { 0 }); + let aliases = if memory { + vec![(0x42000000, false), (0x82000000, wide)] + } else { vec![(0x102000000, false)] }; + for (address, wide) in aliases { + assert_pio_matches_slow(head, &card, address, if swap { 0 } else if wide { 7 } else { 3 }); + } + } + } + } + } +} + +/// The runs the head serves without searching its bus. +fn runs(head: &XBridgeHead) -> Vec<(u64, u64)> { + head.pio.runs.read().iter().map(|run| (run.start, run.end)).collect() +} + +/// A moved, disabled or reset BAR stops reaching its former place. +#[test] +fn bar_moves_disable_and_reset_revoke_native_pages() { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let card = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + chip.bus(15).attach(at(0), card); + head.write32(0x20010, 0x02000000); + head.write32(0x20004, 2); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1022); + assert_eq!(head.read32(0x200000).data, u32::MAX); + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + assert_eq!(head.read32(0x200000).data, 0x03020100); + for i in 0..128 { + let base = 0x02000000 + i * 0x1000; + head.write32(0x20010, base); + assert_eq!(head.read32(0x40000000 + base as u64).data, 0x03020100); + if i != 0 { + assert_eq!(head.read32(0x40000000 + base as u64 - 0x1000).data, u32::MAX); + } + } + head.write8(0x20007, 0); + assert!(runs(head).is_empty()); + assert_eq!(head.read32(0x4207f000).data, u32::MAX); + head.write8(0x20007, 2); + assert!(!runs(head).is_empty()); + chip.crossbar().write32((xbow::LINK_BASE + 7 * xbow::LINK_STRIDE + xbow::LINK_RESET + 4) as u64, 1); + assert!(runs(head).is_empty()); + assert_eq!(head.read32(0x4207f000).data, u32::MAX); + head.write32(0x20010, 0x02000000); + head.write32(0x20004, 2); + assert_eq!(head.read32(0x42000000).data, 0x03020100); +} + +/// BARs that claim the same pages stay on the slow decoder, which takes the +/// first function in device order. +#[test] +fn overlapping_bar_pages_use_the_slow_decoder() { + let (chip, _) = chip(); + let head = chip.pci_head(15); + let small = Card::with_bar(crate::pci::Bar::Mem(0x1000), 0); + let large = Card::with_bar(crate::pci::Bar::Mem(0x2000), 0); + small.bytes.lock().fill(0xa5); + chip.bus(15).attach(at(0), small); + chip.bus(15).attach(at(1), large.clone()); + for slot in 0..2 { + head.write32(0x20010 + slot * 0x1000, 0x02000000); + head.write32(0x20004 + slot * 0x1000, 2); + } + head.write32((reg::DEVICE_BASE + 4) as u64, 0x1020); + assert!(runs(head).is_empty()); + assert_eq!(head.read32(0x42000000).data, 0xa5a5a5a5); + assert_eq!(head.read32(0x42001000).data, 0x03020100, "past the small BAR, only the large one decodes"); + head.write32(0x20004, 0); + assert!(!runs(head).is_empty()); + assert_pio_matches_slow(head, &large, 0x42000000, 3); + head.write32(0x21004, 0); + head.write32(0x20004, 2); + assert!(!runs(head).is_empty()); + assert_eq!(head.read32(0x42000000).data, 0xa5a5a5a5); +} + +/// BAR holes and cross-boundary cycles must use the original slow decoder, +/// including after the host publishes a replacement set of BAR mappings. +#[test] +fn sparse_bar_lookup_rejects_holes_and_cross_boundary_cycles() { + let decode = PioDecode::default(); + let first: Arc = Arc::new(Memory(Mutex::new(vec![0; 0x4000]))); + let second: Arc = Arc::new(Memory(Mutex::new(vec![0; 0x4000]))); + drop(decode.publish(vec![ + PioRun { start: 0x3000, end: 0x4000, dev: second.clone() }, + PioRun { start: 0x1000, end: 0x2000, dev: first.clone() }, + ])); + assert!(Arc::ptr_eq(&decode.target(0x1000, 8).unwrap(), &first)); + assert!(Arc::ptr_eq(&decode.target(0x1ff8, 8).unwrap(), &first)); + assert!(decode.target(0x1ffc, 8).is_none()); + assert!(decode.target(0x2000, 8).is_none()); + assert!(decode.target(0x2ff8, 8).is_none()); + assert!(Arc::ptr_eq(&decode.target(0x3000, 8).unwrap(), &second)); + assert!(decode.target(0x4000, 8).is_none()); + assert!(decode.target(u64::MAX, 8).is_none()); + drop(decode.publish(vec![PioRun { start: 0x2000, end: 0x3000, dev: first.clone() }])); + assert!(decode.target(0x1000, 8).is_none()); + assert!(Arc::ptr_eq(&decode.target(0x2000, 8).unwrap(), &first)); + assert!(decode.target(0x3000, 8).is_none()); +} + +fn xbow_read(x: &Arc, off: u32) -> u32 { + x.read32((off + 4) as u64).data +} + +/// Software probing the crossbar after reset sees these values. Each +/// link's control word resets to 0x80000; that value belongs in the control +/// word, not in the link reset register, which no read can see. +#[test] +fn the_crossbar_resets_to_its_documented_values() { + let (c, _) = chip(); + let x = c.crossbar(); + for (off, value) in [ + (xbow::ERR_UPPER, 0), + (xbow::ERR_LOWER, 0), + (xbow::CONTROL, 0x80), + (xbow::PACKET_TIMEOUT, 0xfffff), + (xbow::INT_UPPER, 0x80000), + (xbow::INT_LOWER, 0), + (xbow::ERR_CMD, 0), + (xbow::LLP_CONTROL, 0x03ff1810), + (xbow::ARB_RELOAD, 2), + (xbow::PERF_A, 0), + (xbow::PERF_B, 0), + (xbow::NIC, 2), + (xbow::FENCE_BASE, 0xff), + (xbow::LOCK, 0), + ] { + assert_eq!(xbow_read(&x, off), value, "{off:#x}"); + } + for link in 0..8 { + let base = xbow::LINK_BASE + link * xbow::LINK_STRIDE; + assert_eq!(xbow_read(&x, xbow::FENCE_BASE + 8 + link * 8), 0xff); + assert_eq!(xbow_read(&x, base + xbow::LINK_CONTROL), 0x80000, "link {}", link + 8); + assert_eq!(xbow_read(&x, base + xbow::LINK_ARB_UPPER), 0); + assert_eq!(xbow_read(&x, base + xbow::LINK_ARB_LOWER), 0); + } +} + +/// Software shares the crossbar between processors with the resource lock: +/// a write takes it only while it is free, so a second holder is refused +/// until the first writes the clear register. +#[test] +fn the_resource_lock_admits_one_holder_until_cleared() { + let (c, _) = chip(); + let x = c.crossbar(); + let other = c.crossbar_for_port(HEAD_E); + x.write32((xbow::LOCK + 4) as u64, 0x5a); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x5a); + other.write32((xbow::LOCK + 4) as u64, 0x3c); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x5a); + assert_eq!(xbow_read(&x, xbow::LOCK_CLEAR), 0x5a); + x.write32((xbow::LOCK_CLEAR + 4) as u64, 0); + assert_eq!(xbow_read(&x, xbow::LOCK), 0); + other.write32((xbow::LOCK + 4) as u64, 0x3c); + assert_eq!(xbow_read(&x, xbow::LOCK), 0x3c); +} + +/// Every reset fence bit is read and write, a link's own bit included, and +/// only the low eight bits exist. +#[test] +fn reset_fences_keep_their_low_eight_bits() { + let (c, _) = chip(); + let x = c.crossbar(); + for off in (xbow::FENCE_BASE..=FENCE_LAST).step_by(8) { + x.write32((off + 4) as u64, 0); + assert_eq!(xbow_read(&x, off), 0); + x.write32((off + 4) as u64, off as u32); + assert_eq!(xbow_read(&x, off), off as u32 & 0xff); + x.write32((off + 4) as u64, u32::MAX); + assert_eq!(xbow_read(&x, off), 0xff); + } +} From 526e17a80c59d5143e18e1081b26466d404f5e1e Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 13/22] l1: add the Fuel system controller The Fuel console first passes through the L1 system controller. Reuse the shared 16550 core and keep controller framing, power commands and identity records in its protocol model. --- debug.md | 5 +- src/dev/l1.rs | 449 ++++++++++++++++++++++++++ src/dev/l1/model.rs | 764 ++++++++++++++++++++++++++++++++++++++++++++ src/dev/l1/tests.rs | 381 ++++++++++++++++++++++ src/dev/mod.rs | 3 +- src/devlog.rs | 7 +- 6 files changed, 1605 insertions(+), 4 deletions(-) create mode 100644 src/dev/l1.rs create mode 100644 src/dev/l1/model.rs create mode 100644 src/dev/l1/tests.rs diff --git a/debug.md b/debug.md index c270b0433..53912da82 100644 --- a/debug.md +++ b/debug.md @@ -593,7 +593,7 @@ Per-module logging; all of it needs a developer build to produce output. | `log file ` | Send a module's log to a file | Modules: `net hpc3 seeq hal2 mc rex3 mips ioc scsi pdma vino dcb vc2 cmap -xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock pci bridge heart nvme`. `pci` logs every +xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock pci bridge heart nvme l1`. `pci` logs every configuration cycle on a PCI bus (MACE's, the Octane Bridge's) and whether a function answered it. `bridge` logs the Octane Bridge's register and translation-table accesses by name, and any transfer a PCI @@ -604,7 +604,8 @@ interrupt vector raised or lowered. Mask categories: `pdma` — `hal enet scsi`; `mips` — `insn tlb mem fpu cp0`; `l1i`/`l1d`/`l2c` — `hit miss op`; `nvme` — `state admin regs io` ([docs/nvme.md](docs/nvme.md#tracing)); every module also takes `on`/`all`, `off`/`none` or a hex mask. -`bedrock` covers the Fuel's devices on its Bedrock hub. +`bedrock` covers the Fuel's devices on its Bedrock hub, and `l1` its system +controller. `IRIS_DEBUG_LOG` or `[debug] debug_log` set the same thing at startup. ### Performance and locks diff --git a/src/dev/l1.rs b/src/dev/l1.rs new file mode 100644 index 000000000..c23b6c256 --- /dev/null +++ b/src/dev/l1.rs @@ -0,0 +1,449 @@ +//! Fuel's L1 system controller and its processor-side UART. +//! +//! The internal L1 connection is separate from IOC3's external serial ports. +//! The Ns16550 owns the UART registers, FIFO, timeout and interrupt state; +//! this device keeps the controller protocol and its eight-byte register stride. +//! Controller identities and power transitions are functional approximations. +//! Remote routing and controller diagnostics are absent. +//! +//! Ported from hw/misc/sgi-l1.c in jamesbraid/qemu-sgi-origami. + +use crate::dev::ns16550::{com_reg, Ns16550}; +use crate::dev::serial_host::{null_serial_backend, SerialBackend}; +use crate::devlog::LogModule; +use crate::irq::IrqLine; +use crate::traits::{BusDevice, BusRead64, Device, Resettable, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::collections::VecDeque; +use std::io; +use std::sync::{ + atomic::{AtomicBool, AtomicU64, Ordering}, + Arc, +}; + +mod model; +use model::State; +pub use model::{parse_revision, L1Config, L1Event, DEFAULT_BRICK_SERIAL, DEFAULT_REVISION}; + +fn fcs(mut crc: u16, byte: u8) -> u16 { + crc ^= byte as u16; + for _ in 0..8 { + crc = (crc >> 1) ^ if crc & 1 != 0 { 0x8408 } else { 0 }; + } + crc +} + +fn encode(payload: &[u8]) -> Vec { + let crc = !payload.iter().fold(0xffff, |value, byte| fcs(value, *byte)); + let mut out = vec![0x7e]; + for byte in payload.iter().copied().chain([crc as u8, (crc >> 8) as u8]) { + if matches!(byte, 0x7e | 0x7d | 0x0d) { + out.extend_from_slice(&[0x7d, byte ^ 0x20]); + } else { + out.push(byte); + } + } + out.push(0x7e); + out +} + +const UART_BAUDBASE: u32 = 115200; +const UART_WINDOW_SIZE: u64 = 0x40; + +mod uart_offset { + use super::com_reg; + pub const DATA: u64 = 0x00; + pub const IER: u64 = 0x08; + pub const IIR_FCR: u64 = 0x10; + pub const LCR: u64 = 0x18; + pub const MCR: u64 = 0x20; + pub const LSR: u64 = 0x28; + pub const MSR: u64 = 0x30; + pub const SCR: u64 = 0x38; + + pub fn register(offset: u64) -> u32 { + match offset { + DATA => com_reg::THR, + IER => com_reg::IER, + IIR_FCR => com_reg::IIR_FCR, + LCR => com_reg::LCR, + MCR => com_reg::MCR, + LSR => com_reg::LSR, + MSR => com_reg::MSR, + SCR => com_reg::SCR, + _ => unreachable!("validated L1 UART offset"), + } + } +} + +type EventHandler = Arc; + +#[derive(Default)] +struct Dispatch { + active: bool, + pending: bool, + level: Option, +} + +/// The UART backend does not connect guest L1 commands to a host terminal. +/// Guest transmit bytes are handed to the protocol parser after the UART lock +/// is released; receive bytes come from the protocol reply queue. +struct ProtocolBackend { + guest_tx: Arc>>, + protocol_rx: Arc>>, +} + +impl SerialBackend for ProtocolBackend { + fn send_byte(&self, byte: u8) { + self.guest_tx.lock().push_back(byte); + } + + fn recv_byte(&self) -> io::Result { + self.protocol_rx + .lock() + .pop_front() + .ok_or_else(|| io::ErrorKind::WouldBlock.into()) + } +} + +/// The internal L1 connection is independent of IOC3's external serial ports. +/// Register offsets are relative to the 0x40-byte UART window. Only aligned +/// 64-bit accesses are decoded; register values occupy the low eight bits. +pub struct L1 { + // UART RX takes UART -> protocol queue. Model replies take state -> protocol + // queue. No path holds the protocol queue while acquiring either lock. + state: Arc>, + // Take dispatch before state when draining, release it before the UART or callbacks. + dispatch: Arc>, + uart: Arc>, + guest_tx: Arc>>, + backend: Arc>>, + irq: Arc>, + handler: Arc>>, + running: Arc, + clock: Arc, + worker: Mutex>>, + wake: Arc<(Mutex<()>, parking_lot::Condvar)>, +} +impl L1 { + pub fn new(config: L1Config) -> Result { + if config.rack > 0x3ff || config.bay > 0x3f || config.node_position > 3 { + return Err("L1 rack, bay or node position out of range".into()); + } + if config.mac == [0; 6] || config.mac[0] & 1 != 0 { + return Err("L1 MAC must be nonzero unicast".into()); + } + if config + .brick_serial + .as_ref() + .is_some_and(|s| s.len() > 37 || s.as_bytes().contains(&0)) + { + return Err("L1 brick serial must fit a 37-byte text field".into()); + } + + let protocol_rx = Arc::new(Mutex::new(VecDeque::new())); + let state = Arc::new(Mutex::new(State::new(config, protocol_rx.clone()))); + let guest_tx = Arc::new(Mutex::new(VecDeque::new())); + let protocol_backend = Arc::new(ProtocolBackend { + guest_tx: guest_tx.clone(), + protocol_rx, + }); + let mut uart = Ns16550::new(UART_BAUDBASE); + uart.set_backend(protocol_backend); + + Ok(Self { + state, + dispatch: Arc::new(Mutex::new(Dispatch::default())), + uart: Arc::new(Mutex::new(uart)), + guest_tx, + backend: Arc::new(Mutex::new(null_serial_backend())), + irq: Arc::new(Mutex::new(IrqLine::unconnected())), + handler: Arc::new(Mutex::new(None)), + running: Arc::new(AtomicBool::new(false)), + clock: Arc::new(AtomicU64::new(0)), + worker: Mutex::new(None), + wake: Arc::new((Mutex::new(()), parking_lot::Condvar::new())), + }) + } + + /// Install the separate host-side L1 console connection. + pub fn set_serial_backend(&self, backend: Arc) { + *self.backend.lock() = backend; + } + + pub fn set_irq(&self, line: IrqLine) { + *self.irq.lock() = line; + self.dispatch.lock().level = None; + self.flush(); + } + + /// Power/reset requests reach the machine only after the reply is queued + /// and after the device state lock is released. + pub fn set_event_handler(&self, handler: EventHandler) { + *self.handler.lock() = Some(handler); + } + + #[cfg(test)] + pub fn node_powered(&self) -> bool { + self.state.lock().powered + } + + pub fn station_mac(&self) -> [u8; 6] { + self.state.lock().config.mac + } + + #[cfg(test)] + pub fn display_line(&self) -> String { + self.state.lock().display.clone() + } + + #[cfg(test)] + pub fn record(&self, component: usize, area: u32) -> Option> { + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return None, + }; + self.state.lock().records.get(component).and_then(|r| { + if r[index].is_empty() { + None + } else { + Some(r[index].clone()) + } + }) + } + + /// Install a raw area before execution. It replaces only the named area; + /// station identity remains the configured MAC, as in the Fuel model. + #[cfg(test)] + pub fn set_record(&self, component: usize, area: u32, bytes: Vec) -> Result<(), String> { + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => return Err("unsupported L1 area".into()), + }; + if component >= 16 || bytes.len() > 65535 { + return Err("L1 record out of range".into()); + } + self.state.lock().records[component][index] = bytes; + Ok(()) + } + + pub fn send_event(&self, code: u32, number: u32, text: &str) { + self.state.lock().event(code, number, text); + self.service_protocol_uart(); + self.flush(); + } + + /// Deterministic input path shared with the polled host serial backend. + pub fn receive_console(&self, bytes: &[u8]) { + for &byte in bytes { + self.state.lock().console_input(byte); + self.service_protocol_uart(); + self.flush(); + } + } + + pub fn poll_serial(&self) { + let backend = self.backend.lock().clone(); + for _ in 0..256 { + if self.state.lock().queue.lock().len() > 4088 { + break; + } + match backend.recv_byte() { + Ok(byte) => self.receive_console(&[byte]), + Err(error) => { + if error.kind() != io::ErrorKind::WouldBlock { + log::debug!("L1 serial input: {error}"); + } + break; + } + } + } + } + + /// Let the native UART pull reply bytes from the model's shared bounded queue. + fn service_protocol_uart(&self) { + let mut uart = self.uart.lock(); + uart.pump(); + uart.check_timeout(std::time::Instant::now()); + } + + /// Parse guest transmit only after the UART lock is released. The UART + /// callback only appends to this independent queue. + fn process_guest_transmit(&self) { + let bytes: Vec<_> = self.guest_tx.lock().drain(..).collect(); + if bytes.is_empty() { + return; + } + { + let mut state = self.state.lock(); + for byte in bytes { + state.parse(byte); + } + } + self.service_protocol_uart(); + } + + fn flush(&self) { + let mut dispatch = self.dispatch.lock(); + dispatch.pending = true; + if dispatch.active { + return; + } + dispatch.active = true; + loop { + dispatch.pending = false; + let (output, actions) = { + let mut state = self.state.lock(); + ( + std::mem::take(&mut state.output), + std::mem::take(&mut state.actions), + ) + }; + drop(dispatch); + let level = self.uart.lock().irq(); + + // Reentrant reads request another drain, but must not repeat an + // unchanged IRQ and recursively trigger the same machine callback. + dispatch = self.dispatch.lock(); + let changed = dispatch.level != Some(level); + dispatch.level = Some(level); + drop(dispatch); + + if changed { + let irq = self.irq.lock().clone(); + irq.set(level); + } + let backend = self.backend.lock().clone(); + for byte in output { + backend.send_byte(byte); + } + let handler = self.handler.lock().clone(); + for action in actions { + if let Some(handler) = &handler { + handler(action); + } else { + crate::dlog!(LogModule::L1, "unconnected L1 action {action:?}"); + } + } + + dispatch = self.dispatch.lock(); + if !dispatch.pending { + dispatch.active = false; + break; + } + } + } +} +impl BusDevice for L1 { + fn read64(&self, addr: u64) -> BusRead64 { + if addr >= UART_WINDOW_SIZE || addr & 7 != 0 { + return BusRead64::err(); + } + let value = self.uart.lock().read(uart_offset::register(addr)); + self.flush(); + BusRead64::ok(value as u64) + } + + fn write64(&self, addr: u64, value: u64) -> u32 { + if addr >= UART_WINDOW_SIZE || addr & 7 != 0 { + return BUS_ERR; + } + let register = uart_offset::register(addr); + let value = value as u8; + if register == com_reg::IIR_FCR && value & com_reg::FCR_RFR != 0 { + self.state.lock().queue.lock().clear(); + } + self.uart.lock().write(register, value); + self.process_guest_transmit(); + self.flush(); + BUS_OK + } +} +impl Resettable for L1 { + fn power_on(&self) { + self.state.lock().reset(); + self.guest_tx.lock().clear(); + self.uart.lock().reset(); + self.clock.store(0, Ordering::Relaxed); + self.flush(); + } +} +impl Device for L1 { + fn start(&self) { + let mut worker = self.worker.lock(); + if worker.is_some() { + return; + } + self.running.store(true, Ordering::Release); + let device = Self { + state: self.state.clone(), + dispatch: self.dispatch.clone(), + uart: self.uart.clone(), + guest_tx: self.guest_tx.clone(), + backend: self.backend.clone(), + irq: self.irq.clone(), + handler: self.handler.clone(), + running: self.running.clone(), + clock: self.clock.clone(), + worker: Mutex::new(None), + wake: self.wake.clone(), + }; + *worker = Some( + std::thread::Builder::new() + .name("l1-serial".into()) + .spawn(move || { + while device.is_running() { + device.poll_serial(); + let mut guard = device.wake.0.lock(); + if device.is_running() { + device + .wake + .1 + .wait_for(&mut guard, std::time::Duration::from_millis(2)); + } + } + }) + .expect("start L1 serial worker"), + ); + } + + fn stop(&self) { + let mut slot = self.worker.lock(); + { + let _guard = self.wake.0.lock(); + self.running.store(false, Ordering::Release); + self.wake.1.notify_all(); + } + let worker = slot.take(); + if let Some(worker) = worker { + if worker.thread().id() != std::thread::current().id() { + let _ = worker.join(); + } + } + } + + fn is_running(&self) -> bool { + self.running.load(Ordering::Acquire) + } + + fn get_clock(&self) -> u64 { + self.clock.load(Ordering::Relaxed) + } + + fn step(&self, cycles: u64) { + if self.is_running() { + self.clock.fetch_add(cycles, Ordering::Relaxed); + } + } +} +impl Drop for L1 { + fn drop(&mut self) { + self.stop(); + } +} +#[cfg(test)] +mod tests; diff --git a/src/dev/l1/model.rs b/src/dev/l1/model.rs new file mode 100644 index 000000000..907fd59dd --- /dev/null +++ b/src/dev/l1/model.rs @@ -0,0 +1,764 @@ +//! Fuel's L1 controller protocol parser and reply model. +//! +//! This state owns packet framing, controller identity records, console +//! commands and power events. The UART register file and receive pacing live +//! in the shared Ns16550 core. +//! +//! Ported from hw/misc/sgi-l1.c in jamesbraid/qemu-sgi-origami. + +use super::*; + +/// Firmware revision the model reports for request 0x0005. +/// +/// Model choice: 1.48.1 is the revision a retained real Fuel L1 session +/// prints (the Fuel contract's L1 section). The model reproduces that +/// firmware's replies, not its code, so the revision is a label. +pub const DEFAULT_REVISION: [u32; 3] = [1, 48, 1]; + +/// Brick serial the model reports for request 0x0004. +/// +/// Model choice: the marked synthetic text form, system-board letter `N`, +/// instance 0 (the component taxonomy's synthetic serials). The real format +/// is unknown, and the value must not pass as a real SGI serial. +pub const DEFAULT_BRICK_SERIAL: &str = "EMUN00"; + +/// Parse a "major.minor.patch" revision string, as a `[fuel] l1_revision` +/// override spells it. Each part is a decimal number. +pub fn parse_revision(text: &str) -> Result<[u32; 3], String> { + let bad = || format!("L1 revision \"{text}\" must be major.minor.patch, each a decimal number"); + let mut parts = text.split('.'); + let mut rev = [0u32; 3]; + for slot in &mut rev { + let part = parts.next().ok_or_else(bad)?; + if part.is_empty() || !part.bytes().all(|b| b.is_ascii_digit()) { + return Err(bad()); + } + *slot = part.parse().map_err(|_| bad())?; + } + if parts.next().is_some() { + return Err(bad()); + } + Ok(rev) +} + +#[derive(Clone, Debug)] +pub struct L1Config { + pub rack: u32, + pub bay: u32, + pub node_position: u8, + pub type_code: u32, + pub revision: Option<[u32; 3]>, + pub mac: [u8; 6], + pub brick_serial: Option, + pub auto_power: bool, +} +impl Default for L1Config { + fn default() -> Self { + Self { + rack: 1, + bay: 1, + node_position: 0, + type_code: 0x41, + revision: Some(DEFAULT_REVISION), + mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + brick_serial: Some(DEFAULT_BRICK_SERIAL.into()), + auto_power: true, + } + } +} + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +pub enum L1Event { + Power(bool), + Reset, + SoftReset, + Nmi, + Shutdown, +} + +pub(super) struct State { + pub config: L1Config, + pub queue: Arc>>, + frame: Vec, + in_frame: bool, + escape: bool, + overflow: bool, + pub display: String, + pub records: [[Vec; 3]; 16], + pub actions: Vec, + pub output: Vec, + pub powered: bool, + console: bool, + escaped: bool, + command: Vec, + port: u8, + debug: u32, + partition: u32, + two_bit: u32, + reasons: [u32; 2], + events: Option, + power_down_pending: bool, +} +fn text_field(out: &mut Vec, text: &[u8]) { + out.push(1); + out.extend_from_slice(text); + out.push(0); +} +pub(super) fn ints(values: &[u32]) -> Vec { + let mut out = Vec::new(); + for v in values { + out.push(0); + out.extend_from_slice(&v.to_be_bytes()); + } + out +} +fn word(p: &[u8]) -> u32 { + u32::from_be_bytes(p[..4].try_into().unwrap()) +} +fn arguments(mut p: &[u8], count: u8) -> Option> { + let mut fields = Vec::new(); + for _ in 0..count { + let tag = *p.first()?; + let len = match tag { + 0 => 5, + 1 => p[1..].iter().position(|b| *b == 0)? + 2, + t if t & 0x80 != 0 => 1 + (t & 0x7f) as usize, + _ => return None, + }; + fields.push(p.get(..len)?); + p = &p[len..]; + } + if p.is_empty() { + Some(fields) + } else { + None + } +} +fn area_finish(mut area: Vec) -> Vec { + while (area.len() + 1) % 8 != 0 { + area.push(0); + } + let sum = area.iter().fold(0u8, |s, b| s.wrapping_add(*b)); + area.push(sum.wrapping_neg()); + area +} +fn ia_text(area: &mut Vec, text: &str) { + area.push(0xc0 | text.len() as u8); + area.extend_from_slice(text.as_bytes()); +} +fn board(mac: Option<[u8; 6]>) -> Vec { + let mut b = vec![0; 6]; + for s in ["SYNTHETIC", "IP34", "EMUN00", "030-1707-003"] { + ia_text(&mut b, s); + } + b.push(0); + ia_text(&mut b, "-G"); + b.extend_from_slice(&[1, 0]); + ia_text(&mut b, "00"); + for _ in 0..3 { + b.extend_from_slice(&[4, 0, 0, 0, 0]); + } + if let Some(mac) = mac { + ia_text( + &mut b, + &mac.iter().map(|v| format!("{v:02X}")).collect::(), + ); + } + b.push(0xc1); + // The count includes the checksum and padding, not a separate header. + b[1] = (b.len() + 1).div_ceil(8) as u8; + area_finish(b) +} +impl State { + pub fn new(config: L1Config, queue: Arc>>) -> Self { + let mut records: [[Vec; 3]; 16] = + std::array::from_fn(|_| std::array::from_fn(|_| Vec::new())); + let chassis = area_finish(vec![0, 2, 0, 0xc2, b'N', b'A', 0xc2, b'N', b'A', 0xc1]); + for c in 0..2 { + records[c][0] = chassis.clone(); + records[c][1] = board(if c == 1 { Some(config.mac) } else { None }); + } + let powered = config.auto_power; + let port = config.node_position << 5; + Self { + config, + queue, + frame: Vec::new(), + in_frame: false, + escape: false, + overflow: false, + display: "L1 running".into(), + records, + actions: Vec::new(), + output: Vec::new(), + powered, + console: powered, + escaped: false, + command: Vec::new(), + port, + debug: 0, + partition: 0, + two_bit: 0, + reasons: [0; 2], + events: None, + power_down_pending: false, + } + } + pub fn reset(&mut self) { + // Controller settings survive the controller's local boot. The UART and + // packet parser reset independently of those persistent settings. + let persistent = (self.debug, self.partition, self.two_bit, self.reasons); + let events = self.events.filter(|&channel| channel >= 4); + let records = std::mem::replace( + &mut self.records, + std::array::from_fn(|_| std::array::from_fn(|_| Vec::new())), + ); + let queue = self.queue.clone(); + queue.lock().clear(); + *self = Self::new(self.config.clone(), queue); + (self.debug, self.partition, self.two_bit, self.reasons) = persistent; + self.events = events; + self.records = records; + } + fn emit(&mut self, payload: &[u8]) { + if payload.len() > 128 { + crate::dlog!(LogModule::L1, "L1 reply exceeds frame capacity"); + return; + } + let bytes = encode(payload); + let mut queue = self.queue.lock(); + if queue.len() + bytes.len() <= 4096 { + queue.extend(bytes); + } else { + crate::dlog!(LogModule::L1, "L1 receive queue full"); + } + } + fn reply(&mut self, channel: u8, code: i32, count: u8, fields: &[u8]) { + let mut p = vec![0x20 | channel]; + p.extend_from_slice(&code.to_be_bytes()); + p.push(count); + p.extend_from_slice(fields); + self.emit(&p); + } + pub fn parse(&mut self, mut byte: u8) { + if byte == 0x7e { + if self.in_frame + && !self.escape + && !self.overflow + && self.frame.len() >= 3 + && self.frame.iter().fold(0xffff, |v, b| fcs(v, *b)) == 0xf0b8 + { + let payload = self.frame[..self.frame.len() - 2].to_vec(); + self.request(&payload); + } + self.in_frame = true; + self.escape = false; + self.overflow = false; + self.frame.clear(); + return; + } + if !self.in_frame || self.overflow { + return; + } + if byte == 0x7d { + self.escape = true; + return; + } + if self.escape { + byte ^= 0x20; + self.escape = false; + } + if self.frame.len() == 258 { + self.overflow = true; + } else { + self.frame.push(byte); + } + } + fn eeprom(&mut self, channel: u8, a: &[&[u8]]) { + if a.len() < 4 || a[..4].iter().any(|v| v[0] != 0) { + self.reply(channel, -100, 0, &[]); + return; + } + let component = word(&a[0][1..]) as usize; + let area = word(&a[1][1..]); + let offset = word(&a[2][1..]) as usize; + let requested = word(&a[3][1..]) as usize; + if component >= 16 || self.records[component].iter().all(Vec::is_empty) { + self.reply(channel, -104, 0, &[]); + return; + } + if area > 4 { + self.reply(channel, -105, 0, &[]); + return; + } + let index = match area { + 1 => 0, + 2 => 1, + 4 => 2, + _ => { + self.reply(channel, -104, 0, &[]); + return; + } + }; + let record = &self.records[component][index]; + if record.is_empty() { + self.reply(channel, -104, 0, &[]); + return; + } + let checksum_len = if area == 4 { + record.len().min(64) + } else { + record.len() + }; + if record[..checksum_len] + .iter() + .fold(0u8, |s, b| s.wrapping_add(*b)) + != 0 + { + self.reply(channel, -107, 0, &[]); + return; + } + if offset >= record.len() { + self.reply(channel, -105, 0, &[]); + return; + } + let len = requested.min(record.len() - offset); + if len > 116 { + crate::dlog!(LogModule::L1, "L1 EEPROM reply exceeds frame capacity"); + self.reply(channel, -105, 0, &[]); + return; + } + let mut fields = ints(&[(record.len() - offset - len) as u32]); + if requested != 0 { + fields.push(0x80 | len as u8); + fields.extend_from_slice(&record[offset..offset + len]); + } + self.reply(channel, 0, if requested == 0 { 1 } else { 2 }, &fields); + } + fn request(&mut self, p: &[u8]) { + let channel = p[0] & 31; + if p[0] & 0xe0 == 0x40 { + self.port = (self.port & 0x60) | channel; + if self.console && self.powered { + self.output.extend_from_slice(&p[1..]); + } else { + crate::dlog!(LogModule::L1, "L1 console output while operator is in L1 mode"); + } + return; + } + if p[0] & 0xe0 != 0 || p.len() < 7 { + return; + } + let address = word(&p[1..]); + let cmd = u16::from_be_bytes([p[5], p[6]]); + let key = address >> 12; + if key != 0xffff && key != ((self.config.rack << 6) | self.config.bay) { + crate::dlog!(LogModule::L1, "L1 unsupported destination {address:#x}"); + self.emit(&[0x20 | channel, 255, 255, 255, 255, 0, 0, 0, 8]); + return; + } + let count = if p.len() == 7 { 0 } else { p[7] }; + let raw = if p.len() > 8 { &p[8..] } else { &[] }; + let task = address & 31; + if task == 5 || task == 21 { + if self.powered { + self.emit(&[ + ((address >> 5) & 31) as u8, + raw.get(4).copied().unwrap_or(0), + ]); + } else { + let mut field = Vec::new(); + text_field( + &mut field, + format!( + "WARNING: power on {:03}c{:02} appears off!", + self.config.rack, self.config.bay + ) + .as_bytes(), + ); + self.reply(channel, -1, 1, &field); + } + return; + } + if task != 6 && task != 3 && task != 18 { + crate::dlog!(LogModule::L1, "L1 unsupported task {task}"); + return; + } + let Some(a) = arguments(raw, count) else { + self.reply(channel, -100, 0, &[]); + return; + }; + if task == 3 { + if a.len() != 1 || a[0][0] != 1 { + self.reply(channel, -100, 0, &[]); + return; + } + let text = String::from_utf8_lossy(&a[0][1..a[0].len() - 1]); + let (code, output) = self.run_line(&text, false); + let mut fields = Vec::new(); + let mut n = 0; + for line in output.split("\r\n").filter(|s| !s.is_empty()) { + if fields.len() + line.len() + 2 > 121 { + break; + } + text_field(&mut fields, line.as_bytes()); + n += 1; + } + self.reply(channel, code, n, &fields); + return; + } + if task == 18 { + let length = a.iter().take(3).map(|v| v.len()).sum::().min(121); + self.reply(channel, cmd as i32, count.min(3), &raw[..length]); + return; + } + let mut values = Vec::new(); + let mut fields = Vec::new(); + let mut field_count = 0; + let mut error = 0; + match cmd { + 1 => values.push(self.debug), + 2 => values.push((self.config.rack << 6) | self.config.bay), + 3 => { + if self.config.type_code == 0 { + error = -104; + } else { + values.extend([ + (self.config.rack << 6) | self.config.bay, + self.config.type_code, + ]); + } + } + 4 => { + if let Some(serial) = &self.config.brick_serial { + text_field(&mut fields, serial.as_bytes()); + field_count = 1; + } else { + error = -104; + } + } + 5 => { + if let Some(rev) = self.config.revision { + values.extend(rev); + } else { + crate::dlog!(LogModule::L1, "L1 revision metadata not configured"); + error = -104; + } + } + 6 => { + self.eeprom(channel, &a); + return; + } + 7 => error = if a.is_empty() { -100 } else { -104 }, + // The system serial is the station address as twelve uppercase + // hexadecimal digits in address order, no separators or OUI + // added: the text the stock PROM converts straight back into the + // Ethernet address (the Fuel contract's MAC section). The + // uppercase case is the contract's choice for the model. + 8 => { + let mac = self + .config + .mac + .iter() + .map(|b| format!("{b:02X}")) + .collect::(); + text_field(&mut fields, mac.as_bytes()); + field_count = 1; + } + 9 => values.push(self.partition), + 0xa => values.extend([0, 0]), + 0xb => values.push(self.two_bit), + 0xc => values.extend([0; 8]), + 0xd => values.extend([0; 4]), + 0xe => values.push((self.port >> 5) as u32 & 3), + 0xf => values.extend(self.reasons), + 0x10 => values.extend([1, 1, 1, 1]), + 0x11 => { + error = -106; + values.push(1); + } + 0x1001 | 0x1002 | 0x1006 | 0x1007 | 0x1008 => { + if a.is_empty() { + error = -100; + } else if a[0][0] != 0 { + error = -105; + } else { + let v = word(&a[0][1..]); + match cmd { + 0x1001 => self.debug = v & 0xffff, + 0x1002 => self.port = ((self.port as u32 & 0x60) | v) as u8 & 0x7f, + 0x1006 => self.partition = v, + 0x1007 => self.events = Some(v as u8), + _ => self.two_bit = v & 3, + } + } + } + 0x1003 => {} + 0x1004 | 0x1005 => { + if a.is_empty() { + error = -100; + } else if a[0][0] != 1 { + error = -102; + } else { + self.display = a[0][1..a[0].len() - 1] + .iter() + .take(72) + .map(|b| { + if (0x20..=0x7e).contains(b) { + *b as char + } else { + '?' + } + }) + .collect(); + } + } + 0x1009 => { + if a.len() < 2 { + error = -100; + } else if a[0][0] != 0 || a[1][0] != 0 || word(&a[1][1..]) & 15 > 2 { + error = -105; + } + } + 0x100a => self.events = None, + 0x100b => { + if a.len() != 3 { + error = -100; + } else { + self.events = None; + } + } + 0x100c => { + if a.len() != 2 { + error = -100; + } + } + 0x2001 | 0x2005 | 0x2006 => { + self.reply(channel, 0, 0, &[]); + match cmd { + 0x2001 => self.reset_node(), + 0x2005 => self.power_down(true, false), + _ => { + self.power_down(false, false); + self.power_up(true); + } + } + return; + } + 0x2002..=0x2004 => error = -100, + 0x2007 | 0x2008 => error = -108, + 0x4000 => error = -104, + 0x4002 => { + self.reply(channel, 0, 0, &[]); + self.reboot(); + return; + } + _ => { + crate::dlog!(LogModule::L1, "L1 unsupported request {cmd:#x}"); + error = -101; + } + } + if !values.is_empty() { + field_count = values.len() as u8; + fields = ints(&values); + } + self.reply(channel, error, field_count, &fields); + } + fn reason(&mut self) { + self.reasons[1] = self.reasons[0]; + self.reasons[0] = 3; + } + fn reset_node(&mut self) { + if self.powered { + self.reason(); + self.display = "Reset done".into(); + self.actions.push(L1Event::Reset); + } + } + fn power_down(&mut self, shutdown: bool, confirm: bool) { + if self.powered { + if confirm && !self.power_down_pending { + self.power_down_pending = true; + self.event( + 0x8000, + 0, + "Power Down: issue 'pwr d' again to immediately power down.", + ); + return; + } + self.power_down_pending = false; + self.reason(); + self.powered = false; + self.display = "Powered Down".into(); + self.actions.push(L1Event::Power(false)); + if shutdown { + self.actions.push(L1Event::Shutdown); + } + } + } + fn power_up(&mut self, restart: bool) { + self.reason(); + self.powered = true; + self.power_down_pending = false; + self.port &= 0x60; + self.console = true; + self.display = "Powered Up".into(); + self.actions.push(L1Event::Power(true)); + if restart { + self.actions.push(L1Event::Reset); + } + } + fn reboot(&mut self) { + self.power_down_pending = false; + self.console = false; + self.escaped = false; + self.command.clear(); + self.frame.clear(); + self.in_frame = false; + self.port = self.config.node_position << 5; + self.display = "L1 running".into(); + if self.events.is_some_and(|c| c < 4) { + self.events = None; + } + } + pub fn event(&mut self, code: u32, number: u32, text: &str) { + let Some(channel) = self.events else { + return; + }; + let mut p = vec![0x40 | (channel & 31)]; + p.extend_from_slice( + &((self.config.rack << 18) | (self.config.bay << 12) | 4).to_be_bytes(), + ); + p.extend_from_slice(&code.to_be_bytes()); + p.push(2); + p.extend(ints(&[number])); + text_field( + &mut p, + format!( + "\r\n{:03}c{:02} ATTN: {text}\r\n", + self.config.rack, self.config.bay + ) + .as_bytes(), + ); + if p.len() <= 127 { + self.emit(&p); + } + } + fn run_line(&mut self, line: &str, interactive: bool) -> (i32, String) { + let words = line.split_whitespace().collect::>(); + let local = if words.first() == Some(&"*") { + &words[1..] + } else { + &words[..] + }; + match local { + ["pwr" | "power", "d" | "down"] => { + self.power_down(true, interactive); + (0, String::new()) + } + ["pwr" | "power", "u" | "up"] => { + if !self.powered { + self.power_up(true); + } + (0, String::new()) + } + ["reset"] => { + self.reset_node(); + (0, String::new()) + } + ["softreset"] => { + self.actions.push(L1Event::SoftReset); + self.display = "Sft rst done".into(); + (0, String::new()) + } + ["nmi"] => { + self.actions.push(L1Event::Nmi); + self.display = "NMI done".into(); + (0, String::new()) + } + ["reboot_l1"] => { + self.reboot(); + (0, String::new()) + } + ["l1"] => { + self.escaped = false; + (0, String::new()) + } + ["version"] => { + if let Some(r) = self.config.revision { + ( + 0, + format!("L1 Controller: Revision {}.{}.{}\r\n", r[0], r[1], r[2]), + ) + } else { + ( + -104, + "L1 firmware revision metadata not configured\r\n".into(), + ) + } + } + ["debug"] if words.len() == 1 => ( + 0, + format!("debug switches set to 0x{:04x}\n\r\n", self.debug), + ), + ["debug", v] if words.len() == 2 => { + let n = if let Some(hex) = v.strip_prefix("0x") { + u32::from_str_radix(hex, 16) + } else { + v.parse() + }; + match n { + Ok(v) => { + self.debug = v & 0xffff; + ( + 0, + format!("debug switches set to 0x{:04x}\n\r\n", self.debug), + ) + } + Err(_) => (-1200, "ERROR: command not found.\n\r\n".into()), + } + } + _ => { + crate::dlog!(LogModule::L1, "L1 unsupported operator command {line:?}"); + (-1, "ERROR: command not found.\r\n".into()) + } + } + } + fn prompt(&mut self) { + self.output.extend_from_slice( + format!("{:03}c{:02}-L1>", self.config.rack, self.config.bay).as_bytes(), + ); + } + pub fn console_input(&mut self, byte: u8) { + if self.console { + if byte == 0x14 { + self.console = false; + self.escaped = true; + self.output + .extend_from_slice(b"escaping to L1 system controller\r\n"); + self.prompt(); + } else if self.powered { + self.emit(&[self.port & 31, byte]); + } else { + crate::dlog!(LogModule::L1, "L1 console input while powered off"); + } + } else if byte == 4 { + self.command.clear(); + self.console = true; + } else if byte == b'\r' || byte == b'\n' { + let line = String::from_utf8_lossy(&self.command).into_owned(); + self.command.clear(); + let (_, output) = self.run_line(&line, true); + self.output.extend_from_slice(output.as_bytes()); + if self.escaped { + self.console = true; + } else { + self.prompt(); + } + } else if byte == 8 || byte == 127 { + self.command.pop(); + } else if self.command.len() < 255 { + self.command.push(byte); + } + } +} diff --git a/src/dev/l1/tests.rs b/src/dev/l1/tests.rs new file mode 100644 index 000000000..64d206fd5 --- /dev/null +++ b/src/dev/l1/tests.rs @@ -0,0 +1,381 @@ +// Cases follow tests/qtest/sn1-l1-test.c in jamesbraid/qemu-sgi-origami: +// uart, frames, identity, eeprom, errors and reset. + +use super::*; + +fn send(l1: &L1, bytes: &[u8]) { + for &b in bytes { + assert_eq!(l1.write64(0, b as u64), BUS_OK); + } +} +fn drain(l1: &L1) -> Vec { + let mut out = Vec::new(); + while l1.read64(0x28).data & 1 != 0 { + out.push(l1.read64(0).data as u8); + } + out +} +fn request(l1: &L1, command: u16, args: &[u8], count: u8) -> Vec { + let mut p = vec![ + 0x10, + 0x0f, + 0xff, + 0xf0, + 6, + (command >> 8) as u8, + command as u8, + count, + ]; + p.extend_from_slice(args); + send(l1, &encode(&p)); + drain(l1) +} +fn integer_reply(values: &[u32]) -> Vec { + let mut p = vec![0x30, 0, 0, 0, 0, values.len() as u8]; + p.extend(model::ints(values)); + encode(&p) +} + +/// The PROM accesses the L1 UART through byte-sized register operations. +#[test] +fn the_l1_uart_frames_bytes_and_prioritizes_interrupts() { + let l1 = L1::new(Default::default()).unwrap(); + assert_eq!(l1.read64(1).status, BUS_ERR); + assert_eq!(l1.read8(0).status, BUS_ERR); + assert_eq!(l1.read32(0).status, BUS_ERR); + assert_eq!(l1.read64(0x28).data, 0x60); + send(&l1, &[0x7e, 0x10, 0x0f, 0xff, 0xf0, 6, 0, 0x0e, 0, 0xf9, 0x72, 0x7e]); + assert_eq!(drain(&l1), [0x7e, 0x30, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0x30, 0x98, 0x7e]); + let (irq, line) = crate::irq::LevelProbe::line(); + l1.set_irq(line); + l1.write64(8, 3); + assert!(irq.level()); + assert_eq!(l1.read64(0x10).data, 2); + assert!(!irq.level()); + l1.write64(0x18, 0x80); + l1.write64(0, 42); + l1.write64(8, 7); + assert_eq!(l1.read64(0).data, 42); + assert_eq!(l1.read64(8).data, 7); + l1.write64(0x18, 3); + send(&l1, &encode(&[0x10, 0x0f, 0xff, 0xf0, 6, 0, 0x0e, 0])); + assert_eq!(l1.read64(0x10).data, 4); + drain(&l1); + assert_eq!(l1.read64(0x10).data, 2); + assert!(!irq.level()); +} + +/// The guest reads controller identity and checks the EEPROM record checksum. +#[test] +fn the_l1_returns_identity_and_validated_eeprom_records() { + let l1 = L1::new(Default::default()).unwrap(); + assert_eq!(request(&l1, 3, &[], 0), integer_reply(&[65, 65])); + let board = l1.record(1, 2).unwrap(); + assert_eq!(board.iter().fold(0u8, |s, x| s.wrapping_add(*x)), 0); + for text in [b"IP34".as_slice(), b"EMUN00", b"080069123456"] { + assert!(board.windows(text.len()).any(|x| x == text)); + } + let mut expected = vec![0x30, 0, 0, 0, 0, 2]; + expected.extend(model::ints(&[0])); + expected.push(0x80 | board.len() as u8); + expected.extend(board); + assert_eq!(request(&l1, 6, &model::ints(&[1, 2, 0, 116]), 4), encode(&expected)); + for (args, count, error) in [([2, 5, 0, 1], 4, 152), ([1, 5, 0, 1], 4, 151), ([1, 2, 0, 1], 3, 156)] { + assert_eq!(request(&l1, 6, &model::ints(&args), count), encode(&[0x30, 255, 255, 255, error, 0])); + } + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[1, 48, 1])); +} + +/// A successful reply carrying one text field. +fn text_reply(text: &str) -> Vec { + let mut p = vec![0x30, 0, 0, 0, 0, 1]; + p.push(1); + p.extend_from_slice(text.as_bytes()); + p.push(0); + encode(&p) +} + +/// With no configuration the controller reports the labelled model defaults: +/// firmware 1.48.1 and the synthetic brick serial. +#[test] +fn the_l1_reports_the_default_revision_and_brick_serial() { + let l1 = L1::new(Default::default()).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[1, 48, 1])); + assert_eq!(request(&l1, 4, &[], 0), text_reply("EMUN00")); +} + +/// Both identity values can be replaced. +#[test] +fn the_l1_revision_and_brick_serial_can_be_overridden() { + let l1 = L1::new(L1Config { + revision: Some(parse_revision("2.13.40").unwrap()), + brick_serial: Some("EMUN07".into()), + ..Default::default() + }) + .unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 5, &[], 0), integer_reply(&[2, 13, 40])); + assert_eq!(request(&l1, 4, &[], 0), text_reply("EMUN07")); + assert_eq!( + L1::new(L1Config { brick_serial: Some("x".repeat(38)), ..Default::default() }).err().as_deref(), + Some("L1 brick serial must fit a 37-byte text field") + ); +} + +/// The revision override is exactly three decimal numbers. +#[test] +fn the_l1_revision_text_is_three_decimal_parts() { + assert_eq!(parse_revision("1.48.1"), Ok([1, 48, 1])); + for bad in ["", "1", "1.48", "1.48.1.0", "1..1", "a.b.c", "1.48.-1", "1.48.+1", " 1.48.1", "1.48.99999999999"] { + assert!(parse_revision(bad).is_err(), "{bad:?}"); + } +} + +/// Request 0x0008 is the configured station address as twelve uppercase +/// hexadecimal digits in address order, the text the PROM turns back into the +/// Ethernet address. +#[test] +fn the_l1_system_serial_is_the_station_address_in_hex() { + let l1 = L1::new(Default::default()).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 8, &[], 0), text_reply("080069123456")); + let l1 = L1::new(L1Config { mac: [0x00, 0xA0, 0xc9, 0x1f, 0x02, 0xbe], ..Default::default() }).unwrap(); + l1.power_on(); + assert_eq!(request(&l1, 8, &[], 0), text_reply("00A0C91F02BE")); +} + +/// The L1 parser recovers after malformed frames and preserves escaped console text. +#[test] +fn the_l1_resynchronizes_after_bad_frames_and_preserves_escaped_text() { + let l1 = L1::new(Default::default()).unwrap(); + send(&l1, &[0x7e, 1, 2, 3, 0x7e]); + send(&l1, &vec![0x41; 259]); + send(&l1, &[0x7e]); + assert!(drain(&l1).is_empty()); + assert_eq!(request(&l1, 0x2000, &[], 0), encode(&[0x30, 255, 255, 255, 155, 0])); + send(&l1, &encode(&[0x10, 0x4f, 0xff, 0xf0, 6, 0, 3, 0])); + assert_eq!(drain(&l1), encode(&[0x30, 255, 255, 255, 255, 0, 0, 0, 8])); + let mut text = vec![1]; + text.extend_from_slice(b"hi~}\r\0"); + assert_eq!(request(&l1, 0x1004, &text, 1), integer_reply(&[])); + assert_eq!(l1.display_line(), "hi~}?"); + text.truncate(1); + text.extend_from_slice(b"abcdefghijkl\x01\x7f"); + text.extend(std::iter::repeat_n(b'z', 70)); + text.push(0); + assert_eq!(request(&l1, 0x1004, &text, 1), integer_reply(&[])); + assert_eq!(l1.display_line(), format!("abcdefghijkl??{}", "z".repeat(58))); +} + +/// The PROM may reset L1 during boot; controller settings and channel configuration must remain available. +#[test] +fn l1_reset_clears_uart_state_but_preserves_settings_and_channels() { + for channel in [3, 23] { + let l1 = L1::new(Default::default()).unwrap(); + let (irq, line) = crate::irq::LevelProbe::line(); + l1.set_irq(line); + for (set, value) in [(0x1001, 0x1234), (0x1006, 42), (0x1008, 2), (0x1007, channel)] { + request(&l1, set, &model::ints(&[value]), 1); + } + request(&l1, 0x2001, &[], 0); + let reasons = request(&l1, 0xf, &[], 0); + assert_eq!(reasons, integer_reply(&[3, 0])); + l1.write64(8, 1); + l1.send_event(0x8000, 7, "before reset"); + assert!(irq.level()); + let stale = encode(&[0x10, 0x0f, 0xff, 0xf0, 6, 0, 9, 0]); + send(&l1, &stale[..4]); + l1.write64(0x18, 0x80); + l1.write64(0, 42); + l1.write64(8, 7); + l1.power_on(); + assert!(!irq.level()); + assert_eq!(l1.read64(0x18).data, 0); + assert_eq!(l1.read64(8).data, 0); + assert!(drain(&l1).is_empty()); + l1.write64(0x18, 0x80); + // Ns16550 resets its divisor latch to 0x000c for 9600 baud. + assert_eq!(l1.read64(0).data, 0x0c); + assert_eq!(l1.read64(8).data, 0); + l1.write64(0x18, 0); + send(&l1, &stale[4..]); + assert!(drain(&l1).is_empty()); + for (get, value) in [(1, 0x1234), (9, 42), (0xb, 2)] { + assert_eq!(request(&l1, get, &[], 0), integer_reply(&[value])); + } + assert_eq!(request(&l1, 0xf, &[], 0), reasons); + l1.write64(8, 1); + l1.send_event(0x8000, 7, "after reset"); + assert_eq!(irq.level(), channel == 23); + let emitted = drain(&l1); + assert_eq!(emitted.is_empty(), channel == 3); + assert!(!irq.level()); + } +} + +/// A local console power-down request requires a second confirmation command. +#[test] +fn interactive_power_down_waits_for_a_second_command() { + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let actions = Arc::new(Mutex::new(Vec::new())); + let observed = actions.clone(); + let weak = Arc::downgrade(&l1); + l1.set_event_handler(Arc::new(move |event| { + assert!(!weak.upgrade().unwrap().node_powered()); + observed.lock().push(event); + })); + request(&l1, 0x1007, &model::ints(&[23]), 1); + l1.receive_console(&[0x14]); + l1.receive_console(b"pwr d\r"); + assert!(l1.node_powered()); + assert!(actions.lock().is_empty()); + let mut warning = vec![0x57, 0, 4, 0x10, 4, 0, 0, 0x80, 0, 2]; + warning.extend(model::ints(&[0])); + warning.push(1); + warning.extend_from_slice(b"\r\n001c01 ATTN: Power Down: issue 'pwr d' again to immediately power down.\r\n\0"); + assert_eq!(drain(&l1), encode(&warning)); + l1.power_on(); + l1.receive_console(&[0x14]); + l1.receive_console(b"pwr d\r"); + assert!(l1.node_powered(), "reset clears the first confirmation"); + drain(&l1); + l1.receive_console(&[0x14]); + l1.receive_console(b"power down\r"); + assert!(!l1.node_powered()); + assert_eq!(*actions.lock(), [L1Event::Power(false), L1Event::Shutdown]); +} + +/// Protocol power-down actions must finish before the L1 request reply. +#[test] +fn protocol_and_task_three_power_down_commands_are_immediate() { + let protocol = vec![0x10, 0x0f, 0xff, 0xf0, 6, 0x20, 5, 0]; + let mut task_three = vec![0x10, 0x0f, 0xff, 0xf0, 3, 0, 0, 1, 1]; + task_three.extend_from_slice(b"pwr d\0"); + for (case, packet) in [ + ("protocol command", protocol), + ("task three console", task_three), + ] { + let l1 = L1::new(Default::default()).unwrap(); + let actions = Arc::new(Mutex::new(Vec::new())); + let observed = actions.clone(); + l1.set_event_handler(Arc::new(move |event| observed.lock().push(event))); + send(&l1, &encode(&packet)); + assert_eq!(drain(&l1), integer_reply(&[]), "{case}"); + assert!(!l1.node_powered(), "{case}"); + assert_eq!( + *actions.lock(), + [L1Event::Power(false), L1Event::Shutdown], + "{case}" + ); + } +} + +/// A CPU may read the UART in its interrupt callback while the worker updates the line. +#[test] +fn l1_worker_irq_delivery_stays_ordered_across_reentrant_cpu_reads() { + use std::sync::mpsc; + struct BlockingIrq { + entered: mpsc::SyncSender<()>, + resume: std::sync::Mutex>, + blocked: AtomicBool, + levels: Mutex>, + device: std::sync::Weak, + } + impl crate::irq::InterruptController for BlockingIrq { + fn set_line(&self, _: u32, level: bool) { + if level && !self.blocked.swap(true, Ordering::AcqRel) { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv().unwrap(); + // A machine interrupt handler may inspect the UART immediately. + assert_eq!(self.device.upgrade().unwrap().read64(0x28).data, 0x60); + } + self.levels.lock().push(level); + } + } + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let backend = Arc::new(crate::dev::serial_host::CiSerialBackend::new()); + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (resume_tx, resume_rx) = mpsc::sync_channel(1); + let irq = Arc::new(BlockingIrq { + entered: entered_tx, + resume: std::sync::Mutex::new(resume_rx), + blocked: AtomicBool::new(false), + levels: Mutex::new(Vec::new()), + device: Arc::downgrade(&l1), + }); + l1.set_irq(IrqLine::new(irq.clone(), 0)); + l1.set_serial_backend(backend.clone()); + l1.write64(8, 1); + l1.start(); + backend.push_host(b"x"); + let entered = entered_rx.recv_timeout(std::time::Duration::from_secs(2)); + if entered.is_ok() { + l1.write64(0x10, 2); + } + resume_tx.send(()).unwrap(); + l1.stop(); + entered.unwrap(); + assert_eq!(*irq.levels.lock(), [false, true, false]); +} + +/// Concurrent worker output and platform actions must preserve their order. +#[test] +fn concurrent_l1_outputs_and_actions_are_delivered_in_order() { + use std::sync::mpsc; + struct BlockingBackend { + entered: mpsc::SyncSender<()>, + resume: std::sync::Mutex>, + output: Mutex>, + } + impl SerialBackend for BlockingBackend { + fn send_byte(&self, byte: u8) { + if byte == b'A' { + self.entered.send(()).unwrap(); + self.resume.lock().unwrap().recv().unwrap(); + } + self.output.lock().push(byte); + } + fn recv_byte(&self) -> std::io::Result { + Err(std::io::ErrorKind::WouldBlock.into()) + } + } + let l1 = Arc::new(L1::new(Default::default()).unwrap()); + let (entered_tx, entered_rx) = mpsc::sync_channel(1); + let (resume_tx, resume_rx) = mpsc::sync_channel(1); + let backend = Arc::new(BlockingBackend { + entered: entered_tx, + resume: std::sync::Mutex::new(resume_rx), + output: Mutex::new(Vec::new()), + }); + l1.set_serial_backend(backend.clone()); + let actions = Arc::new(Mutex::new(Vec::new())); + let captured = actions.clone(); + let weak = Arc::downgrade(&l1); + l1.set_event_handler(Arc::new(move |action| { + // Reentrant register reads must not recursively dispatch notifications. + assert_eq!(weak.upgrade().unwrap().read64(0x28).data, 0x60); + captured.lock().push(action); + })); + { + let mut s = l1.state.lock(); + s.output.extend(b"AB"); + s.actions.push(L1Event::Reset); + } + let worker_device = l1.clone(); + let worker = std::thread::spawn(move || worker_device.flush()); + let entered = entered_rx.recv_timeout(std::time::Duration::from_secs(2)); + if entered.is_ok() { + let mut s = l1.state.lock(); + s.output.extend(b"CD"); + s.actions.push(L1Event::Nmi); + drop(s); + l1.flush(); + } + resume_tx.send(()).unwrap(); + worker.join().unwrap(); + entered.unwrap(); + assert_eq!(*backend.output.lock(), b"ABCD"); + assert_eq!(*actions.lock(), [L1Event::Reset, L1Event::Nmi]); +} diff --git a/src/dev/mod.rs b/src/dev/mod.rs index cf217a2a3..776ff17a6 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -78,7 +78,8 @@ pub mod bedrock; pub mod bedrock_flash; pub mod dp83843; pub mod ics1890; +pub mod isp12160; +pub mod l1; pub mod timekeeper; pub mod prom_image; -pub mod isp12160; pub mod xbridge; diff --git a/src/devlog.rs b/src/devlog.rs index 6cd909a16..fc5cc595f 100644 --- a/src/devlog.rs +++ b/src/devlog.rs @@ -70,6 +70,7 @@ pub enum LogModule { /// The NVM Express controller: register writes, controller state /// changes, and the commands it runs (categories `NVME_LOG_*`). Nvme = 34, + L1 = 35, } // Mask bits shared by L1i / L1d / L2c @@ -84,7 +85,7 @@ pub const NVME_LOG_REGS: u32 = 0x4; // every register read and write below the pub const NVME_LOG_IO: u32 = 0x8; // I/O doorbells, commands and completions impl LogModule { - pub const COUNT: usize = 35; + pub const COUNT: usize = 36; pub fn from_str(s: &str) -> Option { match s { @@ -123,6 +124,7 @@ impl LogModule { "bridge" => Some(Self::Bridge), "heart" => Some(Self::Heart), "nvme" => Some(Self::Nvme), + "l1" => Some(Self::L1), _ => None, } } @@ -164,6 +166,7 @@ impl LogModule { Self::Bridge => "bridge", Self::Heart => "heart", Self::Nvme => "nvme", + Self::L1 => "l1", } } @@ -176,6 +179,7 @@ impl LogModule { Self::L1i, Self::L1d, Self::L2c, Self::Ultra, Self::Scsi1, Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, Self::Hub, Self::Pci, Self::Bridge, Self::Heart, Self::Nvme, + Self::L1, ] } @@ -340,6 +344,7 @@ impl DevLog { ModuleLog::new(), // Bridge ModuleLog::new(), // Heart ModuleLog::new(), // Nvme + ModuleLog::new(), // L1 ], } } From c49724b79be8aa2867dd59adb6b6296ce4606f5d Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 14/22] ohci: add the Lucent USB host controller and HID boot devices Fuel enumerates a Lucent OHCI controller on PCI. Provide its registers and descriptor scheduling through the USB function interface while leaving devices unattached in the machine configuration. The frame counter follows the machine clock the board hands the controller, and every register read and write first delivers the frame boundaries that clock has passed. Advancing it only from the board's poll, which runs between CPU batches, left HcFmNumber at zero after the guest's own 1 ms wait had expired, and the Fuel PROM reported that SOF did not start within 2 ms. The sync holds one lock from reading the clock to applying the frames, so a timer thread cannot leave a concurrent reader behind it. Entering UsbOperational from Reset or Suspend re-anchors the frame: it loads HcFmRemaining from HcFmInterval, toggle included, and increments HcFmNumber, with the first SOF one frame later (OpenHCI 1.0a 6.2.1, 7.3.2, 7.3.3). The tests cover that entry with frame numbers at and around the overflow flag. Add two low-speed USB functions for the OHCI root ports: a HID boot keyboard and a HID boot mouse, each with one configuration, one boot interface and one interrupt-IN endpoint. They answer the standard requests (descriptors, SET_ADDRESS taking effect after its status stage, SET_CONFIGURATION, status and interface queries) and the HID class requests (GET/SET_REPORT, GET/SET_IDLE, GET/SET_PROTOCOL), and report host keys and pointer moves as boot reports. Reports queue one per change so a key tapped between two polls still shows both edges. Written from the USB 1.1 and HID 1.11 specifications (boot descriptors from appendix B) and the HID Usage Tables, with no code from another emulator. The wheel, remote wakeup and the host layout translation that the PS/2 path has are left out. Ohci::attach loses its test-only gate so a machine can plug the devices in. --- debug.md | 6 +- src/dev/mod.rs | 2 + src/dev/ohci.rs | 584 +++++++++++++++++++++++++++++++++++++ src/dev/ohci/engine.rs | 223 ++++++++++++++ src/dev/ohci/schedule.rs | 98 +++++++ src/dev/ohci/tests.rs | 555 +++++++++++++++++++++++++++++++++++ src/dev/ps2.rs | 70 ++++- src/dev/usb_hid.rs | 609 +++++++++++++++++++++++++++++++++++++++ src/dev/usb_hid/tests.rs | 448 ++++++++++++++++++++++++++++ src/devlog.rs | 8 +- 10 files changed, 2596 insertions(+), 7 deletions(-) create mode 100644 src/dev/ohci.rs create mode 100644 src/dev/ohci/engine.rs create mode 100644 src/dev/ohci/schedule.rs create mode 100644 src/dev/ohci/tests.rs create mode 100644 src/dev/usb_hid.rs create mode 100644 src/dev/usb_hid/tests.rs diff --git a/debug.md b/debug.md index 53912da82..65900e3f1 100644 --- a/debug.md +++ b/debug.md @@ -593,7 +593,7 @@ Per-module logging; all of it needs a developer build to produce output. | `log file ` | Send a module's log to a file | Modules: `net hpc3 seeq hal2 mc rex3 mips ioc scsi pdma vino dcb vc2 cmap -xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock pci bridge heart nvme l1`. `pci` logs every +xmap bt445 scc ps2 rtc eeprom l1i l1d l2c bedrock pci bridge heart nvme l1 usb`. `pci` logs every configuration cycle on a PCI bus (MACE's, the Octane Bridge's) and whether a function answered it. `bridge` logs the Octane Bridge's register and translation-table accesses by name, and any transfer a PCI @@ -604,8 +604,8 @@ interrupt vector raised or lowered. Mask categories: `pdma` — `hal enet scsi`; `mips` — `insn tlb mem fpu cp0`; `l1i`/`l1d`/`l2c` — `hit miss op`; `nvme` — `state admin regs io` ([docs/nvme.md](docs/nvme.md#tracing)); every module also takes `on`/`all`, `off`/`none` or a hex mask. -`bedrock` covers the Fuel's devices on its Bedrock hub, and `l1` its system -controller. +`bedrock` covers the Fuel's devices on its Bedrock hub, `l1` its system +controller and `usb` its USB host controller. `IRIS_DEBUG_LOG` or `[debug] debug_log` set the same thing at startup. ### Performance and locks diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 776ff17a6..1a158fa99 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -63,6 +63,7 @@ pub mod seeq8003; pub mod testdev; pub mod timer; pub mod ultra64; +pub mod usb_hid; pub mod vino; pub mod virtio; pub mod wd33c93a; @@ -80,6 +81,7 @@ pub mod dp83843; pub mod ics1890; pub mod isp12160; pub mod l1; +pub mod ohci; pub mod timekeeper; pub mod prom_image; pub mod xbridge; diff --git a/src/dev/ohci.rs b/src/dev/ohci.rs new file mode 100644 index 000000000..edbbb6c62 --- /dev/null +++ b/src/dev/ohci.rs @@ -0,0 +1,584 @@ +// Derived from MAME's xbox_usb.cpp and xbox_usb.h (the OHCI host controller), +// BSD-3-Clause, copyright Samuele Zannoli. + +//! The Lucent USS-312 USB controller: two root ports, OHCI registers and +//! descriptor processing through the PCI bridge's DMA interface. +//! +//! Functions plug into a root port with `attach`: the Fuel fits a keyboard and +//! a mouse (`usb_hid`), and the tests drive the descriptor and packet +//! scheduling engine through a test-only function. Isochronous +//! traffic, suspend/resume, external hubs, USB storage and legacy keyboard/SMI +//! routing are unsupported. Scheduling uses synthetic work units rather than +//! wire time. Register behavior follows OpenHCI 1.0a and the USS-312 revision 5 +//! data sheet. +//! +//! Ported from src/mame/shared/xbox_usb.cpp and xbox_usb.h in MAME, +//! revision 2ee9b1e30299b122b131cab0bd7912abc2b095ba. + +use crate::dev::heart::{host_clock, Clock}; +use crate::irq::IrqLine; +use crate::pci::{Bar, PciConfig, PciConfigPolicy, PciDma, PciFunction, PciFunctionState, PCI_INTA}; +use crate::traits::{Device, Resettable}; +use parking_lot::Mutex; +use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::{Arc, OnceLock}; + +#[derive(Clone, Copy, Debug, PartialEq, Eq)] +#[repr(u32)] +pub enum Pid { + Setup = 0, + Out = 1, + In = 2, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub struct UsbPacket { + pub endpoint: u8, + pub pid: Pid, + pub toggle: bool, + /// IN requests name capacity; OUT and SETUP carry their bytes. + pub capacity: usize, + pub data: Vec, + pub time_ms: u64, +} +#[derive(Clone, Debug, PartialEq, Eq)] +pub enum UsbOutcome { + Ack(Vec), + Nak, + Stall, + NoResponse, + CrcError, + BitStuffError, +} +pub trait UsbFunction: Send + Sync { + fn reset(&self); + fn address(&self) -> u8; + fn low_speed(&self) -> bool { + false + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome; +} + +mod engine; +mod schedule; + +const EVENTS: u32 = 0x4000_007f; +const WDH: u32 = 2; +const UE: u32 = 16; +const RHSC: u32 = 64; +const PPS: u32 = 0x100; +const CHANGES: u32 = 0x1f0000; +#[derive(Clone)] +struct State { + r: [u32; 23], + ports: [Option>; 2], + low_speed: [bool; 2], + reset_ms: [u8; 2], + done_delay: u8, + now_ms: u64, + active_fi: u32, + bit_credit: u64, +} +impl State { + fn new() -> Self { + let mut r = [0; 23]; + r[0] = 0x110; + r[13] = 0x2edf; + r[17] = 0x628; + r[18] = 0x1000_0902; + r[19] = 0x60000; + Self { + r, + ports: [None, None], + low_speed: [false; 2], + reset_ms: [0; 2], + done_delay: 7, + now_ms: 0, + active_fi: 0x2edf, + bit_credit: 0, + } + } + fn soft_reset(&mut self) { + let control = (self.r[1] & 0x300) | 0xc0; + let keep = [self.r[18], self.r[19], self.r[20], self.r[21], self.r[22]]; + let ports = self.ports.clone(); + let low_speed = self.low_speed; + let resets = self.reset_ms; + let time = self.now_ms; + *self = Self::new(); + self.r[1] = control; + self.r[18..23].copy_from_slice(&keep); + self.ports = ports; + self.low_speed = low_speed; + self.reset_ms = resets; + self.now_ms = time; + } + fn irq(&self) -> bool { + self.r[1] & 0x100 == 0 && self.r[4] & 0x80000000 != 0 && self.r[3] & self.r[4] & EVENTS != 0 + } + fn powered(&self, p: usize) -> bool { + self.r[18] & 0x200 != 0 || self.r[21 + p] & PPS != 0 + } + fn individual(&self, p: usize) -> bool { + self.r[18] & 0x100 != 0 && self.r[19] & (1 << (17 + p)) != 0 + } + fn power(&mut self, p: usize, on: bool) { + if self.r[18] & 0x200 != 0 || on { + self.r[21 + p] |= PPS; + if self.ports[p].is_some() && self.r[21 + p] & 1 == 0 { + self.r[21 + p] |= 1 | 0x10000 | if self.low_speed[p] { 0x200 } else { 0 }; + self.r[3] |= RHSC; + } + } else { + let previous = self.r[21 + p]; + self.r[21 + p] &= !(PPS | 0x17); + if previous & 1 != 0 { + self.r[21 + p] |= 0x10000; + } + if previous & 2 != 0 { + self.r[21 + p] |= 0x20000; + } + if previous & 3 != 0 { + self.r[3] |= RHSC; + } + self.reset_ms[p] = 0; + } + } +} + +pub struct Ohci { + pci: PciFunctionState, + state: Mutex, + pass: Mutex<()>, + running: AtomicBool, + /// Source of the frame count: the machine's own clock, so the guest's + /// idea of a millisecond and the controller's agree. + time: OnceLock, + /// `time` reading at the last frame boundary delivered; `None` while stopped. + clock: Mutex>, +} +impl Ohci { + pub fn new() -> Arc { + let config = PciConfig::new_with_policy( + 0x11c1, + 0x5802, + 0x0c0310, + 0x10, + [ + Bar::Mem(0x1000), + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + Bar::Unused, + ], + PCI_INTA, + PciConfigPolicy { + status: 0x0200, + status_w1c_mask: 0x8000, + command_mask: 0x0006, + latency_mask: 0, + cache_line_mask: 0, + registers: &[], + }, + ); + config.write(11, 0xffff_ffff); + Arc::new(Self { + pci: PciFunctionState::new(config), + state: Mutex::new(State::new()), + pass: Mutex::new(()), + running: AtomicBool::new(false), + time: OnceLock::new(), + clock: Mutex::new(None), + }) + } + /// Use `clock` for frame timing. Call before `start`. + pub fn set_clock(&self, clock: Clock) { + assert!( + !self.running.load(Ordering::Acquire) && self.time.set(clock).is_ok(), + "connect the OHCI clock once, before start" + ); + } + fn now(&self) -> u64 { + (self.time.get_or_init(host_clock))() + } + /// Deliver every frame boundary the clock has passed. Every register + /// access calls this, so a read sees the frame count the guest's own + /// clock implies, however late the machine's poll or timer ran. + pub fn sync_clock(&self) { + // Held through delivery: a second caller must not find the boundary + // consumed and read the frame count before the first has applied it. + let mut clock = self.clock.lock(); + let Some(at) = clock.as_mut() else { return }; + let now = self.now(); + let elapsed = now.saturating_sub(*at) / 1_000_000; + *at += elapsed * 1_000_000; + self.advance_frames(elapsed); + } + fn drive_irq(&self) { + loop { + let level = self.state.lock().irq(); + if let Some(line) = self.pci.irq() { + line.set(level); + } + // A callback may acknowledge the interrupt or another writer may + // change it while delivery is outside the state lock. + if self.state.lock().irq() == level { + break; + } + } + } + pub fn attach(&self, port: usize, function: Arc) -> Result<(), String> { + if port >= 2 { + return Err("OHCI has two ports".into()); + } + if self.state.lock().ports[port].is_some() { + return Err("OHCI port occupied".into()); + } + let low_speed = function.low_speed(); + function.reset(); + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + if s.ports[port].is_some() { + return Err("OHCI port occupied".into()); + } + if s.powered(port) { + s.r[21 + port] |= 1 | 0x10000; + } + s.low_speed[port] = low_speed; + if low_speed { + s.r[21 + port] |= 0x200; + } + s.ports[port] = Some(function); + s.r[3] |= RHSC; + } + self.drive_irq(); + Ok(()) + } + #[cfg(test)] + pub fn detach(&self, port: usize) -> Result<(), String> { + if port >= 2 { + return Err("OHCI has two ports".into()); + } + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + let enabled = s.r[21 + port] & 2; + s.ports[port] = None; + s.low_speed[port] = false; + s.reset_ms[port] = 0; + s.r[21 + port] = + (s.r[21 + port] & PPS) | 0x10000 | if enabled != 0 { 0x20000 } else { 0 }; + s.r[3] |= RHSC; + } + self.drive_irq(); + Ok(()) + } + /// Deterministic millisecond ticks; `sync_clock` delivers the same ticks from the clock. + pub fn advance_frames(&self, count: u64) { + for _ in 0..count { + if !self.running.load(Ordering::Acquire) { + break; + } + { + let _pass = self.pass.lock(); + if !self.running.load(Ordering::Acquire) { + break; + } + let mut s = self.state.lock().clone(); + self.frame(&mut s); + *self.state.lock() = s; + } + self.drive_irq(); + } + } + fn frame(&self, s: &mut State) { + s.now_ms = s.now_ms.wrapping_add(1); + for p in 0..2 { + if s.reset_ms[p] != 0 { + s.reset_ms[p] -= 1; + if s.reset_ms[p] == 0 { + s.r[21 + p] = (s.r[21 + p] & !16) | 2 | (1 << 20); + s.r[3] |= RHSC; + } + } + } + if s.r[1] & 0xc0 != 0x80 || s.r[3] & UE != 0 { + return; + } + s.bit_credit += 12000; + while s.bit_credit >= u64::from((s.active_fi & 0x3fff) + 1) { + s.bit_credit -= u64::from((s.active_fi & 0x3fff) + 1); + s.active_fi = s.r[13]; + if self.schedule(s).is_err() { + s.r[3] |= UE; + s.bit_credit = 0; + self.pci.config().set_status_bits(0x8000); + break; + } + } + } +} +impl PciFunction for Ohci { + fn state(&self) -> &PciFunctionState { + &self.pci + } + fn attached(&self) { + self.drive_irq(); + } + fn reset(&self) { + Resettable::power_on(self); + } + fn bar_read(&self, bar: usize, off: u32, width: usize) -> u32 { + if bar != 0 || width != 4 || off & 3 != 0 || off > 0x58 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci unsupported read at register offset {off:#x}, width {width}" + ); + return 0; + } + self.sync_clock(); + let s = self.state.lock(); + let i = (off / 4) as usize; + if i == 5 { + s.r[4] + } else if i == 21 || i == 22 { + s.r[i] | if s.r[18] & 0x200 != 0 { PPS } else { 0 } + } else { + s.r[i] + } + } + fn bar_write(&self, bar: usize, off: u32, val: u32, width: usize) { + if bar != 0 || width != 4 || off & 3 != 0 || off > 0x58 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci unsupported write at register offset {off:#x}, width {width}" + ); + return; + } + self.sync_clock(); + // Taken before `pass`, as `sync_clock` orders them. Only a control + // write can restart the frame phase. + let mut anchor = (off == 4).then(|| self.clock.lock()); + let mut reset = None; + { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + match off / 4 { + 1 => { + let old = s.r[1]; + s.r[1] = val & 0x7ff; + if val & 0xc0 == 0x80 && old & 0xc0 != 0x80 { + // OpenHCI 1.0a 6.2.1, 7.3.2, 7.3.3: entering + // UsbOperational loads HcFmRemaining from the frame + // interval and increments HcFmNumber, with no SOF at + // the load; the first follows one frame later. The + // free-running clock's phase therefore restarts here. + s.bit_credit = 0; + s.active_fi = s.r[13]; + s.r[14] = (s.r[13] & 0x8000_0000) | (s.r[13] & 0x3fff); + let old_frame = s.r[15] as u16; + let frame = old_frame.wrapping_add(1); + s.r[15] = u32::from(frame); + if (old_frame ^ frame) & 0x8000 != 0 { + s.r[3] |= 32; + } + if let Some(Some(at)) = anchor.as_deref_mut() { + *at = self.now(); + } + } + if val & 0xc0 == 0 && old & 0xc0 != 0 { + s.r[18] = 0x10000902; + for p in 0..2 { + s.power(p, false); + } + } + if val & 0xc0 == 0x40 || val & 0xc0 == 0xc0 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci suspend/resume timing unsupported at control register offset 0x04" + ); + } + } + 2 => { + if val & 1 != 0 { + s.soft_reset(); + } else { + s.r[2] |= val & 6; + if val & 8 != 0 { + s.r[3] |= 1 << 30; + } + } + } + 3 => s.r[3] &= !(val & EVENTS), + 4 => s.r[4] |= val & (EVENTS | 0x80000000), + 5 => s.r[4] &= !(val & (EVENTS | 0x80000000)), + 6 => s.r[6] = val & !255, + 7..=11 => s.r[(off / 4) as usize] = val & !15, + 13 => s.r[13] = val & 0xffff3fff, + 16 => s.r[16] = val & 0x3fff, + 17 => s.r[17] = val & 0xfff, + 18 => { + s.r[18] = (s.r[18] & !0xff001b00) | (val & 0xff001b00); + if val & 0x200 != 0 { + for p in 0..2 { + s.power(p, true); + } + } + } + 19 => s.r[19] = val & 0x60006, + 20 => { + if val & (1 << 15) != 0 { + s.r[20] |= 1 << 15; + } + if val & (1 << 31) != 0 { + s.r[20] &= !(1 << 15); + } + if val & (1 << 17) != 0 { + s.r[20] &= !(1 << 17); + } + if s.r[18] & 0x200 == 0 { + for p in 0..2 { + if !s.individual(p) { + if val & 1 != 0 { + s.power(p, false); + } + if val & (1 << 16) != 0 { + s.power(p, true); + } + } + } + } + } + 21 | 22 => { + let p = (off / 4 - 21) as usize; + let i = 21 + p; + s.r[i] &= !(val & CHANGES); + if val & 1 != 0 { + s.r[i] &= !2; + } + if val & 2 != 0 { + if s.r[i] & 1 != 0 && s.powered(p) { + s.r[i] |= 2; + } else { + s.r[i] |= 1 << 16; + } + } + if val & 12 != 0 { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci port suspend/resume unsupported at port register offset {:#x}", + 0x54 + p * 4 + ); + } + if val & 16 != 0 { + if s.r[i] & 1 != 0 && s.powered(p) { + s.r[i] |= 16; + s.r[i] &= !2; + s.reset_ms[p] = 10; + reset = s.ports[p].clone(); + } else { + s.r[i] |= 1 << 16; + } + } + if s.individual(p) && s.r[18] & 0x200 == 0 { + if val & PPS != 0 { + s.power(p, true); + } + if val & 0x200 != 0 { + s.power(p, false); + } + } + if s.r[i] & CHANGES != 0 { + s.r[3] |= RHSC; + } + } + _ => { + crate::dlog!( + crate::devlog::LogModule::Usb, + "ohci write to read-only register offset {off:#x}" + ); + } + } + } + drop(anchor); + if let Some(f) = reset { + f.reset(); + } + self.drive_irq(); + } +} +impl Resettable for Ohci { + fn power_on(&self) { + let ports = { + let _pass = self.pass.lock(); + let mut s = self.state.lock(); + let ports = s.ports.clone(); + let low_speed = s.low_speed; + *s = State::new(); + s.ports = ports.clone(); + s.low_speed = low_speed; + ports + }; + for f in ports.into_iter().flatten() { + f.reset(); + } + self.drive_irq(); + } +} +impl Device for Ohci { + fn start(&self) { + if self.running.swap(true, Ordering::AcqRel) { + return; + } + let now = self.now(); + *self.clock.lock() = Some(now); + } + fn stop(&self) { + self.sync_clock(); + self.running.store(false, Ordering::Release); + *self.clock.lock() = None; + drop(self.pass.lock()); + } + fn is_running(&self) -> bool { + self.running.load(Ordering::Acquire) + } + fn step(&self, _: u64) {} + fn get_clock(&self) -> u64 { + 0 + } +} + +#[cfg(test)] +mod register_tests { + use super::*; + use crate::pci::PciFunction; + use crate::traits::Resettable; + + #[test] + fn lucent_identity_and_reset_contract() { + let hc = Ohci::new(); + assert_eq!(hc.state().config().read(0), 0x5802_11c1); + assert_eq!(hc.state().config().read(2), 0x0c03_1010); + assert_eq!(hc.state().config().read(11), 0xffff_ffff); + assert_eq!(hc.state().config().read(1), 0x0200_0000); + assert_eq!(hc.bar_read(0, 0, 4), 0x110); + assert_eq!(hc.bar_read(0, 0x10, 4), 0); + assert_eq!(hc.bar_read(0, 0x34, 4), 0x2edf); + assert_eq!(hc.bar_read(0, 0x48, 4), 0x1000_0902); + assert_eq!(hc.bar_read(0, 0x4c, 4), 0x0006_0000); + for p in [0x54, 0x58] { + assert_eq!(hc.bar_read(0, p, 4), 0); + } + hc.bar_write(0, 0x10, 0xffff_ffff, 4); + hc.bar_write(0, 0x08, 1, 4); + assert_eq!(hc.bar_read(0, 0x10, 4), 0); + assert_eq!(hc.bar_read(0, 0x04, 4) & 0xc0, 0xc0); + hc.power_on(); + assert_eq!(hc.bar_read(0, 0x04, 4), 0); + } +} + +#[cfg(test)] +#[path = "ohci/tests.rs"] +mod tests; diff --git a/src/dev/ohci/engine.rs b/src/dev/ohci/engine.rs new file mode 100644 index 000000000..7b76cc646 --- /dev/null +++ b/src/dev/ohci/engine.rs @@ -0,0 +1,223 @@ +//! General TD packet processing; guest DMA byte slices are the endianness edge. +use super::*; + +fn read_words(dma: &dyn PciDma, at: u32) -> Result<[u32; 4], ()> { + let mut bytes = [0; 16]; + if !dma.dma_read(at as u64, &mut bytes).is_ok() { + return Err(()); + } + Ok(std::array::from_fn(|i| { + u32::from_le_bytes(bytes[i * 4..i * 4 + 4].try_into().unwrap()) + })) +} +fn write_words(dma: &dyn PciDma, at: u32, words: &[u32]) -> Result<(), ()> { + let bytes: Vec = words.iter().flat_map(|x| x.to_le_bytes()).collect(); + if dma.dma_write(at as u64, &bytes).is_ok() { + Ok(()) + } else { + Err(()) + } +} +fn remaining(cbp: u32, be: u32) -> Result { + if cbp == 0 { + return Ok(0); + } + if cbp & !0xfff == be & !0xfff { + be.checked_sub(cbp).map(|n| n as usize + 1).ok_or(()) + } else { + Ok(0x1000 - (cbp as usize & 0xfff) + (be as usize & 0xfff) + 1) + } +} +fn advance(cbp: u32, be: u32, n: usize) -> u32 { + let first = 0x1000 - (cbp as usize & 0xfff); + if n < first { + cbp.wrapping_add(n as u32) + } else { + (be & !0xfff).wrapping_add((n - first) as u32) + } +} +fn payload_read(dma: &dyn PciDma, cbp: u32, be: u32, len: usize) -> Result, ()> { + let mut bytes = vec![0; len]; + let first = len.min(0x1000 - (cbp as usize & 0xfff)); + if first != 0 && !dma.dma_read(cbp as u64, &mut bytes[..first]).is_ok() { + return Err(()); + } + if len > first + && !dma + .dma_read((be & !0xfff) as u64, &mut bytes[first..]) + .is_ok() + { + return Err(()); + } + Ok(bytes) +} +fn payload_write(dma: &dyn PciDma, cbp: u32, be: u32, bytes: &[u8]) -> Result<(), ()> { + let first = bytes.len().min(0x1000 - (cbp as usize & 0xfff)); + if first != 0 && !dma.dma_write(cbp as u64, &bytes[..first]).is_ok() { + return Err(()); + } + if bytes.len() > first && !dma.dma_write((be & !0xfff) as u64, &bytes[first..]).is_ok() { + return Err(()); + } + Ok(()) +} +impl Ohci { + /// Returns the next ED and whether this ED still holds queued general work. + pub(super) fn endpoint( + &self, + s: &mut State, + dma: &dyn PciDma, + at: u32, + ) -> Result<(u32, bool, bool), ()> { + let mut ed = read_words(dma, at)?; + let next = ed[3] & !15; + if ed[0] & (1 << 15) != 0 { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci isochronous ED unsupported at guest address {at:#x}"); + return Ok((next, false, false)); + } + if ed[0] & (1 << 14) != 0 || ed[2] & 1 != 0 || ed[2] & !15 == ed[1] & !15 { + return Ok((next, false, false)); + } + let td_at = ed[2] & !15; + let mut td = read_words(dma, td_at)?; + let mps = ((ed[0] >> 16) & 0x7ff) as usize; + if mps == 0 || mps > 1023 { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci invalid maximum packet size {mps} at ED address {at:#x}"); + return Ok((next, false, false)); + } + let Ok(n) = remaining(td[1], td[3]) else { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci malformed TD buffer range at ED address {at:#x}"); + return Ok((next, false, false)); + }; + let direction = match (ed[0] >> 11) & 3 { + 1 => 1, + 2 => 2, + _ => (td[0] >> 19) & 3, + }; + let pid = match direction { + 0 => Pid::Setup, + 1 => Pid::Out, + 2 => Pid::In, + _ => { + crate::dlog!(crate::devlog::LogModule::Usb, "ohci reserved PID at ED address {at:#x}"); + return Ok((next, false, false)); + } + }; + let length = n.min(mps); + let toggle = if td[0] & (1 << 25) != 0 { + td[0] & (1 << 24) != 0 + } else { + ed[2] & 2 != 0 + }; + let outgoing = if pid != Pid::In { + payload_read(dma, td[1], td[3], length)? + } else { + vec![] + }; + let address = (ed[0] & 127) as u8; + let mut functions = (0..2) + .filter(|&p| s.powered(p) && s.r[21 + p] & 0x16 == 2) + .filter_map(|p| s.ports[p].clone()) + .filter(|f| f.address() == address); + // Multiple matching functions are an unsupported topology. Do not choose + // a recipient by port order or claim to model electrical collisions. + let first = functions.next(); + let function = if functions.next().is_none() { + first + } else { + None + }; + let reply = if let Some(f) = function { + f.transact(UsbPacket { + endpoint: ((ed[0] >> 7) & 15) as u8, + pid, + toggle, + capacity: if pid == Pid::In { mps } else { length }, + data: outgoing, + time_ms: s.now_ms, + }) + } else { + UsbOutcome::NoResponse + }; + let mut retire = false; + let mut cc = 0; + let mut halt = false; + match reply { + UsbOutcome::Nak => return Ok((next, true, true)), + UsbOutcome::Stall => { + cc = 4; + halt = true; + retire = true; + } + UsbOutcome::NoResponse | UsbOutcome::CrcError | UsbOutcome::BitStuffError => { + cc = match reply { + UsbOutcome::CrcError => 1, + UsbOutcome::BitStuffError => 2, + _ => 5, + }; + let ec = ((td[0] >> 26) & 3) + 1; + td[0] = (td[0] & !(3 << 26)) | ((ec.min(3)) << 26); + if ec >= 3 { + retire = true; + halt = true; + } + } + UsbOutcome::Ack(bytes) => { + let actual = if pid == Pid::In { bytes.len() } else { length }; + if pid == Pid::In { + payload_write(dma, td[1], td[3], &bytes[..actual.min(n).min(mps)])?; + } + td[0] = (td[0] & !((3 << 24) | (3 << 26))) | ((2 | u32::from(!toggle)) << 24); + if actual > length { + cc = 8; + halt = true; + retire = true; + } else { + td[1] = if actual == n { + 0 + } else { + advance(td[1], td[3], actual) + }; + if actual < length { + retire = true; + if td[0] & (1 << 18) == 0 { + cc = 9; + halt = true; + } + } else if actual == n { + retire = true; + } + } + } + } + td[0] = (td[0] & 0x0fff_ffff) | (cc << 28); + if retire { + let next_td = td[2] & !15; + td[2] = s.r[12]; + // Failure stops immediately. Earlier successful writes and USB side + // effects remain visible; descriptor writeback is not a transaction. + ed[2] = next_td | ((td[0] >> 23) & 2) | u32::from(halt); + write_words(dma, at + 8, &[ed[2]])?; + write_words(dma, td_at, &td)?; + s.r[12] = td_at; + let di = ((td[0] >> 21) & 7) as u8; + s.done_delay = s.done_delay.min(if cc != 0 { 0 } else { di }); + Ok((next, next_td != ed[1] & !15 && !halt, true)) + } else { + write_words(dma, td_at, &td)?; + Ok((next, true, true)) + } + } + pub(super) fn publish(&self, s: &mut State, dma: &dyn PciDma) -> Result<(), ()> { + if s.done_delay == 0 && s.r[3] & WDH == 0 && s.r[12] != 0 { + let other = u32::from(s.r[3] & s.r[4] & EVENTS & !WDH != 0); + write_words(dma, s.r[6] + 0x84, &[s.r[12] | other])?; + s.r[12] = 0; + s.done_delay = 7; + s.r[3] |= WDH; + } else if s.done_delay > 0 && s.done_delay < 7 { + s.done_delay -= 1; + } + Ok(()) + } +} diff --git a/src/dev/ohci/schedule.rs b/src/dev/ohci/schedule.rs new file mode 100644 index 000000000..5042b9da1 --- /dev/null +++ b/src/dev/ohci/schedule.rs @@ -0,0 +1,98 @@ +//! Functional scheduling approximation: 256 synthetic units per ED visit. +//! Costs deliberately do not represent wire time. See the module's limitations. +use super::*; +const VISIT: u32 = 256; +impl Ohci { + fn list_ready(s: &mut State, head: usize, current: usize, filled: u32) -> bool { + if s.r[current] == 0 && s.r[2] & filled != 0 { + s.r[current] = s.r[head]; + s.r[2] &= !filled; + } + s.r[current] != 0 + } + fn nonperiodic( + &self, + s: &mut State, + dma: &dyn PciDma, + budget: &mut u32, + cutoff: u32, + credit: &mut u32, + ) -> Result<(), ()> { + let mut yielded: Option = None; + while *budget >= VISIT && *budget > cutoff { + let control = s.r[1] & 16 != 0 && Self::list_ready(s, 8, 9, 2); + let bulk = s.r[1] & 32 != 0 && Self::list_ready(s, 10, 11, 4); + if !control && !bulk { + break; + } + let choose_control = control + && (!bulk + || match yielded { + Some(was_control) => !was_control, + None => *credit > 0, + }); + let (current, filled) = if choose_control { (9, 2) } else { (11, 4) }; + let (next, queued, attempt) = self.endpoint(s, dma, s.r[current])?; + *budget -= VISIT; + s.r[current] = next; + if queued { + s.r[2] |= filled; + } + yielded = if attempt { None } else { Some(choose_control) }; + if attempt { + if choose_control { + *credit = credit.saturating_sub(1); + } else { + *credit = (s.r[1] & 3) + 1; + } + } + } + Ok(()) + } + pub(super) fn schedule(&self, s: &mut State) -> Result<(), ()> { + let old = s.r[15] as u16; + let frame = old.wrapping_add(1); + s.r[15] = u32::from(frame); + if (old ^ frame) & 0x8000 != 0 { + s.r[3] |= 32; + } + s.r[3] |= 4; + let mut budget = (s.active_fi & 0x3fff) + 1; + s.r[14] = (s.active_fi & 0x80000000) | budget.min(0x3fff); + if self.pci.config().command() & 4 == 0 { + return Ok(()); + } + let Some(dma) = self.pci.dma().cloned() else { + return Err(()); + }; + let frame_bytes = [frame as u8, (frame >> 8) as u8, 0, 0]; + if !dma.dma_write((s.r[6] + 0x80) as u64, &frame_bytes).is_ok() { + return Err(()); + } + let mut credit = (s.r[1] & 3) + 1; + let cutoff = s.r[16].min(budget); + self.nonperiodic(s, &*dma, &mut budget, cutoff, &mut credit)?; + if s.r[1] & 4 != 0 { + let mut bytes = [0; 4]; + if !dma + .dma_read((s.r[6] + u32::from(frame & 31) * 4) as u64, &mut bytes) + .is_ok() + { + return Err(()); + } + s.r[7] = u32::from_le_bytes(bytes) & !15; + while s.r[7] != 0 && budget >= VISIT { + let (next, _, _) = self.endpoint(s, &*dma, s.r[7])?; + s.r[7] = next; + budget -= VISIT; + } + if s.r[7] != 0 { + s.r[3] |= 1; + s.r[2] = (s.r[2] & !0x30000) | ((s.r[2].wrapping_add(0x10000)) & 0x30000); + } + } + self.nonperiodic(s, &*dma, &mut budget, 0, &mut credit)?; + s.r[14] = (s.active_fi & 0x80000000) | budget.min(0x3fff); + self.publish(s, &*dma) + } +} diff --git a/src/dev/ohci/tests.rs b/src/dev/ohci/tests.rs new file mode 100644 index 000000000..6717287ea --- /dev/null +++ b/src/dev/ohci/tests.rs @@ -0,0 +1,555 @@ +// The register and clock cases follow tests/qtest/usb-lucent-ohci-test.c in +// jamesbraid/qemu-sgi-origami: root-hub-descriptor-a-is-writable and +// frame-number-tracks-clock-while-timer-starved. The descriptor engine cases +// drive a test-only function and have no qtest counterpart. +use super::*; +use crate::pci::{PciDmaStatus, PciFunction}; +use crate::traits::{Device, Resettable}; +use std::sync::atomic::{AtomicUsize, Ordering}; +use std::sync::Weak; + +struct Memory { + bytes: Mutex>, + refuse_read: Mutex>, + refuse_write: Mutex>, + writes: Mutex>, +} +impl Memory { + fn new() -> Arc { + Arc::new(Self { + bytes: Mutex::new(vec![0xa5; 0x10000]), + refuse_read: Mutex::new(None), + refuse_write: Mutex::new(None), + writes: Mutex::new(vec![]), + }) + } + fn words(&self, at: u32, words: &[u32]) { + for (i, w) in words.iter().enumerate() { + self.bytes.lock()[at as usize + i * 4..at as usize + i * 4 + 4] + .copy_from_slice(&w.to_le_bytes()); + } + } + fn word(&self, at: u32) -> u32 { + u32::from_le_bytes( + self.bytes.lock()[at as usize..at as usize + 4] + .try_into() + .unwrap(), + ) + } +} +impl PciDma for Memory { + fn dma_read(&self, addr: u64, dst: &mut [u8]) -> PciDmaStatus { + if self + .refuse_read + .lock() + .is_some_and(|x| (addr..addr + dst.len() as u64).contains(&(x as u64))) + { + return PciDmaStatus::MasterAbort; + } + let b = self.bytes.lock(); + let Some(s) = b.get(addr as usize..addr as usize + dst.len()) else { + return PciDmaStatus::MasterAbort; + }; + dst.copy_from_slice(s); + PciDmaStatus::Ok + } + fn dma_write(&self, addr: u64, src: &[u8]) -> PciDmaStatus { + if self + .refuse_write + .lock() + .is_some_and(|x| (addr..addr + src.len() as u64).contains(&(x as u64))) + { + return PciDmaStatus::MasterAbort; + } + let mut b = self.bytes.lock(); + let Some(s) = b.get_mut(addr as usize..addr as usize + src.len()) else { + return PciDmaStatus::MasterAbort; + }; + s.copy_from_slice(src); + self.writes.lock().push(addr); + PciDmaStatus::Ok + } +} +struct Function { + address: AtomicUsize, + calls: AtomicUsize, + reply: Mutex, + packets: Mutex>, +} +impl Function { + fn new(reply: UsbOutcome) -> Arc { + Arc::new(Self { + address: AtomicUsize::new(0), + calls: AtomicUsize::new(0), + reply: Mutex::new(reply), + packets: Mutex::new(vec![]), + }) + } +} +impl UsbFunction for Function { + fn reset(&self) {} + fn address(&self) -> u8 { + self.address.load(Ordering::SeqCst) as u8 + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome { + self.calls.fetch_add(1, Ordering::SeqCst); + self.packets.lock().push(packet); + self.reply.lock().clone() + } +} +fn start_port_reset(h: &Ohci, port: u32) { + let status = 0x54 + port * 4; + h.bar_write(0, status, 0x100, 4); + h.bar_write(0, status, 0x10, 4); +} + +fn setup( + pid: Pid, + reply: UsbOutcome, + cbp: u32, + be: u32, + flags: u32, +) -> (Arc, Arc, Arc) { + let h = Ohci::new(); + let m = Memory::new(); + let f = Function::new(reply); + h.state().connect_dma(m.clone()); + h.attach(0, f.clone()).unwrap(); + h.start(); + start_port_reset(&h, 0); + h.advance_frames(10); + m.words(0x1000, &[0; 64]); + m.words(0x2000, &[(8 << 16), 0x3020, 0x3000, 0]); + m.words( + 0x3000, + &[0xf000_0000 | flags | ((pid as u32) << 19), cbp, 0x3020, be], + ); + h.state().config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + h.bar_write(0, 0x20, 0x2000, 4); + h.bar_write(0, 0x08, 2, 4); + h.bar_write(0, 0x04, 0x90, 4); + (h, m, f) +} +/// The guest receives fatal status if the controller cannot access a descriptor. +#[test] +fn dma_refusal_preserves_descriptors_and_reports_a_fatal_error() { + for periodic in [false, true] { + for pid in [Pid::Out, Pid::In] { + let (h, m, f) = setup(pid, UsbOutcome::Ack(vec![5; 8]), 0x4000, 0x4007, 0); + if periodic { + m.words(0x1000, &[0x2000; 32]); + h.bar_write(0, 4, 0x84, 4); + } + if pid == Pid::In { + *m.refuse_write.lock() = Some(0x4000); + } else { + *m.refuse_read.lock() = Some(0x4000); + } + let before = m.bytes.lock()[0x3000..0x3010].to_vec(); + h.advance_frames(1); + let frame = h.bar_read(0, 0x3c, 4); + h.advance_frames(3); + assert_eq!(f.calls.load(Ordering::SeqCst), usize::from(pid == Pid::In)); + assert_eq!(&m.bytes.lock()[0x3000..0x3010], &before); + assert_eq!(m.word(0x2008), 0x3000); + assert_eq!(m.word(0x1084), 0); + assert_eq!(h.bar_read(0, 0x3c, 4), frame); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + assert_eq!(h.state().config().read(1) >> 16, 0x8200); + h.state().config().write(1, 0x8000_0006); + assert_eq!(h.state().config().read(1) >> 16, 0x0200); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + } + } +} + +/// Short and overlong USB replies leave bytes outside the packet untouched. +#[test] +fn short_or_overlong_in_packets_preserve_unwritten_bytes() { + for (name, payload, end, flags, bytes, current, condition) in [ + ("short", vec![1, 2, 3], 0x401f, (1 << 18) | (3 << 24), [1, 2, 3, 0xa5, 0xa5], 0x4003, 0), + ("overrun", vec![1; 8], 0x4003, 3 << 24, [1, 1, 1, 1, 0xa5], 0x4000, 8), + ] { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(payload), 0x4000, end, flags); + h.advance_frames(1); + assert_eq!(&m.bytes.lock()[0x4000..0x4005], &bytes, "{name}"); + assert_eq!(m.word(0x3004), current, "{name}"); + assert_eq!(m.word(0x3000) >> 28, condition, "{name}"); + assert_eq!((m.word(0x3000) >> 24) & 3, 2, "{name}"); + if condition == 0 { + assert_eq!(m.word(0x2008) & !15, 0x3020); + assert_eq!(m.word(0x1084) & !15, 0x3000); + } + } +} + +/// The guest retries NAKs, stops on STALL, and retains consistent short-transfer state. +#[test] +fn nak_stall_and_short_in_completions_keep_the_endpoint_consistent() { + for (name, reply, flags, condition) in [ + ("NAK", UsbOutcome::Nak, 0, 15), + ("STALL", UsbOutcome::Stall, 3 << 24, 4), + ("no response", UsbOutcome::NoResponse, 0, 5), + ("short without rounding", UsbOutcome::Ack(vec![2]), 0, 9), + ] { + let (h, m, f) = setup(Pid::In, reply, 0x4000, 0x4007, flags); + h.bar_write(0, 0x34, 255, 4); + let before = m.bytes.lock()[0x3000..0x3010].to_vec(); + h.advance_frames(1); + if condition == 5 { + assert_eq!(f.calls.load(Ordering::SeqCst), 1); + assert_eq!((m.word(0x3000) >> 26) & 3, 1); + assert_eq!(m.word(0x2008), 0x3000); + h.advance_frames(2); + } + assert_eq!(m.word(0x3000) >> 28, condition, "{name}"); + if condition == 15 { + assert_eq!(&m.bytes.lock()[0x3000..0x3010], &before); + assert_eq!(m.word(0x2008), 0x3000); + } else { + assert_ne!(m.word(0x2008) & 1, 0, "{name}"); + } + if condition == 4 { + assert_eq!(m.word(0x3004), 0x4000); + assert_eq!((m.word(0x3000) >> 24) & 3, 3); + assert_eq!(m.word(0x2008) & 3, 3); + } + } +} + +/// Port and controller reset paths retain attached USB functions. +#[test] +fn attached_functions_survive_port_and_controller_resets() { + let h = Ohci::new(); + h.start(); + h.attach(1, Function::new(UsbOutcome::Nak)).unwrap(); + assert_eq!(h.bar_read(0, 0x58, 4) & 1, 0); + start_port_reset(&h, 1); + h.advance_frames(9); + assert_ne!(h.bar_read(0, 0x58, 4) & 16, 0); + h.advance_frames(1); + assert_eq!(h.bar_read(0, 0x58, 4) & 16, 0); + assert_ne!(h.bar_read(0, 0x58, 4) & (1 << 20), 0); + h.bar_write(0, 8, 1, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); + h.bar_write(0, 0x58, 0x200, 4); + assert_eq!(h.bar_read(0, 0x58, 4) & 0x117, 0); + h.bar_write(0, 0x58, PPS, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); + h.power_on(); + assert_eq!(h.bar_read(0, 0x58, 4), 0); + h.bar_write(0, 0x58, PPS, 4); + assert_ne!(h.bar_read(0, 0x58, 4) & 1, 0); +} + +/// The guest observes descriptor updates before the done bit and acknowledges them later. +#[test] +fn done_publication_orders_writes_and_waits_for_acknowledgement() { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![8; 8]), 0x4000, 0x4007, 0); + h.advance_frames(1); + { + let writes = m.writes.lock(); + assert!(writes.iter().position(|a| *a == 0x2008).unwrap() < writes.iter().position(|a| *a == 0x3000).unwrap()); + } + assert_eq!(m.word(0x1084) & !15, 0x3000); + m.words(0x3020, &[0xf0000000 | (2 << 19), 0x4100, 0x3040, 0x4107]); + m.words(0x2004, &[0x3040]); + h.bar_write(0, 8, 2, 4); + h.advance_frames(1); + assert_eq!(m.word(0x1084) & !15, 0x3000); + assert_eq!(h.bar_read(0, 0x30, 4), 0x3020); + h.bar_write(0, 0xc, 2, 4); + h.advance_frames(1); + assert_eq!(m.word(0x1084) & !15, 0x3020); + for delay in [0, 2, 7] { + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![5; 8]), 0x4000, 0x4007, delay << 21); + h.advance_frames(1); + if delay != 0 { assert_eq!(m.word(0x1084), 0); } + h.advance_frames(u64::from(delay)); + assert_eq!(m.word(0x1084) & !15, if delay == 7 { 0 } else { 0x3000 }); + } + let (h, m, _) = setup(Pid::In, UsbOutcome::Ack(vec![1; 8]), 0x4000, 0x4007, 0); + *m.refuse_write.lock() = Some(0x1084); + h.advance_frames(1); + assert_ne!(h.bar_read(0, 0xc, 4) & UE, 0); + assert_eq!(h.bar_read(0, 0xc, 4) & WDH, 0); + assert_eq!(m.word(0x1084), 0); +} + +/// OHCI must yield to the other transfer list when a guest marks one list skipped. +#[test] +fn scheduler_shares_service_and_yields_past_skipped_lists() { + for (cbsr, skip_control, skip_bulk) in [(0, false, false), (3, false, false), (3, true, false), (0, false, true)] { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + m.words(0x2000, &[(8 << 16) | (1 << 7) | (u32::from(skip_control) << 14), 0x3020, 0x3000, if skip_control { 0x2000 } else { 0 }]); + m.words(0x2100, &[(8 << 16) | (2 << 7) | (u32::from(skip_bulk) << 14), 0x3120, 0x3100, if skip_bulk { 0x2100 } else { 0 }]); + m.words(0x3100, &[0xf0100000, 0x4100, 0x3120, 0x4107]); + h.bar_write(0, 0x28, 0x2100, 4); + h.bar_write(0, 8, 6, 4); + h.bar_write(0, 4, 0xb0 | cbsr, 4); + h.advance_frames(1); + let packets = f.packets.lock(); + let control = packets.iter().filter(|p| p.endpoint == 1).count(); + let bulk = packets.iter().filter(|p| p.endpoint == 2).count(); + assert_eq!(control > 0, !skip_control); + assert_eq!(bulk > 0, !skip_bulk); + if skip_bulk { + assert!(control > 1, "control resumes after its credit is exhausted"); + } + if !skip_control && !skip_bulk { + assert!(control >= bulk * (cbsr as usize + 1)); + assert!(control <= bulk * (cbsr as usize + 1) + cbsr as usize + 1); + } + assert_eq!(h.bar_read(0, 0xc, 4) & UE, 0); + } +} + +/// The controller routes address zero only while one attached function can claim it. +#[test] +fn address_zero_is_routed_only_to_a_unique_eligible_function() { + for (name, address, peer_status, first_selected, peer_selected) in [ + ("ambiguous", 0, PPS | 1 | 2, false, false), + ("disabled peer", 0, PPS | 1, true, false), + ("assigned first", 5, PPS | 1 | 2, false, true), + ] { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + let peer = Function::new(UsbOutcome::Nak); + h.attach(1, peer.clone()).unwrap(); + h.state.lock().r[22] = peer_status; + f.address.store(address, Ordering::SeqCst); + h.advance_frames(1); + assert_eq!(f.calls.load(Ordering::SeqCst) > 0, first_selected, "{name}"); + assert_eq!(peer.calls.load(Ordering::SeqCst) > 0, peer_selected, "{name}"); + if address != 0 { + let peer_calls = peer.calls.load(Ordering::SeqCst); + m.words(0x2000, &[(8 << 16) | 5]); + h.advance_frames(1); + assert!(f.calls.load(Ordering::SeqCst) > 0); + assert_eq!(peer.calls.load(Ordering::SeqCst), peer_calls); + } else if !first_selected { + assert_eq!(m.word(0x3000) >> 28, 5); + assert_eq!((m.word(0x3000) >> 26) & 3, 3); + } + } +} + + +/// A guest USB packet can cross physically nonadjacent DMA pages. +#[test] +fn packet_crosses_nonadjacent_pages() { + let (h, m, _) = setup( + Pid::In, + UsbOutcome::Ack(vec![1, 2, 3, 4, 5, 6, 7, 8]), + 0x4ffc, + 0x9003, + 0, + ); + h.advance_frames(1); + assert_eq!(&m.bytes.lock()[0x4ffc..0x5000], &[1, 2, 3, 4]); + assert_eq!(&m.bytes.lock()[0x9000..0x9004], &[5, 6, 7, 8]); + assert_eq!(m.bytes.lock()[0x5000], 0xa5); + assert_eq!(m.word(0x3004), 0); +} + +/// The frame counter catches up after its host timer was delayed. +#[test] +fn clock_read_catches_up_after_timer_starvation() { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(0)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + h.bar_write(0, 4, 0x80, 4); + now.store(5_000_000, Ordering::Relaxed); + assert!(h.bar_read(0, 0x3c, 4) >= 5); + h.stop(); + let frame = h.bar_read(0, 0x3c, 4); + h.advance_frames(10); + assert_eq!(h.bar_read(0, 0x3c, 4), frame); +} + +/// The frame count follows the guest's clock, not when the machine's poll or +/// a timer thread last ran: a guest that waits 1 ms after going operational +/// and reads HcFmNumber sees the frame, even if nothing else advanced the +/// controller in between (a slow or descheduled host thread). +#[test] +fn frame_number_follows_the_clock_with_no_poll_or_timer() { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(10_000_000)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + // Part-way into a frame period when the guest sets operational. + now.store(10_400_000, Ordering::Relaxed); + h.bar_write(0, 4, 0x80, 4); + // Entering Operational increments the frame number at once. + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + // 1.1 ms on the guest's clock: the first boundary, 1 ms after the write, has passed. + now.store(11_500_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 2); + // A long guest-time gap is delivered in full at the next access. + now.store(31_500_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 22); + // Time that has not passed on the guest's clock delivers nothing. + assert_eq!(h.bar_read(0, 0x3c, 4), 22); + h.stop(); +} + +fn fake_clock_ohci(start_ns: usize) -> (Arc, Arc) { + let h = Ohci::new(); + let now = Arc::new(AtomicUsize::new(start_ns)); + let t = now.clone(); + h.set_clock(Arc::new(move || t.load(Ordering::Relaxed) as u64)); + h.start(); + (h, now) +} + +/// OpenHCI 1.0a sections 6.2.1 and 7.3.3: entering UsbOperational loads +/// HcFmRemaining from the frame interval, increments HcFmNumber, and sends the +/// first SOF one frame later, whatever the phase of the free-running clock. +fn operational_entry_restarts_the_frame(from_hcfs: Option) { + let (h, now) = fake_clock_ohci(10_000_000); + let m = Memory::new(); + h.state().connect_dma(m.clone()); + h.state().config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + // Not operational: no frames count however much time passes. + if let Some(hcfs) = from_hcfs { + h.bar_write(0, 4, hcfs, 4); + } + now.store(15_000_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 0); + now.store(15_400_000, Ordering::Relaxed); + h.bar_write(0, 4, 0x80, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + assert_eq!(h.bar_read(0, 0x38, 4), 0x2edf); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 0, "no SOF at the load"); + assert_eq!(m.bytes.lock()[0x1080], 0xa5, "no HCCA write at the load"); + // Just short of one frame after the write: still nothing. + now.store(16_399_999, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x3c, 4), 1); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 0); + assert_eq!(m.bytes.lock()[0x1080], 0xa5); + // Exactly one frame after: the first SOF and its HCCA frame number. + now.store(16_400_000, Ordering::Relaxed); + assert_eq!(h.bar_read(0, 0x0c, 4) & 4, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 2); + assert_eq!(m.bytes.lock()[0x1080], 2); + h.stop(); +} + +#[test] +fn operational_entry_from_reset_restarts_the_frame() { + operational_entry_restarts_the_frame(None); +} + +#[test] +fn operational_entry_from_suspend_restarts_the_frame() { + // HCFS 11b. + operational_entry_restarts_the_frame(Some(0xc0)); +} + +#[test] +fn operational_entry_from_resume_restarts_the_frame() { + // HCFS 01b. + operational_entry_restarts_the_frame(Some(0x40)); +} + +/// The increment on entry carries into the frame-number overflow flag like a +/// frame's own, and HcFmRemaining takes the frame-interval toggle bit too. +#[test] +fn operational_entry_sets_overflow_and_loads_the_interval_toggle() { + for (before, after, overflow) in [(0x7fff, 0x8000, true), (0xffff, 0, true), (0x1234, 0x1235, false)] { + let (h, _now) = fake_clock_ohci(10_000_000); + h.state.lock().r[15] = before; + h.bar_write(0, 0x34, 0x8000_2edf, 4); + h.bar_write(0, 4, 0x80, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), after, "from {before:#x}"); + assert_eq!(h.bar_read(0, 0x0c, 4) & 32 != 0, overflow, "from {before:#x}"); + assert_eq!(h.bar_read(0, 0x38, 4), 0x8000_2edf); + h.stop(); + } +} + +/// An interrupt handler can acknowledge a line while the controller raises it. +#[test] +fn reentrant_irq_acknowledgement_leaves_final_line_clear() { + struct Ack { + hc: Weak, + level: AtomicBool, + once: AtomicBool, + } + impl crate::irq::InterruptController for Ack { + fn set_line(&self, _: u32, level: bool) { + if level && !self.once.swap(true, Ordering::SeqCst) { + self.hc.upgrade().unwrap().bar_write(0, 0xc, RHSC, 4); + } + self.level.store(level, Ordering::SeqCst); + } + } + let h = Ohci::new(); + let ack = Arc::new(Ack { + hc: Arc::downgrade(&h), + level: AtomicBool::new(false), + once: AtomicBool::new(false), + }); + h.state().connect_irq(IrqLine::new(ack.clone(), 0)); + h.attached(); + h.bar_write(0, 0x10, 0x80000040, 4); + h.attach(0, Function::new(UsbOutcome::Nak)).unwrap(); + assert!(!ack.level.load(Ordering::SeqCst)); +} + +/// The guest sees the documented write masks, HCR exemptions, and ownership bit. +#[test] +fn register_masks_preserve_hcr_exempt_fields_and_ownership_status() { + let h = Ohci::new(); + h.bar_write(0, 0x18, 0x123456ff, 4); + assert_eq!(h.bar_read(0, 0x18, 4), 0x12345600); + h.bar_write(0, 0x20, 0x123456ff, 4); + assert_eq!(h.bar_read(0, 0x20, 4), 0x123456f0); + h.bar_write(0, 0x10, u32::MAX, 4); + assert_eq!(h.bar_read(0, 0x10, 4), 0xc000007f); + h.bar_write(0, 8, 8, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 1 << 30); + h.bar_write(0, 0xc, 1 << 31, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 1 << 30); + h.bar_write(0, 0xc, 1 << 30, 4); + assert_eq!(h.bar_read(0, 0xc, 4), 0); + h.bar_write(0, 0x48, u32::MAX, 4); + assert_eq!(h.bar_read(0, 0x48, 4), 0xff001b02); + h.bar_write(0, 4, 0x700, 4); + h.bar_write(0, 8, 1, 4); + assert_eq!(h.bar_read(0, 4, 4), 0x3c0); + assert_eq!(h.bar_read(0, 0x48, 4), 0xff001b02); + assert_eq!(h.bar_read(0, 0x10, 4), 0); + h.bar_write(0, 0x3c, 0xffff, 4); + assert_eq!(h.bar_read(0, 0x3c, 4), 0); + h.bar_write(0, 0x18, 1, 1); + assert_eq!(h.bar_read(0, 0x18, 4), 0); +} + +/// The periodic phase controls how the guest control and bulk lists share service. +#[test] +fn periodic_phase_preserves_control_and_bulk_credit() { + let (h, m, f) = setup(Pid::In, UsbOutcome::Nak, 0x4000, 0x4007, 0); + m.words(0x2000, &[(8 << 16) | (1 << 7), 0x3020, 0x3000, 0]); + m.words(0x2100, &[(8 << 16) | (2 << 7), 0x3120, 0x3100, 0]); + m.words(0x3100, &[0xf0100000, 0x4100, 0x3120, 0x4107]); + m.words(0x2200, &[(8 << 16) | (3 << 7), 0x3220, 0x3200, 0]); + m.words(0x3200, &[0xf0100000, 0x4200, 0x3220, 0x4207]); + for i in 0..32 { + m.words(0x1000 + i * 4, &[0x2200]); + } + let mut s = h.state.lock().clone(); + s.active_fi = 5 * 256 - 1; + s.r[16] = 3 * 256; + s.r[1] = 0xb5; + s.r[10] = 0x2100; + s.r[2] |= 4; + h.schedule(&mut s).unwrap(); + let order: Vec<_> = f.packets.lock().iter().map(|p| p.endpoint).collect(); + assert_eq!(order, [1, 1, 3, 2, 1]); + assert_eq!(s.r[14], 0); +} diff --git a/src/dev/ps2.rs b/src/dev/ps2.rs index fa24f7b6e..068e70084 100644 --- a/src/dev/ps2.rs +++ b/src/dev/ps2.rs @@ -57,8 +57,25 @@ struct Ps2State { const MOUSE_MAX: i32 = 127; /// Combined PS/2 Keyboard and Mouse Controller +/// Another input device on the same host keyboard and mouse: a machine whose +/// guest may use USB instead of PS/2 (the Fuel) fits one beside the 8042 or +/// IOC3 controller, and host input reaches both. Each device drops what its +/// guest has not enabled, as the PS/2 side does. +pub trait HostInputSink: Send + Sync { + fn key(&self, key: KeyCode, pressed: bool); + /// Buttons in PS/2 order (left 1, right 2, middle 4); `dy` is positive + /// down the screen. + fn mouse(&self, buttons: u8, dx: i32, dy: i32, dz: i32); + /// Whether the guest has enabled this mouse. + fn mouse_enabled(&self) -> bool; + /// Mouse movement queued that the guest has not read yet. + fn mouse_pending(&self) -> usize; +} + pub struct Ps2Controller { state: Mutex, + /// Other devices fed from the same host input. + sinks: Mutex>>, /// The controller's interrupt: up while a byte waits to be read. irq: Option, running: AtomicBool, @@ -91,6 +108,7 @@ impl Ps2Controller { held_keys: [false; 256], }), irq, + sinks: Mutex::new(Vec::new()), running: AtomicBool::new(false), layout: AtomicU8::new(KeyboardLayout::Ansi as u8), } @@ -526,17 +544,27 @@ impl Ps2Controller { /// clicks are dropped. pub fn mouse_enabled(&self) -> bool { let state = self.state.lock(); - state.mouse_enabled && state.config & 0x20 == 0 + (state.mouse_enabled && state.config & 0x20 == 0) + || self.sinks.lock().iter().any(|s| s.mouse_enabled()) + } + + /// Feed host keyboard and mouse input to `sink` as well. + pub fn add_sink(&self, sink: Arc) { + self.sinks.lock().push(sink); } /// Mouse bytes queued that the guest has not read yet. pub fn mouse_pending(&self) -> usize { - self.state.lock().mouse_queue_bytes + let own = self.state.lock().mouse_queue_bytes; + self.sinks.lock().iter().fold(own, |n, s| n.max(s.mouse_pending())) } /// Push a scancode byte to the keyboard queue (called from UI) pub fn push_kb(&self, key: KeyCode, pressed: bool) { if !self.running.load(Ordering::Relaxed) { return; } + for sink in self.sinks.lock().iter() { + sink.key(key, pressed); + } let mut state = self.state.lock(); if !state.scanning_enabled || state.config & 0x10 != 0 { return; @@ -968,6 +996,9 @@ impl Ps2Controller { /// - `dz`: scroll wheel delta (positive = scroll up/away from user) pub fn push_mouse_input(&self, buttons: u8, dx: i32, dy: i32, dz: i32) { if !self.running.load(Ordering::Relaxed) { return; } + for sink in self.sinks.lock().iter() { + sink.mouse(buttons, dx, dy, dz); + } let mut state = self.state.lock(); if !state.mouse_enabled || state.config & 0x20 != 0 { return; } @@ -1348,6 +1379,41 @@ impl Saveable for Ps2Controller { mod tests { use super::*; + /// A sink sees host keys and mouse moves even while the guest has not + /// enabled the PS/2 devices, and its mouse counts toward `mouse_enabled` + /// and `mouse_pending`. + #[test] + fn a_sink_shares_the_host_input() { + #[derive(Default)] + struct Sink(Mutex>); + impl HostInputSink for Sink { + fn key(&self, key: KeyCode, pressed: bool) { + self.0.lock().push(format!("{key:?} {pressed}")); + } + fn mouse(&self, buttons: u8, dx: i32, dy: i32, dz: i32) { + self.0.lock().push(format!("mouse {buttons} {dx} {dy} {dz}")); + } + fn mouse_enabled(&self) -> bool { + true + } + fn mouse_pending(&self) -> usize { + 5 + } + } + let p = Ps2Controller::new(None); + let sink = Arc::new(Sink::default()); + p.add_sink(sink.clone()); + p.push_kb(KeyCode::KeyA, true); + assert!(sink.0.lock().is_empty(), "a stopped controller passes nothing on"); + p.start(); + assert!(p.mouse_enabled(), "the sink's mouse is on though the PS/2 mouse is not"); + assert_eq!(p.mouse_pending(), 5); + p.push_kb(KeyCode::KeyA, true); + p.push_mouse_input(2, 3, -4, 1); + assert_eq!(*sink.0.lock(), ["KeyA true", "mouse 2 3 -4 1"]); + assert_eq!(p.pop_from(false), None, "and the PS/2 keyboard, not enabled, took nothing"); + } + /// `type_char` sends what the keys themselves would: Shift around a /// shifted character, Enter for a newline, and nothing for a character /// with no key. diff --git a/src/dev/usb_hid.rs b/src/dev/usb_hid.rs new file mode 100644 index 000000000..8f417c05b --- /dev/null +++ b/src/dev/usb_hid.rs @@ -0,0 +1,609 @@ +//! USB HID boot-protocol keyboard and mouse: low-speed functions that plug +//! into an OHCI root port and are fed from the host's keyboard and mouse. +//! +//! Written from the public specifications: USB 1.1 (chapter 9 requests and +//! descriptors), Device Class Definition for HID 1.11 (class requests and +//! Appendix B, the boot interface descriptors and report formats) and the HID +//! Usage Tables (keyboard/keypad page). Each device has one configuration, +//! one boot interface and one interrupt-IN endpoint. +//! +//! The control pipe follows USB 1.1 section 8.5.2: a SETUP packet, an optional +//! data stage and a status stage. SET_ADDRESS takes effect once its status +//! stage completes. Interrupt reports queue one per host-side change so a key +//! tapped and released between two polls is still seen twice. Input that +//! arrives before the guest has selected a configuration is dropped, as an +//! unconfigured device would not report it. +//! +//! The data toggle on the wire is not tracked: the controller model keeps +//! each endpoint's toggle and a lost transfer cannot be replayed here. +//! Remote wakeup, the wheel, and the host keyboard layout are not modelled. + +use crate::dev::ohci::{Pid, UsbFunction, UsbOutcome, UsbPacket}; +use crate::dev::ps2::HostInputSink; +use parking_lot::Mutex; +use std::collections::VecDeque; +use std::sync::Arc; +use winit::keyboard::KeyCode; + +/// A vendor and product pair from the pid.codes open test allocation, since +/// the devices are not products of any vendor. +const VENDOR_ID: u16 = 0x1209; +const PRODUCT_ID: u16 = 0x0001; + +// Standard request codes (USB 1.1 table 9-4) and descriptor types (table 9-5). +const GET_STATUS: u8 = 0; +const CLEAR_FEATURE: u8 = 1; +const SET_FEATURE: u8 = 3; +const SET_ADDRESS: u8 = 5; +const GET_DESCRIPTOR: u8 = 6; +const GET_CONFIGURATION: u8 = 8; +const SET_CONFIGURATION: u8 = 9; +const GET_INTERFACE: u8 = 10; +const SET_INTERFACE: u8 = 11; +const DESC_DEVICE: u8 = 1; +const DESC_CONFIGURATION: u8 = 2; +const DESC_STRING: u8 = 3; +const DESC_HID: u8 = 0x21; +const DESC_REPORT: u8 = 0x22; +// HID class requests (HID 1.11 section 7.2). +const GET_REPORT: u8 = 1; +const GET_IDLE: u8 = 2; +const GET_PROTOCOL: u8 = 3; +const SET_REPORT: u8 = 9; +const SET_IDLE: u8 = 10; +const SET_PROTOCOL: u8 = 11; + +/// HID 1.11 appendix B.1: the boot keyboard's report descriptor. Input: a +/// modifier bit per key and six key usages; output: five LED bits. +pub const KEYBOARD_REPORT_DESCRIPTOR: [u8; 63] = [ + 0x05, 0x01, 0x09, 0x06, 0xa1, 0x01, 0x05, 0x07, 0x19, 0xe0, 0x29, 0xe7, 0x15, 0x00, 0x25, 0x01, + 0x75, 0x01, 0x95, 0x08, 0x81, 0x02, 0x95, 0x01, 0x75, 0x08, 0x81, 0x01, 0x95, 0x05, 0x75, 0x01, + 0x05, 0x08, 0x19, 0x01, 0x29, 0x05, 0x91, 0x02, 0x95, 0x01, 0x75, 0x03, 0x91, 0x01, 0x95, 0x06, + 0x75, 0x08, 0x15, 0x00, 0x25, 0x65, 0x05, 0x07, 0x19, 0x00, 0x29, 0x65, 0x81, 0x00, 0xc0, +]; +/// HID 1.11 appendix B.2: the boot mouse's report descriptor. Three buttons +/// and relative X and Y. +pub const MOUSE_REPORT_DESCRIPTOR: [u8; 50] = [ + 0x05, 0x01, 0x09, 0x02, 0xa1, 0x01, 0x09, 0x01, 0xa1, 0x00, 0x05, 0x09, 0x19, 0x01, 0x29, 0x03, + 0x15, 0x00, 0x25, 0x01, 0x95, 0x03, 0x75, 0x01, 0x81, 0x02, 0x95, 0x01, 0x75, 0x05, 0x81, 0x01, + 0x05, 0x01, 0x09, 0x30, 0x09, 0x31, 0x15, 0x81, 0x25, 0x7f, 0x75, 0x08, 0x95, 0x02, 0x81, 0x06, + 0xc0, 0xc0, +]; + +/// Reports held for the guest before the oldest is dropped. +const QUEUE_LIMIT: usize = 64; + +/// What differs between the two devices: the report they send and how host +/// input becomes one. +pub trait Boot: Send { + const PRODUCT: &'static str; + /// bInterfaceProtocol: 1 keyboard, 2 mouse. + const PROTOCOL: u8; + const REPORT_DESCRIPTOR: &'static [u8]; + const MAX_PACKET: u8; + /// The idle rate at reset, in 4 ms units (HID 1.11 section 7.2.4). + const DEFAULT_IDLE: u8; + fn reset(&mut self); + /// The report GET_REPORT returns now. + fn current(&self) -> Vec; + /// The next queued report, if any. + fn next_report(&mut self) -> Option>; + /// An output report; false when the device has none. + fn set_output(&mut self, _data: &[u8]) -> bool { + false + } + /// Host input waiting for the guest, in reports. + fn backlog(&self) -> usize; +} + +enum Stage { + Idle, + /// Data stage toward the host, then an OUT status. + In { data: Vec, sent: usize }, + /// Data stage from the host for a SET_REPORT of `length` bytes. + Out { length: usize, data: Vec }, + /// Status stage of a request with no data toward the host: the next IN + /// answers with an empty packet and the request takes effect. + StatusIn(Effect), +} + +#[derive(Clone, Copy)] +enum Effect { + None, + Address(u8), + Configuration(u8), + Idle(u8), + Protocol(u8), + Output, +} + +struct Inner { + dev: B, + address: u8, + configured: bool, + /// 0 boot, 1 report; a device starts in report protocol. + protocol: u8, + idle: u8, + remote_wakeup: bool, + stage: Stage, + last_report_ms: u64, +} + +pub struct UsbHid { + inner: Mutex>, +} + +impl UsbHid { + fn with(dev: B) -> Arc { + Arc::new(Self { + inner: Mutex::new(Inner { + dev, + address: 0, + configured: false, + protocol: 1, + idle: B::DEFAULT_IDLE, + remote_wakeup: false, + stage: Stage::Idle, + last_report_ms: 0, + }), + }) + } +} + +fn string_descriptor(s: &str) -> Vec { + let units: Vec = s.encode_utf16().flat_map(u16::to_le_bytes).collect(); + let mut d = vec![2 + units.len() as u8, DESC_STRING]; + d.extend(units); + d +} + +impl Inner { + fn device_descriptor() -> Vec { + let mut d = vec![18, DESC_DEVICE, 0x10, 0x01, 0, 0, 0, 8]; + d.extend(VENDOR_ID.to_le_bytes()); + d.extend(PRODUCT_ID.to_le_bytes()); + // bcdDevice, manufacturer and product strings, no serial, one configuration. + d.extend([0x00, 0x01, 1, 2, 0, 1]); + d + } + + fn hid_descriptor() -> Vec { + let n = B::REPORT_DESCRIPTOR.len() as u16; + let mut d = vec![9, DESC_HID, 0x11, 0x01, 0, 1, DESC_REPORT]; + d.extend(n.to_le_bytes()); + d + } + + /// The configuration descriptor followed by its interface, HID and + /// endpoint descriptors, as GET_DESCRIPTOR(configuration) returns them. + fn configuration_descriptor() -> Vec { + let interface = [9, 4, 0, 0, 1, 3, 1, B::PROTOCOL, 0]; + let hid = Self::hid_descriptor(); + // Interrupt IN endpoint 1, polled every 10 ms. + let endpoint = [7, 5, 0x81, 3, B::MAX_PACKET, 0, 10]; + let total = (9 + interface.len() + hid.len() + endpoint.len()) as u16; + let mut d = vec![9, DESC_CONFIGURATION]; + d.extend(total.to_le_bytes()); + // One interface, configuration 1, no string, bus powered with remote + // wakeup, 100 mA. + d.extend([1, 1, 0, 0xa0, 50]); + d.extend(interface); + d.extend(hid); + d.extend(endpoint); + d + } + + fn string(index: u8) -> Option> { + match index { + 0 => Some(vec![4, DESC_STRING, 0x09, 0x04]), + 1 => Some(string_descriptor("iris")), + 2 => Some(string_descriptor(B::PRODUCT)), + _ => None, + } + } + + fn reset(&mut self) { + self.address = 0; + self.configured = false; + self.protocol = 1; + self.idle = B::DEFAULT_IDLE; + self.remote_wakeup = false; + self.stage = Stage::Idle; + self.dev.reset(); + } + + /// Decode a SETUP packet (USB 1.1 table 9-2). None is a request this + /// device does not have, answered with a STALL. + fn setup(&mut self, p: &[u8]) -> Option<()> { + let [rt, req, vl, vh, il, _ih, ll, lh] = <[u8; 8]>::try_from(p).ok()?; + let value = u16::from_le_bytes([vl, vh]); + let length = u16::from_le_bytes([ll, lh]) as usize; + let reply = |data: Vec| Stage::In { sent: 0, data: data[..data.len().min(length)].to_vec() }; + let (kind, recipient) = ((rt >> 5) & 3, rt & 0x1f); + let to_host = rt & 0x80 != 0; + self.stage = match (kind, recipient, to_host, req) { + // Standard requests (USB 1.1 section 9.4). + (0, 0, true, GET_STATUS) => reply(vec![u8::from(self.remote_wakeup) << 1, 0]), + (0, 1 | 2, true, GET_STATUS) if value == 0 => reply(vec![0, 0]), + (0, 0, false, CLEAR_FEATURE) if value == 1 => { + self.remote_wakeup = false; + Stage::StatusIn(Effect::None) + } + (0, 0, false, SET_FEATURE) if value == 1 => { + self.remote_wakeup = true; + Stage::StatusIn(Effect::None) + } + // Clearing a halt on the interrupt endpoint is accepted; nothing halts it. + (0, 2, false, CLEAR_FEATURE) if value == 0 && il & 0x8f == 0x81 => Stage::StatusIn(Effect::None), + (0, 0, false, SET_ADDRESS) if value <= 127 => Stage::StatusIn(Effect::Address(value as u8)), + (0, 0, true, GET_DESCRIPTOR) => match (vh, vl) { + (DESC_DEVICE, 0) => reply(Self::device_descriptor()), + (DESC_CONFIGURATION, 0) => reply(Self::configuration_descriptor()), + (DESC_STRING, n) => reply(Self::string(n)?), + _ => return None, + }, + (0, 0, true, GET_CONFIGURATION) => reply(vec![u8::from(self.configured)]), + (0, 0, false, SET_CONFIGURATION) if value <= 1 => Stage::StatusIn(Effect::Configuration(value as u8)), + (0, 1, true, GET_INTERFACE) if il == 0 => reply(vec![0]), + (0, 1, false, SET_INTERFACE) if il == 0 && value == 0 => Stage::StatusIn(Effect::None), + // The HID descriptor and report descriptor belong to the interface. + (0, 1, true, GET_DESCRIPTOR) if il == 0 => match (vh, vl) { + (DESC_HID, 0) => reply(Self::hid_descriptor()), + (DESC_REPORT, 0) => reply(B::REPORT_DESCRIPTOR.to_vec()), + _ => return None, + }, + // HID class requests (HID 1.11 section 7.2), interface 0. + (1, 1, true, GET_REPORT) if il == 0 && vh == 1 && vl == 0 => reply(self.dev.current()), + (1, 1, false, SET_REPORT) if il == 0 && vh == 2 && vl == 0 && length > 0 => { + Stage::Out { length, data: Vec::new() } + } + (1, 1, true, GET_IDLE) if il == 0 => reply(vec![self.idle]), + (1, 1, false, SET_IDLE) if il == 0 => Stage::StatusIn(Effect::Idle(vh)), + (1, 1, true, GET_PROTOCOL) if il == 0 => reply(vec![self.protocol]), + (1, 1, false, SET_PROTOCOL) if il == 0 && value <= 1 => Stage::StatusIn(Effect::Protocol(value as u8)), + _ => return None, + }; + Some(()) + } + + fn apply(&mut self, effect: Effect, data: &[u8], now_ms: u64) -> bool { + match effect { + Effect::None => {} + Effect::Address(a) => self.address = a, + Effect::Configuration(c) => { + self.configured = c != 0; + self.dev.reset(); + self.last_report_ms = now_ms; + } + Effect::Idle(rate) => self.idle = rate, + Effect::Protocol(p) => self.protocol = p, + Effect::Output => return self.dev.set_output(data), + } + true + } + + fn control(&mut self, p: UsbPacket) -> UsbOutcome { + match p.pid { + Pid::Setup => match self.setup(&p.data) { + Some(()) => UsbOutcome::Ack(Vec::new()), + None => { + self.stage = Stage::Idle; + UsbOutcome::Stall + } + }, + Pid::In => match &mut self.stage { + Stage::In { data, sent } => { + let n = p.capacity.min(data.len() - *sent); + let chunk = data[*sent..*sent + n].to_vec(); + *sent += n; + UsbOutcome::Ack(chunk) + } + Stage::StatusIn(effect) => { + let effect = *effect; + self.stage = Stage::Idle; + if self.apply(effect, &[], p.time_ms) { UsbOutcome::Ack(Vec::new()) } else { UsbOutcome::Stall } + } + _ => UsbOutcome::Stall, + }, + Pid::Out => match &mut self.stage { + // The status stage of a request that sent data to the host. + Stage::In { .. } if p.data.is_empty() => { + self.stage = Stage::Idle; + UsbOutcome::Ack(Vec::new()) + } + Stage::Out { length, data } => { + data.extend(&p.data); + if data.len() >= *length { + let data = std::mem::take(data); + let ok = self.apply(Effect::Output, &data, p.time_ms); + self.stage = if ok { Stage::StatusIn(Effect::None) } else { Stage::Idle }; + if !ok { + return UsbOutcome::Stall; + } + } + UsbOutcome::Ack(Vec::new()) + } + _ => UsbOutcome::Stall, + }, + } + } + + /// Interrupt IN endpoint 1 (USB 1.1 section 5.7): a report when there is + /// one, otherwise NAK. A nonzero idle rate repeats the last state once the + /// period passes without a change. + fn interrupt(&mut self, p: UsbPacket) -> UsbOutcome { + if !self.configured { + return UsbOutcome::Stall; + } + if let Some(r) = self.dev.next_report() { + self.last_report_ms = p.time_ms; + return UsbOutcome::Ack(r); + } + if self.idle != 0 && p.time_ms.saturating_sub(self.last_report_ms) >= u64::from(self.idle) * 4 { + self.last_report_ms = p.time_ms; + return UsbOutcome::Ack(self.dev.current()); + } + UsbOutcome::Nak + } +} + +impl UsbFunction for UsbHid { + fn reset(&self) { + self.inner.lock().reset(); + } + fn address(&self) -> u8 { + self.inner.lock().address + } + fn low_speed(&self) -> bool { + true + } + fn transact(&self, packet: UsbPacket) -> UsbOutcome { + let mut s = self.inner.lock(); + match packet.endpoint { + 0 => s.control(packet), + 1 if packet.pid == Pid::In => s.interrupt(packet), + _ => UsbOutcome::Stall, + } + } +} + +// --------------------------------------------------------------------------- +// Keyboard +// --------------------------------------------------------------------------- + +/// The HID usage (keyboard/keypad page 0x07) of a host key, or None for a +/// key the page has no entry for. +pub fn usage(key: KeyCode) -> Option { + use KeyCode::*; + Some(match key { + KeyA => 0x04, KeyB => 0x05, KeyC => 0x06, KeyD => 0x07, KeyE => 0x08, KeyF => 0x09, + KeyG => 0x0a, KeyH => 0x0b, KeyI => 0x0c, KeyJ => 0x0d, KeyK => 0x0e, KeyL => 0x0f, + KeyM => 0x10, KeyN => 0x11, KeyO => 0x12, KeyP => 0x13, KeyQ => 0x14, KeyR => 0x15, + KeyS => 0x16, KeyT => 0x17, KeyU => 0x18, KeyV => 0x19, KeyW => 0x1a, KeyX => 0x1b, + KeyY => 0x1c, KeyZ => 0x1d, + Digit1 => 0x1e, Digit2 => 0x1f, Digit3 => 0x20, Digit4 => 0x21, Digit5 => 0x22, + Digit6 => 0x23, Digit7 => 0x24, Digit8 => 0x25, Digit9 => 0x26, Digit0 => 0x27, + Enter => 0x28, Escape => 0x29, Backspace => 0x2a, Tab => 0x2b, Space => 0x2c, + Minus => 0x2d, Equal => 0x2e, BracketLeft => 0x2f, BracketRight => 0x30, + Backslash => 0x31, Semicolon => 0x33, Quote => 0x34, Backquote => 0x35, + Comma => 0x36, Period => 0x37, Slash => 0x38, CapsLock => 0x39, + F1 => 0x3a, F2 => 0x3b, F3 => 0x3c, F4 => 0x3d, F5 => 0x3e, F6 => 0x3f, + F7 => 0x40, F8 => 0x41, F9 => 0x42, F10 => 0x43, F11 => 0x44, F12 => 0x45, + PrintScreen => 0x46, ScrollLock => 0x47, Pause => 0x48, Insert => 0x49, + Home => 0x4a, PageUp => 0x4b, Delete => 0x4c, End => 0x4d, PageDown => 0x4e, + ArrowRight => 0x4f, ArrowLeft => 0x50, ArrowDown => 0x51, ArrowUp => 0x52, + NumLock => 0x53, NumpadDivide => 0x54, NumpadMultiply => 0x55, + NumpadSubtract => 0x56, NumpadAdd => 0x57, NumpadEnter => 0x58, + Numpad1 => 0x59, Numpad2 => 0x5a, Numpad3 => 0x5b, Numpad4 => 0x5c, + Numpad5 => 0x5d, Numpad6 => 0x5e, Numpad7 => 0x5f, Numpad8 => 0x60, + Numpad9 => 0x61, Numpad0 => 0x62, NumpadDecimal => 0x63, + IntlBackslash => 0x64, ContextMenu => 0x65, + _ => return None, + }) +} + +/// The bit of a modifier key in the report's first byte (usages 0xe0 to 0xe7). +fn modifier(key: KeyCode) -> Option { + use KeyCode::*; + Some(match key { + ControlLeft => 0x01, ShiftLeft => 0x02, AltLeft => 0x04, SuperLeft => 0x08, + ControlRight => 0x10, ShiftRight => 0x20, AltRight => 0x40, SuperRight => 0x80, + _ => return None, + }) +} + +#[derive(Default)] +pub struct Keyboard { + modifiers: u8, + /// Pressed keys in the order they went down. + keys: Vec, + queue: VecDeque>, + leds: u8, +} + +impl Keyboard { + /// The 8-byte boot report (HID 1.11 B.1): modifiers, reserved, six keys. + /// More than six keys at once is the phantom state, ErrorRollOver in every slot. + fn report(&self) -> Vec { + let mut r = vec![self.modifiers, 0, 0, 0, 0, 0, 0, 0]; + if self.keys.len() > 6 { + r[2..].fill(0x01); + } else { + r[2..2 + self.keys.len()].copy_from_slice(&self.keys); + } + r + } + + fn push(&mut self) { + if self.queue.len() >= QUEUE_LIMIT { + self.queue.pop_front(); + } + let report = self.report(); + self.queue.push_back(report); + } +} + +impl Boot for Keyboard { + const PRODUCT: &'static str = "USB Keyboard"; + const PROTOCOL: u8 = 1; + const REPORT_DESCRIPTOR: &'static [u8] = &KEYBOARD_REPORT_DESCRIPTOR; + const MAX_PACKET: u8 = 8; + // 500 ms, the recommended keyboard default. + const DEFAULT_IDLE: u8 = 125; + fn reset(&mut self) { + *self = Keyboard::default(); + } + fn current(&self) -> Vec { + self.report() + } + fn next_report(&mut self) -> Option> { + self.queue.pop_front() + } + /// The LED output report (HID 1.11 B.1): num lock, caps lock, scroll + /// lock, compose, kana in bits 0 to 4. + fn set_output(&mut self, data: &[u8]) -> bool { + self.leds = data.first().copied().unwrap_or(0) & 0x1f; + true + } + fn backlog(&self) -> usize { + self.queue.len() + } +} + +pub type UsbKeyboard = UsbHid; + +impl UsbKeyboard { + pub fn new() -> Arc { + Self::with(Keyboard::default()) + } + + /// The LEDs the guest last asked for. + pub fn leds(&self) -> u8 { + self.inner.lock().dev.leds + } +} + +impl HostInputSink for UsbKeyboard { + fn key(&self, key: KeyCode, pressed: bool) { + let mut s = self.inner.lock(); + if !s.configured { + return; + } + let k = &mut s.dev; + if let Some(bit) = modifier(key) { + let before = k.modifiers; + if pressed { k.modifiers |= bit } else { k.modifiers &= !bit } + if k.modifiers != before { + k.push(); + } + } else if let Some(u) = usage(key) { + let at = k.keys.iter().position(|&x| x == u); + match (pressed, at) { + (true, None) => k.keys.push(u), + (false, Some(i)) => { + k.keys.remove(i); + } + _ => return, + } + k.push(); + } + } + fn mouse(&self, _: u8, _: i32, _: i32, _: i32) {} + fn mouse_enabled(&self) -> bool { + false + } + fn mouse_pending(&self) -> usize { + 0 + } +} + +// --------------------------------------------------------------------------- +// Mouse +// --------------------------------------------------------------------------- + +#[derive(Default)] +pub struct Mouse { + buttons: u8, + /// Movement not yet in a report. + dx: i32, + dy: i32, + queue: VecDeque>, +} + +impl Mouse { + /// Move accumulated motion into reports, at most a signed byte on each + /// axis apiece (HID 1.11 B.2: buttons, X, Y). `force` sends one even + /// with no motion, for a button change. + fn flush(&mut self, mut force: bool) { + while (force || self.dx != 0 || self.dy != 0) && self.queue.len() < QUEUE_LIMIT { + let (x, y) = (self.dx.clamp(-127, 127), self.dy.clamp(-127, 127)); + self.dx -= x; + self.dy -= y; + self.queue.push_back(vec![self.buttons, x as i8 as u8, y as i8 as u8]); + force = false; + } + } +} + +impl Boot for Mouse { + const PRODUCT: &'static str = "USB Mouse"; + const PROTOCOL: u8 = 2; + const REPORT_DESCRIPTOR: &'static [u8] = &MOUSE_REPORT_DESCRIPTOR; + const MAX_PACKET: u8 = 4; + // A mouse reports only on change. + const DEFAULT_IDLE: u8 = 0; + fn reset(&mut self) { + *self = Mouse::default(); + } + fn current(&self) -> Vec { + vec![self.buttons, 0, 0] + } + fn next_report(&mut self) -> Option> { + if self.queue.is_empty() { + self.flush(false); + } + self.queue.pop_front() + } + fn backlog(&self) -> usize { + self.queue.len() + usize::from(self.dx != 0 || self.dy != 0) + } +} + +pub type UsbMouse = UsbHid; + +impl UsbMouse { + pub fn new() -> Arc { + Self::with(Mouse::default()) + } +} + +impl HostInputSink for UsbMouse { + fn key(&self, _: KeyCode, _: bool) {} + fn mouse(&self, buttons: u8, dx: i32, dy: i32, _dz: i32) { + let mut s = self.inner.lock(); + if !s.configured { + return; + } + let m = &mut s.dev; + m.dx = m.dx.saturating_add(dx); + m.dy = m.dy.saturating_add(dy); + let buttons = buttons & 7; + // Motion up to now travels with the old button state. + if buttons != m.buttons { + m.flush(false); + m.buttons = buttons; + m.flush(true); + } else { + m.flush(false); + } + } + fn mouse_enabled(&self) -> bool { + self.inner.lock().configured + } + fn mouse_pending(&self) -> usize { + self.inner.lock().dev.backlog() + } +} + +#[cfg(test)] +mod tests; diff --git a/src/dev/usb_hid/tests.rs b/src/dev/usb_hid/tests.rs new file mode 100644 index 000000000..393985056 --- /dev/null +++ b/src/dev/usb_hid/tests.rs @@ -0,0 +1,448 @@ +// The cases follow USB 1.1 chapter 9 and HID 1.11: descriptor layout and +// lengths, the control pipe's stages, and the boot report formats. The last +// ones put a device behind the OHCI model to show an interrupt transfer +// returns a report through guest memory. +use super::*; +use crate::dev::ohci::Ohci; +use crate::pci::{PciDma, PciDmaStatus, PciFunction}; +use crate::traits::Device; + +fn packet(endpoint: u8, pid: Pid, capacity: usize, data: &[u8], time_ms: u64) -> UsbPacket { + UsbPacket { endpoint, pid, toggle: false, capacity, data: data.to_vec(), time_ms } +} + +fn ack(o: UsbOutcome) -> Vec { + match o { + UsbOutcome::Ack(b) => b, + other => panic!("expected an ACK, got {other:?}"), + } +} + +/// A control transfer toward the host: SETUP, IN packets of 8 until short, +/// then the OUT status. +fn control_in(f: &dyn UsbFunction, rt: u8, req: u8, value: u16, index: u16, length: u16) -> Result, UsbOutcome> { + let mut setup = vec![rt, req]; + setup.extend(value.to_le_bytes()); + setup.extend(index.to_le_bytes()); + setup.extend(length.to_le_bytes()); + match f.transact(packet(0, Pid::Setup, 8, &setup, 0)) { + UsbOutcome::Ack(_) => {} + other => return Err(other), + } + let mut data = Vec::new(); + loop { + let chunk = ack(f.transact(packet(0, Pid::In, 8, &[], 0))); + let short = chunk.len() < 8; + data.extend(chunk); + if short || data.len() >= length as usize { + break; + } + } + ack(f.transact(packet(0, Pid::Out, 8, &[], 0))); + Ok(data) +} + +/// A control transfer with no data stage: SETUP then the IN status. +fn control_nodata(f: &dyn UsbFunction, rt: u8, req: u8, value: u16, index: u16) -> UsbOutcome { + let mut setup = vec![rt, req]; + setup.extend(value.to_le_bytes()); + setup.extend(index.to_le_bytes()); + setup.extend(0u16.to_le_bytes()); + match f.transact(packet(0, Pid::Setup, 8, &setup, 0)) { + UsbOutcome::Ack(_) => f.transact(packet(0, Pid::In, 8, &[], 0)), + other => other, + } +} + +fn enumerate(f: &dyn UsbFunction, address: u8) { + f.reset(); + assert!(ack(control_nodata(f, 0x00, SET_ADDRESS, address as u16, 0)).is_empty()); + assert_eq!(f.address(), address); + assert!(ack(control_nodata(f, 0x00, SET_CONFIGURATION, 1, 0)).is_empty()); +} + +fn interrupt(f: &dyn UsbFunction, time_ms: u64) -> UsbOutcome { + f.transact(packet(1, Pid::In, 8, &[], time_ms)) +} + +#[test] +fn the_device_descriptor_names_a_low_speed_hid_function() { + for f in [&*UsbKeyboard::new() as &dyn UsbFunction, &*UsbMouse::new()] { + assert!(f.low_speed()); + let d = control_in(f, 0x80, GET_DESCRIPTOR, 0x0100, 0, 64).unwrap(); + assert_eq!(d.len(), 18); + assert_eq!(d[0], 18, "bLength"); + assert_eq!(d[1], 1, "bDescriptorType"); + assert_eq!(u16::from_le_bytes([d[2], d[3]]), 0x0110, "USB 1.1"); + assert_eq!(d[4], 0, "the class is in the interface"); + assert_eq!(d[7], 8, "a low-speed control pipe carries 8 bytes"); + assert_eq!(d[17], 1, "one configuration"); + // The guest may read just the first 8 bytes before addressing. + assert_eq!(control_in(f, 0x80, GET_DESCRIPTOR, 0x0100, 0, 8).unwrap(), d[..8]); + } +} + +/// A configuration, its interface, the HID descriptor and the endpoint come +/// back in one block whose wTotalLength covers them (USB 1.1 9.6.2). +#[test] +fn the_configuration_block_describes_one_boot_interface_and_one_interrupt_endpoint() { + for (f, protocol, report_len, packet_size) in [ + (&*UsbKeyboard::new() as &dyn UsbFunction, 1u8, KEYBOARD_REPORT_DESCRIPTOR.len(), 8u8), + (&*UsbMouse::new(), 2, MOUSE_REPORT_DESCRIPTOR.len(), 4), + ] { + let c = control_in(f, 0x80, GET_DESCRIPTOR, 0x0200, 0, 255).unwrap(); + assert_eq!(c.len(), 34); + assert_eq!(&c[..2], [9, 2]); + assert_eq!(u16::from_le_bytes([c[2], c[3]]) as usize, c.len(), "wTotalLength"); + assert_eq!((c[4], c[5]), (1, 1), "one interface; configuration value 1"); + assert_eq!(&c[9..14], [9, 4, 0, 0, 1], "interface 0, no alternate, one endpoint"); + assert_eq!(&c[14..17], [3, 1, protocol], "HID class, boot subclass, keyboard or mouse"); + assert_eq!(&c[18..20], [9, 0x21], "HID descriptor"); + assert_eq!(u16::from_le_bytes([c[20], c[21]]), 0x0111, "bcdHID 1.11"); + assert_eq!(&c[24..26], [0x22, report_len as u8], "one report descriptor of its length"); + assert_eq!(&c[27..], [7, 5, 0x81, 3, packet_size, 0, 10], "interrupt IN 1, polled at 10 ms"); + // The HID descriptor is also readable on its own through the interface. + assert_eq!(control_in(f, 0x81, GET_DESCRIPTOR, 0x2100, 0, 9).unwrap(), c[18..27]); + } +} + +#[test] +fn the_report_descriptors_are_the_boot_descriptors() { + let k = UsbKeyboard::new(); + let r = control_in(&*k, 0x81, GET_DESCRIPTOR, 0x2200, 0, 255).unwrap(); + assert_eq!(r, KEYBOARD_REPORT_DESCRIPTOR); + // Usage page generic desktop, usage keyboard, application collection. + assert_eq!(&r[..6], [0x05, 0x01, 0x09, 0x06, 0xa1, 0x01]); + assert_eq!(r.last(), Some(&0xc0), "collection ends"); + let m = UsbMouse::new(); + let r = control_in(&*m, 0x81, GET_DESCRIPTOR, 0x2200, 0, 255).unwrap(); + assert_eq!(r, MOUSE_REPORT_DESCRIPTOR); + assert_eq!(&r[..6], [0x05, 0x01, 0x09, 0x02, 0xa1, 0x01]); +} + +#[test] +fn strings_are_utf16_with_a_language_table() { + let k = UsbKeyboard::new(); + assert_eq!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0300, 0, 255).unwrap(), [4, 3, 0x09, 0x04]); + let s = control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0302, 0x0409, 255).unwrap(); + assert_eq!(s[0] as usize, s.len()); + let text: Vec = s[2..].chunks(2).map(|c| u16::from_le_bytes([c[0], c[1]])).collect(); + assert_eq!(String::from_utf16(&text).unwrap(), "USB Keyboard"); + assert!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0309, 0x0409, 255).is_err(), "no such string"); +} + +/// SET_ADDRESS takes effect after its status stage, not before (USB 1.1 9.4.6). +#[test] +fn set_address_applies_after_the_status_stage() { + let m = UsbMouse::new(); + let mut setup = vec![0x00, SET_ADDRESS, 9, 0, 0, 0, 0, 0]; + assert!(ack(m.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert_eq!(m.address(), 0, "still the default address during the status stage"); + assert!(ack(m.transact(packet(0, Pid::In, 8, &[], 0))).is_empty()); + assert_eq!(m.address(), 9); + // An address above 127 is not one. + setup[2] = 200; + assert_eq!(m.transact(packet(0, Pid::Setup, 8, &setup, 0)), UsbOutcome::Stall); + // A bus reset returns the device to the default address. + m.reset(); + assert_eq!(m.address(), 0); +} + +#[test] +fn configuration_state_is_readable_and_unknown_requests_stall() { + let k = UsbKeyboard::new(); + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [0]); + assert!(ack(control_nodata(&*k, 0x00, SET_CONFIGURATION, 1, 0)).is_empty()); + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [1]); + assert_eq!(control_nodata(&*k, 0x00, SET_CONFIGURATION, 2, 0), UsbOutcome::Stall); + assert_eq!(control_in(&*k, 0x81, GET_INTERFACE, 0, 0, 1).unwrap(), [0]); + assert_eq!(control_in(&*k, 0x80, GET_STATUS, 0, 0, 2).unwrap(), [0, 0]); + // No device qualifier on a low-speed-only device; no such request. + assert!(control_in(&*k, 0x80, GET_DESCRIPTOR, 0x0600, 0, 10).is_err()); + assert_eq!(control_nodata(&*k, 0x00, 0x77, 0, 0), UsbOutcome::Stall); + // A stalled request leaves the pipe usable for the next SETUP. + assert_eq!(control_in(&*k, 0x80, GET_CONFIGURATION, 0, 0, 1).unwrap(), [1]); + // The interrupt endpoint is not there before a configuration is selected. + let fresh = UsbKeyboard::new(); + assert_eq!(interrupt(&*fresh, 0), UsbOutcome::Stall); +} + +#[test] +fn hid_class_requests_set_and_report_idle_and_protocol() { + let k = UsbKeyboard::new(); + // A device starts in report protocol with the keyboard's 500 ms idle. + assert_eq!(control_in(&*k, 0xa1, GET_PROTOCOL, 0, 0, 1).unwrap(), [1]); + assert_eq!(control_in(&*k, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [125]); + assert!(ack(control_nodata(&*k, 0x21, SET_PROTOCOL, 0, 0)).is_empty()); + assert_eq!(control_in(&*k, 0xa1, GET_PROTOCOL, 0, 0, 1).unwrap(), [0]); + // SET_IDLE carries the duration in the high byte of wValue. + assert!(ack(control_nodata(&*k, 0x21, SET_IDLE, 0, 0)).is_empty()); + assert_eq!(control_in(&*k, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [0]); + assert_eq!(control_nodata(&*k, 0x21, SET_PROTOCOL, 2, 0), UsbOutcome::Stall); + // A mouse starts with no idle repeat. + let m = UsbMouse::new(); + assert_eq!(control_in(&*m, 0xa1, GET_IDLE, 0, 0, 1).unwrap(), [0]); +} + +#[test] +fn set_report_takes_the_keyboards_leds_and_a_mouse_has_none() { + let k = UsbKeyboard::new(); + let setup = [0x21, SET_REPORT, 0, 2, 0, 0, 1, 0]; + assert!(ack(k.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert!(ack(k.transact(packet(0, Pid::Out, 8, &[0x1f | 0x80], 0))).is_empty()); + assert!(ack(k.transact(packet(0, Pid::In, 8, &[], 0))).is_empty(), "status stage"); + assert_eq!(k.leds(), 0x1f, "five LED bits"); + let m = UsbMouse::new(); + assert!(ack(m.transact(packet(0, Pid::Setup, 8, &setup, 0))).is_empty()); + assert_eq!(m.transact(packet(0, Pid::Out, 8, &[1], 0)), UsbOutcome::Stall); +} + +#[test] +fn keyboard_reports_carry_modifiers_and_keys_in_press_order() { + let k = UsbKeyboard::new(); + enumerate(&*k, 3); + assert_eq!(interrupt(&*k, 1), UsbOutcome::Nak, "nothing to report yet"); + k.key(KeyCode::ShiftLeft, true); + k.key(KeyCode::KeyA, true); + k.key(KeyCode::KeyB, true); + k.key(KeyCode::KeyA, false); + k.key(KeyCode::ShiftLeft, false); + k.key(KeyCode::KeyB, false); + let reports: Vec> = (0..6).map(|i| ack(interrupt(&*k, i))).collect(); + assert_eq!(reports, [ + vec![0x02, 0, 0, 0, 0, 0, 0, 0], + vec![0x02, 0, 0x04, 0, 0, 0, 0, 0], + vec![0x02, 0, 0x04, 0x05, 0, 0, 0, 0], + vec![0x02, 0, 0x05, 0, 0, 0, 0, 0], + vec![0x00, 0, 0x05, 0, 0, 0, 0, 0], + vec![0x00, 0, 0, 0, 0, 0, 0, 0], + ], "a tap between polls still reports both edges"); + assert_eq!(interrupt(&*k, 7), UsbOutcome::Nak); +} + +#[test] +fn keyboard_usages_follow_the_usage_table() { + assert_eq!(usage(KeyCode::KeyA), Some(0x04)); + assert_eq!(usage(KeyCode::KeyZ), Some(0x1d)); + assert_eq!(usage(KeyCode::Digit1), Some(0x1e)); + assert_eq!(usage(KeyCode::Digit0), Some(0x27)); + assert_eq!(usage(KeyCode::Enter), Some(0x28)); + assert_eq!(usage(KeyCode::Escape), Some(0x29)); + assert_eq!(usage(KeyCode::Space), Some(0x2c)); + assert_eq!(usage(KeyCode::F1), Some(0x3a)); + assert_eq!(usage(KeyCode::F12), Some(0x45)); + assert_eq!(usage(KeyCode::ArrowRight), Some(0x4f)); + assert_eq!(usage(KeyCode::ArrowUp), Some(0x52)); + assert_eq!(usage(KeyCode::Numpad0), Some(0x62)); + assert_eq!(usage(KeyCode::ControlLeft), None, "modifiers have a report bit, not a usage slot"); + assert_eq!(modifier(KeyCode::ControlLeft), Some(0x01)); + assert_eq!(modifier(KeyCode::SuperRight), Some(0x80)); +} + +#[test] +fn a_seventh_key_reports_rollover_and_release_recovers() { + let k = UsbKeyboard::new(); + enumerate(&*k, 1); + let keys = [KeyCode::KeyA, KeyCode::KeyB, KeyCode::KeyC, KeyCode::KeyD, KeyCode::KeyE, KeyCode::KeyF, KeyCode::KeyG]; + for key in keys { + k.key(key, true); + } + let reports: Vec> = (0..7).map(|i| ack(interrupt(&*k, i))).collect(); + assert_eq!(reports[5], [0, 0, 4, 5, 6, 7, 8, 9]); + assert_eq!(reports[6], [0, 0, 1, 1, 1, 1, 1, 1], "ErrorRollOver"); + k.key(KeyCode::KeyG, false); + assert_eq!(ack(interrupt(&*k, 8)), [0, 0, 4, 5, 6, 7, 8, 9]); + // A repeated press of a held key is not a new edge. + k.key(KeyCode::KeyA, true); + assert_eq!(interrupt(&*k, 9), UsbOutcome::Nak); +} + +#[test] +fn input_before_configuration_is_dropped_and_get_report_reads_the_state() { + let k = UsbKeyboard::new(); + k.key(KeyCode::KeyA, true); + enumerate(&*k, 2); + assert_eq!(interrupt(&*k, 1), UsbOutcome::Nak, "the early key never reached the report queue"); + k.key(KeyCode::KeyQ, true); + assert_eq!(control_in(&*k, 0xa1, GET_REPORT, 0x0100, 0, 8).unwrap(), [0, 0, 0x14, 0, 0, 0, 0, 0]); +} + +/// With a nonzero idle rate the keyboard repeats its state when the period +/// passes without a change (HID 1.11 7.2.4), and with zero it stays quiet. +#[test] +fn the_idle_rate_repeats_the_last_report() { + let k = UsbKeyboard::new(); + enumerate(&*k, 2); + k.key(KeyCode::KeyA, true); + assert_eq!(ack(interrupt(&*k, 10)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert_eq!(interrupt(&*k, 509), UsbOutcome::Nak, "500 ms not yet passed"); + assert_eq!(ack(interrupt(&*k, 510)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert!(ack(control_nodata(&*k, 0x21, SET_IDLE, 0, 0)).is_empty()); + assert_eq!(interrupt(&*k, 5000), UsbOutcome::Nak); +} + +#[test] +fn mouse_reports_carry_buttons_and_signed_motion() { + let m = UsbMouse::new(); + enumerate(&*m, 4); + assert!(m.mouse_enabled()); + assert_eq!(interrupt(&*m, 1), UsbOutcome::Nak); + // Host dy is positive down the screen, as HID Y is. + m.mouse(0, 10, -20, 0); + assert_eq!(ack(interrupt(&*m, 2)), [0, 10, (-20i8) as u8]); + // A press travels with its own report, then the release. + m.mouse(1, 0, 0, 0); + m.mouse(0, 0, 0, 0); + assert_eq!(ack(interrupt(&*m, 3)), [1, 0, 0]); + assert_eq!(ack(interrupt(&*m, 4)), [0, 0, 0]); + // Buttons in PS/2 order: right is 2, middle is 4. + m.mouse(2 | 4, 0, 0, 0); + assert_eq!(ack(interrupt(&*m, 5)), [6, 0, 0]); + assert_eq!(control_in(&*m, 0xa1, GET_REPORT, 0x0100, 0, 3).unwrap(), [6, 0, 0]); + // Boot protocol has no wheel byte: the report stays 3 bytes. + m.mouse(6, 0, 0, 3); + assert_eq!(interrupt(&*m, 6), UsbOutcome::Nak); +} + +#[test] +fn large_motion_splits_across_reports_without_loss() { + let m = UsbMouse::new(); + enumerate(&*m, 4); + m.mouse(0, 300, -200, 0); + assert_eq!(m.mouse_pending(), 3); + let mut sum = (0i32, 0i32); + let mut n = 0; + while let UsbOutcome::Ack(r) = interrupt(&*m, n) { + sum.0 += r[1] as i8 as i32; + sum.1 += r[2] as i8 as i32; + assert!(r[1] as i8 != -128 && r[2] as i8 != -128); + n += 1; + } + assert_eq!(sum, (300, -200)); + assert_eq!(m.mouse_pending(), 0); +} + +#[test] +fn a_mouse_before_configuration_is_not_enabled_and_drops_motion() { + let m = UsbMouse::new(); + assert!(!m.mouse_enabled()); + m.mouse(1, 5, 5, 0); + assert_eq!(m.mouse_pending(), 0); + enumerate(&*m, 1); + assert_eq!(interrupt(&*m, 1), UsbOutcome::Nak); +} + +/// Host input arrives through the PS/2 controller's sinks, so the one +/// interface the GUI, VNC and CI use feeds both kinds of device. +#[test] +fn host_input_reaches_usb_devices_through_the_ps2_controller() { + use crate::dev::ps2::Ps2Controller; + let ps2 = Ps2Controller::new(None); + Device::start(&ps2); + let (k, m) = (UsbKeyboard::new(), UsbMouse::new()); + ps2.add_sink(k.clone()); + ps2.add_sink(m.clone()); + enumerate(&*k, 1); + enumerate(&*m, 2); + assert!(ps2.mouse_enabled(), "the USB mouse counts: the PS/2 mouse is still off"); + ps2.push_kb(KeyCode::KeyA, true); + ps2.type_char('b'); + ps2.push_mouse_input(1, 3, 4, 0); + assert_eq!(ps2.mouse_pending(), 2, "a move and a button change wait for the guest"); + assert_eq!(ack(interrupt(&*k, 1)), [0, 0, 4, 0, 0, 0, 0, 0]); + assert_eq!(ack(interrupt(&*k, 2)), [0, 0, 4, 5, 0, 0, 0, 0]); + assert_eq!(ack(interrupt(&*m, 1)), [0, 3, 4]); + assert_eq!(ack(interrupt(&*m, 2)), [1, 0, 0]); +} + +// ---- Through the OHCI model ------------------------------------------------ + +struct Memory(Mutex>); +impl Memory { + fn new() -> Arc { + Arc::new(Self(Mutex::new(vec![0xa5; 0x10000]))) + } + fn words(&self, at: usize, words: &[u32]) { + for (i, w) in words.iter().enumerate() { + self.0.lock()[at + i * 4..at + i * 4 + 4].copy_from_slice(&w.to_le_bytes()); + } + } + fn word(&self, at: usize) -> u32 { + u32::from_le_bytes(self.0.lock()[at..at + 4].try_into().unwrap()) + } +} +impl PciDma for Memory { + fn dma_read(&self, addr: u64, dst: &mut [u8]) -> PciDmaStatus { + match self.0.lock().get(addr as usize..addr as usize + dst.len()) { + Some(s) => { + dst.copy_from_slice(s); + PciDmaStatus::Ok + } + None => PciDmaStatus::MasterAbort, + } + } + fn dma_write(&self, addr: u64, src: &[u8]) -> PciDmaStatus { + match self.0.lock().get_mut(addr as usize..addr as usize + src.len()) { + Some(s) => { + s.copy_from_slice(src); + PciDmaStatus::Ok + } + None => PciDmaStatus::MasterAbort, + } + } +} + +/// An OHCI controller with the device on port 0, reset and enabled, with +/// the periodic list holding one interrupt-IN endpoint at `address`. +fn ohci_with(f: Arc, address: u8) -> (Arc, Arc) { + let h = Ohci::new(); + let m = Memory::new(); + h.state().connect_dma(m.clone()); + h.attach(0, f.clone()).unwrap(); + h.start(); + h.bar_write(0, 0x54, 0x100, 4); + h.bar_write(0, 0x54, 0x10, 4); + h.advance_frames(10); + // The port reset returned the device to its default state: enumerate it. + enumerate(&*f, address); + m.words(0x1000, &[0; 64]); + // Interrupt table: every slot points at the endpoint. ED: address, endpoint 1, + // direction from the TD, low speed, max packet 8; TD queue 0x3000 .. tail 0x3020. + m.words(0x1000, &[0x2000; 32]); + m.words(0x2000, &[u32::from(address) | (1 << 7) | (1 << 13) | (8 << 16), 0x3020, 0x3000, 0]); + // TD: direction IN (2), buffer 0x4000..0x4007, buffer rounding allowed, interrupt delay 0. + m.words(0x3000, &[0xf000_0000 | (2 << 19) | (1 << 18), 0x4000, 0x3020, 0x4007]); + h.state().config().write(1, 6); + h.bar_write(0, 0x18, 0x1000, 4); + h.bar_write(0, 0x04, 0x84, 4); + (h, m) +} + +/// A report waiting in the device is carried by an interrupt TD into guest +/// memory, with the TD retired as a success. +#[test] +fn an_interrupt_transfer_returns_a_keyboard_report_through_ohci() { + let k = UsbKeyboard::new(); + let (h, m) = ohci_with(k.clone(), 5); + k.key(KeyCode::KeyA, true); + h.advance_frames(2); + let report = m.0.lock()[0x4000..0x4009].to_vec(); + // The done queue is published once the delay for a low-priority completion passes. + h.advance_frames(10); + assert_eq!(report, vec![0, 0, 4, 0, 0, 0, 0, 0, 0xa5], "the report, and nothing past its end"); + assert_eq!(m.word(0x3000) >> 28, 0, "completed without error"); + assert_eq!(m.word(0x1084) & !15, 0x3000, "the TD is on the done queue"); +} + +/// With nothing to report the device NAKs and the transfer stays queued. +#[test] +fn an_interrupt_transfer_with_no_report_stays_pending() { + let m = UsbMouse::new(); + let (h, mem) = ohci_with(m.clone(), 5); + h.advance_frames(3); + assert_eq!(mem.word(0x3000) >> 28, 0xf, "not accessed"); + m.mouse(2, 1, 1, 0); + h.advance_frames(2); + assert_eq!(&mem.0.lock()[0x4000..0x4003], [0, 1, 1]); + assert_eq!(mem.word(0x3000) >> 28, 0); +} diff --git a/src/devlog.rs b/src/devlog.rs index fc5cc595f..6dd8e2d9f 100644 --- a/src/devlog.rs +++ b/src/devlog.rs @@ -71,6 +71,7 @@ pub enum LogModule { /// changes, and the commands it runs (categories `NVME_LOG_*`). Nvme = 34, L1 = 35, + Usb = 36, } // Mask bits shared by L1i / L1d / L2c @@ -85,7 +86,7 @@ pub const NVME_LOG_REGS: u32 = 0x4; // every register read and write below the pub const NVME_LOG_IO: u32 = 0x8; // I/O doorbells, commands and completions impl LogModule { - pub const COUNT: usize = 36; + pub const COUNT: usize = 37; pub fn from_str(s: &str) -> Option { match s { @@ -125,6 +126,7 @@ impl LogModule { "heart" => Some(Self::Heart), "nvme" => Some(Self::Nvme), "l1" => Some(Self::L1), + "usb" => Some(Self::Usb), _ => None, } } @@ -167,6 +169,7 @@ impl LogModule { Self::Heart => "heart", Self::Nvme => "nvme", Self::L1 => "l1", + Self::Usb => "usb", } } @@ -179,7 +182,7 @@ impl LogModule { Self::L1i, Self::L1d, Self::L2c, Self::Ultra, Self::Scsi1, Self::Nveeprom, Self::Gr2, Self::Rad1, Self::Odyssey, Self::Bedrock, Self::Hub, Self::Pci, Self::Bridge, Self::Heart, Self::Nvme, - Self::L1, + Self::L1, Self::Usb, ] } @@ -345,6 +348,7 @@ impl DevLog { ModuleLog::new(), // Heart ModuleLog::new(), // Nvme ModuleLog::new(), // L1 + ModuleLog::new(), // Usb ], } } From dd172935de467b11fb5e28899f79299cc500b427 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:12:30 +0000 Subject: [PATCH 15/22] cpu: take code and generation counters from the cache's bus The JIT reads code pages, and asks for their generation counters, at plain physical addresses, through the executor's sysad. Everywhere else sysad is also what the cache fills from, so a plain address there is memory. The Fuel's sysad is the uncached connection: it takes bits 58:57 of the address as the uncached attribute and decodes a plain one as Hub-special space, which holds RAM only in its first 256 MiB. Its compile pool was started on the cache's own view of memory instead, so the pool and the CPU's inline compile and page lookup read code through two different objects, and a flush restarted the pool on the CPU's. The executor now keeps one bus for cacheable memory, defaulting to sysad, and the page lookup, the inline compile, a flush's restart and `j2 inline off` all use it. A board whose uncached connection decodes differently gives it the cache's view with set_cached_bus and starts the pool on the same object. No other machine changes: the two are the same object there. --- src/cpu/mips_exec.rs | 34 +++++++++++++++++++++++++++------- src/cpu/mips_exec/monitor.rs | 4 ++-- 2 files changed, 29 insertions(+), 9 deletions(-) diff --git a/src/cpu/mips_exec.rs b/src/cpu/mips_exec.rs index 6e316bed7..a76092ea3 100644 --- a/src/cpu/mips_exec.rs +++ b/src/cpu/mips_exec.rs @@ -1016,6 +1016,14 @@ enum BusErrorOp { pub struct MipsExecutor { pub core: MipsCore, pub sysad: Arc, + /// Where the JIT reads code and generation counters, and the bus its + /// compile pool is started on: cacheable memory at plain physical + /// addresses. The same object as `sysad` unless the board's uncached + /// connection decodes a plain address as something other than memory (the + /// Fuel's decodes it as Hub-special space, with RAM only in its first + /// 256 MiB), in which case [`Self::set_cached_bus`] gives it the view the + /// cache fills from. + cached_bus: Arc, /// cheritest convention (`--cheritest-dump-hook`): a guest write to CP0 26 /// dumps machine state via the test device. Off unless asked for — CP0 26 /// is ECC on a real R4400 and the PROM writes it during cache init. @@ -2596,6 +2604,17 @@ impl MipsExecutor { size * 8, self.core.pc, virt_addr, phys_addr, val, status, self.fault_note(phys_addr, size)); } + /// The bus the JIT reads code from and its compile pool is started on. + pub fn cached_bus(&self) -> &Arc { + &self.cached_bus + } + + /// Give the JIT a cacheable view of memory that differs from `sysad` (see + /// the field). A board that does so builds the cache on the same view. + pub fn set_cached_bus(&mut self, bus: Arc) { + self.cached_bus = bus; + } + /// Create a new executor from a config and a bus (sysad) and a TLB. /// The cache hierarchy is constructed internally as a unified R4000Cache. pub fn new(sysad: Arc, mut tlb: T, cfg: &MipsCpuConfig) -> Self @@ -2751,6 +2770,7 @@ impl MipsExecutor { let mut executor = Self { core, + cached_bus: sysad.clone(), sysad, cheritest_dump_hook: false, tlb, @@ -3445,7 +3465,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cfg(feature = "jitv2")] pub(crate) fn jitv2_flush_from_here(&mut self, only_if_exhausted: bool) { crate::pf_scope!(crate::perfetto::events::JIT_CPU_FLUSH); - let bus = self.sysad.clone(); + let bus = self.cached_bus.clone(); let jit = self.jitv2.clone(); let flush = || { let mut j = jit.lock(); @@ -3587,7 +3607,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", let (mut jit, lookup) = { crate::pf_scope!(crate::perfetto::events::JIT_CPU_TRACK); let mut jit = self.jitv2.lock(); - let lookup = jit.page_for(pfn, page_base, self.sysad.as_ref(), fr1); + let lookup = jit.page_for(pfn, page_base, self.cached_bus.as_ref(), fr1); (jit, lookup) }; match lookup { @@ -3628,7 +3648,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", // exact page this fetch already landed on. self.clear_pcp(); let mut jit = self.jitv2.lock(); - let slot = jit.page_for(pfn, page_base, self.sysad.as_ref(), fr1) + let slot = jit.page_for(pfn, page_base, self.cached_bus.as_ref(), fr1) .expect("flush must leave room for at least one page"); // Direct assignment for the same borrow reason as the // hit path above; `pfn` is what `set_pcp` would mirror. @@ -9330,7 +9350,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", .is_some_and(|c| c.packing_stats().1 > crate::cpu::jitv2::CODEGEN_ARENA_FLUSH_THRESHOLD_BYTES); if over_threshold { let phys_page_base = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - unsafe { self.jitv2.lock().flush_from_cpu_thread(self.sysad.clone()); } + unsafe { self.jitv2.lock().flush_from_cpu_thread(self.cached_bus.clone()); } self.clear_pcp(); self.jitv2_track_pcp(phys_page_base); return EXEC_RETRY; @@ -9352,10 +9372,10 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", #[cfg(feature = "developer")] { let stats = self.jitv2.lock().stats.clone(); - ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.sysad, &mut self.jitv2_inline_analyzer, codegen, &stats); + ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.cached_bus, &mut self.jitv2_inline_analyzer, codegen, &stats); } #[cfg(not(feature = "developer"))] - { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.sysad, &mut self.jitv2_inline_analyzer, codegen); } + { ran_out_of_memory = crate::cpu::jitv2::comp::handle_request(&req, &self.cached_bus, &mut self.jitv2_inline_analyzer, codegen); } } *self.jitv2.lock().codegen.lock() = codegen; { @@ -9366,7 +9386,7 @@ va={:#018x} phys={:#010x} (code pfn {:#x}, page {:#010x}, word {}/{})", if ran_out_of_memory { let page = unsafe { &mut *self.pcp }; let phys_page_base = page.pfn as u64 * crate::cpu::jitv2::PAGE_SIZE as u64; - unsafe { self.jitv2.lock().flush_from_cpu_thread(self.sysad.clone()); } + unsafe { self.jitv2.lock().flush_from_cpu_thread(self.cached_bus.clone()); } self.clear_pcp(); self.jitv2_track_pcp(phys_page_base); return EXEC_RETRY; diff --git a/src/cpu/mips_exec/monitor.rs b/src/cpu/mips_exec/monitor.rs index a4385fad1..c836309ed 100644 --- a/src/cpu/mips_exec/monitor.rs +++ b/src/cpu/mips_exec/monitor.rs @@ -2027,7 +2027,7 @@ impl MipsCpu { #[cfg(not(feature = "jitv2_lockstep"))] { exec.jitv2_inline_compile = false; - let bus = exec.sysad.clone(); + let bus = exec.cached_bus.clone(); let old = exec.jitv2.lock().codegen.lock().take(); match old { Some(old) => { @@ -3010,7 +3010,7 @@ impl MipsCpu { // implementation. let path = actual_args.get(1).copied().unwrap_or("jitv2.html"); let jitv2_arc = exec.jitv2.clone(); - let bus = exec.sysad.clone(); + let bus = exec.cached_bus.clone(); crate::cpu::jitv2_html_j2wp::write_jitv2_html( &jitv2_arc, &bus, &exec.memory.clone(), &mut exec.jitv2_inline_analyzer, path, &mut *writer, )?; From c7e73870a50e9812389b7eb3f25f16d7d2551343 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 16/22] fuel: add the serial IP35 machine Build the Fuel IP34 board around its R14000, Bedrock, XBridge, IOC3 and dual-channel ISP12160. Use one Physical memory table with sparse processor and peripheral views. The Bedrock node every IP35 board shares lives in ip35::node: the processor bus with its four views, the Bedrock chip, the L1, the boot flash and the Hub-special page with its board-id word and LEDs. A board names its inputs in a NodeConfig (bank sizes, fitted processors, firmware record, board-id word, L1 identity) and adds its own widgets; connect_cpus gives the node's memory, interrupts and L1 events to a slice of processors, CPU 0 first. The Fuel passes two 256 MiB banks, one processor in PI0 slot A and its 500 MHz firmware record, from which the CPU's reset Config comes too. The machine file is ip34_machine.rs, after its board, as the Octane's and O2's are named; ip35 is the software platform other IP35 boards share. Bedrock's node resets and the L1's power events reach the system controller on one channel. Only an L1 shutdown powers the machine off and a restart resets it, so a power cycle the PROM asks for restarts the node instead of stopping the machine at its power-down. Each processor's IP2..IP6 are bits of its own interrupt word, which the chip publishes into directly, and the PI comparators run on the machine's timer manager, so the CPU's batch hook no longer polls the PIs. Uncached byte and halfword stores reach a device through BusDevice::write_narrow_from, which each window forwards at the device-local address; only the memory controller's ECC view reads more than the low bits of the register. With the jitv2 feature, start the compile pool after the memory windows are published, as the Octane and O2 do. Each worker copies the cache geometry once as it starts, and without the windows that geometry has no inline load or store path. Workers read code through the CPU's cached bus, the view the cache fills from, which serves plain physical addresses; the uncached views would decode them as node registers. The machine describes where its memory can answer to the CPU: each 1 GiB slot the controller can place a bank in, as far as the largest bank of its group reaches, and the reset PROM. Its disks on both ISP12160 channels are flushed at power-off, quit and exit like the other machines', so a copy-on-write overlay keeps its record of held sectors after a guest `halt -p`. The processor is the R14000 at revision 2.3 (PRId 0x0f23), the revision a public software inventory of a Fuel prints, where the Octane's part reports 2.4. The raw CP0 word, the FIR and the 128-byte lines are taken from the Octane part, not read from a Fuel. Like the Octane's R14000 it reports "R12000" as its name. The machine loads its PROM through prom_image. Feed the IOC3 PS/2 controller from the window, VNC and CI input, and let a USB keyboard on root port 0 and a USB mouse on port 1 share it. The Fuel's manual shows PS/2 as the standard keyboard and mouse, so ps2_input under [fuel] defaults on and usb_input defaults off; setting usb_input fits the USB pair, and ps2_input = false leaves the PS/2 pair out. The monitor's ps2 command reaches the keyboard and mouse too. Config profile (config): The Fuel is a new machine profile with two memory banks, wide SCSI buses, no Ethernet adapter choice beyond its built-in one, and no graphics board by default. The [fuel] section selects the serial console (the L1 controller or either IOC3 port), the L1 TCP port and log, and the persistent boot flash store. l1_revision and brick_serial replace the L1's reported firmware revision (default 1.48.1) and brick serial (default EMUN00). The key names the part: machine.cpu = "r14000" selects the Fuel's r14000_500, and a Fuel config cannot claim another processor. The Octane's R14000 is revision 2.4; the Fuel's is 2.3. --- HELP.md | 6 +- src/config.rs | 260 ++++++++++++- src/cpu/mips_cache_shadow.rs | 1 - src/cpu/parts.rs | 3 + src/dev/bedrock/md.rs | 163 ++++++++ src/dev/l1.rs | 5 + src/dev/mod.rs | 2 +- src/ip34_machine.rs | 731 +++++++++++++++++++++++++++++++++++ src/ip35.rs | 196 +++++++++- src/ip35/node.rs | 276 +++++++++++++ src/ip35/tests.rs | 638 ++++++++++++++++++++++++++++++ src/lib.rs | 1 + src/machine.rs | 8 +- src/workstation.rs | 1 + 14 files changed, 2264 insertions(+), 27 deletions(-) create mode 100644 src/ip34_machine.rs create mode 100644 src/ip35/node.rs create mode 100644 src/ip35/tests.rs diff --git a/HELP.md b/HELP.md index 7619f4939..77ace5234 100644 --- a/HELP.md +++ b/HELP.md @@ -1,7 +1,9 @@ -# IRIS — SGI Indy and Indigo2 emulator +# IRIS — SGI workstation emulator The complete core/GUI feature and configuration inventory is in -[FEATURES.md](FEATURES.md). This guide explains everyday use; the monitor's full command reference is in [debug.md](debug.md). +[FEATURES.md](FEATURES.md). For the Fuel, see the [Fuel guide](docs/fuel-ip34.md). +This guide explains everyday use. The monitor's full command reference is in +[debug.md](debug.md). ## Quick start diff --git a/src/config.rs b/src/config.rs index 3e9cb8fd3..e68b10881 100644 --- a/src/config.rs +++ b/src/config.rs @@ -676,6 +676,8 @@ pub enum MachineProfile { /// a test widget. Configured by the shared keys and `[origin2000]`; see /// `src/ip27_onyx2.rs`. Onyx2Ip27, + /// Serial-only Fuel IP34 board using the IP35 software platform. + FuelIp34, } impl MachineProfile { @@ -699,6 +701,7 @@ impl MachineProfile { Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 => &[0, 64, 128, 256, 512], Self::O2Ip32 => &[0, 32, 64, 128, 256], Self::Indigo2Ip28 => MC_BANK_SIZES, + Self::FuelIp34 => &[0, 256], _ => &[0, 8, 16, 32, 64, 128], } } @@ -723,7 +726,7 @@ impl MachineProfile { /// Every profile, in the order `workstation::build` lists them. `ALL` is /// only the ones the GUI offers. - pub const EVERY: [Self; 8] = [ + pub const EVERY: [Self; 9] = [ Self::IndyIp24, Self::Indigo2Ip22, Self::Indigo2Ip28, @@ -732,6 +735,7 @@ impl MachineProfile { Self::Origin200Ip29, Self::Origin2000Ip27, Self::Onyx2Ip27, + Self::FuelIp34, ]; /// The name `[machine] profile` and `--machine` give this profile. @@ -745,6 +749,7 @@ impl MachineProfile { Self::Origin200Ip29 => "origin200_ip29", Self::Origin2000Ip27 => "origin2000_ip27", Self::Onyx2Ip27 => "onyx2_ip27", + Self::FuelIp34 => "fuel_ip34", } } @@ -802,6 +807,7 @@ impl MachineProfile { Self::Origin200Ip29 => "SGI Origin 200 (IP29)", Self::Origin2000Ip27 => "SGI Origin 2000 (IP27)", Self::Onyx2Ip27 => "SGI Onyx2 (IP27)", + Self::FuelIp34 => "SGI Fuel (IP34)", } } @@ -811,7 +817,7 @@ impl MachineProfile { /// connector), one on the Indy. pub fn scsi_controllers(self) -> u8 { match self { - Self::Indigo2Ip22 | Self::O2Ip32 | Self::OctaneIp30 | Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 => 2, + Self::Indigo2Ip22 | Self::O2Ip32 | Self::OctaneIp30 | Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 | Self::FuelIp34 => 2, _ => 1, } } @@ -820,18 +826,19 @@ impl MachineProfile { /// buses are wide, with ids 0 to 15 and the controller on 0. The Indy's /// and Indigo2's WD33C93A chips are narrow, with ids 0 to 7. The O2's /// Adaptec buses are wide too, but stay at 7 until they are widened on - /// purpose. + /// purpose. The Fuel's ISP12160 buses are wide. pub fn max_scsi_id(self) -> u8 { match self { - Self::OctaneIp30 | Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 => 15, + Self::OctaneIp30 | Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 | Self::FuelIp34 => 15, _ => 7, } } - /// How many memory banks the machine has: four on all of them (the MC's - /// four banks; the O2's four DIMM pairs). `banks` has room for more. + /// How many memory banks the machine has: four on the MC, HEART and O2 + /// (the O2's four DIMM pairs), two on Fuel. `banks` has room for more; + /// unused entries must be zero. pub fn memory_banks(self) -> usize { - 4 + if self == Self::FuelIp34 { 2 } else { 4 } } pub fn guinness(self) -> bool { @@ -846,6 +853,7 @@ impl MachineProfile { Self::Origin200Ip29 | Self::Origin2000Ip27 | Self::Onyx2Ip27 => crate::ip27_machine::CPUS, Self::O2Ip32 => crate::ip32_machine::CPUS, Self::Indigo2Ip28 => crate::machine::IP28_CPUS, + Self::FuelIp34 => crate::ip34_machine::CPUS, Self::IndyIp24 | Self::Indigo2Ip22 => crate::machine::CPUS, } } @@ -896,7 +904,7 @@ pub enum GraphicsBoard { #[clap(name = "v12", alias = "vpro:v12", alias = "vpro_v12")] V12, /// No graphics board: the machine runs on its serial console. Octane - /// only. + /// and Fuel. #[clap(name = "none")] None, } @@ -959,6 +967,7 @@ impl GraphicsBoard { match profile { MachineProfile::OctaneIp30 => self.is_impact() || self.is_vpro() || self == Self::None, MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 => self == Self::None, + MachineProfile::FuelIp34 => self == Self::None, MachineProfile::Indigo2Ip22 => !self.is_vpro() && self != Self::None, _ => !self.is_vpro() && !matches!(self, Self::Extreme | Self::None), } @@ -969,6 +978,7 @@ impl GraphicsBoard { match profile { MachineProfile::OctaneIp30 => Self::SolidImpact, MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 => Self::None, + MachineProfile::FuelIp34 => Self::None, _ => Self::Newport, } } @@ -1028,7 +1038,7 @@ impl std::fmt::Display for GraphicsBoard { #[serde(deny_unknown_fields)] pub struct GraphicsSection { /// Graphics board: Newport, GR2 XZ/Extreme, IMPACT Solid/High/Maximum, - /// or none (Octane only). Unset: the machine's default + /// or none (Octane and Fuel). Unset: the machine's default /// ([`GraphicsBoard::default_for`]); read it with /// [`MachineConfig::graphics_board`]. Extreme is restricted to IP22, /// the Octane to IMPACT or none; non-Newport boards require one head @@ -1796,6 +1806,80 @@ mod scsi_keys { } } +/// Fuel's independent serial connections. The selected console serves CI and serial_log. +#[derive(Debug, Clone, Copy, Default, PartialEq, Eq, Serialize, Deserialize)] +#[serde(rename_all = "snake_case")] +pub enum FuelConsole { + #[default] + L1, + Ioc3A, + Ioc3B, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct FuelSection { + /// Persistent 16 MiB boot flash store; absent means volatile flash. + #[serde(default)] + pub flash: Option, + #[serde(default)] + pub console: FuelConsole, + /// L1 host console TCP port; IOC3 keeps serial_port_a and serial_port_b. + #[serde(default)] + pub l1_port: Option, + /// Separate L1 log when an IOC3 port is selected as console. + #[serde(default)] + pub l1_log: Option, + /// Fit the PS/2 keyboard and mouse controller in the onboard IOC3 and + /// feed it the host's keyboard and mouse. + #[serde(default = "default_fuel_input")] + pub ps2_input: bool, + /// Plug a USB keyboard and mouse into the OHCI root ports and feed them + /// the same host input. Off by default: the Fuel's manual and the PROM + /// treat PS/2 as the standard keyboard and mouse. + #[serde(default)] + pub usb_input: bool, + /// L1 firmware revision as "major.minor.patch", reported to request + /// 0x0005. Unset: 1.48.1. + #[serde(default)] + pub l1_revision: Option, + /// Brick serial reported to L1 request 0x0004, at most 37 bytes. Unset: + /// the marked synthetic `EMUN00`. + #[serde(default)] + pub brick_serial: Option, +} + +fn default_fuel_input() -> bool { true } + +impl Default for FuelSection { + fn default() -> Self { + Self { + flash: None, + console: FuelConsole::default(), + l1_port: None, + l1_log: None, + ps2_input: true, + usb_input: false, + l1_revision: None, + brick_serial: None, + } + } +} + +impl FuelSection { + /// Left out of a written config when nothing in it was set. + fn is_default(&self) -> bool { + self.flash.is_none() + && self.console == FuelConsole::default() + && self.l1_port.is_none() + && self.l1_log.is_none() + && self.ps2_input + && !self.usb_input + && self.l1_revision.is_none() + && self.brick_serial.is_none() + } +} + /// Top-level machine configuration. /// /// Field order matters for TOML export: the `toml` serializer requires every @@ -2050,6 +2134,10 @@ pub struct MachineConfig { #[serde(default, skip_serializing_if = "Origin2000Section::is_default")] pub origin2000: Origin2000Section, + /// Serial console and flash storage for the Fuel. + #[serde(default, skip_serializing_if = "FuelSection::is_default")] + pub fuel: FuelSection, + /// Host performance tuning (`[perf]` section). #[serde(default)] pub perf: PerfConfig, @@ -2288,6 +2376,7 @@ impl Default for MachineConfig { octane: OctaneSection::default(), origin200: Origin200Section::default(), origin2000: Origin2000Section::default(), + fuel: FuelSection::default(), perf: PerfConfig::default(), clock: ClockConfig::default(), rtc_offset: RtcOffset::default(), @@ -2382,6 +2471,9 @@ impl MachineConfig { MachineProfile::Origin200Ip29 => Err("[virtio_serial] is not wired on the Origin 200".into()), MachineProfile::Origin2000Ip27 => Err("[virtio_serial] is not wired on the Origin 2000".into()), MachineProfile::Onyx2Ip27 => Err("[virtio_serial] is not wired on the Onyx2".into()), + MachineProfile::FuelIp34 => Err( + "[virtio_serial] is not wired on the Fuel: its XBridge has no virtio-pci attachment".into(), + ), MachineProfile::O2Ip32 => Err( "[virtio_serial] is not wired on the O2 yet: the virtio-pci transport exists, but the \ MACE PCI bridge has no interrupt route for a third slot (see docs/dev/virtio-serial.md)" @@ -2566,7 +2658,7 @@ impl MachineConfig { /// Validate bank sizes, returns a description of any errors. /// The graphics board: `[graphics] board`, or the machine's default - /// when it is not set (Solid IMPACT on the Octane, Newport elsewhere). + /// (Solid IMPACT on Octane, none on Fuel, Newport elsewhere). pub fn graphics_board(&self) -> GraphicsBoard { self.graphics.board.unwrap_or_else(|| GraphicsBoard::default_for(self.machine.profile)) } @@ -2663,9 +2755,9 @@ impl MachineConfig { "scsi.{} and scsi.{} both name {}", other, key, scsi_label(controller, id) )); } - if dev.is_daynaport() && matches!(self.machine.profile, MachineProfile::O2Ip32 | MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27) { + if dev.is_daynaport() && matches!(self.machine.profile, MachineProfile::O2Ip32 | MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 | MachineProfile::FuelIp34) { return Err(format!( - "scsi.{}: kind = \"daynaport\" is not available on the O2 (its Ethernet is built in)", key + "scsi.{}: kind = \"daynaport\" is not available on {} (its Ethernet is built in)", key, self.machine.profile.label() )); } if dev.is_daynaport() && self.network.mode == NetMode::Stream { @@ -2727,6 +2819,19 @@ impl MachineConfig { return Err(format!("bank{i} size 256 MB: the PROM disables 256 MB banks on the 180 MHz R10000 module (use 128 and 128, or the r12000 module)")); } } + } else if self.machine.profile == MachineProfile::FuelIp34 { + if self.banks[..2] != [256, 256] { + return Err("Fuel currently requires banks = [256, 256] (512 MiB)".into()); + } + if board != GraphicsBoard::None { + return Err("Fuel currently supports graphics.board = \"none\" only".into()); + } + if let Some(text) = &self.fuel.l1_revision { + crate::dev::l1::parse_revision(text).map_err(|e| format!("fuel.l1_revision: {e}"))?; + } + if self.fuel.brick_serial.as_ref().is_some_and(|s| s.len() > 37 || s.contains('\0')) { + return Err("fuel.brick_serial must be at most 37 bytes with no NUL".into()); + } } else if self.machine.profile == MachineProfile::OctaneIp30 { // A HEART bank is a matched DIMM pair: 64 MB to 2 GB, filled from // bank 0 without a gap, 8 GB at most (SGI's memory upgrade guide). @@ -2772,8 +2877,8 @@ impl MachineConfig { } } match (self.network.mode, self.machine.profile) { - (NetMode::Off, p) if p != MachineProfile::O2Ip32 && p != MachineProfile::OctaneIp30 && p != MachineProfile::Origin200Ip29 && p != MachineProfile::Origin2000Ip27 && p != MachineProfile::Onyx2Ip27 => { - return Err("network.mode = \"off\" is only available on the O2, the Octane and the Origin 200 and 2000 for now; the Indy and Indigo2 always have their Ethernet".into()); + (NetMode::Off, p) if p != MachineProfile::O2Ip32 && p != MachineProfile::OctaneIp30 && p != MachineProfile::Origin200Ip29 && p != MachineProfile::Origin2000Ip27 && p != MachineProfile::Onyx2Ip27 && p != MachineProfile::FuelIp34 => { + return Err("network.mode = \"off\" is only available on the O2, the Octane, the Fuel and the Origin 200 and 2000 for now; the Indy and Indigo2 always have their Ethernet".into()); } (NetMode::Stream, _) => { let ep = self.network.stream.as_deref() @@ -3901,7 +4006,7 @@ mod export_tests { assert_eq!(apply(&["--ip22"]).machine.profile, MachineProfile::Indigo2Ip22); assert!(Cli::try_parse_from(["iris", "--ip22", "--machine", "o2_ip32"]).is_err()); let e = MachineProfile::from_name("vax").unwrap_err(); - assert_eq!(e, "unknown machine \"vax\" (valid: indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32, octane_ip30, origin200_ip29, origin2000_ip27, onyx2_ip27)"); + assert_eq!(e, "unknown machine \"vax\" (valid: indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32, octane_ip30, origin200_ip29, origin2000_ip27, onyx2_ip27, fuel_ip34)"); let list = MachineProfile::list(); assert_eq!(list.lines().count(), MachineProfile::EVERY.len()); assert!(list.starts_with("indy_ip24 SGI Indy (IP24)\n"), "{list}"); @@ -3910,7 +4015,7 @@ mod export_tests { for p in MachineProfile::EVERY { match p { MachineProfile::IndyIp24 | MachineProfile::Indigo2Ip22 | MachineProfile::Indigo2Ip28 - | MachineProfile::O2Ip32 | MachineProfile::OctaneIp30 | MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 => {} + | MachineProfile::O2Ip32 | MachineProfile::OctaneIp30 | MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 | MachineProfile::FuelIp34 => {} } let cfg: MachineConfig = toml::from_str(&format!("[machine]\nprofile = \"{}\"", p.name())).unwrap(); assert_eq!(cfg.machine.profile, p); @@ -3925,6 +4030,7 @@ mod export_tests { let accepts = |p: MachineProfile, cpu: &str| p.part(Some(cpu)).is_ok(); assert!(accepts(MachineProfile::OctaneIp30, "r14000") && !accepts(MachineProfile::OctaneIp30, "r5000")); assert!(accepts(MachineProfile::IndyIp24, "r5000") && !accepts(MachineProfile::IndyIp24, "r10000")); + assert!(accepts(MachineProfile::FuelIp34, "r14000") && !accepts(MachineProfile::FuelIp34, "r10000")); // The counts are what the checks in validate() and the Octane // constructor allowed before they moved here: one on every machine @@ -4357,3 +4463,125 @@ mod ci_overlay_tests { assert!(a.contains(&format!("iris-ci-{}-", std::process::id())), "{a}"); } } + +#[cfg(test)] +mod fuel_input_tests { + use super::*; + + /// A config saved for another machine must not grow a `[fuel]` table, + /// and a Fuel's own settings must survive a save and reload. + #[test] + fn only_a_configured_fuel_section_is_saved_and_it_round_trips() { + let other = MachineConfig::default(); + let saved = toml::to_string(&other).unwrap(); + assert!(!saved.contains("[fuel]"), "{saved}"); + let mut fuel = MachineConfig::default(); + fuel.fuel.console = FuelConsole::Ioc3B; + fuel.fuel.flash = Some("flash.bin".into()); + fuel.fuel.l1_port = Some(18882); + fuel.fuel.l1_log = Some("l1.log".into()); + fuel.fuel.usb_input = true; + fuel.fuel.ps2_input = false; + let saved = toml::to_string(&fuel).unwrap(); + assert!(saved.contains("[fuel]"), "{saved}"); + let loaded: MachineConfig = toml::from_str(&saved).unwrap(); + assert_eq!(loaded.fuel.console, FuelConsole::Ioc3B); + assert_eq!(loaded.fuel.flash.as_deref(), Some("flash.bin")); + assert_eq!(loaded.fuel.l1_port, Some(18882)); + assert_eq!(loaded.fuel.l1_log.as_deref(), Some("l1.log")); + assert!(!loaded.fuel.ps2_input && loaded.fuel.usb_input); + assert!(!MachineProfile::ALL.contains(&MachineProfile::FuelIp34)); + } + + /// PS/2 is the Fuel's standard keyboard and mouse; USB is opt-in, and a + /// `[fuel]` table that names neither key keeps both defaults. + #[test] + fn fuel_input_defaults_to_ps2_with_usb_opt_in() { + let none: MachineConfig = toml::from_str("[fuel]\nflash = \"flash.bin\"\n").unwrap(); + assert!(none.fuel.ps2_input && !none.fuel.usb_input); + assert!(MachineConfig::default().fuel.is_default()); + let usb: MachineConfig = toml::from_str("[fuel]\nusb_input = true\n").unwrap(); + assert!(usb.fuel.ps2_input && usb.fuel.usb_input); + assert!(!usb.fuel.is_default(), "an opt-in is written back"); + } + + /// The Fuel's processor is fixed, so `machine.cpu` may only name it (or the + /// family, as the key did before the R14000 had a name); the default + /// `r4400` is refused rather than silently ignored. + /// The L1 identity keys parse, validate and round trip; unset they are + /// left out of a written config. + #[test] + fn fuel_l1_identity_keys_are_validated_and_round_trip() { + let base = "banks = [256, 256]\n[machine]\nprofile = \"fuel_ip34\"\ncpu = \"r14000\"\n"; + let ok: MachineConfig = toml::from_str(&format!( + "{base}[fuel]\nl1_revision = \"1.50.2\"\nbrick_serial = \"EMUN03\"\n" + )).unwrap(); + ok.validate().unwrap(); + let saved = toml::to_string(&ok).unwrap(); + let back: MachineConfig = toml::from_str(&saved).unwrap(); + assert_eq!(back.fuel.l1_revision.as_deref(), Some("1.50.2")); + assert_eq!(back.fuel.brick_serial.as_deref(), Some("EMUN03")); + let plain: MachineConfig = toml::from_str(base).unwrap(); + plain.validate().unwrap(); + assert!(plain.fuel.l1_revision.is_none() && plain.fuel.brick_serial.is_none()); + let bad_rev: MachineConfig = toml::from_str(&format!("{base}[fuel]\nl1_revision = \"1.48\"\n")).unwrap(); + assert!(bad_rev.validate().unwrap_err().contains("fuel.l1_revision")); + let long = "x".repeat(38); + let bad_serial: MachineConfig = toml::from_str(&format!("{base}[fuel]\nbrick_serial = \"{long}\"\n")).unwrap(); + assert!(bad_serial.validate().unwrap_err().contains("fuel.brick_serial")); + } + + #[test] + fn fuel_takes_only_its_r14000() { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = MachineProfile::FuelIp34; + cfg.banks.fill(0); + cfg.banks[0] = 256; + cfg.banks[1] = 256; + cfg.scsi.clear(); + // Leaving `cpu` out takes the Fuel's own part. + cfg.machine.cpu = None; + cfg.validate().unwrap(); + assert_eq!(cfg.cpu(), &crate::cpu::parts::R14000_500); + for name in ["r14000", "r14000_500"] { + cfg.machine.cpu = Some(name.into()); + cfg.validate().unwrap(); + assert_eq!(cfg.cpu(), &crate::cpu::parts::R14000_500, "{name}"); + } + for name in ["r10000", "r12000_360", "r14000_600"] { + cfg.machine.cpu = Some(name.into()); + assert!(cfg.validate().unwrap_err().contains("r14000_500"), "{name}"); + } + } + + #[test] + fn fuel_requires_its_fixed_memory_population_and_no_graphics() { + let mut cfg = MachineConfig::default(); + cfg.machine.profile = MachineProfile::FuelIp34; + cfg.banks.fill(0); + cfg.banks[0] = 256; + cfg.banks[1] = 256; + cfg.scsi.clear(); + assert_eq!(cfg.graphics_board(), GraphicsBoard::None); + cfg.network.mode = NetMode::Off; + cfg.validate().unwrap(); + cfg.network.mode = NetMode::Pcap; + cfg.validate().unwrap(); + cfg.graphics.board = Some(GraphicsBoard::Newport); + assert!(cfg.validate().unwrap_err().contains("none")); + cfg.graphics.board = None; + cfg.banks[1] = 0; + assert!(cfg.validate().unwrap_err().contains("256, 256")); + } + + #[test] + fn fuel_accepts_wide_scsi_ids_on_both_channels_without_changing_narrow_profiles() { + let text = "banks = [256, 256]\n[machine]\nprofile = \"fuel_ip34\"\ncpu = \"r14000\"\n[scsi.15]\npath = \"disk.raw\"\n[scsi.31]\npath = \"disk1.raw\"\n"; + let mut cfg: MachineConfig = toml::from_str(text).unwrap(); + cfg.validate().unwrap(); + // The Octane's buses are wide as well; the Indy's WD33C93A is narrow. + cfg.machine.profile = MachineProfile::IndyIp24; + cfg.machine.cpu = Some("r4400".into()); + assert!(cfg.validate().unwrap_err().contains("out of range")); + } +} diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index c49045c89..3cfccd4f5 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -1146,7 +1146,6 @@ pub type R12000Module4MShadowCache = pub type R12000O2ShadowCache = ShadowCache<32768, 64, 32768, 32, 2097152, 64, true, 0x0000_0e35, 0x0000_0900, 64, { model::R12000 }>; - /// SGI O2 R10000 (IP32): the IP28 model with 64-byte secondary-cache lines. /// /// The O2 PROM (4.18 and earlier) handles only that line size. With Config's diff --git a/src/cpu/parts.rs b/src/cpu/parts.rs index ceda3972c..b3d0479e3 100644 --- a/src/cpu/parts.rs +++ b/src/cpu/parts.rs @@ -52,6 +52,9 @@ pub const R10000_250: Part = Part { mhz: 250, rev: (3, 4), ..R10000_195 }; pub const R12000_360: Part = Part { family: Family::R12000, mhz: 360, rev: (3, 5), l2_kb: 2048 }; pub const R12000_400: Part = Part { mhz: 400, ..R12000_360 }; pub const R14000_600: Part = Part { family: Family::R14000, mhz: 600, rev: (2, 4), l2_kb: 2048 }; +/// The Fuel's R14000A: rev 2.3 at 500 MHz with a 2 MB secondary cache, as the IP35 +/// `hinv` dumps show it. +pub const R14000_500: Part = Part { mhz: 500, rev: (2, 3), ..R14000_600 }; /// Every part, for the bare-metal bench rig, which takes any. A name matches /// the first listed, so the rig runs the later silicon of a clock that has two. diff --git a/src/dev/bedrock/md.rs b/src/dev/bedrock/md.rs index 2428e0037..ad8276e56 100644 --- a/src/dev/bedrock/md.rs +++ b/src/dev/bedrock/md.rs @@ -1741,6 +1741,169 @@ mod tests { assert!(after > before, "{what}: {before} -> {after}"); } } + /// A read-only 4 MiB window standing for the reset PROM, whose first + /// word reads 0x0f1e2d3c4b5a6978. + struct Prom; + impl BusDevice for Prom { + fn read64(&self, a: u64) -> BusRead64 { + if a == 0 { BusRead64::ok(0x0f1e_2d3c_4b5a_6978) } else { BusRead64::ok(0) } + } + fn write64(&self, _: u64, _: u64) -> u32 { BUS_ERR } + } + + /// The Tezro's memory wired as the board wires it: the processor bus with + /// uncached RAM views, the cached view, and the reset PROM mapped over the + /// cached plane where the bank leaves room. + fn board(md: &Arc) -> (crate::ip35::Ip35Bus, Arc) { + let mut bus = crate::ip35::Ip35Bus::new(md.physical().clone()); + md.rebuild_memory(); + bus.map(3, 0, 1 << 33, md.uncached_view()); + bus.map(0, 0, 0x1000_0000, md.uncached_view()); + bus.map(0, 0x1fc0_0000, 0x40_0000, Arc::new(Prom)); + crate::ip35::map_device(md.physical(), 0x1fc0_0000, 0x40_0000, Arc::new(Prom), "test reset PROM"); + let cached = bus.cached_view(md.dma_view(), md.ram_extent()); + (bus, cached) + } + + /// The Tezro's default population, wired and configured as the board does. + fn default_tezro() -> (Arc, crate::ip35::Ip35Bus, Arc) { + let md = TestMd::new(&TEZRO_BANKS).unwrap(); + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG), BUS_OK); + let (bus, cached) = board(&md); + (md, bus, cached) + } + + /// A processor store to banks 4 and 5, above the 4 GiB the direct window + /// reaches, goes through the board's cached view or an uncached one into + /// the bank behind the slot and advances that slot's logical counter, + /// and no other slot's. The JIT page lookup and the cache's own bump read + /// the same counter. + #[test] + fn processor_stores_to_banks_4_and_5_advance_their_slot_counters() { + let (md, bus, cached) = default_tezro(); + let uncached = |address: u64| (3u64 << 57) | address; + for (slot, other) in [(4u64, 5u64), (5, 4)] { + let at = slot * BANK_SPAN + 0x2000; + let elsewhere = counter(&md, other * BANK_SPAN + 0x2000); + #[cfg(feature = "jitv2")] + { + assert_eq!(cached.gen_ptr(at), md.memory.gen_ptr(at)); + } + let mut last = counter(&md, at); + let mut stored = |what: &str, status: u32| { + assert_eq!(status, BUS_OK, "{what}"); + let now = counter(&md, at); + assert_advanced(what, last, now); + last = now; + }; + stored("cached store", cached.write64(at, 0x1111)); + stored("cached block store", cached.write_block(at + 8, &[0x2222, 0x3333])); + stored("cached masked store", cached.write64_masked(at + 0x20, 0x4444, 0xff00)); + stored("cached halfword store", cached.write16(at + 0x30, 0x5555)); + stored("uncached store", bus.write64(uncached(at + 0x40), 0x6666)); + stored("uncached byte store", bus.write8(uncached(at + 0x48), 0x77)); + let bank = &md.physical().banks()[slot as usize]; + let word = |offset: u64| bank.read64(0x2000 + offset).data; + assert_eq!([word(0), word(8), word(0x10), word(0x20), word(0x40)], + [0x1111, 0x2222, 0x3333, 0x4400, 0x6666], "slot {slot}"); + assert_eq!(counter(&md, other * BANK_SPAN + 0x2000), elsewhere, "slot {other} untouched"); + } + } + + /// A checked page above 4 GiB keeps its logical counter honest: marking + /// it advances the counter, a store through the page's slow route does + /// too, and so does the store once the page is back on its bank. + #[test] + fn a_checked_page_above_4_gib_advances_its_counter() { + let (md, _bus, cached) = default_tezro(); + let (page, address) = (4 * BANK_SPAN + 0x6000, 4 * BANK_SPAN + 0x6008); + let untouched = counter(&md, 4 * BANK_SPAN + 0x8000); + let start = counter(&md, page); + mark(&md, page); + let marked = counter(&md, page); + assert_advanced("mark", start, marked); + assert!(crate::ppmem::devmap::entry_eq(md.physical().entry_at(address), md.physical().entry_at(page))); + assert!(!crate::ppmem::devmap::entry_eq(md.physical().entry_at(page), md.physical().entry_at(page + 0x2000))); + assert_eq!(cached.write64(address, 0xaa), BUS_OK); + let stored = counter(&md, page); + assert_advanced("store to the checked page", marked, stored); + lift(&md, page); + md.retire_lifted_marks(); + let retired = counter(&md, page); + assert_advanced("retirement", stored, retired); + assert_eq!(cached.write64(address, 0xbb), BUS_OK); + assert_advanced("store after retirement", retired, counter(&md, page)); + assert_eq!(md.physical().banks()[4].read64(0x6008).data, 0xbb); + assert_eq!(counter(&md, 4 * BANK_SPAN + 0x8000), untouched, "a neighbouring page keeps its counter"); + } + + /// After a bank swap the slot's counter, which does not move, is the one + /// a store advances, whichever bank now sits behind the slot, and the + /// slot the bank left is not touched. + #[test] + fn a_store_after_a_bank_swap_advances_the_counter_of_the_slot_it_reached() { + let (md, bus, cached) = default_tezro(); + let (four, five) = (4 * BANK_SPAN + 0x2000, 5 * BANK_SPAN + 0x2000); + #[cfg(feature = "jitv2")] + let pointers = (md.memory.gen_ptr(four), md.memory.gen_ptr(five)); + // Bit 32 swaps the banks of a pair: slot 4 now reaches bank 5. + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG | (1 << 32)), BUS_OK); + #[cfg(feature = "jitv2")] + assert_eq!((md.memory.gen_ptr(four), md.memory.gen_ptr(five)), pointers); + let (before_four, before_five) = (counter(&md, four), counter(&md, five)); + assert_eq!(cached.write64(four, 0x44), BUS_OK); + assert_eq!(md.physical().banks()[5].read64(0x2000).data, 0x44, "slot 4 reaches bank 5"); + assert_eq!(md.physical().banks()[4].read64(0x2000).data, 0); + let after_four = counter(&md, four); + assert_advanced("cached store to slot 4", before_four, after_four); + assert_eq!(counter(&md, five), before_five); + assert_eq!(bus.write64((3u64 << 57) | five, 0x55), BUS_OK); + assert_eq!(md.physical().banks()[4].read64(0x2000).data, 0x55, "slot 5 reaches bank 4"); + assert_advanced("uncached store to slot 5", before_five, counter(&md, five)); + assert_eq!(counter(&md, four), after_four); + } + + /// With bank 0 at 1 GiB the RAM extent covers the cached PROM window at + /// 0x1fc0_0000, which a Fuel's 256 MiB banks leave to the PROM. Once + /// bank 0 is enabled the cached plane, the uncached RAM view and DMA all + /// see RAM there, and a cached store lands in it; the PROM answers + /// through Hub-special space only. Before bank 0 is enabled nothing is + /// RAM, the cached plane shows the PROM and a store is refused. + /// + /// The QEMU model agrees for those views and differs for one: its reset + /// window is routed by cache attribute, and a noncoherent cached access + /// (what the stock PROM runs with) keeps seeing the PROM after RAM is + /// enabled (hw/mips/sgi/sn1-node.c, sn1_reset_attrs_to_index and + /// sn1_node_map_reset, "Compatibility policy"). The cache here passes + /// no attribute to the bus, so a noncoherent cached access sees RAM. + #[test] + fn a_1_gib_bank_0_covers_the_cached_prom_window_with_ram() { + let md = TestMd::new(&TEZRO_BANKS).unwrap(); + let (bus, cached) = board(&md); + let at = 0x1fc0_0000; + let prom = 0x0f1e_2d3c_4b5a_6978; + let uncached = (3u64 << 57) | at; + assert_eq!(cached.read64(at).data, prom, "before RAM is enabled"); + assert_eq!(cached.write64(at, 0x55), BUS_ERR); + assert_eq!(md.md_bus().write64(reg::CONFIG, TEZRO_CONFIG), BUS_OK); + assert_eq!(bus.read64(at).data, prom, "Hub-special space still shows the PROM"); + for (what, value) in [("cached", cached.read64(at).data), ("uncached RAM", bus.read64(uncached).data), + ("DMA", md.dma_view().read64(at).data)] { + assert_eq!(value, 0, "{what} sees RAM"); + } + assert!(md.physical().mem_ptr(at).is_some(), "the cache fills from RAM too"); + let before = counter(&md, at); + assert_eq!(cached.write64(at, 0x55), BUS_OK); + assert_eq!(md.physical().banks()[0].read64(at).data, 0x55); + assert_advanced("cached store", before, counter(&md, at)); + assert_eq!(bus.read64(at).data, prom, "the PROM is not written"); + // The Fuel's 256 MiB banks never reach it. + let fuel = TestMd::new(&FUEL_BANKS).unwrap(); + let (_bus, cached) = board(&fuel); + assert_eq!(fuel.md_bus().write64(reg::CONFIG, 0x33), BUS_OK); + assert_eq!(cached.read64(at).data, prom); + assert_eq!(cached.write64(at, 0x55), BUS_ERR); + } /// Physical's bulk writers, on a bank directly, advance only that bank's /// counters, which nothing that follows code pages reads on this board. diff --git a/src/dev/l1.rs b/src/dev/l1.rs index c23b6c256..c09e9f6a6 100644 --- a/src/dev/l1.rs +++ b/src/dev/l1.rs @@ -192,6 +192,11 @@ impl L1 { self.state.lock().config.mac } + #[cfg(test)] + pub fn config(&self) -> L1Config { + self.state.lock().config.clone() + } + #[cfg(test)] pub fn display_line(&self) -> String { self.state.lock().display.clone() diff --git a/src/dev/mod.rs b/src/dev/mod.rs index 1a158fa99..6b48d3154 100644 --- a/src/dev/mod.rs +++ b/src/dev/mod.rs @@ -82,6 +82,6 @@ pub mod ics1890; pub mod isp12160; pub mod l1; pub mod ohci; -pub mod timekeeper; pub mod prom_image; +pub mod timekeeper; pub mod xbridge; diff --git a/src/ip34_machine.rs b/src/ip34_machine.rs new file mode 100644 index 000000000..7bb8952b2 --- /dev/null +++ b/src/ip34_machine.rs @@ -0,0 +1,731 @@ +//! The Fuel (IP34 board, IP35 software platform): an R14000 on Bedrock, +//! XBridge and the base I/O devices, with Iris's CPU thread, monitor and serial console. +//! +//! Selected with `[machine] profile = "fuel_ip34"`. Shared keys select the +//! PROM, NVRAM, memory and SCSI devices; `[fuel]` selects the serial console +//! and writable flash. `ip35::Ip35Bus` maps the processor views over the +//! same Physical memory banks. Graphics is not attached, and the IOC3 has +//! no keyboard or mouse ports. +//! +//! The Bedrock node itself, and its approximations, are in `ip35::node`. +//! +//! Board composition ported from hw/mips/sgi/sn1-node.c +//! in jamesbraid/qemu-sgi-origami. + +use crate::config::{scsi_label, FuelConsole, MachineConfig}; +use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; +use crate::cpu::mips_tlb::MipsTlb; +use crate::dev::bedrock::{Bedrock, Population}; +use crate::dev::bedrock_flash::{FirmwareRecord, Flash}; +use crate::dev::heart::{host_clock, Clock}; +use crate::dev::ioc3::{Ioc3, Ioc3Config}; +use crate::dev::ioc3_eth::Ioc3Eth; +use crate::dev::isp12160::Isp12160; +use crate::dev::l1::{L1, L1Config}; +use crate::dev::ohci::Ohci; +use crate::dev::usb_hid::{UsbKeyboard, UsbMouse}; +use crate::dev::prom_image; +use crate::dev::ps2::Ps2Controller; +use crate::dev::serial_host::{null_serial_backend, open_log, tcp_serial_backend, CiSerialBackend, LogMode, SerialBackend, TeeBackend}; +use crate::dev::timekeeper::Timekeeper; +use crate::dev::xbridge::XBridge; +use crate::pci::{BoardIrq, DevFn}; +use crate::devlog::{init_devlog, LogModule}; +use crate::gdb_stub::CpuDebug; +use crate::gfx_display::GfxDisplay; +use crate::hptimer::TimerManager; +use crate::cpu::mips_cache_shadow::R12000ShadowCache; +use crate::cpu::parts::{self, Part}; +use crate::cpu::mips_cache_v2::MipsCache; +use crate::ip35::node::{connect_cpus, Node, NodeConfig}; +use crate::ip35::{Ip35Bus, CPU_PHYS_MASK}; +use crate::machine_state::GuestState; +use crate::monitor::{Monitor, DEFAULT_PORT as DEFAULT_MONITOR_PORT}; +use crate::net::{GatewayConfig, Link, NatControl}; +use crate::perf_monitor::PerfMonitor; +use crate::scsi::{attach_from_config, ScsiBus}; +use crate::traits::{BusDevice, Device, MachineEvent, Resettable, BUS_OK}; +use crate::workstation::{register_system_controller, DiscChanger, MachineRef, MachineSlot, Workstation}; +use parking_lot::Mutex; +use std::net::{Ipv4Addr, SocketAddr}; +use std::path::{Path, PathBuf}; +use std::sync::mpsc::{Receiver, Sender}; +use std::sync::{atomic::{AtomicBool, AtomicU64, Ordering}, Arc}; + +pub struct FuelMachine { + board: Arc, + cpu: Arc, + timers: Arc, + monitor: Arc, + monitor_port: u16, + ci_serial: Option>, + console: usize, + ci: bool, + scratch: Option, + timekeeper: Arc, + nvram: String, + frames: Option>, + nat: Option<( + Arc, + (Ipv4Addr, Ipv4Addr, Ipv4Addr), + )>, + events: Option>, + slot: Option, + lifecycle: GuestState, +} + +impl FuelMachine { + pub fn new(cfg: &MachineConfig) -> Result { + cfg.validate()?; + let prom = prom_image::load(Path::new(&cfg.prom), 8 * 1024 * 1024) + .map_err(|e| format!("Fuel {e}"))?; + let timekeeper = Arc::new(Timekeeper::fuel_with_nvram_file( + Path::new(&cfg.nvram), + )?); + let board = Arc::new(FuelBoard::with_config(FuelBoardConfig { + payload: &prom, + clock: host_clock(), + flash_path: cfg.fuel.flash.as_deref().map(Path::new), + station_mac: cfg.network().mac, + timekeeper: timekeeper.clone(), + ps2_input: cfg.fuel.ps2_input, + usb_input: cfg.fuel.usb_input, + l1: L1Identity { + revision: cfg.fuel.l1_revision.as_deref().map(crate::dev::l1::parse_revision).transpose()?, + brick_serial: cfg.fuel.brick_serial.clone(), + }, + })?); + let timers = Arc::new(TimerManager::new()); + let part = cfg.cpu(); + let mut exec = board.build_cpu(part); + exec.core.set_timer_manager(timers.clone()); + exec.core.set_count_hz( + cfg.clock + .fixed_mhz + .map_or(part.count_hz(), |mhz| (mhz * 1_000_000.0) as u64) + .max(1), + ); + exec.core.count_write_samples = true; + let poll_board = board.clone(); + exec.batch_hook = Some(Box::new(move |_| poll_board.poll())); + let cpu: Arc = Arc::new(MipsCpu::<_, R12000ShadowCache>::new(*exec)); + connect_cpus("Fuel", &board.bedrock, &board.l1, &board.events, std::slice::from_ref(&cpu))?; + board.bedrock.set_timer_manager(&timers); + // After the windows above: each compile worker takes the cache's + // geometry and the CPU model's policies once, as it starts, and + // without the windows the geometry has no inline load or store path. + // Workers read plain physical code addresses through the CPU's cached + // bus, which serves them; the uncached views would decode them as Hub + // registers. + #[cfg(feature = "jitv2")] + { + let cpu_device: Arc = cpu.clone().as_device(); + let jit = cpu.jitv2(); + let mut g = jit.lock(); + g.compile_queue.set_thread_count(cfg.jitv2.threads.max(1)); + g.compile_queue.set_cpu(Arc::downgrade(&cpu_device)); + g.compile_queue.set_owner(Arc::downgrade(&jit)); + // Not under lockstep, as on the other machines: lockstep compiles + // inline on the CPU thread, and a running pool both owns the + // Codegen inline needs and deadlocks the CPU thread's flush. + #[cfg(not(feature = "jitv2_lockstep"))] + { + let stats = g.stats.clone(); + g.compile_queue.start(board.cached_bus(), stats); + } + // The compile pool keeps only a weak handle on the CPU. + std::mem::forget(cpu_device); + } + let scratch = Self::attach_scsi(cfg, &board)?; + let console = match cfg.fuel.console { + FuelConsole::L1 => 0, + FuelConsole::Ioc3A => 1, + FuelConsole::Ioc3B => 2, + }; + let ci_serial = cfg.ci.then(|| board.serial[console].clone()); + Self::connect_serial(cfg, &board, console)?; + let monitor = Monitor::new(); + monitor.register_device(init_devlog()); + monitor.register_device(Arc::new(crate::cpu::cpu_monitor::CpuMonitor::new(vec![cpu.clone()]))); + monitor.register_device(PerfMonitor::new( + cpu.running_flag(), cpu.cycles_ptr(), cpu.fasttick_count(), None, None, + )); + monitor.register_device(board.bedrock.physical().clone()); + monitor.register_device(board.l1.clone()); + monitor.register_device(board.scsi.clone()); + // `ps2 type` and friends: the keyboard and mouse from the monitor. + monitor.register_device(board.ioc3.ps2()); + cpu.register_locks(); + board.scsi.register_locks(); + board.ioc3.register_locks(); + board.xbridge.register_locks(); + board.flash.register_locks(); + board.bedrock.register_locks(); + let (mut nat, mut frames) = (None, None); + let net = GatewayConfig::from_network(&cfg.network()); + if let Some(link) = Link::with_gateway(net.clone())? { + frames = Some(link.frames()); + nat = Some((link.nat(), (net.client_ip, net.gateway_ip, net.netmask))); + board.eth.attach(link); + } + let events = board.receiver.lock().take(); + eprintln!( + "iris: Fuel IP34, {}, 512 MiB, serial console {:?}", + cfg.cpu().label(), cfg.fuel.console + ); + Ok(Self { + board, + cpu, + timers, + monitor: Arc::new(monitor), + monitor_port: cfg.monitor_port.unwrap_or(DEFAULT_MONITOR_PORT), + ci_serial, + console, + ci: cfg.ci, + scratch, + timekeeper, + nvram: cfg.nvram.clone(), + nat, + frames, + events, + slot: None, + lifecycle: GuestState::Idle, + }) + } + + fn attach_scsi(cfg: &MachineConfig, board: &FuelBoard) -> Result, String> { + let attached = attach_from_config(cfg, Some("fuel"), &|channel| { + board.scsi.bus_for_channel(channel) + .ok_or_else(|| format!("controller {channel} (a Fuel has channels 0 and 1)")) + })?; + if let Some(device) = attached.daynaports.first() { + return Err(format!("{}: kind = \"daynaport\" is not available on the Fuel", + scsi_label(device.controller, device.id))); + } + Ok(attached.scratch) + } + + fn connect_serial( + cfg: &MachineConfig, + board: &FuelBoard, + console: usize, + ) -> Result<(), String> { + let [a, b] = cfg.serial_ports(); + let ports = [cfg.fuel.l1_port.unwrap_or(8882), a, b]; + if ports[0] == ports[1] || ports[0] == ports[2] || ports[1] == ports[2] { + return Err("Fuel L1 and IOC3 serial ports must be distinct".into()); + } + let labels = ["L1", "IOC3 A", "IOC3 B"]; + let settings = ["fuel.l1_port", "serial_port_a", "serial_port_b"]; + let mut log_config = cfg.clone(); + for (index, port) in ports.into_iter().enumerate() { + log_config.serial_log = if index == console { + cfg.serial_log.clone() + } else if index == 0 { + cfg.fuel.l1_log.clone() + } else { + None + }; + let tag = format!("Fuel {}", labels[index]); + let log = open_log(&log_config, &tag, LogMode::Fresh); + let backend: Arc = if cfg.ci && index == console { + if let Some(log) = log { + board.serial[index].set_log(log); + } + board.serial[index].clone() + } else { + let mut backend = null_serial_backend(); + match tcp_serial_backend(port) { + Some((socket, actual_port)) => { + eprintln!("iris: {tag} on 127.0.0.1:{actual_port}"); + backend = socket; + } + None => eprintln!( + "iris: {tag}: port {port} unavailable; is another iris using it? ({})", + settings[index] + ), + } + match log { + Some(log) => Arc::new(TeeBackend::new(backend, log)), + None => backend, + } + }; + if index == 0 { + board.l1.set_serial_backend(backend); + } else { + board.ioc3.set_serial_backend(index - 1, backend); + } + } + Ok(()) + } + + #[cfg(test)] + pub(crate) fn connect_serial_for_test(cfg: &MachineConfig, board: &FuelBoard, console: usize) -> Result<(), String> { + Self::connect_serial(cfg, board, console) + } + +} + +impl Workstation for FuelMachine { + fn start(&mut self) { + self.lifecycle = GuestState::Starting; + self.monitor + .clone() + .start_server(format!("127.0.0.1:{}", self.monitor_port)); + self.restart_peripherals(); + if !self.ci { + self.cpu_start(); + } + } + fn stop(&mut self) { + self.cpu.clone().as_device().stop(); + self.board.stop(); + if let Err(e) = self.timekeeper.flush_nvram() { + dlog!(LogModule::Bedrock, "Fuel NVRAM: {e}"); + } + if self.lifecycle != GuestState::Idle { + self.lifecycle = GuestState::Halted; + } + } + fn reset(&mut self) { + self.stop(); + self.cpu.power_on(); + self.board.bedrock.power_on(); + self.board.l1.power_on(); + self.board.flash.power_on(); + self.board.xbridge.power_on(); + self.board.ioc3.power_on(); + self.board.eth.power_on(); + self.board.scsi.power_on(); + self.board.usb.power_on(); + self.timekeeper.power_on(); + self.restart_peripherals(); + } + fn restart_peripherals(&mut self) { + self.board.start(); + } + fn cpu_start(&self) { + self.cpu.clone().as_device().start(); + } + fn cpu_is_running(&self) -> bool { + self.cpu.clone().as_device().is_running() + } + fn machine_slot(&mut self) -> MachineSlot { + let ptr = self as *mut Self as *mut dyn Workstation; + self.slot + .get_or_insert_with(|| Arc::new(Mutex::new(MachineRef(ptr)))) + .clone() + } + fn register_system_controller(&mut self) { + let slot = self.machine_slot(); + let events = self + .events + .take() + .expect("Fuel system controller registered twice"); + register_system_controller(&self.monitor, slot, Some(events)); + } + fn monitor_addr(&self) -> Option { + self.monitor.bound_addr() + } + fn lifecycle(&self) -> GuestState { + self.lifecycle + } + fn set_lifecycle(&mut self, state: GuestState) { + self.lifecycle = state; + } + fn get_display(&self) -> Option> { + None + } + fn register_monitor_device(&self, device: Arc) { + self.monitor.register_device(device); + } + fn get_ps2(&self) -> Arc { + self.board.ioc3.ps2() + } + fn get_timer_manager(&self) -> Arc { + self.timers.clone() + } + fn get_cpu_debug(&self) -> Arc { + self.cpu.clone().debug_adapter() + } + fn fasttick_count(&self) -> Arc { + self.cpu.fasttick_count() + } + fn get_ci_serial(&self) -> Option> { + self.ci_serial.clone() + } + fn inject_serial_console(&self, bytes: &[u8]) { + self.board.serial[self.console].push_host(bytes); + } + fn as_any_mut(&mut self) -> &mut dyn std::any::Any { + self + } + fn save_nvram(&self, path: Option<&str>) -> Result { + if path.is_some_and(|p| p != self.nvram) { + return Err("Fuel NVRAM save uses the configured path".into()); + } + self.timekeeper.flush_nvram()?; + Ok(self.nvram.clone()) + } + fn scratch_path(&self) -> Option<&Path> { + self.scratch.as_deref() + } + fn disc_changer(&self) -> Option> { + Some(self.board.scsi.clone() as Arc) + } + fn pending_chd_sync_count(&self) -> usize { + (0..2).map(|channel| self.board.scsi.bus_for_channel(channel).unwrap().pending_chd_sync_count()).sum() + } + fn flush_disks(&self) -> std::io::Result<()> { + self.board.flush_disks() + } + fn sync_chd_disks( + &self, + only: Option, + progress: &mut dyn FnMut(usize, usize, f32), + cancel: &dyn Fn() -> bool, + ) -> std::io::Result { + let buses: Vec<_> = (0..2).map(|channel| self.board.scsi.bus_for_channel(channel).unwrap()).collect(); + ScsiBus::sync_chd_disks(&buses, only, progress, cancel) + } + fn net_frames(&self) -> Option> { self.frames.clone() } + fn nat_control(&self) -> Option> { + self.nat.as_ref().map(|(n, _)| n.clone()) + } + fn nat_expected(&self) -> Option<(Ipv4Addr, Ipv4Addr, Ipv4Addr)> { + self.nat.as_ref().map(|(_, a)| *a) + } +} + +impl Drop for FuelMachine { + fn drop(&mut self) { + if let Some(slot) = self.slot.take() { + slot.lock().0 = MachineRef::null(); + } + self.stop(); + self.monitor.shutdown(); + } +} + +/// The processors a Fuel can be built with, the default first: the R14000A +/// the Fuel ships with. +pub const CPUS: &[&Part] = &[&parts::R14000_500]; + +/// The cache type `with_cpu!` builds for every entry of [`CPUS`]; a test pins +/// it. The board's devices are not generic over the processor, so it names the +/// type once here. +pub(crate) type FuelCpu = MipsExecutor; + +const FUEL_BOARD_ID: u64 = 0x4004; +/// Two 256 MiB banks, locational banks 0 and 1. +const FUEL_BANKS_MIB: [u32; 2] = [256, 256]; + + +/// Build the Fuel's processor, `part` being the one the config resolved, and +/// set the PROM boot-mode inputs. +pub(crate) fn build_cpu(bus: Arc, part: &Part) -> Box { + let extent = crate::dev::bedrock::RamExtent::new(&FUEL_BANKS_MIB); + let cfg = MipsCpuConfig::for_model::(Arc::new(crate::ip35::memory_map(&extent))) + .with_prid(part.prid()); + let mut cpu = Box::new(MipsExecutor::new( + bus as Arc, + MipsTlb::new(cfg.tlb_entries), + &cfg, + )); + cpu.phys_mask = CPU_PHYS_MASK; + // The flash record's mode word selects these boot-mode inputs. A + // mismatch enters the PROM's error routine before GP is initialized. + cpu.core.cp0_config = FirmwareRecord::FUEL.reset_config0(); + cpu +} + +/// Shared device composition for a single-processor Fuel. +/// Graphics is absent; the serial workstation shares these base I/O devices. +pub(crate) struct FuelBoard { + pub bus: Arc, + /// Cacheable memory at plain physical addresses: what the cache fills + /// from, and the one bus the JIT and its compile pool read code through. + cached: Arc, + pub bedrock: Arc, + pub l1: Arc, + pub flash: Arc, + /// The node's resets and power events, and their receiver until the + /// system controller takes it. + events: Sender, + receiver: Mutex>>, + pub xbridge: Arc, + pub ioc3: Arc, + pub eth: Arc, + pub scsi: Arc, + pub usb: Arc, + pub serial: [Arc; 3], + #[cfg(test)] + pub irq: Arc, + clock: Clock, + last_clock: AtomicU64, + running: AtomicBool, +} + +pub(crate) struct FuelBoardConfig<'a> { + pub payload: &'a [u8], + pub clock: Clock, + pub flash_path: Option<&'a Path>, + pub station_mac: [u8; 6], + pub timekeeper: Arc, + /// The IOC3's PS/2 keyboard and mouse controller. + pub ps2_input: bool, + /// A USB keyboard and mouse on the OHCI root ports. + pub usb_input: bool, + /// Replacements for the L1's reported identity; unset keeps its defaults. + pub l1: L1Identity, +} + +/// What a Fuel config may replace in the L1's identity replies. +#[derive(Clone, Debug, Default)] +pub(crate) struct L1Identity { + pub revision: Option<[u32; 3]>, + pub brick_serial: Option, +} + +impl FuelBoard { + /// Flush the disks on both channels of the storage controller. + pub fn flush_disks(&self) -> std::io::Result<()> { + (0..2).try_for_each(|channel| self.scsi.bus_for_channel(channel).unwrap().flush()) + } + + #[cfg(test)] + pub fn new(payload: &[u8], clock: Clock) -> Result { + Self::with_config(FuelBoardConfig { + payload, clock, flash_path: None, + station_mac: L1Config::default().mac, + timekeeper: Arc::new(Timekeeper::fuel()?), + ps2_input: true, + usb_input: false, + l1: L1Identity::default(), + }) + } + + pub fn with_config(config: FuelBoardConfig<'_>) -> Result { + let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity } = config; + let mut l1_config = L1Config { mac: station_mac, ..Default::default() }; + if let Some(revision) = l1_identity.revision { + l1_config.revision = Some(revision); + } + if let Some(serial) = l1_identity.brick_serial { + l1_config.brick_serial = Some(serial); + } + let node = Node::build(NodeConfig { + name: "Fuel", + payload, + clock: clock.clone(), + flash_path, + banks_mib: &FUEL_BANKS_MIB, + processors: Population::SINGLE, + firmware: FirmwareRecord::FUEL, + board_id: FUEL_BOARD_ID, + l1: l1_config, + })?; + let Node { bus, bedrock, l1, flash, cached, events, receiver } = node; + #[cfg(test)] + let irq = Arc::new(PendingIrq::default()); + #[cfg(test)] + bedrock.set_cpu_interrupts(0, &irq.0); + let serial = std::array::from_fn(|_| Arc::new(CiSerialBackend::new())); + l1.set_serial_backend(serial[0].clone()); + let xbridge = XBridge::new(bedrock.dma_view()); + for widget in [0, 10, 14, 15] { + bedrock.connect_widget(widget, xbridge.widget(widget).unwrap()); + } + let weak = Arc::downgrade(&bedrock); + xbridge.set_interrupt_sender(Arc::new(move |packet| { + if packet.widget != 10 { + return false; + } + weak.upgrade().is_some_and(|bedrock| { + bedrock.inbound_write( + packet.source_widget, + None, + packet.address, + 8, + packet.vector as u64 | if packet.level { 1 << 8 } else { 0 }, + ) == BUS_OK + }) + })); + let weak = Arc::downgrade(&bedrock); + xbridge.set_dma_sender(Arc::new(move |packet, bytes| { + if packet.widget != 10 || packet.half_read_requested || packet.half_write_requested { + dlog!(LogModule::Bedrock, + "Fuel: unsupported XBridge DMA destination or halfword placement {packet:?}" + ); + return false; + } + let Some(bedrock) = weak.upgrade() else { + return false; + }; + // Preserve the width of XBridge's naturally aligned packet legs, + // including Hub registers whose narrower accesses are refused. + let size = bytes.len(); + if matches!(size, 1 | 2 | 4 | 8) + && packet.address & (size as u64 - 1) == 0 + { + if packet.write { + let value = bytes.iter().fold(0u64, |value, byte| (value << 8) | *byte as u64); + return bedrock.inbound_write( + packet.source_widget, Some(packet.slot), packet.address, size as u8, value, + ) == BUS_OK; + } + let value = bedrock.inbound_read( + packet.source_widget, Some(packet.slot), packet.address, size as u8, + ); + if value.status != BUS_OK { + return false; + } + bytes.copy_from_slice(&value.data.to_be_bytes()[8 - size..]); + return true; + } + for (i, byte) in bytes.iter_mut().enumerate() { + let a = packet.address + i as u64; + if packet.write { + if bedrock.inbound_write(packet.source_widget, Some(packet.slot), a, 1, *byte as u64) + != BUS_OK + { + return false; + } + } else { + let value = bedrock.inbound_read(packet.source_widget, Some(packet.slot), a, 1); + if value.status != BUS_OK { + return false; + } + *byte = value.data as u8; + } + } + true + })); + let mac = l1.station_mac(); + let mut ioc3_config = Ioc3Config::fuel(mac); + ioc3_config.keyboard_mouse = ps2_input; + let ioc3 = Arc::new(Ioc3::new(ioc3_config)); + let eth = Ioc3Eth::new_fuel(mac); + ioc3.attach_eth(eth.clone()); + ioc3.attach_timekeeper(timekeeper); + ioc3.set_serial_backend(0, serial[1].clone()); + ioc3.set_serial_backend(1, serial[2].clone()); + let scsi = Arc::new(Isp12160::new( + Isp12160::new_bus(), + Isp12160::new_bus(), + )); + let pci = xbridge.bus(15); + let slot = |device| DevFn { device, function: 0 }; + pci.attach(slot(1), scsi.clone()); + pci.attach(slot(4), ioc3.clone()); + let usb = Ohci::new(); + usb.set_clock(clock.clone()); + if usb_input { + // Host input reaches the USB devices through the same controller + // object the GUI, VNC and CI already drive; each device drops + // what its guest has not enabled. + let keyboard = UsbKeyboard::new(); + let mouse = UsbMouse::new(); + usb.attach(0, keyboard.clone())?; + usb.attach(1, mouse.clone())?; + ioc3.ps2().add_sink(keyboard); + ioc3.ps2().add_sink(mouse); + } + pci.attach(slot(5), usb.clone()); + ioc3.connect_superio_interrupt(pci.board_irq(BoardIrq(0))); + let reset_ioc3 = ioc3.clone(); + let reset_eth = eth.clone(); + let reset_scsi = scsi.clone(); + let reset_usb = usb.clone(); + xbridge.set_reset_output(Arc::new(move |port| { + if port == 15 { + reset_ioc3.power_on(); + reset_eth.power_on(); + reset_scsi.power_on(); + reset_usb.power_on(); + } + })); + Ok(Self { + bus, + cached, + bedrock, + l1, + flash, + events, + receiver: Mutex::new(Some(receiver)), + xbridge, + ioc3, + eth, + scsi, + usb, + serial, + #[cfg(test)] + irq, + last_clock: AtomicU64::new(clock()), + clock, + running: AtomicBool::new(false), + }) + } + + pub fn cached_bus(&self) -> Arc { + self.cached.clone() + } + + pub fn build_cpu(&self, part: &Part) -> Box { + let mut cpu = build_cpu(self.bus.clone(), part); + cpu.cache = R12000ShadowCache::new(self.cached_bus()); + cpu.set_cached_bus(self.cached_bus()); + cpu + } + + pub fn start(&self) { + self.last_clock + .store((self.clock)(), Ordering::Release); + self.running + .store(true, Ordering::Release); + self.bedrock.start(); + self.l1.start(); + self.ioc3.start(); + self.eth.start(); + self.scsi.start(); + self.usb.start(); + } + pub fn poll(&self) { + if self.running.load(Ordering::Acquire) { + let now = (self.clock)(); + let previous = self + .last_clock + .swap(now, Ordering::AcqRel); + self.flash.advance_ns(now.saturating_sub(previous)); + self.usb.sync_clock(); + } + } + pub fn stop(&self) { + self.poll(); + self.running + .store(false, Ordering::Release); + self.usb.stop(); + self.scsi.stop(); + self.eth.stop(); + self.ioc3.stop(); + self.l1.stop(); + self.bedrock.stop(); + } +} + +impl Drop for FuelBoard { + fn drop(&mut self) { + self.stop(); + } +} + +/// CPU 0's interrupt word, for tests that build the board without a CPU. +#[cfg(test)] +#[derive(Default)] +pub(crate) struct PendingIrq(pub(crate) AtomicU64); +#[cfg(test)] +impl PendingIrq { + pub fn bits(&self) -> u64 { + self.0.load(Ordering::Acquire) + } +} diff --git a/src/ip35.rs b/src/ip35.rs index 432d8f497..44875f9d5 100644 --- a/src/ip35.rs +++ b/src/ip35.rs @@ -1,18 +1,61 @@ -//! The IP35 processor bus: windows over small MMIO apertures. +//! The IP35 processor bus: Bedrock's four XKPHYS access-space views. //! -//! Maps replace whole 4 KiB pages as Physical does, while each endpoint still -//! rejects padding and enforces its cell permission. +//! Raw address bits 58:57 select Hub-special, I/O, fetch operations or uncached +//! RAM. Each view has a small fixed set of MMIO apertures, decoded by sorted +//! windows rather than a full physical-address table. Maps replace whole 4 KiB +//! pages as Physical does, while each endpoint still rejects padding and +//! enforces its cell permission. +//! +//! Cached RAM alone uses Physical, including its direct bank pointers, ppmem +//! and generation mappings. The MMIO views and register sub-bus need none of +//! those RAM facilities. Giving each its own full table would consume several +//! entries in the process-wide fault-handler registry per machine. //! //! Ported from hw/mips/sgi/sn1-node.c in jamesbraid/qemu-sgi-origami. +use crate::dev::bedrock::RamExtent; use crate::physical::Physical; use crate::traits::*; #[cfg(feature = "jitv2")] -use std::sync::atomic::AtomicU64; +use std::sync::{atomic::AtomicU64, OnceLock}; use std::sync::Arc; +pub(crate) const CPU_PHYS_MASK: u64 = ((1u64 << 40) - 1) | (3u64 << 57); const PA_MASK: u64 = (1u64 << 40) - 1; +/// Where a Bedrock node has memory: each 1 GiB software slot `extent` says +/// can hold RAM, as far as its largest bank reaches, and the reset PROM. +/// Which bank lands in which slot of a group of four depends on how the PROM +/// programs the memory controller; the Fuel's two 256 MiB banks can be in any +/// of the first four. Uncached fetches of the PROM's own code land in its +/// range. A slot 0 larger than 508 MiB reaches the PROM's range, which the +/// node maps over it, so its RAM is listed on either side. +pub fn memory_map(extent: &RamExtent) -> crate::memory_map::MemoryMap { + use crate::dev::bedrock::{BANK_SPAN, SLOTS}; + use crate::memory_map::{MemKind, MemRange, MemoryMap}; + const NAMES: [&str; SLOTS] = [ + "ram slot 0", "ram slot 1", "ram slot 2", "ram slot 3", + "ram slot 4", "ram slot 5", "ram slot 6", "ram slot 7", + ]; + const PROM: (u64, u64) = (0x1fc0_0000, 0x2000_0000); + let mut ranges = vec![MemRange::new("prom", MemKind::Rom, PROM.0, PROM.1)]; + for (slot, &name) in NAMES.iter().enumerate() { + let (base, end) = (slot as u64 * BANK_SPAN, slot as u64 * BANK_SPAN + extent.prefix(slot)); + if base == end { + continue; + } + if base < PROM.1 && end > PROM.0 { + ranges.push(MemRange::new(name, MemKind::Ram, base, PROM.0)); + if end > PROM.1 { + ranges.push(MemRange::new("ram slot 0 above the prom", MemKind::Ram, PROM.1, end)); + } + } else { + ranges.push(MemRange::new(name, MemKind::Ram, base, end)); + } + } + MemoryMap::new(ranges) +} + struct Unclaimed; impl BusDevice for Unclaimed {} @@ -177,3 +220,148 @@ impl BusDevice for SparseWindowMap { sparse_access!(read64, write64, u64, BusRead64); } +/// The four uncached processor views decode MMIO separately from cached RAM. +pub struct Ip35Bus { + pub phys: Arc, + views: [SparseWindowMap; 4], +} +impl Ip35Bus { + pub fn new(phys: Arc) -> Self { + phys.set_default(Arc::new(Unclaimed), "IP35 unclaimed cached address"); + Self { + phys, + views: std::array::from_fn(|_| SparseWindowMap::default()), + } + } + /// Raw XKPHYS bits 58:57 select Hub-special, I/O, fetchop, or uncached RAM. + pub fn map(&mut self, raw_uc: u8, base: u64, size: u64, device: Arc) { + assert!(raw_uc < 4); + self.views[raw_uc as usize].map(base, size, device); + } + pub fn set_fallback(&mut self, raw_uc: u8, end: u64, device: Arc) { + self.map(raw_uc, 0, end, device); + } + fn target(&self, a: u64, n: u64) -> Option<(&SparseWindowMap, u64)> { + let pa = plain_address(a, n)?; + Some((&self.views[(a >> 57) as usize], pa)) + } + /// Cached RAM, with `plain_memory` answering wherever `extent` says + /// fitted RAM can be. + pub fn cached_view(&self, plain_memory: Arc, extent: RamExtent) -> Arc { + Arc::new(CachedPhysical { + phys: self.phys.clone(), + plain_memory, + extent, + }) + } +} +fn plain_address(a: u64, n: u64) -> Option { + if a & !CPU_PHYS_MASK != 0 { + return None; + } + let pa = a & PA_MASK; + (pa.checked_add(n)? <= PA_MASK + 1).then_some(pa) +} +struct CachedPhysical { + phys: Arc, + plain_memory: Arc, + extent: RamExtent, +} +impl CachedPhysical { + fn memory_target(&self, a: u64, n: u64) -> Option<(&dyn BusDevice, u64)> { + self.target(a, n).map(|(physical, pa)| { + // MD owns every guest write to a possible fitted-RAM prefix, so + // all paths update its stable logical generation counters. PROM + // and other devices retain the cached Physical connection. + let memory: &dyn BusDevice = if self.extent.contains(pa) { + self.plain_memory.as_ref() + } else { physical }; + (memory, pa) + }) + } + fn target(&self, a: u64, n: u64) -> Option<(&Physical, u64)> { + // UC selectors belong to the uncached SysAd connection. Reject them + // here rather than turning a malformed cached request into an alias. + if a & !PA_MASK != 0 || a & (n - 1) != 0 { + return None; + } + Some((self.phys.as_ref(), plain_address(a, n)?)) + } +} +macro_rules! processor_access { + ($r:ident,$w:ident,$t:ty,$out:ident,$n:expr,$writes:ident) => { + fn $r(&self, a: u64) -> $out { + self + .target(a, $n) + .map(|(v, pa)| v.$r(pa)) + .unwrap_or_else($out::err) + } + fn $w(&self, a: u64, v: $t) -> u32 { + self + .$writes(a, $n) + .map(|(view, pa)| view.$w(pa, v)) + .unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for Ip35Bus { + fn write64_masked(&self, a: u64, val: u64, mask: u64) -> u32 { + if a & 7 != 0 { + return BUS_ERR; + } + self.target(a, 8) + .map(|(view, pa)| view.write64_masked(pa, val, mask)) + .unwrap_or(BUS_ERR) + } + fn write_narrow_from(&self, a: u64, size: usize, gpr: u64) -> u32 { + self.target(a, size as u64) + .map(|(view, pa)| view.write_narrow_from(pa, size, gpr)) + .unwrap_or(BUS_ERR) + } + processor_access!(read8, write8, u8, BusRead8, 1, target); + processor_access!(read16, write16, u16, BusRead16, 2, target); + processor_access!(read32, write32, u32, BusRead32, 4, target); + processor_access!(read64, write64, u64, BusRead64, 8, target); +} +impl BusDevice for CachedPhysical { + fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { + self.memory_target(address, 8).map(|(view, pa)| view.write64_masked(pa, value, mask)) + .unwrap_or(BUS_ERR) + } + fn mem_ptr(&self, a: u64) -> Option<*const u64> { + self.target(a, 8).and_then(|(view, pa)| view.mem_ptr(pa)) + } + #[cfg(feature = "jitv2")] + fn gen_ptr(&self, a: u64) -> *const AtomicU64 { + self.target(a, 1).map(|(view, pa)| { + let generation = self.plain_memory.gen_ptr(pa); + if generation.is_null() { view.gen_ptr(pa) } else { generation } + }).unwrap_or(std::ptr::null()) + } + fn read_block(&self, a: u64, out: &mut [u64]) -> u32 { + self.target(a, 8) + .map(|(view, pa)| view.read_block(pa, out)).unwrap_or(BUS_ERR) + } + fn write_block(&self, a: u64, data: &[u64]) -> u32 { + self.memory_target(a, 8) + .map(|(view, pa)| view.write_block(pa, data)).unwrap_or(BUS_ERR) + } + processor_access!(read8, write8, u8, BusRead8, 1, memory_target); + processor_access!(read16, write16, u16, BusRead16, 2, memory_target); + processor_access!(read32, write32, u32, BusRead32, 4, memory_target); + processor_access!(read64, write64, u64, BusRead64, 8, memory_target); + fn dma_read64(&self, a: u64) -> BusRead64 { + plain_address(a, 8) + .map(|pa| self.plain_memory.dma_read64(pa)) + .unwrap_or_else(BusRead64::err) + } + fn dma_write64(&self, a: u64, v: u64) -> u32 { + plain_address(a, 8) + .map(|pa| self.plain_memory.dma_write64(pa, v)) + .unwrap_or(BUS_ERR) + } +} + +pub(crate) mod node; +#[cfg(test)] +mod tests; diff --git a/src/ip35/node.rs b/src/ip35/node.rs new file mode 100644 index 000000000..d92ba1225 --- /dev/null +++ b/src/ip35/node.rs @@ -0,0 +1,276 @@ +//! The Bedrock node every IP35 board shares: the processor bus with its four +//! views, the Bedrock chip with its memory, the L1, the boot flash and the +//! Hub-special page, wired together and to the board's processors. A board +//! names its inputs in [`NodeConfig`] and adds its own crossbar widgets. +//! +//! # What is approximated here +//! +//! The two Hub-special head windows return zero on reads and acknowledge +//! writes without side effects. PROM 6.210 writes the 64-bit value 0x2000 +//! at relative offsets 0 and 0x40 before DRAM discovery. Their hardware +//! meaning is unknown; Bedrock logging records accesses to these windows. +//! +//! Node composition ported from hw/mips/sgi/sn1-node.c in +//! jamesbraid/qemu-sgi-origami. + +use super::{map_device, Ip35Bus, Window}; +use crate::cpu::mips_exec::CpuDevice; +use crate::dev::bedrock::{Bedrock, BedrockConfig, Form, Population}; +use crate::dev::bedrock_flash::{FirmwareRecord, Flash, PART_SIZE}; +use crate::dev::heart::Clock; +use crate::dev::l1::{L1Config, L1Event, L1}; +use crate::devlog::LogModule; +use crate::ppmem::devmap::SLOT_SHIFT; +use crate::traits::{BusDevice, BusRead16, BusRead32, BusRead64, BusRead8, MachineEvent, Signal, BUS_ERR, BUS_OK}; +use parking_lot::Mutex; +use std::path::Path; +use std::sync::mpsc::{channel, Receiver, Sender}; +use std::sync::Arc; + +/// What a board gives its Bedrock node. +pub(crate) struct NodeConfig<'a> { + /// The board's name in logs and map names. + pub name: &'static str, + pub payload: &'a [u8], + pub clock: Clock, + pub flash_path: Option<&'a Path>, + /// The memory banks in MiB, locational bank 0 upward. + pub banks_mib: &'a [u32], + /// The processor slots the board fits. + pub processors: Population, + pub firmware: FirmwareRecord, + /// The word the board reads back at the Hub-special board-id window. + pub board_id: u64, + pub l1: L1Config, +} + +/// One Bedrock node as the IP35 boards share it. +pub(crate) struct Node { + pub bus: Arc, + pub bedrock: Arc, + pub l1: Arc, + pub flash: Arc, + /// Cacheable memory at plain physical addresses: what a cache fills + /// from, and the one bus the JIT and its compile pool read code through. + pub cached: Arc, + /// The node's resets and power events, for the system controller. + pub events: Sender, + pub receiver: Receiver, +} + +impl Node { + pub fn build(config: NodeConfig<'_>) -> Result { + let NodeConfig { name, payload, clock, flash_path, banks_mib, processors, firmware, board_id, l1 } = config; + let (events, receiver) = channel(); + let bedrock = Bedrock::new(BedrockConfig { + name, + clock: clock.clone(), + processors, + banks_mib, + events: Some(events.clone()), + })?; + let phys = bedrock.physical().clone(); + let l1 = Arc::new(L1::new(l1)?); + l1.set_irq(bedrock.uart_irq()); + let flash = Arc::new(match flash_path { + Some(path) => Flash::with_file(payload, path, firmware), + None => Flash::new(payload, firmware), + }.map_err(|e| e.to_string())?); + let mut bus = Ip35Bus::new(phys.clone()); + bedrock.rebuild_memory(); + bus.set_fallback(1, 1 << 32, bedrock.pio()); + // A processor on PI1 finds its own interface at the local form's + // offset 0 (OpenBSD expects the local form to select the requester's + // interface); with processors on PI0 alone the two forms are one. + bus.map(1, 0x0100_0000, 0x80_0000, bedrock.registers(Form::Local)); + bus.map(1, 0x0180_0000, 0x80_0000, bedrock.registers(Form::Node)); + // Cached RAM uses MD's direct bank mappings; diagnostic views stay separate. + bus.map(3, 0, 1 << 33, bedrock.uncached_view()); + // Hub-special low memory: its first 256 MiB is the node memory view. + bus.map(0, 0, 0x1000_0000, bedrock.uncached_view()); + let prom = Arc::new(flash.array_view(0, PART_SIZE).map_err(|e| e.to_string())?); + let boot = Arc::new(flash.boot_view()); + for base in [0x1800_0000, 0x3800_0000] { + bus.map(0, base, 0x800_0000, boot.clone()); + } + // Reset execution always sees stored bytes, independently of commands. + bus.map(0, 0x1fc0_0000, 0x40_0000, prom.clone()); + if !payload.is_empty() { + map_device(&phys, 0x1fc0_0000, payload.len() as u64, prom.clone(), &format!("{name} cached reset PROM")); + } + let cell = bedrock.clone(); + let permission: Arc bool + Send + Sync> = Arc::new(move || cell.cell_permits()); + let leds = Arc::new(Leds { bedrock: bedrock.clone(), values: Mutex::new([0; 4]) }); + // The three windows occupy the same processor register page. + let special = Arc::new(HubSpecialPage { + windows: [ + Window::new(0, 0x80, Arc::new(HubHead(name))).permitted(permission.clone()), + Window::new(0x80, 0x40, l1.clone()).permitted(permission.clone()), + Window::new(0xc0, 0x20, leds).permitted(permission), + ], + }); + for base in [0x1000_0000, 0x3000_0000] { + bus.map(0, base, 1 << SLOT_SHIFT, special.clone()); + bus.map(0, base + 0x400_0000, 8, Arc::new(BoardId { bedrock: bedrock.clone(), word: board_id })); + } + bus.map(0, 0x1_c000_0000, 1 << 30, bedrock.directory_bus()); + bus.map(0, 0x1_4000_0000, 1 << 30, bedrock.ecc_bus()); + bus.map(0, 0x1_0000_0000, 1 << 30, bedrock.ecc_scrub_bus()); + bus.map(2, 0, 1 << 33, bedrock.fetchop_bus()); + let bus = Arc::new(bus); + let cached = bus.cached_view(bedrock.dma_view(), bedrock.ram_extent()); + Ok(Self { bus, bedrock, l1, flash, cached, events, receiver }) + } +} + +/// Give a node's memory and interrupts to its processors, CPU 0 first, each +/// in the slot the board's population gives it: a cached view of memory for +/// every one and its IP2..IP6 from its PI slot. The L1's power events reach +/// the system controller on `events`, and its soft reset every processor. +pub(crate) fn connect_cpus( + name: &'static str, + bedrock: &Bedrock, + l1: &L1, + events: &Sender, + cpus: &[Arc], +) -> Result<(), String> { + if let Some(space) = bedrock.physical().ppmem_space() { + // Each CPU reads memory through a cached view of its own. + for cpu in cpus { + let v = space + .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()]) + .map_err(|e| format!("ppmem: a CPU's cached view: {e}"))?; + unsafe { + cpu.set_tcache_window(space.view_base(v)); + cpu.set_cached_view(space, v); + #[cfg(feature = "jitv2")] + cpu.set_tcache_gen_window(bedrock.generation_base()); + } + } + bedrock.rebuild_memory(); + } + for (n, cpu) in cpus.iter().enumerate() { + bedrock.set_cpu_interrupts(n, cpu.interrupts_ptr()); + } + let tx = events.clone(); + let cpus: Vec<_> = cpus.iter().map(Arc::downgrade).collect(); + l1.set_event_handler(Arc::new(move |event| match event { + L1Event::SoftReset => { + for cpu in cpus.iter().filter_map(|cpu| cpu.upgrade()) { + cpu.as_device().signal(Signal::Reset(true)); + } + } + L1Event::Nmi => dlog!(LogModule::L1, "{name}: NMI entry is not implemented by the CPU"), + _ => { + dlog!(LogModule::L1, "{name}: L1 event {event:?}"); + if let Some(action) = machine_action(event) { + let _ = tx.send(action); + } + } + })); + Ok(()) +} + +/// What the machine does for an L1 power or reset event. A power change +/// alone does nothing: the shutdown or restart the L1 reports with it +/// carries the action. So a power cycle, which the PROM may ask for to retry +/// a processor, restarts the node rather than powering the machine off. +fn machine_action(event: L1Event) -> Option { + match event { + L1Event::Shutdown => Some(MachineEvent::PowerOff), + L1Event::Reset => Some(MachineEvent::HardReset), + L1Event::Power(_) | L1Event::SoftReset | L1Event::Nmi => None, + } +} + +struct HubSpecialPage { + windows: [Window; 3], +} +impl HubSpecialPage { + fn target(&self, a: u64, n: u64) -> Option<&Window> { + self.windows.iter().find(|window| window.contains(a, n)) + } +} +macro_rules! hub_page_access { + ($r:ident,$w:ident,$t:ty,$out:ident,$n:expr) => { + fn $r(&self, a: u64) -> $out { + self.target(a, $n).map(|window| window.$r(a)).unwrap_or_else($out::err) + } + fn $w(&self, a: u64, value: $t) -> u32 { + self.target(a, $n).map(|window| window.$w(a, value)).unwrap_or(BUS_ERR) + } + }; +} +impl BusDevice for HubSpecialPage { + hub_page_access!(read8, write8, u8, BusRead8, 1); + hub_page_access!(read16, write16, u16, BusRead16, 2); + hub_page_access!(read32, write32, u32, BusRead32, 4); + hub_page_access!(read64, write64, u64, BusRead64, 8); +} + +struct BoardId { + bedrock: Arc, + word: u64, +} +impl BusDevice for BoardId { + fn read64(&self, a: u64) -> BusRead64 { + if a == 0 && self.bedrock.cell_permits() { + BusRead64::ok(self.word) + } else { + BusRead64::err() + } + } +} +// Compatibility head windows: their register contract is not established. +// PROM 6.210 writes 0x2000 at offsets 0 and 0x40 before DRAM discovery. +// Retain the existing zero-read/acknowledged-write model, with accesses +// observable through the Bedrock log, until the register behavior is known. +/// The board name for the log. +struct HubHead(&'static str); +macro_rules! head_access { + ($r:ident,$w:ident,$ty:ty,$result:ident) => { + fn $r(&self, a: u64) -> $result { + dlog!(LogModule::Bedrock, + "{}: unidentified Hub-special read {} bytes at {a:#x}", + self.0, + std::mem::size_of::<$ty>() + ); + $result::ok(0) + } + fn $w(&self, a: u64, v: $ty) -> u32 { + dlog!(LogModule::Bedrock, + "{}: unidentified Hub-special write {} bytes at {a:#x} value {v:#x}", + self.0, + std::mem::size_of::<$ty>() + ); + BUS_OK + } + }; +} +impl BusDevice for HubHead { + head_access!(read8, write8, u8, BusRead8); + head_access!(read16, write16, u16, BusRead16); + head_access!(read32, write32, u32, BusRead32); + head_access!(read64, write64, u64, BusRead64); +} +struct Leds { + bedrock: Arc, + values: Mutex<[u8; 4]>, +} +impl BusDevice for Leds { + fn read64(&self, a: u64) -> BusRead64 { + if a < 32 && a & 7 == 0 && self.bedrock.cell_permits() { + BusRead64::ok(self.values.lock()[a as usize / 8] as u64) + } else { + BusRead64::err() + } + } + fn write64(&self, a: u64, v: u64) -> u32 { + if a < 32 && a & 7 == 0 && self.bedrock.cell_permits() { + self.values.lock()[a as usize / 8] = v as u8; + BUS_OK + } else { + BUS_ERR + } + } +} diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs new file mode 100644 index 000000000..796f84fd1 --- /dev/null +++ b/src/ip35/tests.rs @@ -0,0 +1,638 @@ +use super::*; +use crate::dev::bedrock::{reg as bedrock_reg, Form}; +use crate::traits::{BusDevice, BUS_OK}; + +/// A Bedrock unit's registers, at its base in the node's register window. +struct Unit(Arc, u64); +impl Unit { + fn read64(&self, a: u64) -> BusRead64 { + self.0.read64(self.1 + a) + } + fn write64(&self, a: u64, v: u64) -> u32 { + self.0.write64(self.1 + a, v) + } +} +fn md_regs(board: &crate::ip34_machine::FuelBoard) -> Unit { + Unit(board.bedrock.registers(Form::Node), bedrock_reg::MD_BASE) +} +fn pi_regs(board: &crate::ip34_machine::FuelBoard, interface: u64) -> Unit { + Unit(board.bedrock.registers(Form::Node), interface * bedrock_reg::PI1_BASE) +} + +struct Word(u64); +impl BusDevice for Word { + fn read64(&self, _: u64) -> BusRead64 { + BusRead64::ok(self.0) + } +} + +#[derive(Clone, Debug, PartialEq, Eq)] +enum NarrowWriteCall { + Byte(u64, u8), + Half(u64, u16), + From(usize, u64, u64), +} +/// Records every narrow write it sees. With `sees_register` it overrides +/// `write_narrow_from` and keeps the whole source register, as Bedrock's ECC +/// view does; otherwise it takes the trait's default. +#[derive(Default)] +struct NarrowWriteSpy { + sees_register: bool, + calls: std::sync::Mutex>, +} +impl NarrowWriteSpy { + fn new(sees_register: bool) -> Arc { + Arc::new(Self { sees_register, ..Default::default() }) + } + fn calls(&self) -> Vec { + self.calls.lock().unwrap().clone() + } +} +impl BusDevice for NarrowWriteSpy { + fn write8(&self, address: u64, value: u8) -> u32 { + self.calls.lock().unwrap().push(NarrowWriteCall::Byte(address, value)); + BUS_OK + } + fn write16(&self, address: u64, value: u16) -> u32 { + self.calls.lock().unwrap().push(NarrowWriteCall::Half(address, value)); + BUS_OK + } + fn write_narrow_from(&self, address: u64, size: usize, gpr: u64) -> u32 { + if !self.sees_register { + return if size == 1 { self.write8(address, gpr as u8) } else { self.write16(address, gpr as u16) }; + } + self.calls.lock().unwrap().push(NarrowWriteCall::From(size, address, gpr)); + BUS_OK + } +} + +/// An uncached SB or SH reaches a device with the whole source register, at the +/// device-local address; only a device that overrides the call sees more than +/// the low bits, and a window outside every map is a bus error. +/// The Fuel's processor is the R14000 at revision 2.3 and 500 MHz, built as +/// the R12000 type with its PRId set over it; the Octane's own part is +/// revision 2.4. +#[test] +fn the_fuel_runs_the_r14000_at_revision_2_3() { + use crate::cpu::mips_cache_shadow::R12000ShadowCache; + use crate::cpu::mips_cache_v2::CpuModel as Model; + use crate::cpu::parts; + use crate::ip34_machine::CPUS; + assert_eq!(CPUS, &[&parts::R14000_500]); + let part = CPUS[0]; + assert_eq!((part.mhz, part.rev, part.l2_kb), (500, (2, 3), 2048)); + assert_eq!(part.prid(), 0x0f23); + assert_eq!(part.count_hz(), 250_000_000); + // `with_cpu!` builds the type `FuelCpu` names for this part. + let built = crate::with_cpu!(part, true, false, C => std::any::type_name::()); + assert_eq!(built, std::any::type_name::()); + // Typed R12000, it takes the R10000 family's behaviour: reserved + // secondary-tag bits read as zero, and a refused uncached store posts. + assert!(::R10K_FAMILY); + assert_ne!(::SECONDARY_TAG_MASK, u64::MAX); + assert!(::POSTS_UNCACHED_STORES); + assert!(::PARTIAL_LWR_KEEPS_UPPER); + // The executor carries the chip's PRId. + // The executor is large; the default test stack is not. + std::thread::Builder::new() + .stack_size(64 << 20) + .spawn(|| { + let cpu = crate::ip34_machine::build_cpu(Arc::new(Ip35Bus::new(Physical::new(Vec::new()))), crate::ip34_machine::CPUS[0]); + assert_eq!(cpu.core.cp0_prid, 0x0f23); + }) + .unwrap() + .join() + .expect("thread panicked"); +} + +#[test] +fn narrow_stores_carry_the_register_to_devices_that_ask_for_it() { + let mut bus = Ip35Bus::new(Physical::new(Vec::new())); + let plain = NarrowWriteSpy::new(false); + let ecc = NarrowWriteSpy::new(true); + bus.map(3, 0x1000, 0x1000, plain.clone()); + bus.map(3, 0x2000, 0x1000, ecc.clone()); + + let source = 0x1122_3344_5566_77a5u64; + let uc = 3u64 << 57; + assert_eq!(bus.write_narrow_from(uc | 0x1003, 1, source), BUS_OK); + assert_eq!(bus.write_narrow_from(uc | 0x1004, 2, source), BUS_OK); + assert_eq!( + plain.calls(), + vec![NarrowWriteCall::Byte(3, source as u8), NarrowWriteCall::Half(4, source as u16)], + "an ordinary window gets the architectural low bits", + ); + + assert_eq!(bus.write_narrow_from(uc | 0x2004, 2, source), BUS_OK); + assert_eq!(ecc.calls(), vec![NarrowWriteCall::From(2, 4, source)]); + + assert_eq!(bus.write_narrow_from(uc | 0x8000, 2, source), BUS_ERR, "no window there"); +} + +/// Equal physical offsets in different XKPHYS access spaces reach different devices. +#[test] +fn uc_spaces_keep_equal_offsets_distinct() { + let mut bus = Ip35Bus::new(Physical::new(Vec::new())); + for space in 0..4 { + bus.map( + space, + 0x0100_0000, + 0x1000, + Arc::new(Word(0x100 + space as u64)), + ); + } + for space in 0..4 { + let value = bus.read64(((space as u64) << 57) | 0x0100_0000); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0x100 + space as u64); + } +} + +/// The processor must retain both XKPHYS access-space bits through translation. +#[test] +fn xkphys_uc_bits_survive_the_board_cpu_mask() { + // MipsExecutor and the TLB's inline arrays exceed the test worker's stack. + std::thread::Builder::new() + .stack_size(4 * 1024 * 1024) + .spawn(|| { + let bus = Arc::new(Ip35Bus::new(Physical::new(Vec::new()))); + let mut cpu = crate::ip34_machine::build_cpu(bus, crate::ip34_machine::CPUS[0]); + cpu.core.write_cp0( + 12, + (cpu.core.cp0_status | crate::cpu::mips_core::STATUS_KX) as u64, + ); + for raw_uc in 0..4u64 { + let va = 0x9000_0000_0100_0020 | (raw_uc << 57); + for _ in 0..2 { + let translated = cpu.debug_translate(va); + assert_eq!(translated.status, crate::cpu::mips_exec::TR_UNCACHED); + assert_eq!(translated.phys, (raw_uc << 57) | 0x0100_0020); + } + } + }) + .expect("spawn") + .join() + .expect("thread panicked"); +} + +/// RAM placed above the first GiB must remain shared by cached, uncached and DMA paths. +#[test] +fn cached_and_uncached_memory_keep_high_addresses() { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(md_regs(&board).write64(8, 0x33), BUS_OK); + let at = (1 << 30) + 0x1000; + assert_eq!(board.bedrock.dma_view().write64(at, 0x12345678), BUS_OK); + assert_eq!(board.cached_bus().read64(at).data, 0x12345678); + assert_eq!(board.bus.read64((3 << 57) | at).data, 0x12345678); + assert_eq!(board.bus.read64(at).status, BUS_ERR); + assert_eq!(board.bedrock.dma_view().read64(at).data, 0x12345678); +} + +/// Memory diagnostics must notify both processor interfaces and refuse bad uncached data. +#[test] +fn memory_errors_reach_both_interfaces_and_uncached_reads_fail() { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let correctable = 1 << 61; + pi_regs(&board, 0 as u64).write64(0xb0, correctable); + md_regs(&board).write64(8, 0x33); + for directory in [false, true] { + if directory { + md_regs(&board).write64(0x40000, 2); + board.bedrock.directory_bus().write64(0, 1 << 16); + md_regs(&board).write64(0x40000, 0); + } else { + board.bedrock.ecc_bus().write8(7, 1); + } + assert_eq!(board.bus.read64(0).status, BUS_OK); + for pi in [pi_regs(&board, 0), pi_regs(&board, 1)] { + assert_eq!(pi.read64(0xa0).data & correctable, correctable); + } + assert_eq!(board.irq.bits() & (1 << 11), 1 << 11); + for pi in [pi_regs(&board, 0), pi_regs(&board, 1)] { + pi.write64(0x90, 64 + 61); + assert_eq!(pi.read64(0xa0).data & correctable, 0); + } + assert_eq!(board.irq.bits() & (1 << 11), 0); + } + + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + md_regs(&board).write64(8, 0x33); + board.bedrock.ecc_bus().write8(7, 3); + for selector in [0, 3] { + assert_eq!(board.bus.read64(selector << 57).status, BUS_ERR); + } + assert_eq!(board.cached_bus().read64(0).status, BUS_OK); + assert_eq!(board.bedrock.dma_view().read64(0).status, BUS_OK); +} + +/// Firmware resets the PCI link before reinitializing the attached storage controller. +#[test] +fn fuel_link_reset_resets_the_attached_scsi_engine() { + use crate::dev::isp1040::{mbox, reg, CTRL_INT_ENABLE, CTRL_RISC_INT_ENABLE, HOST_CMD_SHIFT}; + use crate::pci::PciFunction; + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let isp = &board.scsi; + isp.state().config().write(1, 6); + isp.state().config().write(5, 0x02000000); + isp.bar_write(1, reg::BUS_CTRL, u32::from(CTRL_INT_ENABLE | CTRL_RISC_INT_ENABLE), 2); + isp.bar_write(1, reg::MAILBOX_BASE, u32::from(mbox::NOP), 2); + isp.bar_write(1, reg::HOST_CMD, 0x5 << HOST_CMD_SHIFT, 2); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE, 2), u32::from(mbox::COMPLETE)); + assert_ne!(isp.bar_read(1, reg::BUS_CTRL, 2), 0); + assert_eq!(board.xbridge.widget(0).unwrap().write32(0x2f4, 1), BUS_OK); + assert_eq!(isp.state().config().command(), 0); + assert_eq!(isp.state().config().bar_base(1), 0); + assert_eq!(isp.bar_read(1, reg::BUS_CTRL, 2), 0); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE, 2), 0); + assert_eq!(isp.bar_read(1, reg::MAILBOX_BASE + 2, 2), 0x4953); +} + +/// The PROM reads the station address from the L1 and IRIX from the +/// Ethernet; both must report the configured one. +#[test] +fn the_configured_station_address_reaches_the_l1_and_the_ethernet() { + let mac = [0x08, 0x00, 0x69, 0x21, 0x43, 0x65]; + let board = crate::ip34_machine::FuelBoard::with_config(crate::ip34_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: mac, + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: true, + l1: Default::default(), + }).unwrap(); + assert_eq!(board.l1.station_mac(), mac); + assert_eq!(board.eth.mac(), mac); +} + +/// The L1's reported revision and brick serial keep their defaults unless +/// the board is built with replacements. +#[test] +fn the_l1_identity_overrides_reach_the_controller() { + use crate::ip34_machine::{FuelBoard, FuelBoardConfig, L1Identity}; + let build = |l1| FuelBoard::with_config(FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: false, l1, + }).unwrap().l1.config(); + let stock = build(L1Identity::default()); + assert_eq!(stock.revision, Some([1, 48, 1])); + assert_eq!(stock.brick_serial.as_deref(), Some("EMUN00")); + let set = build(L1Identity { revision: Some([2, 3, 4]), brick_serial: Some("EMUN09".into()) }); + assert_eq!(set.revision, Some([2, 3, 4])); + assert_eq!(set.brick_serial.as_deref(), Some("EMUN09")); +} + +fn fuel_dma_board() -> crate::ip34_machine::FuelBoard { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(md_regs(&board).write64(8, 0x33), BUS_OK); + board.xbridge.head(15).write32(0x24, 15); + board.xbridge.set_wide_slot_mask(15, 0); + board +} + +fn fuel_dma_address(address: u64) -> u64 { + (10 << 60) | (1 << 55) | address +} + +/// Storage DMA buffers can begin unaligned and cross a physical page boundary. +#[test] +fn dma_preserves_byte_order_and_crosses_ragged_pages() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let memory = board.bedrock.dma_view(); + for (address, length) in [(0x1030, 8), (0x1103, 27), (0x1ffb, 19)] { + let data: Vec = (0..length).map(|i| 0x31 + i as u8).collect(); + assert_eq!(dma.dma_write(fuel_dma_address(address), &data), PciDmaStatus::Ok); + for (i, &byte) in data.iter().enumerate() { + assert_eq!(memory.read8(address + i as u64).data, byte); + } + let mut back = vec![0xff; length]; + assert_eq!(dma.dma_read(fuel_dma_address(address), &mut back), PciDmaStatus::Ok); + assert_eq!(back, data); + } + // Without byte swapping enabled, the bridge reverses each 32-bit word. + assert_eq!(dma.dma_write((10 << 60) | 0x3000, &[0, 1, 2, 3, 4, 5, 6, 7]), PciDmaStatus::Ok); + assert_eq!(memory.read64(0x3000).data, 0x0302010007060504); +} + +/// A failed DMA read must clear only refused chunks and preserve completed data. +#[test] +fn dma_refusals_clear_failed_reads_without_losing_completed_chunks() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let memory = board.bedrock.dma_view(); + // Bank zero has 256 MiB fitted. The second qword is outside its backing. + let last = 0x0fff_fff8; + assert_eq!(memory.write64(last, 0x0102030405060708), BUS_OK); + let mut bytes = [0xff; 16]; + assert_eq!(dma.dma_read(fuel_dma_address(last), &mut bytes), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [1, 2, 3, 4, 5, 6, 7, 8, 0, 0, 0, 0, 0, 0, 0, 0]); + assert_eq!(dma.dma_write(fuel_dma_address(last), &[0x42; 16]), PciDmaStatus::MasterAbort); + assert_eq!(memory.read64(last).data, 0x4242424242424242); + board.bedrock.disable_device(15, 0); + bytes.fill(0xff); + assert_eq!(dma.dma_read(fuel_dma_address(last), &mut bytes), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [0; 16]); + assert_eq!(dma.dma_write(fuel_dma_address(last), &[0x99; 8]), PciDmaStatus::MasterAbort); + assert_eq!(memory.read64(last).data, 0x4242424242424242); +} + +/// A wide DMA leg must report an uncorrectable word once without publishing bad bytes. +#[test] +fn fuel_dma_checks_an_aligned_uncorrectable_word_once() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let summary = |n| pi_regs(&board, n).read64(0x400).data & (1 << 24); + assert_eq!(board.bedrock.dma_view().write64(0, 0), BUS_OK); + assert_eq!(board.bedrock.ecc_bus().write8(7, 3), BUS_OK); + for address in 0..8 { + assert_eq!(board.bedrock.inbound_read(15, Some(0), address, 1).status, BUS_OK); + } + assert_eq!([summary(0), summary(1)], [1 << 24; 2], "both interfaces see the error"); + assert_ne!(md_regs(&board).read64(0x60018).data & (1 << 61), 0); + for n in 0..2 { + pi_regs(&board, n).write64(0x400, 1 << 24); + } + let mut bytes = [0xff; 8]; + assert_eq!(board.xbridge.dma(15, 0).dma_read(fuel_dma_address(0), &mut bytes), PciDmaStatus::Ok); + assert_eq!(bytes, [0; 8]); + assert_eq!([summary(0), summary(1)], [1 << 24; 2]); + let record = md_regs(&board).read64(0x60010).data; + assert_ne!(record & (1 << 60), 0); + assert_eq!(record & (1 << 61), 0, "one error, so no overrun"); + assert_eq!(board.bedrock.uncached_view().read64(0).status, BUS_ERR); +} + +/// Inbound Hub writes must preserve access width and the requesting device identity. +#[test] +fn dma_hub_access_preserves_width_and_source_permissions() { + use crate::pci::PciDmaStatus; + let board = fuel_dma_board(); + let dma = board.xbridge.dma(15, 0); + let hub = (1 << 47) | 0x1800000; + for (interface, offset) in [(0, 0), (1, 0x200000)] { + let address = fuel_dma_address(hub + offset + 0x90); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::Ok); + assert_ne!(pi_regs(&board, interface as u64).read64(0x98).data & (1 << 12), 0); + for width in [1, 2, 4] { + assert_eq!(dma.dma_write(address, &[0; 8][..width]), PciDmaStatus::MasterAbort); + assert_ne!(pi_regs(&board, interface as u64).read64(0x98).data & (1 << 12), 0); + } + assert_eq!(dma.dma_write(address, &12u64.to_be_bytes()), PciDmaStatus::Ok); + assert_eq!(pi_regs(&board, interface as u64).read64(0x98).data & (1 << 12), 0); + } + let address = fuel_dma_address(hub + 0x90); + // I/O modify must retain PI region protection and requesting-slot identity. + assert_eq!(pi_regs(&board, 0 as u64).write64(0x10, 0), BUS_OK); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::MasterAbort); + assert_eq!(pi_regs(&board, 0 as u64).read64(0x98).data & (1 << 12), 0); + assert_eq!(pi_regs(&board, 0 as u64).write64(0x10, 1), BUS_OK); + board.bedrock.disable_device(15, 0); + assert_eq!(dma.dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::MasterAbort); + assert_eq!(pi_regs(&board, 0 as u64).read64(0x98).data & (1 << 12), 0); + assert_eq!(board.xbridge.dma(15, 1).dma_write(address, &0x10cu64.to_be_bytes()), PciDmaStatus::Ok); + assert_ne!(pi_regs(&board, 0 as u64).read64(0x98).data & (1 << 12), 0); + let mut bytes = [0xff; 8]; + assert_eq!(board.xbridge.dma(15, 1).dma_read( + fuel_dma_address(hub + 0x40), &mut bytes, + ), PciDmaStatus::MasterAbort); + assert_eq!(bytes, [0; 8]); + assert_eq!(board.bus.read64((1 << 57) | 0x1800040).status, BUS_OK); +} + +/// Rounded MMIO pages must not expose padding or bypass cell permissions. +#[test] +fn exact_windows_reject_padding_and_revoked_permissions() { + use std::sync::atomic::{AtomicBool, Ordering}; + struct Offset; + impl BusDevice for Offset { + fn read64(&self, offset: u64) -> BusRead64 { BusRead64::ok(offset) } + fn write64(&self, _offset: u64, _value: u64) -> u32 { BUS_OK } + } + let permitted = Arc::new(AtomicBool::new(true)); + let permission = permitted.clone(); + let window = Window::new(0x1000, 0x88, Arc::new(Offset)) + .permitted(Arc::new(move || permission.load(Ordering::Acquire))); + assert_eq!(window.read64(0x1080).data, 0x80); + assert_eq!(window.read64(0x1088).status, BUS_ERR); + assert_eq!(window.read64(0x1084).status, BUS_ERR); + assert_eq!(window.read64(0x0ff8).status, BUS_ERR); + assert_eq!(window.read64(u64::MAX).status, BUS_ERR); + permitted.store(false, Ordering::Release); + assert_eq!(window.read64(0x1000).status, BUS_ERR); + assert_eq!(window.write64(0x1000, 1), BUS_ERR); + permitted.store(true, Ordering::Release); + assert_eq!(window.write64(0x1000, 1), BUS_OK); +} + +/// Board default setup must preserve the direct RAM path and its shared DMA contents. +#[test] +fn direct_cached_banks_remain_visible_after_board_defaults_are_installed() { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_eq!(md_regs(&board).write64(8, 0x33), BUS_OK); + let cached = board.cached_bus(); + assert!(cached.mem_ptr(0x2000).is_some()); + assert_eq!(cached.write64(0x2000, 0x123456789abcdef0), BUS_OK); + assert_eq!(board.bedrock.dma_view().read64(0x2000).data, 0x123456789abcdef0); + assert!(cached.mem_ptr(3 << 57).is_none(), "UC selectors cannot alias cached RAM"); +} + +/// A device overlay replaces its pages, including padding, while its neighbors +/// keep their original address translation. +#[test] +fn sparse_overlays_keep_neighbors_and_refuse_the_new_windows_padding() { + let mut map = SparseWindowMap::default(); + map.map(0x1000, 0x4000, Arc::new(Word(1))); + map.map(0x2040, 0x80, Arc::new(Word(2))); + assert_eq!(map.read64(0x1000).data, 1); + assert_eq!(map.read64(0x2040).data, 2); + assert_eq!(map.read64(0x20b8).data, 2); + assert_eq!(map.read64(0x20bc).status, BUS_ERR); + assert_eq!(map.read64(0x2000).status, BUS_ERR); + assert_eq!(map.read64(0x20c0).status, BUS_ERR); + assert_eq!(map.read64(0x3000).data, 1); + assert_eq!(map.read64(0x5000).status, BUS_ERR); + map.map(0x1000, 0x4000, Arc::new(Word(3))); + assert_eq!(map.ranges.len(), 1); + assert_eq!(map.read64(0x2040).data, 3); +} + +/// Sparse MMIO holes remain bus errors and a revoked cell permission does not +/// fall through to an older mapping beneath the protected window. +#[test] +fn sparse_holes_and_revoked_permissions_never_expose_older_devices() { + use std::sync::atomic::{AtomicBool, Ordering}; + let permission = Arc::new(AtomicBool::new(true)); + let allowed = permission.clone(); + let mut map = SparseWindowMap::default(); + map.map(0, 0x3000, Arc::new(Word(1))); + map.map(0x1000, 8, Arc::new(Window::new(0, 8, Arc::new(Word(2))) + .permitted(Arc::new(move || allowed.load(Ordering::Acquire))))); + assert_eq!(map.read64(0x1000).data, 2); + permission.store(false, Ordering::Release); + assert_eq!(map.read64(0x1000).status, BUS_ERR); + assert_eq!(map.read64(0x1008).status, BUS_ERR); + assert_eq!(map.read64(0x2000).data, 1); + assert_eq!(map.read64(0x3000).status, BUS_ERR); +} + +/// Code-page lookup receives plain physical addresses and reads them through +/// the cached plane. Cached stores must also update the MD view. +#[cfg(feature = "jitv2")] +#[test] +fn code_generations_use_the_stable_ram_endpoint() { + struct Generations { + ram_only: bool, + generation: AtomicU64, + writes: std::sync::Mutex>, + } + impl Generations { + fn new(ram_only: bool) -> Arc { + Arc::new(Self { ram_only, generation: AtomicU64::new(0), writes: Default::default() }) + } + } + impl BusDevice for Generations { + fn gen_ptr(&self, address: u64) -> *const AtomicU64 { + if !self.ram_only || address < 0x1000 { &self.generation } else { std::ptr::null() } + } + fn write64(&self, address: u64, value: u64) -> u32 { + self.writes.lock().unwrap().push((address, value)); + BUS_OK + } + fn write64_masked(&self, address: u64, value: u64, mask: u64) -> u32 { + self.write64(address, value & mask) + } + fn write_block(&self, address: u64, values: &[u64]) -> u32 { + for (index, value) in values.iter().enumerate() { + self.write64(address + index as u64 * 8, *value); + } + BUS_OK + } + } + let physical = Physical::new(Vec::new()); + let bus = Ip35Bus::new(physical.clone()); + let raw_ram = Generations::new(false); + let stable_ram = Generations::new(true); + let rom = Generations::new(false); + map_device(&physical, 0, 0x1000, raw_ram.clone(), "raw RAM generation"); + map_device(&physical, 0x1fc00000, 0x1000, rom.clone(), "ROM generation"); + let cached = bus.cached_view(stable_ram.clone(), crate::dev::bedrock::RamExtent::new(&[256, 256])); + assert_eq!(cached.gen_ptr(0x40), &stable_ram.generation as *const AtomicU64); + assert_eq!(cached.gen_ptr(0x1fc00000), &rom.generation as *const AtomicU64); + assert_eq!(cached.write64(0x40, 0x1234), BUS_OK); + assert_eq!(cached.write_block(0x48, &[0x5678]), BUS_OK); + assert_eq!(cached.write64_masked(0x50, 0xaabb, 0xff), BUS_OK); + assert_eq!(*stable_ram.writes.lock().unwrap(), [(0x40, 0x1234), (0x48, 0x5678), (0x50, 0xbb)]); + assert_eq!(cached.write64(0x1fc00000, 0x4321), BUS_OK); + assert_eq!(*rom.writes.lock().unwrap(), [(0, 0x4321)], "the ROM window translates to a device-local offset"); + assert!(raw_ram.writes.lock().unwrap().is_empty()); +} + +/// The JIT's page lookup and inline compile read code through the CPU's cached +/// bus, which a board whose uncached connection decodes plain addresses as +/// something else sets to the view the cache fills from. The compile pool is +/// started on that same object. +#[test] +fn the_cpu_reads_code_through_the_cached_view_not_the_uncached_connection() { + // An executor is large; a test thread's stack is not. + std::thread::Builder::new().stack_size(64 << 20).spawn(cpu_cached_bus).unwrap().join().unwrap(); +} + +fn cpu_cached_bus() { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let cpu = board.build_cpu(crate::ip34_machine::CPUS[0]); + let cached = board.cached_bus(); + assert!(Arc::ptr_eq(cpu.cached_bus(), &cached)); + let sysad: Arc = board.bus.clone(); + assert!(!Arc::ptr_eq(cpu.cached_bus(), &sysad), "sysad is the uncached connection"); +} + +/// Unmodeled Hub heads retain the PROM compatibility readback without +/// absorbing the adjacent, independently implemented L1 UART window. +#[test] +fn compatibility_hub_heads_do_not_hide_the_l1_uart_window() { + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + assert_ne!(board.bedrock.cell_permits() as u64, 0); + for base in [0x1000_0000, 0x3000_0000] { + for offset in [0, 8, 0x40, 0x78] { + let address = base + offset; + assert_eq!(board.bus.write8(address, 0xa5), BUS_OK); + assert_eq!(board.bus.write16(address, 0xa5a5), BUS_OK); + assert_eq!(board.bus.write32(address, 0xa5a5a5a5), BUS_OK); + assert_eq!(board.bus.write64(address, 0x2000), BUS_OK); + let value = board.bus.read8(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read16(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read32(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + let value = board.bus.read64(address); + assert_eq!(value.status, BUS_OK); + assert_eq!(value.data, 0); + } + assert_eq!(board.bus.read64(base + 0x80).status, BUS_OK); + } +} + +/// A busy host port or unusable transcript path must not prevent a serial +/// machine from starting, including when any of its three ports serves CI. +#[test] +fn unavailable_serial_ports_and_logs_do_not_abort_the_machine() { + use crate::config::{FuelConsole, MachineConfig}; + let listeners: Vec<_> = (0..3).map(|_| std::net::TcpListener::bind(("127.0.0.1", 0)).unwrap()).collect(); + let ports: Vec<_> = listeners.iter().map(|socket| socket.local_addr().unwrap().port()).collect(); + let board = crate::ip34_machine::FuelBoard::new(&[], Arc::new(|| 0)).unwrap(); + let mut config = MachineConfig::default(); + config.fuel.l1_port = Some(ports[0]); + config.serial_port_a = Some(ports[1]); + config.serial_port_b = Some(ports[2]); + config.ci = true; + let missing = std::env::temp_dir().join(format!("iris-fuel-missing-log-{}", std::process::id())).join("absent").join("serial.txt"); + assert!(!missing.parent().unwrap().exists(), "the log parent must be absent"); + config.serial_log = Some(missing.to_string_lossy().into_owned()); + config.fuel.l1_log = config.serial_log.clone(); + for (console, selection) in [FuelConsole::L1, FuelConsole::Ioc3A, FuelConsole::Ioc3B].into_iter().enumerate() { + config.fuel.console = selection; + crate::ip34_machine::FuelMachine::connect_serial_for_test(&config, &board, console).unwrap(); + } + config.ci = false; + crate::ip34_machine::FuelMachine::connect_serial_for_test(&config, &board, 0).unwrap(); +} + +/// The Fuel's OHCI root ports each carry a low-speed device, and host +/// input reaches them through the IOC3's PS/2 controller object; the USB +/// pair is fitted only when asked for. +#[test] +fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { + use crate::pci::PciFunction; + let ports = |usb_input| { + let board = crate::ip34_machine::FuelBoard::with_config(crate::ip34_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input, + l1: Default::default(), + }).unwrap(); + // Power each port: one with a device shows it connected. + board.usb.bar_write(0, 0x54, 0x100, 4); + board.usb.bar_write(0, 0x58, 0x100, 4); + let status = [0x54, 0x58].map(|at| board.usb.bar_read(0, at, 4)); + (board, status) + }; + let (board, status) = ports(true); + for s in status { + assert_eq!(s & 1, 1, "device connected"); + assert_eq!(s & 0x200, 0x200, "low speed"); + } + // The mouse the USB side fits makes the shared controller report a mouse + // only once the guest has configured it. + assert!(!board.ioc3.ps2().mouse_enabled()); + let (_, status) = ports(false); + assert_eq!(status.map(|s| s & 1), [0, 0], "nothing plugged in"); +} + diff --git a/src/lib.rs b/src/lib.rs index 48915580d..3da500cc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -190,6 +190,7 @@ pub mod ip27_onyx2; pub mod ip27_origin2000; /// Fuel has an IP34 board and uses the IP35 software platform and processor views. pub mod ip35; +pub mod ip34_machine; pub mod machine_state; pub mod platform; pub mod physical; diff --git a/src/machine.rs b/src/machine.rs index 62566f107..d343154b1 100644 --- a/src/machine.rs +++ b/src/machine.rs @@ -53,7 +53,7 @@ pub fn set_emulator_profile(profile: MachineProfile) { fn emulator_profile() -> Option { let raw = EMULATOR_PROFILE.load(std::sync::atomic::Ordering::Relaxed); - [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::Origin200Ip29, MachineProfile::Origin2000Ip27, MachineProfile::Onyx2Ip27] + [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::Origin200Ip29, MachineProfile::Origin2000Ip27, MachineProfile::Onyx2Ip27, MachineProfile::FuelIp34] .into_iter().find(|p| *p as u8 == raw) } @@ -71,6 +71,7 @@ fn emulator_name_thirds(profile: Option) -> &'static [&'static s Some(MachineProfile::Origin200Ip29) => &["IRIX", "Iris", "IPsomething", "IP29"], Some(MachineProfile::Origin2000Ip27) => &["IRIX", "Iris", "IPsomething", "IP27"], Some(MachineProfile::Onyx2Ip27) => &["IRIX", "Iris", "IPsomething", "IP27"], + Some(MachineProfile::FuelIp34) => &["IRIX", "Iris", "IPsomething", "IP34"], None => &["IRIX", "Iris", "IPsomething"], } } @@ -2166,11 +2167,12 @@ mod emulator_name_tests { Some(MachineProfile::Origin200Ip29) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], Some(MachineProfile::Origin2000Ip27) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], Some(MachineProfile::Onyx2Ip27) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], + Some(MachineProfile::FuelIp34) => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], None => &["Indy", "Indigo2", "IP24", "IP22", "IP28", "IP30", "IP32", "O2"], } }; for p in [None, Some(MachineProfile::IndyIp24), Some(MachineProfile::Indigo2Ip22), - Some(MachineProfile::Indigo2Ip28), Some(MachineProfile::O2Ip32), Some(MachineProfile::OctaneIp30), Some(MachineProfile::Origin200Ip29), Some(MachineProfile::Origin2000Ip27), Some(MachineProfile::Onyx2Ip27)] { + Some(MachineProfile::Indigo2Ip28), Some(MachineProfile::O2Ip32), Some(MachineProfile::OctaneIp30), Some(MachineProfile::Origin200Ip29), Some(MachineProfile::Origin2000Ip27), Some(MachineProfile::Onyx2Ip27), Some(MachineProfile::FuelIp34)] { let thirds = emulator_name_thirds(p); assert!(!thirds.is_empty()); for w in thirds { @@ -2182,7 +2184,7 @@ mod emulator_name_tests { #[test] fn profile_round_trips_through_the_global() { - for p in [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::Origin200Ip29, MachineProfile::Origin2000Ip27, MachineProfile::Onyx2Ip27] { + for p in [MachineProfile::IndyIp24, MachineProfile::Indigo2Ip22, MachineProfile::Indigo2Ip28, MachineProfile::O2Ip32, MachineProfile::OctaneIp30, MachineProfile::Origin200Ip29, MachineProfile::Origin2000Ip27, MachineProfile::Onyx2Ip27, MachineProfile::FuelIp34] { set_emulator_profile(p); assert_eq!(emulator_profile(), Some(p)); } diff --git a/src/workstation.rs b/src/workstation.rs index 2bd5798da..6194c1b31 100644 --- a/src/workstation.rs +++ b/src/workstation.rs @@ -40,6 +40,7 @@ use crate::traits::{Device, MachineEvent}; pub fn build(cfg: MachineConfig) -> Result, String> { crate::machine::set_emulator_profile(cfg.machine.profile); match cfg.machine.profile { + crate::config::MachineProfile::FuelIp34 => Ok(Box::new(crate::ip34_machine::FuelMachine::new(&cfg)?)), // The O2 is its own machine (CRIME, MACE, GBE), not an MC/HPC3 one. crate::config::MachineProfile::O2Ip32 => Ok(Box::new(crate::ip32_machine::O2Machine::new(&cfg)?)), // So is the Octane (HEART, Xbow, Bridge). From 1890079673d970d3c7b754bf40a0cc4c159eb378 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 04:22:11 -0700 Subject: [PATCH 17/22] odyssey: hold the refused-push test's slice until the test lets go The test slept 150 ms and required the pushing thread to still be unfinished. The board thread's hold was a fixed 4 s sleep, which a loaded host could outlast before the test looked, so the refused store could complete and the assertion fail. The hold now lasts until the test releases it, and the test waits for the push to be refused before it asserts. --- src/dev/odyssey/fifo.rs | 12 ++++++++++++ src/dev/odyssey/tests/fifo.rs | 15 +++++++++++---- src/dev/odyssey/worker.rs | 6 +++++- 3 files changed, 28 insertions(+), 5 deletions(-) diff --git a/src/dev/odyssey/fifo.rs b/src/dev/odyssey/fifo.rs index f582e8cf0..cd7183cdb 100644 --- a/src/dev/odyssey/fifo.rs +++ b/src/dev/odyssey/fifo.rs @@ -174,6 +174,12 @@ pub(super) struct CommandFifo { /// counted after its last look at the hurry request. #[cfg(test)] pub test_parked: AtomicU32, + /// Tests: pushes the full ring has refused. + #[cfg(test)] + pub test_refused: AtomicU32, + /// Tests: ends a long slice's hold (`Board::test_slice_sleep_ms`) early. + #[cfg(test)] + pub test_release: AtomicBool, /// Tests: the next push calls this, once, as soon as its /// store is in the ring: a consumer that takes the store at that /// moment, before the push has returned. @@ -205,6 +211,10 @@ impl Default for CommandFifo { #[cfg(test)] test_parked: AtomicU32::new(0), #[cfg(test)] + test_refused: AtomicU32::new(0), + #[cfg(test)] + test_release: AtomicBool::new(false), + #[cfg(test)] test_after_ring_push: Mutex::new(None), } } @@ -334,6 +344,8 @@ impl CommandFifo { pub fn push(&self, front: &Mutex, addr: u64, value: u64, size: u32) -> bool { let words = store_words(size); if !self.has_room() { + #[cfg(test)] + self.test_refused.fetch_add(1, SeqCst); return false; } let q = QueuedStore { value, addr: addr as u32, size: size as u8 }; diff --git a/src/dev/odyssey/tests/fifo.rs b/src/dev/odyssey/tests/fifo.rs index 8e90e12eb..425a023a0 100644 --- a/src/dev/odyssey/tests/fifo.rs +++ b/src/dev/odyssey/tests/fifo.rs @@ -951,13 +951,14 @@ fn a_refused_command_push_retries_behind_the_board_thread() { let (h, _rec) = threaded_board(); let words = preamble_twice(); - // The ring must fill and the retry be seen waiting while this one slice - // holds the board, however long the test's threads wait for a core. - start_long_slice(&h, &words, 4000); + // The slice holds the board's thread until the test releases it, so the + // ring stays full however long the host takes to get here. + start_long_slice(&h, &words, 600_000); while h.fifo.stores() < RING_STORES { assert!(h.bus_write(consts::cmd::FIFO, 0, 4)); } let stores = h.fifo.stores(); + let refused = h.fifo.test_refused.load(SeqCst); let done = Arc::new(std::sync::atomic::AtomicBool::new(false)); let store = { let (h, done) = (h.clone(), done.clone()); @@ -966,9 +967,15 @@ fn a_refused_command_push_retries_behind_the_board_thread() { done.store(true, SeqCst); }) }; - std::thread::sleep(std::time::Duration::from_millis(150)); + // The store is refused and retries: wait until it has been. + let deadline = std::time::Instant::now() + std::time::Duration::from_secs(20); + while h.fifo.test_refused.load(SeqCst) == refused { + assert!(std::time::Instant::now() < deadline, "the store was never refused"); + std::thread::yield_now(); + } assert!(!done.load(SeqCst), "the store was dropped or ran around the board's thread"); assert_eq!(h.fifo.stores(), stores); + h.fifo.test_release.store(true, SeqCst); store.join().unwrap(); assert!(h.fifo.level() <= cfifo::WORDS); h.fifo.test_words.store(0, SeqCst); diff --git a/src/dev/odyssey/worker.rs b/src/dev/odyssey/worker.rs index b0a44abf0..1f60af2c0 100644 --- a/src/dev/odyssey/worker.rs +++ b/src/dev/odyssey/worker.rs @@ -141,7 +141,11 @@ impl Board { // A run that takes as long as a long draw would. let sleep_ms = std::mem::take(&mut self.test_slice_sleep_ms); if sleep_ms != 0 { - std::thread::sleep(std::time::Duration::from_millis(sleep_ms as u64)); + // Until the time is up or a test releases the hold. + let end = std::time::Instant::now() + std::time::Duration::from_millis(sleep_ms as u64); + while std::time::Instant::now() < end && !self.fifo.test_release.load(SeqCst) { + std::thread::sleep(std::time::Duration::from_millis(1)); + } } } From b3d76e2b8d79bfa5cdcb1682228111b09107e99d Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 04:22:12 -0700 Subject: [PATCH 18/22] odyssey: let a widget-offset machine host the board The board assumed the Octane: its interrupt packets went only to HEART at widget 8, and it took the processor's addresses in the IP30 widget windows. A Fuel's processor reaches widget 13 through Bedrock, which hands the board widget offsets, and the board's packets cross the XBridge's crossbar to Bedrock like any widget's. Add a constructor for such a machine. It takes a packet sender for the interrupts and decodes every address as an aperture offset. The two choices were hard-wired into the board; they are now one packet sender (the Octane's checks for HEART's widget and register and drives its line, and a packet to anywhere else is refused as before) and an explicit aperture mode chosen at construction. The Octane's constructor and behaviour are unchanged, and nothing on its store path is touched. The debug register dump no longer lists the heart widget, which is the Octane sender's business now. A test routes a packet to a widget other than HEART and requires the sender to get it, not HEART's line. --- src/dev/odyssey/board.rs | 2 +- src/dev/odyssey/debug.rs | 2 +- src/dev/odyssey/interface.rs | 36 +++++++++------ src/dev/odyssey/mod.rs | 68 ++++++++++++++++++++++++----- src/dev/odyssey/tests/interrupts.rs | 23 ++++++++++ src/dev/odyssey/tests/mod.rs | 2 +- src/dev/odyssey/tests/monitor.rs | 16 ++++--- 7 files changed, 117 insertions(+), 32 deletions(-) diff --git a/src/dev/odyssey/board.rs b/src/dev/odyssey/board.rs index 8a3cb1dbc..c40746087 100644 --- a/src/dev/odyssey/board.rs +++ b/src/dev/odyssey/board.rs @@ -1,5 +1,5 @@ //! The Odyssey (VPro) board's constants and state. The interrupt packet path -//! is the `irq` field of the host interface (`HostInterface`, interface.rs). +//! is the `packets` field of the host interface (`HostInterface`, interface.rs). //! //! Ported from include/hw/misc/sgi-odyssey.h in jamesbraid/qemu-sgi-origami. diff --git a/src/dev/odyssey/debug.rs b/src/dev/odyssey/debug.rs index ce8b6f677..bb40c2d07 100644 --- a/src/dev/odyssey/debug.rs +++ b/src/dev/odyssey/debug.rs @@ -463,7 +463,7 @@ impl Odyssey { f.route_dest_upper, f.route_dest_lower, f.route_vectors, (f.route_vectors >> 24) & 0xff, (f.route_vectors >> 16) & 0xff, (f.route_vectors >> 8) & 0xff, f.route_vectors & 0xff, f.routed()); let _ = writeln!(out, " swap arm {:#x} pending {:#x} control {:#x} companion {:#x} set {:#x} clear {:#x} displayed {:#010x}{}", f.swap_arm, f.swap_pending, f.swap_control, f.swap_control_companion, f.swap_set, f.swap_clear, f.displayed_buffer, if f.displayed_written { " (written, repaint pending)" } else { "" }); - let _ = writeln!(out, " sw token {:#x} flow-control target {:#x} heart widget {}", f.sw_token, f.flctl_target, f.heart_widget); + let _ = writeln!(out, " sw token {:#x} flow-control target {:#x}", f.sw_token, f.flctl_target); } let g = self.quiesced()?; let wg = &g.widget; diff --git a/src/dev/odyssey/interface.rs b/src/dev/odyssey/interface.rs index 49727c3a8..26fe1557a 100644 --- a/src/dev/odyssey/interface.rs +++ b/src/dev/odyssey/interface.rs @@ -29,14 +29,13 @@ use super::display_fifo::display_fifo_access; use super::fifo::CommandFifo; use super::mmio::{access_waits, command_store, fifo_port}; use super::registers::reg; +use super::PacketSender; use parking_lot::Mutex; use std::sync::Arc; pub(super) struct HostInterface { - /// Where interrupt packets go (the iris glue's HEART). - pub irq: Option>, - /// The widget number of HEART, the packets' destination. - pub heart_widget: u32, + /// Where interrupt packets go: the machine's fabric. + pub packets: Option, /// ODSY_IDENTITY, composed from the revision at reset. pub identity: u32, @@ -88,8 +87,7 @@ pub(super) struct HostInterface { impl Default for HostInterface { fn default() -> Self { HostInterface { - irq: None, - heart_widget: 8, + packets: None, identity: 0, widget_control: 0, xt_req_timeout: 0, @@ -498,18 +496,28 @@ pub(super) fn sgi_xio_compose(who: &str, widget: u32, address: u64, packet: &mut /// HEART's interrupt-packet register in its widget space. const HEART_PACKET: u64 = 0x80; +/// The Octane's packet sender: HEART takes a packet at 0x80 of widget 8 and +/// drives the line the vector names. +pub(super) fn heart_sender(irq: Arc) -> PacketSender { + Arc::new(move |widget, address, vector, level| { + if widget != crate::ip30::WIDGET_HEART || address != HEART_PACKET { + return false; + } + irq.set_line(vector as u32, level); + true + }) +} + impl HostInterface { - /// The interrupt-modify write to the interrupt target. HEART takes it at - /// 0x80 of widget 8. + /// The interrupt-modify write to the interrupt target: the packet goes to + /// the machine's sender, which routes it by widget and address. pub fn send_interrupt(&mut self, packet: u64, number: u8, set: bool) { let widget = (packet >> XIO_PACKET_ADDRESS_BITS) as u32; let address = packet & ((1u64 << XIO_PACKET_ADDRESS_BITS) - 1); - if widget != self.heart_widget || address != HEART_PACKET { - crate::dlog!(LogModule::Odyssey, "odyssey: interrupt packet to widget {widget} address {address:#x} has no receiver"); - return; - } - if let Some(irq) = &self.irq { - irq.set_line(number as u32, set); + if let Some(send) = &self.packets { + if !send(widget as u8, address, number, set) { + crate::dlog!(LogModule::Odyssey, "odyssey: interrupt packet to widget {widget} address {address:#x} has no receiver"); + } } } } diff --git a/src/dev/odyssey/mod.rs b/src/dev/odyssey/mod.rs index e5cad6423..6d036f9c6 100644 --- a/src/dev/odyssey/mod.rs +++ b/src/dev/odyssey/mod.rs @@ -127,19 +127,34 @@ use scanout::monotonic_ns; /// The VPro board on widget `slot`: a `BusDevice` for the widget's /// windows. pub struct Odyssey { - small: u64, - large: u64, + aperture: Aperture, state: BoardLocks, display: Arc, running: Arc, thread: Mutex>>, } +/// Carries a board interrupt packet into a machine's crosstalk fabric: the +/// destination widget, the address within it, the vector, and whether the +/// source rose or fell. False when nothing at the destination takes it. +pub type PacketSender = Arc bool + Send + Sync>; + +/// How the processor's addresses reach the board, chosen when it is built. +#[derive(Clone, Copy)] +enum Aperture { + /// The Octane's: the widget's small and large IP30 windows. + Ip30Windows { small: u64, large: u64 }, + /// Every address is already an aperture offset, as a machine whose I/O + /// interface hands widgets offsets (Bedrock) presents it. + Offsets, +} + impl Odyssey { - /// A board of `product` (v12, v10, v8, v6) at `widget`: its interrupt - /// packets reach `irq` (HEART), its DMA reaches `mem`. `pool` is the - /// CPU raster unit's workers and tile size, `raster` which unit draws - /// and, for the GPU, the wgpu backend (`[graphics]`). + /// A board of `product` (v12, v10, v8, v6) at `widget` on the Octane: + /// the processor reaches it through the IP30 widget windows, its + /// interrupt packets reach `irq` (HEART), its DMA reaches `mem`. `pool` + /// is the CPU raster unit's workers and tile size, `raster` which unit + /// draws and, for the GPU, the wgpu backend (`[graphics]`). #[allow(clippy::too_many_arguments, reason = "the board's wiring")] pub fn new( widget: u8, @@ -149,6 +164,37 @@ impl Odyssey { status: Arc, pool: (usize, u32), raster: (crate::config::RasterUnit, &str), + ) -> Result, String> { + let aperture = Aperture::Ip30Windows { small: widget_small_base(widget), large: widget_large_base(widget) }; + Self::build(widget, product, interface::heart_sender(irq), aperture, mem, status, pool, raster) + } + + /// A board of `product` at `widget` on a machine that hands the board + /// widget-relative addresses, as Bedrock's I/O interface does, and + /// carries its interrupt packets through `packets`. + #[allow(clippy::too_many_arguments, reason = "the board's wiring")] + pub fn new_widget_relative( + widget: u8, + product: &str, + packets: PacketSender, + mem: Arc, + status: Arc, + pool: (usize, u32), + raster: (crate::config::RasterUnit, &str), + ) -> Result, String> { + Self::build(widget, product, packets, Aperture::Offsets, mem, status, pool, raster) + } + + #[allow(clippy::too_many_arguments, reason = "the board's wiring")] + fn build( + widget: u8, + product: &str, + packets: PacketSender, + aperture: Aperture, + mem: Arc, + status: Arc, + pool: (usize, u32), + raster: (crate::config::RasterUnit, &str), ) -> Result, String> { let mut s = Board::new(product, widget as u32)?; // The fragment pipeline's workers and tile size (`[graphics]`). @@ -168,15 +214,14 @@ impl Odyssey { }); let mem: Arc = Arc::new(record::RecordingMem { inner: mem, rec: rec.clone() }); s.guest = Some(mem); - s.front.lock().irq = Some(irq); + s.front.lock().packets = Some(packets); if let Ok(path) = std::env::var("IRIS_ODSY_TRACE") { s.trace.start(&path, trace::TRACE_ALL).map_err(|e| format!("IRIS_ODSY_TRACE {path}: {e}"))?; } let mut state = BoardLocks::new(s); state.rec = rec; Ok(Arc::new(Self { - small: widget_small_base(widget), - large: widget_large_base(widget), + aperture, state, display: Arc::new(Display::new(status)), running: Arc::new(AtomicBool::new(false)), @@ -192,7 +237,10 @@ impl Odyssey { /// The aperture offset of `addr`, in either window. fn offset(&self, addr: u64) -> Option { - let off = if addr >= self.large { addr - self.large } else { addr.wrapping_sub(self.small) }; + let off = match self.aperture { + Aperture::Ip30Windows { small, large } => if addr >= large { addr - large } else { addr.wrapping_sub(small) }, + Aperture::Offsets => addr, + }; (off < ODSY_APERTURE).then_some(off) } diff --git a/src/dev/odyssey/tests/interrupts.rs b/src/dev/odyssey/tests/interrupts.rs index 6bbb67ca9..8f89604cf 100644 --- a/src/dev/odyssey/tests/interrupts.rs +++ b/src/dev/odyssey/tests/interrupts.rs @@ -437,3 +437,26 @@ fn a_held_flag_is_sent_with_the_source_enables_clear() { reg_write(&mut s, route::DEST_UPPER, HEART_UPPER); assert_eq!(rec.events(FLAG_VECTOR), [true]); } + +/// A board built for a machine whose fabric carries packets to any widget +/// hands them to that machine's sender, to the widget the routing names, +/// and never to HEART's line: the Fuel's widget 13 reaches Bedrock this way. +#[test] +fn a_packet_sender_takes_the_interrupt_to_the_routed_widget() { + let (mut s, rec) = board("v12"); + let seen = Arc::new(Mutex::new(Vec::new())); + let log = seen.clone(); + s.front.lock().packets = Some(Arc::new(move |widget, address, vector, level| { + log.lock().push((widget, address, vector, level)); + true + })); + s.mmio_write(ROUTE_DEST_UPPER, 1 << 25, 4); + s.mmio_write(ROUTE_DEST_LOWER, 0x1230, 4); + s.mmio_write(ROUTE_VECTORS, TEST_VECTOR as u64, 4); + s.mmio_write(INTR_ENABLE_SET, INTR_FIFO_LOW_WATER, 4); + assert_eq!(*seen.lock(), vec![(1, 0x1230, TEST_VECTOR as u8, true)]); + assert!(rec.silent(), "the packet did not go to HEART"); + + s.mmio_write(INTR_CLEAR, INTR_FIFO_LOW_WATER, 4); + assert_eq!(seen.lock().last(), Some(&(1, 0x1230, TEST_VECTOR as u8, false))); +} diff --git a/src/dev/odyssey/tests/mod.rs b/src/dev/odyssey/tests/mod.rs index ff2f655b9..96ea31208 100644 --- a/src/dev/odyssey/tests/mod.rs +++ b/src/dev/odyssey/tests/mod.rs @@ -88,7 +88,7 @@ pub(super) fn board(product: &str) -> (Box, Arc) { let rec = Arc::new(Recorder::default()); let mut s = Board::new(product, VPRO_WIDGET).expect("board"); s.test_screen(); - s.front.lock().irq = Some(rec.clone()); + s.front.lock().packets = Some(super::interface::heart_sender(rec.clone())); (s, rec) } diff --git a/src/dev/odyssey/tests/monitor.rs b/src/dev/odyssey/tests/monitor.rs index 01c33271c..9fdbae56f 100644 --- a/src/dev/odyssey/tests/monitor.rs +++ b/src/dev/odyssey/tests/monitor.rs @@ -27,6 +27,12 @@ fn board() -> Arc { v } +/// The base of the board's large IP30 window, where a processor's stores +/// reach its aperture. +fn large() -> u64 { + crate::ip30::widget_large_base(WIDGET) +} + /// The monitor's writer: what a command wrote, read back after it. #[derive(Clone, Default)] struct Sink(Arc>>); @@ -289,9 +295,9 @@ fn rec_starts_marks_and_stops_a_recording_from_the_monitor() { // Two command stores and a front-register load through the bus, as a // processor makes them; then a pixel lands and a mark takes it. - assert_eq!(v.write32(v.large + cmd::FIFO, cmd::REGISTER_TAG | reg::STROBE), crate::traits::BUS_OK); - assert_eq!(v.write32(v.large + cmd::FIFO, cmd::REGISTER_TAG | reg::STROBE), crate::traits::BUS_OK); - let status0 = v.read32(v.large + ODSY_STATUS0); + assert_eq!(v.write32(large() + cmd::FIFO, cmd::REGISTER_TAG | reg::STROBE), crate::traits::BUS_OK); + assert_eq!(v.write32(large() + cmd::FIFO, cmd::REGISTER_TAG | reg::STROBE), crate::traits::BUS_OK); + let status0 = v.read32(large() + ODSY_STATUS0); assert!(status0.is_ok()); pixel_write(v.state.test_board(), 3, 4, 0x1234_5600); let h1 = memory_hash_bytes(v.state.test_board()); @@ -353,12 +359,12 @@ fn trace_needs_gputrace_and_then_writes_the_classes_asked_for() { // A GL begin and three integer vertices through the kernel FIFO port: the // words are FIFO lines, the third vertex completes a triangle (a DRAW // line); three identical status polls are one CPU line. - let fifo = v.large + cmd::FIFO; + let fifo = large() + cmd::FIFO; for word in [cmd::REGISTER_TAG | reg::PATTERN_MODE, 1 << 13, cmd::REGISTER_TAG | reg::GL_BEGIN, gl_cmd::TRIANGLES, gl_cmd::VERTEX_2I, 10, 20, gl_cmd::VERTEX_2I, 30, 20, gl_cmd::VERTEX_2I, 10, 40] { assert_eq!(v.write32(fifo, word), crate::traits::BUS_OK); } for _ in 0..3 { - assert!(v.read32(v.large + ODSY_STATUS0).is_ok()); + assert!(v.read32(large() + ODSY_STATUS0).is_ok()); } assert_eq!(ok(&v, "trace flush").trim(), "trace flushed"); let text = std::fs::read_to_string(&path).unwrap(); From 3e70cb71e749f17cac88609f52a941a9e9052a02 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 04:22:15 -0700 Subject: [PATCH 19/22] ip35: fit the VPro board in the Fuel's graphics slot Every Fuel has a VPro graphics slot: a Fuel IRIX inventory lists its board under xtalk/13 and OpenBSD finds Odyssey at crossbar widget 13. Build the board there when the machine is given a V12 or a V10, through the widget-offset constructor, and keep none, the serial machine, as the default. The XBridge now reports link 13 alive, the board's DMA enters Bedrock's inbound path as widget 13, and a link reset of port 13 resets it. Its interrupt packets cross the crossbar to Bedrock like the PCI heads'. The status bar's activity (SCSI, IOC3 and Ethernet) and the CPU's Compare ticks reach its LEDs as on the Octane. With PROM 6.210 the board passes the graphics diagnostic, and IRIX 6.5.30 with the VPro driver lists Graphics board: V12 and draws the X login screen on it. Input comes from the PS/2 pair the Fuel already has. Configuration accepts the board on the Fuel too, see the config note below. Tests build a Fuel board with each board and check that the processor reaches the board at widget 13 and finds its link alive, that its interrupt packets set and clear a processor interface's pending bit, that a link reset of its port resets it, that a headless Fuel has nothing there, and that the processor interfaces keep their graphics registers. Config (config): A Fuel is reported with both the V12 and the V10, so take either with [graphics] board; the other boards stay refused, with a message that names the Fuel's choices, and none, the serial machine, stays the default. The doc comments for the two boards now say they fit the Fuel as well as the Octane. The Fuel's validation case now accepts both boards and refuses the V6. --- src/config.rs | 28 +++++---- src/ip34_machine.rs | 132 +++++++++++++++++++++++++++++++++++++----- src/ip35/tests.rs | 137 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 274 insertions(+), 23 deletions(-) diff --git a/src/config.rs b/src/config.rs index e68b10881..b6b8c7ef4 100644 --- a/src/config.rs +++ b/src/config.rs @@ -893,14 +893,14 @@ pub enum GraphicsBoard { /// VPro V8: BUZZ A, 128 MB. Octane only. #[clap(name = "v8", alias = "vpro:v8", alias = "vpro_v8")] V8, - /// VPro V10: BUZZ B, 32 MB. Octane only. It shipped only in the Octane2 - /// and needs Crossbow 1.4, so the machine takes the Octane2's Crossbow and - /// Bridge revisions. + /// VPro V10: BUZZ B, 32 MB. Octane and Fuel. On the Octane it shipped + /// only in the Octane2 and needs Crossbow 1.4, so the machine takes the + /// Octane2's Crossbow and Bridge revisions. #[clap(name = "v10", alias = "vpro:v10", alias = "vpro_v10")] V10, - /// VPro V12: BUZZ B, 128 MB. Octane only. It shipped only in the Octane2 - /// and needs Crossbow 1.4, so the machine takes the Octane2's Crossbow and - /// Bridge revisions. + /// VPro V12: BUZZ B, 128 MB. Octane and Fuel. On the Octane it shipped + /// only in the Octane2 and needs Crossbow 1.4, so the machine takes the + /// Octane2's Crossbow and Bridge revisions. #[clap(name = "v12", alias = "vpro:v12", alias = "vpro_v12")] V12, /// No graphics board: the machine runs on its serial console. Octane @@ -967,7 +967,7 @@ impl GraphicsBoard { match profile { MachineProfile::OctaneIp30 => self.is_impact() || self.is_vpro() || self == Self::None, MachineProfile::Origin200Ip29 | MachineProfile::Origin2000Ip27 | MachineProfile::Onyx2Ip27 => self == Self::None, - MachineProfile::FuelIp34 => self == Self::None, + MachineProfile::FuelIp34 => matches!(self, Self::V10 | Self::V12 | Self::None), MachineProfile::Indigo2Ip22 => !self.is_vpro() && self != Self::None, _ => !self.is_vpro() && !matches!(self, Self::Extreme | Self::None), } @@ -2714,6 +2714,8 @@ impl MachineConfig { format!("graphics.board \"{name}\" is not modelled on the Origin 2000 (valid: none)") } else if self.machine.profile == MachineProfile::Onyx2Ip27 { format!("graphics.board \"{name}\" is not modelled on the Onyx2 (valid: none)") + } else if self.machine.profile == MachineProfile::FuelIp34 { + format!("graphics.board \"{name}\" is not a Fuel board (valid: v12, v10, none)") } else { format!("graphics.board \"{name}\" is only valid on the Octane") }); @@ -2823,8 +2825,8 @@ impl MachineConfig { if self.banks[..2] != [256, 256] { return Err("Fuel currently requires banks = [256, 256] (512 MiB)".into()); } - if board != GraphicsBoard::None { - return Err("Fuel currently supports graphics.board = \"none\" only".into()); + if !board.supports(MachineProfile::FuelIp34) { + return Err(format!("graphics.board \"{}\" is not a Fuel board (valid: v12, v10, none)", board.as_str())); } if let Some(text) = &self.fuel.l1_revision { crate::dev::l1::parse_revision(text).map_err(|e| format!("fuel.l1_revision: {e}"))?; @@ -4555,7 +4557,7 @@ mod fuel_input_tests { } #[test] - fn fuel_requires_its_fixed_memory_population_and_no_graphics() { + fn fuel_requires_its_fixed_memory_population_and_a_vpro_or_no_graphics() { let mut cfg = MachineConfig::default(); cfg.machine.profile = MachineProfile::FuelIp34; cfg.banks.fill(0); @@ -4569,6 +4571,12 @@ mod fuel_input_tests { cfg.validate().unwrap(); cfg.graphics.board = Some(GraphicsBoard::Newport); assert!(cfg.validate().unwrap_err().contains("none")); + for board in [GraphicsBoard::V12, GraphicsBoard::V10] { + cfg.graphics.board = Some(board); + cfg.validate().unwrap(); + } + cfg.graphics.board = Some(GraphicsBoard::V6); + assert!(cfg.validate().unwrap_err().contains("not a Fuel board")); cfg.graphics.board = None; cfg.banks[1] = 0; assert!(cfg.validate().unwrap_err().contains("256, 256")); diff --git a/src/ip34_machine.rs b/src/ip34_machine.rs index 7bb8952b2..3d1dc3742 100644 --- a/src/ip34_machine.rs +++ b/src/ip34_machine.rs @@ -4,21 +4,33 @@ //! Selected with `[machine] profile = "fuel_ip34"`. Shared keys select the //! PROM, NVRAM, memory and SCSI devices; `[fuel]` selects the serial console //! and writable flash. `ip35::Ip35Bus` maps the processor views over the -//! same Physical memory banks. Graphics is not attached, and the IOC3 has -//! no keyboard or mouse ports. +//! same Physical memory banks. The IOC3 carries PS/2 keyboard and mouse ports +//! (default) and the OHCI controller can take a USB pair; without graphics the +//! machine runs on a serial console. With `[graphics] board = "v12"` (or +//! `"v10"`) a VPro board answers at crossbar widget 13, the Fuel's graphics +//! slot. //! //! The Bedrock node itself, and its approximations, are in `ip35::node`. //! +//! # What is approximated here +//! +//! The processor interfaces' graphics flow-control registers store what +//! IRIX writes, which is what its VPro driver checks, but count no stores: +//! no uncached store into the graphics page takes a credit and the board +//! returns none, so the credit-starvation interrupt never fires. The board's +//! DMA reaches memory through Bedrock's inbound path as widget 13. +//! //! Board composition ported from hw/mips/sgi/sn1-node.c //! in jamesbraid/qemu-sgi-origami. -use crate::config::{scsi_label, FuelConsole, MachineConfig}; +use crate::config::{scsi_label, FuelConsole, GraphicsBoard, MachineConfig}; use crate::cpu::mips_exec::{CpuDevice, MipsCpu, MipsCpuConfig, MipsExecutor}; use crate::cpu::mips_tlb::MipsTlb; use crate::dev::bedrock::{Bedrock, Population}; use crate::dev::bedrock_flash::{FirmwareRecord, Flash}; use crate::dev::heart::{host_clock, Clock}; use crate::dev::ioc3::{Ioc3, Ioc3Config}; +use crate::dev::odyssey::{Odyssey, PacketSender}; use crate::dev::ioc3_eth::Ioc3Eth; use crate::dev::isp12160::Isp12160; use crate::dev::l1::{L1, L1Config}; @@ -28,7 +40,7 @@ use crate::dev::prom_image; use crate::dev::ps2::Ps2Controller; use crate::dev::serial_host::{null_serial_backend, open_log, tcp_serial_backend, CiSerialBackend, LogMode, SerialBackend, TeeBackend}; use crate::dev::timekeeper::Timekeeper; -use crate::dev::xbridge::XBridge; +use crate::dev::xbridge::{InterruptPacket, XBridge}; use crate::pci::{BoardIrq, DevFn}; use crate::devlog::{init_devlog, LogModule}; use crate::gdb_stub::CpuDebug; @@ -94,10 +106,15 @@ impl FuelMachine { revision: cfg.fuel.l1_revision.as_deref().map(crate::dev::l1::parse_revision).transpose()?, brick_serial: cfg.fuel.brick_serial.clone(), }, + graphics: cfg.graphics_board(), + graphics_pool: (cfg.graphics.threads, cfg.graphics.tile), + graphics_raster: (cfg.graphics.raster, cfg.graphics.gpu_backend.as_str()), })?); let timers = Arc::new(TimerManager::new()); let part = cfg.cpu(); let mut exec = board.build_cpu(part); + // The CPU's Compare matches blink the graphics status bar's LED. + exec.core.fasttick_count = board.fasttick.clone(); exec.core.set_timer_manager(timers.clone()); exec.core.set_count_hz( cfg.clock @@ -109,6 +126,8 @@ impl FuelMachine { let poll_board = board.clone(); exec.batch_hook = Some(Box::new(move |_| poll_board.poll())); let cpu: Arc = Arc::new(MipsCpu::<_, R12000ShadowCache>::new(*exec)); + board.status.set_cpu_cycles(cpu.cycles_ptr()); + board.status.set_count_hz(cpu.count_hz_atomic()); connect_cpus("Fuel", &board.bedrock, &board.l1, &board.events, std::slice::from_ref(&cpu))?; board.bedrock.set_timer_manager(&timers); // After the windows above: each compile worker takes the cache's @@ -150,11 +169,9 @@ impl FuelMachine { monitor.register_device(PerfMonitor::new( cpu.running_flag(), cpu.cycles_ptr(), cpu.fasttick_count(), None, None, )); - monitor.register_device(board.bedrock.physical().clone()); - monitor.register_device(board.l1.clone()); - monitor.register_device(board.scsi.clone()); - // `ps2 type` and friends: the keyboard and mouse from the monitor. - monitor.register_device(board.ioc3.ps2()); + for device in board.monitor_devices() { + monitor.register_device(device); + } cpu.register_locks(); board.scsi.register_locks(); board.ioc3.register_locks(); @@ -170,8 +187,10 @@ impl FuelMachine { } let events = board.receiver.lock().take(); eprintln!( - "iris: Fuel IP34, {}, 512 MiB, serial console {:?}", - cfg.cpu().label(), cfg.fuel.console + "iris: Fuel IP34, {}, 512 MiB, {}, serial console {:?}", + cfg.cpu().label(), + if board.vpro.is_some() { cfg.graphics_board().label() } else { "no graphics" }, + cfg.fuel.console ); Ok(Self { board, @@ -298,6 +317,9 @@ impl Workstation for FuelMachine { self.board.eth.power_on(); self.board.scsi.power_on(); self.board.usb.power_on(); + if let Some(vpro) = &self.board.vpro { + vpro.power_on(); + } self.timekeeper.power_on(); self.restart_peripherals(); } @@ -334,7 +356,7 @@ impl Workstation for FuelMachine { self.lifecycle = state; } fn get_display(&self) -> Option> { - None + self.board.vpro.as_ref().map(|vpro| vpro.display()) } fn register_monitor_device(&self, device: Arc) { self.monitor.register_device(device); @@ -458,6 +480,15 @@ pub(crate) struct FuelBoard { pub eth: Arc, pub scsi: Arc, pub usb: Arc, + /// The VPro board in the graphics slot, widget 13. + pub vpro: Option>, + /// The status bar's activity bits and the CPU's Compare matches, which + /// the graphics board shows. + pub heartbeat: Arc, + pub fasttick: Arc, + /// What the graphics board's status bar shows: the activity bits, the + /// Compare matches and the CPU's cycle count and Count rate. + pub status: Arc, pub serial: [Arc; 3], #[cfg(test)] pub irq: Arc, @@ -478,6 +509,12 @@ pub(crate) struct FuelBoardConfig<'a> { pub usb_input: bool, /// Replacements for the L1's reported identity; unset keeps its defaults. pub l1: L1Identity, + /// `None`, `V12` or `V10`; validation refuses the others. + pub graphics: GraphicsBoard, + /// The VPro's fragment pipeline: worker count and tile size (`[graphics]`). + pub graphics_pool: (usize, u32), + /// Which unit draws and, for the GPU, the wgpu backend (`[graphics]`). + pub graphics_raster: (crate::config::RasterUnit, &'a str), } /// What a Fuel config may replace in the L1's identity replies. @@ -502,11 +539,14 @@ impl FuelBoard { ps2_input: true, usb_input: false, l1: L1Identity::default(), + graphics: GraphicsBoard::None, + graphics_pool: (2, 32), + graphics_raster: (crate::config::RasterUnit::Cpu, "auto"), }) } pub fn with_config(config: FuelBoardConfig<'_>) -> Result { - let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity } = config; + let FuelBoardConfig { payload, clock, flash_path, station_mac, timekeeper, ps2_input, usb_input, l1: l1_identity, graphics, graphics_pool, graphics_raster } = config; let mut l1_config = L1Config { mac: station_mac, ..Default::default() }; if let Some(revision) = l1_identity.revision { l1_config.revision = Some(revision); @@ -614,6 +654,35 @@ impl FuelBoard { Isp12160::new_bus(), Isp12160::new_bus(), )); + let heartbeat = Arc::new(AtomicU64::new(0)); + let fasttick = Arc::new(AtomicU64::new(0)); + scsi.set_heartbeat(heartbeat.clone()); + ioc3.set_heartbeat(heartbeat.clone()); + eth.set_heartbeat(heartbeat.clone()); + let status = crate::disp::StatusSource::new(heartbeat.clone(), fasttick.clone()); + let vpro = if graphics == GraphicsBoard::None { + None + } else { + // Interrupt packets leave through the crossbar as the heads' do; + // DMA enters Bedrock as this widget's inbound traffic. + let crossbar = Arc::downgrade(&xbridge); + let packets: PacketSender = Arc::new(move |widget, address, vector, level| { + crossbar.upgrade().is_some_and(|x| x.send_interrupt(InterruptPacket { + source_widget: WIDGET_VPRO, + widget, + address, + vector, + level, + })) + }); + let dma = bedrock.widget_dma(WIDGET_VPRO); + let board = Odyssey::new_widget_relative( + WIDGET_VPRO, graphics.as_str(), packets, dma, status.clone(), graphics_pool, graphics_raster, + )?; + xbridge.attach_widget(WIDGET_VPRO, board.clone()); + bedrock.connect_widget(WIDGET_VPRO, xbridge.widget(WIDGET_VPRO).expect("attached widget")); + Some(board) + }; let pci = xbridge.bus(15); let slot = |device| DevFn { device, function: 0 }; pci.attach(slot(1), scsi.clone()); @@ -637,6 +706,7 @@ impl FuelBoard { let reset_eth = eth.clone(); let reset_scsi = scsi.clone(); let reset_usb = usb.clone(); + let reset_vpro = vpro.as_ref().map(Arc::downgrade); xbridge.set_reset_output(Arc::new(move |port| { if port == 15 { reset_ioc3.power_on(); @@ -644,6 +714,12 @@ impl FuelBoard { reset_scsi.power_on(); reset_usb.power_on(); } + // A link reset of the graphics port resets the board behind it. + if port == WIDGET_VPRO { + if let Some(vpro) = reset_vpro.as_ref().and_then(|v| v.upgrade()) { + vpro.power_on(); + } + } })); Ok(Self { bus, @@ -658,6 +734,10 @@ impl FuelBoard { eth, scsi, usb, + vpro, + heartbeat, + fasttick, + status, serial, #[cfg(test)] irq, @@ -689,7 +769,26 @@ impl FuelBoard { self.eth.start(); self.scsi.start(); self.usb.start(); + if let Some(vpro) = &self.vpro { + vpro.start(); + } + } + /// The board's devices that take monitor commands: the physical map, the + /// L1, the SCSI controller, the keyboard and mouse (`ps2 type` and + /// friends) and, when one is fitted, the VPro (`vpro …`). + pub fn monitor_devices(&self) -> Vec> { + let mut devices: Vec> = vec![ + self.bedrock.physical().clone(), + self.l1.clone(), + self.scsi.clone(), + self.ioc3.ps2(), + ]; + if let Some(vpro) = &self.vpro { + devices.push(vpro.clone()); + } + devices } + pub fn poll(&self) { if self.running.load(Ordering::Acquire) { let now = (self.clock)(); @@ -704,6 +803,9 @@ impl FuelBoard { self.poll(); self.running .store(false, Ordering::Release); + if let Some(vpro) = &self.vpro { + vpro.stop(); + } self.usb.stop(); self.scsi.stop(); self.eth.stop(); @@ -713,6 +815,10 @@ impl FuelBoard { } } +/// The crossbar widget of the Fuel's graphics slot: a Fuel IRIX inventory +/// lists its VPro under xtalk/13, and OpenBSD finds Odyssey at widget 13. +pub const WIDGET_VPRO: u8 = 13; + impl Drop for FuelBoard { fn drop(&mut self) { self.stop(); diff --git a/src/ip35/tests.rs b/src/ip35/tests.rs index 796f84fd1..268fe1808 100644 --- a/src/ip35/tests.rs +++ b/src/ip35/tests.rs @@ -258,6 +258,9 @@ fn the_configured_station_address_reaches_the_l1_and_the_ethernet() { timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input: true, l1: Default::default(), + graphics: crate::config::GraphicsBoard::None, + graphics_pool: (2, 32), + graphics_raster: (crate::config::RasterUnit::Cpu, "auto"), }).unwrap(); assert_eq!(board.l1.station_mac(), mac); assert_eq!(board.eth.mac(), mac); @@ -273,6 +276,9 @@ fn the_l1_identity_overrides_reach_the_controller() { station_mac: [8, 0, 0x69, 0x12, 0x34, 0x56], timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input: false, l1, + graphics: crate::config::GraphicsBoard::None, + graphics_pool: (2, 32), + graphics_raster: (crate::config::RasterUnit::Cpu, "auto"), }).unwrap().l1.config(); let stock = build(L1Identity::default()); assert_eq!(stock.revision, Some([1, 48, 1])); @@ -617,6 +623,9 @@ fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), ps2_input: true, usb_input, l1: Default::default(), + graphics: crate::config::GraphicsBoard::None, + graphics_pool: (2, 32), + graphics_raster: (crate::config::RasterUnit::Cpu, "auto"), }).unwrap(); // Power each port: one with a device shows it connected. board.usb.bar_write(0, 0x54, 0x100, 4); @@ -636,3 +645,131 @@ fn the_fuel_plugs_a_usb_keyboard_and_mouse_only_when_configured_on() { assert_eq!(status.map(|s| s & 1), [0, 0], "nothing plugged in"); } +fn fuel_with_graphics(graphics: crate::config::GraphicsBoard) -> crate::ip34_machine::FuelBoard { + crate::ip34_machine::FuelBoard::with_config(crate::ip34_machine::FuelBoardConfig { + payload: &[], clock: Arc::new(|| 0), flash_path: None, + station_mac: [0x08, 0x00, 0x69, 0x21, 0x43, 0x65], + timekeeper: Arc::new(crate::dev::timekeeper::Timekeeper::fuel().unwrap()), + ps2_input: true, usb_input: true, + l1: Default::default(), + graphics, + graphics_pool: (2, 32), + graphics_raster: (crate::config::RasterUnit::Cpu, "auto"), + }).unwrap() +} + +/// The VPro's link status in the crossbar, in its slot's data half. +const VPRO_LINK_STATUS: u64 = 0x100 + 5 * 0x40 + 0x10 + 4; +/// A processor access to widget 13 through Bedrock's small window. +const VPRO_PIO: u64 = 13 << 24; + +// The VPro cases follow tests/qtest/sn1-fuel-vpro-test.c in +// jamesbraid/qemu-sgi-origami: widget-identity, profile-identity-and-routing +// and headless-fuel-has-no-board. + +// VPro host-interface registers the tests below touch, as byte offsets in +// the board's widget space. The board's own constants are private to it. +const ODSY_IDENTITY: u64 = 0x00; +const ODSY_CONTROL: u64 = 0x20; +const ODSY_CONTROL_WIDGET_MASK: u32 = 0xf; +const ODSY_INT_DEST_LOWER: u64 = 0x38; +const ODSY_INTR_CLEAR: u64 = 0x101c; +const ODSY_INTR_ENABLE_SET: u64 = 0x1034; +const ODSY_INTR_FIFO_LOW_WATER: u32 = 0x200; +const ODSY_ROUTE_DEST_UPPER: u64 = 0x1044; +const ODSY_ROUTE_DEST_LOWER: u64 = 0x104c; +const ODSY_ROUTE_VECTORS: u64 = 0x1054; +const ODSY_ROUTE_DEST_WIDGET_SHIFT: u32 = 25; + +/// A Fuel IRIX inventory lists its VPro under xtalk/13: the processor must +/// reach the board there, learn its widget number from it, and find the +/// link alive in the crossbar. +#[test] +fn the_vpro_answers_at_widget_13_with_its_link_alive() { + for (graphics, revision) in [(crate::config::GraphicsBoard::V12, 0xa), (crate::config::GraphicsBoard::V10, 0xa)] { + let board = fuel_with_graphics(graphics); + let identity = board.bedrock.pio().read32(VPRO_PIO | (ODSY_IDENTITY + 4)); + assert_eq!(identity.status, BUS_OK); + assert_eq!(identity.data as u32, (revision << 28) | 0x0c01_3047, "{graphics:?}"); + let control = board.bedrock.pio().read32(VPRO_PIO | (ODSY_CONTROL + 4)).data as u32; + assert_eq!(control & ODSY_CONTROL_WIDGET_MASK, 13); + assert_ne!(board.xbridge.crossbar().read32(VPRO_LINK_STATUS).data & (1 << 31), 0); + } +} + +/// The monitor's `vpro` commands resolve on a Fuel that has a VPro, and are +/// unknown on one that has none. +#[test] +fn the_monitor_resolves_vpro_commands_on_a_fuel_with_a_vpro() { + let resolves = |board: &crate::ip34_machine::FuelBoard, command: &str| { + board.monitor_devices().iter().any(|d| d.register_commands().iter().any(|(c, _)| c == command)) + }; + let with = fuel_with_graphics(crate::config::GraphicsBoard::V12); + assert!(resolves(&with, "vpro"), "a Fuel with a V12 answers `vpro`"); + assert!(resolves(&with, "ps2"), "the other board commands stay"); + let without = fuel_with_graphics(crate::config::GraphicsBoard::None); + assert!(!resolves(&without, "vpro"), "a Fuel with no VPro has no `vpro`"); + assert!(resolves(&without, "ps2")); +} + +/// The board's interrupt packets cross the XBridge to Bedrock as the PCI +/// heads' do: a raised source sets its vector's pending bit in the +/// processor interface the destination names, and a cleared one clears it. +#[test] +fn vpro_interrupt_packets_set_and_clear_a_processor_interface_pending_bit() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + // Register space (bit 47), the node's hub window and PI0's pending + // modify register, at the Bedrock's crossbar port 0xa. + let packet = (1u64 << 47) | 0x180_0000 | 0x90; + let vector = 5; + let write = |offset: u64, value: u64| { + assert_eq!(board.bedrock.pio().write32(VPRO_PIO | offset, value as u32), BUS_OK, "{offset:#x}"); + }; + write(ODSY_ROUTE_DEST_UPPER, (0xa << ODSY_ROUTE_DEST_WIDGET_SHIFT) | (packet >> 32)); + write(ODSY_ROUTE_DEST_LOWER, packet & 0xffff_ffff); + write(ODSY_ROUTE_VECTORS, vector); + write(ODSY_INTR_ENABLE_SET, ODSY_INTR_FIFO_LOW_WATER as u64); + let pending = || pi_regs(&board, 0 as u64).read64(0x98).data & (1 << vector); + assert_ne!(pending(), 0); + write(ODSY_INTR_CLEAR, ODSY_INTR_FIFO_LOW_WATER as u64); + assert_eq!(pending(), 0); +} + +/// A link reset of the graphics port resets the board behind it, as the +/// PCI port's does for the functions behind its head. +#[test] +fn a_graphics_link_reset_resets_the_vpro() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + let destination = VPRO_PIO | (ODSY_INT_DEST_LOWER + 4); + assert_eq!(board.bedrock.pio().write32(destination, 0x55 as u32), BUS_OK); + assert_eq!(board.bedrock.pio().read32(destination).data, 0x55); + let link_reset = 0x100 + 5 * 0x40 + 0x30 + 4; + assert_eq!(board.xbridge.crossbar().write32(link_reset, 1), BUS_OK); + assert_eq!(board.bedrock.pio().read32(destination).data, 0); +} + +/// Without graphics nothing answers at widget 13 and its link is down, as +/// on a headless Fuel. +#[test] +fn a_fuel_without_graphics_has_no_widget_13() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::None); + assert!(board.vpro.is_none()); + assert_ne!(board.bedrock.pio().read32(VPRO_PIO | 4).status, BUS_OK); + assert_eq!(board.xbridge.crossbar().read32(VPRO_LINK_STATUS).data & (1 << 31), 0); +} + +/// IRIX's VPro driver writes each processor interface's graphics credit +/// counter at attach and tears the board down if it reads back zero; the +/// registers must keep what the guest writes, per interface. +#[test] +fn processor_interface_graphics_registers_keep_their_writes() { + let board = fuel_with_graphics(crate::config::GraphicsBoard::V12); + // Slot A's credit counter and bias; PI1 keeps its own block. + assert_eq!(pi_regs(&board, 0 as u64).write64(0x308, 100), BUS_OK); + assert_eq!(pi_regs(&board, 0 as u64).write64(0x310, 0x1000 | 68), BUS_OK); + assert_eq!(pi_regs(&board, 0 as u64).read64(0x308).data, 100); + assert_eq!(pi_regs(&board, 0 as u64).read64(0x310).data, 68); + assert_eq!(pi_regs(&board, 1 as u64).read64(0x308).data, 0); + board.bedrock.power_on(); + assert_eq!(pi_regs(&board, 0 as u64).read64(0x308).data, 0); +} From 2f61d8857c2296a5cb364eccdd8e249d190edb61 Mon Sep 17 00:00:00 2001 From: James Braid Date: Fri, 30 Oct 2026 08:13:37 +0000 Subject: [PATCH 20/22] docs: describe the Fuel machine Document the supported serial configuration, image inputs, the host input options and the explicit limits for graphics, USB devices and snapshots, in the feature list, hacking notes, debug notes, the sample config and a Fuel page. The page also says how the PROM's stack in primary data-cache lines is served: by the CPU's staged-line support, with nothing special in the memory controller. The hacking notes say where Bedrock and its node live, and use the chip as the worked case of the lock rules: atomic interrupt inputs, a publish into the CPU's interrupt word that takes no lock, and comparators on host timers. --- CHANGELOG.md | 12 ++++++ FEATURES.md | 33 ++++++++++++---- HACKING.md | 74 +++++++++++++++++++++++++---------- debug.md | 9 ++++- docs/fuel-ip34.md | 98 +++++++++++++++++++++++++++++++++++++++++++++++ fuel.toml.example | 44 +++++++++++++++++++++ 6 files changed, 242 insertions(+), 28 deletions(-) create mode 100644 docs/fuel-ip34.md create mode 100644 fuel.toml.example diff --git a/CHANGELOG.md b/CHANGELOG.md index 22ca48c68..64aebb4d9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -520,6 +520,18 @@ where they differed: - Memory past 4 GB is as fast as the rest: such a machine gets a 16 GB direct window, and the JIT compiles code anywhere in it. - A store-conditional on memory outside the window is now atomic with two CPUs. + +### SGI Fuel (IP34 board, IP35 software platform) + +- **The Fuel takes a VPro board**: `[graphics] board = "v12"` or `"v10"` + fits the Octane's VPro model at crossbar widget 13. IRIX 6.5.30 with the + VPro driver brings the X login screen up on it. + +- Added a headless Fuel profile with the 500 MHz R14000, 512 MiB RAM, + Bedrock, dual-head XBridge, IOC3 networking and two-channel ISP12160 SCSI. + The normal interpreter boot reaches the IRIX 6.5.30 root shell with PROM + 6.210. See the [Fuel guide](docs/fuel-ip34.md). + ### PCI configuration - A PCI function can describe its own writable header fields and diff --git a/FEATURES.md b/FEATURES.md index e3d747504..577c25d7c 100644 --- a/FEATURES.md +++ b/FEATURES.md @@ -175,8 +175,8 @@ the first `[section]`, because TOML tables continue until the next header. | Key | Default | Values and behavior | |---|---|---| -| `machine.profile` | `indy_ip24` | indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32 (configured by `[o2]`, see [the O2 guide](docs/o2-ip32.md)), octane_ip30 (configured by `[octane]`, see [the Octane guide](docs/octane-ip30.md)), origin200_ip29 (configured by `[origin200]`, see [the Origin 200 guide](docs/origin200-ip29.md)), origin2000_ip27 (configured by `[origin2000]`, see [the Origin 2000 guide](docs/origin2000-ip27.md)), onyx2_ip27 (also configured by `[origin2000]`, see [the Onyx2 guide](docs/onyx2-ip27.md)). | -| `machine.cpu` | the machine's first | A family (`r4400`, `r5000`, `r10000`, `r12000`, `r14000`) or a part (`r14000_600`); independent of build features. Indy and `indigo2_ip22`: r4400, r5000; `indigo2_ip28`: r10000; O2: r5000, r10000, r12000; Octane: r10000, r12000, r14000; Origin 200, Origin 2000 and Onyx2: r10000 (`r10000_180`), r12000 (`r12000_270`). The first listed is the default. | +| `machine.profile` | `indy_ip24` | indy_ip24, indigo2_ip22, indigo2_ip28, o2_ip32 (configured by `[o2]`, see [the O2 guide](docs/o2-ip32.md)), octane_ip30 (configured by `[octane]`, see [the Octane guide](docs/octane-ip30.md)), origin200_ip29 (configured by `[origin200]`, see [the Origin 200 guide](docs/origin200-ip29.md)), origin2000_ip27 (configured by `[origin2000]`, see [the Origin 2000 guide](docs/origin2000-ip27.md)), onyx2_ip27 (also configured by `[origin2000]`, see [the Onyx2 guide](docs/onyx2-ip27.md)), fuel_ip34 (see [the Fuel guide](docs/fuel-ip34.md)). | +| `machine.cpu` | the machine's first | A family (`r4400`, `r5000`, `r10000`, `r12000`, `r14000`) or a part (`r14000_600`); independent of build features. Indy and `indigo2_ip22`: r4400, r5000; `indigo2_ip28`: r10000; O2: r5000, r10000, r12000; Octane: r10000, r12000, r14000; Origin 200, Origin 2000 and Onyx2: r10000 (`r10000_180`), r12000 (`r12000_270`); Fuel: r14000 (`r14000_500`). The first listed is the default. | | `machine.cpus` | `1` | Processors fitted. 2 only on the Octane; see [its guide](docs/octane-ip30-multiprocessor.md). | ### [origin2000] @@ -192,11 +192,30 @@ Origin 2000 settings apply when `machine.profile = "origin2000_ip27"` or `"onyx2 An `[origin2000]` table is written to a saved config only when one of these keys is set. See the [Origin 2000 guide](docs/origin2000-ip27.md). +### [fuel] + +Fuel-specific settings apply when `machine.profile = "fuel_ip34"`. + +| Key | Default | Values and behavior | +|---|---|---| +| `flash` | unset | Persistent 16 MiB boot-flash file. Unset, the flash is volatile. | +| `console` | `l1` | `l1`, `ioc3a` or `ioc3b`. Selects the guest console connection. | +| `l1_port` | `8882` | Host TCP port for the L1 console. Used when `console = "l1"`. | +| `l1_log` | unset | Optional serial log path for L1 output when an IOC3 port is selected as console. | +| `l1_revision` | `1.48.1` | L1 firmware revision as `major.minor.patch`, decimal. The guest reads it with L1 request `0x0005`. | +| `brick_serial` | `EMUN00` | L1 brick serial, at most 37 bytes. The guest reads it with L1 request `0x0004`. The default is a marked synthetic serial, not a real one. | + +The machine uses one 500 MHz R14000 and fixed 512 MiB RAM in two 256 MiB +banks. Its IOC3 has no keyboard or mouse ports, so it runs on a serial +console. A `[fuel]` table is written to a saved config only when one of +these keys is set. The sample configuration and supported PROM inputs are in the +[Fuel guide](docs/fuel-ip34.md). + ### [graphics] | Key | Default | Values and behavior | |---|---|---| -| `graphics.board` | `newport`; `solidimpact` on the Octane | newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12, none. Extreme is Indigo2 only. The Octane takes only `solidimpact` (SI), `highimpact` (SI with texture option), `maximpact` (MXI), `v6`, `v8`, `v10`, `v12` (the VPro boards, also `vpro:v12` and so on; need an IRIX install with the VPro driver, IRIX 6.5.10 or later and PROM 4.0, or 4.5 for the V10 and V12, which shipped only in the Octane2 and so give the machine the Octane2's Crossbow 1.4 and Bridge revision) or `none` (no board: the machine runs on its serial console, with no window). | +| `graphics.board` | `newport`; `solidimpact` on the Octane | newport, xz, extreme, solidimpact, highimpact, maximpact, v6, v8, v10, v12, none. Extreme is Indigo2 only. The Octane takes only `solidimpact` (SI), `highimpact` (SI with texture option), `maximpact` (MXI), `v6`, `v8`, `v10`, `v12` (the VPro boards, also `vpro:v12` and so on; need an IRIX install with the VPro driver, IRIX 6.5.10 or later and PROM 4.0, or 4.5 for the V10 and V12, which shipped only in the Octane2 and so give the machine the Octane2's Crossbow 1.4 and Bridge revision) or `none` (no board: the machine runs on its serial console, with no window). The Fuel takes `v12` or `v10` in its graphics slot, or `none`, its default. | | `graphics.heads` | `1` | 1 or 2; dual-head requires Newport. | | `graphics.resolution` | `guest` | guest, 1024x768, 1280x960, 1280x1024; presets require Newport. | | `graphics.threads` | `2` | Worker threads the board's fragment pipeline draws on, at least 1. VPro: the screen tiles' owners, besides its front-end and raster threads; more help on a host with cores to spare. | @@ -231,7 +250,7 @@ An explicit non-disk kind takes precedence over the legacy cdrom flag. | Key | Default | Values and behavior | |---|---|---| -| `network.mode` | `nat` | nat, pcap, stream or off (no Ethernet; O2 and Octane only). Stream needs `network.stream`. PCAP needs build support and host capture permissions. PCAP works on every Ethernet; stream on every onboard one, not on a DaynaPort. | +| `network.mode` | `nat` | nat, pcap, stream or off (no Ethernet; O2, Octane and Fuel only). Stream needs `network.stream`. PCAP needs build support and host capture permissions. PCAP works on every Ethernet; stream on every onboard one, not on a DaynaPort. | | `network.stream` | `unset` | `tcp:IP:PORT` or `unix:PATH`: the server of the private segment for `mode = "stream"` (QEMU `-netdev stream` framing). See NETWORKING.md. | | `network.pcap_interface` | `auto-pick when unset` | Interface name or numeric list index; PCAP only. | | `network.nfs_pcap_ip` | `unset` | Virtual LAN IP for the built-in NFS responder in PCAP mode. | @@ -419,8 +438,8 @@ IP22/IP24 four-bank IRIX 6.5 behavior remains separate from IP28's verified two-bank 1 GB layout. Extreme requires IP22. All non-Newport boards require `heads=1` and `resolution="guest"`. IP28 IRIX needs IMPACT; its PROM has an embedded fallback. -The config validator permits SCSI controller 1 only on IP22, the O2 and the -Octane, the machines with two controllers. +The config validator permits SCSI controller 1 only on IP22, the O2, Octane +and Fuel. These machines have two controllers. Standalone CLI options update the loaded config where supplied; boolean enable switches do not provide a general way to clear every saved true setting. @@ -473,7 +492,7 @@ iris-gui does not parse this standalone option set; use its named-machine UI. | `--list-net-interfaces` | Print the host network interfaces libpcap can bridge onto, then exit. Requires a build with --features pcap. | | `--no-scsi-deferred-int` | Disable deferred SCSI status interrupts (default: enabled for OpenBSD/NetBSD compatibility). | | `--cpu` | Emulated CPU: a family (`r4400`, `r5000`, `r10000`, `r12000`, `r14000`) or a part (`r14000_600`) the machine takes. Overrides `[machine] cpu`. | -| `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on, Octane only). | +| `--graphics` | Graphics board: `newport` (default), `xz`, `extreme`, `solidimpact`, `highimpact`, `maximpact` (or `impact:solid`, `impact:high`, `impact:max`), `v6`, `v8`, `v10`, `v12` (`vpro:v12` and so on, Octane; `v10`/`v12` on the Fuel). | | `--raster` | VPro and IMPACT: `cpu` (default) or `wgpu`, the raster unit on the host GPU (`[graphics] raster`; needs a build with the `wgpu` feature). | | `--renderer` | What draws the window: `gl` (default) or `wgpu` (`renderer`; needs a build with the `wgpu` feature). | | `--gdb-port` | Enable GDB stub on the given TCP port (e.g. --gdb-port 1234). Connect with: target remote localhost: | diff --git a/HACKING.md b/HACKING.md index 28795b501..dc6068c75 100644 --- a/HACKING.md +++ b/HACKING.md @@ -16,13 +16,18 @@ Other documents: ## 1. What IRIS is -IRIS emulates five SGI workstations: the Indy (IP24), the Indigo2 (IP22), the -Indigo2 IMPACT (IP28), the O2 (IP32) and the Octane (IP30). The CPU is an -R4400, R5000, R10000, R12000 or R14000, chosen at runtime from the parts the -machine shipped with (`src/cpu/parts.rs`). -Graphics is Newport (REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and -Indigo2, GBE on the O2 and Solid IMPACT on the Octane. It boots the real PROMs -and IRIX 5.3 and 6.5 to a desktop with networking. +IRIS emulates the Indy (IP24), Indigo2 (IP22), Indigo2 IMPACT (IP28), O2 +(IP32), Octane (IP30) and Fuel (IP34 board, IP35 software platform). The CPU +is an R4400, R5000, R10000, R12000 or R14000, chosen at runtime from the +parts the machine shipped with (`src/cpu/parts.rs`); the Fuel has its +500 MHz R14000. Graphics is Newport +(REX3), GR2 (XZ/Extreme) or IMPACT on the Indy and Indigo2, GBE on the O2, and +Solid IMPACT on the Octane. Fuel has no graphics board and runs headless on +its serial console. It does have a PS/2 keyboard and mouse on its IOC3, fed +from the host's input and the default as in the Fuel's manual, and `[fuel] +usb_input = true` adds a USB keyboard and mouse on its OHCI controller; +`ps2_input = false` turns PS/2 off. The graphical machines boot their real +PROMs and IRIX 5.3 or 6.5 to a desktop with networking. It is **not cycle-accurate**, anywhere. IRIX is interrupt-driven and does not care how many cycles an instruction takes, so IRIS models behaviour @@ -83,17 +88,18 @@ image. A CD-ROM's node is a `ChangerNode`: the queue of discs and the open active one, with the `Removable` trait (present, changed, eject, load) separate from `BlockNode` so that removable media which is not a block device, a tape, can use it too. The WD33C93A (Indy, Indigo2), the Adaptec -7880 (O2) and the ISP1040 (Octane) each make an empty bus of their own kind -(`new_bus`); the machine fills it from config and gives it to the controller's -constructor, so a controller exists with its targets and keeps only its -registers and DMA. What a `BlockNode` must guarantee about durable writes, -and why a disk's `write_cache` setting matters, is in the "Disk write cache" -section of [STORAGE.md](STORAGE.md) and the `block_node.rs` module docs: read -them before writing a backend or changing a flush. `src/scsi_monitor.rs` implements the monitor's `scsi` and `cow` +7880 (O2) and ISP1040 (Octane) each make an empty bus of their own kind +(`new_bus`). Fuel uses an ISP12160 with two SCSI buses behind one PCI +function. The machine fills each bus from config and gives it to the controller +constructor, so the controller keeps its registers and DMA. What a `BlockNode` +must guarantee about durable writes, and why a disk's `write_cache` setting +matters, is in the "Disk write cache" section of [STORAGE.md](STORAGE.md) and +the `block_node.rs` module docs: read them before writing a backend or changing +a flush. `src/scsi_monitor.rs` implements the monitor's `scsi` and `cow` subcommands and the run-time disc changes once, over the bus a controller hands -out through `ScsiTargets::scsi_bus`; chip-specific subcommands (`regs`; the -WD's `wdt`, `defer`, `debug`) stay with the chip, and `DiscChanger` is -implemented once for every `ScsiTargets`. Configuration is shared the same +out through `ScsiTargets::scsi_bus`. Chip-specific subcommands stay with the +chip, including `regs` and the WD's `wdt`, `defer` and `debug`. `DiscChanger` +is implemented once for every `ScsiTargets`. Configuration is shared the same way: `scsi::attach_from_config` walks the `[scsi.N]` tables for every machine, where the key is a slot (controller × 16 + id, see `config::scsi_slot`), and the machine supplies only a route from controller number to bus. @@ -104,8 +110,9 @@ their own command state. `ScsiTarget::bus_reset` defaults to no action. Disk targets record reset as unit attention. Only the ISP12160 calls it; the other controllers keep their earlier behaviour of not reporting resets to targets. -The ISP12160 releases its controller state lock before walking either bus. -Target callbacks run under that bus lock and must not reacquire it. +The ISP12160 defers target resets and releases its controller state lock +before walking the bus. It calls targets while holding the bus lock, so a +target callback must not take that lock again. ## 2. Data path and endianness @@ -311,9 +318,12 @@ Every device that has real work to do runs on its own thread: | `VNC-Grab`, `VNC-Pointer`, `VNC-Client`/`VNC-Read` | with `--vnc`: frame diffing, pointer steering, one pair per viewer | | `machine-events` | power-off and reset requests from devices | | `ISP1040-0`, `ISP1040-1` | Octane SCSI request rings and disk transfers | -| `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane serial input per port; the DS1685 NVRAM file writer (IOC3's other deadlines are `hptimer` one-shots) | +| `ISP12160-0` | Fuel SCSI request ring for both ISP12160 channels | +| `IOC3-RX-A`, `IOC3-RX-B`, `IOC3-NVRAM` | Octane and Fuel IOC3 serial input and NVRAM writer (IOC3's other deadlines are `hptimer` one-shots) | +| `l1-serial` | Fuel L1 console UART worker. Its other serial ports are IOC3 A and B | | `ip30-eth-rx`, `ip30-eth-net` | Octane ethernet receive pump; the network backend | | `odyssey-work`, `odyssey-raster`, `odyssey-pool-N`, `odyssey-display` | VPro: the front end decodes the queued command-FIFO stores (the processor's stores only queue); the raster unit sets each draw up and walks its primitives into 2x2 quads; a pool of workers (`[graphics] threads`, default 2) walks, shades and stores filled triangles and stores every other fragment, one screen tile at a time; and the display composes each whole frame from board memory (64-bit words, as main memory is: a field is a bit range of its word, read by shift and mask with no byte order, `vram.rs`) as the scan-out registers describe it, counts the retrace and presents the window at 60 Hz. No board-wide lock: the front end owns its state, an access that needs the drawing is answered busy until it is done, the display FIFO (PB&J's register file) has its own lock and never waits for the drawing (`display_fifo.rs`), and the display reads board memory while it is drawn, as the board's own scan-out does; nobody tracks which rows changed. The raster thread sets primitives up and writes each as one command in a ring of 64-byte slots (`pipe/ring.rs`): a triangle's record and bounding box, a quad it shaded itself (lines, points), or a clear's rectangle; the slot's index goes on the queue of every screen tile (`[graphics] tile`, default 32 pixels) it touches. A worker takes a ready tile and owns it until its queue is empty, walking each triangle within the tile and shading and storing quad by quad, so every pixel sees its fragments in command order with no other ordering, and the depth pre-test reads depth its own tile stored. A slot is retired once all its tiles have run it. The raster thread drains the ring before any other work on board memory (a texture upload waits for the shader alone). A completion (a fence's token, a query's words) does not drain: it rides the ring and runs once the ring retires past the quads queued before it. A quad's four pixels are lanes: the walk's depth, the texture unit's positions and the colour weights run on the host's SIMD unit (`fearless_simd`, `pipe/lanes.rs`, the level detected at start-up), bit for bit the scalar arithmetic. When its queue runs dry the raster thread does not drain: it keeps one count of the raster unit's pending jobs for the quads still in flight, and a front-end wait or a refused processor read asks it to store them. `IRIS_ODSY_SHADER=inline` makes the raster thread shade every quad itself, and `IRIS_ODSY_BACKEND=inline` stores on the raster thread with no pool. The texture registers a draw reads live in an array of draw state slots (`pipe/slot.rs`): the front end writes the current slot, a draw latches it and holds it until its job is given back, and a write while a queued draw holds the current slot first copies it to a free one, so the stages read their draw's registers with no lock | +| `ioc3-eth-rx`, `ioc3-eth-net` | Fuel IOC3 Ethernet receive pump and network backend | Threads that have nothing to do park instead of spinning: the graphics consumers when their FIFO is empty, and the CPU thread in `idle-pause` builds @@ -368,6 +378,16 @@ When a device has several locks, document their order next to them and keep to it. Graphics recording is an example: `rec` before `trace`, and the consumer thread never takes `rec`. +Bedrock shows rule 4 at the scale of a whole chip. Each processor +interface's interrupt inputs (pending banks, masks, timer and error status) +are atomics, and a store or an inbound packet changes one and then publishes +the slot's levels into Cause bits 10-14 of its CPU's interrupt word with no +lock: a writer that published a stale level re-reads the inputs and corrects +it, so the last write always matches the registers. A mask store, an +interrupt packet, a counter read, inbound DMA and PIO take no lock. The +comparators fire on host timers, as the Hub's do, not from the CPU's batch +hook. + ## 6. The CPU ### State and execution @@ -753,6 +773,20 @@ introspection commands are in [debug.md](debug.md#jit-v2). Crossbow and the Bridge with IOC3 and two ISP1040 SCSI controllers, an R10000, and IMPACT graphics (see the [Octane guide](docs/octane-ip30.md) for the boards). +- **IP34 board, IP35 software platform (Fuel)**: one 500 MHz R14000, 512 MiB + in two 256 MiB banks, Bedrock, two XBridge heads, IOC3 and a two-channel + ISP12160. It runs headless on the L1 or IOC3 serial console. The IOC3 uses + the shared `net::Link` for NAT or PCAP. The writable Bedrock flash and + serial-flash devices share `FlashBacking` storage, with separate command + handling. A VPro board (the Octane's Odyssey model) can sit at crossbar + widget 13; its packets and DMA go through the XBridge and Bedrock's inbound + path, and the PI graphics registers are stored but count no credits. + Bedrock is one device (`src/dev/bedrock/`): its two PIs, memory controller, + I/O, local-bus, network and crossbar blocks are plain fields, and a block's + access returns what it changed elsewhere for the chip to act on. The node + around it (the processor bus, flash, L1 and Hub-special page) is + `src/ip35/node.rs`. Snapshots are unsupported. See the + [Fuel guide](docs/fuel-ip34.md). The Indy and Indigo2 profiles each have an embedded PROM fallback; `src/platform.rs` and `src/machine.rs` wire their differences, and diff --git a/debug.md b/debug.md index 65900e3f1..d635152e6 100644 --- a/debug.md +++ b/debug.md @@ -368,6 +368,13 @@ widget's link, refused Crossbow accesses, and accesses to physical addresses nothing is mapped at, with the PC; a device's own refusal is not listed). The `phys` commands work on the Octane as well. +On the Fuel the one controller is a two-channel ISP12160: `scsi` (or +`scsi0`) and `cow` act on channel 0, `scsi1` and `cow1` on channel 1, and +ids run 0 to 15 on each (`[scsi.17]` is id 1 on channel 1). `scsi notes` +shows the whole chip's command total and anything odd the driver did, and +has no `scsi1` form; `regs`, `wdt`, `defer`, `debug` and `dayna` are not +there. + Queue position `[0]` is the mounted disc, `[1]` the next on eject. A disc change reaches IRIX as a Unit Attention ("medium changed") on its next TEST UNIT READY; IRIX's `mediad` may still need a manual remount @@ -574,7 +581,7 @@ a forced termination; a normal stop saves them automatically. |---|---| | `phys mem [n]` / `mm` | Dump physical memory | | `phys dis [n]` / `md` | Disassemble physical memory | -| `phys map` | The decode table: pages by kind (untouched, filled but not yet mapped, shared, private), memory in use, slots faulted in, and the table as runs of devices. Every machine on `Physical` has the `phys` commands (Indy, Indigo2, O2, Octane) | +| `phys map` | The decode table: pages by kind (untouched, filled but not yet mapped, shared, private), memory in use, slots faulted in, and the table as runs of devices. Every machine on `Physical` has the `phys` commands (Indy, Indigo2, O2, Octane, Fuel) | | `phys trace ` / `trace ` | Log physical bus accesses | | `phys error ` | Stop on bus errors | | `phys hole ` | Stop on accesses to unmapped addresses | diff --git a/docs/fuel-ip34.md b/docs/fuel-ip34.md new file mode 100644 index 000000000..3622c313c --- /dev/null +++ b/docs/fuel-ip34.md @@ -0,0 +1,98 @@ +# SGI Fuel (IP34 board, IP35 software platform) + +Fuel is the IP34 board running the IP35 software platform. The emulated +configuration has one 500 MHz R14000 and 512 MiB of RAM in two fixed 256 MiB +banks. It includes Bedrock, two XBridge heads, an IOC3, and an ISP12160 with +two SCSI channels. The machine runs headless on its L1 or IOC3 serial console. + +## Configuration + +Copy [`fuel.toml.example`](../fuel.toml.example) to `fuel.toml`, provide the +PROM and disk files named there, then run `iris --config fuel.toml` from the +repository root. The sample selects `machine.profile = "fuel_ip34"` and +`machine.cpu = "r14000"`. The machine always runs its one R14000 (`r14000_500`, +the 500 MHz part); `cpu` may be left out, and any other value is refused. The sample +sets `banks = [256, 256]`, `headless = true`, `no_audio = true`, serial +ports 18880 and 18881, and `graphics.board = "none"`. The `[fuel]` settings +select the flash backing file, console, and optional L1 port and log. The +console options are `l1`, `ioc3a` and `ioc3b`. The Fuel's IOC3 has no +keyboard or mouse ports, so the console is always serial. + +The L1 answers firmware revision requests with 1.48.1 and brick serial +requests with the synthetic `EMUN00`; both are model choices, not values read +from a Fuel. `[fuel] l1_revision = "major.minor.patch"` and +`[fuel] brick_serial = "..."` replace them. Its system serial request returns +the machine's Ethernet address (`[network] mac`) as twelve uppercase +hexadecimal digits, the text the PROM converts back into that address. +IRIX's `numastatd` reads the revision at startup; without one it logs +`SNDRV_GET_ELSC_VERSION: I/O error`. + +The PROM decoder accepts a raw payload or a supported version-six, +single-segment stored or gzip container. The sample attaches a raw IRIX disk +at SCSI ID 1 and writes guest changes to its overlay. `[network] mode` defaults +to `nat`. `off` disconnects the link. `pcap` uses the shared `net::Link` backend +with a PCAP-enabled build and host capture support. The example sets `off`. +Set it to `nat` for the built-in gateway. + +## Observed boot and limits + +With the normal interpreter configuration, PROM 6.210 reports +`ALL DIAGS PASSED` and IRIX 6.5.30 has reached the root shell. The guest +reports a 500 MHz R14000 with 512 MiB, detects `ef0`, and passes the observed +NAT networking checks. During initial boot, IRIX's file indexer can keep the +guest busy while the serial console shows little activity. + +The PROM uses the R10000-family primary data cache as temporary memory before +it configures DRAM: it tags lines for the 32 KiB window `0x100000`-`0x107fff` +with Index Store Tag and installs them with Index Store Data, so its stack has +no memory behind it. The CPU model keeps those staged lines and serves cached +loads and stores that hit them, ahead of memory, until the PROM invalidates +them or the machine powers on; the memory controller has no special case for +the window. Without them the PROM takes a data bus error on its first stack +access. The O2's PROM does the same for its POST stack. The model's +approximations are listed in the cache-view note, +[`clanker/cpu/cache-lines-live-in-the-cpus-view.md`](../clanker/cpu/cache-lines-live-in-the-cpus-view.md). + +The ordinary interpreter configuration is the one with recorded Fuel boot +evidence. `lightning` and experimental `jitv2` builds are available, but no +Fuel guest boot with either is claimed here. + +## Graphics + +`[graphics] board = "v12"` (or `"v10"`) fits a VPro board in the graphics +slot, crossbar widget 13: a Fuel IRIX inventory lists its board under +xtalk/13, and Fuels have been reported with both the V12 and the V10. It is +the Octane's VPro model. Its interrupt packets cross the XBridge to Bedrock +like the PCI heads', its DMA enters Bedrock's inbound path as widget 13, and +a link reset of port 13 resets it. The processor interfaces' graphics +flow-control registers keep what IRIX writes but count no stores, so the +credit-starvation interrupt never fires. + +With PROM 6.210 the graphics diagnostic finds the board and passes. An IRIX +6.5.30 disk whose kernel includes the VPro driver lists `Graphics board: V12` +in `hinv` and brings the X login screen up on it. A disk installed for a +machine without graphics has no VPro driver in its kernel and runs as before. + +## Input + +The IOC3 carries a PS/2 keyboard and mouse, fed from host input; they are the +default, as in the Fuel's manual. `[fuel] usb_input = true` also plugs a USB +keyboard and mouse into the OHCI controller, and `ps2_input = false` leaves the +PS/2 pair out. Machine snapshots are unsupported. + +## Source basis + +The literal `0x53474972756c6573` is named `IIO_PROTECT_OVRRD_KEY` in public +Linux revision `db2ddb87143519e20a95aa36c60b36107b736a58`, +`arch/mips/include/asm/sn/sn0/hubio.h`. This establishes the literal's public +provenance, not the mapping of each Fuel register offset that uses it. Those +offsets remain approximate. The board composition port follows the project's +BSD-licensed QEMU SGI model in `hw/mips/sgi/sn1-node.c`. That source is a BSD-licensed implementation +reference, not proof of hardware behavior. + +The Hub-special head windows are compatibility approximations. Reads return +zero and writes are acknowledged without side effects. PROM 6.210 writes the +64-bit word `0x2000` at relative offsets `0x0` and `0x40` before discovering +DRAM; their hardware meaning remains unknown. Bedrock logging records +accesses. This keeps the existing model behavior explicit rather than +claiming a register implementation. diff --git a/fuel.toml.example b/fuel.toml.example new file mode 100644 index 000000000..8ac609a09 --- /dev/null +++ b/fuel.toml.example @@ -0,0 +1,44 @@ +# SGI Fuel (IP34 board, IP35 software platform), serial-only setup. +# PROM 6.210 and IRIX 6.5.30 have been observed reaching the root shell. +# Copy to fuel.toml and run from the repository root: iris --config fuel.toml +# Create ignore/fuel/ and supply your decoded raw PROM payload and IRIX disk. +# PROM also accepts supported version-six single-segment stored/gzip containers. +# No PROM or disk images are included. Convert QCOW2 disks to raw first. + +prom = "ignore/fuel/ip35prom.payload.bin" +nvram = "ignore/fuel/nvram.bin" +banks = [256, 256] +headless = true +no_audio = true +monitor_port = 18888 +serial_port_a = 18880 +serial_port_b = 18881 + +[machine] +profile = "fuel_ip34" +# The Fuel's one processor, the R14000 at 500 MHz with a 2 MiB secondary +# cache (`r14000_500`); other values are refused. Unset takes the same. +cpu = "r14000" + +[graphics] +# "v12" or "v10" fits a VPro in the graphics slot; IRIX needs a kernel with +# the VPro driver. The console stays on serial either way. +board = "none" + +[fuel] +flash = "ignore/fuel/flash.bin" +console = "l1" +l1_port = 18882 + +[network] +# Disconnected cable. For NAT, use mode = "nat". +# Keep the installed disk's working ef0 address and default route. +# NAT can adopt its static gateway before routing begins. +# Initial defaults: guest 192.168.0.2/24, gateway 192.168.0.1. +mode = "off" + +[scsi.1] +path = "ignore/fuel/irix.raw" +cdrom = false +# Keep the base disk unchanged. Guest writes go to irix.raw.overlay. +overlay = true From 06ba0ec87efc1a2f6195e863a276a5e6a9d05848 Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 12:25:00 -0700 Subject: [PATCH 21/22] ppmem: let a machine hold cache lines where no memory is mapped yet The Fuel's and Tezro's PROMs keep their stack in primary data lines for 0x100000-0x107fff and run code from primary instruction lines at the PROM's own addresses before they have configured memory. The cached view refuses a page with no memory behind it, so those lines would hold nothing and the PROM would read the bus. allow_unbacked_lines names ranges where a page may be held with nothing mapped behind it. Such a page starts as what the cache's bus shows there, and its region is the view's while it stands, as for any held page. A remap keeps the page, so memory arriving mid-phase does not lose the stack. Memory Bedrock has only withdrawn from the direct path still backs a page. jitv2 stays off a held page by the mark ppmem puts on the page's generation counter, but only on ppmem's own counters. The IP35 boards' compiles check other ones: Bedrock's for RAM, and for the reset PROM that the bus maps over memory at 0x1fc00000 the flash's, which is neither ppmem's nor Bedrock's. A page held there carried no mark, so jitv2 could compile the image's bytes for a page whose lines hold other words, as a PROM that stages code in instruction lines at its own address makes it. The cache now puts the same mark on whatever counter its bus gives the page when that is not ppmem's, and takes it off with the page's last line. That runs at CACHE instructions only; loads, stores and fetches are unchanged. --- .../cpu/cache-lines-live-in-the-cpus-view.md | 8 +- src/cpu/mips_cache_shadow.rs | 87 ++++++++++++++++++- src/ppmem/ppmem.rs | 41 ++++++++- 3 files changed, 131 insertions(+), 5 deletions(-) diff --git a/clanker/cpu/cache-lines-live-in-the-cpus-view.md b/clanker/cpu/cache-lines-live-in-the-cpus-view.md index d4a157db0..a2830778d 100644 --- a/clanker/cpu/cache-lines-live-in-the-cpus-view.md +++ b/clanker/cpu/cache-lines-live-in-the-cpus-view.md @@ -77,9 +77,11 @@ A page some CPU holds is not compiled: compiles read memory, and compiled code i shared. ppmem puts `GEN_HELD` on the page's generation counter at the first hold and takes it off at the last release, moving it with the page on a remap (`PpMemSpace::mark`). The compile that finds it publishes nothing and denies every -offset until the release. ppmem's counters are the bus's for memory; a machine whose bus -maps a device over memory, as the IP35 boards map their reset PROM, must mark that -device's counter as well. +offset until the release. ppmem's counters are the bus's for memory. Where the bus +maps a device over memory, as the IP35 boards map their reset PROM, the compile checks +that device's counter instead, so the cache puts the same mark there when a page +becomes held and takes it off with the page's last line (`mark_foreign`). That work +is at CACHE instructions only; a load, store or fetch is unchanged. ## Primary instruction lines diff --git a/src/cpu/mips_cache_shadow.rs b/src/cpu/mips_cache_shadow.rs index 3cfccd4f5..8acfcd401 100644 --- a/src/cpu/mips_cache_shadow.rs +++ b/src/cpu/mips_cache_shadow.rs @@ -244,6 +244,10 @@ pub struct ShadowCache< /// secondary line that a refill there would load (`refilled_word`), or /// `u64::MAX`. refill: UnsafeCell>, + /// The held pages whose generation counters are not ppmem's, with the + /// counters carrying their `GEN_HELD`: see [`Self::mark_foreign`]. + #[cfg(feature = "jitv2")] + foreign: UnsafeCell>, } // Safety: the CPU thread is the only accessor, as for every other cache model @@ -303,6 +307,8 @@ impl< view_space: UnsafeCell::new(std::ptr::null()), view: UnsafeCell::new(0), refill: UnsafeCell::new(vec![u64::MAX; Self::PI_SETS].into_boxed_slice()), + #[cfg(feature = "jitv2")] + foreign: UnsafeCell::new(Vec::new()), } } @@ -512,10 +518,23 @@ impl< /// into it from the data array, since Index Store Tag writes the tag /// alone (NEC VR5000/VR10000 instruction manual pp. 24-25). A line with /// no memory behind it holds no page and stays in the arrays, as every - /// line did before. + /// line did before, unless the machine allows lines there. fn line_enters(&self, sp: &PpMemSpace, sel: u32, idx: usize, va: u64, (pa, len): (u64, usize)) { + let unbacked = !sp.backs(pa); let Some(new) = sp.hold_line_page(self.view_no(), pa) else { return }; let base = unsafe { *self.tc_base.get() }; + if new { + self.mark_foreign(sp, pa, true); + } + if new && unbacked { + // No memory behind the page yet: it starts as what the cache's + // bus shows there. + let gran = crate::ppmem::granularity() as u64; + for a in (pa & !(gran - 1)..(pa & !(gran - 1)) + gran).step_by(8) { + let r = self.downstream.read64(a); + unsafe { *(base.add(a as usize) as *mut u64) = if r.is_ok() { r.data.rotate_left(32) } else { 0 } }; + } + } if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> line {pa:#x} holds its page (new: {new})"); } @@ -559,11 +578,48 @@ impl< self.set_refill(s.tags[idx], pa, len, false); } let gone = sp.release_line_page(self.view_no(), pa); + if gone { + self.mark_foreign(sp, pa, false); + } if l2_op_log(CACHE_LOG_OP) { crate::dlog!(LogModule::L2c, "shadow: -> line {pa:#x} leaves (page released: {gone})"); } } + /// ppmem puts `GEN_HELD` on its own generation counters. Where the cache's + /// bus counts a held page's writes on a counter of its own, which compiles + /// check instead (the Fuel's reset PROM, mapped over memory), the mark + /// goes there too, and comes off with the page's last line. A page + /// outside any counter, or on ppmem's, is skipped. + #[cfg_attr(not(feature = "jitv2"), allow(unused_variables))] + fn mark_foreign(&self, sp: &PpMemSpace, pa: u64, held: bool) { + #[cfg(feature = "jitv2")] + { + use crate::cpu::jitv2::jitv2::GEN_HELD; + use crate::ppmem::MappedMemory; + use std::sync::atomic::{AtomicU64, Ordering}; + let page = pa & !(crate::ppmem::granularity() as u64 - 1); + let foreign = unsafe { &mut *self.foreign.get() }; + if held { + for a in (page..page + crate::ppmem::granularity() as u64).step_by(4096) { + let g = self.downstream.gen_ptr(a); + let own = unsafe { sp.gen_window_base().add((a >> 12) as usize) as *const AtomicU64 }; + if !g.is_null() && g != own { + unsafe { (*g).fetch_add(GEN_HELD, Ordering::Release) }; + foreign.push((page, g)); + } + } + } else { + foreign.retain(|&(p, g)| { + if p == page { + unsafe { (*g).fetch_add(1u64.wrapping_sub(GEN_HELD), Ordering::Release) }; + } + p != page + }); + } + } + } + /// Index Store Data into a held line writes through to its page, so the /// next cached access sees it. Same quadword rule as the data array. fn store_into_held_line(&self, sel: u32, va: u64, value: u64) { @@ -1803,4 +1859,33 @@ mod tests { c.cache_op(C_IST | CACH_SD, 0x8000_4f00, 0); assert_eq!(c.cache_op(C_R10K_ILD | CACH_PI, set(0x10), 0), 0x3_4420_0004, "gone with the line: the array"); } + + /// A page the cache's bus counts on a counter of its own (a ROM mapped + /// over memory) carries one `GEN_HELD` there however many lines hold it, + /// and none once they go. A counter that is ppmem's is left to ppmem. + #[cfg(feature = "jitv2")] + #[test] + fn a_held_page_marks_the_buss_own_counter() { + use crate::cpu::jitv2::jitv2::GEN_HELD; + use std::sync::atomic::{AtomicU64, Ordering}; + struct Rom(AtomicU64); + impl BusDevice for Rom { + fn gen_ptr(&self, _addr: u64) -> *const AtomicU64 { &self.0 } + } + let (space, _banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let space = Box::new(space); + space.map_bank(0, 0, 64 << 20, 64 << 20).unwrap(); + let rom = Arc::new(Rom(AtomicU64::new(0))); + let c = Box::new(R10000ShadowCache::from(rom.clone() as Arc)); + attach(&c, &space); + let marks = || rom.0.load(Ordering::Relaxed) / GEN_HELD; + c.cache_op(C_IST | CACH_PD, 0x8000_0000, pd_dirty(0x2000)); + c.cache_op(C_IST | CACH_PD, 0x8000_0040, pd_dirty(0x2040)); + assert_eq!(marks(), 1, "one mark for two lines"); + c.cache_op(C_IST | CACH_PD, 0x8000_0000, 0); + assert_eq!(marks(), 1, "the page is still held"); + c.cache_op(C_IST | CACH_PD, 0x8000_0040, 0); + assert_eq!(marks(), 0, "off with the last line"); + assert_eq!(rom.0.load(Ordering::Relaxed), 1, "by one ordinary bump"); + } } diff --git a/src/ppmem/ppmem.rs b/src/ppmem/ppmem.rs index 35c94b02b..aec49d6cd 100644 --- a/src/ppmem/ppmem.rs +++ b/src/ppmem/ppmem.rs @@ -713,6 +713,9 @@ struct SpaceState { bits: u64, /// Each CPU's cached view of memory: see [`PpMemSpace::add_cached_view`]. views: Vec, + /// Ranges where a held page may stand with no memory mapped behind it: + /// see [`PpMemSpace::allow_unbacked_lines`]. + unbacked: Vec<(u64, u64)>, } /// One CPU's cached view of memory: a window of its own, with every mapping @@ -788,6 +791,7 @@ impl PpMemSpace { sink2, bits: 0, views: Vec::new(), + unbacked: Vec::new(), }), }) } @@ -1058,7 +1062,7 @@ impl PpMemSpace { let st = unsafe { &mut *self.state.get() }; let gran = granularity() as u64; let page = pa & !(gran - 1); - if !st.present().any(|m| m.at <= page && m.at + m.len >= page + gran) { + if !self.backs(page) && !st.unbacked.iter().any(|&(a, b)| page >= a && page + gran <= b) { return None; } let shift = window_shift(st.window); @@ -1085,6 +1089,23 @@ impl PpMemSpace { Some(true) } + /// Whether memory is mapped behind the whole page holding `pa`. A page + /// [`Self::hold_line_page`] takes without it starts as zeros, for the + /// caller to fill. + pub fn backs(&self, pa: u64) -> bool { + let st = unsafe { &*self.state.get() }; + let gran = granularity() as u64; + let page = pa & !(gran - 1); + st.present().any(|m| m.at <= page && m.at + m.len >= page + gran) + } + + /// Let held pages in `[start, end)` stand with no memory mapped behind + /// them: a PROM that keeps its stack and code in cache lines before it + /// has set up memory. + pub fn allow_unbacked_lines(&self, start: u64, end: u64) { + unsafe { (*self.state.get()).unbacked.push((start, end)) }; + } + /// One line of [`Self::hold_line_page`]'s page is no longer valid. /// Returns true when it was the page's last: the view shows memory there /// again, and what the page held is dropped, as a line invalidated @@ -2134,6 +2155,7 @@ mod tests { space.withdraw_range(region, region); assert_eq!(*field, 0b1101, "the view's bitmap follows the withdrawal"); assert_eq!(word(space.view_base(v), region + 0x2040), 0x1234_5678, "its bytes stay mapped"); + assert!(space.backs(region + 0x2000), "withdrawn memory is still there"); // A page held and released while the region is off the path lays the bank again. assert_eq!(space.hold_line_page(v, region + 0x2000), Some(true)); assert_eq!(*field, 0b1111, "a held page makes its region the view's"); @@ -2193,4 +2215,21 @@ mod tests { space.release_line_page(b, 0x2040); assert_eq!(gen(at()) / GEN_HELD, 0); } + + /// With no memory behind it a page is refused unless the machine allows + /// lines there, and then its region is the view's until the page goes. + #[test] + fn an_unbacked_page_needs_the_machines_leave() { + let (space, _banks) = PpMemSpace::with_bank_sizes(&[64]).unwrap(); + let (v, field) = view_of(&space); + assert_eq!(space.hold_line_page(v, 0x10_0000), None); + space.allow_unbacked_lines(0, 64 * MB as u64); + assert!(!space.backs(0x10_0000)); + assert_eq!(space.hold_line_page(v, 0x10_0000), Some(true)); + assert_eq!(*field, 1); + assert_eq!(space.mapped_bitmap(), 0, "the shared window is unchanged"); + assert_eq!(word(space.view_base(v), 0x3ff_fffc), 0, "the region is safe to touch"); + space.release_line_page(v, 0x10_0000); + assert_eq!(*field, 0); + } } From bbcb84db9c72739d1cd88ae60b342a7e70a5d9d7 Mon Sep 17 00:00:00 2001 From: James Braid Date: Sun, 8 Nov 2026 09:18:29 -0700 Subject: [PATCH 22/22] ip35: let lines stand in the Fuel CPU's bank slots The Fuel's PROM keeps its stack in primary data cache lines for 0x100000-0x107fff before it has configured any memory, so those lines have nothing behind them when they are made valid. The cached view that the CPU reads memory through can hold such lines only in places declared for it. Declare every slot that can hold a bank, as far as its largest bank reaches, as a place lines may be held before memory is mapped: on the Fuel, all four 1 GiB slots up to 256 MiB. Without the declaration the PROM's first stack access takes a data bus error. --- src/ip35/node.rs | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/src/ip35/node.rs b/src/ip35/node.rs index d92ba1225..eeb504816 100644 --- a/src/ip35/node.rs +++ b/src/ip35/node.rs @@ -15,7 +15,7 @@ use super::{map_device, Ip35Bus, Window}; use crate::cpu::mips_exec::CpuDevice; -use crate::dev::bedrock::{Bedrock, BedrockConfig, Form, Population}; +use crate::dev::bedrock::{Bedrock, BedrockConfig, Form, Population, BANK_SPAN, SLOTS}; use crate::dev::bedrock_flash::{FirmwareRecord, Flash, PART_SIZE}; use crate::dev::heart::Clock; use crate::dev::l1::{L1Config, L1Event, L1}; @@ -135,7 +135,16 @@ pub(crate) fn connect_cpus( cpus: &[Arc], ) -> Result<(), String> { if let Some(space) = bedrock.physical().ppmem_space() { - // Each CPU reads memory through a cached view of its own. + // Each CPU reads memory through a cached view of its own. The PROM + // keeps its stack in primary data lines before it has set up memory, + // so lines may stand in every slot that can hold a bank without it. + let extent = bedrock.ram_extent(); + for slot in 0..SLOTS { + if extent.prefix(slot) > 0 { + let at = slot as u64 * BANK_SPAN; + space.allow_unbacked_lines(at, at + extent.prefix(slot)); + } + } for cpu in cpus { let v = space .add_cached_view(&[cpu.ppmem_bitmap_ptr(), cpu.tcache_bitmap_ptr()])