From db8d3329ec31ffb8d781fe6df8a5c2df2ae674c4 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 09:42:29 +0900 Subject: [PATCH 01/11] server2: restore only the named members and minutes within reach A restart read the minute pages through latest-first pages, and every page call read the descriptor of every group of the day's minute catalog (up to ~720 a day). The day catalog is now listed once (control objects only), the newest acquisition of each minute within 40 minutes is kept, and only those groups are restored. CatalogStore::lookup_members reads one catalog and then only the named members' groups; lookup_acquisition is its one-member case. Reader::listed_acquisitions, moved from the image forecast, lists a catalog's acquisitions for both callers. ordered_candidates and lookup_page are unchanged for the other cursor readers. Co-Authored-By: Claude Opus 5.5 --- server2/config/tasks/KMA-RESTART-OBSERVE.json | 48 ++++ .../src/domestic/kma/source/forecast_input.rs | 55 +--- server2/src/domestic/kma/source/observe.rs | 122 ++++++-- server2/src/domestic/kma/source/records.rs | 71 ++++- server2/src/storage/catalog.rs | 58 ++-- server2/tests/domestic/observe.rs | 260 ++++++++++++++++++ server2/tests/storage/catalog_wire.rs | 86 ++++++ 7 files changed, 616 insertions(+), 84 deletions(-) create mode 100644 server2/config/tasks/KMA-RESTART-OBSERVE.json diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json new file mode 100644 index 000000000..2d2cd4e2e --- /dev/null +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -0,0 +1,48 @@ +{ + "task": "KMA-RESTART-OBSERVE", + "issue": "https://github.com/WizardFactory/TodayWeather/issues/2614", + "scope": "Observed KMA path: a restarted process restores station observations without restoring every stored page. Targeted member reads and one listing of the minute catalog; a per-day product of the parsed station pages written by any process that folds, read at restart and compared with the day catalogs, with the remaining days restored after the first answer. No deployment or production mutation.", + "server2_paths": [ + "server2/config/tasks/KMA-RESTART-OBSERVE.json", + "server2/src/cache/disk.rs", + "server2/src/domestic/kma/source/doc.rs", + "server2/src/domestic/kma/source/forecast_input.rs", + "server2/src/domestic/kma/source/mod.rs", + "server2/src/domestic/kma/source/observe.rs", + "server2/src/domestic/kma/source/observe_product.rs", + "server2/src/domestic/kma/source/records.rs", + "server2/src/storage/catalog.rs", + "server2/tests/cache.rs", + "server2/tests/domestic.rs", + "server2/tests/domestic/observe.rs", + "server2/tests/domestic/observe_product.rs", + "server2/tests/storage/catalog_wire.rs" + ], + "outside": [ + { + "path": "intent/issue-2614-kma-restart-observe.md", + "category": "documentation", + "reason": "Task intent." + }, + { + "path": "specs/issue-2614-kma-restart-observe.md", + "category": "documentation", + "reason": "Acceptance." + }, + { + "path": "plans/issue-2614-kma-restart-observe.md", + "category": "documentation", + "reason": "Plan." + }, + { + "path": "docs/architecture/server2.md", + "category": "documentation", + "reason": "The observation restart read, the station page product and its S16 note." + }, + { + "path": "docs/operations/server2-infrastructure.md", + "category": "documentation", + "reason": "The station page product is not cached on disk." + } + ] +} diff --git a/server2/src/domestic/kma/source/forecast_input.rs b/server2/src/domestic/kma/source/forecast_input.rs index 34cb97a13..086d67cc9 100644 --- a/server2/src/domestic/kma/source/forecast_input.rs +++ b/server2/src/domestic/kma/source/forecast_input.rs @@ -40,8 +40,8 @@ use crate::gateway::BoxFuture; use crate::history::IndexListing; use crate::resolver::{OwnedConfig, OwnedFlights}; use crate::storage::{ - Catalog, CatalogId, CatalogLimits, CatalogStore, Envelope, GroupDeclaration, GroupMember, - PageOutcome, Pagination, RawRecord, + CatalogId, CatalogLimits, CatalogStore, Envelope, GroupDeclaration, GroupMember, PageOutcome, + Pagination, RawRecord, }; use std::collections::BTreeMap; use std::sync::Arc; @@ -289,44 +289,19 @@ fn image_catalog(kind: &str, day: &str) -> Result { partition: day.into(), }) } -/// The acquisitions one day catalog lists: (completion instant, fetch group, a member envelope), -/// from the catalog control object alone (no record body is read). -async fn listed_acquisitions( - transport: &T, +/// The acquisitions one day catalog lists ([`Reader::listed_acquisitions`]). +async fn listed_acquisitions( + reader: &Reader, id: &CatalogId, deadline: Instant, ) -> Result, String> { - let limits = CatalogLimits::default(); - let object = match tokio::time::timeout_at( - deadline, - transport.get_control(&id.key(), limits.control_bytes), - ) - .await - { - Err(_) => return Err("image catalog deadline".into()), - Ok(Err(crate::storage::Error::NotFound)) => return Ok(Vec::new()), - Ok(Err(e)) => return Err(format!("image catalog: {e}")), - Ok(Ok(object)) => object, - }; - let catalog = Catalog::from_bytes(&object.body, id, &limits) - .map_err(|e| format!("image catalog: {e}"))?; - // One acquisition per fetch group; its completion is its members' latest fetch instant. - let mut groups: BTreeMap = BTreeMap::new(); - for e in catalog.entries { - let group = match &e.fetch_group { - Some(g) => g.descriptor_key().map_err(|e| e.to_string())?, - None => e.identity.raw_sha256.clone(), - }; - let fetched = e.identity.fetched_at_ms; - groups - .entry(group) - .and_modify(|(f, _)| *f = (*f).max(fetched)) - .or_insert((fetched, e)); - } - Ok(groups - .into_iter() - .map(|(group, (fetched, e))| (fetched, group, e)) - .collect()) + reader + .listed_acquisitions(id, deadline) + .await + .map_err(|e| match e { + crate::storage::Error::Timeout => "image catalog deadline".to_owned(), + e => format!("image catalog: {e}"), + }) } impl ImageForecasts { @@ -769,9 +744,7 @@ async fn listed( let mut out = Vec::new(); for day in window(cut).into_iter().filter(|d| listed.contains(d)) { let id = image_catalog(image::KIND, &day)?; - for (fetched, group, member) in - listed_acquisitions(&shared.reader.transport, &id, deadline).await? - { + for (fetched, group, member) in listed_acquisitions(&shared.reader, &id, deadline).await? { if fetched as f64 <= cut { out.push(((fetched, day.clone(), group), id.clone(), member)); } @@ -929,7 +902,7 @@ async fn newest_product( .map_err(|e| format!("product listing: {e}"))?; for day in days.iter().rev().filter(|d| **d >= oldest) { let id = image_catalog(PRODUCT_KIND, day)?; - let newest = listed_acquisitions(&shared.reader.transport, &id, deadline) + let newest = listed_acquisitions(&shared.reader, &id, deadline) .await? .into_iter() .max_by(|a, b| (a.0, &a.1).cmp(&(b.0, &b.1))); diff --git a/server2/src/domestic/kma/source/observe.rs b/server2/src/domestic/kma/source/observe.rs index 4da289329..d532dfc3a 100644 --- a/server2/src/domestic/kma/source/observe.rs +++ b/server2/src/domestic/kma/source/observe.rs @@ -31,7 +31,7 @@ use crate::domestic::publication::Publication; use crate::domestic::{Acquired, DomesticError}; use crate::gateway::{BoxFuture, OrderedJson}; use crate::history::{IndexListing, schedule}; -use crate::storage::{CatalogStore, ProviderKey, RawRecord}; +use crate::storage::{CatalogId, CatalogStore, ProviderKey, RawRecord}; use std::collections::{BTreeMap, BTreeSet, HashMap}; use std::sync::{Arc, Mutex}; use std::time::Duration; @@ -621,33 +621,96 @@ impl Observations { } // Minute pages of the last 40 minutes, newest first. let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); + let wanted = self.minute_entries(now_ms, deadline).await; + self.restore_minutes(wanted, floor, deadline).await; + let mut s = self.state(); + s.pages.minute.retain(|k, _| *k > floor); + s.pages.minute_read = Some(std::time::Instant::now()); + s.pages.dirty = true; + } + /// The minute pages of the readers' reach that this process does not hold, newest first by + /// completion: the day catalogs are listed (control objects only, no record is read), and of + /// each minute only its newest acquisition is kept. A catalog that cannot be listed adds + /// nothing (the next reload lists it again). + async fn minute_entries(&self, now_ms: i64, deadline: Instant) -> Vec { + let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); + let today = schedule::from_ms(now_ms); let mut partitions = vec![format!("{:08}", today.date())]; if floor.date() != today.date() { partitions.push(format!("{:08}", floor.date())); } - let mut found = Vec::new(); + let held: BTreeSet = self.state().pages.minute.keys().copied().collect(); + let mut newest: BTreeMap = BTreeMap::new(); for partition in partitions { - let read = self - .reader - .latest_raw( - "kma-web", - "aws-minute", - &aws_key(), - &partition, - 4, - |read| { - read.last() - .and_then(|a| a.first()) - .and_then(record_publication) - .is_some_and(|p| p < floor) - }, - deadline, - ) - .await; - if let Ok(read) = read { - found.extend(read); + let Ok(id) = super::records::catalog("kma-web", "aws-minute", &aws_key(), &partition) + else { + continue; + }; + let Ok(listed) = self.reader.listed_acquisitions(&id, deadline).await else { + continue; + }; + for (completed, _, e) in listed { + let Some(p) = Publication::new( + e.identity.period.local_date, + e.identity.period.slot.parse().unwrap_or(u16::MAX), + ) else { + continue; + }; + if p <= floor || held.contains(&p) { + continue; + } + let entry = MinuteEntry { + catalog: id.clone(), + member: e.identity, + completed, + }; + // Of one minute, the acquisition a latest-first read meets first. + if newest.get(&p).is_none_or(|old| entry.after(old)) { + newest.insert(p, entry); + } } } + let mut out: Vec = newest.into_values().collect(); + out.sort_by(|a, b| b.order().cmp(&a.order())); + out + } + /// Restore `entries` (one catalog read per catalog, then only their groups) and keep the pages + /// after `floor` that this process does not hold. A group that is not complete leaves its + /// minute out, not the others. + async fn restore_minutes( + &self, + entries: Vec, + floor: Publication, + deadline: Instant, + ) { + let mut by_catalog: Vec<(CatalogId, Vec)> = Vec::new(); + for e in entries { + match by_catalog.iter_mut().find(|(id, _)| *id == e.catalog) { + Some((_, members)) => members.push(e.member), + None => by_catalog.push((e.catalog, vec![e.member])), + } + } + let mut found = Vec::new(); + for (id, members) in by_catalog { + match self.reader.members(&id, &members, deadline).await { + Ok(Some(read)) => found.extend(read), + Ok(None) => { + for member in members { + if let Ok(Some(read)) = self + .reader + .members(&id, std::slice::from_ref(&member), deadline) + .await + { + found.extend(read); + } + } + } + Err(_) => {} + } + } + if found.is_empty() { + return; + } let parsed = crate::observability::spawn_blocking(move || { found .iter() @@ -661,8 +724,6 @@ impl Observations { for (p, rows) in parsed { s.pages.minute.entry(p).or_insert(rows); } - s.pages.minute.retain(|k, _| *k > floor); - s.pages.minute_read = Some(std::time::Instant::now()); s.pages.dirty = true; } /// The newest imported seed that parses and was exported after the carried one: partitions @@ -1511,6 +1572,21 @@ fn object(pairs: Vec<(&str, OrderedJson)>) -> OrderedJson { .collect(), ) } +/// A listed minute acquisition to restore. +struct MinuteEntry { + catalog: CatalogId, + member: crate::storage::RecordId, + completed: u64, +} +impl MinuteEntry { + /// Latest-first catalog order: completion, then the raw digest. + fn order(&self) -> (u64, &str) { + (self.completed, &self.member.raw_sha256) + } + fn after(&self, other: &Self) -> bool { + self.order() > other.order() + } +} type ParsedPage = (Publication, Arc>, f64, String, String); fn parse_hourly(records: &[RawRecord]) -> Option { let r = records.first()?; diff --git a/server2/src/domestic/kma/source/records.rs b/server2/src/domestic/kma/source/records.rs index 091337684..7b3395ec5 100644 --- a/server2/src/domestic/kma/source/records.rs +++ b/server2/src/domestic/kma/source/records.rs @@ -3,10 +3,13 @@ //! in page order and its fetch instant. use crate::history::IndexListing; use crate::storage::{ - CatalogId, CatalogStore, Error, LookupOutcome, PageOutcome, ProviderKey, RawRecord, ReadCursor, - ReadOrder, sha256, + Catalog, CatalogId, CatalogLimits, CatalogStore, Envelope, Error, LookupOutcome, PageOutcome, + ProviderKey, RawRecord, ReadCursor, ReadOrder, sha256, +}; +use std::{ + collections::{BTreeMap, HashMap}, + sync::Arc, }; -use std::{collections::HashMap, sync::Arc}; use tokio::time::Instant; /// One complete acquisition. @@ -43,7 +46,8 @@ impl Clone for Reader { } } } -fn catalog( +/// The catalog of one key and partition. +pub fn catalog( source: &str, kind: &str, key: &ProviderKey, @@ -184,6 +188,45 @@ impl Reader { } Ok(out) } + /// The acquisitions one catalog lists: (completion instant, fetch group, a member envelope), + /// from the catalog control object alone (no descriptor or record body is read); empty when + /// the catalog does not exist. One acquisition per fetch group; its completion is its + /// members' latest fetch instant. + pub async fn listed_acquisitions( + &self, + id: &CatalogId, + deadline: Instant, + ) -> Result, Error> { + let limits = CatalogLimits::default(); + let object = match tokio::time::timeout_at( + deadline, + self.transport.get_control(&id.key(), limits.control_bytes), + ) + .await + { + Err(_) => return Err(Error::Timeout), + Ok(Err(Error::NotFound)) => return Ok(Vec::new()), + Ok(Err(e)) => return Err(e), + Ok(Ok(object)) => object, + }; + let catalog = Catalog::from_bytes(&object.body, id, &limits)?; + let mut groups: BTreeMap = BTreeMap::new(); + for e in catalog.entries { + let group = match &e.fetch_group { + Some(g) => g.descriptor_key()?, + None => e.identity.raw_sha256.clone(), + }; + let fetched = e.identity.fetched_at_ms; + groups + .entry(group) + .and_modify(|(f, _)| *f = (*f).max(fetched)) + .or_insert((fetched, e)); + } + Ok(groups + .into_iter() + .map(|(group, (fetched, e))| (fetched, group, e)) + .collect()) + } /// The hash a key's partitions are listed under. pub fn key_hash(key: &ProviderKey) -> Result { Ok(sha256(&key.canonical_bytes()?)) @@ -286,6 +329,26 @@ impl Reader { Err(e) => Err(e), } } + /// The acquisitions of the named members of one catalog as raw records in member order (pages + /// in page order): one catalog read and the named groups only. None when a member is not + /// listed or its group is not complete. + pub async fn members( + &self, + id: &CatalogId, + members: &[crate::storage::RecordId], + deadline: Instant, + ) -> Result>>, Error> { + match self.store.lookup_members(id, members, deadline).await? { + PageOutcome::Ready(found) => Ok(Some( + found + .acquisitions() + .iter() + .map(|a| ordered(a.records())) + .collect(), + )), + PageOutcome::Incomplete => Ok(None), + } + } /// The newest acquisitions of one catalog, latest first, read in pages of `page` until /// `enough` holds for the acquisitions read so far (or the catalog ends). #[allow(clippy::too_many_arguments)] // One catalog identity plus paging and a stop rule. diff --git a/server2/src/storage/catalog.rs b/server2/src/storage/catalog.rs index 4c8631132..949f65248 100644 --- a/server2/src/storage/catalog.rs +++ b/server2/src/storage/catalog.rs @@ -468,11 +468,29 @@ impl CatalogStore { id: &CatalogId, member: &RecordId, deadline: Instant, + ) -> Result { + self.lookup_members(id, std::slice::from_ref(member), deadline) + .await + } + /// The acquisitions of the named members of one catalog, in member order (a group named + /// twice is read once): one catalog read, then only the named members' groups. Incomplete + /// when a member is not listed or its group is not complete. Never a proof of complete + /// revision coverage. + pub async fn lookup_members( + &self, + id: &CatalogId, + members: &[RecordId], + deadline: Instant, ) -> Result { id.validate()?; - member.validate()?; - if !id.matches(member) { - return Err(Error::Invalid("targeted member namespace")); + if members.is_empty() || members.len() > self.limits.entries { + return Err(Error::Invalid("targeted member count")); + } + for member in members { + member.validate()?; + if !id.matches(member) { + return Err(Error::Invalid("targeted member namespace")); + } } timeout_at(self.deadline(deadline), async { let _admission = self.admit().await?; @@ -481,21 +499,29 @@ impl CatalogStore { let Some(Some(p)) = c.pins.get(id) else { return Ok(PageOutcome::Incomplete); }; - let Some(e) = p - .catalog - .entries - .iter() - .find(|e| &e.identity == member) - .cloned() - else { - return Ok(PageOutcome::Incomplete); - }; - let Some(a) = self.acquisition(id, &e, &mut c).await? else { - return Ok(PageOutcome::Incomplete); - }; + let mut entries = Vec::with_capacity(members.len()); + for member in members { + let Some(e) = p.catalog.entries.iter().find(|e| &e.identity == member) else { + return Ok(PageOutcome::Incomplete); + }; + entries.push(e.clone()); + } + let mut groups = BTreeSet::new(); + let mut acquisitions = Vec::with_capacity(entries.len()); + for e in entries { + if let Some(r) = &e.fetch_group + && !groups.insert(r.group_sha256.clone()) + { + continue; + } + let Some(a) = self.acquisition(id, &e, &mut c).await? else { + return Ok(PageOutcome::Incomplete); + }; + acquisitions.push(a); + } Ok(PageOutcome::Ready(CompletePage { coverage: RevisionCoverage::Targeted, - acquisitions: vec![a], + acquisitions, dependencies: Self::dependencies(&c), excluded: 0, next: None, diff --git a/server2/tests/domestic/observe.rs b/server2/tests/domestic/observe.rs index 814f3e5f3..1cb22ebd0 100644 --- a/server2/tests/domestic/observe.rs +++ b/server2/tests/domestic/observe.rs @@ -1045,3 +1045,263 @@ async fn same_hour_aws_and_city_restore_respects_fetched_order_and_field_presenc assert_eq!(get(hourly, "cloud").to_bytes(), b"3"); } } + +/// Counts S3 requests at the transport boundary by method and object class (`raw/v2/source/kind`, +/// `index/v2/source/kind`, `index/v2/groups` descriptors, `list index/v2/source/kind`); every +/// request passes through to the S06 loopback peer. +#[derive(Clone)] +pub(super) struct Counted { + inner: server2::world::BoundedS3, + counts: Arc>>, +} +impl Counted { + pub(super) fn new(inner: server2::world::BoundedS3) -> Self { + Self { + inner, + counts: Arc::default(), + } + } + fn count(&self, method: &str, key: &str) { + let class = if key.starts_with("index/v2/groups/") { + "index/v2/groups".to_owned() + } else { + key.split('/').take(4).collect::>().join("/") + }; + *self + .counts + .lock() + .unwrap() + .entry(format!("{method} {class}")) + .or_default() += 1; + } + /// Requests so far of `method` and `class` (for example `GET raw/v2/kma-web/aws-minute`). + pub(super) fn get(&self, what: &str) -> usize { + self.counts.lock().unwrap().get(what).copied().unwrap_or(0) + } +} +impl server2::storage::ObjectTransport for Counted { + async fn put( + &self, + record: &server2::storage::PreparedRecord, + ) -> Result { + self.count("PUT", &record.key); + self.inner.put(record).await + } + async fn head(&self, key: &str) -> Result { + self.count("HEAD", key); + self.inner.head(key).await + } + async fn get( + &self, + key: &str, + max: usize, + ) -> Result { + self.count("GET", key); + self.inner.get(key, max).await + } +} +impl server2::storage::CatalogTransport for Counted { + async fn get_control( + &self, + key: &str, + max: usize, + ) -> Result { + self.count("GET", key); + self.inner.get_control(key, max).await + } + async fn put_control( + &self, + key: &str, + body: &[u8], + condition: &server2::storage::WriteCondition, + ) -> Result { + self.count("PUT", key); + self.inner.put_control(key, body, condition).await + } + async fn list_raw_document( + &self, + prefix: &str, + token: Option<&str>, + keys: usize, + bytes: usize, + ) -> Result, server2::storage::Error> { + self.inner + .list_raw_document(prefix, token, keys, bytes) + .await + } +} +impl server2::history::IndexListing for Counted { + async fn list_index( + &self, + prefix: &str, + token: Option<&str>, + ) -> Result { + self.count("LIST", prefix); + self.inner.list_index(prefix, token).await + } +} + +/// A KMA page record as the S13 acquirer stores it. +pub(super) fn page_record( + kind: &str, + hour: Publication, + fetched_at: u64, + body: Vec, +) -> RawRecord { + let key = server2::storage::ProviderKey::Named { + id: if kind == "city-obs" { + "city-national" + } else { + "aws-national" + } + .into(), + }; + let content = if kind == "city-obs" { + "text/html;charset=utf-8" + } else { + "text/html;charset=euc-kr" + }; + RawRecord::new( + "kma-web", + kind, + &key, + hour.period(), + fetched_at, + 200, + content, + None, + body.into(), + &CatalogLimits::default().raw, + ) + .unwrap() +} +fn one_group(record: &RawRecord) -> GroupDeclaration { + let partition = format!("{:08}", record.envelope.identity.period.local_date); + let member = GroupMember { + envelope: record.envelope.clone(), + catalogs: vec![CatalogId::for_record(&record.envelope.identity, &partition).unwrap()], + }; + GroupDeclaration::new(vec![member], &CatalogLimits::default()).unwrap() +} +/// Store many page groups as their publications leave them (descriptor, raw body, one catalog +/// entry each), writing each new day catalog once: the S3 test peer keeps 64 revisions of an +/// object, fewer than the publications of a day. +pub(super) async fn store_pages(transport: &T, records: Vec) +where + T: server2::storage::CatalogTransport + Clone, +{ + use server2::storage::{Catalog, RawRecordStore, WriteCondition}; + let limits = CatalogLimits::default(); + let raw = RawRecordStore::new(transport.clone(), limits.raw.clone()).unwrap(); + let mut catalogs: Vec = Vec::new(); + for record in records { + let d = one_group(&record); + let reference = d.reference(&limits).unwrap(); + let attached = d.attach(vec![record], &limits).unwrap(); + assert_eq!( + transport + .put_control( + &reference.descriptor_key().unwrap(), + &d.bytes(&limits).unwrap(), + &WriteCondition::Absent + ) + .await + .unwrap(), + 200 + ); + raw.publish(attached[0].clone()).await.unwrap(); + let id = d.partitions[0].clone(); + let at = match catalogs.iter().position(|c| c.identity == id) { + Some(at) => at, + None => { + catalogs.push(Catalog::empty(id)); + catalogs.len() - 1 + } + }; + catalogs[at] = catalogs[at] + .union(&[attached[0].envelope.clone()], &limits) + .unwrap(); + } + for catalog in catalogs { + assert_eq!( + transport + .put_control( + &catalog.identity.key(), + &catalog.bytes(&limits).unwrap(), + &WriteCondition::Absent + ) + .await + .unwrap(), + 200 + ); + } +} +/// `YYYY.MM.DD.HH:MM` of a publication (the page's own `pubDate`). +pub(super) fn dotted(p: Publication) -> String { + let d = p.date(); + format!( + "{:04}.{:02}.{:02}.{:02}:{:02}", + d / 10000, + (d / 100) % 100, + d % 100, + p.hour(), + p.minute() + ) +} + +/// A restart reads the minute pages of the readers' reach without reading the descriptor of every +/// minute group of the day: the day catalog is listed once and only the newest acquisition of each +/// minute within 40 minutes is restored (here 20 of 200 stored groups). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restart_restores_only_the_minute_pages_within_reach() { + use server2::domestic::kma::source::observe::Observations; + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = Peer::new(); + // 200 minute pages every two minutes up to 10:49 (legacy cadence), one group each. + let last = server2::history::schedule::to_ms(p(20260924, 1049)); + let pages = (0..200) + .map(|i| { + let at = last - i * 120_000; + let minute = server2::history::schedule::from_ms(at); + page_record( + "aws-minute", + minute, + at as u64 + 30_000, + aws_page(&dotted(minute), i == 0), + ) + }) + .collect(); + store_pages(&peer.transport_limited(4), pages).await; + let counted = Counted::new(peer.transport_limited(4)); + let obs = Arc::new(Observations::new( + counted.clone(), + Arc::new(CatalogStore::new(counted.clone(), CatalogLimits::default()).unwrap()), + Arc::new(Unreachable), + Arc::new(Unreachable), + )); + let folded = obs + .stations(NOW as i64, Instant::now() + Duration::from_secs(3)) + .await + .unwrap(); + assert!(!folded.minute.docs().is_empty(), "the newest minute rows"); + // Wait for any read the request did not wait for. + for _ in 0..40 { + if obs.settled( + NOW as i64, + &obs.stations(NOW as i64, Instant::now()).await.unwrap(), + ) { + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + let descriptors = counted.get("GET index/v2/groups"); + let raws = counted.get("GET raw/v2/kma-web/aws-minute"); + assert!( + descriptors <= 20 && raws <= 20, + "{descriptors} descriptor and {raws} raw GETs for 20 minutes within reach" + ); + assert!( + counted.get("GET index/v2/kma-web/aws-minute") <= 4, + "the minute catalogs are read a bounded number of times" + ); +} diff --git a/server2/tests/storage/catalog_wire.rs b/server2/tests/storage/catalog_wire.rs index 6e798a11a..517387617 100644 --- a/server2/tests/storage/catalog_wire.rs +++ b/server2/tests/storage/catalog_wire.rs @@ -1362,3 +1362,89 @@ async fn repair_scope_foreign_period_partition_cannot_misfile_group() { )); assert_eq!(peer.puts(&foreign.key()).await, 0); } +/// A targeted read of named members reads the catalog once and then only their groups: one +/// descriptor and one raw body per member, never the other groups' descriptors. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn targeted_members_read_one_catalog_and_only_their_groups() { + let peer = Peer::new(); + let store = CatalogStore::new(peer.transport(), CatalogLimits::default()).unwrap(); + let mut members = Vec::new(); + for fetch in 1..=30 { + let (d, raw) = singleton(fetch, 32, None); + members.push(raw[0].envelope.identity.clone()); + store.publish(d, raw, deadline()).await.unwrap(); + } + let (d, _) = singleton(1, 32, None); + let id = d.partitions[0].clone(); + let named = vec![members[29].clone(), members[3].clone(), members[17].clone()]; + let before = peer.requests("GET ").await; + let PageOutcome::Ready(page) = store.lookup_members(&id, &named, deadline()).await.unwrap() + else { + panic!("complete members") + }; + assert_eq!(peer.requests("GET ").await - before, 1 + 2 * named.len()); + let read: Vec<_> = page + .acquisitions() + .iter() + .map(|a| a.records()[0].envelope.identity.clone()) + .collect(); + assert_eq!(read, named, "member order"); + // A member the catalog does not list: nothing is answered. + let (_, unknown) = singleton(99, 32, None); + assert!(matches!( + store + .lookup_members( + &id, + &[members[0].clone(), unknown[0].envelope.identity.clone()], + deadline() + ) + .await + .unwrap(), + PageOutcome::Incomplete + )); + // A member of another kind is refused. + let mut foreign_kind = members[0].clone(); + foreign_kind.kind = "other".into(); + assert!(matches!( + store.lookup_members(&id, &[foreign_kind], deadline()).await, + Err(Error::Invalid(_)) + )); + let (other, foreign) = group(100, 2); + store + .publish(other.clone(), foreign.clone(), deadline()) + .await + .unwrap(); + // Both members of one group: the group is read once, its pages in order. + let PageOutcome::Ready(page) = store + .lookup_members( + &other.partitions[0], + &[ + foreign[1].envelope.identity.clone(), + foreign[0].envelope.identity.clone(), + ], + deadline(), + ) + .await + .unwrap() + else { + panic!("complete group") + }; + assert_eq!(page.acquisitions().len(), 1); + assert_eq!(page.acquisitions()[0].records().len(), 2); + // The store deadline holds across many members. + peer.fault("slow-all", "unused").await; + let bounded = CatalogStore::new( + peer.transport(), + CatalogLimits { + deadline: Duration::from_millis(100), + ..CatalogLimits::default() + }, + ) + .unwrap(); + let started = std::time::Instant::now(); + assert!(matches!( + bounded.lookup_members(&id, &members, deadline()).await, + Err(Error::Timeout) + )); + assert!(started.elapsed() < Duration::from_secs(1)); +} From b7b81b5a5606555c102cdb78a31f1f354c440435 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 10:41:42 +0900 Subject: [PATCH 02/11] server2: restore station observations from day products after a restart A restart restored every stored AWS hourly and city page of the 8-day window and parsed each again, about 1 s per day on a t3.small, so the first request answered without past days (#2866). AK chose the day product design on 2026-11-07. Any process that folds now stores one product per KST day (kma-web/station-pages) holding the parsed inputs of the hourly fold: the newest page per kind and hour with its fetch instant, digest and raw identity, reduced AWS rows as a facts table, the city set they are whole for and the S06 exclusions of the read. It is written after a fold that recomputed the hourly part, never awaited by a request, at most once per ten minutes per day and only when the day's pages changed. Values keep undefined, dates, negative zero and NaN. A restart reads each day's newest product and compares it with the day catalogs: unlisted product pages are dropped and only newer listed pages are restored and parsed. An unusable product (unreadable, older schema or parse revision, a grown city set) or a newer page that is not complete gives the day's raw read. The load ends after the products, today's and yesterday's comparisons, the seed and the four newest minute pages; LoadRest finishes the other days and minutes. settled() counts LoadRest and the product comparison as busy, and an excluded group keeps its day partial. Products are kept out of the disk cache. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/server2-kma-data-design.md | 7 +- docs/architecture/server2.md | 73 ++ docs/operations/server2-infrastructure.md | 4 +- intent/issue-2614-kma-restart-observe.md | 25 + plans/issue-2614-kma-restart-observe.md | 21 + server2/config/tasks/KMA-RESTART-OBSERVE.json | 7 +- server2/src/cache/disk.rs | 6 +- server2/src/domestic/kma/source/mod.rs | 1 + server2/src/domestic/kma/source/observe.rs | 842 +++++++++++++++--- .../domestic/kma/source/observe_product.rs | 461 ++++++++++ server2/tests/cache.rs | 22 + server2/tests/domestic.rs | 2 + server2/tests/domestic/observe.rs | 54 +- server2/tests/domestic/observe_product.rs | 529 +++++++++++ server2/tests/domestic/summary_s3.rs | 44 +- specs/issue-2614-kma-restart-observe.md | 10 + 16 files changed, 1939 insertions(+), 169 deletions(-) create mode 100644 intent/issue-2614-kma-restart-observe.md create mode 100644 plans/issue-2614-kma-restart-observe.md create mode 100644 server2/src/domestic/kma/source/observe_product.rs create mode 100644 server2/tests/domestic/observe_product.rs create mode 100644 specs/issue-2614-kma-restart-observe.md diff --git a/docs/architecture/server2-kma-data-design.md b/docs/architecture/server2-kma-data-design.md index 7977482d2..4e331e312 100644 --- a/docs/architecture/server2-kma-data-design.md +++ b/docs/architecture/server2-kma-data-design.md @@ -389,6 +389,7 @@ representations/v1/{source}/{kind}/{key_hash}/{day}/{manifest_sha256}.json raw/v2/kma-warning/zone-state/{YYYY}/{MM}/{DD}/... raw/v2/data-go-kr/kasi-fold/{YYYY}/{MM}/{DD}/... raw/v2/airkorea-image/airkorea-forecast-hourly/{YYYY}/{MM}/{DD}/... +raw/v2/kma-web/station-pages/{YYYY}/{MM}/{DD}/... # Deferred discovery experiments; not initial serving dependencies: index/v3/{source}/{kind}/{key_hash}/day/{YYYYMMDD}.json @@ -475,7 +476,11 @@ capture pass. These hold provider items or legacy-shaped state, not assembled we process falls back to the original objects when no usable fold exists. The AirKorea image hourly forecast is the one fold requests read without that fallback: `airkorea-forecast-hourly`, the legacy `ArpltnHourlyForecast` rows decoded from every stored image acquisition, is made by the -process timer (below), and no request decodes the GIFs. The observed response also +process timer (below), and no request decodes the GIFs. The station page product +(`station-pages`, one per KST day) holds the parsed AWS hourly and city pages any folding process +keeps; a restarted process reads it, compares it with the day catalogs and restores only the +pages it lacks, falling back to the raw pages when it cannot be used +([restart read](server2.md#s15b-2-restart-read-and-station-page-products-2026-11-07)). The observed response also uses the release's land zones (`data/kma_forecast_land_zones.csv`) until a forecast-zone list is stored, so its first answer can request mid temperatures. Measurements are in the [speed evidence](../evidence/research/kma-speed-options-2026-11-04.md#result-of-the-change). diff --git a/docs/architecture/server2.md b/docs/architecture/server2.md index fdd8ee46c..2b3866f37 100644 --- a/docs/architecture/server2.md +++ b/docs/architecture/server2.md @@ -2265,6 +2265,72 @@ from `/gather` against the retired `KmaStnMinute`; the rules are ported but not decision), and the deployed `ASOS_HISTORY_STATIONS` allowlist (every registry city station is a candidate). +### S15b-2 restart read and station page products (2026-11-07) + +The EC2 restart comparison of #2866 showed the first request after a restart without minute +observations or past days: the load restored every stored hourly and city page of the window one +day after another, parsed each again (about 1.0 to 1.4 s per day on a t3.small), and read the +minute pages last through latest-first pages that read the descriptor of every minute group of the +day. AK chose the day product design on 2026-11-07 ([intent](../../intent/issue-2614-kma-restart-observe.md), +[spec](../../specs/issue-2614-kma-restart-observe.md), [plan](../../plans/issue-2614-kma-restart-observe.md)). + +**Targeted reads.** `CatalogStore::lookup_members` reads one catalog and then only the named +members' groups (one descriptor and one raw body each); `lookup_acquisition` is its one-member +case, and `ordered_candidates`/`lookup_page` are unchanged for the other cursor readers. +`Reader::listed_acquisitions` lists a catalog's acquisitions from the control object alone and is +shared with the image forecast. The minute catalogs of the readers' 40 minutes are listed once; of +each minute only the newest acquisition (completion, then raw digest) is restored, at most four +members per read. + +**Day product.** One product per KST day D: source `kma-web`, kind `station-pages`, the AWS page +key, partition `YYYYMMDD`, period D/0000, fetched at its making instant, published as a one-member +group. It is gzip JSON of the parsed inputs of the hourly fold, not its output: `schema`, +`parseRevision` (bumped when page parsing or the reduction to registry facts changes), `made`, +`day`, `excluded`, `cityIds`, a `facts` table and `pages`. Per kind and hour of D it holds the +newest page as the process keeps it (`kind`, `hour`, `fetchedAt`, `sha256`, `rawId`, `rows`). +AWS rows equal to their registry facts are indices into `facts`; city rows are whole. `cityIds` is +the city set the reduced rows are whole for (the intersection of the sets the day's pages were +reduced with). `excluded` is the number of the day's listed groups S06 excluded when the pages +were read, carried forward while the day is compared rather than read again. Values keep every +stored value (undefined, dates, negative zero, NaN) and key order. A reader refuses a product that +expands past 16 MiB. + +Any process that folds writes products: after a fold that recomputed the hourly part a +`Flight::Product` (never awaited by a request) stores the day whose made-of (sorted page raw +identities, `cityIds`, `parseRevision`) differs from the last product this process read or stored. +Only days read without failure whose every hour is in the window are stored, each at most once per +ten minutes per process. Products are not cached on disk. + +**Restart read.** In parallel: the product, AWS hourly and city partition LISTs, the seed and the +minute catalogs. Then for the due days, newest first and eight at a time, the day's page and +product catalogs are listed (control objects only) and, the first time a process reads the day, +its newest product (completion, then digest) is read and decoded. A product whose `cityIds` does +not cover the current city set (seed and city pages) is not used. Product pages the catalogs do not +list are dropped (S3 is truth). A day is then compared: for each kind and hour whose newest listed +page (latest fetch; of one instant the smaller digest, the page a read of every acquisition keeps) +is newer than the held one, that page alone is restored and parsed. An unreadable product, an older +schema or parse revision, a grown city set, no product, or a newer page that is not complete or +does not parse gives the day's raw read as before. `Flight::Load` ends after the products, today's +and yesterday's comparisons, the seed and the four newest minute pages. `Flight::LoadRest`, which +no request waits for, then compares the other days (each marked read and the pages dirty) and +restores the rest of the minute window. Reloads every five minutes compare with the pages held. + +**S16.** Answers, the station page input identities and the response-first durability set are +unchanged: a page restored from a product carries the raw identity and digest of the stored page. +A day is partial when its held `excluded` is above zero or a newer group is not complete, as a raw +read with excluded groups is. `settled()` counts `LoadRest`, `Product` and a due product comparison +as busy. Not recorded by a product: a listed group older than its pages that is published later; if +S06 excludes it, a restart from the product counts the day complete where a raw read would not. +Right after the first deployment the window's oldest day has no product (some of its hours were +already out of the window) and is read from its raw pages after the first answer. + +Verification (local, S06 loopback S3 peer, no provider): targeted member GET counts, a restart over +200 stored minute groups, the value codec round trip, a restart with unreachable providers answering +168 past hourly pages and minute rows on its first request without raw page GETs, a later page +restored alone and equal to a cold raw read, unusable products replaced by the raw pages and stored +again, two processes' products, unsettled answers during the rest of the read, a day with an +S06-excluded group kept partial, and the disk cache exclusion. + ### S15b-2 s station registry seed (2026-10-15) AK authorized a one-time read-only export of the legacy `kmastninfos` on 2026-10-15. It ran with @@ -3034,6 +3100,13 @@ S16 also changes one S15b-2 behavior: a station-page day whose `aws-hourly` or ` is not listed now reads as empty. Before, the absent catalog's lookup answered "incomplete", so a day without city pages never loaded its hourly pages from S3 and was read again on every load. +Since the [restart read](#s15b-2-restart-read-and-station-page-products-2026-11-07) (2026-11-07) a +station-page day can be read from its day product and the day catalogs instead of every raw page. +The input identities are the same, a product carries the S06 exclusions of the read it was made +from, and a station fold is not complete while the rest of a restart read or a product comparison +is still running. +The first answer after a start therefore records no summary; a later request does. + **Last-year lookup** (`summary::Lookup::last_year`, not wired to a route). The same date a year earlier (Feb 29 → Feb 28), then ±1..±3, earlier first. At each date: the town's grid summary, then the station's ASOS summary (when missing, one data.go.kr daily range `[base-3, base+3]` through the diff --git a/docs/operations/server2-infrastructure.md b/docs/operations/server2-infrastructure.md index 2284a2876..baddb1d5f 100644 --- a/docs/operations/server2-infrastructure.md +++ b/docs/operations/server2-infrastructure.md @@ -264,7 +264,9 @@ is read by the tests or measurement tool. The cache holds verified immutable raw objects for `data-go-kr`, `kma-web`, `vc` and the exact `airkorea-image` / `airkorea-forecast-image` pair. AirKorea's image raws are included; the decoded hourly forecast (`airkorea-forecast-hourly`, made by -the process timer and read from S3) and packs are **not** cached on disk. +the process timer and read from S3), the KMA station page products +(`kma-web` / `station-pages`, made by any process that folds the station pages) and packs +are **not** cached on disk. Mutable catalogs, absence proofs, state, budgets, push tokens, geocode and exact request coordinates are excluded. Catalog/group publication checks still use S3. Filename and namespace hashes do not replace the raw codec's identity validation. diff --git a/intent/issue-2614-kma-restart-observe.md b/intent/issue-2614-kma-restart-observe.md new file mode 100644 index 000000000..8418d68e1 --- /dev/null +++ b/intent/issue-2614-kma-restart-observe.md @@ -0,0 +1,25 @@ +# Station observations after a restart + +AK requires that a request waits at most 3 s and answers from what was gathered by then. The EC2 restart comparison of #2866 showed that the first weather request after a restart misses the minute observations and the past days. The restart read restored every stored AWS hourly and city page of the 8-day window one day after another and parsed each page again: about 1.0 to 1.4 s per day on a t3.small, plus 4 to 5 s of sequential GETs. The minute pages came last, and every page call read the descriptor of every group in the day's minute catalog. The request stopped waiting at 2.5 s with neither. On 2026-11-07 AK chose the day product design ("A로 진행"). + +## Outcomes + +- **MINUTE:** a restart lists the day's minute catalog once and restores only the newest acquisition of each minute within the readers' 40 minutes. The newest four come before the first answer and the rest follow. +- **PRODUCT:** any process that folds the station pages stores one product per KST day holding the parsed inputs of the hourly fold, the newest page of each kind and hour, at most once per ten minutes per day and only when what the day is made of changed. No request waits for it. +- **RESTART:** a restarted process reads each day's newest product and compares it with the day catalogs (S3 is truth). It restores only the pages the product lacks, parses only those, and answers past days and minute rows from its first request. Today and yesterday are compared before the answer, the other days after it. +- **FALLBACK:** an unreadable product, an older schema or parse revision, or a city set that grew is replaced by the raw pages of that day, and the day's product is stored again. A page the catalog does not list is dropped. +- **S16:** legacy answers, the station page input identities and the response-first durability set are unchanged. A day stays partial when its product's read had a group S06 excluded or a newer group is not complete, and answers stay unsettled until the rest of the read ends. +- **PLACEMENT:** the placement gate and the artifact policy pass on the outgoing commits. + +## Decisions for AK + +This adds a stored object that other processes trust as a starting point, so it is not merged without AK: + +- Products are written to S3 under `kma-web` / `station-pages` beside the pages they are made of, never deleted, at most one per day and ten minutes per process. They are excluded from the disk cache, like the image forecast product. +- A day product does not record listed groups older than its pages. If such a group is published after the product and S06 excludes it, a restart from the product counts the day as complete where a raw read would not. +- The first answer after a start is never complete for S16: the days past yesterday are compared after it, so a later request records the daily summaries. A fold that recomputes the hourly part also holds summaries back until its product comparison ends. +- A process at hour H stores products only for days whose every hour is in the window. Right after the first deployment, the oldest day of the window therefore has no product and is read from its raw pages after the first answer. + +## Exclusions + +Not in this change: a real EC2 measurement of the new path, a product for the minute pages, the station ASOS state, and the collector's restart probing (KMA-SR-06 of #2862). diff --git a/plans/issue-2614-kma-restart-observe.md b/plans/issue-2614-kma-restart-observe.md new file mode 100644 index 000000000..5505191ff --- /dev/null +++ b/plans/issue-2614-kma-restart-observe.md @@ -0,0 +1,21 @@ +# Plan for station observations after a restart + +Task declaration: `server2/config/tasks/KMA-RESTART-OBSERVE.json`. Follows `plans/issue-2614-kma-image-production.md` and the EC2 restart comparison of #2866. + +1. Read named members of one catalog (`CatalogStore::lookup_members`), and share the listing of a catalog's acquisitions (`Reader::listed_acquisitions`) between the image forecast and the observations. +2. List the minute catalog once at a restart and restore only the newest acquisition of each minute within reach. +3. Store the parsed station pages of a day as one product with a lossless value codec, and keep it out of the disk cache. +4. Write the changed day products after a fold that recomputed the hourly part, never awaited by a request. +5. Read the products at a restart, compare them with the day catalogs, restore only what they lack, and fall back to the raw pages when a product cannot be used. Finish today and yesterday before the answer and the rest after it, with S16 completeness kept. +6. Update the architecture and operations documents. +7. Open a draft PR and take it through CI and cross-provider review. AK decides on the merge. + +## Scenarios + +| User | Goal | Prerequisites | Actions | Expected | Failure path | Criteria | +| --- | --- | --- | --- | --- | --- | --- | +| App user | See current and past observations right after a deployment | Products of the window stored by the previous process | Request weather from the new process | Past hours and minute rows on the first answer, within 3 s | No product for a day: that day's raw pages follow after the answer | RESTART, MINUTE | +| Operator | Restart without re-parsing every page | Products stored, pages unchanged | Start a new process | No raw page read for the product days; no product stored | A product unreadable or of an older revision: that day's raw pages are read and its product stored again | RESTART, FALLBACK | +| Operator | Two hosts serve at once | Both fold the same pages | Both write their products; a third starts | The third reads the newest product of each day and answers as both | A page only one host has: the catalog comparison restores it | RESTART | +| Summary recorder | Record only complete days | One day had a group S06 excluded | Restart from the products | The day stays partial; no summary recorded from it | The rest of the read still running: answers unsettled until it ends | S16 | +| Operator | Keep S3 writes bounded | A long-running process | Requests every few minutes | A day's product only when its pages changed, at most once per ten minutes | A failed write is tried again ten minutes later | PRODUCT | diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 2d2cd4e2e..8165f43eb 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -5,7 +5,6 @@ "server2_paths": [ "server2/config/tasks/KMA-RESTART-OBSERVE.json", "server2/src/cache/disk.rs", - "server2/src/domestic/kma/source/doc.rs", "server2/src/domestic/kma/source/forecast_input.rs", "server2/src/domestic/kma/source/mod.rs", "server2/src/domestic/kma/source/observe.rs", @@ -16,6 +15,7 @@ "server2/tests/domestic.rs", "server2/tests/domestic/observe.rs", "server2/tests/domestic/observe_product.rs", + "server2/tests/domestic/summary_s3.rs", "server2/tests/storage/catalog_wire.rs" ], "outside": [ @@ -39,6 +39,11 @@ "category": "documentation", "reason": "The observation restart read, the station page product and its S16 note." }, + { + "path": "docs/architecture/server2-kma-data-design.md", + "category": "documentation", + "reason": "The station page product among the server-written folds." + }, { "path": "docs/operations/server2-infrastructure.md", "category": "documentation", diff --git a/server2/src/cache/disk.rs b/server2/src/cache/disk.rs index 315019dd3..d63147d43 100644 --- a/server2/src/cache/disk.rs +++ b/server2/src/cache/disk.rs @@ -90,9 +90,13 @@ struct Header { fn failed() -> io::Error { io::Error::other("disposable cache unavailable") } +/// Provider raw records; the products processes make from them (the AirKorea hourly forecast, +/// the KMA station page products) are read from S3. fn eligible(id: &RecordId) -> bool { id.validate().is_ok() - && (matches!(id.source.as_str(), "data-go-kr" | "kma-web" | "vc") + && (matches!(id.source.as_str(), "data-go-kr" | "vc") + || (id.source == "kma-web" + && id.kind != crate::domestic::kma::source::observe_product::KIND) || (id.source == "airkorea-image" && id.kind == "airkorea-forecast-image")) } fn token(name: &str) -> bool { diff --git a/server2/src/domestic/kma/source/mod.rs b/server2/src/domestic/kma/source/mod.rs index 4fde8fc2f..41bb3a65a 100644 --- a/server2/src/domestic/kma/source/mod.rs +++ b/server2/src/domestic/kma/source/mod.rs @@ -43,6 +43,7 @@ pub mod live; pub mod mfds; mod mid; pub mod observe; +pub mod observe_product; pub mod pages; mod records; mod reference; diff --git a/server2/src/domestic/kma/source/observe.rs b/server2/src/domestic/kma/source/observe.rs index d532dfc3a..2611a4a1b 100644 --- a/server2/src/domestic/kma/source/observe.rs +++ b/server2/src/domestic/kma/source/observe.rs @@ -19,6 +19,7 @@ //! the next request. Nothing runs without a request. use super::asos::{AsosFetched, AsosStore}; use super::doc::{Doc, Val}; +use super::observe_product::{self as product, DayProduct, MadeOf, StoredPage}; use super::pages::{merge_aws_city, parse_aws, parse_city}; use super::records::Reader; use super::representative::{self, Verdict}; @@ -78,6 +79,11 @@ const ONGOING_MS: f64 = 2. * 3_600_000.; const MEMO_KEEP: Duration = Duration::from_secs(7 * 3600); /// Minute pages kept in memory (the readers look back 20 minutes). const MINUTE_KEEP_MS: f64 = 40. * 60_000.; +/// Minute pages restored before the load ends (the newest; the rest follow in +/// [`Flight::LoadRest`]), and the most members of one targeted read. +const MINUTE_FIRST: usize = 4; +/// A day's product is stored at most once per this interval by one process. +const PRODUCT_EVERY: Duration = Duration::from_secs(600); /// Legacy minute collection cadence. const MINUTE_CADENCE: i64 = 2; /// S3 day partitions read again after this long (other processes may have published). @@ -170,6 +176,10 @@ fn page_ids(ids: &PageIds) -> Arc> { #[derive(Debug, Clone, PartialEq, Eq, Hash)] enum Flight { Load, + /// The rest of a load after the answer: the other days' missing pages and minute pages. + LoadRest, + /// Store the day products whose pages changed. + Product, Fold, Geocode, Minute, @@ -201,8 +211,15 @@ struct Pages { ids: PageIds, minute: BTreeMap>>, days: HashMap, - /// Days whose latest S3 read failed (S16 completeness; a later success clears them). + /// Days whose latest S3 read failed or was partial (S16 completeness; a later complete read + /// clears them). failed_days: std::collections::BTreeSet, + /// Days whose latest S3 read failed (no product is stored for them). + unreadable: BTreeSet, + /// Listed groups S06 excluded in each read day (S16; a product carries the count). + excluded: BTreeMap, + /// AWS hourly pages kept reduced to registry facts outside this city set. + reduced: BTreeMap>>, minute_read: Option, /// Pages changed since the last fold. dirty: bool, @@ -234,6 +251,12 @@ struct State { registry_id: Option<(String, f64)>, /// The hourly part of the last fold (registry, hourly rows, national rows). hourly_fold: Option, + /// What the last product of each day this process read or stored is made of. + products: HashMap, + /// When this process last tried to store each day's product. + product_attempts: HashMap, + /// A fold recomputed the hourly part since the products were last compared. + product_due: bool, } #[derive(Clone)] struct HourlyFold { @@ -242,6 +265,8 @@ struct HourlyFold { national: Arc>>, unresolved: Vec, pages: Arc>, + /// The city set the hourly rows were kept in full for. + cities: Arc>, } pub struct Observations { @@ -458,6 +483,10 @@ impl Observations { if let Some(rx) = self.fold_if_dirty(now_ms) { Self::wait(vec![rx], until).await; } + // A product comparison a finishing one missed. + if self.state().product_due && self.flight(&Flight::Product).is_none() { + self.want_products(now_ms); + } // Before the first fold the carried seed alone is the registry (no pages yet). Some(self.state().folded.clone().unwrap_or_else(seed_only)) } @@ -471,8 +500,11 @@ impl Observations { rx.clone() }) } - /// Read hourly/city day partitions of the 8-day window and the latest minute pages from S3 - /// (at start and every [`RELOAD`]). + /// Read the hourly/city pages of the 8-day window and the latest minute pages from S3 (at + /// start and every [`RELOAD`]). The load ends once the request can answer: every day's + /// stored product (the parsed pages of that day), the pages of today and yesterday the + /// products lack, the seed and the newest minute pages. [`Flight::LoadRest`] then restores the + /// other days' missing pages and the rest of the minute window; no request waits for it. fn load(self: &Arc, now_ms: i64) -> Option> { let stale = { let s = self.state(); @@ -489,144 +521,415 @@ impl Observations { let deadline = Instant::now() + READ_DEADLINE; let first = schedule::from_ms(now_ms - HOURLY_WINDOW_HOURS * 3_600_000 - 3_600_000); let today = schedule::from_ms(now_ms); - let mut days = Vec::new(); - let mut d = Publication::new(first.date(), 0); - while let Some(day) = d { - if day.date() > today.date() { - break; - } - days.push(day.date()); - d = day.add_hours(24); - } - // Published day catalogs of both page kinds (S16): a day whose catalog is not listed has - // nothing published and reads as empty, as S14/S15b listings treat absent catalogs. A - // failed listing keeps every day to its catalog lookup. + let days = window_days(now_ms); + // Published day catalogs of both page kinds and of the products (S16): a day whose catalog + // is not listed has nothing published and reads as empty, as S14/S15b listings treat + // absent catalogs. A failed listing keeps every day to its catalog read. let months = super::records::months(schedule::to_ms(first) as f64, now_ms as f64); let (aws_k, city_k) = (aws_key(), city_key()); - let (listed_hourly, listed_city) = tokio::join!( + let (listed_products, listed_hourly, listed_city, seed, mut minutes) = tokio::join!( + self.reader + .partitions("kma-web", product::KIND, &aws_k, &months, deadline), self.reader .partitions("kma-web", "aws-hourly", &aws_k, &months, deadline), self.reader .partitions("kma-web", "city-obs", &city_k, &months, deadline), + self.read_seed(now_ms, deadline), + self.minute_entries(now_ms, deadline), ); + self.apply_seed(seed); let listed = |l: &Result, crate::storage::Error>, p: &str| { l.as_ref().map_or(true, |l| l.iter().any(|x| x == p)) }; - for day in days { - let fresh = { - let s = self.state(); - s.pages.days.get(&day).is_some_and(|at| { - day < today.date() && at.elapsed() < Duration::from_secs(86_400) - || at.elapsed() < RELOAD + // Due days newest first: their catalogs and products, LOOKUPS at a time. + let due: Vec<(u32, [bool; 3])> = { + let s = self.state(); + days.iter() + .rev() + .filter(|day| { + !s.pages.days.get(day).is_some_and(|at| { + **day < today.date() && at.elapsed() < Duration::from_secs(86_400) + || at.elapsed() < RELOAD + }) }) - }; - if fresh { - continue; - } - let partition = format!("{day:08}"); - let (aws, city_k) = (aws_key(), city_key()); - let (want_hourly, want_city) = ( - listed(&listed_hourly, &partition), - listed(&listed_city, &partition), - ); - let (hourly, city) = tokio::join!( - async { - if want_hourly { - self.reader - .raw_counted("kma-web", "aws-hourly", &aws, &partition, deadline) - .await - } else { - Ok((Vec::new(), 0)) + .map(|day| { + let p = format!("{day:08}"); + ( + *day, + [ + listed(&listed_hourly, &p), + listed(&listed_city, &p), + listed(&listed_products, &p), + ], + ) + }) + .collect() + }; + use futures_util::StreamExt; + let mut reads: Vec = futures_util::stream::iter(due) + .map(|(day, want)| self.list_day(day, want, deadline)) + .buffered(super::records::LOOKUPS) + .collect() + .await; + self.apply_products(&mut reads); + // Today and yesterday are compared now; the other days after the answer. + let yesterday = today.add_hours(-24).map_or(today.date(), Publication::date); + let (now, rest): (Vec, Vec) = + reads.drain(..).partition(|r| r.day >= yesterday); + crate::history::join_all( + now.into_iter() + .map(|r| self.complete_day(r, deadline)) + .collect(), + ) + .await; + let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); + let later = minutes.split_off(minutes.len().min(MINUTE_FIRST)); + self.restore_minutes(minutes, floor, deadline).await; + { + let mut s = self.state(); + s.pages.minute.retain(|k, _| *k > floor); + s.pages.minute_read = Some(std::time::Instant::now()); + s.pages.dirty = true; + let oldest = first.date(); + s.pages.days.retain(|d, _| *d >= oldest); + s.pages.failed_days.retain(|d| *d >= oldest); + s.pages.unreadable.retain(|d| *d >= oldest); + s.pages.excluded.retain(|d, _| *d >= oldest); + s.products.retain(|d, _| *d >= oldest); + s.product_attempts.retain(|d, _| *d >= oldest); + } + if !rest.is_empty() || !later.is_empty() { + let _ = self.start(Flight::LoadRest, Duration::ZERO, move |this| { + Box::pin(async move { + for day in rest { + this.complete_day(day, Instant::now() + READ_DEADLINE).await; } - }, - async { - if want_city { - self.reader - .raw_counted("kma-web", "city-obs", &city_k, &partition, deadline) - .await - } else { - Ok((Vec::new(), 0)) + this.restore_minutes(later, floor, Instant::now() + READ_DEADLINE) + .await; + }) + }); + } + } + /// Use the seed read (S16: a failed read leaves the registry unproven; the previous seed stays + /// in use). + fn apply_seed(&self, seed: Result<(Option>, bool), ()>) { + let mut s = self.state(); + s.seed_failed = !matches!(seed, Ok((_, false))); + if let Ok((Some(seed), _)) = seed + && s.seed + .as_ref() + .is_none_or(|old| old.len() != seed.len() || **old != *seed) + { + s.seed = Some(Arc::new(seed)); + s.pages.dirty = true; + s.pages.hourly_dirty = true; + } + } + /// List one day's page catalogs (control objects only) and, the first time this process + /// reads the day, read and decode its newest product. + async fn list_day(&self, day: u32, want: [bool; 3], deadline: Instant) -> DayRead { + let partition = format!("{day:08}"); + let known = self.state().pages.days.contains_key(&day); + let list = |kind: &'static str, key: ProviderKey, wanted: bool| { + let partition = partition.clone(); + async move { + if !wanted { + return Ok(None); + } + let id = super::records::catalog("kma-web", kind, &key, &partition)?; + let entries = self.reader.listed_acquisitions(&id, deadline).await?; + Ok::<_, crate::storage::Error>(Some((id, entries))) + } + }; + let (hourly, city, products) = tokio::join!( + list("aws-hourly", aws_key(), want[0]), + list("city-obs", city_key(), want[1]), + list(product::KIND, aws_key(), want[2] && !known), + ); + let listed = match (hourly, city) { + (Ok(hourly), Ok(city)) => { + let mut pages = Vec::new(); + for (kind, listed) in [("aws-hourly", hourly), ("city-obs", city)] { + let Some((id, entries)) = listed else { + continue; + }; + for (_, _, e) in entries { + if let Some(page) = ListedPage::new(kind, &id, e) { + pages.push(page); + } } - }, - ); - let (Ok((hourly, hourly_excluded)), Ok((city, city_excluded))) = (hourly, city) else { - self.state().pages.failed_days.insert(day); + } + Some(pages) + } + _ => None, + }; + let newest = match (&listed, products) { + (Some(_), Ok(Some((id, entries)))) => entries + .into_iter() + .max_by(|a, b| { + (a.0, &a.2.identity.raw_sha256).cmp(&(b.0, &b.2.identity.raw_sha256)) + }) + .map(|(_, _, e)| (id, e.identity)), + _ => None, + }; + let product = match newest { + Some((id, member)) => self.read_product(&id, member, day, deadline).await, + None => None, + }; + DayRead { + day, + listed, + product, + base: None, + } + } + /// One stored product of `day`, decoded; None when it cannot be read or is not a product of + /// this schema and parse revision. + async fn read_product( + &self, + id: &CatalogId, + member: crate::storage::RecordId, + day: u32, + deadline: Instant, + ) -> Option { + let read = self + .reader + .members(id, std::slice::from_ref(&member), deadline) + .await + .ok() + .flatten()?; + let bytes = read.first()?.first()?.bytes.clone(); + crate::observability::spawn_blocking(move || DayProduct::decode(&bytes, day)) + .await + .ok() + .flatten() + } + /// Hold the pages of each read product that its day catalogs list (S3 is truth), unless the + /// product's city set does not cover the current one (its reduced rows may lack a city + /// station's fields): such a day reads its raw pages. A held product's day is then compared + /// with its catalogs; a day this process read before is compared with what it holds. + fn apply_products(&self, reads: &mut [DayRead]) { + let mut s = self.state(); + let cities = { + let seed = s + .seed + .clone() + .unwrap_or_else(|| Arc::new(super::seed::embedded().docs.clone())); + let held = s.pages.city.values().map(|rows| &**rows); + let stored = reads + .iter() + .filter_map(|r| r.product.as_ref()) + .flat_map(|p| p.pages.iter()) + .filter(|page| page.kind == "city-obs") + .map(|page| &*page.rows); + city_ids(Some(&seed), held.chain(stored)) + }; + for r in reads.iter_mut() { + if s.pages.days.contains_key(&r.day) { + r.base = Some(s.pages.excluded.get(&r.day).copied().unwrap_or(0)); + continue; + } + let (Some(listed), Some(stored)) = (&r.listed, r.product.take()) else { continue; }; - // S16: a group S06 excluded makes the day's read partial. - let partial = hourly_excluded + city_excluded > 0; - let parsed = crate::observability::spawn_blocking(move || { - ( - hourly - .iter() - .filter_map(|a| parse_hourly(a)) - .collect::>(), - city.iter() - .filter_map(|a| parse_city_record(a)) - .collect::>(), - ) - }) - .await - .unwrap_or_default(); - let mut s = self.state(); - for (p, rows, at, sha, raw_id) in parsed.0 { - if s.pages - .hourly_at - .get(&p) - .is_none_or(|previous| *previous < at) - { - s.pages.hourly.insert(p, rows); - s.pages.hourly_at.insert(p, at); - s.pages.ids.insert(("aws-hourly", p), (at, sha, raw_id)); - } + if !stored.city_ids.is_superset(&cities) { + continue; } - for (p, rows, at, sha, raw_id) in parsed.1 { - if s.pages - .ids - .get(&("city-obs", p)) - .is_none_or(|(previous, _, _)| *previous < at) - { - s.pages.city.insert(p, rows); - s.pages.ids.insert(("city-obs", p), (at, sha, raw_id)); + s.products.insert(r.day, stored.made_of()); + let reduced = Arc::new(stored.city_ids.clone()); + for page in stored.pages { + if !listed.iter().any(|l| l.raw_id == page.raw_id) { + continue; } + let reduced = (page.kind == "aws-hourly").then(|| reduced.clone()); + let held = ( + page.hour, + page.rows, + page.fetched_at_ms, + page.sha256, + page.raw_id, + ); + hold(&mut s.pages, page.kind, held, reduced); } - s.pages.days.insert(day, std::time::Instant::now()); - if partial { - s.pages.failed_days.insert(day); - } else { - s.pages.failed_days.remove(&day); - } + r.base = Some(stored.excluded); s.pages.dirty = true; s.pages.hourly_dirty = true; } + } + /// Bring one day up to its catalogs. With pages held for it (a product, or an earlier read), + /// only the newest listed page of each kind and hour that is newer than the held one is + /// restored and parsed; otherwise, or when such a page is not complete or does not parse, + /// every page of the day is read as before (the newest complete page of each hour). + async fn complete_day(&self, r: DayRead, deadline: Instant) { + let DayRead { + day, listed, base, .. + } = r; + let Some(listed) = listed else { + self.day_failed(day); + return; + }; + if let Some(excluded) = base { + match self.restore_newer(&listed, deadline).await { + Ok(Some(changed)) => return self.day_read(day, excluded, changed), + Ok(None) => {} + Err(()) => return self.day_failed(day), + } + } + self.raw_day(day, &listed, deadline).await; + } + /// Restore and hold the newest listed pages newer than the held ones. Ok(None): one is not + /// complete or does not parse. + async fn restore_newer( + &self, + listed: &[ListedPage], + deadline: Instant, + ) -> Result, ()> { + let mut newest: BTreeMap<(&'static str, Publication), &ListedPage> = BTreeMap::new(); + for page in listed { + let slot = newest.entry((page.kind, page.hour)).or_insert(page); + if newer(page.at, &page.sha256, Some((slot.at, slot.sha256.as_str()))) { + *slot = page; + } + } + let wanted: Vec<&ListedPage> = { + let s = self.state(); + newest + .into_values() + .filter(|page| { + newer( + page.at, + &page.sha256, + s.pages + .ids + .get(&(page.kind, page.hour)) + .map(|(at, sha, _)| (*at, sha.as_str())), + ) + }) + .collect() + }; + if wanted.is_empty() { + return Ok(Some(false)); + } + let mut found: Vec<(&'static str, Vec)> = Vec::new(); + for kind in product::PAGE_KINDS { + let of_kind: Vec<&ListedPage> = + wanted.iter().copied().filter(|p| p.kind == kind).collect(); + for chunk in of_kind.chunks(MINUTE_FIRST) { + let members: Vec<_> = chunk.iter().map(|p| p.member.clone()).collect(); + match self + .reader + .members(&chunk[0].catalog, &members, deadline) + .await + { + Ok(Some(read)) => found.extend(read.into_iter().map(|a| (kind, a))), + Ok(None) => return Ok(None), + Err(_) => return Err(()), + } + } + } + let parsed = crate::observability::spawn_blocking(move || { + found + .iter() + .map(|(kind, records)| { + let parsed = if *kind == "aws-hourly" { + parse_hourly(records) + } else { + parse_city_record(records) + }; + parsed.map(|page| (*kind, page)) + }) + .collect::>>() + }) + .await + .ok() + .flatten(); + let Some(parsed) = parsed else { + return Ok(None); + }; + let mut s = self.state(); + let mut changed = false; + for (kind, page) in parsed { + changed |= hold(&mut s.pages, kind, page, None); + } + Ok(Some(changed)) + } + /// Read every page of one day (each complete acquisition is restored and parsed). + async fn raw_day(&self, day: u32, listed: &[ListedPage], deadline: Instant) { + let partition = format!("{day:08}"); + let (aws, city_k) = (aws_key(), city_key()); + let (want_hourly, want_city) = ( + listed.iter().any(|p| p.kind == "aws-hourly"), + listed.iter().any(|p| p.kind == "city-obs"), + ); + let (hourly, city) = tokio::join!( + async { + if want_hourly { + self.reader + .raw_counted("kma-web", "aws-hourly", &aws, &partition, deadline) + .await + } else { + Ok((Vec::new(), 0)) + } + }, + async { + if want_city { + self.reader + .raw_counted("kma-web", "city-obs", &city_k, &partition, deadline) + .await + } else { + Ok((Vec::new(), 0)) + } + }, + ); + let (Ok((hourly, hourly_excluded)), Ok((city, city_excluded))) = (hourly, city) else { + return self.day_failed(day); + }; + let parsed = crate::observability::spawn_blocking(move || { + ( + hourly + .iter() + .filter_map(|a| parse_hourly(a)) + .collect::>(), + city.iter() + .filter_map(|a| parse_city_record(a)) + .collect::>(), + ) + }) + .await + .unwrap_or_default(); { let mut s = self.state(); - s.pages.days.retain(|d, _| *d >= first.date()); - s.pages.failed_days.retain(|d| *d >= first.date()); - } - // The newest registry seed, if one was imported. - let seed = self.read_seed(now_ms, deadline).await; - // S16: a failed seed read leaves the registry unproven (the previous seed stays in use). - self.state().seed_failed = !matches!(seed, Ok((_, false))); - if let Ok((Some(seed), _)) = seed { - let mut s = self.state(); - if s.seed - .as_ref() - .is_none_or(|old| old.len() != seed.len() || **old != *seed) - { - s.seed = Some(Arc::new(seed)); - s.pages.dirty = true; - s.pages.hourly_dirty = true; + for page in parsed.0 { + hold(&mut s.pages, "aws-hourly", page, None); + } + for page in parsed.1 { + hold(&mut s.pages, "city-obs", page, None); } } - // Minute pages of the last 40 minutes, newest first. - let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); - let wanted = self.minute_entries(now_ms, deadline).await; - self.restore_minutes(wanted, floor, deadline).await; + // S16: a group S06 excluded makes the day's read partial. + self.day_read(day, hourly_excluded + city_excluded, true); + } + /// A day read completely, with `excluded` listed groups S06 excluded (S16). + fn day_read(&self, day: u32, excluded: usize, changed: bool) { let mut s = self.state(); - s.pages.minute.retain(|k, _| *k > floor); - s.pages.minute_read = Some(std::time::Instant::now()); - s.pages.dirty = true; + s.pages.days.insert(day, std::time::Instant::now()); + s.pages.unreadable.remove(&day); + s.pages.excluded.insert(day, excluded); + if excluded > 0 { + s.pages.failed_days.insert(day); + } else { + s.pages.failed_days.remove(&day); + } + if changed { + s.pages.dirty = true; + s.pages.hourly_dirty = true; + } + } + /// A day whose catalogs or pages could not be read (S16; nothing is stored for it). + fn day_failed(&self, day: u32) { + let mut s = self.state(); + s.pages.unreadable.insert(day); + s.pages.failed_days.insert(day); } /// The minute pages of the readers' reach that this process does not hold, newest first by /// completion: the day catalogs are listed (control objects only, no record is read), and of @@ -683,9 +986,13 @@ impl Observations { floor: Publication, deadline: Instant, ) { + // A few members per read: each read is bounded by the S06 lookup deadline. let mut by_catalog: Vec<(CatalogId, Vec)> = Vec::new(); for e in entries { - match by_catalog.iter_mut().find(|(id, _)| *id == e.catalog) { + match by_catalog + .iter_mut() + .find(|(id, members)| *id == e.catalog && members.len() < MINUTE_FIRST) + { Some((_, members)) => members.push(e.member), None => by_catalog.push((e.catalog, vec![e.member])), } @@ -836,6 +1143,7 @@ impl Observations { let mut s = self.state(); s.pages.hourly.insert(p, rows); s.pages.hourly_at.insert(p, at); + s.pages.reduced.remove(&p); s.pages.ids.insert(("aws-hourly", p), (at, sha, raw_id)); s.pages.dirty = true; s.pages.hourly_dirty = true; @@ -978,6 +1286,7 @@ impl Observations { s.pages.hourly_at.retain(|h, _| *h >= first); s.pages.city.retain(|h, _| *h >= first); s.pages.ids.retain(|(_, h), _| *h >= first); + s.pages.reduced.retain(|h, _| *h >= first); // Minute pages older than the readers' reach, however they arrived (S3 or local). let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); s.pages.minute.retain(|k, _| *k > floor); @@ -1006,6 +1315,7 @@ impl Observations { (Some(cached), _) => cached, (None, Some((seed, hourly, hourly_at, city, geo, pages))) => { let fold_pages = pages.clone(); + let held = hourly.clone(); let Ok((mut fold, reduced)) = crate::observability::spawn_blocking(move || { fold_hourly( seed.as_deref().map(Vec::as_slice), @@ -1022,13 +1332,23 @@ impl Observations { }; fold.pages = pages; let mut s = self.state(); - // Non-city stations' hourly rows are kept as registry facts only. + // Non-city stations' hourly rows are kept as registry facts only (unless a newer + // page replaced the folded one meanwhile). for (h, rows) in reduced { - if s.pages.hourly.contains_key(&h) { + let same = s + .pages + .hourly + .get(&h) + .zip(held.get(&h)) + .is_some_and(|(now, folded)| Arc::ptr_eq(now, folded)); + if same { s.pages.hourly.insert(h, rows); + s.pages.reduced.insert(h, fold.cities.clone()); } } s.hourly_fold = Some(fold.clone()); + drop(s); + self.want_products(now_ms); fold } (None, None) => return, @@ -1115,6 +1435,76 @@ impl Observations { }); } + // ---- day products ---- + + /// Compare the day products with the pages held (after a fold recomputed the hourly part) and + /// store the changed ones; no request waits for it. + fn want_products(self: &Arc, now_ms: i64) { + self.state().product_due = true; + let _ = self.start(Flight::Product, Duration::ZERO, move |this| { + Box::pin(async move { this.write_products(now_ms).await }) + }); + } + async fn write_products(self: Arc, now_ms: i64) { + loop { + let due = { + let mut s = self.state(); + s.product_due = false; + product_candidates(&mut s, now_ms) + }; + for (stored, made_of) in due { + let day = stored.day; + if self.store_product(stored).await.is_ok() { + self.state().products.insert(day, made_of); + } + } + if !self.state().product_due { + break; + } + } + } + /// Store one day product (one record, one group, the day's product catalog). + async fn store_product(&self, stored: DayProduct) -> Result<(), ()> { + let (day, made) = (stored.day, stored.made); + let bytes = crate::observability::spawn_blocking(move || stored.encode(facts)) + .await + .map_err(|_| ())? + .map_err(|_| ())?; + let limits = crate::storage::CatalogLimits::default(); + let record = RawRecord::new( + "kma-web", + product::KIND, + &aws_key(), + Publication::new(day, 0).ok_or(())?.period(), + made as u64, + 200, + "application/gzip", + Some(crate::storage::Pagination { + page: 1, + pages: 1, + complete: true, + }), + bytes.into(), + &limits.raw, + ) + .map_err(|_| ())?; + let member = crate::storage::GroupMember { + envelope: record.envelope.clone(), + catalogs: vec![ + CatalogId::for_record(&record.envelope.identity, &format!("{day:08}")) + .map_err(|_| ())?, + ], + }; + let declaration = + crate::storage::GroupDeclaration::new(vec![member], &limits).map_err(|_| ())?; + self.reader + .store + .publish(declaration, vec![record], Instant::now() + READ_DEADLINE) + .await + .map(|_| ()) + .map_err(|_| ()) + } + // ---- ASOS history ---- /// `history.loadForTown(town)` at `now_ms`, waiting for the station's acquisitions until @@ -1486,13 +1876,20 @@ pub struct AsosView { impl Observations { /// Whether `folded` is the current fold of a complete read at `now_ms`: every day of the /// 8-day page window was read from S3 and its latest read did not fail, the minute pages were - /// read, no newer page is waiting for a fold, and no S3 load, fold or geocoding is in flight. + /// read, no newer page is waiting for a fold, and no S3 load (nor its rest), fold, geocoding + /// or product comparison is in flight or due. pub fn settled(&self, now_ms: i64, folded: &Arc) -> bool { let busy = { let flights = self.flights.lock().unwrap_or_else(|e| e.into_inner()); - [Flight::Load, Flight::Fold, Flight::Geocode] - .iter() - .any(|f| flights.contains_key(f)) + [ + Flight::Load, + Flight::LoadRest, + Flight::Fold, + Flight::Geocode, + Flight::Product, + ] + .iter() + .any(|f| flights.contains_key(f)) }; if busy { return false; @@ -1501,26 +1898,16 @@ impl Observations { // The snapshot's fold must be the current one: a newer fold or newer pages are inputs // the snapshot did not use. if s.pages.dirty + || s.product_due || s.seed_failed || s.pages.minute_read.is_none() || !s.folded.as_ref().is_some_and(|f| Arc::ptr_eq(f, folded)) { return false; } - let first = schedule::from_ms(now_ms - HOURLY_WINDOW_HOURS * 3_600_000 - 3_600_000); - let today = schedule::from_ms(now_ms).date(); - let mut d = Publication::new(first.date(), 0); - while let Some(day) = d { - if day.date() > today { - break; - } - if !s.pages.days.contains_key(&day.date()) || s.pages.failed_days.contains(&day.date()) - { - return false; - } - d = day.add_hours(24); - } - true + window_days(now_ms) + .iter() + .all(|day| s.pages.days.contains_key(day) && !s.pages.failed_days.contains(day)) } } /// Keep the acquisitions whose range reaches the window (older ones cannot change it). @@ -1628,7 +2015,7 @@ impl StationGeo for Table<'_> { type Page = Arc>; /// Registry facts of a station row (the fields `_saveStnInfo` reads). const FACTS: [&str; 5] = ["stnId", "stnName", "addr", "altitude", "isCityWeather"]; -fn facts(row: &Doc) -> Doc { +pub(super) fn facts(row: &Doc) -> Doc { let mut out = Doc::new(); for k in FACTS { if let Some(v) = row.get(k) { @@ -1637,6 +2024,81 @@ fn facts(row: &Doc) -> Doc { } out } +/// The day products to store at `now_ms` (each attempt is recorded): per day whose every hour +/// is in the window, that was read without failure and holds pages, the held pages unless the +/// last product this process read or stored for it is made of the same, at most once per +/// [`PRODUCT_EVERY`]. +fn product_candidates(s: &mut State, now_ms: i64) -> Vec<(DayProduct, MadeOf)> { + let Some(cities) = s.hourly_fold.as_ref().map(|f| f.cities.clone()) else { + return Vec::new(); + }; + let first = latest_hourly(now_ms) + .add_hours(-HOURLY_WINDOW_HOURS) + .unwrap_or_else(|| latest_hourly(now_ms)); + let made = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map_or(0., |d| d.as_millis() as f64); + let mut by_day: BTreeMap> = BTreeMap::new(); + for ((kind, hour), (at, sha, raw_id)) in &s.pages.ids { + let rows = if *kind == "aws-hourly" { + s.pages.hourly.get(hour) + } else { + s.pages.city.get(hour) + }; + let Some(rows) = rows else { + continue; + }; + by_day.entry(hour.date()).or_default().push(StoredPage { + kind, + hour: *hour, + fetched_at_ms: *at, + sha256: sha.clone(), + raw_id: raw_id.clone(), + rows: rows.clone(), + }); + } + let mut out = Vec::new(); + for (day, pages) in by_day { + if Publication::new(day, 0).is_none_or(|start| start < first) + || !s.pages.days.contains_key(&day) + || s.pages.unreadable.contains(&day) + { + continue; + } + // The rows of a reduced page are whole for its city set only. + let city_ids = pages + .iter() + .filter(|p| p.kind == "aws-hourly") + .filter_map(|p| s.pages.reduced.get(&p.hour)) + .fold(None::>, |all, set| { + Some(match all { + None => (**set).clone(), + Some(all) => all.intersection(set).cloned().collect(), + }) + }) + .unwrap_or_else(|| (*cities).clone()); + let made_of = MadeOf::new(pages.iter().map(|p| p.raw_id.as_str()), &city_ids); + if s.products.get(&day) == Some(&made_of) + || s.product_attempts + .get(&day) + .is_some_and(|at| at.elapsed() < PRODUCT_EVERY) + { + continue; + } + s.product_attempts.insert(day, std::time::Instant::now()); + out.push(( + DayProduct { + made, + day, + excluded: s.pages.excluded.get(&day).copied().unwrap_or(0), + city_ids, + pages, + }, + made_of, + )); + } + out +} /// Rebuild the hourly collections: runs in hour order (AWS rows merged with the city rows of the /// same hour, or a `cityUnavailable` run). Hourly reads select city stations only, so rows of /// other stations are needed for registration only; they are returned reduced to registry facts @@ -1650,16 +2112,7 @@ fn fold_hourly( geo: &HashMap, page_ids: &[PageId], ) -> (HourlyFold, Vec<(Publication, Page)>) { - let city_ids: BTreeSet = city - .values() - .flat_map(|rows| rows.iter()) - .filter_map(|r| r.get("stnId").and_then(Val::as_str).map(str::to_owned)) - .chain(seed.unwrap_or_default().iter().filter_map(|d| { - (d.get("isCityWeather") == Some(&Val::Bool(true))) - .then(|| d.get("stnId").and_then(Val::as_str).map(str::to_owned)) - .flatten() - })) - .collect(); + let city_ids = city_ids(seed, city.values().map(|rows| &**rows)); let city_id = |r: &Doc| { r.get("stnId") .and_then(Val::as_str) @@ -1743,10 +2196,105 @@ fn fold_hourly( national: Arc::new(national), unresolved: unresolved.into_iter().collect(), pages: Arc::new(Vec::new()), + cities: Arc::new(city_ids), }, reduced, ) } +/// The city stations: those of the city pages and the seed's `isCityWeather` stations. +fn city_ids<'a>( + seed: Option<&[Doc]>, + city: impl Iterator>, +) -> BTreeSet { + city.flat_map(|rows| rows.iter()) + .filter_map(|r| r.get("stnId").and_then(Val::as_str).map(str::to_owned)) + .chain(seed.unwrap_or_default().iter().filter_map(|d| { + (d.get("isCityWeather") == Some(&Val::Bool(true))) + .then(|| d.get("stnId").and_then(Val::as_str).map(str::to_owned)) + .flatten() + })) + .collect() +} +/// Whether a page fetched at `at` with raw digest `sha` replaces `held`: a later fetch, or of one +/// instant the smaller digest (the page a read of every acquisition in catalog order keeps). +fn newer(at: f64, sha: &str, held: Option<(f64, &str)>) -> bool { + held.is_none_or(|(held_at, held_sha)| at > held_at || (at == held_at && sha < held_sha)) +} +/// Hold a page of `kind` unless the held one is newer ([`newer`]); `reduced`: the city set an +/// AWS page's rows are kept in full for. True when held. +fn hold( + pages: &mut Pages, + kind: &'static str, + (p, rows, at, sha, raw_id): ParsedPage, + reduced: Option>>, +) -> bool { + let current = pages + .ids + .get(&(kind, p)) + .map(|(at, sha, _)| (*at, sha.as_str())); + if !newer(at, &sha, current) { + return false; + } + if kind == "aws-hourly" { + pages.hourly.insert(p, rows); + pages.hourly_at.insert(p, at); + match reduced { + Some(cities) => pages.reduced.insert(p, cities), + None => pages.reduced.remove(&p), + }; + } else { + pages.city.insert(p, rows); + } + pages.ids.insert((kind, p), (at, sha, raw_id)); + true +} +/// The KST days of the 8-day page window and the hour before it, oldest first. +fn window_days(now_ms: i64) -> Vec { + let first = schedule::from_ms(now_ms - HOURLY_WINDOW_HOURS * 3_600_000 - 3_600_000); + let today = schedule::from_ms(now_ms).date(); + let mut days = Vec::new(); + let mut d = Publication::new(first.date(), 0); + while let Some(day) = d { + if day.date() > today { + break; + } + days.push(day.date()); + d = day.add_hours(24); + } + days +} +/// One day read: its listed pages (None: a catalog could not be listed), its stored product, and +/// the S06 exclusions of the pages held for it (None: none held, the raw pages are read). +struct DayRead { + day: u32, + listed: Option>, + product: Option, + base: Option, +} +/// A page a day catalog lists. +struct ListedPage { + kind: &'static str, + hour: Publication, + at: f64, + sha256: String, + raw_id: String, + member: crate::storage::RecordId, + catalog: CatalogId, +} +impl ListedPage { + fn new(kind: &'static str, catalog: &CatalogId, e: crate::storage::Envelope) -> Option { + let period = &e.identity.period; + Some(Self { + kind, + hour: Publication::new(period.local_date, period.slot.parse().ok()?)?, + at: e.identity.fetched_at_ms as f64, + sha256: e.identity.raw_sha256.clone(), + raw_id: e.identity.object_key().ok()?, + member: e.identity, + catalog: catalog.clone(), + }) + } +} /// The collections before any page: the carried seed's registry. fn seed_only() -> Arc { diff --git a/server2/src/domestic/kma/source/observe_product.rs b/server2/src/domestic/kma/source/observe_product.rs new file mode 100644 index 000000000..eedb2f98f --- /dev/null +++ b/server2/src/domestic/kma/source/observe_product.rs @@ -0,0 +1,461 @@ +//! S15b-2 restart read: the station page product of one KST day (AK 2026-11-07, "A로 진행"). +//! +//! A restarted process restored every stored AWS hourly and city page of the 8-day window and +//! parsed each again (about 1 s per day on a t3.small) before its first answer had past days. The +//! product holds the parsed inputs of the hourly fold (not its output) for the pages of one day: +//! per page kind and hour the newest page, as the process keeps it, with its fetch instant, raw +//! digest and canonical raw identity. AWS rows of stations outside the city set are kept as their +//! registry facts (the reduced form the fold keeps them in), each distinct facts document once. +//! +//! A product is never the truth: a reader compares it with the day catalogs and restores what it +//! lacks. `cityIds` is the city set its reduced rows keep in full; `excluded` is the number of the +//! day's listed groups S06 excluded when its pages were read (S16 completeness). Values keep +//! every [`Val`] (undefined, dates, negative zero and NaN) through a tagged string form. +use super::doc::{Doc, Val}; +use crate::domestic::publication::Publication; +use serde::de::{self, Deserializer, MapAccess, SeqAccess, Visitor}; +use serde::ser::{SerializeMap, SerializeSeq, Serializer}; +use serde::{Deserialize, Serialize}; +use std::collections::{BTreeSet, HashMap}; +use std::sync::Arc; + +/// Kind of the product (source `kma-web`, the AWS page key, partition `YYYYMMDD`). +pub const KIND: &str = "station-pages"; +/// The product's layout. +pub const SCHEMA: u64 = 1; +/// Bumped whenever page parsing (`parse_aws`, `parse_city`) or the reduction to registry facts +/// changes: an older product is then read as absent. +pub const PARSE_REVISION: u64 = 1; +/// The largest product JSON a reader expands. +pub const JSON_BYTES: u64 = 16 << 20; +/// Marks a tagged value in a string (a stored string starting with it is escaped). +const TAG: char = '\u{1}'; +/// The kinds of pages a product holds. +pub const PAGE_KINDS: [&str; 2] = ["aws-hourly", "city-obs"]; + +/// One stored page of a day. +#[derive(Debug, Clone, PartialEq)] +pub struct StoredPage { + pub kind: &'static str, + pub hour: Publication, + pub fetched_at_ms: f64, + pub sha256: String, + pub raw_id: String, + pub rows: Arc>, +} +/// The pages of one KST day. +#[derive(Debug, Clone, PartialEq)] +pub struct DayProduct { + pub made: f64, + pub day: u32, + /// Listed groups S06 excluded when these pages were read. + pub excluded: usize, + pub city_ids: BTreeSet, + pub pages: Vec, +} +/// What a product is made of: its pages, city set and parse revision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct MadeOf { + pages: Vec, + city_ids: Vec, + revision: u64, +} +impl MadeOf { + pub fn new<'a>(raw_ids: impl Iterator, city_ids: &BTreeSet) -> Self { + let mut pages: Vec = raw_ids.map(str::to_owned).collect(); + pages.sort(); + Self { + pages, + city_ids: city_ids.iter().cloned().collect(), + revision: PARSE_REVISION, + } + } +} + +impl DayProduct { + pub fn made_of(&self) -> MadeOf { + MadeOf::new(self.pages.iter().map(|p| p.raw_id.as_str()), &self.city_ids) + } + /// The stored form: gzip of the product JSON. An AWS row equal to its registry facts + /// (`facts`) is stored as an index into the facts table. + pub fn encode(&self, facts: impl Fn(&Doc) -> Doc) -> std::io::Result> { + use std::io::Write; + let mut table: Vec<&Doc> = Vec::new(); + let mut index: HashMap, usize> = HashMap::new(); + let mut pages = Vec::with_capacity(self.pages.len()); + for page in &self.pages { + let mut rows = Vec::with_capacity(page.rows.len()); + for row in page.rows.iter() { + if page.kind == "aws-hourly" && facts(row) == *row { + let key = serde_json::to_vec(&V(&Val::Doc(row.clone())))?; + let at = *index.entry(key).or_insert_with(|| { + table.push(row); + table.len() - 1 + }); + rows.push(RowOut::Fact(at)); + } else { + rows.push(RowOut::Full(row)); + } + } + pages.push(PageOut { + kind: page.kind, + hour: page.hour.time(), + fetched_at: page.fetched_at_ms, + sha256: &page.sha256, + raw_id: &page.raw_id, + rows, + }); + } + let out = ProductOut { + schema: SCHEMA, + parse_revision: PARSE_REVISION, + made: self.made, + day: self.day, + excluded: self.excluded, + city_ids: &self.city_ids, + facts: table.into_iter().map(DocOut).collect(), + pages, + }; + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + serde_json::to_writer(&mut gzip, &out)?; + gzip.flush()?; + gzip.finish() + } + /// A stored product of `day`; None when it is not one of this schema and parse revision, or + /// expands past [`JSON_BYTES`]. + pub fn decode(stored: &[u8], day: u32) -> Option { + use std::io::Read; + let mut bytes = Vec::new(); + flate2::read::GzDecoder::new(stored) + .take(JSON_BYTES + 1) + .read_to_end(&mut bytes) + .ok()?; + if bytes.len() as u64 > JSON_BYTES { + return None; + } + let stored: ProductIn = serde_json::from_slice(&bytes).ok()?; + if stored.schema != SCHEMA || stored.parse_revision != PARSE_REVISION || stored.day != day { + return None; + } + let facts: Vec = stored + .facts + .into_iter() + .map(|v| match v.0 { + Val::Doc(d) => Some(d), + _ => None, + }) + .collect::>()?; + let mut seen = BTreeSet::new(); + let mut pages = Vec::with_capacity(stored.pages.len()); + for page in stored.pages { + let kind = *PAGE_KINDS.iter().find(|k| **k == page.kind)?; + let hour = Publication::new(day, page.hour).filter(|h| h.minute() == 0)?; + if !seen.insert((kind, hour)) || !page.fetched_at.is_finite() { + return None; + } + let rows = page + .rows + .into_iter() + .map(|row| match row.0 { + Val::Doc(d) => Some(d), + Val::Num(n) if kind == "aws-hourly" && n >= 0. && n.fract() == 0. => { + facts.get(n as usize).cloned() + } + _ => None, + }) + .collect::>>()?; + pages.push(StoredPage { + kind, + hour, + fetched_at_ms: page.fetched_at, + sha256: page.sha256, + raw_id: page.raw_id, + rows: Arc::new(rows), + }); + } + Some(Self { + made: stored.made, + day, + excluded: stored.excluded, + city_ids: stored.city_ids, + pages, + }) + } +} + +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct ProductOut<'a> { + schema: u64, + parse_revision: u64, + made: f64, + day: u32, + excluded: usize, + city_ids: &'a BTreeSet, + facts: Vec>, + pages: Vec>, +} +#[derive(Serialize)] +#[serde(rename_all = "camelCase")] +struct PageOut<'a> { + kind: &'a str, + hour: u16, + fetched_at: f64, + sha256: &'a str, + raw_id: &'a str, + rows: Vec>, +} +enum RowOut<'a> { + Fact(usize), + Full(&'a Doc), +} +impl Serialize for RowOut<'_> { + fn serialize(&self, s: S) -> Result { + match self { + Self::Fact(i) => s.serialize_u64(*i as u64), + Self::Full(d) => DocOut(d).serialize(s), + } + } +} +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct ProductIn { + schema: u64, + parse_revision: u64, + made: f64, + day: u32, + excluded: usize, + city_ids: BTreeSet, + facts: Vec, + pages: Vec, +} +#[derive(Deserialize)] +#[serde(rename_all = "camelCase")] +struct PageIn { + kind: String, + hour: u16, + fetched_at: f64, + sha256: String, + raw_id: String, + rows: Vec, +} + +/// A document in key order. +struct DocOut<'a>(&'a Doc); +impl Serialize for DocOut<'_> { + fn serialize(&self, s: S) -> Result { + let mut map = s.serialize_map(Some(self.0.entries().len()))?; + for (k, v) in self.0.entries() { + map.serialize_entry(k, &V(v))?; + } + map.end() + } +} +/// A value: JSON for numbers, strings, booleans, null, arrays and documents; a tagged string for +/// undefined, dates, non-finite numbers and negative zero; a string starting with the tag gets one +/// more. +struct V<'a>(&'a Val); +impl Serialize for V<'_> { + fn serialize(&self, s: S) -> Result { + match self.0 { + Val::Num(n) if n.is_finite() && !(*n == 0. && n.is_sign_negative()) => { + if n.fract() == 0. && n.abs() < 9_007_199_254_740_992. { + s.serialize_i64(*n as i64) + } else { + s.serialize_f64(*n) + } + } + Val::Num(n) => s.serialize_str(&format!("{TAG}n{:016x}", n.to_bits())), + Val::Str(t) if t.starts_with(TAG) => s.serialize_str(&format!("{TAG}{t}")), + Val::Str(t) => s.serialize_str(t), + Val::Bool(b) => s.serialize_bool(*b), + Val::Null => s.serialize_unit(), + Val::Undef => s.serialize_str(&format!("{TAG}u")), + Val::Date(ms) => s.serialize_str(&format!("{TAG}d{:016x}", ms.to_bits())), + Val::Arr(a) => { + let mut seq = s.serialize_seq(Some(a.len()))?; + for v in a { + seq.serialize_element(&V(v))?; + } + seq.end() + } + Val::Doc(d) => DocOut(d).serialize(s), + } + } +} +fn tagged(t: &str) -> Option { + let Some(rest) = t.strip_prefix(TAG) else { + return Some(Val::Str(t.to_owned())); + }; + let bits = |hex: &str| { + (hex.len() == 16) + .then(|| u64::from_str_radix(hex, 16).ok()) + .flatten() + .map(f64::from_bits) + }; + if rest.starts_with(TAG) { + Some(Val::Str(rest.to_owned())) + } else if rest == "u" { + Some(Val::Undef) + } else if let Some(hex) = rest.strip_prefix('d') { + bits(hex).map(Val::Date) + } else if let Some(hex) = rest.strip_prefix('n') { + bits(hex).map(Val::Num) + } else { + None + } +} +/// A stored value read back ([`V`]'s form). +struct ValIn(Val); +impl<'de> Deserialize<'de> for ValIn { + fn deserialize>(d: D) -> Result { + struct Of; + impl<'de> Visitor<'de> for Of { + type Value = Val; + fn expecting(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result { + f.write_str("a stored value") + } + fn visit_bool(self, b: bool) -> Result { + Ok(Val::Bool(b)) + } + fn visit_i64(self, n: i64) -> Result { + Ok(Val::Num(n as f64)) + } + fn visit_u64(self, n: u64) -> Result { + Ok(Val::Num(n as f64)) + } + fn visit_f64(self, n: f64) -> Result { + Ok(Val::Num(n)) + } + fn visit_str(self, t: &str) -> Result { + tagged(t).ok_or_else(|| E::custom("tagged value")) + } + fn visit_unit(self) -> Result { + Ok(Val::Null) + } + fn visit_seq>(self, mut seq: A) -> Result { + let mut out = Vec::new(); + while let Some(v) = seq.next_element::()? { + out.push(v.0); + } + Ok(Val::Arr(out)) + } + fn visit_map>(self, mut map: A) -> Result { + let mut doc = Doc::new(); + while let Some((k, v)) = map.next_entry::()? { + if doc.has(&k) { + return Err(de::Error::custom("repeated key")); + } + doc.set(&k, v.0); + } + Ok(Val::Doc(doc)) + } + } + d.deserialize_any(Of).map(ValIn) + } +} + +#[cfg(test)] +mod tests { + use super::*; + fn facts(d: &Doc) -> Doc { + let mut out = Doc::new(); + for k in ["stnId", "stnName"] { + if let Some(v) = d.get(k) { + out.set(k, v.clone()); + } + } + out + } + #[test] + fn every_value_and_key_order_round_trips() { + let nested = Doc::new() + .with("z", Val::Num(1.)) + .with("a", Val::Arr(vec![Val::Null, Val::Bool(false)])); + let full = Doc::new() + .with("stnId", Val::Num(108.)) + .with("zeta", Val::Str("서울".into())) + .with("alpha", Val::Num(-0.)) + .with("nan", Val::Num(f64::NAN)) + .with("inf", Val::Num(f64::NEG_INFINITY)) + .with("big", Val::Num(1e300)) + .with("frac", Val::Num(18.4)) + .with("huge", Val::Num(9_007_199_254_740_993.)) + .with("undef", Val::Undef) + .with("date", Val::Date(1_790_214_600_000.)) + .with("bad date", Val::Date(f64::NAN)) + .with("tag", Val::Str(format!("{TAG}u"))) + .with("tags", Val::Str(format!("{TAG}{TAG}"))) + .with("doc", Val::Doc(nested)) + .with("null", Val::Null); + let reduced = Doc::new() + .with("stnId", Val::Num(400.)) + .with("stnName", Val::Str("강남".into())); + let product = DayProduct { + made: 1_790_300_000_000.5, + day: 20260924, + excluded: 2, + city_ids: ["108".to_owned()].into(), + pages: vec![ + StoredPage { + kind: "aws-hourly", + hour: Publication::new(20260924, 1000).unwrap(), + fetched_at_ms: 1_790_211_900_000., + sha256: "a".repeat(64), + raw_id: "raw/a".into(), + rows: Arc::new(vec![full.clone(), reduced.clone(), reduced.clone()]), + }, + StoredPage { + kind: "city-obs", + hour: Publication::new(20260924, 1000).unwrap(), + fetched_at_ms: 1_790_211_900_001., + sha256: "b".repeat(64), + raw_id: "raw/b".into(), + rows: Arc::new(vec![reduced.clone()]), + }, + ], + }; + let bytes = product.encode(facts).unwrap(); + let back = DayProduct::decode(&bytes, 20260924).unwrap(); + assert_eq!(back.made_of(), product.made_of()); + assert_eq!(back.city_ids, product.city_ids); + assert_eq!((back.made, back.excluded), (product.made, product.excluded)); + for (a, b) in back.pages.iter().zip(&product.pages) { + assert_eq!( + (a.kind, a.hour, a.fetched_at_ms, &a.sha256, &a.raw_id), + (b.kind, b.hour, b.fetched_at_ms, &b.sha256, &b.raw_id) + ); + assert_eq!(a.rows.len(), b.rows.len()); + for (x, y) in a.rows.iter().zip(b.rows.iter()) { + // NaN is not equal to itself: compare the stored bits. + assert_eq!(bits(x), bits(y)); + } + } + // One facts document for the two reduced AWS rows; the city row stays whole. + let mut raw = Vec::new(); + std::io::Read::read_to_end(&mut flate2::read::GzDecoder::new(&bytes[..]), &mut raw) + .unwrap(); + let json: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + assert_eq!(json["facts"].as_array().unwrap().len(), 1); + assert_eq!(json["pages"][0]["rows"][1], serde_json::json!(0)); + assert!(json["pages"][1]["rows"][0].is_object()); + // Another day, schema or revision is not this product. + assert!(DayProduct::decode(&bytes, 20260923).is_none()); + assert!(DayProduct::decode(b"not gzip", 20260924).is_none()); + } + fn bits(d: &Doc) -> Vec<(String, String)> { + fn val(v: &Val) -> String { + match v { + Val::Num(n) => format!("n{:x}", n.to_bits()), + Val::Date(n) => format!("d{:x}", n.to_bits()), + Val::Doc(d) => format!("{:?}", bits(d)), + Val::Arr(a) => format!("{:?}", a.iter().map(val).collect::>()), + other => format!("{other:?}"), + } + } + d.entries() + .iter() + .map(|(k, v)| (k.clone(), val(v))) + .collect() + } +} diff --git a/server2/tests/cache.rs b/server2/tests/cache.rs index e2d669877..c758fdac7 100644 --- a/server2/tests/cache.rs +++ b/server2/tests/cache.rs @@ -430,3 +430,25 @@ async fn publication_reconciliation_never_uses_cache_to_mask_same_length_s3_corr ); assert_eq!(gets.load(Ordering::SeqCst), 2); } +/// The station page products (`kma-web` / `station-pages`, made and read again by processes) stay +/// out of the disk cache beside the cached station pages, as the image forecast product does. +#[tokio::test] +async fn the_station_page_product_is_not_cached() { + let d = Directory::new(); + let page = record("kma-web", "aws-hourly", 1, b"page".to_vec()); + let product = record("kma-web", "station-pages", 2, b"product".to_vec()); + let w = wire(&[page.clone(), product.clone()]); + for _ in 0..2 { + let c = Arc::new(DiskCache::new(&d.0, "bucket-a/seoul", 1048576, 16).unwrap()); + let s = store(w.clone(), Some(c.clone())); + s.load(&page.envelope.identity).await.unwrap(); + s.load(&product.envelope.identity).await.unwrap(); + idle(&c).await; + } + assert_eq!(d.files().len(), 1, "the page only"); + assert_eq!( + w.gets.load(Ordering::SeqCst), + 3, + "the product is read each time" + ); +} diff --git a/server2/tests/domestic.rs b/server2/tests/domestic.rs index 3fb9de6e8..74628bad5 100644 --- a/server2/tests/domestic.rs +++ b/server2/tests/domestic.rs @@ -16,6 +16,8 @@ mod life; mod life_s3; #[path = "domestic/observe.rs"] mod observe; +#[path = "domestic/observe_product.rs"] +mod observe_product; #[path = "domestic/observed_history.rs"] mod observed_history; #[path = "domestic/parity.rs"] diff --git a/server2/tests/domestic/observe.rs b/server2/tests/domestic/observe.rs index 1cb22ebd0..2f5cbe666 100644 --- a/server2/tests/domestic/observe.rs +++ b/server2/tests/domestic/observe.rs @@ -1053,26 +1053,42 @@ async fn same_hour_aws_and_city_restore_respects_fetched_order_and_field_presenc pub(super) struct Counted { inner: server2::world::BoundedS3, counts: Arc>>, + /// GETs of one class after the first `n` are held this long. + delay: Arc>>, } impl Counted { pub(super) fn new(inner: server2::world::BoundedS3) -> Self { Self { inner, counts: Arc::default(), + delay: Arc::default(), } } - fn count(&self, method: &str, key: &str) { - let class = if key.starts_with("index/v2/groups/") { + /// Hold every GET of `class` after its first `after` for `hold`. + pub(super) fn delay_gets(&self, class: &str, after: usize, hold: Duration) { + *self.delay.lock().unwrap() = Some((class.to_owned(), after, hold)); + } + fn held(&self, key: &str) -> Option { + let delay = self.delay.lock().unwrap().clone()?; + let class = Self::class(key); + (class == delay.0 && self.get(&format!("GET {class}")) > delay.1).then_some(delay.2) + } + fn class(key: &str) -> String { + if key.starts_with("index/v2/groups/") { "index/v2/groups".to_owned() } else { key.split('/').take(4).collect::>().join("/") - }; - *self - .counts - .lock() - .unwrap() - .entry(format!("{method} {class}")) - .or_default() += 1; + } + } + fn count(&self, method: &str, key: &str) { + let class = Self::class(key); + let mut counts = self.counts.lock().unwrap(); + *counts.entry(format!("{method} {class}")).or_default() += 1; + // Raw records also by KST day (`raw/v2/source/kind/YYYY/MM/DD`). + if key.starts_with("raw/") { + let day = key.split('/').take(7).collect::>().join("/"); + *counts.entry(format!("{method} {day}")).or_default() += 1; + } } /// Requests so far of `method` and `class` (for example `GET raw/v2/kma-web/aws-minute`). pub(super) fn get(&self, what: &str) -> usize { @@ -1097,6 +1113,9 @@ impl server2::storage::ObjectTransport for Counted { max: usize, ) -> Result { self.count("GET", key); + if let Some(hold) = self.held(key) { + tokio::time::sleep(hold).await; + } self.inner.get(key, max).await } } @@ -1189,6 +1208,16 @@ fn one_group(record: &RawRecord) -> GroupDeclaration { pub(super) async fn store_pages(transport: &T, records: Vec) where T: server2::storage::CatalogTransport + Clone, +{ + store_pages_without(transport, records, &[]).await; +} +/// [`store_pages`], leaving out the raw bodies of `missing` (S06 then excludes their groups). +pub(super) async fn store_pages_without( + transport: &T, + records: Vec, + missing: &[server2::storage::RecordId], +) where + T: server2::storage::CatalogTransport + Clone, { use server2::storage::{Catalog, RawRecordStore, WriteCondition}; let limits = CatalogLimits::default(); @@ -1209,7 +1238,9 @@ where .unwrap(), 200 ); - raw.publish(attached[0].clone()).await.unwrap(); + if !missing.contains(&attached[0].envelope.identity) { + raw.publish(attached[0].clone()).await.unwrap(); + } let id = d.partitions[0].clone(); let at = match catalogs.iter().position(|c| c.identity == id) { Some(at) => at, @@ -1300,8 +1331,9 @@ async fn a_restart_restores_only_the_minute_pages_within_reach() { descriptors <= 20 && raws <= 20, "{descriptors} descriptor and {raws} raw GETs for 20 minutes within reach" ); + // One listing, then one catalog read per targeted read of at most four members. assert!( - counted.get("GET index/v2/kma-web/aws-minute") <= 4, + counted.get("GET index/v2/kma-web/aws-minute") <= 1 + 20 / 4, "the minute catalogs are read a bounded number of times" ); } diff --git a/server2/tests/domestic/observe_product.rs b/server2/tests/domestic/observe_product.rs new file mode 100644 index 000000000..c01ecca6d --- /dev/null +++ b/server2/tests/domestic/observe_product.rs @@ -0,0 +1,529 @@ +//! S15b-2 restart read (AK 2026-11-07, "A로 진행"): a process that folds the station pages +//! stores one product per KST day holding the parsed pages, and a restarted process answers from +//! the products and the day catalogs instead of restoring and parsing every stored page. The +//! products are compared with the catalogs (S3 is truth): a page published later is restored, an +//! unusable product is replaced by the raw pages, and S16 completeness is kept. The S06 loopback +//! S3 test peer holds the pages; the acquirer never reaches a provider. +use super::observe::{Counted, aws_page, city_page, dotted, page_record, store_pages_without}; +use super::source_s3::{NOW, Peer, Unreachable, deadline}; +use serde_json::Value; +use server2::domestic::kma::source::observe::{Folded, Observations}; +use server2::domestic::kma::source::stations::date_of; +use server2::domestic::kma::source::{Doc, Val}; +use server2::domestic::publication::Publication; +use server2::history::schedule; +use server2::storage::{ + CatalogId, CatalogLimits, CatalogStore, GroupDeclaration, GroupMember, Pagination, ProviderKey, + RawRecord, RecordId, +}; +use std::sync::Arc; +use std::time::Duration; +use tokio::time::Instant; + +type Obs = Observations; +const PRODUCTS: &str = "raw/v2/kma-web/station-pages"; +const HOUR_MS: i64 = 3_600_000; + +fn p(date: u32, time: u16) -> Publication { + Publication::new(date, time).unwrap() +} +/// The city page's own `pubDate` form. +fn city_date(h: Publication) -> String { + let d = h.date(); + format!( + "{}.{}.{}.{:02}:00", + d / 10000, + (d / 100) % 100, + d % 100, + h.hour() + ) +} +/// Every hour from 2026-09-16 00:00 to 10:00 of 2026-09-24 (the nine day partitions a process at +/// 10:50 reads), each with its AWS hourly and city page acquired 15 minutes into the hour, and the +/// minute pages of the last 40 minutes. +fn station_records() -> Vec { + let mut out = Vec::new(); + let mut h = p(20260916, 0); + while h <= p(20260924, 1000) { + let at = schedule::to_ms(h) as u64 + 15 * 60_000; + out.push(page_record( + "aws-hourly", + h, + at, + aws_page(&dotted(h), h.hour().is_multiple_of(5)), + )); + out.push(page_record("city-obs", h, at, city_page(&city_date(h)))); + h = h.add_hours(1).unwrap(); + } + for minute in (11..=49).step_by(2) { + let m = p(20260924, 1000 + minute); + out.push(page_record( + "aws-minute", + m, + schedule::to_ms(m) as u64 + 30_000, + aws_page(&dotted(m), minute == 49), + )); + } + out +} +async fn seeded_peer(missing: &[RecordId], extra: Vec) -> Peer { + let peer = Peer::new(); + let mut records = station_records(); + records.extend(extra); + store_pages_without(&peer.transport_limited(4), records, missing).await; + peer +} +fn process(peer: &Peer) -> (Arc, Counted) { + let counted = Counted::new(peer.transport_limited(4)); + let obs = Arc::new(Observations::new( + counted.clone(), + Arc::new(CatalogStore::new(counted.clone(), CatalogLimits::default()).unwrap()), + Arc::new(Unreachable), + Arc::new(Unreachable), + )); + (obs, counted) +} +async fn answer(obs: &Arc) -> Arc { + obs.stations(NOW as i64, Instant::now() + Duration::from_millis(2500)) + .await + .unwrap() +} +/// A later answer whose inputs were read completely (S16), or a panic. +async fn settle(obs: &Arc) -> Arc { + for _ in 0..120 { + let folded = answer(obs).await; + if obs.settled(NOW as i64, &folded) { + return folded; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + panic!("the station inputs never settled"); +} +/// What an answer is made of (the registry's first-seen instant is per process). +fn fingerprint(f: &Folded) -> Vec { + let docs = |d: &[Doc]| { + d.iter() + .map(|d| String::from_utf8_lossy(&d.json().to_bytes()).into_owned()) + .collect::>() + }; + let mut out = vec!["registry".to_owned()]; + out.extend(docs(f.registry.docs())); + out.push("hourly".into()); + out.extend(docs(f.hourly.docs())); + out.push("minute".into()); + out.extend(docs(f.minute.docs())); + out.push("national".into()); + let mut national: Vec<_> = f.national.iter().collect(); + national.sort_by(|a, b| a.0.cmp(b.0)); + for (id, rows) in national { + for (at, row) in rows { + out.push(format!( + "{id} {at} {}", + String::from_utf8_lossy(&row.json().to_bytes()) + )); + } + } + out.push("pages".into()); + for page in f.pages.iter() { + out.push(format!("{page:?}")); + } + out +} +/// The answers are made of the same (the first difference of each part otherwise). +#[track_caller] +fn same(a: &Folded, b: &Folded, what: &str) { + let (a, b) = (fingerprint(a), fingerprint(b)); + if a == b { + return; + } + let parts = |f: &[String]| { + let mut out: Vec<(String, Vec)> = Vec::new(); + for line in f { + if ["registry", "hourly", "minute", "national", "pages"].contains(&line.as_str()) { + out.push((line.clone(), Vec::new())); + } else { + out.last_mut().unwrap().1.push(line.clone()); + } + } + out + }; + let mut report = String::new(); + for ((name, x), (_, y)) in parts(&a).iter().zip(parts(&b).iter()) { + if let Some(i) = (0..x.len().max(y.len())).find(|i| x.get(*i) != y.get(*i)) { + report += &format!( + "\n{name} entry {i} of {} and {}:\n {:?}\n {:?}", + x.len(), + y.len(), + x.get(i), + y.get(i) + ); + } + } + panic!("{what}:{report}"); +} +/// Raw page GETs of `kind` outside 2026-09-16 (a process at 10:50 on 09-24 stores no product +/// of that day: some of its hours are already out of the window). +fn raw_gets(counted: &Counted, kind: &str) -> usize { + counted.get(&format!("GET raw/v2/kma-web/{kind}")) + - counted.get(&format!("GET raw/v2/kma-web/{kind}/2026/09/16")) +} +/// Hourly pages from hour `from` to `to` with a row of station 108. A page's rows are dated nine +/// hours after its KST hour (the page time read as UTC, as the parsed pages carry it). +fn hours_of_108(f: &Folded, from: Publication, to: Publication) -> usize { + let shift = 9. * HOUR_MS as f64; + let (from, to) = ( + schedule::to_ms(from) as f64 + shift, + schedule::to_ms(to) as f64 + shift, + ); + f.hourly + .docs() + .iter() + .filter(|d| d.get("stnId") == Some(&Val::Num(108.))) + .map(date_of) + .filter(|t| (from..=to).contains(t)) + .count() +} +/// Call until `done` holds (folds and products follow requests). +async fn until(obs: &Arc, mut done: impl FnMut() -> bool) { + for _ in 0..120 { + if done() { + return; + } + let _ = answer(obs).await; + tokio::time::sleep(Duration::from_millis(250)).await; + } + panic!("never reached"); +} +fn product_catalog(day: u32) -> CatalogId { + let key = ProviderKey::Named { + id: "aws-national".into(), + }; + CatalogId { + source: "kma-web".into(), + kind: "station-pages".into(), + key_sha256: server2::storage::sha256(&key.canonical_bytes().unwrap()), + partition: format!("{day:08}"), + } +} +/// The stored products of one day, oldest first, as JSON. +async fn products(peer: &Peer, day: u32) -> Vec { + product_texts(peer, day) + .await + .iter() + .map(|text| serde_json::from_str(text).unwrap()) + .collect() +} +/// The stored products of one day, oldest first, as JSON text. +async fn product_texts(peer: &Peer, day: u32) -> Vec { + let store = CatalogStore::new(peer.transport_limited(4), CatalogLimits::default()).unwrap(); + match store.lookup(&product_catalog(day), deadline()).await { + Ok(server2::storage::LookupOutcome::Ready(set)) => set + .acquisitions() + .iter() + .map(|a| { + let mut raw = Vec::new(); + std::io::Read::read_to_end( + &mut flate2::read::GzDecoder::new(&a.records()[0].bytes[..]), + &mut raw, + ) + .unwrap(); + String::from_utf8(raw).unwrap() + }) + .collect(), + // No catalog yet. + Ok(server2::storage::LookupOutcome::Incomplete) + | Err(server2::storage::Error::NotFound) => Vec::new(), + Err(e) => panic!("product catalog: {e:?}"), + } +} +fn gzip(bytes: &[u8]) -> Vec { + let mut gzip = flate2::write::GzEncoder::new(Vec::new(), flate2::Compression::fast()); + std::io::Write::write_all(&mut gzip, bytes).unwrap(); + gzip.finish().unwrap() +} +/// Store `bytes` as a product of `day` made at `made`. +async fn publish_product(peer: &Peer, day: u32, made: u64, bytes: Vec) { + let store = CatalogStore::new(peer.transport_limited(4), CatalogLimits::default()).unwrap(); + let limits = CatalogLimits::default(); + let record = RawRecord::new( + "kma-web", + "station-pages", + &ProviderKey::Named { + id: "aws-national".into(), + }, + p(day, 0).period(), + made, + 200, + "application/gzip", + Some(Pagination { + page: 1, + pages: 1, + complete: true, + }), + bytes.into(), + &limits.raw, + ) + .unwrap(); + let member = GroupMember { + envelope: record.envelope.clone(), + catalogs: vec![product_catalog(day)], + }; + store + .publish( + GroupDeclaration::new(vec![member], &limits).unwrap(), + vec![record], + deadline(), + ) + .await + .unwrap(); +} +fn wall_ms() -> u64 { + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_millis() as u64 +} +/// The days a process at 10:50 on 2026-09-24 stores products for: those whose every hour is +/// within the 8-day window. +const DAYS: [u32; 8] = [ + 20260917, 20260918, 20260919, 20260920, 20260921, 20260922, 20260923, 20260924, +]; +/// A process that read every page from the raw records and stored its products. +async fn producing(peer: &Peer) -> (Arc, Counted, Arc) { + let (obs, counted) = process(peer); + let folded = settle(&obs).await; + (obs, counted, folded) +} + +/// A process that folds stores one product per day whose hours are all in the window, and stores +/// none again while nothing changed; a restarted process with no provider answers 168 past hours +/// and the minute rows from its first request, reads no raw page the products cover and stores +/// nothing. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_restart_answers_from_the_day_products() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let (a, a_counted, a_folded) = producing(&peer).await; + for day in DAYS { + assert_eq!(products(&peer, day).await.len(), 1, "one product of {day}"); + } + assert!(products(&peer, 20260916).await.is_empty(), "a partial day"); + let written = a_counted.get(&format!("PUT {PRODUCTS}")); + for _ in 0..4 { + let again = answer(&a).await; + assert!(a.settled(NOW as i64, &again)); + } + assert_eq!(a_counted.get(&format!("PUT {PRODUCTS}")), written); + // The restart. + let (b, b_counted) = process(&peer); + let started = std::time::Instant::now(); + let first = answer(&b).await; + assert!(started.elapsed() < Duration::from_millis(2600)); + assert_eq!( + hours_of_108(&first, p(20260917, 0), p(20260923, 2300)), + 168, + "168 past hours on the first answer" + ); + assert!(!first.minute.docs().is_empty(), "minute rows"); + let b_folded = settle(&b).await; + same(&b_folded, &a_folded, "restart"); + for kind in ["aws-hourly", "city-obs"] { + assert_eq!( + raw_gets(&b_counted, kind), + 0, + "{kind} raw pages are in the products" + ); + } + assert_eq!(b_counted.get(&format!("PUT {PRODUCTS}")), 0, "nothing new"); +} + +/// A page published after the product (another process) is restored by itself, the answers equal +/// a cold read of the raw pages, and the day's product is stored again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_page_published_after_the_product_is_restored() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let _ = producing(&peer).await; + // A newer acquisition of 09-24 08:00 with rain at 410, by another process. + let hour = p(20260924, 800); + let later = page_record( + "aws-hourly", + hour, + schedule::to_ms(hour) as u64 + 50 * 60_000, + aws_page(&dotted(hour), true), + ); + let store = CatalogStore::new(peer.transport_limited(4), CatalogLimits::default()).unwrap(); + let declaration = GroupDeclaration::new( + vec![GroupMember { + envelope: later.envelope.clone(), + catalogs: vec![CatalogId::for_record(&later.envelope.identity, "20260924").unwrap()], + }], + &CatalogLimits::default(), + ) + .unwrap(); + store + .publish(declaration, vec![later.clone()], deadline()) + .await + .unwrap(); + let (c, c_counted) = process(&peer); + let c_folded = settle(&c).await; + assert_eq!(raw_gets(&c_counted, "aws-hourly"), 1); + assert_eq!(raw_gets(&c_counted, "city-obs"), 0); + until(&c, || c_counted.get(&format!("PUT {PRODUCTS}")) == 1).await; + // The same raw pages without any product. + let cold_peer = seeded_peer(&[], vec![later]).await; + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + same(&c_folded, &d_folded, "cold read"); +} + +/// A product that cannot be used as it is (unreadable, an older schema or parse revision, a page +/// the catalog does not list, a city set that grew) gives the answers of the raw pages, and the +/// day's product is stored again. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn unusable_products_are_replaced_by_the_raw_pages() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let (_, _, reference) = producing(&peer).await; + let day = 20260922; + // Edited as text: the key order of the stored rows is kept. + let stored = product_texts(&peer, day).await.pop().unwrap(); + let edit = |from: &str, to: &str| { + assert!(stored.contains(from), "{from}"); + gzip(stored.replacen(from, to, 1).as_bytes()) + }; + let hour3 = { + let v: Value = serde_json::from_str(&stored).unwrap(); + let page = v["pages"] + .as_array() + .unwrap() + .iter() + .find(|p| p["kind"] == "aws-hourly" && p["hour"] == 300) + .unwrap() + .clone(); + page["rawId"].as_str().unwrap().to_owned() + }; + let unlisted = hour3.replace(".raw.gz", "0.raw.gz"); + let variants: Vec<(&str, Vec, bool)> = vec![ + ("unreadable", b"not a product".to_vec(), true), + ("older schema", edit("\"schema\":1,", "\"schema\":0,"), true), + ( + "older parse revision", + edit("\"parseRevision\":1,", "\"parseRevision\":0,"), + true, + ), + ( + "a page the catalog does not list", + edit(&format!("\"{hour3}\""), &format!("\"{unlisted}\"")), + false, + ), + ("a grown city set", edit("\"108\",", ""), true), + ]; + for (i, (name, bytes, raw)) in variants.into_iter().enumerate() { + publish_product( + &peer, + day, + wall_ms() + (i as u64 + 1) * HOUR_MS as u64, + bytes, + ) + .await; + let (e, e_counted) = process(&peer); + let folded = settle(&e).await; + same(&folded, &reference, name); + let pages = raw_gets(&e_counted, "aws-hourly") + raw_gets(&e_counted, "city-obs"); + if raw { + assert!(pages >= 48, "{name}: the day's raw pages ({pages})"); + } else { + assert_eq!(pages, 1, "{name}: the unlisted page's hour"); + } + until(&e, || e_counted.get(&format!("PUT {PRODUCTS}")) == 1).await; + } +} + +/// Two processes store their products; a third reads the newest of each day, stores none and +/// answers as they do. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_newest_of_two_processes_products_is_read() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let ((_, _, one), (_, _, two)) = tokio::join!(producing(&peer), producing(&peer)); + same(&one, &two, "two processes"); + for day in DAYS { + assert_eq!(products(&peer, day).await.len(), 2, "{day}"); + } + let (f, f_counted) = process(&peer); + let folded = settle(&f).await; + same(&folded, &one, "the third"); + assert_eq!(f_counted.get(&format!("GET {PRODUCTS}")), DAYS.len()); + assert_eq!(raw_gets(&f_counted, "aws-hourly"), 0); + assert_eq!(f_counted.get(&format!("PUT {PRODUCTS}")), 0); +} + +/// The answer after the first is not complete (S16) while the remaining stored pages are still +/// being restored, and is once they are. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn the_rest_of_the_restart_read_keeps_answers_unsettled() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let _ = producing(&peer).await; + let (b, b_counted) = process(&peer); + // The four newest minute pages come at once, the other sixteen slowly. + b_counted.delay_gets("raw/v2/kma-web/aws-minute", 4, Duration::from_millis(400)); + let _ = answer(&b).await; + let minutes = || b_counted.get("GET raw/v2/kma-web/aws-minute"); + let mut checked = false; + for _ in 0..80 { + let folded = answer(&b).await; + let read = minutes(); + if (8..18).contains(&read) { + assert!( + !b.settled(NOW as i64, &folded), + "unsettled with {read} of 20 minute pages" + ); + checked = true; + } + if read >= 20 { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert!(checked, "the rest of the read was observed"); + let _ = settle(&b).await; +} + +/// A day read with a group S06 excluded stays partial through its product: a restart from the +/// product is never complete, as a read of the raw pages is not. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_excluded_group_keeps_its_day_partial_through_the_product() { + let _peers = super::source_s3::S3_PEERS.lock().await; + // An earlier acquisition of 09-21 05:00 whose raw body is missing. + let hour = p(20260921, 500); + let lost = page_record( + "aws-hourly", + hour, + schedule::to_ms(hour) as u64 + 5 * 60_000, + aws_page(&dotted(hour), false), + ); + let peer = seeded_peer( + std::slice::from_ref(&lost.envelope.identity), + vec![lost.clone()], + ) + .await; + let (a, a_counted) = process(&peer); + until(&a, || { + a_counted.get(&format!("PUT {PRODUCTS}")) >= DAYS.len() + }) + .await; + let stored = products(&peer, 20260921).await; + assert_eq!(stored.len(), 1); + assert_eq!(stored[0]["excluded"], Value::from(1)); + let (b, b_counted) = process(&peer); + for _ in 0..40 { + let folded = answer(&b).await; + assert!(!b.settled(NOW as i64, &folded), "the 09-21 read is partial"); + tokio::time::sleep(Duration::from_millis(100)).await; + } + assert_eq!(raw_gets(&b_counted, "aws-hourly"), 0); + let last = answer(&b).await; + assert_eq!(hours_of_108(&last, p(20260917, 0), p(20260923, 2300)), 168); +} diff --git a/server2/tests/domestic/summary_s3.rs b/server2/tests/domestic/summary_s3.rs index ce070fc8a..b8a4c1189 100644 --- a/server2/tests/domestic/summary_s3.rs +++ b/server2/tests/domestic/summary_s3.rs @@ -253,6 +253,22 @@ async fn serve(backend: &KmaBackend) { .unwrap(); assert!(matches!(served, Native::Served(_)), "served natively"); } +/// Serve until `date` has a stored summary: the first answer after a start is not complete (the +/// days past yesterday are read after it), a later one is. +async fn serve_until( + backend: &KmaBackend, + recorder: &Recorder>, + date: u32, +) { + for _ in 0..40 { + serve(backend).await; + assert!(recorder.idle(deadline()).await); + if names(recorder, date).await > 0 { + return; + } + tokio::time::sleep(Duration::from_millis(100)).await; + } +} async fn names(recorder: &Recorder>, date: u32) -> usize { recorder .store() @@ -293,8 +309,7 @@ async fn served_requests_record_complete_grid_days_and_late_revisions() { .await .unwrap(); let (backend, recorder) = backend(&peer, Arc::new(Unreachable)); - serve(&backend).await; - assert!(recorder.idle(deadline()).await); + serve_until(&backend, &recorder, 20260923).await; let store = recorder.store(); let d23 = store .newest(&town(), 20260923, deadline()) @@ -339,8 +354,7 @@ async fn served_requests_record_complete_grid_days_and_late_revisions() { day(&catalogs, 20260922, [18]).await; current(&catalogs, 20260922, 0, 30., NOW - 60_000.).await; let (backend, recorder) = backend_again(&peer); - serve(&backend).await; - assert!(recorder.idle(deadline()).await); + serve_until(&backend, &recorder, 20260922).await; let store = recorder.store(); let d22 = store .newest(&town(), 20260922, deadline()) @@ -560,7 +574,15 @@ async fn only_the_current_fold_of_a_complete_read_is_settled() { Arc::new(Unreachable), )); let now = NOW as i64; - let folded = observations.stations(now, deadline()).await.unwrap(); + // The rest of the read and the product comparison follow the first answer. + let mut folded = observations.stations(now, deadline()).await.unwrap(); + for _ in 0..20 { + if observations.settled(now, &folded) { + break; + } + tokio::time::sleep(Duration::from_millis(100)).await; + folded = observations.stations(now, deadline()).await.unwrap(); + } assert!( observations.settled(now, &folded), "every window day read, fold current" @@ -621,8 +643,16 @@ async fn seed_settled(ats: &[f64], fault: &str) -> bool { Arc::new(Unreachable), )); let now = NOW as i64; - let folded = observations.stations(now, deadline()).await.unwrap(); - observations.settled(now, &folded) + // The days past yesterday are read after the first answer: a read that can settle does so + // within a few requests. + for _ in 0..20 { + let folded = observations.stations(now, deadline()).await.unwrap(); + if observations.settled(now, &folded) { + return true; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + false } #[tokio::test(flavor = "multi_thread", worker_threads = 2)] diff --git a/specs/issue-2614-kma-restart-observe.md b/specs/issue-2614-kma-restart-observe.md new file mode 100644 index 000000000..531e08bc1 --- /dev/null +++ b/specs/issue-2614-kma-restart-observe.md @@ -0,0 +1,10 @@ +# Acceptance for station observations after a restart + +- **MINUTE:** `CatalogStore::lookup_members` reads one catalog and then only the named members' groups: one descriptor and one raw body per member. An unlisted member answers incomplete, a member of another namespace is refused, and the store deadline holds. A restart over 200 stored minute groups restores at most the 20 within reach, with the minute catalogs read a bounded number of times. Checked by `targeted_members_read_one_catalog_and_only_their_groups` (`tests/storage/catalog_wire.rs`) and `a_restart_restores_only_the_minute_pages_within_reach` (`tests/domestic/observe.rs`). +- **PRODUCT:** a product keeps every stored value and key order, each distinct registry facts row once, and refuses another day, schema or parse revision. A folding process stores one product per day whose hours are all in the window and stores none again while nothing changed. Checked by `every_value_and_key_order_round_trips` (unit test in `observe_product.rs`) and `a_restart_answers_from_the_day_products`. +- **RESTART:** a restarted process with no provider answers the 168 past hourly pages and minute rows on its first request, within the bound. It reads no raw page that the products cover, stores no product and answers as the producing process. A page published after the product is restored by itself and the answers equal a cold read of the raw pages. Of two processes' products the newest is read. Checked by `a_restart_answers_from_the_day_products`, `a_page_published_after_the_product_is_restored` and `the_newest_of_two_processes_products_is_read` (`tests/domestic/observe_product.rs`). +- **FALLBACK:** an unreadable product, an older schema, an older parse revision and a grown city set each give that day's raw pages; a page the catalog does not list is dropped and its hour restored. Each answers as the raw pages do and stores the day's product again. Checked by `unusable_products_are_replaced_by_the_raw_pages`. +- **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1) without reading its raw pages. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. +- **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json`, and the artifact policy passes on outgoing commits. + +Failure paths: a day whose catalogs cannot be listed or whose raw read fails stays failed (S16) and stores no product. A product that cannot be stored is tried again after ten minutes. A newer page that is not complete or does not parse gives the day's raw read. From cd52dc9402ba00386ad858998aef777efe8d5fd9 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 12:02:29 +0900 Subject: [PATCH 03/11] docs: add station pages to the KMA data design restart Co-Authored-By: Claude Opus 5.5 --- .../architecture/diagrams/server2-kma-data-design.html | 4 ++-- .../architecture/diagrams/server2-kma-data-design.json | 4 ++-- server2/config/tasks/KMA-RESTART-OBSERVE.json | 10 ++++++++++ 3 files changed, 14 insertions(+), 4 deletions(-) diff --git a/docs/architecture/diagrams/server2-kma-data-design.html b/docs/architecture/diagrams/server2-kma-data-design.html index 97fde9d53..39f850477 100644 --- a/docs/architecture/diagrams/server2-kma-data-design.html +++ b/docs/architecture/diagrams/server2-kma-data-design.html @@ -5431,7 +5431,7 @@

KMA response-first contract and qualification

Read and bounded repair - Zone state and KASI restore fromone stored fold; a restore holdsa request at most 500 ms. + Zone state, KASI and stationpages restore from stored folds;a restore holds a request at most500 ms. @@ -5758,7 +5758,7 @@

Canonical reuse; conditional candidates

  • Canonical v2 raw/catalog/group controls remain the restore authority. Same selected original closure must replay exactly.
    • -
  • The collector stores zone-state daily, KASI capture stores kasi-fold and the image timer airkorea-forecast-hourly: folds of raw inputs, never assembled weather.
    • +
  • The collector stores zone-state daily, KASI capture stores kasi-fold, folding processes store station-pages per day and the image timer airkorea-forecast-hourly: folds of raw inputs, never assembled weather.
  • Per-grid optimal combinations, source plans, day locators, heads and packs remain deferred. Single EC2 runs only: the cold p95 gate is not qualified.
diff --git a/docs/architecture/diagrams/server2-kma-data-design.json b/docs/architecture/diagrams/server2-kma-data-design.json index 84eaa0a35..163965e7d 100644 --- a/docs/architecture/diagrams/server2-kma-data-design.json +++ b/docs/architecture/diagrams/server2-kma-data-design.json @@ -240,7 +240,7 @@ "to": "s3", "y": 1713, "label": "Read and bounded repair", - "note": "Zone state and KASI restore from one stored fold; a restore holds a request at most 500 ms." + "note": "Zone state, KASI and station pages restore from stored folds; a restore holds a request at most 500 ms." }, { "id": "reacquire", @@ -285,7 +285,7 @@ "title": "Canonical reuse; conditional candidates", "items": [ "Canonical v2 raw/catalog/group controls remain the restore authority. Same selected original closure must replay exactly.", - "The collector stores zone-state daily, KASI capture stores kasi-fold and the image timer airkorea-forecast-hourly: folds of raw inputs, never assembled weather.", + "The collector stores zone-state daily, KASI capture stores kasi-fold, folding processes store station-pages per day and the image timer airkorea-forecast-hourly: folds of raw inputs, never assembled weather.", "Per-grid optimal combinations, source plans, day locators, heads and packs remain deferred. Single EC2 runs only: the cold p95 gate is not qualified." ] } diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 8165f43eb..58b1c7a9d 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -48,6 +48,16 @@ "path": "docs/operations/server2-infrastructure.md", "category": "documentation", "reason": "The station page product is not cached on disk." + }, + { + "path": "docs/architecture/diagrams/server2-kma-data-design.json", + "category": "documentation", + "reason": "Archify source: station pages restore from a stored fold." + }, + { + "path": "docs/architecture/diagrams/server2-kma-data-design.html", + "category": "documentation", + "reason": "Archify output regenerated from the source." } ] } From 78886d20b8e7c4cc56cd6a29f54ae0b0a81b9805 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 12:50:55 +0900 Subject: [PATCH 04/11] server2: account for every listed station group after a restart Review of #2872 (CHANGES_REQUESTED) found three answers that differed from a cold raw read. RO-city-growth: a city page restored after the products could name a station whose AWS rows the products had reduced to registry facts. A fold whose city set grew past the set an AWS page was reduced for now starts Flight::Expand, which reads those pages in full again by their raw identity (RecordId::from_object_key); the next fold reduces them for the new set. RO-minute-fallback: the minute read kept only the newest acquisition of each minute. Up to three are kept, newest first, and a minute whose acquisition is not complete or does not parse is read from the next one. RO-sticky-excluded: a product carried only the count of excluded groups, so a repaired group kept its day partial. Products (schema 2) now record knownIds and excludedIds, and each read of a day restores the groups that are neither held nor known complete: the excluded ones again and those published since. A repaired group completes the day and the product is stored again; a group published later and older than the pages is checked too. S06 excluded members come from CompleteSet. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/server2.md | 43 +- intent/issue-2614-kma-restart-observe.md | 3 +- server2/config/tasks/KMA-RESTART-OBSERVE.json | 1 + server2/src/domestic/kma/source/observe.rs | 421 +++++++++++++----- .../domestic/kma/source/observe_product.rs | 57 ++- server2/src/domestic/kma/source/records.rs | 23 + server2/src/storage/catalog.rs | 13 +- server2/src/storage/codec.rs | 29 ++ server2/tests/domestic/observe.rs | 25 ++ server2/tests/domestic/observe_product.rs | 130 +++++- server2/tests/domestic/summary_s3.rs | 62 +++ specs/issue-2614-kma-restart-observe.md | 5 +- 12 files changed, 665 insertions(+), 147 deletions(-) diff --git a/docs/architecture/server2.md b/docs/architecture/server2.md index 2b3866f37..0f10d497d 100644 --- a/docs/architecture/server2.md +++ b/docs/architecture/server2.md @@ -2279,25 +2279,27 @@ members' groups (one descriptor and one raw body each); `lookup_acquisition` is case, and `ordered_candidates`/`lookup_page` are unchanged for the other cursor readers. `Reader::listed_acquisitions` lists a catalog's acquisitions from the control object alone and is shared with the image forecast. The minute catalogs of the readers' 40 minutes are listed once; of -each minute only the newest acquisition (completion, then raw digest) is restored, at most four +each minute the newest acquisition (completion, then raw digest) is restored, and when it is not +complete or does not parse the next one of that minute (at most three per minute), at most four members per read. **Day product.** One product per KST day D: source `kma-web`, kind `station-pages`, the AWS page key, partition `YYYYMMDD`, period D/0000, fetched at its making instant, published as a one-member group. It is gzip JSON of the parsed inputs of the hourly fold, not its output: `schema`, `parseRevision` (bumped when page parsing or the reduction to registry facts changes), `made`, -`day`, `excluded`, `cityIds`, a `facts` table and `pages`. Per kind and hour of D it holds the +`day`, `excluded`, `knownIds`, `excludedIds`, `cityIds`, a `facts` table and `pages`. Per kind and hour of D it holds the newest page as the process keeps it (`kind`, `hour`, `fetchedAt`, `sha256`, `rawId`, `rows`). AWS rows equal to their registry facts are indices into `facts`; city rows are whole. `cityIds` is the city set the reduced rows are whole for (the intersection of the sets the day's pages were -reduced with). `excluded` is the number of the day's listed groups S06 excluded when the pages -were read, carried forward while the day is compared rather than read again. Values keep every +reduced with). Every listed group of the day is accounted for: its pages, `knownIds` (the raw +identities of the other groups found complete, such as superseded acquisitions) and `excludedIds` +(a member of each group S06 excluded; `excluded` is their number). Values keep every stored value (undefined, dates, negative zero, NaN) and key order. A reader refuses a product that expands past 16 MiB. Any process that folds writes products: after a fold that recomputed the hourly part a `Flight::Product` (never awaited by a request) stores the day whose made-of (sorted page raw -identities, `cityIds`, `parseRevision`) differs from the last product this process read or stored. +identities, `cityIds`, excluded identities, `parseRevision`) differs from the last product this process read or stored. Only days read without failure whose every hour is in the window are stored, each at most once per ten minutes per process. Products are not cached on disk. @@ -2306,21 +2308,30 @@ minute catalogs. Then for the due days, newest first and eight at a time, the da product catalogs are listed (control objects only) and, the first time a process reads the day, its newest product (completion, then digest) is read and decoded. A product whose `cityIds` does not cover the current city set (seed and city pages) is not used. Product pages the catalogs do not -list are dropped (S3 is truth). A day is then compared: for each kind and hour whose newest listed -page (latest fetch; of one instant the smaller digest, the page a read of every acquisition keeps) -is newer than the held one, that page alone is restored and parsed. An unreadable product, an older -schema or parse revision, a grown city set, no product, or a newer page that is not complete or -does not parse gives the day's raw read as before. `Flight::Load` ends after the products, today's +list are dropped (S3 is truth). A day is then compared: every listed group that is neither held nor +known complete (the excluded ones again, and those published since) is restored, four per catalog +read; a complete one becomes known, an incomplete one excluded. Of each kind and hour the newest +complete new page (latest fetch; of one instant the smaller digest, the page a read of every +acquisition keeps) that is newer than the held one is parsed and held. An unreadable product, an +older schema or parse revision, a grown city set, no product, or a new newest page that does not +parse gives the day's raw read as before, which records every complete group as known and every +excluded one. When a fold's city set has grown past the set an AWS page was reduced for (a city +page restored after the products, or acquired since, names another station), `Flight::Expand` +reads those pages in full again by their raw identity (no request waits for it; a failed read is +tried again after ten minutes), and the next fold reduces them for the new set. `Flight::Load` ends after the products, today's and yesterday's comparisons, the seed and the four newest minute pages. `Flight::LoadRest`, which no request waits for, then compares the other days (each marked read and the pages dirty) and restores the rest of the minute window. Reloads every five minutes compare with the pages held. **S16.** Answers, the station page input identities and the response-first durability set are unchanged: a page restored from a product carries the raw identity and digest of the stored page. -A day is partial when its held `excluded` is above zero or a newer group is not complete, as a raw -read with excluded groups is. `settled()` counts `LoadRest`, `Product` and a due product comparison -as busy. Not recorded by a product: a listed group older than its pages that is published later; if -S06 excludes it, a restart from the product counts the day complete where a raw read would not. +A day is partial while one of its listed groups is excluded, as a raw read with excluded groups is. +Excluded groups are checked again at every read of the day (a restart, and each reload), so a +repaired group completes the day and its product is stored again without it. A group published +after a product, older than its pages, is unaccounted for and checked like any new group, so a +restart from products counts a day complete exactly when a raw read would, given that a complete +raw object is never deleted. `settled()` counts `LoadRest`, `Product`, `Expand` and a due product +comparison as busy. Right after the first deployment the window's oldest day has no product (some of its hours were already out of the window) and is read from its raw pages after the first answer. @@ -2329,7 +2340,9 @@ Verification (local, S06 loopback S3 peer, no provider): targeted member GET cou 168 past hourly pages and minute rows on its first request without raw page GETs, a later page restored alone and equal to a cold raw read, unusable products replaced by the raw pages and stored again, two processes' products, unsettled answers during the rest of the read, a day with an -S06-excluded group kept partial, and the disk cache exclusion. +S06-excluded group kept partial, a repaired group completing its day (and recording summaries), a +station joining the city set after the products, an incomplete newest minute acquisition, and the +disk cache exclusion. ### S15b-2 s station registry seed (2026-10-15) diff --git a/intent/issue-2614-kma-restart-observe.md b/intent/issue-2614-kma-restart-observe.md index 8418d68e1..0bc4fe2d8 100644 --- a/intent/issue-2614-kma-restart-observe.md +++ b/intent/issue-2614-kma-restart-observe.md @@ -7,6 +7,7 @@ AK requires that a request waits at most 3 s and answers from what was gathered - **MINUTE:** a restart lists the day's minute catalog once and restores only the newest acquisition of each minute within the readers' 40 minutes. The newest four come before the first answer and the rest follow. - **PRODUCT:** any process that folds the station pages stores one product per KST day holding the parsed inputs of the hourly fold, the newest page of each kind and hour, at most once per ten minutes per day and only when what the day is made of changed. No request waits for it. - **RESTART:** a restarted process reads each day's newest product and compares it with the day catalogs (S3 is truth). It restores only the pages the product lacks, parses only those, and answers past days and minute rows from its first request. Today and yesterday are compared before the answer, the other days after it. +- **ACCOUNTED:** a repaired group completes its day after a restart, a station that joins the city set after the products has its AWS rows read in full again, and a minute whose newest acquisition is not complete is read from the next one, each as a raw read gives it. - **FALLBACK:** an unreadable product, an older schema or parse revision, or a city set that grew is replaced by the raw pages of that day, and the day's product is stored again. A page the catalog does not list is dropped. - **S16:** legacy answers, the station page input identities and the response-first durability set are unchanged. A day stays partial when its product's read had a group S06 excluded or a newer group is not complete, and answers stay unsettled until the rest of the read ends. - **PLACEMENT:** the placement gate and the artifact policy pass on the outgoing commits. @@ -16,7 +17,7 @@ AK requires that a request waits at most 3 s and answers from what was gathered This adds a stored object that other processes trust as a starting point, so it is not merged without AK: - Products are written to S3 under `kma-web` / `station-pages` beside the pages they are made of, never deleted, at most one per day and ten minutes per process. They are excluded from the disk cache, like the image forecast product. -- A day product does not record listed groups older than its pages. If such a group is published after the product and S06 excludes it, a restart from the product counts the day as complete where a raw read would not. +- A day product records every listed group of its day: the pages, the other complete groups and the S06-excluded ones. Each read checks the excluded groups and any group published since, so a restart counts a day complete exactly when a raw read would, as long as no complete raw object is ever deleted. While a group stays excluded, every read of its day spends one catalog read and the group's descriptor and raw GETs on it. - The first answer after a start is never complete for S16: the days past yesterday are compared after it, so a later request records the daily summaries. A fold that recomputes the hourly part also holds summaries back until its product comparison ends. - A process at hour H stores products only for days whose every hour is in the window. Right after the first deployment, the oldest day of the window therefore has no product and is read from its raw pages after the first answer. diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 58b1c7a9d..354cff35e 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -11,6 +11,7 @@ "server2/src/domestic/kma/source/observe_product.rs", "server2/src/domestic/kma/source/records.rs", "server2/src/storage/catalog.rs", + "server2/src/storage/codec.rs", "server2/tests/cache.rs", "server2/tests/domestic.rs", "server2/tests/domestic/observe.rs", diff --git a/server2/src/domestic/kma/source/observe.rs b/server2/src/domestic/kma/source/observe.rs index 2611a4a1b..8d9d79b65 100644 --- a/server2/src/domestic/kma/source/observe.rs +++ b/server2/src/domestic/kma/source/observe.rs @@ -84,6 +84,11 @@ const MINUTE_KEEP_MS: f64 = 40. * 60_000.; const MINUTE_FIRST: usize = 4; /// A day's product is stored at most once per this interval by one process. const PRODUCT_EVERY: Duration = Duration::from_secs(600); +/// A reduced AWS hourly page whose full read failed is read again after this long. +const EXPAND_RETRY: Duration = Duration::from_secs(600); +/// Acquisitions of one minute tried, newest first, when a newer one is not complete or does not +/// parse. +const MINUTE_CANDIDATES: usize = 3; /// Legacy minute collection cadence. const MINUTE_CADENCE: i64 = 2; /// S3 day partitions read again after this long (other processes may have published). @@ -180,6 +185,8 @@ enum Flight { LoadRest, /// Store the day products whose pages changed. Product, + /// Read AWS hourly pages in full again whose reduction misses a city station. + Expand, Fold, Geocode, Minute, @@ -216,8 +223,12 @@ struct Pages { failed_days: std::collections::BTreeSet, /// Days whose latest S3 read failed (no product is stored for them). unreadable: BTreeSet, - /// Listed groups S06 excluded in each read day (S16; a product carries the count). - excluded: BTreeMap, + /// Raw identities of the listed groups of each read day found complete beyond the held + /// pages (superseded ones; S16 accounts for every listed group). + known: BTreeMap>, + /// Raw identities of a member of each listed group of a read day S06 excluded (S16; checked + /// again at every read of the day). + excluded: BTreeMap>, /// AWS hourly pages kept reduced to registry facts outside this city set. reduced: BTreeMap>>, minute_read: Option, @@ -257,6 +268,8 @@ struct State { product_attempts: HashMap, /// A fold recomputed the hourly part since the products were last compared. product_due: bool, + /// When this process last read each reduced AWS hourly page in full again. + expand_attempts: HashMap, } #[derive(Clone)] struct HourlyFold { @@ -595,6 +608,7 @@ impl Observations { s.pages.failed_days.retain(|d| *d >= oldest); s.pages.unreadable.retain(|d| *d >= oldest); s.pages.excluded.retain(|d, _| *d >= oldest); + s.pages.known.retain(|d, _| *d >= oldest); s.products.retain(|d, _| *d >= oldest); s.product_attempts.retain(|d, _| *d >= oldest); } @@ -726,7 +740,10 @@ impl Observations { }; for r in reads.iter_mut() { if s.pages.days.contains_key(&r.day) { - r.base = Some(s.pages.excluded.get(&r.day).copied().unwrap_or(0)); + r.base = Some(Accounted { + known: s.pages.known.get(&r.day).cloned().unwrap_or_default(), + excluded: s.pages.excluded.get(&r.day).cloned().unwrap_or_default(), + }); continue; } let (Some(listed), Some(stored)) = (&r.listed, r.product.take()) else { @@ -737,7 +754,8 @@ impl Observations { } s.products.insert(r.day, stored.made_of()); let reduced = Arc::new(stored.city_ids.clone()); - for page in stored.pages { + let mut stored = stored; + for page in std::mem::take(&mut stored.pages) { if !listed.iter().any(|l| l.raw_id == page.raw_id) { continue; } @@ -751,15 +769,19 @@ impl Observations { ); hold(&mut s.pages, page.kind, held, reduced); } - r.base = Some(stored.excluded); + r.base = Some(Accounted { + known: stored.known, + excluded: stored.excluded, + }); s.pages.dirty = true; s.pages.hourly_dirty = true; } } /// Bring one day up to its catalogs. With pages held for it (a product, or an earlier read), - /// only the newest listed page of each kind and hour that is newer than the held one is - /// restored and parsed; otherwise, or when such a page is not complete or does not parse, - /// every page of the day is read as before (the newest complete page of each hour). + /// only the listed groups not accounted for are restored: the excluded ones again and the new + /// ones, and a new page newer than the held one of its kind and hour is parsed and held. + /// Otherwise, or when such a page does not parse, every page of the day is read as before + /// (the newest complete page of each hour). async fn complete_day(&self, r: DayRead, deadline: Instant) { let DayRead { day, listed, base, .. @@ -768,34 +790,73 @@ impl Observations { self.day_failed(day); return; }; - if let Some(excluded) = base { - match self.restore_newer(&listed, deadline).await { - Ok(Some(changed)) => return self.day_read(day, excluded, changed), + if let Some(base) = base { + match self.compare_day(day, &listed, base, deadline).await { + Ok(Some((changed, accounted))) => return self.day_read(day, accounted, changed), Ok(None) => {} Err(()) => return self.day_failed(day), } } self.raw_day(day, &listed, deadline).await; } - /// Restore and hold the newest listed pages newer than the held ones. Ok(None): one is not - /// complete or does not parse. - async fn restore_newer( + /// [`Self::complete_day`] with pages held. Ok(None): a new newest page does not parse. + async fn compare_day( &self, + day: u32, listed: &[ListedPage], + base: Accounted, deadline: Instant, - ) -> Result, ()> { - let mut newest: BTreeMap<(&'static str, Publication), &ListedPage> = BTreeMap::new(); - for page in listed { - let slot = newest.entry((page.kind, page.hour)).or_insert(page); - if newer(page.at, &page.sha256, Some((slot.at, slot.sha256.as_str()))) { - *slot = page; + ) -> Result, ()> { + let Accounted { mut known, .. } = base; + let unknown: Vec<&ListedPage> = { + let s = self.state(); + let held: BTreeSet<&str> = s + .pages + .ids + .iter() + .filter(|((_, hour), _)| hour.date() == day) + .map(|(_, (_, _, raw_id))| raw_id.as_str()) + .collect(); + listed + .iter() + .filter(|p| !held.contains(p.raw_id.as_str()) && !known.contains(&p.raw_id)) + .collect() + }; + let wanted: Vec<(CatalogId, crate::storage::RecordId)> = unknown + .iter() + .map(|p| (p.catalog.clone(), p.member.clone())) + .collect(); + let read = self.restore(&wanted, deadline).await?; + let mut excluded = BTreeSet::new(); + let mut complete: Vec<(&ListedPage, Vec)> = Vec::new(); + for (page, records) in unknown.into_iter().zip(read) { + match records { + Some(records) => { + known.insert(page.raw_id.clone()); + complete.push((page, records)); + } + None => { + excluded.insert(page.raw_id.clone()); + } + } + } + // Of each kind and hour the newest complete new page, when newer than the held one. + let mut newest: BTreeMap<(&'static str, Publication), (&ListedPage, Vec)> = + BTreeMap::new(); + for (page, records) in complete { + match newest.get(&(page.kind, page.hour)) { + Some((other, _)) + if !newer(page.at, &page.sha256, Some((other.at, &other.sha256))) => {} + _ => { + newest.insert((page.kind, page.hour), (page, records)); + } } } - let wanted: Vec<&ListedPage> = { + let found: Vec<(&'static str, Vec)> = { let s = self.state(); newest .into_values() - .filter(|page| { + .filter(|(page, _)| { newer( page.at, &page.sha256, @@ -805,28 +866,9 @@ impl Observations { .map(|(at, sha, _)| (*at, sha.as_str())), ) }) + .map(|(page, records)| (page.kind, records)) .collect() }; - if wanted.is_empty() { - return Ok(Some(false)); - } - let mut found: Vec<(&'static str, Vec)> = Vec::new(); - for kind in product::PAGE_KINDS { - let of_kind: Vec<&ListedPage> = - wanted.iter().copied().filter(|p| p.kind == kind).collect(); - for chunk in of_kind.chunks(MINUTE_FIRST) { - let members: Vec<_> = chunk.iter().map(|p| p.member.clone()).collect(); - match self - .reader - .members(&chunk[0].catalog, &members, deadline) - .await - { - Ok(Some(read)) => found.extend(read.into_iter().map(|a| (kind, a))), - Ok(None) => return Ok(None), - Err(_) => return Err(()), - } - } - } let parsed = crate::observability::spawn_blocking(move || { found .iter() @@ -851,7 +893,50 @@ impl Observations { for (kind, page) in parsed { changed |= hold(&mut s.pages, kind, page, None); } - Ok(Some(changed)) + // Pages a newer one replaced stay accounted for. + known.extend(s.pages.known.get(&day).into_iter().flatten().cloned()); + Ok(Some((changed, Accounted { known, excluded }))) + } + /// Restore `wanted` (catalog and member), a few members per catalog read (each read is bounded + /// by the S06 lookup deadline): the records of each, or None when its group is not complete. + /// Err: a read failed. + async fn restore( + &self, + wanted: &[(CatalogId, crate::storage::RecordId)], + deadline: Instant, + ) -> Result>>, ()> { + let mut out: Vec>> = vec![None; wanted.len()]; + let mut at = 0; + while at < wanted.len() { + let catalog = &wanted[at].0; + let mut end = at + 1; + while end < wanted.len() && end - at < MINUTE_FIRST && wanted[end].0 == *catalog { + end += 1; + } + let members: Vec<_> = wanted[at..end].iter().map(|(_, m)| m.clone()).collect(); + match self.reader.members(catalog, &members, deadline).await { + Ok(Some(read)) if read.len() == members.len() => { + for (slot, records) in out[at..end].iter_mut().zip(read) { + *slot = Some(records); + } + } + Ok(_) if members.len() == 1 => {} + Ok(_) => { + // One of them is not complete: each alone. + for (slot, member) in out[at..end].iter_mut().zip(&members) { + *slot = self + .reader + .members(catalog, std::slice::from_ref(member), deadline) + .await + .map_err(|_| ())? + .and_then(|mut read| read.pop()); + } + } + Err(_) => return Err(()), + } + at = end; + } + Ok(out) } /// Read every page of one day (each complete acquisition is restored and parsed). async fn raw_day(&self, day: u32, listed: &[ListedPage], deadline: Instant) { @@ -865,25 +950,39 @@ impl Observations { async { if want_hourly { self.reader - .raw_counted("kma-web", "aws-hourly", &aws, &partition, deadline) + .raw_excluded("kma-web", "aws-hourly", &aws, &partition, deadline) .await } else { - Ok((Vec::new(), 0)) + Ok((Vec::new(), Vec::new())) } }, async { if want_city { self.reader - .raw_counted("kma-web", "city-obs", &city_k, &partition, deadline) + .raw_excluded("kma-web", "city-obs", &city_k, &partition, deadline) .await } else { - Ok((Vec::new(), 0)) + Ok((Vec::new(), Vec::new())) } }, ); let (Ok((hourly, hourly_excluded)), Ok((city, city_excluded))) = (hourly, city) else { return self.day_failed(day); }; + // S16: every listed group is complete (known) or excluded. + let key = |r: &RawRecord| r.envelope.identity.object_key().ok(); + let accounted = Accounted { + known: hourly + .iter() + .chain(&city) + .filter_map(|a| a.first().and_then(key)) + .collect(), + excluded: hourly_excluded + .iter() + .chain(&city_excluded) + .filter_map(|id| id.object_key().ok()) + .collect(), + }; let parsed = crate::observability::spawn_blocking(move || { ( hourly @@ -907,15 +1006,17 @@ impl Observations { } } // S16: a group S06 excluded makes the day's read partial. - self.day_read(day, hourly_excluded + city_excluded, true); + self.day_read(day, accounted, true); } - /// A day read completely, with `excluded` listed groups S06 excluded (S16). - fn day_read(&self, day: u32, excluded: usize, changed: bool) { + /// A day read completely: its listed groups found complete and those S06 excluded (S16). + fn day_read(&self, day: u32, accounted: Accounted, changed: bool) { let mut s = self.state(); s.pages.days.insert(day, std::time::Instant::now()); s.pages.unreadable.remove(&day); - s.pages.excluded.insert(day, excluded); - if excluded > 0 { + let partial = !accounted.excluded.is_empty(); + s.pages.known.insert(day, accounted.known); + s.pages.excluded.insert(day, accounted.excluded); + if partial { s.pages.failed_days.insert(day); } else { s.pages.failed_days.remove(&day); @@ -932,10 +1033,11 @@ impl Observations { s.pages.failed_days.insert(day); } /// The minute pages of the readers' reach that this process does not hold, newest first by - /// completion: the day catalogs are listed (control objects only, no record is read), and of - /// each minute only its newest acquisition is kept. A catalog that cannot be listed adds - /// nothing (the next reload lists it again). - async fn minute_entries(&self, now_ms: i64, deadline: Instant) -> Vec { + /// completion: the day catalogs are listed (control objects only, no record is read). Of each + /// minute up to [`MINUTE_CANDIDATES`] acquisitions are kept, newest first (the order a + /// latest-first read meets them). A catalog that cannot be listed adds nothing (the next reload + /// lists it again). + async fn minute_entries(&self, now_ms: i64, deadline: Instant) -> Vec> { let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); let today = schedule::from_ms(now_ms); let mut partitions = vec![format!("{:08}", today.date())]; @@ -943,7 +1045,7 @@ impl Observations { partitions.push(format!("{:08}", floor.date())); } let held: BTreeSet = self.state().pages.minute.keys().copied().collect(); - let mut newest: BTreeMap = BTreeMap::new(); + let mut minutes: BTreeMap> = BTreeMap::new(); for partition in partitions { let Ok(id) = super::records::catalog("kma-web", "aws-minute", &aws_key(), &partition) else { @@ -962,73 +1064,75 @@ impl Observations { if p <= floor || held.contains(&p) { continue; } - let entry = MinuteEntry { + minutes.entry(p).or_default().push(MinuteEntry { catalog: id.clone(), member: e.identity, completed, - }; - // Of one minute, the acquisition a latest-first read meets first. - if newest.get(&p).is_none_or(|old| entry.after(old)) { - newest.insert(p, entry); - } + }); } } - let mut out: Vec = newest.into_values().collect(); - out.sort_by(|a, b| b.order().cmp(&a.order())); + let mut out: Vec> = minutes + .into_values() + .map(|mut candidates| { + candidates.sort_by(|a, b| b.order().cmp(&a.order())); + candidates.truncate(MINUTE_CANDIDATES); + candidates + }) + .collect(); + out.sort_by(|a, b| b[0].order().cmp(&a[0].order())); out } - /// Restore `entries` (one catalog read per catalog, then only their groups) and keep the pages - /// after `floor` that this process does not hold. A group that is not complete leaves its - /// minute out, not the others. + /// Restore the minutes (candidates newest first) and keep the pages after `floor` that this + /// process does not hold: the newest acquisition of each, and for a minute whose acquisition + /// is not complete or does not parse, the next one. async fn restore_minutes( &self, - entries: Vec, + minutes: Vec>, floor: Publication, deadline: Instant, ) { - // A few members per read: each read is bounded by the S06 lookup deadline. - let mut by_catalog: Vec<(CatalogId, Vec)> = Vec::new(); - for e in entries { - match by_catalog + let mut pending: Vec> = + minutes.into_iter().map(Vec::into_iter).collect(); + let mut done = vec![false; pending.len()]; + let mut kept = Vec::new(); + for _ in 0..MINUTE_CANDIDATES { + let round: Vec<(usize, MinuteEntry)> = pending .iter_mut() - .find(|(id, members)| *id == e.catalog && members.len() < MINUTE_FIRST) - { - Some((_, members)) => members.push(e.member), - None => by_catalog.push((e.catalog, vec![e.member])), + .enumerate() + .filter(|(i, _)| !done[*i]) + .filter_map(|(i, c)| c.next().map(|e| (i, e))) + .collect(); + if round.is_empty() { + break; } - } - let mut found = Vec::new(); - for (id, members) in by_catalog { - match self.reader.members(&id, &members, deadline).await { - Ok(Some(read)) => found.extend(read), - Ok(None) => { - for member in members { - if let Ok(Some(read)) = self - .reader - .members(&id, std::slice::from_ref(&member), deadline) - .await - { - found.extend(read); - } + let wanted: Vec<_> = round + .iter() + .map(|(_, e)| (e.catalog.clone(), e.member.clone())) + .collect(); + let Ok(read) = self.restore(&wanted, deadline).await else { + break; + }; + let parsed = crate::observability::spawn_blocking(move || { + read.iter() + .map(|a| a.as_deref().and_then(parse_minute)) + .collect::>() + }) + .await + .unwrap_or_default(); + for ((i, _), page) in round.iter().zip(parsed) { + if let Some((p, rows)) = page { + done[*i] = true; + if p > floor { + kept.push((p, rows)); } } - Err(_) => {} } } - if found.is_empty() { + if kept.is_empty() { return; } - let parsed = crate::observability::spawn_blocking(move || { - found - .iter() - .filter_map(|a| parse_minute(a)) - .filter(|(p, _)| *p > floor) - .collect::>() - }) - .await - .unwrap_or_default(); let mut s = self.state(); - for (p, rows) in parsed { + for (p, rows) in kept { s.pages.minute.entry(p).or_insert(rows); } s.pages.dirty = true; @@ -1347,7 +1451,27 @@ impl Observations { } } s.hourly_fold = Some(fold.clone()); + // Pages reduced for a smaller city set lack a city station's rows (a station + // joined the set since): they are read in full again. + let stale: Vec<(Publication, String)> = s + .pages + .reduced + .iter() + .filter(|(_, set)| !set.is_superset(&fold.cities)) + .filter_map(|(h, _)| { + let (_, _, raw_id) = s.pages.ids.get(&("aws-hourly", *h))?; + Some((*h, raw_id.clone())) + }) + .filter(|(h, _)| { + s.expand_attempts + .get(h) + .is_none_or(|at| at.elapsed() >= EXPAND_RETRY) + }) + .collect(); drop(s); + if !stale.is_empty() { + self.expand(stale); + } self.want_products(now_ms); fold } @@ -1435,6 +1559,61 @@ impl Observations { }); } + // ---- reduced pages ---- + + /// Read the AWS hourly `pages` (hour and raw identity) in full again and hold their whole + /// rows while still the held pages; no request waits for it. + fn expand(self: &Arc, pages: Vec<(Publication, String)>) { + let _ = self.start(Flight::Expand, Duration::ZERO, move |this| { + Box::pin(async move { this.read_full(pages).await }) + }); + } + async fn read_full(&self, pages: Vec<(Publication, String)>) { + let deadline = Instant::now() + READ_DEADLINE; + let mut wanted = Vec::new(); + { + let mut s = self.state(); + for (hour, raw_id) in &pages { + s.expand_attempts.insert(*hour, std::time::Instant::now()); + let (Ok(member), Ok(catalog)) = ( + crate::storage::RecordId::from_object_key(raw_id), + super::records::catalog("kma-web", "aws-hourly", &aws_key(), &hour.base_date()), + ) else { + continue; + }; + wanted.push((catalog, member)); + } + } + let Ok(read) = self.restore(&wanted, deadline).await else { + return; + }; + let parsed = crate::observability::spawn_blocking(move || { + read.iter() + .filter_map(|a| a.as_deref().and_then(parse_hourly)) + .collect::>() + }) + .await + .unwrap_or_default(); + let mut s = self.state(); + let mut changed = false; + for (p, rows, _, _, raw_id) in parsed { + let same = s + .pages + .ids + .get(&("aws-hourly", p)) + .is_some_and(|(_, _, held)| *held == raw_id); + if same { + s.pages.hourly.insert(p, rows); + s.pages.reduced.remove(&p); + changed = true; + } + } + if changed { + s.pages.dirty = true; + s.pages.hourly_dirty = true; + } + } + // ---- day products ---- /// Compare the day products with the pages held (after a fold recomputed the hourly part) and @@ -1887,6 +2066,7 @@ impl Observations { Flight::Fold, Flight::Geocode, Flight::Product, + Flight::Expand, ] .iter() .any(|f| flights.contains_key(f)) @@ -1970,9 +2150,6 @@ impl MinuteEntry { fn order(&self) -> (u64, &str) { (self.completed, &self.member.raw_sha256) } - fn after(&self, other: &Self) -> bool { - self.order() > other.order() - } } type ParsedPage = (Publication, Arc>, f64, String, String); fn parse_hourly(records: &[RawRecord]) -> Option { @@ -2077,7 +2254,12 @@ fn product_candidates(s: &mut State, now_ms: i64) -> Vec<(DayProduct, MadeOf)> { }) }) .unwrap_or_else(|| (*cities).clone()); - let made_of = MadeOf::new(pages.iter().map(|p| p.raw_id.as_str()), &city_ids); + let excluded = s.pages.excluded.get(&day).cloned().unwrap_or_default(); + let made_of = MadeOf::new( + pages.iter().map(|p| p.raw_id.as_str()), + &city_ids, + &excluded, + ); if s.products.get(&day) == Some(&made_of) || s.product_attempts .get(&day) @@ -2090,7 +2272,16 @@ fn product_candidates(s: &mut State, now_ms: i64) -> Vec<(DayProduct, MadeOf)> { DayProduct { made, day, - excluded: s.pages.excluded.get(&day).copied().unwrap_or(0), + known: s + .pages + .known + .get(&day) + .into_iter() + .flatten() + .filter(|id| !pages.iter().any(|p| p.raw_id == **id)) + .cloned() + .collect(), + excluded, city_ids, pages, }, @@ -2235,6 +2426,13 @@ fn hold( if !newer(at, &sha, current) { return false; } + if let Some((_, _, replaced)) = pages.ids.get(&(kind, p)) { + pages + .known + .entry(p.date()) + .or_default() + .insert(replaced.clone()); + } if kind == "aws-hourly" { pages.hourly.insert(p, rows); pages.hourly_at.insert(p, at); @@ -2269,7 +2467,12 @@ struct DayRead { day: u32, listed: Option>, product: Option, - base: Option, + base: Option, +} +/// The listed groups of a day beyond its held pages: found complete, and S06 excluded. +struct Accounted { + known: BTreeSet, + excluded: BTreeSet, } /// A page a day catalog lists. struct ListedPage { diff --git a/server2/src/domestic/kma/source/observe_product.rs b/server2/src/domestic/kma/source/observe_product.rs index eedb2f98f..6b276f3f5 100644 --- a/server2/src/domestic/kma/source/observe_product.rs +++ b/server2/src/domestic/kma/source/observe_product.rs @@ -8,9 +8,11 @@ //! registry facts (the reduced form the fold keeps them in), each distinct facts document once. //! //! A product is never the truth: a reader compares it with the day catalogs and restores what it -//! lacks. `cityIds` is the city set its reduced rows keep in full; `excluded` is the number of the -//! day's listed groups S06 excluded when its pages were read (S16 completeness). Values keep -//! every [`Val`] (undefined, dates, negative zero and NaN) through a tagged string form. +//! lacks. `cityIds` is the city set its reduced rows keep in full. Every listed group of the day is +//! accounted for (S16 completeness): its pages, `knownIds` (the other groups found complete) and +//! `excludedIds` (a member of each group S06 excluded, `excluded` of them); a reader checks the +//! excluded and the unaccounted groups again. Values keep every [`Val`] (undefined, dates, negative +//! zero and NaN) through a tagged string form. use super::doc::{Doc, Val}; use crate::domestic::publication::Publication; use serde::de::{self, Deserializer, MapAccess, SeqAccess, Visitor}; @@ -22,7 +24,7 @@ use std::sync::Arc; /// Kind of the product (source `kma-web`, the AWS page key, partition `YYYYMMDD`). pub const KIND: &str = "station-pages"; /// The product's layout. -pub const SCHEMA: u64 = 1; +pub const SCHEMA: u64 = 2; /// Bumped whenever page parsing (`parse_aws`, `parse_city`) or the reduction to registry facts /// changes: an older product is then read as absent. pub const PARSE_REVISION: u64 = 1; @@ -48,25 +50,33 @@ pub struct StoredPage { pub struct DayProduct { pub made: f64, pub day: u32, - /// Listed groups S06 excluded when these pages were read. - pub excluded: usize, + /// Raw identities of the other listed groups found complete. + pub known: BTreeSet, + /// Raw identities of a member of each listed group S06 excluded. + pub excluded: BTreeSet, pub city_ids: BTreeSet, pub pages: Vec, } -/// What a product is made of: its pages, city set and parse revision. +/// What a product is made of: its pages, city set, excluded groups and parse revision. #[derive(Debug, Clone, PartialEq, Eq)] pub struct MadeOf { pages: Vec, city_ids: Vec, + excluded: Vec, revision: u64, } impl MadeOf { - pub fn new<'a>(raw_ids: impl Iterator, city_ids: &BTreeSet) -> Self { + pub fn new<'a>( + raw_ids: impl Iterator, + city_ids: &BTreeSet, + excluded: &BTreeSet, + ) -> Self { let mut pages: Vec = raw_ids.map(str::to_owned).collect(); pages.sort(); Self { pages, city_ids: city_ids.iter().cloned().collect(), + excluded: excluded.iter().cloned().collect(), revision: PARSE_REVISION, } } @@ -74,7 +84,11 @@ impl MadeOf { impl DayProduct { pub fn made_of(&self) -> MadeOf { - MadeOf::new(self.pages.iter().map(|p| p.raw_id.as_str()), &self.city_ids) + MadeOf::new( + self.pages.iter().map(|p| p.raw_id.as_str()), + &self.city_ids, + &self.excluded, + ) } /// The stored form: gzip of the product JSON. An AWS row equal to its registry facts /// (`facts`) is stored as an index into the facts table. @@ -111,7 +125,9 @@ impl DayProduct { parse_revision: PARSE_REVISION, made: self.made, day: self.day, - excluded: self.excluded, + excluded: self.excluded.len(), + known_ids: &self.known, + excluded_ids: &self.excluded, city_ids: &self.city_ids, facts: table.into_iter().map(DocOut).collect(), pages, @@ -134,7 +150,11 @@ impl DayProduct { return None; } let stored: ProductIn = serde_json::from_slice(&bytes).ok()?; - if stored.schema != SCHEMA || stored.parse_revision != PARSE_REVISION || stored.day != day { + if stored.schema != SCHEMA + || stored.parse_revision != PARSE_REVISION + || stored.day != day + || stored.excluded != stored.excluded_ids.len() + { return None; } let facts: Vec = stored @@ -176,7 +196,8 @@ impl DayProduct { Some(Self { made: stored.made, day, - excluded: stored.excluded, + known: stored.known_ids, + excluded: stored.excluded_ids, city_ids: stored.city_ids, pages, }) @@ -191,6 +212,8 @@ struct ProductOut<'a> { made: f64, day: u32, excluded: usize, + known_ids: &'a BTreeSet, + excluded_ids: &'a BTreeSet, city_ids: &'a BTreeSet, facts: Vec>, pages: Vec>, @@ -225,6 +248,8 @@ struct ProductIn { made: f64, day: u32, excluded: usize, + known_ids: BTreeSet, + excluded_ids: BTreeSet, city_ids: BTreeSet, facts: Vec, pages: Vec, @@ -394,7 +419,8 @@ mod tests { let product = DayProduct { made: 1_790_300_000_000.5, day: 20260924, - excluded: 2, + known: ["raw/c".to_owned()].into(), + excluded: ["raw/d".to_owned(), "raw/e".to_owned()].into(), city_ids: ["108".to_owned()].into(), pages: vec![ StoredPage { @@ -419,7 +445,10 @@ mod tests { let back = DayProduct::decode(&bytes, 20260924).unwrap(); assert_eq!(back.made_of(), product.made_of()); assert_eq!(back.city_ids, product.city_ids); - assert_eq!((back.made, back.excluded), (product.made, product.excluded)); + assert_eq!( + (back.made, &back.known, &back.excluded), + (product.made, &product.known, &product.excluded) + ); for (a, b) in back.pages.iter().zip(&product.pages) { assert_eq!( (a.kind, a.hour, a.fetched_at_ms, &a.sha256, &a.raw_id), diff --git a/server2/src/domestic/kma/source/records.rs b/server2/src/domestic/kma/source/records.rs index 7b3395ec5..69233f443 100644 --- a/server2/src/domestic/kma/source/records.rs +++ b/server2/src/domestic/kma/source/records.rs @@ -306,6 +306,29 @@ impl Reader { .await .map(|(r, _)| r) } + /// [`Self::raw`] and a listed member of each group S06 excluded (S16 completeness). + pub async fn raw_excluded( + &self, + source: &str, + kind: &str, + key: &ProviderKey, + partition: &str, + deadline: Instant, + ) -> Result<(Vec>, Vec), Error> { + let id = catalog(source, kind, key, partition)?; + match self.store.lookup(&id, deadline).await { + Ok(LookupOutcome::Ready(set)) => Ok(( + set.acquisitions() + .iter() + .map(|a| ordered(a.records())) + .collect(), + set.excluded_members().to_vec(), + )), + Ok(LookupOutcome::Incomplete) => Err(Error::Corrupt("incomplete catalog")), + Err(Error::NotFound) => Ok((Vec::new(), Vec::new())), + Err(e) => Err(e), + } + } /// [`Self::raw`] and the number of listed groups S06 excluded (S16 completeness). pub async fn raw_counted( &self, diff --git a/server2/src/storage/catalog.rs b/server2/src/storage/catalog.rs index 949f65248..5e2b58ca8 100644 --- a/server2/src/storage/catalog.rs +++ b/server2/src/storage/catalog.rs @@ -40,6 +40,7 @@ pub struct CompleteSet { acquisitions: Vec, dependencies: Vec, excluded: usize, + excluded_members: Vec, } impl CompleteSet { pub fn coverage(&self) -> RevisionCoverage { @@ -54,6 +55,10 @@ impl CompleteSet { pub fn excluded_groups(&self) -> usize { self.excluded } + /// A listed member of each group S06 excluded. + pub fn excluded_members(&self) -> &[RecordId] { + &self.excluded_members + } /// Parse each complete acquisition's ordered pages in memory. No result is persisted. pub fn fold( &self, @@ -667,6 +672,7 @@ impl CatalogStore { let mut groups = BTreeSet::new(); let mut acquisitions = Vec::new(); let mut excluded = 0; + let mut excluded_members = Vec::new(); for e in entries { if let Some(r) = &e.fetch_group && !groups.insert(r.group_sha256.clone()) @@ -675,7 +681,10 @@ impl CatalogStore { } match self.acquisition(id, &e, c).await? { Some(a) => acquisitions.push(a), - None => excluded += 1, + None => { + excluded += 1; + excluded_members.push(e.identity.clone()); + } } } let acquisitions = self @@ -705,6 +714,7 @@ impl CatalogStore { acquisitions, dependencies, excluded, + excluded_members, })) } pub async fn lookup(&self, id: &CatalogId, deadline: Instant) -> Result { @@ -1030,6 +1040,7 @@ impl CatalogStore { }], dependencies: Self::dependencies(&context), excluded: 0, + excluded_members: Vec::new(), }) }) .await; diff --git a/server2/src/storage/codec.rs b/server2/src/storage/codec.rs index 1bc8a7245..092b12e02 100644 --- a/server2/src/storage/codec.rs +++ b/server2/src/storage/codec.rs @@ -144,6 +144,35 @@ impl RecordId { self.raw_sha256 )) } + /// The identity whose [`Self::object_key`] is `key`. + pub fn from_object_key(key: &str) -> Result { + let invalid = Error::Invalid("record object key"); + let rest = key.strip_prefix("raw/v2/").ok_or(invalid.clone())?; + let parts: Vec<&str> = rest.split('/').collect(); + let [source, kind, y, m, d, slot, key_sha256, file] = parts[..] else { + return Err(invalid); + }; + let (fetched, raw) = file + .strip_suffix(".raw.gz") + .and_then(|f| f.split_once('-')) + .ok_or(invalid.clone())?; + let number = |t: &str| t.parse::().map_err(|_| invalid.clone()); + let id = Self { + source: source.into(), + kind: kind.into(), + key_sha256: key_sha256.into(), + period: Period { + local_date: number(y)? * 10000 + number(m)? * 100 + number(d)?, + slot: slot.into(), + }, + fetched_at_ms: fetched.parse().map_err(|_| invalid.clone())?, + raw_sha256: raw.into(), + }; + if id.object_key()? != key { + return Err(invalid); + } + Ok(id) + } } #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[serde(deny_unknown_fields)] diff --git a/server2/tests/domestic/observe.rs b/server2/tests/domestic/observe.rs index 2f5cbe666..5f202b5d8 100644 --- a/server2/tests/domestic/observe.rs +++ b/server2/tests/domestic/observe.rs @@ -1202,6 +1202,31 @@ fn one_group(record: &RawRecord) -> GroupDeclaration { }; GroupDeclaration::new(vec![member], &CatalogLimits::default()).unwrap() } +/// Publish one KMA page record as the S13 acquirer does (one group, the day catalog). +pub(super) async fn publish_record( + store: &CatalogStore, + record: RawRecord, +) { + store + .publish(one_group(&record), vec![record], deadline()) + .await + .unwrap(); +} +/// Store the raw body [`store_pages_without`] left out (a repair of its group). +pub(super) async fn repair_raw(transport: &T, record: &RawRecord) +where + T: server2::storage::CatalogTransport + Clone, +{ + let limits = CatalogLimits::default(); + let attached = one_group(record) + .attach(vec![record.clone()], &limits) + .unwrap(); + server2::storage::RawRecordStore::new(transport.clone(), limits.raw.clone()) + .unwrap() + .publish(attached[0].clone()) + .await + .unwrap(); +} /// Store many page groups as their publications leave them (descriptor, raw body, one catalog /// entry each), writing each new day catalog once: the S3 test peer keeps 64 revisions of an /// object, fewer than the publications of a day. diff --git a/server2/tests/domestic/observe_product.rs b/server2/tests/domestic/observe_product.rs index c01ecca6d..1ffae0a53 100644 --- a/server2/tests/domestic/observe_product.rs +++ b/server2/tests/domestic/observe_product.rs @@ -4,7 +4,10 @@ //! products are compared with the catalogs (S3 is truth): a page published later is restored, an //! unusable product is replaced by the raw pages, and S16 completeness is kept. The S06 loopback //! S3 test peer holds the pages; the acquirer never reaches a provider. -use super::observe::{Counted, aws_page, city_page, dotted, page_record, store_pages_without}; +use super::observe::{ + Counted, aws_page, city_page, dotted, page_record, publish_record, repair_raw, + store_pages_without, +}; use super::source_s3::{NOW, Peer, Unreachable, deadline}; use serde_json::Value; use server2::domestic::kma::source::observe::{Folded, Observations}; @@ -41,7 +44,7 @@ fn city_date(h: Publication) -> String { /// Every hour from 2026-09-16 00:00 to 10:00 of 2026-09-24 (the nine day partitions a process at /// 10:50 reads), each with its AWS hourly and city page acquired 15 minutes into the hour, and the /// minute pages of the last 40 minutes. -fn station_records() -> Vec { +pub(super) fn station_records() -> Vec { let mut out = Vec::new(); let mut h = p(20260916, 0); while h <= p(20260924, 1000) { @@ -206,7 +209,7 @@ fn product_catalog(day: u32) -> CatalogId { } } /// The stored products of one day, oldest first, as JSON. -async fn products(peer: &Peer, day: u32) -> Vec { +pub(super) async fn products(peer: &Peer, day: u32) -> Vec { product_texts(peer, day) .await .iter() @@ -406,7 +409,7 @@ async fn unusable_products_are_replaced_by_the_raw_pages() { let unlisted = hour3.replace(".raw.gz", "0.raw.gz"); let variants: Vec<(&str, Vec, bool)> = vec![ ("unreadable", b"not a product".to_vec(), true), - ("older schema", edit("\"schema\":1,", "\"schema\":0,"), true), + ("older schema", edit("\"schema\":2,", "\"schema\":1,"), true), ( "older parse revision", edit("\"parseRevision\":1,", "\"parseRevision\":0,"), @@ -523,7 +526,124 @@ async fn an_excluded_group_keeps_its_day_partial_through_the_product() { assert!(!b.settled(NOW as i64, &folded), "the 09-21 read is partial"); tokio::time::sleep(Duration::from_millis(100)).await; } - assert_eq!(raw_gets(&b_counted, "aws-hourly"), 0); + // Only the excluded group is checked again (its missing raw body), no raw read of the day. + assert_eq!(raw_gets(&b_counted, "aws-hourly"), 1); let last = answer(&b).await; assert_eq!(hours_of_108(&last, p(20260917, 0), p(20260923, 2300)), 168); } + +/// A city page of `h` naming `stations` (`(id, name)`), as `city_page` does for 108 alone. +fn city_page_of(h: Publication, stations: &[(&str, &str)]) -> Vec { + let rows: String = stations + .iter() + .map(|(id, name)| { + format!( + r#"{name}맑음318.060남서2.0"# + ) + }) + .collect(); + format!( + r#"
기상실황표{}
{rows}
이름현재일기운량
1/10		현재
기온		습도
%		풍향풍속
"#, + city_date(h) + ) + .into_bytes() +} + +/// RO-city-growth: a city page published after the products names a station the products +/// reduced to its registry facts (강남, 400). Its AWS rows of every stored page are read in full +/// again, and the answers equal a cold read of the raw pages. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_station_joining_the_city_set_after_the_products_is_read_in_full() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let peer = seeded_peer(&[], Vec::new()).await; + let _ = producing(&peer).await; + let hour = p(20260924, 800); + let later = page_record( + "city-obs", + hour, + schedule::to_ms(hour) as u64 + 50 * 60_000, + city_page_of(hour, &[("108", "서울"), ("400", "강남")]), + ); + let store = CatalogStore::new(peer.transport_limited(4), CatalogLimits::default()).unwrap(); + publish_record(&store, later.clone()).await; + let (c, _) = process(&peer); + let c_folded = settle(&c).await; + let cold_peer = seeded_peer(&[], vec![later]).await; + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + assert!( + d_folded + .hourly + .docs() + .iter() + .any(|r| r.get("stnId") == Some(&Val::Num(400.))), + "a cold read holds the new city station's rows" + ); + same(&c_folded, &d_folded, "cold read"); +} + +/// RO-minute-fallback: when the newest acquisition of a minute is not complete (S06 excludes it), +/// an older complete acquisition of that minute is used, as a latest-first read of every +/// acquisition does. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let minute = p(20260924, 1049); + let lost = page_record( + "aws-minute", + minute, + schedule::to_ms(minute) as u64 + 40_000, + aws_page(&dotted(minute), false), + ); + let missing = lost.envelope.identity.clone(); + let peer = seeded_peer(std::slice::from_ref(&missing), vec![lost]).await; + let (b, _) = process(&peer); + let b_folded = settle(&b).await; + let cold_peer = seeded_peer(&[], Vec::new()).await; + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + same(&b_folded, &d_folded, "the 10:49 minute page"); +} + +/// RO-sticky-excluded: a product made while a group's raw body was missing records it; once the +/// group is repaired, a restart reads the day as complete (as a raw read does) and stores the +/// day's product again without the exclusion. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_repaired_group_completes_its_day_after_the_product() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let hour = p(20260921, 500); + let lost = page_record( + "aws-hourly", + hour, + schedule::to_ms(hour) as u64 + 5 * 60_000, + aws_page(&dotted(hour), false), + ); + let peer = seeded_peer( + std::slice::from_ref(&lost.envelope.identity), + vec![lost.clone()], + ) + .await; + let (a, a_counted) = process(&peer); + until(&a, || { + a_counted.get(&format!("PUT {PRODUCTS}")) >= DAYS.len() + }) + .await; + assert_eq!( + products(&peer, 20260921).await[0]["excluded"], + Value::from(1) + ); + repair_raw(&peer.transport_limited(4), &lost).await; + let (b, b_counted) = process(&peer); + let b_folded = settle(&b).await; + let cold_peer = seeded_peer(&[], vec![lost]).await; + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + same(&b_folded, &d_folded, "cold read"); + until(&b, || b_counted.get(&format!("PUT {PRODUCTS}")) >= 1).await; + let stored = products(&peer, 20260921).await; + assert_eq!(stored.len(), 2); + assert!( + stored.iter().any(|v| v["excluded"] == 0), + "the day's product is stored again without the exclusion" + ); +} diff --git a/server2/tests/domestic/summary_s3.rs b/server2/tests/domestic/summary_s3.rs index b8a4c1189..8916c8f23 100644 --- a/server2/tests/domestic/summary_s3.rs +++ b/server2/tests/domestic/summary_s3.rs @@ -1041,3 +1041,65 @@ async fn older_native_routes_record_only_compatible_daily_rows() { } } } + +/// RO-sticky-excluded end to end: a station page group whose raw body was missing when the day +/// products were made blocks every summary; once it is repaired, a new process records the +/// complete grid day as a process reading the raw pages would. +#[tokio::test(flavor = "multi_thread", worker_threads = 2)] +async fn a_repaired_station_group_lets_a_restart_record_summaries() { + use super::observe::{aws_page, dotted, page_record, repair_raw, store_pages_without}; + let _peers = S3_PEERS.lock().await; + let peer = Peer::persistent(); + let catalogs = CatalogStore::new(peer.transport(), CatalogLimits::default()).unwrap(); + seed(&catalogs, &[]).await; + day(&catalogs, 20260923, 1..=24).await; + // 09-21 05:00: a complete AWS hourly page and an earlier acquisition without its raw body. + let hour = p(20260921, 500); + let at = server2::history::schedule::to_ms(hour) as u64; + let kept = page_record( + "aws-hourly", + hour, + at + 15 * 60_000, + aws_page(&dotted(hour), false), + ); + let lost = page_record( + "aws-hourly", + hour, + at + 5 * 60_000, + aws_page(&dotted(hour), true), + ); + store_pages_without( + &peer.transport_limited(4), + vec![kept, lost.clone()], + std::slice::from_ref(&lost.envelope.identity), + ) + .await; + let (backend, recorder) = backend(&peer, Arc::new(Unreachable)); + let mut made = false; + for _ in 0..40 { + serve(&backend).await; + assert!(recorder.idle(deadline()).await); + if !super::observe_product::products(&peer, 20260921) + .await + .is_empty() + { + made = true; + break; + } + tokio::time::sleep(Duration::from_millis(250)).await; + } + assert!(made, "the 09-21 product is stored"); + assert_eq!( + names(&recorder, 20260923).await, + 0, + "the 09-21 read is partial" + ); + repair_raw(&peer.transport_limited(4), &lost).await; + let (backend, recorder) = backend_again(&peer); + serve_until(&backend, &recorder, 20260923).await; + assert_eq!( + names(&recorder, 20260923).await, + 1, + "recorded after the repair" + ); +} diff --git a/specs/issue-2614-kma-restart-observe.md b/specs/issue-2614-kma-restart-observe.md index 531e08bc1..7287613b9 100644 --- a/specs/issue-2614-kma-restart-observe.md +++ b/specs/issue-2614-kma-restart-observe.md @@ -4,7 +4,8 @@ - **PRODUCT:** a product keeps every stored value and key order, each distinct registry facts row once, and refuses another day, schema or parse revision. A folding process stores one product per day whose hours are all in the window and stores none again while nothing changed. Checked by `every_value_and_key_order_round_trips` (unit test in `observe_product.rs`) and `a_restart_answers_from_the_day_products`. - **RESTART:** a restarted process with no provider answers the 168 past hourly pages and minute rows on its first request, within the bound. It reads no raw page that the products cover, stores no product and answers as the producing process. A page published after the product is restored by itself and the answers equal a cold read of the raw pages. Of two processes' products the newest is read. Checked by `a_restart_answers_from_the_day_products`, `a_page_published_after_the_product_is_restored` and `the_newest_of_two_processes_products_is_read` (`tests/domestic/observe_product.rs`). - **FALLBACK:** an unreadable product, an older schema, an older parse revision and a grown city set each give that day's raw pages; a page the catalog does not list is dropped and its hour restored. Each answers as the raw pages do and stores the day's product again. Checked by `unusable_products_are_replaced_by_the_raw_pages`. -- **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1) without reading its raw pages. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. -- **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json`, and the artifact policy passes on outgoing commits. +- **ACCOUNTED** (review RO-city-growth, RO-minute-fallback, RO-sticky-excluded): a city page published after the products that names another station makes the restart read the reduced AWS pages in full again, and its answers equal a cold raw read. A minute whose newest acquisition S06 excludes is read from the next acquisition of that minute, as a cold read gives it. A group excluded when the product was made and repaired later completes its day on a restart: the answers equal a cold read, they settle, the day's product is stored again without the exclusion, and a new process records the daily summaries. Checked by `a_station_joining_the_city_set_after_the_products_is_read_in_full`, `an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one`, `a_repaired_group_completes_its_day_after_the_product` (`tests/domestic/observe_product.rs`) and `a_repaired_station_group_lets_a_restart_record_summaries` (`tests/domestic/summary_s3.rs`). +- **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1); a restart checks only that group again and reads no other raw page of the day. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. +- **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json` (base `origin/feat/2614-kma-restart-reads`), and the artifact policy passes on outgoing commits. Failure paths: a day whose catalogs cannot be listed or whose raw read fails stays failed (S16) and stores no product. A product that cannot be stored is tried again after ten minutes. A newer page that is not complete or does not parse gives the day's raw read. From 9c16049169ac5180dec5792073f39f708ce52db5 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 13:51:23 +0900 Subject: [PATCH 05/11] server2: keep reduced station pages unsettled until read in full Second review of #2872 (RO-city-growth, RO-expand-progress, RO-record-key-overflow). A held AWS page reduced for a smaller city set than the last fold's now keeps settled() false, so no summary is recorded from its missing rows. The full read of such pages is started after every fold and request (not only after an hourly recompute), reads four pages per catalog read and keeps each read as it completes; only started pages wait for the retry, now one minute as for a failed ASOS S3 read, and pages not reached before the 30 s limit stay due for the next run. RecordId::from_object_key requires the key's four, two and two date digits before computing the date, so out-of-range numbers are refused instead of overflowing. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/server2.md | 14 +- server2/config/tasks/KMA-RESTART-OBSERVE.json | 3 +- server2/src/domestic/kma/source/observe.rs | 176 +++++++++++------- server2/src/storage/codec.rs | 10 +- server2/tests/domestic/observe.rs | 39 +++- server2/tests/domestic/observe_product.rs | 81 ++++++++ server2/tests/storage/codec.rs | 21 +++ specs/issue-2614-kma-restart-observe.md | 4 +- 8 files changed, 265 insertions(+), 83 deletions(-) diff --git a/docs/architecture/server2.md b/docs/architecture/server2.md index 0f10d497d..94e911f80 100644 --- a/docs/architecture/server2.md +++ b/docs/architecture/server2.md @@ -2317,8 +2317,11 @@ older schema or parse revision, a grown city set, no product, or a new newest pa parse gives the day's raw read as before, which records every complete group as known and every excluded one. When a fold's city set has grown past the set an AWS page was reduced for (a city page restored after the products, or acquired since, names another station), `Flight::Expand` -reads those pages in full again by their raw identity (no request waits for it; a failed read is -tried again after ten minutes), and the next fold reduces them for the new set. `Flight::Load` ends after the products, today's +reads those pages in full again by their raw identity, four per catalog read, and keeps each read +as it completes; no request waits for it. A page whose read was started waits a minute before it +is tried again; pages not reached before the read's 30 s limit, and due retries, are started by the +next fold or request without waiting for a new hourly page. The next fold reduces the pages read +in full for the new set. `Flight::Load` ends after the products, today's and yesterday's comparisons, the seed and the four newest minute pages. `Flight::LoadRest`, which no request waits for, then compares the other days (each marked read and the pages dirty) and restores the rest of the minute window. Reloads every five minutes compare with the pages held. @@ -2331,7 +2334,8 @@ repaired group completes the day and its product is stored again without it. A g after a product, older than its pages, is unaccounted for and checked like any new group, so a restart from products counts a day complete exactly when a raw read would, given that a complete raw object is never deleted. `settled()` counts `LoadRest`, `Product`, `Expand` and a due product -comparison as busy. +comparison as busy, and is false while a held AWS page is reduced for a city set that does not +cover the last fold's (its past rows of a city station would be missing). Right after the first deployment the window's oldest day has no product (some of its hours were already out of the window) and is read from its raw pages after the first answer. @@ -2341,8 +2345,8 @@ Verification (local, S06 loopback S3 peer, no provider): targeted member GET cou restored alone and equal to a cold raw read, unusable products replaced by the raw pages and stored again, two processes' products, unsettled answers during the rest of the read, a day with an S06-excluded group kept partial, a repaired group completing its day (and recording summaries), a -station joining the city set after the products, an incomplete newest minute acquisition, and the -disk cache exclusion. +station joining the city set after the products (also with failing and with slow full reads), an +incomplete newest minute acquisition, and the disk cache exclusion. ### S15b-2 s station registry seed (2026-10-15) diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 354cff35e..3589fd122 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -17,7 +17,8 @@ "server2/tests/domestic/observe.rs", "server2/tests/domestic/observe_product.rs", "server2/tests/domestic/summary_s3.rs", - "server2/tests/storage/catalog_wire.rs" + "server2/tests/storage/catalog_wire.rs", + "server2/tests/storage/codec.rs" ], "outside": [ { diff --git a/server2/src/domestic/kma/source/observe.rs b/server2/src/domestic/kma/source/observe.rs index 8d9d79b65..6c47e081b 100644 --- a/server2/src/domestic/kma/source/observe.rs +++ b/server2/src/domestic/kma/source/observe.rs @@ -84,8 +84,9 @@ const MINUTE_KEEP_MS: f64 = 40. * 60_000.; const MINUTE_FIRST: usize = 4; /// A day's product is stored at most once per this interval by one process. const PRODUCT_EVERY: Duration = Duration::from_secs(600); -/// A reduced AWS hourly page whose full read failed is read again after this long. -const EXPAND_RETRY: Duration = Duration::from_secs(600); +/// A reduced AWS hourly page whose full read was tried is read again after this long (an S3 +/// read, retried as a failed ASOS S3 read is). +const EXPAND_RETRY: Duration = Duration::from_secs(60); /// Acquisitions of one minute tried, newest first, when a newer one is not complete or does not /// parse. const MINUTE_CANDIDATES: usize = 3; @@ -496,10 +497,11 @@ impl Observations { if let Some(rx) = self.fold_if_dirty(now_ms) { Self::wait(vec![rx], until).await; } - // A product comparison a finishing one missed. + // A product comparison a finishing one missed, and a due full read of reduced pages. if self.state().product_due && self.flight(&Flight::Product).is_none() { self.want_products(now_ms); } + self.expand_due(); // Before the first fold the carried seed alone is the registry (no pages yet). Some(self.state().folded.clone().unwrap_or_else(seed_only)) } @@ -1391,6 +1393,7 @@ impl Observations { s.pages.city.retain(|h, _| *h >= first); s.pages.ids.retain(|(_, h), _| *h >= first); s.pages.reduced.retain(|h, _| *h >= first); + s.expand_attempts.retain(|h, _| *h >= first); // Minute pages older than the readers' reach, however they arrived (S3 or local). let floor = schedule::from_ms(now_ms - MINUTE_KEEP_MS as i64); s.pages.minute.retain(|k, _| *k > floor); @@ -1451,27 +1454,8 @@ impl Observations { } } s.hourly_fold = Some(fold.clone()); - // Pages reduced for a smaller city set lack a city station's rows (a station - // joined the set since): they are read in full again. - let stale: Vec<(Publication, String)> = s - .pages - .reduced - .iter() - .filter(|(_, set)| !set.is_superset(&fold.cities)) - .filter_map(|(h, _)| { - let (_, _, raw_id) = s.pages.ids.get(&("aws-hourly", *h))?; - Some((*h, raw_id.clone())) - }) - .filter(|(h, _)| { - s.expand_attempts - .get(h) - .is_none_or(|at| at.elapsed() >= EXPAND_RETRY) - }) - .collect(); drop(s); - if !stale.is_empty() { - self.expand(stale); - } + self.expand_due(); self.want_products(now_ms); fold } @@ -1561,56 +1545,99 @@ impl Observations { // ---- reduced pages ---- - /// Read the AWS hourly `pages` (hour and raw identity) in full again and hold their whole - /// rows while still the held pages; no request waits for it. - fn expand(self: &Arc, pages: Vec<(Publication, String)>) { + /// Pages reduced for a smaller city set than the last fold's lack a city station's rows (a + /// station joined the set since): read the due ones in full again, unless a read runs. Called + /// after every fold and request, so a retry does not wait for a new hourly page. + fn expand_due(self: &Arc) { + if self.flight(&Flight::Expand).is_some() { + return; + } + let due: Vec<(Publication, String)> = { + let s = self.state(); + stale_pages(&s) + .into_iter() + .filter(|(h, _)| { + s.expand_attempts + .get(h) + .is_none_or(|at| at.elapsed() >= EXPAND_RETRY) + }) + .collect() + }; + if due.is_empty() { + return; + } let _ = self.start(Flight::Expand, Duration::ZERO, move |this| { - Box::pin(async move { this.read_full(pages).await }) + Box::pin(async move { this.read_full(due).await }) }); } - async fn read_full(&self, pages: Vec<(Publication, String)>) { + /// Read the AWS hourly `pages` (hour and raw identity) in full again, a few at a time, and hold + /// the whole rows of each read while still the held page. A read is kept as it completes; the + /// pages not started before the deadline stay due (only started pages wait for + /// [`EXPAND_RETRY`]). + async fn read_full(&self, mut pages: Vec<(Publication, String)>) { let deadline = Instant::now() + READ_DEADLINE; - let mut wanted = Vec::new(); - { - let mut s = self.state(); - for (hour, raw_id) in &pages { - s.expand_attempts.insert(*hour, std::time::Instant::now()); - let (Ok(member), Ok(catalog)) = ( - crate::storage::RecordId::from_object_key(raw_id), - super::records::catalog("kma-web", "aws-hourly", &aws_key(), &hour.base_date()), - ) else { - continue; - }; - wanted.push((catalog, member)); + pages.sort(); + let mut batches: Vec> = Vec::new(); + for (hour, raw_id) in pages { + let (Ok(member), Ok(catalog)) = ( + crate::storage::RecordId::from_object_key(&raw_id), + super::records::catalog("kma-web", "aws-hourly", &aws_key(), &hour.base_date()), + ) else { + // Not a readable identity: tried again after the retry interval. + self.state() + .expand_attempts + .insert(hour, std::time::Instant::now()); + continue; + }; + match batches.last_mut() { + Some(batch) if batch.len() < MINUTE_FIRST && batch[0].1 == catalog => { + batch.push((hour, catalog, member)) + } + _ => batches.push(vec![(hour, catalog, member)]), } } - let Ok(read) = self.restore(&wanted, deadline).await else { - return; - }; - let parsed = crate::observability::spawn_blocking(move || { - read.iter() - .filter_map(|a| a.as_deref().and_then(parse_hourly)) - .collect::>() - }) - .await - .unwrap_or_default(); - let mut s = self.state(); - let mut changed = false; - for (p, rows, _, _, raw_id) in parsed { - let same = s - .pages - .ids - .get(&("aws-hourly", p)) - .is_some_and(|(_, _, held)| *held == raw_id); - if same { - s.pages.hourly.insert(p, rows); - s.pages.reduced.remove(&p); - changed = true; + for batch in batches { + if Instant::now() >= deadline { + break; + } + { + let mut s = self.state(); + for (hour, _, _) in &batch { + s.expand_attempts.insert(*hour, std::time::Instant::now()); + } + } + let wanted: Vec<_> = batch + .into_iter() + .map(|(_, catalog, member)| (catalog, member)) + .collect(); + let Ok(read) = self.restore(&wanted, deadline).await else { + continue; + }; + let parsed = crate::observability::spawn_blocking(move || { + read.iter() + .filter_map(|a| a.as_deref().and_then(parse_hourly)) + .collect::>() + }) + .await + .unwrap_or_default(); + let mut s = self.state(); + let mut changed = false; + for (p, rows, _, _, raw_id) in parsed { + let same = s + .pages + .ids + .get(&("aws-hourly", p)) + .is_some_and(|(_, _, held)| *held == raw_id); + if same { + s.pages.hourly.insert(p, rows); + s.pages.reduced.remove(&p); + changed = true; + } + } + if changed { + s.pages.dirty = true; + s.pages.hourly_dirty = true; } - } - if changed { - s.pages.dirty = true; - s.pages.hourly_dirty = true; } } @@ -2079,6 +2106,7 @@ impl Observations { // the snapshot did not use. if s.pages.dirty || s.product_due + || !stale_pages(&s).is_empty() || s.seed_failed || s.pages.minute_read.is_none() || !s.folded.as_ref().is_some_and(|f| Arc::ptr_eq(f, folded)) @@ -2201,6 +2229,22 @@ pub(super) fn facts(row: &Doc) -> Doc { } out } +/// The held AWS hourly pages (hour and raw identity) reduced for a city set that does not cover +/// the last fold's. +fn stale_pages(s: &State) -> Vec<(Publication, String)> { + let Some(fold) = &s.hourly_fold else { + return Vec::new(); + }; + s.pages + .reduced + .iter() + .filter(|(_, set)| !set.is_superset(&fold.cities)) + .filter_map(|(h, _)| { + let (_, _, raw_id) = s.pages.ids.get(&("aws-hourly", *h))?; + Some((*h, raw_id.clone())) + }) + .collect() +} /// The day products to store at `now_ms` (each attempt is recorded): per day whose every hour /// is in the window, that was read without failure and holds pages, the held pages unless the /// last product this process read or stored for it is made of the same, at most once per diff --git a/server2/src/storage/codec.rs b/server2/src/storage/codec.rs index 092b12e02..8fa8a3cc0 100644 --- a/server2/src/storage/codec.rs +++ b/server2/src/storage/codec.rs @@ -156,13 +156,19 @@ impl RecordId { .strip_suffix(".raw.gz") .and_then(|f| f.split_once('-')) .ok_or(invalid.clone())?; - let number = |t: &str| t.parse::().map_err(|_| invalid.clone()); + // Four, two and two digits (the key's own widths), so the date cannot overflow. + let number = |t: &str, width: usize| { + (t.len() == width && t.bytes().all(|b| b.is_ascii_digit())) + .then(|| t.parse::().ok()) + .flatten() + .ok_or(invalid.clone()) + }; let id = Self { source: source.into(), kind: kind.into(), key_sha256: key_sha256.into(), period: Period { - local_date: number(y)? * 10000 + number(m)? * 100 + number(d)?, + local_date: number(y, 4)? * 10000 + number(m, 2)? * 100 + number(d, 2)?, slot: slot.into(), }, fetched_at_ms: fetched.parse().map_err(|_| invalid.clone())?, diff --git a/server2/tests/domestic/observe.rs b/server2/tests/domestic/observe.rs index 5f202b5d8..9685ba6dc 100644 --- a/server2/tests/domestic/observe.rs +++ b/server2/tests/domestic/observe.rs @@ -1049,12 +1049,16 @@ async fn same_hour_aws_and_city_restore_respects_fetched_order_and_field_presenc /// Counts S3 requests at the transport boundary by method and object class (`raw/v2/source/kind`, /// `index/v2/source/kind`, `index/v2/groups` descriptors, `list index/v2/source/kind`); every /// request passes through to the S06 loopback peer. +/// Held GETs: the key prefix, how many pass first, the hold and how many were seen. +type Delay = (String, usize, Duration, usize); #[derive(Clone)] pub(super) struct Counted { inner: server2::world::BoundedS3, counts: Arc>>, - /// GETs of one class after the first `n` are held this long. - delay: Arc>>, + /// GETs of keys under a prefix after the first `n` (counted here) are held this long. + delay: Arc>>, + /// GETs of keys under this prefix fail. + fail: Arc>>, } impl Counted { pub(super) fn new(inner: server2::world::BoundedS3) -> Self { @@ -1062,16 +1066,32 @@ impl Counted { inner, counts: Arc::default(), delay: Arc::default(), + fail: Arc::default(), } } - /// Hold every GET of `class` after its first `after` for `hold`. - pub(super) fn delay_gets(&self, class: &str, after: usize, hold: Duration) { - *self.delay.lock().unwrap() = Some((class.to_owned(), after, hold)); + /// Hold every GET of a key under `prefix` after its first `after` for `hold`. + pub(super) fn delay_gets(&self, prefix: &str, after: usize, hold: Duration) { + *self.delay.lock().unwrap() = Some((prefix.to_owned(), after, hold, 0)); + } + /// Fail every GET of a key under `prefix` (None: none). + pub(super) fn fail_gets(&self, prefix: Option<&str>) { + *self.fail.lock().unwrap() = prefix.map(str::to_owned); } fn held(&self, key: &str) -> Option { - let delay = self.delay.lock().unwrap().clone()?; - let class = Self::class(key); - (class == delay.0 && self.get(&format!("GET {class}")) > delay.1).then_some(delay.2) + let mut delay = self.delay.lock().unwrap(); + let (prefix, after, hold, seen) = delay.as_mut()?; + if !key.starts_with(prefix.as_str()) { + return None; + } + *seen += 1; + (*seen > *after).then_some(*hold) + } + fn failed(&self, key: &str) -> bool { + self.fail + .lock() + .unwrap() + .as_deref() + .is_some_and(|prefix| key.starts_with(prefix)) } fn class(key: &str) -> String { if key.starts_with("index/v2/groups/") { @@ -1116,6 +1136,9 @@ impl server2::storage::ObjectTransport for Counted { if let Some(hold) = self.held(key) { tokio::time::sleep(hold).await; } + if self.failed(key) { + return Err(server2::storage::Error::Transport); + } self.inner.get(key, max).await } } diff --git a/server2/tests/domestic/observe_product.rs b/server2/tests/domestic/observe_product.rs index 1ffae0a53..00ebc8d2b 100644 --- a/server2/tests/domestic/observe_product.rs +++ b/server2/tests/domestic/observe_product.rs @@ -647,3 +647,84 @@ async fn a_repaired_group_completes_its_day_after_the_product() { "the day's product is stored again without the exclusion" ); } + +/// Products, then a later city page naming 강남 (400), whose AWS rows the products keep reduced: +/// the peer and the cold peer holding the same raw pages without products. +async fn grown_city_set() -> (Peer, Peer) { + let peer = seeded_peer(&[], Vec::new()).await; + let _ = producing(&peer).await; + let hour = p(20260924, 800); + let later = page_record( + "city-obs", + hour, + schedule::to_ms(hour) as u64 + 50 * 60_000, + city_page_of(hour, &[("108", "서울"), ("400", "강남")]), + ); + let store = CatalogStore::new(peer.transport_limited(4), CatalogLimits::default()).unwrap(); + publish_record(&store, later.clone()).await; + (peer, seeded_peer(&[], vec![later]).await) +} + +/// RO-city-growth: while pages reduced for a smaller city set cannot be read in full again (here +/// the 09-17 pages fail), answers stay unsettled; the next attempt follows a later request without +/// any new hourly page, and the answers then equal a cold read. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_failed_full_read_keeps_answers_unsettled_until_it_is_tried_again() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let (peer, cold_peer) = grown_city_set().await; + let (c, c_counted) = process(&peer); + const FAILING: &str = "raw/v2/kma-web/aws-hourly/2026/09/17"; + c_counted.fail_gets(Some(FAILING)); + let attempts = || c_counted.get(&format!("GET {FAILING}")); + until(&c, || attempts() > 0).await; + for _ in 0..20 { + let folded = answer(&c).await; + assert!( + !c.settled(NOW as i64, &folded), + "the 09-17 pages are still reduced" + ); + tokio::time::sleep(Duration::from_millis(250)).await; + } + c_counted.fail_gets(None); + let tried = attempts(); + let mut folded = answer(&c).await; + for _ in 0..240 { + if c.settled(NOW as i64, &folded) { + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + folded = answer(&c).await; + } + assert!(attempts() > tried, "read again without a new hourly page"); + assert!(c.settled(NOW as i64, &folded), "settled after the retry"); + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + same(&folded, &d_folded, "cold read"); +} + +/// RO-expand-progress: reading the reduced pages in full again takes longer than one read's +/// limit (each AWS page GET of 09-20..24 takes 400 ms); what was read is kept and the rest follows, +/// so the answers settle and equal a cold read. +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_slow_full_read_progresses_over_several_runs() { + let _peers = super::source_s3::S3_PEERS.lock().await; + let (peer, cold_peer) = grown_city_set().await; + let (c, c_counted) = process(&peer); + c_counted.delay_gets( + "raw/v2/kma-web/aws-hourly/2026/09/2", + 0, + Duration::from_millis(400), + ); + let mut folded = answer(&c).await; + for _ in 0..300 { + if c.settled(NOW as i64, &folded) { + break; + } + tokio::time::sleep(Duration::from_millis(500)).await; + folded = answer(&c).await; + } + assert!(c.settled(NOW as i64, &folded), "the full read completed"); + let (d, _) = process(&cold_peer); + let d_folded = settle(&d).await; + same(&folded, &d_folded, "cold read"); +} diff --git a/server2/tests/storage/codec.rs b/server2/tests/storage/codec.rs index 13370dce8..01a8d4959 100644 --- a/server2/tests/storage/codec.rs +++ b/server2/tests/storage/codec.rs @@ -230,3 +230,24 @@ fn validated_group_builder_requires_s06_descriptor_for_completeness() { invalid.members = 0; assert!(r.with_fetch_group(invalid, &limits).is_err()); } +/// A raw key with out-of-range numbers is not a record identity (no arithmetic overflow). +#[test] +fn absurd_record_keys_are_refused() { + let hash = "a".repeat(64); + for key in [ + format!("raw/v2/kma-web/aws-hourly/500000/01/01/0000/{hash}/1-{hash}.raw.gz"), + format!("raw/v2/kma-web/aws-hourly/2026/4294967295/01/0000/{hash}/1-{hash}.raw.gz"), + format!("raw/v2/kma-web/aws-hourly/2026/09/99999/0000/{hash}/1-{hash}.raw.gz"), + format!( + "raw/v2/kma-web/aws-hourly/2026/09/24/0000/{hash}/99999999999999999999-{hash}.raw.gz" + ), + ] { + assert!( + server2::storage::RecordId::from_object_key(&key).is_err(), + "{key}" + ); + } + let good = format!("raw/v2/kma-web/aws-hourly/2026/09/24/1000/{hash}/1-{hash}.raw.gz"); + let id = server2::storage::RecordId::from_object_key(&good).unwrap(); + assert_eq!(id.object_key().unwrap(), good); +} diff --git a/specs/issue-2614-kma-restart-observe.md b/specs/issue-2614-kma-restart-observe.md index 7287613b9..d58085d4e 100644 --- a/specs/issue-2614-kma-restart-observe.md +++ b/specs/issue-2614-kma-restart-observe.md @@ -4,8 +4,10 @@ - **PRODUCT:** a product keeps every stored value and key order, each distinct registry facts row once, and refuses another day, schema or parse revision. A folding process stores one product per day whose hours are all in the window and stores none again while nothing changed. Checked by `every_value_and_key_order_round_trips` (unit test in `observe_product.rs`) and `a_restart_answers_from_the_day_products`. - **RESTART:** a restarted process with no provider answers the 168 past hourly pages and minute rows on its first request, within the bound. It reads no raw page that the products cover, stores no product and answers as the producing process. A page published after the product is restored by itself and the answers equal a cold read of the raw pages. Of two processes' products the newest is read. Checked by `a_restart_answers_from_the_day_products`, `a_page_published_after_the_product_is_restored` and `the_newest_of_two_processes_products_is_read` (`tests/domestic/observe_product.rs`). - **FALLBACK:** an unreadable product, an older schema, an older parse revision and a grown city set each give that day's raw pages; a page the catalog does not list is dropped and its hour restored. Each answers as the raw pages do and stores the day's product again. Checked by `unusable_products_are_replaced_by_the_raw_pages`. -- **ACCOUNTED** (review RO-city-growth, RO-minute-fallback, RO-sticky-excluded): a city page published after the products that names another station makes the restart read the reduced AWS pages in full again, and its answers equal a cold raw read. A minute whose newest acquisition S06 excludes is read from the next acquisition of that minute, as a cold read gives it. A group excluded when the product was made and repaired later completes its day on a restart: the answers equal a cold read, they settle, the day's product is stored again without the exclusion, and a new process records the daily summaries. Checked by `a_station_joining_the_city_set_after_the_products_is_read_in_full`, `an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one`, `a_repaired_group_completes_its_day_after_the_product` (`tests/domestic/observe_product.rs`) and `a_repaired_station_group_lets_a_restart_record_summaries` (`tests/domestic/summary_s3.rs`). +- **ACCOUNTED** (review RO-city-growth, RO-minute-fallback, RO-sticky-excluded): a city page published after the products that names another station makes the restart read the reduced AWS pages in full again, and its answers equal a cold raw read. A minute whose newest acquisition S06 excludes is read from the next acquisition of that minute, as a cold read gives it. A group excluded when the product was made and repaired later completes its day on a restart: the answers equal a cold read, they settle, the day's product is stored again without the exclusion, and a new process records the daily summaries. While those pages cannot be read in full the answers stay unsettled, and a later request tries again without a new hourly page; a read slower than one run's limit keeps what it read and finishes over later runs. Checked by `a_station_joining_the_city_set_after_the_products_is_read_in_full`, `a_failed_full_read_keeps_answers_unsettled_until_it_is_tried_again`, `a_slow_full_read_progresses_over_several_runs`, `an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one`, `a_repaired_group_completes_its_day_after_the_product` (`tests/domestic/observe_product.rs`) and `a_repaired_station_group_lets_a_restart_record_summaries` (`tests/domestic/summary_s3.rs`). - **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1); a restart checks only that group again and reads no other raw page of the day. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. - **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json` (base `origin/feat/2614-kma-restart-reads`), and the artifact policy passes on outgoing commits. +A raw key with out-of-range numbers is not a record identity (`absurd_record_keys_are_refused`, `tests/storage/codec.rs`). + Failure paths: a day whose catalogs cannot be listed or whose raw read fails stays failed (S16) and stores no product. A product that cannot be stored is tried again after ten minutes. A newer page that is not complete or does not parse gives the day's raw read. From b34c00180dd086cf5f263b1306a4137ecbb68310 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 22:47:06 +0900 Subject: [PATCH 06/11] server2: order the producers of the_newest_of_two_processes_products_is_read observe_product::the_newest_of_two_processes_products_is_read failed 9 times in 20 at 0fa360e6 with one product for a day. Both producing processes stored every day's product (8 PUTs each), but two processes started together run in step and make each day's product within 0 to 3 ms of each other. The product's instant (made, product_candidates) is its record's fetched time and is in its bytes, so two made in the same millisecond are one record identity and S06 keeps one catalog entry. That is right for S3; the test assumed two instants. The second process now runs on a test transport that holds its AWS hourly LISTs: it lists the (absent) products, waits there while the first process stores its products, then makes its own, so each day holds two products of different instants and the third process reads the newer. 20 consecutive passes, and 10 with the CPU saturated. Co-Authored-By: Claude Opus 5.5 --- server2/tests/domestic/observe_product.rs | 140 +++++++++++++++++++++- 1 file changed, 135 insertions(+), 5 deletions(-) diff --git a/server2/tests/domestic/observe_product.rs b/server2/tests/domestic/observe_product.rs index 00ebc8d2b..8a48ee657 100644 --- a/server2/tests/domestic/observe_product.rs +++ b/server2/tests/domestic/observe_product.rs @@ -86,13 +86,19 @@ fn process(peer: &Peer) -> (Arc, Counted) { )); (obs, counted) } -async fn answer(obs: &Arc) -> Arc { +async fn answer(obs: &Arc>) -> Arc +where + T: server2::history::IndexListing + Clone + Send + Sync + 'static, +{ obs.stations(NOW as i64, Instant::now() + Duration::from_millis(2500)) .await .unwrap() } /// A later answer whose inputs were read completely (S16), or a panic. -async fn settle(obs: &Arc) -> Arc { +async fn settle(obs: &Arc>) -> Arc +where + T: server2::history::IndexListing + Clone + Send + Sync + 'static, +{ for _ in 0..120 { let folded = answer(obs).await; if obs.settled(NOW as i64, &folded) { @@ -443,16 +449,140 @@ async fn unusable_products_are_replaced_by_the_raw_pages() { } } +/// A process's transport that holds its LISTs under `held` until the test opens them and records +/// the prefixes of the LISTs answered, so a test can order two processes' restores. +#[derive(Clone)] +struct Listing { + inner: Counted, + held: String, + open: Arc>, + answered: Arc>>, +} +impl Listing { + /// Wait until a LIST under `prefix` was answered. + async fn answered(&self, prefix: &str) { + let _ = self + .answered + .subscribe() + .wait_for(|a| a.iter().any(|p| p.starts_with(prefix))) + .await; + } + fn open(&self) { + self.open.send_replace(true); + } +} +impl server2::storage::ObjectTransport for Listing { + async fn put( + &self, + record: &server2::storage::PreparedRecord, + ) -> Result { + self.inner.put(record).await + } + async fn head(&self, key: &str) -> Result { + self.inner.head(key).await + } + async fn get( + &self, + key: &str, + max: usize, + ) -> Result { + server2::storage::ObjectTransport::get(&self.inner, key, max).await + } +} +impl server2::storage::CatalogTransport for Listing { + async fn get_control( + &self, + key: &str, + max: usize, + ) -> Result { + self.inner.get_control(key, max).await + } + async fn put_control( + &self, + key: &str, + body: &[u8], + condition: &server2::storage::WriteCondition, + ) -> Result { + self.inner.put_control(key, body, condition).await + } + async fn list_raw_document( + &self, + prefix: &str, + token: Option<&str>, + keys: usize, + bytes: usize, + ) -> Result, server2::storage::Error> { + self.inner + .list_raw_document(prefix, token, keys, bytes) + .await + } +} +impl server2::history::IndexListing for Listing { + async fn list_index( + &self, + prefix: &str, + token: Option<&str>, + ) -> Result { + if prefix.starts_with(self.held.as_str()) { + let _ = self.open.subscribe().wait_for(|open| *open).await; + } + let page = self.inner.list_index(prefix, token).await; + self.answered.send_modify(|a| a.push(prefix.to_owned())); + page + } +} +/// A process on a [`Listing`] transport that holds the LISTs under `held`. +fn listing_process(peer: &Peer, held: &str) -> (Arc>, Listing) { + let listing = Listing { + inner: Counted::new(peer.transport_limited(4)), + held: held.to_owned(), + open: Arc::new(tokio::sync::watch::channel(false).0), + answered: Arc::new(tokio::sync::watch::channel(Vec::new()).0), + }; + let obs = Arc::new(Observations::new( + listing.clone(), + Arc::new(CatalogStore::new(listing.clone(), CatalogLimits::default()).unwrap()), + Arc::new(Unreachable), + Arc::new(Unreachable), + )); + (obs, listing) +} + /// Two processes store their products; a third reads the newest of each day, stores none and -/// answers as they do. +/// answers as they do. Two processes that run in step make each day's product within a +/// millisecond or two, and one made in the same millisecond is the same record (instant and +/// bytes), which S06 stores once: so the second process lists the products (none yet), waits at +/// its page listing while the first stores its products, and then makes its own. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn the_newest_of_two_processes_products_is_read() { let _peers = super::source_s3::S3_PEERS.lock().await; let peer = seeded_peer(&[], Vec::new()).await; - let ((_, _, one), (_, _, two)) = tokio::join!(producing(&peer), producing(&peer)); + let (b, b_listing) = listing_process(&peer, "index/v2/kma-web/aws-hourly/"); + let first = tokio::spawn({ + let b = b.clone(); + async move { answer(&b).await } + }); + b_listing.answered("index/v2/kma-web/station-pages/").await; + let (_, _, one) = producing(&peer).await; + for day in DAYS { + assert_eq!( + products(&peer, day).await.len(), + 1, + "the first process's {day}" + ); + } + b_listing.open(); + first.await.unwrap(); + let two = settle(&b).await; same(&one, &two, "two processes"); for day in DAYS { - assert_eq!(products(&peer, day).await.len(), 2, "{day}"); + let made: Vec = products(&peer, day) + .await + .iter() + .map(|p| p["made"].as_f64().unwrap()) + .collect(); + assert_eq!(made.len(), 2, "{day}"); + assert!(made[0] < made[1], "{day}: {made:?}"); } let (f, f_counted) = process(&peer); let folded = settle(&f).await; From 52a270c8e21c1f65b3525c8334dbc3a8d5d5c4b3 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 23:00:54 +0900 Subject: [PATCH 07/11] server2: make the province test independent of the zone list's arrival The test double answered the forecast-zone page with Seoul only. When that list was stored before the request's zone refresh, towns outside Seoul had no city zone and their mid temperature was not collected, so a_province_under_its_new_name_finds_its_town_and_mid_forecast failed in about 60% of runs on this branch (the day product reads at the start of a request make the race more likely) and occasionally on master. The double now answers every packaged land zone, as the provider's list does, and the test runs each town with and without the list stored first. Same test change as 0c76e778 on fix/2614-kma-test-clock, declared here so this PR's CI keeps one task declaration. Co-Authored-By: Claude Opus 5.5 --- server2/config/tasks/KMA-RESTART-OBSERVE.json | 1 + server2/tests/domestic/full_response.rs | 59 +++++++++++++++++-- 2 files changed, 56 insertions(+), 4 deletions(-) diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 3589fd122..7014771fd 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -14,6 +14,7 @@ "server2/src/storage/codec.rs", "server2/tests/cache.rs", "server2/tests/domestic.rs", + "server2/tests/domestic/full_response.rs", "server2/tests/domestic/observe.rs", "server2/tests/domestic/observe_product.rs", "server2/tests/domestic/summary_s3.rs", diff --git a/server2/tests/domestic/full_response.rs b/server2/tests/domestic/full_response.rs index 3949aef61..d108655ea 100644 --- a/server2/tests/domestic/full_response.rs +++ b/server2/tests/domestic/full_response.rs @@ -235,7 +235,7 @@ impl ForecastAcquire for Forecasts { }, p, p.base_date(), - zone_page(), + land_zone_page(), "application/json", ) } @@ -259,6 +259,39 @@ impl ForecastAcquire for Forecasts { .map(|r| r.records.clone()) } } +/// The forecast-zone list the double answers: like the provider's, it holds every land zone the +/// release carries (`Zones::packaged`, the city zones of `getMidTa`) besides the Seoul region of +/// [`zone_page`]. A request reads this list when it is stored before its zone read and the +/// release's land zones otherwise, so a test of a town outside Seoul answers the same either way +/// (a Seoul-only list left the mid temperatures of such towns uncollected whenever it was stored +/// first). +fn land_zone_page() -> Vec { + use server2::domestic::kma::source::{Val, Zones}; + let seoul: serde_json::Value = serde_json::from_slice(&zone_page()).unwrap(); + let mut items: Vec = seoul["response"]["body"]["items"]["item"] + .as_array() + .unwrap() + .clone(); + for zone in Zones::packaged().docs() { + let text = |k: &str| zone.get(k).and_then(Val::as_str).unwrap().to_owned(); + let id = text("regId"); + if items.iter().any(|item| item["regId"] == id.as_str()) { + continue; + } + items.push(json!({ + "regId": id, + "tmSt": "201601010000", + "tmEd": "210012310000", + "regSp": text("regSp"), + "regName": text("regName"), + })); + } + serde_json::to_vec(&json!({"response": { + "header": {"resultCode": "00", "resultMsg": "NORMAL_SERVICE"}, + "body": {"dataType": "JSON", "items": {"item": items}, "pageNo": 1, "numOfRows": 1000, + "totalCount": items.len()}}})) + .unwrap() +} struct FixedGeo; impl Geocoder for FixedGeo { fn coord(&self, _: [f64; 2], _: String) -> BoxFuture<'_, Result> { @@ -1214,20 +1247,38 @@ async fn a_town_outside_the_list_gets_its_forecast_within_the_bound() { } /// The geocoder's name for the province must reach the listed town and its mid forecast. +/// +/// The request starts the process's forecast-zone acquisition without waiting for it, and its +/// zone read then uses that list when it is already stored, the release's land zones otherwise. +/// Both orders happen (they depend on how fast the stored list is published), so the test runs +/// each town with the list stored before the request and without it. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { let _peers = S3_PEERS.lock().await; - for (region, city, town) in [ + for ((region, city, town), stored_first) in [ ("강원특별자치도", "강릉시", "홍제동"), ("전북특별자치도", "군산시", "조촌동"), ("전남광주통합특별시", "여수시", "쌍봉동"), ("전남광주통합특별시", "광산구", "송정2동"), - ] { + ] + .into_iter() + .flat_map(|town| [(town, true), (town, false)]) + { let peer = Peer::persistent(); let store = Arc::new( CatalogStore::new(peer.transport_limited(3), CatalogLimits::default()).unwrap(), ); let writer = Writer::new(store, Config::default()).unwrap(); + if stored_first { + Forecasts { + writer: writer.clone(), + calls: Arc::new(AtomicUsize::new(0)), + } + .acquire(ForecastRequest::Zones) + .await + .unwrap() + .expect("the forecast-zone list is stored"); + } let source = build_source(&peer, Arc::new(Providers::default())) .with_forecast(Some(Arc::new(Forecasts { writer: writer.clone(), @@ -1265,7 +1316,7 @@ async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { "future-mid-unavailable" ] .contains(&reason.as_str().unwrap()), - "{region} {city}: {status}" + "{region} {city} (zone list stored first: {stored_first}): {status}" ); } let daily = body["midData"]["dailyData"].as_array().unwrap(); From 06018c20beb9532f3d2436f69eec0f2931d9dfda Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sat, 7 Nov 2026 23:40:48 +0900 Subject: [PATCH 08/11] server2: hold the zone request in the province test's fallback case (TC-force-fallback-order) full_response::a_province_under_its_new_name_finds_its_town_and_mid_forecast asked each town with the forecast-zone list stored first and "without it", but nothing kept the request's own zone acquisition from storing the list before the answer: counting the lists S3 held when the answer was made, the "without" case had one in 10 of 10 runs, so it did not pin the release's land-zone fallback. In that case the double's zone request now waits at a barrier (HeldZones) until the answer has been checked, and the test asserts the lists S3 held at the answer (none, or the one stored first) and that the held request stores its list once released. 10/10 passes. The same test change as 64541ef9 on fix/2614-kma-test-clock. Co-Authored-By: Claude Opus 5.5 --- server2/tests/domestic/full_response.rs | 76 +++++++++++++++++++++++-- 1 file changed, 70 insertions(+), 6 deletions(-) diff --git a/server2/tests/domestic/full_response.rs b/server2/tests/domestic/full_response.rs index d108655ea..aa5bf6c10 100644 --- a/server2/tests/domestic/full_response.rs +++ b/server2/tests/domestic/full_response.rs @@ -1246,12 +1246,53 @@ async fn a_town_outside_the_list_gets_its_forecast_within_the_bound() { ); } +/// The test double's forecast-zone request held until [`HeldZones::release`]; every other request +/// goes straight to [`Forecasts`]. +struct HeldZones { + inner: Forecasts, + open: tokio::sync::watch::Sender, +} +impl HeldZones { + fn release(&self) { + self.open.send_replace(true); + } +} +impl ForecastAcquire for HeldZones { + fn acquire(&self, request: ForecastRequest) -> BoxFuture<'static, Result> { + let mut open = self.open.subscribe(); + let zones = request == ForecastRequest::Zones; + let acquired = self.inner.acquire(request); + Box::pin(async move { + if zones { + let _ = open.wait_for(|open| *open).await; + } + acquired.await + }) + } + fn cached(&self, request: &ForecastRequest) -> Published { + self.inner.cached(request) + } +} +/// The forecast-zone lists S3 holds now. +async fn stored_zone_lists(peer: &Peer) -> usize { + server2::history::IndexListing::list_index( + &peer.transport_limited(2), + "index/v2/data-go-kr/forecast-zone/", + None, + ) + .await + .unwrap() + .keys + .len() +} + /// The geocoder's name for the province must reach the listed town and its mid forecast. /// /// The request starts the process's forecast-zone acquisition without waiting for it, and its -/// zone read then uses that list when it is already stored, the release's land zones otherwise. -/// Both orders happen (they depend on how fast the stored list is published), so the test runs -/// each town with the list stored before the request and without it. +/// zone read then uses the stored list when there is one, the release's land zones otherwise. +/// Each town is asked both ways: with the list stored before the request, and with the zone +/// request held until the answer is checked, so that no list is stored before it and the land +/// zones answer (TC-force-fallback-order). #[tokio::test(flavor = "multi_thread", worker_threads = 4)] async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { let _peers = S3_PEERS.lock().await; @@ -1279,11 +1320,15 @@ async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { .unwrap() .expect("the forecast-zone list is stored"); } - let source = build_source(&peer, Arc::new(Providers::default())) - .with_forecast(Some(Arc::new(Forecasts { + let zones = Arc::new(HeldZones { + inner: Forecasts { writer: writer.clone(), calls: Arc::new(AtomicUsize::new(0)), - }))) + }, + open: tokio::sync::watch::channel(stored_first).0, + }); + let source = build_source(&peer, Arc::new(Providers::default())) + .with_forecast(Some(zones.clone())) .with_raw_writer(writer.clone()); let backend = KmaBackend::new( Arc::new(source), @@ -1298,6 +1343,12 @@ async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { .outcome(request, Instant::now() + Duration::from_secs(8)) .await .unwrap(); + // The lists the answer could read: the held request stored none before it. + assert_eq!( + stored_zone_lists(&peer).await, + usize::from(stored_first), + "{region} {city}: zone lists stored at the answer (stored first: {stored_first})" + ); let server2::domestic::kma::Outcome::Json(body) = outcome else { panic!("{outcome:?}") }; @@ -1321,6 +1372,19 @@ async fn a_province_under_its_new_name_finds_its_town_and_mid_forecast() { } let daily = body["midData"]["dailyData"].as_array().unwrap(); assert_eq!(daily.last().unwrap()["date"], json!("20261004")); + // The held request then stores its list. + zones.release(); + for _ in 0..100 { + if stored_zone_lists(&peer).await == 1 { + break; + } + tokio::time::sleep(Duration::from_millis(50)).await; + } + assert_eq!( + stored_zone_lists(&peer).await, + 1, + "{region} {city}: the list" + ); } } From e708d12ae3b9006d9d030cd833bcfa3254e091e1 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sun, 8 Nov 2026 00:19:08 +0900 Subject: [PATCH 09/11] server2: count response-first waiters before a batch is ready CI run 42379684133 (52a270c8) failed full_response::empty_weather_store_serves_seven_days_before_raw_publication_and_replays_identically: one raw publication before the body terminal. Its trace shows a group published 2 ms into the owned flight that acquired it, while the request was still running. Writer::acquire counted a waiter only by its watch receiver, and run() asked receiver_count() after sending Ready: a waiter that took the batch and returned before that check, or a creating caller not yet polled again when an instant provider was done (it subscribed after spawning the job), left the batch looking unwaited, and NoWaiters released it, so the response's own raw was stored before the response finished (against provider -> assemble -> respond -> store). The writer is unchanged from master; the day products do not use it. The caller now subscribes before the job is spawned (or under the table lock when it joins), and the waiters are counted before Ready is sent. The new tests/response_first.rs a_creating_waiter_holds_its_instant_provider_until_the_body_ends (2,000 scoped instant acquisitions) failed 17/20 runs with 30 busy loops on 10 cores (NoWaiters releases before the body ended) and passes 20/20 under the same load; both full-cycle tests pass 20/20 under it. Co-Authored-By: Claude Opus 5.5 --- docs/architecture/server2.md | 6 +- server2/config/tasks/KMA-RESTART-OBSERVE.json | 2 + server2/src/storage/response_first.rs | 18 ++-- server2/tests/response_first.rs | 83 +++++++++++++++++++ specs/issue-2614-kma-restart-observe.md | 1 + 5 files changed, 104 insertions(+), 6 deletions(-) diff --git a/docs/architecture/server2.md b/docs/architecture/server2.md index 94e911f80..d81703335 100644 --- a/docs/architecture/server2.md +++ b/docs/architecture/server2.md @@ -342,7 +342,11 @@ instance headroom remains unqualified. Normal retries retain the same original bytes (9 s attempts, backoff capped at 30 s). Deterministic invalid input parks until shutdown; corrupt stored/control reads remain eligible for retry. Body end/error/drop releases publication; a 10 s dormancy cap, no-waiter detection -and shutdown also release it. The hook proves server body termination, not remote +and shutdown also release it. A caller is a waiter from the moment it creates or joins an +acquisition, and the waiters are counted before they can see the batch ready, so a waiter that +takes the batch and returns at once (or one not yet polled again when a fast provider is done) +does not make its response's raw look unwaited: on a loaded runner either order published a +group before the body ended (CI run 42379684133, 2026-11-07). The hook proves server body termination, not remote receipt. Quiesce rejects queued misses and a 34 s shutdown drain counts abandoned served originals. No durable copy means exact crash recovery can be impossible. diff --git a/server2/config/tasks/KMA-RESTART-OBSERVE.json b/server2/config/tasks/KMA-RESTART-OBSERVE.json index 7014771fd..c67dcc12d 100644 --- a/server2/config/tasks/KMA-RESTART-OBSERVE.json +++ b/server2/config/tasks/KMA-RESTART-OBSERVE.json @@ -12,12 +12,14 @@ "server2/src/domestic/kma/source/records.rs", "server2/src/storage/catalog.rs", "server2/src/storage/codec.rs", + "server2/src/storage/response_first.rs", "server2/tests/cache.rs", "server2/tests/domestic.rs", "server2/tests/domestic/full_response.rs", "server2/tests/domestic/observe.rs", "server2/tests/domestic/observe_product.rs", "server2/tests/domestic/summary_s3.rs", + "server2/tests/response_first.rs", "server2/tests/storage/catalog_wire.rs", "server2/tests/storage/codec.rs" ], diff --git a/server2/src/storage/response_first.rs b/server2/src/storage/response_first.rs index 67816fe84..9b2357324 100644 --- a/server2/src/storage/response_first.rs +++ b/server2/src/storage/response_first.rs @@ -595,11 +595,15 @@ impl Writer { F: Future, Error>> + Send + 'static, { let scope = ResponseScope::current(); - let entry = { + // The caller is a waiter from the moment it creates or joins the entry: its receiver + // exists before the provider can finish, so a provider that is done before the caller + // is polled again is not released as one nobody waits for (NoWaiters), which would + // publish a response's raw before its body ends. + let mut receiver = { let mut table = self.table.lock().unwrap_or_else(|e| e.into_inner()); Self::prune(&mut table); if let Some(entry) = table.entries.get(&key) { - entry.clone() + entry.state.subscribe() } else { if self.closing.load(SeqCst) || reserve == 0 @@ -618,6 +622,7 @@ impl Writer { if scope.is_none() { entry.gate.release(ReleaseReason::Background); } + let receiver = entry.state.subscribe(); table.entries.insert(key, entry.clone()); let this = self.clone(); let owned = entry.clone(); @@ -659,10 +664,9 @@ impl Writer { this.run(owned, lease, reserve, provider_until, provider) .await; })); - entry + receiver } }; - let mut receiver = entry.state.subscribe(); loop { let state = receiver.borrow_and_update().clone(); match state { @@ -772,9 +776,13 @@ impl Writer { _lease: lease, }); self.counters.ready.fetch_add(1, SeqCst); + // Waiters are counted before they can observe Ready: one that takes the batch and + // returns at once must not leave it looking unwaited (NoWaiters), which would publish + // its response's raw before the body ends. + let waiters = entry.state.receiver_count(); // The supervisor owns the charged batch before a waiter can observe Ready. entry.state.send_replace(Stage::Ready(ready.clone())); - if entry.state.receiver_count() == 0 { + if waiters == 0 { entry.gate.release(ReleaseReason::NoWaiters); } tokio::select! { diff --git a/server2/tests/response_first.rs b/server2/tests/response_first.rs index 643a8c462..e9328e4df 100644 --- a/server2/tests/response_first.rs +++ b/server2/tests/response_first.rs @@ -830,3 +830,86 @@ fn inheriting_a_scope_keeps_nested_futures_small() { let size = std::mem::size_of_val(&nested); assert!(size < 4 * 1024, "{size} bytes on the caller's stack"); } + +/// A batch of one grid's current conditions (distinct natural keys for many acquisitions). +fn batch_grid(nx: u16, ny: u16) -> Batch { + let r = RawRecord::new( + "kma", + "current", + &ProviderKey::Grid { nx, ny }, + Period { + local_date: 20261025, + slot: "1200".into(), + }, + 1792897200000, + 200, + "application/json", + None, + Arc::from(&b"{\"t1h\":17}"[..]), + &Limits::default(), + ) + .unwrap(); + let d = GroupDeclaration::new( + vec![GroupMember { + envelope: r.envelope.clone(), + catalogs: vec![CatalogId::for_record(&r.envelope.identity, "20261025-1200").unwrap()], + }], + &CatalogLimits::default(), + ) + .unwrap(); + Batch { + declaration: d, + records: vec![r], + } +} +/// A response's own acquisition is held until its body ends even when its provider is done +/// before the caller that created it is polled again (CI run 42379684133: a raw group published +/// before the body terminal). The caller is the test thread, so the provider runs on a worker at +/// once; on a busy runner the caller can be descheduled between creating the acquisition and +/// waiting for it, and a waiter counted only then made the writer release it as one nobody +/// waits for (NoWaiters). +#[tokio::test(flavor = "multi_thread", worker_threads = 4)] +async fn a_creating_waiter_holds_its_instant_provider_until_the_body_ends() { + let sink = Arc::new(Sink::default()); + let writer = Writer::new( + sink.clone(), + Config { + retained_bytes: 64 << 20, + group_bytes: 16384, + entries: 8192, + ..Config::default() + }, + ) + .unwrap(); + let scope = ResponseScope::default(); + scope + .run(async { + for i in 0..2000u16 { + writer + .acquire( + format!("current/{i}"), + 16384, + Instant::now() + Duration::from_secs(5), + async move { Ok(Some(batch_grid(1 + i % 140, 1 + i / 140))) }, + ) + .await + .unwrap(); + } + }) + .await; + let early: Vec<_> = scope + .batches() + .into_iter() + .filter_map(|b| b.released.map(|r| r.1)) + .collect(); + assert_eq!(early, Vec::new(), "released before the body ended"); + assert_eq!( + sink.attempts.load(SeqCst), + 0, + "published before the body ended" + ); + drop(scope.body(axum::body::Body::from("weather"))); + sink.allow.add_permits(2000); + assert!(writer.drain(Instant::now() + Duration::from_secs(10)).await); + assert_eq!(writer.stats().durable, 2000); +} diff --git a/specs/issue-2614-kma-restart-observe.md b/specs/issue-2614-kma-restart-observe.md index d58085d4e..8b4532401 100644 --- a/specs/issue-2614-kma-restart-observe.md +++ b/specs/issue-2614-kma-restart-observe.md @@ -6,6 +6,7 @@ - **FALLBACK:** an unreadable product, an older schema, an older parse revision and a grown city set each give that day's raw pages; a page the catalog does not list is dropped and its hour restored. Each answers as the raw pages do and stores the day's product again. Checked by `unusable_products_are_replaced_by_the_raw_pages`. - **ACCOUNTED** (review RO-city-growth, RO-minute-fallback, RO-sticky-excluded): a city page published after the products that names another station makes the restart read the reduced AWS pages in full again, and its answers equal a cold raw read. A minute whose newest acquisition S06 excludes is read from the next acquisition of that minute, as a cold read gives it. A group excluded when the product was made and repaired later completes its day on a restart: the answers equal a cold read, they settle, the day's product is stored again without the exclusion, and a new process records the daily summaries. While those pages cannot be read in full the answers stay unsettled, and a later request tries again without a new hourly page; a read slower than one run's limit keeps what it read and finishes over later runs. Checked by `a_station_joining_the_city_set_after_the_products_is_read_in_full`, `a_failed_full_read_keeps_answers_unsettled_until_it_is_tried_again`, `a_slow_full_read_progresses_over_several_runs`, `an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one`, `a_repaired_group_completes_its_day_after_the_product` (`tests/domestic/observe_product.rs`) and `a_repaired_station_group_lets_a_restart_record_summaries` (`tests/domestic/summary_s3.rs`). - **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1); a restart checks only that group again and reads no other raw page of the day. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. +- **RESPONSE-FIRST:** a response's own raw groups are published only after its body ends, also when a provider is done before its caller is polled again or a waiter returns before the writer counts it. Checked under CPU saturation by `a_creating_waiter_holds_its_instant_provider_until_the_body_ends` (`tests/response_first.rs`; it failed 17 of 20 runs before the change) and by `empty_weather_store_serves_seven_days_before_raw_publication_and_replays_identically` (`tests/domestic/full_response.rs`, failed in CI run 42379684133). - **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json` (base `origin/feat/2614-kma-restart-reads`), and the artifact policy passes on outgoing commits. A raw key with out-of-range numbers is not a record identity (`absurd_record_keys_are_refused`, `tests/storage/codec.rs`). From 3374e105a3aeaf6c78c912438293ece769d0f472 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sun, 8 Nov 2026 01:02:37 +0900 Subject: [PATCH 10/11] server2: settle response-first waiters with Ready under the join lock (RFW-zero-count-join) run() counted the receivers, then announced Ready and decided NoWaiters, without the table lock under which callers join entries. A response that joined after its creator was cancelled (count 0) but before Ready got the batch while the stored 0 released it as unwaited, so its raw was published while its body was still being sent. Who waits is now settled under the table lock together with the Ready announcement and the NoWaiters decision. Publisher::checkpoint is a test seam (a default that is ready at once; no setting). The new a_response_joining_before_ready_holds_a_cancelled_creators_batch forces that order; with the count taken before the checkpoint it failed 3/3 ([NoWaiters]). Same writer and test code as 7d275888 on fix/2614-response-first-waiters (#2903). Co-Authored-By: Claude Opus 5.5 --- docs/architecture/server2.md | 9 +- server2/src/storage/response_first.rs | 35 +++++-- server2/tests/response_first.rs | 132 +++++++++++++++++++++++++- 3 files changed, 163 insertions(+), 13 deletions(-) diff --git a/docs/architecture/server2.md b/docs/architecture/server2.md index d81703335..1b351d589 100644 --- a/docs/architecture/server2.md +++ b/docs/architecture/server2.md @@ -343,10 +343,11 @@ bytes (9 s attempts, backoff capped at 30 s). Deterministic invalid input parks until shutdown; corrupt stored/control reads remain eligible for retry. Body end/error/drop releases publication; a 10 s dormancy cap, no-waiter detection and shutdown also release it. A caller is a waiter from the moment it creates or joins an -acquisition, and the waiters are counted before they can see the batch ready, so a waiter that -takes the batch and returns at once (or one not yet polled again when a fast provider is done) -does not make its response's raw look unwaited: on a loaded runner either order published a -group before the body ended (CI run 42379684133, 2026-11-07). The hook proves server body termination, not remote +acquisition, and the waiters are counted together with the Ready announcement, under the lock +callers create and join entries under. A waiter that takes the batch and returns at once, one not +yet polled again when a fast provider is done, or a response that joins after the creator was +cancelled but before Ready therefore never makes its response's raw look unwaited: on a loaded +runner such an order published a group before the body ended (CI run 42379684133, 2026-11-07). The hook proves server body termination, not remote receipt. Quiesce rejects queued misses and a 34 s shutdown drain counts abandoned served originals. No durable copy means exact crash recovery can be impossible. diff --git a/server2/src/storage/response_first.rs b/server2/src/storage/response_first.rs index 9b2357324..43ad8b46c 100644 --- a/server2/src/storage/response_first.rs +++ b/server2/src/storage/response_first.rs @@ -78,9 +78,22 @@ pub struct Batch { pub declaration: GroupDeclaration, pub records: Vec, } +/// A step of an acquisition at which the writer awaits [`Publisher::checkpoint`]. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Checkpoint { + /// The job's batch is valid and charged; who waits for it is not settled and it is not + /// Ready yet. + Validated, +} /// Only this publisher can confer durable eligibility. RAM records carry no S06 token. pub trait Publisher: Send + Sync + 'static { fn publish(&self, batch: Arc, deadline: Instant) -> BoxFuture<'_, Result<(), Error>>; + /// A test seam, not a setting: the writer awaits it at `point` of every acquisition, so a + /// test can order responses against the writer's steps. Production publishers keep this + /// default, which is ready at once. + fn checkpoint(&self, _point: Checkpoint) -> BoxFuture<'_, ()> { + Box::pin(std::future::ready(())) + } } impl Publisher for CatalogStore { fn publish(&self, batch: Arc, deadline: Instant) -> BoxFuture<'_, Result<(), Error>> { @@ -776,14 +789,20 @@ impl Writer { _lease: lease, }); self.counters.ready.fetch_add(1, SeqCst); - // Waiters are counted before they can observe Ready: one that takes the batch and - // returns at once must not leave it looking unwaited (NoWaiters), which would publish - // its response's raw before the body ends. - let waiters = entry.state.receiver_count(); - // The supervisor owns the charged batch before a waiter can observe Ready. - entry.state.send_replace(Stage::Ready(ready.clone())); - if waiters == 0 { - entry.gate.release(ReleaseReason::NoWaiters); + self.publisher.checkpoint(Checkpoint::Validated).await; + // Who waits is settled under the table lock, the one callers create and join entries + // under, together with the Ready announcement: a response that joins before Ready is + // counted, one that joins after it finds the batch Ready, and a waiter cannot take the + // batch and leave before it is counted. Released as unwaited (NoWaiters), a response's + // own raw would be published before its body ends. + { + let _joins = self.table.lock().unwrap_or_else(|e| e.into_inner()); + let waiters = entry.state.receiver_count(); + // The supervisor owns the charged batch before a waiter can observe Ready. + entry.state.send_replace(Stage::Ready(ready.clone())); + if waiters == 0 { + entry.gate.release(ReleaseReason::NoWaiters); + } } tokio::select! { _=entry.gate.wait()=>(), diff --git a/server2/tests/response_first.rs b/server2/tests/response_first.rs index e9328e4df..fe9316428 100644 --- a/server2/tests/response_first.rs +++ b/server2/tests/response_first.rs @@ -4,7 +4,9 @@ use server2::{ storage::{ CatalogId, CatalogLimits, Error, GroupDeclaration, GroupMember, Limits, Period, ProviderKey, RawRecord, - response_first::{Batch, Config, Publisher, ResponseScope, Writer}, + response_first::{ + Batch, Checkpoint, Config, Publisher, ReleaseReason, ResponseScope, Writer, + }, }, }; use std::{ @@ -913,3 +915,131 @@ async fn a_creating_waiter_holds_its_instant_provider_until_the_body_ends() { assert!(writer.drain(Instant::now() + Duration::from_secs(10)).await); assert_eq!(writer.stats().durable, 2000); } + +/// A publisher that also holds the writer at one [`Checkpoint`]: the test learns when the writer +/// got there ([`Held::reached`]) and lets it go on ([`Held::go`]). Publications are counted and +/// wait for `allow`, as [`Sink`]'s. +struct Held { + at: Checkpoint, + reached: tokio::sync::watch::Sender, + open: tokio::sync::watch::Sender, + sink: Sink, +} +impl Held { + fn at(at: Checkpoint) -> Self { + Self { + at, + reached: tokio::sync::watch::channel(false).0, + open: tokio::sync::watch::channel(false).0, + sink: Sink::default(), + } + } + async fn reached(&self) { + let _ = self.reached.subscribe().wait_for(|r| *r).await; + } + fn go(&self) { + self.open.send_replace(true); + } +} +impl Publisher for Held { + fn publish(&self, batch: Arc, deadline: Instant) -> BoxFuture<'_, Result<(), Error>> { + self.sink.publish(batch, deadline) + } + fn checkpoint(&self, point: Checkpoint) -> BoxFuture<'_, ()> { + Box::pin(async move { + if point == self.at { + self.reached.send_replace(true); + let _ = self.open.subscribe().wait_for(|o| *o).await; + } + }) + } +} +/// A writer whose checks before a body ends stay clear of its 10 s dormancy cap. +fn held_writer(held: Arc) -> Arc { + Writer::new( + held, + Config { + retained_bytes: 32768, + group_bytes: 16384, + dormancy: Duration::from_secs(10), + ..Config::default() + }, + ) + .unwrap() +} +/// The batches `scope` used that were released before its body ended, and why. +fn released_early(scope: &ResponseScope) -> Vec { + scope + .batches() + .into_iter() + .filter_map(|b| b.released.map(|r| r.1)) + .collect() +} + +/// RFW-zero-count-join: the response that created an acquisition is cancelled, so nobody waits +/// for it; another response joins it after its batch is valid and before it is Ready. That +/// response is a waiter: the batch is held until its body ends, not released as unwaited +/// (NoWaiters) and published while it is still being sent. +#[tokio::test] +async fn a_response_joining_before_ready_holds_a_cancelled_creators_batch() { + let held = Arc::new(Held::at(Checkpoint::Validated)); + let writer = held_writer(held.clone()); + let key = "grid/current/60/127/20261025-1200"; + let (provide, provided) = tokio::sync::oneshot::channel::<()>(); + let creator = tokio::spawn({ + let writer = writer.clone(); + async move { + let scope = ResponseScope::default(); + scope + .run(writer.acquire( + key.into(), + 16384, + Instant::now() + Duration::from_secs(5), + async move { + let _ = provided.await; + Ok(Some(batch())) + }, + )) + .await + } + }); + tokio::task::yield_now().await; + creator.abort(); + match creator.await { + Err(e) => assert!(e.is_cancelled()), + Ok(_) => panic!("the creator ran to its end"), + } + provide.send(()).unwrap(); + held.reached().await; + let joiner = ResponseScope::default(); + let joined = tokio::spawn({ + let (writer, joiner) = (writer.clone(), joiner.clone()); + async move { + joiner + .run(writer.acquire( + key.into(), + 16384, + Instant::now() + Duration::from_secs(5), + async { panic!("the joiner starts no provider") }, + )) + .await + } + }); + tokio::task::yield_now().await; + held.go(); + assert!(joined.await.unwrap().unwrap().is_some()); + assert_eq!( + released_early(&joiner), + Vec::new(), + "released before the body ended" + ); + assert_eq!( + held.sink.attempts.load(SeqCst), + 0, + "published before the body ended" + ); + drop(joiner.body(axum::body::Body::from("weather"))); + held.sink.allow.add_permits(1); + assert!(writer.drain(Instant::now() + Duration::from_secs(5)).await); + assert_eq!(writer.stats().durable, 1); +} From 76add9e8b618f552069c31d0653c5b0d3095c948 Mon Sep 17 00:00:00 2001 From: hwanjjang Date: Sun, 8 Nov 2026 01:02:53 +0900 Subject: [PATCH 11/11] server2: force the response-first waiter orders in tests (RFW-test-scheduling) The regression test for the waiter count relied on the scheduler to preempt at the right step over 2,000 instant acquisitions, and its "nothing released" check could trip on a slow runner's 10 s dormancy cap. Two more Publisher::checkpoint points (Created, Announced) let a_creator_not_yet_waiting_when_its_batch_is_ready_holds_it and a_waiter_that_takes_the_batch_at_once_holds_it force the orders; each failed 3/3 with the old order ([NoWaiters]). The mass run is an ignored stress check that fails only on NoWaiters. The spec's RESPONSE-FIRST criterion names the forced-order tests. Same writer and test code as 03f7b575 on fix/2614-response-first-waiters (#2903). Co-Authored-By: Claude Opus 5.5 --- server2/src/storage/response_first.rs | 14 ++- server2/tests/response_first.rs | 123 ++++++++++++++++++++---- specs/issue-2614-kma-restart-observe.md | 2 +- 3 files changed, 117 insertions(+), 22 deletions(-) diff --git a/server2/src/storage/response_first.rs b/server2/src/storage/response_first.rs index 43ad8b46c..7c21fb062 100644 --- a/server2/src/storage/response_first.rs +++ b/server2/src/storage/response_first.rs @@ -81,9 +81,13 @@ pub struct Batch { /// A step of an acquisition at which the writer awaits [`Publisher::checkpoint`]. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum Checkpoint { + /// The creating caller has created the entry and started its job, and waits for it next. + Created, /// The job's batch is valid and charged; who waits for it is not settled and it is not /// Ready yet. Validated, + /// The batch is Ready and who waits for it is settled. + Announced, } /// Only this publisher can confer durable eligibility. RAM records carry no S06 token. pub trait Publisher: Send + Sync + 'static { @@ -612,11 +616,11 @@ impl Writer { // exists before the provider can finish, so a provider that is done before the caller // is polled again is not released as one nobody waits for (NoWaiters), which would // publish a response's raw before its body ends. - let mut receiver = { + let (mut receiver, created) = { let mut table = self.table.lock().unwrap_or_else(|e| e.into_inner()); Self::prune(&mut table); if let Some(entry) = table.entries.get(&key) { - entry.state.subscribe() + (entry.state.subscribe(), false) } else { if self.closing.load(SeqCst) || reserve == 0 @@ -677,9 +681,12 @@ impl Writer { this.run(owned, lease, reserve, provider_until, provider) .await; })); - receiver + (receiver, true) } }; + if created { + self.publisher.checkpoint(Checkpoint::Created).await; + } loop { let state = receiver.borrow_and_update().clone(); match state { @@ -804,6 +811,7 @@ impl Writer { entry.gate.release(ReleaseReason::NoWaiters); } } + self.publisher.checkpoint(Checkpoint::Announced).await; tokio::select! { _=entry.gate.wait()=>(), _=tokio::time::sleep(self.config.dormancy)=>entry.gate.release(ReleaseReason::DormancyCap), diff --git a/server2/tests/response_first.rs b/server2/tests/response_first.rs index fe9316428..6df9d7c1b 100644 --- a/server2/tests/response_first.rs +++ b/server2/tests/response_first.rs @@ -864,14 +864,14 @@ fn batch_grid(nx: u16, ny: u16) -> Batch { records: vec![r], } } -/// A response's own acquisition is held until its body ends even when its provider is done -/// before the caller that created it is polled again (CI run 42379684133: a raw group published -/// before the body terminal). The caller is the test thread, so the provider runs on a worker at -/// once; on a busy runner the caller can be descheduled between creating the acquisition and -/// waiting for it, and a waiter counted only then made the writer release it as one nobody -/// waits for (NoWaiters). +/// Stress check (not a regression test: its orders depend on the scheduler; the deterministic +/// ones are the `Checkpoint` tests below). 2,000 scoped acquisitions with instant providers, +/// created from the test thread; run on a saturated machine with +/// `cargo test --test response_first -- --ignored`. None may be released as unwaited +/// (NoWaiters) before the body ends; a dormancy release on a slow machine is not this defect. #[tokio::test(flavor = "multi_thread", worker_threads = 4)] -async fn a_creating_waiter_holds_its_instant_provider_until_the_body_ends() { +#[ignore = "stress check: run under CPU saturation"] +async fn stress_scoped_instant_acquisitions_are_never_released_unwaited() { let sink = Arc::new(Sink::default()); let writer = Writer::new( sink.clone(), @@ -899,19 +899,13 @@ async fn a_creating_waiter_holds_its_instant_provider_until_the_body_ends() { } }) .await; - let early: Vec<_> = scope - .batches() + let unwaited = released_early(&scope) .into_iter() - .filter_map(|b| b.released.map(|r| r.1)) - .collect(); - assert_eq!(early, Vec::new(), "released before the body ended"); - assert_eq!( - sink.attempts.load(SeqCst), - 0, - "published before the body ended" - ); + .filter(|r| *r == ReleaseReason::NoWaiters) + .count(); + assert_eq!(unwaited, 0, "released as unwaited before the body ended"); drop(scope.body(axum::body::Body::from("weather"))); - sink.allow.add_permits(2000); + sink.allow.add_permits(4000); assert!(writer.drain(Instant::now() + Duration::from_secs(10)).await); assert_eq!(writer.stats().durable, 2000); } @@ -923,6 +917,8 @@ struct Held { at: Checkpoint, reached: tokio::sync::watch::Sender, open: tokio::sync::watch::Sender, + /// Every checkpoint passed so far. + passed: tokio::sync::watch::Sender>, sink: Sink, } impl Held { @@ -931,12 +927,21 @@ impl Held { at, reached: tokio::sync::watch::channel(false).0, open: tokio::sync::watch::channel(false).0, + passed: tokio::sync::watch::channel(Vec::new()).0, sink: Sink::default(), } } async fn reached(&self) { let _ = self.reached.subscribe().wait_for(|r| *r).await; } + /// Wait until the writer passed `point`. + async fn passed(&self, point: Checkpoint) { + let _ = self + .passed + .subscribe() + .wait_for(|p| p.contains(&point)) + .await; + } fn go(&self) { self.open.send_replace(true); } @@ -951,6 +956,7 @@ impl Publisher for Held { self.reached.send_replace(true); let _ = self.open.subscribe().wait_for(|o| *o).await; } + self.passed.send_modify(|p| p.push(point)); }) } } @@ -1043,3 +1049,84 @@ async fn a_response_joining_before_ready_holds_a_cancelled_creators_batch() { assert!(writer.drain(Instant::now() + Duration::from_secs(5)).await); assert_eq!(writer.stats().durable, 1); } + +/// One scoped acquisition of `writer` by a response of its own (`scope`), with an instant +/// provider. +fn respond( + writer: &Arc, + scope: &ResponseScope, +) -> tokio::task::JoinHandle>, Error>> +{ + let (writer, scope) = (writer.clone(), scope.clone()); + tokio::spawn(async move { + scope + .run(writer.acquire( + "grid/current/60/127/20261025-1200".into(), + 16384, + Instant::now() + Duration::from_secs(5), + async { Ok(Some(batch())) }, + )) + .await + }) +} +/// The body of `scope` ends and its batch becomes durable. +async fn body_ends(held: &Held, writer: &Arc, scope: &ResponseScope) { + drop(scope.body(axum::body::Body::from("weather"))); + held.sink.allow.add_permits(1); + assert!(writer.drain(Instant::now() + Duration::from_secs(5)).await); + assert_eq!(writer.stats().durable, 1); +} + +/// RFW-test-scheduling: the creating caller is paused right after it created the entry and +/// started its job, and the job makes the batch Ready meanwhile. The caller was a waiter from the +/// start, so its batch is held until its body ends. +#[tokio::test] +async fn a_creator_not_yet_waiting_when_its_batch_is_ready_holds_it() { + let held = Arc::new(Held::at(Checkpoint::Created)); + let writer = held_writer(held.clone()); + let scope = ResponseScope::default(); + let creator = respond(&writer, &scope); + held.reached().await; + held.passed(Checkpoint::Announced).await; + held.go(); + assert!(creator.await.unwrap().unwrap().is_some()); + assert_eq!( + released_early(&scope), + Vec::new(), + "released before the body ended" + ); + assert_eq!( + held.sink.attempts.load(SeqCst), + 0, + "published before the body ended" + ); + body_ends(&held, &writer, &scope).await; +} + +/// RFW-test-scheduling: the job is paused right after it made the batch Ready, and the waiting +/// response takes the batch and returns meanwhile. It was counted before Ready, so its batch is +/// held until its body ends. +#[tokio::test] +async fn a_waiter_that_takes_the_batch_at_once_holds_it() { + let held = Arc::new(Held::at(Checkpoint::Announced)); + let writer = held_writer(held.clone()); + let scope = ResponseScope::default(); + let waiter = respond(&writer, &scope); + held.reached().await; + assert!(waiter.await.unwrap().unwrap().is_some()); + held.go(); + for _ in 0..8 { + tokio::task::yield_now().await; + } + assert_eq!( + released_early(&scope), + Vec::new(), + "released before the body ended" + ); + assert_eq!( + held.sink.attempts.load(SeqCst), + 0, + "published before the body ended" + ); + body_ends(&held, &writer, &scope).await; +} diff --git a/specs/issue-2614-kma-restart-observe.md b/specs/issue-2614-kma-restart-observe.md index 8b4532401..2cd4dd6dc 100644 --- a/specs/issue-2614-kma-restart-observe.md +++ b/specs/issue-2614-kma-restart-observe.md @@ -6,7 +6,7 @@ - **FALLBACK:** an unreadable product, an older schema, an older parse revision and a grown city set each give that day's raw pages; a page the catalog does not list is dropped and its hour restored. Each answers as the raw pages do and stores the day's product again. Checked by `unusable_products_are_replaced_by_the_raw_pages`. - **ACCOUNTED** (review RO-city-growth, RO-minute-fallback, RO-sticky-excluded): a city page published after the products that names another station makes the restart read the reduced AWS pages in full again, and its answers equal a cold raw read. A minute whose newest acquisition S06 excludes is read from the next acquisition of that minute, as a cold read gives it. A group excluded when the product was made and repaired later completes its day on a restart: the answers equal a cold read, they settle, the day's product is stored again without the exclusion, and a new process records the daily summaries. While those pages cannot be read in full the answers stay unsettled, and a later request tries again without a new hourly page; a read slower than one run's limit keeps what it read and finishes over later runs. Checked by `a_station_joining_the_city_set_after_the_products_is_read_in_full`, `a_failed_full_read_keeps_answers_unsettled_until_it_is_tried_again`, `a_slow_full_read_progresses_over_several_runs`, `an_incomplete_newest_minute_acquisition_falls_back_to_an_older_one`, `a_repaired_group_completes_its_day_after_the_product` (`tests/domestic/observe_product.rs`) and `a_repaired_station_group_lets_a_restart_record_summaries` (`tests/domestic/summary_s3.rs`). - **S16:** answers are unsettled while the rest of the restart read runs and settle after it. A day read with an S06-excluded group stays partial through its product (`excluded` 1); a restart checks only that group again and reads no other raw page of the day. The station page products are not cached on disk. Checked by `the_rest_of_the_restart_read_keeps_answers_unsettled`, `an_excluded_group_keeps_its_day_partial_through_the_product` and `the_station_page_product_is_not_cached` (`tests/cache.rs`); the existing `tests/domestic/observe.rs` cases keep the legacy answers, and the `tests/domestic/summary_s3.rs` cases record their summaries from the first complete answer after a start. -- **RESPONSE-FIRST:** a response's own raw groups are published only after its body ends, also when a provider is done before its caller is polled again or a waiter returns before the writer counts it. Checked under CPU saturation by `a_creating_waiter_holds_its_instant_provider_until_the_body_ends` (`tests/response_first.rs`; it failed 17 of 20 runs before the change) and by `empty_weather_store_serves_seven_days_before_raw_publication_and_replays_identically` (`tests/domestic/full_response.rs`, failed in CI run 42379684133). +- **RESPONSE-FIRST:** a response's own raw groups are published only after its body ends: also when a provider is done before its caller waits, when a waiter takes the batch and returns at once, and when a response joins after the creator was cancelled but before Ready. Checked with forced orders by `a_creator_not_yet_waiting_when_its_batch_is_ready_holds_it`, `a_waiter_that_takes_the_batch_at_once_holds_it` and `a_response_joining_before_ready_holds_a_cancelled_creators_batch` (`tests/response_first.rs`; each failed with the old order), and by `empty_weather_store_serves_seven_days_before_raw_publication_and_replays_identically` (`tests/domestic/full_response.rs`, failed in CI run 42379684133). The mass run is the ignored stress check `stress_scoped_instant_acquisitions_are_never_released_unwaited`. - **PLACEMENT:** the placement gate passes against `server2/config/tasks/KMA-RESTART-OBSERVE.json` (base `origin/feat/2614-kma-restart-reads`), and the artifact policy passes on outgoing commits. A raw key with out-of-range numbers is not a record identity (`absurd_record_keys_are_refused`, `tests/storage/codec.rs`).