From e160a3ffbb475d8b6868388b16fea1615f9a391b Mon Sep 17 00:00:00 2001 From: Franklin-Qi Date: Sat, 10 Oct 2026 18:12:39 +0800 Subject: [PATCH] interpreter: implement FIELDWIDTHS fixed-width splitting Parse FIELDWIDTHS once on assignment (tokenized blanks / skip:width), cache designators for record splits, fatal on invalid specs, and match gawk when a skip past EOF still yields an empty field (including *). Closes: #123 --- interpreter/src/lib.rs | 13 ++- interpreter/src/vm.rs | 38 +++++-- interpreter/src/vm/symbols.rs | 180 +++++++++++++++++++++++++++++++++- tests/by-util/test_awk.rs | 143 +++++++++++++++++++++++++++ 4 files changed, 358 insertions(+), 16 deletions(-) diff --git a/interpreter/src/lib.rs b/interpreter/src/lib.rs index 3354048..384d957 100644 --- a/interpreter/src/lib.rs +++ b/interpreter/src/lib.rs @@ -34,6 +34,8 @@ pub enum InterpreterError { ScalarUseOfArrary(AriadneSpan), #[error("Attempted to use a scalar as an array value!")] ArrayUseOfScalar(AriadneSpan), + #[error("Invalid FIELDWIDTHS value!")] + InvalidFieldwidths(AriadneSpan), #[error("{1}")] Regex(AriadneSpan, RegexError), } @@ -48,7 +50,8 @@ impl InterpreterError { | Self::UnknownIndFunction(span, _) | Self::RecursionDepth(span) | Self::ArityMismatch(span, _, _) - | Self::UnknownFunction(span) => self.add_diagnostic_cached(store, span.clone()), + | Self::UnknownFunction(span) + | Self::InvalidFieldwidths(span) => self.add_diagnostic_cached(store, span.clone()), } } } @@ -66,7 +69,8 @@ impl Diagnostic for InterpreterError { | &Self::UnknownIndFunction(AriadneSpan(_, span), _) | &Self::RecursionDepth(AriadneSpan(_, span)) | &Self::ArityMismatch(AriadneSpan(_, span), _, _) - | &Self::UnknownFunction(AriadneSpan(_, span)) => Some(span), + | &Self::UnknownFunction(AriadneSpan(_, span)) + | &Self::InvalidFieldwidths(AriadneSpan(_, span)) => Some(span), } } fn add_labels(&self, span: AriadneSpan, report: &mut ReportBuilder) { @@ -107,6 +111,11 @@ impl Diagnostic for InterpreterError { operator, which forces values into numbers. Otherwise, some\nvalues of `b`, like \ an empty string, would still trigger this error. It also parses the \"+nan\"." } + Self::InvalidFieldwidths(_) => { + "FIELDWIDTHS must be a blank-separated list of positive widths, optional \ + `skip:width` pairs, and at most one trailing `*` / `skip:*`.\nZero, negative, \ + non-integer, or misplaced `*` values are rejected (matching gawk)." + } _ => return, }; report.set_help(note); diff --git a/interpreter/src/vm.rs b/interpreter/src/vm.rs index 40c001e..0cb6fe5 100644 --- a/interpreter/src/vm.rs +++ b/interpreter/src/vm.rs @@ -185,7 +185,7 @@ impl<'a> Interpreter<'a> { }; (new_val, observed) })?; - Place::new(arg, ty).write(self, new_val); + Place::new(arg, ty).write(self, new_val, self.get_span(metadata))?; self.write_reg(dest, observed); } Instruction::CopyS { dest, arg, ty } => { @@ -310,7 +310,7 @@ impl<'a> Interpreter<'a> { debug_assert!(matches!(ty_place, PlaceTy::UserVal | PlaceTy::BtInVal)); let val = self.get_val(arg, ty, metadata, Value::clone)?; - Place::new(var, ty_place).write(self, val.clone()); + Place::new(var, ty_place).write(self, val.clone(), self.get_span(metadata))?; self.write_reg(dest, val); } Instruction::StoreF { dest, src, arg, ty, tys } => { @@ -478,22 +478,31 @@ impl<'a> Interpreter<'a> { } /// Writes and triggers side-effects of built-in variables on write. - pub fn sync_btin_write(&mut self, sys: BuiltInVar, val: Value<'a>) { + pub fn sync_btin_write( + &mut self, + sys: BuiltInVar, + val: Value<'a>, + span: AriadneSpan, + ) -> Result<()> { // Apply side effects before writing the value match sys { BuiltInVar::Nf => { // TODO: errors let n = usize::try_from(val.to_int()).unwrap(); let _ = self.record.resize(n, &mut self.symbols, self.mode); - return; // auto-updated + return Ok(()); // auto-updated } BuiltInVar::Fs => self.record.enable_fs_splitting(&val, self.mode), BuiltInVar::Fpat => self.record.enable_fpat_splitting(), - BuiltInVar::Fieldwidths => self.record.enable_fieldwidths_splitting(), + BuiltInVar::Fieldwidths => self + .record + .enable_fieldwidths_splitting(&val) + .map_err(|()| InterpreterError::InvalidFieldwidths(span))?, BuiltInVar::Ofs => self.record.invalidate(), _ => {} } *self.symbols.get_btin_mut(sys) = val; + Ok(()) } /// Convenience wrapper to add errors from context metadata. @@ -821,11 +830,22 @@ impl Place { } #[inline(always)] - fn write<'a>(self, intrp: &mut Interpreter<'a>, val: Value<'a>) { + fn write<'a>( + self, + intrp: &mut Interpreter<'a>, + val: Value<'a>, + span: AriadneSpan, + ) -> Result<()> { match self.ty { - PlaceTy::Reg => *intrp.read_reg_mut(unsafe { self.arg.reg }) = val, - PlaceTy::UserVal => *intrp.symbols.user_mut(unsafe { self.arg.usr }) = val, - PlaceTy::BtInVal => intrp.sync_btin_write(unsafe { self.arg.sys }, val), + PlaceTy::Reg => { + *intrp.read_reg_mut(unsafe { self.arg.reg }) = val; + Ok(()) + } + PlaceTy::UserVal => { + *intrp.symbols.user_mut(unsafe { self.arg.usr }) = val; + Ok(()) + } + PlaceTy::BtInVal => intrp.sync_btin_write(unsafe { self.arg.sys }, val, span), } } } diff --git a/interpreter/src/vm/symbols.rs b/interpreter/src/vm/symbols.rs index 19abff9..9955edb 100644 --- a/interpreter/src/vm/symbols.rs +++ b/interpreter/src/vm/symbols.rs @@ -111,6 +111,8 @@ pub struct Record { raw: Vec, fields: Option>, split_mode: SplitMode, + /// Parsed `FIELDWIDTHS` designators; filled on assignment, used per record. + fieldwidths: SmallVec<[FieldWidth; 8]>, } // TODO: add variants for overridden modes (GNU CSV ext) and/or to `ExecMode`. @@ -347,9 +349,14 @@ impl Record { self.invalidate(); } - pub fn enable_fieldwidths_splitting(&mut self) { + /// Enables FIELDWIDTHS mode after validating/parsing `val` once. + /// + /// Returns `Err(())` for an invalid spec (caller should fatal like gawk). + pub fn enable_fieldwidths_splitting(&mut self, val: &Value) -> Result<(), ()> { + self.fieldwidths = parse_fieldwidths(&val.to_bytes()).ok_or(())?; self.split_mode = SplitMode::Fieldwidths; self.invalidate(); + Ok(()) } /// Splits the fields if unsplit and grants access to the inner buffer of @@ -373,14 +380,58 @@ impl Record { SplitMode::Fpat => { return self.fpat_regex_split(&symbols.fpat.to_bytes(), mode); } - SplitMode::Fieldwidths => { - let _fieldwidths = symbols.fieldwidths.to_bytes(); - todo!() - } + SplitMode::Fieldwidths => self.fieldwidths_split(), } }) } + /// Fixed-width splitting via `FIELDWIDTHS` (gawk extension). + /// + /// Uses the designators cached at assignment time. Counts are in **bytes** + /// (matching `LANG=C` gawk). A skip that reaches EOF still yields one empty + /// field for that designator (including a trailing `*`). + fn fieldwidths_split(&mut self) -> &mut Vec { + let specs = self.fieldwidths.clone(); + let (raw, buf) = self.init_fields(); + let len = raw.len(); + let mut pos = 0usize; + + for spec in specs { + if pos >= len { + break; + } + + if spec.skip > 0 { + pos = pos.saturating_add(spec.skip as usize).min(len); + } + + match spec.kind { + FieldWidthKind::Rest => { + // Empty when skip already consumed the record (gawk: NF=1, `$1=""`). + buf.push(Span::from(pos..len)); + break; + } + FieldWidthKind::Width(width) => { + let width = width as usize; + if pos >= len { + // A leading skip consumed the rest of the record: still + // materialize an empty field, then stop. + buf.push(Span::from(pos..pos)); + break; + } + let end = pos.saturating_add(width).min(len); + buf.push(Span::from(pos..end)); + let got = end - pos; + pos = end; + if got < width { + break; + } + } + } + } + buf + } + /// Initializes fields and assigns `$0`. Reuses existing buffer allocation. /// Used to be yet another monad, but we don't need to wrap everything in /// closures if we already have the `Option` monad internally. @@ -670,6 +721,84 @@ const fn to_span(start: usize) -> Span { Span { start, end: start + 1 } } +#[derive(Clone, Copy, Debug)] +enum FieldWidthKind { + Width(u32), + Rest, +} + +#[derive(Clone, Copy, Debug)] +struct FieldWidth { + skip: u32, + kind: FieldWidthKind, +} + +/// Parses a `FIELDWIDTHS` value into skip/width designators. +/// +/// Grammar: blank-separated tokens, each `width`, `skip:width`, `*`, or +/// `skip:*`. `*` / `skip:*` may appear only as the final token. Returns `None` +/// for invalid specs (negative/zero widths, misplaced `*`, junk, etc.). +fn parse_fieldwidths(spec: &[u8]) -> Option> { + let mut specs: SmallVec<[FieldWidth; 8]> = SmallVec::new(); + for token in spec + .split(|&b| matches!(b, b' ' | b'\t')) + .filter(|t| !t.is_empty()) + { + if specs + .last() + .is_some_and(|s| matches!(s.kind, FieldWidthKind::Rest)) + { + return None; + } + specs.push(parse_fieldwidths_token(token)?); + } + Some(specs) +} + +fn parse_fieldwidths_token(token: &[u8]) -> Option { + if token == b"*" { + return Some(FieldWidth { skip: 0, kind: FieldWidthKind::Rest }); + } + + if let Some(colon) = memchr::memchr(b':', token) { + let skip = parse_fieldwidths_u32(&token[..colon])?; + if skip == 0 { + return None; + } + let rest = &token[colon + 1..]; + if rest == b"*" { + Some(FieldWidth { skip, kind: FieldWidthKind::Rest }) + } else { + let width = parse_fieldwidths_u32(rest)?; + (width != 0).then_some(FieldWidth { skip, kind: FieldWidthKind::Width(width) }) + } + } else { + let width = parse_fieldwidths_u32(token)?; + (width != 0).then_some(FieldWidth { skip: 0, kind: FieldWidthKind::Width(width) }) + } +} + +/// Parses a whole token as an optional-`+` unsigned integer (no trailing junk). +fn parse_fieldwidths_u32(bytes: &[u8]) -> Option { + let mut i = 0; + if i < bytes.len() && bytes[i] == b'+' { + i += 1; + } + if i >= bytes.len() || !bytes[i].is_ascii_digit() { + return None; + } + + let mut n = 0u32; + while i < bytes.len() { + if !bytes[i].is_ascii_digit() { + return None; + } + n = n.checked_mul(10)?.checked_add(u32::from(bytes[i] - b'0'))?; + i += 1; + } + Some(n) +} + #[cfg(test)] mod tests { use super::*; @@ -729,4 +858,45 @@ mod tests { ); assert_eq!(st.argv.array_len(), Some(3)); } + + #[test] + fn parse_fieldwidths_basic_and_extensions() { + let specs = parse_fieldwidths(b"2 3 4").unwrap(); + assert_eq!(specs.len(), 3); + assert!(matches!( + specs.as_slice(), + [ + FieldWidth { skip: 0, kind: FieldWidthKind::Width(2) }, + FieldWidth { skip: 0, kind: FieldWidthKind::Width(3) }, + FieldWidth { skip: 0, kind: FieldWidthKind::Width(4) }, + ] + )); + + let specs = parse_fieldwidths(b"3:2 1:2 *").unwrap(); + assert!(matches!( + specs.as_slice(), + [ + FieldWidth { skip: 3, kind: FieldWidthKind::Width(2) }, + FieldWidth { skip: 1, kind: FieldWidthKind::Width(2) }, + FieldWidth { skip: 0, kind: FieldWidthKind::Rest }, + ] + )); + + let specs = parse_fieldwidths(b"2:*").unwrap(); + assert!(matches!( + specs.as_slice(), + [FieldWidth { skip: 2, kind: FieldWidthKind::Rest }] + )); + + assert!(parse_fieldwidths(b"").unwrap().is_empty()); + assert!(parse_fieldwidths(b" 2 3 ").is_some()); + assert!(parse_fieldwidths(b"+2 +3").is_some()); + + assert!(parse_fieldwidths(b"2 -1 2").is_none()); + assert!(parse_fieldwidths(b"2 0 2").is_none()); + assert!(parse_fieldwidths(b"0:2").is_none()); + assert!(parse_fieldwidths(b"2 * 2").is_none()); + assert!(parse_fieldwidths(b"2 x 2").is_none()); + assert!(parse_fieldwidths(b"2.5 2").is_none()); + } } diff --git a/tests/by-util/test_awk.rs b/tests/by-util/test_awk.rs index 481b35c..eca47a1 100644 --- a/tests/by-util/test_awk.rs +++ b/tests/by-util/test_awk.rs @@ -602,6 +602,149 @@ fn ascii_char_fs_field_split() { .stdout_is("foo bar baz\n"); } +#[test] +fn fieldwidths_basic_split() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = "2 3 4"; $0 = "aabbbcccc"; print NF, $1, $2, $3 }"#) + .succeeds() + .stdout_is("3 aa bbb cccc\n"); +} + +#[test] +fn fieldwidths_partial_and_short_records() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "2 3 4" + $0 = "aabbb"; print NF, $1, $2 + $0 = "aab"; print NF, $1, $2 + $0 = "aabbbccccddd"; print NF, $1, $2, $3 + }"#, + ) + .succeeds() + .stdout_is("2 aa bbb\n2 aa b\n3 aa bbb cccc\n"); +} + +#[test] +fn fieldwidths_trailing_star() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "2 3 4 *" + $0 = "aabbbccccddd"; print NF, $1, $2, $3, $4 + $0 = "aabbbcccc"; print NF, $1, $2, $3 + }"#, + ) + .succeeds() + .stdout_is("4 aa bbb cccc ddd\n3 aa bbb cccc\n"); +} + +/// gawk manual "Allowing trailing data": `FIELDWIDTHS = "2 2 *"`. +#[test] +fn fieldwidths_manual_trailing_star_example() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = "2 2 *"; $0 = "1234abcdefghi"; print NF, $1, $2, $3 }"#) + .succeeds() + .stdout_is("3 12 34 abcdefghi\n"); +} + +#[test] +fn fieldwidths_skip_and_star() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "3:2 1:2 *" + $0 = "1234567890" + print NF, $1, $2, $3 + }"#, + ) + .succeeds() + .stdout_is("3 45 78 90\n"); +} + +/// gawk manual "Skipping intervening": multi-field `skip:width` list. +#[test] +fn fieldwidths_manual_skip_intervening_example() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "8 1:5 4:7 6 1:6 1:6 2:33" + $0 = "hzang ttyV3 6:37pm 50 -csh" + print $1, $2, $4 + $0 = "eklye ttyV5 9:53pm 7 1 em thes.tex" + print $1, $2, $4 + }"#, + ) + .succeeds() + .stdout_is("hzang ttyV3 50\neklye ttyV5 \n"); +} + +/// gawk manual "Processing Fixed-Width Data": `FIELDWIDTHS = "9 6 10 6 7 7 35"`. +/// Prints fields only (idle conversion uses `sub()`, which is not implemented yet). +#[test] +fn fieldwidths_manual_fixed_width_columns_example() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "9 6 10 6 7 7 35" + $0 = "hzang ttyV3 6:37pm 50 -csh" + print "[" $1 "]", "[" $2 "]", "[" $4 "]" + $0 = "eklye ttyV5 9:53pm 7 1 em thes.tex" + print "[" $1 "]", "[" $2 "]", "[" $4 "]" + }"#, + ) + .succeeds() + .stdout_is("[hzang ] [ttyV3 ] [ 50]\n[eklye ] [ttyV5 ] [ ]\n"); +} + +#[test] +fn fieldwidths_skip_past_end_yields_empty_field() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = "5:2"; $0 = "ab"; print NF; print "[" $1 "]" }"#) + .succeeds() + .stdout_is("1\n[]\n"); +} + +/// gawk: a skip past EOF with trailing `*` still yields one empty field. +#[test] +fn fieldwidths_skip_star_past_end_yields_empty_field() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = "5:*"; $0 = "ab"; print NF; print "[" $1 "]" }"#) + .succeeds() + .stdout_is("1\n[]\n"); +} + +#[test] +fn fieldwidths_invalid_spec_is_fatal() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = "2 -1"; print "alive" }"#) + .fails_with_code(1) + .stderr_contains("Invalid FIELDWIDTHS value"); +} + +#[test] +fn fieldwidths_empty_disables_fs_fields() { + ucmd() + .arg(r#"BEGIN { FIELDWIDTHS = ""; $0 = "a b c"; print NF }"#) + .succeeds() + .stdout_is("0\n"); +} + +#[test] +fn fieldwidths_overridden_by_fs() { + ucmd() + .arg( + r#"BEGIN { + FIELDWIDTHS = "1 1 1" + FS = ":" + $0 = "a:b:c" + print NF, $1, $2, $3 + }"#, + ) + .succeeds() + .stdout_is("3 a b c\n"); +} + #[test] fn return_stmnt_sets_typedness_eagerly() { ucmd()