diff --git a/interpreter/src/ir.rs b/interpreter/src/ir.rs index 8d338d3..359e025 100644 --- a/interpreter/src/ir.rs +++ b/interpreter/src/ir.rs @@ -178,6 +178,7 @@ pub enum BuiltInVar { Fpat = var_n(Variable::Fpat), Fieldwidths = var_n(Variable::Fieldwidths), Rs = var_n(Variable::Rs), + Rt = var_n(Variable::Rt), Ofs = var_n(Variable::Ofs), Ors = var_n(Variable::Ors), Filename = var_n(Variable::Filename), @@ -296,6 +297,7 @@ impl<'a, 'r> TryFrom<&'r Variable<'a>> for BuiltInVar { Variable::Fpat => Ok(Self::Fpat), Variable::Fieldwidths => Ok(Self::Fieldwidths), Variable::Rs => Ok(Self::Rs), + Variable::Rt => Ok(Self::Rt), Variable::Ofs => Ok(Self::Ofs), Variable::Ors => Ok(Self::Ors), Variable::Filename => Ok(Self::Filename), diff --git a/interpreter/src/vm.rs b/interpreter/src/vm.rs index 40c001e..d980fc1 100644 --- a/interpreter/src/vm.rs +++ b/interpreter/src/vm.rs @@ -54,6 +54,10 @@ pub struct Interpreter<'a> { registers: Registers<'a>, pub(crate) symbols: SymbolTable<'a>, pub(crate) record: Record, + /// Unconsumed input left after a regexp `RS` match; cleared on each new file. + rs_leftover: StdVec, + /// Cached regexp `RS` automaton (`pattern`, `regex`); invalidated on change. + rs_regex_cache: Option<(StdVec, minrx::Regex)>, consts: Consts<'a>, mode: ExecMode, frames: StdVec, @@ -114,6 +118,8 @@ impl<'a> Interpreter<'a> { registers: Registers(bumpalo::vec![in code.arena; Value::new_untyped(); n_regs + 1]), symbols: code.symbols, record: Record::new(), + rs_leftover: StdVec::new(), + rs_regex_cache: None, consts: code.consts, mode, frames: StdVec::new(), diff --git a/interpreter/src/vm/io.rs b/interpreter/src/vm/io.rs index 29022ef..596a48f 100644 --- a/interpreter/src/vm/io.rs +++ b/interpreter/src/vm/io.rs @@ -6,15 +6,16 @@ #[cfg(unix)] use std::{ffi::OsStr, os::unix::ffi::OsStrExt}; use std::{ - io::{self, BufRead, Result}, + io::{self, BufRead, ErrorKind}, num::NonZero, + ops::Range, path::{Path, PathBuf}, }; use atoi::atoi; use super::ExecMode; -use crate::{Interpreter, first_char, vm::types::Value}; +use crate::{Interpreter, exactly_one_char, first_char, vm::regex, vm::types::Value}; #[derive(Debug)] pub enum FilePath { @@ -66,12 +67,11 @@ impl Interpreter<'_> { self.symbols.errno = Value::new_int(errno); self.symbols.fnr = Value::new_int(0); self.record.clear(); + self.rs_leftover.clear(); } - // TODO: consider caching the result of all this validation, so we can - // directly dispatch to the correct functions and avoid extra writes to RT. // TODO: on Windows, we must strip `\r` depending on BINMODE. - pub fn read_record(&mut self, reader: impl BufRead) -> Result { + pub fn read_record(&mut self, reader: impl BufRead) -> io::Result { // Update vars // TODO: optimize and make more ergonomic self.symbols.nr = &self.symbols.fnr + &Value::new_int(1); @@ -79,58 +79,269 @@ impl Interpreter<'_> { let rs = self.symbols.rs.clone(); let rs_bytes = &*rs.to_bytes(); - let first = first_char(rs_bytes); - match self.mode { - // Regex matching (GNU extension) - ExecMode::Uu | ExecMode::Gnu if first.is_none() => { - self.read_record_regex(rs_bytes, reader) - } - // Single char matching - // TODO: what if POSIX & invalid UTF-8? - _ if let Some(c) = first => { - self.symbols.rt.clone_from(&rs); + if rs_bytes.is_empty() { + // Blank-line separated records (`RS = ""`). + self.read_record_blank_lines(reader) + } else if self.mode == ExecMode::Posix { + // POSIX: never a regexp — only the first character (or first byte). + if let Some(c) = first_char(rs_bytes) { self.read_record_until_char(c, reader) + } else { + self.read_record_until_byte(rs_bytes[0], reader) } - // Empty RS - _ => self.read_record_blank_lines(reader), + } else if let Some(c) = exactly_one_char(rs_bytes) { + self.read_record_until_char(c, reader) + } else { + // Multi-character (or non-UTF-8) `RS` is a regexp (gawk extension). + self.read_record_regex(rs_bytes, reader) } } - pub fn read_record_regex(&mut self, _rs: &[u8], mut _reader: impl BufRead) -> Result { - todo!() + pub fn read_record_regex(&mut self, rs: &[u8], mut reader: impl BufRead) -> io::Result { + self.ensure_rs_regex(rs)?; + + // Grow the leftover buffer until we see a usable non-empty match or EOF. + // Empty matches are ignored: `RS = "()"` must not split between chars. + // A match that ends at the buffer end may still grow (e.g. `RS = "\n+"`), + // so keep reading unless we have reached EOF. + let mut at_eof = false; + loop { + let range = { + let re = &self.rs_regex_cache.as_ref().unwrap().1; + find_nonempty_match(re, &self.rs_leftover)? + }; + if let Some(range) = range { + let touches_end = range.end == self.rs_leftover.len(); + if !touches_end || at_eof { + let rec = self.record.write_new(); + rec.extend_from_slice(&self.rs_leftover[..range.start]); + self.symbols.rt = + Value::new_string(self.rs_leftover[range.start..range.end].into()); + self.rs_leftover.drain(..range.end); + return Ok(true); + } + // Match reaches the end of the buffered input; try to extend it. + } + + if at_eof { + if self.rs_leftover.is_empty() { + self.symbols.rt = Value::new_str(b""); + return Ok(false); + } + let rec = self.record.write_new(); + rec.append(&mut self.rs_leftover); + self.symbols.rt = Value::new_str(b""); + return Ok(true); + } + + let buf = reader.fill_buf()?; + if buf.is_empty() { + at_eof = true; + continue; + } + let n = buf.len(); + self.rs_leftover.extend_from_slice(buf); + reader.consume(n); + } } - pub fn read_record_until_char(&mut self, c: char, mut reader: impl BufRead) -> Result { + /// Compiles `RS` once and reuses the automaton while the pattern is unchanged. + fn ensure_rs_regex(&mut self, rs: &[u8]) -> io::Result<()> { + let needs_compile = self + .rs_regex_cache + .as_ref() + .is_none_or(|(pat, _)| pat.as_slice() != rs); + if needs_compile { + let re = regex::automaton(rs, self.mode, false) + .map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))?; + self.rs_regex_cache = Some((rs.to_vec(), re)); + } + Ok(()) + } + + pub fn read_record_until_char( + &mut self, + c: char, + mut reader: impl BufRead, + ) -> io::Result { let mut bytes = [0; 4]; let bytes = c.encode_utf8(&mut bytes).as_bytes(); - let rec = self.record.write_new(); + self.read_record_until_delim(bytes, &mut reader) + } + + fn read_record_until_byte(&mut self, byte: u8, mut reader: impl BufRead) -> io::Result { + self.read_record_until_delim(&[byte], &mut reader) + } + + /// Character / byte `RS`, consuming any `rs_leftover` from a previous regexp + /// `RS` before reading the underlying stream. + fn read_record_until_delim( + &mut self, + delim: &[u8], + reader: &mut impl BufRead, + ) -> io::Result { + // Drain leftover from a prior regexp RS first so mid-file RS changes + // do not drop already-buffered input. + if let Some(pos) = find_delim(&self.rs_leftover, delim) { + let rec = self.record.write_new(); + rec.extend_from_slice(&self.rs_leftover[..pos]); + self.symbols.rt = Value::new_string(delim.into()); + self.rs_leftover.drain(..pos + delim.len()); + return Ok(true); + } // It is correct behavior that we terminate the record on EOF too, // even if the text file is malformed (no trailing newline). - match *bytes { - [byte] => { - // fast path: single search - let read = reader.read_until(byte, rec)?; - rec.pop_if(|&mut last| last == byte); - Ok(read > 0) + let (read_ok, found_delim) = { + let rec = self.record.write_new(); + rec.append(&mut self.rs_leftover); + match *delim { + [byte] => { + // fast path: single search + let read = reader.read_until(byte, rec)?; + let found_delim = rec.pop_if(|&mut last| last == byte).is_some(); + (read > 0 || !rec.is_empty() || found_delim, found_delim) + } + [.., last] => { + // slow path: loop searching multiple bytes or EOF. + let mut found = false; + let mut read_any = false; + while reader.read_until(last, rec)? > 0 { + read_any = true; + if rec.ends_with(delim) { + rec.truncate(rec.len() - delim.len()); + found = true; + break; + } + } + (found || read_any || !rec.is_empty(), found) + } + // This would be a bug in the std. It is often optimized away. + _ => unreachable!(), + } + }; + self.symbols.rt = if found_delim { + Value::new_string(delim.into()) + } else { + Value::new_str(b"") + }; + Ok(read_ok) + } + + /// Blank-line record separation for `RS = ""`. + /// + /// Leading newlines are skipped. A record ends at the first blank line; the + /// run of separating newlines becomes `RT`. A final newline at EOF (without + /// a following blank line) is stripped from the record and stored in `RT`. + /// + /// Any `rs_leftover` from a previous regexp `RS` is consumed first. + pub fn read_record_blank_lines(&mut self, mut reader: impl BufRead) -> io::Result { + // Merge leftover into a working buffer so mid-file RS changes keep bytes. + let mut pending = std::mem::take(&mut self.rs_leftover); + let start = pending + .iter() + .position(|&b| b != b'\n') + .unwrap_or(pending.len()); + pending.drain(..start); + + if pending.is_empty() { + // Skip leading newlines on the stream. + loop { + let buf = reader.fill_buf()?; + if buf.is_empty() { + self.symbols.rt = Value::new_str(b""); + return Ok(false); + } + if buf[0] == b'\n' { + reader.consume(1); + continue; + } + break; } - [.., last] => { - // slow path: loop searching multiple bytes or EOF. - while reader.read_until(last, rec)? > 0 { - if rec.ends_with(bytes) { - rec.truncate(rec.len() - bytes.len()); - return Ok(true); + } + + let (rt, tail) = { + let rec = self.record.write_new(); + rec.append(&mut pending); + + loop { + if let Some((content_len, rt_len)) = blank_line_split(rec) { + let rt = rec[content_len..content_len + rt_len].to_vec(); + let tail = rec.split_off(content_len + rt_len); + rec.truncate(content_len); + break (rt, tail); + } + + let n = reader.read_until(b'\n', rec)?; + if n == 0 || !rec.ends_with(b"\n") { + break (Vec::new(), Vec::new()); + } + + let buf = reader.fill_buf()?; + if buf.is_empty() { + rec.pop(); + break (Vec::from(b"\n".as_slice()), Vec::new()); + } + if buf[0] == b'\n' { + rec.pop(); + let mut rt = Vec::from(b"\n".as_slice()); + while { + let b = reader.fill_buf()?; + !b.is_empty() && b[0] == b'\n' + } { + rt.push(b'\n'); + reader.consume(1); } + break (rt, Vec::new()); } - Ok(!rec.is_empty()) } - // This would be a bug in the std. It is often optimized away. - _ => unreachable!(), + }; + + self.rs_leftover = tail; + self.symbols.rt = Value::new_string(rt.as_slice().into()); + Ok(true) + } +} + +/// Finds a blank-line boundary in `buf`. +/// +/// Returns `(content_len, rt_len)` where `buf[..content_len]` is the record and +/// `buf[content_len..content_len + rt_len]` is the run of separator newlines. +fn blank_line_split(buf: &[u8]) -> Option<(usize, usize)> { + let mut i = 0; + while i < buf.len() { + if buf[i] != b'\n' { + i += 1; + continue; + } + if i + 1 < buf.len() && buf[i + 1] == b'\n' { + let mut j = i; + while j < buf.len() && buf[j] == b'\n' { + j += 1; + } + return Some((i, j - i)); } + i += 1; } + None +} - pub fn read_record_blank_lines(&mut self, mut _reader: impl BufRead) -> Result { - todo!() +fn find_delim(haystack: &[u8], delim: &[u8]) -> Option { + match delim { + [] => None, + [b] => haystack.iter().position(|x| x == b), + d => haystack.windows(d.len()).position(|w| w == d), + } +} + +fn find_nonempty_match(re: &minrx::Regex, haystack: &[u8]) -> io::Result>> { + for m in re.find_iter(haystack) { + let m = m.map_err(|e| io::Error::new(ErrorKind::InvalidInput, e))?; + let range = m.range(); + if !range.is_empty() { + return Ok(Some(range.into())); + } } + Ok(None) } diff --git a/interpreter/src/vm/symbols.rs b/interpreter/src/vm/symbols.rs index 19abff9..b4dc483 100644 --- a/interpreter/src/vm/symbols.rs +++ b/interpreter/src/vm/symbols.rs @@ -285,6 +285,7 @@ impl<'a> SymbolTable<'a> { BuiltInVar::Fpat => &self.fpat, BuiltInVar::Fieldwidths => &self.fieldwidths, BuiltInVar::Rs => &self.rs, + BuiltInVar::Rt => &self.rt, BuiltInVar::Ofs => &self.ofs, BuiltInVar::Ors => &self.ors, BuiltInVar::Filename => &self.filename, @@ -307,6 +308,7 @@ impl<'a> SymbolTable<'a> { BuiltInVar::Fpat => &mut self.fpat, BuiltInVar::Fieldwidths => &mut self.fieldwidths, BuiltInVar::Rs => &mut self.rs, + BuiltInVar::Rt => &mut self.rt, BuiltInVar::Ofs => &mut self.ofs, BuiltInVar::Ors => &mut self.ors, BuiltInVar::Filename => &mut self.filename, diff --git a/lexer/src/lib.rs b/lexer/src/lib.rs index d7eabad..38ba14e 100644 --- a/lexer/src/lib.rs +++ b/lexer/src/lib.rs @@ -195,6 +195,8 @@ pub enum Token<'a> { FieldwidthsVariable, #[regex("(awk::)?RS", accept_expression)] RsVariable, + #[regex("(awk::)?RT", accept_expression)] + RtVariable, #[regex("(awk::)?OFS", accept_expression)] OfsVariable, #[regex("(awk::)?ORS", accept_expression)] diff --git a/lexer/src/tests.rs b/lexer/src/tests.rs index c998bca..072c6eb 100644 --- a/lexer/src/tests.rs +++ b/lexer/src/tests.rs @@ -539,7 +539,7 @@ fn lexer_test_control_flow_keywords() { #[test] fn lexer_test_builtin_variables() { let arena = Bump::new(); - let str = b"NR FNR NF FS FPAT FIELDWIDTHS RS OFS ORS FILENAME ARGC ARGV SUBSEP ARGIND OFMT \ + let str = b"NR FNR NF FS FPAT FIELDWIDTHS RS RT OFS ORS FILENAME ARGC ARGV SUBSEP ARGIND OFMT \ RSTART RLENGTH ENVIRON PROCINFO SYMTAB FUNCTAB"; assert_eq!( &lex(str, &arena, false, false), @@ -551,6 +551,7 @@ fn lexer_test_builtin_variables() { Token::FpatVariable, Token::FieldwidthsVariable, Token::RsVariable, + Token::RtVariable, Token::OfsVariable, Token::OrsVariable, Token::FilenameVariable, diff --git a/parser/src/ast.rs b/parser/src/ast.rs index a5c77d2..49984b9 100644 --- a/parser/src/ast.rs +++ b/parser/src/ast.rs @@ -79,6 +79,7 @@ pub enum Variable<'a> { Fpat, Fieldwidths, Rs, + Rt, Ofs, Ors, Filename, diff --git a/parser/src/lex.rs b/parser/src/lex.rs index 17cced4..1f2a418 100644 --- a/parser/src/lex.rs +++ b/parser/src/lex.rs @@ -262,6 +262,7 @@ impl TokenExt for Token<'_> { | Token::FpatVariable | Token::FieldwidthsVariable | Token::RsVariable + | Token::RtVariable | Token::OfsVariable | Token::OrsVariable | Token::FilenameVariable diff --git a/parser/src/lib.rs b/parser/src/lib.rs index f96debf..ff3d84a 100644 --- a/parser/src/lib.rs +++ b/parser/src/lib.rs @@ -924,6 +924,7 @@ impl<'a> Parser<'a> { Token::FpatVariable => Ok(Variable::Fpat), Token::FieldwidthsVariable => Ok(Variable::Fieldwidths), Token::RsVariable => Ok(Variable::Rs), + Token::RtVariable => Ok(Variable::Rt), Token::OfsVariable => Ok(Variable::Ofs), Token::OrsVariable => Ok(Variable::Ors), Token::FilenameVariable => Ok(Variable::Filename), diff --git a/tests/by-util/test_awk.rs b/tests/by-util/test_awk.rs index 481b35c..84f2f3a 100644 --- a/tests/by-util/test_awk.rs +++ b/tests/by-util/test_awk.rs @@ -602,6 +602,117 @@ fn ascii_char_fs_field_split() { .stdout_is("foo bar baz\n"); } +/// gawk manual "Record Splitting with Standard awk": `RS = "u"`. +#[test] +fn rs_single_char_splits_and_sets_rt() { + ucmd() + .arg(r#"BEGIN { RS = "u" } { print "[" $0 "] RT=[" RT "]" }"#) + .pipe_in("AmeliauAnthonyuBill") + .succeeds() + .stdout_is("[Amelia] RT=[u]\n[Anthony] RT=[u]\n[Bill] RT=[]\n"); +} + +/// gawk manual "Record Splitting with gawk": regexp `RS` and `RT`. +#[test] +fn rs_regex_manual_example_sets_rt() { + ucmd() + .arg(r#"BEGIN { RS = "\n|( *[[:upper:]]+ *)" } { print "Record =", $0,"and RT = [" RT "]" }"#) + .pipe_in("record 1 AAAA record 2 BBBB record 3\n") + .succeeds() + .stdout_is( + "Record = record 1 and RT = [ AAAA ]\n\ + Record = record 2 and RT = [ BBBB ]\n\ + Record = record 3 and RT = [\n\ + ]\n", + ); +} + +/// gawk manual "Multiple-Line Records" / empty `RS`: blank-line separated addresses. +#[test] +fn rs_empty_blank_line_records_manual_example() { + ucmd() + .arg(r#"BEGIN { RS = "" ; FS = "\n" } { print "Name is:", $1; print "Address is:", $2; print "City and State are:", $3; print "" }"#) + .pipe_in( + "Jane Doe\n\ + 123 Main Street\n\ + Anywhere, SE 12345-6789\n\ + \n\ + John Smith\n\ + 456 Tree-lined Avenue\n\ + Smallville, MW 98765-4321\n", + ) + .succeeds() + .stdout_is( + "Name is: Jane Doe\n\ + Address is: 123 Main Street\n\ + City and State are: Anywhere, SE 12345-6789\n\ + \n\ + Name is: John Smith\n\ + Address is: 456 Tree-lined Avenue\n\ + City and State are: Smallville, MW 98765-4321\n\ + \n", + ); +} + +#[test] +fn rs_empty_match_does_not_split_characters() { + ucmd() + .arg(r#"BEGIN { RS = "()" } { print "[" $0 "] RT=[" RT "]" }"#) + .pipe_in("abc") + .succeeds() + .stdout_is("[abc] RT=[]\n"); +} + +/// Mid-file switch from regexp `RS` to a single-char `RS` must still consume +/// bytes left in `rs_leftover` (not re-read past them from the stream). +#[test] +fn rs_switch_to_char_consumes_leftover() { + ucmd() + .arg( + r#"BEGIN { RS = "XY" } + NR == 1 { print "[" $0 "]"; RS = "\n"; next } + { print "[" $0 "]" }"#, + ) + .pipe_in("fooXYbar\nbaz\n") + .succeeds() + .stdout_is("[foo]\n[bar]\n[baz]\n"); +} + +/// `RS = "\n+"` with a separator that straddles the default `BufReader` buffer +/// (8 KiB) must not emit an extra empty record. +#[test] +fn rs_regex_match_at_buffer_end_extends_before_split() { + let mut input = vec![b'x'; 8191]; + input.extend_from_slice(b"\n\ny\n"); + ucmd() + .arg(r#"BEGIN { RS = "\n+" } { print "[" $0 "] RTL=" length(RT) }"#) + .pipe_in(input) + .succeeds() + .stdout_is(format!("[{}] RTL=2\n[y] RTL=1\n", "x".repeat(8191))); +} + +/// `RS = ""`: leading newlines are skipped; RT captures separator / trailing runs. +#[test] +fn rs_empty_rt_leading_several_and_trailing_newlines() { + ucmd() + .arg(r#"BEGIN { RS = "" } { print "[" $0 "] RTL=" length(RT) }"#) + .pipe_in("\n\na\n\n\nb\n") + .succeeds() + .stdout_is("[a] RTL=3\n[b] RTL=1\n"); + + ucmd() + .arg(r#"BEGIN { RS = "" } { print "[" $0 "] RTL=" length(RT) }"#) + .pipe_in("a\n\n\nb\n\n\nc\n") + .succeeds() + .stdout_is("[a] RTL=3\n[b] RTL=3\n[c] RTL=1\n"); + + ucmd() + .arg(r#"BEGIN { RS = "" } { print "[" $0 "] RTL=" length(RT) }"#) + .pipe_in("a\n\n\n") + .succeeds() + .stdout_is("[a] RTL=3\n"); +} + #[test] fn return_stmnt_sets_typedness_eagerly() { ucmd()