feat: invalid UTF-8 bytes match as U+FFFD width 1

CHANGELOG.md

@@ -8,12 +8,20 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ## [Unreleased]
 
 ### Fixed
+- **Invalid UTF-8 bytes now match as U+FFFD width 1** ([#179](https://github.com/coregx/coregex/issues/179)):
+  `.`, `\S`, `\D`, `\W`, `[^x]`, `\P{Han}`, and literal `\x{FFFD}` now match
+  invalid UTF-8 bytes (orphan continuations, overlong leads, out-of-range bytes)
+  as U+FFFD replacement character with width 1, matching stdlib semantics.
+  New `LookInvalidUTF8` assertion ensures matches fire only at rune boundaries,
+  not inside valid multi-byte sequences. All four engines (PikeVM, BoundedBacktracker,
+  OnePass DFA, lazy DFA) handle the assertion correctly. Lazy DFA uses temporary
+  quit-to-PikeVM fallback on bytes ≥ 0x80 (phase 3 will add UTF-8 validator product).
+  Harness divergence: 14 → 1 (only `FindReaderIndex` remains)
 - **Negated classes matched bytes, not runes** ([#174](https://github.com/coregx/coregex/issues/174)):
   `\S{2}` matched single Cyrillic `К` (2 UTF-8 bytes, 1 rune) by treating each
   byte as a separate character. Same for `\D`, `\W`, `[^x]` on any multi-byte
   rune. Root cause: `0x80–0xFF` single-byte range in NFA compiler's
-  `coversAllNonASCII` path. Invalid UTF-8 byte handling deferred to
-  [#179](https://github.com/coregx/coregex/issues/179).
+  `coversAllNonASCII` path.
   Random-fuzz divergence vs stdlib: 0.75% → 0.21%
 
 ### Planned

dfa/lazy/builder.go

@@ -70,22 +70,26 @@ func (b *Builder) Build() (*DFA, error) {
 	// Check if the pattern is always anchored (has ^ prefix)
 	isAlwaysAnchored := b.nfa.IsAlwaysAnchored()
 
+	// Check if the NFA contains LookInvalidUTF8 assertions
+	hasInvalidUTF8Look := b.checkHasInvalidUTF8Look()
+
 	// Build the immutable start byte map
 	var startByteMap [256]StartKind
 	initByteMap(&startByteMap)
 
 	// Create DFA — fully immutable after this point
 	dfa := &DFA{
-		nfa:              b.nfa,
-		config:           b.config,
-		prefilter:        pf,
-		pikevm:           nfa.NewPikeVM(b.nfa),
-		byteClasses:      b.nfa.ByteClasses(),
-		unanchoredStart:  b.nfa.StartUnanchored(),
-		hasWordBoundary:  hasWordBoundary,
-		hasEndLine:       hasEndLine,
-		isAlwaysAnchored: isAlwaysAnchored,
-		startByteMap:     startByteMap,
+		nfa:                b.nfa,
+		config:             b.config,
+		prefilter:          pf,
+		pikevm:             nfa.NewPikeVM(b.nfa),
+		byteClasses:        b.nfa.ByteClasses(),
+		unanchoredStart:    b.nfa.StartUnanchored(),
+		hasWordBoundary:    hasWordBoundary,
+		hasEndLine:         hasEndLine,
+		isAlwaysAnchored:   isAlwaysAnchored,
+		hasInvalidUTF8Look: hasInvalidUTF8Look,
+		startByteMap:       startByteMap,
 	}
 
 	return dfa, nil
@@ -654,7 +658,7 @@ func detectAccelFromTransitions(selfID StateID, stride int, transitionFn func(in
 			continue
 		}
 
-		if nextID == selfID || nextID == DeadState {
+		if nextID == selfID || nextID == DeadState || nextID == QuitState {
 			continue
 		}
 
@@ -748,3 +752,22 @@ func (b *Builder) checkHasEndLine() bool {
 	}
 	return false
 }
+
+// checkHasInvalidUTF8Look checks if the NFA contains LookInvalidUTF8 assertions.
+// When true, determinize quits on bytes >= 0x80 (temporary 3c fallback).
+func (b *Builder) checkHasInvalidUTF8Look() bool {
+	numStates := b.nfa.States()
+	for i := nfa.StateID(0); int(i) < numStates; i++ {
+		state := b.nfa.State(i)
+		if state == nil {
+			continue
+		}
+		if state.Kind() == nfa.StateLook {
+			look, _ := state.Look()
+			if look == nfa.LookInvalidUTF8 {
+				return true
+			}
+		}
+	}
+	return false
+}

dfa/lazy/cache.go

@@ -126,10 +126,10 @@ func (c *DFACache) Insert(key StateKey, state *State) (StateID, error) {
 }
 
 // safeOffset computes flat table offset from premultiplied StateID.
-// For tagged states (dead/invalid), returns MaxInt so bounds check always
-// fails safely. For normal and match-tagged states, returns sid.Offset() + classIdx.
+// For special tagged states (dead/invalid/quit), returns MaxInt so bounds
+// check always fails safely. Match and start tags pass through.
 func safeOffset(sid StateID, classIdx int) int {
-	if sid.IsDeadTag() || sid.IsInvalidTag() {
+	if sid.IsDeadTag() || sid.IsInvalidTag() || sid.IsQuitTag() {
 		return int(^uint(0) >> 1) // MaxInt
 	}
 	return sid.Offset() + classIdx
@@ -307,8 +307,8 @@ func (c *DFACache) ResetClearCount() {
 // getState retrieves a state from the stateList by premultiplied ID.
 // Converts premultiplied offset to state index for stateList lookup.
 func (c *DFACache) getState(id StateID) *State {
-	// Guard against tagged special states
-	if id.IsTagged() && (id.IsDeadTag() || id.IsInvalidTag()) {
+	// Guard against tagged special states (dead/invalid/quit)
+	if id.IsTagged() && (id.IsDeadTag() || id.IsInvalidTag() || id.IsQuitTag()) {
 		return nil
 	}
 	if c.stride == 0 {

dfa/lazy/lazy.go

@@ -87,6 +87,11 @@ type DFA struct {
 	// When true, we only need to try matching from position 0.
 	isAlwaysAnchored bool
 
+	// hasInvalidUTF8Look is true if the NFA contains LookInvalidUTF8 assertions.
+	// When true, determinize sets QuitState for all bytes >= 0x80 (temporary 3c).
+	// Phase 3 replaces this with a UTF-8 validator product.
+	hasInvalidUTF8Look bool
+
 	// startByteMap is the immutable byte-to-StartKind mapping used to initialize
 	// DFACache.startTable. Computed once during compilation.
 	startByteMap [256]StartKind
@@ -298,6 +303,9 @@ func (d *DFA) SearchAtAnchored(cache *DFACache, haystack []byte, at int) int {
 		case DeadState:
 			return lastMatch
 
+		case QuitState:
+			return d.nfaFallbackAnchored(haystack, at)
+
 		default:
 			sid = nextID
 		}
@@ -493,6 +501,8 @@ func (d *DFA) searchFirstAt(cache *DFACache, haystack []byte, startPos int) int
 			ftLen = len(ft)
 		case DeadState:
 			return lastMatch
+		case QuitState:
+			return d.nfaFallback(haystack, startPos)
 		default:
 			sid = nextID
 		}
@@ -705,7 +715,7 @@ func (d *DFA) searchEarliestMatch(cache *DFACache, haystack []byte, startPos int
 			offset := sid.Offset() + classIdx
 			if offset < ftLen {
 				nextID := ft[offset]
-				if nextID != InvalidState && nextID != DeadState {
+				if !nextID.IsTagged() {
 					sid = nextID
 					pos++
 					if cache.IsMatchState(sid) {
@@ -714,7 +724,7 @@ func (d *DFA) searchEarliestMatch(cache *DFACache, haystack []byte, startPos int
 					continue
 				}
 			}
-			// InvalidState/DeadState: fall through to full slow path
+			// Tagged (Invalid/Dead/Quit): fall through to full slow path
 		}
 
 		// Try lazy acceleration detection if not yet checked
@@ -774,6 +784,10 @@ func (d *DFA) searchEarliestMatch(cache *DFACache, haystack []byte, startPos int
 		case DeadState:
 			goto earliestPreSkip
 
+		case QuitState:
+			start, end, matched := d.pikevm.SearchAt(haystack, startPos)
+			return matched && start >= 0 && end >= start
+
 		default:
 			sid = nextID
 		}
@@ -901,6 +915,10 @@ func (d *DFA) searchEarliestMatchAnchored(cache *DFACache, haystack []byte, star
 		case DeadState:
 			return false
 
+		case QuitState:
+			start, end, matched := d.pikevm.SearchAt(haystack, startPos)
+			return matched && start == startPos && end >= start
+
 		default:
 			sid = nextID
 		}
@@ -1045,6 +1063,9 @@ func (d *DFA) findWithPrefilterAt(cache *DFACache, haystack []byte, startAt int)
 			lastMatch = -1
 			continue
 
+		case QuitState:
+			return d.nfaFallback(haystack, 0)
+
 		default:
 			sid = nextID
 		}
@@ -1231,7 +1252,7 @@ func (d *DFA) searchAt(cache *DFACache, haystack []byte, startPos int) int { //n
 			offset := sid.Offset() + classIdx
 			if offset < ftLen {
 				nextID := ft[offset]
-				if nextID != InvalidState && nextID != DeadState {
+				if !nextID.IsTagged() {
 					sid = nextID
 					if cache.IsMatchState(sid) {
 						lastMatch = pos
@@ -1240,7 +1261,7 @@ func (d *DFA) searchAt(cache *DFACache, haystack []byte, startPos int) int { //n
 					continue
 				}
 			}
-			// InvalidState/DeadState: fall through to full slow path
+			// Tagged (Invalid/Dead/Quit): fall through to full slow path
 		}
 
 		// Resolve State for slow path (acceleration, word boundary, determinize).
@@ -1289,6 +1310,8 @@ func (d *DFA) searchAt(cache *DFACache, haystack []byte, startPos int) int { //n
 			ftLen = len(ft)
 		case DeadState:
 			return lastMatch
+		case QuitState:
+			return d.nfaFallback(haystack, startPos)
 		default:
 			sid = nextID
 		}
@@ -1334,6 +1357,16 @@ func (d *DFA) searchAt(cache *DFACache, haystack []byte, startPos int) int { //n
 //
 //	or if determinization limit exceeded.
 func (d *DFA) determinize(cache *DFACache, current *State, b byte) (*State, error) {
+	// Temporary 3c: when the NFA has LookInvalidUTF8 assertions, the DFA
+	// cannot evaluate them (requires look-ahead/look-behind). Quit on any
+	// byte >= 0x80 and let PikeVM handle it. Phase 3 replaces this with a
+	// UTF-8 validator product that quits only on structurally invalid bytes.
+	if d.hasInvalidUTF8Look && b >= 0x80 {
+		classIdx := d.byteToClass(b)
+		cache.SetFlatTransition(current.id, int(classIdx), QuitState)
+		return nil, &DFAError{Kind: NFAFallback, Message: "quit: byte >= 0x80 with LookInvalidUTF8"}
+	}
+
 	// Need builder for move operations.
 	// Use NewBuilderWithWordBoundary to pass pre-computed flag and avoid O(states) scan.
 	builder := NewBuilderWithWordBoundary(d.nfa, d.config, d.hasWordBoundary)
@@ -1632,6 +1665,17 @@ func (d *DFA) nfaFallback(haystack []byte, startPos int) int {
 	return end
 }
 
+// nfaFallbackAnchored is like nfaFallback but requires the match to start
+// exactly at startPos. Used by SearchAtAnchored where unanchored fallback
+// would produce false positives from matches after the expected position.
+func (d *DFA) nfaFallbackAnchored(haystack []byte, startPos int) int {
+	start, end, matched := d.pikevm.SearchAt(haystack, startPos)
+	if !matched || start != startPos {
+		return -1
+	}
+	return end
+}
+
 // matchesEmpty checks if the pattern matches an empty string
 func (d *DFA) matchesEmpty(cache *DFACache) bool {
 	// With 1-byte match delay, the start state is never tagged as match.
@@ -1894,6 +1938,9 @@ func (d *DFA) SearchReverse(cache *DFACache, haystack []byte, start, end int) in
 		case DeadState:
 			return lastMatch
 
+		case QuitState:
+			return d.nfaFallbackReverse(haystack, start, end)
+
 		default:
 			sid = nextID
 		}
@@ -2011,6 +2058,9 @@ func (d *DFA) SearchReverseLimited(cache *DFACache, haystack []byte, start, end,
 		case DeadState:
 			return lastMatch
 
+		case QuitState:
+			return d.nfaFallbackReverse(haystack, start, end)
+
 		default:
 			sid = nextID
 		}
@@ -2103,6 +2153,10 @@ func (d *DFA) IsMatchReverse(cache *DFACache, haystack []byte, start, end int) b
 		case DeadState:
 			return false
 
+		case QuitState:
+			_, _, matched := d.pikevm.SearchBetween(haystack, start, end)
+			return matched
+
 		default:
 			sid = nextID
 		}
@@ -2158,11 +2212,12 @@ func (d *DFA) getStartStateForReverse(cache *DFACache, haystack []byte, end int)
 }
 
 // nfaFallbackReverse handles NFA fallback for reverse search.
+// Uses SearchBetween to preserve full haystack context (word boundaries,
+// LookInvalidUTF8 rune-boundary check need bytes before start).
 func (d *DFA) nfaFallbackReverse(haystack []byte, start, end int) int {
-	// For reverse fallback, we need to search the region and find match start
-	matchStart, _, matched := d.pikevm.Search(haystack[start:end])
+	matchStart, _, matched := d.pikevm.SearchBetween(haystack, start, end)
 	if !matched {
 		return -1
 	}
-	return start + matchStart
+	return matchStart
 }