From e3a7977dcd2c666735451465ebc8b22c61b6733f Mon Sep 17 00:00:00 2001 From: Affluent Date: Wed, 9 Sep 2026 21:37:16 -0700 Subject: [PATCH] =?UTF-8?q?feat:=20$LogFile=20parser=20+=20tool,=20AFF4=20?= =?UTF-8?q?reader/writer,=20VHD/VHDX=20output,=20any=E2=86=94any=20convers?= =?UTF-8?q?ion?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit $LogFile - NtfsLogFile: restart-page geometry, fixup-protected RCRD pages, records reassembled across page boundaries, LSN↔offset validation that separates live records from circular-log slack (kept, flagged stale), buffer-page duplicates dropped. Names, parent records and $FILE_NAME timestamps recovered from InitializeFileRecordSegment, Add/DeleteIndexEntry (delete's undo side) and Create/DeleteAttribute payloads. Microsoft's operation names. - $LogFile tool in Examine: every NTFS volume in an image, or an extracted file. AFF4 - Aff4Writer: AFF4 Standard v1.0 container — ZIP volume, container.description, version.txt, information.turtle, zlib bevies with standard (uint64 offset, uint32 length) indexes, single-range Map, Image object with MD5/SHA-1/SHA-256 and case metadata. Verified with pyaff4 0.34: opens as a PhysicalImageContainer, reads every byte, all three stored hashes match, metadata present. - Aff4Reader: Image → Map → ImageStream chains with multiple ranges and zero/unreadable targets, standard and pre-standard index layouts, zlib, snappy and LZ4 (both decoders in-tree, checked against python-snappy / lz4.block vectors, bounds-checked against hostile input) and stored chunks, rdflib-style turtle. EvidenceOpener detects AFF4 so every image-taking tool accepts .aff4; Verify tool checks all recorded digests. VHD / VHDX output through DiscUtils' dynamic-disk writers. Imager writes all five formats in-process; the Python imager sidecar is no longer needed for any of them. ImageConverter.ConvertAsync: any source (raw, E01, AFF4, VHD, VHDX) to any target, output re-read and hashed after writing, container sources checked against their recorded hash. Convert tool exposes it. Tests: NtfsLogFileTests (synthetic log with fixups, multi-page record, stale slack, torn page, garbage), Aff4Tests (round trip both compressions, random access across bevies, pyaff4-style container, snappy/LZ4 vectors, imager AFF4, VHD and VHDX read back through DiscUtils). Docs and tool help updated. --- CHANGELOG.md | 30 + LIMITATIONS.md | 8 + ROADMAP.md | 26 +- docs/FEATURES.md | 12 +- .../ViewModels/Tools/BuiltInToolsHelp.cs | 29 +- .../ViewModels/Tools/LogFileTool.cs | 210 +++++ .../ViewModels/Tools/ToolImplementations.cs | 198 ++-- .../ViewModels/Tools/WorkspaceViewModel.cs | 1 + src/Cinder.Filesystems/NtfsLogFile.cs | 542 +++++++++++ src/Cinder.Imaging/Aff4/Aff4Reader.cs | 888 ++++++++++++++++++ src/Cinder.Imaging/Aff4/Aff4Writer.cs | 258 +++++ src/Cinder.Imaging/Cinder.Imaging.csproj | 5 + src/Cinder.Imaging/EvidenceOpener.cs | 30 + src/Cinder.Imaging/InProcessImager.cs | 256 ++++- tests/Cinder.Core.Tests/NtfsLogFileTests.cs | 296 ++++++ tests/Cinder.Imaging.Tests/Aff4Tests.cs | 242 +++++ .../InProcessImagerTests.cs | 8 +- 17 files changed, 2917 insertions(+), 122 deletions(-) create mode 100644 src/Cinder.App/ViewModels/Tools/LogFileTool.cs create mode 100644 src/Cinder.Filesystems/NtfsLogFile.cs create mode 100644 src/Cinder.Imaging/Aff4/Aff4Reader.cs create mode 100644 src/Cinder.Imaging/Aff4/Aff4Writer.cs create mode 100644 tests/Cinder.Core.Tests/NtfsLogFileTests.cs create mode 100644 tests/Cinder.Imaging.Tests/Aff4Tests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 89ea186..334e838 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,36 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 Correctness and evidence-integrity pass, followed by the features a competitive gap analysis showed every established tool has and Cinder lacked. +### Added — $LogFile, AFF4, VHD/VHDX + +- **`$LogFile` parser and tool.** `NtfsLogFile` reads the NTFS transaction + journal: restart-page geometry, fixup-protected `RCRD` pages, records + reassembled across page boundaries, the LSN ↔ offset check that separates live + records from circular-log slack (kept, flagged stale — that slack is often + where a long-deleted name survives), buffer-page duplicates dropped. Names, + parent records and `$FILE_NAME` timestamps are recovered from the payloads + that carry them: file-record initialisation, index-entry add / delete + (delete's undo side), attribute create / delete. Operation names match + Microsoft's. New **$LogFile** tool in Examine reads it from every NTFS volume + in an image or from an extracted file. +- **AFF4 in-process, both ways.** `Aff4Writer` produces AFF4 Standard v1.0 + containers — ZIP volume, `container.description`, `version.txt`, + `information.turtle`, zlib-compressed bevies with standard (offset, length) indexes, a + single-range `Map`, and an `Image` object with MD5 / SHA-1 / SHA-256 and case + metadata — the structure pyaff4 and Evimetry emit, verified readable by pyaff4. + `Aff4Reader` opens what Cinder, pyaff4, libaff4 and Evimetry write for disk + images: standard and pre-standard (Evimetry) index layouts, `Image → Map → ImageStream` chains with + multiple ranges and zero / unreadable targets, zlib, **snappy**, **LZ4** and + stored chunks (both decoders in-tree, checked against the reference + implementations), rdflib-style turtle. `EvidenceOpener` detects AFF4, so every + tool that takes an image takes an `.aff4`; the Verify tool checks all three + recorded digests. +- **VHD / VHDX output** through DiscUtils' managed writers (dynamic disks). +- **Convert tool: any → any** across raw, E01, AFF4, VHD and VHDX, with the + output re-read and hashed after writing and a container source checked + against its recorded hash. +- The Python imager sidecar is no longer needed for any format. + ### Added — EWF writer, trusted timestamps - **E01 acquisition in-process.** `EwfWriter` produces EnCase 6-layout chains: diff --git a/LIMITATIONS.md b/LIMITATIONS.md index d4fe1ae..38ff5ee 100644 --- a/LIMITATIONS.md +++ b/LIMITATIONS.md @@ -116,6 +116,14 @@ fixture-vs-reference diff is in CI. - **Blockers:** PDF/A-specific conformance (embedded font subsetting, XMP metadata, the PDF/A-2b identifier) is not implemented — output is ordinary PDF, not archival PDF/A. +## AFF4 + +Read support covers physical disk images as pyaff4, libaff4 and Evimetry write them: `Image → +Map → ImageStream` chains with zlib / snappy / LZ4 / stored chunks in either index layout. +Logical containers (one object per file), encrypted volumes and striped multi-volume sets are +not read. The writer produces v1.0 containers with a single map range; unreadable sectors are +zero-filled in the stream and listed in the `.log.json` rather than as a separate map target. + ## Chain-of-custody log has no external anchor - **Where:** [`Cinder.Core.Custody.CustodyLog`](src/Cinder.Core/Custody/CustodyLog.cs) diff --git a/ROADMAP.md b/ROADMAP.md index 6e0156f..2d5c354 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -87,17 +87,19 @@ Not in the original plan; landed alongside Phase 1. UI surfaces exist for every tool. Image acquisition / mounting / shadow copies are placeholders until the platform-specific drivers ship. -- ✅ Disk imager (raw, E01) — `InProcessImager`: file / block device / E01 - chain → `.dd` or an EnCase 6 `.E01` chain (`EwfWriter`, libewf-verified) - with hash-on-read, retry + sector-level fallback with bad-sector offsets - logged (and in the E01's error section), `.log.json` companion, custody - entry. Needs Administrator / root for devices. 🟡 AFF4 still via sidecar. -- ✅ Image verify — in-process: E01 against recorded digests, raw against a - companion digest or SHA256SUMS; verified / failed / unverifiable. +- ✅ Disk imager (raw, E01, AFF4, VHD, VHDX) — `InProcessImager`: file / + block device / E01 / AFF4 → any of the five, all in-process. E01 via + `EwfWriter` (libewf-verified), AFF4 via `Aff4Writer` (pyaff4-verified), + VHD/VHDX via DiscUtils. Hash-on-read, retry + sector-level fallback with + bad-sector offsets logged (and in the E01's error section), `.log.json` + companion, custody entry. Needs Administrator / root for devices. +- ✅ Image verify — in-process: E01 and AFF4 against recorded digests, raw + against a companion digest or SHA256SUMS; verified / failed / unverifiable. - 🟡 Mount image — VHD/VHDX/ISO via PowerShell `Mount-DiskImage` on Windows works; E01 mount requires Arsenal Image Mounter (free, external). -- ✅ Convert format — E01 ↔ raw in-process; E01 → raw compared with the - container's recorded hash, raw → E01 re-read and checked after writing. +- ✅ Convert format — raw / E01 / AFF4 / VHD / VHDX in any direction; output + re-read and checked after writing, container sources checked against + their recorded hash. - 🟡 Write-blocker (Windows) — placeholder, real version blocked on a signed kernel driver in `drivers/cinder-wb-windows`. - 🟡 Write-blocker (Linux) — `blockdev --setro` wrapper, works. @@ -114,8 +116,10 @@ copies are placeholders until the platform-specific drivers ship. surfaces name, size, MAC/creation times, MFT index and sequence as `IsDeleted` rows. Names and timestamps only; contents via the carver. - ✅ $UsnJrnl:$J — USN_RECORD_V2/V3 parser + USN journal tool (image or - extracted `$J`); timeline ingest of `$J` from triage folders. $LogFile - still tracked. + extracted `$J`); timeline ingest of `$J` from triage folders. +- ✅ $LogFile — transaction-log parser + tool: records reassembled across + pages, stale slack kept and flagged, names / parents / timestamps + recovered from FILE-record and index-entry payloads. ## Phase 4 — Windows artifacts ✅ (most) / 🟡 (some) diff --git a/docs/FEATURES.md b/docs/FEATURES.md index ec528fc..b5ae164 100644 --- a/docs/FEATURES.md +++ b/docs/FEATURES.md @@ -36,7 +36,7 @@ long tail (PST via pypff, Volatility 3, pytsk for non-DiscUtils filesystems). |---|---|---| | `Cinder.App` | Avalonia shell, every view-model, every tool implementation, app services | ~55% of all code. **Parsing logic lives here**, in `ViewModels/Tools/*.cs`, not in the library projects — the biggest structural debt (see §9). | | `Cinder.Core` | Case store (SQLite + Dapper + migrations), custody log, hash service (MD5/SHA-1/SHA-256/BLAKE3 streaming), signature scanner (60+ magics), encrypted-container heuristic, `ExecutableResolver`, `TabularExporter`, `FeatureExtractor` | The genuinely reusable core. | -| `Cinder.Imaging` | `EwfReader`/`EwfStream` (in-process E01, multi-segment, verification), `EvidenceOpener` (E01-or-raw → `Stream`), mounters (VHD/VHDX/ISO via PowerShell; Linux loop), shadow-copy enumeration, write-blocker wrappers, sidecar imager/verifier | EWF reader is hardened against malformed input; damaged chunks are zero-filled and reported, never silently truncated. | +| `Cinder.Imaging` | `EwfReader`/`EwfStream` (in-process E01, multi-segment, verification), `Aff4Reader`/`Aff4Writer` (AFF4 v1.0/v1.1, zlib/snappy/LZ4), `EwfWriter`, `InProcessImager` (raw/E01/AFF4/VHD/VHDX), `EvidenceOpener` (E01 / AFF4 / raw → `Stream`), mounters (VHD/VHDX/ISO via PowerShell; Linux loop), shadow-copy enumeration, write-blocker wrappers, sidecar imager/verifier | EWF reader is hardened against malformed input; damaged chunks are zero-filled and reported, never silently truncated. | | `Cinder.Carving` | `FileCarver` (header/footer, vectorised, 30+ signatures), `SlackUnallocCarver` | | | `Cinder.Hex` | `HexViewer` control (virtualised, `ILogicalScrollable`), `MmapHexBuffer`, `HexSearch` (streaming find), bookmarks, overlays | | | `Cinder.Search` | Lucene.NET `CaseIndex`, `SuperTimeline` + `TimelineExporter` + `MitreTagger`, `HashSetService` (NSRL, SQLite), `CommunicationGraph`, `GeoPoint` index, `VirusTotalClient`, `YaraScanner` (sidecar stub — real scanning is `Cinder.App/Services/YaraLite`) | | @@ -87,6 +87,7 @@ unkeyed and stored beside the data; SECURITY.md explains. | **Gallery** | works | Image viewer with EXIF panel (MetadataExtractor); GPS → Map | `GalleryTool.cs` | | **Documents** | works | Text extraction for DOCX/DOCM, XLSX/XLSM, PPTX, ODT/ODS/ODP, EPUB, RTF, PDF (PdfPig), HTML/XML, 20+ text/code formats; 50 MB in / 2 MB out caps; XXE closed | `Services/DocumentReader` | | **Filesystem** | works | `DiscUtilsWalker` (in `Cinder.Filesystems`, tested against a checked-in NTFS image on Windows and Linux): NTFS / FAT / ext2-4 / ISO9660, whole-disk images per partition, VHD/VHDX, E01 via `EwfReader`; all timestamps; **deleted-file recovery** from the NTFS $MFT (`IsDeleted = true`); **optional hashing + hash-set verdict** per file (Settings ▸ Hash sets) so `Unknown` in the row filter is the known-good filter; metadata row shows recorded E01 hashes labelled UNVERIFIED | `Cinder.Filesystems/DiscUtilsWalker.cs`, `Phase3To10ParserTools.cs` | +| **$LogFile** | works | `NtfsLogFile` parses the NTFS transaction log from any NTFS volume in an image or an extracted file: fixups, multi-page records, LSN ↔ offset validation, stale slack flagged, names / parent / `$FILE_NAME` timestamps from InitializeFileRecordSegment, Add/DeleteIndexEntry and Create/DeleteAttribute payloads; Microsoft's operation names | `Cinder.Filesystems/NtfsLogFile.cs`, `LogFileTool.cs` | | **USN journal** | works | `UsnJournal` parses `$UsnJrnl:$J` (V2/V3) from any NTFS volume in an image or an extracted `$J`; reason flags in MFTECmd/Plaso naming, MFT + parent references; 200k-row budget with banner. Also ingested into the timeline from triage folders | `Cinder.Filesystems/UsnJournal.cs`, `UsnJournalTool.cs` | | **Registry** | works | Eric Zimmerman `Registry` lib walk of any hive (NTUSER/SYSTEM/SOFTWARE/SAM/Amcache) with transaction-log replay; row and depth budgets with truncation banner | `WindowsParserTools.cs` | | **Event Log** | works | `evtx` lib; every record with time, channel, provider, id, level, user, computer, mapped description | " | @@ -131,10 +132,10 @@ annotation; every run is written to the custody log. | Tool | Status | What it does | |---|---|---| -| **Disk imager** | works (raw, E01) | `InProcessImager`: file / block device / E01 chain → `.dd` or an EnCase 6 `.E01` chain (`EwfWriter`: zlib chunks, segments, metadata header, MD5+SHA-1 inside, `error2` bad-sector section; verified readable by libewf). Hash-on-read, retry then sector-level fallback with bad-sector offsets in `.log.json`, custody entry. Devices need Administrator/root. AFF4 still via the Python sidecar | +| **Disk imager** | works (raw, E01, AFF4, VHD, VHDX) | `InProcessImager`: file / block device / E01 / AFF4 → `.dd`, EnCase 6 `.E01` chain (`EwfWriter`, libewf-verified), AFF4 v1.0 container (`Aff4Writer`, pyaff4-verified), or a dynamic VHD/VHDX (DiscUtils). Hash-on-read, retry then sector-level fallback with bad-sector offsets in `.log.json` (and the E01 error section), custody entry. Devices need Administrator/root. No sidecar for any format | | **Image verify** | works | In-process. E01: `EwfReader.VerifyAsync` re-reads the decoded media and compares to the recorded MD5/SHA-1. Raw: hashes and compares to a `.sha256/.sha1/.md5` companion or a `SHA256SUMS` line. Three distinct outcomes — verified / failed / **unverifiable** — and the result is written to custody with digests | | **Mount image** | partial | VHD/VHDX/ISO via `Mount-DiskImage` (Windows); Linux `losetup`+`mount` read-only; E01 needs Arsenal Image Mounter | -| **Convert format** | works (E01 ↔ raw) | `ImageConverter.EwfToRawAsync` decodes, hashes, writes flat, compares with the recorded acquisition hash; `RawToEwfAsync` writes the chain then re-reads it and checks the digest reproduces. Both directions double as a verification | +| **Convert format** | works (any ↔ any) | `ImageConverter.ConvertAsync`: raw / E01 / AFF4 / VHD / VHDX in any direction; output re-read through its own reader and checked against the digest computed while writing; a container source is checked against its recorded acquisition hash too | | **Shadow copies** | works | `vssadmin` (Windows), btrfs/LVM/ZFS snapshots (Linux) | | **RAM capture** | shell | winpmem / LiME fallbacks when present; no bundled driver | | **File carver** | works | Vectorised header/footer carving, 30+ signatures, validators for JPEG/PNG/PE, exact window ownership (no duplicate hits), slack/unallocated regions | @@ -207,8 +208,9 @@ publish unless the suite passes on both platforms; Windows binary is currently u `Cinder.Filesystems`; the same move for registry/EVTX/etc. is what would make them testable. 3. **Custody attestations need a configured TSA to bind time**; without one the examiner's own key is the only anchor, and publishing the export out of their reach is a process step. -4. **Deleted-file recovery is names only**; contents via the carver. `$LogFile` is not parsed. -5. **Imaging writes raw and E01**; AFF4 output still needs the Python sidecar. +4. **Deleted-file recovery is names only**; contents via the carver. +5. **AFF4 read support covers disk images**, not logical (file-per-object) containers; encrypted + AFF4 is not supported. 6. **IOC matching is folder-scoped**, not against the Lucene index or an open case's parsed grids; hash indicators require exact digests. 7. RAM capture and cloud pull are shells or externally dependent; VirusTotal needs a key. diff --git a/src/Cinder.App/ViewModels/Tools/BuiltInToolsHelp.cs b/src/Cinder.App/ViewModels/Tools/BuiltInToolsHelp.cs index 91c6b37..edd810a 100644 --- a/src/Cinder.App/ViewModels/Tools/BuiltInToolsHelp.cs +++ b/src/Cinder.App/ViewModels/Tools/BuiltInToolsHelp.cs @@ -882,7 +882,13 @@ and the run is recorded in the custody log. header, MD5 + SHA-1 in the container, unreadable sectors in its error section. Verified readable by libewf, which is what FTK Imager, Autopsy, X-Ways and EnCase-compatible tools use. - - **AFF4** is handed to the Python imager sidecar, which must be installed. + - **AFF4** in-process: AFF4 Standard v1.0 ZIP volume, zlib-compressed 32 KiB + chunks in 64 MiB bevies, `Image → Map → ImageStream` objects, MD5 / SHA-1 / + SHA-256 and case metadata in `information.turtle`. Verified readable by + pyaff4. + - **VHD / VHDX** in-process: dynamic disks through DiscUtils' managed + writers, mountable in Hyper-V or Disk Management. These need the source + size up front, so a device that does not report one goes raw or E01. 4. Read errors: Cinder retries, then re-reads the failing block one sector at a time so a bad sector costs 512 zero bytes rather than a megabyte. Every zero-filled sector is counted, and its offset is listed in the log. @@ -910,8 +916,9 @@ defensive sanity check that catches storage corruption. 1. Pick the image file. 2. Cinder reads every block and compares against a reference digest. For an E01 that reference is the MD5 / SHA-1 the acquisition tool wrote inside the - container; for a raw image it's a .sha256 / .sha1 / .md5 companion file, or a - SHA256SUMS line naming the image, sitting next to it. + container; for an AFF4 container it is the MD5 / SHA-1 / SHA-256 recorded in + its `information.turtle`; for a raw image it's a .sha256 / .sha1 / .md5 + companion file, or a SHA256SUMS line naming the image, sitting next to it. ## Reading the result There are three distinct outcomes, and the difference matters. @@ -952,10 +959,10 @@ public sealed partial class ConvertTool { public override string HelpMarkdown => """ ## What this is -Conversion between raw and E01 in both directions, in-process. E01 → raw decodes -every chunk of the EWF container, hashes as it streams, and writes flat. Raw → -E01 chunks, compresses and hashes the source into an EnCase 6 chain, then -re-reads the finished chain and checks it reproduces the digest. +Conversion between forensic image formats — raw, E01, AFF4, VHD, VHDX — in any +direction, in-process. The source is decoded through whichever reader +understands it, hashed as it streams, and written through the target's writer; +the finished output is then re-read and checked against that digest. ## When you'd use it When the tool you need next only understands raw images — a loop-mount, a @@ -980,9 +987,11 @@ the count is in the result and the log. the faithful decode of what was in the E01, but nothing ties it to the media. 4. The conversion is recorded in the custody log with both digests. -## Not yet -AFF4 in either direction; VHD/VHDX output. AFF4 acquisition still goes through -the Python sidecar from the Imager tool. +## Formats +Source: raw, E01 chain, AFF4, VHD, VHDX. Target: any of the same five. Every +target is re-read through its own reader after writing and must hash back to +what was written; a container source is also checked against its recorded +acquisition hash. """; } diff --git a/src/Cinder.App/ViewModels/Tools/LogFileTool.cs b/src/Cinder.App/ViewModels/Tools/LogFileTool.cs new file mode 100644 index 0000000..1c13505 --- /dev/null +++ b/src/Cinder.App/ViewModels/Tools/LogFileTool.cs @@ -0,0 +1,210 @@ +using System.Globalization; +using Cinder.Filesystems; +using Cinder.Imaging; +using DiscUtils; +using DiscUtils.Ntfs; + +namespace Cinder.App.ViewModels.Tools; + +/// +/// NTFS transaction-log viewer. Accepts an image (raw / E01 / VHD / VHDX, single volume or whole +/// disk) and reads $LogFile from each NTFS volume in it, or an extracted $LogFile +/// straight from a triage collection. +/// +public sealed partial class LogFileTool : SidecarToolViewModel +{ + public override string Id => "logfile"; + public override string Title => "$LogFile"; + public override string Icon => "🧾"; + public override string Subtitle => "NTFS transaction log — file records and index entries created, deleted and renamed, with names recovered from the payloads. Image or extracted $LogFile."; + public override string Phase => "3"; + public override string Kind => "logfile"; + + private const int RowBudget = 200_000; + + public override string HelpMarkdown => """ +## What this is +`$LogFile` is NTFS's transaction journal: before the filesystem changes an MFT record +or a directory index it writes what it is about to do (redo) and how to reverse it +(undo). Every file creation, deletion and rename passes through it, and the payloads +carry a full `$FILE_NAME` — so the log still holds a deleted file's name and parent +after the MFT record has been reused and the USN journal has rolled over. + +## When you'd use it +- The USN journal has rolled past the window you care about (it usually covers days; + `$LogFile` is 64 MB by default and covers the last hours to days of metadata churn). +- You need the *order* of operations inside one transaction, or to see that a rename + and a delete were the same transaction. +- To recover names from slack: the log is circular, and pages keep records from + earlier cycles past their current end. Those rows are flagged **stale**. + +## How to use it in Cinder +1. Pick evidence: a disk image (every NTFS volume in it is read) or an extracted + `$LogFile` (KAPE and most triage tools collect it). +2. Each row is one log record: LSN, transaction, redo and undo operation (the names + Microsoft uses — `InitializeFileRecordSegment`, `AddIndexEntryAllocation`, + `DeleteIndexEntryAllocation`, …), the MFT record it targets, and — when the payload + carried one — the file name, its parent record and the timestamps in the name + attribute. +3. Filter on an operation or a name, export, or bookmark a row as a finding. + +## Reading it +- **InitializeFileRecordSegment** with a name: a file record was created (new file, or + a reused record). +- **AddIndexEntryAllocation / Root** with a name: the name was added to its parent + directory — creation or the new side of a rename. +- **DeleteIndexEntryAllocation / Root** with a name: removed from the directory — + deletion or the old side of a rename. The name comes from the undo payload. +- **DeallocateFileRecordSegment**: the MFT record was freed. +- Records in the same **transaction** happened together. + +## Limits +Timestamps are only those inside `$FILE_NAME` payloads (creation and last-modified of +the file at that moment); the log itself does not stamp records. The MFT record number +is derived from the target VCN assuming the volume's cluster size (read from the +volume when an image is opened; 4 KiB assumed for an extracted file). The grid stops at +200,000 rows and says so; export to get everything. +"""; + + protected override async Task LoadAsync(string evidencePath, CancellationToken ct) + { + var (rows, truncated) = await Task.Run(() => Parse(evidencePath, ct), ct); + AddRows(rows, budget: int.MaxValue); + IsTruncated = truncated; + } + + private static (List Rows, bool Truncated) Parse(string path, CancellationToken ct) + { + var rows = new List(); + var truncated = false; + + var name = Path.GetFileName(path); + if (name.Equals("$LogFile", StringComparison.OrdinalIgnoreCase)) + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20); + truncated = Append(NtfsLogFile.Parse(fs, 4096, ct), "", rows, ct); + return (rows, truncated); + } + + var ext = Path.GetExtension(path).ToLowerInvariant(); + using var stream = ext is ".vhd" or ".vhdx" ? null : EvidenceOpener.Open(path); + + if (stream is not null) + { + truncated = WalkVolumes(stream, rows, ct); + } + else + { + using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + using VirtualDisk disk = ext == ".vhd" + ? new DiscUtils.Vhd.Disk(file, DiscUtils.Streams.Ownership.None) + : new DiscUtils.Vhdx.Disk(file, DiscUtils.Streams.Ownership.None); + foreach (var part in disk.Partitions.Partitions) + { + ct.ThrowIfCancellationRequested(); + using var ps = part.Open(); + TryVolume(ps, $"[part {part.FirstSector:N0}]", rows, ct); + if (rows.Count >= RowBudget) { truncated = true; break; } + } + } + + if (rows.Count == 0) + { + rows.Add(new + { + Lsn = 0L, + Transaction = 0u, + Redo = "(no $LogFile found)", + Undo = "", + Mft = "", + FileName = "", + Parent = "", + Created = "", + Modified = "", + Flags = "", + Volume = "", + Note = "No NTFS volume with a $LogFile was found in this evidence.", + }); + } + return (rows, truncated); + } + + private static bool WalkVolumes(Stream image, List rows, CancellationToken ct) + { + if (TryVolume(image, "", rows, ct)) + { + return rows.Count >= RowBudget; + } + image.Position = 0; + var vm = new VolumeManager(); + vm.AddDisk(image); + foreach (var vol in vm.GetLogicalVolumes()) + { + ct.ThrowIfCancellationRequested(); + try + { + using var vs = vol.Open(); + TryVolume(vs, $"[{vol.Identity}]", rows, ct); + } + catch (OperationCanceledException) { throw; } + catch { /* not NTFS or unreadable — try the next partition */ } + if (rows.Count >= RowBudget) return true; + } + return false; + } + +#pragma warning disable CA1416 // read-only NTFS access; see DiscUtilsWalker + private static bool TryVolume(Stream s, string prefix, List rows, CancellationToken ct) + { + try + { + s.Position = 0; + if (!NtfsFileSystem.Detect(s)) return false; + using var fs = new NtfsFileSystem(s); + using var log = NtfsLogFile.Open(fs, out var clusterSize); + if (log is null) + { + return true; + } + Append(NtfsLogFile.Parse(log, clusterSize, ct), prefix, rows, ct); + return true; + } + catch (OperationCanceledException) { throw; } + catch { return false; } + } +#pragma warning restore CA1416 + + private static bool Append(IEnumerable records, string volume, List rows, CancellationToken ct) + { + foreach (var r in records) + { + ct.ThrowIfCancellationRequested(); + if (rows.Count >= RowBudget) + { + return true; + } + var flags = string.Join(' ', new[] + { + r.IsStale ? "stale" : null, + r.IsTruncated ? "truncated" : null, + r.IsCheckpoint ? "checkpoint" : null, + }.Where(f => f is not null)); + rows.Add(new + { + r.Lsn, + Transaction = r.TransactionId, + Redo = r.IsCheckpoint ? "Checkpoint" : r.RedoText, + Undo = r.IsCheckpoint ? "" : r.UndoText, + Mft = r.MftRecordNumber?.ToString(CultureInfo.InvariantCulture) ?? "", + FileName = r.FileName ?? "", + Parent = r.ParentMftRecord?.ToString(CultureInfo.InvariantCulture) ?? "", + Created = r.Created?.ToString("u", CultureInfo.InvariantCulture) ?? "", + Modified = r.Modified?.ToString("u", CultureInfo.InvariantCulture) ?? "", + Flags = flags, + Volume = volume, + Note = $"vcn {r.TargetVcn} blk {r.ClusterBlockOffset} attr 0x{r.TargetAttribute:X} prev {r.PreviousLsn}", + }); + } + return false; + } +} diff --git a/src/Cinder.App/ViewModels/Tools/ToolImplementations.cs b/src/Cinder.App/ViewModels/Tools/ToolImplementations.cs index 801b59c..00d8f54 100644 --- a/src/Cinder.App/ViewModels/Tools/ToolImplementations.cs +++ b/src/Cinder.App/ViewModels/Tools/ToolImplementations.cs @@ -1422,6 +1422,10 @@ private async Task PickAndRunAsync(CancellationToken ct) { await VerifyEwfAsync(path, ct); } + else if (EvidenceOpener.IsAff4(path)) + { + await VerifyAff4Async(path, ct); + } else { await VerifyRawAsync(path, ct); @@ -1478,6 +1482,75 @@ private async Task VerifyEwfAsync(string path, CancellationToken ct) }, ct); } + /// + /// AFF4 records MD5 / SHA-1 / SHA-256 as aff4:hash literals on the image. Re-read + /// the whole map and compare every digest that was recorded. + /// + private async Task VerifyAff4Async(string path, CancellationToken ct) + { + using var reader = Cinder.Imaging.Aff4.Aff4Reader.Open(path); + await using var stream = reader.OpenStream(); + using var md5 = IncrementalHash.CreateHash(HashAlgorithmName.MD5); + using var sha1 = IncrementalHash.CreateHash(HashAlgorithmName.SHA1); + using var sha256 = IncrementalHash.CreateHash(HashAlgorithmName.SHA256); + var buffer = new byte[4 * 1024 * 1024]; + long total = 0; + while (true) + { + ct.ThrowIfCancellationRequested(); + var n = await stream.ReadAsync(buffer, ct); + if (n == 0) break; + md5.AppendData(buffer, 0, n); + sha1.AppendData(buffer, 0, n); + sha256.AppendData(buffer, 0, n); + total += n; + StatusLine = $"Verifying — {total:N0} bytes read…"; + } + var cMd5 = Convert.ToHexStringLower(md5.GetHashAndReset()); + var cSha1 = Convert.ToHexStringLower(sha1.GetHashAndReset()); + var cSha256 = Convert.ToHexStringLower(sha256.GetHashAndReset()); + + bool? md5Ok = reader.RecordedMd5 is null ? null : reader.RecordedMd5 == cMd5; + bool? sha1Ok = reader.RecordedSha1 is null ? null : reader.RecordedSha1 == cSha1; + bool? sha256Ok = reader.RecordedSha256 is null ? null : reader.RecordedSha256 == cSha256; + var any = md5Ok is not null || sha1Ok is not null || sha256Ok is not null; + var verified = any && md5Ok != false && sha1Ok != false && sha256Ok != false && reader.DamagedChunks.Count == 0; + + ResultText = $""" + Source : AFF4 container, image {reader.ImageUrn} + Bytes verified : {total:N0} + Damaged chunks : {reader.DamagedChunks.Count:N0} + + Recorded MD5 : {reader.RecordedMd5 ?? "— none recorded —"} + Computed MD5 : {cMd5} + Recorded SHA-1 : {reader.RecordedSha1 ?? "— none recorded —"} + Computed SHA-1 : {cSha1} + Recorded SHA-256 : {reader.RecordedSha256 ?? "— none recorded —"} + Computed SHA-256 : {cSha256} + + {(!any ? "UNVERIFIABLE: the container records no hash." : verified ? "VERIFIED: every recorded digest reproduces." : "FAILED: a recorded digest does not reproduce, or chunks were damaged.")} + """; + StatusLine = !any + ? "⚠ unverifiable — container records no acquisition hash" + : verified ? "✓ verified against the acquisition hash" : "✗ VERIFICATION FAILED"; + + await ActiveCaseContext.LogAsync(CustodyAction.EvidenceHashed, new + { + Image = path, + Container = "aff4", + reader.ImageUrn, + BytesVerified = total, + reader.RecordedMd5, + ComputedMd5 = cMd5, + reader.RecordedSha1, + ComputedSha1 = cSha1, + reader.RecordedSha256, + ComputedSha256 = cSha256, + DamagedChunks = reader.DamagedChunks.Count, + Verdict = !any ? "unverifiable" : verified ? "verified" : "failed", + }, ct); + } + private async Task VerifyRawAsync(string path, CancellationToken ct) { var hashes = new HashService(); @@ -1634,32 +1707,41 @@ public sealed partial class ConvertTool [ObservableProperty] private string _format = "raw"; [ObservableProperty] private string? _statusLine; - /// Both directions are in-process: E01 → raw through the reader, raw → E01 through . - public IReadOnlyList Formats { get; } = ["raw", "E01"]; + /// Every target is written in-process; the source can be raw, E01, AFF4, VHD or VHDX. + public IReadOnlyList Formats { get; } = ["raw", "E01", "AFF4", "VHD", "VHDX"]; [ObservableProperty] private long _bytesConverted; + private ImageFormat TargetFormat => Format switch + { + "E01" => ImageFormat.Ewf, + "AFF4" => ImageFormat.Aff4, + "VHD" => ImageFormat.Vhd, + "VHDX" => ImageFormat.Vhdx, + _ => ImageFormat.Raw, + }; + [RelayCommand] private async Task PickSourceAsync(CancellationToken ct) { - var p = await ToolDialog.PickFileAsync("Source image (.E01 chain or raw)"); + var p = await ToolDialog.PickFileAsync("Source image (raw, .E01 chain, .aff4, .vhd, .vhdx)"); if (!string.IsNullOrEmpty(p)) Source = p; } [RelayCommand] private async Task PickOutputAsync(CancellationToken ct) { - var ewf = Format == "E01"; - var p = await ToolDialog.SaveFileAsync("Output", ewf ? "out.E01" : "out.dd", ewf ? "E01" : "dd"); + var ext = TargetFormat.DefaultExtension(); + var p = await ToolDialog.SaveFileAsync("Output", "out" + ext, ext.TrimStart('.')); if (!string.IsNullOrEmpty(p)) Output = p; } /// - /// E01 → raw: every chunk decoded through , hashed - /// as it streams, written flat, and compared with the digests the container recorded. - /// Raw → E01: chunked, compressed and hashed through , - /// then the finished chain is re-read and must reproduce the digest. Either way the - /// conversion doubles as a verification. + /// Reads the source through whichever reader understands it, writes the target through + /// (hashing as it streams), then re-reads the finished output + /// and checks it hashes back to the same digest. When the source was a container with a + /// recorded acquisition hash, that is compared too — so the conversion is also a + /// verification of both ends. /// [RelayCommand] private async Task RunAsync(CancellationToken ct) @@ -1670,79 +1752,41 @@ private async Task RunAsync(CancellationToken ct) return; } BytesConverted = 0; - StatusLine = "Converting…"; + StatusLine = $"Converting to {Format} — hashing as it streams, then re-reading the output to verify…"; try { var progress = new Progress(p => BytesConverted = p.BytesRead); - if (Format == "E01") + var r = await ImageConverter.ConvertAsync(Source, Output, TargetFormat, Environment.UserName, progress: progress, ct: ct); + + var outputVerdict = r.MatchesRecorded switch { - if (EvidenceOpener.IsEwf(Source)) - { - StatusLine = "The source is already an EWF container; pick raw as the output format to unpack it."; - return; - } - StatusLine = "Converting to E01 — compressing, hashing, then re-reading the chain to verify…"; - var r = await ImageConverter.RawToEwfAsync(Source, Output, Environment.UserName, progress: progress, ct: ct); - var verdict = r.MatchesRecorded == true - ? "✓ chain re-read and hashes back to the recorded digest" - : $"✗ re-read of the written chain does not reproduce the digest ({r.DamagedChunks} damaged chunk(s)) — do not use it"; - StatusLine = $"Done. {r.Image.BytesWritten:N0} bytes → {Output} · MD5 {r.Image.Md5} · SHA-1 {r.Image.Sha1} · {verdict}"; - await ActiveCaseContext.LogAsync(CustodyAction.EvidenceImaged, new - { - Operation = "convert-raw-to-ewf", - Source, - Output, - r.Image.BytesWritten, - r.Image.Md5, - r.Image.Sha1, - r.Image.Sha256, - r.Image.BadSectors, - Verified = r.MatchesRecorded, - }, ct); - } - else if (EvidenceOpener.IsEwf(Source)) + true => "✓ output re-read and hashes back to what was written", + false => $"✗ output re-read does NOT reproduce the digest ({r.DamagedChunks} damaged chunk(s)) — do not use it", + null => "⚠ output could not be re-read for verification", + }; + var sourceVerdict = r.SourceMatchesRecorded switch { - var r = await ImageConverter.EwfToRawAsync(Source, Output, Environment.UserName, progress, ct); - var verdict = r.MatchesRecorded switch - { - true => "✓ output hash matches the recorded acquisition hash", - false when r.DamagedChunks > 0 => $"✗ {r.DamagedChunks:N0} damaged chunk(s) zero-filled — output does not match the acquisition", - false => "✗ output hash does NOT match the recorded acquisition hash", - null => "⚠ container recorded no hash — nothing to compare against", - }; - StatusLine = $"Done. {r.Image.BytesWritten:N0} bytes → {Output} · MD5 {r.Image.Md5} · SHA-256 {r.Image.Sha256} · {verdict}"; - await ActiveCaseContext.LogAsync(CustodyAction.EvidenceImaged, new - { - Operation = "convert-ewf-to-raw", - Source, - Output, - r.Image.BytesWritten, - r.Image.Md5, - r.Image.Sha1, - r.Image.Sha256, - r.RecordedMd5, - r.RecordedSha1, - r.DamagedChunks, - r.MatchesRecorded, - }, ct); - } - else + true => "; source matched its recorded acquisition hash", + false => "; ✗ source did NOT match its recorded acquisition hash", + null => r.RecordedMd5 is null ? "" : "", + }; + StatusLine = $"Done. {r.Image.BytesWritten:N0} bytes → {Output} · MD5 {r.Image.Md5} · SHA-256 {r.Image.Sha256} · {outputVerdict}{sourceVerdict}"; + await ActiveCaseContext.LogAsync(CustodyAction.EvidenceImaged, new { - var job = new ImageJob(Source, Output, ImageFormat.Raw, ExaminerName: Environment.UserName, Description: "Raw copy"); - var r = await new InProcessImager().ImageAsync(job, progress, ct); - StatusLine = $"Done. {r.BytesWritten:N0} bytes → {Output} · SHA-256 {r.Sha256} · {r.BadSectors} bad sector(s)"; - await ActiveCaseContext.LogAsync(CustodyAction.EvidenceImaged, new - { - Operation = "copy-raw", - Source, - Output, - r.BytesWritten, - r.Md5, - r.Sha1, - r.Sha256, - r.BadSectors, - }, ct); - } + Operation = "convert", + Source, + Output, + Format = TargetFormat.ToString(), + r.Image.BytesWritten, + r.Image.Md5, + r.Image.Sha1, + r.Image.Sha256, + r.Image.BadSectors, + OutputVerified = r.MatchesRecorded, + SourceRecordedMd5 = r.RecordedMd5, + SourceRecordedSha1 = r.RecordedSha1, + SourceVerified = r.SourceMatchesRecorded, + }, ct); } catch (OperationCanceledException) { diff --git a/src/Cinder.App/ViewModels/Tools/WorkspaceViewModel.cs b/src/Cinder.App/ViewModels/Tools/WorkspaceViewModel.cs index f98a7a6..94bd12b 100644 --- a/src/Cinder.App/ViewModels/Tools/WorkspaceViewModel.cs +++ b/src/Cinder.App/ViewModels/Tools/WorkspaceViewModel.cs @@ -44,6 +44,7 @@ public WorkspaceViewModel() new DocumentsTool(), new FilesystemTool(), new UsnJournalTool(), + new LogFileTool(), new RegistryTool(), new EventLogTool(), new PrefetchTool(), diff --git a/src/Cinder.Filesystems/NtfsLogFile.cs b/src/Cinder.Filesystems/NtfsLogFile.cs new file mode 100644 index 0000000..c220cc3 --- /dev/null +++ b/src/Cinder.Filesystems/NtfsLogFile.cs @@ -0,0 +1,542 @@ +using System.Buffers.Binary; +using System.Text; +using DiscUtils.Ntfs; + +namespace Cinder.Filesystems; + +/// One NTFS transaction-log record: an operation against an MFT record or index. +public sealed record LogFileRecord( + long Lsn, + long PreviousLsn, + long UndoNextLsn, + uint TransactionId, + int RecordType, + ushort RedoOp, + ushort UndoOp, + long TargetVcn, + long? TargetLcn, + ushort ClusterBlockOffset, + ushort RecordOffset, + ushort AttributeOffset, + ushort TargetAttribute, + long? MftRecordNumber, + string? FileName, + long? ParentMftRecord, + DateTimeOffset? Created, + DateTimeOffset? Modified, + bool IsStale, + bool IsTruncated, + long FileOffset) +{ + public string RedoText => NtfsLogFile.DescribeOperation(RedoOp); + public string UndoText => NtfsLogFile.DescribeOperation(UndoOp); + public bool IsCheckpoint => RecordType == 2; +} + +/// +/// Parser for $LogFile, the NTFS transaction journal: every metadata change the +/// filesystem made — file records initialised and deallocated, index entries added and +/// removed, attributes created, resized and deleted — as redo / undo pairs grouped by +/// transaction. It is the third NTFS journal after $MFT and $UsnJrnl, and the +/// one that still holds a deleted file's name and parent after both of those have moved on, +/// because the index-entry and file-record payloads carry a full $FILE_NAME. +/// +/// Layout: two restart pages (RSTR), two buffer pages that mirror the most +/// recently written pages, then a circular run of 4 KiB record pages (RCRD). Every +/// page has update-sequence fixups. Records are 48-byte headers followed by client data; +/// a record can continue across page boundaries. Bytes past a page's +/// next_record_offset are slack from earlier cycles of the circular log — parsed +/// too, flagged , because that slack is often the only +/// place a long-deleted name survives. +/// +/// Names are recovered opportunistically from the payloads that carry a +/// $FILE_NAME structure: file-record initialisation, index-entry add / delete, and +/// attribute create / delete. Everything is bounds-checked; the input is evidence and +/// may be corrupt or hostile. +/// +public static class NtfsLogFile +{ + private const int DefaultPageSize = 4096; + private const int RecordHeaderLength = 0x30; + private const int MaxClientDataLength = 1 << 20; + + /// Opens $LogFile on an NTFS volume (null if absent) and reports the volume's cluster size. +#pragma warning disable CA1416 // read-only; see DiscUtilsWalker for the rationale + public static Stream? Open(NtfsFileSystem fs, out int clusterSize) + { + ArgumentNullException.ThrowIfNull(fs); + clusterSize = (int)fs.ClusterSize; + try + { + return fs.OpenFile("$LogFile", FileMode.Open, FileAccess.Read); + } + catch (FileNotFoundException) { return null; } + catch (DirectoryNotFoundException) { return null; } + catch (IOException) { return null; } + } +#pragma warning restore CA1416 + + /// + /// Parses every record page. is only used to turn a + /// target VCN + cluster-block offset into an MFT record number for operations that + /// address the MFT; 4096 is right for almost every volume. + /// + public static IEnumerable Parse(Stream logFile, int clusterSize = 4096, CancellationToken ct = default) + { + ArgumentNullException.ThrowIfNull(logFile); + if (!logFile.CanSeek) + { + throw new ArgumentException("$LogFile parsing needs a seekable stream.", nameof(logFile)); + } + + var length = logFile.Length; + var pageSize = DefaultPageSize; + var dataOffset = 0x40; + var seqBits = 0; + + // Restart page: page size, log-page data offset, sequence-number bits. + var first = ReadPage(logFile, 0, pageSize); + if (first is not null && first.AsSpan(0, 4).SequenceEqual("RSTR"u8)) + { + var logPageSize = BinaryPrimitives.ReadUInt32LittleEndian(first.AsSpan(0x14)); + if (logPageSize is >= 512 and <= 65536 && (logPageSize & (logPageSize - 1)) == 0) + { + pageSize = (int)logPageSize; + } + var raOffset = BinaryPrimitives.ReadUInt16LittleEndian(first.AsSpan(0x18)); + if (raOffset + 0x30 <= first.Length) + { + seqBits = (int)BinaryPrimitives.ReadUInt32LittleEndian(first.AsSpan(raOffset + 0x0C)); + var dOff = BinaryPrimitives.ReadUInt16LittleEndian(first.AsSpan(raOffset + 0x1A)); + if (dOff >= 0x28 && dOff < pageSize) + { + dataOffset = dOff; + } + } + } + + var seen = new HashSet(); + var pageCount = length / pageSize; + var page = new byte[pageSize]; + var payloadPerPage = pageSize - dataOffset; + + for (long p = 2; p < pageCount; p++) + { + ct.ThrowIfCancellationRequested(); + if (!TryReadPage(logFile, p * pageSize, page)) + { + yield break; + } + if (!page.AsSpan(0, 4).SequenceEqual("RCRD"u8)) + { + continue; + } + ApplyFixups(page); + var nextRecordOffset = NextRecordOffset(page, dataOffset, pageSize); + + var pos = dataOffset; + while (pos + RecordHeaderLength <= pageSize) + { + ct.ThrowIfCancellationRequested(); + var stale = pos >= nextRecordOffset; + var lsn = BinaryPrimitives.ReadInt64LittleEndian(page.AsSpan(pos)); + if (lsn <= 0) + { + break; + } + var dataLen = (int)Math.Min(int.MaxValue, BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(pos + 0x18))); + var recordType = BinaryPrimitives.ReadUInt32LittleEndian(page.AsSpan(pos + 0x20)); + if (dataLen > MaxClientDataLength || recordType is not (1 or 2)) + { + break; // not a record — the rest of the page is noise + } + + // A record's LSN encodes its own file offset; a mismatch means we are reading + // slack that has been partially overwritten — still worth a row, but one whose + // header we no longer trust enough to follow across pages. + var recordFileOffset = p * pageSize + pos; + if (seqBits > 0 && seqBits < 64 && (long)(((ulong)lsn << seqBits) >> seqBits) * 8 != recordFileOffset) + { + stale = true; + } + + var header = page.AsSpan(pos, RecordHeaderLength).ToArray(); + var available = pageSize - (pos + RecordHeaderLength); + + if (dataLen <= available) + { + var data = page.AsSpan(pos + RecordHeaderLength, dataLen).ToArray(); + if (seen.Add(lsn)) + { + yield return Build(header, data, lsn, recordType, clusterSize, stale, truncated: false, recordFileOffset); + } + pos = Align8(pos + RecordHeaderLength + dataLen); + continue; + } + + if (stale) + { + // Slack that runs off the page: what follows belongs to a newer cycle. + if (seen.Add(lsn)) + { + yield return Build(header, page.AsSpan(pos + RecordHeaderLength, available).ToArray(), lsn, recordType, clusterSize, stale, truncated: true, recordFileOffset); + } + break; + } + + // Live record continuing into the following page(s): the tail of the client + // data sits at each next page's data offset. + var full = new byte[dataLen]; + page.AsSpan(pos + RecordHeaderLength, available).CopyTo(full); + var filled = available; + var q = p + 1; + var lastTake = 0; + var cont = new byte[pageSize]; + while (filled < dataLen && q < pageCount) + { + if (!TryReadPage(logFile, q * pageSize, cont) || !cont.AsSpan(0, 4).SequenceEqual("RCRD"u8)) + { + break; + } + ApplyFixups(cont); + lastTake = Math.Min(payloadPerPage, dataLen - filled); + cont.AsSpan(dataOffset, lastTake).CopyTo(full.AsSpan(filled)); + filled += lastTake; + q++; + } + + if (filled < dataLen) + { + if (seen.Add(lsn)) + { + yield return Build(header, full[..filled], lsn, recordType, clusterSize, stale, truncated: true, recordFileOffset); + } + break; + } + + if (seen.Add(lsn)) + { + yield return Build(header, full, lsn, recordType, clusterSize, stale, truncated: false, recordFileOffset); + } + + // Resume inside the last continuation page, after this record's tail. + p = q - 1; + cont.CopyTo(page, 0); + nextRecordOffset = NextRecordOffset(page, dataOffset, pageSize); + pos = lastTake == payloadPerPage ? pageSize : Align8(dataOffset + lastTake); + } + } + } + + private static int NextRecordOffset(byte[] page, int dataOffset, int pageSize) + { + int next = BinaryPrimitives.ReadUInt16LittleEndian(page.AsSpan(0x18)); + return next < dataOffset || next > pageSize ? pageSize : next; + } + + private static int Align8(int v) => (v + 7) & ~7; + + private static byte[]? ReadPage(Stream s, long offset, int size) + { + var b = new byte[size]; + return TryReadPage(s, offset, b) ? b : null; + } + + private static bool TryReadPage(Stream s, long offset, byte[] page) + { + if (offset + page.Length > s.Length) + { + return false; + } + s.Position = offset; + var filled = 0; + while (filled < page.Length) + { + var n = s.Read(page, filled, page.Length - filled); + if (n <= 0) + { + return false; + } + filled += n; + } + return true; + } + + /// Undoes the update-sequence protection: the last two bytes of every 512-byte sector hold the USN; the originals live in the array. + internal static bool ApplyFixups(Span page) + { + var usaOfs = BinaryPrimitives.ReadUInt16LittleEndian(page[4..]); + var usaCount = BinaryPrimitives.ReadUInt16LittleEndian(page[6..]); + if (usaCount < 2 || usaOfs + usaCount * 2 > page.Length || (usaCount - 1) * 512 > page.Length) + { + return false; + } + var usn = page.Slice(usaOfs, 2); + for (int i = 1; i < usaCount; i++) + { + var sectorEnd = i * 512 - 2; + if (!page.Slice(sectorEnd, 2).SequenceEqual(usn)) + { + return false; // torn write or not a real page — leave it + } + page.Slice(usaOfs + i * 2, 2).CopyTo(page.Slice(sectorEnd, 2)); + } + return true; + } + + private static LogFileRecord Build(byte[] header, byte[] data, long lsn, uint recordType, int clusterSize, + bool stale, bool truncated, long fileOffset) + { + var prevLsn = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(0x08)); + var undoNext = BinaryPrimitives.ReadInt64LittleEndian(header.AsSpan(0x10)); + var txn = BinaryPrimitives.ReadUInt32LittleEndian(header.AsSpan(0x24)); + + if (recordType == 2 || data.Length < 0x20) + { + return new LogFileRecord(lsn, prevLsn, undoNext, txn, (int)recordType, 0, 0, 0, null, 0, 0, 0, 0, + null, null, null, null, null, stale, truncated, fileOffset); + } + + var d = data.AsSpan(); + var redoOp = BinaryPrimitives.ReadUInt16LittleEndian(d); + var undoOp = BinaryPrimitives.ReadUInt16LittleEndian(d[2..]); + var redoOff = BinaryPrimitives.ReadUInt16LittleEndian(d[4..]); + var redoLen = BinaryPrimitives.ReadUInt16LittleEndian(d[6..]); + var undoOff = BinaryPrimitives.ReadUInt16LittleEndian(d[8..]); + var undoLen = BinaryPrimitives.ReadUInt16LittleEndian(d[10..]); + var targetAttr = BinaryPrimitives.ReadUInt16LittleEndian(d[12..]); + var lcnsToFollow = BinaryPrimitives.ReadUInt16LittleEndian(d[14..]); + var recordOffset = BinaryPrimitives.ReadUInt16LittleEndian(d[16..]); + var attrOffset = BinaryPrimitives.ReadUInt16LittleEndian(d[18..]); + var clusterBlock = BinaryPrimitives.ReadUInt16LittleEndian(d[20..]); + var targetVcn = BinaryPrimitives.ReadInt64LittleEndian(d[24..]); + long? targetLcn = lcnsToFollow > 0 && d.Length >= 0x28 ? BinaryPrimitives.ReadInt64LittleEndian(d[32..]) : null; + + long? mftRecord = null; + if (IsMftOperation(redoOp) || IsMftOperation(undoOp)) + { + // MFT record = (VCN × cluster + block × 512) / 1024. Correct whenever the target + // is the MFT, which these operations always address. + var bytes = targetVcn * clusterSize + (long)clusterBlock * 512; + if (bytes >= 0) + { + mftRecord = bytes / 1024; + } + } + + string? name = null; + long? parent = null; + DateTimeOffset? created = null, modified = null; + + var redo = Slice(d, redoOff, redoLen); + var undo = Slice(d, undoOff, undoLen); + switch (redoOp) + { + case 0x02: // InitializeFileRecordSegment — payload is a FILE record + (name, parent, created, modified, var recNo) = FromFileRecord(redo); + if (recNo is not null) mftRecord = recNo; + break; + case 0x0C: // AddIndexEntryRoot + case 0x0E: // AddIndexEntryAllocation + (name, parent, created, modified) = FromIndexEntry(redo, out var childRef); + mftRecord ??= childRef; + break; + case 0x05: // CreateAttribute + (name, parent, created, modified) = FromAttribute(redo); + break; + } + if (name is null) + { + switch (undoOp) + { + case 0x0C: + case 0x0E: // undo of a delete is an add: the entry being removed + (name, parent, created, modified) = FromIndexEntry(undo, out var childRef); + mftRecord ??= childRef; + break; + case 0x05: // undo of DeleteAttribute is CreateAttribute: the attribute removed + (name, parent, created, modified) = FromAttribute(undo); + break; + } + } + if (name is null && (redoOp is 0x0D or 0x0F) && undo.Length == 0 && redo.Length >= 0x52) + { + // Some writers put the deleted entry in the redo payload. + (name, parent, created, modified) = FromIndexEntry(redo, out _); + } + + return new LogFileRecord(lsn, prevLsn, undoNext, txn, (int)recordType, redoOp, undoOp, targetVcn, targetLcn, + clusterBlock, recordOffset, attrOffset, targetAttr, mftRecord, name, parent, created, modified, stale, truncated, fileOffset); + } + + private static ReadOnlySpan Slice(ReadOnlySpan d, int off, int len) => + off >= 0 && len > 0 && off + len <= d.Length ? d.Slice(off, len) : ReadOnlySpan.Empty; + + private static bool IsMftOperation(ushort op) => op is 0x02 or 0x03 or 0x04 or 0x05 or 0x06 or 0x07 or 0x09 or 0x0B or 0x0C or 0x0D or 0x11 or 0x13 or 0x21 or 0x23 or 0x25; + + // ---- payload decoders ---------------------------------------------------------------- + + /// A FILE record: walk its attributes for the first $FILE_NAME. + private static (string? Name, long? Parent, DateTimeOffset? Created, DateTimeOffset? Modified, long? RecordNumber) FromFileRecord(ReadOnlySpan r) + { + if (r.Length < 0x30 || !r[..4].SequenceEqual("FILE"u8)) + { + return (null, null, null, null, null); + } + long? recNo = BinaryPrimitives.ReadUInt32LittleEndian(r[0x2C..]); + var attrOff = BinaryPrimitives.ReadUInt16LittleEndian(r[0x14..]); + var pos = (int)attrOff; + for (int guard = 0; guard < 64 && pos + 0x18 <= r.Length; guard++) + { + var type = BinaryPrimitives.ReadUInt32LittleEndian(r[pos..]); + if (type == 0xFFFFFFFF) + { + break; + } + var len = BinaryPrimitives.ReadUInt32LittleEndian(r[(pos + 4)..]); + if (len < 0x18 || pos + len > r.Length) + { + break; + } + var nonResident = r[pos + 8] != 0; + if (type == 0x30 && !nonResident) + { + var valueLen = BinaryPrimitives.ReadUInt32LittleEndian(r[(pos + 0x10)..]); + var valueOff = BinaryPrimitives.ReadUInt16LittleEndian(r[(pos + 0x14)..]); + if (valueOff + valueLen <= len) + { + var (n, p, c, m) = FromFileNameValue(r.Slice(pos + valueOff, (int)valueLen)); + if (n is not null) + { + return (n, p, c, m, recNo); + } + } + } + pos += (int)len; + } + return (null, null, null, null, recNo); + } + + /// An index entry: file reference, lengths, then a $FILE_NAME key. + private static (string? Name, long? Parent, DateTimeOffset? Created, DateTimeOffset? Modified) FromIndexEntry(ReadOnlySpan e, out long? childRecord) + { + childRecord = null; + if (e.Length < 0x10 + 0x42) + { + return (null, null, null, null); + } + var fileRef = BinaryPrimitives.ReadUInt64LittleEndian(e); + var keyLen = BinaryPrimitives.ReadUInt16LittleEndian(e[10..]); + if (keyLen < 0x42 || 0x10 + keyLen > e.Length) + { + return (null, null, null, null); + } + childRecord = (long)(fileRef & 0x0000FFFFFFFFFFFF); + return FromFileNameValue(e.Slice(0x10, keyLen)); + } + + /// An attribute record; only $FILE_NAME (0x30) carries a name. + private static (string? Name, long? Parent, DateTimeOffset? Created, DateTimeOffset? Modified) FromAttribute(ReadOnlySpan a) + { + if (a.Length < 0x18) + { + return (null, null, null, null); + } + var type = BinaryPrimitives.ReadUInt32LittleEndian(a); + if (type != 0x30 || a[8] != 0) + { + return (null, null, null, null); + } + var valueLen = BinaryPrimitives.ReadUInt32LittleEndian(a[0x10..]); + var valueOff = BinaryPrimitives.ReadUInt16LittleEndian(a[0x14..]); + if (valueOff + valueLen > a.Length) + { + return (null, null, null, null); + } + return FromFileNameValue(a.Slice(valueOff, (int)valueLen)); + } + + private static (string? Name, long? Parent, DateTimeOffset? Created, DateTimeOffset? Modified) FromFileNameValue(ReadOnlySpan v) + { + if (v.Length < 0x42) + { + return (null, null, null, null); + } + var parentRef = BinaryPrimitives.ReadUInt64LittleEndian(v); + var created = FileTime(BinaryPrimitives.ReadInt64LittleEndian(v[8..])); + var modified = FileTime(BinaryPrimitives.ReadInt64LittleEndian(v[16..])); + int nameLen = v[0x40]; + if (nameLen == 0 || 0x42 + nameLen * 2 > v.Length) + { + return (null, null, null, null); + } + var name = Encoding.Unicode.GetString(v.Slice(0x42, nameLen * 2)); + foreach (var ch in name) + { + if (ch == '\0' || (ch < 0x20 && ch != '\t')) + { + return (null, null, null, null); // not text — a payload that only looked like $FILE_NAME + } + } + return (name, (long)(parentRef & 0x0000FFFFFFFFFFFF), created, modified); + } + + private static DateTimeOffset? FileTime(long ft) + { + if (ft <= 0 || ft > 0x7FFF_FFFF_FFFF_FFFF - 1) + { + return null; + } + try + { + var t = DateTimeOffset.FromFileTime(ft); + return t.Year is >= 1980 and <= 2200 ? t : null; + } + catch (ArgumentOutOfRangeException) { return null; } + } + + // ---- operation names -------------------------------------------------------------------- + + public static string DescribeOperation(ushort op) => op switch + { + 0x00 => "Noop", + 0x01 => "CompensationLogRecord", + 0x02 => "InitializeFileRecordSegment", + 0x03 => "DeallocateFileRecordSegment", + 0x04 => "WriteEndOfFileRecordSegment", + 0x05 => "CreateAttribute", + 0x06 => "DeleteAttribute", + 0x07 => "UpdateResidentValue", + 0x08 => "UpdateNonresidentValue", + 0x09 => "UpdateMappingPairs", + 0x0A => "DeleteDirtyClusters", + 0x0B => "SetNewAttributeSizes", + 0x0C => "AddIndexEntryRoot", + 0x0D => "DeleteIndexEntryRoot", + 0x0E => "AddIndexEntryAllocation", + 0x0F => "DeleteIndexEntryAllocation", + 0x10 => "WriteEndOfIndexBuffer", + 0x11 => "SetIndexEntryVcnRoot", + 0x12 => "SetIndexEntryVcnAllocation", + 0x13 => "UpdateFileNameRoot", + 0x14 => "UpdateFileNameAllocation", + 0x15 => "SetBitsInNonresidentBitMap", + 0x16 => "ClearBitsInNonresidentBitMap", + 0x17 => "HotFix", + 0x18 => "EndTopLevelAction", + 0x19 => "PrepareTransaction", + 0x1A => "CommitTransaction", + 0x1B => "ForgetTransaction", + 0x1C => "OpenNonresidentAttribute", + 0x1D => "OpenAttributeTableDump", + 0x1E => "AttributeNamesDump", + 0x1F => "DirtyPageTableDump", + 0x20 => "TransactionTableDump", + 0x21 => "UpdateRecordDataRoot", + 0x22 => "UpdateRecordDataAllocation", + 0x23 => "UpdateRelativeDataIndex", + 0x24 => "UpdateRelativeDataAllocation", + 0x25 => "ZeroEndOfFileRecord", + _ => $"0x{op:X2}", + }; +} diff --git a/src/Cinder.Imaging/Aff4/Aff4Reader.cs b/src/Cinder.Imaging/Aff4/Aff4Reader.cs new file mode 100644 index 0000000..c6b1f17 --- /dev/null +++ b/src/Cinder.Imaging/Aff4/Aff4Reader.cs @@ -0,0 +1,888 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; +using System.Text.RegularExpressions; + +namespace Cinder.Imaging.Aff4; + +/// +/// Reads AFF4 physical-image containers: the ZIP volume, its information.turtle, the +/// Image → Map → ImageStream object chain, and bevies in the standard (uint64 offset + +/// uint32 length) or pre-standard (uint32 offset list) index layout with zlib, snappy, LZ4 or +/// stored chunks. Covers what +/// Cinder, pyaff4, libaff4 and Evimetry write for disk images; logical (file-per-object) +/// containers are not in scope. +/// +/// The turtle parser is deliberately small — subjects, predicate lists, IRIs, prefixed +/// names, literals with datatypes — because that is all these files contain, and a full RDF +/// stack would be a large dependency for it. Everything read from the container is treated +/// as untrusted: sizes are bounded, offsets are range-checked, and a chunk that fails to +/// decode is zero-filled and recorded in rather than aborting. +/// +public sealed class Aff4Reader : IDisposable +{ + private const int MaxChunkSize = 1 << 24; + private const long MaxBevyBytes = 1L << 31; + + private readonly ZipArchive _zip; + private readonly FileStream _file; + private readonly List<(string S, string P, string O, bool Literal)> _triples = []; + private readonly Dictionary _streams = new(StringComparer.Ordinal); + private readonly List _damaged = []; + private readonly int _minor; + + private Aff4Reader(FileStream file, ZipArchive zip) + { + _file = file; + _zip = zip; + + VolumeUrn = ReadText("container.description")?.Trim() ?? zip.Comment?.Trim() ?? ""; + var version = ReadText("version.txt") ?? ""; + var m = Regex.Match(version, @"minor\s*=\s*(\d+)"); + _minor = m.Success ? int.Parse(m.Groups[1].Value, CultureInfo.InvariantCulture) : 0; + + var turtle = ReadText("information.turtle") ?? throw new InvalidDataException("AFF4: no information.turtle in the container."); + ParseTurtle(turtle); + + // Images: anything typed aff4:Image (or DiskImage) with a dataStream; fall back to a + // bare Map or ImageStream so a container without the Image object still opens. + Images = _triples.Where(t => t.P == Aff4Type && t.O is Aff4Image or Aff4DiskImage or Aff4ContiguousImage) + .Select(t => t.S).Distinct().Where(s => Get(s, "dataStream") is not null).ToList(); + if (Images.Count == 0) + { + Images = _triples.Where(t => t.P == Aff4Type && t.O == Aff4Map).Select(t => t.S).Distinct().ToList(); + } + if (Images.Count == 0) + { + Images = _triples.Where(t => t.P == Aff4Type && t.O == Aff4ImageStream).Select(t => t.S).Distinct().ToList(); + } + if (Images.Count == 0) + { + throw new InvalidDataException("AFF4: the container describes no image, map or image stream."); + } + ImageUrn = Images[0]; + Ranges = ResolveRanges(ImageUrn, out var size); + Size = size; + + foreach (var (_, p, o, lit) in _triples.Where(t => t.S == ImageUrn)) + { + if (!lit) continue; + var (value, type) = SplitLiteral(o); + if (p == Aff4 + "hash") + { + if (type == Aff4 + "MD5") RecordedMd5 = value.ToLowerInvariant(); + else if (type == Aff4 + "SHA1") RecordedSha1 = value.ToLowerInvariant(); + else if (type == Aff4 + "SHA256") RecordedSha256 = value.ToLowerInvariant(); + } + else if (p.StartsWith(Aff4, StringComparison.Ordinal)) + { + Metadata[p[Aff4.Length..]] = value; + } + } + } + + public static bool IsAff4(string path) + { + try + { + using var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + Span magic = stackalloc byte[4]; + if (fs.Read(magic) < 4 || magic[0] != 'P' || magic[1] != 'K' || magic[2] != 3 || magic[3] != 4) + { + return false; + } + fs.Position = 0; + using var zip = new ZipArchive(fs, ZipArchiveMode.Read, leaveOpen: true); + return zip.GetEntry("container.description") is not null || zip.GetEntry("information.turtle") is not null; + } + catch { return false; } + } + + public static Aff4Reader Open(string path) + { + var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20); + try + { + var zip = new ZipArchive(fs, ZipArchiveMode.Read, leaveOpen: true); + return new Aff4Reader(fs, zip); + } + catch + { + fs.Dispose(); + throw; + } + } + + public string VolumeUrn { get; } + public string ImageUrn { get; } + public IReadOnlyList Images { get; } + public long Size { get; } + public string? RecordedMd5 { get; } + public string? RecordedSha1 { get; } + public string? RecordedSha256 { get; } + public Dictionary Metadata { get; } = new(StringComparer.Ordinal); + public IReadOnlyList DamagedChunks => _damaged; + internal IReadOnlyList Ranges { get; } + + public Stream OpenStream() => new Aff4Stream(this, ownsReader: false); + internal Stream OpenOwningStream() => new Aff4Stream(this, ownsReader: true); + + public void Dispose() + { + _zip.Dispose(); + _file.Dispose(); + } + + // ---- object graph ------------------------------------------------------------------------- + + internal sealed record MapRange(long MapOffset, long Length, long TargetOffset, ImageStreamInfo? Target); + + internal sealed class ImageStreamInfo + { + public required string Urn { get; init; } + public required int ChunkSize { get; init; } + public required int ChunksPerSegment { get; init; } + public required string Compression { get; init; } + public required long Size { get; init; } + public int LoadedBevy = -1; + public byte[]? BevyData; + public long[]? BevyOffsets; + public int[]? BevyLengths; + } + + private List ResolveRanges(string imageUrn, out long size) + { + var target = Get(imageUrn, "dataStream") ?? imageUrn; + var types = _triples.Where(t => t.S == target && t.P == Aff4Type).Select(t => t.O).ToHashSet(StringComparer.Ordinal); + size = ParseLong(Get(imageUrn, "size")) ?? ParseLong(Get(target, "size")) ?? 0; + + if (types.Contains(Aff4Map)) + { + var ranges = ReadMap(target); + if (size == 0 && ranges.Count > 0) + { + size = ranges[^1].MapOffset + ranges[^1].Length; + } + return ranges; + } + + var stream = StreamInfo(target) ?? throw new InvalidDataException($"AFF4: {target} is neither a map nor an image stream."); + if (size == 0) size = stream.Size; + return [new MapRange(0, stream.Size, 0, stream)]; + } + + private List ReadMap(string mapUrn) + { + var idx = ReadText(Member(mapUrn, "idx")) ?? ReadText(MemberRaw(mapUrn, "idx")) ?? ""; + var targets = idx.Split('\n', StringSplitOptions.RemoveEmptyEntries).Select(t => t.Trim()).ToList(); + var raw = ReadBytes(Member(mapUrn, "map")) ?? ReadBytes(MemberRaw(mapUrn, "map")) + ?? throw new InvalidDataException("AFF4: map has no range data."); + if (raw.Length % 28 != 0) + { + throw new InvalidDataException("AFF4: map range table has an unexpected size."); + } + + var ranges = new List(raw.Length / 28); + for (int i = 0; i + 28 <= raw.Length; i += 28) + { + var mapOffset = BitConverter.ToInt64(raw, i); + var length = BitConverter.ToInt64(raw, i + 8); + var targetOffset = BitConverter.ToInt64(raw, i + 16); + var targetId = BitConverter.ToUInt32(raw, i + 24); + if (mapOffset < 0 || length < 0 || targetOffset < 0) + { + throw new InvalidDataException("AFF4: map range with negative offsets."); + } + ImageStreamInfo? target = null; + if (targetId < targets.Count) + { + var t = targets[(int)targetId]; + if (!IsZeroTarget(t)) + { + target = StreamInfo(t); // null → treated as unreadable (zeros) + } + } + ranges.Add(new MapRange(mapOffset, length, targetOffset, target)); + } + ranges.Sort((a, b) => a.MapOffset.CompareTo(b.MapOffset)); + return ranges; + } + + private static bool IsZeroTarget(string t) => + t.EndsWith("Zero", StringComparison.Ordinal) || t.EndsWith("UnknownData", StringComparison.Ordinal) || + t.EndsWith("UnreadableData", StringComparison.Ordinal) || t == "aff4:Zero"; + + private ImageStreamInfo? StreamInfo(string urn) + { + if (_streams.TryGetValue(urn, out var cached)) + { + return cached; + } + var chunkSize = (int)(ParseLong(Get(urn, "chunkSize")) ?? 32 * 1024); + var perSegment = (int)(ParseLong(Get(urn, "chunksInSegment")) ?? 2048); + if (chunkSize is < 512 or > MaxChunkSize || perSegment is < 1 or > 1 << 20) + { + throw new InvalidDataException($"AFF4: {urn} declares implausible chunk geometry ({chunkSize} × {perSegment})."); + } + var compression = Get(urn, "compressionMethod") ?? Aff4Writer.StoredCompression; + var size = ParseLong(Get(urn, "size")) ?? 0; + if (_zip.GetEntry(Member(urn, "00000000")) is null && _zip.GetEntry(MemberRaw(urn, "00000000")) is null && size > 0) + { + return null; // referenced but not stored here (a foreign container) + } + var info = new ImageStreamInfo { Urn = urn, ChunkSize = chunkSize, ChunksPerSegment = perSegment, Compression = compression, Size = size }; + _streams[urn] = info; + return info; + } + + // ---- chunk access ------------------------------------------------------------------------- + + /// Copies bytes [offset, offset+count) of an image stream into ; zero-fills past its end. + internal int ReadStream(ImageStreamInfo s, long offset, Span dest) + { + var done = 0; + while (done < dest.Length) + { + var pos = offset + done; + if (pos >= s.Size) + { + dest[done..].Clear(); + return dest.Length; + } + var chunkIndex = (int)(pos / s.ChunkSize); + var within = (int)(pos % s.ChunkSize); + var chunk = ReadChunk(s, chunkIndex); + var n = Math.Min(dest.Length - done, chunk.Length - within); + if (n <= 0) + { + dest[done..].Clear(); + return dest.Length; + } + chunk.AsSpan(within, n).CopyTo(dest[done..]); + done += n; + } + return done; + } + + private byte[] ReadChunk(ImageStreamInfo s, int chunkIndex) + { + var bevy = chunkIndex / s.ChunksPerSegment; + var inBevy = chunkIndex % s.ChunksPerSegment; + if (s.LoadedBevy != bevy) + { + LoadBevy(s, bevy); + } + var expected = (int)Math.Min(s.ChunkSize, s.Size - (long)chunkIndex * s.ChunkSize); + if (expected <= 0) + { + return []; + } + if (s.BevyData is null || s.BevyOffsets is null || inBevy >= s.BevyOffsets.Length) + { + return Damaged(chunkIndex, expected); + } + + var start = s.BevyOffsets[inBevy]; + var length = s.BevyLengths is not null + ? s.BevyLengths[inBevy] + : (int)((inBevy + 1 < s.BevyOffsets.Length ? s.BevyOffsets[inBevy + 1] : s.BevyData.Length) - start); + if (start < 0 || length < 0 || start + length > s.BevyData.Length) + { + return Damaged(chunkIndex, expected); + } + var raw = s.BevyData.AsSpan((int)start, length); + try + { + var outBuf = new byte[expected]; + var produced = s.Compression switch + { + Aff4Writer.StoredCompression or "http://aff4.org/Schema#NullCompressor" or "https://aff4.org/Schema#NullCompressor" => Copy(raw, outBuf), + Aff4Writer.ZlibCompression or "http://www.ietf.org/rfc/rfc1950.txt" => Inflate(raw, outBuf), + "https://code.google.com/p/snappy/" or "http://code.google.com/p/snappy/" => SnappyDecoder.Decode(raw, outBuf), + "https://code.google.com/p/lz4/" or "http://code.google.com/p/lz4/" => Lz4Decoder.Decode(raw, outBuf), + _ => throw new NotSupportedException($"AFF4: compression method {s.Compression} is not supported."), + }; + if (produced < expected) + { + // A stored chunk may legitimately be exactly the raw bytes even under a + // compressing stream (libaff4 does this for incompressible data). + if (raw.Length == expected) + { + raw.CopyTo(outBuf); + return outBuf; + } + RecordDamage(chunkIndex); + outBuf.AsSpan(produced).Clear(); + } + return outBuf; + } + catch (NotSupportedException) { throw; } + catch (Exception) + { + if (raw.Length == expected) + { + return raw.ToArray(); + } + return Damaged(chunkIndex, expected); + } + } + + private static int Copy(ReadOnlySpan raw, byte[] outBuf) + { + var n = Math.Min(raw.Length, outBuf.Length); + raw[..n].CopyTo(outBuf); + return n; + } + + private static int Inflate(ReadOnlySpan raw, byte[] outBuf) + { + using var ms = new MemoryStream(raw.ToArray()); + using var z = new ZLibStream(ms, CompressionMode.Decompress); + var filled = 0; + while (filled < outBuf.Length) + { + var n = z.Read(outBuf, filled, outBuf.Length - filled); + if (n <= 0) break; + filled += n; + } + return filled; + } + + private void LoadBevy(ImageStreamInfo s, int bevy) + { + s.LoadedBevy = bevy; + s.BevyData = null; + s.BevyOffsets = null; + s.BevyLengths = null; + + var name = bevy.ToString("D8", CultureInfo.InvariantCulture); + var data = ReadBytes(Member(s.Urn, name), MaxBevyBytes) ?? ReadBytes(MemberRaw(s.Urn, name), MaxBevyBytes); + var index = ReadBytes(Member(s.Urn, name + ".index")) ?? ReadBytes(MemberRaw(s.Urn, name + ".index")); + if (data is null || index is null) + { + return; + } + s.BevyData = data; + + // Standard layout is { uint64 offset; uint32 length; } per chunk; pre-standard + // (Evimetry / early pyaff4) is a uint32 offset list. Twelve is a multiple of four, so + // the length alone cannot tell them apart — decode as standard and keep it only if + // every entry lies inside the bevy in order, which a uint32 list never does. + if (index.Length % 12 == 0 && index.Length > 0 && TryStandardIndex(index, data.Length, out var offsets, out var lengths)) + { + s.BevyOffsets = offsets; + s.BevyLengths = lengths; + } + else + { + var n = index.Length / 4; + s.BevyOffsets = new long[n]; + for (int i = 0; i < n; i++) + { + s.BevyOffsets[i] = BitConverter.ToUInt32(index, i * 4); + } + } + } + + private static bool TryStandardIndex(byte[] index, long bevyLength, out long[] offsets, out int[] lengths) + { + var n = index.Length / 12; + offsets = new long[n]; + lengths = new int[n]; + long previous = -1; + for (int i = 0; i < n; i++) + { + var off = BitConverter.ToInt64(index, i * 12); + var len = BitConverter.ToInt32(index, i * 12 + 8); + if (off < previous || len < 0 || off + len > bevyLength) + { + return false; + } + offsets[i] = off; + lengths[i] = len; + previous = off; + } + return true; + } + + private byte[] Damaged(int chunkIndex, int length) + { + RecordDamage(chunkIndex); + return new byte[length]; + } + + private void RecordDamage(int chunkIndex) + { + if (_damaged.Count < 100_000 && (_damaged.Count == 0 || _damaged[^1] != chunkIndex)) + { + _damaged.Add(chunkIndex); + } + } + + // ---- members --------------------------------------------------------------------------- + + private static string Member(string urn, string segment) => Aff4Writer.MemberName(urn, segment); + private static string MemberRaw(string urn, string segment) => urn + "/" + segment; + + private string? ReadText(string name) + { + var b = ReadBytes(name, 64 * 1024 * 1024); + return b is null ? null : Encoding.UTF8.GetString(b); + } + + private byte[]? ReadBytes(string name, long limit = 64L * 1024 * 1024) + { + var entry = _zip.GetEntry(name); + if (entry is null) + { + return null; + } + if (entry.Length > limit) + { + throw new InvalidDataException($"AFF4: member {name} is {entry.Length:N0} bytes, above the {limit:N0}-byte ceiling."); + } + using var s = entry.Open(); + var buf = new byte[entry.Length]; + var filled = 0; + while (filled < buf.Length) + { + var n = s.Read(buf, filled, buf.Length - filled); + if (n <= 0) break; + filled += n; + } + return filled == buf.Length ? buf : buf[..filled]; + } + + // ---- turtle ------------------------------------------------------------------------------ + + private const string Aff4 = "http://aff4.org/Schema#"; + private const string Aff4Type = "http://www.w3.org/1999/02/22-rdf-syntax-ns#type"; + private const string Aff4Image = Aff4 + "Image"; + private const string Aff4DiskImage = Aff4 + "DiskImage"; + private const string Aff4ContiguousImage = Aff4 + "ContiguousImage"; + private const string Aff4Map = Aff4 + "Map"; + private const string Aff4ImageStream = Aff4 + "ImageStream"; + + private string? Get(string subject, string aff4Predicate) + { + var p = Aff4 + aff4Predicate; + foreach (var t in _triples) + { + if (t.S == subject && t.P == p) + { + return t.Literal ? SplitLiteral(t.O).Value : t.O; + } + } + return null; + } + + private static (string Value, string? Type) SplitLiteral(string o) + { + var i = o.IndexOf("^^", StringComparison.Ordinal); + return i < 0 ? (o, null) : (o[..i], o[(i + 2)..]); + } + + private static long? ParseLong(string? s) => + s is not null && long.TryParse(s, NumberStyles.Integer, CultureInfo.InvariantCulture, out var v) ? v : null; + + private void ParseTurtle(string text) + { + var prefixes = new Dictionary(StringComparer.Ordinal) + { + ["rdf"] = "http://www.w3.org/1999/02/22-rdf-syntax-ns#", + ["xsd"] = "http://www.w3.org/2001/XMLSchema#", + ["aff4"] = Aff4, + }; + var tokens = Tokenize(text); + var i = 0; + while (i < tokens.Count) + { + var t = tokens[i]; + if (t.Kind == Tok.Word && (t.Text == "@prefix" || t.Text.Equals("PREFIX", StringComparison.OrdinalIgnoreCase))) + { + if (i + 2 < tokens.Count) + { + var name = tokens[i + 1].Text.TrimEnd(':'); + prefixes[name] = tokens[i + 2].Text; + } + i += 3; + if (i < tokens.Count && tokens[i].Kind == Tok.Dot) i++; + continue; + } + if (t.Kind == Tok.Word && t.Text == "@base") + { + i += 2; + if (i < tokens.Count && tokens[i].Kind == Tok.Dot) i++; + continue; + } + + // subject predicateObjectList . + var subject = Expand(t, prefixes); + i++; + string? predicate = null; + while (i < tokens.Count && tokens[i].Kind != Tok.Dot) + { + var tk = tokens[i]; + if (tk.Kind == Tok.Semi) { predicate = null; i++; continue; } + if (tk.Kind == Tok.Comma) { i++; continue; } + if (predicate is null) + { + predicate = tk.Kind == Tok.Word && tk.Text == "a" ? Aff4Type : Expand(tk, prefixes); + i++; + continue; + } + if (tk.Kind == Tok.Literal) + { + var value = tk.Text; + var type = tk.Datatype is null ? null : Expand(new Token(tk.Datatype.StartsWith('<') ? Tok.Iri : Tok.Word, tk.Datatype.Trim('<', '>')), prefixes); + _triples.Add((subject, predicate, type is null ? value : value + "^^" + type, true)); + } + else if (tk.Kind == Tok.Number) + { + _triples.Add((subject, predicate, tk.Text, true)); + } + else + { + _triples.Add((subject, predicate, Expand(tk, prefixes), false)); + } + i++; + } + i++; // the dot + } + } + + private static string Expand(Token t, Dictionary prefixes) + { + if (t.Kind == Tok.Iri) + { + return t.Text; + } + var colon = t.Text.IndexOf(':'); + if (colon > 0 && prefixes.TryGetValue(t.Text[..colon], out var ns)) + { + return ns + t.Text[(colon + 1)..]; + } + return t.Text; + } + + private enum Tok { Iri, Word, Literal, Number, Dot, Semi, Comma } + + private sealed record Token(Tok Kind, string Text, string? Datatype = null); + + private static List Tokenize(string s) + { + var tokens = new List(); + var i = 0; + while (i < s.Length) + { + var c = s[i]; + if (char.IsWhiteSpace(c)) { i++; continue; } + if (c == '#') { while (i < s.Length && s[i] != '\n') i++; continue; } + if (c == '<') + { + var end = s.IndexOf('>', i + 1); + if (end < 0) break; + tokens.Add(new Token(Tok.Iri, s[(i + 1)..end])); + i = end + 1; + continue; + } + if (c == '"') + { + var sb = new StringBuilder(); + var j = i + 1; + var triple = j + 1 < s.Length && s[j] == '"' && s[j + 1] == '"'; + if (triple) j += 2; + while (j < s.Length) + { + if (s[j] == '\\' && j + 1 < s.Length) + { + sb.Append(s[j + 1] switch { 'n' => '\n', 't' => '\t', 'r' => '\r', var x => x }); + j += 2; + continue; + } + if (s[j] == '"' && (!triple || (j + 2 < s.Length && s[j + 1] == '"' && s[j + 2] == '"'))) + { + j += triple ? 3 : 1; + break; + } + sb.Append(s[j]); + j++; + } + string? datatype = null; + if (j + 1 < s.Length && s[j] == '^' && s[j + 1] == '^') + { + j += 2; + var start = j; + if (j < s.Length && s[j] == '<') + { + var end = s.IndexOf('>', j); + datatype = s[j..(end + 1)]; + j = end + 1; + } + else + { + while (j < s.Length && !char.IsWhiteSpace(s[j]) && s[j] != ';' && s[j] != ',' && !(s[j] == '.' && (j + 1 >= s.Length || char.IsWhiteSpace(s[j + 1])))) j++; + datatype = s[start..j]; + } + } + else if (j < s.Length && s[j] == '@') + { + while (j < s.Length && !char.IsWhiteSpace(s[j]) && s[j] != ';' && s[j] != ',') j++; + } + tokens.Add(new Token(Tok.Literal, sb.ToString(), datatype)); + i = j; + continue; + } + if (c == ';') { tokens.Add(new Token(Tok.Semi, ";")); i++; continue; } + if (c == ',') { tokens.Add(new Token(Tok.Comma, ",")); i++; continue; } + if (c == '.' && (i + 1 >= s.Length || char.IsWhiteSpace(s[i + 1]))) + { + tokens.Add(new Token(Tok.Dot, ".")); + i++; + continue; + } + var st = i; + while (i < s.Length && !char.IsWhiteSpace(s[i]) && s[i] != ';' && s[i] != ',' && !(s[i] == '.' && (i + 1 >= s.Length || char.IsWhiteSpace(s[i + 1])))) i++; + var word = s[st..i]; + if (word.Length > 0 && (char.IsDigit(word[0]) || (word[0] == '-' && word.Length > 1))) + { + tokens.Add(new Token(Tok.Number, word)); + } + else + { + tokens.Add(new Token(Tok.Word, word)); + } + } + return tokens; + } + + // ---- stream ----------------------------------------------------------------------------- + + private sealed class Aff4Stream(Aff4Reader reader, bool ownsReader) : Stream + { + private long _pos; + + public override bool CanRead => true; + public override bool CanSeek => true; + public override bool CanWrite => false; + public override long Length => reader.Size; + public override long Position { get => _pos; set => _pos = value; } + + public override int Read(byte[] buffer, int offset, int count) => Read(buffer.AsSpan(offset, count)); + + public override int Read(Span buffer) + { + if (_pos >= reader.Size) + { + return 0; + } + var n = (int)Math.Min(buffer.Length, reader.Size - _pos); + var dest = buffer[..n]; + var done = 0; + while (done < n) + { + var pos = _pos + done; + var r = FindRange(pos); + if (r is null) + { + // Gap in the map: zeros up to the next range. + var next = reader.Ranges.FirstOrDefault(x => x.MapOffset > pos); + var gap = (int)Math.Min(n - done, (next?.MapOffset ?? reader.Size) - pos); + dest.Slice(done, gap).Clear(); + done += gap; + continue; + } + var within = pos - r.MapOffset; + var take = (int)Math.Min(n - done, r.Length - within); + if (r.Target is null) + { + dest.Slice(done, take).Clear(); + } + else + { + reader.ReadStream(r.Target, r.TargetOffset + within, dest.Slice(done, take)); + } + done += take; + } + _pos += done; + return done; + } + + private MapRange? FindRange(long pos) + { + var ranges = reader.Ranges; + int lo = 0, hi = ranges.Count - 1; + while (lo <= hi) + { + var mid = (lo + hi) / 2; + var r = ranges[mid]; + if (pos < r.MapOffset) hi = mid - 1; + else if (pos >= r.MapOffset + r.Length) lo = mid + 1; + else return r; + } + return null; + } + + public override long Seek(long offset, SeekOrigin origin) + { + _pos = origin switch + { + SeekOrigin.Begin => offset, + SeekOrigin.Current => _pos + offset, + _ => reader.Size + offset, + }; + return _pos; + } + + public override void Flush() { } + public override void SetLength(long value) => throw new NotSupportedException(); + public override void Write(byte[] buffer, int offset, int count) => throw new NotSupportedException(); + + protected override void Dispose(bool disposing) + { + if (disposing && ownsReader) + { + reader.Dispose(); + } + base.Dispose(disposing); + } + } +} + +/// Snappy raw-block decoder (the format pyaff4 / libaff4 / Evimetry use per chunk). +internal static class SnappyDecoder +{ + public static int Decode(ReadOnlySpan src, byte[] dst) + { + var i = 0; + // Preamble: varint uncompressed length. + long declared = 0; + var shift = 0; + while (i < src.Length) + { + var b = src[i++]; + declared |= (long)(b & 0x7F) << shift; + if ((b & 0x80) == 0) break; + shift += 7; + if (shift > 35) throw new InvalidDataException("snappy: bad length varint"); + } + var o = 0; + var limit = (int)Math.Min(dst.Length, declared); + while (i < src.Length && o < limit) + { + var tag = src[i++]; + switch (tag & 3) + { + case 0: + { + var len = tag >> 2; + if (len >= 60) + { + var extra = len - 59; + if (i + extra > src.Length) throw new InvalidDataException("snappy: truncated literal length"); + len = 0; + for (int k = 0; k < extra; k++) len |= src[i + k] << (8 * k); + i += extra; + } + len += 1; + if (i + len > src.Length || o + len > dst.Length) throw new InvalidDataException("snappy: literal overruns"); + src.Slice(i, len).CopyTo(dst.AsSpan(o)); + i += len; + o += len; + break; + } + case 1: + { + var len = 4 + ((tag >> 2) & 7); + if (i >= src.Length) throw new InvalidDataException("snappy: truncated copy"); + var offset = ((tag >> 5) << 8) | src[i++]; + o = CopyBack(dst, o, offset, len); + break; + } + case 2: + { + var len = 1 + (tag >> 2); + if (i + 2 > src.Length) throw new InvalidDataException("snappy: truncated copy"); + var offset = src[i] | (src[i + 1] << 8); + i += 2; + o = CopyBack(dst, o, offset, len); + break; + } + default: + { + var len = 1 + (tag >> 2); + if (i + 4 > src.Length) throw new InvalidDataException("snappy: truncated copy"); + var offset = (int)BitConverter.ToUInt32(src.Slice(i, 4)); + i += 4; + o = CopyBack(dst, o, offset, len); + break; + } + } + } + return o; + } + + private static int CopyBack(byte[] dst, int o, int offset, int len) + { + if (offset <= 0 || offset > o || o + len > dst.Length) throw new InvalidDataException("snappy: bad copy"); + for (int k = 0; k < len; k++) + { + dst[o + k] = dst[o - offset + k]; + } + return o + len; + } +} + +/// LZ4 raw-block decoder (no frame header), as libaff4 stores chunks. +internal static class Lz4Decoder +{ + public static int Decode(ReadOnlySpan src, byte[] dst) + { + var i = 0; + var o = 0; + while (i < src.Length) + { + var token = src[i++]; + var literal = token >> 4; + if (literal == 15) + { + byte b; + do + { + if (i >= src.Length) throw new InvalidDataException("lz4: truncated literal length"); + b = src[i++]; + literal += b; + } while (b == 255); + } + if (i + literal > src.Length || o + literal > dst.Length) throw new InvalidDataException("lz4: literal overruns"); + src.Slice(i, literal).CopyTo(dst.AsSpan(o)); + i += literal; + o += literal; + if (i >= src.Length) + { + break; // last sequence has no match + } + if (i + 2 > src.Length) throw new InvalidDataException("lz4: truncated offset"); + var offset = src[i] | (src[i + 1] << 8); + i += 2; + var match = (token & 0xF) + 4; + if ((token & 0xF) == 15) + { + byte b; + do + { + if (i >= src.Length) throw new InvalidDataException("lz4: truncated match length"); + b = src[i++]; + match += b; + } while (b == 255); + } + if (offset <= 0 || offset > o || o + match > dst.Length) throw new InvalidDataException("lz4: bad match"); + for (int k = 0; k < match; k++) + { + dst[o + k] = dst[o - offset + k]; + } + o += match; + } + return o; + } +} diff --git a/src/Cinder.Imaging/Aff4/Aff4Writer.cs b/src/Cinder.Imaging/Aff4/Aff4Writer.cs new file mode 100644 index 0000000..d35bed9 --- /dev/null +++ b/src/Cinder.Imaging/Aff4/Aff4Writer.cs @@ -0,0 +1,258 @@ +using System.Globalization; +using System.IO.Compression; +using System.Text; + +namespace Cinder.Imaging.Aff4; + +/// +/// Writes AFF4 physical-image containers (AFF4 Standard v1.0 layout, the one both libaff4 and +/// pyaff4 read): a ZIP volume holding container.description, version.txt, +/// information.turtle, an ImageStream stored as bevies of zlib-compressed 32 KiB +/// chunks with a (uint64 offset, uint32 length) index per bevy, a single-range Map over the +/// stream, and an Image whose aff4:dataStream is the map — the object structure +/// Evimetry and pyaff4's imager produce, so the result opens as a disk image rather than a +/// bare stream. +/// +/// Digests are the caller's, recorded as aff4:hash literals with the MD5 / SHA1 / +/// SHA256 datatypes. Case metadata goes into the turtle as literals on the image. Media size +/// need not be known up front; the turtle is written last. +/// +public sealed class Aff4Writer : IDisposable +{ + public sealed record Options + { + public bool Compress { get; init; } = true; + public int ChunkSize { get; init; } = 32 * 1024; + public int ChunksPerSegment { get; init; } = 2048; + public string? CaseNumber { get; init; } + public string? EvidenceNumber { get; init; } + public string? Examiner { get; init; } + public string? Description { get; init; } + public string? Notes { get; init; } + public string? Source { get; init; } + public string Tool { get; init; } = "Cinder"; + } + + public const string ZlibCompression = "https://www.ietf.org/rfc/rfc1950.txt"; + public const string StoredCompression = "http://aff4.org/Schema#NullCompressor"; + + private readonly Options _o; + private readonly FileStream _file; + private readonly ZipArchive _zip; + private readonly byte[] _pending; + private int _pendingLength; + + private readonly MemoryStream _bevy = new(); + private readonly List _bevyIndex = []; + private int _bevyNumber; + private long _size; + private bool _finished; + + public Aff4Writer(string path, Options? options = null) + { + ArgumentException.ThrowIfNullOrWhiteSpace(path); + _o = options ?? new Options(); + if (_o.ChunkSize is < 512 or > (1 << 24) || _o.ChunksPerSegment is < 1 or > 65536) + { + throw new ArgumentOutOfRangeException(nameof(options)); + } + Path = System.IO.Path.GetFullPath(path); + Directory.CreateDirectory(System.IO.Path.GetDirectoryName(Path)!); + + VolumeUrn = "aff4://" + Guid.NewGuid().ToString("D"); + ImageUrn = "aff4://" + Guid.NewGuid().ToString("D"); + MapUrn = "aff4://" + Guid.NewGuid().ToString("D"); + StreamUrn = "aff4://" + Guid.NewGuid().ToString("D"); + + _file = new FileStream(Path, FileMode.Create, FileAccess.ReadWrite, FileShare.None, 1 << 20); + _zip = new ZipArchive(_file, ZipArchiveMode.Create, leaveOpen: true); + _zip.Comment = VolumeUrn; // pyaff4 and libaff4 take the volume URN from the archive comment + _pending = new byte[_o.ChunkSize]; + + WriteMember("container.description", Encoding.UTF8.GetBytes(VolumeUrn)); + WriteMember("version.txt", Encoding.ASCII.GetBytes("major=1\nminor=0\ntool=" + Clean(_o.Tool) + "\n")); + } + + public string Path { get; } + public string VolumeUrn { get; } + public string ImageUrn { get; } + public string MapUrn { get; } + public string StreamUrn { get; } + public long BytesWritten => _size; + + public void Write(ReadOnlySpan data) + { + if (_finished) + { + throw new InvalidOperationException("The container has been finished."); + } + while (data.Length > 0) + { + var take = Math.Min(_o.ChunkSize - _pendingLength, data.Length); + data[..take].CopyTo(_pending.AsSpan(_pendingLength)); + _pendingLength += take; + _size += take; + data = data[take..]; + if (_pendingLength == _o.ChunkSize) + { + EmitChunk(_pending.AsSpan(0, _pendingLength)); + _pendingLength = 0; + } + } + } + + public void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256) + { + if (_finished) + { + return; + } + _finished = true; + if (_pendingLength > 0) + { + EmitChunk(_pending.AsSpan(0, _pendingLength)); + _pendingLength = 0; + } + FlushBevy(); + + // Map: one range covering the whole stream. Range = map_offset u64, length u64, + // target_offset u64, target_id u32 (little-endian); idx lists targets by id. + var range = new byte[28]; + BitConverter.TryWriteBytes(range.AsSpan(0), 0L); + BitConverter.TryWriteBytes(range.AsSpan(8), _size); + BitConverter.TryWriteBytes(range.AsSpan(16), 0L); + BitConverter.TryWriteBytes(range.AsSpan(24), 0u); + WriteMember(MemberName(MapUrn, "map"), range); + WriteMember(MemberName(MapUrn, "idx"), Encoding.UTF8.GetBytes(StreamUrn + "\n")); + + WriteMember("information.turtle", Encoding.UTF8.GetBytes(Turtle(md5, sha1, sha256))); + + _zip.Dispose(); + _file.Flush(); + _file.Dispose(); + } + + public void Dispose() + { + try { _zip.Dispose(); } catch { } + _file.Dispose(); + } + + // ---- chunks and bevies -------------------------------------------------------------------- + + private void EmitChunk(ReadOnlySpan plain) + { + _bevyIndex.Add((uint)_bevy.Length); + if (_o.Compress) + { + // With zlib declared every chunk is zlib — readers do not probe per chunk. + using var z = new ZLibStream(_bevy, CompressionLevel.Optimal, leaveOpen: true); + z.Write(plain); + } + else + { + _bevy.Write(plain); + } + if (_bevyIndex.Count >= _o.ChunksPerSegment) + { + FlushBevy(); + } + if (_bevy.Length > int.MaxValue - (1 << 25)) + { + throw new InvalidOperationException("AFF4: bevy exceeds the 32-bit index; lower chunks per segment."); + } + } + + private void FlushBevy() + { + if (_bevyIndex.Count == 0) + { + return; + } + var name = _bevyNumber.ToString("D8", CultureInfo.InvariantCulture); + // Standard index: one { uint64 offset; uint32 length; } per chunk (the uint32 offset + // list is the pre-standard layout, which the reader still accepts). + var index = new byte[_bevyIndex.Count * 12]; + for (int i = 0; i < _bevyIndex.Count; i++) + { + var end = i + 1 < _bevyIndex.Count ? _bevyIndex[i + 1] : (uint)_bevy.Length; + BitConverter.TryWriteBytes(index.AsSpan(i * 12), (ulong)_bevyIndex[i]); + BitConverter.TryWriteBytes(index.AsSpan(i * 12 + 8), end - _bevyIndex[i]); + } + WriteMember(MemberName(StreamUrn, name + ".index"), index); + WriteMember(MemberName(StreamUrn, name), _bevy.GetBuffer().AsSpan(0, (int)_bevy.Length)); + _bevy.SetLength(0); + _bevyIndex.Clear(); + _bevyNumber++; + } + + private void WriteMember(string name, ReadOnlySpan data) + { + // Stored: chunk payloads are already compressed, and libaff4 reads stored members fastest. + var entry = _zip.CreateEntry(name, CompressionLevel.NoCompression); + using var s = entry.Open(); + s.Write(data); + } + + /// v1.0 member naming: the object URN percent-encoded, then a plain path segment. + public static string MemberName(string urn, string segment) => Uri.EscapeDataString(urn) + "/" + segment; + + // ---- RDF ---------------------------------------------------------------------------------- + + private string Turtle(byte[]? md5, byte[]? sha1, byte[]? sha256) + { + var sb = new StringBuilder(); + sb.Append("@prefix aff4: .\n"); + sb.Append("@prefix rdf: .\n"); + sb.Append("@prefix xsd: .\n\n"); + + sb.Append('<').Append(VolumeUrn).Append(">\n"); + sb.Append(" a aff4:ZipVolume ;\n"); + sb.Append(" aff4:creationTime \"").Append(DateTimeOffset.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.ffffffzzz", CultureInfo.InvariantCulture)).Append("\"^^xsd:dateTime .\n\n"); + + sb.Append('<').Append(ImageUrn).Append(">\n"); + sb.Append(" a aff4:DiskImage, aff4:Image, aff4:ContiguousImage ;\n"); + sb.Append(" aff4:dataStream <").Append(MapUrn).Append("> ;\n"); + sb.Append(" aff4:size \"").Append(_size).Append("\"^^xsd:long ;\n"); + sb.Append(" aff4:stored <").Append(VolumeUrn).Append("> ;\n"); + if (md5 is not null) sb.Append(" aff4:hash \"").Append(Convert.ToHexStringLower(md5)).Append("\"^^aff4:MD5 ;\n"); + if (sha1 is not null) sb.Append(" aff4:hash \"").Append(Convert.ToHexStringLower(sha1)).Append("\"^^aff4:SHA1 ;\n"); + if (sha256 is not null) sb.Append(" aff4:hash \"").Append(Convert.ToHexStringLower(sha256)).Append("\"^^aff4:SHA256 ;\n"); + Literal(sb, "aff4:caseNumber", _o.CaseNumber); + Literal(sb, "aff4:evidenceNumber", _o.EvidenceNumber); + Literal(sb, "aff4:examiner", _o.Examiner); + Literal(sb, "aff4:caseDescription", _o.Description); + Literal(sb, "aff4:caseNotes", _o.Notes); + Literal(sb, "aff4:acquisitionSource", _o.Source); + Literal(sb, "aff4:acquisitionTool", _o.Tool); + sb.Append(" aff4:acquisitionTime \"").Append(DateTimeOffset.UtcNow.ToString("yyyy-MM-ddTHH:mm:ss.ffffffzzz", CultureInfo.InvariantCulture)).Append("\"^^xsd:dateTime .\n\n"); + + sb.Append('<').Append(MapUrn).Append(">\n"); + sb.Append(" a aff4:Map ;\n"); + sb.Append(" aff4:dependentStream <").Append(StreamUrn).Append("> ;\n"); + sb.Append(" aff4:size \"").Append(_size).Append("\"^^xsd:long ;\n"); + sb.Append(" aff4:stored <").Append(VolumeUrn).Append("> .\n\n"); + + sb.Append('<').Append(StreamUrn).Append(">\n"); + sb.Append(" a aff4:ImageStream ;\n"); + sb.Append(" aff4:chunkSize \"").Append(_o.ChunkSize).Append("\"^^xsd:int ;\n"); + sb.Append(" aff4:chunksInSegment \"").Append(_o.ChunksPerSegment).Append("\"^^xsd:int ;\n"); + sb.Append(" aff4:compressionMethod <").Append(_o.Compress ? ZlibCompression : StoredCompression).Append("> ;\n"); + sb.Append(" aff4:size \"").Append(_size).Append("\"^^xsd:long ;\n"); + sb.Append(" aff4:stored <").Append(VolumeUrn).Append("> ;\n"); + sb.Append(" aff4:version \"1\"^^xsd:int .\n"); + return sb.ToString(); + } + + private static void Literal(StringBuilder sb, string predicate, string? value) + { + if (string.IsNullOrEmpty(value)) + { + return; + } + sb.Append(" ").Append(predicate).Append(" \"").Append(Escape(value)).Append("\" ;\n"); + } + + private static string Escape(string s) => s.Replace("\\", "\\\\").Replace("\"", "\\\"").Replace("\n", "\\n").Replace("\r", "\\r").Replace("\t", "\\t"); + private static string Clean(string s) => s.Replace('\n', ' ').Replace('\r', ' '); +} diff --git a/src/Cinder.Imaging/Cinder.Imaging.csproj b/src/Cinder.Imaging/Cinder.Imaging.csproj index 6dc1a66..7a10453 100644 --- a/src/Cinder.Imaging/Cinder.Imaging.csproj +++ b/src/Cinder.Imaging/Cinder.Imaging.csproj @@ -1,6 +1,11 @@ + + + + + diff --git a/src/Cinder.Imaging/EvidenceOpener.cs b/src/Cinder.Imaging/EvidenceOpener.cs index 8c804cf..e7bcd3e 100644 --- a/src/Cinder.Imaging/EvidenceOpener.cs +++ b/src/Cinder.Imaging/EvidenceOpener.cs @@ -22,6 +22,11 @@ public static Stream Open(string path) // Sniff first 8 bytes; if EVF magic, hand back the EWF-backed Stream. // Otherwise it's a raw file — return a FileStream so callers see byte-for-byte. var fs = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + if (IsAff4Magic(fs)) + { + fs.Dispose(); + return Aff4.Aff4Reader.Open(path).OpenOwningStream(); + } if (IsEwfMagic(fs)) { try @@ -41,6 +46,31 @@ public static Stream Open(string path) return fs; } + /// True for an AFF4 container (a ZIP volume with a container.description). + public static bool IsAff4(string path) => Aff4.Aff4Reader.IsAff4(path); + + private static bool IsAff4Magic(FileStream s) + { + var save = s.Position; + try + { + s.Position = 0; + Span head = stackalloc byte[4]; + if (s.Read(head) < 4 || head[0] != 'P' || head[1] != 'K' || head[2] != 3 || head[3] != 4) + { + return false; + } + s.Position = 0; + using var zip = new System.IO.Compression.ZipArchive(s, System.IO.Compression.ZipArchiveMode.Read, leaveOpen: true); + return zip.GetEntry("container.description") is not null || zip.GetEntry("information.turtle") is not null; + } + catch (InvalidDataException) { return false; } + finally + { + s.Position = save; + } + } + public static bool IsEwf(string path) { try diff --git a/src/Cinder.Imaging/InProcessImager.cs b/src/Cinder.Imaging/InProcessImager.cs index da67829..46684ef 100644 --- a/src/Cinder.Imaging/InProcessImager.cs +++ b/src/Cinder.Imaging/InProcessImager.cs @@ -28,14 +28,15 @@ public sealed class InProcessImager : IDiskImager /// Bad sectors listed individually in the log before the list is capped. private const int MaxLoggedBadSectors = 10_000; - public static bool Supports(ImageFormat format) => format is ImageFormat.Raw or ImageFormat.Ewf; + public static bool Supports(ImageFormat format) => + format is ImageFormat.Raw or ImageFormat.Ewf or ImageFormat.Vhd or ImageFormat.Vhdx or ImageFormat.Aff4; public async Task ImageAsync(ImageJob job, IProgress? progress = null, CancellationToken ct = default) { ArgumentNullException.ThrowIfNull(job); if (!Supports(job.Format)) { - throw new NotSupportedException($"InProcessImager writes raw and EWF images; {job.Format} is not supported in-process."); + throw new NotSupportedException($"InProcessImager writes raw, EWF, AFF4, VHD and VHDX images; {job.Format} is not supported in-process."); } await using var source = OpenSource(job.SourceDevice); @@ -52,16 +53,17 @@ public async Task ImageStreamAsync(Stream source, ImageJob job, ArgumentNullException.ThrowIfNull(job); if (!Supports(job.Format)) { - throw new NotSupportedException($"InProcessImager writes raw and EWF images; {job.Format} is not supported in-process."); + throw new NotSupportedException($"InProcessImager writes raw, EWF, AFF4, VHD and VHDX images; {job.Format} is not supported in-process."); } var started = DateTimeOffset.UtcNow; var sw = Stopwatch.StartNew(); long? total = TryLength(source); - // EWF records MD5 and SHA-1 in the container; make sure they exist for that format. - var wantMd5 = job.ComputeMd5 || job.Format == ImageFormat.Ewf; - var wantSha1 = job.ComputeSha1 || job.Format == ImageFormat.Ewf; + // EWF and AFF4 record digests in the container; make sure they exist for those formats. + var container = job.Format is ImageFormat.Ewf or ImageFormat.Aff4; + var wantMd5 = job.ComputeMd5 || container; + var wantSha1 = job.ComputeSha1 || container; using var md5 = wantMd5 ? IncrementalHash.CreateHash(HashAlgorithmName.MD5) : null; using var sha1 = wantSha1 ? IncrementalHash.CreateHash(HashAlgorithmName.SHA1) : null; using var sha256 = job.ComputeSha256 ? IncrementalHash.CreateHash(HashAlgorithmName.SHA256) : null; @@ -72,7 +74,7 @@ public async Task ImageStreamAsync(Stream source, ImageJob job, var block = new byte[BlockSize]; Directory.CreateDirectory(Path.GetDirectoryName(Path.GetFullPath(job.OutputPath))!); - using var sink = OpenSink(job); + using var sink = OpenSink(job, total); long position = 0; while (true) { @@ -97,14 +99,16 @@ public async Task ImageStreamAsync(Stream source, ImageJob job, var md5Bytes = md5?.GetHashAndReset(); var sha1Bytes = sha1?.GetHashAndReset(); + var sha256Bytes = sha256?.GetHashAndReset(); var md5Hex = md5Bytes is null ? null : Convert.ToHexStringLower(md5Bytes); var sha1Hex = sha1Bytes is null ? null : Convert.ToHexStringLower(sha1Bytes); - var sha256Hex = sha256 is null ? null : Convert.ToHexStringLower(sha256.GetHashAndReset()); + var sha256Hex = sha256Bytes is null ? null : Convert.ToHexStringLower(sha256Bytes); - sink.Finish(md5Bytes, sha1Bytes); + sink.Finish(md5Bytes, sha1Bytes, sha256Bytes); - // Companions: the digest file the Verify tool reads (raw only), and the acquisition log. - if (sha256Hex is not null && job.Format == ImageFormat.Raw) + // Companions: the digest file the Verify tool reads (formats that do not carry their + // own digest), and the acquisition log. + if (sha256Hex is not null && !container) { await File.WriteAllTextAsync(job.OutputPath + ".sha256", $"{sha256Hex} {Path.GetFileName(job.OutputPath)}\n", ct).ConfigureAwait(false); @@ -251,9 +255,11 @@ public static bool IsDevicePath(string path) => // ---- output sinks -------------------------------------------------------------------- - private static ImageSink OpenSink(ImageJob job) => job.Format switch + private static ImageSink OpenSink(ImageJob job, long? sourceLength) => job.Format switch { ImageFormat.Ewf => new EwfSink(job), + ImageFormat.Aff4 => new Aff4Sink(job), + ImageFormat.Vhd or ImageFormat.Vhdx => new VirtualDiskSink(job, sourceLength), _ => new RawSink(job.OutputPath), }; @@ -262,10 +268,93 @@ private abstract class ImageSink : IDisposable public abstract IReadOnlyList Paths { get; } public abstract ValueTask WriteAsync(ReadOnlyMemory data, CancellationToken ct); public virtual void NoteBadSector(long sector) { } - public abstract void Finish(byte[]? md5, byte[]? sha1); + public abstract void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256); public abstract void Dispose(); } + /// + /// VHD / VHDX through DiscUtils' managed writers. Dynamic disks, so unallocated zero + /// runs cost nothing on disk. Both formats need the capacity up front, which is why a + /// source of unknown length cannot go to these formats — image it raw or E01 first. + /// + private sealed class VirtualDiskSink : ImageSink + { + private readonly FileStream _file; + private readonly DiscUtils.VirtualDisk _disk; + private readonly Stream _content; + private readonly string _path; + + public VirtualDiskSink(ImageJob job, long? sourceLength) + { + if (sourceLength is not { } capacity || capacity <= 0) + { + throw new NotSupportedException($"{job.Format} needs the source size up front; this source does not report one. Image it raw or E01."); + } + // Both formats are sector-granular; round a ragged tail up so the last bytes fit. + capacity = (capacity + 511) / 512 * 512; + + _path = job.OutputPath; + _file = new FileStream(_path, FileMode.Create, FileAccess.ReadWrite, FileShare.None, BlockSize); + _disk = job.Format == ImageFormat.Vhd + ? DiscUtils.Vhd.Disk.InitializeDynamic(_file, DiscUtils.Streams.Ownership.None, capacity) + : DiscUtils.Vhdx.Disk.InitializeDynamic(_file, DiscUtils.Streams.Ownership.None, capacity); + _content = _disk.Content; + _content.Position = 0; + } + + public override IReadOnlyList Paths => [_path]; + + public override ValueTask WriteAsync(ReadOnlyMemory data, CancellationToken ct) + { + _content.Write(data.Span); + return ValueTask.CompletedTask; + } + + public override void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256) + { + // DiscUtils' VHDX content stream does not implement Flush; disposing the disk + // writes the block allocation table and metadata for both formats. + _disk.Dispose(); + _file.Flush(); + } + + public override void Dispose() + { + try { _disk.Dispose(); } catch { } + _file.Dispose(); + } + } + + private sealed class Aff4Sink : ImageSink + { + private readonly Aff4.Aff4Writer _writer; + + public Aff4Sink(ImageJob job) + { + _writer = new Aff4.Aff4Writer(job.OutputPath, new Aff4.Aff4Writer.Options + { + Compress = job.CompressionLevel > 0, + CaseNumber = job.CaseNumber, + EvidenceNumber = job.EvidenceNumber, + Examiner = job.ExaminerName, + Description = job.Description, + Notes = job.Notes, + Source = job.SourceDevice, + }); + } + + public override IReadOnlyList Paths => [_writer.Path]; + + public override ValueTask WriteAsync(ReadOnlyMemory data, CancellationToken ct) + { + _writer.Write(data.Span); + return ValueTask.CompletedTask; + } + + public override void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256) => _writer.Finish(md5, sha1, sha256); + public override void Dispose() => _writer.Dispose(); + } + private sealed class RawSink : ImageSink { private readonly FileStream _out; @@ -279,7 +368,7 @@ public RawSink(string path) public override IReadOnlyList Paths => [_path]; public override ValueTask WriteAsync(ReadOnlyMemory data, CancellationToken ct) => _out.WriteAsync(data, ct); - public override void Finish(byte[]? md5, byte[]? sha1) => _out.Flush(); + public override void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256) => _out.Flush(); public override void Dispose() => _out.Dispose(); } @@ -314,7 +403,7 @@ public override ValueTask WriteAsync(ReadOnlyMemory data, CancellationToke } public override void NoteBadSector(long sector) => _writer.AddBadSectors(sector, 1); - public override void Finish(byte[]? md5, byte[]? sha1) => _writer.Finish(md5, sha1); + public override void Finish(byte[]? md5, byte[]? sha1, byte[]? sha256) => _writer.Finish(md5, sha1); public override void Dispose() => _writer.Dispose(); } } @@ -328,12 +417,147 @@ public override ValueTask WriteAsync(ReadOnlyMemory data, CancellationToke /// public static class ImageConverter { + /// + /// : the written output, re-read, reproduces the digest computed + /// while writing. : the source container's own recorded + /// acquisition hash reproduced while reading (null when the source recorded none). + /// public sealed record ConversionResult( ImageJobResult Image, string? RecordedMd5, string? RecordedSha1, int DamagedChunks, - bool? MatchesRecorded); + bool? MatchesRecorded, + bool? SourceMatchesRecorded = null); + + /// + /// Any supported source (raw, E01, AFF4, VHD, VHDX) to any in-process target. The output + /// is re-read through the matching reader and must hash back to what was written; a + /// container source is additionally checked against its recorded acquisition hash. + /// + public static async Task ConvertAsync( + string source, string output, ImageFormat format, string? examiner = null, + int compressionLevel = 1, long segmentSizeMiB = 1500, bool verify = true, + IProgress? progress = null, CancellationToken ct = default) + { + if (!InProcessImager.Supports(format)) + { + throw new NotSupportedException($"{format} cannot be written in-process."); + } + + string? recordedMd5 = null, recordedSha1 = null; + var sourceDamaged = 0; + ImageJobResult result; + var job = new ImageJob(source, output, format, CompressionLevel: compressionLevel, SegmentSizeMiB: segmentSizeMiB, + ExaminerName: examiner, Description: $"Converted from {Path.GetFileName(source)}"); + + var ext = Path.GetExtension(source).ToLowerInvariant(); + if (ext is ".vhd" or ".vhdx") + { + using var file = new FileStream(source, FileMode.Open, FileAccess.Read, FileShare.Read); + using DiscUtils.VirtualDisk disk = ext == ".vhd" + ? new DiscUtils.Vhd.Disk(file, DiscUtils.Streams.Ownership.None) + : new DiscUtils.Vhdx.Disk(file, DiscUtils.Streams.Ownership.None); + result = await new InProcessImager().ImageStreamAsync(disk.Content, job, progress, ct).ConfigureAwait(false); + } + else if (EvidenceOpener.IsEwf(source)) + { + using var ewf = EwfReader.Open(source); + await using var stream = ewf.OpenStream(); + result = await new InProcessImager().ImageStreamAsync(stream, job, progress, ct).ConfigureAwait(false); + recordedMd5 = ewf.RecordedMd5; + recordedSha1 = ewf.RecordedSha1; + sourceDamaged = ewf.DamagedChunks.Count; + } + else if (EvidenceOpener.IsAff4(source)) + { + using var aff4 = Aff4.Aff4Reader.Open(source); + await using var stream = aff4.OpenStream(); + result = await new InProcessImager().ImageStreamAsync(stream, job, progress, ct).ConfigureAwait(false); + recordedMd5 = aff4.RecordedMd5; + recordedSha1 = aff4.RecordedSha1; + sourceDamaged = aff4.DamagedChunks.Count; + } + else + { + result = await new InProcessImager().ImageAsync(job, progress, ct).ConfigureAwait(false); + } + + bool? sourceOk = null; + if (recordedMd5 is not null && result.Md5 is not null) + { + sourceOk = string.Equals(recordedMd5, result.Md5, StringComparison.OrdinalIgnoreCase) && sourceDamaged == 0; + } + else if (recordedSha1 is not null && result.Sha1 is not null) + { + sourceOk = string.Equals(recordedSha1, result.Sha1, StringComparison.OrdinalIgnoreCase) && sourceDamaged == 0; + } + + if (!verify) + { + return new ConversionResult(result, recordedMd5, recordedSha1, sourceDamaged, null, sourceOk); + } + + var (rereadMd5, damaged) = await RereadMd5Async(output, format, ct).ConfigureAwait(false); + var outputOk = rereadMd5 is null ? (bool?)null + : string.Equals(rereadMd5, result.Md5, StringComparison.OrdinalIgnoreCase) && damaged == 0; + return new ConversionResult(result, recordedMd5, recordedSha1, damaged, outputOk, sourceOk); + } + + private static async Task<(string? Md5, int Damaged)> RereadMd5Async(string path, ImageFormat format, CancellationToken ct) + { + using var md5 = System.Security.Cryptography.IncrementalHash.CreateHash(System.Security.Cryptography.HashAlgorithmName.MD5); + var buffer = new byte[1 << 20]; + var damaged = 0; + + async Task HashAsync(Stream s) + { + while (true) + { + ct.ThrowIfCancellationRequested(); + var n = await s.ReadAsync(buffer, ct).ConfigureAwait(false); + if (n == 0) break; + md5.AppendData(buffer, 0, n); + } + } + + switch (format) + { + case ImageFormat.Vhd: + case ImageFormat.Vhdx: + { + using var file = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read); + using DiscUtils.VirtualDisk disk = format == ImageFormat.Vhd + ? new DiscUtils.Vhd.Disk(file, DiscUtils.Streams.Ownership.None) + : new DiscUtils.Vhdx.Disk(file, DiscUtils.Streams.Ownership.None); + await HashAsync(disk.Content).ConfigureAwait(false); + break; + } + case ImageFormat.Ewf: + { + using var ewf = EwfReader.Open(path); + await using var s = ewf.OpenStream(); + await HashAsync(s).ConfigureAwait(false); + damaged = ewf.DamagedChunks.Count; + break; + } + case ImageFormat.Aff4: + { + using var aff4 = Aff4.Aff4Reader.Open(path); + await using var s = aff4.OpenStream(); + await HashAsync(s).ConfigureAwait(false); + damaged = aff4.DamagedChunks.Count; + break; + } + default: + { + await using var s = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, 1 << 20, FileOptions.SequentialScan); + await HashAsync(s).ConfigureAwait(false); + break; + } + } + return (Convert.ToHexStringLower(md5.GetHashAndReset()), damaged); + } public static async Task EwfToRawAsync( string sourceE01, string outputRaw, string? examiner = null, diff --git a/tests/Cinder.Core.Tests/NtfsLogFileTests.cs b/tests/Cinder.Core.Tests/NtfsLogFileTests.cs new file mode 100644 index 0000000..66195da --- /dev/null +++ b/tests/Cinder.Core.Tests/NtfsLogFileTests.cs @@ -0,0 +1,296 @@ +using System.Buffers.Binary; +using System.Text; +using Cinder.Filesystems; +using FluentAssertions; +using Xunit; + +namespace Cinder.Core.Tests; + +/// +/// Builds a synthetic $LogFile — restart page, fixup-protected RCRD pages, records that +/// carry FILE-record and index-entry payloads, one record spanning two pages, and stale slack +/// from an earlier cycle — and checks the parser recovers every field and every name. +/// +public sealed class NtfsLogFileTests +{ + private const int PageSize = 4096; + private const int DataOffset = 0x40; + private const int SeqBits = 44; + private const ushort Usn = 0x1234; + + [Fact] + public void Parses_records_names_transactions_and_multi_page_continuation() + { + var log = new LogBuilder(); + var t1 = 0x11u; + + // Page 2: a file record initialised (name from the FILE record), the entry added to + // its parent's index, then a long record that spills into page 3. + var lsnInit = log.Add(t1, redoOp: 0x02, undoOp: 0x03, redo: FileRecord(recordNo: 45, name: "invoice.pdf.exe", parent: 5), targetVcn: 11, clusterBlock: 2); + var lsnAdd = log.Add(t1, redoOp: 0x0E, undoOp: 0x0F, redo: IndexEntry(childRecord: 45, name: "invoice.pdf.exe", parent: 5), targetVcn: 900, clusterBlock: 0); + var lsnLong = log.Add(t1, redoOp: 0x08, undoOp: 0x08, redo: new byte[3000], targetVcn: 1234, clusterBlock: 0); + + // Page 3 (after the continuation tail): a delete whose undo payload carries the name. + var lsnDel = log.Add(0x12, redoOp: 0x0F, undoOp: 0x0E, redo: [], undo: IndexEntry(childRecord: 45, name: "invoice.pdf.exe", parent: 5), targetVcn: 900, clusterBlock: 0); + var lsnCommit = log.Add(0x12, redoOp: 0x1B, undoOp: 0x00, redo: [], targetVcn: 0, clusterBlock: 0); + + // Stale slack past next_record_offset on the last page, from an "earlier cycle". + var bytes = log.Build(staleName: "secret-plan.docx"); + + using var ms = new MemoryStream(bytes); + var records = NtfsLogFile.Parse(ms, clusterSize: 4096).ToList(); + + records.Should().HaveCountGreaterThanOrEqualTo(6); + var init = records.Single(r => r.Lsn == lsnInit); + init.RedoText.Should().Be("InitializeFileRecordSegment"); + init.UndoText.Should().Be("DeallocateFileRecordSegment"); + init.FileName.Should().Be("invoice.pdf.exe"); + init.ParentMftRecord.Should().Be(5); + init.MftRecordNumber.Should().Be(45, "the FILE record's own number wins over the VCN estimate"); + init.TransactionId.Should().Be(t1); + init.Created.Should().NotBeNull(); + init.IsStale.Should().BeFalse(); + + var add = records.Single(r => r.Lsn == lsnAdd); + add.RedoText.Should().Be("AddIndexEntryAllocation"); + add.FileName.Should().Be("invoice.pdf.exe"); + add.MftRecordNumber.Should().Be(45, "the index entry's file reference"); + + var lng = records.Single(r => r.Lsn == lsnLong); + lng.IsTruncated.Should().BeFalse("the record continues into the next page and is reassembled"); + lng.TargetVcn.Should().Be(1234); + + var del = records.Single(r => r.Lsn == lsnDel); + del.RedoText.Should().Be("DeleteIndexEntryAllocation"); + del.FileName.Should().Be("invoice.pdf.exe", "the undo payload of a delete is the entry that was removed"); + + records.Single(r => r.Lsn == lsnCommit).RedoText.Should().Be("ForgetTransaction"); + + var stale = records.Where(r => r.IsStale).ToList(); + stale.Should().NotBeEmpty(); + stale.Should().Contain(r => r.FileName == "secret-plan.docx"); + + records.Select(r => r.Lsn).Should().OnlyHaveUniqueItems("buffer pages mirror record pages and must not double-count"); + } + + [Fact] + public void Torn_page_fixups_are_not_applied_and_garbage_does_not_throw() + { + var log = new LogBuilder(); + log.Add(1, 0x02, 0x03, FileRecord(7, "a.txt", 5), 0, 0); + var bytes = log.Build(); + + // Corrupt one sector's USN on page 2: the fixup check fails and the page is left alone. + bytes[2 * PageSize + 512 - 2] ^= 0xFF; + using var ms = new MemoryStream(bytes); + var act = () => NtfsLogFile.Parse(ms).ToList(); + act.Should().NotThrow(); + + // Random bytes are not a log file either. + var junk = new byte[PageSize * 6]; + new Random(3).NextBytes(junk); + using var ms2 = new MemoryStream(junk); + NtfsLogFile.Parse(ms2).ToList().Should().BeEmpty(); + } + + [Fact] + public void Operation_names_match_microsofts() + { + NtfsLogFile.DescribeOperation(0x02).Should().Be("InitializeFileRecordSegment"); + NtfsLogFile.DescribeOperation(0x0F).Should().Be("DeleteIndexEntryAllocation"); + NtfsLogFile.DescribeOperation(0x1B).Should().Be("ForgetTransaction"); + NtfsLogFile.DescribeOperation(0x77).Should().Be("0x77"); + } + + // ---- payload builders ---------------------------------------------------------------- + + private static byte[] FileNameValue(string name, long parent) + { + var n = Encoding.Unicode.GetBytes(name); + var v = new byte[0x42 + n.Length]; + BinaryPrimitives.WriteUInt64LittleEndian(v, (ulong)parent | (5UL << 48)); + var now = DateTimeOffset.UtcNow.ToFileTime(); + for (int i = 0; i < 4; i++) BinaryPrimitives.WriteInt64LittleEndian(v.AsSpan(8 + i * 8), now); + v[0x40] = (byte)name.Length; + v[0x41] = 3; + n.CopyTo(v, 0x42); + return v; + } + + private static byte[] FileRecord(long recordNo, string name, long parent) + { + var fn = FileNameValue(name, parent); + var attrLen = (0x18 + fn.Length + 7) & ~7; + var r = new byte[0x38 + attrLen + 8]; + "FILE"u8.CopyTo(r); + BinaryPrimitives.WriteUInt16LittleEndian(r.AsSpan(0x14), 0x38); + BinaryPrimitives.WriteUInt32LittleEndian(r.AsSpan(0x2C), (uint)recordNo); + var a = r.AsSpan(0x38); + BinaryPrimitives.WriteUInt32LittleEndian(a, 0x30); + BinaryPrimitives.WriteUInt32LittleEndian(a[4..], (uint)attrLen); + a[8] = 0; + BinaryPrimitives.WriteUInt32LittleEndian(a[0x10..], (uint)fn.Length); + BinaryPrimitives.WriteUInt16LittleEndian(a[0x14..], 0x18); + fn.CopyTo(a[0x18..]); + BinaryPrimitives.WriteUInt32LittleEndian(r.AsSpan(0x38 + attrLen), 0xFFFFFFFF); + return r; + } + + private static byte[] IndexEntry(long childRecord, string name, long parent) + { + var fn = FileNameValue(name, parent); + var e = new byte[0x10 + fn.Length]; + BinaryPrimitives.WriteUInt64LittleEndian(e, (ulong)childRecord | (2UL << 48)); + BinaryPrimitives.WriteUInt16LittleEndian(e.AsSpan(8), (ushort)e.Length); + BinaryPrimitives.WriteUInt16LittleEndian(e.AsSpan(10), (ushort)fn.Length); + fn.CopyTo(e, 0x10); + return e; + } + + /// Lays records out page by page exactly as NTFS does, including continuation and fixups. + private sealed class LogBuilder + { + private readonly List _pages = []; + private byte[] _page = NewPage(); + private int _pos = DataOffset; + private long _prevLsn; + + private static byte[] NewPage() + { + var p = new byte[PageSize]; + "RCRD"u8.CopyTo(p); + BinaryPrimitives.WriteUInt16LittleEndian(p.AsSpan(4), 0x28); // USA offset + BinaryPrimitives.WriteUInt16LittleEndian(p.AsSpan(6), 9); // USA count: 1 + 8 sectors + return p; + } + + private long CurrentFileOffset => (2 + _pages.Count) * (long)PageSize + _pos; + private static long LsnFor(long fileOffset) => (fileOffset / 8) | (1L << SeqBits); + + public long Add(uint txn, ushort redoOp, ushort undoOp, byte[] redo, long targetVcn, ushort clusterBlock, byte[]? undo = null) + { + undo ??= []; + var body = new byte[0x20 + Pad8(redo.Length) + Pad8(undo.Length)]; + BinaryPrimitives.WriteUInt16LittleEndian(body, redoOp); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(2), undoOp); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(4), 0x20); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(6), (ushort)redo.Length); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(8), (ushort)(0x20 + Pad8(redo.Length))); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(10), (ushort)undo.Length); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(20), clusterBlock); + BinaryPrimitives.WriteInt64LittleEndian(body.AsSpan(24), targetVcn); + redo.CopyTo(body, 0x20); + undo.CopyTo(body, 0x20 + Pad8(redo.Length)); + + if (_pos + 0x30 > PageSize) + { + Flush(); + } + var lsn = LsnFor(CurrentFileOffset); + var header = new byte[0x30]; + BinaryPrimitives.WriteInt64LittleEndian(header, lsn); + BinaryPrimitives.WriteInt64LittleEndian(header.AsSpan(8), _prevLsn); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x18), (uint)body.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x20), 1); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x24), txn); + header.CopyTo(_page, _pos); + _pos += 0x30; + + var written = 0; + while (written < body.Length) + { + var room = PageSize - _pos; + if (room == 0) + { + Flush(); + room = PageSize - _pos; + } + var take = Math.Min(room, body.Length - written); + body.AsSpan(written, take).CopyTo(_page.AsSpan(_pos)); + _pos += take; + written += take; + } + _pos = (_pos + 7) & ~7; + BinaryPrimitives.WriteUInt16LittleEndian(_page.AsSpan(0x18), (ushort)Math.Min(_pos, PageSize)); + BinaryPrimitives.WriteInt64LittleEndian(_page.AsSpan(0x20), lsn); // last_end_lsn + _prevLsn = lsn; + return lsn; + } + + private void Flush() + { + _pages.Add(_page); + _page = NewPage(); + _pos = DataOffset; + } + + public byte[] Build(string? staleName = null) + { + if (staleName is not null) + { + // A record from an earlier cycle sitting right after the live data, past + // next_record_offset — exactly where circular-log slack lives. + var entry = IndexEntry(childRecord: 99, name: staleName, parent: 5); + var body = new byte[0x20 + entry.Length]; + if (_pos + 0x30 + body.Length > PageSize) + { + Flush(); + BinaryPrimitives.WriteUInt16LittleEndian(_page.AsSpan(0x18), DataOffset); + } + var stalePos = _pos; + BinaryPrimitives.WriteUInt16LittleEndian(body, 0x0E); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(2), 0x0F); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(4), 0x20); + BinaryPrimitives.WriteUInt16LittleEndian(body.AsSpan(6), (ushort)entry.Length); + entry.CopyTo(body, 0x20); + var header = new byte[0x30]; + BinaryPrimitives.WriteInt64LittleEndian(header, LsnFor(1_000_000)); // wrong offset: an old cycle + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x18), (uint)body.Length); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x20), 1); + BinaryPrimitives.WriteUInt32LittleEndian(header.AsSpan(0x24), 0x0A); + header.CopyTo(_page, stalePos); + body.CopyTo(_page, stalePos + 0x30); + } + _pages.Add(_page); + + var restart = new byte[PageSize]; + "RSTR"u8.CopyTo(restart); + BinaryPrimitives.WriteUInt16LittleEndian(restart.AsSpan(4), 0x1E); + BinaryPrimitives.WriteUInt16LittleEndian(restart.AsSpan(6), 9); + BinaryPrimitives.WriteUInt32LittleEndian(restart.AsSpan(0x10), PageSize); + BinaryPrimitives.WriteUInt32LittleEndian(restart.AsSpan(0x14), PageSize); + BinaryPrimitives.WriteUInt16LittleEndian(restart.AsSpan(0x18), 0x30); + BinaryPrimitives.WriteUInt32LittleEndian(restart.AsSpan(0x30 + 0x0C), SeqBits); + BinaryPrimitives.WriteUInt16LittleEndian(restart.AsSpan(0x30 + 0x1A), DataOffset); + Fixup(restart, 0x1E); + + var all = new List { restart, (byte[])restart.Clone() }; + foreach (var p in _pages) + { + Fixup(p, 0x28); + all.Add(p); + } + // Pages 2 and 3 in a real log are buffer copies of the newest pages; mimic that + // by inserting duplicates of the first two record pages ahead of the run. + all.Insert(2, (byte[])all[2].Clone()); + if (_pages.Count > 1) all.Insert(3, (byte[])all[4].Clone()); + + var result = new byte[all.Count * PageSize]; + for (int i = 0; i < all.Count; i++) all[i].CopyTo(result, i * PageSize); + return result; + } + + private static void Fixup(byte[] page, int usaOfs) + { + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(usaOfs), Usn); + for (int i = 1; i <= 8; i++) + { + var end = i * 512 - 2; + page.AsSpan(end, 2).CopyTo(page.AsSpan(usaOfs + i * 2)); + BinaryPrimitives.WriteUInt16LittleEndian(page.AsSpan(end), Usn); + } + } + + private static int Pad8(int n) => (n + 7) & ~7; + } +} diff --git a/tests/Cinder.Imaging.Tests/Aff4Tests.cs b/tests/Cinder.Imaging.Tests/Aff4Tests.cs new file mode 100644 index 0000000..6d69657 --- /dev/null +++ b/tests/Cinder.Imaging.Tests/Aff4Tests.cs @@ -0,0 +1,242 @@ +using System.IO.Compression; +using System.Security.Cryptography; +using System.Text; +using Cinder.Imaging.Aff4; +using FluentAssertions; +using Xunit; + +namespace Cinder.Imaging.Tests; + +public sealed class Aff4Tests : IDisposable +{ + private readonly string _dir = Directory.CreateTempSubdirectory("cinder-aff4").FullName; + + public void Dispose() + { + try { Directory.Delete(_dir, recursive: true); } catch { } + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Round_trips_through_the_reader_with_digests_and_metadata(bool compress) + { + var media = EwfBuilder.Media(2 * 1024 * 1024 + 4321); // ragged tail, several bevies + var path = Path.Combine(_dir, "rt.aff4"); + using (var w = new Aff4Writer(path, new Aff4Writer.Options + { + Compress = compress, + ChunkSize = 4096, + ChunksPerSegment = 64, + CaseNumber = "C-9", + Examiner = "dana", + Description = "round \"trip\"\nwith newline", + })) + { + var pos = 0; + while (pos < media.Length) + { + var n = Math.Min(7777, media.Length - pos); + w.Write(media.AsSpan(pos, n)); + pos += n; + } + w.Finish(MD5.HashData(media), SHA1.HashData(media), SHA256.HashData(media)); + } + + Aff4Reader.IsAff4(path).Should().BeTrue(); + using var r = Aff4Reader.Open(path); + r.Size.Should().Be(media.Length); + r.RecordedMd5.Should().Be(Convert.ToHexStringLower(MD5.HashData(media))); + r.RecordedSha256.Should().Be(Convert.ToHexStringLower(SHA256.HashData(media))); + r.Metadata["caseNumber"].Should().Be("C-9"); + r.Metadata["examiner"].Should().Be("dana"); + r.Metadata["caseDescription"].Should().Be("round \"trip\"\nwith newline"); + + await using var s = r.OpenStream(); + var back = new byte[media.Length]; + var filled = 0; + while (filled < back.Length) + { + var n = await s.ReadAsync(back.AsMemory(filled)); + if (n == 0) break; + filled += n; + } + filled.Should().Be(media.Length); + back.Should().Equal(media); + r.DamagedChunks.Should().BeEmpty(); + + // Random access across a bevy boundary. + s.Position = 64 * 4096 - 100; + var window = new byte[300]; + s.Read(window).Should().Be(300); + window.Should().Equal(media.AsSpan(64 * 4096 - 100, 300).ToArray()); + + // Structure a third-party reader relies on. + using var zip = ZipFile.OpenRead(path); + zip.Comment.Should().Be(r.VolumeUrn); + zip.GetEntry("container.description").Should().NotBeNull(); + zip.GetEntry("information.turtle").Should().NotBeNull(); + zip.Entries.Should().Contain(e => e.FullName.StartsWith("aff4%3A%2F%2F") && e.FullName.EndsWith("/00000000")); + zip.Entries.Should().Contain(e => e.FullName.EndsWith("/00000000.index")); + zip.Entries.Should().Contain(e => e.FullName.EndsWith("/map")); + } + + [Fact] + public async Task Evidence_opener_and_imager_handle_aff4() + { + var data = EwfBuilder.Media(1024 * 1024); + var src = Path.Combine(_dir, "src.dd"); + await File.WriteAllBytesAsync(src, data); + var out1 = Path.Combine(_dir, "img.aff4"); + + var result = await new InProcessImager().ImageAsync(new ImageJob(src, out1, ImageFormat.Aff4, ExaminerName: "eve", CaseNumber: "C-1")); + result.Sha256.Should().Be(Convert.ToHexStringLower(SHA256.HashData(data))); + File.Exists(out1 + ".log.json").Should().BeTrue(); + File.Exists(out1 + ".sha256").Should().BeFalse("AFF4 carries its digests inside"); + + EvidenceOpener.IsAff4(out1).Should().BeTrue(); + EvidenceOpener.IsEwf(out1).Should().BeFalse(); + await using var s = EvidenceOpener.Open(out1); + s.Length.Should().Be(data.Length); + var back = new byte[data.Length]; + var filled = 0; + while (filled < back.Length) + { + var n = await s.ReadAsync(back.AsMemory(filled)); + if (n == 0) break; + filled += n; + } + back.Should().Equal(data); + + // AFF4 → raw through the imager's source opener too. + var raw = Path.Combine(_dir, "back.dd"); + var r2 = await new InProcessImager().ImageAsync(new ImageJob(out1, raw, ImageFormat.Raw)); + (await File.ReadAllBytesAsync(raw)).Should().Equal(data); + r2.Md5.Should().Be(result.Md5); + } + + [Fact] + public void Reads_unencoded_member_names_and_rdflib_style_turtle() + { + // A container as pyaff4 0.3x writes it: unencoded member names, bare integer literals, + // no Image object — just a Map over a stream. + var media = EwfBuilder.Media(10_000); + var path = Path.Combine(_dir, "v11.aff4"); + const string vol = "aff4://11111111-1111-1111-1111-111111111111"; + const string map = "aff4://22222222-2222-2222-2222-222222222222"; + const string stream = "aff4://33333333-3333-3333-3333-333333333333"; + + using (var zip = ZipFile.Open(path, ZipArchiveMode.Create)) + { + zip.Comment = vol; + Add(zip, "container.description", Encoding.UTF8.GetBytes(vol)); + Add(zip, "version.txt", Encoding.ASCII.GetBytes("major=1\nminor=1\ntool=pyaff4\n")); + + var bevy = new MemoryStream(); + var index = new MemoryStream(); + for (int off = 0; off < media.Length; off += 4096) + { + var start = bevy.Length; + var chunk = media.AsSpan(off, Math.Min(4096, media.Length - off)); + using (var z = new ZLibStream(bevy, CompressionLevel.Optimal, leaveOpen: true)) z.Write(chunk); + index.Write(BitConverter.GetBytes(start)); + index.Write(BitConverter.GetBytes((int)(bevy.Length - start))); + } + Add(zip, stream + "/00000000", bevy.ToArray()); + Add(zip, stream + "/00000000.index", index.ToArray()); + + var range = new byte[28]; + BitConverter.GetBytes((long)media.Length).CopyTo(range, 8); + Add(zip, map + "/map", range); + Add(zip, map + "/idx", Encoding.UTF8.GetBytes(stream + "\n")); + + Add(zip, "information.turtle", Encoding.UTF8.GetBytes($""" + @prefix aff4: . + @prefix rdf: . + @prefix xsd: . + + <{map}> a aff4:Map ; + aff4:dependentStream <{stream}> ; + aff4:size {media.Length} ; + aff4:stored <{vol}> . + + <{stream}> a aff4:ImageStream ; + aff4:chunkSize 4096 ; + aff4:chunksInSegment 2048 ; + aff4:compressionMethod ; + aff4:size {media.Length} ; + aff4:stored <{vol}> ; + aff4:version 1 . + """)); + } + + using var r = Aff4Reader.Open(path); + r.ImageUrn.Should().Be(map); + r.Size.Should().Be(media.Length); + using var s = r.OpenStream(); + var back = new byte[media.Length]; + s.Read(back).Should().Be(media.Length); + back.Should().Equal(media); + } + + [Fact] + public void Snappy_and_lz4_decoders_match_the_reference_implementations() + { + // Vectors produced by python-snappy and lz4.block (store_size=False) over the same input. + var expectedSha = "b7af7ca825f35bfa74502e09e9819e9b04f1c8504089b188bec76046317fc7a5"; + const int length = 2568; + var snappy = Convert.FromBase64String("iBSwVGhlIHF1aWNrIGJyb3duIGZveCBqdW1wcyBvdmVyIHRoZSBsYXp5IGRvZy4g/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0A/i0Aai0A9AUBAAECAwQFBgcICQoLDA0ODxAREhMUFRYXGBkaGxwdHh8gISIjJCUmJygpKissLS4vMDEyMzQ1Njc4OTo7PD0+P0BBQkNERUZHSElKS0xNTk9QUVJTVFVWV1hZWltcXV5fYGFiY2RlZmdoaWprbG1ub3BxcnN0dXZ3eHl6e3x9fn+AgYKDhIWGh4iJiouMjY6PkJGSk5SVlpeYmZqbnJ2en6ChoqOkpaanqKmqq6ytrq+wsbKztLW2t7i5uru8vb6/wMHCw8TFxsfIycrLzM3Oz9DR0tPU1dbX2Nna29zd3t/g4eLj5OXm5+jp6uvs7e7v8PHy8/T19vf4+fr7/P3+/wABAgMEBf4AAf4AAf4AAf4AAf4AAf4AAf4AAeYAAQ=="); + var lz4 = Convert.FromBase64String("/x5UaGUgcXVpY2sgYnJvd24gZm94IGp1bXBzIG92ZXIgdGhlIGxhenkgZG9nLiAtAP///////87/8QABAgMEBQYHCAkKCwwNDg8QERITFBUWFxgZGhscHR4fICEiIyQlJicoKSorLC0uLzAxMjM0NTY3ODk6Ozw9Pj9AQUJDREVGR0hJSktMTU5PUFFSU1RVVldYWVpbXF1eX2BhYmNkZWZnaGlqa2xtbm9wcXJzdHV2d3h5ent8fX5/gIGCg4SFhoeIiYqLjI2Oj5CRkpOUlZaXmJmam5ydnp+goaKjpKWmp6ipqqusra6vsLGys7S1tre4ubq7vL2+v8DBwsPExcbHyMnKy8zNzs/Q0dLT1NXW19jZ2tvc3d7f4OHi4+Tl5ufo6err7O3u7/Dx8vP09fb3+Pn6+/z9/v8AAf/pUPv8/f7/"); + + var a = new byte[length]; + SnappyDecoder.Decode(snappy, a).Should().Be(length); + Convert.ToHexStringLower(SHA256.HashData(a)).Should().Be(expectedSha); + + var b = new byte[length]; + Lz4Decoder.Decode(lz4, b).Should().Be(length); + Convert.ToHexStringLower(SHA256.HashData(b)).Should().Be(expectedSha); + + // Hostile input must not escape the output buffer. + var act = () => SnappyDecoder.Decode(new byte[] { 0x10, 0xFC, 0xFF, 0xFF, 0xFF }, new byte[16]); + act.Should().Throw(); + var act2 = () => Lz4Decoder.Decode(new byte[] { 0x0F, 0x01, 0xFF, 0xFF }, new byte[16]); + act2.Should().Throw(); + } + + [Theory] + [InlineData(ImageFormat.Vhd)] + [InlineData(ImageFormat.Vhdx)] + public async Task Imager_writes_virtual_disks_that_discutils_reads_back(ImageFormat format) + { + var data = EwfBuilder.Media(3 * 1024 * 1024 + 512); + var src = Path.Combine(_dir, "src.dd"); + await File.WriteAllBytesAsync(src, data); + var outPath = Path.Combine(_dir, "disk" + format.DefaultExtension()); + + var result = await new InProcessImager().ImageAsync(new ImageJob(src, outPath, format)); + result.Sha256.Should().Be(Convert.ToHexStringLower(SHA256.HashData(data))); + + using var file = File.OpenRead(outPath); + using DiscUtils.VirtualDisk disk = format == ImageFormat.Vhd + ? new DiscUtils.Vhd.Disk(file, DiscUtils.Streams.Ownership.None) + : new DiscUtils.Vhdx.Disk(file, DiscUtils.Streams.Ownership.None); + disk.Capacity.Should().Be(data.Length); + var back = new byte[data.Length]; + disk.Content.Position = 0; + var filled = 0; + while (filled < back.Length) + { + var n = disk.Content.Read(back, filled, back.Length - filled); + if (n <= 0) break; + filled += n; + } + back.Should().Equal(data); + } + + private static void Add(ZipArchive zip, string name, byte[] data) + { + var e = zip.CreateEntry(name, CompressionLevel.NoCompression); + using var s = e.Open(); + s.Write(data); + } +} diff --git a/tests/Cinder.Imaging.Tests/InProcessImagerTests.cs b/tests/Cinder.Imaging.Tests/InProcessImagerTests.cs index c6bf3e8..6b5a6cb 100644 --- a/tests/Cinder.Imaging.Tests/InProcessImagerTests.cs +++ b/tests/Cinder.Imaging.Tests/InProcessImagerTests.cs @@ -108,10 +108,12 @@ public void Device_paths_are_recognised_on_both_platforms() } [Fact] - public async Task Refuses_formats_it_cannot_write() + public void Writes_every_image_format_in_process() { - var act = async () => await new InProcessImager().ImageAsync(new ImageJob("x", Path.Combine(_dir, "y"), ImageFormat.Aff4)); - await act.Should().ThrowAsync(); + foreach (var f in Enum.GetValues()) + { + InProcessImager.Supports(f).Should().BeTrue($"{f} should no longer need the sidecar"); + } } }