Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ Options:
--runtime-url <URL> URL to download uruntime from [env: URUNTIME_LINK]
-u, --update-info <UPINFO> Update information string [env: UPINFO]
--dwarfs-comp <COMP> DWARFS compression options [env: DWARFS_COMP]
--source-date-epoch <EPOCH> Pin every timestamp in the image [env: SOURCE_DATE_EPOCH]
--optimize-launch Enable DWARFS profile optimization (also via OPTIMIZE_LAUNCH=1)
--profile-timeout <SECS> Profiling timeout in seconds [env: OPTIMIZE_LAUNCH_TIMEOUT] [default: 10]
--keep-mount Keep the FUSE mount alive after exit (also via URUNTIME_PRELOAD=1)
Expand Down Expand Up @@ -111,6 +112,22 @@ Every CLI option has a matching env var (shown above). A few extra knobs that ar
| `OPTIMIZE_LAUNCH_TIMEOUT` | Profiling timeout in seconds (default `10`). |
| `SKIP_INTEGRITY_CHECKS` | Set to `1` to skip the pinned uruntime SHA-256 verification. |

### Reproducible builds

Set `SOURCE_DATE_EPOCH` (or `--source-date-epoch`) to a unix timestamp to pin
the timestamps of every entry in the image. Without it the image inherits
whatever timestamps the build host produced, plus the `.env` and desktop entry
that appimagetool rewrites moments before packing, so two builds of the same
AppDir differ. With `--order=path`, `--no-history`, `--no-create-timestamp`,
pinned owner/group and a pinned segmenter-worker count — all passed already —
this makes the DWARFS image bit-identical for a given AppDir, pinned
`mkdwarfs`/uruntime and compression settings.

`OPTIMIZE_LAUNCH` does not combine with reproducibility: the profile pass
records which files the AppImage touches while running, which varies from run
to run. Supply a fixed `DWARFSPROF` instead if you want the categorization
without the profiling pass.

### AppDir requirements

The AppDir must contain:
Expand Down
9 changes: 9 additions & 0 deletions src/appimage.rs
Original file line number Diff line number Diff line change
Expand Up @@ -93,6 +93,14 @@ pub fn build(config: &Config) -> Result<()> {

// DWARFS profile optimization (optional)
let profile = if config.optimize_launch {
if config.source_date_epoch.is_some() {
crate::log_warn!(
"SOURCE_DATE_EPOCH is set, but OPTIMIZE_LAUNCH records a hotness \
profile by running the AppImage; the recorded profile is not \
reproducible"
);
}

// Profiling launches the AppImage, which mounts via FUSE. Fail fast in
// environments (typically minimal containers) where FUSE isn't usable.
dwarfs::check_fuse_available()?;
Expand Down Expand Up @@ -129,6 +137,7 @@ pub fn build(config: &Config) -> Result<()> {
&runtime_path,
&output_path,
&config.dwarfs_comp,
config.source_date_epoch,
profile.as_deref(),
)?;

Expand Down
59 changes: 59 additions & 0 deletions src/config.rs
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,8 @@
use std::env;
use std::path::PathBuf;

use crate::error::{Error, Result};

fn env_opt(name: &str) -> Option<String> {
env::var(name).ok()
}
Expand Down Expand Up @@ -38,6 +40,9 @@ pub struct Config {
pub runtime_url: Option<String>,
/// Compression options passed to `mkdwarfs`.
pub dwarfs_comp: String,
/// Unix timestamp applied to every entry in the image. Makes the build
/// reproducible; see [`CliArgs::source_date_epoch`].
pub source_date_epoch: Option<u64>,
/// `upd_info` ELF section payload (zsync URL or similar).
pub update_info: Option<String>,
/// Permanent env-var lines to bake into the runtime's `.envs` section.
Expand Down Expand Up @@ -90,6 +95,9 @@ pub struct CliArgs {
pub update_info: Option<String>,
/// `mkdwarfs` compression option string.
pub dwarfs_comp: Option<String>,
/// Pin every timestamp in the image to this unix timestamp.
/// Also `SOURCE_DATE_EPOCH`.
pub source_date_epoch: Option<String>,
/// Enable the DWARFS profiling pass.
pub optimize_launch: bool,
/// Path to an existing DWARFS profile to feed into the build.
Expand Down Expand Up @@ -174,6 +182,11 @@ impl Config {
.dwarfs_comp
.unwrap_or_else(|| "zstd:level=22 -S26 -B6".to_string());

let source_date_epoch = parse_source_date_epoch(
args.source_date_epoch
.or_else(|| env_opt("SOURCE_DATE_EPOCH")),
)?;

Ok(Config {
appdir,
output_dir: args.output.unwrap_or_else(|| PathBuf::from(".")),
Expand All @@ -183,6 +196,7 @@ impl Config {
runtime: args.runtime,
runtime_url: args.runtime_url,
dwarfs_comp,
source_date_epoch,
update_info,
env_vars,
dwarfs_profile,
Expand Down Expand Up @@ -221,6 +235,27 @@ fn dirs_home() -> PathBuf {
.unwrap_or_else(|_| PathBuf::from("~"))
}

/// Parse `SOURCE_DATE_EPOCH` into a unix timestamp.
///
/// The spec defines the value as the output of `date +%s`, so only ASCII
/// digits are accepted — `u64::from_str` would also take a leading `+`.
/// Failing here rather than in `mkdwarfs` keeps the AppDir untouched when the
/// value is malformed.
fn parse_source_date_epoch(value: Option<String>) -> Result<Option<u64>> {
let Some(value) = value.filter(|v| !v.is_empty()) else {
return Ok(None);
};
if !value.bytes().all(|b| b.is_ascii_digit()) {
return Err(Error::Config(format!(
"SOURCE_DATE_EPOCH must be a unix timestamp in seconds, got '{value}'"
)));
}
value
.parse()
.map(Some)
.map_err(|_| Error::Config(format!("SOURCE_DATE_EPOCH is out of range: '{value}'")))
}

/// Strip the epoch prefix from a version string.
/// Shell equivalent: `${VERSION#*:}` — removes everything up to
/// and including the first colon. `"1:2.0.1"` → `"2.0.1"`.
Expand All @@ -244,6 +279,30 @@ mod tests {
assert_eq!(strip_epoch("0:1.0.0-alpha"), "1.0.0-alpha");
}

#[test]
fn test_parse_source_date_epoch() {
assert_eq!(parse_source_date_epoch(None).unwrap(), None);
assert_eq!(parse_source_date_epoch(Some(String::new())).unwrap(), None);
assert_eq!(
parse_source_date_epoch(Some("1700000000".to_string())).unwrap(),
Some(1_700_000_000)
);
}

#[test]
fn test_parse_source_date_epoch_rejects_anything_else() {
// `date +%s` never emits these, and a leading `+` would otherwise be
// accepted by `u64::from_str`.
for bad in ["now", "-1", "+1700000000", "1.5", "1700000000s", " "] {
assert!(
parse_source_date_epoch(Some(bad.to_string())).is_err(),
"expected {bad:?} to be rejected"
);
}
// More digits than u64 holds is a range error, not a format error.
assert!(parse_source_date_epoch(Some("9".repeat(21))).is_err());
}

fn args_with_appdir() -> CliArgs {
CliArgs {
appdir: Some(PathBuf::from("/tmp/AppDir")),
Expand Down
17 changes: 17 additions & 0 deletions src/dwarfs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,17 @@ pub fn resolve_mkdwarfs(config: &Config) -> Result<PathBuf> {
}

/// Build a DWARFS AppImage. The runtime is embedded via `--header`.
///
/// `source_date_epoch` comes from [`crate::config::Config`]; when set, every
/// stored timestamp is pinned to it so packing the same AppDir twice gives the
/// same image. See "Producing bit-identical images" in mkdwarfs(1).
pub fn build_appimage(
mkdwarfs: &Path,
appdir: &Path,
runtime: &Path,
output: &Path,
compression: &str,
source_date_epoch: Option<u64>,
profile: Option<&Path>,
) -> Result<()> {
crate::log_info!("Building DWARFS AppImage...");
Expand All @@ -91,6 +96,10 @@ pub fn build_appimage(
.arg("--input")
.arg(appdir);

if let Some(epoch) = source_date_epoch {
cmd.arg("--set-time").arg(epoch.to_string());
}

// Add profile optimization if available
if let Some(profile) = profile
&& profile.exists()
Expand All @@ -100,6 +109,14 @@ pub fn build_appimage(
.arg(format!("--hotness-list={}", profile.display()));
}

// mkdwarfs only produces bit-identical categorized images when the number
// of segmenter workers is fixed; it defaults to the host's CPU count, which
// differs between build machines. Pin it whenever reproducibility is asked
// for, including when `DWARFS_COMP` smuggles in a `--categorize` of its own.
if source_date_epoch.is_some() {
cmd.arg("--num-segmenter-workers").arg("1");
}

// Add compression options. The string can contain multiple space-separated
// args like "zstd:level=22 -S26 -B6". The first part goes to -C, the rest
// are passed as separate args.
Expand Down
5 changes: 5 additions & 0 deletions src/main.rs
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,10 @@ struct Cli {
#[arg(long, env = "DWARFS_COMP")]
dwarfs_comp: Option<String>,

/// Pin every timestamp in the image to this unix timestamp (reproducible builds)
#[arg(long, env = "SOURCE_DATE_EPOCH")]
source_date_epoch: Option<String>,

/// Enable DWARFS profile optimization
#[arg(long)]
optimize_launch: bool,
Expand Down Expand Up @@ -110,6 +114,7 @@ fn main() {
runtime_url: cli.runtime_url,
update_info: cli.update_info,
dwarfs_comp: cli.dwarfs_comp,
source_date_epoch: cli.source_date_epoch,
optimize_launch: cli.optimize_launch,
dwarfs_profile: cli.dwarfs_profile,
mkdwarfs: cli.mkdwarfs,
Expand Down
100 changes: 100 additions & 0 deletions tests/integration.rs
Original file line number Diff line number Diff line change
Expand Up @@ -409,6 +409,97 @@ fn test_full_build_pipeline() {
assert!(found, "no .AppImage file found in output directory");
}

// ─── Reproducible builds (needs mkdwarfs, run with --ignored) ────────

/// Set the mtime of every entry under `dir`.
fn set_all_mtimes(dir: &std::path::Path, epoch: u64) {
use std::time::{Duration, UNIX_EPOCH};

let modified = UNIX_EPOCH + Duration::from_secs(epoch);
let mut stack = vec![dir.to_path_buf()];
while let Some(path) = stack.pop() {
if path.is_dir() {
for entry in fs::read_dir(&path).unwrap() {
stack.push(entry.unwrap().path());
}
}
// Opening a directory read-only works on Linux, and this suite is
// Unix-only anyway.
fs::File::open(&path)
.and_then(|file| file.set_times(fs::FileTimes::new().set_modified(modified)))
.unwrap_or_else(|err| panic!("failed to set the mtime of {}: {err}", path.display()));
}
}

/// Pack `appdir` twice under one `SOURCE_DATE_EPOCH` while changing its own
/// mtimes in between, then once more under a different epoch: the first two
/// images must match and the third must not. `header` is only prepended to the
/// image, so a dummy file is enough — no uruntime download and no FUSE.
fn assert_reproducible_build(
mkdwarfs: &std::path::Path,
appdir: &std::path::Path,
header: &std::path::Path,
out_dir: &std::path::Path,
) {
let pack = |epoch: u64, name: &str| -> Vec<u8> {
let out = out_dir.join(name);
appimagetool::dwarfs::build_appimage(
mkdwarfs,
appdir,
header,
&out,
"zstd:level=1",
Some(epoch),
None,
)
.unwrap();
fs::read(&out).unwrap()
};

let first = pack(1_700_000_000, "first.AppImage");

// Same contents from an older "build host". Without --set-time this alone
// changes the image.
set_all_mtimes(appdir, 1_500_000_000);

let same_epoch = pack(1_700_000_000, "same-epoch.AppImage");
let other_epoch = pack(1_600_000_000, "other-epoch.AppImage");

assert_eq!(
first, same_epoch,
"the same SOURCE_DATE_EPOCH must ignore the AppDir's own mtimes"
);
assert_ne!(
first, other_epoch,
"a different SOURCE_DATE_EPOCH must change the image"
);
}

/// Standalone entry point, so the check can be run with `--ignored` without the
/// smoke test's runtime download. CI installs mkdwarfs only for the smoke job,
/// which calls the same helper.
#[test]
#[ignore = "needs mkdwarfs; the smoke test carries it in CI"]
fn reproducible_build_pins_timestamps() {
let tmp = TempDir::new("appimagetool-repro");
let appdir = tmp.path().join("AppDir");
let runtime = tmp.path().join("dummy-runtime");
fs::create_dir_all(&appdir).unwrap();
create_mock_appdir(&appdir, "Repro");
fs::write(&runtime, b"not a real runtime\n").unwrap();

let config = Config::from_cli_args(CliArgs {
appdir: Some(appdir.clone()),
tmpdir: Some(tmp.path().to_path_buf()),
mkdwarfs: std::env::var_os("APPIMAGETOOL_SMOKE_MKDWARFS").map(PathBuf::from),
..Default::default()
})
.unwrap();
let mkdwarfs = appimagetool::dwarfs::resolve_mkdwarfs(&config).unwrap();

assert_reproducible_build(&mkdwarfs, &appdir, &runtime, tmp.path());
}

// ─── Smoke test: package a trivial AppDir and run the AppImage ───────

/// A minimal AppDir whose `AppRun` prints `success`.
Expand Down Expand Up @@ -622,4 +713,13 @@ fn smoke_builds_and_runs_appimage() {
"{arch} AppImage did not print `success`\n--- stdout ---\n{}",
String::from_utf8_lossy(&output.stdout),
);

// The smoke job is the only CI job with mkdwarfs installed, so it also
// carries the reproducible-build check.
let repro = tmp.path().join("repro");
fs::create_dir_all(&repro).unwrap();
let header = tmp.path().join("dummy-header");
fs::write(&header, b"not a real header\n").unwrap();
let mkdwarfs = appimagetool::dwarfs::resolve_mkdwarfs(&config).unwrap();
assert_reproducible_build(&mkdwarfs, &config.appdir, &header, &repro);
}