Route Clang LTO to ThinLTO and honor linker job budgets

.github/workflows/ci.yml

@@ -32,14 +32,24 @@ jobs:
         cc: [gcc-14, clang-19]
     env:
       CC: ${{ matrix.cc }}
+      FORGE_TEST_REQUIRE_LTO: '1'
+      FORGE_TEST_LLD_PATH: /usr/bin/ld.lld-19
     steps:
       - uses: actions/checkout@fbc6f3992d24b796d5a048ff273f7fcc4a7b6c09 # v5
         with:
           persist-credentials: false
       - name: Install both supported toolchains
+        timeout-minutes: 5
         run: |
-          sudo apt-get update
-          sudo apt-get install --no-install-recommends -y gcc-14 clang-19 llvm-19 libclang-rt-19-dev binutils pkg-config
+          # Use Ubuntu's official HTTPS mirror for LLVM package downloads.
+          if test -f /etc/apt/apt-mirrors.txt; then
+            printf '%s\n' 'https://archive.ubuntu.com/ubuntu' | sudo tee /etc/apt/apt-mirrors.txt > /dev/null
+          fi
+          sudo apt-get -o Acquire::Retries=3 -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 update
+          sudo env DEBIAN_FRONTEND=noninteractive apt-get \
+            -o DPkg::Lock::Timeout=120 -o Acquire::Retries=3 \
+            -o Acquire::http::Timeout=30 -o Acquire::https::Timeout=30 \
+            install --no-install-recommends -y gcc-14 clang-19 llvm-19 lld-19 libclang-rt-19-dev binutils pkg-config
           sudo update-alternatives --install /usr/bin/gcc gcc /usr/bin/gcc-14 100
           sudo update-alternatives --set gcc /usr/bin/gcc-14
           sudo update-alternatives --install /usr/bin/clang clang /usr/bin/clang-19 190

README.md

@@ -83,6 +83,10 @@ compiler=GCC
 compiler_path=/usr/bin/gcc
 ```
 
+`TARGET_LTO` remains an enable/disable switch: GCC uses native LTO and Clang
+uses ThinLTO. See [LTO and linker parallelism](docs/LTO.md) for linker selection,
+tool dependencies and the backend-specific mapping of `./make -j N`.
+
 Configuration keys can be overridden per platform by suffixing the key
 name (`compiler_windows=...`, `compiler_linux=...`, `compiler_mac_os=...`),
 falling back to the bare key for the platform's default. Every setting

README.zh-CN.md

@@ -68,6 +68,10 @@ compiler=GCC
 compiler_path=/usr/bin/gcc
 ```
 
+`TARGET_LTO` 保持启用/禁用开关：GCC 使用原生 LTO，Clang 使用 ThinLTO。
+链接器选择、工具依赖及 `./make -j N` 的后端映射见
+[LTO 与链接并行度](docs/LTO.md)。
+
 支持按平台加后缀覆盖(`compiler_windows=…`、`compiler_linux=…`、
 `compiler_mac_os=…`),缺省值平台的同名键兜底。所有配置项会以
 `-DCONF_<KEY>=<VALUE>` 的形式,作为编译 build.c 时的宏注入(与

docs/LTO.md

@@ -0,0 +1,63 @@
+# LTO and linker parallelism
+
+The public contract is `set(TARGET_LTO, 1, ENABLE)` or `DISABLE`, not a shared
+set of toolchain-specific LTO modes. `ENABLE` selects GCC's native LTO on GCC
+and ThinLTO on Clang. LTO is enabled by default. The existing `COMPILER_LTO`
+and `LINKER_LTO` switches still control the individual phases when needed.
+
+This changes Clang targets from the driver's implicit Full LTO to explicit
+ThinLTO at both compilation and link time. Clean old output directories when
+moving between compiler families or compiler versions; their LTO objects are
+not interchangeable. Re-bootstrap Forge after updating its headers/runtime.
+
+## Job budget on Linux
+
+`./make -j N` sets a stage's worker budget, including `-j 1`. File compilation
+and final linking run in separate stages; this is not a process-wide thread
+or memory limit. Each backend translates the budget into its supported flags:
+
+| Backend | Compile option | Link options |
+| --- | --- | --- |
+| GCC, default linker | `-flto` | `-flto=N` when N > 1; otherwise `-flto` |
+| Clang, default GNU linker with LLVM plugin | `-flto=thin` | `-flto=thin -Wl,-plugin-opt,jobs=N` |
+| Clang, explicitly selected ELF LLD | `-flto=thin` | `-flto=thin -Wl,--thinlto-jobs=N -Wl,--threads=N` |
+
+The Clang default-linker route needs the matching LLVM linker plugin
+(`LLVMgold.so`). The ELF LLD route handles LLVM bitcode directly and needs no
+GNU linker plugin. A missing linker or plugin fails the build; Forge does not
+silently disable LTO or switch linkers.
+
+`DISABLE` removes the generated LTO and LTO-job options. Explicit ELF LLD still
+receives `--threads=N` because its ordinary linker workers also use the budget.
+The generated Linux linker flags are not promised for other object formats;
+macOS and Windows backends remain disabled in this release.
+
+## Select Clang and LLD
+
+Inside a target block:
+
+```c
+set_toolchain(CLANG);
+set(TARGET_LINKER, 1, LLD);
+set(TARGET_LTO, 1, ENABLE);
+```
+
+`TARGET_LINKER = LLD` now actually selects LLD via the Clang driver using
+`-fuse-ld=lld`. To select a specific installed version, also set:
+
+```c
+set(TARGET_LINKER_PATH, 1, "/usr/bin/ld.lld-19");
+```
+
+Clang receives the path as one `--ld-path=...` argument, including any spaces.
+The selected path must match the configured linker family; Forge does not
+guess capabilities from executable names. Without `LLD`, the Clang driver
+keeps its default linker and uses the LLVM plugin's ThinLTO job option.
+
+Use a matching Clang/LLVM/LLD installation. CI uses GCC 14 and Clang/LLVM/LLD 19,
+and requires real multi-module links through both the default GNU linker and
+LLD, executable execution, LTO-disabled builds and relinking after `-j` changes.
+
+Tool-specific overrides still belong in `TARGET_OPTIONS` / `LINKER_OPTIONS`,
+which are appended after generated options. Their compatibility and any
+additional parallelism controls belong to that explicit configuration.

lib/include/build.h

@@ -335,7 +335,8 @@ typedef enum {
   TARGET_OUTPUT_PATH,
   TARGET_OUTPUT_FINAL_PATH,
   TARGET_OUTPUT_INTERMEDIATE_PATH,
-  TARGET_LTO,
+  TARGET_LTO, /**< ENABLE / DISABLE for compile and link: GCC native LTO,
+                  Clang ThinLTO; parallelism follows the backend's -j mapping */
   TARGET_SOURCES,
   TARGET_SOURCE_PATHS,
   TARGET_SOURCE_PATHS_R,

lib/include/forge_os.h

@@ -137,9 +137,9 @@ extern FORGE_TLS int os_verbose;
  * Range: >= 1. Affects two places:
  * - build.c's compile pass runs N-way in parallel (N = min(os_jobs,
  *   number of files to compile));
- * - the link pass: for gcc/clang with LTO enabled and N > 1,
- *   `-flto` is rewritten to `-flto=N`; for MSVC with N > 1,
- *   `/CGTHREADS:N` is appended (cap 8).
+ * - the link pass: GCC LTO uses `-flto=N` when N > 1; Clang selects
+ *   ThinLTO and on Linux passes N to the LLVM GNU linker plugin or
+ *   explicitly selected ELF LLD. ELF LLD's worker pool also uses N.
  * Set by os_set_jobs(); defined in lib/src/args.c.
  */
 extern int os_jobs;

lib/src/compiler/clang.c

@@ -4,15 +4,71 @@
 
 /**
  * @file compiler/clang.c
- * @brief clang_compiler() implementation: the current CC-compatible
- *        subset simply reuses the gcc backend.
+ * @brief Clang policy over the shared GCC-style command builder.
  *
- * The flag subset this project currently uses (-std/-O/-D/-I/-f
- * series/-MMD/-o etc.) has identical semantics in gcc and clang; a
- * separate backend is deferred until a difference shows up.
+ * Common flags share the CC builder; LTO and linker selection stay here.
+ * The public LTO switch selects ThinLTO for Clang, independently of the
+ * GCC backend's implementation of the same build intent.
  */
 #include "forge_compiler.h"
+#include "forge_def.h"
+#include "forge_linker.h"
+#include "internal.h"
+
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+int forge_clang_options(strv_t *args, const forge_context_t context) {
+  const toolchain_t *tc = &context.target->toolchain;
+  if (context.source_type == FORGE_SOURCE)
+    return tc->compiler_args->lto_enabled == ENABLE
+               ? strv_append(args, "-flto=thin") : 0;
+
+  if (tc->linker == LLD && strv_append(args, "-fuse-ld=lld") != 0)
+    return -1;
+  if (tc->linker_path && *tc->linker_path) {
+    const char prefix[] = "--ld-path=";
+    size_t size = sizeof(prefix) + strlen(tc->linker_path);
+    char *option = malloc(size);
+    if (!option)
+      return -1;
+    snprintf(option, size, "%s%s", prefix, tc->linker_path);
+    int rc = strv_append(args, option);
+    free(option);
+    if (rc != 0)
+      return -1;
+  }
+
+#if defined(__linux__)
+  /* LLD's worker budget applies even when LTO is disabled. The selector,
+     rather than a guessed executable name, identifies ELF LLD here. */
+  if (tc->linker == LLD) {
+    char option[64];
+    snprintf(option, sizeof(option), "-Wl,--threads=%d", os_jobs);
+    if (strv_append(args, option) != 0)
+      return -1;
+  }
+#endif
+
+  if (tc->linker_args->lto_enabled != ENABLE)
+    return 0;
+  if (strv_append(args, "-flto=thin") != 0)
+    return -1;
+
+#if defined(__linux__)
+  char option[64];
+  /* LLVM's GNU linker plugin and ELF LLD use different spellings for
+     the same ThinLTO backend budget. Honor -j 1 as well as -j N. */
+  snprintf(option, sizeof(option), tc->linker == LLD
+               ? "-Wl,--thinlto-jobs=%d" : "-Wl,-plugin-opt,jobs=%d", os_jobs);
+  return strv_append(args, option);
+#else
+  /* Other platform backends are not enabled in this Linux-only release. */
+  return 0;
+#endif
+}
 
 strv_t *clang_compiler(const forge_context_t context) {
   return gcc_compiler(context);
-}
+}

lib/src/compiler/gcc.c

@@ -17,8 +17,8 @@
  *   optimization -> -D -> -I (target+libs) -> per-file cflags ->
  *   visibility/lto/pic flags -> -c|-S -> source file -> -o obj ->
  *   -MMD -MF obj.d -> trailing options;
- * - Link branch (FORGE_OBJ, EXE/SHARED): cc -> -flto(=N for GCC when
- *   os_jobs>1) -> -shared -> obj list -> -L (incl. libs) -> -l -> -o ->
+ * - Link branch (FORGE_OBJ, EXE/SHARED): cc -> backend LTO/linker
+ *   options -> -shared -> obj list -> -L (incl. libs) -> -l -> -o ->
  *   export symbol (os_export_flag) -> trailing linker options.
  * FORGE_STATIC_LIB belongs to the archiver; NULL is returned here.
  *
@@ -98,7 +98,9 @@ static strv_t *gcc_compile(const forge_context_t ctx) {
 
   if (!failed && ca->visibility == HIDDEN)
     failed = strv_append(&args, "-fvisibility=hidden") != 0;
-  if (!failed && ca->lto_enabled == ENABLE)
+  if (!failed && t->toolchain.compiler == CLANG)
+    failed = forge_clang_options(&args, ctx) != 0;
+  else if (!failed && ca->lto_enabled == ENABLE)
     failed = strv_append(&args, "-flto") != 0;
   if (!failed && (ca->pic_enabled != 0 ||
                   ctx.target_type == FORGE_SHARED_LIB))
@@ -178,8 +180,10 @@ static strv_t *gcc_link(const forge_context_t ctx) {
 
   failed = strv_append(&args, forge_default_cc(t)) != 0;
 
-  if (!failed && la->lto_enabled == ENABLE) {
-    if (os_jobs > 1 && t->toolchain.compiler != CLANG) {
+  if (!failed && t->toolchain.compiler == CLANG) {
+    failed = forge_clang_options(&args, ctx) != 0;
+  } else if (!failed && la->lto_enabled == ENABLE) {
+    if (os_jobs > 1) {
       char flto[32];
       snprintf(flto, sizeof(flto), "-flto=%d", os_jobs);
       failed = strv_append(&args, flto) != 0;

lib/src/internal.h

@@ -82,6 +82,16 @@ int strv_append_args(strv_t *args, const char *text);
  */
 const char *forge_default_cc(const target_t *t);
 
+/**
+ * @brief Append Clang-specific options to the shared CC command builder.
+ *
+ * ENABLE selects ThinLTO. Link options select an explicitly requested LLD
+ * or linker path and translate os_jobs for the supported platform/linker.
+ * Does not execute or probe tools; returns 0 on success, -1 on allocation
+ * failure. The caller retains ownership of args.
+ */
+int forge_clang_options(strv_t *args, const forge_context_t context);
+
 /**
  * @brief Return whether the current MSVC compiler supports
  * /sourceDependencies (generating .d dependencies).

lib/src/linker/clang.c

@@ -4,15 +4,14 @@
 
 /**
  * @file linker/clang.c
- * @brief clang_linker() implementation: a placeholder -- reuses the
- *        compiler link branch.
+ * @brief Clang link command generation through the compiler driver.
  *
- * Same as linker/gcc.c: linking actually goes through the compiler
- * (the clang backend); a real lld wrapper will replace it later.
+ * The shared link branch uses Clang's backend policy to select an
+ * explicitly requested LLD and to translate LTO/parallelism options.
  */
 #include "forge_linker.h"
 #include "forge_compiler.h"
 
 strv_t *clang_linker(const forge_context_t context) {
   return clang_compiler(context);
-}
+}

tests/edge/test_ed_lto.c

@@ -0,0 +1,108 @@
+// Source lib/src/os/line.c
+// Source lib/src/os/path.c
+// Source lib/src/os/unix.c
+// Include lib/include
+#include <build.h>
+#include "tests_common.h"
+
+/* Real multi-module links catch driver/plugin incompatibilities that argv
+   tests cannot. All files remain under the ignored test/edge_* trees. */
+static int tool_available(const char *tool) {
+  char *argv[] = {(char *)tool, "--version", NULL};
+  int status = -1;
+  char *output = os_execute_capture_all_status(tool, argv, "", NULL, &status);
+  free(output);
+  return status == 0;
+}
+
+static int build_case(const char *name, int lld, int lto) {
+  char *dir = edge_dir(name);
+  TEST(dir != NULL);
+  char script[2048];
+  snprintf(script, sizeof(script),
+      "#include <build.h>\n"
+      "function(build) {\n"
+      "  parse_args(builtin_args);\n"
+      "  int result = 0;\n"
+      "  target(\"app\") {\n"
+      "    set_toolchain(CLANG);\n"
+      "    set(TARGET_LINKER, 1, %s);\n"
+      "    set(TARGET_LTO, 1, %s);\n"
+      "    const char *path = getenv(\"FORGE_TEST_LLD_PATH\");\n"
+      "    if (%d && path && *path) set(TARGET_LINKER_PATH, 1, path);\n"
+      "    set_optimization(OPT_SPEED);\n"
+      "    add_sources_r(\"src\");\n"
+      "    result = compile();\n"
+      "  }\n"
+      "  return result;\n"
+      "}\n"
+      "set_default(build);\n", lld ? "LLD" : "LD",
+      lto ? "ENABLE" : "DISABLE", lld);
+  TEST(edge_write(dir, "build.c", script) == 0);
+  TEST(edge_write(dir, "build.conf", "compiler=GCC\n") == 0);
+  TEST(edge_write(dir, "src/main.c",
+                  "int answer(void);\n"
+                  "int main(void) { return answer() != 42; }\n") == 0);
+  TEST(edge_write(dir, "src/answer.c",
+                  "int answer(void) { return 42; }\n") == 0);
+
+  char output[16384];
+  TEST(edge_forge(dir, output, sizeof(output)) == 0);
+  char *argv[] = {"./make", "-j", "2", NULL};
+  int status = edge_run(argv, output, sizeof(output), dir);
+  if (status != 0)
+    fprintf(stderr, "LTO case %s failed:\n%s\n", name, output);
+  TEST(status == 0);
+  TEST((strstr(output, "-flto=thin") != NULL) == lto);
+  TEST(strstr(output, "-flto=2") == NULL);
+  if (lld) {
+    TEST(strstr(output, "-fuse-ld=lld") != NULL);
+    TEST(strstr(output, "-Wl,--threads=2") != NULL);
+    TEST((strstr(output, "-Wl,--thinlto-jobs=2") != NULL) == lto);
+    TEST(strstr(output, "-plugin-opt,jobs=") == NULL);
+    const char *path = getenv("FORGE_TEST_LLD_PATH");
+    if (path && *path) {
+      TEST(strstr(output, "--ld-path=") != NULL);
+      TEST(strstr(output, path) != NULL);
+    }
+  } else {
+    TEST(strstr(output, "-Wl,-plugin-opt,jobs=2") != NULL);
+    TEST(strstr(output, "--thinlto-jobs=") == NULL);
+  }
+
+  char *app = os_path_join(dir, "output/app");
+  TEST(app != NULL);
+  char *run[] = {app, NULL};
+  TEST(edge_run(run, output, sizeof(output), dir) == 0);
+  free(app);
+
+  if (lld && lto) {
+    /* A changed job budget must invalidate the link command's cache. */
+    argv[2] = "1";
+    TEST(edge_run(argv, output, sizeof(output), dir) == 0);
+    TEST(strstr(output, "-Wl,--thinlto-jobs=1") != NULL);
+    TEST(strstr(output, "-Wl,--threads=1") != NULL);
+    TEST(strstr(output, "-c src/") == NULL);
+  }
+  free(dir);
+  return 0;
+}
+
+int main(void) {
+  const char *require = getenv("FORGE_TEST_REQUIRE_LTO");
+  const char *lld = getenv("FORGE_TEST_LLD_PATH");
+  int available = edge_available() && tool_available("clang") &&
+                  tool_available(lld && *lld ? lld : "ld.lld");
+  if (require && strcmp(require, "1") == 0)
+    TEST(available);
+  if (!available) {
+    fprintf(stderr, "SKIP: Clang/LLD LTO scenarios require a bootstrapped "
+                    "Forge, Clang and LLD\n");
+  } else {
+    TEST(build_case("lto_gnu", 0, 1) == 0);
+    TEST(build_case("lto_lld", 1, 1) == 0);
+    TEST(build_case("lto_disabled", 1, 0) == 0);
+  }
+  PASS();
+  return 0;
+}