Add experimental debugger support for JIT-compiled C++

.github/workflows/ci.yml

@@ -91,6 +91,57 @@ jobs:
           cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp"
           cmake -S . -B build-external -DLLVM_DIR="$LLVM_DIR" -DCppInterOp_DIR="$RUNNER_TEMP/cppinterop-build/lib/cmake/CppInterOp"
 
+  # The experimental debugger support is off in the other cells, which skip the
+  # gdb tests of test_debugging.py.
+  debugger:
+    name: debugger/ubuntu-24.04/llvm22/py3.14
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@v7
+      - uses: actions/setup-python@v7
+        with:
+          python-version: '3.14'
+      - name: Install dependencies
+        run: |
+          sudo apt-get update -qq
+          sudo apt-get install -y --no-install-recommends cmake ninja-build g++ git libedit-dev
+          python -m pip install --upgrade pip
+          python -m pip install -r requirements.txt
+      # The DAP tests need gdb 16; Ubuntu 24.04 has gdb 15.
+      - uses: mamba-org/setup-micromamba@f457c30a868e4760d3a6fcea5f25dc655b8edf39 # v3.2.1
+        with:
+          environment-name: gdb-env
+          create-args: gdb=16.3
+          condarc: |
+            channels:
+              - conda-forge
+      - name: Add gdb to PATH
+        run: |
+          # Only gdb: the environment's python must not shadow setup-python's.
+          mkdir -p "$RUNNER_TEMP/gdb-bin"
+          ln -s "${MAMBA_ROOT_PREFIX:-$HOME/micromamba}/envs/gdb-env/bin/gdb" "$RUNNER_TEMP/gdb-bin/gdb"
+          echo "$RUNNER_TEMP/gdb-bin" >> "$GITHUB_PATH"
+      - uses: compiler-research/ci-workflows/actions/setup-llvm@main
+        with:
+          version: '22'
+          os: ubuntu-24.04
+      - name: Build and install CppJIT with the debugger support
+        run: |
+          CMAKE_BUILD_PARALLEL_LEVEL="$(nproc)" python -m pip install . -v \
+            --config-settings=cmake.define.LLVM_DIR="$LLVM_DIR" \
+            --config-settings=cmake.define.CPPJIT_ENABLE_DEBUGGER_SUPPORT=ON \
+            --config-settings=install.strip=false
+      - name: Run the debugging tests
+        working-directory: test
+        env:
+          CPPINTEROP_EXTRA_INTERPRETER_ARGS: -std=c++20
+        run: |
+          set -o pipefail
+          gdb --version | head -1
+          python -m pytest -ra --tb=short -p no:cacheprovider test_debugging.py | tee "$RUNNER_TEMP/debugging.log"
+          # A skip here means gdb or ptrace is not usable, so the cell tests nothing.
+          ! grep -q "^SKIPPED" "$RUNNER_TEMP/debugging.log"
+
   report:
     if: ${{ always() && github.event_name == 'pull_request' }}
     needs: build-test

.gitignore

@@ -37,4 +37,6 @@ MODULE.bazel.lock
 # Editors / OS
 .idea/
 .vscode/
+# the examples ship their launch configurations
+!/examples/*/.vscode/
 .DS_Store

CMakeLists.txt

@@ -10,8 +10,11 @@ include(GNUInstallDirs)
 # This option won't make a lot of sense since we only ship the shared library in site-packages
 # Perhaps this should permanently be OFF and users can build their own CppInterOp if they want to run the tests?
 option(CPPJIT_ENABLE_CPPINTEROP_TESTS "enable CppInterOp tests" OFF)
-set(CPPINTEROP_GIT_REPOSITORY "https://github.com/compiler-research/CppInterOp.git" CACHE STRING "")
-set(CPPINTEROP_GIT_TAG "d3c8cf7aff7796e1539582f8e52910f351a48977" CACHE STRING "")
+# Experimental: debugging of JIT-compiled C++ with gdb (cppjit.debugger, the
+# gdb extension and the libcppjit hooks). Off, the build has none of it.
+option(CPPJIT_ENABLE_DEBUGGER_SUPPORT "build the experimental debugger support" OFF)
+set(CPPINTEROP_GIT_REPOSITORY "https://github.com/chicagotrading/CppInterOp.git" CACHE STRING "")
+set(CPPINTEROP_GIT_TAG "upstreaming/jit-debugging" CACHE STRING "")
 set(CPPINTEROP_SOURCE_DIR "" CACHE PATH
     "Override default CppInterOp built by ExternalProject_Add, with a path to local CppInterOp source")
 
@@ -77,6 +80,7 @@ target_compile_definitions(cppjit PRIVATE
     CPPJIT_CLANG_MAJOR="${CPPJIT_INTEROP_CLANG_MAJOR}"
     CPPJIT_CLANG_INCLUDE_DIR="${CPPJIT_INTEROP_CLANG_DIR}"
     $<$<BOOL:${CPPJIT_USE_CLING}>:CPPJIT_USE_CLING>
+    $<$<BOOL:${CPPJIT_ENABLE_DEBUGGER_SUPPORT}>:CPPJIT_WITH_DEBUGGER_SUPPORT>
 )
 
 target_include_directories(cppjit PRIVATE
@@ -116,10 +120,22 @@ if(NOT SKBUILD)
     set_target_properties(cppjit PROPERTIES
         LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/python/cppjit"
     )
+    set(_cppjit_debugger_tree)
+    if(CPPJIT_ENABLE_DEBUGGER_SUPPORT)
+        set(_cppjit_debugger_tree
+            COMMAND ${CMAKE_COMMAND} -E copy_directory
+                "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit-debugger"
+                "${CMAKE_BINARY_DIR}/python/cppjit/debugger"
+            COMMAND ${CMAKE_COMMAND} -E rename
+                "${CMAKE_BINARY_DIR}/python/cppjit/debugger/libcppjit.so-gdb.py"
+                "${CMAKE_BINARY_DIR}/python/cppjit/libcppjit.so-gdb.py"
+        )
+    endif()
     add_custom_target(cppjit-python-tree ALL
         COMMAND ${CMAKE_COMMAND} -E copy_directory
             "${CMAKE_CURRENT_SOURCE_DIR}/python/cppjit"
             "${CMAKE_BINARY_DIR}/python/cppjit"
+        ${_cppjit_debugger_tree}
     )
 endif()
 
@@ -137,6 +153,19 @@ if(NOT SKBUILD)
     )
 endif()
 
+# The experimental debugger support installs as cppjit.debugger, with the
+# gdb auto-load script beside the extension.
+if(CPPJIT_ENABLE_DEBUGGER_SUPPORT)
+    install(DIRECTORY python/cppjit-debugger/
+        DESTINATION cppjit/debugger
+        PATTERN "__pycache__" EXCLUDE
+        PATTERN "libcppjit.so-gdb.py" EXCLUDE
+    )
+    install(FILES python/cppjit-debugger/libcppjit.so-gdb.py
+        DESTINATION cppjit
+    )
+endif()
+
 # An external CppInterOp is consumed in place and nothing of it ships;
 # the bundled one installs with the builtin headers of the build clang,
 # laid out as a headers-only resource dir (only include/ ships).

README.md

@@ -121,6 +121,29 @@ reflection and JIT compilation API for cppjit.
 - Domain-specific languages: user-defined "pythonizations" adapt the
   bindings into Pythonic libraries.
 
+### Debugging JIT-compiled C++ (experimental)
+
+Build cppjit with `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` to debug the
+JIT-compiled C++ at the source level with gdb, also from Python. The
+default build does not have this support. In a build with it, set
+`CPPJIT_DEBUGGER_SUPPORT=1` before `import cppjit`. cppjit then compiles
+with `-g -O0 -fstandalone-debug` and writes each C++ input to a file in
+`$TMPDIR/cppjit-debug-<uid>` (`CPPINTEROP_JIT_SOURCE_DIR` selects another
+directory), so that gdb shows the source and binds `file:line`
+breakpoints.
+
+The gdb extension in `cppjit/debugger/gdb` puts the Python frames between
+the C++ frames in backtraces, with their variables (CPython 3.13 or
+later), and adds the `cppjit` commands, such as `cppjit py-bt`:
+
+```bash
+CPPJIT_DEBUGGER_SUPPORT=1 gdb -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \
+    -ex 'break -qualified my_function' -ex run --args python my_script.py
+```
+
+For VS Code, with Python breakpoints and steps between Python and C++,
+see [examples/vscode-gdb](examples/vscode-gdb/README.md).
+
 ### Requirements
 
 A package manager installation of cppjit (such as pip) requires

docs/ReleaseNotes.md

@@ -16,7 +16,13 @@ This release supports Python 3.12-3.14 and LLVM 21-22.
 
 ## New features
 
-- ...
+- Experimental debugger support, in builds with
+  `-DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON` only. `CPPJIT_DEBUGGER_SUPPORT=1`
+  starts a debug session: the JIT compiles with `-g -O0`, and the C++
+  inputs go to files that gdb can show. A gdb extension
+  (`cppjit/debugger/gdb`) shows mixed Python and C++ backtraces with the
+  variables of both. `examples/vscode-gdb` adds Python breakpoints and
+  steps between Python and C++ in VS Code, through gdb's DAP server.
 
 ## Deprecated features
 

examples/vscode-gdb/.vscode/launch.json

@@ -0,0 +1,24 @@
+{
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "cppjit: demo.py (gdb)",
+            "type": "cppjit-gdb",
+            "request": "launch",
+            "program": "python3",
+            "args": ["${workspaceFolder}/demo.py"],
+            "cwd": "${workspaceFolder}",
+            "env": {"CPPJIT_DEBUGGER_SUPPORT": "1"}
+        },
+        {
+            "name": "cppjit: attach (gdb)",
+            "type": "cppjit-gdb",
+            "request": "attach",
+            "program": "python3",
+            "pid": "${input:pid}"
+        }
+    ],
+    "inputs": [
+        {"id": "pid", "type": "promptString", "description": "Process ID of the Python process"}
+    ]
+}

examples/vscode-gdb/AGENTS.md

@@ -0,0 +1,159 @@
+# Notes for agents: the cppjit debugger support
+
+These notes are for a coding agent (or a person) that continues the work on
+source-level debugging of JIT-compiled C++. Read them with
+[CONTRIBUTING.md](../../CONTRIBUTING.md) and the example's
+[README](README.md), which describes the features
+for users.
+
+The support is experimental and opt-in. With
+`CPPJIT_ENABLE_DEBUGGER_SUPPORT=OFF` (the default), the build must not change:
+no `cppjit.debugger` package, no gdb extension, no hooks. Keep it that way.
+
+## Map
+
+| Part | Where |
+|------|-------|
+| Debug session: `-g -O0 -fstandalone-debug`, the source directory, line-buffered stdout | `src/interop/interop_wrapper.cxx` (`CPPJIT_WITH_DEBUGGER_SUPPORT`) |
+| Native hooks that gdb breaks on: `cppjit_debug_break_python`, `cppjit_debug_enter_jit`, `cppjit_debug_leave_jit` | `src/interop/interop_wrapper.cxx` |
+| Import of `cppjit.debugger` at module init | `src/cpyrt/cpyrtModule.cxx` |
+| `cppjit.debugger`: source files, Python breakpoints and steps with `sys.monitoring`, `gdb_script()` | `python/cppjit-debugger/__init__.py` |
+| gdb extension: one module for each concern, see each docstring | `python/cppjit-debugger/gdb/` |
+| CPython state read from memory (`_Py_DebugOffsets`, frames, objects) | `gdb/cpython.py`, `gdb/inferior.py`, `gdb/values.py` |
+| Mixed backtraces and hidden internal frames | `gdb/frames.py` |
+| `cppjit` gdb commands (`py-step`, `py-bt`, `sources`, `up`, `down`, `gil`) | `gdb/commands.py` |
+| Breakpoints in code that the JIT has not compiled yet | `gdb/lines.py`, `gdb/breakpoints.py` |
+| A limit on the children of a value in DAP | `gdb/limits.py` |
+| VS Code extension, and the DAP proxy that adds Python breakpoints and steps | `examples/vscode-gdb/extension/` |
+| The demo | `examples/vscode-gdb/` |
+| Tests (gdb CLI and DAP, with and without the proxy) | `test/test_debugging.py` |
+| CI job | `debugger` in `.github/workflows/ci.yml` |
+
+CppInterOp writes each interpreter input to a file in
+`CPPINTEROP_JIT_SOURCE_DIR`, so that gdb can show the source and bind
+`FILE:LINE` breakpoints in `cppdef` code. That part is in CppInterOp
+(compiler-research/CppInterOp#1156). Until it is in the pinned CppInterOp,
+build with `CPPINTEROP_SOURCE_DIR` set to a checkout of that branch.
+
+## Set up
+
+Requirements: Linux, gdb 16 or later (for its DAP server), CPython 3.13 or
+later, `node` (for the DAP proxy tests and for VS Code), and an LLVM that
+cppjit supports.
+
+```bash
+export LLVM_DIR=<your LLVM>/lib/cmake/llvm
+cmake -S . -B build -G Ninja -DCMAKE_BUILD_TYPE=RelWithDebInfo -DLLVM_DIR=$LLVM_DIR \
+    -DCPPJIT_ENABLE_DEBUGGER_SUPPORT=ON \
+    -DCPPINTEROP_SOURCE_DIR=$PWD/../CppInterOp   # a CppInterOp with #1156
+cmake --build build -j
+export PYTHONPATH=$PWD/build/python
+```
+
+`RelWithDebInfo` gives source for the cppjit and CppInterOp frames when you
+show the internal frames. gdb loads the copy of the package in
+`build/python/cppjit/debugger`. A build copies a change in
+`python/cppjit-debugger` there; `--target cppjit-python-tree` copies only
+the Python files.
+
+gdb loads the libstdc++ pretty-printers, and on some systems
+`libthread_db`, only from its auto-load safe path. Without the printers,
+`std::vector` shows its raw fields. Add
+`add-auto-load-safe-path <directory>` to `~/.config/gdb/gdbinit` for each.
+
+## Run the demo
+
+```bash
+cd examples/vscode-gdb
+lib/build.sh                      # libhistogram.so, without its source
+# demo.h needs nlohmann/json: install it, or put nlohmann/json.hpp here
+CPPJIT_DEBUGGER_SUPPORT=1 CPPINTEROP_EXTRA_INTERPRETER_ARGS=-std=c++20 python demo.py
+```
+
+An agent cannot click in VS Code. Use one of these:
+
+- gdb in batch mode, with the extension loaded:
+
+  ```bash
+  CPPJIT_DEBUGGER_SUPPORT=1 CPPINTEROP_EXTRA_INTERPRETER_ARGS=-std=c++20 \
+  gdb -batch -iex "source $(python -c 'import cppjit.debugger as d; print(d.gdb_script())')" \
+      -iex 'set breakpoint pending on' \
+      -ex 'break demo::Histogram::add' -ex run -ex bt -ex 'info locals' -ex 'print *this' \
+      -ex 'directory lib' -ex list -ex kill --args python demo.py
+  ```
+
+- The DAP server, through the proxy, as VS Code uses it:
+  `node extension/dap-proxy.js gdb -i dap -iex "source <gdb_script()>"`.
+  `_Dap` in `test/test_debugging.py` is a small client to copy. Send
+  `initialize`, `launch`, `setBreakpoints` or `setFunctionBreakpoints`,
+  `configurationDone`, then wait for the `stopped` event and send
+  `stackTrace`, `scopes` and `variables`.
+- `set debug dap-log-file <file>` (in DAP mode) writes all DAP traffic of a
+  VS Code session to a file. Use it to see what VS Code sends.
+
+A person can follow the steps in the example's README to run the demo in
+VS Code.
+
+## Test
+
+```bash
+cd test
+make -j4
+python -m pytest -ra test_debugging.py
+```
+
+The gdb tests skip without gdb, the DAP tests without gdb 16, and the proxy
+tests without `node`. They also skip if gdb cannot trace a process
+(`ptrace_scope` 3, or a container that denies ptrace). For a bug,
+write a test in `test_debugging.py` first and see it fail.
+
+## Lessons
+
+- gdb's DAP server answers one request at a time. A slow request blocks all
+  of them, also the steps and the pause. `limits.py` stops the count of the
+  children of a value at 10000 (garbage memory, for example a member before
+  its constructor runs), and the proxy cancels an `evaluate` or `variables`
+  request after 5 s.
+- Do not call functions in the process from the extension. A pretty-printer,
+  a frame filter or a value lookup can cause an inferior call, for example
+  through the address of a thread-local variable. The DAP server reports the
+  call as a stop with the reason "function call". Read memory instead.
+- The Python state comes from memory through `_Py_DebugOffsets`, without
+  CPython debug information, so it also works in a core file. Each CPython
+  version can change the layouts. CPython 3.14 changed the owner of the
+  entry frame and the layout of split dictionaries.
+- The frame names have prefixes that the proxy uses: `[py] ` for a Python
+  frame and `[internal] ` for an internal frame at the top of the stack.
+- gdb moves a `FILE:LINE` breakpoint in an inline function that the JIT has
+  not compiled yet to the next line with code. `lines.py` disables such a
+  location until the JIT compiles the function.
+- Clang leaves out the types that a library instantiates, such as
+  `std::ifstream`, unless it has `-fstandalone-debug`. gdb then shows
+  `<incomplete type>`.
+- If the source file of a frame is missing, the proxy removes the source of
+  the frame, so VS Code shows the disassembly and not an error.
+- A gdb that is busy in a long Python call did not stop on SIGTERM. To stop
+  a DAP gdb, close its stdin, or use SIGKILL.
+
+## Open work
+
+- **lldb.** Not started. These are the facts so far:
+  - clang `-gdwarf-5 -gembed-source` puts the source text in the DWARF line
+    table. gdb 16.3 ignores it. lldb 21 shows it (`source list`), also when
+    the file is missing. A library without its source could then show source
+    in lldb.
+  - Questions to answer first: Does lldb see the JIT code through the GDB
+    JIT interface on Linux, with its symbols and line tables? Do `FILE:LINE`
+    breakpoints bind in the files in `CPPINTEROP_JIT_SOURCE_DIR`? How do the
+    Python frames get into the stack? lldb has no frame filter API like
+    gdb's. Options to look at: a DAP proxy in front of `lldb-dap` that
+    changes `stackTrace`, or lldb's Python API.
+  - The CPython reader (`cpython.py`) does not use the gdb API: it gets
+    functions that read memory, from `inferior.py`. An lldb version of
+    `inferior.py` can use the same reader.
+- The limits in the example's README: Python variables are read only,
+  Python breakpoints need a launch (not an attach), objects with `__slots__`
+  show no attributes.
+- Under gdb's `set environment`, libc's `environ` in the process did not
+  have the variables that `/proc/PID/environ` had. `inferior.environment()`
+  merges both. The cause is not known.