diff --git a/.github/workflows/build_and_test.yml b/.github/workflows/build_and_test.yml index 7f5b393e2..54382989d 100644 --- a/.github/workflows/build_and_test.yml +++ b/.github/workflows/build_and_test.yml @@ -310,20 +310,12 @@ jobs: # warm, but a cache-cold rerun on a slow runner legitimately # needs ~40 (observed 2026-07-05: all tests PASSED, then the # 40-min axe fell during post-job cache packing) - env: - CONDA_PKGS_DIRS: ${{ github.workspace }}\conda_pkgs steps: - uses: actions/checkout@v5 - - name: Cache conda packages - uses: actions/cache@v5 - with: - path: ${{ github.workspace }}\conda_pkgs - key: windows-conda-cpp-${{ hashFiles('environment-win.yml', '.github/workflows/build_and_test.yml') }} - restore-keys: | - windows-conda-cpp- - windows-conda- - + # The conda environment is not cached (ADR-0078): measured, a restored cache of its package + # archives saved no time (conda fetched every package again), and downloading the whole + # ~325 MB environment from conda-forge takes about as long as restoring a cache of it. - uses: conda-incubator/setup-miniconda@v4 with: activate-environment: b2-windows @@ -331,10 +323,14 @@ jobs: auto-activate: false channel-priority: strict + - name: Versions of the main conda packages + shell: pwsh + run: conda list -n b2-windows '^(python|clang|libboost-devel|libboost-headers|gmp|mpfr|mpc|libflint|eigen|eigenpy|numpy|scipy|libblas|cmake|ninja|ccache|msmpi|vs2022_win-64|vc14_runtime|ucrt)$' + # Persist the compiler cache for the C++ test build (mirrors the Unix test jobs and the # Windows wheel job). Loose key -- always save fresh, restore newest; ccache self-invalidates. - - name: Cache ccache (Windows C++ tests) - uses: actions/cache@v5 + - name: Restore ccache (Windows C++ tests) + uses: actions/cache/restore@v5 with: path: ${{ github.workspace }}\.ccache key: ccache-win-cpp-${{ github.sha }} @@ -351,12 +347,8 @@ jobs: $env:CMAKE_GENERATOR_TOOLSET = '' $env:CMAKE_PREFIX_PATH = "$env:CONDA_PREFIX\Library" - # The GitHub Windows runner now ships Visual Studio 2026 (MSVC 14.51), whose STL - # hard-asserts (STL1000) that the compiler is Clang >= 20; the conda - # clang-cl is older. The only errors are that version gate -- the C++17 code compiles - # fine -- so we opt out of the gate. (Proper fix: ship a clang >= 20 toolchain.) - $env:CXXFLAGS = '-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH' - $env:CFLAGS = '-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH' + # The runner's Visual Studio 2026 STL (, STL1000) requires Clang >= 20, so + # environment-win.yml asks for clang >=20. $boostHeader = "$env:CONDA_PREFIX\Library\include\boost\archive\archive_exception.hpp" (Get-Content $boostHeader).Replace('public virtual std::exception', 'public std::exception') | Set-Content $boostHeader @@ -368,17 +360,49 @@ jobs: # build, never a break. $env:CCACHE_DIR = "${{ github.workspace }}\.ccache" $env:CCACHE_MAXSIZE = '400M' + # ccache declines a compile that uses a precompiled header (CMake's cmake_pch) unless + # told to be sloppy about PCH defines and time macros, and it must know the compiler + # is clang-cl. The reason every call was uncacheable until 2026-11-04 (0 of 149) was + # CMake's -clang:-MD dependency flags, which the root CMakeLists now replaces by + # /showIncludes when ccache drives clang-cl; the verbose stats below show hits and + # name any remaining refusal. + $env:CCACHE_SLOPPINESS = 'pch_defines,time_macros' + $env:CCACHE_COMPILERTYPE = 'clang-cl' ccache --zero-stats 2>$null + # Link with the Visual Studio the STL comes from, not the runner's own LLVM, which CMake + # takes otherwise (ADR-0078; tools/msvc_link_tools.ps1 says how they are found). + $msvcTools = & "${{ github.workspace }}\tools\msvc_link_tools.ps1" + $msvcToolArgs = $msvcTools.GetEnumerator() | ForEach-Object { "-D$($_.Key)=$($_.Value.Replace('\', '/'))" } + Get-Command cmake, ninja, clang-cl, ccache | ForEach-Object { "$($_.Name) = $($_.Source)" } + cmake -B build ` + @msvcToolArgs ` -DCMAKE_PREFIX_PATH="$env:CONDA_PREFIX\Library" ` -DBUILD_PYTHON_BINDINGS=OFF ` -DENABLE_UNIT_TESTING=ON ` -DINSTALL_DOCUMENTATION=OFF ` -DCMAKE_DISABLE_FIND_PACKAGE_Doxygen=ON ` -DCMAKE_BUILD_TYPE=Release + # What the build was configured with: the compiler and the four link tools CMake will use. + clang-cl --version + Select-String -Path build\CMakeCache.txt -Pattern '^CMAKE_(C|CXX)_COMPILER:|^CMAKE_(LINKER|AR|RC_COMPILER|MT):' | ForEach-Object Line cmake --build build --parallel - ccache --show-stats + # A native command's exit code does not stop a pwsh step: keep the build's, print the + # stats, then fail here rather than as "Not Run" in the test step. + $buildExitCode = $LASTEXITCODE + ccache --show-stats -v + ccache --version + if ($buildExitCode -ne 0) { Write-Error "the build failed (exit code $buildExitCode)"; exit $buildExitCode } + + # Saved here rather than in actions/cache's post step, which a failed job skips: what compiled + # is worth keeping when a test then fails, or the next attempt compiles cold again. + - name: Save ccache (Windows C++ tests) + if: ${{ !cancelled() }} + uses: actions/cache/save@v5 + with: + path: ${{ github.workspace }}\.ccache + key: ccache-win-cpp-${{ github.sha }} # Run under both per-solve thread settings (see the unix C++ test job for the rationale). - name: Test (threaded + BERTINI_NUM_THREADS=1) @@ -396,6 +420,68 @@ jobs: if ($LASTEXITCODE -ne 0) { exit $LASTEXITCODE } Write-Host "::endgroup::" + # Windows reports a crash as a bare exit code (0xc0000374 is heap corruption), with no stack + # and with the test's buffered stdout lost. When a suite failed, rerun it under cdb, Boost.Test + # naming each test case on stderr as it starts, under both thread settings, so the log says + # which test died and where; and list what the binary imports, in case two C runtimes meet. + # For names inside our own code, add /Z7 to the compile flags and /DEBUG to the link flags for + # one run: that is how ADR-0079's crash was found. cdb runs under the debug heap, which fills + # freed memory, so a read of freed memory that normally goes unnoticed can fail here. + - name: On a failed suite, rerun it under cdb for the stack + if: ${{ failure() }} + timeout-minutes: 20 + shell: pwsh + working-directory: build\core + run: | + $failedLog = '..\Testing\Temporary\LastTestsFailed.log' + if (-not (Test-Path $failedLog)) { Write-Host "no ${failedLog}: the failure was not a test"; exit 0 } + $failed = Select-String -Path $failedLog -Pattern '^\d+:(\S+)' | ForEach-Object { $_.Matches[0].Groups[1].Value } + Write-Host "failed suites: $failed" + + $cdb = 'C:\Program Files (x86)\Windows Kits\10\Debuggers\x64\cdb.exe' + if (-not (Test-Path $cdb)) { + Write-Host "no cdb at $cdb; installing WinDbg with winget" + winget install --id Microsoft.WinDbg --exact --silent --accept-source-agreements --accept-package-agreements | Out-Host + $cdb = (Get-Command cdb.exe -ErrorAction SilentlyContinue).Source + if (-not $cdb) { Write-Host "no cdb after the install either; the suites run plainly" } + } + $env:_NT_SYMBOL_PATH = "srv*$env:TEMP\symbols*https://msdl.microsoft.com/download/symbols" + + $msvcTools = & "${{ github.workspace }}\tools\msvc_link_tools.ps1" + $dumpbin = Join-Path (Split-Path $msvcTools.CMAKE_LINKER) 'dumpbin.exe' + + foreach ($suite in $failed) { + $exe = Join-Path (Get-Location) "$suite.exe" + Write-Host "::group::$suite imports" + & $dumpbin /dependents $exe | Select-String -Pattern '\.dll' | ForEach-Object Line + Write-Host "::endgroup::" + foreach ($threads in @('', '1')) { + if ($threads) { $env:BERTINI_NUM_THREADS = $threads } else { Remove-Item Env:\BERTINI_NUM_THREADS -ErrorAction SilentlyContinue } + Write-Host "::group::$suite with BERTINI_NUM_THREADS='$threads'" + if ($cdb) { + & $cdb -g -G -lines -c '!analyze -v; ~*kv 40; q' $exe --log_level=test_suite --log_sink=stderr + } else { + & $exe --log_level=test_suite --log_sink=stderr + } + Write-Host "exit code $LASTEXITCODE" + Write-Host "::endgroup::" + } + # Heap corruption is reported when the heap next looks, often far from the bad write. + # Full page heap puts a guard page after every allocation, so the write itself faults. + if ($cdb) { + $imageOptions = "HKLM:\SOFTWARE\Microsoft\Windows NT\CurrentVersion\Image File Execution Options\$suite.exe" + New-Item -Path $imageOptions -Force | Out-Null + New-ItemProperty -Path $imageOptions -Name GlobalFlag -Value 0x02000000 -PropertyType DWord -Force | Out-Null + New-ItemProperty -Path $imageOptions -Name PageHeapFlags -Value 3 -PropertyType DWord -Force | Out-Null + Remove-Item Env:\BERTINI_NUM_THREADS -ErrorAction SilentlyContinue + Write-Host "::group::$suite under full page heap" + & $cdb -g -G -lines -c '!analyze -v; ~*kv 40; q' $exe --log_level=test_suite --log_sink=stderr + Write-Host "exit code $LASTEXITCODE" + Write-Host "::endgroup::" + Remove-Item -Path $imageOptions -Recurse + } + } + # The wheel build is an INDEPENDENT full compile (prebuilt deps image/tarball + # cibuildwheel, ccache namespace ccache-wheels-*), sharing nothing with the C++ # test job (boost-base from source, ccache namespace ccache-cpp-*). So it does @@ -624,8 +710,6 @@ jobs: needs: [doc_lint] if: ${{ inputs.minimal != true && !failure() && !cancelled() && (inputs.os == '' || inputs.os == 'all' || inputs.os == 'windows') }} runs-on: windows-latest - env: - CONDA_PKGS_DIRS: ${{ github.workspace }}\conda_pkgs strategy: fail-fast: false matrix: @@ -634,17 +718,7 @@ jobs: - uses: actions/checkout@v5 with: persist-credentials: false - - name: Cache conda packages + pip - uses: actions/cache@v5 - continue-on-error: true - with: - path: | - ${{ github.workspace }}\conda_pkgs - ~\AppData\Local\pip\Cache - key: windows-conda-${{ matrix.python-version }}-${{ hashFiles('environment-win.yml', '.github/workflows/build_and_test.yml') }} - restore-keys: | - windows-conda-${{ matrix.python-version }}- - windows-conda- + # Not cached, as in test_cpp_windows (see there; ADR-0078). - uses: conda-incubator/setup-miniconda@v4 with: activate-environment: b2-windows @@ -678,12 +752,8 @@ jobs: $env:CMAKE_GENERATOR_TOOLSET = '' $env:CMAKE_PREFIX_PATH = "$env:CONDA_PREFIX\Library" - # The GitHub Windows runner now ships Visual Studio 2026 (MSVC 14.51), whose STL - # hard-asserts (STL1000) that the compiler is Clang >= 20; the conda - # clang-cl is older. The only errors are that version gate -- the C++17 code compiles - # fine -- so we opt out of the gate. (Proper fix: ship a clang >= 20 toolchain.) - $env:CXXFLAGS = '-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH' - $env:CFLAGS = '-D_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH' + # The runner's Visual Studio 2026 STL (, STL1000) requires Clang >= 20, so + # environment-win.yml asks for clang >=20. $boostHeader = "$env:CONDA_PREFIX\Library\include\boost\archive\archive_exception.hpp" (Get-Content $boostHeader).Replace('public virtual std::exception', 'public std::exception') | Set-Content $boostHeader @@ -693,11 +763,33 @@ jobs: # A cold cache is just a normal build, so this can only help -- never break. $env:CCACHE_DIR = "${{ github.workspace }}\.ccache" $env:CCACHE_MAXSIZE = '400M' + # ccache declines a compile that uses a precompiled header (CMake's cmake_pch) unless + # told to be sloppy about PCH defines and time macros, and it must know the compiler + # is clang-cl. The reason every call was uncacheable until 2026-11-04 (0 of 149) was + # CMake's -clang:-MD dependency flags, which the root CMakeLists now replaces by + # /showIncludes when ccache drives clang-cl; the verbose stats below show hits and + # name any remaining refusal. + $env:CCACHE_SLOPPINESS = 'pch_defines,time_macros' + $env:CCACHE_COMPILERTYPE = 'clang-cl' + + # Link with the Visual Studio the STL comes from, as test_cpp_windows does (see there). + # scikit-build-core splits CMAKE_ARGS as a POSIX shell would: forward slashes, quoted. + $msvcTools = & "${{ github.workspace }}\tools\msvc_link_tools.ps1" + $env:CMAKE_ARGS = ($msvcTools.GetEnumerator() | ForEach-Object { "-D$($_.Key)=`"$($_.Value.Replace('\', '/'))`"" }) -join ' ' + Write-Host "CMAKE_ARGS=$env:CMAKE_ARGS" + Get-Command cmake, ninja, clang-cl, ccache | ForEach-Object { "$($_.Name) = $($_.Source)" } cd ${{ github.workspace }} pip install build delvewheel ccache --zero-stats 2>$null python -m build --wheel --no-isolation - ccache --show-stats + # A native command's exit code does not stop a pwsh step: keep the build's, print what it + # was configured with and the cache stats, then fail. + $buildExitCode = $LASTEXITCODE + clang-cl --version + Select-String -Path bld\CMakeCache.txt -Pattern '^CMAKE_(C|CXX)_COMPILER:|^CMAKE_(LINKER|AR|RC_COMPILER|MT):' -ErrorAction Continue | ForEach-Object Line + ccache --show-stats -v + ccache --version + if ($buildExitCode -ne 0) { Write-Error "the wheel build failed (exit code $buildExitCode)"; exit $buildExitCode } $wheelPath = (Get-ChildItem dist/*.whl | Select-Object -First 1).FullName if (-not $wheelPath) { Write-Error "No wheel found in dist/"; exit 1 } @@ -705,6 +797,13 @@ jobs: Remove-Item dist/*.whl Move-Item repaired/*.whl dist/ + # After the build, whose `conda install python=... --update-deps` can change the environment; + # also when the build failed. + - name: Versions of the main conda packages + if: ${{ !cancelled() }} + shell: pwsh + run: conda list -n b2-windows '^(python|clang|libboost-devel|gmp|mpfr|mpc|libflint|eigen|eigenpy|numpy|scipy|libblas|cmake|ninja|ccache|msmpi|vs2022_win-64)$' + - uses: actions/upload-artifact@v6 with: name: wheels-windows-${{ matrix.python-version }} diff --git a/CHANGELOG.md b/CHANGELOG.md index ef507237f..645c87231 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -74,18 +74,30 @@ The arm64 macOS wheels move to macOS 15, and the fixes made since 4.0.0. ### Added - FLINT 3 (which carries Arb, arbitrary-precision ball arithmetic) is a dependency of the library, the groundwork for interval evaluation and certification (#412). Wheels bundle it; a source build needs `libflint` (Homebrew `flint`, conda-forge `libflint`, apt `libflint-dev`). `bertini.info.flint_version()` reports the linked version. +- The random state can be got and set (#518, ADR-0077). `RandomState` holds everything the calling thread's next draws depend on: the global seed and whether one was chosen, the ordinal the next `AdvanceSolveSeed` takes, and the thread's `b2rand/1` draw stream. `GetRandomState()` returns it and `SetRandomState(state)` sets it, so the draws after setting a state are the draws that followed getting it, and a computation written out between its stages continues with the numbers an uninterrupted run would have drawn. States compare with `==`, and their fields are public. `ToText()` and `RandomState::FromText(text)` write and read it as versioned plaintext (`b2randstate/1`), refusing any other text with `std::invalid_argument`; Boost.Serialization archives carry that text, so a state can go in an MPI message. In Python: `bertini.random.RandomState`, `get_random_state()` and `set_random_state(state)`; a state pickles as its text, and a refused text raises `ValueError`. Per-path and per-worker streams are reseeded from the seed at each use, so they are not part of the state. +- A solution's metadata can be written as exact JSON and read back (#521). `records::SolutionMetaDataToJson` and `records::SolutionMetaDataFromJson` (`records/solution_metadata_json.hpp`) write and read every field of `SolutionMetaData`, both precisions, in the format `b2meta/1`, with the encodings of the track records: doubles as exact decimals, multiprecision values at their own precision, success codes by name. The metadata read back equals the written one in every field, `path_time_seconds` included. A text of another format or numeric type, or one lacking a field, is refused with `std::invalid_argument`. In Python the metadata types gain `to_json()` and `from_json(text)`, and pickle and copy through them, so a solution's metadata crosses processes whole. ### Changed - The arm64 macOS wheels are built on GitHub's `macos-15` runners, and require macOS 15, as the Intel wheels already did. The `macos-14` runner images were retired on 2026-11-02, and a wheel's minimum macOS follows the runner it is built on, since the Homebrew libraries bundled into it are built for that macOS. The prebuilt macOS dependency tarballs are rebuilt on the same runners. +### Deprecated + +- `DeriveSolveSeed()` is `AdvanceSolveSeed()`, and in Python `bertini.random.derive_solve_seed()` is `advance_solve_seed()` (#518). "Derive" read as a derivative; the function advances the solve-seed chain one step and returns its new seed. The old names still work, the C++ one marked `[[deprecated]]` and the Python one warning with a `DeprecationWarning`, and are removed in 5.0. + +### Removed + +- The thread's `std::mt19937` engine and `ThreadEngine()` (ADR-0077). Every draw comes from the `b2rand/1` draw stream (ADR-0044), and nothing in the library drew from the engine, so no result changes; a second piece of random state was a trap for anyone saving the state. + ### Fixed +- Pickling or copying a solution's metadata in Python keeps every field (#521). It went through the fields that can be set, so the two read-only ones, `singular_values` and `latest_path_point`, came back empty. A class that writes itself exactly (`to_json` / `from_json`) now pickles and copies through that. - A `HomotopySolver` solve's records ask now includes its start points (#498). It named the homotopy, target, settings and seed but not the given start points, so with recording on (the default) a second solve of the same homotopy from different start points was the same ask, and was answered from the record of the first: it silently returned the first solve's endpoints. The start system now reports the identity of start points it was given (`StartSystem::GivenStartIdentity`, a digest of their exact values), and the solve adds it to its ask. A solve from given start points therefore has a different run id than in 4.0.0, so records written by 4.0.0 for such solves are asked again rather than recalled. - Differentiating an expression differentiates each distinct subexpression once, however many paths in the expression reach it (#505). It used to differentiate a shared subexpression once per path, so a determinant built by cofactor expansion with its sub-minors shared took time proportional to k! -- 24 s for the three partial derivatives of a 7 x 7 determinant of quadratics, all of it paid on a `System`'s first evaluation, which builds its Jacobian. `Node::Differentiate` now keeps each derivative for the length of the outermost call, and a Jacobian is built under one `node::DerivativeScope`, so subexpressions its functions share are differentiated once among all of them. A node type's own rule is now `DifferentiateImpl`; a Python subclass of `Node` still overrides `Differentiate`. - Building a sum or product no longer walks its operands (#505). Putting the operands in canonical order asks each one's variables, multidegree and, on a tie, canonical encoding, and each answer was a walk of the operand's DAG -- of its whole expansion, for `Degree(v)` under a negation -- so building an expression cost the square of its size, or worse. Each node now keeps a degree record, made once from its operands' records: its variables, its degree and multidegree in each variable it mentions, its total degree, and its answer for every variable it does not mention. `Degree`, `MultiDegree` and `GatherVariables` read it. On the Barth sextic's 50 deflated critical-point systems, the first evaluations took 121 s and take 0.6 s. A node type's degree rule is now `DegreeImpl`; a Python subclass of `Node` still overrides `Degree`. - **The canonicalization tie-break is a content key, and the system encoding version is `b2sysenc/3`** (#505, ADR-0073). Operands that tie on multidegree were ordered by their whole canonical encoding, which costs a walk of the operand. They are now ordered by a content key: the SHA-256 of the operand's own canonical encoding with each of its operands written as that operand's key -- a Merkle hash, made from the operands' keys once per node. The first evaluation of a 7 x 7 determinant of quadratics took 24.6 s in 4.0.0 and takes 0.04 s. Where operands tie, their order in a system's canonical encoding changes, so the encoding version moves to `b2sysenc/3` and every system digest changes: records written by 4.0.0 are asked again rather than recalled, and `System.from_canonical` refuses `b2sysenc/2` text. - Every step a track takes is at most `max_step_size`, the first one included (#508). A track opened with `initial_step_size` whatever `max_step_size` said, and `max_step_size` bounded only the step's growth after that, so with `max_step_size` 1e-3 and the default `initial_step_size` 1/10 the adaptive tracker took 17 steps above the maximum before reaching it: six of 0.1, six of 0.05, then halving down. A track asked to cross a stretch of the path in small steps could cross it in one, and land on another path. A track now opens with the smaller of the two (`Tracker::FirstStepSize`). With the defaults, where both are 1/10, nothing changes. +- On Windows, a worker thread's exit no longer reads freed memory in an executable (ADR-0079). Boost.Multiprecision freed MPFR's per-thread caches from a `thread_local` destructor, which in an executable ran after the thread's MPFR state had gone, and corrupted the heap. b2's worker threads now free those caches themselves before they exit. _______________________________________________________________________________ diff --git a/CMakeLists.txt b/CMakeLists.txt index 8266d9f0d..a7a8efa1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -120,6 +120,19 @@ if(USE_CCACHE) set(CMAKE_CXX_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") set(CMAKE_C_COMPILER_LAUNCHER "${CCACHE_PROGRAM}") message(STATUS "ccache enabled: ${CCACHE_PROGRAM}") + # clang-cl: CMake's Windows-Clang platform module prefers clang's gcc-style dependency + # files, passed through as -clang:-MD -clang:-MT -clang:-MF, and ccache + # (4.14.1 and later, by design) refuses every /clang: option, so with that form NOTHING is + # ever cached -- the Windows CI jobs reported 0 of 149 calls cacheable, "Unsupported + # compiler option", and compiled cold on every run. ccache does support cl's /showIncludes, + # so with clang-cl and ccache use that form, exactly as CMake's Windows-MSVC module sets it. + if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang" AND CMAKE_CXX_COMPILER_FRONTEND_VARIANT STREQUAL "MSVC") + foreach(lang C CXX) + set(CMAKE_DEPFILE_FLAGS_${lang} "/showIncludes") + set(CMAKE_${lang}_DEPFILE_FORMAT msvc) + endforeach() + message(STATUS "ccache with clang-cl: dependency files via /showIncludes, not -clang:-MD") + endif() endif() endif() diff --git a/core/CMakeLists.txt b/core/CMakeLists.txt index eccb45f00..e6b4802d2 100644 --- a/core/CMakeLists.txt +++ b/core/CMakeLists.txt @@ -156,6 +156,7 @@ set(detail_headers include/bertini2/records/output_directory.hpp include/bertini2/records/load_system.hpp include/bertini2/records/draw_functions.hpp + include/bertini2/records/solution_metadata_json.hpp ) set(function_tree_headers @@ -487,6 +488,12 @@ if(WIN32) target_compile_definitions(bertini2 PUBLIC "HAVE_SNPRINTF") endif() +# Boost.JSON is used header-only (records/output_directory.cpp includes ), so +# no boost_json library is linked. Without this, Boost's auto-linking on Windows asks the linker +# for boost_json.lib by name in every translation unit that includes , and the link +# fails wherever the linker does not happen to search a folder holding one. +target_compile_definitions(bertini2 PUBLIC BOOST_JSON_NO_LIB) + # The full PEP 440 version (e.g. 3.0.0.dev6), including any .devN/rc suffix. # jrl-cmakemodules' generated config.hpp bakes in the *stripped* numeric # BERTINI2_VERSION (for cmake/SOVERSION semantics); this carries the complete diff --git a/core/include/bertini2/double_extensions.hpp b/core/include/bertini2/double_extensions.hpp index c5a07175b..97bfa9478 100644 --- a/core/include/bertini2/double_extensions.hpp +++ b/core/include/bertini2/double_extensions.hpp @@ -37,10 +37,6 @@ namespace bertini{ -// Forward declaration — full definition in random.hpp / random.cpp. -// Avoids pulling mpfr_complex.hpp into the Boost.Multiprecision include chain. -std::mt19937& ThreadEngine(); - // The two double-precision scalar types, named to parallel real_mp / complex_mp. // real_dbl is an explicit alias for double so a future change lives in one place. using real_dbl = double; ///< The double-precision real number type. diff --git a/core/include/bertini2/mpfr_extensions.hpp b/core/include/bertini2/mpfr_extensions.hpp index cfac75393..028047953 100644 --- a/core/include/bertini2/mpfr_extensions.hpp +++ b/core/include/bertini2/mpfr_extensions.hpp @@ -34,6 +34,23 @@ Particularly includes Boost.Serialize code for the real_mp, gmp_rational, and gm #include +// Boost frees MPFR's per-thread caches from a thread_local destructor. On Windows, MPFR is built +// by gcc and keeps those caches in libgcc's emulated TLS, and in an executable that destructor runs +// after the DLLs have detached from the thread, so MPFR reads memory already freed: heap corruption +// when a worker thread exits. So on Windows that destructor is never registered, and a thread +// frees its own caches with bertini::ReleaseThreadMpfrCaches (ADR-0079). Only the thread_local +// half is dropped: this keeps Boost's process-wide cleanup, and this header must stay the one +// place b2 reaches Boost's MPFR backend from. +#if defined(_WIN32) +namespace boost { namespace multiprecision { namespace backends { namespace detail { + template <> + inline void mpfr_cleanup::force_instantiate() + { + init.force_instantiate(); + } +}}}} +#endif + #ifdef B2_FORBID_MIXED_ARITHMETIC #include "bertini2/forbid_mixed_arithmetic.hpp" #endif @@ -80,6 +97,19 @@ namespace bertini{ num.precision(prec); } + /** + \brief Free the calling thread's MPFR caches (constants, Bernoulli numbers). + + A thread b2 starts calls this as the last thing it does, while its thread-local storage is + still alive. Calling it from a thread_local destructor defeats the purpose (ADR-0079). + */ + inline void ReleaseThreadMpfrCaches() + { +#if MPFR_VERSION_MAJOR >= 4 + mpfr_free_cache2(MPFR_FREE_LOCAL_CACHE); +#endif + } + } diff --git a/core/include/bertini2/nag_algorithms/zero_dim_solve.hpp b/core/include/bertini2/nag_algorithms/zero_dim_solve.hpp index 5b07f68cd..b4252d438 100644 --- a/core/include/bertini2/nag_algorithms/zero_dim_solve.hpp +++ b/core/include/bertini2/nag_algorithms/zero_dim_solve.hpp @@ -666,20 +666,17 @@ run the endgame, classify the endpoints, report. See the forward-declare doc ab struct ScopedSessionState { unsigned precision_; ///< The ambient default precision at solve entry. - std::mt19937 engine_; ///< The calling thread's legacy engine state at entry. records::DrawStream stream_; ///< The calling thread's pinned draw stream at entry. /// \brief Capture the ambient session state. ScopedSessionState() : precision_(DefaultPrecision()), - engine_(ThreadEngine()), stream_(records::ThreadDrawStream()) {} /// \brief Restore it, whatever the solve did in between. ~ScopedSessionState() { DefaultPrecision(precision_); - ThreadEngine() = engine_; records::ThreadDrawStream() = stream_; } ScopedSessionState(ScopedSessionState const&) = delete; ///< Non-copyable. diff --git a/core/include/bertini2/parallel/thread_pool.hpp b/core/include/bertini2/parallel/thread_pool.hpp index 12aa65ff8..b8330eb16 100644 --- a/core/include/bertini2/parallel/thread_pool.hpp +++ b/core/include/bertini2/parallel/thread_pool.hpp @@ -74,6 +74,8 @@ of the thread. #include #include +#include "bertini2/mpfr_extensions.hpp" // ReleaseThreadMpfrCaches + namespace bertini { namespace parallel { @@ -217,20 +219,25 @@ class WorkerThreadPool { threads_.emplace_back([this, state_factory, track_fn]() mutable { - // Build thread-local state once (System copy + Tracker copy). - StateT state = state_factory(); - - while (true) { - WorkItem item = task_queue_.pop(); + // Build thread-local state once (System copy + Tracker copy). + StateT state = state_factory(); - if (std::holds_alternative(item)) - break; + while (true) + { + WorkItem item = task_queue_.pop(); - TaskT const& task = std::get(item); - ResultT result = track_fn(state, task); - result_queue_.push(std::move(result)); + if (std::holds_alternative(item)) + break; + + TaskT const& task = std::get(item); + ResultT result = track_fn(state, task); + result_queue_.push(std::move(result)); + } } + + // Last, with the state gone and this thread's TLS still alive (ADR-0079). + ReleaseThreadMpfrCaches(); }); } } diff --git a/core/include/bertini2/random.hpp b/core/include/bertini2/random.hpp index 5bdf9634a..ec662577d 100644 --- a/core/include/bertini2/random.hpp +++ b/core/include/bertini2/random.hpp @@ -34,21 +34,16 @@ #include "bertini2/mpfr_complex.hpp" #include "bertini2/records/derive.hpp" #include +#include +#include #include #include +#include namespace bertini { - /** - Returns this thread's canonical mt19937 engine. Every random draw of every - type routes through here — integer/rational (RandomInt/RandomRat), double - (rand_complex/RandReal), and multiprecision (RandomMp and everything built on - it) — so SetGlobalSeed()/ReseedThisThread() control them all uniformly. - */ - std::mt19937& ThreadEngine(); - /** Set the global RNG seed. Every value is a seed, 0 included, as in numpy and Python's random; SetGlobalSeedFromEntropy asks for an unpredictable one. Must be called before @@ -83,7 +78,7 @@ namespace bertini unsigned long GetGlobalSeed(); /** - Reseed this thread's engine deterministically from the global seed mixed with + Reseed this thread's draw stream deterministically from the global seed mixed with stream_key. Call at the top of each TrackSinglePath* with soln_ind as the key so per-step random draws (condition-number probe, PSEG rand-vector) are path-indexed and mode-independent. @@ -99,7 +94,9 @@ namespace bertini unsigned long DerivedWorkerSeed(uint64_t worker_index); /** - Derive an EFFECTIVE PER-SOLVE SEED from the current master. + Advance the solve-seed chain one step and return its new seed: the EFFECTIVE PER-SOLVE SEED + for the next seedless solve. Each call takes the next solve ordinal, so the call after it + returns the next seed of the chain. A seedless solve's setup draws (gamma, patch, start coefficients) come sequentially off the session stream, so they depend on everything drawn before -- recording the @@ -119,7 +116,87 @@ namespace bertini \return A nonzero seed for this solve, ready for SetGlobalSeed and the ask record. */ - unsigned long DeriveSolveSeed(); + unsigned long AdvanceSolveSeed(); + + /** + The former name of AdvanceSolveSeed, which read as a derivative; removed in 5.0. + \return What AdvanceSolveSeed returns. + */ + [[deprecated("DeriveSolveSeed is AdvanceSolveSeed; the old name is removed in 5.0")]] + inline unsigned long DeriveSolveSeed() { return AdvanceSolveSeed(); } + + /** + \brief The random state of a thread: everything its next draws depend on. + + GetRandomState returns the calling thread's, and SetRandomState returns the thread to one, so + the draws after setting a state are the draws that followed getting it -- a computation + written out between its stages continues with the numbers an uninterrupted run would have + drawn (bertiniteam/b2#518). + + Per-path and per-worker streams are not part of it: each is reseeded from the global seed + and its index when it is used (ReseedThisThread, DerivedWorkerSeed). + */ + struct RandomState + { + unsigned long global_seed = 0; ///< The global seed (GetGlobalSeed). + bool seed_chosen = false; ///< Whether a global seed has been chosen yet. + std::uint64_t solve_ordinal = 0; ///< The ordinal the next AdvanceSolveSeed takes. + records::DrawStream draw_stream; ///< The thread's `b2rand/1` draw stream, the source of every draw. + + /** + \brief The state as versioned plaintext, lines beginning `b2randstate/1`, which + FromText reads back. + */ + std::string ToText() const; + + /** + \brief The state a ToText text describes. + \param text The text ToText returned. + \throws std::invalid_argument if `text` is not such a text, of this version. + */ + static RandomState FromText(std::string const& text); + + /// \brief Whether the two states make the same draws: every field equal. + /// \param other The state to compare with. + bool operator==(RandomState const& other) const; + + /// \brief Whether the two states differ in any field. + /// \param other The state to compare with. + bool operator!=(RandomState const& other) const { return !(*this == other); } + + /// Write the state to a Boost.Serialization archive -- for MPI, and beside a system in an + /// archive -- as its ToText text, so every archive carries the one versioned form and a + /// load refuses a state of another version. + template + void save(Archive& ar, const unsigned /*version*/) const + { + std::string const text = ToText(); + ar & text; + } + + /// Read a state an archive holds (see save); throws std::invalid_argument as FromText does. + template + void load(Archive& ar, const unsigned /*version*/) + { + std::string text; + ar & text; + *this = FromText(text); + } + + BOOST_SERIALIZATION_SPLIT_MEMBER() + }; + + /** + \brief The calling thread's random state. + */ + RandomState GetRandomState(); + + /** + \brief Set the calling thread's random state, so its next draws are the ones that followed + getting `state`. + \param state A state GetRandomState returned, or RandomState::FromText read. + */ + void SetRandomState(RandomState const& state); /** diff --git a/core/include/bertini2/records/derive.hpp b/core/include/bertini2/records/derive.hpp index 3a78532a5..6fdb8de69 100644 --- a/core/include/bertini2/records/derive.hpp +++ b/core/include/bertini2/records/derive.hpp @@ -56,6 +56,7 @@ Draw specifications (exact, forever): #pragma once #include +#include #include "bertini2/detail/sha256.hpp" #include "bertini2/mpfr_extensions.hpp" // mpz_int / real_mp (NOT num_traits: that would cycle through random.hpp) @@ -104,6 +105,24 @@ class DrawStream /// k = ceil(digits*log2(10)) + 1, materialized at `digits` digits of precision. real_mp UnitRealMp(unsigned digits); + /// \brief The stream as one line of text: whether it is seeded, its key, its block + /// counter, its current block and the offset into it, the key and block in hex. + std::string ToText() const; + + /// \brief The stream a ToText line describes, at the same position. + /// \param text The line ToText returned. + /// \throws std::invalid_argument if `text` is not such a line. + static DrawStream FromText(std::string const& text); + + /// \brief Whether the two streams make the same draws: same key, counter, block and + /// offset, and both seeded or both not. + /// \param other The stream to compare with. + bool operator==(DrawStream const& other) const; + + /// \brief Whether the two streams differ. + /// \param other The stream to compare with. + bool operator!=(DrawStream const& other) const { return !(*this == other); } + private: // Refill block_ with SHA-256(key || counter) and advance the counter. void NextBlock(); diff --git a/core/include/bertini2/records/solution_metadata_json.hpp b/core/include/bertini2/records/solution_metadata_json.hpp new file mode 100644 index 000000000..bd34878aa --- /dev/null +++ b/core/include/bertini2/records/solution_metadata_json.hpp @@ -0,0 +1,249 @@ +//This file is part of Bertini 2. +// +//solution_metadata_json.hpp is free software: you can redistribute it and/or modify +//it under the terms of the GNU General Public License as published by +//the Free Software Foundation, either version 3 of the License, or +//(at your option) any later version. +// +//solution_metadata_json.hpp is distributed in the hope that it will be useful, +//but WITHOUT ANY WARRANTY; without even the implied warranty of +//MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +//GNU General Public License for more details. +// +//You should have received a copy of the GNU General Public License +//along with solution_metadata_json.hpp. If not, see . +// +// Copyright(C) Bertini2 Development Team +// +// See for a copy of the license, +// as well as COPYING. Bertini2 is provided with permitted +// additional terms in the b2/licenses/ directory. + +/** +\file solution_metadata_json.hpp + +\brief A solution's metadata as exact JSON, and back (bertiniteam/b2#521). + +Every field of algorithm::SolutionMetaData is written, with the scalar encodings of the track +records (solver_recording.hpp): a double as its exact decimal, a multiprecision value as +full-precision decimal with its precision, a success code by its fixed name. Reading the text +back gives metadata equal to the written one in every field, path_time_seconds included, so a +program that keeps solutions between runs keeps what b2 knew about them. The text names its +format, `b2meta/1`, and its numeric type, and a reader refuses any other. +*/ + +#pragma once + +#include +#include + +#include + +#include "bertini2/nag_algorithms/zero_dim_solve.hpp" +#include "bertini2/records/solver_recording.hpp" + +namespace bertini { +namespace records { + +/// \brief The format a solution's metadata is written in; a new encoding is a new version. +constexpr char SolutionMetaDataFormat[] = "b2meta/1"; + +/** +\brief The success code a CanonicalName names. +\param name The name CanonicalName gave. +\return The code. +\throws std::invalid_argument if no code has that name. +*/ +inline SuccessCode SuccessCodeFromName(std::string const& name) +{ + for (auto code : {SuccessCode::NeverStarted, SuccessCode::Success, + SuccessCode::HigherPrecisionNecessary, SuccessCode::ReduceStepSize, + SuccessCode::GoingToInfinity, SuccessCode::FailedToConverge, + SuccessCode::MatrixSolveFailure, + SuccessCode::MatrixSolveFailureFirstPartOfPrediction, + SuccessCode::MaxNumStepsTaken, SuccessCode::MaxPrecisionReached, + SuccessCode::MinStepSizeReached, SuccessCode::Failure, + SuccessCode::SingularStartPoint, SuccessCode::ExternallyTerminated, + SuccessCode::MinTrackTimeReached, SuccessCode::SecurityMaxNormReached, + SuccessCode::CycleNumTooHigh, SuccessCode::FailedToSelectPrecisionAndStepsize, + SuccessCode::WallClockLimitReached}) + if (CanonicalName(code) == name) + return code; + throw std::invalid_argument("no success code is named \"" + name + "\""); +} + +/// \brief The name of a solution's numeric type in its metadata's text. +/// \return "double" or "multiprecision". +template +inline std::string NumericTypeName(); + +/// \brief The double-precision name. +template <> +inline std::string NumericTypeName() { return "double"; } + +/// \brief The multiprecision name. +template <> +inline std::string NumericTypeName() { return "multiprecision"; } + +/** +\brief A solution's metadata as a JSON object: every field, exactly. +\param m The metadata. +\return The object, beginning with its format and numeric type. +*/ +template +boost::json::object EncodeSolutionMetaData(algorithm::SolutionMetaData const& m) +{ + boost::json::object out; + out["format"] = SolutionMetaDataFormat; + out["numeric_type"] = NumericTypeName(); + out["path_index"] = static_cast(m.path_index); + out["solution_index"] = static_cast(m.solution_index); + out["precision_changed"] = m.precision_changed; + out["time_of_first_prec_increase"] = EncodeComplexScalar(m.time_of_first_prec_increase); + out["max_precision_used"] = static_cast(m.max_precision_used); + out["path_time_seconds"] = ExactDoubleText(m.path_time_seconds); + out["num_successful_steps"] = static_cast(m.num_successful_steps); + out["num_failed_steps"] = static_cast(m.num_failed_steps); + out["wall_clock_limit_seconds"] = ExactDoubleText(m.wall_clock_limit_seconds); + out["latest_path_point"] = EncodePoint(m.latest_path_point); + out["pre_endgame_success_code"] = CanonicalName(m.pre_endgame_success_code); + out["condition_number"] = ExactDoubleText(m.condition_number); + { + boost::json::array values; + for (Eigen::Index ii = 0; ii < m.singular_values.size(); ++ii) + values.push_back(boost::json::value(ExactDoubleText(m.singular_values(ii)))); + out["singular_values"] = std::move(values); + } + out["newton_residual"] = ExactDoubleText(m.newton_residual); + out["final_time_used"] = EncodeComplexScalar(m.final_time_used); + out["accuracy_estimate_internal_coords"] = ExactDoubleText(m.accuracy_estimate_internal_coords); + out["accuracy_estimate_user_coords"] = ExactDoubleText(m.accuracy_estimate_user_coords); + out["cycle_num"] = static_cast(m.cycle_num); + out["precision_digits"] = static_cast(m.precision_digits); + out["accuracy_digits"] = static_cast(m.accuracy_digits); + out["endgame_success_code"] = CanonicalName(m.endgame_success_code); + out["function_residual"] = ExactDoubleText(m.function_residual); + out["multiplicity"] = static_cast(m.multiplicity); + out["multiplicity_representative"] = m.multiplicity_representative; + out["is_real"] = m.is_real; + out["is_finite"] = m.is_finite; + out["is_singular"] = m.is_singular; + out["crossing_unresolved"] = m.crossing_unresolved; + out["is_nonsolution"] = m.is_nonsolution; + return out; +} + +/** +\brief The metadata an EncodeSolutionMetaData object describes. +\param in The object. +\return The metadata, equal to the encoded one in every field. +\throws std::invalid_argument if the object is of another format or numeric type, or lacks a +field or holds one of the wrong kind. +*/ +template +algorithm::SolutionMetaData DecodeSolutionMetaData(boost::json::object const& in) +{ + auto const* format = in.if_contains("format"); + if (!format || !format->is_string() || format->as_string() != SolutionMetaDataFormat) + throw std::invalid_argument(std::string("not a ") + SolutionMetaDataFormat + + " solution metadata object"); + auto const* numeric = in.if_contains("numeric_type"); + if (!numeric || !numeric->is_string() || numeric->as_string() != NumericTypeName()) + throw std::invalid_argument(std::string("solution metadata of another numeric type; this " + "reader reads ") + NumericTypeName()); + + auto field = [&in](char const* name) -> boost::json::value const& { + auto const* v = in.if_contains(name); + if (!v) + throw std::invalid_argument(std::string(SolutionMetaDataFormat) + + " solution metadata lacks the field " + name); + return *v; + }; + auto number = [&field](char const* name) { + return field(name).template to_number(); + }; + auto text = [&field](char const* name) { return std::string(field(name).as_string()); }; + auto real = [&text](char const* name) { return DoubleFromText(text(name)); }; + + algorithm::SolutionMetaData m; + try + { + m.path_index = number("path_index"); + m.solution_index = number("solution_index"); + m.precision_changed = field("precision_changed").as_bool(); + m.time_of_first_prec_increase = + DecodeComplexScalar(field("time_of_first_prec_increase").as_array()); + m.max_precision_used = static_cast(number("max_precision_used")); + m.path_time_seconds = real("path_time_seconds"); + m.num_successful_steps = static_cast(number("num_successful_steps")); + m.num_failed_steps = static_cast(number("num_failed_steps")); + m.wall_clock_limit_seconds = real("wall_clock_limit_seconds"); + m.latest_path_point = DecodePoint(field("latest_path_point").as_array()); + m.pre_endgame_success_code = SuccessCodeFromName(text("pre_endgame_success_code")); + m.condition_number = real("condition_number"); + auto const& values = field("singular_values").as_array(); + m.singular_values.resize(static_cast(values.size())); + for (std::size_t ii = 0; ii < values.size(); ++ii) + m.singular_values(static_cast(ii)) = + DoubleFromText(std::string(values[ii].as_string())); + m.newton_residual = real("newton_residual"); + m.final_time_used = DecodeComplexScalar(field("final_time_used").as_array()); + m.accuracy_estimate_internal_coords = real("accuracy_estimate_internal_coords"); + m.accuracy_estimate_user_coords = real("accuracy_estimate_user_coords"); + m.cycle_num = static_cast(number("cycle_num")); + m.precision_digits = static_cast(number("precision_digits")); + m.accuracy_digits = static_cast(number("accuracy_digits")); + m.endgame_success_code = SuccessCodeFromName(text("endgame_success_code")); + m.function_residual = real("function_residual"); + m.multiplicity = static_cast(field("multiplicity").template to_number()); + m.multiplicity_representative = field("multiplicity_representative").as_bool(); + m.is_real = field("is_real").as_bool(); + m.is_finite = field("is_finite").as_bool(); + m.is_singular = field("is_singular").as_bool(); + m.crossing_unresolved = field("crossing_unresolved").as_bool(); + m.is_nonsolution = field("is_nonsolution").as_bool(); + } + catch (std::invalid_argument const&) + { + throw; + } + catch (std::exception const& e) + { + // boost::json's accessors throw system_error for a value of the wrong kind + throw std::invalid_argument(std::string(SolutionMetaDataFormat) + + " solution metadata has a field of the wrong kind: " + e.what()); + } + return m; +} + +/** +\brief A solution's metadata as JSON text. +\param m The metadata. +\return The text of EncodeSolutionMetaData's object. +*/ +template +std::string SolutionMetaDataToJson(algorithm::SolutionMetaData const& m) +{ + return boost::json::serialize(EncodeSolutionMetaData(m)); +} + +/** +\brief The metadata a SolutionMetaDataToJson text describes. +\param text The text. +\return The metadata. +\throws std::invalid_argument if the text is not JSON, or is not such metadata (see +DecodeSolutionMetaData). +*/ +template +algorithm::SolutionMetaData SolutionMetaDataFromJson(std::string const& text) +{ + boost::system::error_code error; + auto const value = boost::json::parse(text, error); + if (error || !value.is_object()) + throw std::invalid_argument(std::string("not a ") + SolutionMetaDataFormat + + " solution metadata text: not a JSON object"); + return DecodeSolutionMetaData(value.as_object()); +} + +} // namespace records +} // namespace bertini diff --git a/core/src/basics/random.cpp b/core/src/basics/random.cpp index e614928b7..f707fa13c 100644 --- a/core/src/basics/random.cpp +++ b/core/src/basics/random.cpp @@ -32,6 +32,9 @@ #include #include #include +#include +#include +#include namespace bertini { @@ -65,32 +68,13 @@ constexpr uint64_t kDomainPath = 0x9a7400000000ULL; // "path" -- per-path tr constexpr uint64_t kDomainWorker = 0x107ce000000ULL; // "worker" -- per-rank child seeds (MPI) constexpr uint64_t kDomainSolve = 0x501e000000000ULL; // "solve" -- effective per-solve seeds -// Ordinal of the next DeriveSolveSeed call under the current master; reset by SetGlobalSeed +// Ordinal the next AdvanceSolveSeed call takes under the current master; reset by SetGlobalSeed // so (master, ordinal) is well-defined. Since a solve rekeys to its own effective seed, // consecutive seedless solves form a deterministic seed CHAIN from the initial master. std::atomic g_solve_ordinal{0}; -// Seed the engine from the FULL (master, domain, index) tuple via std::seed_seq. Using the whole -// 64-bit words (not a uint32_t truncation) means distinct tuples set distinct mt19937 states, so -// streams cannot collide modulo 2^32 (the old bug). -inline void SeedEngine(std::mt19937& eng, uint64_t master, uint64_t domain, uint64_t index) -{ - std::seed_seq seq{ - static_cast(master), static_cast(master >> 32), - static_cast(domain), static_cast(domain >> 32), - static_cast(index), static_cast(index >> 32) - }; - eng.seed(seq); -} - } // anon namespace -// one mt19937 per thread — seeded from entropy on first use (overwritten deterministically by -// SetGlobalSeed / ReseedThisThread before any draw in a seeded run). -thread_local std::mt19937 g_thread_engine{std::random_device{}()}; - -std::mt19937& ThreadEngine() { return g_thread_engine; } - unsigned long GetGlobalSeed() { if (g_global_seed_chosen.load(std::memory_order_acquire)) @@ -128,9 +112,7 @@ void SetGlobalSeed(unsigned long seed) g_solve_ordinal.store(0, std::memory_order_relaxed); // (master, ordinal) restarts here // the setup stream: domain = setup, index = 0. Path/worker streams use other domains, so none // of them can ever reproduce this stream (the old ReseedThisThread(0) == SetGlobalSeed collision). - SeedEngine(g_thread_engine, static_cast(seed), kDomainSetup, 0); - // the pinned stream (b2rand/1, ADR-0044) is the actual source of identity-relevant - // draws; the mt19937 engine above remains only for legacy/non-identity consumers + // The pinned stream (b2rand/1, ADR-0044) is the source of every draw. records::ThreadDrawStream().Reseed(static_cast(seed), kDomainSetup, 0); } @@ -138,7 +120,6 @@ void ReseedThisThread(uint64_t stream_key) { // per-path / per-thread stream: domain = path, index = stream_key. Deterministic from the global // seed and distinct for every stream_key (and distinct from the setup stream). - SeedEngine(g_thread_engine, static_cast(GetGlobalSeed()), kDomainPath, stream_key); records::ThreadDrawStream().Reseed(static_cast(GetGlobalSeed()), kDomainPath, stream_key); } @@ -153,7 +134,7 @@ unsigned long DerivedWorkerSeed(uint64_t worker_index) return static_cast(h); } -unsigned long DeriveSolveSeed() +unsigned long AdvanceSolveSeed() { // (master, solve ordinal) -> child seed, pure integer arithmetic (splitmix64): // identical on every platform, and DELIBERATELY independent of the thread-local @@ -173,6 +154,96 @@ unsigned long DeriveSolveSeed() return s; } +namespace { + +constexpr char RandomStateVersion[] = "b2randstate/1"; + +// the rest of `line` after `label` and one space, or false if the line is not labelled so +bool AfterLabel(std::string const& line, std::string const& label, std::string& rest) +{ + if (line.compare(0, label.size() + 1, label + ' ') != 0) + return false; + rest = line.substr(label.size() + 1); + return true; +} + +} // anon namespace + +std::string RandomState::ToText() const +{ + std::ostringstream out; + out << RandomStateVersion << '\n' + << "global_seed " << global_seed << '\n' + << "seed_chosen " << (seed_chosen ? 1 : 0) << '\n' + << "solve_ordinal " << solve_ordinal << '\n' + << "draw_stream " << draw_stream.ToText() << '\n'; + return out.str(); +} + +RandomState RandomState::FromText(std::string const& text) +{ + auto refuse = [](std::string const& why) { + throw std::invalid_argument(std::string("not a ") + RandomStateVersion + + " random state: " + why); + }; + std::istringstream in(text); + std::string line, rest; + if (!std::getline(in, line) || line != RandomStateVersion) + refuse("the first line is \"" + line + "\""); + + RandomState state; + int chosen; + if (!std::getline(in, line) || !AfterLabel(line, "global_seed", rest) + || !(std::istringstream(rest) >> state.global_seed)) + refuse("expected the line \"global_seed \", read \"" + line + "\""); + if (!std::getline(in, line) || !AfterLabel(line, "seed_chosen", rest) + || !(std::istringstream(rest) >> chosen) || (chosen != 0 && chosen != 1)) + refuse("expected the line \"seed_chosen <0 or 1>\", read \"" + line + "\""); + state.seed_chosen = (chosen == 1); + if (!std::getline(in, line) || !AfterLabel(line, "solve_ordinal", rest) + || !(std::istringstream(rest) >> state.solve_ordinal)) + refuse("expected the line \"solve_ordinal \", read \"" + line + "\""); + + if (!std::getline(in, line) || !AfterLabel(line, "draw_stream", rest)) + refuse("expected the line \"draw_stream ...\", read \"" + line + "\""); + try + { + state.draw_stream = records::DrawStream::FromText(rest); + } + catch (std::invalid_argument const& e) + { + refuse(e.what()); + } + + if (std::getline(in, line) && !line.empty()) + refuse("unexpected text after the draw_stream line: \"" + line + "\""); + return state; +} + +bool RandomState::operator==(RandomState const& other) const +{ + return global_seed == other.global_seed && seed_chosen == other.seed_chosen + && solve_ordinal == other.solve_ordinal && draw_stream == other.draw_stream; +} + +RandomState GetRandomState() +{ + RandomState state; + state.global_seed = g_global_seed.load(std::memory_order_relaxed); + state.seed_chosen = g_global_seed_chosen.load(std::memory_order_acquire); + state.solve_ordinal = g_solve_ordinal.load(std::memory_order_relaxed); + state.draw_stream = records::ThreadDrawStream(); + return state; +} + +void SetRandomState(RandomState const& state) +{ + g_global_seed.store(state.global_seed, std::memory_order_relaxed); + g_global_seed_chosen.store(state.seed_chosen, std::memory_order_release); + g_solve_ordinal.store(state.solve_ordinal, std::memory_order_relaxed); + records::ThreadDrawStream() = state.draw_stream; +} + real_mp RandomMp() diff --git a/core/src/records/derive.cpp b/core/src/records/derive.cpp index 38a16b56d..29efad3f6 100644 --- a/core/src/records/derive.cpp +++ b/core/src/records/derive.cpp @@ -29,7 +29,13 @@ seed means. #include "bertini2/records/derive.hpp" +#include + +#include +#include #include +#include +#include #include namespace bertini { @@ -43,6 +49,34 @@ namespace { s.push_back(static_cast((v >> shift) & 0xff)); } + // the stream's key and its current block are 32 bytes each, written as lowercase hex + constexpr std::size_t kStreamBytes = 32; + + std::string AsHex(unsigned char const* bytes) + { + std::string out; + boost::algorithm::hex_lower(bytes, bytes + kStreamBytes, std::back_inserter(out)); + return out; + } + + // false if the text is not exactly 32 bytes of hex + bool ReadHex(std::string const& hex, unsigned char* bytes) + { + std::string decoded; + try + { + boost::algorithm::unhex(hex, std::back_inserter(decoded)); + } + catch (boost::algorithm::hex_decode_error const&) + { + return false; + } + if (decoded.size() != kStreamBytes) + return false; + std::copy(decoded.begin(), decoded.end(), bytes); + return true; + } + } // unnamed namespace @@ -83,6 +117,39 @@ unsigned char DrawStream::NextByte() return block_[offset_++]; } +std::string DrawStream::ToText() const +{ + // the block is written, not recomputed from the key and counter, so the text says + // exactly what the next draws are made of + std::ostringstream out; + out << (seeded_ ? 1 : 0) << ' ' << AsHex(key_.bytes.data()) << ' ' << counter_ << ' ' + << AsHex(block_) << ' ' << offset_; + return out.str(); +} + +DrawStream DrawStream::FromText(std::string const& text) +{ + std::istringstream in(text); + int seeded; + std::string key_hex, block_hex, rest; + unsigned offset; + DrawStream stream; + if (!(in >> seeded >> key_hex >> stream.counter_ >> block_hex >> offset) || (in >> rest) + || (seeded != 0 && seeded != 1) || offset > 32 + || !ReadHex(key_hex, stream.key_.bytes.data()) || !ReadHex(block_hex, stream.block_)) + throw std::invalid_argument("not a b2rand/1 draw stream: \"" + text + "\""); + stream.seeded_ = (seeded == 1); + stream.offset_ = offset; + return stream; +} + +bool DrawStream::operator==(DrawStream const& other) const +{ + return seeded_ == other.seeded_ && key_.bytes == other.key_.bytes + && counter_ == other.counter_ && offset_ == other.offset_ + && std::equal(block_, block_ + 32, other.block_); +} + std::uint64_t DrawStream::Uint64() { std::uint64_t v = 0; diff --git a/core/test/classes/fundamentals_test.cpp b/core/test/classes/fundamentals_test.cpp index 65bdc8cdd..abbcb00f8 100644 --- a/core/test/classes/fundamentals_test.cpp +++ b/core/test/classes/fundamentals_test.cpp @@ -293,7 +293,7 @@ BOOST_AUTO_TEST_CASE(RandomMP_nondefault_precision_100) } // The multiprecision RandomMp generator now draws from the single per-thread -// engine (ThreadEngine) shared by every random type, so SetGlobalSeed controls +// draw stream shared by every random type, so SetGlobalSeed controls // it. Before this it used a private, unseedable independent_bits_engine. These // tests pin that contract: same seed -> identical mp draws; different seed -> // different draws. diff --git a/core/test/classes/seeded_randomness_test.cpp b/core/test/classes/seeded_randomness_test.cpp index 076032030..b1eef341f 100644 --- a/core/test/classes/seeded_randomness_test.cpp +++ b/core/test/classes/seeded_randomness_test.cpp @@ -31,6 +31,10 @@ draw reproducibility. #include #include +#include +#include +#include +#include #include "bertini2/records/derive.hpp" #include "bertini2/random.hpp" @@ -52,9 +56,16 @@ System ParseCircleLine() } // build the whole random cascade for one seed: TD start + gamma homotopy + patch +System HomotopyFromTheCurrentState(); System SeededHomotopy(unsigned long seed) { SetGlobalSeed(seed); + return HomotopyFromTheCurrentState(); +} + +// ... drawing from wherever the random state is now +System HomotopyFromTheCurrentState() +{ auto sys = ParseCircleLine(); sys.Homogenize(); sys.AutoPatch(); @@ -95,34 +106,48 @@ BOOST_AUTO_TEST_CASE(seeded_session_draws_are_pinned) // ---- reproducibility semantics ---- -// The effective per-solve seed: DeriveSolveSeed captures the session stream's current -// position, so a seedless solve records a seed that reproduces it STANDALONE instead +// The effective per-solve seed: AdvanceSolveSeed takes the next link of a chain rooted at +// the master seed, so a seedless solve records a seed that reproduces it STANDALONE instead // of a session master that silently under-determines a mid-session run. -BOOST_AUTO_TEST_CASE(derived_solve_seeds_capture_the_stream_deterministically) +BOOST_AUTO_TEST_CASE(solve_seeds_advance_along_a_chain_from_the_master_seed) { SetGlobalSeed(42); - auto const s1 = DeriveSolveSeed(); - auto const s2 = DeriveSolveSeed(); + auto const s1 = AdvanceSolveSeed(); + auto const s2 = AdvanceSolveSeed(); BOOST_CHECK(s1 != 0u); // 0 means "entropy" to a classic input file BOOST_CHECK(s1 != s2); // consecutive solves get distinct seeds BOOST_CHECK(s1 <= 0xFFFFFFFFul); // 32-bit portable (Windows unsigned long) BOOST_CHECK(s2 <= 0xFFFFFFFFul); SetGlobalSeed(42); // same master => - BOOST_CHECK_EQUAL(DeriveSolveSeed(), s1); // the same derived seeds, in order - BOOST_CHECK_EQUAL(DeriveSolveSeed(), s2); + BOOST_CHECK_EQUAL(AdvanceSolveSeed(), s1); // the same seeds, in order + BOOST_CHECK_EQUAL(AdvanceSolveSeed(), s2); // DELIBERATELY independent of the draw streams: a multithreaded solve leaves the // calling thread's stream at a scheduling-dependent position, so the derivation // must not read it -- intervening draws change nothing SetGlobalSeed(42); (void)RandomRat(); - BOOST_CHECK_EQUAL(DeriveSolveSeed(), s1); + BOOST_CHECK_EQUAL(AdvanceSolveSeed(), s1); SetGlobalSeed(43); // different master, different chain - BOOST_CHECK(DeriveSolveSeed() != s1); + BOOST_CHECK(AdvanceSolveSeed() != s1); } +// the former name advances the same chain until it is removed in 5.0 +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wdeprecated-declarations" +BOOST_AUTO_TEST_CASE(the_former_name_derive_solve_seed_advances_the_same_chain) +{ + SetGlobalSeed(42); + auto const first = AdvanceSolveSeed(); + auto const second = AdvanceSolveSeed(); + SetGlobalSeed(42); + BOOST_CHECK_EQUAL(DeriveSolveSeed(), first); + BOOST_CHECK_EQUAL(AdvanceSolveSeed(), second); +} +#pragma GCC diagnostic pop + BOOST_AUTO_TEST_CASE(same_seed_same_draws_different_seed_different_draws) { SetGlobalSeed(42); @@ -271,4 +296,141 @@ BOOST_AUTO_TEST_CASE(seed_42_homotopy_digest_is_pinned_cross_platform) BOOST_CHECK_EQUAL(hex, pinned); } +// ---- getting and setting the random state (bertiniteam/b2#518) ---- + +namespace { + +// a mix of every kind of draw: multiprecision numbers, a double, the stream's raw bytes +// (21 bits, so the position ends partway into a block), and the solve-seed chain +std::string DrawsFromHere() +{ + std::ostringstream out; + out << RandomMp(30) << ' ' << records::ThreadDrawStream().Bits(21) << ' ' + << RandReal() << ' ' << AdvanceSolveSeed() << ' ' << RandomMp(60); + return out.str(); +} + +} // unnamed namespace + +BOOST_AUTO_TEST_CASE(a_state_set_again_draws_what_followed_getting_it) +{ + SetGlobalSeed(1042); + DrawsFromHere(); + auto const state = GetRandomState(); + auto const after_getting_it = DrawsFromHere(); + + SetGlobalSeed(77); // another seed, another solve ordinal, other positions + DrawsFromHere(); + SetRandomState(state); + BOOST_CHECK_EQUAL(DrawsFromHere(), after_getting_it); + BOOST_CHECK_EQUAL(GetGlobalSeed(), 1042ul); +} + +BOOST_AUTO_TEST_CASE(a_homotopy_built_after_setting_a_state_is_the_one_built_after_getting_it) +{ + SetGlobalSeed(42); + DrawsFromHere(); + auto const state = GetRandomState(); + auto const after_getting_it = HomotopyFromTheCurrentState().ContentDigest().Hex(); + + HomotopyFromTheCurrentState(); + SetRandomState(state); + BOOST_CHECK_EQUAL(HomotopyFromTheCurrentState().ContentDigest().Hex(), after_getting_it); +} + +BOOST_AUTO_TEST_CASE(a_state_holds_the_seed_and_ordinal_and_compares_by_its_fields) +{ + SetGlobalSeed(5); + DrawsFromHere(); + auto const state = GetRandomState(); + BOOST_CHECK_EQUAL(state.global_seed, 5ul); + BOOST_CHECK(state.seed_chosen); + BOOST_CHECK_EQUAL(state.solve_ordinal, 1u); + BOOST_CHECK(GetRandomState() == state); + + DrawsFromHere(); + BOOST_CHECK(GetRandomState() != state); + auto moved_stream = state; + moved_stream.draw_stream.Uint64(); + BOOST_CHECK(moved_stream != state); +} + +BOOST_AUTO_TEST_CASE(a_state_built_field_by_field_draws_as_one_got) +{ + SetGlobalSeed(11); + DrawsFromHere(); + auto const got = GetRandomState(); + auto const from_got = DrawsFromHere(); + + RandomState built; + built.global_seed = got.global_seed; + built.seed_chosen = got.seed_chosen; + built.solve_ordinal = got.solve_ordinal; + built.draw_stream = got.draw_stream; + SetGlobalSeed(12); + SetRandomState(built); + BOOST_CHECK_EQUAL(DrawsFromHere(), from_got); +} + +BOOST_AUTO_TEST_CASE(a_state_written_as_text_reads_back_equal) +{ + SetGlobalSeed(5); + DrawsFromHere(); + auto const state = GetRandomState(); + auto const text = state.ToText(); + BOOST_CHECK_EQUAL(text.rfind("b2randstate/1\nglobal_seed 5\nseed_chosen 1\nsolve_ordinal 1\n" + "draw_stream 1 ", 0), 0u); + BOOST_CHECK(RandomState::FromText(text) == state); + BOOST_CHECK_EQUAL(RandomState::FromText(text).ToText(), text); +} + +BOOST_AUTO_TEST_CASE(a_text_that_is_not_a_state_is_refused) +{ + SetGlobalSeed(9); + DrawsFromHere(); + auto const text = GetRandomState().ToText(); + + std::string other_version = text; + other_version.replace(0, std::string("b2randstate/1").size(), "b2randstate/2"); + std::string short_key = text; + auto const at = short_key.find("draw_stream 1 ") + std::string("draw_stream 1 ").size(); + short_key.erase(at, 2); + std::string no_stream = text.substr(0, text.find("draw_stream")); + + for (auto const& bad : {std::string(), std::string("global_seed 9"), other_version, short_key, + no_stream, text + "more\n"}) + BOOST_CHECK_THROW(RandomState::FromText(bad), std::invalid_argument); +} + +BOOST_AUTO_TEST_CASE(a_state_goes_through_text_and_binary_archives) +{ + SetGlobalSeed(13); + DrawsFromHere(); + auto const state = GetRandomState(); + + std::stringstream text_archive; + { + boost::archive::text_oarchive out(text_archive); + out << state; + } + RandomState from_text; + { + boost::archive::text_iarchive in(text_archive); + in >> from_text; + } + BOOST_CHECK(from_text == state); + + std::stringstream binary_archive; + { + boost::archive::binary_oarchive out(binary_archive); + out << state; + } + RandomState from_binary; + { + boost::archive::binary_iarchive in(binary_archive); + in >> from_binary; + } + BOOST_CHECK(from_binary == state); +} + BOOST_AUTO_TEST_SUITE_END() diff --git a/core/test/classes/system_test.cpp b/core/test/classes/system_test.cpp index 62ed45e47..e2056995f 100644 --- a/core/test/classes/system_test.cpp +++ b/core/test/classes/system_test.cpp @@ -1285,7 +1285,7 @@ BOOST_AUTO_TEST_CASE(system_estimate_coeff_bound_homogenized_quartic) // AutoPatch coefficients come from RandomMp; seeding makes this test's draws // (and hence the <10 threshold below) deterministic regardless of which other - // tests ran first. RandomMp now honors SetGlobalSeed (it shares ThreadEngine). + // tests ran first. RandomMp honors SetGlobalSeed (it draws from the thread's draw stream). bertini::SetGlobalSeed(1u); bertini::System sys; diff --git a/core/test/nag_algorithms/threaded_solve.cpp b/core/test/nag_algorithms/threaded_solve.cpp index ba9e7719b..66ed1dcb2 100644 --- a/core/test/nag_algorithms/threaded_solve.cpp +++ b/core/test/nag_algorithms/threaded_solve.cpp @@ -30,6 +30,7 @@ on every platform (no MPI required), which is what gives us cross-OS threading c #include #include +#include #include #include #include @@ -88,6 +89,37 @@ BOOST_AUTO_TEST_CASE(thread_pool_runs_every_task_once) BOOST_CHECK_EQUAL(got, expected); } +// A worker that filled MPFR's per-thread caches must exit cleanly. On Windows, freeing them from +// a thread_local destructor read freed memory and corrupted the heap (ADR-0079). lgamma fills the +// Bernoulli cache, the first one MPFR frees, and acos of a negative fills the pi cache. +BOOST_AUTO_TEST_CASE(thread_pool_workers_that_filled_mpfr_caches_exit_cleanly) +{ + using bertini::parallel::WorkerThreadPool; + using bertini::real_mp; + + auto factory = []() { return std::unique_ptr(new int(0)); }; + auto track = [](std::unique_ptr& state, int const& task) { + ++(*state); + bertini::SetThreadPrecision(50); + real_mp const log_of_factorial_plus_pi = lgamma(real_mp(task + 1)) + acos(real_mp(-1)); + return log_of_factorial_plus_pi.convert_to(); + }; + + WorkerThreadPool pool(4, factory, track); + + const int num_tasks = 40; + for (int i = 0; i < num_tasks; ++i) + pool.submit(4); + + double total = 0; + for (int k = 0; k < num_tasks; ++k) + total += pool.collect(); + + pool.shutdown(); // the workers exit here: the crash this test guards against was at thread exit + + BOOST_CHECK_CLOSE(total, num_tasks * (std::log(24.0) + std::acos(-1.0)), 1e-10); +} + // Each thread builds its own state via the factory; states must not be shared. We hand each // thread a distinct heap counter and confirm the per-thread increments sum to the task count // (a shared-state race would lose increments under a data race / give a wrong total). diff --git a/core/test/nag_algorithms/zero_dim_records.cpp b/core/test/nag_algorithms/zero_dim_records.cpp index a0e44d8f6..6a41a20e5 100644 --- a/core/test/nag_algorithms/zero_dim_records.cpp +++ b/core/test/nag_algorithms/zero_dim_records.cpp @@ -45,6 +45,7 @@ homotopy identical, so recalled and computed results are directly comparable. #include "bertini2/records/load_system.hpp" #include "bertini2/records/solver_recording.hpp" #include "bertini2/records/config_encoding.hpp" +#include "bertini2/records/solution_metadata_json.hpp" #include "test/utility/encoding_registry.hpp" @@ -959,7 +960,8 @@ BOOST_AUTO_TEST_CASE(the_accuracy_estimates_are_recorded_under_names_that_say_th // an earlier record: the same value, under the key it was written with auto earlier = encoded; - earlier["accuracy_estimate"] = earlier.at("accuracy_estimate_internal_coords"); + boost::json::value const internal_coords = earlier.at("accuracy_estimate_internal_coords"); // a copy: the insert below reallocates the object + earlier["accuracy_estimate"] = internal_coords; earlier.erase("accuracy_estimate_internal_coords"); auto const from_earlier = records::DecodeFullPathResult(earlier, 3); BOOST_CHECK_EQUAL(from_earlier.accuracy_estimate_internal_coords, 3.5e-12); @@ -967,3 +969,122 @@ BOOST_AUTO_TEST_CASE(the_accuracy_estimates_are_recorded_under_names_that_say_th } BOOST_AUTO_TEST_SUITE_END() + + +// ---- a solution's metadata as exact JSON, and back (bertiniteam/b2#521) ---- + +BOOST_AUTO_TEST_SUITE(solution_metadata_json) + +BOOST_AUTO_TEST_CASE(a_solves_metadata_reads_back_equal_in_every_field) +{ + SetGlobalSeed(42); + auto sys = TwoQuadrics(); + ZD solver(sys); + solver.DefaultSetup(); + solver.Solve(); + + auto const& metadata = solver.SolutionMetadata(); + BOOST_REQUIRE_EQUAL(metadata.size(), 4u); + for (auto m : metadata) + { + auto const text = records::SolutionMetaDataToJson(m); + auto metadata_loaded = records::SolutionMetaDataFromJson(text); + BOOST_CHECK(metadata_loaded == m); + BOOST_CHECK_EQUAL(metadata_loaded.path_time_seconds, m.path_time_seconds); // not part of == + BOOST_CHECK_EQUAL(records::SolutionMetaDataToJson(metadata_loaded), text); + } +} + +BOOST_AUTO_TEST_CASE(multiprecision_metadata_reads_back_at_its_own_precision) +{ + DefaultPrecision(30); + algorithm::SolutionMetaData m; + m.path_index = 7; + m.solution_index = 5; + m.precision_changed = true; + m.time_of_first_prec_increase = complex_mp( + real_mp("0.123456789012345678901234567890123456789012345678901234567", 60), + real_mp("-0.25", 60)); + m.time_of_first_prec_increase.precision(60); + m.max_precision_used = 60; + m.path_time_seconds = 0.013671875; + m.num_successful_steps = 21; + m.num_failed_steps = 3; + m.wall_clock_limit_seconds = 2.5; + m.latest_path_point = Vec(2); + m.latest_path_point(0) = m.time_of_first_prec_increase; + m.latest_path_point(1) = complex_mp(real_mp("1.75", 60), real_mp("3.0e-41", 60)); + m.latest_path_point(1).precision(60); + m.pre_endgame_success_code = SuccessCode::MaxPrecisionReached; + m.condition_number = 144181.98972868928; + m.singular_values = Vec(2); + m.singular_values << 3.25, 1.5e-9; + m.newton_residual = 2.5e-31; + m.final_time_used = complex_mp(real_mp("0.0625", 60), real_mp("0", 60)); + m.final_time_used.precision(60); + m.accuracy_estimate_internal_coords = 1.990268618987469e-12; + m.accuracy_estimate_user_coords = 7.25e-8; + m.cycle_num = 2; + m.precision_digits = 60; + m.accuracy_digits = 11; + m.endgame_success_code = SuccessCode::CycleNumTooHigh; + m.function_residual = 6.5e-29; + m.multiplicity = 3; + m.multiplicity_representative = false; + m.is_real = true; + m.is_finite = true; + m.is_singular = true; + m.crossing_unresolved = true; + m.is_nonsolution = true; + + auto metadata_loaded = + records::SolutionMetaDataFromJson(records::SolutionMetaDataToJson(m)); + BOOST_CHECK(metadata_loaded == m); + BOOST_CHECK_EQUAL(metadata_loaded.path_time_seconds, m.path_time_seconds); + BOOST_CHECK_EQUAL(metadata_loaded.time_of_first_prec_increase.precision(), 60u); + BOOST_CHECK_EQUAL(metadata_loaded.latest_path_point(0).real().str(0, std::ios::scientific), + m.latest_path_point(0).real().str(0, std::ios::scientific)); + BOOST_CHECK_EQUAL(metadata_loaded.latest_path_point(1).precision(), 60u); +} + +BOOST_AUTO_TEST_CASE(text_that_is_not_such_metadata_is_refused) +{ + algorithm::SolutionMetaData m; + m.path_index = 1; + m.solution_index = 1; + m.time_of_first_prec_increase = complex_dbl(0.5, 0); + m.condition_number = 4.5; + m.newton_residual = 1.5e-15; + m.final_time_used = complex_dbl(0, 0); + m.accuracy_estimate_internal_coords = 2.5e-13; + m.accuracy_estimate_user_coords = 2.5e-13; + m.cycle_num = 1; + m.function_residual = 3.5e-16; + auto const object = records::EncodeSolutionMetaData(m); + + auto with = [&object](char const* key, boost::json::value value) { + auto changed = object; + changed[key] = std::move(value); + return boost::json::serialize(changed); + }; + auto without = [&object](char const* key) { + auto changed = object; + changed.erase(key); + return boost::json::serialize(changed); + }; + + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson("not json"), std::invalid_argument); + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson(with("format", "b2meta/2")), + std::invalid_argument); + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson(boost::json::serialize(object)), + std::invalid_argument); + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson(without("cycle_num")), + std::invalid_argument); + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson(with("is_real", "yes")), + std::invalid_argument); + BOOST_CHECK_THROW(records::SolutionMetaDataFromJson( + with("endgame_success_code", "Succeeded")), + std::invalid_argument); +} + +BOOST_AUTO_TEST_SUITE_END() diff --git a/docs/adr/0044-seed-rooted-randomness.md b/docs/adr/0044-seed-rooted-randomness.md index 6d1a7335a..3ed230bb9 100644 --- a/docs/adr/0044-seed-rooted-randomness.md +++ b/docs/adr/0044-seed-rooted-randomness.md @@ -44,7 +44,8 @@ draw routes: dependency-free header (`records/draw_functions.hpp`) breaks the include cycle `num_traits → random.hpp → derive.hpp → num_traits`. - The `mt19937` engine and its seeding remain for any non-identity consumer, but no - identity-relevant draw touches it. + identity-relevant draw touches it. (Removed by ADR-0077: no draw used it, and a second + piece of random state was a trap for anyone saving the state.) ## Consequences diff --git a/docs/adr/0076-ccache-with-clang-cl-uses-showincludes.md b/docs/adr/0076-ccache-with-clang-cl-uses-showincludes.md new file mode 100644 index 000000000..c0e12d5df --- /dev/null +++ b/docs/adr/0076-ccache-with-clang-cl-uses-showincludes.md @@ -0,0 +1,21 @@ +# ADR-0076: With ccache, clang-cl writes dependency files through /showIncludes + +**Status:** Accepted +**Date:** 2026-11-04 + +## Context + +Both Windows CI jobs compile with clang-cl through ccache, and for as long as the stats step has existed it has reported every call uncacheable: `Cacheable calls: 0 / 149`, `Unsupported compiler option: 149 / 149`. The saved cache was 13 KB, and the C++ test job compiled cold on every run, 32 minutes. The history was three commits that never read that number: ccache was switched off for the C++ job "as a precaution" (87bb4c612), the wheel job's cache directory was persisted "to measure the hit rate" (b4884c790), and ccache was switched back on because the wheel job had "run clang-cl through ccache for months without issue" (0d4d2b0e8, #322). Nothing broke; nothing was cached either. + +The ccache log names the refused option: `-clang:-MD`, with `-clang:-MT` and `-clang:-MF`. CMake's `Platform/Windows-Clang.cmake` sets those for clang-cl under Ninja on purpose ("Prefer clang-cl's gcc-style depfile over cl-style /showIncludes"), overriding the `/showIncludes` form its MSVC base module sets. ccache 4.14.1 (2026-09-27) made ccache reject every `/clang:` pass-through option by design, to avoid caching compilations whose pass-through options it cannot see, and 4.14.2 keeps that. The two meet on every translation unit, so nothing is cacheable. Precompiled headers and debug information, the usual suspects, were not involved: PCH is used by only two targets, and the refusal was universal. + +## Decision + +When ccache drives clang-cl, the root `CMakeLists.txt` sets `CMAKE_DEPFILE_FLAGS_` to `/showIncludes` and `CMAKE__DEPFILE_FORMAT` to `msvc` for C and C++, the values CMake's own Windows-MSVC module uses for cl. ccache supports `/showIncludes` (it stores the include notes with the result and replays them on a hit, so Ninja's `deps = msvc` parsing still works). The Windows jobs also set `CCACHE_SLOPPINESS=pch_defines,time_macros` so the two targets that use a precompiled header are cacheable, `CCACHE_COMPILERTYPE=clang-cl`, and print `ccache --show-stats -v`, which lists uncacheable calls by reason. + +## Consequences + +- Do not remove the `/showIncludes` override because "CMake prefers the gcc-style depfile": with ccache, that preference costs every cache hit on Windows. If ccache ever accepts `-clang:` options again, the override can go, and the verbose stats will show it. +- Do not switch ccache off on Windows as a precaution, and do not conclude it works from the absence of failures; read `Cacheable calls` in the stats step. A cache that saves kilobytes is a cache that saves nothing. +- Keep the verbose stats step: it is the measurement the three earlier commits lacked. +- The first run after the fix: C++ tests 133 of 149 calls cacheable (all misses, as a first run must be) and a 135 MB cache saved against 13 KB before; the wheel job 90 of 106 and 197 MB. The 16 calls still refused are mimalloc's C files: it is a FetchContent subproject whose own `project()` call re-applies the platform defaults in its directory, so those few seconds of compiles keep `-clang:-MD`. Every b2 translation unit is cached. diff --git a/docs/adr/0077-the-random-state-is-one-value.md b/docs/adr/0077-the-random-state-is-one-value.md new file mode 100644 index 000000000..0670692ee --- /dev/null +++ b/docs/adr/0077-the-random-state-is-one-value.md @@ -0,0 +1,27 @@ +# ADR-0077: The random state is one value, and every draw comes from one stream + +**Status:** Accepted +**Date:** 2026-11-03 + +## Context + +A long computation that is written out between its stages must continue, after a restart, with the numbers an uninterrupted run would have drawn (#518). The cellular decomposition of the Barth decic on the 3-sphere runs for many hours in eight stages, and its stages draw from b2's global random state: `random_vector`, `AdvanceSolveSeed` (formerly `DeriveSolveSeed`), the gammas and patches of every solve. b2 could set a seed and read it back, but could not get the state the draws had reached, nor set it. + +The state was split across interfaces. ADR-0044 moved every draw onto the pinned `b2rand/1` draw stream but kept the thread's `std::mt19937` engine "for any non-identity consumer", seeded alongside the stream; nothing drew from it any more, yet `SetGlobalSeed`, `ReseedThisThread` and the solve's `ScopedSessionState` still seeded, saved and restored it. A caller saving "the random state" had to know to take both, and the global seed and the solve-seed ordinal besides. + +## Decision + +- **One stream.** The `mt19937` engine and `ThreadEngine()` are removed. Every draw comes from the thread's `b2rand/1` draw stream. No draw changes: none used the engine. +- **One value.** `bertini::RandomState` (`random.hpp`) is everything the calling thread's next draws depend on: `global_seed`, `seed_chosen`, `solve_ordinal`, and `draw_stream`. `GetRandomState()` returns the calling thread's; `SetRandomState(state)` sets it, so the draws after setting a state are the draws that followed getting it. States compare with `==`. +- **The calling thread only.** Per-path and per-worker streams are not part of it: each is reseeded from the global seed and its index when it is used (`ReseedThisThread`, `DerivedWorkerSeed`, ADR-0071). +- **Public fields.** A caller may build a state field by field, including one no run reached. +- **Text is serialization, at the edge.** `ToText()` and `FromText(text)` write and read versioned plaintext, `b2randstate/1`, one labelled line per field (the draw stream's key and current block in hex), for plaintext records. `FromText` refuses any other text with `std::invalid_argument`. The state version goes with the draw derivation: a new `b2rand` version means a new `b2randstate` version. +- **Archives carry the text.** Boost.Serialization (`save`/`load`) writes the `ToText` text, so every archive -- text, binary, an MPI message -- carries the one versioned form, and a load refuses a state of another version. +- **Python.** `bertini.random.RandomState` with the same fields, `to_text()`, `from_text()`, `==`, and pickling as its text; `get_random_state()` and `set_random_state(state)`, named beside `set_random_seed`. + +## Consequences + +- Do not add a second source of randomness beside the draw stream. A new kind of draw is a new function of the stream; a new piece of per-thread random state is a new field of `RandomState`, with its line in the text and a new `b2randstate` version. +- Do not save or restore part of the state by hand. Code that brackets a computation's randomness takes a `RandomState`. `ScopedSessionState` (a solve leaves the session as it found it) still restores only the draw stream and the precision, deliberately not the seed or the ordinal: a seedless solve rekeys to a derived seed and an MPI solve sets rank 0's, and whether a solve should leave those as it found them is an open question, not decided here. +- "The calling thread only" may prove wrong: a program that draws setup randomness on several threads of its own has a state per thread, and a run continued from one of them draws differently on the others. If such a program appears, the state grows to cover it; until then the restriction is documented on `RandomState`. +- A state written by one version of b2 is read only by a version with the same `b2randstate` version; a record carrying one is otherwise asked again, as records are across encoding versions. diff --git a/docs/adr/0078-windows-ci-installs-a-slim-conda-environment-and-caches-none-of-it.md b/docs/adr/0078-windows-ci-installs-a-slim-conda-environment-and-caches-none-of-it.md new file mode 100644 index 000000000..6c4ae6f75 --- /dev/null +++ b/docs/adr/0078-windows-ci-installs-a-slim-conda-environment-and-caches-none-of-it.md @@ -0,0 +1,42 @@ +# ADR-0078: The Windows CI jobs install a slim conda environment and cache none of it + +**Status:** Accepted +**Date:** 2026-11-06 + +## Context + +GitHub keeps 10 GB of Actions caches per repository and evicts the least recently used beyond that. On 2026-11-04 the repository held 8.9 GB in 23 caches, all on develop, and no Linux cache at all. Each Windows job cached its conda package folder under its own key (`windows-conda-cpp-...` and `windows-conda--...`), about 2.6 GB apiece: six of them do not fit, and while they took turns evicting each other, they evicted every other platform's caches and every pull request's. A PR restores only its own caches and develop's, so PRs compiled cold. + +Three things made the folders large and many: + +- **What the environment installs.** `environment-win.yml` listed the `compilers` metapackage, whose Fortran compiler (flang) brings llvm-tools, lld and clang: 830 MB of a 1153 MB download. The builds use only clang-cl from it. conda's git (115 MB) duplicated the runner's, and numpy and scipy took MKL (109 MB) as their BLAS. +- **Unpacked copies.** A conda package folder holds each package twice, the `.conda` archive and its unpacked folder, so 1.15 GB of downloads made a 2.6 GB cache. +- **One cache per Python version, keyed by the workflow file.** The solves for the Python versions share 282 MB of ~320 MB. And `hashFiles(..., '.github/workflows/build_and_test.yml')` in the key made every workflow edit a new set of caches. + +## What was measured + +With the environment slimmed (below), PR #527 cached the package archives only, ~325 MB per job, keyed by the SHA-256 of `conda list --explicit --md5`, and logged for every job how many archives it restored, how many conda reused, and how many it downloaded. Over three runs (2026-11-05): + +| | jobs | conda install (setup-miniconda) | +|---|---|---| +| no cache to restore | 16 | 116-252 s; 130-179 s in the last run | +| cache restored (81 archives, 326.7 MB, in 5 s) | 2 | 211 s and 178 s | + +In the job that restored the cache, conda installed the same 81 packages, and all 81 archives were rewritten after the restore: conda used none of them and fetched every package again. The install time varies by more than two minutes from runner to runner, so runner speed, not the cache, decides it. And the most a cache that conda did reuse could save is the download of ~325 MB from conda-forge's CDN, which costs about what restoring a cache of the same size does (5 s here). + +## Decision + +- **No conda cache on Windows.** The three Windows jobs (C++ tests, wheels, wheel tests) install the environment from conda-forge on every run; `CONDA_PKGS_DIRS` is not set. The ~1.9 GB the per-Python caches would hold stays free for the caches that pay: ccache on every platform, and the Linux and macOS caches. +- **The environment is slim.** `environment-win.yml` asks for `clang >=20` (clang-cl) instead of `compilers`, drops git, and pins numpy's BLAS to OpenBLAS (`libblas=*=*openblas`). The download is ~325 MB instead of 1153 MB. The Clang floor is the one Visual Studio 2026's STL demands (STL1000), so the builds no longer define `_ALLOW_COMPILER_AND_STL_VERSION_MISMATCH`. +- **The link tools are Visual Studio's.** Without conda's lld, CMake took `lld-link`, `llvm-rc` and `llvm-mt` from the runner's own LLVM (`C:\Program Files\LLVM`), which nothing pins. `tools/msvc_link_tools.ps1` enters the developer environment of the newest Visual Studio (found with `vswhere`; the one clang-cl finds by itself, whose STL it compiles against) for `LIB`, `LIBPATH` and `INCLUDE`, which `link.exe` needs and the conda activation does not set in PowerShell, puts PATH back so the build keeps conda's cmake, ninja, ccache and clang-cl, and returns `link.exe` and `lib.exe` from its MSVC toolset and `rc.exe` and `mt.exe` from its Windows SDK. The builds pass them as `CMAKE_LINKER`, `CMAKE_AR`, `CMAKE_RC_COMPILER` and `CMAKE_MT`, by full path, since Git's `link.exe` can come first on PATH. +- **Boost.JSON does not auto-link.** The first run without lld failed with `could not open 'boost_json.lib'`: b2 uses Boost.JSON header-only, but on Windows Boost's auto-linking asked for the library by name, and only the old linker happened to find one in conda's library folder. `bertini2` defines `BOOST_JSON_NO_LIB`. +- **The Windows build steps log what they used and fail where they fail**: the main conda package versions, which cmake, ninja, clang-cl and ccache run, `clang-cl --version`, and the compiler and link tools in `CMakeCache.txt`. A native command's exit code does not stop a PowerShell step, so the first failed link surfaced only as ten tests "Not Run"; the build steps now exit with the build's code. + +## Consequences + +- Do not add a conda cache back to the Windows jobs without measuring that it saves time: log, as PR #527 did, how many restored archives conda reuses and how long the install takes with and without it. A cache that only mirrors a fast download costs cache space and saves nothing. +- If a cache is ever measured to pay, do not cache the whole package folder (~2.6 GB per Python version crowds the other platforms out of the 10 GB) or key it on the workflow file (every edit makes new caches). +- Do not reinstate `compilers` to get clang-cl; ask for `clang`. If the STL raises its Clang floor, raise the pin. +- Do not drop the explicit MSVC tools for "CMake finds a linker anyway": it finds whatever LLVM the runner image ships. Do not pass them as bare names: Git's `link.exe` is not a linker. Do not leave Visual Studio's PATH in place after entering its developer shell: its CMake and Ninja would replace conda's. +- Do not remove `BOOST_JSON_NO_LIB` while Boost.JSON is used header-only. +- The cibuildwheel key of the Linux and macOS wheel jobs still hashes the workflow file; it caches downloads only and is small, and it was not measured here. diff --git a/docs/adr/0079-a-thread-frees-its-own-mpfr-caches.md b/docs/adr/0079-a-thread-frees-its-own-mpfr-caches.md new file mode 100644 index 000000000..65c3b3edc --- /dev/null +++ b/docs/adr/0079-a-thread-frees-its-own-mpfr-caches.md @@ -0,0 +1,38 @@ +# ADR-0079: A thread frees its own MPFR caches + +**Status:** Accepted +**Date:** 2026-11-07 + +## Context + +`test_nag_algorithms` crashed on Windows with heap corruption (0xc0000374) about two seconds in, deterministically, when PR #527 moved the C++ job to clang 22 and `link.exe`. Rerun under cdb, a worker thread faulted reading `0xfeeefeee`, the debug heap's fill for freed memory, inside `mpfr_bernoulli_freecache`; under full page heap the same stack faulted one frame deeper, in libgcc's `_emutls_get_address`: + +``` +libgcc_s_seh_1!_emutls_get_address +libmpfr_6!mpfr_bernoulli_freecache +libmpfr_6!mpfr_free_cache2 +test_nag_algorithms!__dyn_tls_dtor +ntdll!LdrpCallTlsInitializers +ntdll!LdrShutdownThread +``` + +Boost.Multiprecision frees MPFR's per-thread caches from a `thread_local` object (`mpfr_cleanup::thread_initializer`), which every destroyed `real_mp` registers. conda-forge's MPFR for Windows is built by gcc, so its thread-local variables live in libgcc's emulated TLS, whose per-thread memory goes when the DLLs detach from the thread. In an executable, C++ `thread_local` destructors run from the executable's own TLS callback, which the stack above shows running inside thread shutdown, and by then the emulated TLS block has been freed. MPFR reads its cache pointers out of freed memory and frees what they appear to point at. + +The wheels did not crash: there the destructor belongs to the extension DLL, which sits on top of MPFR in the load order, and every Windows wheel's pytest run passed with the same compiler and linker. That reasoning about the order is inferred from the stack and the wheels' results, not measured. + +The read of freed memory does not depend on the compiler, so it was present before #527. What made it fatal there is not established: whether freed memory still looks harmless depends on heap layout, and the C++ job passed with clang 19 and lld-link the day before. + +## Decision + +- On Windows, `bertini2/mpfr_extensions.hpp` explicitly specializes `mpfr_cleanup::force_instantiate` to skip Boost's `thread_local` object and keep its process-wide cleanup. +- A thread b2 starts frees its own caches with `ReleaseThreadMpfrCaches()` as the last thing it does, while its TLS is alive. `WorkerThreadPool` is the only place b2 starts threads, and it does this on every platform, so the behavior has one description. +- A thread b2 did not start, on Windows, keeps its caches until the process ends. They are small and bounded by the thread count; an uncatchable crash is not. + +## Consequences + +- Do not remove the specialization because `ReleaseThreadMpfrCaches` looks sufficient, or the reverse. Without the specialization Boost's destructor still runs after the thread's own cleanup, and without the release the caches leak per solve. +- Do not add a `thread_local` destructor that calls an MPFR function reading MPFR's thread-local state, in b2 or in a test. On Windows that state is already gone when the destructor runs. +- A new place that starts a thread that uses `real_mp` calls `ReleaseThreadMpfrCaches()` last. +- `bertini2/mpfr_extensions.hpp` stays the first b2 header to reach Boost's MPFR backend in any translation unit that instantiates it, or the specialization comes too late. If a Boost release renames `mpfr_cleanup` or `force_instantiate`, the specialization stops compiling, which is the signal to look again. +- This can go if conda-forge's Windows MPFR stops using emulated TLS, or Boost stops freeing the caches from a `thread_local`. +- `thread_pool_workers_that_filled_mpfr_caches_exit_cleanly` fills the Bernoulli and pi caches on pool workers and exits them. On Windows, the C++ job's crash step (`cdb`, then full page heap) is how to read a failure like this one; it runs only when a suite failed. diff --git a/docs/adr/README.md b/docs/adr/README.md index c6812ec69..8a01b9dd1 100644 --- a/docs/adr/README.md +++ b/docs/adr/README.md @@ -92,3 +92,7 @@ Each ADR follows the template: | [0073](0073-a-node-keeps-what-canonicalization-asks-of-it.md) | A NODE KEEPS WHAT CANONICALIZATION ASKS OF IT, and differentiation visits each node once: `Differentiate` is non-virtual with a memo for the outermost call (`DifferentiateImpl` is the rule; a `DerivativeScope` spans a Jacobian); each node makes a degree record once from its operands' -- variables, degree and multidegree per mentioned variable, total degree, and the rules' answer for every unmentioned variable (not always 0: a negative constant power is -1 everywhere; computed with a never-interned sentinel variable) -- and the tie-break is a CONTENT KEY, the SHA-256 of the node's shallow canonical encoding (operands written as their keys), a Merkle hash made once per node (replaces ADR-0059's key, the whole encoding, which walked); published once by atomic compare-and-swap; building a sum or product no longer walks its operands (#505: Barth sextic deflations 121 s to 0.6 s, a 7 x 7 determinant's first evaluation 24.6 s to 0.04 s); tied operands reorder, so `b2sysenc/3` and every digest moves in 4.1.0; a `DegreeImpl` never asks its own node's `Degree`; don't assume 0, don't order by anything that walks, don't make `Differentiate`/`Degree(v)` virtual again | Core / function tree; system identity | | [0074](0074-integration-branches-are-first-class-in-ci.md) | INTEGRATION BRANCHES ARE FIRST-CLASS IN CI: a branch named `integration/` is a line of work that stays off develop until it has earned its merge; the push trigger includes `integration/**` (merges run the full matrix and save the caches its PRs restore, which a PR never gets from another PR) and `set_matrix` gives PRs based on one the full matrix; the rule names a kind of branch, like `release/x.y`, never one branch; a heavy dependency becomes a build option, default off, before it reaches develop; the first is `integration/intervals` (#412); don't name a branch in the triggers, don't drop the glob, don't take such PRs back to the one-platform subset | CI | | [0075](0075-interval-arithmetic-is-flints-arb.md) | INTERVAL ARITHMETIC IS FLINT'S ARB, through a thin wrapper of ours that forwards every operation (`real_iv_mp`/`complex_iv_mp` over `arb_t`/`acb_t`), with a Boost.Interval complex rectangle (`real_iv_dbl`/`complex_iv_dbl`) for polynomial arithmetic only, rounding mode set once per evaluation; rejected: our own type (she ripped out her own complex once Boost had one), MPFI via `complex_adaptor` (non-enclosures across branch cuts, no win-64), C-XSC; the spike measured `acb` at 0.23-0.83 of `complex_mp` and the rectangle at 7-30x `complex_dbl` (55-591x with per-op rounding); FLINT is a dependency: GMP 6.3.0 fat + MPFR + MPC + FLINT from source in the Linux image, stripped, `-flint` in the tag; `info.flint_version` keeps FLINT linked into the extension; don't write enclosures of our own, don't switch rounding per op, don't give the double type transcendentals, don't run Eigen decompositions on intervals, don't drop the FlintVersion reference, don't ship libflint unstripped | Core / numerics; CI; Python bindings | +| [0076](0076-ccache-with-clang-cl-uses-showincludes.md) | WITH CCACHE, CLANG-CL WRITES DEPENDENCY FILES THROUGH /showIncludes: CMake's Windows-Clang module passes gcc-style depfile flags as `-clang:-MD -clang:-MT -clang:-MF`, and ccache 4.14.1+ refuses every `/clang:` option by design, so every Windows compile was uncacheable (0 of 149) and the jobs compiled cold for months while three commits assumed the cache worked without reading its stats; the root CMakeLists sets `CMAKE_DEPFILE_FLAGS_=/showIncludes` and the `msvc` depfile format when ccache drives clang-cl, plus `CCACHE_SLOPPINESS=pch_defines,time_macros` and verbose stats in the jobs; don't revert the override for CMake's preference, don't switch ccache off as a precaution, read `Cacheable calls` (0075 is on `integration/intervals`) | CI; build system | +| [0077](0077-the-random-state-is-one-value.md) | THE RANDOM STATE IS ONE VALUE, and every draw comes from one stream: the `mt19937` engine and `ThreadEngine()` are gone (nothing drew from it after 0044); `RandomState` is the calling thread's global seed, seed-chosen flag, solve-seed ordinal and `b2rand/1` draw stream, public fields, `==`; `GetRandomState`/`SetRandomState`, so a computation written out between stages continues with the draws an uninterrupted run makes (#518); text `b2randstate/1` via `ToText`/`FromText` is the serialization, and Boost archives (MPI) carry that text; Python `RandomState`, `get_random_state`/`set_random_state`, pickles as text; per-path and per-worker streams are not part of it; don't add a second source of randomness, don't save part of the state by hand (`ScopedSessionState` restoring only the stream is a stated open question) | Core / random | +| [0078](0078-windows-ci-installs-a-slim-conda-environment-and-caches-none-of-it.md) | THE WINDOWS CI JOBS INSTALL A SLIM CONDA ENVIRONMENT AND CACHE NONE OF IT: per-job caches of the whole conda package folder (~2.6 GB per Python version) overflowed the repository's 10 GB and evicted every Linux, macOS and PR cache; `environment-win.yml` asks for `clang >=20` (clang-cl) instead of `compilers` (whose flang brought 830 MB of LLVM), drops git, takes OpenBLAS for MKL (~325 MB download); measured over three runs, a restored cache of the archives saved nothing (conda rewrote all 81; installs 178-211 s with it, 116-252 s without, runner speed decides) and could at best replace a download as fast as its own restore, so there is none; the STL-mismatch define is gone; the builds enter Visual Studio's developer environment (`tools/msvc_link_tools.ps1`: LIB/INCLUDE, PATH restored) and link with its `link.exe`/`lib.exe` and the SDK's `rc.exe`/`mt.exe` by full path, not the runner's LLVM; `bertini2` defines `BOOST_JSON_NO_LIB`; the build steps log their tools and fail where the build fails; don't add a conda cache back unmeasured, don't reinstate `compilers` for clang-cl, don't keep Visual Studio's PATH | CI | +| [0079](0079-a-thread-frees-its-own-mpfr-caches.md) | A THREAD FREES ITS OWN MPFR CACHES: Boost.Multiprecision frees them from a `thread_local` destructor; conda-forge's Windows MPFR keeps its thread-local state in libgcc's emulated TLS, and in an executable that destructor ran after the state was freed, so a worker's exit read freed memory and corrupted the heap (`test_nag_algorithms`, 0xc0000374); on Windows `mpfr_extensions.hpp` specializes `mpfr_cleanup::force_instantiate` to skip the `thread_local`, and `WorkerThreadPool`, the only place b2 starts threads, calls `ReleaseThreadMpfrCaches()` last on every platform; don't remove either half, don't read MPFR's thread-local state from a `thread_local` destructor, a new thread that uses `real_mp` releases last, `mpfr_extensions.hpp` stays the way into Boost's MPFR backend | Core / threading; Windows | \ No newline at end of file diff --git a/environment-win.yml b/environment-win.yml index 5d43c5207..d886bf873 100644 --- a/environment-win.yml +++ b/environment-win.yml @@ -6,9 +6,15 @@ dependencies: - cmake - ccache - cxx-compiler - - compilers + # clang provides clang-cl, the compiler the Windows builds use. Not `compilers`, which brought + # clang only as a dependency of its Fortran compiler (flang) and added ~600 MB of LLVM tools + # and flang to every install (ADR-0078). At least 20: the Windows runners' Visual Studio 2026 STL + # refuses an older Clang (STL1000). + - clang >=20 - ninja - numpy + # numpy's and scipy's BLAS: OpenBLAS (a few MB) rather than MKL (109 MB). b2 itself links no BLAS. + - libblas=*=*openblas - gmp - mpfr - mpc @@ -24,7 +30,6 @@ dependencies: - python-build - scikit-build-core - pkg-config - - git - msmpi - mpi4py - pip diff --git a/python/bertini/_repr.py b/python/bertini/_repr.py index 867c23858..dfcaccabd 100644 --- a/python/bertini/_repr.py +++ b/python/bertini/_repr.py @@ -291,6 +291,16 @@ def _decomposition_repr(self): return _described(self, '%dx%d' % (rows, cols)) +# ----- the random state -------------------------------------------------------------- + +def _random_state_repr(self): + if not _ask(lambda: self.seed_chosen, False): + return _described(self, 'no seed chosen') + return _described(self, 'global seed %s, solve ordinal %s' + % (_ask(lambda: self.global_seed, '?'), + _ask(lambda: self.solve_ordinal, '?'))) + + # ----- the rest ---------------------------------------------------------------------- def _midpath_report_repr(self): @@ -459,6 +469,9 @@ def _install_containers(native): def _install_remainder(native): + _attach(getattr(native.random, 'RandomState', None), _random_state_repr) + # a draw stream's position is a key and a counter, nothing a person reads; to_text() has them + _attach(getattr(native.random, 'DrawStream', None), _bare_repr) _attach(getattr(native.nag_algorithms, 'MidpathCheckReport', None), _midpath_report_repr) _attach(getattr(native.nag_algorithms, 'UserStartSystem', None), _user_start_system_repr) _each(_classes(native.nag_algorithms, ('StartSystemFactory', 'AnyZeroDim')), _bare_repr) diff --git a/python/bertini/config.py b/python/bertini/config.py index 5bb518c4f..7b5a90a4d 100644 --- a/python/bertini/config.py +++ b/python/bertini/config.py @@ -228,7 +228,25 @@ def _reconstruct_enhanced(cls, mapping): return cls.from_dict(mapping) -def _make_reduce(): +def _reconstruct_from_json(cls, text): + """Pickle reconstructor for a class that writes itself exactly: rebuild from its JSON text.""" + return cls.from_json(text) + + +def _writes_itself_exactly(cls): + """Whether the class has its own exact text form (``to_json`` / ``from_json``), as a + solution's metadata does (#521). Such a class pickles and copies through it: the + writable-field dict misses its read-only fields (a metadata's singular values and the + point a failed path stopped at), so a pickle through the dict silently dropped them.""" + return callable(getattr(cls, 'to_json', None)) and callable(getattr(cls, 'from_json', None)) + + +def _make_reduce(exact=False): + if exact: + def __reduce__(self): + return (_reconstruct_from_json, (type(self), self.to_json())) + return __reduce__ + def __reduce__(self): # Pickle via the (to_dict / from_dict) round-trip. Works for any enhanced class; the field # values must themselves be picklable (numbers, including multiprec.Complex/Float). @@ -236,7 +254,18 @@ def __reduce__(self): return __reduce__ -def _make_copy(): +def _make_copy(exact=False): + if exact: + def __copy__(self): + return type(self).from_json(self.to_json()) + + def __deepcopy__(self, memo): + new = type(self).from_json(self.to_json()) + memo[id(self)] = new + return new + + return __copy__, __deepcopy__ + # copy.copy / copy.deepcopy rebuild through the same to_dict/from_dict round-trip as pickle. # Defining __deepcopy__ explicitly is load-bearing: the generic copy.deepcopy fallback would # deep-copy the to_dict() mapping's *values* (the multiprecision scalars), which aliases/corrupts @@ -269,8 +298,9 @@ def _enhance_config_class(cls): cls.__eq__ = _make_eq(fields) # Make the class picklable (copy.copy/deepcopy too) via to_dict/from_dict, overriding the # Boost.Python "pickling not enabled" default. - cls.__reduce__ = _make_reduce() - cls.__copy__, cls.__deepcopy__ = _make_copy() + exact = _writes_itself_exactly(cls) + cls.__reduce__ = _make_reduce(exact) + cls.__copy__, cls.__deepcopy__ = _make_copy(exact) except (TypeError, AttributeError): # some bound types may refuse dunder assignment; the rest still apply pass diff --git a/python/bertini/random/__init__.py b/python/bertini/random/__init__.py index 2f4548e1e..4512ed369 100644 --- a/python/bertini/random/__init__.py +++ b/python/bertini/random/__init__.py @@ -1,9 +1,21 @@ +import warnings as _warnings + import numpy as _np from bertini._pybertini import random as _pybrand from bertini._pybertini.random import * +def derive_solve_seed(): + """The former name of :func:`advance_solve_seed`, which read as a derivative; removed in 5.0. + + Returns what :func:`advance_solve_seed` returns, and warns with a ``DeprecationWarning``. + """ + _warnings.warn("derive_solve_seed is advance_solve_seed; the old name is removed in 5.0", + DeprecationWarning, stacklevel=2) + return _pybrand.advance_solve_seed() + + def random_real(symbolic=False): """A random real number of bounded modulus, at the current default precision. @@ -143,4 +155,4 @@ def random_matrix(rows, cols, real=False, units=False, orthonormal=True, symboli return M -__all__ = dir(_pybrand) + ['random_matrix'] +__all__ = dir(_pybrand) + ['random_matrix', 'derive_solve_seed'] diff --git a/python/bertini/records.py b/python/bertini/records.py index 8464d19fa..4860c75f7 100644 --- a/python/bertini/records.py +++ b/python/bertini/records.py @@ -438,7 +438,7 @@ def solve(system, seed=None, directory=None, mptype='adaptive', precision=None, the run id -- a claim ticket, safe to drop. """ from bertini import nag_algorithm as _nag - from bertini.random import set_random_seed as _set_seed, derive_solve_seed as _derive_seed + from bertini.random import set_random_seed as _set_seed, advance_solve_seed as _advance_seed if (homotopy is None) != (start is None): raise ValueError("solve: homotopy= and start= go together (a chained solve " @@ -453,7 +453,7 @@ def solve(system, seed=None, directory=None, mptype='adaptive', precision=None, # history. (A chained solve's homotopy was built before this line -- its # exact coefficients are archived in definitions/ by the run header, so # nothing is lost there either.) - seed = _derive_seed() + seed = _advance_seed() _set_seed(seed) # precision= is an integer number of digits (applied via default_precision); mptype= is the diff --git a/python/docs/source/tutorials/formulating_and_solving/all_solutions/index.rst b/python/docs/source/tutorials/formulating_and_solving/all_solutions/index.rst index 9317d210e..536470b04 100644 --- a/python/docs/source/tutorials/formulating_and_solving/all_solutions/index.rst +++ b/python/docs/source/tutorials/formulating_and_solving/all_solutions/index.rst @@ -111,6 +111,23 @@ is then a one-line filter on the frame, and you have the metadata right alongsid :mod:`pandas` is an optional dependency: ``to_dataframe()`` raises a helpful :class:`ImportError` if it is not installed, and the point accessors above are the no-pandas path. +Keeping what the solver knew about a solution +============================================= + +Each solution comes with its metadata -- its accuracy, condition number, cycle number, multiplicity, whether it is real, finite or singular. A program that keeps solutions between runs can keep their metadata too: ``to_json()`` writes it as exact JSON, every field, and ``from_json`` reads it back equal. + +.. testcode:: + + import pickle + + metadata = solver.solution_metadata()[0] + text = metadata.to_json() + metadata_loaded = type(metadata).from_json(text) + assert metadata_loaded.to_json() == text + assert pickle.loads(pickle.dumps(metadata)).to_json() == text + +Doubles are written as their exact decimals and multiprecision values at their own precision, so nothing is rounded on the way out. A pickle of the metadata goes through the same text. + A note on precision =================== diff --git a/python/docs/source/tutorials/record_keeping/continuing_a_computation/index.rst b/python/docs/source/tutorials/record_keeping/continuing_a_computation/index.rst new file mode 100644 index 000000000..1fe4c54fe --- /dev/null +++ b/python/docs/source/tutorials/record_keeping/continuing_a_computation/index.rst @@ -0,0 +1,87 @@ +🎲 Continuing a computation with the same random draws +******************************************************* + +.. testsetup:: * + + import pickle + import bertini + +A long computation runs in stages, and every stage draws random numbers: the gamma of a homotopy, a patch, the coefficients of a start system, a random projection. If the computation writes itself out between stages, so that it can stop and continue later, it must also write out where its random draws had got to. Otherwise the stages after a restart draw different numbers than an uninterrupted run would have, and the continued run is a different computation. + +A seed is not enough for this: it says where the draws *began*, not where they have reached. The **random state** says where they have reached. + +Getting and setting the random state +==================================== + +:func:`bertini.random.get_random_state` returns the calling thread's random state, and :func:`bertini.random.set_random_state` sets it. The draws after setting a state are the draws that followed getting it: + +.. testcode:: + + bertini.random.set_random_seed(1042) + bertini.random.random_vector(3) # the first stage's draws + + state = bertini.random.get_random_state() + after_the_first_stage = list(bertini.random.random_vector(3)) + + bertini.random.set_random_seed(7) # something else draws in between + bertini.random.random_vector(5) + + bertini.random.set_random_state(state) + assert list(bertini.random.random_vector(3)) == after_the_first_stage + +Solves draw from the same state, so a solve after setting a state builds the same homotopy, and finds the same solutions, as the solve that followed getting it. + +What the state holds +==================== + +A :class:`~bertini.random.RandomState` holds everything the thread's next draws depend on: the global seed, whether one has been chosen, the ordinal of the next solve seed (:func:`~bertini.random.advance_solve_seed`), and the thread's draw stream. Two states are equal when all of these are: + +.. testcode:: + + bertini.random.set_random_seed(5) + bertini.random.advance_solve_seed() + state = bertini.random.get_random_state() + print(state.global_seed, state.seed_chosen, state.solve_ordinal) + assert bertini.random.get_random_state() == state + +.. testoutput:: + + 5 True 1 + +The fields can be set, so a state can be built field by field. Each path of a solve, and each worker of a parallel solve, draws from its own stream, reseeded from the global seed and its index when it starts, so those streams are not part of the state. + +Writing the state out +===================== + +A state is written as versioned plain text, and read back from it, so it can sit beside the rest of a computation's output: + +.. testcode:: + + text = state.to_text() + print(text.splitlines()[0]) + assert bertini.random.RandomState.from_text(text) == state + +.. testoutput:: + + b2randstate/1 + +A state also pickles (as its text), so it can go wherever a pickle goes: + +.. testcode:: + + assert pickle.loads(pickle.dumps(state)) == state + +Text of another version, or text that is not a state, is refused: + +.. testcode:: + + try: + bertini.random.RandomState.from_text("not a random state") + except ValueError as e: + print("refused") + +.. testoutput:: + + refused + +In C++, the same state is ``bertini::RandomState``, with ``GetRandomState()``, ``SetRandomState(state)``, ``ToText()`` and ``RandomState::FromText(text)``; it also goes through Boost.Serialization archives, and so through MPI. diff --git a/python/docs/source/tutorials/record_keeping/index.rst b/python/docs/source/tutorials/record_keeping/index.rst index 229c8c359..2a4c10d20 100644 --- a/python/docs/source/tutorials/record_keeping/index.rst +++ b/python/docs/source/tutorials/record_keeping/index.rst @@ -15,3 +15,4 @@ them. I hope you like it! automatic_record_keeping/index chained_homotopies/index + continuing_a_computation/index diff --git a/python/test/classes/random_state_test.py b/python/test/classes/random_state_test.py new file mode 100644 index 000000000..62245a828 --- /dev/null +++ b/python/test/classes/random_state_test.py @@ -0,0 +1,113 @@ +"""Getting and setting the random state (bertiniteam/b2#518): the draws after set_random_state are +the draws that followed get_random_state, so a computation written out between its stages continues +with the numbers an uninterrupted run would have drawn.""" +import pickle + +import pytest + +import bertini +from bertini.random import RandomState, get_random_state, set_random_state + + +def _draws_from_here(): + """A homotopy's worth of draws of every kind: vectors, a solve seed, a matrix.""" + return (repr(list(bertini.random.random_vector(3))), + bertini.random.advance_solve_seed(), + repr(bertini.random.random_matrix(2, 3, orthonormal=False).tolist()), + repr(bertini.random.random_real())) + + +def test_a_state_set_again_draws_what_followed_getting_it(): + bertini.random.set_random_seed(1042) + _draws_from_here() + state = get_random_state() + after_getting_it = _draws_from_here() + + bertini.random.set_random_seed(77) + _draws_from_here() + set_random_state(state) + assert _draws_from_here() == after_getting_it + assert bertini.random.get_random_seed() == 1042 + + +def test_a_solve_after_setting_a_state_finds_what_the_solve_after_getting_it_found(): + x, y = bertini.variables(['x', 'y']) + system = bertini.System() + system.add_variable_group(x, y) + system.add_functions([x**2 + y**2 - 1, x - y]) + + def solved(): + solver = bertini.nag_algorithm.ZeroDimSolver(system) + solver.solve() + return sorted(repr([complex(c) for c in s]) for s in solver.all_solutions()) + + bertini.random.set_random_seed(3) + _draws_from_here() + state = get_random_state() + after_getting_it = (solved(), _draws_from_here()) + + _draws_from_here() + set_random_state(state) + assert (solved(), _draws_from_here()) == after_getting_it + + +def test_a_state_holds_the_seed_and_ordinal_and_compares_by_its_fields(): + bertini.random.set_random_seed(5) + bertini.random.advance_solve_seed() + state = get_random_state() + assert (state.global_seed, state.seed_chosen, state.solve_ordinal) == (5, True, 1) + assert get_random_state() == state + _draws_from_here() + assert get_random_state() != state + + +def test_a_state_prints_its_seed_and_ordinal(): + bertini.random.set_random_seed(5) + bertini.random.advance_solve_seed() + assert repr(get_random_state()) == '' + assert repr(RandomState()) == '' + + +def test_the_former_name_derive_solve_seed_warns_and_advances_the_same_chain(): + bertini.random.set_random_seed(42) + first = bertini.random.advance_solve_seed() + second = bertini.random.advance_solve_seed() + bertini.random.set_random_seed(42) + with pytest.warns(DeprecationWarning, match='advance_solve_seed'): + assert bertini.random.derive_solve_seed() == first + assert bertini.random.advance_solve_seed() == second + + +def test_a_state_built_field_by_field_draws_as_one_got(): + bertini.random.set_random_seed(11) + _draws_from_here() + got = get_random_state() + after_getting_it = _draws_from_here() + + built = RandomState() + built.global_seed = got.global_seed + built.seed_chosen = got.seed_chosen + built.solve_ordinal = got.solve_ordinal + built.draw_stream = got.draw_stream + bertini.random.set_random_seed(12) + set_random_state(built) + assert _draws_from_here() == after_getting_it + + +def test_a_state_goes_through_text_and_pickle(): + bertini.random.set_random_seed(13) + _draws_from_here() + state = get_random_state() + text = state.to_text() + assert text.startswith('b2randstate/1\nglobal_seed 13\nseed_chosen 1\nsolve_ordinal 1\n') + assert RandomState.from_text(text) == state + assert pickle.loads(pickle.dumps(state)) == state + + +def test_a_text_that_is_not_a_state_is_refused(): + bertini.random.set_random_seed(9) + text = get_random_state().to_text() + for bad in ('', 'global_seed 9', text.replace('b2randstate/1', 'b2randstate/2'), + text.split('draw_stream')[0]): + with pytest.raises(ValueError, match='b2randstate/1'): + RandomState.from_text(bad) diff --git a/python/test/zero_dim/solution_metadata_json_test.py b/python/test/zero_dim/solution_metadata_json_test.py new file mode 100644 index 000000000..42ee57833 --- /dev/null +++ b/python/test/zero_dim/solution_metadata_json_test.py @@ -0,0 +1,64 @@ +"""A solution's metadata as exact JSON, and back (bertiniteam/b2#521): every field reads back +equal, so a program that keeps solutions between runs keeps what b2 knew about them.""" +import copy +import pickle + +import pytest + +import bertini as pb +from bertini import ZeroDimSolver + +FIELDS = ('path_index', 'solution_index', 'precision_changed', 'time_of_first_prec_increase', + 'max_precision_used', 'path_time_seconds', 'num_successful_steps', 'num_failed_steps', + 'wall_clock_limit_seconds', 'pre_endgame_success_code', 'condition_number', + 'newton_residual', 'final_time_used', 'accuracy_estimate_internal_coords', + 'accuracy_estimate_user_coords', 'cycle_num', 'precision_digits', 'accuracy_digits', + 'endgame_success_code', 'function_residual', 'multiplicity', + 'multiplicity_representative', 'is_real', 'is_finite', 'is_singular', + 'crossing_unresolved', 'is_nonsolution') + + +def _metadata(mptype): + pb.random.set_random_seed(4) + x, y = pb.variables(['x', 'y']) + system = pb.System() + system.add_variable_group([x, y]) + system.add_functions([x ** 2 + y ** 2 - 1, x - y]) + solver = ZeroDimSolver(system, mptype=mptype) + solver.solve(show_progress=False) + return list(solver.solution_metadata()) + + +def _same(a, b): + for name in FIELDS: + assert repr(getattr(a, name)) == repr(getattr(b, name)), name + for name in ('singular_values', 'latest_path_point'): + assert repr(list(getattr(a, name))) == repr(list(getattr(b, name))), name + + +@pytest.mark.parametrize('mptype', ['double', 'adaptive']) +def test_a_solves_metadata_reads_back_equal_in_every_field(mptype): + for m in _metadata(mptype): + text = m.to_json() + metadata_loaded = type(m).from_json(text) + _same(metadata_loaded, m) + assert metadata_loaded.to_json() == text + + +@pytest.mark.parametrize('mptype', ['double', 'adaptive']) +def test_metadata_pickles_and_copies_whole(mptype): + """Every field, the read-only ones too: the singular values and the point a path stopped at + were dropped when pickling went through the writable fields.""" + metadata = _metadata(mptype) + assert any(len(m.singular_values) for m in metadata) + for m in metadata: + for metadata_loaded in (pickle.loads(pickle.dumps(m)), copy.copy(m), copy.deepcopy(m)): + _same(metadata_loaded, m) + + +def test_text_that_is_not_such_metadata_is_refused(): + m = _metadata('double')[0] + with pytest.raises(ValueError, match='b2meta/1'): + type(m).from_json('{"format": "b2meta/2"}') + with pytest.raises(ValueError): + type(m).from_json('not json') diff --git a/python_bindings/include/zero_dim_export.hpp b/python_bindings/include/zero_dim_export.hpp index 195091992..95744cf7d 100644 --- a/python_bindings/include/zero_dim_export.hpp +++ b/python_bindings/include/zero_dim_export.hpp @@ -36,6 +36,7 @@ #include #include +#include #include #include #include // blackbox::type::Start (start-system selector enum) @@ -206,6 +207,17 @@ void ExposeSolutionMetaData(std::string const& class_name){ "was NOT resolved by re-tracking. Its success codes may still read Success, but the solver " "has warned that this endpoint may be wrong; check this before building on the endpoint. " "See endgame_boundary_metadata() for the whole-solve report.") + .def("to_json", +[](MDT const& m){ return records::SolutionMetaDataToJson(m); }, + "The metadata as exact JSON text, format b2meta/1: every field, doubles as exact decimals, " + "multiprecision values at their own precision, success codes by name. from_json reads it " + "back equal in every field.") + .def("from_json", +[](std::string const& text){ return records::SolutionMetaDataFromJson(text); }, + (arg("text")), + "The metadata a to_json text describes. Raises ValueError if the text is not b2meta/1 " + "metadata of this numeric type.") + .staticmethod("from_json") + // pickling and copying go through to_json / from_json: bertini/config.py's enhancement of the + // metadata classes uses them, since they hold every field (the writable ones do not) ; } diff --git a/python_bindings/src/random_export.cpp b/python_bindings/src/random_export.cpp index d1060c729..aae8a8a4d 100644 --- a/python_bindings/src/random_export.cpp +++ b/python_bindings/src/random_export.cpp @@ -7,6 +7,24 @@ namespace bertini{ namespace python{ + // a RandomState pickles as its versioned text, so an unpickle of another version is refused + struct RandomStateTextPickle : boost::python::pickle_suite + { + static boost::python::tuple getinitargs(bertini::RandomState const&) + { + return boost::python::make_tuple(); + } + + static boost::python::object getstate(bertini::RandomState const& state) + { + return boost::python::str(state.ToText()); + } + + static void setstate(bertini::RandomState& state, boost::python::object text) + { + state = bertini::RandomState::FromText(boost::python::extract(text)()); + } + }; void ExportRandom(){ @@ -92,12 +110,54 @@ void ExportRandom(){ def("get_random_seed", &bertini::GetGlobalSeed, "Return the global RNG seed in effect. If none has been set, draws one from entropy " "on the first call and keeps it."); - def("derive_solve_seed", &bertini::DeriveSolveSeed, - "Capture the session stream's current position as an effective per-solve seed: " - "one draw off the current stream (advancing it), nonzero, 32-bit portable. " - "bertini.solve uses this when no explicit seed is given, so a recorded run's " - "seed reproduces that run standalone -- deterministic from the session master " - "when one was set, random for a never-seeded session."); + def("advance_solve_seed", &bertini::AdvanceSolveSeed, + "Advance the solve-seed chain one step and return its new seed: the seed for the next " + "seedless solve, nonzero and 32-bit portable. The chain is rooted at the global seed " + "(set_random_seed restarts it), and each call takes the next link, so the call after " + "returns the next seed; it draws no random numbers. bertini.solve uses this when no " + "explicit seed is given, so a recorded run's seed reproduces that run standalone."); + using bertini::records::DrawStream; + class_("DrawStream", + "A b2rand/1 draw stream at a position: the source of every random draw. Compared with ==; " + "written and read as one line of text.", + init<>()) + .def("to_text", &DrawStream::ToText, "The stream as one line of text, which from_text reads back.") + .def("from_text", &DrawStream::FromText, arg("text"), + "The stream a to_text line describes. Raises ValueError if the text is not such a line.") + .staticmethod("from_text") + .def(self == self) + .def(self != self); + + class_("RandomState", + "The random state of a thread: everything its next draws depend on. get_random_state() " + "returns the calling thread's, and set_random_state(state) returns the thread to one, so " + "the draws after setting a state are the draws that followed getting it -- a computation " + "written out between its stages continues with the numbers an uninterrupted run would " + "have drawn. Per-path and per-worker streams are reseeded from the seed at each use, so " + "they are not part of it. Pickles as its to_text() text.", + init<>()) + .def_readwrite("global_seed", &bertini::RandomState::global_seed, "The global seed.") + .def_readwrite("seed_chosen", &bertini::RandomState::seed_chosen, + "Whether a global seed has been chosen yet.") + .def_readwrite("solve_ordinal", &bertini::RandomState::solve_ordinal, + "The ordinal the next advance_solve_seed() takes.") + .def_readwrite("draw_stream", &bertini::RandomState::draw_stream, + "The thread's draw stream, the source of every draw.") + .def("to_text", &bertini::RandomState::ToText, + "The state as versioned plaintext, lines beginning b2randstate/1, which from_text reads back.") + .def("from_text", &bertini::RandomState::FromText, arg("text"), + "The state a to_text text describes. Raises ValueError if the text is not such a text, " + "of this version.") + .staticmethod("from_text") + .def(self == self) + .def(self != self) + .def_pickle(RandomStateTextPickle()); + + def("get_random_state", &bertini::GetRandomState, + "Return the calling thread's random state, a RandomState."); + def("set_random_state", &bertini::SetRandomState, arg("state"), + "Set the calling thread's random state, so its next draws are the ones that followed " + "getting state."); } diff --git a/tools/msvc_link_tools.ps1 b/tools/msvc_link_tools.ps1 new file mode 100644 index 000000000..ad7ee3a3d --- /dev/null +++ b/tools/msvc_link_tools.ps1 @@ -0,0 +1,44 @@ +# Enters the developer environment of the Visual Studio whose STL clang-cl compiles against, for the +# Windows CI builds (ADR-0078), and returns its link tools: a hashtable of CMake variable -> full path +# for link.exe and lib.exe (its MSVC toolset) and rc.exe and mt.exe (its Windows SDK). +# +# clang-cl finds Visual Studio by itself, and lld-link finds the Windows libraries by itself, but +# link.exe needs the developer environment (LIB above all), and the conda activation does not run +# Visual Studio's setup in PowerShell. Left to itself, CMake takes the runner's own LLVM tools from +# C:\Program Files\LLVM, which nothing pins. Run it with `&` in the step that configures: the +# environment it sets lasts for that step's process. Full paths, because Git's link.exe, which is not +# a linker, can come first on PATH. + +$ErrorActionPreference = 'Stop' + +$vswhere = Join-Path ${env:ProgramFiles(x86)} 'Microsoft Visual Studio\Installer\vswhere.exe' +$vsPath = & $vswhere -latest -products * -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 -property installationPath +if (-not $vsPath) { throw "vswhere found no Visual Studio with the MSVC x64 tools" } +Write-Host "Visual Studio: $vsPath" + +# The developer shell also puts Visual Studio's own CMake, Ninja and tools first on PATH; the build +# must keep conda's cmake, ninja, ccache and clang-cl, so PATH goes back as it was. What stays is +# LIB, LIBPATH and INCLUDE (the libraries and headers of one MSVC toolset and one SDK) and the +# variables naming them; the tools below are passed to CMake by full path. +$pathBefore = $env:PATH +& (Join-Path $vsPath 'Common7\Tools\Launch-VsDevShell.ps1') -Arch amd64 -HostArch amd64 -SkipAutomaticLocation | Out-Host +$env:PATH = $pathBefore +foreach ($name in 'VCToolsInstallDir', 'WindowsSdkVerBinPath', 'LIB') { + if (-not (Get-Item "Env:$name" -ErrorAction SilentlyContinue)) { throw "the developer shell did not set $name" } +} +Write-Host "LIB = $env:LIB" +Write-Host "INCLUDE = $env:INCLUDE" + +$msvcBin = Join-Path $env:VCToolsInstallDir 'bin\Hostx64\x64' +$sdkBin = Join-Path $env:WindowsSdkVerBinPath 'x64' +$tools = [ordered]@{ + CMAKE_LINKER = Join-Path $msvcBin 'link.exe' + CMAKE_AR = Join-Path $msvcBin 'lib.exe' + CMAKE_RC_COMPILER = Join-Path $sdkBin 'rc.exe' + CMAKE_MT = Join-Path $sdkBin 'mt.exe' +} +foreach ($entry in $tools.GetEnumerator()) { + if (-not (Test-Path $entry.Value)) { throw "$($entry.Key): not found at $($entry.Value)" } + Write-Host "$($entry.Key) = $($entry.Value)" +} +$tools