diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2194eed..fb9c998 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -30,7 +30,7 @@ jobs: run: cmake -S . -B build-bench -DLOGIT_BENCH_ENABLE=ON -DLOGIT_BENCH_WITH_SPDLOG=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_CXX_STANDARD=${{ matrix.std }} -DLOGIT_WITH_SYSLOG=ON -DLOGIT_WITH_WIN_EVENT_LOG=OFF - name: Build benchmarks # if: ${{ github.event_name == 'pull_request' || (github.event_name == 'push' && github.ref == 'refs/heads/stable') }} - run: cmake --build build-bench --target logit_bench logit_bench_flush_test logit_public_macro_bench logit_public_macro_formatted_bench logit_hotpath_bench logit_hotpath_bench_legacy logit_exec_mx_bench logit_exec_mx_bench_concurrent benchmark_validation_test + run: cmake --build build-bench --target logit_bench logit_bench_async_contract logit_bench_async_payload_contract_test logit_bench_flush_test logit_public_macro_bench logit_public_macro_formatted_bench logit_hotpath_bench logit_hotpath_bench_legacy logit_exec_mx_bench logit_exec_mx_bench_concurrent benchmark_validation_test - name: Run spdlog async flush regression run: ./build-bench/logit_bench_flush_test - name: Run public macro benchmark smoke @@ -47,6 +47,8 @@ jobs: run: ./build-bench/logit_public_macro_formatted_bench - name: Run benchmark validation tests run: ./build-bench/benchmark_validation_test + - name: Run async payload contract regression + run: ./build-bench/logit_bench_async_payload_contract_test - name: Run logger hot-path A/B smoke env: LOGIT_HOTPATH_BENCH_TOTAL: 20000 @@ -67,6 +69,17 @@ jobs: LOGIT_BENCH_TOTAL: 20000 LOGIT_BENCH_WARMUP: 2000 run: ./build-bench/logit_bench + - name: Run contract-matched async benchmark smoke + env: + LOGIT_BENCH_FILTER_ASYNC: "1" + LOGIT_BENCH_FILTER_SINK: "null" + LOGIT_BENCH_FILTER_PRODUCERS: "4" + LOGIT_BENCH_FILTER_BYTES: "200" + LOGIT_BENCH_TOTAL: 2000 + LOGIT_BENCH_WARMUP: 200 + LOGIT_BENCH_TIMEOUT_SEC: 120 + LOGIT_BENCH_OUTPUT: bench/results/latency-async-contract.csv + run: ./build-bench/logit_bench_async_contract - name: Configure consumer project run: cmake -S tests/install_consumer -B build-consumer -DCMAKE_PREFIX_PATH=${{ github.workspace }}/install -DCMAKE_CXX_STANDARD=${{ matrix.std }} - name: Build consumer project diff --git a/README-RU.md b/README-RU.md index 0be0c50..7be0e39 100644 --- a/README-RU.md +++ b/README-RU.md @@ -939,6 +939,14 @@ LogIt++ включает библиотеку *fmt* для форматиров Запустите `./build/bench/logit_bench`, чтобы получить полный набор измерений (sync/async × null/file × количество продюсеров × размер сообщений). Результаты дописываются в `bench/results/latency.csv` по одной строке на каждую библиотеку/комбинацию. При необходимости сократите нагрузку с помощью переменных окружения `LOGIT_BENCH_TOTAL` и `LOGIT_BENCH_WARMUP`. +Для сравнения async/null с одинаковой стоимостью передачи payload со spdlog +соберите и запустите `logit_bench_async_contract`. В этом режиме полное +сообщение проходит через очередь LogIt++ и результат записывается в отдельный +CSV `bench/results/latency-async-contract.csv` (или в путь из +`LOGIT_BENCH_OUTPUT`). Не смешивайте эти строки с обычным async/null режимом, +где LogIt++ передаёт только marker. Сам target запускает только сценарий +`async=1`, `sink=null`. + ### Что на самом деле измеряет бенчмарк Полная методика, ограничения сравнения и правила интерпретации находятся в diff --git a/README.md b/README.md index 8e30e5a..081ce94 100644 --- a/README.md +++ b/README.md @@ -1167,6 +1167,13 @@ Run `./build/bench/logit_bench` to record the full matrix (sync/async × null/fi are appended to `bench/results/latency.csv` with one row per library/combination. Override the workload via `LOGIT_BENCH_TOTAL` and `LOGIT_BENCH_WARMUP` environment variables if you need a lighter run. +For a payload-matched async/null comparison with spdlog, build and run +`logit_bench_async_contract`; it carries the full message through the LogIt++ +worker queue and writes by default to `bench/results/latency-async-contract.csv` +(or to the path selected by `LOGIT_BENCH_OUTPUT`). Do not combine those rows +with the default marker-only async/null workload. The target itself runs only +the `async=1`, `sink=null` scenario. + ### What this benchmark measures See the canonical [benchmark guide](docs/benchmarks.md) for the measurement diff --git a/bench/BenchmarkMetadata.hpp b/bench/BenchmarkMetadata.hpp index 494520c..4adb22f 100644 --- a/bench/BenchmarkMetadata.hpp +++ b/bench/BenchmarkMetadata.hpp @@ -27,6 +27,7 @@ struct BenchmarkMetadata { std::string queue_policy; std::string latency_completion; std::string flush_barrier; + std::string workload_contract; }; inline std::string benchmark_env(const char* name, const char* fallback) { @@ -138,7 +139,8 @@ inline BenchmarkMetadata make_benchmark_metadata( std::string queue_capacity, std::string queue_policy, std::string latency_completion, - std::string flush_barrier) { + std::string flush_barrier, + std::string workload_contract) { const std::string compiler = benchmark_compiler(); const std::string compiler_version = benchmark_compiler_version(); std::string toolchain = compiler + "-" + compiler_version; @@ -157,7 +159,8 @@ inline BenchmarkMetadata make_benchmark_metadata( std::move(queue_capacity), std::move(queue_policy), std::move(latency_completion), - std::move(flush_barrier)}; + std::move(flush_barrier), + std::move(workload_contract)}; } inline bool benchmark_value_unknown(const std::string& value) { @@ -187,7 +190,8 @@ inline void validate_comparable_metadata(const BenchmarkMetadata& metadata, {"queue_capacity", &metadata.queue_capacity, true}, {"queue_policy", &metadata.queue_policy, true}, {"latency_completion", &metadata.latency_completion, false}, - {"flush_barrier", &metadata.flush_barrier, false}}; + {"flush_barrier", &metadata.flush_barrier, false}, + {"workload_contract", &metadata.workload_contract, false}}; for (const auto& value : values) { if (benchmark_value_unknown(*value.value) || @@ -210,7 +214,7 @@ inline void print_benchmark_metadata(std::ostream& out, const BenchmarkMetadata& metadata, std::size_t total_messages, std::size_t warmup_messages) { - out << "benchmark-fixture version=1" + out << "benchmark-fixture version=2" << " source_commit=" << metadata.source_commit << " compiler=" << metadata.compiler << " compiler_version=" << metadata.compiler_version @@ -225,6 +229,7 @@ inline void print_benchmark_metadata(std::ostream& out, << " queue_policy=" << metadata.queue_policy << " latency_completion=" << metadata.latency_completion << " flush_barrier=" << metadata.flush_barrier + << " workload_contract=" << metadata.workload_contract << " total=" << total_messages << " warmup=" << warmup_messages << '\n'; } diff --git a/bench/BenchmarkValidation.hpp b/bench/BenchmarkValidation.hpp index 5fd739a..e247b2b 100644 --- a/bench/BenchmarkValidation.hpp +++ b/bench/BenchmarkValidation.hpp @@ -15,7 +15,7 @@ inline void validate_queue_capacity(std::size_t capacity) { inline const char* latency_csv_header() { return "lib,async,sink,producers,msg_bytes,total,queue_capacity," - "p50_ns,p99_ns,p999_ns,throughput"; + "workload_contract,p50_ns,p99_ns,p999_ns,throughput"; } inline void validate_latency_csv_header(std::string header) { @@ -24,7 +24,7 @@ inline void validate_latency_csv_header(std::string header) { } if (header != latency_csv_header()) { throw std::runtime_error( - "Unsupported bench/results/latency.csv schema; rename or remove " + "Unsupported benchmark CSV schema; rename or remove " "the existing file before running this benchmark"); } } diff --git a/bench/CMakeLists.txt b/bench/CMakeLists.txt index 8863655..c7f697c 100644 --- a/bench/CMakeLists.txt +++ b/bench/CMakeLists.txt @@ -26,6 +26,23 @@ endforeach() target_link_libraries(logit_bench PRIVATE log-it-cpp::log-it-cpp) +add_executable(logit_bench_async_contract ${LOGIT_BENCH_SOURCES}) +target_include_directories(logit_bench_async_contract PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_definitions(logit_bench_async_contract PRIVATE + LOGIT_BENCH_BUILD_TYPE="$" + LOGIT_BENCH_CONTRACT_MATCHED_ASYNC=1 +) +target_compile_features(logit_bench_async_contract PRIVATE cxx_std_17) +set_target_properties(logit_bench_async_contract PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) +foreach(config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set_target_properties(logit_bench_async_contract PROPERTIES + RUNTIME_OUTPUT_DIRECTORY_${config} ${CMAKE_BINARY_DIR} + ) +endforeach() +target_link_libraries(logit_bench_async_contract PRIVATE log-it-cpp::log-it-cpp) + add_executable(logit_public_macro_bench public_macro_bench.cpp) target_compile_features(logit_public_macro_bench PRIVATE cxx_std_17) target_compile_definitions(logit_public_macro_bench PRIVATE LOGIT_BENCH_BUILD_TYPE=\"$\") @@ -72,8 +89,27 @@ target_compile_features(benchmark_validation_test PRIVATE cxx_std_17) set_target_properties(benchmark_validation_test PROPERTIES RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}) add_test(NAME benchmark_validation_test COMMAND benchmark_validation_test) +add_executable(logit_bench_async_payload_contract_test + async_payload_contract_test.cpp + adapters/LogItAdapter.cpp +) +target_include_directories(logit_bench_async_payload_contract_test PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}) +target_compile_features(logit_bench_async_payload_contract_test PRIVATE cxx_std_17) +target_link_libraries(logit_bench_async_payload_contract_test PRIVATE log-it-cpp::log-it-cpp) +set_target_properties(logit_bench_async_payload_contract_test PROPERTIES + RUNTIME_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR} +) +foreach(config IN ITEMS DEBUG RELEASE RELWITHDEBINFO MINSIZEREL) + set_target_properties(logit_bench_async_payload_contract_test PROPERTIES + RUNTIME_OUTPUT_DIRECTORY_${config} ${CMAKE_BINARY_DIR} + ) +endforeach() +add_test(NAME logit_bench_async_payload_contract_test + COMMAND logit_bench_async_payload_contract_test) + if(LOGIT_BENCH_WITH_SPDLOG) target_compile_definitions(logit_bench PRIVATE LOGIT_BENCH_HAVE_SPDLOG=1) + target_compile_definitions(logit_bench_async_contract PRIVATE LOGIT_BENCH_HAVE_SPDLOG=1) if(NOT TARGET spdlog::spdlog) include(FetchContent) FetchContent_Declare(spdlog @@ -83,6 +119,7 @@ if(LOGIT_BENCH_WITH_SPDLOG) FetchContent_MakeAvailable(spdlog) endif() target_link_libraries(logit_bench PRIVATE spdlog::spdlog) + target_link_libraries(logit_bench_async_contract PRIVATE spdlog::spdlog) add_executable(logit_bench_flush_test spdlog_flush_test.cpp diff --git a/bench/Scenario.hpp b/bench/Scenario.hpp index 05dd1bd..9811979 100644 --- a/bench/Scenario.hpp +++ b/bench/Scenario.hpp @@ -1,7 +1,9 @@ #pragma once #include +#include #include +#include namespace logit_bench { @@ -10,6 +12,11 @@ enum class SinkKind { File, }; +enum class AsyncPayloadMode { + MarkerOnly, + FullMessage, +}; + inline std::string sink_name(SinkKind sink) { switch (sink) { case SinkKind::Null: return "null"; @@ -21,6 +28,8 @@ inline std::string sink_name(SinkKind sink) { struct Scenario { bool async = false; SinkKind sink = SinkKind::Null; + AsyncPayloadMode async_payload = AsyncPayloadMode::MarkerOnly; + std::function async_payload_observer; std::size_t producers = 1; std::size_t message_bytes = 0; std::size_t total_messages = 0; diff --git a/bench/adapters/LogItAdapter.cpp b/bench/adapters/LogItAdapter.cpp index e7bb037..ee7d06d 100644 --- a/bench/adapters/LogItAdapter.cpp +++ b/bench/adapters/LogItAdapter.cpp @@ -38,6 +38,8 @@ namespace logit_bench { void configure(const Scenario& scenario, LatencyRecorder& recorder) { m_async = scenario.async; m_sink = scenario.sink; + m_async_payload = scenario.async_payload; + m_async_payload_observer = scenario.async_payload_observer; m_recorder = &recorder; if (m_sink == SinkKind::File) { @@ -60,6 +62,17 @@ namespace logit_bench { } if (m_sink == SinkKind::Null) { + if (m_async_payload == AsyncPayloadMode::FullMessage) { + AsyncPayload payload; + payload.slot_line = slot_line; + payload.text = message; + logit::detail::TaskExecutor::get_instance().add_task( + [this, payload = std::move(payload)]() mutable { + consume(payload.slot_line, payload.text); + }); + return; + } + logit::detail::TaskExecutor::get_instance().add_task([this, slot_line]() { consume(slot_line, std::string_view{}); }); @@ -121,6 +134,10 @@ namespace logit_bench { if (slot_line >= 0 && m_recorder) { m_recorder->complete_slot(static_cast(slot_line)); } + + if (m_async_payload_observer) { + m_async_payload_observer(text); + } if (m_sink == SinkKind::File) { std::lock_guard lock(m_file_mutex); @@ -137,6 +154,8 @@ namespace logit_bench { bool m_async = false; SinkKind m_sink = SinkKind::Null; + AsyncPayloadMode m_async_payload = AsyncPayloadMode::MarkerOnly; + std::function m_async_payload_observer; LatencyRecorder* m_recorder = nullptr; std::ofstream m_file; diff --git a/bench/async_payload_contract_test.cpp b/bench/async_payload_contract_test.cpp new file mode 100644 index 0000000..61c48cd --- /dev/null +++ b/bench/async_payload_contract_test.cpp @@ -0,0 +1,33 @@ +#include "adapters/LogItAdapter.hpp" + +#include +#include +#include +#include + +int main() { + using namespace logit_bench; + + const std::string expected(200, 'P'); + std::mutex observer_mutex; + std::string observed; + + Scenario scenario; + scenario.async = true; + scenario.sink = SinkKind::Null; + scenario.async_payload = AsyncPayloadMode::FullMessage; + scenario.async_payload_observer = [&](std::string_view payload) { + std::lock_guard lock(observer_mutex); + observed.assign(payload.data(), payload.size()); + }; + + LatencyRecorder recorder(1); + LogItAdapter adapter; + adapter.prepare(scenario, recorder); + adapter.log(recorder.begin(true), expected); + adapter.flush(); + recorder.wait_for_all(); + + std::lock_guard lock(observer_mutex); + return observed == expected ? 0 : 1; +} diff --git a/bench/benchmark_validation_test.cpp b/bench/benchmark_validation_test.cpp index e84e1c4..4cd03e0 100644 --- a/bench/benchmark_validation_test.cpp +++ b/bench/benchmark_validation_test.cpp @@ -25,11 +25,22 @@ int main() { } if (!rejected_legacy_schema) return 2; + bool rejected_pre_contract_schema = false; + try { + validate_latency_csv_header( + "lib,async,sink,producers,msg_bytes,total,queue_capacity," + "p50_ns,p99_ns,p999_ns,throughput"); + } catch (const std::runtime_error&) { + rejected_pre_contract_schema = true; + } + if (!rejected_pre_contract_schema) return 3; + validate_latency_csv_header(std::string(latency_csv_header()) + "\r"); validate_latency_csv_header(latency_csv_header()); const auto comparable = make_benchmark_metadata( - "8192", "block", "sink-entry", "all-prior-work-drained"); + "8192", "block", "sink-entry", "all-prior-work-drained", + "test/workload"); auto complete = comparable; complete.source_commit = "test-commit"; complete.compiler = "test-compiler"; @@ -51,7 +62,7 @@ int main() { } catch (const std::runtime_error&) { rejected_unknown_metadata = true; } - if (!rejected_unknown_metadata) return 3; + if (!rejected_unknown_metadata) return 4; return 0; } diff --git a/bench/logit_bench.cpp b/bench/logit_bench.cpp index fdd820f..ad0283d 100644 --- a/bench/logit_bench.cpp +++ b/bench/logit_bench.cpp @@ -1,6 +1,7 @@ #include #include #include +#include #include #include #include @@ -35,6 +36,16 @@ namespace { std::atomic* g_watchdog_progress = nullptr; constexpr std::size_t k_watchdog_stride = 256; +#if defined(LOGIT_BENCH_CONTRACT_MATCHED_ASYNC) +constexpr AsyncPayloadMode kAsyncPayloadMode = AsyncPayloadMode::FullMessage; +constexpr const char* kWorkloadContract = + "prepared-message/async-full-message"; +#else +constexpr AsyncPayloadMode kAsyncPayloadMode = AsyncPayloadMode::MarkerOnly; +constexpr const char* kWorkloadContract = + "prepared-message/direct-dispatch"; +#endif + std::string make_message(std::size_t bytes, std::size_t index) { if (bytes == 0) return {}; const char fill = static_cast('A' + static_cast(index % 26)); @@ -52,6 +63,17 @@ std::size_t get_env_size_t(const char* name, std::size_t def) { return def; } +std::filesystem::path benchmark_output_path() { + if (const char* value = std::getenv("LOGIT_BENCH_OUTPUT")) { + if (*value != '\0') return value; + } +#if defined(LOGIT_BENCH_CONTRACT_MATCHED_ASYNC) + return "bench/results/latency-async-contract.csv"; +#else + return "bench/results/latency.csv"; +#endif +} + struct BenchFilter { std::optional library; std::optional async; @@ -300,15 +322,18 @@ ScenarioResult execute_scenario( } void append_csv( + const std::filesystem::path& csv_path, const std::string& library, const Scenario& scenario, + const BenchmarkMetadata& metadata, const LatencyRecorder::Summary& summary, double throughput) { namespace fs = std::filesystem; - const fs::path csv_path{"bench/results/latency.csv"}; const std::string expected_header = latency_csv_header(); - fs::create_directories(csv_path.parent_path()); + if (!csv_path.parent_path().empty()) { + fs::create_directories(csv_path.parent_path()); + } const bool write_header = !fs::exists(csv_path) || fs::file_size(csv_path) == 0; @@ -316,13 +341,17 @@ void append_csv( std::ifstream in(csv_path); std::string header; if (!in || !std::getline(in, header)) { - throw std::runtime_error("Failed to read latency.csv schema header"); + throw std::runtime_error( + "Failed to read benchmark CSV schema header"); } validate_latency_csv_header(header); } std::ofstream out(csv_path, std::ios::app); - if (!out) throw std::runtime_error("Failed to open latency.csv for writing"); + if (!out) { + throw std::runtime_error( + "Failed to open benchmark CSV for writing"); + } if (write_header) { out << expected_header << '\n'; @@ -334,6 +363,7 @@ void append_csv( << scenario.message_bytes << ',' << scenario.total_messages << ',' << scenario.queue_capacity << ',' + << metadata.workload_contract << ',' << summary.p50_ns << ',' << summary.p99_ns << ',' << summary.p999_ns << ',' @@ -379,8 +409,13 @@ int main() { #endif // Matrix +#if defined(LOGIT_BENCH_CONTRACT_MATCHED_ASYNC) + const std::array async_modes{true}; + const std::array sinks{SinkKind::Null}; +#else const std::array async_modes{false, true}; const std::array sinks{SinkKind::Null, SinkKind::File}; +#endif const std::array producer_counts{1, 4, 16, 32}; const std::array message_sizes{40, 200, 1024}; @@ -392,6 +427,7 @@ int main() { "LOGIT_BENCH_QUEUE_CAPACITY", std::max(8192, total_messages * 2)); validate_queue_capacity(queue_capacity); + const auto csv_path = benchmark_output_path(); const BenchFilter filter = load_filter(); @@ -399,7 +435,8 @@ int main() { std::to_string(queue_capacity), "block", "sink-entry", - "all-prior-work-drained"); + "all-prior-work-drained", + kWorkloadContract); validate_comparable_metadata(metadata); print_benchmark_metadata(std::cout, metadata, total_messages, warmup_messages); @@ -435,6 +472,7 @@ int main() { Scenario scenario; scenario.async = async_mode; scenario.sink = sink; + scenario.async_payload = kAsyncPayloadMode; scenario.producers = producers; scenario.message_bytes = msg_bytes; scenario.total_messages = total_messages; @@ -457,7 +495,8 @@ int main() { } auto result = execute_scenario(*adapter, scenario, warmup_messages); - append_csv(adapter->library_name(), scenario, result.summary, result.throughput); + append_csv(csv_path, adapter->library_name(), scenario, metadata, + result.summary, result.throughput); print_summary(adapter->library_name(), scenario, result); } } diff --git a/bench/public_macro_bench.cpp b/bench/public_macro_bench.cpp index 73a3336..fc1f692 100644 --- a/bench/public_macro_bench.cpp +++ b/bench/public_macro_bench.cpp @@ -121,7 +121,8 @@ int main() { "not-applicable", "not-applicable", "backend-count", - "logger-wait"); + "logger-wait", + std::string("public-macro/") + benchmark_mode()); logit_bench::validate_comparable_metadata(metadata); logit_bench::print_benchmark_metadata(std::cout, metadata, total, warmup); diff --git a/bench/results/benchmark-fixture-v2.json b/bench/results/benchmark-fixture-v2.json new file mode 100644 index 0000000..279ec14 --- /dev/null +++ b/bench/results/benchmark-fixture-v2.json @@ -0,0 +1,61 @@ +{ + "fixture_version": 2, + "name": "log-it-cpp-release-benchmark", + "metadata_required": [ + "source_commit", + "compiler", + "compiler_version", + "toolchain", + "cxx_standard", + "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", + "queue_capacity", + "queue_policy", + "latency_completion", + "flush_barrier", + "workload_contract" + ], + "metadata_must_match": [ + "compiler", + "compiler_version", + "toolchain", + "cxx_standard", + "platform", + "build_type", + "architecture", + "machine_id", + "cpu_model", + "queue_capacity", + "queue_policy", + "latency_completion", + "flush_barrier", + "workload_contract" + ], + "latency_workload": { + "total_messages": 200000, + "warmup_messages": 4096, + "producer_counts": [1, 4, 16, 32], + "message_bytes": [40, 200, 1024], + "async_modes": [false, true], + "sinks": ["null", "file"], + "queue_policy": "block", + "latency_completion": "sink-entry", + "flush_barrier": "all-prior-work-drained" + }, + "public_macro_workload": { + "total_messages": 20000, + "producer_counts": [1, 4, 16, 32], + "queue_capacity": "not-applicable", + "queue_policy": "not-applicable", + "latency_completion": "backend-count", + "flush_barrier": "logger-wait", + "scenarios": [ + "public macro record construction and dispatch with passthrough formatter", + "public macro record construction, dispatch, and SimpleLogFormatter formatting" + ] + }, + "measurement_rule": "All required metadata fields must be present. Compare runs only when metadata_must_match fields match; source_commit is provenance and may differ between before/after runs. Set LOGIT_BENCH_REQUIRE_COMPARABLE=1 for publication runs so unknown metadata is rejected; CI runs are regression smoke, not universal performance claims." +} diff --git a/docs/adr/0005-benchmark-methodology.md b/docs/adr/0005-benchmark-methodology.md index ffa343e..98066d8 100644 --- a/docs/adr/0005-benchmark-methodology.md +++ b/docs/adr/0005-benchmark-methodology.md @@ -30,6 +30,12 @@ measurements, while the fields listed in `metadata_must_match` must match. The values; it does not enforce the canonical fixture workload unless a separate workload-validation mode is requested. +Asynchronous payload handling is part of the workload contract. The default +prepared-message benchmark keeps the LogIt++ null-sink marker-only path as a +separate diagnostic scenario. Cross-library async/null payload comparisons use +the dedicated matched target, which carries the full message through both +queues and writes to a separate result CSV. + ## Consequences Benchmark documentation remains comparable and honest across changes. New diff --git a/docs/benchmarks.md b/docs/benchmarks.md index d2a7224..d33c8d8 100644 --- a/docs/benchmarks.md +++ b/docs/benchmarks.md @@ -12,6 +12,14 @@ cmake --build build --target logit_bench ./build/bench/logit_bench ``` +For a contract-matched asynchronous null-sink comparison, build +`logit_bench_async_contract` and run it with +an explicit producer count and message size when comparing the two libraries. +This target is structurally limited to `async=1`, `sink=null` and defaults to the separate +`bench/results/latency-async-contract.csv` output; `LOGIT_BENCH_OUTPUT` can +select another path when needed. In this target LogIt++ carries the full +message through its async queue, matching spdlog's async payload contract. + The harness records latency from the logging call until the adapter enters its sink callback, together with aggregate throughput. This is a **sink-entry latency** metric: for the file scenario the marker is recorded before the @@ -19,7 +27,9 @@ latency** metric: for the file scenario the marker is recorded before the It compares synchronous and asynchronous modes, null and file sinks, producer counts, and message sizes. Results are appended to `bench/results/latency.csv`; workload size can be -reduced with `LOGIT_BENCH_TOTAL` and `LOGIT_BENCH_WARMUP`. +reduced with `LOGIT_BENCH_TOTAL` and `LOGIT_BENCH_WARMUP`. Set +`LOGIT_BENCH_OUTPUT` to keep a separate workload contract from being appended +to an existing CSV. ## Interpreting results @@ -37,6 +47,11 @@ are not a universal speed ranking. The asynchronous measurement includes enqueue, worker wake-up/scheduling, and sink time. Results are sensitive to queue capacity, overflow policy, worker count, filesystem cache state, compiler, operating system, and hardware. +The default LogIt++ async/null adapter transports only a slot marker because +the null sink does not consume the message. That path is intentionally not a +cross-library payload-cost comparison: spdlog transports the message through +its async queue. Use `logit_bench_async_contract` for a matched full-message +async contract. ## Historical snapshot @@ -77,14 +92,31 @@ sink callback. A queue capacity of `0` is rejected because it would mean an unlimited LogIt++ queue but a bounded spdlog queue and invalidate the comparison. -The current CSV schema includes `queue_capacity`. Before appending, the harness -validates the existing `bench/results/latency.csv` header and fails with a -rename/remove instruction when it finds an older schema. Existing result files -are never silently rewritten or mixed with rows from a different schema. - -The prepared-message/direct-dispatch pipeline and a true public macro benchmark that -calls `LOGIT_INFO(...)` are separate scenarios with different work contracts; -their results must not be presented as one number. +The current CSV schema includes `queue_capacity` and `workload_contract`. +Before appending, the harness validates the selected output CSV header and +fails with a rename/remove instruction when it finds an older schema. Existing +result files are never silently rewritten or mixed with rows from a different +schema. Each CSV row retains its comparison provenance. + +The prepared-message/direct-dispatch pipeline, the matched full-message async +target, and a true public macro benchmark that calls `LOGIT_INFO(...)` are +separate scenarios with different work contracts; their results must not be +presented as one number. + +`logit_bench_async_contract` uses workload contract +`prepared-message/async-full-message`. For async/null it copies the same +message payload into the LogIt++ worker task that spdlog carries in its async +queue. Its CSV uses a separate output path (the default is +`bench/results/latency-async-contract.csv`, overridable with +`LOGIT_BENCH_OUTPUT`) and should be compared only with runs carrying the same +workload contract. This matches the queued payload contract, not every +instruction or allocation performed by the two libraries; LogIt++ still builds +and dispatches its `LogRecord` before the sink task is queued. + +`logit_bench_async_payload_contract_test` is a functional regression test, not +a performance measurement. It configures the matched null-sink path with a +200-byte payload and verifies the exact payload observed by the worker-side +sink callback. `logit_exec_mx_bench` and `logit_exec_mx_bench_concurrent` are a guarded lock-elision experiment. They use the same prepared `LogRecord` and a small @@ -113,12 +145,12 @@ experiments on identical hardware. Both targets print the same fixture metadata line as `logit_bench`, using `not-applicable` for queue settings and explicit `backend-count` / `logger-wait` completion semantics. -The checked-in [`benchmark-fixture-v1.json`](https://github.com/LimiNode/log-it-cpp/blob/main/bench/results/benchmark-fixture-v1.json) +The checked-in [`benchmark-fixture-v2.json`](https://github.com/LimiNode/log-it-cpp/blob/main/bench/results/benchmark-fixture-v2.json) defines the required metadata and workload contract. All publication-capable benchmark binaries print a versioned `benchmark-fixture` metadata line for each run, including source commit, compiler/version, toolchain, C++ standard, platform, build type, architecture, machine identity, CPU model, queue -settings, latency completion, and flush barrier. The commit defaults to +settings, latency completion, flush barrier, and workload contract. The commit defaults to `LOGIT_BENCH_COMMIT` or `GITHUB_SHA`; machine identity and CPU model can be provided through `LOGIT_BENCH_MACHINE_ID` and `LOGIT_BENCH_CPU_MODEL`. For a comparable/publication run, set `LOGIT_BENCH_REQUIRE_COMPARABLE=1` and @@ -128,7 +160,8 @@ provide all required metadata; public-macro runs may explicitly use that must match between runs: `source_commit` is required provenance and is expected to differ in before/after comparisons, while the fields listed in `metadata_must_match` (compiler, toolchain, platform, build, hardware, queue, -and completion semantics) must be identical. `LOGIT_BENCH_REQUIRE_COMPARABLE=1` +completion semantics, and workload contract) must be identical. +`LOGIT_BENCH_REQUIRE_COMPARABLE=1` checks metadata completeness and known values; it does not enforce the canonical fixture workload values such as total messages, warmup, or producer matrix.