From 04c3e65a4cd7d4722c1ab3b402cd46b057f575b5 Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Fri, 15 May 2026 09:53:20 +0000 Subject: [PATCH 1/4] events: Document metric_export_sample_by Adds a Sampling section to the Event Export page and settings entries for metric_export_sample_by / _field / _percent, and a stats_sample_by anchor for cross-linking. Sampling hashes a configured event field (typically session, or trace_id for whole traces) and forwards only the percentage set by percent to the exporter, while in-process statistics counters keep counting every event. Multiple sample_by blocks combine with AND semantics. --- data/settings.js | 65 +++++++++++++++++++++++++++ data/updates.js | 1 + docs/core/config/events/export.md | 75 +++++++++++++++++++++++++++++++ 3 files changed, 141 insertions(+) diff --git a/data/settings.js b/data/settings.js index eac3fafa0..f31157a52 100644 --- a/data/settings.js +++ b/data/settings.js @@ -10476,6 +10476,71 @@ Human-readable description of the metric. This is included in the HELP text sent to OpenMetrics.` }, + metric_export_sample_by: { + added: { + settings_metric_export_sample_by_added: false, + }, + tags: [ 'metrics' ], + values: setting_types.NAMED_LIST_FILTER, + seealso: [ '[[link,stats_sample_by]]', + 'metric_export_sample_by_field', + 'metric_export_sample_by_percent' ], + text: ` +Defines a [[link,stats_sample_by,sampling rule]] that decides whether an +event matching this metric is forwarded to the +[[setting,metric_exporter]]. The filter name is an arbitrary identifier +for the rule; the actual sampled field is the +[[setting,metric_export_sample_by_field]] setting inside the block. + +Multiple [[setting,metric_export_sample_by]] blocks combine with AND +semantics: an event is exported only if every rule passes. Sampling is +deterministic: events that share the same field value are always sampled +in or out together. Statistics counters are unaffected; sampling applies +only to exported events.` + }, + + metric_export_sample_by_field: { + added: { + settings_metric_export_sample_by_added: false, + }, + tags: [ 'metrics' ], + values: setting_types.STRING, + seealso: [ '[[link,stats_sample_by]]', + 'metric_export_sample_by', + 'metric_export_sample_by_percent' ], + text: ` +Name of the event field whose value is hashed to make the sampling +decision. A common choice is \`session\`, which causes every event in +the same session to be sampled in or out together. + +The value \`trace_id\` always uses the trace id of the event; an event field named +\`trace_id\` is not looked up. Sampling by \`trace_id\` keeps every event of a +trace together. It also keeps a \`doveadm\` command whole: the command's own +events have no \`session\` field, so sampling by \`session\` drops them. + +An event without the field, or for \`trace_id\` without a trace id, is dropped +from the export.` + }, + + metric_export_sample_by_percent: { + added: { + settings_metric_export_sample_by_added: false, + }, + tags: [ 'metrics' ], + values: setting_types.STRING, + default: '100', + seealso: [ '[[link,stats_sample_by]]', + 'metric_export_sample_by', + 'metric_export_sample_by_field' ], + text: ` +Percentage of distinct field values that are exported. The valid range is +\`0.001\`..\`100\`; \`0\` is rejected. The default \`100\` exports every event, +which makes the [[setting,metric_export_sample_by]] block a no-op. + +For example, \`percent = 10\` exports approximately 10% of distinct field +values, and \`percent = 0.5\` exports approximately 0.5%.` + }, + oauth2: { tags: [ 'oauth2' ], values: setting_types.NAMED_FILTER, diff --git a/data/updates.js b/data/updates.js index c50ce7003..34e6577b6 100644 --- a/data/updates.js +++ b/data/updates.js @@ -232,6 +232,7 @@ export const updates = { settings_mailbox_defaults_english_changed: '2.4.3', settings_mailbox_directory_name_legacy_deprecated: '2.4.2', settings_mailbox_special_use_changed: '2.4.0', + settings_metric_export_sample_by_added: '2.4.6', settings_metric_fields_changed: '2.4.0', settings_passdb_default_password_scheme_changed: '2.4.3', settings_passdb_mechanisms_filter_added: '2.4.0', diff --git a/docs/core/config/events/export.md b/docs/core/config/events/export.md index 2dc147338..8a796283b 100644 --- a/docs/core/config/events/export.md +++ b/docs/core/config/events/export.md @@ -12,6 +12,9 @@ dovecotlinks: event_export_drivers: hash: drivers text: "Event Export: Drivers" + stats_sample_by: + hash: sampling + text: "Event Export: Sampling" --- # Event Export @@ -144,6 +147,78 @@ One uses the `metric` block settings documented in [[link,stats]] to select and filter the event to be exported. See [[setting,metric_exporter]] and [[setting,metric_exporter_include]] settings. +## Sampling + +[[added,settings_metric_export_sample_by_added]] + +When a metric's exporter would emit too many events (for example, every +IMAP command from every session), the +[[setting,metric_export_sample_by]] block selects a deterministic subset +to forward to the exporter. + +For each block, Dovecot hashes the configured event field with SHA-1, +folds the digest to a 64-bit integer, and exports the event when +`hash % 100000 < percent * 1000`. Because the decision is a pure function of +the field value, every event that shares the same value (e.g. every event +in the same IMAP session, when sampling by `session`) is either +exported in full or dropped in full. This is the property that makes +the resulting export usable for tracing: a sampled session has all of +its events, not a random scattering. + +To keep every event of a trace together, or a `doveadm` command whole, sample +by `trace_id`, see [[setting,metric_export_sample_by_field]]. + +::: tip +Sampling only affects what reaches the [[setting,metric_exporter]]. +The metric's in-process counters (`doveadm stats dump`, +OpenMetrics) still count every event. +::: + +Multiple [[setting,metric_export_sample_by]] blocks combine with AND +semantics: an event is exported only if every rule passes. This lets a +metric require, say, both a sampled session *and* a sampled user. + +An event without the configured [[setting,metric_export_sample_by_field]], or +for `trace_id` without a trace id, is dropped from the export. + +### Example: sample 10% of IMAP sessions + +```doveconf[dovecot.conf] +event_exporter datalake { + driver = http-post + http_post_url = https://datalake.example.com/api/endpoint + format = json + time_format = rfc3339 +} + +metric imap_traced { + exporter = datalake + filter = event=imap_command_finished + + metric_export_sample_by sess { + field = session + percent = 10 + } +} +``` + +Roughly 10% of distinct session IDs are forwarded to the exporter, and +the same session always produces the same decision across restarts. + +### Example: sample 1% of traces + +```doveconf[dovecot.conf] +metric traces { + exporter = datalake + filter = event=* + + metric_export_sample_by traces { + field = trace_id + percent = 1 + } +} +``` + ## Settings From d95893ad873e861a2a4e06ab4a40336d7a0fe348 Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Fri, 15 May 2026 12:49:08 +0000 Subject: [PATCH 2/4] events: Document the opentelemetry driver Adds settings entries for event_exporter_opentelemetry_url, _flush_interval, _flush_count and _max_pending_requests, and an OpenTelemetry section: the span model (one span per span owner), span kinds, status and timing, trace correlation, resource attributes, batching, and the protobuf and JSON wire formats. Whitelist the new spell-check terms in .github/actions/spelling/allow.txt. --- .github/actions/spelling/allow.txt | 7 + data/settings.js | 70 +++++++++- data/updates.js | 1 + docs/core/config/events/export.md | 216 ++++++++++++++++++++++++++++- 4 files changed, 292 insertions(+), 2 deletions(-) diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index e33898afe..3ffe4fb87 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -13,12 +13,14 @@ fcf frop fuzzer github +Grafana hardenings https iostream iostreams istream istreams +Jaeger jvm JVM JVMs @@ -27,6 +29,7 @@ learntype lfill libcap libpcre +libprotobuf libstemmer libtextcat lri @@ -36,13 +39,17 @@ mmaped mtls ond onlinedocs +opentelemetry ostream ostreams +otel +otlp plugin plugins postgre Postgre postgres +protobuf retpoline retuns rfill diff --git a/data/settings.js b/data/settings.js index f31157a52..e7adc969e 100644 --- a/data/settings.js +++ b/data/settings.js @@ -6371,7 +6371,7 @@ Creates a new event exporter. The filter name refers to the event_exporter_driver: { tags: [ 'event-export' ], values: setting_types.ENUM, - values_enum: [ 'log', 'file', 'unix', 'http-post', 'drop' ], + values_enum: [ 'log', 'file', 'unix', 'http-post', 'drop', 'opentelemetry' ], default: 'log', seealso: [ '[[link,event_export_drivers]]' ], text: ` @@ -6445,6 +6445,74 @@ Timeout when connecting to unix socket with Path to event unix socket with [[setting,event_exporter_driver,unix]].` }, + event_exporter_opentelemetry_url: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.STRING, + default: 'http://localhost:4318', + seealso: [ '[[link,event_export_opentelemetry]]' ], + text: ` +Base URL of the OTLP/HTTP collector when using +[[setting,event_exporter_driver,opentelemetry]]. The exporter POSTs +spans to \`/v1/traces\`. The wire format and +\`Content-Type\` are selected by [[setting,event_exporter_format]]: + +* \`protobuf\` → \`application/x-protobuf\` (OTLP/HTTP+protobuf binary) +* \`json\` → \`application/json\` (OTLP/HTTP+JSON, proto3 ProtoJSON + canonical encoding) + +Spans are sent in batches, see +[[setting,event_exporter_opentelemetry_flush_interval]]. + +Only OTLP/HTTP is implemented. OTLP/gRPC is not supported; point this +setting at a collector that accepts OTLP/HTTP (the OpenTelemetry +Collector, Jaeger v2, Grafana Tempo, etc.).` + }, + + event_exporter_opentelemetry_flush_interval: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.TIME_MSECS, + default: '5 secs', + seealso: [ 'event_exporter_opentelemetry_flush_count' ], + text: ` +Spans are sent at the latest this long after the first one was queued, +even when [[setting,event_exporter_opentelemetry_flush_count]] has not been +reached. Must not be \`0\`.` + }, + + event_exporter_opentelemetry_flush_count: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.UINT, + default: 512, + seealso: [ 'event_exporter_opentelemetry_flush_interval', + 'event_exporter_opentelemetry_max_pending_requests' ], + text: ` +Spans are sent as soon as this many are queued, without waiting for +[[setting,event_exporter_opentelemetry_flush_interval]]. Must not be \`0\`.` + }, + + event_exporter_opentelemetry_max_pending_requests: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.UINT, + default: 4, + seealso: [ 'event_exporter_opentelemetry_flush_count' ], + text: ` +Maximum number of export requests that can be waiting for the collector's +response. While this many requests are pending, a further batch is dropped +and an error is logged with the number of dropped records. Must not be \`0\`.` + }, + execute: { tags: [ 'execute' ], values: setting_types.NAMED_LIST_FILTER, diff --git a/data/updates.js b/data/updates.js index 34e6577b6..c28e6f0ba 100644 --- a/data/updates.js +++ b/data/updates.js @@ -200,6 +200,7 @@ export const updates = { settings_dovecot_config_version_added: '2.4.0', settings_dovecot_storage_version_added: '2.4.0', settings_dovecot_storage_version_thread_index_changed: '2.4.5', + settings_event_exporter_opentelemetry_added: '2.4.6', settings_imapc_features_auto_capabilities: '2.4.3', settings_fifo_listener_type_added: '2.4.0', settings_fts_decoder_tika_url_changed: '2.4.0', diff --git a/docs/core/config/events/export.md b/docs/core/config/events/export.md index 8a796283b..32f6813df 100644 --- a/docs/core/config/events/export.md +++ b/docs/core/config/events/export.md @@ -12,6 +12,9 @@ dovecotlinks: event_export_drivers: hash: drivers text: "Event Export: Drivers" + event_export_opentelemetry: + hash: opentelemetry + text: "Event Export: OpenTelemetry" stats_sample_by: hash: sampling text: "Event Export: Sampling" @@ -61,8 +64,9 @@ Supported Formats: | Formats | Description | | ------- | ----------- | -| `json` | JSON output | +| `json` | JSON output. With the [[setting,event_exporter_driver,opentelemetry]] driver this produces OTLP/HTTP+JSON `TracesData` payload (proto3 canonical encoding); with all other drivers, the legacy event-envelope JSON shown below. | | `tab-text` | TAB-separated text fields | +| `protobuf` | OTLP/HTTP+protobuf `TracesData` payload. Only valid with [[setting,event_exporter_driver,opentelemetry]] driver.
[[added,settings_event_exporter_opentelemetry_added]] | #### Example: JSON @@ -115,6 +119,7 @@ Supported drivers: | `http-post` | Send the serialized event as a HTTP POST payload to [[setting,event_exporter_http_post_url]]. The driver defaults to [[setting,http_client_request_absolute_timeout,250 milliseconds]]. | | `file` | Send serialized events to a file specified in [[setting,event_exporter_file_path]]
[[added,event_export_drivers_file_unix_added]] | | `unix` | Send serialised events to a unix socket specified in [[setting,event_exporter_unix_path]]. The [[setting,event_exporter_unix_connect_timeout]] setting is used to specify how long the unix socket connection can take. Default is `250 milliseconds`.
[[added,event_export_drivers_file_unix_added]] | +| `opentelemetry` | Send events as OTLP/HTTP spans to an OpenTelemetry collector at [[setting,event_exporter_opentelemetry_url]]. Wire format is selected by [[setting,event_exporter_format]] (`protobuf` or `json`). See [[link,event_export_opentelemetry]].
[[added,settings_event_exporter_opentelemetry_added]] | The `drop` driver is useful when one wants to disable the event exporter temporarily. Note that serialization still occurs, but the resulting @@ -147,6 +152,215 @@ One uses the `metric` block settings documented in [[link,stats]] to select and filter the event to be exported. See [[setting,metric_exporter]] and [[setting,metric_exporter_include]] settings. +## OpenTelemetry + +[[added,settings_event_exporter_opentelemetry_added]] + +The `opentelemetry` driver sends spans to an OTLP/HTTP collector (Jaeger v2, +Grafana Tempo, the OpenTelemetry Collector, etc.) as serialized +[OTLP `TracesData`](https://opentelemetry.io/docs/concepts/signals/traces/) +payloads. Spans are queued and sent in batches, see +[Batching](#batching). + +The `opentelemetry` driver requires [[setting,event_exporter_format]] +to be set to one of: + +| Format | Wire format | `Content-Type` | +| ------ | ----------- | -------------- | +| `protobuf` | OTLP/HTTP+protobuf (binary `TracesData`) | `application/x-protobuf` | +| `json` | OTLP/HTTP+JSON (proto3 ProtoJSON canonical encoding) | `application/json` | + +The `protobuf` format produces a compact binary payload; the `json` +format is easier to inspect with `curl` / `jq`. Both POST to the same +`/v1/traces` path and carry identical trace/span content; they +differ only in serialization. + +Any other format value is rejected at config load time. `protobuf` may +only be paired with this driver. + +::: tip +Only OTLP/HTTP is implemented. OTLP/gRPC is **not** supported; point +[[setting,event_exporter_opentelemetry_url]] at a collector that +accepts OTLP/HTTP (most collectors expose both endpoints on different +ports — use the HTTP one, typically port `4318`). + +The driver is available only when Dovecot is built with `libprotobuf-c` +support. +::: + +Minimal configuration (protobuf): + +```doveconf[dovecot.conf] +event_exporter otlp { + driver = opentelemetry + format = protobuf + event_exporter_opentelemetry_url = http://collector.example.com:4318 +} + +metric traces { + filter = event=* + exporter = otlp +} +``` + +To export as JSON instead, set `format = json`. + +The driver accepts the standard `http_client_*` settings (for example +[[setting,http_client_request_timeout]]) directly under the exporter block. The exporter overrides some defaults: + +| Setting | Exporter default | +| ------- | ---------------- | +| [[setting,http_client_request_timeout]] | `2s` | +| [[setting,http_client_request_absolute_timeout]] | `5s` | + +[[setting,http_client_request_absolute_timeout]] must not be `0`; the +configuration is rejected otherwise. + +```doveconf[dovecot.conf] +event_exporter otlp { + driver = opentelemetry + format = json + event_exporter_opentelemetry_url = http://collector.example.com:4318 + http_client_request_timeout = 1s +} +``` + +The JSON output follows the +[OTLP/HTTP+JSON encoding](https://opentelemetry.io/docs/specs/otlp/#json-protobuf-encoding): +`trace_id` / `span_id` are lowercase hex strings (OTLP deviates from +proto3 ProtoJSON here, which would use base64), `int64` values including +`startTimeUnixNano` / `endTimeUnixNano` / `intValue` are JSON strings, +and enums (`SpanKind`, `Status.code`) are name strings (e.g. +`"SPAN_KIND_SERVER"`, `"STATUS_CODE_ERROR"`). + +### Spans + +Spans are produced per *span owner*, not per event. A span owner is an +object that has a lifetime, and each owner is one span, which is sent when +the owner ends: + +| Span name | Owner | Span kind | +| --------- | ----- | --------- | +| `login_client` | Client connection in a login process | `SERVER` | +| `imap_client` | IMAP client session | `SERVER` | +| `pop3_client` | POP3 client session | `SERVER` | +| `submission_client` | Submission client session | `SERVER` | +| `lmtp_client` | LMTP client connection | `SERVER` | +| `imap_command` | IMAP command | `INTERNAL` | +| `auth_request` | Authentication request in the auth process | `SERVER` | +| `auth_client_request` | Authentication request made by a service | `CLIENT` | +| `auth_master_request` | Userdb lookup the auth process does for a service after a successful login | `SERVER` | +| `smtp_server_transaction` | SMTP/LMTP server transaction | `INTERNAL` | +| `doveadm_command` | doveadm command | `INTERNAL` for the CLI, otherwise `SERVER` | + +Other events that match the metric's filter become child spans of the +span they happened in. They have the kind `INTERNAL`, are named after the +event and have a random span id. + +The `auth_master_request` span is a child of the span that did the login, such +as `login_client`. + +A record that has no span of its own belongs to the IMAP command that is running +when the record is sent, not when it was created. For example +`mail_cache_decision_changed` sent during an `APPEND` is a child of that +`imap_command` span. Records sent outside a command stay children of the +`imap_client` span. + +The span of an owner is named after the owner's span name (for example +`imap_command` or `auth_request`). The name of the event that ended the +owner (for example `imap_command_finished`) is the name of the span's +`Span.Event`. + +If a process disconnects unexpectedly before an owner ends, the span is sent +when the connection to the stats process is closed, with the attribute +`dovecot.span.truncated = true` and the closing time as its end time. + +The span status is `ERROR` if the event has an `error` field (the field +value becomes the status message), if an IMAP command was replied to with +`BAD` (the status message is the tagged reply text), or if a POP3 reply was +`-ERR`. An IMAP `NO` reply is a normal protocol result and leaves the +status `UNSET`, as does everything else. + +Event fields and the category list are sent as span `attributes`, subject to +[[setting,metric_exporter_include]] and [[setting,metric_fields]]. Fields whose +name begins with `_` are not sent. + +#### Span timing + +The span `start_time` is the creation time of the event that owns the span. For +an IMAP command this is when its first data arrives, not when the full line is +received. Other exported events start at their own creation time, except events +that only extend another event (for example `mail_user_session_finished` and +the auth and SMTP `*_finished` events), which start at the creation time of the +event they extend. A record's span never starts before the span it belongs to: +its start is the later of the two creation times. + +The span `end_time` is the time the event that ends the span was sent (for +example `imap_command_finished`). An owner with no such event ends when its +last reference is released, or when the stats connection closes for truncated +spans. + +#### Which spans are exported + +An owner span is exported only if at least one record under it was exported +by the same exporter: an event of the owner itself, or of a span nested +inside it in the same process. With a filter such as + +```doveconf[dovecot.conf] +metric imap_commands { + filter = event=imap_command_finished + exporter = otlp +} +``` + +the `imap_command` and `imap_client` spans are emitted, but the +`login_client` span of the login process is not, because no event of that +process matched. The collector then shows the trace with a missing parent. + +For complete traces use `filter = event=*`, or include the events of the +session level (login, authentication and client sessions) in the filter. +Combine it with [[link,stats_sample_by]] to keep the volume bounded. + +### Trace correlation + +Every session gets a random `trace_id`, and every span owner a random +`span_id`. The ids are not derived from the session id. The id of the trace +is created by the first Dovecot process that handles the connection +(normally a login process, or `doveadm` for its own commands) and is passed +on, with the span id of the sender as the parent of the receiving span: + +* login process to the IMAP/POP3/submission process: in the login + request, together with the id of the `login_client` span as the parent. +* services to the auth process: the `trace_id` and `parent_span_id` + fields of the auth protocol request. Userdb lookups carry them as well. + +This ties the events of one mail session, and the work it causes in other +processes, into one trace. The OpenTelemetry exporter does not export events +that have no trace id. + +### Resource attributes + +Each batch carries an OTel +[Resource](https://opentelemetry.io/docs/concepts/resources/) with +`service.name = "Dovecot"`, `service.version` set to the Dovecot version, +`service.instance.id` set to a random version 4 UUID, one per exporter +(a new one after a configuration reload), and `host.name` set to the +hostname. Sub-system context (`imap`, `imap-login`, `auth`, `indexer-worker`, +…) lands on the span's InstrumentationScope rather than on `service.name`, so +all traces show up under a single `Dovecot` service in collector UIs. + +### Batching + +Spans are queued and sent in batches, one HTTP request per batch, rather than +one request per event. A batch is sent when it has +[[setting,event_exporter_opentelemetry_flush_count]] spans, and at the latest +[[setting,event_exporter_opentelemetry_flush_interval]] after its first span +was queued. + +While [[setting,event_exporter_opentelemetry_max_pending_requests]] requests +are waiting for the collector, a further batch is dropped and an error with the +number of dropped records is logged. + ## Sampling [[added,settings_metric_export_sample_by_added]] From 36238ec8d4cfff665256829e2127a703d2d63fa9 Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Wed, 4 Nov 2026 09:21:44 +0000 Subject: [PATCH 3/4] events: Document the opentelemetry log records feature Add event_exporter_opentelemetry_emit_spans and event_exporter_opentelemetry_emit_logs settings, and describe the OTLP LogRecord output shape, severity mapping and trace/span correlation, and the reserved _-prefixed event fields. --- data/settings.js | 68 ++++++++++++++++--- docs/core/config/events/export.md | 105 ++++++++++++++++++++++++++---- 2 files changed, 150 insertions(+), 23 deletions(-) diff --git a/data/settings.js b/data/settings.js index e7adc969e..0bde4a000 100644 --- a/data/settings.js +++ b/data/settings.js @@ -6452,18 +6452,21 @@ Path to event unix socket with [[setting,event_exporter_driver,unix]].` tags: [ 'event-export' ], values: setting_types.STRING, default: 'http://localhost:4318', - seealso: [ '[[link,event_export_opentelemetry]]' ], + seealso: [ '[[link,event_export_opentelemetry]]', + 'event_exporter_opentelemetry_emit_logs' ], text: ` Base URL of the OTLP/HTTP collector when using [[setting,event_exporter_driver,opentelemetry]]. The exporter POSTs -spans to \`/v1/traces\`. The wire format and +spans to \`/v1/traces\` and (when +[[setting,event_exporter_opentelemetry_emit_logs]] is enabled) log +records to \`/v1/logs\`. The wire format and \`Content-Type\` are selected by [[setting,event_exporter_format]]: * \`protobuf\` → \`application/x-protobuf\` (OTLP/HTTP+protobuf binary) * \`json\` → \`application/json\` (OTLP/HTTP+JSON, proto3 ProtoJSON canonical encoding) -Spans are sent in batches, see +Spans and log records use the same format and are sent in batches, see [[setting,event_exporter_opentelemetry_flush_interval]]. Only OTLP/HTTP is implemented. OTLP/gRPC is not supported; point this @@ -6471,6 +6474,49 @@ setting at a collector that accepts OTLP/HTTP (the OpenTelemetry Collector, Jaeger v2, Grafana Tempo, etc.).` }, + event_exporter_opentelemetry_emit_spans: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.BOOLEAN, + default: 'yes', + seealso: [ '[[link,event_export_opentelemetry]]', + 'event_exporter_opentelemetry_emit_logs' ], + text: ` +When enabled (the default), the exporter emits OTLP Spans to +\`/v1/traces\`. Disable only when you want the exporter to +ship log records exclusively (set together with +[[setting,event_exporter_opentelemetry_emit_logs,yes]]).` + }, + + event_exporter_opentelemetry_emit_logs: { + added: { + settings_event_exporter_opentelemetry_added: false, + }, + tags: [ 'event-export' ], + values: setting_types.BOOLEAN, + default: 'no', + seealso: [ '[[link,event_export_opentelemetry_logs]]', + 'event_exporter_opentelemetry_emit_spans' ], + text: ` +When enabled, the exporter emits an OTLP LogRecord to +\`/v1/logs\` for every matched event that carries a +formatted log message (\`e_info()\`, \`e_warning()\`, \`e_error()\`, +\`e_debug()\` call sites). The LogRecord shares its \`trace_id\` with the +trace of the span it belongs to, and its \`span_id\` with that span, so +collectors link logs and traces. + +Log records use the wire format selected by +[[setting,event_exporter_format]], like spans. Severity is mapped to the OTel +severity model (DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21; +PANIC is sent as FATAL4=24). + +Enabling this setting opts the stats process into receiving the +formatted message text from each dovecot service; the wire cost is +zero when the corresponding metric's filter does not match an event.` + }, + event_exporter_opentelemetry_flush_interval: { added: { settings_event_exporter_opentelemetry_added: false, @@ -6480,9 +6526,9 @@ Collector, Jaeger v2, Grafana Tempo, etc.).` default: '5 secs', seealso: [ 'event_exporter_opentelemetry_flush_count' ], text: ` -Spans are sent at the latest this long after the first one was queued, -even when [[setting,event_exporter_opentelemetry_flush_count]] has not been -reached. Must not be \`0\`.` +Spans and log records are sent at the latest this long after the first one +was queued, even when [[setting,event_exporter_opentelemetry_flush_count]] has +not been reached. Must not be \`0\`.` }, event_exporter_opentelemetry_flush_count: { @@ -6495,8 +6541,9 @@ reached. Must not be \`0\`.` seealso: [ 'event_exporter_opentelemetry_flush_interval', 'event_exporter_opentelemetry_max_pending_requests' ], text: ` -Spans are sent as soon as this many are queued, without waiting for -[[setting,event_exporter_opentelemetry_flush_interval]]. Must not be \`0\`.` +Spans and log records are sent as soon as this many are queued, without +waiting for [[setting,event_exporter_opentelemetry_flush_interval]]. Spans and +log records are queued separately. Must not be \`0\`.` }, event_exporter_opentelemetry_max_pending_requests: { @@ -6509,8 +6556,9 @@ Spans are sent as soon as this many are queued, without waiting for seealso: [ 'event_exporter_opentelemetry_flush_count' ], text: ` Maximum number of export requests that can be waiting for the collector's -response. While this many requests are pending, a further batch is dropped -and an error is logged with the number of dropped records. Must not be \`0\`.` +response. Spans and log records share this limit. While this many requests +are pending, a further batch is dropped and an error is logged with the +number of dropped records. Must not be \`0\`.` }, execute: { diff --git a/docs/core/config/events/export.md b/docs/core/config/events/export.md index 32f6813df..ea0f05824 100644 --- a/docs/core/config/events/export.md +++ b/docs/core/config/events/export.md @@ -15,6 +15,9 @@ dovecotlinks: event_export_opentelemetry: hash: opentelemetry text: "Event Export: OpenTelemetry" + event_export_opentelemetry_logs: + hash: log-records + text: "Event Export: OpenTelemetry LogRecords" stats_sample_by: hash: sampling text: "Event Export: Sampling" @@ -64,9 +67,9 @@ Supported Formats: | Formats | Description | | ------- | ----------- | -| `json` | JSON output. With the [[setting,event_exporter_driver,opentelemetry]] driver this produces OTLP/HTTP+JSON `TracesData` payload (proto3 canonical encoding); with all other drivers, the legacy event-envelope JSON shown below. | +| `json` | JSON output. With the [[setting,event_exporter_driver,opentelemetry]] driver this produces OTLP/HTTP+JSON `TracesData` (and `LogsData`) payloads (proto3 canonical encoding); with all other drivers, the event JSON shown below. | | `tab-text` | TAB-separated text fields | -| `protobuf` | OTLP/HTTP+protobuf `TracesData` payload. Only valid with [[setting,event_exporter_driver,opentelemetry]] driver.
[[added,settings_event_exporter_opentelemetry_added]] | +| `protobuf` | OTLP/HTTP+protobuf `TracesData` (and `LogsData`) payloads. Only valid with [[setting,event_exporter_driver,opentelemetry]] driver.
[[added,settings_event_exporter_opentelemetry_added]] | #### Example: JSON @@ -119,7 +122,7 @@ Supported drivers: | `http-post` | Send the serialized event as a HTTP POST payload to [[setting,event_exporter_http_post_url]]. The driver defaults to [[setting,http_client_request_absolute_timeout,250 milliseconds]]. | | `file` | Send serialized events to a file specified in [[setting,event_exporter_file_path]]
[[added,event_export_drivers_file_unix_added]] | | `unix` | Send serialised events to a unix socket specified in [[setting,event_exporter_unix_path]]. The [[setting,event_exporter_unix_connect_timeout]] setting is used to specify how long the unix socket connection can take. Default is `250 milliseconds`.
[[added,event_export_drivers_file_unix_added]] | -| `opentelemetry` | Send events as OTLP/HTTP spans to an OpenTelemetry collector at [[setting,event_exporter_opentelemetry_url]]. Wire format is selected by [[setting,event_exporter_format]] (`protobuf` or `json`). See [[link,event_export_opentelemetry]].
[[added,settings_event_exporter_opentelemetry_added]] | +| `opentelemetry` | Send spans, and optionally log records, as OTLP/HTTP to an OpenTelemetry collector at [[setting,event_exporter_opentelemetry_url]]. Wire format is selected by [[setting,event_exporter_format]] (`protobuf` or `json`). See [[link,event_export_opentelemetry]].
[[added,settings_event_exporter_opentelemetry_added]] | The `drop` driver is useful when one wants to disable the event exporter temporarily. Note that serialization still occurs, but the resulting @@ -283,7 +286,7 @@ status `UNSET`, as does everything else. Event fields and the category list are sent as span `attributes`, subject to [[setting,metric_exporter_include]] and [[setting,metric_fields]]. Fields whose -name begins with `_` are not sent. +name begins with `_` are not sent, see [Reserved fields](#reserved-fields). #### Span timing @@ -336,7 +339,7 @@ on, with the span id of the sender as the parent of the receiving span: This ties the events of one mail session, and the work it causes in other processes, into one trace. The OpenTelemetry exporter does not export events -that have no trace id. +and log records that have no trace id. ### Resource attributes @@ -351,15 +354,91 @@ all traces show up under a single `Dovecot` service in collector UIs. ### Batching -Spans are queued and sent in batches, one HTTP request per batch, rather than -one request per event. A batch is sent when it has -[[setting,event_exporter_opentelemetry_flush_count]] spans, and at the latest -[[setting,event_exporter_opentelemetry_flush_interval]] after its first span -was queued. +Spans and log records are queued and sent in batches, one HTTP request per +batch, rather than one request per event. A batch is sent when it has +[[setting,event_exporter_opentelemetry_flush_count]] records, and at the latest +[[setting,event_exporter_opentelemetry_flush_interval]] after its first record +was queued. Spans and log records are queued separately. + +Spans and log records share the +[[setting,event_exporter_opentelemetry_max_pending_requests]] export requests +that can be waiting for the collector. While that many are pending, a further +batch is dropped and an error is logged with the endpoint and the number of +dropped records, e.g. +`Too many pending requests for http://collector.example.com:4318/v1/traces, dropped 512 records`. + +### Log records + +When [[setting,event_exporter_opentelemetry_emit_logs,yes]] is set, the +exporter additionally sends an OTLP +[LogRecord](https://opentelemetry.io/docs/specs/otel/logs/data-model/) +to `/v1/logs` for every matched event that carries a +formatted log message (i.e. events produced by `e_info()`, +`e_warning()`, `e_error()`, `e_debug()` and friends). + +Log records use the wire format selected by +[[setting,event_exporter_format]]: `protobuf` or `json`, as for spans. +They are batched like spans. + +Each LogRecord carries: + +* the `trace_id` of the trace it belongs to, and the `span_id` of the span + that was exported for the same event, so the collector can navigate from a + log line to its span and vice versa. With + [[setting,event_exporter_opentelemetry_emit_spans,no]] the `span_id` is + that of the span the event happened in. Events outside of any span carry + no `span_id`, +* a `severityNumber` and `severityText` derived from the dovecot + log type (DEBUG=5, INFO=9, WARN=13, ERROR=17, FATAL=21; + PANIC is sent as FATAL4=24), +* `body.stringValue` with the formatted message text from the + `e_*()` call site, +* the event's fields and category list as `attributes`, identical to + what the matching span carries. + +The exporter ships log records only for events whose corresponding +metric's `filter` matches. Use the metric filter to scope log +forwarding to the sessions you want to trace - typically a session +predicate plus a category restriction: -While [[setting,event_exporter_opentelemetry_max_pending_requests]] requests -are waiting for the collector, a further batch is dropped and an error with the -number of dropped records is logged. +```doveconf[dovecot.conf] +event_exporter otlp_logs { + driver = opentelemetry + format = json + event_exporter_opentelemetry_url = http://collector.example.com:4318 + event_exporter_opentelemetry_emit_logs = yes +} + +metric traced_session_logs { + exporter = otlp_logs + # Pick whichever events should ship as both spans and log records. + filter = event=imap_command_finished +} +``` + +Only events logged with `e_*()` have a log record. `imap_command_finished` +is a debug-level record. + +Disable [[setting,event_exporter_opentelemetry_emit_spans,no]] +together with `emit_logs = yes` to ship log records only. + +When the metric filter does not match an event, no log message is +formatted or transmitted - the wire cost of enabling log records on a +heavy-traffic service is therefore proportional to the matched +volume, not the total event volume. + +The corresponding ResourceLogs carries the same `service.name`, +`service.instance.id`, `service.version` and `host.name` attributes as +ResourceSpans, so collectors group traces and logs under the same +service. + +### Reserved fields + +Event fields whose name begins with `_` are reserved for the stats +process's internal use. `_log_message` and `_log_severity` carry the log +message and severity to the exporter. These fields are not exported by the +`json` and `tab-text` formats or as OpenTelemetry attributes, unless they are +named explicitly in [[setting,metric_fields]]. ## Sampling From 8e0c7d6ebf8f7ae315cfe0f8d541253b3901093b Mon Sep 17 00:00:00 2001 From: Aki Tuomi Date: Wed, 4 Nov 2026 09:21:54 +0000 Subject: [PATCH 4/4] events: Document trace context propagation Add http_client_trace_context and http_server_trace_context_trusted_networks, and the x-trace-id / x-parent-span-id (IMAP ID) and TRACE-ID / PARENT-SPAN-ID (XCLIENT) forwarding fields. Describe in the OpenTelemetry section how the trace id and parent span id are passed between processes and servers. Add an upgrade note: with imap_id_retain, upgrade proxies before backends. --- .github/actions/spelling/allow.txt | 1 + data/settings.js | 39 ++++++++++++++++++- data/updates.js | 1 + docs/core/config/events/export.md | 20 +++++++++- docs/core/config/proxy/forwarding.md | 6 +++ ....4.6-list-imap-id-retain-trace-context.inc | 5 +++ 6 files changed, 70 insertions(+), 2 deletions(-) create mode 100644 docs/installation/upgrade/include/2.4.6-list-imap-id-retain-trace-context.inc diff --git a/.github/actions/spelling/allow.txt b/.github/actions/spelling/allow.txt index 3ffe4fb87..f00e33983 100644 --- a/.github/actions/spelling/allow.txt +++ b/.github/actions/spelling/allow.txt @@ -57,6 +57,7 @@ sls sni spnego ssh +traceparent ubsan ubuntu Uppercases diff --git a/data/settings.js b/data/settings.js index 0bde4a000..b00dbf940 100644 --- a/data/settings.js +++ b/data/settings.js @@ -7155,6 +7155,24 @@ connection before trying the next IP in parallel. \`0\` = wait until current connection attempt finishes.` }, + http_client_trace_context: { + tags: [ 'http', 'http_client' ], + values: setting_types.BOOLEAN, + default: 'no', + added: { + trace_context_added: false, + }, + seealso: [ '[[link,event_export_opentelemetry]]' ], + text: ` +Send a W3C \`traceparent\` header with each request, carrying the trace id +and span id of the event the request was made under. A request that sets its +own \`traceparent\` header keeps it. Requests made under an event without a +trace id or a span id carry no header. + +The header lets the receiving service continue the same trace. It also +reveals the trace id to the server, so enable it only for servers you trust.` + }, + http_client_user_agent: { advanced: true, tags: [ 'http', 'http_client' ], @@ -7281,6 +7299,23 @@ The kernel send buffer size for the connection sockets. \`0\` = kernel defaults.` }, + http_server_trace_context_trusted_networks: { + tags: [ 'http', 'http-server' ], + values: setting_types.BOOLLIST, + added: { + trace_context_added: false, + }, + seealso: [ '[[link,event_export_opentelemetry]]' ], + text: ` +List of IP addresses or network ranges in CIDR notation (e.g. \`192.168.0.0/24\` +or \`2001:db8::/32\`) whose W3C \`traceparent\` request header is accepted. +The request then continues the trace of the header, so the spans of the +request share the trace id of the caller. Requests from other addresses have +the header ignored. By default no network is trusted. + +An invalid network is rejected when the configuration is loaded.` + }, + http_server_rawlog_dir: { changed: { settings_path_types_added: `Setting type changed. A leading \`~/\` in the value is now expanded to the user's home directory.`, @@ -10631,7 +10666,9 @@ the same session to be sampled in or out together. The value \`trace_id\` always uses the trace id of the event; an event field named \`trace_id\` is not looked up. Sampling by \`trace_id\` keeps every event of a -trace together. It also keeps a \`doveadm\` command whole: the command's own +trace together, including events of other systems sharing the trace id (a +\`traceparent\` header from a trusted network, or IMAP \`ID\` / \`XCLIENT\` from a +trusted proxy). It also keeps a \`doveadm\` command whole: the command's own events have no \`session\` field, so sampling by \`session\` drops them. An event without the field, or for \`trace_id\` without a trace id, is dropped diff --git a/data/updates.js b/data/updates.js index c28e6f0ba..70736764a 100644 --- a/data/updates.js +++ b/data/updates.js @@ -125,6 +125,7 @@ export const updates = { setting_sqlite_synchronous_added: '2.4.3', setting_sqlite_busy_timeout_added: '2.4.5', ssl_sni_settings_reload_added: '2.4.0', + trace_context_added: '2.4.6', weak_password_schemes: '2.4.0', variables_auth_variables_protocol: '2.4.0', variables_login_variables_protocol: '2.4.0', diff --git a/docs/core/config/events/export.md b/docs/core/config/events/export.md index ea0f05824..57b145615 100644 --- a/docs/core/config/events/export.md +++ b/docs/core/config/events/export.md @@ -215,6 +215,7 @@ The driver accepts the standard `http_client_*` settings (for example | ------- | ---------------- | | [[setting,http_client_request_timeout]] | `2s` | | [[setting,http_client_request_absolute_timeout]] | `5s` | +| [[setting,http_client_trace_context]] | `no` | [[setting,http_client_request_absolute_timeout]] must not be `0`; the configuration is rejected otherwise. @@ -335,7 +336,24 @@ on, with the span id of the sender as the parent of the receiving span: * login process to the IMAP/POP3/submission process: in the login request, together with the id of the `login_client` span as the parent. * services to the auth process: the `trace_id` and `parent_span_id` - fields of the auth protocol request. Userdb lookups carry them as well. + fields of the auth protocol request. Userdb lookups carry them as well. The + auth process starts a new trace for a request that arrives without one. +* auth process to its worker processes: the worker spans are children of the + `auth_request` span. +* IMAP hibernation: the resumed `imap_client` span is a child of the + hibernated one. +* proxying between Dovecot servers: IMAP `ID` `x-trace-id` / `x-parent-span-id`, + or `XCLIENT` `TRACE-ID` / `PARENT-SPAN-ID`. The values are accepted only + from [[setting,login_trusted_networks]]. See [[link,forwarding_parameters]]. +* HTTP: the W3C `traceparent` header. It is sent when + [[setting,http_client_trace_context,yes]] is set, and accepted by HTTP + servers (such as the doveadm HTTP API) from + [[setting,http_server_trace_context_trusted_networks]]. + +The parent span id sent to the auth process and in the `traceparent` header +follows the same rule: a lookup or request made while an IMAP command runs +(for example the home userdb lookup for a shared namespace) is a child of the +`imap_command` span. This ties the events of one mail session, and the work it causes in other processes, into one trace. The OpenTelemetry exporter does not export events diff --git a/docs/core/config/proxy/forwarding.md b/docs/core/config/proxy/forwarding.md index a9563512e..d0aa29237 100644 --- a/docs/core/config/proxy/forwarding.md +++ b/docs/core/config/proxy/forwarding.md @@ -52,6 +52,8 @@ The parameters are forwarded as part of the ID command field-value list. | `x-connected-name` | TLS SNI provided name. | | `x-proxy-ttl` | TTL which is reduced by each hop, loop prevention. When TTL drops to 0, the connection is dropped. | | `x-session-id`,
`x-session-ext-id` | Session ID to be used. 1 to 64 bytes of printable ASCII without space, `"` or `\`. An invalid value fails the `ID` command. [[changed,login_session_id_validated]] | +| `x-trace-id` | [[added,trace_context_added]] Trace id of the session, 32 hexadecimal characters. See [[link,event_export_opentelemetry]]. | +| `x-parent-span-id` | [[added,trace_context_added]] Span id of the sender's span, 16 hexadecimal characters. The receiver uses it as the parent of its own span. | | `x-forward-` | Forwarded variable, see [[variable]] | ## POP3 @@ -78,6 +80,8 @@ other AUTH error. | `SESSION` | Session ID. 1 to 64 bytes of printable ASCII without space, `"` or `\`. An invalid or empty value fails the `XCLIENT` command. [[changed,login_session_id_validated]] | | `TTL` | TTL which is reduced by each hop, loop prevention. When TTL drops to 0, the connection is dropped. | | `DESTNAME` | Name provided by TLS SNI. | +| `TRACE-ID` | [[added,trace_context_added]] Trace id of the session, 32 hexadecimal characters. See [[link,event_export_opentelemetry]]. | +| `PARENT-SPAN-ID` | [[added,trace_context_added]] Span id of the sender's span, 16 hexadecimal characters. | | `FORWARD` | Base64-encoded, tab-separated list of `key=value` pairs to be forwarded to auth process. The keys and values are escaped using Dovecot's tab-escape format. | ## SMTP/LMTP @@ -99,6 +103,8 @@ See [XCLIENT information](https://www.postfix.org/XCLIENT_README.html). | `PROTO` | Forwarded protocol: `SMTP`, `ESTMP`, or `LMTP`. | | `SESSION` | Session ID. Printable ASCII without space, `"` or `\`. Submission: 1 to 64 bytes. LMTP: any length, because every proxy hop appends `:P` to the ID it forwards. An invalid or empty value is rejected with `501 5.5.4 Invalid SESSION parameter`. [[changed,login_session_id_validated]] | | `DESTNAME` | Name provided by TLS SNI. | +| `TRACE-ID` | [[added,trace_context_added]] Trace id of the session, 32 hexadecimal characters. See [[link,event_export_opentelemetry]]. | +| `PARENT-SPAN-ID` | [[added,trace_context_added]] Span id of the sender's span, 16 hexadecimal characters. | ### Supported Fields (*SMTP/Submission* ONLY) diff --git a/docs/installation/upgrade/include/2.4.6-list-imap-id-retain-trace-context.inc b/docs/installation/upgrade/include/2.4.6-list-imap-id-retain-trace-context.inc new file mode 100644 index 000000000..8a6939cd4 --- /dev/null +++ b/docs/installation/upgrade/include/2.4.6-list-imap-id-retain-trace-context.inc @@ -0,0 +1,5 @@ + * If [[setting,imap_id_retain]] is enabled on the proxies, upgrade them + before the backends. An older proxy forwards the `x-trace-id` and + `x-parent-span-id` fields of a client's `ID` command to the backend inside + its own trusted `ID` command, and the backend uses them as the session's + trace context.