diff --git a/CHANGELOG.md b/CHANGELOG.md index 2fab517..fbcce62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,10 @@ --- develop --- + +* feature: Add poller-driven pipeline health alerts for stalled processing, incoming backlogs, remote synchronization failures or delays, and blocked partition maintenance; expand the existing Status health card with check results and email delivery status, with optional problem, reminder, and recovery notifications +* bug: Treat an empty incoming queue as drained rather than missing data, and evaluate stale queued messages even when new messages continue arriving + * feature: Add an auto-refresh interval dropdown and a manual refresh glyph to the Syslog Status toolbar (mirroring the intropage refresh concept); the interval persists through the page's auto-reload and the refresh glyphs spin while their cards reload * bug: Show the Syslog Status card refresh glyph again by using Font Awesome's fa-sync-alt (the bundled Font Awesome 6 no longer renders the Font Awesome 4 fa-refresh alias) * change: Repaint the Syslog Status card headers with the active theme's native html_start_box title bar (sampled from the live cactiTableTitle at runtime) instead of a flat surface tint so they match Cacti on every theme, widen the card grid from 6 to 8 columns, drop the card corner radius, and give each header one of four translucent severity overlays - a neutral grey "no severity" default plus intropage's good / warning / error palette (#94b36b / #ff8f1e / #cc0000) diff --git a/README.md b/README.md index 265e7f3..1fb4729 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,38 @@ be quieted using syslog's 'Re-Alert' setting. * Ability to run alert specific commands at Alert and Re-alert times +## Pipeline Health Alerts + +The Status tab includes a **Pipeline health** card in the existing Collector +health card position. It reports stalled processing, excessive or old incoming +backlogs, failed or delayed remote synchronization, and blocked partition +maintenance. An empty incoming queue is healthy when processing is completing; +quiet devices do not trigger source-silence alerts. Unavailable queue metrics +are reported as problems rather than healthy values. + +Checks run from Cacti's poller before launching the Syslog processor, so a +stalled processor can still be detected. The delay is the greater of the +Collector Staleness Threshold, three poller intervals, and 300 seconds. New or +re-enabled monitoring gets a startup grace period. Incoming age uses the +configured incoming timestamp field; delayed historical events or incorrect +source clocks can therefore trigger an old-backlog warning. + +In **Settings > Syslog**, select a **Pipeline Health Notification List** from +Thold and/or set **Pipeline Health Email Recipients** to enable problem, +changed-problem, reminder, and recovery emails using Cacti's mail settings. +Enabled lists add their current recipients on each check, preserving BCC +recipients; disabled or missing lists contribute no addresses. Direct email +recipients work without Thold. Email is disabled by default. Persistent problems repeat at the +configured reminder interval, and failed delivery attempts retry no more than +once every five minutes. The card shows the last health check, email attempt +status, and last successful send. Health state changes also appear in Cacti's +log. Intentionally disabled processing does not generate health notifications. + +Monitoring requires a running Cacti poller and an accessible Syslog database. +Use external monitoring for a complete Cacti or database outage. Remote health +checks run on each enabled remote collector; this does not infer a collector +outage merely from a quiet stream of device messages. + ## Table Architecture and Storage Engine Support New installs always create partitioned `syslog` tables. The traditional diff --git a/css/dashboard.css b/css/dashboard.css index 3eca9f7..68bd835 100644 --- a/css/dashboard.css +++ b/css/dashboard.css @@ -86,55 +86,14 @@ align-items: start; } +/* Card surfaces, headers, titles and tools reuse the Status tab classes. */ .syslogDashboardCard { - box-sizing: border-box; position: relative; - border: 1px solid var(--search-border, #cbd8d1); - border-radius: 8px; - background: var(--search-card, #fff); - color: var(--search-text, #23362f); - box-shadow: 0 1px 2px rgba(35, 54, 47, 0.08); - overflow: hidden; -} - -.syslogDashboardCardHeader { - display: flex; - align-items: center; - justify-content: space-between; - gap: 8px; - padding: 8px 12px; - border-bottom: 1px solid var(--search-tint, #e2ede6); - background: var(--search-tint, #e2ede6); -} - -.syslogDashboardCardTitle { - margin: 0; - font-size: 13px; - font-weight: 600; - line-height: 1.4; - overflow-wrap: anywhere; -} - -.syslogDashboardCardHeader .syslogDashboardCardButton { - border: 0; - background: transparent; - color: var(--search-muted, #596d64); - cursor: pointer; - padding: 3px 4px; - border-radius: 4px; - font-size: 12px; -} - -.syslogDashboardCardHeader .syslogDashboardCardButton:hover, -.syslogDashboardCardHeader .syslogDashboardCardButton:focus { - background: var(--search-accent, #276448); - color: #fff; - outline: none; } .syslogChart { min-height: 260px; - padding: 6px 6px 0 0; + padding: 6px 0 0; } /* Bottom-right drag handle; expands the pointer target to the corner. */ @@ -182,7 +141,7 @@ } .syslogDashboardCardStatus { - margin: 40px 12px; + margin: 8px 0; font-size: 12px; color: var(--search-muted, #596d64); } @@ -204,12 +163,8 @@ } /* Drag-and-drop reorder affordances. */ -.syslogDashboardCardHeader { - cursor: grab; -} - .syslogDashboardCardDragging { - opacity: 0.45; + opacity: 0.5; } .syslogDashboardCardDropBefore { @@ -266,7 +221,12 @@ grid-template-columns: 1fr; } + .syslogDashboardCard { + /* Fit saved multi-column panels into the mobile grid. */ + grid-column: 1 / -1 !important; + } + .syslogDashboardPanelActions { margin-left: 0; } -} \ No newline at end of file +} diff --git a/includes/functions.php b/includes/functions.php index 28c67fb..642a10d 100644 --- a/includes/functions.php +++ b/includes/functions.php @@ -1577,6 +1577,7 @@ function syslog_status_get(): array { syslog_status_ensure_table(); $status = [ + 'pipeline_health_monitor' => '', 'last_polling_time' => '', 'last_start_time' => '', 'last_end_time' => '', @@ -1596,6 +1597,8 @@ function syslog_status_get(): array { 'partition_maintenance_last_attempt' => '', 'partition_maintenance_last_success' => '', 'partition_maintenance_outcome' => '', + 'partition_maintenance_blocked' => '', + 'partition_maintenance_reason' => '', 'partition_recovery_progress' => '', 'partition_maintenance_history' => '', 'replication_state' => '', @@ -1607,6 +1610,7 @@ function syslog_status_get(): array { $rows = syslog_db_fetch_assoc("SELECT `name`, `value`, `updated` FROM `$syslogdb_default`.`syslog_status` WHERE `name` IN ( + 'pipeline_health_monitor', 'last_polling_time', 'last_start_time', 'last_end_time', @@ -1626,6 +1630,8 @@ function syslog_status_get(): array { 'partition_maintenance_last_attempt', 'partition_maintenance_last_success', 'partition_maintenance_outcome', + 'partition_maintenance_blocked', + 'partition_maintenance_reason', 'partition_recovery_progress', 'partition_maintenance_history', 'replication_state', @@ -1641,6 +1647,161 @@ function syslog_status_get(): array { return $status; } +/** Read pipeline health without changing state or sending notifications. */ +function syslog_pipeline_health(): array { + global $config, $syslogdb_default, $syslog_incoming_config; + + $status = syslog_status_get(); + $monitor = json_decode($status['pipeline_health_monitor'], true); + $monitor = is_array($monitor) ? $monitor : []; + $health = ['state' => 'up', 'issues' => [], 'checks' => [], 'monitor' => $monitor]; + + if (read_config_option('syslog_enabled') !== 'on' + || ((int) ($config['poller_id'] ?? 1) > 1 && read_config_option('syslog_remote_enabled') !== 'on')) { + $health['state'] = 'disabled'; + return $health; + } + + $now = time(); + $interval = max(60, (int) read_config_option('poller_interval')); + $delay = max($interval * 3, (int) read_config_option('syslog_stale_threshold'), 300); + $limit = (int) read_config_option('syslog_backlog_threshold'); + $limit = $limit > 0 ? $limit : 10000; + $completed = (int) $status['last_end_time']; + $baseline = max($completed, (int) ($monitor['first_checked'] ?? 0)); + if (($monitor['signature'] ?? '') === 'disabled:') { + $baseline = 0; + } + $health['checks']['processing'] = $completed === 0 || $baseline === 0 + ? __('Waiting for a completed processing run.', 'syslog') + : __('Processing is completing within the allowed delay.', 'syslog'); + if ($baseline > 0 && $now - $baseline > $delay) { + $health['issues']['processing'] = __('No processing run has completed for more than %d seconds.', $delay, 'syslog'); + } + + $column = $syslog_incoming_config['timeField'] ?? 'logtime'; + $incoming = []; + if (is_string($column) && preg_match('/^[a-zA-Z_][a-zA-Z0-9_]*$/', $column)) { + $incoming = syslog_db_fetch_row("SELECT COUNT(*) AS pending, UNIX_TIMESTAMP(MIN(`$column`)) AS oldest + FROM `$syslogdb_default`.`syslog_incoming`", false); + } + $health['checks']['backlog'] = __('Incoming messages are within the configured limits.', 'syslog'); + if (!isset($incoming['pending']) || !is_numeric($incoming['pending'])) { + $health['issues']['backlog'] = __('Incoming queue metrics are unavailable.', 'syslog'); + } elseif ((int) $incoming['pending'] > $limit) { + $health['issues']['backlog'] = __('Incoming backlog of %s messages exceeds the limit of %s.', number_format((int) $incoming['pending']), number_format($limit), 'syslog'); + } elseif ((int) $incoming['pending'] > 0) { + if (!isset($incoming['oldest']) || !is_numeric($incoming['oldest'])) { + $health['issues']['backlog'] = __('The oldest incoming message timestamp is unavailable.', 'syslog'); + } elseif ($now - (int) $incoming['oldest'] > $delay) { + $health['issues']['backlog'] = __('Unprocessed messages are older than %d seconds.', $delay, 'syslog'); + } + } + + $replication = syslog_replication_operational_status(); + $health['checks']['replication'] = __('Synchronization is not enabled on this collector.', 'syslog'); + if (!empty($replication['enabled'])) { + $health['checks']['replication'] = __('Synchronization is within the allowed delay.', 'syslog'); + $oldest = strtotime($replication['oldest_pending']); + if ($replication['pending'] === null) { + $health['issues']['replication'] = __('Synchronization queue metrics are unavailable.', 'syslog'); + } elseif ($replication['pending'] > 0 && $oldest === false) { + $health['issues']['replication'] = __('The oldest synchronization record timestamp is unavailable.', 'syslog'); + } elseif ($replication['pending'] > 0 && ($replication['last_error'] !== '' + || ($oldest !== false && $now - $oldest > $delay))) { + $health['issues']['replication'] = __('Synchronization has failed or queued records exceed the allowed delay (%s pending).', number_format($replication['pending']), 'syslog'); + } + } + + $health['checks']['partition'] = __('Partition maintenance is not blocked.', 'syslog'); + if ((int) ($status['partition_maintenance_blocked'] ?? 0) === 1) { + $health['issues']['partition'] = __('Partition maintenance is blocked: %s', $status['partition_maintenance_reason'] ?? '', 'syslog'); + } + + if ($health['issues']) { + $health['state'] = 'down'; + } elseif ($completed === 0 || $baseline === 0) { + $health['state'] = 'recovering'; + } + return $health; +} + +/** Check independently of the Syslog processor; serialize notifications per collector. */ +function syslog_pipeline_health_check(): void { + global $config, $syslogdb_default; + + $lock = 'syslog_health_' . sha1($syslogdb_default . ':' . (int) ($config['poller_id'] ?? 1)); + if ((int) syslog_db_fetch_cell_prepared('SELECT GET_LOCK(?, 0)', [$lock], false) !== 1) { + return; + } + + try { + $health = syslog_pipeline_health(); + $monitor = $health['monitor']; + $now = time(); + $keys = array_keys($health['issues']); + sort($keys); + $signature = $health['state'] . ':' . implode(',', $keys); + $prior = $monitor['signature'] ?? ''; + $monitor['first_checked'] = $prior === 'disabled:' ? $now : ($monitor['first_checked'] ?? $now); + $monitor['checked'] = $now; + $monitor['signature'] = $signature; + + if ($signature !== $prior && $health['state'] !== 'disabled' && ($health['issues'] || str_starts_with($prior, 'down:'))) { + cacti_log('SYSLOG PIPELINE: ' . ($health['issues'] ? implode(' ', $health['issues']) : 'Pipeline recovered'), false, 'SYSLOG'); + } + + $recipients = trim((string) read_config_option('syslog_pipeline_health_email'), ' ,'); + $bcc = ''; + $list_id = (int) read_config_option('syslog_pipeline_health_notify'); + if ($list_id > 0 && db_table_exists('plugin_notification_lists')) { + $bcc_column = db_column_exists('plugin_notification_lists', 'bcc_emails') ? 'bcc_emails' : "'' AS bcc_emails"; + $list = db_fetch_row_prepared("SELECT emails, $bcc_column FROM plugin_notification_lists WHERE id = ? AND enabled = ?", [$list_id, 'on']); + $recipients = implode(', ', array_filter([$recipients, trim((string) ($list['emails'] ?? ''), ' ,')])); + $bcc = trim((string) ($list['bcc_emails'] ?? ''), ' ,'); + } + if (($recipients === '' && $bcc === '') || $health['state'] === 'disabled') { + $monitor['notification'] = 'disabled'; + unset($monitor['notified_signature']); + } else { + $notified = $monitor['notified_signature'] ?? ''; + $repeat = max(300, (int) (read_config_option('syslog_pipeline_health_repeat') ?: 3600)); + $due = ($health['issues'] && ($signature !== $notified || $now - (int) ($monitor['last_notified'] ?? 0) >= $repeat)) + || (!$health['issues'] && str_starts_with($notified, 'down:')); + // Bound failed mail retries even if poller hooks overlap or run more often. + if ($due && $now - (int) ($monitor['last_attempt'] ?? 0) >= 300) { + $monitor['last_attempt'] = $now; + $monitor['notification'] = 'pending'; + if (!syslog_status_set('pipeline_health_monitor', (string) json_encode($monitor))) { + return; + } + $subject = __('Syslog pipeline health - collector %d', (int) ($config['poller_id'] ?? 1), 'syslog'); + $message = $health['issues'] ? implode("\n", $health['issues']) : __('The Syslog pipeline has recovered.', 'syslog'); + $from = [(string) (read_config_option('settings_from_email') ?: 'Cacti@cacti.net'), (string) (read_config_option('settings_from_name') ?: 'Cacti')]; + try { + $error = mailer($from, $recipients, '', $bcc, '', $subject, '', $message, '', '', false); + $monitor['notification'] = $error === '' ? 'sent' : 'failed'; + } catch (Throwable $error) { + $monitor['notification'] = 'failed'; + } + if ($monitor['notification'] === 'sent') { + $monitor['notified_signature'] = $signature; + $monitor['last_notified'] = $now; + } else { + cacti_log('SYSLOG PIPELINE: Health notification failed; retrying on a later poller run', false, 'SYSLOG'); + } + } elseif ($due && ($monitor['notification'] ?? '') !== 'failed') { + $monitor['notification'] = 'pending'; + } elseif (!isset($monitor['notification']) || $monitor['notification'] === 'disabled') { + $monitor['notification'] = 'ready'; + } + } + syslog_status_set('pipeline_health_monitor', (string) json_encode($monitor)); + } finally { + syslog_db_fetch_cell_prepared('SELECT RELEASE_LOCK(?)', [$lock], false); + } +} + /** * syslog_worker_stats_get - Collect the parallel worker statistics shown * on the Syslog Status tab. diff --git a/includes/processing.php b/includes/processing.php index d645c3d..51b7789 100644 --- a/includes/processing.php +++ b/includes/processing.php @@ -78,6 +78,7 @@ function syslog_poller_bottom(): void { if (syslog_config_safe()) { syslog_connect(); + syslog_pipeline_health_check(); syslog_status_set('last_polling_time', time()); $command_string = (string) read_config_option('path_php_binary'); diff --git a/includes/settings.php b/includes/settings.php index bf1a423..2b6fe67 100644 --- a/includes/settings.php +++ b/includes/settings.php @@ -52,6 +52,13 @@ function syslog_config_settings(): void { $tabs['syslog'] = __('Syslog', 'syslog'); + $notification_lists = [0 => __('None', 'syslog')]; + if (db_table_exists('plugin_notification_lists')) { + foreach (db_fetch_assoc('SELECT id, name FROM plugin_notification_lists ORDER BY name') as $list) { + $notification_lists[(int) $list['id']] = $list['name']; + } + } + $temp = [ 'syslog_header' => [ 'friendly_name' => __('General Settings', 'syslog'), @@ -191,7 +198,7 @@ function syslog_config_settings(): void { ], 'syslog_stale_threshold' => [ 'friendly_name' => __('Collector Staleness Threshold', 'syslog'), - 'description' => __('The Syslog Status page shows a warning when no new message has arrived within this many seconds. Set to 0 to use the default of 300 seconds.', 'syslog'), + 'description' => __('Pipeline health alerts warn when unprocessed messages, processing completion, or queued synchronization exceed this delay. The effective delay is at least three poller intervals and 300 seconds. Set to 0 to use 300 seconds.', 'syslog'), 'method' => 'drop_array', 'default' => '300', 'array' => [ @@ -216,6 +223,37 @@ function syslog_config_settings(): void { '50000' => __('%d Messages', 50000, 'syslog') ] ], + 'syslog_pipeline_health_header' => [ + 'friendly_name' => __('Pipeline Health Alerts', 'syslog'), + 'method' => 'spacer', + ], + 'syslog_pipeline_health_notify' => [ + 'friendly_name' => __('Pipeline Health Notification List', 'syslog'), + 'description' => __('Select an existing Thold notification list for pipeline health alerts. Enabled list recipients are combined with the email addresses below, preserving BCC recipients. Lists require the Thold notification-list table.', 'syslog'), + 'method' => 'drop_array', + 'array' => $notification_lists, + 'default' => '0' + ], + 'syslog_pipeline_health_email' => [ + 'friendly_name' => __('Pipeline Health Email Recipients', 'syslog'), + 'description' => __('Comma-separated email recipients for pipeline failures and recovery. Leave blank and select no notification list to disable email; health remains visible on the Status tab and state changes are logged. Checks require the Cacti poller to run.', 'syslog'), + 'method' => 'textbox', + 'size' => 80, + 'max_length' => 255, + 'default' => '' + ], + 'syslog_pipeline_health_repeat' => [ + 'friendly_name' => __('Pipeline Health Reminder Interval', 'syslog'), + 'description' => __('Repeat notifications for ongoing pipeline problems after this interval. Failed email attempts are retried no more than once every five minutes.', 'syslog'), + 'method' => 'drop_array', + 'default' => '3600', + 'array' => [ + '900' => __('%d minutes', 15, 'syslog'), + '3600' => __('%d hours', 1, 'syslog'), + '21600' => __('%d hours', 6, 'syslog'), + '86400' => __('%d days', 1, 'syslog') + ] + ], 'syslog_alert_retention' => [ 'friendly_name' => __('Syslog Alert Retention', 'syslog'), 'description' => __('This is the number of days to keep alert logs.', 'syslog'), diff --git a/js/dashboard.js b/js/dashboard.js index bc17892..bf929eb 100644 --- a/js/dashboard.js +++ b/js/dashboard.js @@ -50,35 +50,47 @@ function syslogDashboardLoad(data) { function syslogDashboardPanelCard(panel) { var card = document.createElement('article'); - card.className = 'syslogDashboardCard'; + card.className = 'syslogDashboardCard syslogStatusCard'; card.id = 'syslog_panel_' + panel.id; var header = document.createElement('header'); - header.className = 'syslogDashboardCardHeader'; - if (syslogDashboard.canManage) header.setAttribute('draggable', 'true'); + header.className = 'syslogDashboardCardHeader syslogStatusCardHeader syslogStatusCardHeader--none'; + if (syslogDashboard.canManage) { + var handle = document.createElement('button'); + handle.type = 'button'; + handle.className = 'syslogStatusCardDrag'; + handle.setAttribute('aria-label', syslogDashboard.text.dragPanel); + handle.innerHTML = ''; + header.append(handle); + } var title = document.createElement('h3'); - title.className = 'syslogDashboardCardTitle'; + title.className = 'syslogDashboardCardTitle syslogStatusCardTitle'; title.textContent = panel.title; header.append(title); var actions = document.createElement('div'); + actions.className = 'syslogStatusCardTools'; actionButtons().forEach(function(button) { actions.append(button); }); header.append(actions); card.append(header); + var body = document.createElement('div'); + body.className = 'syslogStatusCardBody'; + card.append(body); + var chart = document.createElement('div'); chart.className = 'syslogChart'; chart.id = 'syslog_chart_' + panel.id; - card.append(chart); + body.append(chart); var status = document.createElement('p'); status.className = 'syslogDashboardCardStatus'; status.setAttribute('role', 'status'); status.textContent = '...'; - card.append(status); + body.append(status); // Shared dashboards are view-only: no resize or drag affordances. if (syslogDashboard.canManage) { @@ -223,7 +235,7 @@ function actionButtons() { actions.forEach(function(entry) { var button = document.createElement('button'); button.type = 'button'; - button.className = 'syslogDashboardCardButton'; + button.className = 'syslogDashboardCardButton syslogStatusCardTool'; button.dataset.action = entry[0]; button.innerHTML = ''; button.title = entry[2]; @@ -337,7 +349,7 @@ function syslogDashboardRenderGrid() { /* ===================== Drag-and-drop reorder ===================== */ /** - * Make a card reorderable: dragging its header shows a placeholder, and + * Make a card reorderable: dragging its handle shows a placeholder, and * dropping posts the new 1-based position. Drop targets are the card * midpoints, so moving a panel underneath another works in both * directions without up/down buttons. @@ -345,19 +357,45 @@ function syslogDashboardRenderGrid() { function syslogDashboardInitDrag(card, panel) { if (!syslogDashboard.canManage) return; - var header = card.querySelector('.syslogDashboardCardHeader'); - if (!header) return; + var handle = card.querySelector('.syslogStatusCardDrag'); + if (!handle) return; + + var disarm = function() { card.draggable = false; }; + handle.addEventListener('mousedown', function() { + card.draggable = true; + document.addEventListener('mouseup', disarm, {once: true}); + }); + handle.addEventListener('touchstart', function() { + card.draggable = true; + document.addEventListener('touchend', disarm, {once: true}); + document.addEventListener('touchcancel', disarm, {once: true}); + }, {passive: true}); + handle.addEventListener('keydown', function(event) { + var direction = ['ArrowLeft', 'ArrowUp'].includes(event.key) ? -1 + : (['ArrowRight', 'ArrowDown'].includes(event.key) ? 1 : 0); + if (!direction) return; + event.preventDefault(); + var order = syslogDashboard.panels.map(function(item) { return item.id; }); + var from = order.indexOf(panel.id); + var to = from + direction; + if (from < 0 || to < 0 || to >= order.length) return; + order.splice(from, 1); + order.splice(to, 0, panel.id); + syslogDashboardApplyOrder(order, panel.id, to + 1); + handle.focus(); + }); - header.addEventListener('dragstart', function(event) { + card.addEventListener('dragstart', function(event) { event.dataTransfer.effectAllowed = 'move'; event.dataTransfer.setData('text/plain', String(panel.id)); card.classList.add('syslogDashboardCardDragging'); }); - header.addEventListener('dragend', function() { + card.addEventListener('dragend', function() { + card.draggable = false; card.classList.remove('syslogDashboardCardDragging'); - document.querySelectorAll('.syslogDashboardCardDropBefore') - .forEach(function(el) { el.classList.remove('syslogDashboardCardDropBefore'); }); + document.querySelectorAll('.syslogDashboardCardDropBefore, .syslogDashboardCardDropAfter') + .forEach(function(el) { el.classList.remove('syslogDashboardCardDropBefore', 'syslogDashboardCardDropAfter'); }); }); card.addEventListener('dragover', function(event) { @@ -366,7 +404,7 @@ function syslogDashboardInitDrag(card, panel) { event.dataTransfer.dropEffect = 'move'; var rect = card.getBoundingClientRect(); - var before = event.clientY < rect.top + rect.height / 2; + var before = event.clientY < rect.top || (event.clientY <= rect.bottom && event.clientX < rect.left + rect.width / 2); card.classList.toggle('syslogDashboardCardDropBefore', before); card.classList.toggle('syslogDashboardCardDropAfter', !before); }); @@ -390,7 +428,7 @@ function syslogDashboardInitDrag(card, panel) { if (draggedId === panel.id) return; var rect = card.getBoundingClientRect(); - var before = event.clientY < rect.top + rect.height / 2; + var before = event.clientY < rect.top || (event.clientY <= rect.bottom && event.clientX < rect.left + rect.width / 2); var panels = syslogDashboard.panels || []; var order = panels.map(function(p) { return p.id; }); var from = order.indexOf(draggedId); diff --git a/lib/syslog_dashboard.php b/lib/syslog_dashboard.php index 41d6532..35fdf9f 100644 --- a/lib/syslog_dashboard.php +++ b/lib/syslog_dashboard.php @@ -1363,6 +1363,7 @@ function syslog_dashboard(): void { canShare: , shared: , text: { + dragPanel: , refreshPanel: , totalMessages: , libraryUnavailable: , diff --git a/locales/po/cacti.pot b/locales/po/cacti.pot index 58a080f..fd5d930 100644 --- a/locales/po/cacti.pot +++ b/locales/po/cacti.pot @@ -8,7 +8,7 @@ msgid "" msgstr "" "Project-Id-Version: Cacti 1.2.32\n" "Report-Msgid-Bugs-To: developers@cacti.net\n" -"POT-Creation-Date: 2026-10-10 09:25-0400\n" +"POT-Creation-Date: 2026-10-12 16:41-0400\n" "PO-Revision-Date: YEAR-MO-DA HO:MI+ZONE\n" "Last-Translator: Cacti Developers >\n" "Language-Team: Cacti Developers \n" @@ -81,6 +81,80 @@ msgstr "" msgid "The Syslog tables are not partitioned. Traditional (non-partitioned) tables are deprecated, and partitioned tables are required for new installs. Your data will continue to be collected, but you should migrate the syslog table to a partitioned architecture." msgstr "" +#: includes/functions.php +msgid "Waiting for a completed processing run." +msgstr "" + +#: includes/functions.php +msgid "Processing is completing within the allowed delay." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "No processing run has completed for more than %d seconds." +msgstr "" + +#: includes/functions.php +msgid "Incoming messages are within the configured limits." +msgstr "" + +#: includes/functions.php +msgid "Incoming queue metrics are unavailable." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Incoming backlog of %s messages exceeds the limit of %s." +msgstr "" + +#: includes/functions.php +msgid "The oldest incoming message timestamp is unavailable." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Unprocessed messages are older than %d seconds." +msgstr "" + +#: includes/functions.php +msgid "Synchronization is not enabled on this collector." +msgstr "" + +#: includes/functions.php +msgid "Synchronization is within the allowed delay." +msgstr "" + +#: includes/functions.php +msgid "Synchronization queue metrics are unavailable." +msgstr "" + +#: includes/functions.php +msgid "The oldest synchronization record timestamp is unavailable." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Synchronization has failed or queued records exceed the allowed delay (%s pending)." +msgstr "" + +#: includes/functions.php +msgid "Partition maintenance is not blocked." +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Partition maintenance is blocked: %s" +msgstr "" + +#: includes/functions.php +#, php-format +msgid "Syslog pipeline health - collector %d" +msgstr "" + +#: includes/functions.php +msgid "The Syslog pipeline has recovered." +msgstr "" + #: includes/functions.php #, php-format msgid "Partition layout for %s is invalid or incomplete; partition maintenance stopped. Verify the partition metadata with SHOW CREATE TABLE and, if required, rebuild the partitions; writes continue into the dMaxValue safety partition." @@ -537,11 +611,16 @@ msgstr "" msgid "Every day" msgstr "" -#: includes/settings.php syslog_alerts.php syslog_device_rules.php +#: includes/settings.php syslog.php syslog_alerts.php syslog_device_rules.php #: syslog_removal.php syslog_reports.php msgid "Disabled" msgstr "" +#: includes/settings.php lib/syslog_dashboard.php syslog.php syslog_alerts.php +#: syslog_device_rules.php syslog_reports.php +msgid "None" +msgstr "" + #: includes/settings.php msgid "General Settings" msgstr "" @@ -702,7 +781,7 @@ msgid "Collector Staleness Threshold" msgstr "" #: includes/settings.php -msgid "The Syslog Status page shows a warning when no new message has arrived within this many seconds. Set to 0 to use the default of 300 seconds." +msgid "Pipeline health alerts warn when unprocessed messages, processing completion, or queued synchronization exceed this delay. The effective delay is at least three poller intervals and 300 seconds. Set to 0 to use 300 seconds." msgstr "" #: includes/settings.php @@ -731,6 +810,49 @@ msgstr "" msgid "%d Messages" msgstr "" +#: includes/settings.php +msgid "Pipeline Health Alerts" +msgstr "" + +#: includes/settings.php +msgid "Pipeline Health Notification List" +msgstr "" + +#: includes/settings.php +msgid "Select an existing Thold notification list for pipeline health alerts. Enabled list recipients are combined with the email addresses below, preserving BCC recipients. Lists require the Thold notification-list table." +msgstr "" + +#: includes/settings.php +msgid "Pipeline Health Email Recipients" +msgstr "" + +#: includes/settings.php +msgid "Comma-separated email recipients for pipeline failures and recovery. Leave blank and select no notification list to disable email; health remains visible on the Status tab and state changes are logged. Checks require the Cacti poller to run." +msgstr "" + +#: includes/settings.php +msgid "Pipeline Health Reminder Interval" +msgstr "" + +#: includes/settings.php +msgid "Repeat notifications for ongoing pipeline problems after this interval. Failed email attempts are retried no more than once every five minutes." +msgstr "" + +#: includes/settings.php +#, php-format +msgid "%d minutes" +msgstr "" + +#: includes/settings.php +#, php-format +msgid "%d hours" +msgstr "" + +#: includes/settings.php +#, php-format +msgid "%d days" +msgstr "" + #: includes/settings.php msgid "Syslog Alert Retention" msgstr "" @@ -1079,6 +1201,10 @@ msgstr "" msgid "A title of up to 128 characters is required." msgstr "" +#: lib/syslog_dashboard.php syslog.php +msgid "Drag to reorder card" +msgstr "" + #: lib/syslog_dashboard.php msgid "Refresh Panel" msgstr "" @@ -1322,11 +1448,6 @@ msgstr "" msgid "Start from Saved Search" msgstr "" -#: lib/syslog_dashboard.php syslog.php syslog_alerts.php -#: syslog_device_rules.php syslog_reports.php -msgid "None" -msgstr "" - #: lib/syslog_dashboard.php syslog.php syslog_saved_searches.php msgid "Enter message text…" msgstr "" @@ -1403,7 +1524,7 @@ msgstr "" #: syslog.php #, php-format -msgid "no message received for %s" +msgid "unprocessed messages are %s old" msgstr "" #: syslog.php @@ -1411,10 +1532,6 @@ msgstr "" msgid "incoming backlog of %s messages" msgstr "" -#: syslog.php -msgid "no incoming data is available to evaluate" -msgstr "" - #: syslog.php msgid "day" msgstr "" @@ -1500,7 +1617,7 @@ msgid "Recent partition maintenance activity" msgstr "" #: syslog.php -msgid "Collector health" +msgid "Pipeline health" msgstr "" #: syslog.php @@ -1677,23 +1794,83 @@ msgid "No action required." msgstr "" #: syslog.php -msgid "Last received log" +msgid "Incoming backlog" msgstr "" #: syslog.php -msgid "Oldest unprocessed message" +msgid "Processing" msgstr "" #: syslog.php -msgid "Incoming backlog" +msgid "Partition maintenance" msgstr "" #: syslog.php -msgid "Records processed in latest run" +msgid "Synchronization" +msgstr "" + +#: syslog.php +msgid "Attention required" +msgstr "" + +#: syslog.php +msgid "Healthy" +msgstr "" + +#: syslog.php +msgid "Processing disabled" +msgstr "" + +#: syslog.php +msgid "Waiting for processing" +msgstr "" + +#: syslog.php +msgid "Health" msgstr "" #: syslog.php -msgid "Collector warning" +msgid "Ready" +msgstr "" + +#: syslog.php +msgid "Sent" +msgstr "" + +#: syslog.php +msgid "Failed; retry pending" +msgstr "" + +#: syslog.php +msgid "Pending; next poller check" +msgstr "" + +#: syslog.php +msgid "Last health check" +msgstr "" + +#: syslog.php +msgid "Email notifications" +msgstr "" + +#: syslog.php +msgid "Not checked yet" +msgstr "" + +#: syslog.php +msgid "Last notification sent" +msgstr "" + +#: syslog.php +msgid "Last received log" +msgstr "" + +#: syslog.php +msgid "Oldest unprocessed message" +msgstr "" + +#: syslog.php +msgid "Records processed in latest run" msgstr "" #: syslog.php @@ -1736,10 +1913,6 @@ msgstr "" msgid "Records" msgstr "" -#: syslog.php -msgid "Partition maintenance" -msgstr "" - #: syslog.php msgid "Reference updates" msgstr "" @@ -1808,10 +1981,6 @@ msgstr "" msgid "Remove from page" msgstr "" -#: syslog.php -msgid "Drag to reorder card" -msgstr "" - #: syslog.php msgid "Unknown card." msgstr "" diff --git a/syslog.php b/syslog.php index c2308a6..368b69f 100644 --- a/syslog.php +++ b/syslog.php @@ -422,14 +422,13 @@ function syslog_status_collector_health(): array { FROM `$syslogdb_default`.`syslog_incoming`"); $last_received = __('Unavailable', 'syslog'); - $seconds_since = null; + $oldest_seconds = null; if (is_string($last_received_raw) && $last_received_raw !== '' && $last_received_raw !== 'NULL') { $epoch = strtotime($last_received_raw); if ($epoch !== false) { $last_received = date('Y-m-d H:i:s', $epoch); - $seconds_since = time() - $epoch; } } @@ -446,6 +445,7 @@ function syslog_status_collector_health(): array { $age = time() - $oldest_epoch; if ($age >= 0) { + $oldest_seconds = $age; $oldest_age = syslog_status_format_age($age); } } @@ -476,19 +476,14 @@ function syslog_status_collector_health(): array { $backlog_threshold = SYSLOG_COLLECTOR_BACKLOG_THRESHOLD; } - if ($seconds_since !== null && $seconds_since > $stale_threshold) { + if (is_numeric($backlog_raw) && (int) $backlog_raw > 0 && $oldest_seconds !== null && $oldest_seconds > $stale_threshold) { $warning = true; - $warn_parts[] = sprintf(__('no message received for %s', 'syslog'), syslog_status_format_age($seconds_since)); + $warn_parts[] = __('unprocessed messages are %s old', syslog_status_format_age($oldest_seconds), 'syslog'); } if (is_numeric($backlog_raw) && (int) $backlog_raw > $backlog_threshold) { $warning = true; - $warn_parts[] = sprintf(__('incoming backlog of %s messages', 'syslog'), number_format((int) $backlog_raw)); - } - - if (is_numeric($backlog_raw) && (int) $backlog_raw === 0 && $seconds_since === null) { - $warning = true; - $warn_parts[] = __('no incoming data is available to evaluate', 'syslog'); + $warn_parts[] = __('incoming backlog of %s messages', number_format((int) $backlog_raw), 'syslog'); } return [ @@ -659,7 +654,7 @@ function syslog_status_card_meta(): array { 'partition_health' => ['title' => __('Partition health', 'syslog'), 'span' => 4, 'expandable' => false], 'maintenance' => ['title' => __('Partition maintenance activity', 'syslog'), 'span' => 2, 'expandable' => false], 'maintenance_recent' => ['title' => __('Recent partition maintenance activity', 'syslog'), 'span' => 4, 'expandable' => true], - 'collector_health' => ['title' => __('Collector health', 'syslog'), 'span' => 2, 'expandable' => false], + 'collector_health' => ['title' => __('Pipeline health', 'syslog'), 'span' => 4, 'expandable' => false, 'state' => syslog_status_pipeline()['state']], 'workers' => ['title' => __('Parallel workers', 'syslog'), 'span' => 2, 'expandable' => false], 'worker_processes' => ['title' => __('Worker processes', 'syslog'), 'span' => 2, 'expandable' => true], 'phases' => ['title' => __('Processing phases', 'syslog'), 'span' => 4, 'expandable' => true], @@ -741,6 +736,12 @@ function syslog_status_replication(): array { return $replication; } +/** Share the same live health snapshot between the card header and body. */ +function syslog_status_pipeline(): array { + static $health = null; + return $health ??= syslog_pipeline_health(); +} + /** * Render the inner body HTML of a single Status card. * @@ -842,6 +843,29 @@ function syslog_status_card_body(string $key): string { break; case 'collector_health': + $pipeline = syslog_status_pipeline(); + $labels = [ + 'processing' => __('Processing', 'syslog'), 'backlog' => __('Incoming backlog', 'syslog'), + 'replication' => __('Synchronization', 'syslog'), 'partition' => __('Partition maintenance', 'syslog') + ]; + print ''; + $states = [ + 'up' => __('Healthy', 'syslog'), 'down' => __('Attention required', 'syslog'), + 'disabled' => __('Processing disabled', 'syslog'), 'recovering' => __('Waiting for processing', 'syslog') + ]; + $kv(__('Health', 'syslog'), $states[$pipeline['state']]); + foreach ($pipeline['checks'] as $check => $detail) { + $kv($labels[$check], $pipeline['issues'][$check] ?? $detail); + } + $monitor = $pipeline['monitor']; + $notifications = [ + 'disabled' => __('Disabled', 'syslog'), 'ready' => __('Ready', 'syslog'), 'sent' => __('Sent', 'syslog'), + 'failed' => __('Failed; retry pending', 'syslog'), 'pending' => __('Pending; next poller check', 'syslog') + ]; + $kv(__('Last health check', 'syslog'), syslog_status_format_time((string) ($monitor['checked'] ?? ''))); + $kv(__('Email notifications', 'syslog'), $notifications[$monitor['notification'] ?? ''] ?? __('Not checked yet', 'syslog')); + $kv(__('Last notification sent', 'syslog'), syslog_status_format_time((string) ($monitor['last_notified'] ?? ''))); + print '
'; $health = syslog_status_collector_health(); print ''; $kv(__('Last received log', 'syslog'), (string) $health['last_received']); @@ -849,9 +873,6 @@ function syslog_status_card_body(string $key): string { $kv(__('Incoming backlog', 'syslog'), (string) $health['backlog']); $kv(__('Records processed in latest run', 'syslog'), (string) $health['processed']); print '
'; - if ($health['warning']) { - print '

' . __esc('Collector warning', 'syslog') . ': ' . html_escape($health['warning_text']) . '

'; - } break; case 'workers': diff --git a/tests/Unit/CollectorHealthTest.php b/tests/Unit/CollectorHealthTest.php index 05f0b83..3cde9ec 100644 --- a/tests/Unit/CollectorHealthTest.php +++ b/tests/Unit/CollectorHealthTest.php @@ -69,6 +69,7 @@ function collector_health_load_functions(): void { } function collector_health_setup_config(): void { + syslog_test_reset_globals(); $GLOBALS['syslogdb_default'] = 'syslogdb'; $GLOBALS['syslog_incoming_config'] = [ 'timeField' => 'logtime', @@ -268,3 +269,22 @@ function collector_health_capture_db(array &$calls, array $overrides = []): void expect(syslog_status_format_age(5400))->toContain('hour'); expect(syslog_status_format_age(90000))->toContain('day'); }); + + +it('does not warn for an empty queue after processing drains incoming messages', function () { + syslog_load_plugin_source('includes/functions.php'); + collector_health_load_functions(); + collector_health_setup_config(); + $calls = []; + collector_health_capture_db($calls, ['count' => '0']); + expect(syslog_status_collector_health()['warning'])->toBeFalse(); +}); + +it('warns about old queued messages even while new messages arrive', function () { + syslog_load_plugin_source('includes/functions.php'); + collector_health_load_functions(); + collector_health_setup_config(); + $calls = []; + collector_health_capture_db($calls, ['max' => date('Y-m-d H:i:s'), 'min' => date('Y-m-d H:i:s', time() - 600), 'count' => '2']); + expect(syslog_status_collector_health()['warning'])->toBeTrue(); +}); diff --git a/tests/Unit/DashboardDefaultTest.php b/tests/Unit/DashboardDefaultTest.php index 18e531c..0ce24b5 100644 --- a/tests/Unit/DashboardDefaultTest.php +++ b/tests/Unit/DashboardDefaultTest.php @@ -2,12 +2,10 @@ it('seeds valid starter panels atomically and does not reseed an existing default', function () { syslog_load_plugin_source('setup.php'); + syslog_load_plugin_source('includes/schema.php'); syslog_load_plugin_source('lib/syslog_dashboard.php'); - test_override('syslog_ensure_dashboard_tables', fn () => null); - test_override('syslog_db_table_exists', fn () => true); - test_override('syslog_db_column_exists', fn () => true); + test_override('db_update_table', fn () => true); $GLOBALS['syslogdb_default'] = 'syslog'; - test_override('syslog_dashboard_username', fn () => 'admin'); test_override('syslog_db_fetch_insert_id', fn () => 42); $existing = false; $calls = []; diff --git a/tests/Unit/DashboardRenderTest.php b/tests/Unit/DashboardRenderTest.php new file mode 100644 index 0000000..81aa1e5 --- /dev/null +++ b/tests/Unit/DashboardRenderTest.php @@ -0,0 +1,26 @@ + 'Never']; + $GLOBALS['request']['refresh'] = 0; + $GLOBALS['request']['dashboard_id'] = 0; + test_override('syslog_db_fetch_assoc_prepared', fn () => []); + test_override('syslog_db_fetch_assoc', fn () => []); + test_override('syslog_db_fetch_row_prepared', fn () => []); + + ob_start(); + try { + syslog_dashboard(); + $html = ob_get_contents(); + } finally { + ob_end_clean(); + } + + expect($html)->toContain('dragPanel: "Drag to reorder card"') + ->toContain('refreshPanel: "Refresh Panel"') + ->toContain("id='syslog_dashboard_grid'"); +}); diff --git a/tests/Unit/PipelineHealthTest.php b/tests/Unit/PipelineHealthTest.php new file mode 100644 index 0000000..bada38a --- /dev/null +++ b/tests/Unit/PipelineHealthTest.php @@ -0,0 +1,384 @@ + 'received_at']; + $GLOBALS['pipeline_fixture'] = [ + 'settings' => ['syslog_enabled' => 'on', 'poller_interval' => '300'], + 'status' => ['last_end_time' => (string) time()], + 'incoming' => ['pending' => '0', 'oldest' => null], + 'outbox' => ['pending' => '0', 'oldest_pending' => ''], + 'mails' => [], 'locks' => [], 'queries' => [], 'logs' => [], 'mail_error' => '', 'lock' => 1, + ]; + test_override('read_config_option', fn ($key) => $GLOBALS['pipeline_fixture']['settings'][$key] ?? ''); + test_override('syslog_db_table_exists', fn () => true); + test_override('syslog_db_fetch_assoc', function ($sql) { + $GLOBALS['pipeline_fixture']['queries'][] = $sql; + $rows = []; + foreach ($GLOBALS['pipeline_fixture']['status'] as $key => $value) { + $rows[] = ['name' => $key, 'value' => $value, 'updated' => time()]; + } + return $rows; + }); + test_override('syslog_db_fetch_row', function ($sql) { + $GLOBALS['pipeline_fixture']['queries'][] = $sql; + if (str_contains($sql, 'syslog_incoming')) return $GLOBALS['pipeline_fixture']['incoming']; + if (str_contains($sql, 'syslog_replication_output')) return $GLOBALS['pipeline_fixture']['outbox']; + return []; + }); + test_override('syslog_db_fetch_cell_prepared', function ($sql, $params) { + $GLOBALS['pipeline_fixture']['locks'][] = [$sql, $params]; + return $GLOBALS['pipeline_fixture']['lock']; + }); + test_override('syslog_db_execute_prepared', function ($sql, $params) { + if ($params[0] === 'pipeline_health_monitor') { + if (!empty($GLOBALS['pipeline_fixture']['write_failed'])) return false; + $GLOBALS['pipeline_fixture']['status'][$params[0]] = $params[1]; + } + return true; + }); + test_override('mailer', function (...$args) { + $GLOBALS['pipeline_fixture']['mails'][] = $args; + return $GLOBALS['pipeline_fixture']['mail_error']; + }); + test_override('cacti_log', function ($text) { $GLOBALS['pipeline_fixture']['logs'][] = $text; }); +} + +it('treats an empty drained queue and recent processing as healthy without writes', function () { + pipeline_health_fixture(); + $health = syslog_pipeline_health(); + expect($health['state'])->toBe('up')->and($health['issues'])->toBeEmpty() + ->and($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty() + ->and($GLOBALS['__test_db_calls'])->toBeEmpty(); + expect(implode(' ', $GLOBALS['pipeline_fixture']['queries']))->toContain('MIN(`received_at`)') + ->toContain('`health_test`.`syslog_incoming`')->toContain("'partition_maintenance_blocked'"); +}); + +it('detects stalled processing even with an empty queue', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['status']['last_end_time'] = (string) (time() - 1000); + expect(syslog_pipeline_health()['issues'])->toHaveKey('processing'); +}); + +it('allows three poller intervals before declaring delayed processing', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['poller_interval'] = '600'; + $GLOBALS['pipeline_fixture']['status']['last_end_time'] = (string) (time() - 1000); + expect(syslog_pipeline_health()['issues'])->toBeEmpty(); +}); + +it('gives a new processor a grace period but detects failure to ever complete', function () { + pipeline_health_fixture(); + unset($GLOBALS['pipeline_fixture']['status']['last_end_time']); + syslog_pipeline_health_check(); + expect(syslog_pipeline_health()['issues'])->toBeEmpty(); + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode(['first_checked' => time() - 1000]); + expect(syslog_pipeline_health()['issues'])->toHaveKey('processing'); +}); + +it('detects old queued events even when fresh messages are arriving', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 10, 'oldest' => time() - 1000]; + expect(syslog_pipeline_health()['issues'])->toHaveKey('backlog'); +}); + +it('reports queued events when the oldest message timestamp is unavailable', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 10, 'oldest' => null]; + expect(syslog_pipeline_health()['issues']['backlog'])->toContain('timestamp is unavailable'); +}); + +it('uses the configured backlog threshold', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_backlog_threshold'] = '1000'; + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 1001, 'oldest' => time()]; + expect(syslog_pipeline_health()['issues']['backlog'])->toContain('1,001')->toContain('1,000'); +}); + +it('never reports unavailable metrics or unsafe column mappings as healthy', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['incoming'] = false; + expect(syslog_pipeline_health()['issues']['backlog'])->toContain('unavailable'); + $GLOBALS['syslog_incoming_config']['timeField'] = 'bad`; DROP TABLE syslog'; + $GLOBALS['pipeline_fixture']['queries'] = []; + expect(syslog_pipeline_health()['issues']['backlog'])->toContain('unavailable'); + expect(implode(' ', $GLOBALS['pipeline_fixture']['queries']))->not->toContain('DROP TABLE'); +}); + +it('detects failed and delayed remote delivery without treating ordinary recovery as failure', function () { + pipeline_health_fixture(); + $GLOBALS['config']['poller_id'] = 2; + $GLOBALS['config']['connection'] = 'offline'; + $GLOBALS['pipeline_fixture']['settings']['syslog_remote_enabled'] = 'on'; + $GLOBALS['pipeline_fixture']['outbox'] = ['pending' => 10, 'oldest_pending' => date('Y-m-d H:i:s')]; + expect(syslog_pipeline_health()['issues'])->toBeEmpty(); + $GLOBALS['pipeline_fixture']['outbox']['oldest_pending'] = date('Y-m-d H:i:s', time() - 1000); + expect(syslog_pipeline_health()['issues'])->toHaveKey('replication'); + $GLOBALS['pipeline_fixture']['outbox']['oldest_pending'] = date('Y-m-d H:i:s'); + $GLOBALS['pipeline_fixture']['status']['replication_last_error'] = 'Central commit failed'; + expect(syslog_pipeline_health()['issues'])->toHaveKey('replication'); + $GLOBALS['pipeline_fixture']['outbox']['pending'] = null; + expect(syslog_pipeline_health()['issues']['replication'])->toContain('unavailable'); +}); + +it('reports queued replication records when the oldest timestamp is invalid', function () { + pipeline_health_fixture(); + $GLOBALS['config']['poller_id'] = 2; + $GLOBALS['pipeline_fixture']['settings']['syslog_remote_enabled'] = 'on'; + $GLOBALS['pipeline_fixture']['outbox'] = ['pending' => 10, 'oldest_pending' => 'not-a-timestamp']; + expect(syslog_pipeline_health()['issues']['replication'])->toContain('timestamp is unavailable'); +}); + +it('reports blocked partition maintenance with its reason', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['status']['partition_maintenance_blocked'] = '1'; + $GLOBALS['pipeline_fixture']['status']['partition_maintenance_reason'] = 'DDL failed'; + expect(syslog_pipeline_health()['issues']['partition'])->toContain('DDL failed'); +}); + +it('sends one problem notification and one recovery notification without duplicate polling emails', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_pipeline_health_email'] = 'ops@example.test'; + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 20000, 'oldest' => time()]; + syslog_pipeline_health_check(); + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['mails'])->toHaveCount(1); + $monitor = json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true); + $monitor['last_attempt'] -= 301; + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode($monitor); + $GLOBALS['pipeline_fixture']['incoming']['pending'] = 0; + syslog_pipeline_health_check(); + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['mails'])->toHaveCount(2) + ->and($GLOBALS['pipeline_fixture']['mails'][1][7])->toContain('recovered'); + expect($GLOBALS['pipeline_fixture']['locks'][1][0])->toContain('RELEASE_LOCK'); +}); + +it('retries failed mail after five minutes without claiming success', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_pipeline_health_email'] = 'ops@example.test'; + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 20000, 'oldest' => time()]; + $GLOBALS['pipeline_fixture']['mail_error'] = 'SMTP unavailable'; + syslog_pipeline_health_check(); + syslog_pipeline_health_check(); + $monitor = json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true); + expect($monitor['notification'])->toBe('failed')->and($monitor)->not->toHaveKey('last_notified') + ->and($GLOBALS['pipeline_fixture']['mails'])->toHaveCount(1); + $monitor['last_attempt'] -= 301; + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode($monitor); + $GLOBALS['pipeline_fixture']['mail_error'] = ''; + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['mails'])->toHaveCount(2); + expect(json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true)['notification'])->toBe('sent'); +}); + +it('reminds for ongoing problems only after the configured interval', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings'] += ['syslog_pipeline_health_email' => 'ops@example.test', 'syslog_pipeline_health_repeat' => '900']; + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 20000, 'oldest' => time()]; + syslog_pipeline_health_check(); + $monitor = json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true); + $monitor['last_attempt'] -= 901; + $monitor['last_notified'] -= 901; + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode($monitor); + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['mails'])->toHaveCount(2); +}); + +it('keeps a due notification pending while the retry cooldown is active', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_pipeline_health_email'] = 'ops@example.test'; + $GLOBALS['pipeline_fixture']['incoming']['pending'] = 20000; + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode([ + 'signature' => 'up:', + 'notified_signature' => 'previous:issue', + 'last_attempt' => time(), + 'notification' => 'ready', + ]); + + syslog_pipeline_health_check(); + + $monitor = json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true); + expect($monitor['notification'])->toBe('pending') + ->and($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty(); +}); + +it('checks and logs health with email disabled', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['incoming'] = ['pending' => 20000, 'oldest' => time()]; + syslog_pipeline_health_check(); + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['logs'])->toHaveCount(1) + ->and($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty(); +}); + +it('does not alert for intentionally disabled processing and gives re-enablement a grace period', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings'] += ['syslog_pipeline_health_email' => 'ops@example.test']; + $GLOBALS['pipeline_fixture']['settings']['syslog_enabled'] = ''; + $GLOBALS['pipeline_fixture']['status']['last_end_time'] = (string) (time() - 1000); + syslog_pipeline_health_check(); + expect(syslog_pipeline_health()['state'])->toBe('disabled')->and($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty(); + $GLOBALS['pipeline_fixture']['settings']['syslog_enabled'] = 'on'; + syslog_pipeline_health_check(); + expect(syslog_pipeline_health()['issues'])->toBeEmpty(); +}); + +it('skips notifications when another monitor owns the lock', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['lock'] = 0; + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['queries'])->toBeEmpty()->and($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty(); +}); + +it('runs the monitor from the independent poller before launching the processor', function () { + $source = plugin_test_read_source('includes/processing.php'); + expect(strpos($source, 'syslog_pipeline_health_check();'))->toBeLessThan(strpos($source, 'exec_background($command_string, $extra_args);')); +}); + + +it('does not send email when the retry state cannot be persisted', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_pipeline_health_email'] = 'ops@example.test'; + $GLOBALS['pipeline_fixture']['incoming']['pending'] = 20000; + $GLOBALS['pipeline_fixture']['write_failed'] = true; + syslog_pipeline_health_check(); + expect($GLOBALS['pipeline_fixture']['mails'])->toBeEmpty() + ->and($GLOBALS['pipeline_fixture']['locks'][1][0])->toContain('RELEASE_LOCK'); +}); + +it('keeps mail exceptions from blocking processing and releases the monitor lock', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['settings']['syslog_pipeline_health_email'] = 'ops@example.test'; + $GLOBALS['pipeline_fixture']['incoming']['pending'] = 20000; + test_override('mailer', function () { throw new RuntimeException('Mail transport failed'); }); + syslog_pipeline_health_check(); + expect(json_decode($GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'], true)['notification'])->toBe('failed') + ->and($GLOBALS['pipeline_fixture']['locks'][1][0])->toContain('RELEASE_LOCK'); +}); + +it('renders pipeline problems and notification status in the existing Status card with escaped details', function () { + pipeline_health_fixture(); + $GLOBALS['pipeline_fixture']['status']['partition_maintenance_blocked'] = '1'; + $GLOBALS['pipeline_fixture']['status']['partition_maintenance_reason'] = ''; + $GLOBALS['pipeline_fixture']['status']['pipeline_health_monitor'] = json_encode(['checked' => time(), 'notification' => 'failed']); + $source = plugin_test_read_source('syslog.php'); + foreach (['syslog_status_data', 'syslog_status_workers', 'syslog_status_pipeline', 'syslog_status_card_meta', 'syslog_status_card_body', 'syslog_status_collector_health', 'syslog_status_collector_unavailable', 'syslog_status_format_age', 'syslog_status_format_count', 'syslog_status_format_time'] as $name) { + if (!function_exists($name)) { + $start = strpos($source, "function $name("); + $end = strpos($source, "\n}", $start); + eval(substr($source, $start, $end - $start + 2)); + } + } + if (!defined('SYSLOG_COLLECTOR_STALE_SECONDS')) define('SYSLOG_COLLECTOR_STALE_SECONDS', 300); + if (!defined('SYSLOG_COLLECTOR_BACKLOG_THRESHOLD')) define('SYSLOG_COLLECTOR_BACKLOG_THRESHOLD', 10000); + $html = syslog_status_card_body('collector_health'); + expect(syslog_status_card_meta()['collector_health']['title'])->toBe('Pipeline health') + ->and(syslog_status_card_meta()['collector_health']['state'])->toBe('down') + ->and($html)->toContain('Attention required')->toContain('Partition maintenance') + ->toContain('Failed; retry pending')->toContain('Last health check') + ->toContain('<script>')->not->toContain('