diff --git a/docs/changelog.md b/docs/changelog.md index 89543dcca..860d951ac 100644 --- a/docs/changelog.md +++ b/docs/changelog.md @@ -1,5 +1,12 @@ # Changelog +## v0.89.4 + +### Fixed + +- Fixed `alerts/count` check undercounting alerts for rules with + `keep_firing_for` - [#2004](https://github.com/cloudflare/pint/issues/2004). + ## v0.89.3 ### Fixed diff --git a/internal/checks/alerts_count.go b/internal/checks/alerts_count.go index 838dbe740..a8c92ae5e 100644 --- a/internal/checks/alerts_count.go +++ b/internal/checks/alerts_count.go @@ -98,12 +98,27 @@ func (c AlertsCheck) Check(ctx context.Context, entry *discovery.Entry, _ []*dis keepFiringForDur = entry.Rule.AlertingRule.KeepFiringFor.Value } - var alerts int + // An alert fires once a range is longer than `for`. + // If `keep_firing_for` is set it will delay when the alert resolves. + var ( + alerts int + firingFP uint64 + firingUntil time.Time + ) for _, r := range qr.Series.Ranges { - // If `keepFiringFor` is not defined its Duration will be 0 - if r.End.Sub(r.Start) > (forDur + keepFiringForDur) { - alerts++ + if r.Fingerprint == firingFP && !r.Start.After(firingUntil) { + firingUntil = r.End.Add(time.Second + keepFiringForDur) + // Separate range of an already firing alert and within keep_firing_for value. + // This does not count as a new alert. + continue + } + if r.End.Sub(r.Start) <= forDur { + // Too short to fire due to `for` value, wouldn't fire at all. + continue } + alerts++ + firingFP = r.Fingerprint + firingUntil = r.End.Add(time.Second + keepFiringForDur) } if alerts < c.minCount { diff --git a/internal/checks/alerts_count_test.go b/internal/checks/alerts_count_test.go index 269de1fbc..bcb9417d8 100644 --- a/internal/checks/alerts_count_test.go +++ b/internal/checks/alerts_count_test.go @@ -16,21 +16,25 @@ func newAlertsCheck(prom *promapi.FailoverGroup) checks.RuleChecker { func TestAlertsCountCheck(t *testing.T) { content := "- alert: Foo Is Down\n expr: up{job=\"foo\"} == 0\n" + now := time.Now() testCases := []checkTest{ { + // A recording rule is not checked, so no problems are reported. description: "ignores recording rules", content: "- record: foo\n expr: up == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, }, { + // The expression does not parse, so no problems are reported. description: "ignores rules with syntax errors", content: "- alert: Foo Is Down\n expr: sum(\n", checker: newAlertsCheck, prometheus: newSimpleProm, }, { + // The server rejects the query, so the check reports a problem with the error. description: "bad request", content: content, checker: newAlertsCheck, @@ -47,6 +51,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { + // The optional server is unreachable, so queries report a warning problem. description: "connection refused / upstream not required / warning", content: content, checker: newAlertsCheck, @@ -56,7 +61,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "empty response", + description: "empty response / 0 alerts", content: content, checker: newAlertsCheck, prometheus: newSimpleProm, @@ -72,7 +77,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "multiple alerts", + description: "multiple alerts / seven ranges for one series / 7 alerts", content: content, checker: newAlertsCheck, prometheus: newSimpleProm, @@ -147,7 +152,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "for: 10m", + description: "for: 10m / 2 ranges over 10m / 2 alerts", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, @@ -209,7 +214,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "minCount=2", + description: "minCount=2 / 2 alerts reported", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: func(prom *promapi.FailoverGroup) checks.RuleChecker { return checks.NewAlertsCheck(prom, time.Hour*24, time.Minute, time.Minute*5, 2, "rule comment", checks.Information) @@ -273,7 +278,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "minCount=2 severity=bug", + description: "minCount=2 severity=bug / 2 alerts reported as bug", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: func(prom *promapi.FailoverGroup) checks.RuleChecker { return checks.NewAlertsCheck(prom, time.Hour*24, time.Minute, time.Minute*5, 2, "", checks.Bug) @@ -337,7 +342,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "minCount=10", + description: "minCount=10 / 2 alerts below minCount, nothing reported", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: func(prom *promapi.FailoverGroup) checks.RuleChecker { return checks.NewAlertsCheck(prom, time.Hour*24, time.Minute, time.Minute*5, 10, "", checks.Information) @@ -400,7 +405,7 @@ func TestAlertsCountCheck(t *testing.T) { }, }, { - description: "{__name__=}", + description: "{__name__=} / three ranges / 3 alerts", content: ` - alert: foo expr: '{__name__="up", job="foo"} == 0' @@ -447,7 +452,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "{__name__=~}", + description: "{__name__=~} / three ranges / 3 alerts", content: ` - alert: foo expr: '{__name__=~"(up|foo)", job="foo"} == 0' @@ -494,7 +499,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "uptime query error", + description: "uptime query error / gap detection skipped / 3 alerts", content: "- alert: Foo Is Down\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, @@ -538,7 +543,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "keep_firing_for: 10m", + description: "keep_firing_for: 10m / 6 ranges 1h apart / 6 alerts", content: "- alert: Foo Is Down\n keep_firing_for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, @@ -600,7 +605,7 @@ func TestAlertsCountCheck(t *testing.T) { problems: true, }, { - description: "for: 10m + keep_firing_for: 10m", + description: "for: 10m + keep_firing_for: 10m / 2 ranges over 10m / 2 alerts", content: "- alert: Foo Is Down\n for: 10m\n keep_firing_for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, @@ -661,10 +666,189 @@ func TestAlertsCountCheck(t *testing.T) { }, problems: true, }, + { + description: "keep_firing_for=30m merges 15m gaps between ranges / 2 alerts", + content: "- alert: Foo Is Down\n keep_firing_for: 30m\n expr: up{job=\"foo\"} == 0\n", + checker: newAlertsCheck, + prometheus: newSimpleProm, + mocks: []*prometheusMock{ + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: `up{job="foo"} == 0`}, + }, + resp: matrixResponse{ + samples: []*model.SampleStream{ + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20), + now.Add(time.Hour*-20).Add(time.Minute*5), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*20), + now.Add(time.Hour*-20).Add(time.Minute*25), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*40), + now.Add(time.Hour*-20).Add(time.Minute*45), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-16), + now.Add(time.Hour*-16).Add(time.Minute*5), + time.Minute, + ), + }, + }, + }, + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: "count(\nup\n)"}, + }, + resp: respondWithSingleRangeVector1D(), + }, + }, + problems: true, + }, + { + description: "keep_firing_for=2m covers a 2m gap but not wider gaps / 3 alerts", + content: "- alert: Foo Is Down\n keep_firing_for: 2m\n expr: up{job=\"foo\"} == 0\n", + checker: newAlertsCheck, + prometheus: newSimpleProm, + mocks: []*prometheusMock{ + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: `up{job="foo"} == 0`}, + }, + resp: matrixResponse{ + samples: []*model.SampleStream{ + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20), + now.Add(time.Hour*-20), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*2), + now.Add(time.Hour*-20).Add(time.Minute*2), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*8), + now.Add(time.Hour*-20).Add(time.Minute*8), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*12), + now.Add(time.Hour*-20).Add(time.Minute*12), + time.Minute, + ), + }, + }, + }, + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: "count(\nup\n)"}, + }, + // Prometheus is up the whole window, so any gap is real. + resp: matrixResponse{ + samples: []*model.SampleStream{ + generateSampleStream( + map[string]string{}, + now.Add(time.Hour*-24), + now, + time.Minute, + ), + }, + }, + }, + }, + problems: true, + }, + { + // The ranges at +40m and +70m start before keep_firing_for: 30m + // runs out, so foo has one alert. The last 2m range is too short + // for `for`. job=bar fires its own alert. + description: "for=10m + keep_firing_for=30m gives one alert per series / 2 alerts", + content: "- alert: Foo Is Down\n for: 10m\n keep_firing_for: 30m\n expr: up{job=\"foo\"} == 0\n", + checker: newAlertsCheck, + prometheus: newSimpleProm, + mocks: []*prometheusMock{ + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: `up{job="foo"} == 0`}, + }, + resp: matrixResponse{ + samples: []*model.SampleStream{ + generateSampleStream( + map[string]string{"job": "bar"}, + now.Add(time.Hour*-20), + now.Add(time.Hour*-20).Add(time.Minute*15), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20), + now.Add(time.Hour*-20).Add(time.Minute*15), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*40), + now.Add(time.Hour*-20).Add(time.Minute*42), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-20).Add(time.Minute*70), + now.Add(time.Hour*-20).Add(time.Minute*85), + time.Minute, + ), + generateSampleStream( + map[string]string{"job": "foo"}, + now.Add(time.Hour*-16), + now.Add(time.Hour*-16).Add(time.Minute*2), + time.Minute, + ), + }, + }, + }, + { + conds: []requestCondition{ + requireRangeQueryPath, + formCond{key: "query", value: "count(\nup\n)"}, + }, + // Prometheus is up the whole window, so any gap is real. + resp: matrixResponse{ + samples: []*model.SampleStream{ + generateSampleStream( + map[string]string{}, + now.Add(time.Hour*-24), + now, + time.Minute, + ), + }, + }, + }, + }, + problems: true, + }, { // Two 7m ranges separated by 3m, Prometheus was down during those 3m. // Since Prometheus was down there's no gap, so ranges merge into ~17m, exceeding for: 10m. - description: "ranges merge when separated by Prometheus downtime", + description: "Prometheus down during gap / ranges merge / 1 alert", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, @@ -724,7 +908,7 @@ func TestAlertsCountCheck(t *testing.T) { { // Two 7m ranges but Prometheus was up the entire time. // There's a gap between them, so they don't merge. - description: "ranges stay separate when gap is confirmed by uptime", + description: "Prometheus up during gap / ranges stay separate / 0 alerts", content: "- alert: Foo Is Down\n for: 10m\n expr: up{job=\"foo\"} == 0\n", checker: newAlertsCheck, prometheus: newSimpleProm, diff --git a/internal/checks/alerts_count_test.snap b/internal/checks/alerts_count_test.snap index e08b25862..15a1b100b 100644 --- a/internal/checks/alerts_count_test.snap +++ b/internal/checks/alerts_count_test.snap @@ -1,4 +1,60 @@ +[TestAlertsCountCheck/Prometheus_down_during_gap_/_ranges_merge_/_1_alert - 1] +- description: Prometheus down during gap / ranges merge / 1 alert + content: | + - alert: Foo Is Down + for: 10m + expr: up{job="foo"} == 0 + output: | + 3 | expr: up{job="foo"} == 0 + ^^^^^^^^^^^^^^^^^^ + `prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last + 1d. + problem: + reporter: alerts/count + summary: alert count estimate + details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). + diagnostics: + - message: '`prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last 1d.' + firstcolumn: 1 + lastcolumn: 18 + kind: 0 + lines: + first: 3 + last: 3 + severity: 0 + anchor: 0 + +--- + +[TestAlertsCountCheck/Prometheus_up_during_gap_/_ranges_stay_separate_/_0_alerts - 1] +- description: Prometheus up during gap / ranges stay separate / 0 alerts + content: | + - alert: Foo Is Down + for: 10m + expr: up{job="foo"} == 0 + output: | + 3 | expr: up{job="foo"} == 0 + ^^^^^^^^^^^^^^^^^^ + `prom` Prometheus server at https://simple.example.com would trigger 0 alert(s) in the last + 1d. + problem: + reporter: alerts/count + summary: alert count estimate + details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). + diagnostics: + - message: '`prom` Prometheus server at https://simple.example.com would trigger 0 alert(s) in the last 1d.' + firstcolumn: 1 + lastcolumn: 18 + kind: 0 + lines: + first: 3 + last: 3 + severity: 0 + anchor: 0 + +--- + [TestAlertsCountCheck/bad_request - 1] - description: bad request content: | @@ -53,8 +109,8 @@ --- -[TestAlertsCountCheck/empty_response - 1] -- description: empty response +[TestAlertsCountCheck/empty_response_/_0_alerts - 1] +- description: empty response / 0 alerts content: | - alert: Foo Is Down expr: up{job="foo"} == 0 @@ -80,8 +136,37 @@ --- -[TestAlertsCountCheck/for:_10m - 1] -- description: 'for: 10m' +[TestAlertsCountCheck/for:_10m_+_keep_firing_for:_10m_/_2_ranges_over_10m_/_2_alerts - 1] +- description: 'for: 10m + keep_firing_for: 10m / 2 ranges over 10m / 2 alerts' + content: | + - alert: Foo Is Down + for: 10m + keep_firing_for: 10m + expr: up{job="foo"} == 0 + output: | + 4 | expr: up{job="foo"} == 0 + ^^^^^^^^^^^^^^^^^^ + `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last + 1d. + problem: + reporter: alerts/count + summary: alert count estimate + details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). + diagnostics: + - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' + firstcolumn: 1 + lastcolumn: 18 + kind: 0 + lines: + first: 4 + last: 4 + severity: 0 + anchor: 0 + +--- + +[TestAlertsCountCheck/for:_10m_/_2_ranges_over_10m_/_2_alerts - 1] +- description: 'for: 10m / 2 ranges over 10m / 2 alerts' content: | - alert: Foo Is Down for: 10m @@ -108,24 +193,24 @@ --- -[TestAlertsCountCheck/for:_10m_+_keep_firing_for:_10m - 1] -- description: 'for: 10m + keep_firing_for: 10m' +[TestAlertsCountCheck/for=10m_+_keep_firing_for=30m_gives_one_alert_per_series_/_2_alerts - 1] +- description: for=10m + keep_firing_for=30m gives one alert per series / 2 alerts content: | - alert: Foo Is Down for: 10m - keep_firing_for: 10m + keep_firing_for: 30m expr: up{job="foo"} == 0 output: | 4 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 @@ -147,8 +232,8 @@ --- -[TestAlertsCountCheck/keep_firing_for:_10m - 1] -- description: 'keep_firing_for: 10m' +[TestAlertsCountCheck/keep_firing_for:_10m_/_6_ranges_1h_apart_/_6_alerts - 1] +- description: 'keep_firing_for: 10m / 6 ranges 1h apart / 6 alerts' content: | - alert: Foo Is Down keep_firing_for: 10m @@ -156,14 +241,14 @@ output: | 3 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 6 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 6 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 @@ -175,25 +260,23 @@ --- -[TestAlertsCountCheck/minCount=2 - 1] -- description: minCount=2 +[TestAlertsCountCheck/keep_firing_for=2m_covers_a_2m_gap_but_not_wider_gaps_/_3_alerts - 1] +- description: keep_firing_for=2m covers a 2m gap but not wider gaps / 3 alerts content: | - alert: Foo Is Down - for: 10m + keep_firing_for: 2m expr: up{job="foo"} == 0 output: | 3 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 3 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate - details: |- - To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). - Rule comment: rule comment + details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 3 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 @@ -205,11 +288,11 @@ --- -[TestAlertsCountCheck/minCount=2_severity=bug - 1] -- description: minCount=2 severity=bug +[TestAlertsCountCheck/keep_firing_for=30m_merges_15m_gaps_between_ranges_/_2_alerts - 1] +- description: keep_firing_for=30m merges 15m gaps between ranges / 2 alerts content: | - alert: Foo Is Down - for: 10m + keep_firing_for: 30m expr: up{job="foo"} == 0 output: | 3 | expr: up{job="foo"} == 0 @@ -228,45 +311,43 @@ lines: first: 3 last: 3 - severity: 2 + severity: 0 anchor: 0 --- -[TestAlertsCountCheck/minCount=10 - 1] -[] - ---- - -[TestAlertsCountCheck/multiple_alerts - 1] -- description: multiple alerts +[TestAlertsCountCheck/minCount=2_/_2_alerts_reported - 1] +- description: minCount=2 / 2 alerts reported content: | - alert: Foo Is Down + for: 10m expr: up{job="foo"} == 0 output: | - 2 | expr: up{job="foo"} == 0 + 3 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 7 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate - details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). + details: |- + To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). + Rule comment: rule comment diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 7 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 lines: - first: 2 - last: 2 + first: 3 + last: 3 severity: 0 anchor: 0 --- -[TestAlertsCountCheck/ranges_merge_when_separated_by_Prometheus_downtime - 1] -- description: ranges merge when separated by Prometheus downtime +[TestAlertsCountCheck/minCount=2_severity=bug_/_2_alerts_reported_as_bug - 1] +- description: minCount=2 severity=bug / 2 alerts reported as bug content: | - alert: Foo Is Down for: 10m @@ -274,55 +355,59 @@ output: | 3 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 1 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 2 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 lines: first: 3 last: 3 - severity: 0 + severity: 2 anchor: 0 --- -[TestAlertsCountCheck/ranges_stay_separate_when_gap_is_confirmed_by_uptime - 1] -- description: ranges stay separate when gap is confirmed by uptime +[TestAlertsCountCheck/minCount=10_/_2_alerts_below_minCount,_nothing_reported - 1] +[] + +--- + +[TestAlertsCountCheck/multiple_alerts_/_seven_ranges_for_one_series_/_7_alerts - 1] +- description: multiple alerts / seven ranges for one series / 7 alerts content: | - alert: Foo Is Down - for: 10m expr: up{job="foo"} == 0 output: | - 3 | expr: up{job="foo"} == 0 + 2 | expr: up{job="foo"} == 0 ^^^^^^^^^^^^^^^^^^ - `prom` Prometheus server at https://simple.example.com would trigger 0 alert(s) in the last + `prom` Prometheus server at https://simple.example.com would trigger 7 alert(s) in the last 1d. problem: reporter: alerts/count summary: alert count estimate details: To get a preview of the alerts that would fire please [click here](https://simple.example.com/graph?g0.expr=up%7Bjob%3D%22foo%22%7D+%3D%3D+0&g0.tab=0&g0.range_input=1d). diagnostics: - - message: '`prom` Prometheus server at https://simple.example.com would trigger 0 alert(s) in the last 1d.' + - message: '`prom` Prometheus server at https://simple.example.com would trigger 7 alert(s) in the last 1d.' firstcolumn: 1 lastcolumn: 18 kind: 0 lines: - first: 3 - last: 3 + first: 2 + last: 2 severity: 0 anchor: 0 --- -[TestAlertsCountCheck/uptime_query_error - 1] -- description: uptime query error +[TestAlertsCountCheck/uptime_query_error_/_gap_detection_skipped_/_3_alerts - 1] +- description: uptime query error / gap detection skipped / 3 alerts content: | - alert: Foo Is Down expr: up{job="foo"} == 0 @@ -348,8 +433,8 @@ --- -[TestAlertsCountCheck/{__name__=} - 1] -- description: '{__name__=}' +[TestAlertsCountCheck/{__name__=}_/_three_ranges_/_3_alerts - 1] +- description: '{__name__=} / three ranges / 3 alerts' content: |4 - alert: foo @@ -376,8 +461,8 @@ --- -[TestAlertsCountCheck/{__name__=~} - 1] -- description: '{__name__=~}' +[TestAlertsCountCheck/{__name__=~}_/_three_ranges_/_3_alerts - 1] +- description: '{__name__=~} / three ranges / 3 alerts' content: |4 - alert: foo