Replace OpenCensus with OpenTelemetry for metrics

.github/workflows/codeql.yml

@@ -41,16 +41,16 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
         with:
           egress-policy: audit
 
       - name: Checkout repository
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
 
       # Initializes the CodeQL tools for scanning.
       - name: Initialize CodeQL
-        uses: github/codeql-action/init@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
+        uses: github/codeql-action/init@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
         with:
           languages: ${{ matrix.language }}
           # If you wish to specify custom queries, you can do so here or in a config file.
@@ -60,7 +60,7 @@ jobs:
       # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
       # If this step fails, then you should remove it and run the build manually (see below)
       - name: Autobuild
-        uses: github/codeql-action/autobuild@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
+        uses: github/codeql-action/autobuild@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
 
       # ℹ️ Command-line programs to run using the OS shell.
       # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
@@ -73,6 +73,6 @@ jobs:
       #   ./location_of_script_within_repo/buildscript.sh
 
       - name: Perform CodeQL Analysis
-        uses: github/codeql-action/analyze@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
+        uses: github/codeql-action/analyze@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
         with:
           category: "/language:${{matrix.language}}"

.github/workflows/dependency-review.yml

@@ -17,11 +17,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
         with:
           egress-policy: audit
 
       - name: 'Checkout Repository'
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: 'Dependency Review'
-        uses: actions/dependency-review-action@3c4e3dcb1aa7874d2c16be7d79418e9b7efd6261 # v4.8.2
+        uses: actions/dependency-review-action@a1d282b36b6f3519aa1f3fc636f609c47dddb294 # v5.0.0

.github/workflows/lint.yml

@@ -5,9 +5,10 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@v6.0.1
+      - name: Checkout Repository
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       - name: Set up Go
-        uses: actions/setup-go@v6
+        uses: actions/setup-go@b7ad1dad31e06c5925ef5d2fc7ad053ef454303e # v7.0.0
         with:
           go-version-file: go.mod
       - name: golangci-lint

.github/workflows/scorecards.yml

@@ -31,17 +31,17 @@ jobs:
 
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
         with:
           egress-policy: audit
 
       - name: "Checkout code"
-        uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           persist-credentials: false
 
       - name: "Run analysis"
-        uses: ossf/scorecard-action@4eaacf0543bb3f2c246792bd56e8cdeffafb205a # v2.4.3
+        uses: ossf/scorecard-action@2d1146689b8cda280b9bc96326124645441f03bc # v2.4.4
         with:
           results_file: results.sarif
           results_format: sarif
@@ -63,14 +63,14 @@ jobs:
       # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
       # format to the repository Actions tab.
       - name: "Upload artifact"
-        uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0
+        uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
         with:
           name: SARIF file
           path: results.sarif
           retention-days: 5
 
       # Upload the results to GitHub's code scanning dashboard.
       - name: "Upload to code-scanning"
-        uses: github/codeql-action/upload-sarif@5d4e8d1aca955e8d8589aabd499c5cae939e33c7 # v4.31.9
+        uses: github/codeql-action/upload-sarif@b96794f015dfd88f77b49b1c93e0fa7110f94c63 # v4.38.0
         with:
           sarif_file: results.sarif

.github/workflows/tag-release.yml

@@ -19,10 +19,10 @@ jobs:
       contents: write
     steps:
       - name: Harden Runner
-        uses: step-security/harden-runner@20cf305ff2072d973412fa9b1e3a4f227bda3c76 # v2.14.0
+        uses: step-security/harden-runner@e14015d583714f6e62063499dc959a02595150a1 # v2.21.1
         with:
           egress-policy: audit
-      - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1
+      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
         with:
           fetch-depth: 0
       - run: /usr/bin/git config --global user.email actions@github.com

.gitignore

@@ -9,3 +9,5 @@ debug.test
 /output/
 coverage.out
 .idea/
+*.DS_Store
+*.iml

.golangci.yml

@@ -114,6 +114,13 @@ linters:
     # - wsl                       403
     # - wsl_v5                    58
     - zerologlint
+  settings:
+    goconst:
+      # goconst >= 1.9.0 (golangci-lint >= v2.12.0) also inspects composite
+      # literals. Keep the previous scope; the default is [Call].
+      exclude-types:
+        - Call
+        - CompositeLit
   exclusions:
     generated: lax
     paths:

Dockerfile

@@ -5,7 +5,7 @@
 # Must override builder-base, not builder, since the latter is referred to later in the file and so must not be
 # directly replaced. See here, and note that "stage" parameter mentioned there has been renamed to
 # "build-context": https://github.com/docker/buildx/pull/904#issuecomment-1005871838
-FROM golang:1.25.5-bookworm@sha256:09f53deea14d4019922334afe6258b7b776afc1d57952be2012f2c8c4076db05 AS builder-base
+FROM golang:1.27.2-bookworm AS builder-base
 FROM --platform=$BUILDPLATFORM builder-base AS builder
 
 ARG TARGETARCH
@@ -20,9 +20,9 @@ COPY . /src/
 WORKDIR /src
 RUN GOARCH=${TARGETARCH} make bin/node-problem-detector bin/health-checker bin/log-counter
 
-FROM registry.k8s.io/build-image/debian-base:bookworm-v1.0.6@sha256:e100119ba6cf265b29957046f178e9374f9a9419133284c9b883e64c5b463d73 AS base
+FROM registry.k8s.io/build-image/debian-base:bookworm-v1.0.9@sha256:628de94e72732b63ddd84a79f961ef1045779bb83edd170e8306f4ba28a511e4 AS base
 
-RUN clean-install util-linux bash libsystemd-dev
+RUN apt-get update && apt-get dist-upgrade -y && clean-install util-linux bash libsystemd0
 
 # Avoid symlink of /etc/localtime.
 RUN test -h /etc/localtime && rm -f /etc/localtime && cp /usr/share/zoneinfo/UTC /etc/localtime || true

Dockerfile.windows

@@ -5,7 +5,7 @@
 # Must override builder-base, not builder, since the latter is referred to later in the file and so must not be
 # directly replaced. See here, and note that "stage" parameter mentioned there has been renamed to
 # "build-context": https://github.com/docker/buildx/pull/904#issuecomment-1005871838
-FROM --platform=linux/$BUILDARCH golang:1.25.5-bookworm@sha256:09f53deea14d4019922334afe6258b7b776afc1d57952be2012f2c8c4076db05 AS builder-base
+FROM --platform=linux/$BUILDARCH golang:1.27.2-bookworm AS builder-base
 FROM --platform=linux/$BUILDARCH builder-base AS builder
 
 ARG TARGETARCH

Makefile

@@ -15,7 +15,7 @@
 # Build the node-problem-detector image.
 
 .PHONY: all \
-        lint vet fmt version test e2e-test \
+        lint vet fmt fmt-configs version test e2e-test \
         build-binaries build-container build-tar build \
         docker-builder build-in-docker \
         push-container push-tar push release clean depup \
@@ -123,7 +123,7 @@ else
 	LOGCOUNTER=*dont-include-log-counter
 endif
 
-GOLANGCI_LINT_VERSION := v2.6.2
+GOLANGCI_LINT_VERSION := v2.14.0
 GOLANGCI_LINT := ./.bin/golangci-lint
 
 lint: $(GOLANGCI_LINT)
@@ -132,7 +132,7 @@ lint: $(GOLANGCI_LINT)
 $(GOLANGCI_LINT):
 	@echo "golangci-lint not found, downloading..."
 	@mkdir -p ./.bin
-	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./.bin $(GOLANGCI_LINT_VERSION)
+	curl -sSfL https://golangci-lint.run/install.sh | sh -s -- -b ./.bin $(GOLANGCI_LINT_VERSION)
 
 
 vet:
@@ -144,6 +144,9 @@ fmt: $(GOLANGCI_LINT)
 	find . -type f -name "*.go" | grep -v "./vendor/*" | xargs gofmt -s -w -l
 	$(GOLANGCI_LINT) run --config .golangci.yml --fix ./...
 
+fmt-configs:
+	UPDATE_EXPECTED=y go test -run TestShippedConfigs ./pkg/systemlogmonitor
+
 version:
 	@echo $(VERSION)
 

README.md

@@ -1,7 +1,5 @@
 # node-problem-detector
 
-[![Build Status](https://travis-ci.org/kubernetes/node-problem-detector.svg?branch=master)](https://travis-ci.org/kubernetes/node-problem-detector)  [![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/node-problem-detector)](https://goreportcard.com/report/github.com/kubernetes/node-problem-detector)
-
 node-problem-detector aims to make various node problems visible to the upstream
 layers in the cluster management stack.
 It is a daemon that runs on each node, detects node
@@ -12,6 +10,7 @@ Now it is running as a
 [Kubernetes Addon](https://github.com/kubernetes/kubernetes/tree/master/cluster/addons)
 enabled by default in the GKE cluster. It is also enabled by default in AKS as part of the
 [AKS Linux Extension](https://learn.microsoft.com/en-us/azure/aks/faq#what-is-the-purpose-of-the-aks-linux-extension-i-see-installed-on-my-linux-vmss-instances).
+
 # Background
 
 There are tons of node problems that could possibly affect the pods running on the
@@ -174,8 +173,7 @@ to another registry.
 The easiest way to install node-problem-detector into your cluster is to use the [Helm](https://helm.sh/) [chart](https://github.com/deliveryhero/helm-charts/tree/master/stable/node-problem-detector):
 
 ```
-helm repo add deliveryhero https://charts.deliveryhero.io/
-helm install --generate-name deliveryhero/node-problem-detector
+helm install --generate-name oci://ghcr.io/deliveryhero/helm-charts/node-problem-detector
 ```
 
 Alternatively, to install node-problem-detector manually:

SECURITY.md

@@ -0,0 +1,30 @@
+# Security Policy
+
+## Security Announcements
+
+Join the [kubernetes-security-announce] group for security and vulnerability announcements.
+
+You can also subscribe to an RSS feed of the above using [this link][kubernetes-security-announce-rss].
+
+## Reporting a Vulnerability
+
+Instructions for reporting a vulnerability can be found on the [Kubernetes Security and Disclosure Information] page.
+
+## Dependency CVEs and vulnerability scanner findings
+
+node-problem-detector is a Go program with a large dependency tree, and vulnerability scanners regularly flag CVEs in its dependencies. The project follows the Kubernetes-wide guidance for handling these reports — see [CVEs in our dependencies] in the Kubernetes security guide. In short:
+
+- Scanners match dependency versions; they do not check whether the vulnerable code is reachable. The large majority of CVEs flagged against node-problem-detector dependencies are in code paths its binaries never execute. Reachability evidence (for example, `govulncheck` output showing a call stack to the vulnerable symbol) is what determines whether a finding is treated as a vulnerability in node-problem-detector.
+- Dependency updates land on `master` continuously and ship with the next scheduled release (see [versioning](docs/versioning.md) for the release cadence). The project is maintained by a small group of volunteers and does not cut out-of-band releases solely to update dependency version strings for findings that are not reachable.
+- If you have evidence that a CVE is reachable and exploitable in node-problem-detector as deployed, please report it through the process above. That is treated as a vulnerability in node-problem-detector itself, and a fix will be prioritized and backported to supported release branches.
+
+If your compliance program requires container images with zero scanner findings on a faster cadence than the project's releases, rebuild node-problem-detector from the latest release tag or from `master` in your own build pipeline — everything needed is in this repository (see the [Makefile](Makefile) and [Dockerfile](Dockerfile)). No upstream release cadence can track every scanner database, and rebuilding downstream is standard practice for organizations with strict scanning requirements.
+
+## Supported Versions
+
+Fixes land on `master` first. Release lines follow supported Kubernetes minor versions as described in [versioning](docs/versioning.md), and fixes for vulnerabilities that are reachable in node-problem-detector may be backported to supported release branches.
+
+[kubernetes-security-announce]: https://groups.google.com/forum/#!forum/kubernetes-security-announce
+[kubernetes-security-announce-rss]: https://groups.google.com/forum/feed/kubernetes-security-announce/msgs/rss_v2_0.xml?num=50
+[Kubernetes Security and Disclosure Information]: https://kubernetes.io/docs/reference/issues-security/security/#report-a-vulnerability
+[CVEs in our dependencies]: https://github.com/kubernetes/community/blob/master/contributors/guide/security.md#cves-in-our-dependencies

cmd/nodeproblemdetector/node_problem_detector.go

@@ -18,6 +18,7 @@ package main
 
 import (
 	"context"
+	"time"
 
 	"k8s.io/klog/v2"
 
@@ -29,7 +30,9 @@ import (
 	"k8s.io/node-problem-detector/pkg/exporters/prometheusexporter"
 	"k8s.io/node-problem-detector/pkg/problemdaemon"
 	"k8s.io/node-problem-detector/pkg/problemdetector"
+	"k8s.io/node-problem-detector/pkg/problemmetrics"
 	"k8s.io/node-problem-detector/pkg/types"
+	otelutil "k8s.io/node-problem-detector/pkg/util/otel"
 	"k8s.io/node-problem-detector/pkg/version"
 )
 
@@ -43,13 +46,7 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro
 	npdo.SetConfigFromDeprecatedOptionsOrDie()
 	npdo.ValidOrDie()
 
-	// Initialize problem daemons.
-	problemDaemons := problemdaemon.NewProblemDaemons(npdo.MonitorConfigPaths)
-	if len(problemDaemons) == 0 {
-		klog.Fatalf("No problem daemon is configured")
-	}
-
-	// Initialize exporters.
+	// Initialize exporters first to set up the OpenTelemetry readers.
 	defaultExporters := []types.Exporter{}
 	if ke := k8sexporter.NewExporterOrDie(ctx, npdo); ke != nil {
 		defaultExporters = append(defaultExporters, ke)
@@ -59,9 +56,28 @@ func npdMain(ctx context.Context, npdo *options.NodeProblemDetectorOptions) erro
 		defaultExporters = append(defaultExporters, pe)
 		klog.Info("Prometheus exporter started.")
 	}
-
 	plugableExporters := exporters.NewExporters()
 
+	// Initialize OpenTelemetry meter provider with all registered readers
+	// This must be called after all exporters have been created and registered their readers
+	meterProvider := otelutil.InitializeMeterProvider()
+	defer func() {
+		// Drop the cancellation of ctx because it is likely already canceled
+		// at shutdown, which would prevent flushing pending metrics.
+		shutdownCtx, cancel := context.WithTimeout(context.WithoutCancel(ctx), 10*time.Second)
+		defer cancel()
+		if err := meterProvider.Shutdown(shutdownCtx); err != nil {
+			klog.Errorf("Failed to shut down OpenTelemetry meter provider: %v", err)
+		}
+	}()
+	problemmetrics.InitializeGlobalProblemMetricsManager()
+
+	// Initialize problem daemons.
+	problemDaemons := problemdaemon.NewProblemDaemons(npdo.MonitorConfigPaths)
+	if len(problemDaemons) == 0 {
+		klog.Fatalf("No problem daemon is configured")
+	}
+
 	npdExporters := []types.Exporter{}
 	npdExporters = append(npdExporters, defaultExporters...)
 	npdExporters = append(npdExporters, plugableExporters...)

cmd/nodeproblemdetector/node_problem_detector_test.go

@@ -127,17 +127,17 @@ func setupNPD(t *testing.T) (*options.NodeProblemDetectorOptions, func()) {
 	}
 
 	return &options.NodeProblemDetectorOptions{
-			MonitorConfigPaths: map[types.ProblemDaemonType]*[]string{
-				"system-log-monitor": {
-					fakeConfigFileName,
-				},
+		MonitorConfigPaths: map[types.ProblemDaemonType]*[]string{
+			"system-log-monitor": {
+				fakeConfigFileName,
 			},
-		}, func() {
-			if err := os.Remove(fakeLogFileName); err != nil {
-				t.Logf("Failed to remove temporary file %s: %v", fakeLogFileName, err)
-			}
-			if err := os.Remove(fakeConfigFileName); err != nil {
-				t.Logf("Failed to remove temporary file %s: %v", fakeConfigFileName, err)
-			}
+		},
+	}, func() {
+		if err := os.Remove(fakeLogFileName); err != nil {
+			t.Logf("Failed to remove temporary file %s: %v", fakeLogFileName, err)
 		}
+		if err := os.Remove(fakeConfigFileName); err != nil {
+			t.Logf("Failed to remove temporary file %s: %v", fakeConfigFileName, err)
+		}
+	}
 }

cmd/nodeproblemdetector/node_problem_detector_unix.go

@@ -21,6 +21,9 @@ package main
 import (
 	"context"
 	"flag"
+	"os"
+	"os/signal"
+	"syscall"
 
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
@@ -31,6 +34,9 @@ import (
 func main() {
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
+	// Opt into fixed stderrthreshold behavior (kubernetes/klog#212).
+	_ = klogFlags.Set("legacy_stderr_threshold_behavior", "false")
+	_ = klogFlags.Set("stderrthreshold", "INFO")
 	klogFlags.VisitAll(func(f *flag.Flag) {
 		switch f.Name {
 		case "v", "vmodule", "logtostderr":
@@ -51,7 +57,9 @@ func main() {
 	}
 
 	pflag.Parse()
-	if err := npdMain(context.Background(), npdo); err != nil {
+	ctx, stop := signal.NotifyContext(context.Background(), os.Interrupt, syscall.SIGTERM)
+	defer stop()
+	if err := npdMain(ctx, npdo); err != nil {
 		klog.Fatalf("Problem detector failed with error: %v", err)
 	}
 }

cmd/nodeproblemdetector/node_problem_detector_windows.go

@@ -46,6 +46,9 @@ var (
 func main() {
 	klogFlags := flag.NewFlagSet("klog", flag.ExitOnError)
 	klog.InitFlags(klogFlags)
+	// Opt into fixed stderrthreshold behavior (kubernetes/klog#212).
+	_ = klogFlags.Set("legacy_stderr_threshold_behavior", "false")
+	_ = klogFlags.Set("stderrthreshold", "INFO")
 	klogFlags.VisitAll(func(f *flag.Flag) {
 		switch f.Name {
 		case "v", "vmodule", "logtostderr":