From ee68d0665340c3382026d965bac664b79efcb79a Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Thu, 8 Oct 2026 14:47:50 -0700 Subject: [PATCH 1/8] [INFOPLAT-19425] Send legacy telemetry via a dedicated chip-ingress batch client With TelemetryIngress.ChipIngressEnabled, the telemetry manager now uses an injected chip-ingress service instead of Beholder's chip client, which was a no-op unless Telemetry.ChipIngressEndpoint was also set. - Add TelemetryIngress.ChipIngressEndpoint and ChipIngressInsecureConnection, validated when chip ingress is enabled. - Build a dedicated chipingress.Client with CSA auth and NOP lookup. - Replace the per-(contract, type) workers with one shared chainlink-common batch client; Close now flushes. - Emit OTI-parity CloudEvent extensions (legacytelemetry, csapublickey, partitionkey, sentat in ns, receivedat in ms); stop sending empty nodeoperatorname/nodename/nodecsapublickey. - Count telemetry_client_messages_sent/send_errors per message. - Record telemetry_ingress.config.info. --- .changeset/telemetry-chip-ingress-endpoint.md | 5 + core/config/docs/core.toml | 10 +- core/config/mocks/telemetry_ingress.go | 90 ++++ core/config/telemetry_ingress_config.go | 2 + core/config/toml/types.go | 36 +- core/services/chainlink/application.go | 25 +- .../chainlink/config_telemetry_ingress.go | 8 + core/services/chainlink/config_test.go | 28 +- .../testdata/config-empty-effective.toml | 2 + .../chainlink/testdata/config-full.toml | 2 + .../config-multi-chain-effective.toml | 2 + .../chip_ingress_batch_client.go | 197 +++++--- .../chip_ingress_batch_client_test.go | 457 +++++++----------- .../chip_ingress_batch_worker.go | 159 ------ .../chip_ingress_batch_worker_test.go | 131 ----- .../synchronization/chip_ingress_client.go | 62 +++ .../chip_ingress_client_test.go | 62 +++ core/services/synchronization/helpers_test.go | 6 +- core/services/telemetry/manager.go | 108 +++-- core/services/telemetry/manager_test.go | 136 ++++-- .../testdata/config-empty-effective.toml | 2 + core/web/resolver/testdata/config-full.toml | 2 + .../config-multi-chain-effective.toml | 2 + docs/CONFIG.md | 20 +- .../scripts/config/merge_raw_configs.txtar | 2 + testdata/scripts/node/validate/default.txtar | 2 + .../node/validate/defaults-override.txtar | 2 + .../disk-based-logging-disabled.txtar | 2 + .../validate/disk-based-logging-no-dir.txtar | 2 + .../node/validate/disk-based-logging.txtar | 2 + .../node/validate/fallback-override.txtar | 2 + .../node/validate/invalid-ocr-p2p.txtar | 2 + testdata/scripts/node/validate/invalid.txtar | 2 + testdata/scripts/node/validate/valid.txtar | 2 + testdata/scripts/node/validate/warnings.txtar | 2 + 35 files changed, 817 insertions(+), 759 deletions(-) create mode 100644 .changeset/telemetry-chip-ingress-endpoint.md delete mode 100644 core/services/synchronization/chip_ingress_batch_worker.go delete mode 100644 core/services/synchronization/chip_ingress_batch_worker_test.go create mode 100644 core/services/synchronization/chip_ingress_client.go create mode 100644 core/services/synchronization/chip_ingress_client_test.go diff --git a/.changeset/telemetry-chip-ingress-endpoint.md b/.changeset/telemetry-chip-ingress-endpoint.md new file mode 100644 index 00000000000..193d87f737d --- /dev/null +++ b/.changeset/telemetry-chip-ingress-endpoint.md @@ -0,0 +1,5 @@ +--- +"chainlink": patch +--- + +Telemetry: with `TelemetryIngress.ChipIngressEnabled = true`, the node now sends legacy telemetry to chip-ingress through its own CSA-authenticated client and one shared batch client, instead of relying on Beholder's chip client (which silently dropped telemetry unless `Telemetry.ChipIngressEndpoint` was also set). Added `TelemetryIngress.ChipIngressEndpoint` (default `legacy-telemetry.prod.telemetry.chain.link:443`) and `TelemetryIngress.ChipIngressInsecureConnection`. `TelemetryIngress.Endpoints` entries still select which chains send telemetry. `telemetry_client_messages_sent{endpoint="chip-ingress"}` now counts messages instead of batches. #added diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 30bc5c82a23..3a1b9632d34 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -104,9 +104,9 @@ LeaseRefreshInterval = '1s' # Default UniConn = false # Default # Logging toggles verbose logging of the raw telemetry messages being sent. Logging = false # Default -# BufferSize is the number of telemetry messages to buffer before dropping new ones. +# BufferSize is the number of telemetry messages to buffer before dropping new ones. Ignored when ChipIngressEnabled is true. BufferSize = 100 # Default -# MaxBatchSize is the maximum number of messages to batch into one telemetry request. +# MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. MaxBatchSize = 50 # Default # SendInterval determines how often batched telemetry is sent to the ingress server. SendInterval = '500ms' # Default @@ -114,8 +114,12 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default # UseBatchSend toggles sending telemetry to the ingress server using the batch client. UseBatchSend = true # Default -# ChipIngressEnabled enables sending telemetry to CHIP Ingress. +# ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry. ChipIngressEnabled = false # Default +# ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint used when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default +# ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. +ChipIngressInsecureConnection = false # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index 09e13a510cc..d3c07519469 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -112,6 +112,96 @@ func (_c *TelemetryIngress_ChipIngressEnabled_Call) RunAndReturn(run func() bool return _c } +// ChipIngressEndpoint provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressEndpoint() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressEndpoint") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// TelemetryIngress_ChipIngressEndpoint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressEndpoint' +type TelemetryIngress_ChipIngressEndpoint_Call struct { + *mock.Call +} + +// ChipIngressEndpoint is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressEndpoint() *TelemetryIngress_ChipIngressEndpoint_Call { + return &TelemetryIngress_ChipIngressEndpoint_Call{Call: _e.mock.On("ChipIngressEndpoint")} +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) Run(run func()) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) Return(_a0 string) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) RunAndReturn(run func() string) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressInsecureConnection provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressInsecureConnection() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressInsecureConnection") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// TelemetryIngress_ChipIngressInsecureConnection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressInsecureConnection' +type TelemetryIngress_ChipIngressInsecureConnection_Call struct { + *mock.Call +} + +// ChipIngressInsecureConnection is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressInsecureConnection() *TelemetryIngress_ChipIngressInsecureConnection_Call { + return &TelemetryIngress_ChipIngressInsecureConnection_Call{Call: _e.mock.On("ChipIngressInsecureConnection")} +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) Run(run func()) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) Return(_a0 bool) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) RunAndReturn(run func() bool) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Return(run) + return _c +} + // Endpoints provides a mock function with no fields func (_m *TelemetryIngress) Endpoints() []config.TelemetryIngressEndpoint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index 918af269601..d7b7e5c0ca5 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -15,6 +15,8 @@ type TelemetryIngress interface { UseBatchSend() bool Endpoints() []TelemetryIngressEndpoint ChipIngressEnabled() bool + ChipIngressEndpoint() string + ChipIngressInsecureConnection() bool } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 589233efaf2..f21ec9f63d1 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -973,15 +973,17 @@ func (d *DatabaseBackup) setFrom(f *DatabaseBackup) { } type TelemetryIngress struct { - UniConn *bool - Logging *bool - BufferSize *uint16 - MaxBatchSize *uint16 - SendInterval *commonconfig.Duration - SendTimeout *commonconfig.Duration - UseBatchSend *bool - Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` - ChipIngressEnabled *bool + UniConn *bool + Logging *bool + BufferSize *uint16 + MaxBatchSize *uint16 + SendInterval *commonconfig.Duration + SendTimeout *commonconfig.Duration + UseBatchSend *bool + Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` + ChipIngressEnabled *bool + ChipIngressEndpoint *string + ChipIngressInsecureConnection *bool } type TelemetryIngressEndpoint struct { @@ -1019,6 +1021,22 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressEnabled; v != nil { t.ChipIngressEnabled = v } + if v := f.ChipIngressEndpoint; v != nil { + t.ChipIngressEndpoint = v + } + if v := f.ChipIngressInsecureConnection; v != nil { + t.ChipIngressInsecureConnection = v + } +} + +func (t *TelemetryIngress) ValidateConfig() (err error) { + if t.ChipIngressEnabled == nil || !*t.ChipIngressEnabled { + return nil + } + if t.ChipIngressEndpoint == nil || *t.ChipIngressEndpoint == "" { + err = errors.Join(err, configutils.MissingError{Name: "ChipIngressEndpoint", Msg: "must be set when ChipIngressEnabled is true"}) + } + return err } type AuditLogger struct { diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index efa992654a7..09bcf8c97c0 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -85,6 +85,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities" "github.com/smartcontractkit/chainlink/v2/core/services/streams" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" "github.com/smartcontractkit/chainlink/v2/core/services/vrf" workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" @@ -539,7 +540,29 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err globalLogger.Info("Nurse service (automatic pprof profiling) is disabled") } - telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + var telemChipService synchronization.ChipIngressService + if ti := cfg.TelemetryIngress(); ti.ChipIngressEnabled() { + chipClient, chipErr := synchronization.NewChipIngressClient(synchronization.ChipIngressClientConfig{ + Endpoint: ti.ChipIngressEndpoint(), + InsecureConnection: ti.ChipIngressInsecureConnection(), + AuthHeaders: beholderAuthHeaders, + AuthHeadersTTL: cfg.Telemetry().AuthHeadersTTL(), + AuthPublicKeyHex: csaPubKeyHex, + AuthKeySigner: csaKeystore, + MeterProvider: beholder.GetClient().MeterProvider, + TracerProvider: beholder.GetClient().TracerProvider, + }) + if chipErr != nil { + return nil, fmt.Errorf("failed to create telemetry chip-ingress client: %w", chipErr) + } + telemChipService, chipErr = synchronization.NewChipIngressBatchClient(chipClient, csaPubKeyHex, ti.Logging(), globalLogger, ti.SendInterval(), ti.SendTimeout()) + if chipErr != nil { + _ = chipClient.Close() // the batch client never took ownership of chipClient + return nil, fmt.Errorf("failed to create telemetry chip-ingress batch client: %w", chipErr) + } + globalLogger.Infow("Telemetry will be sent via ChIP ingress", "endpoint", ti.ChipIngressEndpoint()) + } + telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, telemChipService, globalLogger) srvcs = append(srvcs, telemetryManager) backupCfg := cfg.Database().Backup() diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 21c50d967a0..5fe384cd5f1 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -60,6 +60,14 @@ func (t *telemetryIngressConfig) ChipIngressEnabled() bool { return *t.c.ChipIngressEnabled } +func (t *telemetryIngressConfig) ChipIngressEndpoint() string { + return *t.c.ChipIngressEndpoint +} + +func (t *telemetryIngressConfig) ChipIngressInsecureConnection() bool { + return *t.c.ChipIngressInsecureConnection +} + func (t *telemetryIngressEndpointConfig) Network() string { return *t.c.Network } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 7aa5645fccd..114980d3943 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -246,14 +246,16 @@ func TestConfig_Marshal(t *testing.T) { }, } full.TelemetryIngress = toml.TelemetryIngress{ - UniConn: new(false), - Logging: new(true), - BufferSize: new(uint16(1234)), - MaxBatchSize: new(uint16(4321)), - SendInterval: commonconfig.MustNewDuration(time.Minute), - SendTimeout: commonconfig.MustNewDuration(5 * time.Second), - UseBatchSend: new(true), - ChipIngressEnabled: new(false), + UniConn: new(false), + Logging: new(true), + BufferSize: new(uint16(1234)), + MaxBatchSize: new(uint16(4321)), + SendInterval: commonconfig.MustNewDuration(time.Minute), + SendTimeout: commonconfig.MustNewDuration(5 * time.Second), + UseBatchSend: new(true), + ChipIngressEnabled: new(false), + ChipIngressEndpoint: new("legacy-telemetry.example:443"), + ChipIngressInsecureConnection: new(true), Endpoints: []toml.TelemetryIngressEndpoint{ { Network: new("EVM"), @@ -939,6 +941,8 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.example:443' +ChipIngressInsecureConnection = true [[TelemetryIngress.Endpoints]] Network = 'EVM' @@ -1504,7 +1508,13 @@ func TestConfig_Validate(t *testing.T) { - 0.Nodes.1.Name: invalid value (ton-test): duplicate - must be unique - 0: 2 errors: - Enabled: invalid value (1): expected bool - - ChainID: missing: required for all chains`}, + - ChainID: missing: required for all chains`, + }, + {name: "TelemetryIngress chip enabled without endpoint", toml: ` +[TelemetryIngress] +ChipIngressEnabled = true +ChipIngressEndpoint = '' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: missing: must be set when ChipIngressEnabled is true`}, } { t.Run(tt.name, func(t *testing.T) { var c Config diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index c1459f2bd6e..3a5406cce6a 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -44,6 +44,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 1e3ea6346ee..4335f8bcbba 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -44,6 +44,8 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.example:443' +ChipIngressInsecureConnection = true [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index dba938357aa..7528256f31e 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -44,6 +44,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = true diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index b2b385a0573..8a487f35ea7 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -3,106 +3,173 @@ package synchronization import ( "context" "fmt" - "sync" + "strconv" + "sync/atomic" "time" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + chipingress "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + batch "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + chipingresspb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink-common/pkg/timeutil" ) +const ( + chipIngress = "chip-ingress" // endpoint label on telemetry_client_* metrics + // legacyTelemetryBatchClientName is the client_name label on chip_ingress.batch.* metrics. + legacyTelemetryBatchClientName = "legacy_telemetry" + // Shared-buffer sizing mirrors chainlink-common beholder.DefaultConfig() chip batch defaults. + chipIngressMessageBufferSize = 10_000 + chipIngressMaxBatchSize = 1_000 + chipIngressMaxConcurrentSends = 10 +) + type chipIngressBatchClient struct { services.Service eng *services.Engine - chipClient chipingress.Client - - logging bool - - telemBufferSize uint - telemMaxBatchSize uint - telemSendInterval time.Duration - telemSendTimeout time.Duration - - workers map[string]*chipIngressBatchWorker - workersMutex sync.RWMutex + chipClient chipingress.Client // used only for health Ping; owned and closed by batchClient.Stop + batchClient *batch.Client + csaPubKeyHex string + logging bool + dropCount atomic.Uint32 + errorCount atomic.Uint32 } -// NewChipIngressBatchClient returns a client backed by chipingress.Client that -// can send telemetry to the chip ingress server -func NewChipIngressBatchClient(chipClient chipingress.Client, logging bool, lggr logger.Logger, telemBufferSize uint, telemMaxBatchSize uint, telemSendInterval time.Duration, telemSendTimeout time.Duration) ChipIngressService { +// NewChipIngressBatchClient returns a client backed by the shared +// chainlink-common chip-ingress batch client that can send telemetry to the +// chip ingress server. The batch client owns chipClient: Stop flushes the +// buffer and then closes the underlying gRPC connection. +func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, logging bool, lggr logger.Logger, sendInterval, sendTimeout time.Duration) (ChipIngressService, error) { + batchClient, err := batch.NewBatchClient(chipClient, + batch.WithMessageBuffer(chipIngressMessageBufferSize), + batch.WithBatchSize(chipIngressMaxBatchSize), + batch.WithMaxConcurrentSends(chipIngressMaxConcurrentSends), + batch.WithBatchInterval(sendInterval), + batch.WithMaxPublishTimeout(sendTimeout), + batch.WithClientName(legacyTelemetryBatchClientName), + // Send builds a fresh event per message and never touches it after queueing, + // so skip the defensive per-message proto.Clone. + batch.WithEventClone(false), + ) + if err != nil { + return nil, fmt.Errorf("failed to create chip-ingress batch client: %w", err) + } + c := &chipIngressBatchClient{ - telemBufferSize: telemBufferSize, - telemMaxBatchSize: telemMaxBatchSize, - telemSendInterval: telemSendInterval, - telemSendTimeout: telemSendTimeout, - chipClient: chipClient, - logging: logging, - workers: make(map[string]*chipIngressBatchWorker), + chipClient: chipClient, + batchClient: batchClient, + csaPubKeyHex: csaPubKeyHex, + logging: logging, } c.Service, c.eng = services.Config{ Name: "ChipIngressBatchClient", Start: c.start, + Close: c.close, }.NewServiceEngine(lggr) - return c + return c, nil } // start initializes the chip ingress batch client and starts health monitoring func (cc *chipIngressBatchClient) start(ctx context.Context) error { + cc.batchClient.Start(ctx) cc.startHealthMonitoring(ctx, cc.chipClient) return nil } -// Send directs incoming telemetry messages to the worker responsible for pushing it to -// the ingress server. If the worker telemetry buffer is full, messages are dropped -// and a warning is logged. +// close stops the batch client: it flushes queued messages (up to the shutdown +// timeout) and then closes the underlying chip-ingress client. +func (cc *chipIngressBatchClient) close() error { + cc.batchClient.Stop() + return nil +} + +// Send converts a telemetry payload into a CloudEvent and queues it on the +// shared batch client. Queueing is non-blocking: if the shared buffer is full +// the message is dropped and a warning is logged with exponential back-off. func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload) { - worker := cc.findOrCreateWorker(payload) + now := time.Now() + ev, err := cc.payloadToEvent(payload, now) + if err != nil { + TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() + cc.eng.Warnw("failed to build CloudEvent for ChIP ingress", "error", err, "contractID", payload.ContractID, "telemType", payload.TelemType) + return + } - select { - case worker.chTelemetry <- payload: - worker.dropMessageCount.Store(0) - case <-ctx.Done(): + err = cc.batchClient.QueueMessage(ev, func(sendErr error) { + if sendErr == nil { + TelemetryClientMessagesSent.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() + if cc.logging { + cc.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", payload.ContractID, "telemType", payload.TelemType) + } + return + } + TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() + count := cc.errorCount.Add(1) + if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + cc.eng.Warnw("Could not send telemetry via ChIP ingress", + "error", sendErr, + "errorCode", batch.ErrorCodeFor(sendErr), + "telemType", payload.TelemType, + "errorCount", count) + } + }) + if err != nil { + // batch.ErrMessageBufferFull or batch.ErrClientShutdown + TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() + count := cc.dropCount.Add(1) + if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + cc.eng.Warnw("dropping telemetry message for ChIP ingress", + "error", err, + "contractID", payload.ContractID, + "telemType", payload.TelemType, + "droppedCount", count) + } return - default: - worker.logBufferFullWithExpBackoff(payload) } + cc.dropCount.Store(0) } -// findOrCreateWorker finds a worker by ContractID or creates a new one if none exists. -// The number of workers is naturally bounded by the number of unique (ContractID, TelemetryType) -// pairs, which is determined by the jobs configured on the node. Each job has a fixed contract -// and telemetry type, so the worker count is proportional to the number of active jobs. -func (cc *chipIngressBatchClient) findOrCreateWorker(payload TelemPayload) *chipIngressBatchWorker { - cc.workersMutex.Lock() - defer cc.workersMutex.Unlock() - - workerKey := fmt.Sprintf("%s_%s", payload.ContractID, payload.TelemType) - worker, found := cc.workers[workerKey] - - if !found { - worker = NewChipIngressBatchWorker( - cc.telemMaxBatchSize, - cc.telemSendTimeout, - cc.chipClient, - make(chan TelemPayload, cc.telemBufferSize), - payload.ContractID, - payload.TelemType, - cc.eng, - cc.logging, - ) - cc.eng.GoTick(timeutil.NewTicker(func() time.Duration { - return cc.telemSendInterval - }), worker.Send) - cc.workers[workerKey] = worker - - TelemetryClientWorkers.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() +// payloadToEvent converts a telemetry payload into a CloudEvent with the +// OTI-parity extension set. sentat and receivedat are both stamped with now +// (enqueue time); chip-ingress legacy mode may overwrite receivedat with the +// true server-receive time later. +func (cc *chipIngressBatchClient) payloadToEvent(payload TelemPayload, now time.Time) (*chipingress.CloudEventPb, error) { + domain := payload.Domain + entity := payload.Entity + if domain == "" || entity == "" { + var err error + domain, entity, err = TelemetryTypeToDomainAndEntity(payload.TelemType) + if err != nil { + return nil, fmt.Errorf("missing domain/entity for telemType %s: %w", payload.TelemType, err) + } + } + + event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, map[string]any{ + "time": now.UTC(), + "datacontenttype": "application/protobuf", + }) + if err != nil { + return nil, fmt.Errorf("failed creating CloudEvent: %w", err) } - return worker + event.SetExtension("legacytelemetry", "true") + event.SetExtension("telemetrytype", string(payload.TelemType)) + event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) + event.SetExtension("networkname", payload.Network) + event.SetExtension("contractid", payload.ContractID) + event.SetExtension("csapublickey", cc.csaPubKeyHex) + event.SetExtension("partitionkey", payload.ContractID+"-"+cc.csaPubKeyHex) + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) + event.SetExtension("receivedat", strconv.FormatInt(now.UnixMilli(), 10)) + // nodeoperatorname / nodename are stamped server-side by chip-ingress via + // csa-auth (WithNOPLookup); the node must not send them, or chip-ingress + // would emit duplicate conflicting headers. nodeoperatorkey is not known to + // the node at all (spike INFOPLAT-19426). + + return chipingress.EventToProto(event) } // startHealthMonitoring starts a goroutine to monitor the connection state and update other relevant metrics every 5 seconds @@ -112,7 +179,7 @@ func (cc *chipIngressBatchClient) startHealthMonitoring(_ context.Context, chipC }), func(ctx context.Context) { connected := float64(0) pingCtx, pingCancel := context.WithTimeout(ctx, 2*time.Second) - _, err := chipClient.Ping(pingCtx, &pb.EmptyRequest{}) + _, err := chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}) pingCancel() if err == nil { connected = float64(1) diff --git a/core/services/synchronization/chip_ingress_batch_client_test.go b/core/services/synchronization/chip_ingress_batch_client_test.go index 691602f427f..8cdc2f0e46c 100644 --- a/core/services/synchronization/chip_ingress_batch_client_test.go +++ b/core/services/synchronization/chip_ingress_batch_client_test.go @@ -1,347 +1,220 @@ -package synchronization_test +package synchronization import ( "context" - "sync/atomic" + "slices" + "strconv" + "sync" "testing" "time" - "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" chipingressmocks "github.com/smartcontractkit/chainlink-common/pkg/chipingress/mocks" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) -func TestChipIngressBatchClient_HappyPath(t *testing.T) { - g := gomega.NewWithT(t) - - // Create mocks - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - // Wire up the chip ingress client with instant interval for testing - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - // Create telemetry payloads for different contracts - telemPayload1 := synchronization.TelemPayload{ - Telemetry: []byte("Mock telem 1"), - ContractID: "0x1", - TelemType: synchronization.OCR, +// newTestPayload returns a payload with Domain/Entity unset so the +// TelemetryTypeToDomainAndEntity fallback applies. +func newTestPayload(contractID string) TelemPayload { + return TelemPayload{ + Telemetry: []byte("telemetry-payload"), + TelemType: OCR2Median, + ContractID: contractID, ChainSelector: 12345, - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - Network: "ethereum-mainnet", + Network: "EVM", } - telemPayload3 := synchronization.TelemPayload{ - Telemetry: []byte("Mock telem 3"), - ContractID: "0x3", - TelemType: synchronization.OCR2Functions, - ChainSelector: 67890, - Domain: "functions", - Entity: "ocr.v2.functions.telemetry", - Network: "polygon-mainnet", - } - - // Assert telemetry payloads for each contract are correctly sent to chip ingress - var contractCounter1 atomic.Uint32 - var contractCounter3 atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - - for _, protoEvent := range batch.Events { - event, err := chipingress.ProtoToEvent(protoEvent) - require.NoError(t, err) - - attrs := protoEvent.GetAttributes() - contractID := attrs["contractid"].GetCeString() - - if contractID == "0x1" { - contractCounter1.Add(1) - assert.Equal(t, telemPayload1.Telemetry, event.Data()) - assert.Equal(t, string(synchronization.OCR), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, telemPayload1.Network, attrs["networkname"].GetCeString()) - assert.Empty(t, attrs["nodeoperatorname"].GetCeString()) - assert.Empty(t, attrs["nodename"].GetCeString()) - assert.Empty(t, attrs["nodecsapublickey"].GetCeString()) - } - if contractID == "0x3" { - contractCounter3.Add(1) - assert.Equal(t, telemPayload3.Telemetry, event.Data()) - assert.Equal(t, string(synchronization.OCR2Functions), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, telemPayload3.Network, attrs["networkname"].GetCeString()) - assert.Empty(t, attrs["nodeoperatorname"].GetCeString()) - assert.Empty(t, attrs["nodename"].GetCeString()) - assert.Empty(t, attrs["nodecsapublickey"].GetCeString()) - } - } - }) +} - // Send telemetry - testCtx := t.Context() - chipIngressClient.Send(testCtx, telemPayload1) - chipIngressClient.Send(testCtx, telemPayload3) - time.Sleep(sendInterval * 2) - chipIngressClient.Send(testCtx, telemPayload1) - chipIngressClient.Send(testCtx, telemPayload1) +// publishedBatches collects PublishBatch requests; the batch client publishes +// from its own goroutines, so access is guarded. +type publishedBatches struct { + mu sync.Mutex + batches [][]*chipingress.CloudEventPb +} - // Wait for the telemetry to be handled - g.Eventually(func() []uint32 { - return []uint32{contractCounter1.Load(), contractCounter3.Load()} - }, testutils.WaitTimeout(t)).Should(gomega.Equal([]uint32{3, 1})) +func (p *publishedBatches) add(events []*chipingress.CloudEventPb) { + p.mu.Lock() + defer p.mu.Unlock() + p.batches = append(p.batches, events) } -func TestChipIngressBatchClient_MultipleBatches(t *testing.T) { - g := gomega.NewWithT(t) +func (p *publishedBatches) snapshot() [][]*chipingress.CloudEventPb { + p.mu.Lock() + defer p.mu.Unlock() + return slices.Clone(p.batches) +} +func setupBatchClientTest(t *testing.T) (*chipingressmocks.Client, ChipIngressService, *publishedBatches) { + t.Helper() chipClient := chipingressmocks.NewClient(t) chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() + // batch.Client.Stop() closes the underlying client. + chipClient.On("Close").Return(nil).Maybe() + published := &publishedBatches{} + chipClient.EXPECT().PublishBatch(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(_ context.Context, batch *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + published.add(batch.Events) + // The batch client validates positional results: one per event. + results := make([]*chipingress.PublishResult, len(batch.Events)) + for i := range results { + results[i] = &chipingress.PublishResult{} + } + return &chipingress.PublishResponse{Results: results}, nil + }).Maybe() - sendInterval := time.Millisecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xabc", - TelemType: synchronization.OCR2Median, - ChainSelector: 12345, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", - } + client, err := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) + require.NoError(t, err) + servicetest.Run(t, client) + return chipClient, client, published +} - var batchCount atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batchCount.Add(1) - }) +// awaitFirstEvent waits for exactly one published batch holding one event and decodes it. +func awaitFirstEvent(t *testing.T, published *publishedBatches) chipingress.CloudEvent { + t.Helper() + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Len(c, published.snapshot(), 1) + }, testutils.WaitTimeout(t), 10*time.Millisecond) + batches := published.snapshot() + require.Len(t, batches[0], 1) + event, err := chipingress.ProtoToEvent(batches[0][0]) + require.NoError(t, err) + return event +} - testCtx := t.Context() - // Send multiple messages to trigger multiple batches - for i := range 10 { - chipIngressClient.Send(testCtx, telemPayload) - if i%3 == 0 { - time.Sleep(sendInterval * 2) // Allow batch to be sent - } - } +func sentCount(telemType string) float64 { + return testutil.ToFloat64(TelemetryClientMessagesSent.WithLabelValues(chipIngress, telemType)) +} - // Wait for batches to be sent - g.Eventually(batchCount.Load, testutils.WaitTimeout(t)).Should(gomega.BeNumerically(">=", 2)) +func sendErrorCount(telemType string) float64 { + return testutil.ToFloat64(TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, telemType)) } -func TestChipIngressBatchClient_DifferentTelemetryTypes(t *testing.T) { - g := gomega.NewWithT(t) +func droppedCount(telemType string) float64 { + return testutil.ToFloat64(TelemetryClientMessagesDropped.WithLabelValues(chipIngress, telemType)) +} - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() +func TestChipIngressBatchClient_Extensions(t *testing.T) { + t.Parallel() + _, client, published := setupBatchClientTest(t) - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) + client.Send(t.Context(), newTestPayload("0xabc")) + event := awaitFirstEvent(t, published) - // Create payloads with different telemetry types but same contract - payloadOCR := synchronization.TelemPayload{ - Telemetry: []byte("OCR telemetry"), - ContractID: "0x123", - TelemType: synchronization.OCR, - ChainSelector: 1, - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - Network: "ethereum-mainnet", + // Exact extension set from the OTI-parity table plus the ones the + // library stamps (seqnum by the batch client, recordedtime by NewEvent). + extNames := make([]string, 0, len(event.Extensions())) + for name := range event.Extensions() { + extNames = append(extNames, name) } - - payloadOCR2 := synchronization.TelemPayload{ - Telemetry: []byte("OCR2 telemetry"), - ContractID: "0x123", - TelemType: synchronization.OCR2Median, - ChainSelector: 1, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", + assert.ElementsMatch(t, []string{ + "legacytelemetry", "telemetrytype", "chainselector", "networkname", + "contractid", "csapublickey", "partitionkey", "sentat", "receivedat", + "seqnum", "recordedtime", + }, extNames) + + exts := event.Extensions() + assert.Equal(t, "true", exts["legacytelemetry"]) + assert.Equal(t, string(OCR2Median), exts["telemetrytype"]) + assert.Equal(t, "12345", exts["chainselector"]) + assert.Equal(t, "EVM", exts["networkname"]) + assert.Equal(t, "0xabc", exts["contractid"]) + assert.Equal(t, "deadbeef", exts["csapublickey"]) + assert.Equal(t, "0xabc-deadbeef", exts["partitionkey"]) + + // sentat is unix nanos; receivedat is unix millis of the same instant. + sentAt, err := strconv.ParseInt(exts["sentat"].(string), 10, 64) + require.NoError(t, err) + receivedAt, err := strconv.ParseInt(exts["receivedat"].(string), 10, 64) + require.NoError(t, err) + assert.Positive(t, sentAt) + assert.Equal(t, sentAt/1_000_000, receivedAt) + assert.WithinDuration(t, time.Now(), time.Unix(0, sentAt), time.Minute) + + // nodeoperator* / nodename / nodecsapublickey must be absent: chip-ingress + // stamps them server-side; sending them would duplicate headers. + for _, absent := range []string{"nodeoperatorname", "nodename", "nodecsapublickey", "nodeoperatorkey"} { + assert.NotContains(t, exts, absent) } - - var ocrCount, ocr2Count atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - for _, protoEvent := range batch.Events { - attrs := protoEvent.GetAttributes() - telemType := attrs["telemetrytype"].GetCeString() - if telemType == string(synchronization.OCR) { - ocrCount.Add(1) - } else if telemType == string(synchronization.OCR2Median) { - ocr2Count.Add(1) - } - } - }) - - testCtx := t.Context() - chipIngressClient.Send(testCtx, payloadOCR) - chipIngressClient.Send(testCtx, payloadOCR2) - - g.Eventually(func() []uint32 { - return []uint32{ocrCount.Load(), ocr2Count.Load()} - }, testutils.WaitTimeout(t)).Should(gomega.Equal([]uint32{1, 1})) } -func TestChipIngressBatchClient_ContextCancellation(t *testing.T) { - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Maybe() - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xdef", - TelemType: synchronization.OCR2Functions, - ChainSelector: 67890, - Domain: "functions", - Entity: "ocr.v2.functions.telemetry", - Network: "ethereum-mainnet", - } +func TestChipIngressBatchClient_DomainEntityFallback(t *testing.T) { + t.Parallel() + _, client, published := setupBatchClientTest(t) - // Create a cancelled context - ctx, cancel := context.WithCancel(t.Context()) - cancel() - - // Should not panic or block when context is cancelled - chipIngressClient.Send(ctx, telemPayload) + client.Send(t.Context(), newTestPayload("0xabc")) + event := awaitFirstEvent(t, published) + domain, entity, err := TelemetryTypeToDomainAndEntity(OCR2Median) + require.NoError(t, err) + assert.Equal(t, domain, event.Source()) + assert.Equal(t, entity, event.Type()) } -func TestChipIngressBatchClient_WorkerReuse(t *testing.T) { - g := gomega.NewWithT(t) +func TestChipIngressBatchClient_UnmappedTypeDropped(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics + _, client, published := setupBatchClientTest(t) - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) + payload := newTestPayload("0xabc") + payload.TelemType = LLOReport // no chip domain/entity mapping + payload.Domain = "" + payload.Entity = "" + before := droppedCount(string(payload.TelemType)) + // The event is rejected synchronously in Send, before it reaches the queue. + client.Send(t.Context(), payload) - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xabc", - TelemType: synchronization.OCR2Median, - ChainSelector: 99999, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", - } + assert.InDelta(t, 1, droppedCount(string(payload.TelemType))-before, 0.001) + assert.Empty(t, published.snapshot(), "PublishBatch must never be called for unmapped types") +} - var messageCount atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - // #nosec G115 -- len() returns non-negative int, safe to convert to uint32 - messageCount.Add(uint32(len(batch.Events))) - }) +func TestChipIngressBatchClient_SentCountsMessages(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics + _, client, _ := setupBatchClientTest(t) - testCtx := t.Context() - // Send multiple messages with same contract and type - should reuse worker - for range 5 { - chipIngressClient.Send(testCtx, telemPayload) + payload := newTestPayload("0xabc") + const n = 5 + before := sentCount(string(payload.TelemType)) + for range n { + client.Send(t.Context(), payload) } - - // Wait for all messages to be sent - g.Eventually(messageCount.Load, testutils.WaitTimeout(t)).Should(gomega.Equal(uint32(5))) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, float64(n), sentCount(string(payload.TelemType))-before, 0.001, + "Sent must count messages, not batches") + }, testutils.WaitTimeout(t), 10*time.Millisecond) } -func TestChipIngressBatchClient_ChainSelectorInAttributes(t *testing.T) { - g := gomega.NewWithT(t) - +func TestChipIngressBatchClient_SendErrorCountsMessages(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics chipClient := chipingressmocks.NewClient(t) chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - expectedChainSelector := uint64(123456789) - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xtest", - TelemType: synchronization.OCR3CCIPCommit, - ChainSelector: expectedChainSelector, - Domain: "ccip", - Entity: "ocr.v3.ccip.commit.telemetry", - Network: "ethereum-mainnet", + chipClient.On("Close").Return(nil).Maybe() + chipClient.EXPECT().PublishBatch(mock.Anything, mock.Anything, mock.Anything). + Return(nil, assert.AnError).Maybe() + + client, err := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) + require.NoError(t, err) + servicetest.Run(t, client) + + payload := newTestPayload("0xabc") + const n = 3 + before := sendErrorCount(string(payload.TelemType)) + for range n { + client.Send(t.Context(), payload) } - - var capturedChainSelector atomic.Value - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - if len(batch.Events) > 0 { - attrs := batch.Events[0].GetAttributes() - capturedChainSelector.Store(attrs["chainselector"].GetCeString()) - } - }) - - testCtx := t.Context() - chipIngressClient.Send(testCtx, telemPayload) - - g.Eventually(func() string { - v := capturedChainSelector.Load() - if v == nil { - return "" - } - return v.(string) - }, testutils.WaitTimeout(t)).Should(gomega.Equal("123456789")) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, float64(n), sendErrorCount(string(payload.TelemType))-before, 0.001, + "SendErrors must count messages, not batches") + }, testutils.WaitTimeout(t), 10*time.Millisecond) } -func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - g := gomega.NewWithT(t) - - chipClient := chipingressmocks.NewClient(t) - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - - // Mock Ping to succeed initially - var pingCallCount atomic.Uint32 - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Run(func(args mock.Arguments) { - pingCallCount.Add(1) - }) - - // Start the client which should start health monitoring - servicetest.Run(t, chipIngressClient) - - // Wait for at least 2 ping calls to verify health monitoring is running - g.Eventually(pingCallCount.Load, testutils.WaitTimeout(t), 100*time.Millisecond).Should(gomega.BeNumerically(">=", 2)) -} - -func TestChipIngressBatchClient_HealthMonitoring_PingFailure(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - g := gomega.NewWithT(t) - - chipClient := chipingressmocks.NewClient(t) - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - - // Mock Ping to fail - var pingCallCount atomic.Uint32 - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(nil, assert.AnError).Run(func(args mock.Arguments) { - pingCallCount.Add(1) - }) - - // Start the client which should start health monitoring - servicetest.Run(t, chipIngressClient) +func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // resets a package-global prometheus gauge + // The status gauge is package-global; reset it so a value left by another test can't satisfy the check. + TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(0) + _, _, _ = setupBatchClientTest(t) - // Wait for at least 2 ping calls to verify health monitoring continues despite failures - g.Eventually(pingCallCount.Load, testutils.WaitTimeout(t), 100*time.Millisecond).Should(gomega.BeNumerically(">=", 2)) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, 1, testutil.ToFloat64(TelemetryClientConnectionStatus.WithLabelValues(chipIngress)), 0.001) + }, testutils.WaitTimeout(t), 10*time.Millisecond) } diff --git a/core/services/synchronization/chip_ingress_batch_worker.go b/core/services/synchronization/chip_ingress_batch_worker.go deleted file mode 100644 index c59252a79d6..00000000000 --- a/core/services/synchronization/chip_ingress_batch_worker.go +++ /dev/null @@ -1,159 +0,0 @@ -package synchronization - -import ( - "context" - "fmt" - "strconv" - "sync/atomic" - "time" - - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -const ( - chipIngress = "chip-ingress" -) - -// chipIngressBatchWorker mirrors telemetryIngressBatchWorker but targets the ChIP ingress client. -// A worker is created per (contractID, telemetry type) pair. -type chipIngressBatchWorker struct { - services.Service - - maxBatchSize uint - sendTimeout time.Duration - chipClient chipingress.Client - chTelemetry chan TelemPayload - contractID string - telemType TelemetryType - logging bool - lggr logger.Logger - dropMessageCount atomic.Uint32 -} - -// NewChipIngressBatchWorker returns a worker for a given contractID that can send -// telemetry to the chip ingress server via PublishBatch. -func NewChipIngressBatchWorker( - maxBatchSize uint, - sendTimeout time.Duration, - chipClient chipingress.Client, - chTelemetry chan TelemPayload, - contractID string, - telemType TelemetryType, - lggr logger.Logger, - logging bool, -) *chipIngressBatchWorker { - return &chipIngressBatchWorker{ - maxBatchSize: maxBatchSize, - sendTimeout: sendTimeout, - chipClient: chipClient, - chTelemetry: chTelemetry, - contractID: contractID, - telemType: telemType, - logging: logging, - lggr: logger.Named(lggr, "ChipIngressBatchWorker"), - } -} - -// Send sends batched telemetry to the chip ingress server on an interval. -func (cw *chipIngressBatchWorker) Send(ctx context.Context) { - if len(cw.chTelemetry) == 0 { - return - } - - batch := cw.BuildCloudEventBatch() - if batch == nil || len(batch.Events) == 0 { - return - } - - ctx, cancel := context.WithTimeout(ctx, cw.sendTimeout) - defer cancel() - - _, err := cw.chipClient.PublishBatch(ctx, batch) - if err != nil { - cw.lggr.Warnf("Could not send telemetry via ChIP ingress: %v", err) - TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - return - } - - TelemetryClientMessagesSent.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - if cw.logging { - cw.lggr.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", cw.contractID, "telemType", cw.telemType, "batchSize", len(batch.Events)) - } -} - -// logBufferFullWithExpBackoff logs messages at 1,2,4,8,16,32,64,100,200,300,... when the buffer is full. -func (cw *chipIngressBatchWorker) logBufferFullWithExpBackoff(payload TelemPayload) { - count := cw.dropMessageCount.Add(1) - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { - cw.lggr.Warnw("chip ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) - } -} - -// BuildCloudEventBatch reads telemetry off the worker channel and packages it into a CloudEvent batch. -func (cw *chipIngressBatchWorker) BuildCloudEventBatch() *chipingress.CloudEventBatch { - var events []chipingress.CloudEvent - - // #nosec G115 -- maxBatchSize is uint, safe to convert to int for comparison - for len(cw.chTelemetry) > 0 && len(events) < int(cw.maxBatchSize) { - payload := <-cw.chTelemetry - event, err := cw.payloadToEvent(payload) - if err != nil { - cw.lggr.Warnw("failed to build CloudEvent for ChIP ingress", "error", err, "contractID", payload.ContractID, "telemType", payload.TelemType) - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - continue - } - events = append(events, event) - } - - if len(events) == 0 { - return nil - } - - batch, err := chipingress.EventsToBatch(events) - if err != nil { - cw.lggr.Warnw("failed to convert CloudEvents to batch", "error", err) - return nil - } - - return batch -} - -func (cw *chipIngressBatchWorker) payloadToEvent(payload TelemPayload) (chipingress.CloudEvent, error) { - domain := payload.Domain - entity := payload.Entity - if domain == "" || entity == "" { - var err error - domain, entity, err = TelemetryTypeToDomainAndEntity(payload.TelemType) - if err != nil { - return chipingress.CloudEvent{}, fmt.Errorf("missing domain/entity for telemType %s: %w", payload.TelemType, err) - } - } - - now := time.Now().UTC() - - attrs := map[string]any{ - "time": now, - "datacontenttype": "application/protobuf", - } - - event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, attrs) - if err != nil { - return chipingress.CloudEvent{}, fmt.Errorf("failed creating CloudEvent: %w", err) - } - - event.SetExtension("telemetrytype", string(payload.TelemType)) - event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) - event.SetExtension("contractid", payload.ContractID) - event.SetExtension("networkname", payload.Network) - event.SetExtension("sentat", now.Format(time.RFC3339)) - // These attributes need to be set in chip-ingress server side - event.SetExtension("nodeoperatorname", "") - event.SetExtension("nodename", "") - event.SetExtension("nodecsapublickey", "") - - return event, nil -} diff --git a/core/services/synchronization/chip_ingress_batch_worker_test.go b/core/services/synchronization/chip_ingress_batch_worker_test.go deleted file mode 100644 index 8dc68ad5fbd..00000000000 --- a/core/services/synchronization/chip_ingress_batch_worker_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package synchronization - -import ( - "context" - "testing" - "time" - - cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" - "github.com/smartcontractkit/chainlink/v2/core/logger" -) - -type noopChipIngressPublisher struct{} - -func (noopChipIngressPublisher) Publish(ctx context.Context, event *cepb.CloudEvent, opts ...grpc.CallOption) (*pb.PublishResponse, error) { - return &pb.PublishResponse{}, nil -} - -func (noopChipIngressPublisher) PublishBatch(ctx context.Context, batch *pb.CloudEventBatch, opts ...grpc.CallOption) (*pb.PublishResponse, error) { - return &pb.PublishResponse{}, nil -} - -func (noopChipIngressPublisher) Ping(ctx context.Context, req *pb.EmptyRequest, opts ...grpc.CallOption) (*pb.PingResponse, error) { - return &pb.PingResponse{}, nil -} - -func (noopChipIngressPublisher) StreamEvents(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[pb.StreamEventsRequest, pb.StreamEventsResponse], error) { - return nil, nil -} - -func (noopChipIngressPublisher) RegisterSchema(ctx context.Context, req *pb.RegisterSchemaRequest, opts ...grpc.CallOption) (*pb.RegisterSchemaResponse, error) { - return &pb.RegisterSchemaResponse{}, nil -} - -func (noopChipIngressPublisher) Close() error { - return nil -} - -func (noopChipIngressPublisher) RegisterSchemas(ctx context.Context, schemas ...*pb.Schema) (map[string]int, error) { - return nil, nil -} - -func TestChipIngressBatchWorker_BuildCloudEventBatch(t *testing.T) { - maxBatchSize := 3 - chTelemetry := make(chan TelemPayload, 10) - // #nosec G115 -- maxBatchSize is a small positive constant, safe to convert to uint - worker := NewChipIngressBatchWorker( - uint(maxBatchSize), - time.Second, - noopChipIngressPublisher{}, - chTelemetry, - "0xabc", - OCR, - logger.TestLogger(t), - false, - ) - - payload := TelemPayload{ - Telemetry: []byte("payload-1"), - TelemType: OCR, - ContractID: "0xabc", - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - ChainSelector: 7700, - Network: "EVM", - ChainID: "1", - } - - // enqueue more payloads than maxBatchSize to ensure batching occurs - for range 5 { - chTelemetry <- payload - } - - batch1 := worker.BuildCloudEventBatch() - require.NotNil(t, batch1) - require.Len(t, batch1.Events, maxBatchSize) - assert.Len(t, chTelemetry, 5-maxBatchSize) - - evt, err := chipingress.ProtoToEvent(batch1.Events[0]) - require.NoError(t, err) - assert.Equal(t, "data-feeds", evt.Source()) - assert.Equal(t, "ocr.v1.telemetry", evt.Type()) - assert.Equal(t, []byte("payload-1"), evt.Data()) - - attrs := batch1.Events[0].GetAttributes() - require.Contains(t, attrs, "telemetrytype") - require.Contains(t, attrs, "chainselector") - assert.Equal(t, string(OCR), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, "7700", attrs["chainselector"].GetCeString()) - - batch2 := worker.BuildCloudEventBatch() - require.NotNil(t, batch2) - require.Len(t, batch2.Events, 2) - assert.Empty(t, chTelemetry) -} - -func TestChipIngressBatchWorker_BuildCloudEventBatchUsesMapping(t *testing.T) { - chTelemetry := make(chan TelemPayload, 1) - worker := NewChipIngressBatchWorker( - 1, - time.Second, - noopChipIngressPublisher{}, - chTelemetry, - "0xdef", - OCR, - logger.TestLogger(t), - false, - ) - - chTelemetry <- TelemPayload{ - Telemetry: []byte("payload"), - TelemType: OCR, - ContractID: "0xdef", - ChainSelector: 9001, - Network: "EVM", - ChainID: "137", - } - - batch := worker.BuildCloudEventBatch() - require.NotNil(t, batch) - require.Len(t, batch.Events, 1) - - evt, err := chipingress.ProtoToEvent(batch.Events[0]) - require.NoError(t, err) - assert.Equal(t, "data-feeds.telemetry.ocr", evt.Source()) -} diff --git a/core/services/synchronization/chip_ingress_client.go b/core/services/synchronization/chip_ingress_client.go new file mode 100644 index 00000000000..e9b5a0715f4 --- /dev/null +++ b/core/services/synchronization/chip_ingress_client.go @@ -0,0 +1,62 @@ +package synchronization + +import ( + "fmt" + "time" + + "go.opentelemetry.io/otel/metric" + "go.opentelemetry.io/otel/trace" + + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" +) + +// ChipIngressClientConfig configures the gRPC client for the legacy telemetry chip-ingress endpoint. +type ChipIngressClientConfig struct { + Endpoint string // host:port + InsecureConnection bool + AuthHeaders map[string]string // static CSA auth headers (keystore.BuildBeholderAuth) + AuthHeadersTTL time.Duration // >0 selects rotating CSA tokens (must be >= 10m); same value Beholder uses + AuthPublicKeyHex string + AuthKeySigner chipingress.Signer + MeterProvider metric.MeterProvider + TracerProvider trace.TracerProvider +} + +// NewChipIngressClient builds the chip-ingress client used for legacy telemetry. +func NewChipIngressClient(cfg ChipIngressClientConfig) (chipingress.Client, error) { + auth, err := chipingress.NewHeaderProvider(chipingress.HeaderProviderConfig{ + AuthHeaders: cfg.AuthHeaders, + AuthHeadersTTL: cfg.AuthHeadersTTL, + AuthPublicKeyHex: cfg.AuthPublicKeyHex, + AuthKeySigner: cfg.AuthKeySigner, + InsecureConnection: cfg.InsecureConnection, + }) + if err != nil { + return nil, fmt.Errorf("failed to build chip-ingress auth: %w", err) + } + + // The transport option must be applied before WithTokenAuth: WithTokenAuth + // reads the connection security at application time to decide whether its + // per-RPC credentials require TLS. + opts := []chipingress.Opt{} + if cfg.InsecureConnection { + opts = append(opts, chipingress.WithInsecureConnection()) + } else { + opts = append(opts, chipingress.WithTLS()) + } + if auth != nil { + // NewHeaderProvider returns nil only when there are no headers and no TTL, + // which cannot happen for the node (BuildBeholderAuth always returns headers), + // but keep the nil check for safety. + opts = append(opts, chipingress.WithTokenAuth(auth)) + } + opts = append(opts, + chipingress.WithNOPLookup(), + chipingress.WithMeterProvider(cfg.MeterProvider), + chipingress.WithTracerProvider(cfg.TracerProvider), + ) + + // grpc.NewClient connects lazily, so an unreachable endpoint does not fail + // startup; a malformed host:port does (that is intended). + return chipingress.NewClient(cfg.Endpoint, opts...) +} diff --git a/core/services/synchronization/chip_ingress_client_test.go b/core/services/synchronization/chip_ingress_client_test.go new file mode 100644 index 00000000000..c59f403d92c --- /dev/null +++ b/core/services/synchronization/chip_ingress_client_test.go @@ -0,0 +1,62 @@ +package synchronization + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + chipingresspb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" +) + +type capturingChipIngressServer struct { + chipingresspb.UnimplementedChipIngressServer + lastMD metadata.MD +} + +func (s *capturingChipIngressServer) Ping(ctx context.Context, _ *chipingresspb.EmptyRequest) (*chipingresspb.PingResponse, error) { + s.lastMD, _ = metadata.FromIncomingContext(ctx) + return &chipingresspb.PingResponse{}, nil +} + +func TestChipIngressClient_SendAuthHeaders(t *testing.T) { + t.Parallel() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + defer lis.Close() + + srv := grpc.NewServer() + capture := &capturingChipIngressServer{} + chipingresspb.RegisterChipIngressServer(srv, capture) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + client, err := NewChipIngressClient(ChipIngressClientConfig{ + Endpoint: lis.Addr().String(), + InsecureConnection: true, + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "x"}, + }) + require.NoError(t, err) + defer client.Close() + + _, err = client.Ping(t.Context(), &chipingresspb.EmptyRequest{}) + require.NoError(t, err) + + assert.NotNil(t, capture.lastMD) + assert.Equal(t, []string{"true"}, capture.lastMD.Get("x-include-nop-info")) + assert.Equal(t, []string{"x"}, capture.lastMD.Get("X-Beholder-Node-Auth-Token")) +} + +func TestChipIngressClient_InvalidAddress(t *testing.T) { + t.Parallel() + _, err := NewChipIngressClient(ChipIngressClientConfig{ + Endpoint: "no-port", + InsecureConnection: true, + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "x"}, + }) + require.Error(t, err) +} diff --git a/core/services/synchronization/helpers_test.go b/core/services/synchronization/helpers_test.go index 598ce4f31a9..b6dc38d3493 100644 --- a/core/services/synchronization/helpers_test.go +++ b/core/services/synchronization/helpers_test.go @@ -26,7 +26,7 @@ func NewTestTelemetryIngressBatchClient(t *testing.T, url *url.URL, serverPubKey return tc } -// NewTestChipIngressBatchClient calls NewChipIngressBatchClient and injects chipClient. -func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) ChipIngressService { - return NewChipIngressBatchClient(chipClient, logging, logger.TestLogger(t), 100, 50, sendInterval, time.Second) +// NewTestChipIngressBatchClient calls NewChipIngressBatchClient with a fixed test CSA key. +func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) (ChipIngressService, error) { + return NewChipIngressBatchClient(chipClient, "deadbeef", logging, logger.TestLogger(t), sendInterval, time.Second) } diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index 9b24d46c627..baaaca53591 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -1,16 +1,19 @@ package telemetry import ( + "context" + "fmt" "net/url" "strings" "time" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" "github.com/smartcontractkit/libocr/commontypes" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" common "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/config" @@ -26,15 +29,17 @@ type Manager struct { endpoints []*telemetryEndpoint ks keystore.CSA - logging bool - maxBatchSize uint - sendInterval time.Duration - sendTimeout time.Duration - uniConn bool - useBatchSend bool - MonitoringEndpointGenerator MonitoringEndpointGenerator + logging bool + maxBatchSize uint + sendInterval time.Duration + sendTimeout time.Duration + uniConn bool + useBatchSend bool - chipIngressClient chipingress.Client + chipService synchronization.ChipIngressService // nil when chip ingress is disabled + + chipIngressEndpoint string + configGauge otelmetric.Int64Gauge } type telemetryEndpoint struct { @@ -47,31 +52,50 @@ type telemetryEndpoint struct { } // NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints -func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *Manager { - var chipIngressClient chipingress.Client - if cfg.ChipIngressEnabled() { +// chipService is the shared chip-ingress service to use when ChipIngressEnabled is true; nil when the flag is off. +func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, chipService synchronization.ChipIngressService, lggr common.Logger) *Manager { + var chipIngressEndpoint string // "" when disabled + if chipService != nil { lggr.Info("ChIP Ingress is enabled for telemetry") - chipIngressClient = beholder.GetClient().Chip + chipIngressEndpoint = cfg.ChipIngressEndpoint() } m := &Manager{ - bufferSize: cfg.BufferSize(), - ks: csaKeyStore, - logging: cfg.Logging(), - maxBatchSize: cfg.MaxBatchSize(), - sendInterval: cfg.SendInterval(), - sendTimeout: cfg.SendTimeout(), - uniConn: cfg.UniConn(), - useBatchSend: cfg.UseBatchSend(), - chipIngressClient: chipIngressClient, + bufferSize: cfg.BufferSize(), + ks: csaKeyStore, + logging: cfg.Logging(), + maxBatchSize: cfg.MaxBatchSize(), + sendInterval: cfg.SendInterval(), + sendTimeout: cfg.SendTimeout(), + uniConn: cfg.UniConn(), + useBatchSend: cfg.UseBatchSend(), + chipService: chipService, + chipIngressEndpoint: chipIngressEndpoint, } m.Service, m.eng = services.Config{ Name: "TelemetryManager", + Start: func(ctx context.Context) error { + if err := m.recordConfigMetric(ctx); err != nil { + m.eng.Errorw("failed to record telemetry ingress config metric", "err", err) + } + m.eng.GoTick(services.TickerConfig{}.NewTicker(time.Hour), func(ctx context.Context) { + if err := m.recordConfigMetric(ctx); err != nil { + m.eng.Errorw("failed to record telemetry ingress config metric", "err", err) + } + }) + return nil + }, NewSubServices: func(lggr common.Logger) (subs []services.Service) { + if m.chipService != nil { + subs = append(subs, m.chipService) + } for _, e := range cfg.Endpoints() { - if sub, err := m.newEndpoint(e, lggr, cfg); err != nil { + sub, err := m.newEndpoint(e, lggr, cfg) + if err != nil { lggr.Error(err) - } else { + continue + } + if sub != nil { subs = append(subs, sub) } } @@ -91,7 +115,7 @@ func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contract return &NoopAgent{} } - if m.chipIngressClient != nil { + if m.chipService != nil { lggr := m.eng.Named("chip-ingress") adapter, err := NewChipIngressAgent(e.chipIngressClient, network, chainID, contractID, telemType, lggr) if err != nil { @@ -116,7 +140,7 @@ func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, return &NoopAgent{} } - if m.chipIngressClient != nil { + if m.chipService != nil { lggr := m.eng.Named("chip-ingress-multipletype") adapter, err := NewChipIngressAgentMultitype(e.chipIngressClient, network, chainID, contractID, lggr) if err != nil { @@ -156,19 +180,19 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) - if m.chipIngressClient != nil { - lggr.Infof("Using chip-ingress client for network %q chainID %q", e.Network(), e.ChainID()) - // When ChIP ingress is enabled, we create a ChipIngressService - chipIngressService := synchronization.NewChipIngressBatchClient(m.chipIngressClient, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout()) + if m.chipService != nil { + lggr.Infof("Using chip-ingress service for network %q chainID %q", e.Network(), e.ChainID()) + // When ChIP ingress is enabled, all endpoints share the one ChipIngressService + // (injected by the application); entries only select which chains send telemetry. te := telemetryEndpoint{ Network: strings.ToUpper(e.Network()), ChainID: strings.ToUpper(e.ChainID()), URL: e.URL(), PubKey: e.ServerPubKey(), - chipIngressClient: chipIngressService, + chipIngressClient: m.chipService, } m.endpoints = append(m.endpoints, &te) - return chipIngressService, nil + return nil, nil } // Otherwise use the traditional telemetry service @@ -191,6 +215,26 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log return te.client, nil } +// recordConfigMetric records the telemetry ingress config info metric, mirroring +// beholder's ConfigRecorder "config.info" pattern. +func (m *Manager) recordConfigMetric(ctx context.Context) error { + if m.configGauge == nil { + gauge, err := beholder.GetMeter().Int64Gauge("telemetry_ingress.config.info", + otelmetric.WithDescription("Telemetry ingress config info metric"), + otelmetric.WithUnit("{info}")) + if err != nil { + return fmt.Errorf("failed to create telemetry ingress config metric: %w", err) + } + m.configGauge = gauge + } + m.configGauge.Record(ctx, 1, otelmetric.WithAttributes( + attribute.Bool("chip_ingress_enabled", m.chipService != nil), + // "" when disabled + attribute.String("chip_ingress_endpoint", m.chipIngressEndpoint), + )) + return nil +} + func (m *Manager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { for _, e := range m.endpoints { if e.Network == strings.ToUpper(network) && e.ChainID == strings.ToUpper(chainID) { diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index 143066e0894..48b096c43bd 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -23,7 +23,7 @@ import ( mocks2 "github.com/smartcontractkit/chainlink/v2/core/services/synchronization/mocks" ) -func setupMockConfig(t *testing.T, useBatchSend, chipIngressEnabled bool) *mocks.TelemetryIngress { +func setupMockConfig(t *testing.T, useBatchSend bool) *mocks.TelemetryIngress { tic := mocks.NewTelemetryIngress(t) tic.On("BufferSize").Return(uint(123)) tic.On("Logging").Return(true) @@ -32,13 +32,14 @@ func setupMockConfig(t *testing.T, useBatchSend, chipIngressEnabled bool) *mocks tic.On("SendTimeout").Return(time.Second * 7) tic.On("UniConn").Return(true) tic.On("UseBatchSend").Return(useBatchSend) - tic.On("ChipIngressEnabled").Return(chipIngressEnabled) + // chip-ingress is selected by passing a non-nil ChipIngressService to NewManager. + tic.On("ChipIngressEndpoint").Return("chip-ingress.test:443").Maybe() return tic } func TestManagerAgents(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("network-1") te.On("ChainID").Return("network-1-chainID-1") @@ -51,14 +52,14 @@ func TestManagerAgents(t *testing.T) { ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) require.Equal(t, "*synchronization.telemetryIngressBatchClient", reflect.TypeOf(tm.endpoints[0].client).String()) me := tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) - tic = setupMockConfig(t, false, false) + tic = setupMockConfig(t, false) tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) - tm = NewManager(tic, ks, lggr) + tm = NewManager(tic, ks, nil, lggr) require.Equal(t, "*synchronization.telemetryIngressClient", reflect.TypeOf(tm.endpoints[0].client).String()) me = tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgent", reflect.TypeOf(me).String()) @@ -144,7 +145,7 @@ func TestNewManager(t *testing.T) { mockEndpoints = append(mockEndpoints, te) } - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(mockEndpoints) lggr, logObs := logger.TestLoggerObserved(t, zapcore.InfoLevel) @@ -154,7 +155,7 @@ func TestNewManager(t *testing.T) { key := csakey.MustNewV2XXXTestingOnly(big.NewInt(0)) ks.On("GetAll").Return([]csakey.KeyV2{key}, nil) ks.On("Get", key.ID()).Return(key, nil) - m := NewManager(tic, ks, lggr) + m := NewManager(tic, ks, nil, lggr) require.Equal(t, uint(123), m.bufferSize) require.Equal(t, ks, m.ks) @@ -197,13 +198,13 @@ func TestNewManager(t *testing.T) { } func TestCorrectEndpointRouting(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(nil) lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) type testEndpoint struct { network string @@ -283,50 +284,71 @@ func TestCorrectEndpointRouting(t *testing.T) { } } -func TestManager_ChipIngressClient(t *testing.T) { +func TestManager_ChipIngressService(t *testing.T) { + t.Parallel() t.Run("disabled chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + t.Parallel() + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(nil) lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) - assert.Nil(t, tm.chipIngressClient) + tm := NewManager(tic, ks, nil, lggr) + assert.Nil(t, tm.chipService) }) t.Run("enabled chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + t.Parallel() + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(nil) lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) - assert.NotNil(t, tm.chipIngressClient) + chipSvc := &MockChipIngressService{} + tm := NewManager(tic, ks, chipSvc, lggr) + assert.Same(t, chipSvc, tm.chipService) }) } func TestManager_ChipIngressEndpoint(t *testing.T) { - t.Run("creates chip ingress endpoint when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) - te := mocks.NewTelemetryIngressEndpoint(t) - te.On("Network").Return("EVM") - te.On("ChainID").Return("1") - te.On("ServerPubKey").Return("some-pubkey") - u, _ := url.Parse("http://chip-ingress.test") - te.On("URL").Return(u) - tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) + t.Parallel() + t.Run("all endpoints share one chip ingress service, started once", func(t *testing.T) { + t.Parallel() + tic := setupMockConfig(t, true) + endpoints := make([]config.TelemetryIngressEndpoint, 0, 2) + for _, chainID := range []string{"1", "137"} { + te := mocks.NewTelemetryIngressEndpoint(t) + te.On("Network").Return("EVM") + te.On("ChainID").Return(chainID) + te.On("ServerPubKey").Return("some-pubkey") + u, _ := url.Parse("http://chip-ingress.test") + te.On("URL").Return(u) + endpoints = append(endpoints, te) + } + tic.On("Endpoints").Return(endpoints) lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) + chipSvc := &MockChipIngressService{} + chipSvc.On("Start", mock.Anything).Return(nil).Once() + chipSvc.On("Close").Return(nil).Once() + chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() + + tm := NewManager(tic, ks, chipSvc, lggr) + require.Len(t, tm.endpoints, 2) + for _, e := range tm.endpoints { + assert.Same(t, chipSvc, e.chipIngressClient) + assert.Nil(t, e.client) + } - tm := NewManager(tic, ks, lggr) - require.Len(t, tm.endpoints, 1) - assert.NotNil(t, tm.endpoints[0].chipIngressClient) - assert.Nil(t, tm.endpoints[0].client) + require.NoError(t, tm.Start(t.Context())) + require.NoError(t, tm.Close()) + chipSvc.AssertExpectations(t) }) t.Run("creates traditional endpoint when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + t.Parallel() + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -341,7 +363,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) require.Len(t, tm.endpoints, 1) assert.Nil(t, tm.endpoints[0].chipIngressClient) assert.NotNil(t, tm.endpoints[0].client) @@ -350,7 +372,7 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { t.Run("returns chip ingress agent when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -362,13 +384,13 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) t.Run("SendLog sends telemetry through chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -379,9 +401,16 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.DebugLevel) ks := keymocks.NewCSA(t) - - tm := NewManager(tic, ks, lggr) - // Start and cleanup the manager to ensure background goroutines are properly stopped + chipSvc := &MockChipIngressService{} + chipSvc.On("Start", mock.Anything).Return(nil) + chipSvc.On("Close").Return(nil) + chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() + var sent synchronization.TelemPayload + chipSvc.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + sent = args.Get(1).(synchronization.TelemPayload) + }).Once() + + tm := NewManager(tic, ks, chipSvc, lggr) require.NoError(t, tm.Start(t.Context())) t.Cleanup(func() { require.NoError(t, tm.Close()) @@ -390,15 +419,18 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) require.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) - // Verify SendLog doesn't panic and telemetry is queued testPayload := []byte("test telemetry payload") - assert.NotPanics(t, func() { - me.SendLog(testPayload) - }) + me.SendLog(testPayload) + + chipSvc.AssertNumberOfCalls(t, "Send", 1) + assert.Equal(t, testPayload, sent.Telemetry) + assert.Equal(t, synchronization.OCR2Median, sent.TelemType) + assert.Equal(t, "0x123", sent.ContractID) + assert.Equal(t, "EVM", sent.Network) }) t.Run("returns noop agent for invalid chain when chip ingress enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("INVALID") te.On("ChainID").Return("999") @@ -410,7 +442,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) // Invalid network/chainID combination that doesn't exist in chain-selectors me := tm.GenMonitoringEndpoint("INVALID", "999", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -430,7 +462,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { }) t.Run("returns traditional agent when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -445,7 +477,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) }) @@ -453,7 +485,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { t.Run("returns chip ingress multitype agent when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("137") @@ -465,13 +497,13 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) t.Run("returns noop agent for invalid chain when chip ingress enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("INVALID") te.On("ChainID").Return("888") @@ -483,7 +515,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) // Invalid network/chainID combination me := tm.GenMultitypeMonitoringEndpoint("INVALID", "888", "0x456") assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -503,7 +535,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { }) t.Run("returns traditional multitype agent when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("137") @@ -518,7 +550,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.MultiIngressAgentBatch", reflect.TypeOf(me).String()) }) diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index c1459f2bd6e..3a5406cce6a 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -44,6 +44,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 5cc3fd21058..064cc32c6c3 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -44,6 +44,8 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index f8d2dede0c2..433048c65ea 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -44,6 +44,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = true diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 26292423f9c..43b1462d948 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -281,6 +281,8 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default UseBatchSend = true # Default ChipIngressEnabled = false # Default +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default +ChipIngressInsecureConnection = false # Default ``` @@ -300,13 +302,13 @@ Logging toggles verbose logging of the raw telemetry messages being sent. ```toml BufferSize = 100 # Default ``` -BufferSize is the number of telemetry messages to buffer before dropping new ones. +BufferSize is the number of telemetry messages to buffer before dropping new ones. Ignored when ChipIngressEnabled is true. ### MaxBatchSize ```toml MaxBatchSize = 50 # Default ``` -MaxBatchSize is the maximum number of messages to batch into one telemetry request. +MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. ### SendInterval ```toml @@ -330,7 +332,19 @@ UseBatchSend toggles sending telemetry to the ingress server using the batch cli ```toml ChipIngressEnabled = false # Default ``` -ChipIngressEnabled enables sending telemetry to CHIP Ingress. +ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry. + +### ChipIngressEndpoint +```toml +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default +``` +ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint used when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. + +### ChipIngressInsecureConnection +```toml +ChipIngressInsecureConnection = false # Default +``` +ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. ## TelemetryIngress.Endpoints ```toml diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 2b791076b25..60b7c588d63 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -191,6 +191,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index f8159eee279..b92a6651020 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -56,6 +56,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index a993c74f0e9..b227af8e8a5 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -117,6 +117,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 61d90066b0c..8b556e20464 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -100,6 +100,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 46bab3914a9..182630a9484 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -100,6 +100,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index d396eca2fcd..442222e789c 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -100,6 +100,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index bd18f73fab6..fa5ef5cd045 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -199,6 +199,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index eeed39b4c78..f858c20f8f0 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -85,6 +85,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 67667496ed0..b95ae60f735 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -96,6 +96,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 04d1fe5f530..24a2bc4ba3b 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -97,6 +97,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 91f24317518..820089c5ba0 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -79,6 +79,8 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = false [AuditLogger] Enabled = false From 2cf0de266e020acf6a6938c9e38d3017aa248e06 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Thu, 8 Oct 2026 14:56:02 -0700 Subject: [PATCH 2/8] [INFOPLAT-19425] go mod tidy: cloudevents protobuf binding is now indirect --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 628fd553b9a..256cae129a8 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/buger/jsonparser v1.2.0 github.com/buraksezer/consistent v0.10.0 github.com/cespare/xxhash/v2 v2.3.0 - github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.2 github.com/coreos/go-oidc/v3 v3.11.0 github.com/danielkov/gin-helmet v0.0.0-20171108135313-1387e224435e github.com/deckarep/golang-set/v2 v2.9.0 @@ -185,6 +184,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.2 // indirect github.com/cloudevents/sdk-go/v2 v2.16.2 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect From 6b297dd514d76577424d89eaf8d96287128e002a Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Thu, 8 Oct 2026 15:15:39 -0700 Subject: [PATCH 3/8] [INFOPLAT-19425] Simplify chip-ingress telemetry path - Map telemetry type to CloudEvent domain/entity only in the batch client; drop Domain/Entity from TelemPayload and ChipIngressAgent. - Manager: drop the per-endpoint chip service field (endpoints are only an allowlist in chip mode), share endpoint construction, and simplify the config-info metric. - Batch client: inline the health ping, drop the redundant datacontenttype, and share a shouldLogCount helper with the WSRPC clients. - Chip client: use the global OTel providers Beholder installs. - Tests: generate the ChipIngressService mock with mockery, drop wiring and library-behaviour tests, and fold duplicated setup and metric helpers. --- .mockery.yaml | 1 + core/services/chainlink/application.go | 2 - .../chip_ingress_batch_client.go | 56 +-- .../chip_ingress_batch_client_test.go | 152 +++----- .../synchronization/chip_ingress_client.go | 32 +- .../chip_ingress_client_test.go | 10 - core/services/synchronization/common.go | 8 +- core/services/synchronization/helpers_test.go | 8 +- .../mocks/chip_ingress_service.go | 299 ++++++++++++++++ .../telemetry_ingress_batch_worker.go | 2 +- .../telemetry_ingress_client.go | 2 +- core/services/telemetry/chip_ingress.go | 21 +- core/services/telemetry/chip_ingress_test.go | 325 +++--------------- core/services/telemetry/manager.go | 100 ++---- core/services/telemetry/manager_test.go | 46 +-- 15 files changed, 495 insertions(+), 569 deletions(-) create mode 100644 core/services/synchronization/mocks/chip_ingress_service.go diff --git a/.mockery.yaml b/.mockery.yaml index 4ef1f9853e6..c07ecb79b1c 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -215,6 +215,7 @@ packages: github.com/smartcontractkit/chainlink/v2/core/services/synchronization: interfaces: TelemetryService: + ChipIngressService: github.com/smartcontractkit/chainlink/v2/core/services/synchronization/telem: config: dir: "{{ .InterfaceDir }}/../mocks" diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 09bcf8c97c0..381d376504a 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -549,8 +549,6 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err AuthHeadersTTL: cfg.Telemetry().AuthHeadersTTL(), AuthPublicKeyHex: csaPubKeyHex, AuthKeySigner: csaKeystore, - MeterProvider: beholder.GetClient().MeterProvider, - TracerProvider: beholder.GetClient().TracerProvider, }) if chipErr != nil { return nil, fmt.Errorf("failed to create telemetry chip-ingress client: %w", chipErr) diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index 8a487f35ea7..09beb4b517c 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -12,7 +12,6 @@ import ( chipingresspb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/timeutil" ) const ( @@ -72,10 +71,20 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin return c, nil } -// start initializes the chip ingress batch client and starts health monitoring +// start starts the batch client and a 5s health ping that drives +// telemetry_client_connection_status{endpoint="chip-ingress"}. func (cc *chipIngressBatchClient) start(ctx context.Context) error { cc.batchClient.Start(ctx) - cc.startHealthMonitoring(ctx, cc.chipClient) + cc.eng.GoTick(services.TickerConfig{}.NewTicker(5*time.Second), func(ctx context.Context) { + pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + connected := float64(1) + if _, err := cc.chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}); err != nil { + connected = 0 + cc.eng.EmitHealthErr(err) + } + TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(connected) + }) return nil } @@ -107,8 +116,7 @@ func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload return } TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - count := cc.errorCount.Add(1) - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if count := cc.errorCount.Add(1); shouldLogCount(count) { cc.eng.Warnw("Could not send telemetry via ChIP ingress", "error", sendErr, "errorCode", batch.ErrorCodeFor(sendErr), @@ -119,8 +127,7 @@ func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload if err != nil { // batch.ErrMessageBufferFull or batch.ErrClientShutdown TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - count := cc.dropCount.Add(1) - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if count := cc.dropCount.Add(1); shouldLogCount(count) { cc.eng.Warnw("dropping telemetry message for ChIP ingress", "error", err, "contractID", payload.ContractID, @@ -137,20 +144,11 @@ func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload // (enqueue time); chip-ingress legacy mode may overwrite receivedat with the // true server-receive time later. func (cc *chipIngressBatchClient) payloadToEvent(payload TelemPayload, now time.Time) (*chipingress.CloudEventPb, error) { - domain := payload.Domain - entity := payload.Entity - if domain == "" || entity == "" { - var err error - domain, entity, err = TelemetryTypeToDomainAndEntity(payload.TelemType) - if err != nil { - return nil, fmt.Errorf("missing domain/entity for telemType %s: %w", payload.TelemType, err) - } + domain, entity, err := TelemetryTypeToDomainAndEntity(payload.TelemType) + if err != nil { + return nil, err } - - event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, map[string]any{ - "time": now.UTC(), - "datacontenttype": "application/protobuf", - }) + event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, map[string]any{"time": now.UTC()}) if err != nil { return nil, fmt.Errorf("failed creating CloudEvent: %w", err) } @@ -171,21 +169,3 @@ func (cc *chipIngressBatchClient) payloadToEvent(payload TelemPayload, now time. return chipingress.EventToProto(event) } - -// startHealthMonitoring starts a goroutine to monitor the connection state and update other relevant metrics every 5 seconds -func (cc *chipIngressBatchClient) startHealthMonitoring(_ context.Context, chipClient chipingress.Client) { - cc.eng.GoTick(timeutil.NewTicker(func() time.Duration { - return 5 * time.Second - }), func(ctx context.Context) { - connected := float64(0) - pingCtx, pingCancel := context.WithTimeout(ctx, 2*time.Second) - _, err := chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}) - pingCancel() - if err == nil { - connected = float64(1) - } else { - cc.eng.EmitHealthErr(err) - } - TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(connected) - }) -} diff --git a/core/services/synchronization/chip_ingress_batch_client_test.go b/core/services/synchronization/chip_ingress_batch_client_test.go index 8cdc2f0e46c..3e818fc456b 100644 --- a/core/services/synchronization/chip_ingress_batch_client_test.go +++ b/core/services/synchronization/chip_ingress_batch_client_test.go @@ -2,12 +2,14 @@ package synchronization import ( "context" + "maps" "slices" "strconv" "sync" "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" @@ -20,8 +22,6 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" ) -// newTestPayload returns a payload with Domain/Entity unset so the -// TelemetryTypeToDomainAndEntity fallback applies. func newTestPayload(contractID string) TelemPayload { return TelemPayload{ Telemetry: []byte("telemetry-payload"), @@ -51,28 +51,34 @@ func (p *publishedBatches) snapshot() [][]*chipingress.CloudEventPb { return slices.Clone(p.batches) } -func setupBatchClientTest(t *testing.T) (*chipingressmocks.Client, ChipIngressService, *publishedBatches) { +type publishFunc func(*chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) + +// publishOK acknowledges every event; the batch client expects one positional result per event. +func publishOK(b *chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) { + results := make([]*chipingress.PublishResult, len(b.Events)) + for i := range results { + results[i] = &chipingress.PublishResult{} + } + return &chipingress.PublishResponse{Results: results}, nil +} + +// startBatchClient starts a batch client over a mock chip client whose PublishBatch +// records each request and then answers with publish. +func startBatchClient(t *testing.T, publish publishFunc) (ChipIngressService, *publishedBatches) { t.Helper() chipClient := chipingressmocks.NewClient(t) chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - // batch.Client.Stop() closes the underlying client. - chipClient.On("Close").Return(nil).Maybe() + chipClient.On("Close").Return(nil).Maybe() // batch.Client.Stop() closes the underlying client published := &publishedBatches{} chipClient.EXPECT().PublishBatch(mock.Anything, mock.Anything, mock.Anything). - RunAndReturn(func(_ context.Context, batch *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { - published.add(batch.Events) - // The batch client validates positional results: one per event. - results := make([]*chipingress.PublishResult, len(batch.Events)) - for i := range results { - results[i] = &chipingress.PublishResult{} - } - return &chipingress.PublishResponse{Results: results}, nil + RunAndReturn(func(_ context.Context, b *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + published.add(b.Events) + return publish(b) }).Maybe() - client, err := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) - require.NoError(t, err) + client := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) servicetest.Run(t, client) - return chipClient, client, published + return client, published } // awaitFirstEvent waits for exactly one published batch holding one event and decodes it. @@ -88,38 +94,31 @@ func awaitFirstEvent(t *testing.T, published *publishedBatches) chipingress.Clou return event } -func sentCount(telemType string) float64 { - return testutil.ToFloat64(TelemetryClientMessagesSent.WithLabelValues(chipIngress, telemType)) +func counterValue(vec *prometheus.CounterVec, telemType TelemetryType) float64 { + return testutil.ToFloat64(vec.WithLabelValues(chipIngress, string(telemType))) } -func sendErrorCount(telemType string) float64 { - return testutil.ToFloat64(TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, telemType)) -} - -func droppedCount(telemType string) float64 { - return testutil.ToFloat64(TelemetryClientMessagesDropped.WithLabelValues(chipIngress, telemType)) -} - -func TestChipIngressBatchClient_Extensions(t *testing.T) { +func TestChipIngressBatchClient_Event(t *testing.T) { t.Parallel() - _, client, published := setupBatchClientTest(t) + client, published := startBatchClient(t, publishOK) client.Send(t.Context(), newTestPayload("0xabc")) event := awaitFirstEvent(t, published) + // Source/type come from the telemetry type and pick the Kafka topic. + domain, entity, err := TelemetryTypeToDomainAndEntity(OCR2Median) + require.NoError(t, err) + assert.Equal(t, domain, event.Source()) + assert.Equal(t, entity, event.Type()) + // Exact extension set from the OTI-parity table plus the ones the // library stamps (seqnum by the batch client, recordedtime by NewEvent). - extNames := make([]string, 0, len(event.Extensions())) - for name := range event.Extensions() { - extNames = append(extNames, name) - } + exts := event.Extensions() assert.ElementsMatch(t, []string{ "legacytelemetry", "telemetrytype", "chainselector", "networkname", "contractid", "csapublickey", "partitionkey", "sentat", "receivedat", "seqnum", "recordedtime", - }, extNames) - - exts := event.Extensions() + }, slices.Collect(maps.Keys(exts))) assert.Equal(t, "true", exts["legacytelemetry"]) assert.Equal(t, string(OCR2Median), exts["telemetrytype"]) assert.Equal(t, "12345", exts["chainselector"]) @@ -133,86 +132,49 @@ func TestChipIngressBatchClient_Extensions(t *testing.T) { require.NoError(t, err) receivedAt, err := strconv.ParseInt(exts["receivedat"].(string), 10, 64) require.NoError(t, err) - assert.Positive(t, sentAt) assert.Equal(t, sentAt/1_000_000, receivedAt) assert.WithinDuration(t, time.Now(), time.Unix(0, sentAt), time.Minute) - - // nodeoperator* / nodename / nodecsapublickey must be absent: chip-ingress - // stamps them server-side; sending them would duplicate headers. - for _, absent := range []string{"nodeoperatorname", "nodename", "nodecsapublickey", "nodeoperatorkey"} { - assert.NotContains(t, exts, absent) - } -} - -func TestChipIngressBatchClient_DomainEntityFallback(t *testing.T) { - t.Parallel() - _, client, published := setupBatchClientTest(t) - - client.Send(t.Context(), newTestPayload("0xabc")) - event := awaitFirstEvent(t, published) - domain, entity, err := TelemetryTypeToDomainAndEntity(OCR2Median) - require.NoError(t, err) - assert.Equal(t, domain, event.Source()) - assert.Equal(t, entity, event.Type()) } func TestChipIngressBatchClient_UnmappedTypeDropped(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics - _, client, published := setupBatchClientTest(t) + client, published := startBatchClient(t, publishOK) payload := newTestPayload("0xabc") payload.TelemType = LLOReport // no chip domain/entity mapping - payload.Domain = "" - payload.Entity = "" - before := droppedCount(string(payload.TelemType)) + before := counterValue(TelemetryClientMessagesDropped, payload.TelemType) // The event is rejected synchronously in Send, before it reaches the queue. client.Send(t.Context(), payload) - assert.InDelta(t, 1, droppedCount(string(payload.TelemType))-before, 0.001) + assert.InDelta(t, 1, counterValue(TelemetryClientMessagesDropped, payload.TelemType)-before, 0.001) assert.Empty(t, published.snapshot(), "PublishBatch must never be called for unmapped types") } -func TestChipIngressBatchClient_SentCountsMessages(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics - _, client, _ := setupBatchClientTest(t) - - payload := newTestPayload("0xabc") - const n = 5 - before := sentCount(string(payload.TelemType)) - for range n { - client.Send(t.Context(), payload) - } - require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.InDelta(c, float64(n), sentCount(string(payload.TelemType))-before, 0.001, - "Sent must count messages, not batches") - }, testutils.WaitTimeout(t), 10*time.Millisecond) -} - -func TestChipIngressBatchClient_SendErrorCountsMessages(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - chipClient.On("Close").Return(nil).Maybe() - chipClient.EXPECT().PublishBatch(mock.Anything, mock.Anything, mock.Anything). - Return(nil, assert.AnError).Maybe() - - client, err := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) - require.NoError(t, err) - servicetest.Run(t, client) - - payload := newTestPayload("0xabc") - const n = 3 - before := sendErrorCount(string(payload.TelemType)) - for range n { - client.Send(t.Context(), payload) +func TestChipIngressBatchClient_CountsPerMessage(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics + for name, tc := range map[string]struct { //nolint:paralleltest // subtests share package-global prometheus metrics + publish publishFunc + counter *prometheus.CounterVec + }{ + "sent": {publishOK, TelemetryClientMessagesSent}, + "send errors": {func(*chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) { return nil, assert.AnError }, TelemetryClientMessagesSendErrors}, + } { + t.Run(name, func(t *testing.T) { + client, _ := startBatchClient(t, tc.publish) + const n = 3 + before := counterValue(tc.counter, OCR2Median) + for range n { + client.Send(t.Context(), newTestPayload("0xabc")) + } + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, float64(n), counterValue(tc.counter, OCR2Median)-before, 0.001, "must count messages, not batches") + }, testutils.WaitTimeout(t), 10*time.Millisecond) + }) } - require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.InDelta(c, float64(n), sendErrorCount(string(payload.TelemType))-before, 0.001, - "SendErrors must count messages, not batches") - }, testutils.WaitTimeout(t), 10*time.Millisecond) } func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // resets a package-global prometheus gauge // The status gauge is package-global; reset it so a value left by another test can't satisfy the check. TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(0) - _, _, _ = setupBatchClientTest(t) + startBatchClient(t, publishOK) require.EventuallyWithT(t, func(c *assert.CollectT) { assert.InDelta(c, 1, testutil.ToFloat64(TelemetryClientConnectionStatus.WithLabelValues(chipIngress)), 0.001) diff --git a/core/services/synchronization/chip_ingress_client.go b/core/services/synchronization/chip_ingress_client.go index e9b5a0715f4..4d7f0879b3b 100644 --- a/core/services/synchronization/chip_ingress_client.go +++ b/core/services/synchronization/chip_ingress_client.go @@ -4,13 +4,11 @@ import ( "fmt" "time" - "go.opentelemetry.io/otel/metric" - "go.opentelemetry.io/otel/trace" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" ) // ChipIngressClientConfig configures the gRPC client for the legacy telemetry chip-ingress endpoint. +// Metrics and traces use the global OTel providers, which Beholder installs at startup. type ChipIngressClientConfig struct { Endpoint string // host:port InsecureConnection bool @@ -18,8 +16,6 @@ type ChipIngressClientConfig struct { AuthHeadersTTL time.Duration // >0 selects rotating CSA tokens (must be >= 10m); same value Beholder uses AuthPublicKeyHex string AuthKeySigner chipingress.Signer - MeterProvider metric.MeterProvider - TracerProvider trace.TracerProvider } // NewChipIngressClient builds the chip-ingress client used for legacy telemetry. @@ -35,28 +31,18 @@ func NewChipIngressClient(cfg ChipIngressClientConfig) (chipingress.Client, erro return nil, fmt.Errorf("failed to build chip-ingress auth: %w", err) } - // The transport option must be applied before WithTokenAuth: WithTokenAuth - // reads the connection security at application time to decide whether its - // per-RPC credentials require TLS. - opts := []chipingress.Opt{} + // The transport option must come before WithTokenAuth, which reads it to + // decide whether its per-RPC credentials require TLS. + opts := []chipingress.Opt{chipingress.WithTLS()} if cfg.InsecureConnection { - opts = append(opts, chipingress.WithInsecureConnection()) - } else { - opts = append(opts, chipingress.WithTLS()) + opts = []chipingress.Opt{chipingress.WithInsecureConnection()} } - if auth != nil { - // NewHeaderProvider returns nil only when there are no headers and no TTL, - // which cannot happen for the node (BuildBeholderAuth always returns headers), - // but keep the nil check for safety. + if auth != nil { // nil only when there are no headers and no TTL opts = append(opts, chipingress.WithTokenAuth(auth)) } - opts = append(opts, - chipingress.WithNOPLookup(), - chipingress.WithMeterProvider(cfg.MeterProvider), - chipingress.WithTracerProvider(cfg.TracerProvider), - ) + opts = append(opts, chipingress.WithNOPLookup()) - // grpc.NewClient connects lazily, so an unreachable endpoint does not fail - // startup; a malformed host:port does (that is intended). + // grpc.NewClient connects lazily: an unreachable endpoint does not fail + // startup, a malformed host:port does. return chipingress.NewClient(cfg.Endpoint, opts...) } diff --git a/core/services/synchronization/chip_ingress_client_test.go b/core/services/synchronization/chip_ingress_client_test.go index c59f403d92c..3bedc627b16 100644 --- a/core/services/synchronization/chip_ingress_client_test.go +++ b/core/services/synchronization/chip_ingress_client_test.go @@ -50,13 +50,3 @@ func TestChipIngressClient_SendAuthHeaders(t *testing.T) { assert.Equal(t, []string{"true"}, capture.lastMD.Get("x-include-nop-info")) assert.Equal(t, []string{"x"}, capture.lastMD.Get("X-Beholder-Node-Auth-Token")) } - -func TestChipIngressClient_InvalidAddress(t *testing.T) { - t.Parallel() - _, err := NewChipIngressClient(ChipIngressClientConfig{ - Endpoint: "no-port", - InsecureConnection: true, - AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "x"}, - }) - require.Error(t, err) -} diff --git a/core/services/synchronization/common.go b/core/services/synchronization/common.go index e0ef75a8bd2..93dd51cb2ea 100644 --- a/core/services/synchronization/common.go +++ b/core/services/synchronization/common.go @@ -39,13 +39,17 @@ type TelemPayload struct { Telemetry []byte TelemType TelemetryType ContractID string - Domain string - Entity string ChainSelector uint64 Network string ChainID string } +// shouldLogCount reports whether the n-th occurrence of a repeated warning +// should be logged: at 1, 2, 4, 8, 16, 32, 64, then every 100th. +func shouldLogCount(n uint32) bool { + return n > 0 && (n%100 == 0 || n&(n-1) == 0) +} + // TelemetryService encapsulates all the functionality needed to // send telemetry to the ingress server using wsrpc type TelemetryService interface { diff --git a/core/services/synchronization/helpers_test.go b/core/services/synchronization/helpers_test.go index b6dc38d3493..239a88f60bc 100644 --- a/core/services/synchronization/helpers_test.go +++ b/core/services/synchronization/helpers_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" @@ -27,6 +29,8 @@ func NewTestTelemetryIngressBatchClient(t *testing.T, url *url.URL, serverPubKey } // NewTestChipIngressBatchClient calls NewChipIngressBatchClient with a fixed test CSA key. -func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) (ChipIngressService, error) { - return NewChipIngressBatchClient(chipClient, "deadbeef", logging, logger.TestLogger(t), sendInterval, time.Second) +func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) ChipIngressService { + c, err := NewChipIngressBatchClient(chipClient, "deadbeef", logging, logger.TestLogger(t), sendInterval, time.Second) + require.NoError(t, err) + return c } diff --git a/core/services/synchronization/mocks/chip_ingress_service.go b/core/services/synchronization/mocks/chip_ingress_service.go new file mode 100644 index 00000000000..6d5d5cb2edf --- /dev/null +++ b/core/services/synchronization/mocks/chip_ingress_service.go @@ -0,0 +1,299 @@ +// Code generated by mockery v2.53.7. DO NOT EDIT. + +package mocks + +import ( + context "context" + + synchronization "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" + mock "github.com/stretchr/testify/mock" +) + +// ChipIngressService is an autogenerated mock type for the ChipIngressService type +type ChipIngressService struct { + mock.Mock +} + +type ChipIngressService_Expecter struct { + mock *mock.Mock +} + +func (_m *ChipIngressService) EXPECT() *ChipIngressService_Expecter { + return &ChipIngressService_Expecter{mock: &_m.Mock} +} + +// Close provides a mock function with no fields +func (_m *ChipIngressService) Close() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Close") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type ChipIngressService_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Close() *ChipIngressService_Close_Call { + return &ChipIngressService_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *ChipIngressService_Close_Call) Run(run func()) *ChipIngressService_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Close_Call) Return(_a0 error) *ChipIngressService_Close_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Close_Call) RunAndReturn(run func() error) *ChipIngressService_Close_Call { + _c.Call.Return(run) + return _c +} + +// HealthReport provides a mock function with no fields +func (_m *ChipIngressService) HealthReport() map[string]error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for HealthReport") + } + + var r0 map[string]error + if rf, ok := ret.Get(0).(func() map[string]error); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]error) + } + } + + return r0 +} + +// ChipIngressService_HealthReport_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HealthReport' +type ChipIngressService_HealthReport_Call struct { + *mock.Call +} + +// HealthReport is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) HealthReport() *ChipIngressService_HealthReport_Call { + return &ChipIngressService_HealthReport_Call{Call: _e.mock.On("HealthReport")} +} + +func (_c *ChipIngressService_HealthReport_Call) Run(run func()) *ChipIngressService_HealthReport_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_HealthReport_Call) Return(_a0 map[string]error) *ChipIngressService_HealthReport_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_HealthReport_Call) RunAndReturn(run func() map[string]error) *ChipIngressService_HealthReport_Call { + _c.Call.Return(run) + return _c +} + +// Name provides a mock function with no fields +func (_m *ChipIngressService) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// ChipIngressService_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' +type ChipIngressService_Name_Call struct { + *mock.Call +} + +// Name is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Name() *ChipIngressService_Name_Call { + return &ChipIngressService_Name_Call{Call: _e.mock.On("Name")} +} + +func (_c *ChipIngressService_Name_Call) Run(run func()) *ChipIngressService_Name_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Name_Call) Return(_a0 string) *ChipIngressService_Name_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Name_Call) RunAndReturn(run func() string) *ChipIngressService_Name_Call { + _c.Call.Return(run) + return _c +} + +// Ready provides a mock function with no fields +func (_m *ChipIngressService) Ready() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Ready") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Ready_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Ready' +type ChipIngressService_Ready_Call struct { + *mock.Call +} + +// Ready is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Ready() *ChipIngressService_Ready_Call { + return &ChipIngressService_Ready_Call{Call: _e.mock.On("Ready")} +} + +func (_c *ChipIngressService_Ready_Call) Run(run func()) *ChipIngressService_Ready_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Ready_Call) Return(_a0 error) *ChipIngressService_Ready_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Ready_Call) RunAndReturn(run func() error) *ChipIngressService_Ready_Call { + _c.Call.Return(run) + return _c +} + +// Send provides a mock function with given fields: ctx, payload +func (_m *ChipIngressService) Send(ctx context.Context, payload synchronization.TelemPayload) { + _m.Called(ctx, payload) +} + +// ChipIngressService_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send' +type ChipIngressService_Send_Call struct { + *mock.Call +} + +// Send is a helper method to define mock.On call +// - ctx context.Context +// - payload synchronization.TelemPayload +func (_e *ChipIngressService_Expecter) Send(ctx interface{}, payload interface{}) *ChipIngressService_Send_Call { + return &ChipIngressService_Send_Call{Call: _e.mock.On("Send", ctx, payload)} +} + +func (_c *ChipIngressService_Send_Call) Run(run func(ctx context.Context, payload synchronization.TelemPayload)) *ChipIngressService_Send_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(synchronization.TelemPayload)) + }) + return _c +} + +func (_c *ChipIngressService_Send_Call) Return() *ChipIngressService_Send_Call { + _c.Call.Return() + return _c +} + +func (_c *ChipIngressService_Send_Call) RunAndReturn(run func(context.Context, synchronization.TelemPayload)) *ChipIngressService_Send_Call { + _c.Run(run) + return _c +} + +// Start provides a mock function with given fields: _a0 +func (_m *ChipIngressService) Start(_a0 context.Context) error { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Start") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' +type ChipIngressService_Start_Call struct { + *mock.Call +} + +// Start is a helper method to define mock.On call +// - _a0 context.Context +func (_e *ChipIngressService_Expecter) Start(_a0 interface{}) *ChipIngressService_Start_Call { + return &ChipIngressService_Start_Call{Call: _e.mock.On("Start", _a0)} +} + +func (_c *ChipIngressService_Start_Call) Run(run func(_a0 context.Context)) *ChipIngressService_Start_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ChipIngressService_Start_Call) Return(_a0 error) *ChipIngressService_Start_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Start_Call) RunAndReturn(run func(context.Context) error) *ChipIngressService_Start_Call { + _c.Call.Return(run) + return _c +} + +// NewChipIngressService creates a new instance of ChipIngressService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewChipIngressService(t interface { + mock.TestingT + Cleanup(func()) +}) *ChipIngressService { + mock := &ChipIngressService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/services/synchronization/telemetry_ingress_batch_worker.go b/core/services/synchronization/telemetry_ingress_batch_worker.go index 1718941ebf9..d62f9de80d8 100644 --- a/core/services/synchronization/telemetry_ingress_batch_worker.go +++ b/core/services/synchronization/telemetry_ingress_batch_worker.go @@ -94,7 +94,7 @@ func (tw *telemetryIngressBatchWorker) logBufferFullWithExpBackoff(payload Telem count := tw.dropMessageCount.Add(1) TelemetryClientMessagesDropped.WithLabelValues(tw.endpointURL, string(tw.telemType)).Inc() - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if shouldLogCount(count) { tw.lggr.Warnw("telemetry ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) } } diff --git a/core/services/synchronization/telemetry_ingress_client.go b/core/services/synchronization/telemetry_ingress_client.go index fe5c3909760..16d0cf2a3d4 100644 --- a/core/services/synchronization/telemetry_ingress_client.go +++ b/core/services/synchronization/telemetry_ingress_client.go @@ -156,7 +156,7 @@ func (tc *telemetryIngressClient) handleTelemetry() { // etc... func (tc *telemetryIngressClient) logBufferFullWithExpBackoff(payload TelemPayload) { count := tc.dropMessageCount.Add(1) - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if shouldLogCount(count) { tc.eng.Warnw("telemetry ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) } } diff --git a/core/services/telemetry/chip_ingress.go b/core/services/telemetry/chip_ingress.go index c78a0b8f96c..5f32a3e2d41 100644 --- a/core/services/telemetry/chip_ingress.go +++ b/core/services/telemetry/chip_ingress.go @@ -32,8 +32,6 @@ type ChipIngressAgent struct { lggr logger.Logger TelemType synchronization.TelemetryType // Empty for multitype endpoints - Domain string // Derived from TelemetryType (empty for multitype) - Entity string // Derived from TelemetryType (empty for multitype) } // NewChipIngressAgent creates a new agent for a single telemetry type endpoint. @@ -55,8 +53,8 @@ func NewChipIngressAgent( return nil, fmt.Errorf("failed to get chain details for chainID %s and network %s: %w", chainID, network, err) } - domain, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) - if err != nil { + // Reject unmapped telemetry types up front so the caller gets a NoopAgent. + if _, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType); err != nil { return nil, fmt.Errorf("failed to map telemetry type to domain/entity: %w", err) } @@ -65,8 +63,6 @@ func NewChipIngressAgent( ChainID: chainID, ContractID: contractID, ChainSelector: details.ChainSelector, - Domain: domain, - Entity: entity, TelemType: telemType, telemService: telemService, lggr: lggr, @@ -99,7 +95,7 @@ func NewChipIngressAgentMultitype( ChainSelector: details.ChainSelector, telemService: telemService, lggr: lggr, - // TelemType, Domain, Entity left empty for multitype + // TelemType left empty for multitype }, nil } @@ -122,8 +118,6 @@ func (a *ChipIngressAgent) SendLog(log []byte) { TelemType: a.TelemType, ContractID: a.ContractID, ChainSelector: a.ChainSelector, - Domain: a.Domain, - Entity: a.Entity, Network: a.Network, } a.telemService.Send(ctx, payload) @@ -137,20 +131,11 @@ func (a *ChipIngressAgent) SendLog(log []byte) { // MonitoringEndpoint.SendLog signature for consistency. A background context is used internally. func (a *ChipIngressAgent) SendTypedLog(telemType synchronization.TelemetryType, log []byte) { ctx := context.Background() - - domain, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) - if err != nil { - a.lggr.Errorw("failed to map telemetry type to domain/entity", "error", err, "telemType", telemType) - return - } - payload := synchronization.TelemPayload{ Telemetry: log, TelemType: telemType, ContractID: a.ContractID, ChainSelector: a.ChainSelector, - Domain: domain, - Entity: entity, Network: a.Network, } a.telemService.Send(ctx, payload) diff --git a/core/services/telemetry/chip_ingress_test.go b/core/services/telemetry/chip_ingress_test.go index c8f492ef0c5..3bce22ffe6d 100644 --- a/core/services/telemetry/chip_ingress_test.go +++ b/core/services/telemetry/chip_ingress_test.go @@ -1,7 +1,6 @@ package telemetry import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -10,52 +9,15 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" + syncmocks "github.com/smartcontractkit/chainlink/v2/core/services/synchronization/mocks" ) -// MockChipIngressService is a mock implementation of the ChipIngressService interface -type MockChipIngressService struct { - mock.Mock -} - -func (m *MockChipIngressService) Start(ctx context.Context) error { - args := m.Called(ctx) - return args.Error(0) -} - -func (m *MockChipIngressService) Close() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockChipIngressService) Send(ctx context.Context, payload synchronization.TelemPayload) { - m.Called(ctx, payload) -} - -func (m *MockChipIngressService) HealthReport() map[string]error { - args := m.Called() - return args.Get(0).(map[string]error) -} - -func (m *MockChipIngressService) Name() string { - args := m.Called() - return args.String(0) -} - -func (m *MockChipIngressService) Ready() error { - args := m.Called() - return args.Error(0) -} - func TestNewChipIngressAgent(t *testing.T) { t.Run("Success - Ethereum Mainnet", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - chainID := "1" - network := "EVM" contractID := "0x1234" - telemType := synchronization.OCR2Median - adapter, err := NewChipIngressAgent(mockTelemService, network, chainID, contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1", contractID, synchronization.OCR2Median, lggr) require.NoError(t, err) require.NotNil(t, adapter) @@ -64,34 +26,24 @@ func TestNewChipIngressAgent(t *testing.T) { assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "1", adapter.ChainID) assert.Equal(t, contractID, adapter.ContractID) - - // Verify domain and entity were derived correctly - assert.Equal(t, "data-feeds.telemetry.ocr2-median", adapter.Domain) - assert.Equal(t, "offchainreporting2.TelemetryWrapper", adapter.Entity) }) t.Run("Success - Polygon", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "137", "0xabc", synchronization.OCR3CCIPCommit, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "137", "0xabc", synchronization.OCR3CCIPCommit, lggr) require.NoError(t, err) // Verify chain selector was derived correctly (Polygon) assert.Equal(t, uint64(4051577828743386545), adapter.ChainSelector) assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "137", adapter.ChainID) - - // Verify domain and entity for CCIP commit - assert.Equal(t, "ccip.telemetry.ocr3", adapter.Domain) - assert.Equal(t, "offchainreporting3.TelemetryWrapper", adapter.Entity) }) t.Run("Success - Arbitrum One", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "42161", "0xdef", synchronization.OCR2CCIPExec, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "42161", "0xdef", synchronization.OCR2CCIPExec, lggr) require.NoError(t, err) // Verify chain selector was derived correctly (Arbitrum One) @@ -108,30 +60,27 @@ func TestNewChipIngressAgent(t *testing.T) { }) t.Run("Error - invalid chainID", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1234567890123456", "0x1234", synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1234567890123456", "0x1234", synchronization.OCR2Median, lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") }) t.Run("Error - invalid network family", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "INVALID_NETWORK", "1", "0x1234", synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "INVALID_NETWORK", "1", "0x1234", synchronization.OCR2Median, lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") }) t.Run("Error - invalid telemetry type", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", "0x1234", synchronization.TelemetryType("unknown"), lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1", "0x1234", synchronization.TelemetryType("unknown"), lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to map telemetry type to domain/entity") @@ -140,10 +89,9 @@ func TestNewChipIngressAgent(t *testing.T) { func TestNewChipIngressAgentMultitype(t *testing.T) { t.Run("Success - creates multitype agent", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(syncmocks.NewChipIngressService(t), "EVM", "1", "0x1234", lggr) require.NoError(t, err) require.NotNil(t, adapter) @@ -151,11 +99,7 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { assert.Equal(t, uint64(5009297550715157269), adapter.ChainSelector) assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "1", adapter.ChainID) - - // Verify TelemType, Domain, Entity are empty for multitype assert.Empty(t, adapter.TelemType) - assert.Empty(t, adapter.Domain) - assert.Empty(t, adapter.Entity) }) t.Run("Error - nil telemetry service", func(t *testing.T) { @@ -168,10 +112,9 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { }) t.Run("Error - invalid chainID", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "invalid", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(syncmocks.NewChipIngressService(t), "EVM", "invalid", "0x1234", lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") @@ -180,285 +123,113 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { func TestChipIngressAgent_SendLog(t *testing.T) { t.Run("Success - sends to telemetry service", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x1234567890abcdef" telemType := synchronization.OCR2Median telemetryLog := []byte("test telemetry data") - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "1", contractID, telemType, lggr) require.NoError(t, err) - expectedPayload := synchronization.TelemPayload{ + svc.On("Send", mock.Anything, synchronization.TelemPayload{ Telemetry: telemetryLog, TelemType: telemType, ContractID: contractID, ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() + }).Once() - // Call SendLog adapter.SendLog(telemetryLog) - - // Verify Send was called - mockTelemService.AssertExpectations(t) - }) - - t.Run("SendLog continues even if send fails", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0xabcdef" - telemType := synchronization.OCR3Mercury - telemetryLog := []byte("test data") - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "137", contractID, telemType, lggr) - require.NoError(t, err) - - expectedPayload := synchronization.TelemPayload{ - Telemetry: telemetryLog, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() - - // Call SendLog - should not panic - assert.NotPanics(t, func() { - adapter.SendLog(telemetryLog) - }) - - mockTelemService.AssertExpectations(t) }) t.Run("Multiple SendLog calls", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x999" telemType := synchronization.OCR2CCIPCommit - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "42161", contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "42161", contractID, telemType, lggr) require.NoError(t, err) - // Setup expectations for multiple calls - log1 := []byte("log 1") - log2 := []byte("log 2") - log3 := []byte("log 3") - - payload1 := synchronization.TelemPayload{ - Telemetry: log1, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - payload2 := synchronization.TelemPayload{ - Telemetry: log2, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - payload3 := synchronization.TelemPayload{ - Telemetry: log3, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, + logs := [][]byte{[]byte("log 1"), []byte("log 2"), []byte("log 3")} + for _, l := range logs { + svc.On("Send", mock.Anything, synchronization.TelemPayload{ + Telemetry: l, + TelemType: telemType, + ContractID: contractID, + ChainSelector: adapter.ChainSelector, + Network: adapter.Network, + }).Once() } - mockTelemService.On("Send", mock.Anything, payload1).Once() - mockTelemService.On("Send", mock.Anything, payload2).Once() - mockTelemService.On("Send", mock.Anything, payload3).Once() - - // Send multiple logs - adapter.SendLog(log1) - adapter.SendLog(log2) - adapter.SendLog(log3) - - mockTelemService.AssertExpectations(t) + for _, l := range logs { + adapter.SendLog(l) + } }) - t.Run("SendLog on multitype agent logs warning", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + t.Run("SendLog on multitype agent does not send", func(t *testing.T) { + t.Parallel() + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "1", "0x1234", lggr) require.NoError(t, err) - // SendLog should not call Send on multitype agent - assert.NotPanics(t, func() { - adapter.SendLog([]byte("test")) - }) - - // Verify Send was NOT called - mockTelemService.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) + adapter.SendLog([]byte("test")) + svc.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) }) } func TestChipIngressAgent_SendTypedLog(t *testing.T) { t.Run("Success - sends typed log", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x1234" - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", contractID, lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "1", contractID, lggr) require.NoError(t, err) telemType := synchronization.OCR2Median telemetryLog := []byte("test data") - expectedPayload := synchronization.TelemPayload{ + svc.On("Send", mock.Anything, synchronization.TelemPayload{ Telemetry: telemetryLog, TelemType: telemType, ContractID: contractID, ChainSelector: adapter.ChainSelector, - Domain: "data-feeds.telemetry.ocr2-median", - Entity: "offchainreporting2.TelemetryWrapper", Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() + }).Once() adapter.SendTypedLog(telemType, telemetryLog) - - mockTelemService.AssertExpectations(t) }) t.Run("SendTypedLog with different types", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0x5678" - - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "137", contractID, lggr) - require.NoError(t, err) - - // Send with OCR2Median - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { - return p.TelemType == synchronization.OCR2Median && p.Domain == "data-feeds.telemetry.ocr2-median" - })).Once() - adapter.SendTypedLog(synchronization.OCR2Median, []byte("median data")) - - // Send with OCR3Mercury - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { - return p.TelemType == synchronization.OCR3Mercury && p.Domain == "data-streams.telemetry.ocr3-mercury" - })).Once() - adapter.SendTypedLog(synchronization.OCR3Mercury, []byte("mercury data")) - - mockTelemService.AssertExpectations(t) - }) - - t.Run("SendTypedLog with invalid type logs error", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "137", "0x5678", lggr) require.NoError(t, err) - // Should not panic, just log error - assert.NotPanics(t, func() { - adapter.SendTypedLog(synchronization.TelemetryType("invalid"), []byte("test")) - }) - - // Verify Send was NOT called due to invalid type - mockTelemService.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) + for _, telemType := range []synchronization.TelemetryType{synchronization.OCR2Median, synchronization.OCR3Mercury} { + svc.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { + return p.TelemType == telemType + })).Once() + adapter.SendTypedLog(telemType, []byte("data")) + } }) t.Run("SendTypedLog works on single-type agent too", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - contractID := "0x1234" - // Create single-type agent - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "1", "0x1234", synchronization.OCR2Median, lggr) require.NoError(t, err) - // SendTypedLog should still work - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { + svc.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { return p.TelemType == synchronization.OCR3Mercury })).Once() adapter.SendTypedLog(synchronization.OCR3Mercury, []byte("different type")) - - mockTelemService.AssertExpectations(t) - }) -} - -func TestChipIngressAgent_ExportedFields(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0x1234567890" - telemType := synchronization.OCR3Mercury - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, telemType, lggr) - require.NoError(t, err) - - t.Run("ChainSelector", func(t *testing.T) { - assert.Equal(t, uint64(5009297550715157269), adapter.ChainSelector) - }) - - t.Run("Network", func(t *testing.T) { - assert.Equal(t, "EVM", adapter.Network) - }) - - t.Run("ChainID", func(t *testing.T) { - assert.Equal(t, "1", adapter.ChainID) - }) - - t.Run("ContractID", func(t *testing.T) { - assert.Equal(t, contractID, adapter.ContractID) - }) - - t.Run("Domain", func(t *testing.T) { - assert.Equal(t, "data-streams.telemetry.ocr3-mercury", adapter.Domain) - }) - - t.Run("Entity", func(t *testing.T) { - assert.Equal(t, "offchainreporting3.TelemetryWrapper", adapter.Entity) - }) -} - -func TestChipIngressAgent_InterfaceCompliance(t *testing.T) { - t.Run("single-type agent implements MonitoringEndpoint", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", "0x123", synchronization.OCR2Median, lggr) - require.NoError(t, err) - - // Verify it can be assigned to the interface - var _ any = adapter - - // Call the interface method - mockTelemService.On("Send", mock.Anything, mock.Anything) - adapter.SendLog([]byte("test")) - mockTelemService.AssertCalled(t, "Send", mock.Anything, mock.Anything) - }) - - t.Run("multitype agent implements MultitypeMonitoringEndpoint", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x123", lggr) - require.NoError(t, err) - - // Verify it implements MultitypeMonitoringEndpoint - var me MultitypeMonitoringEndpoint = adapter - require.NotNil(t, me) - - // Call the interface method - mockTelemService.On("Send", mock.Anything, mock.Anything) - adapter.SendTypedLog(synchronization.OCR2Median, []byte("test")) - mockTelemService.AssertCalled(t, "Send", mock.Anything, mock.Anything) }) } diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index baaaca53591..607d9cfc9c2 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -2,7 +2,6 @@ package telemetry import ( "context" - "fmt" "net/url" "strings" "time" @@ -36,27 +35,25 @@ type Manager struct { uniConn bool useBatchSend bool - chipService synchronization.ChipIngressService // nil when chip ingress is disabled - - chipIngressEndpoint string - configGauge otelmetric.Int64Gauge + chipService synchronization.ChipIngressService // nil when chip ingress is disabled + chipIngressEndpoint string // "" when disabled } +// telemetryEndpoint is one [[TelemetryIngress.Endpoints]] entry. With chip +// ingress enabled it only allowlists the chain and client is nil. type telemetryEndpoint struct { - ChainID string - Network string - URL *url.URL - client synchronization.TelemetryService - PubKey string - chipIngressClient synchronization.ChipIngressService + ChainID string + Network string + URL *url.URL + client synchronization.TelemetryService + PubKey string } // NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints // chipService is the shared chip-ingress service to use when ChipIngressEnabled is true; nil when the flag is off. func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, chipService synchronization.ChipIngressService, lggr common.Logger) *Manager { - var chipIngressEndpoint string // "" when disabled + var chipIngressEndpoint string if chipService != nil { - lggr.Info("ChIP Ingress is enabled for telemetry") chipIngressEndpoint = cfg.ChipIngressEndpoint() } @@ -75,14 +72,8 @@ func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, chipServi m.Service, m.eng = services.Config{ Name: "TelemetryManager", Start: func(ctx context.Context) error { - if err := m.recordConfigMetric(ctx); err != nil { - m.eng.Errorw("failed to record telemetry ingress config metric", "err", err) - } - m.eng.GoTick(services.TickerConfig{}.NewTicker(time.Hour), func(ctx context.Context) { - if err := m.recordConfigMetric(ctx); err != nil { - m.eng.Errorw("failed to record telemetry ingress config metric", "err", err) - } - }) + m.recordConfigMetric(ctx) + m.eng.GoTick(services.TickerConfig{}.NewTicker(time.Hour), m.recordConfigMetric) return nil }, NewSubServices: func(lggr common.Logger) (subs []services.Service) { @@ -117,7 +108,7 @@ func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contract if m.chipService != nil { lggr := m.eng.Named("chip-ingress") - adapter, err := NewChipIngressAgent(e.chipIngressClient, network, chainID, contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(m.chipService, network, chainID, contractID, telemType, lggr) if err != nil { m.eng.Errorw("failed to create ChIP ingress agent, falling back to noop", "error", err, "network", network, "chainID", chainID, "contractID", contractID, "telemType", telemType) return &NoopAgent{} @@ -142,7 +133,7 @@ func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, if m.chipService != nil { lggr := m.eng.Named("chip-ingress-multipletype") - adapter, err := NewChipIngressAgentMultitype(e.chipIngressClient, network, chainID, contractID, lggr) + adapter, err := NewChipIngressAgentMultitype(m.chipService, network, chainID, contractID, lggr) if err != nil { m.eng.Errorw("failed to create ChIP ingress multitype agent, falling back to noop", "error", err, "network", network, "chainID", chainID, "contractID", contractID) return &NoopAgent{} @@ -178,61 +169,42 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log return nil, errors.Errorf("cannot add telemetry endpoint for network %q and chainID %q, endpoint already exists", e.Network(), e.ChainID()) } - lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) + te := &telemetryEndpoint{ + Network: strings.ToUpper(e.Network()), + ChainID: strings.ToUpper(e.ChainID()), + URL: e.URL(), + PubKey: e.ServerPubKey(), + } + m.endpoints = append(m.endpoints, te) if m.chipService != nil { - lggr.Infof("Using chip-ingress service for network %q chainID %q", e.Network(), e.ChainID()) - // When ChIP ingress is enabled, all endpoints share the one ChipIngressService - // (injected by the application); entries only select which chains send telemetry. - te := telemetryEndpoint{ - Network: strings.ToUpper(e.Network()), - ChainID: strings.ToUpper(e.ChainID()), - URL: e.URL(), - PubKey: e.ServerPubKey(), - chipIngressClient: m.chipService, - } - m.endpoints = append(m.endpoints, &te) + // All chains share the injected chip service; the entry is only an allowlist. return nil, nil } - // Otherwise use the traditional telemetry service - var tClient synchronization.TelemetryService + lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) if m.useBatchSend { - tClient = synchronization.NewTelemetryIngressBatchClient(e.URL(), e.ServerPubKey(), m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) + te.client = synchronization.NewTelemetryIngressBatchClient(e.URL(), e.ServerPubKey(), m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) } else { - tClient = synchronization.NewTelemetryIngressClient(e.URL(), e.ServerPubKey(), m.ks, lggr, cfg.BufferSize()) + te.client = synchronization.NewTelemetryIngressClient(e.URL(), e.ServerPubKey(), m.ks, lggr, cfg.BufferSize()) } - - te := telemetryEndpoint{ - Network: strings.ToUpper(e.Network()), - ChainID: strings.ToUpper(e.ChainID()), - URL: e.URL(), - client: tClient, - PubKey: e.ServerPubKey(), - } - - m.endpoints = append(m.endpoints, &te) return te.client, nil } -// recordConfigMetric records the telemetry ingress config info metric, mirroring -// beholder's ConfigRecorder "config.info" pattern. -func (m *Manager) recordConfigMetric(ctx context.Context) error { - if m.configGauge == nil { - gauge, err := beholder.GetMeter().Int64Gauge("telemetry_ingress.config.info", - otelmetric.WithDescription("Telemetry ingress config info metric"), - otelmetric.WithUnit("{info}")) - if err != nil { - return fmt.Errorf("failed to create telemetry ingress config metric: %w", err) - } - m.configGauge = gauge - } - m.configGauge.Record(ctx, 1, otelmetric.WithAttributes( +// recordConfigMetric records the telemetry_ingress.config.info gauge, mirroring +// Beholder's "config.info" pattern. OTel caches the instrument by name. +func (m *Manager) recordConfigMetric(ctx context.Context) { + gauge, err := beholder.GetMeter().Int64Gauge("telemetry_ingress.config.info", + otelmetric.WithDescription("Telemetry ingress config info metric"), + otelmetric.WithUnit("{info}")) + if err != nil { + m.eng.Errorw("failed to create telemetry ingress config metric", "err", err) + return + } + gauge.Record(ctx, 1, otelmetric.WithAttributes( attribute.Bool("chip_ingress_enabled", m.chipService != nil), - // "" when disabled attribute.String("chip_ingress_endpoint", m.chipIngressEndpoint), )) - return nil } func (m *Manager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index 48b096c43bd..a292e24491d 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -284,32 +284,6 @@ func TestCorrectEndpointRouting(t *testing.T) { } } -func TestManager_ChipIngressService(t *testing.T) { - t.Parallel() - t.Run("disabled chip ingress", func(t *testing.T) { - t.Parallel() - tic := setupMockConfig(t, true) - tic.On("Endpoints").Return(nil) - - lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) - ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, nil, lggr) - assert.Nil(t, tm.chipService) - }) - - t.Run("enabled chip ingress", func(t *testing.T) { - t.Parallel() - tic := setupMockConfig(t, true) - tic.On("Endpoints").Return(nil) - - lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) - ks := keymocks.NewCSA(t) - chipSvc := &MockChipIngressService{} - tm := NewManager(tic, ks, chipSvc, lggr) - assert.Same(t, chipSvc, tm.chipService) - }) -} - func TestManager_ChipIngressEndpoint(t *testing.T) { t.Parallel() t.Run("all endpoints share one chip ingress service, started once", func(t *testing.T) { @@ -329,21 +303,22 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - chipSvc := &MockChipIngressService{} + chipSvc := mocks2.NewChipIngressService(t) chipSvc.On("Start", mock.Anything).Return(nil).Once() chipSvc.On("Close").Return(nil).Once() chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() + chipSvc.On("Send", mock.Anything, mock.Anything).Twice() tm := NewManager(tic, ks, chipSvc, lggr) require.Len(t, tm.endpoints, 2) for _, e := range tm.endpoints { - assert.Same(t, chipSvc, e.chipIngressClient) - assert.Nil(t, e.client) + assert.Nil(t, e.client, "chip mode must not create per-endpoint WSRPC clients") } require.NoError(t, tm.Start(t.Context())) + tm.GenMonitoringEndpoint("EVM", "1", "0xa", synchronization.OCR2Median).SendLog([]byte("a")) + tm.GenMonitoringEndpoint("EVM", "137", "0xb", synchronization.OCR2Median).SendLog([]byte("b")) require.NoError(t, tm.Close()) - chipSvc.AssertExpectations(t) }) t.Run("creates traditional endpoint when chip ingress disabled", func(t *testing.T) { @@ -365,7 +340,6 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { tm := NewManager(tic, ks, nil, lggr) require.Len(t, tm.endpoints, 1) - assert.Nil(t, tm.endpoints[0].chipIngressClient) assert.NotNil(t, tm.endpoints[0].client) }) } @@ -384,7 +358,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -401,7 +375,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.DebugLevel) ks := keymocks.NewCSA(t) - chipSvc := &MockChipIngressService{} + chipSvc := mocks2.NewChipIngressService(t) chipSvc.On("Start", mock.Anything).Return(nil) chipSvc.On("Close").Return(nil) chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() @@ -442,7 +416,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) // Invalid network/chainID combination that doesn't exist in chain-selectors me := tm.GenMonitoringEndpoint("INVALID", "999", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -497,7 +471,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) @@ -515,7 +489,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, &MockChipIngressService{}, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) // Invalid network/chainID combination me := tm.GenMultitypeMonitoringEndpoint("INVALID", "888", "0x456") assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) From b35b55bbb228f095cf3c2c36d7ab3732388d7815 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Thu, 8 Oct 2026 15:29:28 -0700 Subject: [PATCH 4/8] [INFOPLAT-19425] Make chip-ingress telemetry client easier to read - Split Send into newEvent, onPublished and drop so it reads top to bottom; build failures now share the rate-limited drop log. - Document what the client is for and which Kafka header each CloudEvent extension becomes. - Name the health ping interval/timeout and the endpoint metric label; rename the receiver to c. - Rewrite the NewManager and telemetryEndpoint doc comments. --- .../chip_ingress_batch_client.go | 148 ++++++++++-------- .../chip_ingress_batch_client_test.go | 6 +- core/services/telemetry/manager.go | 12 +- 3 files changed, 90 insertions(+), 76 deletions(-) diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index 09beb4b517c..fa50b20da73 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -15,15 +15,24 @@ import ( ) const ( - chipIngress = "chip-ingress" // endpoint label on telemetry_client_* metrics + // chipIngressEndpointLabel is the "endpoint" label on telemetry_client_* metrics. + chipIngressEndpointLabel = "chip-ingress" // legacyTelemetryBatchClientName is the client_name label on chip_ingress.batch.* metrics. legacyTelemetryBatchClientName = "legacy_telemetry" + // Shared-buffer sizing mirrors chainlink-common beholder.DefaultConfig() chip batch defaults. chipIngressMessageBufferSize = 10_000 chipIngressMaxBatchSize = 1_000 chipIngressMaxConcurrentSends = 10 + + healthPingInterval = 5 * time.Second + healthPingTimeout = 2 * time.Second ) +// chipIngressBatchClient sends the node's legacy OCR telemetry to chip-ingress, +// replacing the WSRPC path to OTI (OCR Telemetry Ingress). One instance is +// shared by every job. Each message becomes a CloudEvent whose extensions +// chip-ingress turns into the same Kafka key and headers OTI used to produce. type chipIngressBatchClient struct { services.Service eng *services.Engine @@ -36,10 +45,8 @@ type chipIngressBatchClient struct { errorCount atomic.Uint32 } -// NewChipIngressBatchClient returns a client backed by the shared -// chainlink-common chip-ingress batch client that can send telemetry to the -// chip ingress server. The batch client owns chipClient: Stop flushes the -// buffer and then closes the underlying gRPC connection. +// NewChipIngressBatchClient wraps chipClient in the chainlink-common batch client. +// The batch client owns chipClient: Close flushes the buffer, then closes the connection. func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, logging bool, lggr logger.Logger, sendInterval, sendTimeout time.Duration) (ChipIngressService, error) { batchClient, err := batch.NewBatchClient(chipClient, batch.WithMessageBuffer(chipIngressMessageBufferSize), @@ -71,79 +78,88 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin return c, nil } -// start starts the batch client and a 5s health ping that drives +// start starts the batch client and a periodic health ping that drives // telemetry_client_connection_status{endpoint="chip-ingress"}. -func (cc *chipIngressBatchClient) start(ctx context.Context) error { - cc.batchClient.Start(ctx) - cc.eng.GoTick(services.TickerConfig{}.NewTicker(5*time.Second), func(ctx context.Context) { - pingCtx, cancel := context.WithTimeout(ctx, 2*time.Second) +func (c *chipIngressBatchClient) start(ctx context.Context) error { + c.batchClient.Start(ctx) + c.eng.GoTick(services.TickerConfig{}.NewTicker(healthPingInterval), func(ctx context.Context) { + pingCtx, cancel := context.WithTimeout(ctx, healthPingTimeout) defer cancel() connected := float64(1) - if _, err := cc.chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}); err != nil { + if _, err := c.chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}); err != nil { connected = 0 - cc.eng.EmitHealthErr(err) + c.eng.EmitHealthErr(err) } - TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(connected) + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(connected) }) return nil } -// close stops the batch client: it flushes queued messages (up to the shutdown -// timeout) and then closes the underlying chip-ingress client. -func (cc *chipIngressBatchClient) close() error { - cc.batchClient.Stop() +// close flushes queued messages (up to the batch client's shutdown timeout), +// then closes the underlying chip-ingress client. +func (c *chipIngressBatchClient) close() error { + c.batchClient.Stop() return nil } -// Send converts a telemetry payload into a CloudEvent and queues it on the -// shared batch client. Queueing is non-blocking: if the shared buffer is full -// the message is dropped and a warning is logged with exponential back-off. -func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload) { - now := time.Now() - ev, err := cc.payloadToEvent(payload, now) +// Send builds the CloudEvent and queues it. It never blocks the caller (a libocr +// goroutine): if the event can't be built or the buffer is full, the message is +// dropped and counted. +func (c *chipIngressBatchClient) Send(_ context.Context, payload TelemPayload) { + event, err := c.newEvent(payload, time.Now()) + if err == nil { + err = c.batchClient.QueueMessage(event, c.onPublished(payload)) + } if err != nil { - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - cc.eng.Warnw("failed to build CloudEvent for ChIP ingress", "error", err, "contractID", payload.ContractID, "telemType", payload.TelemType) + c.drop(payload, err) return } + c.dropCount.Store(0) +} - err = cc.batchClient.QueueMessage(ev, func(sendErr error) { - if sendErr == nil { - TelemetryClientMessagesSent.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - if cc.logging { - cc.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", payload.ContractID, "telemType", payload.TelemType) +// onPublished returns the callback the batch client runs once the message's batch +// has been published, counting each message as sent or failed. +func (c *chipIngressBatchClient) onPublished(payload TelemPayload) func(error) { + return func(err error) { + if err != nil { + TelemetryClientMessagesSendErrors.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() + if count := c.errorCount.Add(1); shouldLogCount(count) { + c.eng.Warnw("Could not send telemetry via ChIP ingress", + "error", err, + "errorCode", batch.ErrorCodeFor(err), + "telemType", payload.TelemType, + "errorCount", count) } return } - TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - if count := cc.errorCount.Add(1); shouldLogCount(count) { - cc.eng.Warnw("Could not send telemetry via ChIP ingress", - "error", sendErr, - "errorCode", batch.ErrorCodeFor(sendErr), - "telemType", payload.TelemType, - "errorCount", count) + TelemetryClientMessagesSent.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() + if c.logging { + c.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", payload.ContractID, "telemType", payload.TelemType) } - }) - if err != nil { - // batch.ErrMessageBufferFull or batch.ErrClientShutdown - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() - if count := cc.dropCount.Add(1); shouldLogCount(count) { - cc.eng.Warnw("dropping telemetry message for ChIP ingress", - "error", err, - "contractID", payload.ContractID, - "telemType", payload.TelemType, - "droppedCount", count) - } - return } - cc.dropCount.Store(0) } -// payloadToEvent converts a telemetry payload into a CloudEvent with the -// OTI-parity extension set. sentat and receivedat are both stamped with now -// (enqueue time); chip-ingress legacy mode may overwrite receivedat with the -// true server-receive time later. -func (cc *chipIngressBatchClient) payloadToEvent(payload TelemPayload, now time.Time) (*chipingress.CloudEventPb, error) { +// drop counts a message that never reached the queue: its telemetry type has no +// chip-ingress mapping, the buffer is full, or the client is shutting down. +func (c *chipIngressBatchClient) drop(payload TelemPayload, err error) { + TelemetryClientMessagesDropped.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() + if count := c.dropCount.Add(1); shouldLogCount(count) { + c.eng.Warnw("Dropping telemetry message for ChIP ingress", + "error", err, + "contractID", payload.ContractID, + "telemType", payload.TelemType, + "droppedCount", count) + } +} + +// newEvent converts a payload into the CloudEvent chip-ingress expects for legacy +// telemetry. The event source/type select the Kafka topic. Each extension becomes a +// "ce_" Kafka header; the comments name the header OTI used for the same value. +// +// The node does not send nodeoperatorname/nodename: chip-ingress looks them up from +// the CSA key (WithNOPLookup), and sending them too would duplicate those headers. +// nodeoperatorkey is unknown to the node (spike INFOPLAT-19426). +func (c *chipIngressBatchClient) newEvent(payload TelemPayload, now time.Time) (*chipingress.CloudEventPb, error) { domain, entity, err := TelemetryTypeToDomainAndEntity(payload.TelemType) if err != nil { return nil, err @@ -153,19 +169,15 @@ func (cc *chipIngressBatchClient) payloadToEvent(payload TelemPayload, now time. return nil, fmt.Errorf("failed creating CloudEvent: %w", err) } - event.SetExtension("legacytelemetry", "true") - event.SetExtension("telemetrytype", string(payload.TelemType)) - event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) - event.SetExtension("networkname", payload.Network) - event.SetExtension("contractid", payload.ContractID) - event.SetExtension("csapublickey", cc.csaPubKeyHex) - event.SetExtension("partitionkey", payload.ContractID+"-"+cc.csaPubKeyHex) - event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) - event.SetExtension("receivedat", strconv.FormatInt(now.UnixMilli(), 10)) - // nodeoperatorname / nodename are stamped server-side by chip-ingress via - // csa-auth (WithNOPLookup); the node must not send them, or chip-ingress - // would emit duplicate conflicting headers. nodeoperatorkey is not known to - // the node at all (spike INFOPLAT-19426). + event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry + event.SetExtension("telemetrytype", string(payload.TelemType)) // "telemetry-type" + event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) // chain-selectors ID of the job's chain + event.SetExtension("networkname", payload.Network) // endpoint network, e.g. "EVM" + event.SetExtension("contractid", payload.ContractID) // "contract-address" + event.SetExtension("csapublickey", c.csaPubKeyHex) // "node-operator-csa-public-key" + event.SetExtension("partitionkey", payload.ContractID+"-"+c.csaPubKeyHex) // Kafka record key + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns + event.SetExtension("receivedat", strconv.FormatInt(now.UnixMilli(), 10)) // "received-at", unix ms; chip-ingress may overwrite with its receive time return chipingress.EventToProto(event) } diff --git a/core/services/synchronization/chip_ingress_batch_client_test.go b/core/services/synchronization/chip_ingress_batch_client_test.go index 3e818fc456b..fe8bcb21ebb 100644 --- a/core/services/synchronization/chip_ingress_batch_client_test.go +++ b/core/services/synchronization/chip_ingress_batch_client_test.go @@ -95,7 +95,7 @@ func awaitFirstEvent(t *testing.T, published *publishedBatches) chipingress.Clou } func counterValue(vec *prometheus.CounterVec, telemType TelemetryType) float64 { - return testutil.ToFloat64(vec.WithLabelValues(chipIngress, string(telemType))) + return testutil.ToFloat64(vec.WithLabelValues(chipIngressEndpointLabel, string(telemType))) } func TestChipIngressBatchClient_Event(t *testing.T) { @@ -173,10 +173,10 @@ func TestChipIngressBatchClient_CountsPerMessage(t *testing.T) { //nolint:parall func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // resets a package-global prometheus gauge // The status gauge is package-global; reset it so a value left by another test can't satisfy the check. - TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(0) + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(0) startBatchClient(t, publishOK) require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.InDelta(c, 1, testutil.ToFloat64(TelemetryClientConnectionStatus.WithLabelValues(chipIngress)), 0.001) + assert.InDelta(c, 1, testutil.ToFloat64(TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel)), 0.001) }, testutils.WaitTimeout(t), 10*time.Millisecond) } diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index 607d9cfc9c2..a5a9fbedcc3 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -39,18 +39,20 @@ type Manager struct { chipIngressEndpoint string // "" when disabled } -// telemetryEndpoint is one [[TelemetryIngress.Endpoints]] entry. With chip -// ingress enabled it only allowlists the chain and client is nil. +// telemetryEndpoint is one [[TelemetryIngress.Endpoints]] entry, keyed by Network and ChainID. type telemetryEndpoint struct { ChainID string Network string URL *url.URL - client synchronization.TelemetryService + client synchronization.TelemetryService // WSRPC client; nil in chip mode, where the entry only allowlists the chain PubKey string } -// NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints -// chipService is the shared chip-ingress service to use when ChipIngressEnabled is true; nil when the flag is off. +// NewManager creates the telemetry manager, which hands each job a monitoring +// endpoint for its chain and runs the clients behind them. +// +// A non-nil chipService puts the manager in chip mode: every allowlisted chain sends +// through that one shared service. With nil, each endpoint gets its own WSRPC client. func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, chipService synchronization.ChipIngressService, lggr common.Logger) *Manager { var chipIngressEndpoint string if chipService != nil { From 983d64f812d555724517cc2bd7b5a7ae80e4f033 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Fri, 9 Oct 2026 11:30:14 -0700 Subject: [PATCH 5/8] [INFOPLAT-19425] Make chip-ingress telemetry batching configurable Add TelemetryIngress.ChipIngressBufferSize (10000), ChipIngressMaxBatchSize (1000), ChipIngressMaxConcurrentSends (10) and ChipIngressDrainTimeout (5s), validated > 0 when chip ingress is enabled. Defaults match the previous hard-coded values and Beholder's chip batch defaults. SendInterval and SendTimeout continue to drive the batch interval and per-RPC timeout. NewChipIngressBatchClient now takes a ChipIngressBatchConfig. --- .changeset/telemetry-chip-ingress-endpoint.md | 2 +- core/config/docs/core.toml | 12 +- core/config/mocks/telemetry_ingress.go | 180 ++++++++++++++++++ core/config/telemetry_ingress_config.go | 4 + core/config/toml/types.go | 28 +++ core/services/chainlink/application.go | 10 +- .../chainlink/config_telemetry_ingress.go | 16 ++ core/services/chainlink/config_test.go | 20 ++ .../testdata/config-empty-effective.toml | 4 + .../chainlink/testdata/config-full.toml | 4 + .../config-multi-chain-effective.toml | 4 + .../chip_ingress_batch_client.go | 32 ++-- core/services/synchronization/helpers_test.go | 12 +- .../testdata/config-empty-effective.toml | 4 + core/web/resolver/testdata/config-full.toml | 4 + .../config-multi-chain-effective.toml | 4 + docs/CONFIG.md | 32 +++- .../scripts/config/merge_raw_configs.txtar | 4 + testdata/scripts/node/validate/default.txtar | 4 + .../node/validate/defaults-override.txtar | 4 + .../disk-based-logging-disabled.txtar | 4 + .../validate/disk-based-logging-no-dir.txtar | 4 + .../node/validate/disk-based-logging.txtar | 4 + .../node/validate/fallback-override.txtar | 4 + .../node/validate/invalid-ocr-p2p.txtar | 4 + testdata/scripts/node/validate/invalid.txtar | 4 + testdata/scripts/node/validate/valid.txtar | 4 + testdata/scripts/node/validate/warnings.txtar | 4 + 28 files changed, 396 insertions(+), 20 deletions(-) diff --git a/.changeset/telemetry-chip-ingress-endpoint.md b/.changeset/telemetry-chip-ingress-endpoint.md index 193d87f737d..6ffe7dae131 100644 --- a/.changeset/telemetry-chip-ingress-endpoint.md +++ b/.changeset/telemetry-chip-ingress-endpoint.md @@ -2,4 +2,4 @@ "chainlink": patch --- -Telemetry: with `TelemetryIngress.ChipIngressEnabled = true`, the node now sends legacy telemetry to chip-ingress through its own CSA-authenticated client and one shared batch client, instead of relying on Beholder's chip client (which silently dropped telemetry unless `Telemetry.ChipIngressEndpoint` was also set). Added `TelemetryIngress.ChipIngressEndpoint` (default `legacy-telemetry.prod.telemetry.chain.link:443`) and `TelemetryIngress.ChipIngressInsecureConnection`. `TelemetryIngress.Endpoints` entries still select which chains send telemetry. `telemetry_client_messages_sent{endpoint="chip-ingress"}` now counts messages instead of batches. #added +Telemetry: with `TelemetryIngress.ChipIngressEnabled = true`, the node now sends legacy telemetry to chip-ingress through its own CSA-authenticated client and one shared batch client, instead of relying on Beholder's chip client (which silently dropped telemetry unless `Telemetry.ChipIngressEndpoint` was also set). Added `TelemetryIngress.ChipIngressEndpoint` (default `legacy-telemetry.prod.telemetry.chain.link:443`) and `TelemetryIngress.ChipIngressInsecureConnection`, plus batching settings `ChipIngressBufferSize` (10000), `ChipIngressMaxBatchSize` (1000), `ChipIngressMaxConcurrentSends` (10) and `ChipIngressDrainTimeout` (5s); `SendInterval` and `SendTimeout` also apply to the chip-ingress batch client. `TelemetryIngress.Endpoints` entries still select which chains send telemetry. `telemetry_client_messages_sent{endpoint="chip-ingress"}` now counts messages instead of batches. #added diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 3a1b9632d34..389156d798d 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -108,9 +108,9 @@ Logging = false # Default BufferSize = 100 # Default # MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. MaxBatchSize = 50 # Default -# SendInterval determines how often batched telemetry is sent to the ingress server. +# SendInterval determines how often batched telemetry is sent to the ingress server. With ChipIngressEnabled, it is the max wait before flushing an incomplete batch. SendInterval = '500ms' # Default -# SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. +# SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. With ChipIngressEnabled, it is the per-RPC timeout for PublishBatch. SendTimeout = '10s' # Default # UseBatchSend toggles sending telemetry to the ingress server using the batch client. UseBatchSend = true # Default @@ -120,6 +120,14 @@ ChipIngressEnabled = false # Default ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default # ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. ChipIngressInsecureConnection = false # Default +# ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. +ChipIngressBufferSize = 10000 # Default +# ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. +ChipIngressMaxBatchSize = 1000 # Default +# ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. +ChipIngressMaxConcurrentSends = 10 # Default +# ChipIngressDrainTimeout is the max time to flush buffered chip-ingress telemetry on shutdown. +ChipIngressDrainTimeout = '5s' # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index d3c07519469..f3c017cb48f 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -67,6 +67,96 @@ func (_c *TelemetryIngress_BufferSize_Call) RunAndReturn(run func() uint) *Telem return _c } +// ChipIngressBufferSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressBufferSize() uint { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressBufferSize") + } + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +// TelemetryIngress_ChipIngressBufferSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressBufferSize' +type TelemetryIngress_ChipIngressBufferSize_Call struct { + *mock.Call +} + +// ChipIngressBufferSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressBufferSize() *TelemetryIngress_ChipIngressBufferSize_Call { + return &TelemetryIngress_ChipIngressBufferSize_Call{Call: _e.mock.On("ChipIngressBufferSize")} +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) Run(run func()) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) Return(_a0 uint) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) RunAndReturn(run func() uint) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressDrainTimeout provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressDrainTimeout() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressDrainTimeout") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressDrainTimeout_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressDrainTimeout' +type TelemetryIngress_ChipIngressDrainTimeout_Call struct { + *mock.Call +} + +// ChipIngressDrainTimeout is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressDrainTimeout() *TelemetryIngress_ChipIngressDrainTimeout_Call { + return &TelemetryIngress_ChipIngressDrainTimeout_Call{Call: _e.mock.On("ChipIngressDrainTimeout")} +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) Run(run func()) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Return(run) + return _c +} + // ChipIngressEnabled provides a mock function with no fields func (_m *TelemetryIngress) ChipIngressEnabled() bool { ret := _m.Called() @@ -202,6 +292,96 @@ func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) RunAndReturn(run return _c } +// ChipIngressMaxBatchSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxBatchSize() uint { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxBatchSize") + } + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxBatchSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxBatchSize' +type TelemetryIngress_ChipIngressMaxBatchSize_Call struct { + *mock.Call +} + +// ChipIngressMaxBatchSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxBatchSize() *TelemetryIngress_ChipIngressMaxBatchSize_Call { + return &TelemetryIngress_ChipIngressMaxBatchSize_Call{Call: _e.mock.On("ChipIngressMaxBatchSize")} +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) Return(_a0 uint) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) RunAndReturn(run func() uint) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressMaxConcurrentSends provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxConcurrentSends() int { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxConcurrentSends") + } + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxConcurrentSends_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxConcurrentSends' +type TelemetryIngress_ChipIngressMaxConcurrentSends_Call struct { + *mock.Call +} + +// ChipIngressMaxConcurrentSends is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxConcurrentSends() *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + return &TelemetryIngress_ChipIngressMaxConcurrentSends_Call{Call: _e.mock.On("ChipIngressMaxConcurrentSends")} +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) Return(_a0 int) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) RunAndReturn(run func() int) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Return(run) + return _c +} + // Endpoints provides a mock function with no fields func (_m *TelemetryIngress) Endpoints() []config.TelemetryIngressEndpoint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index d7b7e5c0ca5..46d121585c2 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -17,6 +17,10 @@ type TelemetryIngress interface { ChipIngressEnabled() bool ChipIngressEndpoint() string ChipIngressInsecureConnection() bool + ChipIngressBufferSize() uint + ChipIngressMaxBatchSize() uint + ChipIngressMaxConcurrentSends() int + ChipIngressDrainTimeout() time.Duration } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index f21ec9f63d1..5baaa2fffc0 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -984,6 +984,10 @@ type TelemetryIngress struct { ChipIngressEnabled *bool ChipIngressEndpoint *string ChipIngressInsecureConnection *bool + ChipIngressBufferSize *uint + ChipIngressMaxBatchSize *uint + ChipIngressMaxConcurrentSends *int + ChipIngressDrainTimeout *commonconfig.Duration } type TelemetryIngressEndpoint struct { @@ -1027,6 +1031,18 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressInsecureConnection; v != nil { t.ChipIngressInsecureConnection = v } + if v := f.ChipIngressBufferSize; v != nil { + t.ChipIngressBufferSize = v + } + if v := f.ChipIngressMaxBatchSize; v != nil { + t.ChipIngressMaxBatchSize = v + } + if v := f.ChipIngressMaxConcurrentSends; v != nil { + t.ChipIngressMaxConcurrentSends = v + } + if v := f.ChipIngressDrainTimeout; v != nil { + t.ChipIngressDrainTimeout = v + } } func (t *TelemetryIngress) ValidateConfig() (err error) { @@ -1036,6 +1052,18 @@ func (t *TelemetryIngress) ValidateConfig() (err error) { if t.ChipIngressEndpoint == nil || *t.ChipIngressEndpoint == "" { err = errors.Join(err, configutils.MissingError{Name: "ChipIngressEndpoint", Msg: "must be set when ChipIngressEnabled is true"}) } + if v := t.ChipIngressBufferSize; v != nil && *v == 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressBufferSize", Value: *v, Msg: "must be greater than 0"}) + } + if v := t.ChipIngressMaxBatchSize; v != nil && *v == 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxBatchSize", Value: *v, Msg: "must be greater than 0"}) + } + if v := t.ChipIngressMaxConcurrentSends; v != nil && *v <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxConcurrentSends", Value: *v, Msg: "must be greater than 0"}) + } + if v := t.ChipIngressDrainTimeout; v != nil && v.Duration() <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressDrainTimeout", Value: v.Duration(), Msg: "must be greater than 0"}) + } return err } diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 381d376504a..2500608523a 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -553,7 +553,15 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err if chipErr != nil { return nil, fmt.Errorf("failed to create telemetry chip-ingress client: %w", chipErr) } - telemChipService, chipErr = synchronization.NewChipIngressBatchClient(chipClient, csaPubKeyHex, ti.Logging(), globalLogger, ti.SendInterval(), ti.SendTimeout()) + telemChipService, chipErr = synchronization.NewChipIngressBatchClient(chipClient, csaPubKeyHex, synchronization.ChipIngressBatchConfig{ + BufferSize: ti.ChipIngressBufferSize(), + MaxBatchSize: ti.ChipIngressMaxBatchSize(), + MaxConcurrentSends: ti.ChipIngressMaxConcurrentSends(), + SendInterval: ti.SendInterval(), + SendTimeout: ti.SendTimeout(), + DrainTimeout: ti.ChipIngressDrainTimeout(), + Logging: ti.Logging(), + }, globalLogger) if chipErr != nil { _ = chipClient.Close() // the batch client never took ownership of chipClient return nil, fmt.Errorf("failed to create telemetry chip-ingress batch client: %w", chipErr) diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 5fe384cd5f1..47707bf3b5b 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -68,6 +68,22 @@ func (t *telemetryIngressConfig) ChipIngressInsecureConnection() bool { return *t.c.ChipIngressInsecureConnection } +func (t *telemetryIngressConfig) ChipIngressBufferSize() uint { + return *t.c.ChipIngressBufferSize +} + +func (t *telemetryIngressConfig) ChipIngressMaxBatchSize() uint { + return *t.c.ChipIngressMaxBatchSize +} + +func (t *telemetryIngressConfig) ChipIngressMaxConcurrentSends() int { + return *t.c.ChipIngressMaxConcurrentSends +} + +func (t *telemetryIngressConfig) ChipIngressDrainTimeout() time.Duration { + return t.c.ChipIngressDrainTimeout.Duration() +} + func (t *telemetryIngressEndpointConfig) Network() string { return *t.c.Network } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 114980d3943..3e7eec76ad4 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -256,6 +256,10 @@ func TestConfig_Marshal(t *testing.T) { ChipIngressEnabled: new(false), ChipIngressEndpoint: new("legacy-telemetry.example:443"), ChipIngressInsecureConnection: new(true), + ChipIngressBufferSize: new(uint(2000)), + ChipIngressMaxBatchSize: new(uint(200)), + ChipIngressMaxConcurrentSends: new(4), + ChipIngressDrainTimeout: commonconfig.MustNewDuration(3 * time.Second), Endpoints: []toml.TelemetryIngressEndpoint{ { Network: new("EVM"), @@ -943,6 +947,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.example:443' ChipIngressInsecureConnection = true +ChipIngressBufferSize = 2000 +ChipIngressMaxBatchSize = 200 +ChipIngressMaxConcurrentSends = 4 +ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] Network = 'EVM' @@ -1515,6 +1523,18 @@ func TestConfig_Validate(t *testing.T) { ChipIngressEnabled = true ChipIngressEndpoint = '' `, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: missing: must be set when ChipIngressEnabled is true`}, + {name: "TelemetryIngress chip batching must be positive", toml: ` +[TelemetryIngress] +ChipIngressEnabled = true +ChipIngressBufferSize = 0 +ChipIngressMaxBatchSize = 0 +ChipIngressMaxConcurrentSends = 0 +ChipIngressDrainTimeout = '0s' +`, exp: `invalid configuration: TelemetryIngress: 4 errors: + - ChipIngressBufferSize: invalid value (0): must be greater than 0 + - ChipIngressMaxBatchSize: invalid value (0): must be greater than 0 + - ChipIngressMaxConcurrentSends: invalid value (0): must be greater than 0 + - ChipIngressDrainTimeout: invalid value (0s): must be greater than 0`}, } { t.Run(tt.name, func(t *testing.T) { var c Config diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 3a5406cce6a..f0d1ff6d8f9 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 4335f8bcbba..c2ddd604a5a 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.example:443' ChipIngressInsecureConnection = true +ChipIngressBufferSize = 2000 +ChipIngressMaxBatchSize = 200 +ChipIngressMaxConcurrentSends = 4 +ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 7528256f31e..e11e5fa694c 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = true diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index fa50b20da73..45a16aad0d3 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -20,11 +20,6 @@ const ( // legacyTelemetryBatchClientName is the client_name label on chip_ingress.batch.* metrics. legacyTelemetryBatchClientName = "legacy_telemetry" - // Shared-buffer sizing mirrors chainlink-common beholder.DefaultConfig() chip batch defaults. - chipIngressMessageBufferSize = 10_000 - chipIngressMaxBatchSize = 1_000 - chipIngressMaxConcurrentSends = 10 - healthPingInterval = 5 * time.Second healthPingTimeout = 2 * time.Second ) @@ -45,15 +40,28 @@ type chipIngressBatchClient struct { errorCount atomic.Uint32 } +// ChipIngressBatchConfig sizes the shared chip-ingress batch client. Values come from +// [TelemetryIngress] (ChipIngress* fields plus SendInterval/SendTimeout). +type ChipIngressBatchConfig struct { + BufferSize uint // messages buffered across all jobs before new ones are dropped + MaxBatchSize uint // messages per PublishBatch request + MaxConcurrentSends int // PublishBatch requests in flight + SendInterval time.Duration // max wait before flushing an incomplete batch + SendTimeout time.Duration // per-request PublishBatch timeout + DrainTimeout time.Duration // max time to flush the buffer on Close + Logging bool // debug-log every successful send +} + // NewChipIngressBatchClient wraps chipClient in the chainlink-common batch client. // The batch client owns chipClient: Close flushes the buffer, then closes the connection. -func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, logging bool, lggr logger.Logger, sendInterval, sendTimeout time.Duration) (ChipIngressService, error) { +func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, cfg ChipIngressBatchConfig, lggr logger.Logger) (ChipIngressService, error) { batchClient, err := batch.NewBatchClient(chipClient, - batch.WithMessageBuffer(chipIngressMessageBufferSize), - batch.WithBatchSize(chipIngressMaxBatchSize), - batch.WithMaxConcurrentSends(chipIngressMaxConcurrentSends), - batch.WithBatchInterval(sendInterval), - batch.WithMaxPublishTimeout(sendTimeout), + batch.WithMessageBuffer(int(cfg.BufferSize)), + batch.WithBatchSize(int(cfg.MaxBatchSize)), + batch.WithMaxConcurrentSends(cfg.MaxConcurrentSends), + batch.WithBatchInterval(cfg.SendInterval), + batch.WithMaxPublishTimeout(cfg.SendTimeout), + batch.WithShutdownTimeout(cfg.DrainTimeout), batch.WithClientName(legacyTelemetryBatchClientName), // Send builds a fresh event per message and never touches it after queueing, // so skip the defensive per-message proto.Clone. @@ -67,7 +75,7 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin chipClient: chipClient, batchClient: batchClient, csaPubKeyHex: csaPubKeyHex, - logging: logging, + logging: cfg.Logging, } c.Service, c.eng = services.Config{ Name: "ChipIngressBatchClient", diff --git a/core/services/synchronization/helpers_test.go b/core/services/synchronization/helpers_test.go index 239a88f60bc..20c95f485d2 100644 --- a/core/services/synchronization/helpers_test.go +++ b/core/services/synchronization/helpers_test.go @@ -28,9 +28,17 @@ func NewTestTelemetryIngressBatchClient(t *testing.T, url *url.URL, serverPubKey return tc } -// NewTestChipIngressBatchClient calls NewChipIngressBatchClient with a fixed test CSA key. +// NewTestChipIngressBatchClient calls NewChipIngressBatchClient with a fixed test CSA key and default sizing. func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) ChipIngressService { - c, err := NewChipIngressBatchClient(chipClient, "deadbeef", logging, logger.TestLogger(t), sendInterval, time.Second) + c, err := NewChipIngressBatchClient(chipClient, "deadbeef", ChipIngressBatchConfig{ + BufferSize: 10_000, + MaxBatchSize: 1_000, + MaxConcurrentSends: 10, + SendInterval: sendInterval, + SendTimeout: time.Second, + DrainTimeout: 5 * time.Second, + Logging: logging, + }, logger.TestLogger(t)) require.NoError(t, err) return c } diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 3a5406cce6a..f0d1ff6d8f9 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 064cc32c6c3..c1ae818d0e6 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 433048c65ea..c5998e566a5 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -46,6 +46,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = true diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 43b1462d948..fabf5e2fc00 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -283,6 +283,10 @@ UseBatchSend = true # Default ChipIngressEnabled = false # Default ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default ChipIngressInsecureConnection = false # Default +ChipIngressBufferSize = 10000 # Default +ChipIngressMaxBatchSize = 1000 # Default +ChipIngressMaxConcurrentSends = 10 # Default +ChipIngressDrainTimeout = '5s' # Default ``` @@ -314,13 +318,13 @@ MaxBatchSize is the maximum number of messages to batch into one telemetry reque ```toml SendInterval = '500ms' # Default ``` -SendInterval determines how often batched telemetry is sent to the ingress server. +SendInterval determines how often batched telemetry is sent to the ingress server. With ChipIngressEnabled, it is the max wait before flushing an incomplete batch. ### SendTimeout ```toml SendTimeout = '10s' # Default ``` -SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. +SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. With ChipIngressEnabled, it is the per-RPC timeout for PublishBatch. ### UseBatchSend ```toml @@ -346,6 +350,30 @@ ChipIngressInsecureConnection = false # Default ``` ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. +### ChipIngressBufferSize +```toml +ChipIngressBufferSize = 10000 # Default +``` +ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. + +### ChipIngressMaxBatchSize +```toml +ChipIngressMaxBatchSize = 1000 # Default +``` +ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. + +### ChipIngressMaxConcurrentSends +```toml +ChipIngressMaxConcurrentSends = 10 # Default +``` +ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. + +### ChipIngressDrainTimeout +```toml +ChipIngressDrainTimeout = '5s' # Default +``` +ChipIngressDrainTimeout is the max time to flush buffered chip-ingress telemetry on shutdown. + ## TelemetryIngress.Endpoints ```toml [[TelemetryIngress.Endpoints]] # Example diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 60b7c588d63..4a7b0b4f07a 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -193,6 +193,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index b92a6651020..9c4969dcb6d 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -58,6 +58,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index b227af8e8a5..9f7fe300620 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -119,6 +119,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 8b556e20464..8ea96354b13 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -102,6 +102,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 182630a9484..51ea24d01a3 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -102,6 +102,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 442222e789c..042c9e0afe5 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -102,6 +102,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index fa5ef5cd045..d537ff13f8d 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -201,6 +201,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index f858c20f8f0..f5216106071 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -87,6 +87,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index b95ae60f735..a8732bfb5c5 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -98,6 +98,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 24a2bc4ba3b..a9a4c01fe21 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -99,6 +99,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 820089c5ba0..6faacaa1ef7 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -81,6 +81,10 @@ UseBatchSend = true ChipIngressEnabled = false ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 1000 +ChipIngressMaxConcurrentSends = 10 +ChipIngressDrainTimeout = '5s' [AuditLogger] Enabled = false From d4ee8431b749d9d5cdae88e91ed1b9d8360ed598 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Tue, 13 Oct 2026 13:22:02 -0700 Subject: [PATCH 6/8] [INFOPLAT-19425] Document that Close discards telemetry still queued With the pinned chainlink-common batch client, Stop flushes only the batch being filled; messages queued behind in-flight sends are discarded without a callback (found in local load testing). Correct the Close and ChipIngressDrainTimeout docs. Fix upstream: smartcontractkit/chainlink-common#2461. --- core/config/docs/core.toml | 2 +- .../synchronization/chip_ingress_batch_client.go | 10 ++++++---- docs/CONFIG.md | 2 +- 3 files changed, 8 insertions(+), 6 deletions(-) diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 389156d798d..9470e1b16d8 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -126,7 +126,7 @@ ChipIngressBufferSize = 10000 # Default ChipIngressMaxBatchSize = 1000 # Default # ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. ChipIngressMaxConcurrentSends = 10 # Default -# ChipIngressDrainTimeout is the max time to flush buffered chip-ingress telemetry on shutdown. +# ChipIngressDrainTimeout is the max time shutdown waits for in-flight chip-ingress sends to finish. Messages still queued behind them at shutdown are discarded. ChipIngressDrainTimeout = '5s' # Default [[TelemetryIngress.Endpoints]] # Example diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index 45a16aad0d3..f336e2136a4 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -48,12 +48,12 @@ type ChipIngressBatchConfig struct { MaxConcurrentSends int // PublishBatch requests in flight SendInterval time.Duration // max wait before flushing an incomplete batch SendTimeout time.Duration // per-request PublishBatch timeout - DrainTimeout time.Duration // max time to flush the buffer on Close + DrainTimeout time.Duration // max time Close waits for in-flight sends and callbacks Logging bool // debug-log every successful send } // NewChipIngressBatchClient wraps chipClient in the chainlink-common batch client. -// The batch client owns chipClient: Close flushes the buffer, then closes the connection. +// The batch client owns chipClient: Close finishes in-flight sends, then closes the connection. func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, cfg ChipIngressBatchConfig, lggr logger.Logger) (ChipIngressService, error) { batchClient, err := batch.NewBatchClient(chipClient, batch.WithMessageBuffer(int(cfg.BufferSize)), @@ -103,8 +103,10 @@ func (c *chipIngressBatchClient) start(ctx context.Context) error { return nil } -// close flushes queued messages (up to the batch client's shutdown timeout), -// then closes the underlying chip-ingress client. +// close sends the batch currently being filled, waits (up to the batch client's shutdown +// timeout) for in-flight sends and callbacks, then closes the chip-ingress client. +// With the pinned chainlink-common version, messages still queued behind in-flight sends +// are discarded without a callback, so telemetry_client_messages_dropped does not count them. func (c *chipIngressBatchClient) close() error { c.batchClient.Stop() return nil diff --git a/docs/CONFIG.md b/docs/CONFIG.md index fabf5e2fc00..bae7d96667f 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -372,7 +372,7 @@ ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests ```toml ChipIngressDrainTimeout = '5s' # Default ``` -ChipIngressDrainTimeout is the max time to flush buffered chip-ingress telemetry on shutdown. +ChipIngressDrainTimeout is the max time shutdown waits for in-flight chip-ingress sends to finish. Messages still queued behind them at shutdown are discarded. ## TelemetryIngress.Endpoints ```toml From 1f61d2fda368bce5559480ec155f836488828f82 Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Tue, 13 Oct 2026 13:46:34 -0700 Subject: [PATCH 7/8] [INFOPLAT-19425] Explain why WithNOPLookup is load-bearing --- core/services/synchronization/chip_ingress_client.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/core/services/synchronization/chip_ingress_client.go b/core/services/synchronization/chip_ingress_client.go index 4d7f0879b3b..768661e4d42 100644 --- a/core/services/synchronization/chip_ingress_client.go +++ b/core/services/synchronization/chip_ingress_client.go @@ -40,6 +40,10 @@ func NewChipIngressClient(cfg ChipIngressClientConfig) (chipingress.Client, erro if auth != nil { // nil only when there are no headers and no TTL opts = append(opts, chipingress.WithTokenAuth(auth)) } + // Load-bearing: chip-ingress derives the legacy telemetry identity headers + // (ce_nodeoperatorname/node-operator-name, ce_nodename/node-name) from the + // NOP info this option requests via x-include-nop-info. The node does not + // send them itself, so removing this silently blanks those headers downstream. opts = append(opts, chipingress.WithNOPLookup()) // grpc.NewClient connects lazily: an unreachable endpoint does not fail From 2b9a66e8677c39c444261cd1d3767fb099d7a9bc Mon Sep 17 00:00:00 2001 From: Chet Sharma Date: Thu, 15 Oct 2026 13:15:05 -0700 Subject: [PATCH 8/8] Address chip-ingress telemetry review feedback - Remove the default ChipIngressEndpoint; it is required when the flag is on. - Validate chip settings even with the flag off: bounds, host:port syntax, no insecure connection to chain.link hosts, and DrainTimeout under half of ShutdownGracePeriod. - Add chip-specific SendInterval, SendTimeout and MaxGRPCRequestSize. - Lower defaults: MaxBatchSize 100, DrainTimeout 1s. - Stop sending receivedat; chip-ingress owns it. - Count unsupported telemetry types separately from drops and rate-limit repeated warnings by time. - Mark the service unhealthy only after a sustained ping outage. - Chip mode endpoints need only Network and ChainID; skip the chip client when no endpoints are configured. - Remove the unused TelemPayload.ChainID and the changeset. --- .changeset/telemetry-chip-ingress-endpoint.md | 5 - core/config/docs/core.toml | 30 ++-- core/config/mocks/telemetry_ingress.go | 135 ++++++++++++++++ core/config/telemetry_ingress_config.go | 3 + core/config/toml/types.go | 100 ++++++++++-- core/services/chainlink/application.go | 9 +- .../chainlink/config_telemetry_ingress.go | 18 ++- core/services/chainlink/config_test.go | 46 +++++- .../testdata/config-empty-effective.toml | 9 +- .../chainlink/testdata/config-full.toml | 3 + .../config-multi-chain-effective.toml | 9 +- .../chip_ingress_batch_client.go | 146 +++++++++++------- .../chip_ingress_batch_client_test.go | 104 ++++++++++--- core/services/synchronization/common.go | 1 - core/services/synchronization/metrics.go | 5 + core/services/telemetry/manager.go | 26 ++-- core/services/telemetry/manager_test.go | 6 +- .../testdata/config-empty-effective.toml | 9 +- core/web/resolver/testdata/config-full.toml | 9 +- .../config-multi-chain-effective.toml | 9 +- docs/CONFIG.md | 51 ++++-- .../scripts/config/merge_raw_configs.txtar | 9 +- testdata/scripts/node/validate/default.txtar | 9 +- .../node/validate/defaults-override.txtar | 9 +- .../disk-based-logging-disabled.txtar | 9 +- .../validate/disk-based-logging-no-dir.txtar | 9 +- .../node/validate/disk-based-logging.txtar | 9 +- .../node/validate/fallback-override.txtar | 9 +- .../node/validate/invalid-ocr-p2p.txtar | 9 +- testdata/scripts/node/validate/invalid.txtar | 9 +- testdata/scripts/node/validate/valid.txtar | 9 +- testdata/scripts/node/validate/warnings.txtar | 9 +- 32 files changed, 636 insertions(+), 196 deletions(-) delete mode 100644 .changeset/telemetry-chip-ingress-endpoint.md diff --git a/.changeset/telemetry-chip-ingress-endpoint.md b/.changeset/telemetry-chip-ingress-endpoint.md deleted file mode 100644 index 6ffe7dae131..00000000000 --- a/.changeset/telemetry-chip-ingress-endpoint.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -"chainlink": patch ---- - -Telemetry: with `TelemetryIngress.ChipIngressEnabled = true`, the node now sends legacy telemetry to chip-ingress through its own CSA-authenticated client and one shared batch client, instead of relying on Beholder's chip client (which silently dropped telemetry unless `Telemetry.ChipIngressEndpoint` was also set). Added `TelemetryIngress.ChipIngressEndpoint` (default `legacy-telemetry.prod.telemetry.chain.link:443`) and `TelemetryIngress.ChipIngressInsecureConnection`, plus batching settings `ChipIngressBufferSize` (10000), `ChipIngressMaxBatchSize` (1000), `ChipIngressMaxConcurrentSends` (10) and `ChipIngressDrainTimeout` (5s); `SendInterval` and `SendTimeout` also apply to the chip-ingress batch client. `TelemetryIngress.Endpoints` entries still select which chains send telemetry. `telemetry_client_messages_sent{endpoint="chip-ingress"}` now counts messages instead of batches. #added diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 63618fe51b7..29442e333f3 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -108,26 +108,32 @@ Logging = false # Default BufferSize = 100 # Default # MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. MaxBatchSize = 50 # Default -# SendInterval determines how often batched telemetry is sent to the ingress server. With ChipIngressEnabled, it is the max wait before flushing an incomplete batch. +# SendInterval determines how often batched telemetry is sent to the ingress server. SendInterval = '500ms' # Default -# SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. With ChipIngressEnabled, it is the per-RPC timeout for PublishBatch. +# SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. SendTimeout = '10s' # Default # UseBatchSend toggles sending telemetry to the ingress server using the batch client. UseBatchSend = true # Default -# ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry. +# ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry; in this mode only their Network and ChainID are used. ChipIngressEnabled = false # Default -# ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint used when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default -# ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. +# ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint. Required when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. +ChipIngressEndpoint = '' # Default +# ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups; not allowed for chain.link hosts. ChipIngressInsecureConnection = false # Default -# ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. +# ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. Between 1 and 100000. ChipIngressBufferSize = 10000 # Default -# ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. -ChipIngressMaxBatchSize = 1000 # Default -# ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. +# ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. Between 1 and 500. +ChipIngressMaxBatchSize = 100 # Default +# ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. Between 1 and 32. ChipIngressMaxConcurrentSends = 10 # Default -# ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. -ChipIngressDrainTimeout = '5s' # Default +# ChipIngressSendInterval is the max wait before flushing an incomplete chip-ingress batch. Between 50ms and 10s. +ChipIngressSendInterval = '500ms' # Default +# ChipIngressSendTimeout is the per-request timeout for chip-ingress PublishBatch. Between 1s and 1m. +ChipIngressSendTimeout = '10s' # Default +# ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes; larger batches are split. Between 1 MiB and 10 MiB, and must not exceed the server's receive limit. +ChipIngressMaxGRPCRequestSize = 10485760 # Default +# ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. With ChipIngressEnabled, must be less than half of ShutdownGracePeriod. +ChipIngressDrainTimeout = '1s' # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index f3c017cb48f..fdd4c2e5d8d 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -382,6 +382,141 @@ func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) RunAndReturn(run return _c } +// ChipIngressMaxGRPCRequestSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxGRPCRequestSize() int { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxGRPCRequestSize") + } + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxGRPCRequestSize' +type TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call struct { + *mock.Call +} + +// ChipIngressMaxGRPCRequestSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxGRPCRequestSize() *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + return &TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call{Call: _e.mock.On("ChipIngressMaxGRPCRequestSize")} +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) Return(_a0 int) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) RunAndReturn(run func() int) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressSendInterval provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressSendInterval() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressSendInterval") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressSendInterval_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressSendInterval' +type TelemetryIngress_ChipIngressSendInterval_Call struct { + *mock.Call +} + +// ChipIngressSendInterval is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressSendInterval() *TelemetryIngress_ChipIngressSendInterval_Call { + return &TelemetryIngress_ChipIngressSendInterval_Call{Call: _e.mock.On("ChipIngressSendInterval")} +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) Run(run func()) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressSendTimeout provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressSendTimeout() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressSendTimeout") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressSendTimeout_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressSendTimeout' +type TelemetryIngress_ChipIngressSendTimeout_Call struct { + *mock.Call +} + +// ChipIngressSendTimeout is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressSendTimeout() *TelemetryIngress_ChipIngressSendTimeout_Call { + return &TelemetryIngress_ChipIngressSendTimeout_Call{Call: _e.mock.On("ChipIngressSendTimeout")} +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) Run(run func()) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Return(run) + return _c +} + // Endpoints provides a mock function with no fields func (_m *TelemetryIngress) Endpoints() []config.TelemetryIngressEndpoint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index 46d121585c2..199e0aae760 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -21,6 +21,9 @@ type TelemetryIngress interface { ChipIngressMaxBatchSize() uint ChipIngressMaxConcurrentSends() int ChipIngressDrainTimeout() time.Duration + ChipIngressSendInterval() time.Duration + ChipIngressSendTimeout() time.Duration + ChipIngressMaxGRPCRequestSize() int } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 1458e0ed73e..7c66ba8f491 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1,6 +1,7 @@ package toml import ( + "cmp" "context" "errors" "fmt" @@ -9,6 +10,7 @@ import ( "net/url" "reflect" "regexp" + "strconv" "strings" "time" @@ -141,6 +143,16 @@ func (c *Core) ValidateConfig() (err error) { } } + // The chip-ingress drain runs inside the shutdown grace period. If it can use most of + // it, a hung telemetry server makes the node force-exit before other services close. + // Only checked with the flag on, so configs with a short grace period keep working. + if ti := c.TelemetryIngress; ti.ChipIngressEnabled != nil && *ti.ChipIngressEnabled && + ti.ChipIngressDrainTimeout != nil && c.ShutdownGracePeriod != nil && + ti.ChipIngressDrainTimeout.Duration() >= c.ShutdownGracePeriod.Duration()/2 { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.ChipIngressDrainTimeout", Value: ti.ChipIngressDrainTimeout.Duration(), + Msg: fmt.Sprintf("must be less than half of ShutdownGracePeriod (%s)", c.ShutdownGracePeriod.Duration())}) + } + return err } @@ -987,6 +999,9 @@ type TelemetryIngress struct { ChipIngressBufferSize *uint ChipIngressMaxBatchSize *uint ChipIngressMaxConcurrentSends *int + ChipIngressSendInterval *commonconfig.Duration + ChipIngressSendTimeout *commonconfig.Duration + ChipIngressMaxGRPCRequestSize *int ChipIngressDrainTimeout *commonconfig.Duration } @@ -1043,30 +1058,89 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressDrainTimeout; v != nil { t.ChipIngressDrainTimeout = v } + if v := f.ChipIngressSendInterval; v != nil { + t.ChipIngressSendInterval = v + } + if v := f.ChipIngressSendTimeout; v != nil { + t.ChipIngressSendTimeout = v + } + if v := f.ChipIngressMaxGRPCRequestSize; v != nil { + t.ChipIngressMaxGRPCRequestSize = v + } } +// Bounds for the chip-ingress batching settings. They are checked even when +// ChipIngressEnabled is false, so a bad staged value can't stop the node from +// booting at the moment the flag is turned on. +const ( + chipIngressMaxBufferSize = 100_000 + chipIngressMaxBatchSizeLimit = 500 // the pinned library's batch split is O(n²) in batch size + chipIngressMaxConcurrentSendsCap = 32 + chipIngressMinSendInterval = 50 * time.Millisecond + chipIngressMaxSendInterval = 10 * time.Second + chipIngressMinSendTimeout = time.Second + chipIngressMaxSendTimeout = time.Minute + chipIngressMinGRPCRequestSize = 1 << 20 // the batch client clamps lower values up to this + chipIngressMaxGRPCRequestSize = 10 << 20 // chip-ingress's default receive limit +) + func (t *TelemetryIngress) ValidateConfig() (err error) { - if t.ChipIngressEnabled == nil || !*t.ChipIngressEnabled { - return nil - } - if t.ChipIngressEndpoint == nil || *t.ChipIngressEndpoint == "" { + enabled := t.ChipIngressEnabled != nil && *t.ChipIngressEnabled + insecure := t.ChipIngressInsecureConnection != nil && *t.ChipIngressInsecureConnection + switch { + case t.ChipIngressEndpoint != nil && *t.ChipIngressEndpoint != "": + err = errors.Join(err, validateChipIngressEndpoint(*t.ChipIngressEndpoint, insecure)) + case enabled: err = errors.Join(err, configutils.MissingError{Name: "ChipIngressEndpoint", Msg: "must be set when ChipIngressEnabled is true"}) } - if v := t.ChipIngressBufferSize; v != nil && *v == 0 { - err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressBufferSize", Value: *v, Msg: "must be greater than 0"}) - } - if v := t.ChipIngressMaxBatchSize; v != nil && *v == 0 { - err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxBatchSize", Value: *v, Msg: "must be greater than 0"}) - } - if v := t.ChipIngressMaxConcurrentSends; v != nil && *v <= 0 { - err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressMaxConcurrentSends", Value: *v, Msg: "must be greater than 0"}) - } + err = errors.Join(err, + inRange("ChipIngressBufferSize", t.ChipIngressBufferSize, 1, chipIngressMaxBufferSize), + inRange("ChipIngressMaxBatchSize", t.ChipIngressMaxBatchSize, 1, chipIngressMaxBatchSizeLimit), + inRange("ChipIngressMaxConcurrentSends", t.ChipIngressMaxConcurrentSends, 1, chipIngressMaxConcurrentSendsCap), + inRange("ChipIngressMaxGRPCRequestSize", t.ChipIngressMaxGRPCRequestSize, chipIngressMinGRPCRequestSize, chipIngressMaxGRPCRequestSize), + durationInRange("ChipIngressSendInterval", t.ChipIngressSendInterval, chipIngressMinSendInterval, chipIngressMaxSendInterval), + durationInRange("ChipIngressSendTimeout", t.ChipIngressSendTimeout, chipIngressMinSendTimeout, chipIngressMaxSendTimeout), + ) if v := t.ChipIngressDrainTimeout; v != nil && v.Duration() <= 0 { err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressDrainTimeout", Value: v.Duration(), Msg: "must be greater than 0"}) } return err } +// validateChipIngressEndpoint requires a bare host:port, which is what the gRPC client dials. +func validateChipIngressEndpoint(endpoint string, insecure bool) error { + invalid := configutils.InvalidError{Name: "ChipIngressEndpoint", Value: endpoint, Msg: "must be host:port, without a scheme or whitespace"} + if strings.TrimSpace(endpoint) != endpoint || strings.Contains(endpoint, "://") { + return invalid + } + host, port, err := net.SplitHostPort(endpoint) + if err != nil || host == "" { + return invalid + } + if p, err := strconv.Atoi(port); err != nil || p < 1 || p > 65535 { + return configutils.InvalidError{Name: "ChipIngressEndpoint", Value: endpoint, Msg: "port must be between 1 and 65535"} + } + if insecure && (host == "chain.link" || strings.HasSuffix(host, ".chain.link")) { + return configutils.InvalidError{Name: "ChipIngressInsecureConnection", Value: true, Msg: "must be false for a chain.link endpoint, or the CSA auth token is sent unencrypted"} + } + return nil +} + +func inRange[T cmp.Ordered](name string, v *T, lo, hi T) error { + if v == nil || (*v >= lo && *v <= hi) { + return nil + } + return configutils.InvalidError{Name: name, Value: *v, Msg: fmt.Sprintf("must be between %v and %v", lo, hi)} +} + +func durationInRange(name string, v *commonconfig.Duration, lo, hi time.Duration) error { + if v == nil { + return nil + } + d := v.Duration() + return inRange(name, &d, lo, hi) +} + type AuditLogger struct { Enabled *bool ForwardToUrl *commonconfig.URL //nolint:revive // TOML key compatibility diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index bab6284e5b5..81e0ae258b1 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -544,7 +544,9 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err } var telemChipService synchronization.ChipIngressService - if ti := cfg.TelemetryIngress(); ti.ChipIngressEnabled() { + if ti := cfg.TelemetryIngress(); ti.ChipIngressEnabled() && len(ti.Endpoints()) == 0 { + globalLogger.Warn("TelemetryIngress.ChipIngressEnabled is true but no TelemetryIngress.Endpoints are configured; no telemetry will be sent") + } else if ti.ChipIngressEnabled() { chipClient, chipErr := synchronization.NewChipIngressClient(synchronization.ChipIngressClientConfig{ Endpoint: ti.ChipIngressEndpoint(), InsecureConnection: ti.ChipIngressInsecureConnection(), @@ -560,8 +562,9 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err BufferSize: ti.ChipIngressBufferSize(), MaxBatchSize: ti.ChipIngressMaxBatchSize(), MaxConcurrentSends: ti.ChipIngressMaxConcurrentSends(), - SendInterval: ti.SendInterval(), - SendTimeout: ti.SendTimeout(), + SendInterval: ti.ChipIngressSendInterval(), + SendTimeout: ti.ChipIngressSendTimeout(), + MaxGRPCRequestSize: ti.ChipIngressMaxGRPCRequestSize(), DrainTimeout: ti.ChipIngressDrainTimeout(), Logging: ti.Logging(), }, globalLogger) diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 47707bf3b5b..16dd9ed1ec7 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -84,6 +84,18 @@ func (t *telemetryIngressConfig) ChipIngressDrainTimeout() time.Duration { return t.c.ChipIngressDrainTimeout.Duration() } +func (t *telemetryIngressConfig) ChipIngressSendInterval() time.Duration { + return t.c.ChipIngressSendInterval.Duration() +} + +func (t *telemetryIngressConfig) ChipIngressSendTimeout() time.Duration { + return t.c.ChipIngressSendTimeout.Duration() +} + +func (t *telemetryIngressConfig) ChipIngressMaxGRPCRequestSize() int { + return *t.c.ChipIngressMaxGRPCRequestSize +} + func (t *telemetryIngressEndpointConfig) Network() string { return *t.c.Network } @@ -92,13 +104,17 @@ func (t *telemetryIngressEndpointConfig) ChainID() string { return *t.c.ChainID } +// URL and ServerPubKey are optional in chip-ingress mode, where only Network and ChainID are used. func (t *telemetryIngressEndpointConfig) URL() *url.URL { - if t.c.URL.IsZero() { + if t.c.URL == nil || t.c.URL.IsZero() { return nil } return t.c.URL.URL() } func (t *telemetryIngressEndpointConfig) ServerPubKey() string { + if t.c.ServerPubKey == nil { + return "" + } return *t.c.ServerPubKey } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 9a03d538070..0b6275bd861 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -259,6 +259,9 @@ func TestConfig_Marshal(t *testing.T) { ChipIngressBufferSize: new(uint(2000)), ChipIngressMaxBatchSize: new(uint(200)), ChipIngressMaxConcurrentSends: new(4), + ChipIngressSendInterval: commonconfig.MustNewDuration(time.Second), + ChipIngressSendTimeout: commonconfig.MustNewDuration(5 * time.Second), + ChipIngressMaxGRPCRequestSize: new(4194304), ChipIngressDrainTimeout: commonconfig.MustNewDuration(3 * time.Second), Endpoints: []toml.TelemetryIngressEndpoint{ { @@ -922,6 +925,9 @@ ChipIngressInsecureConnection = true ChipIngressBufferSize = 2000 ChipIngressMaxBatchSize = 200 ChipIngressMaxConcurrentSends = 4 +ChipIngressSendInterval = '1s' +ChipIngressSendTimeout = '5s' +ChipIngressMaxGRPCRequestSize = 4194304 ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] @@ -1495,18 +1501,44 @@ func TestConfig_Validate(t *testing.T) { ChipIngressEnabled = true ChipIngressEndpoint = '' `, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: missing: must be set when ChipIngressEnabled is true`}, - {name: "TelemetryIngress chip batching must be positive", toml: ` + {name: "TelemetryIngress chip settings validated with the flag off", toml: ` [TelemetryIngress] -ChipIngressEnabled = true +ChipIngressEnabled = false ChipIngressBufferSize = 0 -ChipIngressMaxBatchSize = 0 +ChipIngressMaxBatchSize = 501 ChipIngressMaxConcurrentSends = 0 +ChipIngressSendInterval = '10ms' +ChipIngressSendTimeout = '2m' +ChipIngressMaxGRPCRequestSize = 1024 ChipIngressDrainTimeout = '0s' -`, exp: `invalid configuration: TelemetryIngress: 4 errors: - - ChipIngressBufferSize: invalid value (0): must be greater than 0 - - ChipIngressMaxBatchSize: invalid value (0): must be greater than 0 - - ChipIngressMaxConcurrentSends: invalid value (0): must be greater than 0 +`, exp: `invalid configuration: TelemetryIngress: 7 errors: + - ChipIngressBufferSize: invalid value (0): must be between 1 and 100000 + - ChipIngressMaxBatchSize: invalid value (501): must be between 1 and 500 + - ChipIngressMaxConcurrentSends: invalid value (0): must be between 1 and 32 + - ChipIngressMaxGRPCRequestSize: invalid value (1024): must be between 1048576 and 10485760 + - ChipIngressSendInterval: invalid value (10ms): must be between 50ms and 10s + - ChipIngressSendTimeout: invalid value (2m0s): must be between 1s and 1m0s - ChipIngressDrainTimeout: invalid value (0s): must be greater than 0`}, + {name: "TelemetryIngress chip endpoint must be host:port", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'https://chip.example:443' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: invalid value (https://chip.example:443): must be host:port, without a scheme or whitespace`}, + {name: "TelemetryIngress chip endpoint port in range", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'chip.example:0' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: invalid value (chip.example:0): port must be between 1 and 65535`}, + {name: "TelemetryIngress chip insecure not allowed for chain.link", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = true +`, exp: `invalid configuration: TelemetryIngress.ChipIngressInsecureConnection: invalid value (true): must be false for a chain.link endpoint, or the CSA auth token is sent unencrypted`}, + {name: "TelemetryIngress chip drain must fit in shutdown grace period", toml: ` +ShutdownGracePeriod = '4s' +[TelemetryIngress] +ChipIngressEnabled = true +ChipIngressEndpoint = 'chip.example:443' +ChipIngressDrainTimeout = '2s' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressDrainTimeout: invalid value (2s): must be less than half of ShutdownGracePeriod (4s)`}, } { t.Run(tt.name, func(t *testing.T) { var c Config diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index eb69155d33e..b5b062a54a8 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -44,12 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index 2c48796c741..abe761ba245 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -49,6 +49,9 @@ ChipIngressInsecureConnection = true ChipIngressBufferSize = 2000 ChipIngressMaxBatchSize = 200 ChipIngressMaxConcurrentSends = 4 +ChipIngressSendInterval = '1s' +ChipIngressSendTimeout = '5s' +ChipIngressMaxGRPCRequestSize = 4194304 ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index d360149eb41..5b7fe133320 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -44,12 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = true diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index e34fe67bbf4..1b80fb1181a 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -4,7 +4,7 @@ import ( "context" "fmt" "strconv" - "sync/atomic" + "sync" "time" chipingress "github.com/smartcontractkit/chainlink-common/pkg/chipingress" @@ -22,6 +22,13 @@ const ( healthPingInterval = 5 * time.Second healthPingTimeout = 2 * time.Second + // unhealthyAfterPingFailures marks the service unhealthy only after about a minute of + // failed pings, so a brief telemetry-server blip doesn't flip the node's /health. + unhealthyAfterPingFailures = 12 + chipIngressUnreachable = "chip-ingress unreachable" + + // logEvery limits each kind of repeated warning (per telemetry type) to one line per interval. + logEvery = 30 * time.Second ) // chipIngressBatchClient sends the node's legacy OCR telemetry to chip-ingress, @@ -36,18 +43,19 @@ type chipIngressBatchClient struct { batchClient *batch.Client csaPubKeyHex string logging bool - dropCount atomic.Uint32 - errorCount atomic.Uint32 + logs logLimiter + pingInterval time.Duration // healthPingInterval; shortened by tests } // ChipIngressBatchConfig sizes the shared chip-ingress batch client. Values come from -// [TelemetryIngress] (ChipIngress* fields plus SendInterval/SendTimeout). +// the [TelemetryIngress] ChipIngress* settings. type ChipIngressBatchConfig struct { BufferSize uint // messages buffered across all jobs before new ones are dropped MaxBatchSize uint // messages per PublishBatch request MaxConcurrentSends int // PublishBatch requests in flight SendInterval time.Duration // max wait before flushing an incomplete batch SendTimeout time.Duration // per-request PublishBatch timeout + MaxGRPCRequestSize int // max serialized PublishBatch request size; larger batches are split DrainTimeout time.Duration // max time Close waits to publish queued messages and run their callbacks Logging bool // debug-log every successful send } @@ -61,6 +69,7 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin batch.WithMaxConcurrentSends(cfg.MaxConcurrentSends), batch.WithBatchInterval(cfg.SendInterval), batch.WithMaxPublishTimeout(cfg.SendTimeout), + batch.WithMaxGRPCRequestSize(cfg.MaxGRPCRequestSize), batch.WithShutdownTimeout(cfg.DrainTimeout), batch.WithClientName(legacyTelemetryBatchClientName), // Send builds a fresh event per message and never touches it after queueing, @@ -76,6 +85,8 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin batchClient: batchClient, csaPubKeyHex: csaPubKeyHex, logging: cfg.Logging, + logs: logLimiter{every: logEvery}, + pingInterval: healthPingInterval, } c.Service, c.eng = services.Config{ Name: "ChipIngressBatchClient", @@ -86,19 +97,25 @@ func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex strin return c, nil } -// start starts the batch client and a periodic health ping that drives -// telemetry_client_connection_status{endpoint="chip-ingress"}. +// start starts the batch client and a periodic health ping. The ping drives +// telemetry_client_connection_status{endpoint="chip-ingress"} on every probe, and +// marks the service unhealthy only after a sustained outage. func (c *chipIngressBatchClient) start(ctx context.Context) error { c.batchClient.Start(ctx) - c.eng.GoTick(services.TickerConfig{}.NewTicker(healthPingInterval), func(ctx context.Context) { + failures := 0 // only touched by the ticker goroutine + c.eng.GoTick(services.TickerConfig{}.NewTicker(c.pingInterval), func(ctx context.Context) { pingCtx, cancel := context.WithTimeout(ctx, healthPingTimeout) defer cancel() - connected := float64(1) if _, err := c.chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}); err != nil { - connected = 0 - c.eng.EmitHealthErr(err) + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(0) + if failures++; failures >= unhealthyAfterPingFailures { + c.eng.SetHealthCond(chipIngressUnreachable, err) + } + return } - TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(connected) + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(1) + failures = 0 + c.eng.ClearHealthCond(chipIngressUnreachable) }) return nil } @@ -112,67 +129,59 @@ func (c *chipIngressBatchClient) close() error { } // Send builds the CloudEvent and queues it. It never blocks the caller (a libocr -// goroutine): if the event can't be built or the buffer is full, the message is -// dropped and counted. +// goroutine). Types chip-ingress has no mapping for are counted as unsupported; +// a full buffer or a shutting-down client drops the message. func (c *chipIngressBatchClient) Send(_ context.Context, payload TelemPayload) { - event, err := c.newEvent(payload, time.Now()) + telemType := string(payload.TelemType) + domain, entity, err := TelemetryTypeToDomainAndEntity(payload.TelemType) + if err != nil { + // Expected on LLO nodes (llo-report); OTI drops these types too. + TelemetryClientMessagesUnsupported.WithLabelValues(chipIngressEndpointLabel, telemType).Inc() + if ok, suppressed := c.logs.allow("unsupported:" + telemType); ok { + c.eng.Debugw("Telemetry type not supported by ChIP ingress; not sending", "telemType", telemType, "suppressed", suppressed) + } + return + } + event, err := c.newEvent(payload, domain, entity, time.Now()) if err == nil { - err = c.batchClient.QueueMessage(event, c.onPublished(payload)) + err = c.batchClient.QueueMessage(event, c.onPublished(payload.TelemType, payload.ContractID)) } if err != nil { - c.drop(payload, err) - return + TelemetryClientMessagesDropped.WithLabelValues(chipIngressEndpointLabel, telemType).Inc() + if ok, suppressed := c.logs.allow("drop:" + telemType); ok { + c.eng.Warnw("Dropping telemetry message for ChIP ingress", "error", err, "telemType", telemType, "suppressed", suppressed) + } } - c.dropCount.Store(0) } // onPublished returns the callback the batch client runs once the message's batch // has been published, counting each message as sent or failed. -func (c *chipIngressBatchClient) onPublished(payload TelemPayload) func(error) { +func (c *chipIngressBatchClient) onPublished(telemType TelemetryType, contractID string) func(error) { return func(err error) { if err != nil { - TelemetryClientMessagesSendErrors.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() - if count := c.errorCount.Add(1); shouldLogCount(count) { - c.eng.Warnw("Could not send telemetry via ChIP ingress", - "error", err, - "errorCode", batch.ErrorCodeFor(err), - "telemType", payload.TelemType, - "errorCount", count) + TelemetryClientMessagesSendErrors.WithLabelValues(chipIngressEndpointLabel, string(telemType)).Inc() + code := batch.ErrorCodeFor(err) + if ok, suppressed := c.logs.allow("error:" + string(telemType) + ":" + code); ok { + c.eng.Warnw("Could not send telemetry via ChIP ingress", "error", err, "errorCode", code, "telemType", telemType, "suppressed", suppressed) } return } - TelemetryClientMessagesSent.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() + TelemetryClientMessagesSent.WithLabelValues(chipIngressEndpointLabel, string(telemType)).Inc() if c.logging { - c.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", payload.ContractID, "telemType", payload.TelemType) + c.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", contractID, "telemType", telemType) } } } -// drop counts a message that never reached the queue: its telemetry type has no -// chip-ingress mapping, the buffer is full, or the client is shutting down. -func (c *chipIngressBatchClient) drop(payload TelemPayload, err error) { - TelemetryClientMessagesDropped.WithLabelValues(chipIngressEndpointLabel, string(payload.TelemType)).Inc() - if count := c.dropCount.Add(1); shouldLogCount(count) { - c.eng.Warnw("Dropping telemetry message for ChIP ingress", - "error", err, - "contractID", payload.ContractID, - "telemType", payload.TelemType, - "droppedCount", count) - } -} - // newEvent converts a payload into the CloudEvent chip-ingress expects for legacy -// telemetry. The event source/type select the Kafka topic. Each extension becomes a -// "ce_" Kafka header; the comments name the header OTI used for the same value. +// telemetry. Each extension becomes a "ce_" Kafka header; the comments name the +// header OTI used for the same value. A legacy-mode chip-ingress routes on telemetrytype +// and chainselector; source/type (domain/entity) only matter to a non-legacy deployment. // -// The node does not send nodeoperatorname/nodename: chip-ingress looks them up from -// the CSA key (WithNOPLookup), and sending them too would duplicate those headers. -// nodeoperatorkey is unknown to the node (spike INFOPLAT-19426). -func (c *chipIngressBatchClient) newEvent(payload TelemPayload, now time.Time) (*chipingress.CloudEventPb, error) { - domain, entity, err := TelemetryTypeToDomainAndEntity(payload.TelemType) - if err != nil { - return nil, err - } +// The node does not send nodeoperatorname/nodename or receivedat: chip-ingress derives +// them from the CSA key's NOP info (WithNOPLookup) and its own clock. nodeoperatorkey is +// unknown to the node (spike INFOPLAT-19426). +func (c *chipIngressBatchClient) newEvent(payload TelemPayload, domain, entity string, now time.Time) (*chipingress.CloudEventPb, error) { event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, map[string]any{"time": now.UTC()}) if err != nil { return nil, fmt.Errorf("failed creating CloudEvent: %w", err) @@ -186,7 +195,40 @@ func (c *chipIngressBatchClient) newEvent(payload TelemPayload, now time.Time) ( event.SetExtension("csapublickey", c.csaPubKeyHex) // "node-operator-csa-public-key" event.SetExtension("partitionkey", payload.ContractID+"-"+c.csaPubKeyHex) // Kafka record key event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns - event.SetExtension("receivedat", strconv.FormatInt(now.UnixMilli(), 10)) // "received-at", unix ms; chip-ingress may overwrite with its receive time return chipingress.EventToProto(event) } + +// logLimiter allows one log line per key per interval and counts what it suppressed, +// so repeated warnings stay readable however fast messages arrive. +type logLimiter struct { + every time.Duration + mu sync.Mutex + keys map[string]*logLimiterKey +} + +type logLimiterKey struct { + last time.Time + suppressed int +} + +func (l *logLimiter) allow(key string) (ok bool, suppressed int) { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.keys == nil { + l.keys = make(map[string]*logLimiterKey) + } + k, seen := l.keys[key] + if !seen { + l.keys[key] = &logLimiterKey{last: now} + return true, 0 + } + if now.Sub(k.last) < l.every { + k.suppressed++ + return false, 0 + } + suppressed = k.suppressed + k.last, k.suppressed = now, 0 + return true, suppressed +} diff --git a/core/services/synchronization/chip_ingress_batch_client_test.go b/core/services/synchronization/chip_ingress_batch_client_test.go index fe8bcb21ebb..3aec511f47e 100644 --- a/core/services/synchronization/chip_ingress_batch_client_test.go +++ b/core/services/synchronization/chip_ingress_batch_client_test.go @@ -6,6 +6,7 @@ import ( "slices" "strconv" "sync" + "sync/atomic" "testing" "time" @@ -105,18 +106,19 @@ func TestChipIngressBatchClient_Event(t *testing.T) { client.Send(t.Context(), newTestPayload("0xabc")) event := awaitFirstEvent(t, published) - // Source/type come from the telemetry type and pick the Kafka topic. + // Source/type are the telemetry type's domain/entity. domain, entity, err := TelemetryTypeToDomainAndEntity(OCR2Median) require.NoError(t, err) assert.Equal(t, domain, event.Source()) assert.Equal(t, entity, event.Type()) - // Exact extension set from the OTI-parity table plus the ones the - // library stamps (seqnum by the batch client, recordedtime by NewEvent). + // The exact extension set on the wire: the legacy-telemetry contract plus the two + // the library stamps (seqnum by the batch client, recordedtime by NewEvent). + // Anything else, e.g. receivedat, which chip-ingress owns, must not be sent. exts := event.Extensions() assert.ElementsMatch(t, []string{ "legacytelemetry", "telemetrytype", "chainselector", "networkname", - "contractid", "csapublickey", "partitionkey", "sentat", "receivedat", + "contractid", "csapublickey", "partitionkey", "sentat", "seqnum", "recordedtime", }, slices.Collect(maps.Keys(exts))) assert.Equal(t, "true", exts["legacytelemetry"]) @@ -127,26 +129,58 @@ func TestChipIngressBatchClient_Event(t *testing.T) { assert.Equal(t, "deadbeef", exts["csapublickey"]) assert.Equal(t, "0xabc-deadbeef", exts["partitionkey"]) - // sentat is unix nanos; receivedat is unix millis of the same instant. - sentAt, err := strconv.ParseInt(exts["sentat"].(string), 10, 64) + sentAt, err := strconv.ParseInt(exts["sentat"].(string), 10, 64) // unix nanos require.NoError(t, err) - receivedAt, err := strconv.ParseInt(exts["receivedat"].(string), 10, 64) - require.NoError(t, err) - assert.Equal(t, sentAt/1_000_000, receivedAt) assert.WithinDuration(t, time.Now(), time.Unix(0, sentAt), time.Minute) } -func TestChipIngressBatchClient_UnmappedTypeDropped(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics +func TestChipIngressBatchClient_UnsupportedTypesCountedNotDropped(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics client, published := startBatchClient(t, publishOK) - payload := newTestPayload("0xabc") - payload.TelemType = LLOReport // no chip domain/entity mapping - before := counterValue(TelemetryClientMessagesDropped, payload.TelemType) - // The event is rejected synchronously in Send, before it reaches the queue. - client.Send(t.Context(), payload) + // LLO nodes interleave llo-report (no chip mapping) with supported types. + const n = 50 + unsupportedBefore := counterValue(TelemetryClientMessagesUnsupported, LLOReport) + droppedBefore := counterValue(TelemetryClientMessagesDropped, LLOReport) + for range n { + unsupported := newTestPayload("0xabc") + unsupported.TelemType = LLOReport + client.Send(t.Context(), unsupported) + client.Send(t.Context(), newTestPayload("0xabc")) + } - assert.InDelta(t, 1, counterValue(TelemetryClientMessagesDropped, payload.TelemType)-before, 0.001) - assert.Empty(t, published.snapshot(), "PublishBatch must never be called for unmapped types") + assert.InDelta(t, n, counterValue(TelemetryClientMessagesUnsupported, LLOReport)-unsupportedBefore, 0.001) + assert.InDelta(t, 0, counterValue(TelemetryClientMessagesDropped, LLOReport)-droppedBefore, 0.001, + "unsupported-by-design must not count as dropped, so dropped stays alertable") + require.EventuallyWithT(t, func(c *assert.CollectT) { + total := 0 + for _, b := range published.snapshot() { + for _, e := range b { + assert.NotEqual(c, string(LLOReport), e.Attributes["telemetrytype"].GetCeString()) + total++ + } + } + assert.Equal(c, n, total, "only the supported messages are published") + }, testutils.WaitTimeout(t), 10*time.Millisecond) +} + +func TestLogLimiter(t *testing.T) { + t.Parallel() + l := logLimiter{every: 50 * time.Millisecond} + + ok, suppressed := l.allow("a") + assert.True(t, ok) + assert.Zero(t, suppressed) + for range 5 { + ok, _ = l.allow("a") + assert.False(t, ok, "repeats within the interval are suppressed") + } + ok, _ = l.allow("b") + assert.True(t, ok, "keys are limited independently") + + time.Sleep(60 * time.Millisecond) + ok, suppressed = l.allow("a") + assert.True(t, ok) + assert.Equal(t, 5, suppressed, "the next line reports how many were suppressed") } func TestChipIngressBatchClient_CountsPerMessage(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics @@ -171,12 +205,36 @@ func TestChipIngressBatchClient_CountsPerMessage(t *testing.T) { //nolint:parall } } -func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // resets a package-global prometheus gauge - // The status gauge is package-global; reset it so a value left by another test can't satisfy the check. - TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(0) - startBatchClient(t, publishOK) +func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // drives a package-global prometheus gauge + gauge := TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel) + var reachable atomic.Bool + var failedPings atomic.Int32 + chipClient := chipingressmocks.NewClient(t) + chipClient.On("Close").Return(nil).Maybe() + chipClient.EXPECT().Ping(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(context.Context, *chipingress.EmptyRequest, ...grpc.CallOption) (*chipingress.PingResponse, error) { + if reachable.Load() { + return &chipingress.PingResponse{}, nil + } + failedPings.Add(1) + return nil, assert.AnError + }).Maybe() + client := NewTestChipIngressBatchClient(t, chipClient, false, time.Second) + client.(*chipIngressBatchClient).pingInterval = time.Millisecond + gauge.Set(1) + servicetest.Run(t, client) + + // A failed ping only drops the gauge; the service turns unhealthy after a sustained outage. + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, 0, testutil.ToFloat64(gauge), 0.001) + assert.Error(c, client.HealthReport()[client.Name()]) + }, testutils.WaitTimeout(t), time.Millisecond) + assert.GreaterOrEqual(t, failedPings.Load(), int32(unhealthyAfterPingFailures)) + // One good ping restores both. + reachable.Store(true) require.EventuallyWithT(t, func(c *assert.CollectT) { - assert.InDelta(c, 1, testutil.ToFloat64(TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel)), 0.001) - }, testutils.WaitTimeout(t), 10*time.Millisecond) + assert.InDelta(c, 1, testutil.ToFloat64(gauge), 0.001) + assert.NoError(c, client.HealthReport()[client.Name()]) + }, testutils.WaitTimeout(t), time.Millisecond) } diff --git a/core/services/synchronization/common.go b/core/services/synchronization/common.go index ddd87a8900c..69afe0493b9 100644 --- a/core/services/synchronization/common.go +++ b/core/services/synchronization/common.go @@ -43,7 +43,6 @@ type TelemPayload struct { ContractID string ChainSelector uint64 Network string - ChainID string } // shouldLogCount reports whether the n-th occurrence of a repeated warning diff --git a/core/services/synchronization/metrics.go b/core/services/synchronization/metrics.go index d6bafb7b0d6..75f907fcbbe 100644 --- a/core/services/synchronization/metrics.go +++ b/core/services/synchronization/metrics.go @@ -26,6 +26,11 @@ var ( Help: "Number of telemetry messages dropped", }, []string{"endpoint", "telemetry_type"}) + TelemetryClientMessagesUnsupported = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "telemetry_client_messages_unsupported", + Help: "Number of telemetry messages not sent because the endpoint does not support their telemetry type", + }, []string{"endpoint", "telemetry_type"}) + TelemetryClientWorkers = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "telemetry_client_workers", Help: "Number of telemetry workers", diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index a5a9fbedcc3..23c47bcf573 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -158,15 +158,6 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log if e.ChainID() == "" { return nil, errors.New("cannot add telemetry endpoint, chainID cannot be empty") } - - if e.URL() == nil { - return nil, errors.New("cannot add telemetry endpoint, URL cannot be empty") - } - - if e.ServerPubKey() == "" { - return nil, errors.New("cannot add telemetry endpoint, ServerPubKey cannot be empty") - } - if _, found := m.getEndpoint(e.Network(), e.ChainID()); found { return nil, errors.Errorf("cannot add telemetry endpoint for network %q and chainID %q, endpoint already exists", e.Network(), e.ChainID()) } @@ -177,19 +168,28 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log URL: e.URL(), PubKey: e.ServerPubKey(), } - m.endpoints = append(m.endpoints, te) if m.chipService != nil { - // All chains share the injected chip service; the entry is only an allowlist. + // All chains share the injected chip service; the entry is only an allowlist, + // so the OTI-only URL and ServerPubKey are not required. + m.endpoints = append(m.endpoints, te) return nil, nil } + if te.URL == nil { + return nil, errors.New("cannot add telemetry endpoint, URL cannot be empty") + } + if te.PubKey == "" { + return nil, errors.New("cannot add telemetry endpoint, ServerPubKey cannot be empty") + } + lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) if m.useBatchSend { - te.client = synchronization.NewTelemetryIngressBatchClient(e.URL(), e.ServerPubKey(), m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) + te.client = synchronization.NewTelemetryIngressBatchClient(te.URL, te.PubKey, m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) } else { - te.client = synchronization.NewTelemetryIngressClient(e.URL(), e.ServerPubKey(), m.ks, lggr, cfg.BufferSize()) + te.client = synchronization.NewTelemetryIngressClient(te.URL, te.PubKey, m.ks, lggr, cfg.BufferSize()) } + m.endpoints = append(m.endpoints, te) return te.client, nil } diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index a292e24491d..2856b15d2f5 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -291,12 +291,12 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { tic := setupMockConfig(t, true) endpoints := make([]config.TelemetryIngressEndpoint, 0, 2) for _, chainID := range []string{"1", "137"} { + // Chip mode uses only Network and ChainID; the WSRPC URL and ServerPubKey may be unset. te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return(chainID) - te.On("ServerPubKey").Return("some-pubkey") - u, _ := url.Parse("http://chip-ingress.test") - te.On("URL").Return(u) + te.On("ServerPubKey").Return("") + te.On("URL").Return(nil) endpoints = append(endpoints, te) } tic.On("Endpoints").Return(endpoints) diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index eb69155d33e..b5b062a54a8 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -44,12 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index e27c721ea7b..8790085c495 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -44,12 +44,15 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index c1f91098e3d..6095ce35ddc 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -44,12 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = true diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 30d5ec1e6b3..f0a3f85d7a1 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -281,12 +281,15 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default UseBatchSend = true # Default ChipIngressEnabled = false # Default -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default +ChipIngressEndpoint = '' # Default ChipIngressInsecureConnection = false # Default ChipIngressBufferSize = 10000 # Default -ChipIngressMaxBatchSize = 1000 # Default +ChipIngressMaxBatchSize = 100 # Default ChipIngressMaxConcurrentSends = 10 # Default -ChipIngressDrainTimeout = '5s' # Default +ChipIngressSendInterval = '500ms' # Default +ChipIngressSendTimeout = '10s' # Default +ChipIngressMaxGRPCRequestSize = 10485760 # Default +ChipIngressDrainTimeout = '1s' # Default ``` @@ -318,13 +321,13 @@ MaxBatchSize is the maximum number of messages to batch into one telemetry reque ```toml SendInterval = '500ms' # Default ``` -SendInterval determines how often batched telemetry is sent to the ingress server. With ChipIngressEnabled, it is the max wait before flushing an incomplete batch. +SendInterval determines how often batched telemetry is sent to the ingress server. ### SendTimeout ```toml SendTimeout = '10s' # Default ``` -SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. With ChipIngressEnabled, it is the per-RPC timeout for PublishBatch. +SendTimeout is the max duration to wait for the request to complete when sending batch telemetry. ### UseBatchSend ```toml @@ -336,43 +339,61 @@ UseBatchSend toggles sending telemetry to the ingress server using the batch cli ```toml ChipIngressEnabled = false # Default ``` -ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry. +ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry; in this mode only their Network and ChainID are used. ### ChipIngressEndpoint ```toml -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' # Default +ChipIngressEndpoint = '' # Default ``` -ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint used when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. +ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint. Required when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. ### ChipIngressInsecureConnection ```toml ChipIngressInsecureConnection = false # Default ``` -ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups. +ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups; not allowed for chain.link hosts. ### ChipIngressBufferSize ```toml ChipIngressBufferSize = 10000 # Default ``` -ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. +ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. Between 1 and 100000. ### ChipIngressMaxBatchSize ```toml -ChipIngressMaxBatchSize = 1000 # Default +ChipIngressMaxBatchSize = 100 # Default ``` -ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. +ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. Between 1 and 500. ### ChipIngressMaxConcurrentSends ```toml ChipIngressMaxConcurrentSends = 10 # Default ``` -ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. +ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. Between 1 and 32. + +### ChipIngressSendInterval +```toml +ChipIngressSendInterval = '500ms' # Default +``` +ChipIngressSendInterval is the max wait before flushing an incomplete chip-ingress batch. Between 50ms and 10s. + +### ChipIngressSendTimeout +```toml +ChipIngressSendTimeout = '10s' # Default +``` +ChipIngressSendTimeout is the per-request timeout for chip-ingress PublishBatch. Between 1s and 1m. + +### ChipIngressMaxGRPCRequestSize +```toml +ChipIngressMaxGRPCRequestSize = 10485760 # Default +``` +ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes; larger batches are split. Between 1 MiB and 10 MiB, and must not exceed the server's receive limit. ### ChipIngressDrainTimeout ```toml -ChipIngressDrainTimeout = '5s' # Default +ChipIngressDrainTimeout = '1s' # Default ``` -ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. +ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. With ChipIngressEnabled, must be less than half of ShutdownGracePeriod. ## TelemetryIngress.Endpoints ```toml diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 4422a97176e..44a51dab6c9 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -191,12 +191,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index ea198e4d503..2c7328bdcc0 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -56,12 +56,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index dc590ce4100..1587cf083d0 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -117,12 +117,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 624d35d0b0f..a6541a4ea65 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -100,12 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index 06384b0d35d..0441125ac87 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -100,12 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index 05b34ee4e5c..866b1f0e316 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -100,12 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 616aac49c0e..7381291760a 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -199,12 +199,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index db8ad415986..3f0ff08261e 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -85,12 +85,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 636d3f72c93..6cd3b311abc 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -96,12 +96,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index 2bf46fd010d..6e887c3cb99 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -97,12 +97,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 8645004fd90..5b382405901 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -79,12 +79,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false -ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressEndpoint = '' ChipIngressInsecureConnection = false ChipIngressBufferSize = 10000 -ChipIngressMaxBatchSize = 1000 +ChipIngressMaxBatchSize = 100 ChipIngressMaxConcurrentSends = 10 -ChipIngressDrainTimeout = '5s' +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false