diff --git a/.mockery.yaml b/.mockery.yaml index 4ef1f9853e6..c07ecb79b1c 100644 --- a/.mockery.yaml +++ b/.mockery.yaml @@ -215,6 +215,7 @@ packages: github.com/smartcontractkit/chainlink/v2/core/services/synchronization: interfaces: TelemetryService: + ChipIngressService: github.com/smartcontractkit/chainlink/v2/core/services/synchronization/telem: config: dir: "{{ .InterfaceDir }}/../mocks" diff --git a/core/config/docs/core.toml b/core/config/docs/core.toml index 2c8d936d08a..29442e333f3 100644 --- a/core/config/docs/core.toml +++ b/core/config/docs/core.toml @@ -104,9 +104,9 @@ LeaseRefreshInterval = '1s' # Default UniConn = false # Default # Logging toggles verbose logging of the raw telemetry messages being sent. Logging = false # Default -# BufferSize is the number of telemetry messages to buffer before dropping new ones. +# BufferSize is the number of telemetry messages to buffer before dropping new ones. Ignored when ChipIngressEnabled is true. BufferSize = 100 # Default -# MaxBatchSize is the maximum number of messages to batch into one telemetry request. +# MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. MaxBatchSize = 50 # Default # SendInterval determines how often batched telemetry is sent to the ingress server. SendInterval = '500ms' # Default @@ -114,8 +114,26 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default # UseBatchSend toggles sending telemetry to the ingress server using the batch client. UseBatchSend = true # Default -# ChipIngressEnabled enables sending telemetry to CHIP Ingress. +# ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry; in this mode only their Network and ChainID are used. ChipIngressEnabled = false # Default +# ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint. Required when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. +ChipIngressEndpoint = '' # Default +# ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups; not allowed for chain.link hosts. +ChipIngressInsecureConnection = false # Default +# ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. Between 1 and 100000. +ChipIngressBufferSize = 10000 # Default +# ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. Between 1 and 500. +ChipIngressMaxBatchSize = 100 # Default +# ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. Between 1 and 32. +ChipIngressMaxConcurrentSends = 10 # Default +# ChipIngressSendInterval is the max wait before flushing an incomplete chip-ingress batch. Between 50ms and 10s. +ChipIngressSendInterval = '500ms' # Default +# ChipIngressSendTimeout is the per-request timeout for chip-ingress PublishBatch. Between 1s and 1m. +ChipIngressSendTimeout = '10s' # Default +# ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes; larger batches are split. Between 1 MiB and 10 MiB, and must not exceed the server's receive limit. +ChipIngressMaxGRPCRequestSize = 10485760 # Default +# ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. With ChipIngressEnabled, must be less than half of ShutdownGracePeriod. +ChipIngressDrainTimeout = '1s' # Default [[TelemetryIngress.Endpoints]] # Example # Network aka EVM, Solana, Starknet diff --git a/core/config/mocks/telemetry_ingress.go b/core/config/mocks/telemetry_ingress.go index 09e13a510cc..fdd4c2e5d8d 100644 --- a/core/config/mocks/telemetry_ingress.go +++ b/core/config/mocks/telemetry_ingress.go @@ -67,6 +67,96 @@ func (_c *TelemetryIngress_BufferSize_Call) RunAndReturn(run func() uint) *Telem return _c } +// ChipIngressBufferSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressBufferSize() uint { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressBufferSize") + } + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +// TelemetryIngress_ChipIngressBufferSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressBufferSize' +type TelemetryIngress_ChipIngressBufferSize_Call struct { + *mock.Call +} + +// ChipIngressBufferSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressBufferSize() *TelemetryIngress_ChipIngressBufferSize_Call { + return &TelemetryIngress_ChipIngressBufferSize_Call{Call: _e.mock.On("ChipIngressBufferSize")} +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) Run(run func()) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) Return(_a0 uint) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressBufferSize_Call) RunAndReturn(run func() uint) *TelemetryIngress_ChipIngressBufferSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressDrainTimeout provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressDrainTimeout() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressDrainTimeout") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressDrainTimeout_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressDrainTimeout' +type TelemetryIngress_ChipIngressDrainTimeout_Call struct { + *mock.Call +} + +// ChipIngressDrainTimeout is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressDrainTimeout() *TelemetryIngress_ChipIngressDrainTimeout_Call { + return &TelemetryIngress_ChipIngressDrainTimeout_Call{Call: _e.mock.On("ChipIngressDrainTimeout")} +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) Run(run func()) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressDrainTimeout_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressDrainTimeout_Call { + _c.Call.Return(run) + return _c +} + // ChipIngressEnabled provides a mock function with no fields func (_m *TelemetryIngress) ChipIngressEnabled() bool { ret := _m.Called() @@ -112,6 +202,321 @@ func (_c *TelemetryIngress_ChipIngressEnabled_Call) RunAndReturn(run func() bool return _c } +// ChipIngressEndpoint provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressEndpoint() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressEndpoint") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// TelemetryIngress_ChipIngressEndpoint_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressEndpoint' +type TelemetryIngress_ChipIngressEndpoint_Call struct { + *mock.Call +} + +// ChipIngressEndpoint is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressEndpoint() *TelemetryIngress_ChipIngressEndpoint_Call { + return &TelemetryIngress_ChipIngressEndpoint_Call{Call: _e.mock.On("ChipIngressEndpoint")} +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) Run(run func()) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) Return(_a0 string) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressEndpoint_Call) RunAndReturn(run func() string) *TelemetryIngress_ChipIngressEndpoint_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressInsecureConnection provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressInsecureConnection() bool { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressInsecureConnection") + } + + var r0 bool + if rf, ok := ret.Get(0).(func() bool); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(bool) + } + + return r0 +} + +// TelemetryIngress_ChipIngressInsecureConnection_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressInsecureConnection' +type TelemetryIngress_ChipIngressInsecureConnection_Call struct { + *mock.Call +} + +// ChipIngressInsecureConnection is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressInsecureConnection() *TelemetryIngress_ChipIngressInsecureConnection_Call { + return &TelemetryIngress_ChipIngressInsecureConnection_Call{Call: _e.mock.On("ChipIngressInsecureConnection")} +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) Run(run func()) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) Return(_a0 bool) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressInsecureConnection_Call) RunAndReturn(run func() bool) *TelemetryIngress_ChipIngressInsecureConnection_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressMaxBatchSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxBatchSize() uint { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxBatchSize") + } + + var r0 uint + if rf, ok := ret.Get(0).(func() uint); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(uint) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxBatchSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxBatchSize' +type TelemetryIngress_ChipIngressMaxBatchSize_Call struct { + *mock.Call +} + +// ChipIngressMaxBatchSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxBatchSize() *TelemetryIngress_ChipIngressMaxBatchSize_Call { + return &TelemetryIngress_ChipIngressMaxBatchSize_Call{Call: _e.mock.On("ChipIngressMaxBatchSize")} +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) Return(_a0 uint) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxBatchSize_Call) RunAndReturn(run func() uint) *TelemetryIngress_ChipIngressMaxBatchSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressMaxConcurrentSends provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxConcurrentSends() int { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxConcurrentSends") + } + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxConcurrentSends_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxConcurrentSends' +type TelemetryIngress_ChipIngressMaxConcurrentSends_Call struct { + *mock.Call +} + +// ChipIngressMaxConcurrentSends is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxConcurrentSends() *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + return &TelemetryIngress_ChipIngressMaxConcurrentSends_Call{Call: _e.mock.On("ChipIngressMaxConcurrentSends")} +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) Return(_a0 int) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxConcurrentSends_Call) RunAndReturn(run func() int) *TelemetryIngress_ChipIngressMaxConcurrentSends_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressMaxGRPCRequestSize provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressMaxGRPCRequestSize() int { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressMaxGRPCRequestSize") + } + + var r0 int + if rf, ok := ret.Get(0).(func() int); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(int) + } + + return r0 +} + +// TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressMaxGRPCRequestSize' +type TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call struct { + *mock.Call +} + +// ChipIngressMaxGRPCRequestSize is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressMaxGRPCRequestSize() *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + return &TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call{Call: _e.mock.On("ChipIngressMaxGRPCRequestSize")} +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) Run(run func()) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) Return(_a0 int) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call) RunAndReturn(run func() int) *TelemetryIngress_ChipIngressMaxGRPCRequestSize_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressSendInterval provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressSendInterval() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressSendInterval") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressSendInterval_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressSendInterval' +type TelemetryIngress_ChipIngressSendInterval_Call struct { + *mock.Call +} + +// ChipIngressSendInterval is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressSendInterval() *TelemetryIngress_ChipIngressSendInterval_Call { + return &TelemetryIngress_ChipIngressSendInterval_Call{Call: _e.mock.On("ChipIngressSendInterval")} +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) Run(run func()) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendInterval_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressSendInterval_Call { + _c.Call.Return(run) + return _c +} + +// ChipIngressSendTimeout provides a mock function with no fields +func (_m *TelemetryIngress) ChipIngressSendTimeout() time.Duration { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for ChipIngressSendTimeout") + } + + var r0 time.Duration + if rf, ok := ret.Get(0).(func() time.Duration); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(time.Duration) + } + + return r0 +} + +// TelemetryIngress_ChipIngressSendTimeout_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'ChipIngressSendTimeout' +type TelemetryIngress_ChipIngressSendTimeout_Call struct { + *mock.Call +} + +// ChipIngressSendTimeout is a helper method to define mock.On call +func (_e *TelemetryIngress_Expecter) ChipIngressSendTimeout() *TelemetryIngress_ChipIngressSendTimeout_Call { + return &TelemetryIngress_ChipIngressSendTimeout_Call{Call: _e.mock.On("ChipIngressSendTimeout")} +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) Run(run func()) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) Return(_a0 time.Duration) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *TelemetryIngress_ChipIngressSendTimeout_Call) RunAndReturn(run func() time.Duration) *TelemetryIngress_ChipIngressSendTimeout_Call { + _c.Call.Return(run) + return _c +} + // Endpoints provides a mock function with no fields func (_m *TelemetryIngress) Endpoints() []config.TelemetryIngressEndpoint { ret := _m.Called() diff --git a/core/config/telemetry_ingress_config.go b/core/config/telemetry_ingress_config.go index 918af269601..199e0aae760 100644 --- a/core/config/telemetry_ingress_config.go +++ b/core/config/telemetry_ingress_config.go @@ -15,6 +15,15 @@ type TelemetryIngress interface { UseBatchSend() bool Endpoints() []TelemetryIngressEndpoint ChipIngressEnabled() bool + ChipIngressEndpoint() string + ChipIngressInsecureConnection() bool + ChipIngressBufferSize() uint + ChipIngressMaxBatchSize() uint + ChipIngressMaxConcurrentSends() int + ChipIngressDrainTimeout() time.Duration + ChipIngressSendInterval() time.Duration + ChipIngressSendTimeout() time.Duration + ChipIngressMaxGRPCRequestSize() int } type TelemetryIngressEndpoint interface { diff --git a/core/config/toml/types.go b/core/config/toml/types.go index 89d544d6d97..7c66ba8f491 100644 --- a/core/config/toml/types.go +++ b/core/config/toml/types.go @@ -1,6 +1,7 @@ package toml import ( + "cmp" "context" "errors" "fmt" @@ -9,6 +10,7 @@ import ( "net/url" "reflect" "regexp" + "strconv" "strings" "time" @@ -141,6 +143,16 @@ func (c *Core) ValidateConfig() (err error) { } } + // The chip-ingress drain runs inside the shutdown grace period. If it can use most of + // it, a hung telemetry server makes the node force-exit before other services close. + // Only checked with the flag on, so configs with a short grace period keep working. + if ti := c.TelemetryIngress; ti.ChipIngressEnabled != nil && *ti.ChipIngressEnabled && + ti.ChipIngressDrainTimeout != nil && c.ShutdownGracePeriod != nil && + ti.ChipIngressDrainTimeout.Duration() >= c.ShutdownGracePeriod.Duration()/2 { + err = errors.Join(err, configutils.InvalidError{Name: "TelemetryIngress.ChipIngressDrainTimeout", Value: ti.ChipIngressDrainTimeout.Duration(), + Msg: fmt.Sprintf("must be less than half of ShutdownGracePeriod (%s)", c.ShutdownGracePeriod.Duration())}) + } + return err } @@ -973,15 +985,24 @@ func (d *DatabaseBackup) setFrom(f *DatabaseBackup) { } type TelemetryIngress struct { - UniConn *bool - Logging *bool - BufferSize *uint16 - MaxBatchSize *uint16 - SendInterval *commonconfig.Duration - SendTimeout *commonconfig.Duration - UseBatchSend *bool - Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` - ChipIngressEnabled *bool + UniConn *bool + Logging *bool + BufferSize *uint16 + MaxBatchSize *uint16 + SendInterval *commonconfig.Duration + SendTimeout *commonconfig.Duration + UseBatchSend *bool + Endpoints []TelemetryIngressEndpoint `toml:",omitempty"` + ChipIngressEnabled *bool + ChipIngressEndpoint *string + ChipIngressInsecureConnection *bool + ChipIngressBufferSize *uint + ChipIngressMaxBatchSize *uint + ChipIngressMaxConcurrentSends *int + ChipIngressSendInterval *commonconfig.Duration + ChipIngressSendTimeout *commonconfig.Duration + ChipIngressMaxGRPCRequestSize *int + ChipIngressDrainTimeout *commonconfig.Duration } type TelemetryIngressEndpoint struct { @@ -1019,6 +1040,105 @@ func (t *TelemetryIngress) setFrom(f *TelemetryIngress) { if v := f.ChipIngressEnabled; v != nil { t.ChipIngressEnabled = v } + if v := f.ChipIngressEndpoint; v != nil { + t.ChipIngressEndpoint = v + } + if v := f.ChipIngressInsecureConnection; v != nil { + t.ChipIngressInsecureConnection = v + } + if v := f.ChipIngressBufferSize; v != nil { + t.ChipIngressBufferSize = v + } + if v := f.ChipIngressMaxBatchSize; v != nil { + t.ChipIngressMaxBatchSize = v + } + if v := f.ChipIngressMaxConcurrentSends; v != nil { + t.ChipIngressMaxConcurrentSends = v + } + if v := f.ChipIngressDrainTimeout; v != nil { + t.ChipIngressDrainTimeout = v + } + if v := f.ChipIngressSendInterval; v != nil { + t.ChipIngressSendInterval = v + } + if v := f.ChipIngressSendTimeout; v != nil { + t.ChipIngressSendTimeout = v + } + if v := f.ChipIngressMaxGRPCRequestSize; v != nil { + t.ChipIngressMaxGRPCRequestSize = v + } +} + +// Bounds for the chip-ingress batching settings. They are checked even when +// ChipIngressEnabled is false, so a bad staged value can't stop the node from +// booting at the moment the flag is turned on. +const ( + chipIngressMaxBufferSize = 100_000 + chipIngressMaxBatchSizeLimit = 500 // the pinned library's batch split is O(n²) in batch size + chipIngressMaxConcurrentSendsCap = 32 + chipIngressMinSendInterval = 50 * time.Millisecond + chipIngressMaxSendInterval = 10 * time.Second + chipIngressMinSendTimeout = time.Second + chipIngressMaxSendTimeout = time.Minute + chipIngressMinGRPCRequestSize = 1 << 20 // the batch client clamps lower values up to this + chipIngressMaxGRPCRequestSize = 10 << 20 // chip-ingress's default receive limit +) + +func (t *TelemetryIngress) ValidateConfig() (err error) { + enabled := t.ChipIngressEnabled != nil && *t.ChipIngressEnabled + insecure := t.ChipIngressInsecureConnection != nil && *t.ChipIngressInsecureConnection + switch { + case t.ChipIngressEndpoint != nil && *t.ChipIngressEndpoint != "": + err = errors.Join(err, validateChipIngressEndpoint(*t.ChipIngressEndpoint, insecure)) + case enabled: + err = errors.Join(err, configutils.MissingError{Name: "ChipIngressEndpoint", Msg: "must be set when ChipIngressEnabled is true"}) + } + err = errors.Join(err, + inRange("ChipIngressBufferSize", t.ChipIngressBufferSize, 1, chipIngressMaxBufferSize), + inRange("ChipIngressMaxBatchSize", t.ChipIngressMaxBatchSize, 1, chipIngressMaxBatchSizeLimit), + inRange("ChipIngressMaxConcurrentSends", t.ChipIngressMaxConcurrentSends, 1, chipIngressMaxConcurrentSendsCap), + inRange("ChipIngressMaxGRPCRequestSize", t.ChipIngressMaxGRPCRequestSize, chipIngressMinGRPCRequestSize, chipIngressMaxGRPCRequestSize), + durationInRange("ChipIngressSendInterval", t.ChipIngressSendInterval, chipIngressMinSendInterval, chipIngressMaxSendInterval), + durationInRange("ChipIngressSendTimeout", t.ChipIngressSendTimeout, chipIngressMinSendTimeout, chipIngressMaxSendTimeout), + ) + if v := t.ChipIngressDrainTimeout; v != nil && v.Duration() <= 0 { + err = errors.Join(err, configutils.InvalidError{Name: "ChipIngressDrainTimeout", Value: v.Duration(), Msg: "must be greater than 0"}) + } + return err +} + +// validateChipIngressEndpoint requires a bare host:port, which is what the gRPC client dials. +func validateChipIngressEndpoint(endpoint string, insecure bool) error { + invalid := configutils.InvalidError{Name: "ChipIngressEndpoint", Value: endpoint, Msg: "must be host:port, without a scheme or whitespace"} + if strings.TrimSpace(endpoint) != endpoint || strings.Contains(endpoint, "://") { + return invalid + } + host, port, err := net.SplitHostPort(endpoint) + if err != nil || host == "" { + return invalid + } + if p, err := strconv.Atoi(port); err != nil || p < 1 || p > 65535 { + return configutils.InvalidError{Name: "ChipIngressEndpoint", Value: endpoint, Msg: "port must be between 1 and 65535"} + } + if insecure && (host == "chain.link" || strings.HasSuffix(host, ".chain.link")) { + return configutils.InvalidError{Name: "ChipIngressInsecureConnection", Value: true, Msg: "must be false for a chain.link endpoint, or the CSA auth token is sent unencrypted"} + } + return nil +} + +func inRange[T cmp.Ordered](name string, v *T, lo, hi T) error { + if v == nil || (*v >= lo && *v <= hi) { + return nil + } + return configutils.InvalidError{Name: name, Value: *v, Msg: fmt.Sprintf("must be between %v and %v", lo, hi)} +} + +func durationInRange(name string, v *commonconfig.Duration, lo, hi time.Duration) error { + if v == nil { + return nil + } + d := v.Duration() + return inRange(name, &d, lo, hi) } type AuditLogger struct { diff --git a/core/scripts/go.mod b/core/scripts/go.mod index 983ea3a7210..7137bc6f9d8 100644 --- a/core/scripts/go.mod +++ b/core/scripts/go.mod @@ -502,7 +502,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.14.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect diff --git a/core/scripts/go.sum b/core/scripts/go.sum index ae73a29cb9c..f566bfaf1eb 100644 --- a/core/scripts/go.sum +++ b/core/scripts/go.sum @@ -1692,8 +1692,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/core/services/chainlink/application.go b/core/services/chainlink/application.go index 933b91bd6e7..81e0ae258b1 100644 --- a/core/services/chainlink/application.go +++ b/core/services/chainlink/application.go @@ -86,6 +86,7 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/services/shardorchestrator" "github.com/smartcontractkit/chainlink/v2/core/services/standardcapabilities" "github.com/smartcontractkit/chainlink/v2/core/services/streams" + "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" "github.com/smartcontractkit/chainlink/v2/core/services/telemetry" "github.com/smartcontractkit/chainlink/v2/core/services/vrf" workflowstore "github.com/smartcontractkit/chainlink/v2/core/services/workflows/store" @@ -542,7 +543,38 @@ func NewApplication(ctx context.Context, opts ApplicationOpts) (Application, err globalLogger.Info("Nurse service (automatic pprof profiling) is disabled") } - telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, globalLogger) + var telemChipService synchronization.ChipIngressService + if ti := cfg.TelemetryIngress(); ti.ChipIngressEnabled() && len(ti.Endpoints()) == 0 { + globalLogger.Warn("TelemetryIngress.ChipIngressEnabled is true but no TelemetryIngress.Endpoints are configured; no telemetry will be sent") + } else if ti.ChipIngressEnabled() { + chipClient, chipErr := synchronization.NewChipIngressClient(synchronization.ChipIngressClientConfig{ + Endpoint: ti.ChipIngressEndpoint(), + InsecureConnection: ti.ChipIngressInsecureConnection(), + AuthHeaders: beholderAuthHeaders, + AuthHeadersTTL: cfg.Telemetry().AuthHeadersTTL(), + AuthPublicKeyHex: csaPubKeyHex, + AuthKeySigner: csaKeystore, + }) + if chipErr != nil { + return nil, fmt.Errorf("failed to create telemetry chip-ingress client: %w", chipErr) + } + telemChipService, chipErr = synchronization.NewChipIngressBatchClient(chipClient, csaPubKeyHex, synchronization.ChipIngressBatchConfig{ + BufferSize: ti.ChipIngressBufferSize(), + MaxBatchSize: ti.ChipIngressMaxBatchSize(), + MaxConcurrentSends: ti.ChipIngressMaxConcurrentSends(), + SendInterval: ti.ChipIngressSendInterval(), + SendTimeout: ti.ChipIngressSendTimeout(), + MaxGRPCRequestSize: ti.ChipIngressMaxGRPCRequestSize(), + DrainTimeout: ti.ChipIngressDrainTimeout(), + Logging: ti.Logging(), + }, globalLogger) + if chipErr != nil { + _ = chipClient.Close() // the batch client never took ownership of chipClient + return nil, fmt.Errorf("failed to create telemetry chip-ingress batch client: %w", chipErr) + } + globalLogger.Infow("Telemetry will be sent via ChIP ingress", "endpoint", ti.ChipIngressEndpoint()) + } + telemetryManager := telemetry.NewManager(cfg.TelemetryIngress(), csaKeystore, telemChipService, globalLogger) srvcs = append(srvcs, telemetryManager) backupCfg := cfg.Database().Backup() diff --git a/core/services/chainlink/config_telemetry_ingress.go b/core/services/chainlink/config_telemetry_ingress.go index 21c50d967a0..16dd9ed1ec7 100644 --- a/core/services/chainlink/config_telemetry_ingress.go +++ b/core/services/chainlink/config_telemetry_ingress.go @@ -60,6 +60,42 @@ func (t *telemetryIngressConfig) ChipIngressEnabled() bool { return *t.c.ChipIngressEnabled } +func (t *telemetryIngressConfig) ChipIngressEndpoint() string { + return *t.c.ChipIngressEndpoint +} + +func (t *telemetryIngressConfig) ChipIngressInsecureConnection() bool { + return *t.c.ChipIngressInsecureConnection +} + +func (t *telemetryIngressConfig) ChipIngressBufferSize() uint { + return *t.c.ChipIngressBufferSize +} + +func (t *telemetryIngressConfig) ChipIngressMaxBatchSize() uint { + return *t.c.ChipIngressMaxBatchSize +} + +func (t *telemetryIngressConfig) ChipIngressMaxConcurrentSends() int { + return *t.c.ChipIngressMaxConcurrentSends +} + +func (t *telemetryIngressConfig) ChipIngressDrainTimeout() time.Duration { + return t.c.ChipIngressDrainTimeout.Duration() +} + +func (t *telemetryIngressConfig) ChipIngressSendInterval() time.Duration { + return t.c.ChipIngressSendInterval.Duration() +} + +func (t *telemetryIngressConfig) ChipIngressSendTimeout() time.Duration { + return t.c.ChipIngressSendTimeout.Duration() +} + +func (t *telemetryIngressConfig) ChipIngressMaxGRPCRequestSize() int { + return *t.c.ChipIngressMaxGRPCRequestSize +} + func (t *telemetryIngressEndpointConfig) Network() string { return *t.c.Network } @@ -68,13 +104,17 @@ func (t *telemetryIngressEndpointConfig) ChainID() string { return *t.c.ChainID } +// URL and ServerPubKey are optional in chip-ingress mode, where only Network and ChainID are used. func (t *telemetryIngressEndpointConfig) URL() *url.URL { - if t.c.URL.IsZero() { + if t.c.URL == nil || t.c.URL.IsZero() { return nil } return t.c.URL.URL() } func (t *telemetryIngressEndpointConfig) ServerPubKey() string { + if t.c.ServerPubKey == nil { + return "" + } return *t.c.ServerPubKey } diff --git a/core/services/chainlink/config_test.go b/core/services/chainlink/config_test.go index 3ae19c039e0..0b6275bd861 100644 --- a/core/services/chainlink/config_test.go +++ b/core/services/chainlink/config_test.go @@ -246,14 +246,23 @@ func TestConfig_Marshal(t *testing.T) { }, } full.TelemetryIngress = toml.TelemetryIngress{ - UniConn: new(false), - Logging: new(true), - BufferSize: new(uint16(1234)), - MaxBatchSize: new(uint16(4321)), - SendInterval: commonconfig.MustNewDuration(time.Minute), - SendTimeout: commonconfig.MustNewDuration(5 * time.Second), - UseBatchSend: new(true), - ChipIngressEnabled: new(false), + UniConn: new(false), + Logging: new(true), + BufferSize: new(uint16(1234)), + MaxBatchSize: new(uint16(4321)), + SendInterval: commonconfig.MustNewDuration(time.Minute), + SendTimeout: commonconfig.MustNewDuration(5 * time.Second), + UseBatchSend: new(true), + ChipIngressEnabled: new(false), + ChipIngressEndpoint: new("legacy-telemetry.example:443"), + ChipIngressInsecureConnection: new(true), + ChipIngressBufferSize: new(uint(2000)), + ChipIngressMaxBatchSize: new(uint(200)), + ChipIngressMaxConcurrentSends: new(4), + ChipIngressSendInterval: commonconfig.MustNewDuration(time.Second), + ChipIngressSendTimeout: commonconfig.MustNewDuration(5 * time.Second), + ChipIngressMaxGRPCRequestSize: new(4194304), + ChipIngressDrainTimeout: commonconfig.MustNewDuration(3 * time.Second), Endpoints: []toml.TelemetryIngressEndpoint{ { Network: new("EVM"), @@ -911,6 +920,15 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.example:443' +ChipIngressInsecureConnection = true +ChipIngressBufferSize = 2000 +ChipIngressMaxBatchSize = 200 +ChipIngressMaxConcurrentSends = 4 +ChipIngressSendInterval = '1s' +ChipIngressSendTimeout = '5s' +ChipIngressMaxGRPCRequestSize = 4194304 +ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] Network = 'EVM' @@ -1476,7 +1494,51 @@ func TestConfig_Validate(t *testing.T) { - 0.Nodes.1.Name: invalid value (ton-test): duplicate - must be unique - 0: 2 errors: - Enabled: invalid value (1): expected bool - - ChainID: missing: required for all chains`}, + - ChainID: missing: required for all chains`, + }, + {name: "TelemetryIngress chip enabled without endpoint", toml: ` +[TelemetryIngress] +ChipIngressEnabled = true +ChipIngressEndpoint = '' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: missing: must be set when ChipIngressEnabled is true`}, + {name: "TelemetryIngress chip settings validated with the flag off", toml: ` +[TelemetryIngress] +ChipIngressEnabled = false +ChipIngressBufferSize = 0 +ChipIngressMaxBatchSize = 501 +ChipIngressMaxConcurrentSends = 0 +ChipIngressSendInterval = '10ms' +ChipIngressSendTimeout = '2m' +ChipIngressMaxGRPCRequestSize = 1024 +ChipIngressDrainTimeout = '0s' +`, exp: `invalid configuration: TelemetryIngress: 7 errors: + - ChipIngressBufferSize: invalid value (0): must be between 1 and 100000 + - ChipIngressMaxBatchSize: invalid value (501): must be between 1 and 500 + - ChipIngressMaxConcurrentSends: invalid value (0): must be between 1 and 32 + - ChipIngressMaxGRPCRequestSize: invalid value (1024): must be between 1048576 and 10485760 + - ChipIngressSendInterval: invalid value (10ms): must be between 50ms and 10s + - ChipIngressSendTimeout: invalid value (2m0s): must be between 1s and 1m0s + - ChipIngressDrainTimeout: invalid value (0s): must be greater than 0`}, + {name: "TelemetryIngress chip endpoint must be host:port", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'https://chip.example:443' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: invalid value (https://chip.example:443): must be host:port, without a scheme or whitespace`}, + {name: "TelemetryIngress chip endpoint port in range", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'chip.example:0' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressEndpoint: invalid value (chip.example:0): port must be between 1 and 65535`}, + {name: "TelemetryIngress chip insecure not allowed for chain.link", toml: ` +[TelemetryIngress] +ChipIngressEndpoint = 'legacy-telemetry.prod.telemetry.chain.link:443' +ChipIngressInsecureConnection = true +`, exp: `invalid configuration: TelemetryIngress.ChipIngressInsecureConnection: invalid value (true): must be false for a chain.link endpoint, or the CSA auth token is sent unencrypted`}, + {name: "TelemetryIngress chip drain must fit in shutdown grace period", toml: ` +ShutdownGracePeriod = '4s' +[TelemetryIngress] +ChipIngressEnabled = true +ChipIngressEndpoint = 'chip.example:443' +ChipIngressDrainTimeout = '2s' +`, exp: `invalid configuration: TelemetryIngress.ChipIngressDrainTimeout: invalid value (2s): must be less than half of ShutdownGracePeriod (4s)`}, } { t.Run(tt.name, func(t *testing.T) { var c Config diff --git a/core/services/chainlink/testdata/config-empty-effective.toml b/core/services/chainlink/testdata/config-empty-effective.toml index 09123407bae..b5b062a54a8 100644 --- a/core/services/chainlink/testdata/config-empty-effective.toml +++ b/core/services/chainlink/testdata/config-empty-effective.toml @@ -44,6 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/core/services/chainlink/testdata/config-full.toml b/core/services/chainlink/testdata/config-full.toml index e8112b49a45..abe761ba245 100644 --- a/core/services/chainlink/testdata/config-full.toml +++ b/core/services/chainlink/testdata/config-full.toml @@ -44,6 +44,15 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = 'legacy-telemetry.example:443' +ChipIngressInsecureConnection = true +ChipIngressBufferSize = 2000 +ChipIngressMaxBatchSize = 200 +ChipIngressMaxConcurrentSends = 4 +ChipIngressSendInterval = '1s' +ChipIngressSendTimeout = '5s' +ChipIngressMaxGRPCRequestSize = 4194304 +ChipIngressDrainTimeout = '3s' [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/services/chainlink/testdata/config-multi-chain-effective.toml b/core/services/chainlink/testdata/config-multi-chain-effective.toml index 8d069c17353..5b7fe133320 100644 --- a/core/services/chainlink/testdata/config-multi-chain-effective.toml +++ b/core/services/chainlink/testdata/config-multi-chain-effective.toml @@ -44,6 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = true diff --git a/core/services/synchronization/chip_ingress_batch_client.go b/core/services/synchronization/chip_ingress_batch_client.go index b2b385a0573..1b80fb1181a 100644 --- a/core/services/synchronization/chip_ingress_batch_client.go +++ b/core/services/synchronization/chip_ingress_batch_client.go @@ -3,122 +3,232 @@ package synchronization import ( "context" "fmt" + "strconv" "sync" "time" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" + chipingress "github.com/smartcontractkit/chainlink-common/pkg/chipingress" + batch "github.com/smartcontractkit/chainlink-common/pkg/chipingress/batch" + chipingresspb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" - "github.com/smartcontractkit/chainlink-common/pkg/timeutil" ) +const ( + // chipIngressEndpointLabel is the "endpoint" label on telemetry_client_* metrics. + chipIngressEndpointLabel = "chip-ingress" + // legacyTelemetryBatchClientName is the client_name label on chip_ingress.batch.* metrics. + legacyTelemetryBatchClientName = "legacy_telemetry" + + healthPingInterval = 5 * time.Second + healthPingTimeout = 2 * time.Second + // unhealthyAfterPingFailures marks the service unhealthy only after about a minute of + // failed pings, so a brief telemetry-server blip doesn't flip the node's /health. + unhealthyAfterPingFailures = 12 + chipIngressUnreachable = "chip-ingress unreachable" + + // logEvery limits each kind of repeated warning (per telemetry type) to one line per interval. + logEvery = 30 * time.Second +) + +// chipIngressBatchClient sends the node's legacy OCR telemetry to chip-ingress, +// replacing the WSRPC path to OTI (OCR Telemetry Ingress). One instance is +// shared by every job. Each message becomes a CloudEvent whose extensions +// chip-ingress turns into the same Kafka key and headers OTI used to produce. type chipIngressBatchClient struct { services.Service eng *services.Engine - chipClient chipingress.Client - - logging bool - - telemBufferSize uint - telemMaxBatchSize uint - telemSendInterval time.Duration - telemSendTimeout time.Duration + chipClient chipingress.Client // used only for health Ping; owned and closed by batchClient.Stop + batchClient *batch.Client + csaPubKeyHex string + logging bool + logs logLimiter + pingInterval time.Duration // healthPingInterval; shortened by tests +} - workers map[string]*chipIngressBatchWorker - workersMutex sync.RWMutex +// ChipIngressBatchConfig sizes the shared chip-ingress batch client. Values come from +// the [TelemetryIngress] ChipIngress* settings. +type ChipIngressBatchConfig struct { + BufferSize uint // messages buffered across all jobs before new ones are dropped + MaxBatchSize uint // messages per PublishBatch request + MaxConcurrentSends int // PublishBatch requests in flight + SendInterval time.Duration // max wait before flushing an incomplete batch + SendTimeout time.Duration // per-request PublishBatch timeout + MaxGRPCRequestSize int // max serialized PublishBatch request size; larger batches are split + DrainTimeout time.Duration // max time Close waits to publish queued messages and run their callbacks + Logging bool // debug-log every successful send } -// NewChipIngressBatchClient returns a client backed by chipingress.Client that -// can send telemetry to the chip ingress server -func NewChipIngressBatchClient(chipClient chipingress.Client, logging bool, lggr logger.Logger, telemBufferSize uint, telemMaxBatchSize uint, telemSendInterval time.Duration, telemSendTimeout time.Duration) ChipIngressService { +// NewChipIngressBatchClient wraps chipClient in the chainlink-common batch client. +// The batch client owns chipClient: Close publishes queued messages, then closes the connection. +func NewChipIngressBatchClient(chipClient chipingress.Client, csaPubKeyHex string, cfg ChipIngressBatchConfig, lggr logger.Logger) (ChipIngressService, error) { + batchClient, err := batch.NewBatchClient(chipClient, + batch.WithMessageBuffer(int(cfg.BufferSize)), + batch.WithBatchSize(int(cfg.MaxBatchSize)), + batch.WithMaxConcurrentSends(cfg.MaxConcurrentSends), + batch.WithBatchInterval(cfg.SendInterval), + batch.WithMaxPublishTimeout(cfg.SendTimeout), + batch.WithMaxGRPCRequestSize(cfg.MaxGRPCRequestSize), + batch.WithShutdownTimeout(cfg.DrainTimeout), + batch.WithClientName(legacyTelemetryBatchClientName), + // Send builds a fresh event per message and never touches it after queueing, + // so skip the defensive per-message proto.Clone. + batch.WithEventClone(false), + ) + if err != nil { + return nil, fmt.Errorf("failed to create chip-ingress batch client: %w", err) + } + c := &chipIngressBatchClient{ - telemBufferSize: telemBufferSize, - telemMaxBatchSize: telemMaxBatchSize, - telemSendInterval: telemSendInterval, - telemSendTimeout: telemSendTimeout, - chipClient: chipClient, - logging: logging, - workers: make(map[string]*chipIngressBatchWorker), + chipClient: chipClient, + batchClient: batchClient, + csaPubKeyHex: csaPubKeyHex, + logging: cfg.Logging, + logs: logLimiter{every: logEvery}, + pingInterval: healthPingInterval, } c.Service, c.eng = services.Config{ Name: "ChipIngressBatchClient", Start: c.start, + Close: c.close, }.NewServiceEngine(lggr) - return c + return c, nil } -// start initializes the chip ingress batch client and starts health monitoring -func (cc *chipIngressBatchClient) start(ctx context.Context) error { - cc.startHealthMonitoring(ctx, cc.chipClient) +// start starts the batch client and a periodic health ping. The ping drives +// telemetry_client_connection_status{endpoint="chip-ingress"} on every probe, and +// marks the service unhealthy only after a sustained outage. +func (c *chipIngressBatchClient) start(ctx context.Context) error { + c.batchClient.Start(ctx) + failures := 0 // only touched by the ticker goroutine + c.eng.GoTick(services.TickerConfig{}.NewTicker(c.pingInterval), func(ctx context.Context) { + pingCtx, cancel := context.WithTimeout(ctx, healthPingTimeout) + defer cancel() + if _, err := c.chipClient.Ping(pingCtx, &chipingresspb.EmptyRequest{}); err != nil { + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(0) + if failures++; failures >= unhealthyAfterPingFailures { + c.eng.SetHealthCond(chipIngressUnreachable, err) + } + return + } + TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel).Set(1) + failures = 0 + c.eng.ClearHealthCond(chipIngressUnreachable) + }) return nil } -// Send directs incoming telemetry messages to the worker responsible for pushing it to -// the ingress server. If the worker telemetry buffer is full, messages are dropped -// and a warning is logged. -func (cc *chipIngressBatchClient) Send(ctx context.Context, payload TelemPayload) { - worker := cc.findOrCreateWorker(payload) +// close publishes everything still queued, waiting up to the batch client's shutdown +// timeout for sends and callbacks, then closes the chip-ingress client. Batches still +// draining after the timeout fail against the closed client and count as send errors. +func (c *chipIngressBatchClient) close() error { + c.batchClient.Stop() + return nil +} - select { - case worker.chTelemetry <- payload: - worker.dropMessageCount.Store(0) - case <-ctx.Done(): +// Send builds the CloudEvent and queues it. It never blocks the caller (a libocr +// goroutine). Types chip-ingress has no mapping for are counted as unsupported; +// a full buffer or a shutting-down client drops the message. +func (c *chipIngressBatchClient) Send(_ context.Context, payload TelemPayload) { + telemType := string(payload.TelemType) + domain, entity, err := TelemetryTypeToDomainAndEntity(payload.TelemType) + if err != nil { + // Expected on LLO nodes (llo-report); OTI drops these types too. + TelemetryClientMessagesUnsupported.WithLabelValues(chipIngressEndpointLabel, telemType).Inc() + if ok, suppressed := c.logs.allow("unsupported:" + telemType); ok { + c.eng.Debugw("Telemetry type not supported by ChIP ingress; not sending", "telemType", telemType, "suppressed", suppressed) + } return - default: - worker.logBufferFullWithExpBackoff(payload) + } + event, err := c.newEvent(payload, domain, entity, time.Now()) + if err == nil { + err = c.batchClient.QueueMessage(event, c.onPublished(payload.TelemType, payload.ContractID)) + } + if err != nil { + TelemetryClientMessagesDropped.WithLabelValues(chipIngressEndpointLabel, telemType).Inc() + if ok, suppressed := c.logs.allow("drop:" + telemType); ok { + c.eng.Warnw("Dropping telemetry message for ChIP ingress", "error", err, "telemType", telemType, "suppressed", suppressed) + } } } -// findOrCreateWorker finds a worker by ContractID or creates a new one if none exists. -// The number of workers is naturally bounded by the number of unique (ContractID, TelemetryType) -// pairs, which is determined by the jobs configured on the node. Each job has a fixed contract -// and telemetry type, so the worker count is proportional to the number of active jobs. -func (cc *chipIngressBatchClient) findOrCreateWorker(payload TelemPayload) *chipIngressBatchWorker { - cc.workersMutex.Lock() - defer cc.workersMutex.Unlock() - - workerKey := fmt.Sprintf("%s_%s", payload.ContractID, payload.TelemType) - worker, found := cc.workers[workerKey] - - if !found { - worker = NewChipIngressBatchWorker( - cc.telemMaxBatchSize, - cc.telemSendTimeout, - cc.chipClient, - make(chan TelemPayload, cc.telemBufferSize), - payload.ContractID, - payload.TelemType, - cc.eng, - cc.logging, - ) - cc.eng.GoTick(timeutil.NewTicker(func() time.Duration { - return cc.telemSendInterval - }), worker.Send) - cc.workers[workerKey] = worker - - TelemetryClientWorkers.WithLabelValues(chipIngress, string(payload.TelemType)).Inc() +// onPublished returns the callback the batch client runs once the message's batch +// has been published, counting each message as sent or failed. +func (c *chipIngressBatchClient) onPublished(telemType TelemetryType, contractID string) func(error) { + return func(err error) { + if err != nil { + TelemetryClientMessagesSendErrors.WithLabelValues(chipIngressEndpointLabel, string(telemType)).Inc() + code := batch.ErrorCodeFor(err) + if ok, suppressed := c.logs.allow("error:" + string(telemType) + ":" + code); ok { + c.eng.Warnw("Could not send telemetry via ChIP ingress", "error", err, "errorCode", code, "telemType", telemType, "suppressed", suppressed) + } + return + } + TelemetryClientMessagesSent.WithLabelValues(chipIngressEndpointLabel, string(telemType)).Inc() + if c.logging { + c.eng.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", contractID, "telemType", telemType) + } + } +} + +// newEvent converts a payload into the CloudEvent chip-ingress expects for legacy +// telemetry. Each extension becomes a "ce_" Kafka header; the comments name the +// header OTI used for the same value. A legacy-mode chip-ingress routes on telemetrytype +// and chainselector; source/type (domain/entity) only matter to a non-legacy deployment. +// +// The node does not send nodeoperatorname/nodename or receivedat: chip-ingress derives +// them from the CSA key's NOP info (WithNOPLookup) and its own clock. nodeoperatorkey is +// unknown to the node (spike INFOPLAT-19426). +func (c *chipIngressBatchClient) newEvent(payload TelemPayload, domain, entity string, now time.Time) (*chipingress.CloudEventPb, error) { + event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, map[string]any{"time": now.UTC()}) + if err != nil { + return nil, fmt.Errorf("failed creating CloudEvent: %w", err) } - return worker + event.SetExtension("legacytelemetry", "true") // marks node-sent legacy telemetry + event.SetExtension("telemetrytype", string(payload.TelemType)) // "telemetry-type" + event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) // chain-selectors ID of the job's chain + event.SetExtension("networkname", payload.Network) // endpoint network, e.g. "EVM" + event.SetExtension("contractid", payload.ContractID) // "contract-address" + event.SetExtension("csapublickey", c.csaPubKeyHex) // "node-operator-csa-public-key" + event.SetExtension("partitionkey", payload.ContractID+"-"+c.csaPubKeyHex) // Kafka record key + event.SetExtension("sentat", strconv.FormatInt(now.UnixNano(), 10)) // "sent-at", unix ns + + return chipingress.EventToProto(event) } -// startHealthMonitoring starts a goroutine to monitor the connection state and update other relevant metrics every 5 seconds -func (cc *chipIngressBatchClient) startHealthMonitoring(_ context.Context, chipClient chipingress.Client) { - cc.eng.GoTick(timeutil.NewTicker(func() time.Duration { - return 5 * time.Second - }), func(ctx context.Context) { - connected := float64(0) - pingCtx, pingCancel := context.WithTimeout(ctx, 2*time.Second) - _, err := chipClient.Ping(pingCtx, &pb.EmptyRequest{}) - pingCancel() - if err == nil { - connected = float64(1) - } else { - cc.eng.EmitHealthErr(err) - } - TelemetryClientConnectionStatus.WithLabelValues(chipIngress).Set(connected) - }) +// logLimiter allows one log line per key per interval and counts what it suppressed, +// so repeated warnings stay readable however fast messages arrive. +type logLimiter struct { + every time.Duration + mu sync.Mutex + keys map[string]*logLimiterKey +} + +type logLimiterKey struct { + last time.Time + suppressed int +} + +func (l *logLimiter) allow(key string) (ok bool, suppressed int) { + now := time.Now() + l.mu.Lock() + defer l.mu.Unlock() + if l.keys == nil { + l.keys = make(map[string]*logLimiterKey) + } + k, seen := l.keys[key] + if !seen { + l.keys[key] = &logLimiterKey{last: now} + return true, 0 + } + if now.Sub(k.last) < l.every { + k.suppressed++ + return false, 0 + } + suppressed = k.suppressed + k.last, k.suppressed = now, 0 + return true, suppressed } diff --git a/core/services/synchronization/chip_ingress_batch_client_test.go b/core/services/synchronization/chip_ingress_batch_client_test.go index 691602f427f..3aec511f47e 100644 --- a/core/services/synchronization/chip_ingress_batch_client_test.go +++ b/core/services/synchronization/chip_ingress_batch_client_test.go @@ -1,347 +1,240 @@ -package synchronization_test +package synchronization import ( "context" + "maps" + "slices" + "strconv" + "sync" "sync/atomic" "testing" "time" - "github.com/onsi/gomega" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/testutil" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "github.com/stretchr/testify/require" + "google.golang.org/grpc" "github.com/smartcontractkit/chainlink-common/pkg/chipingress" chipingressmocks "github.com/smartcontractkit/chainlink-common/pkg/chipingress/mocks" "github.com/smartcontractkit/chainlink-common/pkg/services/servicetest" "github.com/smartcontractkit/chainlink/v2/core/internal/testutils" - "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" ) -func TestChipIngressBatchClient_HappyPath(t *testing.T) { - g := gomega.NewWithT(t) - - // Create mocks - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - // Wire up the chip ingress client with instant interval for testing - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - // Create telemetry payloads for different contracts - telemPayload1 := synchronization.TelemPayload{ - Telemetry: []byte("Mock telem 1"), - ContractID: "0x1", - TelemType: synchronization.OCR, +func newTestPayload(contractID string) TelemPayload { + return TelemPayload{ + Telemetry: []byte("telemetry-payload"), + TelemType: OCR2Median, + ContractID: contractID, ChainSelector: 12345, - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - Network: "ethereum-mainnet", + Network: "EVM", } - telemPayload3 := synchronization.TelemPayload{ - Telemetry: []byte("Mock telem 3"), - ContractID: "0x3", - TelemType: synchronization.OCR2Functions, - ChainSelector: 67890, - Domain: "functions", - Entity: "ocr.v2.functions.telemetry", - Network: "polygon-mainnet", - } - - // Assert telemetry payloads for each contract are correctly sent to chip ingress - var contractCounter1 atomic.Uint32 - var contractCounter3 atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - - for _, protoEvent := range batch.Events { - event, err := chipingress.ProtoToEvent(protoEvent) - require.NoError(t, err) - - attrs := protoEvent.GetAttributes() - contractID := attrs["contractid"].GetCeString() - - if contractID == "0x1" { - contractCounter1.Add(1) - assert.Equal(t, telemPayload1.Telemetry, event.Data()) - assert.Equal(t, string(synchronization.OCR), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, telemPayload1.Network, attrs["networkname"].GetCeString()) - assert.Empty(t, attrs["nodeoperatorname"].GetCeString()) - assert.Empty(t, attrs["nodename"].GetCeString()) - assert.Empty(t, attrs["nodecsapublickey"].GetCeString()) - } - if contractID == "0x3" { - contractCounter3.Add(1) - assert.Equal(t, telemPayload3.Telemetry, event.Data()) - assert.Equal(t, string(synchronization.OCR2Functions), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, telemPayload3.Network, attrs["networkname"].GetCeString()) - assert.Empty(t, attrs["nodeoperatorname"].GetCeString()) - assert.Empty(t, attrs["nodename"].GetCeString()) - assert.Empty(t, attrs["nodecsapublickey"].GetCeString()) - } - } - }) - - // Send telemetry - testCtx := t.Context() - chipIngressClient.Send(testCtx, telemPayload1) - chipIngressClient.Send(testCtx, telemPayload3) - time.Sleep(sendInterval * 2) - chipIngressClient.Send(testCtx, telemPayload1) - chipIngressClient.Send(testCtx, telemPayload1) - - // Wait for the telemetry to be handled - g.Eventually(func() []uint32 { - return []uint32{contractCounter1.Load(), contractCounter3.Load()} - }, testutils.WaitTimeout(t)).Should(gomega.Equal([]uint32{3, 1})) } -func TestChipIngressBatchClient_MultipleBatches(t *testing.T) { - g := gomega.NewWithT(t) - - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - sendInterval := time.Millisecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xabc", - TelemType: synchronization.OCR2Median, - ChainSelector: 12345, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", - } - - var batchCount atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batchCount.Add(1) - }) - - testCtx := t.Context() - // Send multiple messages to trigger multiple batches - for i := range 10 { - chipIngressClient.Send(testCtx, telemPayload) - if i%3 == 0 { - time.Sleep(sendInterval * 2) // Allow batch to be sent - } - } - - // Wait for batches to be sent - g.Eventually(batchCount.Load, testutils.WaitTimeout(t)).Should(gomega.BeNumerically(">=", 2)) +// publishedBatches collects PublishBatch requests; the batch client publishes +// from its own goroutines, so access is guarded. +type publishedBatches struct { + mu sync.Mutex + batches [][]*chipingress.CloudEventPb } -func TestChipIngressBatchClient_DifferentTelemetryTypes(t *testing.T) { - g := gomega.NewWithT(t) - - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - // Create payloads with different telemetry types but same contract - payloadOCR := synchronization.TelemPayload{ - Telemetry: []byte("OCR telemetry"), - ContractID: "0x123", - TelemType: synchronization.OCR, - ChainSelector: 1, - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - Network: "ethereum-mainnet", - } - - payloadOCR2 := synchronization.TelemPayload{ - Telemetry: []byte("OCR2 telemetry"), - ContractID: "0x123", - TelemType: synchronization.OCR2Median, - ChainSelector: 1, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", - } - - var ocrCount, ocr2Count atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - for _, protoEvent := range batch.Events { - attrs := protoEvent.GetAttributes() - telemType := attrs["telemetrytype"].GetCeString() - if telemType == string(synchronization.OCR) { - ocrCount.Add(1) - } else if telemType == string(synchronization.OCR2Median) { - ocr2Count.Add(1) - } - } - }) - - testCtx := t.Context() - chipIngressClient.Send(testCtx, payloadOCR) - chipIngressClient.Send(testCtx, payloadOCR2) - - g.Eventually(func() []uint32 { - return []uint32{ocrCount.Load(), ocr2Count.Load()} - }, testutils.WaitTimeout(t)).Should(gomega.Equal([]uint32{1, 1})) +func (p *publishedBatches) add(events []*chipingress.CloudEventPb) { + p.mu.Lock() + defer p.mu.Unlock() + p.batches = append(p.batches, events) } -func TestChipIngressBatchClient_ContextCancellation(t *testing.T) { - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Maybe() +func (p *publishedBatches) snapshot() [][]*chipingress.CloudEventPb { + p.mu.Lock() + defer p.mu.Unlock() + return slices.Clone(p.batches) +} - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) +type publishFunc func(*chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xdef", - TelemType: synchronization.OCR2Functions, - ChainSelector: 67890, - Domain: "functions", - Entity: "ocr.v2.functions.telemetry", - Network: "ethereum-mainnet", +// publishOK acknowledges every event; the batch client expects one positional result per event. +func publishOK(b *chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) { + results := make([]*chipingress.PublishResult, len(b.Events)) + for i := range results { + results[i] = &chipingress.PublishResult{} } - - // Create a cancelled context - ctx, cancel := context.WithCancel(t.Context()) - cancel() - - // Should not panic or block when context is cancelled - chipIngressClient.Send(ctx, telemPayload) + return &chipingress.PublishResponse{Results: results}, nil } -func TestChipIngressBatchClient_WorkerReuse(t *testing.T) { - g := gomega.NewWithT(t) - +// startBatchClient starts a batch client over a mock chip client whose PublishBatch +// records each request and then answers with publish. +func startBatchClient(t *testing.T, publish publishFunc) (ChipIngressService, *publishedBatches) { + t.Helper() chipClient := chipingressmocks.NewClient(t) chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) - - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xabc", - TelemType: synchronization.OCR2Median, - ChainSelector: 99999, - Domain: "data-feeds", - Entity: "ocr.v2.median.telemetry", - Network: "ethereum-mainnet", - } - - var messageCount atomic.Uint32 - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - // #nosec G115 -- len() returns non-negative int, safe to convert to uint32 - messageCount.Add(uint32(len(batch.Events))) - }) - - testCtx := t.Context() - // Send multiple messages with same contract and type - should reuse worker - for range 5 { - chipIngressClient.Send(testCtx, telemPayload) - } - - // Wait for all messages to be sent - g.Eventually(messageCount.Load, testutils.WaitTimeout(t)).Should(gomega.Equal(uint32(5))) + chipClient.On("Close").Return(nil).Maybe() // batch.Client.Stop() closes the underlying client + published := &publishedBatches{} + chipClient.EXPECT().PublishBatch(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(_ context.Context, b *chipingress.CloudEventBatch, _ ...grpc.CallOption) (*chipingress.PublishResponse, error) { + published.add(b.Events) + return publish(b) + }).Maybe() + + client := NewTestChipIngressBatchClient(t, chipClient, false, time.Nanosecond) + servicetest.Run(t, client) + return client, published } -func TestChipIngressBatchClient_ChainSelectorInAttributes(t *testing.T) { - g := gomega.NewWithT(t) +// awaitFirstEvent waits for exactly one published batch holding one event and decodes it. +func awaitFirstEvent(t *testing.T, published *publishedBatches) chipingress.CloudEvent { + t.Helper() + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.Len(c, published.snapshot(), 1) + }, testutils.WaitTimeout(t), 10*time.Millisecond) + batches := published.snapshot() + require.Len(t, batches[0], 1) + event, err := chipingress.ProtoToEvent(batches[0][0]) + require.NoError(t, err) + return event +} - chipClient := chipingressmocks.NewClient(t) - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Maybe() +func counterValue(vec *prometheus.CounterVec, telemType TelemetryType) float64 { + return testutil.ToFloat64(vec.WithLabelValues(chipIngressEndpointLabel, string(telemType))) +} - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - servicetest.Run(t, chipIngressClient) +func TestChipIngressBatchClient_Event(t *testing.T) { + t.Parallel() + client, published := startBatchClient(t, publishOK) + + client.Send(t.Context(), newTestPayload("0xabc")) + event := awaitFirstEvent(t, published) + + // Source/type are the telemetry type's domain/entity. + domain, entity, err := TelemetryTypeToDomainAndEntity(OCR2Median) + require.NoError(t, err) + assert.Equal(t, domain, event.Source()) + assert.Equal(t, entity, event.Type()) + + // The exact extension set on the wire: the legacy-telemetry contract plus the two + // the library stamps (seqnum by the batch client, recordedtime by NewEvent). + // Anything else, e.g. receivedat, which chip-ingress owns, must not be sent. + exts := event.Extensions() + assert.ElementsMatch(t, []string{ + "legacytelemetry", "telemetrytype", "chainselector", "networkname", + "contractid", "csapublickey", "partitionkey", "sentat", + "seqnum", "recordedtime", + }, slices.Collect(maps.Keys(exts))) + assert.Equal(t, "true", exts["legacytelemetry"]) + assert.Equal(t, string(OCR2Median), exts["telemetrytype"]) + assert.Equal(t, "12345", exts["chainselector"]) + assert.Equal(t, "EVM", exts["networkname"]) + assert.Equal(t, "0xabc", exts["contractid"]) + assert.Equal(t, "deadbeef", exts["csapublickey"]) + assert.Equal(t, "0xabc-deadbeef", exts["partitionkey"]) + + sentAt, err := strconv.ParseInt(exts["sentat"].(string), 10, 64) // unix nanos + require.NoError(t, err) + assert.WithinDuration(t, time.Now(), time.Unix(0, sentAt), time.Minute) +} - expectedChainSelector := uint64(123456789) - telemPayload := synchronization.TelemPayload{ - Telemetry: []byte("Test telemetry"), - ContractID: "0xtest", - TelemType: synchronization.OCR3CCIPCommit, - ChainSelector: expectedChainSelector, - Domain: "ccip", - Entity: "ocr.v3.ccip.commit.telemetry", - Network: "ethereum-mainnet", +func TestChipIngressBatchClient_UnsupportedTypesCountedNotDropped(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics + client, published := startBatchClient(t, publishOK) + + // LLO nodes interleave llo-report (no chip mapping) with supported types. + const n = 50 + unsupportedBefore := counterValue(TelemetryClientMessagesUnsupported, LLOReport) + droppedBefore := counterValue(TelemetryClientMessagesDropped, LLOReport) + for range n { + unsupported := newTestPayload("0xabc") + unsupported.TelemType = LLOReport + client.Send(t.Context(), unsupported) + client.Send(t.Context(), newTestPayload("0xabc")) } - var capturedChainSelector atomic.Value - chipClient.On("PublishBatch", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PublishResponse{}, nil).Run(func(args mock.Arguments) { - batch := args.Get(1).(*chipingress.CloudEventBatch) - if len(batch.Events) > 0 { - attrs := batch.Events[0].GetAttributes() - capturedChainSelector.Store(attrs["chainselector"].GetCeString()) - } - }) - - testCtx := t.Context() - chipIngressClient.Send(testCtx, telemPayload) - - g.Eventually(func() string { - v := capturedChainSelector.Load() - if v == nil { - return "" + assert.InDelta(t, n, counterValue(TelemetryClientMessagesUnsupported, LLOReport)-unsupportedBefore, 0.001) + assert.InDelta(t, 0, counterValue(TelemetryClientMessagesDropped, LLOReport)-droppedBefore, 0.001, + "unsupported-by-design must not count as dropped, so dropped stays alertable") + require.EventuallyWithT(t, func(c *assert.CollectT) { + total := 0 + for _, b := range published.snapshot() { + for _, e := range b { + assert.NotEqual(c, string(LLOReport), e.Attributes["telemetrytype"].GetCeString()) + total++ + } } - return v.(string) - }, testutils.WaitTimeout(t)).Should(gomega.Equal("123456789")) + assert.Equal(c, n, total, "only the supported messages are published") + }, testutils.WaitTimeout(t), 10*time.Millisecond) } -func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") - } - - g := gomega.NewWithT(t) +func TestLogLimiter(t *testing.T) { + t.Parallel() + l := logLimiter{every: 50 * time.Millisecond} - chipClient := chipingressmocks.NewClient(t) - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - - // Mock Ping to succeed initially - var pingCallCount atomic.Uint32 - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(&chipingress.PingResponse{}, nil).Run(func(args mock.Arguments) { - pingCallCount.Add(1) - }) - - // Start the client which should start health monitoring - servicetest.Run(t, chipIngressClient) + ok, suppressed := l.allow("a") + assert.True(t, ok) + assert.Zero(t, suppressed) + for range 5 { + ok, _ = l.allow("a") + assert.False(t, ok, "repeats within the interval are suppressed") + } + ok, _ = l.allow("b") + assert.True(t, ok, "keys are limited independently") - // Wait for at least 2 ping calls to verify health monitoring is running - g.Eventually(pingCallCount.Load, testutils.WaitTimeout(t), 100*time.Millisecond).Should(gomega.BeNumerically(">=", 2)) + time.Sleep(60 * time.Millisecond) + ok, suppressed = l.allow("a") + assert.True(t, ok) + assert.Equal(t, 5, suppressed, "the next line reports how many were suppressed") } -func TestChipIngressBatchClient_HealthMonitoring_PingFailure(t *testing.T) { - if testing.Short() { - t.Skip("too slow for testing.Short") +func TestChipIngressBatchClient_CountsPerMessage(t *testing.T) { //nolint:paralleltest // asserts deltas on package-global prometheus metrics + for name, tc := range map[string]struct { //nolint:paralleltest // subtests share package-global prometheus metrics + publish publishFunc + counter *prometheus.CounterVec + }{ + "sent": {publishOK, TelemetryClientMessagesSent}, + "send errors": {func(*chipingress.CloudEventBatch) (*chipingress.PublishResponse, error) { return nil, assert.AnError }, TelemetryClientMessagesSendErrors}, + } { + t.Run(name, func(t *testing.T) { + client, _ := startBatchClient(t, tc.publish) + const n = 3 + before := counterValue(tc.counter, OCR2Median) + for range n { + client.Send(t.Context(), newTestPayload("0xabc")) + } + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, float64(n), counterValue(tc.counter, OCR2Median)-before, 0.001, "must count messages, not batches") + }, testutils.WaitTimeout(t), 10*time.Millisecond) + }) } +} - g := gomega.NewWithT(t) - +func TestChipIngressBatchClient_HealthMonitoring(t *testing.T) { //nolint:paralleltest // drives a package-global prometheus gauge + gauge := TelemetryClientConnectionStatus.WithLabelValues(chipIngressEndpointLabel) + var reachable atomic.Bool + var failedPings atomic.Int32 chipClient := chipingressmocks.NewClient(t) - - sendInterval := time.Nanosecond - chipIngressClient := synchronization.NewTestChipIngressBatchClient(t, chipClient, false, sendInterval) - - // Mock Ping to fail - var pingCallCount atomic.Uint32 - chipClient.On("Ping", mock.Anything, mock.Anything, mock.Anything).Return(nil, assert.AnError).Run(func(args mock.Arguments) { - pingCallCount.Add(1) - }) - - // Start the client which should start health monitoring - servicetest.Run(t, chipIngressClient) - - // Wait for at least 2 ping calls to verify health monitoring continues despite failures - g.Eventually(pingCallCount.Load, testutils.WaitTimeout(t), 100*time.Millisecond).Should(gomega.BeNumerically(">=", 2)) + chipClient.On("Close").Return(nil).Maybe() + chipClient.EXPECT().Ping(mock.Anything, mock.Anything, mock.Anything). + RunAndReturn(func(context.Context, *chipingress.EmptyRequest, ...grpc.CallOption) (*chipingress.PingResponse, error) { + if reachable.Load() { + return &chipingress.PingResponse{}, nil + } + failedPings.Add(1) + return nil, assert.AnError + }).Maybe() + client := NewTestChipIngressBatchClient(t, chipClient, false, time.Second) + client.(*chipIngressBatchClient).pingInterval = time.Millisecond + gauge.Set(1) + servicetest.Run(t, client) + + // A failed ping only drops the gauge; the service turns unhealthy after a sustained outage. + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, 0, testutil.ToFloat64(gauge), 0.001) + assert.Error(c, client.HealthReport()[client.Name()]) + }, testutils.WaitTimeout(t), time.Millisecond) + assert.GreaterOrEqual(t, failedPings.Load(), int32(unhealthyAfterPingFailures)) + + // One good ping restores both. + reachable.Store(true) + require.EventuallyWithT(t, func(c *assert.CollectT) { + assert.InDelta(c, 1, testutil.ToFloat64(gauge), 0.001) + assert.NoError(c, client.HealthReport()[client.Name()]) + }, testutils.WaitTimeout(t), time.Millisecond) } diff --git a/core/services/synchronization/chip_ingress_batch_worker.go b/core/services/synchronization/chip_ingress_batch_worker.go deleted file mode 100644 index c59252a79d6..00000000000 --- a/core/services/synchronization/chip_ingress_batch_worker.go +++ /dev/null @@ -1,159 +0,0 @@ -package synchronization - -import ( - "context" - "fmt" - "strconv" - "sync/atomic" - "time" - - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/logger" - "github.com/smartcontractkit/chainlink-common/pkg/services" -) - -const ( - chipIngress = "chip-ingress" -) - -// chipIngressBatchWorker mirrors telemetryIngressBatchWorker but targets the ChIP ingress client. -// A worker is created per (contractID, telemetry type) pair. -type chipIngressBatchWorker struct { - services.Service - - maxBatchSize uint - sendTimeout time.Duration - chipClient chipingress.Client - chTelemetry chan TelemPayload - contractID string - telemType TelemetryType - logging bool - lggr logger.Logger - dropMessageCount atomic.Uint32 -} - -// NewChipIngressBatchWorker returns a worker for a given contractID that can send -// telemetry to the chip ingress server via PublishBatch. -func NewChipIngressBatchWorker( - maxBatchSize uint, - sendTimeout time.Duration, - chipClient chipingress.Client, - chTelemetry chan TelemPayload, - contractID string, - telemType TelemetryType, - lggr logger.Logger, - logging bool, -) *chipIngressBatchWorker { - return &chipIngressBatchWorker{ - maxBatchSize: maxBatchSize, - sendTimeout: sendTimeout, - chipClient: chipClient, - chTelemetry: chTelemetry, - contractID: contractID, - telemType: telemType, - logging: logging, - lggr: logger.Named(lggr, "ChipIngressBatchWorker"), - } -} - -// Send sends batched telemetry to the chip ingress server on an interval. -func (cw *chipIngressBatchWorker) Send(ctx context.Context) { - if len(cw.chTelemetry) == 0 { - return - } - - batch := cw.BuildCloudEventBatch() - if batch == nil || len(batch.Events) == 0 { - return - } - - ctx, cancel := context.WithTimeout(ctx, cw.sendTimeout) - defer cancel() - - _, err := cw.chipClient.PublishBatch(ctx, batch) - if err != nil { - cw.lggr.Warnf("Could not send telemetry via ChIP ingress: %v", err) - TelemetryClientMessagesSendErrors.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - return - } - - TelemetryClientMessagesSent.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - if cw.logging { - cw.lggr.Debugw("Successfully sent telemetry to ChIP ingress", "contractID", cw.contractID, "telemType", cw.telemType, "batchSize", len(batch.Events)) - } -} - -// logBufferFullWithExpBackoff logs messages at 1,2,4,8,16,32,64,100,200,300,... when the buffer is full. -func (cw *chipIngressBatchWorker) logBufferFullWithExpBackoff(payload TelemPayload) { - count := cw.dropMessageCount.Add(1) - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { - cw.lggr.Warnw("chip ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) - } -} - -// BuildCloudEventBatch reads telemetry off the worker channel and packages it into a CloudEvent batch. -func (cw *chipIngressBatchWorker) BuildCloudEventBatch() *chipingress.CloudEventBatch { - var events []chipingress.CloudEvent - - // #nosec G115 -- maxBatchSize is uint, safe to convert to int for comparison - for len(cw.chTelemetry) > 0 && len(events) < int(cw.maxBatchSize) { - payload := <-cw.chTelemetry - event, err := cw.payloadToEvent(payload) - if err != nil { - cw.lggr.Warnw("failed to build CloudEvent for ChIP ingress", "error", err, "contractID", payload.ContractID, "telemType", payload.TelemType) - TelemetryClientMessagesDropped.WithLabelValues(chipIngress, string(cw.telemType)).Inc() - continue - } - events = append(events, event) - } - - if len(events) == 0 { - return nil - } - - batch, err := chipingress.EventsToBatch(events) - if err != nil { - cw.lggr.Warnw("failed to convert CloudEvents to batch", "error", err) - return nil - } - - return batch -} - -func (cw *chipIngressBatchWorker) payloadToEvent(payload TelemPayload) (chipingress.CloudEvent, error) { - domain := payload.Domain - entity := payload.Entity - if domain == "" || entity == "" { - var err error - domain, entity, err = TelemetryTypeToDomainAndEntity(payload.TelemType) - if err != nil { - return chipingress.CloudEvent{}, fmt.Errorf("missing domain/entity for telemType %s: %w", payload.TelemType, err) - } - } - - now := time.Now().UTC() - - attrs := map[string]any{ - "time": now, - "datacontenttype": "application/protobuf", - } - - event, err := chipingress.NewEvent(domain, entity, payload.Telemetry, attrs) - if err != nil { - return chipingress.CloudEvent{}, fmt.Errorf("failed creating CloudEvent: %w", err) - } - - event.SetExtension("telemetrytype", string(payload.TelemType)) - event.SetExtension("chainselector", strconv.FormatUint(payload.ChainSelector, 10)) - event.SetExtension("contractid", payload.ContractID) - event.SetExtension("networkname", payload.Network) - event.SetExtension("sentat", now.Format(time.RFC3339)) - // These attributes need to be set in chip-ingress server side - event.SetExtension("nodeoperatorname", "") - event.SetExtension("nodename", "") - event.SetExtension("nodecsapublickey", "") - - return event, nil -} diff --git a/core/services/synchronization/chip_ingress_batch_worker_test.go b/core/services/synchronization/chip_ingress_batch_worker_test.go deleted file mode 100644 index 8dc68ad5fbd..00000000000 --- a/core/services/synchronization/chip_ingress_batch_worker_test.go +++ /dev/null @@ -1,131 +0,0 @@ -package synchronization - -import ( - "context" - "testing" - "time" - - cepb "github.com/cloudevents/sdk-go/binding/format/protobuf/v2/pb" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - "google.golang.org/grpc" - - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" - "github.com/smartcontractkit/chainlink/v2/core/logger" -) - -type noopChipIngressPublisher struct{} - -func (noopChipIngressPublisher) Publish(ctx context.Context, event *cepb.CloudEvent, opts ...grpc.CallOption) (*pb.PublishResponse, error) { - return &pb.PublishResponse{}, nil -} - -func (noopChipIngressPublisher) PublishBatch(ctx context.Context, batch *pb.CloudEventBatch, opts ...grpc.CallOption) (*pb.PublishResponse, error) { - return &pb.PublishResponse{}, nil -} - -func (noopChipIngressPublisher) Ping(ctx context.Context, req *pb.EmptyRequest, opts ...grpc.CallOption) (*pb.PingResponse, error) { - return &pb.PingResponse{}, nil -} - -func (noopChipIngressPublisher) StreamEvents(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[pb.StreamEventsRequest, pb.StreamEventsResponse], error) { - return nil, nil -} - -func (noopChipIngressPublisher) RegisterSchema(ctx context.Context, req *pb.RegisterSchemaRequest, opts ...grpc.CallOption) (*pb.RegisterSchemaResponse, error) { - return &pb.RegisterSchemaResponse{}, nil -} - -func (noopChipIngressPublisher) Close() error { - return nil -} - -func (noopChipIngressPublisher) RegisterSchemas(ctx context.Context, schemas ...*pb.Schema) (map[string]int, error) { - return nil, nil -} - -func TestChipIngressBatchWorker_BuildCloudEventBatch(t *testing.T) { - maxBatchSize := 3 - chTelemetry := make(chan TelemPayload, 10) - // #nosec G115 -- maxBatchSize is a small positive constant, safe to convert to uint - worker := NewChipIngressBatchWorker( - uint(maxBatchSize), - time.Second, - noopChipIngressPublisher{}, - chTelemetry, - "0xabc", - OCR, - logger.TestLogger(t), - false, - ) - - payload := TelemPayload{ - Telemetry: []byte("payload-1"), - TelemType: OCR, - ContractID: "0xabc", - Domain: "data-feeds", - Entity: "ocr.v1.telemetry", - ChainSelector: 7700, - Network: "EVM", - ChainID: "1", - } - - // enqueue more payloads than maxBatchSize to ensure batching occurs - for range 5 { - chTelemetry <- payload - } - - batch1 := worker.BuildCloudEventBatch() - require.NotNil(t, batch1) - require.Len(t, batch1.Events, maxBatchSize) - assert.Len(t, chTelemetry, 5-maxBatchSize) - - evt, err := chipingress.ProtoToEvent(batch1.Events[0]) - require.NoError(t, err) - assert.Equal(t, "data-feeds", evt.Source()) - assert.Equal(t, "ocr.v1.telemetry", evt.Type()) - assert.Equal(t, []byte("payload-1"), evt.Data()) - - attrs := batch1.Events[0].GetAttributes() - require.Contains(t, attrs, "telemetrytype") - require.Contains(t, attrs, "chainselector") - assert.Equal(t, string(OCR), attrs["telemetrytype"].GetCeString()) - assert.Equal(t, "7700", attrs["chainselector"].GetCeString()) - - batch2 := worker.BuildCloudEventBatch() - require.NotNil(t, batch2) - require.Len(t, batch2.Events, 2) - assert.Empty(t, chTelemetry) -} - -func TestChipIngressBatchWorker_BuildCloudEventBatchUsesMapping(t *testing.T) { - chTelemetry := make(chan TelemPayload, 1) - worker := NewChipIngressBatchWorker( - 1, - time.Second, - noopChipIngressPublisher{}, - chTelemetry, - "0xdef", - OCR, - logger.TestLogger(t), - false, - ) - - chTelemetry <- TelemPayload{ - Telemetry: []byte("payload"), - TelemType: OCR, - ContractID: "0xdef", - ChainSelector: 9001, - Network: "EVM", - ChainID: "137", - } - - batch := worker.BuildCloudEventBatch() - require.NotNil(t, batch) - require.Len(t, batch.Events, 1) - - evt, err := chipingress.ProtoToEvent(batch.Events[0]) - require.NoError(t, err) - assert.Equal(t, "data-feeds.telemetry.ocr", evt.Source()) -} diff --git a/core/services/synchronization/chip_ingress_client.go b/core/services/synchronization/chip_ingress_client.go new file mode 100644 index 00000000000..768661e4d42 --- /dev/null +++ b/core/services/synchronization/chip_ingress_client.go @@ -0,0 +1,52 @@ +package synchronization + +import ( + "fmt" + "time" + + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" +) + +// ChipIngressClientConfig configures the gRPC client for the legacy telemetry chip-ingress endpoint. +// Metrics and traces use the global OTel providers, which Beholder installs at startup. +type ChipIngressClientConfig struct { + Endpoint string // host:port + InsecureConnection bool + AuthHeaders map[string]string // static CSA auth headers (keystore.BuildBeholderAuth) + AuthHeadersTTL time.Duration // >0 selects rotating CSA tokens (must be >= 10m); same value Beholder uses + AuthPublicKeyHex string + AuthKeySigner chipingress.Signer +} + +// NewChipIngressClient builds the chip-ingress client used for legacy telemetry. +func NewChipIngressClient(cfg ChipIngressClientConfig) (chipingress.Client, error) { + auth, err := chipingress.NewHeaderProvider(chipingress.HeaderProviderConfig{ + AuthHeaders: cfg.AuthHeaders, + AuthHeadersTTL: cfg.AuthHeadersTTL, + AuthPublicKeyHex: cfg.AuthPublicKeyHex, + AuthKeySigner: cfg.AuthKeySigner, + InsecureConnection: cfg.InsecureConnection, + }) + if err != nil { + return nil, fmt.Errorf("failed to build chip-ingress auth: %w", err) + } + + // The transport option must come before WithTokenAuth, which reads it to + // decide whether its per-RPC credentials require TLS. + opts := []chipingress.Opt{chipingress.WithTLS()} + if cfg.InsecureConnection { + opts = []chipingress.Opt{chipingress.WithInsecureConnection()} + } + if auth != nil { // nil only when there are no headers and no TTL + opts = append(opts, chipingress.WithTokenAuth(auth)) + } + // Load-bearing: chip-ingress derives the legacy telemetry identity headers + // (ce_nodeoperatorname/node-operator-name, ce_nodename/node-name) from the + // NOP info this option requests via x-include-nop-info. The node does not + // send them itself, so removing this silently blanks those headers downstream. + opts = append(opts, chipingress.WithNOPLookup()) + + // grpc.NewClient connects lazily: an unreachable endpoint does not fail + // startup, a malformed host:port does. + return chipingress.NewClient(cfg.Endpoint, opts...) +} diff --git a/core/services/synchronization/chip_ingress_client_test.go b/core/services/synchronization/chip_ingress_client_test.go new file mode 100644 index 00000000000..3bedc627b16 --- /dev/null +++ b/core/services/synchronization/chip_ingress_client_test.go @@ -0,0 +1,52 @@ +package synchronization + +import ( + "context" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "google.golang.org/grpc" + "google.golang.org/grpc/metadata" + + chipingresspb "github.com/smartcontractkit/chainlink-common/pkg/chipingress/pb" +) + +type capturingChipIngressServer struct { + chipingresspb.UnimplementedChipIngressServer + lastMD metadata.MD +} + +func (s *capturingChipIngressServer) Ping(ctx context.Context, _ *chipingresspb.EmptyRequest) (*chipingresspb.PingResponse, error) { + s.lastMD, _ = metadata.FromIncomingContext(ctx) + return &chipingresspb.PingResponse{}, nil +} + +func TestChipIngressClient_SendAuthHeaders(t *testing.T) { + t.Parallel() + lis, err := (&net.ListenConfig{}).Listen(t.Context(), "tcp", "127.0.0.1:0") + require.NoError(t, err) + defer lis.Close() + + srv := grpc.NewServer() + capture := &capturingChipIngressServer{} + chipingresspb.RegisterChipIngressServer(srv, capture) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + + client, err := NewChipIngressClient(ChipIngressClientConfig{ + Endpoint: lis.Addr().String(), + InsecureConnection: true, + AuthHeaders: map[string]string{"X-Beholder-Node-Auth-Token": "x"}, + }) + require.NoError(t, err) + defer client.Close() + + _, err = client.Ping(t.Context(), &chipingresspb.EmptyRequest{}) + require.NoError(t, err) + + assert.NotNil(t, capture.lastMD) + assert.Equal(t, []string{"true"}, capture.lastMD.Get("x-include-nop-info")) + assert.Equal(t, []string{"x"}, capture.lastMD.Get("X-Beholder-Node-Auth-Token")) +} diff --git a/core/services/synchronization/common.go b/core/services/synchronization/common.go index cd0917c7c80..69afe0493b9 100644 --- a/core/services/synchronization/common.go +++ b/core/services/synchronization/common.go @@ -41,11 +41,14 @@ type TelemPayload struct { Telemetry []byte TelemType TelemetryType ContractID string - Domain string - Entity string ChainSelector uint64 Network string - ChainID string +} + +// shouldLogCount reports whether the n-th occurrence of a repeated warning +// should be logged: at 1, 2, 4, 8, 16, 32, 64, then every 100th. +func shouldLogCount(n uint32) bool { + return n > 0 && (n%100 == 0 || n&(n-1) == 0) } // TelemetryService encapsulates all the functionality needed to diff --git a/core/services/synchronization/helpers_test.go b/core/services/synchronization/helpers_test.go index 598ce4f31a9..20c95f485d2 100644 --- a/core/services/synchronization/helpers_test.go +++ b/core/services/synchronization/helpers_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/stretchr/testify/require" + "github.com/smartcontractkit/chainlink-common/pkg/chipingress" "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/keystore" @@ -26,7 +28,17 @@ func NewTestTelemetryIngressBatchClient(t *testing.T, url *url.URL, serverPubKey return tc } -// NewTestChipIngressBatchClient calls NewChipIngressBatchClient and injects chipClient. +// NewTestChipIngressBatchClient calls NewChipIngressBatchClient with a fixed test CSA key and default sizing. func NewTestChipIngressBatchClient(t *testing.T, chipClient chipingress.Client, logging bool, sendInterval time.Duration) ChipIngressService { - return NewChipIngressBatchClient(chipClient, logging, logger.TestLogger(t), 100, 50, sendInterval, time.Second) + c, err := NewChipIngressBatchClient(chipClient, "deadbeef", ChipIngressBatchConfig{ + BufferSize: 10_000, + MaxBatchSize: 1_000, + MaxConcurrentSends: 10, + SendInterval: sendInterval, + SendTimeout: time.Second, + DrainTimeout: 5 * time.Second, + Logging: logging, + }, logger.TestLogger(t)) + require.NoError(t, err) + return c } diff --git a/core/services/synchronization/metrics.go b/core/services/synchronization/metrics.go index d6bafb7b0d6..75f907fcbbe 100644 --- a/core/services/synchronization/metrics.go +++ b/core/services/synchronization/metrics.go @@ -26,6 +26,11 @@ var ( Help: "Number of telemetry messages dropped", }, []string{"endpoint", "telemetry_type"}) + TelemetryClientMessagesUnsupported = promauto.NewCounterVec(prometheus.CounterOpts{ + Name: "telemetry_client_messages_unsupported", + Help: "Number of telemetry messages not sent because the endpoint does not support their telemetry type", + }, []string{"endpoint", "telemetry_type"}) + TelemetryClientWorkers = promauto.NewCounterVec(prometheus.CounterOpts{ Name: "telemetry_client_workers", Help: "Number of telemetry workers", diff --git a/core/services/synchronization/mocks/chip_ingress_service.go b/core/services/synchronization/mocks/chip_ingress_service.go new file mode 100644 index 00000000000..6d5d5cb2edf --- /dev/null +++ b/core/services/synchronization/mocks/chip_ingress_service.go @@ -0,0 +1,299 @@ +// Code generated by mockery v2.53.7. DO NOT EDIT. + +package mocks + +import ( + context "context" + + synchronization "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" + mock "github.com/stretchr/testify/mock" +) + +// ChipIngressService is an autogenerated mock type for the ChipIngressService type +type ChipIngressService struct { + mock.Mock +} + +type ChipIngressService_Expecter struct { + mock *mock.Mock +} + +func (_m *ChipIngressService) EXPECT() *ChipIngressService_Expecter { + return &ChipIngressService_Expecter{mock: &_m.Mock} +} + +// Close provides a mock function with no fields +func (_m *ChipIngressService) Close() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Close") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Close_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Close' +type ChipIngressService_Close_Call struct { + *mock.Call +} + +// Close is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Close() *ChipIngressService_Close_Call { + return &ChipIngressService_Close_Call{Call: _e.mock.On("Close")} +} + +func (_c *ChipIngressService_Close_Call) Run(run func()) *ChipIngressService_Close_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Close_Call) Return(_a0 error) *ChipIngressService_Close_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Close_Call) RunAndReturn(run func() error) *ChipIngressService_Close_Call { + _c.Call.Return(run) + return _c +} + +// HealthReport provides a mock function with no fields +func (_m *ChipIngressService) HealthReport() map[string]error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for HealthReport") + } + + var r0 map[string]error + if rf, ok := ret.Get(0).(func() map[string]error); ok { + r0 = rf() + } else { + if ret.Get(0) != nil { + r0 = ret.Get(0).(map[string]error) + } + } + + return r0 +} + +// ChipIngressService_HealthReport_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'HealthReport' +type ChipIngressService_HealthReport_Call struct { + *mock.Call +} + +// HealthReport is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) HealthReport() *ChipIngressService_HealthReport_Call { + return &ChipIngressService_HealthReport_Call{Call: _e.mock.On("HealthReport")} +} + +func (_c *ChipIngressService_HealthReport_Call) Run(run func()) *ChipIngressService_HealthReport_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_HealthReport_Call) Return(_a0 map[string]error) *ChipIngressService_HealthReport_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_HealthReport_Call) RunAndReturn(run func() map[string]error) *ChipIngressService_HealthReport_Call { + _c.Call.Return(run) + return _c +} + +// Name provides a mock function with no fields +func (_m *ChipIngressService) Name() string { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Name") + } + + var r0 string + if rf, ok := ret.Get(0).(func() string); ok { + r0 = rf() + } else { + r0 = ret.Get(0).(string) + } + + return r0 +} + +// ChipIngressService_Name_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Name' +type ChipIngressService_Name_Call struct { + *mock.Call +} + +// Name is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Name() *ChipIngressService_Name_Call { + return &ChipIngressService_Name_Call{Call: _e.mock.On("Name")} +} + +func (_c *ChipIngressService_Name_Call) Run(run func()) *ChipIngressService_Name_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Name_Call) Return(_a0 string) *ChipIngressService_Name_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Name_Call) RunAndReturn(run func() string) *ChipIngressService_Name_Call { + _c.Call.Return(run) + return _c +} + +// Ready provides a mock function with no fields +func (_m *ChipIngressService) Ready() error { + ret := _m.Called() + + if len(ret) == 0 { + panic("no return value specified for Ready") + } + + var r0 error + if rf, ok := ret.Get(0).(func() error); ok { + r0 = rf() + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Ready_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Ready' +type ChipIngressService_Ready_Call struct { + *mock.Call +} + +// Ready is a helper method to define mock.On call +func (_e *ChipIngressService_Expecter) Ready() *ChipIngressService_Ready_Call { + return &ChipIngressService_Ready_Call{Call: _e.mock.On("Ready")} +} + +func (_c *ChipIngressService_Ready_Call) Run(run func()) *ChipIngressService_Ready_Call { + _c.Call.Run(func(args mock.Arguments) { + run() + }) + return _c +} + +func (_c *ChipIngressService_Ready_Call) Return(_a0 error) *ChipIngressService_Ready_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Ready_Call) RunAndReturn(run func() error) *ChipIngressService_Ready_Call { + _c.Call.Return(run) + return _c +} + +// Send provides a mock function with given fields: ctx, payload +func (_m *ChipIngressService) Send(ctx context.Context, payload synchronization.TelemPayload) { + _m.Called(ctx, payload) +} + +// ChipIngressService_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send' +type ChipIngressService_Send_Call struct { + *mock.Call +} + +// Send is a helper method to define mock.On call +// - ctx context.Context +// - payload synchronization.TelemPayload +func (_e *ChipIngressService_Expecter) Send(ctx interface{}, payload interface{}) *ChipIngressService_Send_Call { + return &ChipIngressService_Send_Call{Call: _e.mock.On("Send", ctx, payload)} +} + +func (_c *ChipIngressService_Send_Call) Run(run func(ctx context.Context, payload synchronization.TelemPayload)) *ChipIngressService_Send_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context), args[1].(synchronization.TelemPayload)) + }) + return _c +} + +func (_c *ChipIngressService_Send_Call) Return() *ChipIngressService_Send_Call { + _c.Call.Return() + return _c +} + +func (_c *ChipIngressService_Send_Call) RunAndReturn(run func(context.Context, synchronization.TelemPayload)) *ChipIngressService_Send_Call { + _c.Run(run) + return _c +} + +// Start provides a mock function with given fields: _a0 +func (_m *ChipIngressService) Start(_a0 context.Context) error { + ret := _m.Called(_a0) + + if len(ret) == 0 { + panic("no return value specified for Start") + } + + var r0 error + if rf, ok := ret.Get(0).(func(context.Context) error); ok { + r0 = rf(_a0) + } else { + r0 = ret.Error(0) + } + + return r0 +} + +// ChipIngressService_Start_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Start' +type ChipIngressService_Start_Call struct { + *mock.Call +} + +// Start is a helper method to define mock.On call +// - _a0 context.Context +func (_e *ChipIngressService_Expecter) Start(_a0 interface{}) *ChipIngressService_Start_Call { + return &ChipIngressService_Start_Call{Call: _e.mock.On("Start", _a0)} +} + +func (_c *ChipIngressService_Start_Call) Run(run func(_a0 context.Context)) *ChipIngressService_Start_Call { + _c.Call.Run(func(args mock.Arguments) { + run(args[0].(context.Context)) + }) + return _c +} + +func (_c *ChipIngressService_Start_Call) Return(_a0 error) *ChipIngressService_Start_Call { + _c.Call.Return(_a0) + return _c +} + +func (_c *ChipIngressService_Start_Call) RunAndReturn(run func(context.Context) error) *ChipIngressService_Start_Call { + _c.Call.Return(run) + return _c +} + +// NewChipIngressService creates a new instance of ChipIngressService. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewChipIngressService(t interface { + mock.TestingT + Cleanup(func()) +}) *ChipIngressService { + mock := &ChipIngressService{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/core/services/synchronization/telemetry_ingress_batch_worker.go b/core/services/synchronization/telemetry_ingress_batch_worker.go index 1718941ebf9..d62f9de80d8 100644 --- a/core/services/synchronization/telemetry_ingress_batch_worker.go +++ b/core/services/synchronization/telemetry_ingress_batch_worker.go @@ -94,7 +94,7 @@ func (tw *telemetryIngressBatchWorker) logBufferFullWithExpBackoff(payload Telem count := tw.dropMessageCount.Add(1) TelemetryClientMessagesDropped.WithLabelValues(tw.endpointURL, string(tw.telemType)).Inc() - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if shouldLogCount(count) { tw.lggr.Warnw("telemetry ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) } } diff --git a/core/services/synchronization/telemetry_ingress_client.go b/core/services/synchronization/telemetry_ingress_client.go index fe5c3909760..16d0cf2a3d4 100644 --- a/core/services/synchronization/telemetry_ingress_client.go +++ b/core/services/synchronization/telemetry_ingress_client.go @@ -156,7 +156,7 @@ func (tc *telemetryIngressClient) handleTelemetry() { // etc... func (tc *telemetryIngressClient) logBufferFullWithExpBackoff(payload TelemPayload) { count := tc.dropMessageCount.Add(1) - if count > 0 && (count%100 == 0 || count&(count-1) == 0) { + if shouldLogCount(count) { tc.eng.Warnw("telemetry ingress client buffer full, dropping message", "telemetry", payload.Telemetry, "droppedCount", count) } } diff --git a/core/services/telemetry/chip_ingress.go b/core/services/telemetry/chip_ingress.go index c78a0b8f96c..5f32a3e2d41 100644 --- a/core/services/telemetry/chip_ingress.go +++ b/core/services/telemetry/chip_ingress.go @@ -32,8 +32,6 @@ type ChipIngressAgent struct { lggr logger.Logger TelemType synchronization.TelemetryType // Empty for multitype endpoints - Domain string // Derived from TelemetryType (empty for multitype) - Entity string // Derived from TelemetryType (empty for multitype) } // NewChipIngressAgent creates a new agent for a single telemetry type endpoint. @@ -55,8 +53,8 @@ func NewChipIngressAgent( return nil, fmt.Errorf("failed to get chain details for chainID %s and network %s: %w", chainID, network, err) } - domain, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) - if err != nil { + // Reject unmapped telemetry types up front so the caller gets a NoopAgent. + if _, _, err := synchronization.TelemetryTypeToDomainAndEntity(telemType); err != nil { return nil, fmt.Errorf("failed to map telemetry type to domain/entity: %w", err) } @@ -65,8 +63,6 @@ func NewChipIngressAgent( ChainID: chainID, ContractID: contractID, ChainSelector: details.ChainSelector, - Domain: domain, - Entity: entity, TelemType: telemType, telemService: telemService, lggr: lggr, @@ -99,7 +95,7 @@ func NewChipIngressAgentMultitype( ChainSelector: details.ChainSelector, telemService: telemService, lggr: lggr, - // TelemType, Domain, Entity left empty for multitype + // TelemType left empty for multitype }, nil } @@ -122,8 +118,6 @@ func (a *ChipIngressAgent) SendLog(log []byte) { TelemType: a.TelemType, ContractID: a.ContractID, ChainSelector: a.ChainSelector, - Domain: a.Domain, - Entity: a.Entity, Network: a.Network, } a.telemService.Send(ctx, payload) @@ -137,20 +131,11 @@ func (a *ChipIngressAgent) SendLog(log []byte) { // MonitoringEndpoint.SendLog signature for consistency. A background context is used internally. func (a *ChipIngressAgent) SendTypedLog(telemType synchronization.TelemetryType, log []byte) { ctx := context.Background() - - domain, entity, err := synchronization.TelemetryTypeToDomainAndEntity(telemType) - if err != nil { - a.lggr.Errorw("failed to map telemetry type to domain/entity", "error", err, "telemType", telemType) - return - } - payload := synchronization.TelemPayload{ Telemetry: log, TelemType: telemType, ContractID: a.ContractID, ChainSelector: a.ChainSelector, - Domain: domain, - Entity: entity, Network: a.Network, } a.telemService.Send(ctx, payload) diff --git a/core/services/telemetry/chip_ingress_test.go b/core/services/telemetry/chip_ingress_test.go index c8f492ef0c5..3bce22ffe6d 100644 --- a/core/services/telemetry/chip_ingress_test.go +++ b/core/services/telemetry/chip_ingress_test.go @@ -1,7 +1,6 @@ package telemetry import ( - "context" "testing" "github.com/stretchr/testify/assert" @@ -10,52 +9,15 @@ import ( "github.com/smartcontractkit/chainlink/v2/core/logger" "github.com/smartcontractkit/chainlink/v2/core/services/synchronization" + syncmocks "github.com/smartcontractkit/chainlink/v2/core/services/synchronization/mocks" ) -// MockChipIngressService is a mock implementation of the ChipIngressService interface -type MockChipIngressService struct { - mock.Mock -} - -func (m *MockChipIngressService) Start(ctx context.Context) error { - args := m.Called(ctx) - return args.Error(0) -} - -func (m *MockChipIngressService) Close() error { - args := m.Called() - return args.Error(0) -} - -func (m *MockChipIngressService) Send(ctx context.Context, payload synchronization.TelemPayload) { - m.Called(ctx, payload) -} - -func (m *MockChipIngressService) HealthReport() map[string]error { - args := m.Called() - return args.Get(0).(map[string]error) -} - -func (m *MockChipIngressService) Name() string { - args := m.Called() - return args.String(0) -} - -func (m *MockChipIngressService) Ready() error { - args := m.Called() - return args.Error(0) -} - func TestNewChipIngressAgent(t *testing.T) { t.Run("Success - Ethereum Mainnet", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - chainID := "1" - network := "EVM" contractID := "0x1234" - telemType := synchronization.OCR2Median - adapter, err := NewChipIngressAgent(mockTelemService, network, chainID, contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1", contractID, synchronization.OCR2Median, lggr) require.NoError(t, err) require.NotNil(t, adapter) @@ -64,34 +26,24 @@ func TestNewChipIngressAgent(t *testing.T) { assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "1", adapter.ChainID) assert.Equal(t, contractID, adapter.ContractID) - - // Verify domain and entity were derived correctly - assert.Equal(t, "data-feeds.telemetry.ocr2-median", adapter.Domain) - assert.Equal(t, "offchainreporting2.TelemetryWrapper", adapter.Entity) }) t.Run("Success - Polygon", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "137", "0xabc", synchronization.OCR3CCIPCommit, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "137", "0xabc", synchronization.OCR3CCIPCommit, lggr) require.NoError(t, err) // Verify chain selector was derived correctly (Polygon) assert.Equal(t, uint64(4051577828743386545), adapter.ChainSelector) assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "137", adapter.ChainID) - - // Verify domain and entity for CCIP commit - assert.Equal(t, "ccip.telemetry.ocr3", adapter.Domain) - assert.Equal(t, "offchainreporting3.TelemetryWrapper", adapter.Entity) }) t.Run("Success - Arbitrum One", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "42161", "0xdef", synchronization.OCR2CCIPExec, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "42161", "0xdef", synchronization.OCR2CCIPExec, lggr) require.NoError(t, err) // Verify chain selector was derived correctly (Arbitrum One) @@ -108,30 +60,27 @@ func TestNewChipIngressAgent(t *testing.T) { }) t.Run("Error - invalid chainID", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1234567890123456", "0x1234", synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1234567890123456", "0x1234", synchronization.OCR2Median, lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") }) t.Run("Error - invalid network family", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "INVALID_NETWORK", "1", "0x1234", synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "INVALID_NETWORK", "1", "0x1234", synchronization.OCR2Median, lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") }) t.Run("Error - invalid telemetry type", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", "0x1234", synchronization.TelemetryType("unknown"), lggr) + adapter, err := NewChipIngressAgent(syncmocks.NewChipIngressService(t), "EVM", "1", "0x1234", synchronization.TelemetryType("unknown"), lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to map telemetry type to domain/entity") @@ -140,10 +89,9 @@ func TestNewChipIngressAgent(t *testing.T) { func TestNewChipIngressAgentMultitype(t *testing.T) { t.Run("Success - creates multitype agent", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(syncmocks.NewChipIngressService(t), "EVM", "1", "0x1234", lggr) require.NoError(t, err) require.NotNil(t, adapter) @@ -151,11 +99,7 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { assert.Equal(t, uint64(5009297550715157269), adapter.ChainSelector) assert.Equal(t, "EVM", adapter.Network) assert.Equal(t, "1", adapter.ChainID) - - // Verify TelemType, Domain, Entity are empty for multitype assert.Empty(t, adapter.TelemType) - assert.Empty(t, adapter.Domain) - assert.Empty(t, adapter.Entity) }) t.Run("Error - nil telemetry service", func(t *testing.T) { @@ -168,10 +112,9 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { }) t.Run("Error - invalid chainID", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "invalid", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(syncmocks.NewChipIngressService(t), "EVM", "invalid", "0x1234", lggr) require.Error(t, err) assert.Nil(t, adapter) assert.Contains(t, err.Error(), "failed to get chain details") @@ -180,285 +123,113 @@ func TestNewChipIngressAgentMultitype(t *testing.T) { func TestChipIngressAgent_SendLog(t *testing.T) { t.Run("Success - sends to telemetry service", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x1234567890abcdef" telemType := synchronization.OCR2Median telemetryLog := []byte("test telemetry data") - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "1", contractID, telemType, lggr) require.NoError(t, err) - expectedPayload := synchronization.TelemPayload{ + svc.On("Send", mock.Anything, synchronization.TelemPayload{ Telemetry: telemetryLog, TelemType: telemType, ContractID: contractID, ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() + }).Once() - // Call SendLog adapter.SendLog(telemetryLog) - - // Verify Send was called - mockTelemService.AssertExpectations(t) - }) - - t.Run("SendLog continues even if send fails", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0xabcdef" - telemType := synchronization.OCR3Mercury - telemetryLog := []byte("test data") - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "137", contractID, telemType, lggr) - require.NoError(t, err) - - expectedPayload := synchronization.TelemPayload{ - Telemetry: telemetryLog, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() - - // Call SendLog - should not panic - assert.NotPanics(t, func() { - adapter.SendLog(telemetryLog) - }) - - mockTelemService.AssertExpectations(t) }) t.Run("Multiple SendLog calls", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x999" telemType := synchronization.OCR2CCIPCommit - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "42161", contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "42161", contractID, telemType, lggr) require.NoError(t, err) - // Setup expectations for multiple calls - log1 := []byte("log 1") - log2 := []byte("log 2") - log3 := []byte("log 3") - - payload1 := synchronization.TelemPayload{ - Telemetry: log1, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - payload2 := synchronization.TelemPayload{ - Telemetry: log2, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, - } - payload3 := synchronization.TelemPayload{ - Telemetry: log3, - TelemType: telemType, - ContractID: contractID, - ChainSelector: adapter.ChainSelector, - Domain: adapter.Domain, - Entity: adapter.Entity, - Network: adapter.Network, + logs := [][]byte{[]byte("log 1"), []byte("log 2"), []byte("log 3")} + for _, l := range logs { + svc.On("Send", mock.Anything, synchronization.TelemPayload{ + Telemetry: l, + TelemType: telemType, + ContractID: contractID, + ChainSelector: adapter.ChainSelector, + Network: adapter.Network, + }).Once() } - mockTelemService.On("Send", mock.Anything, payload1).Once() - mockTelemService.On("Send", mock.Anything, payload2).Once() - mockTelemService.On("Send", mock.Anything, payload3).Once() - - // Send multiple logs - adapter.SendLog(log1) - adapter.SendLog(log2) - adapter.SendLog(log3) - - mockTelemService.AssertExpectations(t) + for _, l := range logs { + adapter.SendLog(l) + } }) - t.Run("SendLog on multitype agent logs warning", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + t.Run("SendLog on multitype agent does not send", func(t *testing.T) { + t.Parallel() + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "1", "0x1234", lggr) require.NoError(t, err) - // SendLog should not call Send on multitype agent - assert.NotPanics(t, func() { - adapter.SendLog([]byte("test")) - }) - - // Verify Send was NOT called - mockTelemService.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) + adapter.SendLog([]byte("test")) + svc.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) }) } func TestChipIngressAgent_SendTypedLog(t *testing.T) { t.Run("Success - sends typed log", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) contractID := "0x1234" - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", contractID, lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "1", contractID, lggr) require.NoError(t, err) telemType := synchronization.OCR2Median telemetryLog := []byte("test data") - expectedPayload := synchronization.TelemPayload{ + svc.On("Send", mock.Anything, synchronization.TelemPayload{ Telemetry: telemetryLog, TelemType: telemType, ContractID: contractID, ChainSelector: adapter.ChainSelector, - Domain: "data-feeds.telemetry.ocr2-median", - Entity: "offchainreporting2.TelemetryWrapper", Network: adapter.Network, - } - mockTelemService.On("Send", mock.Anything, expectedPayload).Once() + }).Once() adapter.SendTypedLog(telemType, telemetryLog) - - mockTelemService.AssertExpectations(t) }) t.Run("SendTypedLog with different types", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0x5678" - - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "137", contractID, lggr) - require.NoError(t, err) - - // Send with OCR2Median - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { - return p.TelemType == synchronization.OCR2Median && p.Domain == "data-feeds.telemetry.ocr2-median" - })).Once() - adapter.SendTypedLog(synchronization.OCR2Median, []byte("median data")) - - // Send with OCR3Mercury - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { - return p.TelemType == synchronization.OCR3Mercury && p.Domain == "data-streams.telemetry.ocr3-mercury" - })).Once() - adapter.SendTypedLog(synchronization.OCR3Mercury, []byte("mercury data")) - - mockTelemService.AssertExpectations(t) - }) - - t.Run("SendTypedLog with invalid type logs error", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x1234", lggr) + adapter, err := NewChipIngressAgentMultitype(svc, "EVM", "137", "0x5678", lggr) require.NoError(t, err) - // Should not panic, just log error - assert.NotPanics(t, func() { - adapter.SendTypedLog(synchronization.TelemetryType("invalid"), []byte("test")) - }) - - // Verify Send was NOT called due to invalid type - mockTelemService.AssertNotCalled(t, "Send", mock.Anything, mock.Anything) + for _, telemType := range []synchronization.TelemetryType{synchronization.OCR2Median, synchronization.OCR3Mercury} { + svc.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { + return p.TelemType == telemType + })).Once() + adapter.SendTypedLog(telemType, []byte("data")) + } }) t.Run("SendTypedLog works on single-type agent too", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) + svc := syncmocks.NewChipIngressService(t) lggr := logger.TestLogger(t) - contractID := "0x1234" - // Create single-type agent - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, synchronization.OCR2Median, lggr) + adapter, err := NewChipIngressAgent(svc, "EVM", "1", "0x1234", synchronization.OCR2Median, lggr) require.NoError(t, err) - // SendTypedLog should still work - mockTelemService.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { + svc.On("Send", mock.Anything, mock.MatchedBy(func(p synchronization.TelemPayload) bool { return p.TelemType == synchronization.OCR3Mercury })).Once() adapter.SendTypedLog(synchronization.OCR3Mercury, []byte("different type")) - - mockTelemService.AssertExpectations(t) - }) -} - -func TestChipIngressAgent_ExportedFields(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - contractID := "0x1234567890" - telemType := synchronization.OCR3Mercury - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", contractID, telemType, lggr) - require.NoError(t, err) - - t.Run("ChainSelector", func(t *testing.T) { - assert.Equal(t, uint64(5009297550715157269), adapter.ChainSelector) - }) - - t.Run("Network", func(t *testing.T) { - assert.Equal(t, "EVM", adapter.Network) - }) - - t.Run("ChainID", func(t *testing.T) { - assert.Equal(t, "1", adapter.ChainID) - }) - - t.Run("ContractID", func(t *testing.T) { - assert.Equal(t, contractID, adapter.ContractID) - }) - - t.Run("Domain", func(t *testing.T) { - assert.Equal(t, "data-streams.telemetry.ocr3-mercury", adapter.Domain) - }) - - t.Run("Entity", func(t *testing.T) { - assert.Equal(t, "offchainreporting3.TelemetryWrapper", adapter.Entity) - }) -} - -func TestChipIngressAgent_InterfaceCompliance(t *testing.T) { - t.Run("single-type agent implements MonitoringEndpoint", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - - adapter, err := NewChipIngressAgent(mockTelemService, "EVM", "1", "0x123", synchronization.OCR2Median, lggr) - require.NoError(t, err) - - // Verify it can be assigned to the interface - var _ any = adapter - - // Call the interface method - mockTelemService.On("Send", mock.Anything, mock.Anything) - adapter.SendLog([]byte("test")) - mockTelemService.AssertCalled(t, "Send", mock.Anything, mock.Anything) - }) - - t.Run("multitype agent implements MultitypeMonitoringEndpoint", func(t *testing.T) { - mockTelemService := new(MockChipIngressService) - lggr := logger.TestLogger(t) - - adapter, err := NewChipIngressAgentMultitype(mockTelemService, "EVM", "1", "0x123", lggr) - require.NoError(t, err) - - // Verify it implements MultitypeMonitoringEndpoint - var me MultitypeMonitoringEndpoint = adapter - require.NotNil(t, me) - - // Call the interface method - mockTelemService.On("Send", mock.Anything, mock.Anything) - adapter.SendTypedLog(synchronization.OCR2Median, []byte("test")) - mockTelemService.AssertCalled(t, "Send", mock.Anything, mock.Anything) }) } diff --git a/core/services/telemetry/manager.go b/core/services/telemetry/manager.go index 9b24d46c627..23c47bcf573 100644 --- a/core/services/telemetry/manager.go +++ b/core/services/telemetry/manager.go @@ -1,16 +1,18 @@ package telemetry import ( + "context" "net/url" "strings" "time" "github.com/pkg/errors" + "go.opentelemetry.io/otel/attribute" + otelmetric "go.opentelemetry.io/otel/metric" "github.com/smartcontractkit/libocr/commontypes" "github.com/smartcontractkit/chainlink-common/pkg/beholder" - "github.com/smartcontractkit/chainlink-common/pkg/chipingress" common "github.com/smartcontractkit/chainlink-common/pkg/logger" "github.com/smartcontractkit/chainlink-common/pkg/services" "github.com/smartcontractkit/chainlink/v2/core/config" @@ -26,52 +28,67 @@ type Manager struct { endpoints []*telemetryEndpoint ks keystore.CSA - logging bool - maxBatchSize uint - sendInterval time.Duration - sendTimeout time.Duration - uniConn bool - useBatchSend bool - MonitoringEndpointGenerator MonitoringEndpointGenerator + logging bool + maxBatchSize uint + sendInterval time.Duration + sendTimeout time.Duration + uniConn bool + useBatchSend bool - chipIngressClient chipingress.Client + chipService synchronization.ChipIngressService // nil when chip ingress is disabled + chipIngressEndpoint string // "" when disabled } +// telemetryEndpoint is one [[TelemetryIngress.Endpoints]] entry, keyed by Network and ChainID. type telemetryEndpoint struct { - ChainID string - Network string - URL *url.URL - client synchronization.TelemetryService - PubKey string - chipIngressClient synchronization.ChipIngressService + ChainID string + Network string + URL *url.URL + client synchronization.TelemetryService // WSRPC client; nil in chip mode, where the entry only allowlists the chain + PubKey string } -// NewManager create a new telemetry manager that is responsible for configuring telemetry agents and generating the defined telemetry endpoints and monitoring endpoints -func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, lggr common.Logger) *Manager { - var chipIngressClient chipingress.Client - if cfg.ChipIngressEnabled() { - lggr.Info("ChIP Ingress is enabled for telemetry") - chipIngressClient = beholder.GetClient().Chip +// NewManager creates the telemetry manager, which hands each job a monitoring +// endpoint for its chain and runs the clients behind them. +// +// A non-nil chipService puts the manager in chip mode: every allowlisted chain sends +// through that one shared service. With nil, each endpoint gets its own WSRPC client. +func NewManager(cfg config.TelemetryIngress, csaKeyStore keystore.CSA, chipService synchronization.ChipIngressService, lggr common.Logger) *Manager { + var chipIngressEndpoint string + if chipService != nil { + chipIngressEndpoint = cfg.ChipIngressEndpoint() } m := &Manager{ - bufferSize: cfg.BufferSize(), - ks: csaKeyStore, - logging: cfg.Logging(), - maxBatchSize: cfg.MaxBatchSize(), - sendInterval: cfg.SendInterval(), - sendTimeout: cfg.SendTimeout(), - uniConn: cfg.UniConn(), - useBatchSend: cfg.UseBatchSend(), - chipIngressClient: chipIngressClient, + bufferSize: cfg.BufferSize(), + ks: csaKeyStore, + logging: cfg.Logging(), + maxBatchSize: cfg.MaxBatchSize(), + sendInterval: cfg.SendInterval(), + sendTimeout: cfg.SendTimeout(), + uniConn: cfg.UniConn(), + useBatchSend: cfg.UseBatchSend(), + chipService: chipService, + chipIngressEndpoint: chipIngressEndpoint, } m.Service, m.eng = services.Config{ Name: "TelemetryManager", + Start: func(ctx context.Context) error { + m.recordConfigMetric(ctx) + m.eng.GoTick(services.TickerConfig{}.NewTicker(time.Hour), m.recordConfigMetric) + return nil + }, NewSubServices: func(lggr common.Logger) (subs []services.Service) { + if m.chipService != nil { + subs = append(subs, m.chipService) + } for _, e := range cfg.Endpoints() { - if sub, err := m.newEndpoint(e, lggr, cfg); err != nil { + sub, err := m.newEndpoint(e, lggr, cfg) + if err != nil { lggr.Error(err) - } else { + continue + } + if sub != nil { subs = append(subs, sub) } } @@ -91,9 +108,9 @@ func (m *Manager) GenMonitoringEndpoint(network string, chainID string, contract return &NoopAgent{} } - if m.chipIngressClient != nil { + if m.chipService != nil { lggr := m.eng.Named("chip-ingress") - adapter, err := NewChipIngressAgent(e.chipIngressClient, network, chainID, contractID, telemType, lggr) + adapter, err := NewChipIngressAgent(m.chipService, network, chainID, contractID, telemType, lggr) if err != nil { m.eng.Errorw("failed to create ChIP ingress agent, falling back to noop", "error", err, "network", network, "chainID", chainID, "contractID", contractID, "telemType", telemType) return &NoopAgent{} @@ -116,9 +133,9 @@ func (m *Manager) GenMultitypeMonitoringEndpoint(network string, chainID string, return &NoopAgent{} } - if m.chipIngressClient != nil { + if m.chipService != nil { lggr := m.eng.Named("chip-ingress-multipletype") - adapter, err := NewChipIngressAgentMultitype(e.chipIngressClient, network, chainID, contractID, lggr) + adapter, err := NewChipIngressAgentMultitype(m.chipService, network, chainID, contractID, lggr) if err != nil { m.eng.Errorw("failed to create ChIP ingress multitype agent, falling back to noop", "error", err, "network", network, "chainID", chainID, "contractID", contractID) return &NoopAgent{} @@ -141,56 +158,57 @@ func (m *Manager) newEndpoint(e config.TelemetryIngressEndpoint, lggr common.Log if e.ChainID() == "" { return nil, errors.New("cannot add telemetry endpoint, chainID cannot be empty") } - - if e.URL() == nil { - return nil, errors.New("cannot add telemetry endpoint, URL cannot be empty") + if _, found := m.getEndpoint(e.Network(), e.ChainID()); found { + return nil, errors.Errorf("cannot add telemetry endpoint for network %q and chainID %q, endpoint already exists", e.Network(), e.ChainID()) } - if e.ServerPubKey() == "" { - return nil, errors.New("cannot add telemetry endpoint, ServerPubKey cannot be empty") + te := &telemetryEndpoint{ + Network: strings.ToUpper(e.Network()), + ChainID: strings.ToUpper(e.ChainID()), + URL: e.URL(), + PubKey: e.ServerPubKey(), } - if _, found := m.getEndpoint(e.Network(), e.ChainID()); found { - return nil, errors.Errorf("cannot add telemetry endpoint for network %q and chainID %q, endpoint already exists", e.Network(), e.ChainID()) + if m.chipService != nil { + // All chains share the injected chip service; the entry is only an allowlist, + // so the OTI-only URL and ServerPubKey are not required. + m.endpoints = append(m.endpoints, te) + return nil, nil } - lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) - - if m.chipIngressClient != nil { - lggr.Infof("Using chip-ingress client for network %q chainID %q", e.Network(), e.ChainID()) - // When ChIP ingress is enabled, we create a ChipIngressService - chipIngressService := synchronization.NewChipIngressBatchClient(m.chipIngressClient, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout()) - te := telemetryEndpoint{ - Network: strings.ToUpper(e.Network()), - ChainID: strings.ToUpper(e.ChainID()), - URL: e.URL(), - PubKey: e.ServerPubKey(), - chipIngressClient: chipIngressService, - } - m.endpoints = append(m.endpoints, &te) - return chipIngressService, nil + if te.URL == nil { + return nil, errors.New("cannot add telemetry endpoint, URL cannot be empty") + } + if te.PubKey == "" { + return nil, errors.New("cannot add telemetry endpoint, ServerPubKey cannot be empty") } - // Otherwise use the traditional telemetry service - var tClient synchronization.TelemetryService + lggr = common.Sugared(lggr).Named(e.Network()).Named(e.ChainID()) if m.useBatchSend { - tClient = synchronization.NewTelemetryIngressBatchClient(e.URL(), e.ServerPubKey(), m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) + te.client = synchronization.NewTelemetryIngressBatchClient(te.URL, te.PubKey, m.ks, cfg.Logging(), lggr, cfg.BufferSize(), cfg.MaxBatchSize(), cfg.SendInterval(), cfg.SendTimeout(), cfg.UniConn()) } else { - tClient = synchronization.NewTelemetryIngressClient(e.URL(), e.ServerPubKey(), m.ks, lggr, cfg.BufferSize()) + te.client = synchronization.NewTelemetryIngressClient(te.URL, te.PubKey, m.ks, lggr, cfg.BufferSize()) } - - te := telemetryEndpoint{ - Network: strings.ToUpper(e.Network()), - ChainID: strings.ToUpper(e.ChainID()), - URL: e.URL(), - client: tClient, - PubKey: e.ServerPubKey(), - } - - m.endpoints = append(m.endpoints, &te) + m.endpoints = append(m.endpoints, te) return te.client, nil } +// recordConfigMetric records the telemetry_ingress.config.info gauge, mirroring +// Beholder's "config.info" pattern. OTel caches the instrument by name. +func (m *Manager) recordConfigMetric(ctx context.Context) { + gauge, err := beholder.GetMeter().Int64Gauge("telemetry_ingress.config.info", + otelmetric.WithDescription("Telemetry ingress config info metric"), + otelmetric.WithUnit("{info}")) + if err != nil { + m.eng.Errorw("failed to create telemetry ingress config metric", "err", err) + return + } + gauge.Record(ctx, 1, otelmetric.WithAttributes( + attribute.Bool("chip_ingress_enabled", m.chipService != nil), + attribute.String("chip_ingress_endpoint", m.chipIngressEndpoint), + )) +} + func (m *Manager) getEndpoint(network string, chainID string) (*telemetryEndpoint, bool) { for _, e := range m.endpoints { if e.Network == strings.ToUpper(network) && e.ChainID == strings.ToUpper(chainID) { diff --git a/core/services/telemetry/manager_test.go b/core/services/telemetry/manager_test.go index 143066e0894..2856b15d2f5 100644 --- a/core/services/telemetry/manager_test.go +++ b/core/services/telemetry/manager_test.go @@ -23,7 +23,7 @@ import ( mocks2 "github.com/smartcontractkit/chainlink/v2/core/services/synchronization/mocks" ) -func setupMockConfig(t *testing.T, useBatchSend, chipIngressEnabled bool) *mocks.TelemetryIngress { +func setupMockConfig(t *testing.T, useBatchSend bool) *mocks.TelemetryIngress { tic := mocks.NewTelemetryIngress(t) tic.On("BufferSize").Return(uint(123)) tic.On("Logging").Return(true) @@ -32,13 +32,14 @@ func setupMockConfig(t *testing.T, useBatchSend, chipIngressEnabled bool) *mocks tic.On("SendTimeout").Return(time.Second * 7) tic.On("UniConn").Return(true) tic.On("UseBatchSend").Return(useBatchSend) - tic.On("ChipIngressEnabled").Return(chipIngressEnabled) + // chip-ingress is selected by passing a non-nil ChipIngressService to NewManager. + tic.On("ChipIngressEndpoint").Return("chip-ingress.test:443").Maybe() return tic } func TestManagerAgents(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("network-1") te.On("ChainID").Return("network-1-chainID-1") @@ -51,14 +52,14 @@ func TestManagerAgents(t *testing.T) { ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) require.Equal(t, "*synchronization.telemetryIngressBatchClient", reflect.TypeOf(tm.endpoints[0].client).String()) me := tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) - tic = setupMockConfig(t, false, false) + tic = setupMockConfig(t, false) tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) - tm = NewManager(tic, ks, lggr) + tm = NewManager(tic, ks, nil, lggr) require.Equal(t, "*synchronization.telemetryIngressClient", reflect.TypeOf(tm.endpoints[0].client).String()) me = tm.GenMonitoringEndpoint("network-1", "network-1-chainID-1", "", "") assert.Equal(t, "*telemetry.TypedIngressAgent", reflect.TypeOf(me).String()) @@ -144,7 +145,7 @@ func TestNewManager(t *testing.T) { mockEndpoints = append(mockEndpoints, te) } - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(mockEndpoints) lggr, logObs := logger.TestLoggerObserved(t, zapcore.InfoLevel) @@ -154,7 +155,7 @@ func TestNewManager(t *testing.T) { key := csakey.MustNewV2XXXTestingOnly(big.NewInt(0)) ks.On("GetAll").Return([]csakey.KeyV2{key}, nil) ks.On("Get", key.ID()).Return(key, nil) - m := NewManager(tic, ks, lggr) + m := NewManager(tic, ks, nil, lggr) require.Equal(t, uint(123), m.bufferSize) require.Equal(t, ks, m.ks) @@ -197,13 +198,13 @@ func TestNewManager(t *testing.T) { } func TestCorrectEndpointRouting(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) tic.On("Endpoints").Return(nil) lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) type testEndpoint struct { network string @@ -283,50 +284,46 @@ func TestCorrectEndpointRouting(t *testing.T) { } } -func TestManager_ChipIngressClient(t *testing.T) { - t.Run("disabled chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, false) - tic.On("Endpoints").Return(nil) - - lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) - ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) - assert.Nil(t, tm.chipIngressClient) - }) - - t.Run("enabled chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, true) - tic.On("Endpoints").Return(nil) - - lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) - ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) - assert.NotNil(t, tm.chipIngressClient) - }) -} - func TestManager_ChipIngressEndpoint(t *testing.T) { - t.Run("creates chip ingress endpoint when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) - te := mocks.NewTelemetryIngressEndpoint(t) - te.On("Network").Return("EVM") - te.On("ChainID").Return("1") - te.On("ServerPubKey").Return("some-pubkey") - u, _ := url.Parse("http://chip-ingress.test") - te.On("URL").Return(u) - tic.On("Endpoints").Return([]config.TelemetryIngressEndpoint{te}) + t.Parallel() + t.Run("all endpoints share one chip ingress service, started once", func(t *testing.T) { + t.Parallel() + tic := setupMockConfig(t, true) + endpoints := make([]config.TelemetryIngressEndpoint, 0, 2) + for _, chainID := range []string{"1", "137"} { + // Chip mode uses only Network and ChainID; the WSRPC URL and ServerPubKey may be unset. + te := mocks.NewTelemetryIngressEndpoint(t) + te.On("Network").Return("EVM") + te.On("ChainID").Return(chainID) + te.On("ServerPubKey").Return("") + te.On("URL").Return(nil) + endpoints = append(endpoints, te) + } + tic.On("Endpoints").Return(endpoints) lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) + chipSvc := mocks2.NewChipIngressService(t) + chipSvc.On("Start", mock.Anything).Return(nil).Once() + chipSvc.On("Close").Return(nil).Once() + chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() + chipSvc.On("Send", mock.Anything, mock.Anything).Twice() + + tm := NewManager(tic, ks, chipSvc, lggr) + require.Len(t, tm.endpoints, 2) + for _, e := range tm.endpoints { + assert.Nil(t, e.client, "chip mode must not create per-endpoint WSRPC clients") + } - tm := NewManager(tic, ks, lggr) - require.Len(t, tm.endpoints, 1) - assert.NotNil(t, tm.endpoints[0].chipIngressClient) - assert.Nil(t, tm.endpoints[0].client) + require.NoError(t, tm.Start(t.Context())) + tm.GenMonitoringEndpoint("EVM", "1", "0xa", synchronization.OCR2Median).SendLog([]byte("a")) + tm.GenMonitoringEndpoint("EVM", "137", "0xb", synchronization.OCR2Median).SendLog([]byte("b")) + require.NoError(t, tm.Close()) }) t.Run("creates traditional endpoint when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + t.Parallel() + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -341,16 +338,15 @@ func TestManager_ChipIngressEndpoint(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) require.Len(t, tm.endpoints, 1) - assert.Nil(t, tm.endpoints[0].chipIngressClient) assert.NotNil(t, tm.endpoints[0].client) }) } func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { t.Run("returns chip ingress agent when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -362,13 +358,13 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) t.Run("SendLog sends telemetry through chip ingress", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -379,9 +375,16 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.DebugLevel) ks := keymocks.NewCSA(t) - - tm := NewManager(tic, ks, lggr) - // Start and cleanup the manager to ensure background goroutines are properly stopped + chipSvc := mocks2.NewChipIngressService(t) + chipSvc.On("Start", mock.Anything).Return(nil) + chipSvc.On("Close").Return(nil) + chipSvc.On("Name").Return("ChipIngressBatchClient").Maybe() + var sent synchronization.TelemPayload + chipSvc.On("Send", mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + sent = args.Get(1).(synchronization.TelemPayload) + }).Once() + + tm := NewManager(tic, ks, chipSvc, lggr) require.NoError(t, tm.Start(t.Context())) t.Cleanup(func() { require.NoError(t, tm.Close()) @@ -390,15 +393,18 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) require.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) - // Verify SendLog doesn't panic and telemetry is queued testPayload := []byte("test telemetry payload") - assert.NotPanics(t, func() { - me.SendLog(testPayload) - }) + me.SendLog(testPayload) + + chipSvc.AssertNumberOfCalls(t, "Send", 1) + assert.Equal(t, testPayload, sent.Telemetry) + assert.Equal(t, synchronization.OCR2Median, sent.TelemType) + assert.Equal(t, "0x123", sent.ContractID) + assert.Equal(t, "EVM", sent.Network) }) t.Run("returns noop agent for invalid chain when chip ingress enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("INVALID") te.On("ChainID").Return("999") @@ -410,7 +416,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) // Invalid network/chainID combination that doesn't exist in chain-selectors me := tm.GenMonitoringEndpoint("INVALID", "999", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -430,7 +436,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { }) t.Run("returns traditional agent when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("1") @@ -445,7 +451,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) me := tm.GenMonitoringEndpoint("EVM", "1", "0x123", synchronization.OCR2Median) assert.Equal(t, "*telemetry.TypedIngressAgentBatch", reflect.TypeOf(me).String()) }) @@ -453,7 +459,7 @@ func TestManager_GenMonitoringEndpoint_ChipIngress(t *testing.T) { func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { t.Run("returns chip ingress multitype agent when enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("137") @@ -465,13 +471,13 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, _ := logger.TestLoggerObserved(t, zapcore.InfoLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.ChipIngressAgent", reflect.TypeOf(me).String()) }) t.Run("returns noop agent for invalid chain when chip ingress enabled", func(t *testing.T) { - tic := setupMockConfig(t, true, true) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("INVALID") te.On("ChainID").Return("888") @@ -483,7 +489,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { lggr, obsLogs := logger.TestLoggerObserved(t, zapcore.ErrorLevel) ks := keymocks.NewCSA(t) - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, mocks2.NewChipIngressService(t), lggr) // Invalid network/chainID combination me := tm.GenMultitypeMonitoringEndpoint("INVALID", "888", "0x456") assert.Equal(t, "*telemetry.NoopAgent", reflect.TypeOf(me).String()) @@ -503,7 +509,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { }) t.Run("returns traditional multitype agent when chip ingress disabled", func(t *testing.T) { - tic := setupMockConfig(t, true, false) + tic := setupMockConfig(t, true) te := mocks.NewTelemetryIngressEndpoint(t) te.On("Network").Return("EVM") te.On("ChainID").Return("137") @@ -518,7 +524,7 @@ func TestManager_GenMultitypeMonitoringEndpoint_ChipIngress(t *testing.T) { ks.On("GetAll").Return([]csakey.KeyV2{key}, nil).Maybe() ks.On("Get", key.ID()).Return(key, nil).Maybe() - tm := NewManager(tic, ks, lggr) + tm := NewManager(tic, ks, nil, lggr) me := tm.GenMultitypeMonitoringEndpoint("EVM", "137", "0x456") assert.Equal(t, "*telemetry.MultiIngressAgentBatch", reflect.TypeOf(me).String()) }) diff --git a/core/web/resolver/testdata/config-empty-effective.toml b/core/web/resolver/testdata/config-empty-effective.toml index 09123407bae..b5b062a54a8 100644 --- a/core/web/resolver/testdata/config-empty-effective.toml +++ b/core/web/resolver/testdata/config-empty-effective.toml @@ -44,6 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/core/web/resolver/testdata/config-full.toml b/core/web/resolver/testdata/config-full.toml index 02717a33b29..8790085c495 100644 --- a/core/web/resolver/testdata/config-full.toml +++ b/core/web/resolver/testdata/config-full.toml @@ -44,6 +44,15 @@ SendInterval = '1m0s' SendTimeout = '5s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [[TelemetryIngress.Endpoints]] Network = 'EVM' diff --git a/core/web/resolver/testdata/config-multi-chain-effective.toml b/core/web/resolver/testdata/config-multi-chain-effective.toml index 091d5a84368..6095ce35ddc 100644 --- a/core/web/resolver/testdata/config-multi-chain-effective.toml +++ b/core/web/resolver/testdata/config-multi-chain-effective.toml @@ -44,6 +44,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = true diff --git a/deployment/go.mod b/deployment/go.mod index f9a4249eb9c..7eaa27bf45b 100644 --- a/deployment/go.mod +++ b/deployment/go.mod @@ -431,7 +431,7 @@ require ( github.com/smartcontractkit/chainlink-canton/authentication v1.0.0 // indirect github.com/smartcontractkit/chainlink-canton/contracts/v2 v2.1.0 // indirect github.com/smartcontractkit/chainlink-ccv v0.14.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-feeds v0.1.2-0.20250227211209-7cd000095135 // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect diff --git a/deployment/go.sum b/deployment/go.sum index 9e32bfd7adb..ff4a77feb05 100644 --- a/deployment/go.sum +++ b/deployment/go.sum @@ -1413,8 +1413,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/devenv/go.mod b/devenv/go.mod index fb55bb3b874..b7d08d01b95 100644 --- a/devenv/go.mod +++ b/devenv/go.mod @@ -320,7 +320,7 @@ require ( github.com/shirou/gopsutil/v4 v4.26.5 // indirect github.com/sirupsen/logrus v1.9.4 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-framework/chains v0.0.0-20260724153515-bb6a2de39bcb // indirect github.com/smartcontractkit/chainlink-framework/metrics v0.0.0-20260723212800-b2f21d31b1d2 // indirect github.com/smartcontractkit/chainlink-framework/multinode v0.0.0-20260521164805-26d78d5e1243 // indirect diff --git a/devenv/go.sum b/devenv/go.sum index 9516c1b996b..6abdb766599 100644 --- a/devenv/go.sum +++ b/devenv/go.sum @@ -1004,8 +1004,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20260915135448-9ec0150ecb github.com/smartcontractkit/chainlink-common v0.11.2-0.20260915135448-9ec0150ecb5d/go.mod h1:ks0kniCCg8Siv2XflmwLOq5ib9Ifoh6VHXHwfEWIMQA= github.com/smartcontractkit/chainlink-common/keystore v1.3.0 h1:V05Rp9/dTc4Wyipsk1EYnW3riZyjDVKmhz3769JpnHU= github.com/smartcontractkit/chainlink-common/keystore v1.3.0/go.mod h1:vHV8BGm6TN7jBbMsWxq1Hqm3HbCtYFwzvKS0CCczxG8= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260914143713-424babc506ab h1:NWZzKhUqh56rmOMWMz9urvBm6jKKktdW3GBGFcJjNJE= github.com/smartcontractkit/chainlink-evm v0.3.4-0.20260914143713-424babc506ab/go.mod h1:XionVLmbYKsgR7Xi6hoC4D4c/uxhhBvNny8qgA39D4M= github.com/smartcontractkit/chainlink-evm/gethwrappers v0.0.0-20260512150409-b4068bf735e6 h1:JFo7C3FilwhfwGBLAyj2umbL+P4QxGmVi/b8yt9kqvI= diff --git a/docs/CONFIG.md b/docs/CONFIG.md index 1d23d16a0d8..f0a3f85d7a1 100644 --- a/docs/CONFIG.md +++ b/docs/CONFIG.md @@ -281,6 +281,15 @@ SendInterval = '500ms' # Default SendTimeout = '10s' # Default UseBatchSend = true # Default ChipIngressEnabled = false # Default +ChipIngressEndpoint = '' # Default +ChipIngressInsecureConnection = false # Default +ChipIngressBufferSize = 10000 # Default +ChipIngressMaxBatchSize = 100 # Default +ChipIngressMaxConcurrentSends = 10 # Default +ChipIngressSendInterval = '500ms' # Default +ChipIngressSendTimeout = '10s' # Default +ChipIngressMaxGRPCRequestSize = 10485760 # Default +ChipIngressDrainTimeout = '1s' # Default ``` @@ -300,13 +309,13 @@ Logging toggles verbose logging of the raw telemetry messages being sent. ```toml BufferSize = 100 # Default ``` -BufferSize is the number of telemetry messages to buffer before dropping new ones. +BufferSize is the number of telemetry messages to buffer before dropping new ones. Ignored when ChipIngressEnabled is true. ### MaxBatchSize ```toml MaxBatchSize = 50 # Default ``` -MaxBatchSize is the maximum number of messages to batch into one telemetry request. +MaxBatchSize is the maximum number of messages to batch into one telemetry request. Ignored when ChipIngressEnabled is true. ### SendInterval ```toml @@ -330,7 +339,61 @@ UseBatchSend toggles sending telemetry to the ingress server using the batch cli ```toml ChipIngressEnabled = false # Default ``` -ChipIngressEnabled enables sending telemetry to CHIP Ingress. +ChipIngressEnabled sends all telemetry to ChipIngressEndpoint instead of the WSRPC Endpoints. Endpoints entries are still required and select which chains send telemetry; in this mode only their Network and ChainID are used. + +### ChipIngressEndpoint +```toml +ChipIngressEndpoint = '' # Default +``` +ChipIngressEndpoint is the host:port of the chip-ingress legacy telemetry endpoint. Required when ChipIngressEnabled is true. Separate from Telemetry.ChipIngressEndpoint, which is Beholder's endpoint. + +### ChipIngressInsecureConnection +```toml +ChipIngressInsecureConnection = false # Default +``` +ChipIngressInsecureConnection disables TLS to ChipIngressEndpoint. Only for local/test setups; not allowed for chain.link hosts. + +### ChipIngressBufferSize +```toml +ChipIngressBufferSize = 10000 # Default +``` +ChipIngressBufferSize is the number of telemetry messages buffered for chip-ingress across all jobs; new messages are dropped when it is full. Between 1 and 100000. + +### ChipIngressMaxBatchSize +```toml +ChipIngressMaxBatchSize = 100 # Default +``` +ChipIngressMaxBatchSize is the max number of messages per chip-ingress PublishBatch request. Between 1 and 500. + +### ChipIngressMaxConcurrentSends +```toml +ChipIngressMaxConcurrentSends = 10 # Default +``` +ChipIngressMaxConcurrentSends limits parallel chip-ingress PublishBatch requests. Between 1 and 32. + +### ChipIngressSendInterval +```toml +ChipIngressSendInterval = '500ms' # Default +``` +ChipIngressSendInterval is the max wait before flushing an incomplete chip-ingress batch. Between 50ms and 10s. + +### ChipIngressSendTimeout +```toml +ChipIngressSendTimeout = '10s' # Default +``` +ChipIngressSendTimeout is the per-request timeout for chip-ingress PublishBatch. Between 1s and 1m. + +### ChipIngressMaxGRPCRequestSize +```toml +ChipIngressMaxGRPCRequestSize = 10485760 # Default +``` +ChipIngressMaxGRPCRequestSize is the max serialized PublishBatch request size in bytes; larger batches are split. Between 1 MiB and 10 MiB, and must not exceed the server's receive limit. + +### ChipIngressDrainTimeout +```toml +ChipIngressDrainTimeout = '1s' # Default +``` +ChipIngressDrainTimeout is the max time shutdown waits to publish buffered chip-ingress telemetry. Messages not sent by then count as send errors. With ChipIngressEnabled, must be less than half of ShutdownGracePeriod. ## TelemetryIngress.Endpoints ```toml diff --git a/go.mod b/go.mod index 6a758efbe97..1e83d4dc524 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,6 @@ require ( github.com/buger/jsonparser v1.2.0 github.com/buraksezer/consistent v0.10.0 github.com/cespare/xxhash/v2 v2.3.0 - github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.2 github.com/coreos/go-oidc/v3 v3.11.0 github.com/danielkov/gin-helmet v0.0.0-20171108135313-1387e224435e github.com/deckarep/golang-set/v2 v2.9.0 @@ -84,7 +83,7 @@ require ( github.com/smartcontractkit/chainlink-ccv v0.14.0 github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 github.com/smartcontractkit/chainlink-evm v0.3.4-0.20261005112317-b723176adfe8 github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 @@ -185,6 +184,7 @@ require ( github.com/cenkalti/backoff v2.2.1+incompatible // indirect github.com/cenkalti/backoff/v5 v5.0.3 // indirect github.com/clipperhouse/uax29/v2 v2.7.0 // indirect + github.com/cloudevents/sdk-go/binding/format/protobuf/v2 v2.16.2 // indirect github.com/cloudevents/sdk-go/v2 v2.16.2 // indirect github.com/cloudwego/base64x v0.1.4 // indirect github.com/cloudwego/iasm v0.2.0 // indirect diff --git a/go.sum b/go.sum index 7b21043eb1c..5a1eea10e0b 100644 --- a/go.sum +++ b/go.sum @@ -1108,8 +1108,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/integration-tests/go.mod b/integration-tests/go.mod index d41ea0fd944..be12bac0729 100644 --- a/integration-tests/go.mod +++ b/integration-tests/go.mod @@ -407,7 +407,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/deployment v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.14.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/integration-tests/go.sum b/integration-tests/go.sum index 7761db22357..2aa671a6fd1 100644 --- a/integration-tests/go.sum +++ b/integration-tests/go.sum @@ -1398,8 +1398,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/integration-tests/load/go.mod b/integration-tests/load/go.mod index 0f219c60465..1d31f39dfb5 100644 --- a/integration-tests/load/go.mod +++ b/integration-tests/load/go.mod @@ -478,7 +478,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.14.0 // indirect github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/integration-tests/load/go.sum b/integration-tests/load/go.sum index 59d3ef9b2d2..4dcdfd4d12e 100644 --- a/integration-tests/load/go.sum +++ b/integration-tests/load/go.sum @@ -1638,8 +1638,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 h1:HeQlHO9MrUXTJmxYCNX4xPX14glLAdwIN3igc9Am/g8= diff --git a/system-tests/lib/go.mod b/system-tests/lib/go.mod index 5b0398921f7..c830888eab2 100644 --- a/system-tests/lib/go.mod +++ b/system-tests/lib/go.mod @@ -487,7 +487,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana/gobindings v0.0.0-20260916222901-720a003dab50 // indirect github.com/smartcontractkit/chainlink-ccip/deployment v0.0.0-20261001213322-a2d686f610ca // indirect github.com/smartcontractkit/chainlink-ccv v0.14.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa // indirect github.com/smartcontractkit/chainlink-data-streams v1.1.2-0.20261009064810-68424c714170 // indirect github.com/smartcontractkit/chainlink-evm/contracts/cre/gobindings v0.0.0-20260403151002-2c91155b5501 // indirect diff --git a/system-tests/lib/go.sum b/system-tests/lib/go.sum index 69264131433..49fb0586a06 100644 --- a/system-tests/lib/go.sum +++ b/system-tests/lib/go.sum @@ -1663,8 +1663,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.mod b/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.mod index 7b4f60ceeb6..d8fcf906539 100644 --- a/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.mod +++ b/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.mod @@ -53,7 +53,7 @@ require ( github.com/prometheus/procfs v0.20.1 // indirect github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 // indirect github.com/shopspring/decimal v1.4.0 // indirect - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 // indirect + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd // indirect github.com/smartcontractkit/libocr v0.0.0-20260810200708-618b5bf7f342 // indirect github.com/stretchr/testify v1.12.1 // indirect github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect diff --git a/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.sum b/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.sum index 57713d77a8b..e38fe87b7f4 100644 --- a/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.sum +++ b/system-tests/tests/canaries_sentinels/proof-of-reserve/cron-based/go.sum @@ -102,8 +102,8 @@ github.com/smartcontractkit/chain-selectors v1.0.107 h1:LlQI72DqaFrJXWRmcv78H48M github.com/smartcontractkit/chain-selectors v1.0.107/go.mod h1:qy7whtgG5g+7z0jt0nRyii9bLND9m15NZTzuQPkMZ5w= github.com/smartcontractkit/chainlink-common v0.11.2-0.20260915135448-9ec0150ecb5d h1:4heqJxOTXBHR3425SuXe9EzkZq+CqWOdKe4KWzAoGYA= github.com/smartcontractkit/chainlink-common v0.11.2-0.20260915135448-9ec0150ecb5d/go.mod h1:ks0kniCCg8Siv2XflmwLOq5ib9Ifoh6VHXHwfEWIMQA= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72 h1:uWEwl7i2ryuRVoV4DmIKm6mqYevf1lH/8cQYhw/JXko= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20260724142814-45996a1bcb72/go.mod h1:UYcRMb4dZcoaIPgZJ3hckCySTqtJc9K4Q+tOKErwTq0= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648 h1:WEUMkKQPAgcNMRgES6CBWrRUiII+HKEWQjulKQBSuMA= github.com/smartcontractkit/chainlink-protos/cre/go v0.0.0-20260804191526-b7a850ae7648/go.mod h1:/i8hjTPFdVWHiY+QjeSiVS2Z3GB3WAZznGgXHstC02E= github.com/smartcontractkit/cre-sdk-go v1.5.0 h1:kepW3QDKARrOOHjXwWAZ9j5KLk6bxLzvi6OMrLsFwVo= diff --git a/system-tests/tests/go.mod b/system-tests/tests/go.mod index 6ae0f5a3ecf..baebc22b540 100644 --- a/system-tests/tests/go.mod +++ b/system-tests/tests/go.mod @@ -76,7 +76,7 @@ require ( github.com/smartcontractkit/chainlink-ccip/chains/solana v0.0.0-20260916222901-720a003dab50 github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d - github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 + github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 github.com/smartcontractkit/chainlink-confidential-compute/tests/testhelpers v0.0.0-20260812145307-d77342c53d7d github.com/smartcontractkit/chainlink-deployments-framework v0.123.3 diff --git a/system-tests/tests/go.sum b/system-tests/tests/go.sum index 9aaf57b7ca7..f50c45a3e7b 100644 --- a/system-tests/tests/go.sum +++ b/system-tests/tests/go.sum @@ -1842,8 +1842,8 @@ github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88fa github.com/smartcontractkit/chainlink-common v0.11.2-0.20261014202458-d375ab88face/go.mod h1:QeHe1mlcVzzg+awYAkRfl/8mDLRAhsl18IIOg4MVZec= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d h1:M9xuiIvdiHtim4nlxCVajyC/Mk63tAg9ESWb/biAhNA= github.com/smartcontractkit/chainlink-common/keystore v1.3.1-0.20260903141829-ef07b52a737d/go.mod h1:rI2njmTlyTvh+xhmORmK6qX3+V1ycDAqGJLu8LS8WpU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549 h1:J4vYRJxkBO8Yf//17wScKjYWmS45VZaFWx0zt+4ETEU= -github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261005132539-b7d63d5fd549/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd h1:Ab7BgyC9HhObeAVesSdhorm8biIyLbpulIWZ+JaRTyg= +github.com/smartcontractkit/chainlink-common/pkg/chipingress v0.0.11-0.20261015170356-b866ad79d5fd/go.mod h1:Scyk7O80fde5rJF3Ty3fEE3xiS6j84eWbu8/2gl3pIY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa h1:7yIjZpk1mYYu630yXcGR7N+a+pcAHvn4OHlfhJK8GGY= github.com/smartcontractkit/chainlink-common/x/config v0.0.0-20260928140953-a9b0d3a04daa/go.mod h1:nfT8vdE44iAlhDWDV6PaBWucxZZAcDsgGkZ+D0kHpAI= github.com/smartcontractkit/chainlink-confidential-compute v1.3.0 h1:y64hOzM9H61E4EYRzQDtRwqunDBlKpsZc1SvFEUlSLE= diff --git a/testdata/scripts/config/merge_raw_configs.txtar b/testdata/scripts/config/merge_raw_configs.txtar index 5ddd071f7e1..44a51dab6c9 100644 --- a/testdata/scripts/config/merge_raw_configs.txtar +++ b/testdata/scripts/config/merge_raw_configs.txtar @@ -191,6 +191,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/default.txtar b/testdata/scripts/node/validate/default.txtar index bc833cd589e..2c7328bdcc0 100644 --- a/testdata/scripts/node/validate/default.txtar +++ b/testdata/scripts/node/validate/default.txtar @@ -56,6 +56,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/defaults-override.txtar b/testdata/scripts/node/validate/defaults-override.txtar index be8ac8dbbae..1587cf083d0 100644 --- a/testdata/scripts/node/validate/defaults-override.txtar +++ b/testdata/scripts/node/validate/defaults-override.txtar @@ -117,6 +117,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar index 44caccc50fe..a6541a4ea65 100644 --- a/testdata/scripts/node/validate/disk-based-logging-disabled.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-disabled.txtar @@ -100,6 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar index c0cb1a621a9..0441125ac87 100644 --- a/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar +++ b/testdata/scripts/node/validate/disk-based-logging-no-dir.txtar @@ -100,6 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/disk-based-logging.txtar b/testdata/scripts/node/validate/disk-based-logging.txtar index e8fa6a2935a..866b1f0e316 100644 --- a/testdata/scripts/node/validate/disk-based-logging.txtar +++ b/testdata/scripts/node/validate/disk-based-logging.txtar @@ -100,6 +100,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/fallback-override.txtar b/testdata/scripts/node/validate/fallback-override.txtar index 5b65e5ab396..7381291760a 100644 --- a/testdata/scripts/node/validate/fallback-override.txtar +++ b/testdata/scripts/node/validate/fallback-override.txtar @@ -199,6 +199,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar index f889bed5b78..3f0ff08261e 100644 --- a/testdata/scripts/node/validate/invalid-ocr-p2p.txtar +++ b/testdata/scripts/node/validate/invalid-ocr-p2p.txtar @@ -85,6 +85,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/invalid.txtar b/testdata/scripts/node/validate/invalid.txtar index 1bf7371ec5b..6cd3b311abc 100644 --- a/testdata/scripts/node/validate/invalid.txtar +++ b/testdata/scripts/node/validate/invalid.txtar @@ -96,6 +96,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/valid.txtar b/testdata/scripts/node/validate/valid.txtar index fcb5fb210ce..6e887c3cb99 100644 --- a/testdata/scripts/node/validate/valid.txtar +++ b/testdata/scripts/node/validate/valid.txtar @@ -97,6 +97,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false diff --git a/testdata/scripts/node/validate/warnings.txtar b/testdata/scripts/node/validate/warnings.txtar index 6484b803498..5b382405901 100644 --- a/testdata/scripts/node/validate/warnings.txtar +++ b/testdata/scripts/node/validate/warnings.txtar @@ -79,6 +79,15 @@ SendInterval = '500ms' SendTimeout = '10s' UseBatchSend = true ChipIngressEnabled = false +ChipIngressEndpoint = '' +ChipIngressInsecureConnection = false +ChipIngressBufferSize = 10000 +ChipIngressMaxBatchSize = 100 +ChipIngressMaxConcurrentSends = 10 +ChipIngressSendInterval = '500ms' +ChipIngressSendTimeout = '10s' +ChipIngressMaxGRPCRequestSize = 10485760 +ChipIngressDrainTimeout = '1s' [AuditLogger] Enabled = false