From 7e59d719538e199b0d398746e0efbd180dab6737 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 25 Mar 2026 16:03:11 +0100 Subject: [PATCH 01/28] deterministic state sync --- consensus/bor/bor.go | 35 ++- consensus/bor/bor_test.go | 206 +++++++++++++++++- consensus/bor/heimdall.go | 2 + consensus/bor/heimdall/client.go | 97 +++++++++ consensus/bor/heimdall/failover_client.go | 14 ++ .../bor/heimdall/failover_client_test.go | 10 + consensus/bor/heimdall/metrics.go | 16 ++ consensus/bor/heimdall/state_sync_url_test.go | 82 +++++++ consensus/bor/heimdallapp/state_sync.go | 42 +++- consensus/bor/heimdallgrpc/state_sync.go | 91 ++++++++ consensus/bor/span_store_test.go | 32 +++ eth/ethconfig/config_test.go | 6 + eth/handler_bor_test.go | 7 + go.mod | 1 + params/config.go | 32 +-- params/config_test.go | 44 ++++ tests/bor/mocks/IHeimdallClient.go | 30 +++ 17 files changed, 726 insertions(+), 21 deletions(-) create mode 100644 consensus/bor/heimdall/state_sync_url_test.go diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 2b0ca4fcb7..d520fa39a7 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1760,12 +1760,37 @@ func (c *Bor) CommitStates( // Wait for heimdall to be synced before fetching state sync events c.spanStore.waitUntilHeimdallIsSynced(c.ctx) - eventRecords, err = c.HeimdallClient.StateSyncEvents(c.ctx, from, to.Unix()) - if err != nil { - log.Error("Error occurred when fetching state sync events", "fromID", from, "to", to.Unix(), "err", err) + queryCtx := c.ctx + + if c.config.IsDeterministicStateSync(header.Number) { + heimdallHeight, err := c.HeimdallClient.GetBlockHeightByTime(queryCtx, to.Unix()) + if err != nil { + log.Error("Failed to get Heimdall height for deterministic state sync", "to", to.Unix(), "err", err) + // TODO marcello double check: if we silently return “no state syncs” here, + // different validators could end up deriving different state sync sets from different Heimdall views, + // which is what we are trying to solve + return nil, fmt.Errorf("deterministic state sync: failed to resolve Heimdall height: %w", err) + } + + log.Info("Using deterministic state sync", "cutoff", to.Unix(), "heimdallHeight", heimdallHeight) - stateSyncs := make([]*types.StateSyncData, 0) - return stateSyncs, nil + eventRecords, err = c.HeimdallClient.StateSyncEventsAtHeight(queryCtx, from, to.Unix(), heimdallHeight) + if err != nil { + log.Error("Failed to fetch state sync events at height", "fromID", from, "to", to.Unix(), "heimdallHeight", heimdallHeight, "err", err) + // TODO marcello double check: if we silently return “no state syncs” here, + // different validators could end up deriving different state sync sets from different Heimdall views, + // which is what we are trying to solve + return nil, fmt.Errorf("deterministic state sync: failed to fetch events at height %d: %w", heimdallHeight, err) + } + } else { + eventRecords, err = c.HeimdallClient.StateSyncEvents(queryCtx, from, to.Unix()) + if err != nil { + // Pre-fork: preserve existing behavior (returning empty, no error) + log.Error("Error occurred when fetching state sync events", "fromID", from, "to", to.Unix(), "err", err) + + stateSyncs := make([]*types.StateSyncData, 0) + return stateSyncs, nil + } } // This if statement checks if there are any state sync record overrides configured for the current block number. diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index 4db081af2c..b5bc835f1e 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -119,6 +119,12 @@ func (f *failingHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, func (f *failingHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return nil, errors.New("fetch status failed") } +func (f *failingHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) { + return 0, errors.New("get block height by time failed") +} +func (f *failingHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, errors.New("state sync events at height failed") +} // newStateDBForTest creates a fresh state database for testing. func newStateDBForTest(t *testing.T, root common.Hash) *state.StateDB { @@ -2974,6 +2980,12 @@ func (m *mockHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, er func (m *mockHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (m *mockHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) { + return 0, nil +} +func (m *mockHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestEncodeSigHeader_WithBaseFee(t *testing.T) { t.Parallel() h := &types.Header{ @@ -4969,7 +4981,6 @@ func TestVerifyHeaderRejectsInvalidBlockNumber(t *testing.T) { t.Fatalf("expected ErrInvalidNumber for overflow, got %v", err) } } - // giuglianoBorConfig returns a BorConfig with Giugliano enabled at genesis. func giuglianoBorConfig() *params.BorConfig { return ¶ms.BorConfig{ @@ -5259,3 +5270,196 @@ func TestVerifyHeader_PreGiugliano_NoCheck(t *testing.T) { require.NotErrorIs(t, err, errMissingGiuglianoFields) } } + +// trackingHeimdallClient records which IHeimdallClient methods were called. +// It returns configurable results and tracks call counts for assertions. +type trackingHeimdallClient struct { + // Call counters + stateSyncEventsCalled int + getBlockHeightByTimeCalled int + stateSyncEventsAtHeightCalled int + + // Configurable return values + blockHeight int64 + blockHeightErr error + events []*clerk.EventRecordWithTime + eventsErr error + eventsAtHeight []*clerk.EventRecordWithTime + eventsAtHeightErr error +} + +func (t *trackingHeimdallClient) Close() {} +func (t *trackingHeimdallClient) StateSyncEvents(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + t.stateSyncEventsCalled++ + return t.events, t.eventsErr +} +func (t *trackingHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + t.stateSyncEventsAtHeightCalled++ + return t.eventsAtHeight, t.eventsAtHeightErr +} +func (t *trackingHeimdallClient) GetSpan(context.Context, uint64) (*borTypes.Span, error) { + return nil, nil +} +func (t *trackingHeimdallClient) GetLatestSpan(context.Context) (*borTypes.Span, error) { + return nil, nil +} +func (t *trackingHeimdallClient) FetchCheckpoint(context.Context, int64) (*checkpoint.Checkpoint, error) { + return nil, nil +} +func (t *trackingHeimdallClient) FetchCheckpointCount(context.Context) (int64, error) { + return 0, nil +} +func (t *trackingHeimdallClient) FetchMilestone(context.Context) (*milestone.Milestone, error) { + return nil, nil +} +func (t *trackingHeimdallClient) FetchMilestoneCount(context.Context) (int64, error) { + return 0, nil +} +func (t *trackingHeimdallClient) FetchStatus(context.Context) (*ctypes.SyncInfo, error) { + return &ctypes.SyncInfo{CatchingUp: false}, nil +} +func (t *trackingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + t.getBlockHeightByTimeCalled++ + return t.blockHeight, t.blockHeightErr +} + +// deterministicBorConfig returns a BorConfig with DeterministicStateSyncBlock set. +func deterministicBorConfig(forkBlock int64) *params.BorConfig { + return ¶ms.BorConfig{ + Sprint: map[string]uint64{"0": 16}, + Period: map[string]uint64{"0": 2}, + IndoreBlock: big.NewInt(0), + StateSyncConfirmationDelay: map[string]uint64{"0": 0}, + RioBlock: big.NewInt(1000000), + DeterministicStateSyncBlock: big.NewInt(forkBlock), + } +} + +func TestCommitStates_DeterministicForkSwitch(t *testing.T) { + t.Parallel() + + addr1 := common.HexToAddress("0x1") + sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}} + mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0} + + // Fork activates at block 100 + borCfg := deterministicBorConfig(100) + genesisTime := uint64(time.Now().Unix()) - 200 + chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime) + b.GenesisContractsClient = mockGC + + genesis := chain.HeaderChain().GetHeaderByNumber(0) + now := time.Now() + + // Pre-fork: block 16 should use StateSyncEvents (old legacy path) + tracker := &trackingHeimdallClient{ + events: []*clerk.EventRecordWithTime{}, + } + b.SetHeimdallClient(tracker) + + stateDb := newStateDBForTest(t, genesis.Root) + h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} + + _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + require.NoError(t, err) + require.Equal(t, 1, tracker.stateSyncEventsCalled, "pre-fork should call StateSyncEvents") + require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, "pre-fork should not call GetBlockHeightByTime") + require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, "pre-fork should not call StateSyncEventsAtHeight") + + // Post-fork: block 112 should use GetBlockHeightByTime + StateSyncEventsAtHeight (deterministi state sync) + tracker2 := &trackingHeimdallClient{ + blockHeight: 500, + eventsAtHeight: []*clerk.EventRecordWithTime{}, + } + b.SetHeimdallClient(tracker2) + + stateDb2 := newStateDBForTest(t, genesis.Root) + h2 := &types.Header{Number: big.NewInt(112), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} + + _, err = b.CommitStates(stateDb2, h2, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + require.NoError(t, err) + require.Equal(t, 0, tracker2.stateSyncEventsCalled, "post-fork should not call StateSyncEvents") + require.Equal(t, 1, tracker2.getBlockHeightByTimeCalled, "post-fork should call GetBlockHeightByTime") + require.Equal(t, 1, tracker2.stateSyncEventsAtHeightCalled, "post-fork should call StateSyncEventsAtHeight") +} + +func TestCommitStates_FailLoudPostFork(t *testing.T) { + t.Parallel() + + addr1 := common.HexToAddress("0x1") + sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}} + mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0} + + // Fork activates at block 0 so all blocks are post-fork + borCfg := deterministicBorConfig(0) + genesisTime := uint64(time.Now().Unix()) - 200 + chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime) + b.GenesisContractsClient = mockGC + + genesis := chain.HeaderChain().GetHeaderByNumber(0) + now := time.Now() + + // GetBlockHeightByTime returns an error + tracker := &trackingHeimdallClient{ + blockHeightErr: errors.New("heimdall height lookup failed"), + } + b.SetHeimdallClient(tracker) + + stateDb := newStateDBForTest(t, genesis.Root) + h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} + + _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + + // Must return a non-nil error + require.Error(t, err, "post-fork should fail loudly when GetBlockHeightByTime errors") + require.Contains(t, err.Error(), "deterministic state sync") + + // Must not fallback to StateSyncEvents + require.Equal(t, 0, tracker.stateSyncEventsCalled, + "post-fork should NOT fall back to StateSyncEvents on GetBlockHeightByTime error") + require.Equal(t, 1, tracker.getBlockHeightByTimeCalled, + "GetBlockHeightByTime should have been called once") + require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, + "StateSyncEventsAtHeight should NOT be called when GetBlockHeightByTime fails") +} + +func TestCommitStates_FailLoudPostFork_EventsAtHeightError(t *testing.T) { + t.Parallel() + + addr1 := common.HexToAddress("0x1") + sp := &fakeSpanner{vals: []*valset.Validator{{Address: addr1, VotingPower: 1}}} + mockGC := &mockGenesisContractForCommitStatesIndore{lastStateID: 0} + + borCfg := deterministicBorConfig(0) + genesisTime := uint64(time.Now().Unix()) - 200 + chain, b := newChainAndBorForTest(t, sp, borCfg, true, addr1, genesisTime) + b.GenesisContractsClient = mockGC + + genesis := chain.HeaderChain().GetHeaderByNumber(0) + now := time.Now() + + // GetBlockHeightByTime succeeds, but StateSyncEventsAtHeight fails + tracker := &trackingHeimdallClient{ + blockHeight: 500, + eventsAtHeightErr: errors.New("HTTP 503: service unavailable"), + } + b.SetHeimdallClient(tracker) + + stateDb := newStateDBForTest(t, genesis.Root) + h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} + + _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + + // Must return a non-nil error (not silently return empty list) + require.Error(t, err, "post-fork should fail loudly when StateSyncEventsAtHeight errors") + require.Contains(t, err.Error(), "deterministic state sync") + + // Both methods should have been called + require.Equal(t, 1, tracker.getBlockHeightByTimeCalled, + "GetBlockHeightByTime should have been called") + require.Equal(t, 1, tracker.stateSyncEventsAtHeightCalled, + "StateSyncEventsAtHeight should have been called") + // Old path should not have been called as fallback + require.Equal(t, 0, tracker.stateSyncEventsCalled, + "post-fork should not fall back to StateSyncEvents") +} diff --git a/consensus/bor/heimdall.go b/consensus/bor/heimdall.go index 37405e9cee..edf4c9404f 100644 --- a/consensus/bor/heimdall.go +++ b/consensus/bor/heimdall.go @@ -14,6 +14,7 @@ import ( //go:generate mockgen -source=heimdall.go -destination=../../tests/bor/mocks/IHeimdallClient.go -package=mocks type IHeimdallClient interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) + StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) @@ -21,6 +22,7 @@ type IHeimdallClient interface { FetchMilestone(ctx context.Context) (*milestone.Milestone, error) FetchMilestoneCount(ctx context.Context) (int64, error) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) + GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) Close() } diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index d8a4878d83..2aa20cf206 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -95,6 +95,12 @@ const ( fetchLatestSpan = "bor/spans/latest" fetchStatus = "/status" + + fetchBlockHeightByTimePath = "clerk/block-height-by-time" + fetchBlockHeightByTimeFormat = "cutoff_time=%d" + + fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" + fetchStateSyncsAtHeightFormat = "from_id=%d&heimdall_height=%d&to_time=%s&pagination.limit=%d" ) // StateSyncEvents fetches the state sync events from heimdall @@ -265,6 +271,85 @@ func (h *HeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, err return response, nil } +// BlockHeightByTimeResponse is the response from the Heimdall clerk/block-height-by-time endpoint. +type BlockHeightByTimeResponse struct { + Height int64 `json:"height"` +} + +// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff unix timestamp. +func (h *HeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { + heightByTimeURL, err := blockHeightByTimeURL(h.urlString, cutoffTime) + if err != nil { + return 0, err + } + + ctx = WithRequestType(ctx, BlockHeightByTimeRequest) + + response, err := FetchWithRetry[BlockHeightByTimeResponse](ctx, h.client, heightByTimeURL, h.closeCh) + if err != nil { + return 0, err + } + + return response.Height, nil +} + +// RecordListVisibleAtHeightResponse is the response from the Heimdall clerk/state-syncs-at-height endpoint. +type RecordListVisibleAtHeightResponse struct { + EventRecords []clerkTypes.EventRecord `json:"event_records"` +} + +// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, +// using the new query endpoint that queries the latest state with immutable visibility_height indexes. +func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + eventRecords := make([]*clerk.EventRecordWithTime, 0) + + for { + u, err := visibleAtHeightURL(h.urlString, fromID, heimdallHeight, toTime) + if err != nil { + return nil, err + } + + log.Info("Fetching state sync events at height", "queryParams", u.RawQuery) + + ctx = WithRequestType(ctx, StateSyncAtHeightRequest) + + request := &Request{client: h.client, url: u, start: time.Now()} + response, err := Fetch[RecordListVisibleAtHeightResponse](ctx, request) + if err != nil { + return nil, err + } + + for _, e := range response.EventRecords { + if e.Id >= fromID && e.RecordTime.Before(time.Unix(toTime, 0)) { + record := &clerk.EventRecordWithTime{ + EventRecord: clerk.EventRecord{ + ID: e.Id, + ChainID: e.BorChainId, + Contract: common.HexToAddress(e.Contract), + Data: e.Data, + LogIndex: e.LogIndex, + TxHash: common.HexToHash(e.TxHash), + }, + Time: e.RecordTime, + } + eventRecords = append(eventRecords, record) + } + } + + if len(response.EventRecords) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + sort.SliceStable(eventRecords, func(i, j int) bool { + return eventRecords[i].ID < eventRecords[j].ID + }) + + return eventRecords, nil +} + func FetchOnce[T any](ctx context.Context, client http.Client, url *url.URL, closeCh chan struct{}) (*T, error) { request := &Request{client: client, url: url, start: time.Now()} return Fetch[T](ctx, request) @@ -437,6 +522,18 @@ func statusURL(urlString string) (*url.URL, error) { return makeURL(urlString, fetchStatus, "") } +func blockHeightByTimeURL(urlString string, cutoffTime int64) (*url.URL, error) { + queryParams := fmt.Sprintf(fetchBlockHeightByTimeFormat, cutoffTime) + return makeURL(urlString, fetchBlockHeightByTimePath, queryParams) +} + +func visibleAtHeightURL(urlString string, fromID uint64, heimdallHeight int64, toTime int64) (*url.URL, error) { + t := time.Unix(toTime, 0).UTC() + formattedTime := t.Format(time.RFC3339Nano) + queryParams := fmt.Sprintf(fetchStateSyncsAtHeightFormat, fromID, heimdallHeight, formattedTime, stateFetchLimit) + return makeURL(urlString, fetchStateSyncsAtHeightPath, queryParams) +} + func makeURL(urlString, rawPath, rawQuery string) (*url.URL, error) { u, err := url.Parse(urlString) if err != nil { diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 9b20269ff2..98a8699dd0 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -29,6 +29,7 @@ const ( // running into Go's covariant-slice restriction. type Endpoint interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) + StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) @@ -36,6 +37,7 @@ type Endpoint interface { FetchMilestone(ctx context.Context) (*milestone.Milestone, error) FetchMilestoneCount(ctx context.Context) (int64, error) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) + GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) Close() } @@ -109,6 +111,12 @@ func (f *MultiHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64 }) } +func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) { + return c.StateSyncEventsAtHeight(ctx, fromID, toTime, heimdallHeight) + }) +} + func (f *MultiHeimdallClient) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) { return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) (*types.Span, error) { return c.GetSpan(ctx, spanID) @@ -151,6 +159,12 @@ func (f *MultiHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo }) } +func (f *MultiHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { + return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) (int64, error) { + return c.GetBlockHeightByTime(ctx, cutoffTime) + }) +} + func (f *MultiHeimdallClient) Close() { f.probeCancel() // cancel in-flight probes first f.registry.Stop() diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 1ed5740ddd..dbd9282a08 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -122,6 +122,16 @@ func (m *mockHeimdallClient) Close() { } } +func (m *mockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + m.hits.Add(1) + return 0, nil +} + +func (m *mockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + m.hits.Add(1) + return []*clerk.EventRecordWithTime{}, nil +} + // testConnErr is a reusable connection-refused error for tests. var testConnErr = &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} diff --git a/consensus/bor/heimdall/metrics.go b/consensus/bor/heimdall/metrics.go index 7d0e2a65a3..a4ab09151c 100644 --- a/consensus/bor/heimdall/metrics.go +++ b/consensus/bor/heimdall/metrics.go @@ -29,6 +29,8 @@ const ( MilestoneLastNoAckRequest requestType = "milestone-last-no-ack" MilestoneIDRequest requestType = "milestone-id" StatusRequest requestType = "status" + BlockHeightByTimeRequest requestType = "block-height-by-time" + StateSyncAtHeightRequest requestType = "state-sync-at-height" ) func WithRequestType(ctx context.Context, reqType requestType) context.Context { @@ -112,6 +114,20 @@ var ( }, timer: metrics.NewRegisteredTimer("client/requests/milestoneid/duration", nil), }, + BlockHeightByTimeRequest: { + request: map[bool]*metrics.Meter{ + true: metrics.NewRegisteredMeter("client/requests/blockheightbytime/valid", nil), + false: metrics.NewRegisteredMeter("client/requests/blockheightbytime/invalid", nil), + }, + timer: metrics.NewRegisteredTimer("client/requests/blockheightbytime/duration", nil), + }, + StateSyncAtHeightRequest: { + request: map[bool]*metrics.Meter{ + true: metrics.NewRegisteredMeter("client/requests/statesyncatheight/valid", nil), + false: metrics.NewRegisteredMeter("client/requests/statesyncatheight/invalid", nil), + }, + timer: metrics.NewRegisteredTimer("client/requests/statesyncatheight/duration", nil), + }, } ) diff --git a/consensus/bor/heimdall/state_sync_url_test.go b/consensus/bor/heimdall/state_sync_url_test.go new file mode 100644 index 0000000000..c6dfbce71f --- /dev/null +++ b/consensus/bor/heimdall/state_sync_url_test.go @@ -0,0 +1,82 @@ +package heimdall + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/stretchr/testify/require" +) + +func TestStateSyncsAtHeightURL_Format(t *testing.T) { + t.Parallel() + + fromID := uint64(42) + heimdallHeight := int64(9999) + toTime := int64(1700000000) // 2023-11-14T22:13:20Z + + u, err := visibleAtHeightURL("http://bor0", fromID, heimdallHeight, toTime) + require.NoError(t, err) + + // Path must be clerk/state-syncs-at-height + require.True(t, strings.HasSuffix(u.Path, "clerk/state-syncs-at-height"), + "expected path to end with clerk/state-syncs-at-height, got %s", u.Path) + + // to_time must be RFC3339Nano, NOT raw unix seconds + expectedTime := time.Unix(toTime, 0).UTC().Format(time.RFC3339Nano) + require.Contains(t, u.RawQuery, fmt.Sprintf("to_time=%s", expectedTime), + "to_time should be RFC3339Nano formatted") + require.NotContains(t, u.RawQuery, fmt.Sprintf("to_time=%d", toTime), + "to_time should NOT be raw unix seconds") + + // from_id parameter + require.Contains(t, u.RawQuery, fmt.Sprintf("from_id=%d", fromID)) + + // heimdall_height parameter + require.Contains(t, u.RawQuery, fmt.Sprintf("heimdall_height=%d", heimdallHeight)) + + // pagination.limit parameter + require.Contains(t, u.RawQuery, fmt.Sprintf("pagination.limit=%d", stateFetchLimit)) + + // Full URL sanity check + expected := fmt.Sprintf( + "http://bor0/clerk/state-syncs-at-height?from_id=%d&heimdall_height=%d&to_time=%s&pagination.limit=%d", + fromID, heimdallHeight, expectedTime, stateFetchLimit, + ) + require.Equal(t, expected, u.String()) +} + +func TestBlockHeightByTimeURL_Format(t *testing.T) { + t.Parallel() + + cutoffTime := int64(1700000000) + + u, err := blockHeightByTimeURL("http://bor0", cutoffTime) + require.NoError(t, err) + + // Path must be clerk/block-height-by-time + require.True(t, strings.HasSuffix(u.Path, "clerk/block-height-by-time"), + "expected path to end with clerk/block-height-by-time, got %s", u.Path) + + // cutoff_time should be raw unix seconds (integer) + require.Contains(t, u.RawQuery, fmt.Sprintf("cutoff_time=%d", cutoffTime)) + + // Full URL sanity check + expected := fmt.Sprintf("http://bor0/clerk/block-height-by-time?cutoff_time=%d", cutoffTime) + require.Equal(t, expected, u.String()) +} + +func TestStateSyncURL_ToTimeIsRFC3339(t *testing.T) { + t.Parallel() + + fromID := uint64(10) + to := int64(100) + + u, err := stateSyncURL("http://bor0", fromID, to) + require.NoError(t, err) + + expectedTime := time.Unix(to, 0).UTC().Format(time.RFC3339Nano) + require.Contains(t, u.RawQuery, fmt.Sprintf("to_time=%s", expectedTime), + "to_time should be RFC3339Nano formatted") +} diff --git a/consensus/bor/heimdallapp/state_sync.go b/consensus/bor/heimdallapp/state_sync.go index ab8ba4dfab..0df6fae35d 100644 --- a/consensus/bor/heimdallapp/state_sync.go +++ b/consensus/bor/heimdallapp/state_sync.go @@ -4,10 +4,11 @@ import ( "context" "time" + "github.com/0xPolygon/heimdall-v2/x/clerk/keeper" + "github.com/0xPolygon/heimdall-v2/x/clerk/types" + "github.com/cosmos/cosmos-sdk/types/query" "github.com/ethereum/go-ethereum/common" "github.com/ethereum/go-ethereum/consensus/bor/clerk" - - "github.com/0xPolygon/heimdall-v2/x/clerk/types" ) func (h *HeimdallAppClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { @@ -36,6 +37,43 @@ func (h *HeimdallAppClient) StateSyncEvents(ctx context.Context, fromID uint64, return totalRecords, nil } +// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height. +// Uses the clerk query server to apply the same visibility_height filtering as gRPC/HTTP paths. +func (h *HeimdallAppClient) StateSyncEventsAtHeight(_ context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + totalRecords := make([]*clerk.EventRecordWithTime, 0) + + queryServer := keeper.NewQueryServer(&h.hApp.ClerkKeeper) + + for { + req := &types.RecordListVisibleAtHeightRequest{ + FromId: fromID, + HeimdallHeight: heimdallHeight, + ToTime: time.Unix(toTime, 0), + Pagination: query.PageRequest{Limit: stateFetchLimit}, + } + + res, err := queryServer.GetRecordListVisibleAtHeight(h.NewContext(), req) + if err != nil { + return nil, err + } + + totalRecords = append(totalRecords, toEvents(res.EventRecords)...) + + if len(res.EventRecords) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + return totalRecords, nil +} + +// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff unix timestamp. +func (h *HeimdallAppClient) GetBlockHeightByTime(_ context.Context, cutoffTime int64) (int64, error) { + return h.hApp.ClerkKeeper.GetBlockHeightByTime(h.NewContext(), cutoffTime) +} + func toEvents(hdEvents []types.EventRecord) []*clerk.EventRecordWithTime { events := make([]*clerk.EventRecordWithTime, len(hdEvents)) diff --git a/consensus/bor/heimdallgrpc/state_sync.go b/consensus/bor/heimdallgrpc/state_sync.go index fa4098ddd3..13eaae8574 100644 --- a/consensus/bor/heimdallgrpc/state_sync.go +++ b/consensus/bor/heimdallgrpc/state_sync.go @@ -84,3 +84,94 @@ func (h *HeimdallGRPCClient) StateSyncEvents(ctx context.Context, fromID uint64, return eventRecords, nil } + +// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height +// using the native gRPC GetRecordListVisibleAtHeight endpoint. +func (h *HeimdallGRPCClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + log.Info("Fetching state sync events at height (gRPC)", "fromID", fromID, "toTime", toTime, "heimdallHeight", heimdallHeight) + + var err error + + globalCtx, cancel := context.WithTimeout(ctx, stateSyncTotalTimeout) + defer cancel() + + start := time.Now() + ctx = heimdall.WithRequestType(globalCtx, heimdall.StateSyncAtHeightRequest) + + defer func() { + heimdall.SendMetrics(ctx, start, err == nil) + }() + + eventRecords := make([]*clerk.EventRecordWithTime, 0) + + for { + req := &types.RecordListVisibleAtHeightRequest{ + FromId: fromID, + HeimdallHeight: heimdallHeight, + ToTime: time.Unix(toTime, 0), + Pagination: query.PageRequest{Limit: stateFetchLimit}, + } + + var res *types.RecordListVisibleAtHeightResponse + pageCtx, pageCancel := context.WithTimeout(ctx, defaultTimeout) + res, err = h.clerkQueryClient.GetRecordListVisibleAtHeight(pageCtx, req) + pageCancel() + if err != nil { + return nil, err + } + + events := res.GetEventRecords() + + for _, event := range events { + eventRecord := &clerk.EventRecordWithTime{ + EventRecord: clerk.EventRecord{ + ID: event.Id, + Contract: common.HexToAddress(event.Contract), + Data: event.Data, + TxHash: common.HexToHash(event.TxHash), + LogIndex: event.LogIndex, + ChainID: event.BorChainId, + }, + Time: event.RecordTime, + } + eventRecords = append(eventRecords, eventRecord) + } + + if len(events) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + return eventRecords, nil +} + +// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff +// unix timestamp using the native gRPC GetBlockHeightByTime endpoint. +func (h *HeimdallGRPCClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { + log.Info("Fetching block height by time (gRPC)", "cutoffTime", cutoffTime) + + var err error + + start := time.Now() + ctx = heimdall.WithRequestType(ctx, heimdall.BlockHeightByTimeRequest) + + defer func() { + heimdall.SendMetrics(ctx, start, err == nil) + }() + + reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout) + defer cancel() + + req := &types.BlockHeightByTimeRequest{ + CutoffTime: cutoffTime, + } + + res, err := h.clerkQueryClient.GetBlockHeightByTime(reqCtx, req) + if err != nil { + return 0, err + } + + return res.Height, nil +} diff --git a/consensus/bor/span_store_test.go b/consensus/bor/span_store_test.go index d62a0995c3..bbea14b354 100644 --- a/consensus/bor/span_store_test.go +++ b/consensus/bor/span_store_test.go @@ -85,6 +85,13 @@ func (h *MockHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (h *MockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (h *MockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} + func TestSpanStore_SpanById(t *testing.T) { spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337") defer spanStore.Close() @@ -398,6 +405,13 @@ func (h *MockOverlappingHeimdallClient) FetchStatus(ctx context.Context) (*ctype return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (h *MockOverlappingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (h *MockOverlappingHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} + func TestSpanStore_SpanByBlockNumber_OverlappingSpans(t *testing.T) { spanStore := NewSpanStore(&MockOverlappingHeimdallClient{}, nil, "1337") defer spanStore.Close() @@ -958,6 +972,12 @@ func (d *dynamicHeimdallClient) Close() {} func (d *dynamicHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (d *dynamicHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (d *dynamicHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func makeTestSpan(id, start, end uint64, producerAddr string) *types.Span { producer := stakeTypes.Validator{ @@ -1075,6 +1095,12 @@ func (m *MockSyncStatusClient) FetchMilestoneID(ctx context.Context, milestoneID panic("not implemented") } func (m *MockSyncStatusClient) Close() {} +func (m *MockSyncStatusClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (m *MockSyncStatusClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_WaitUntilHeimdallIsSynced(t *testing.T) { t.Run("heimdall already synced", func(t *testing.T) { @@ -1459,6 +1485,12 @@ func (h *TimeoutHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, panic("not implemented") } func (h *TimeoutHeimdallClient) Close() {} +func (h *TimeoutHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (h *TimeoutHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_HeimdallDownTimeout(t *testing.T) { t.Run("heimdallStatus set to nil on FetchStatus error", func(t *testing.T) { diff --git a/eth/ethconfig/config_test.go b/eth/ethconfig/config_test.go index 302a570834..8b971292dc 100644 --- a/eth/ethconfig/config_test.go +++ b/eth/ethconfig/config_test.go @@ -49,6 +49,12 @@ func (m *mockHeimdallClient) FetchMilestoneCount(context.Context) (int64, error) func (m *mockHeimdallClient) FetchStatus(context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (m *mockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (m *mockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} // newTestBorChainConfig creates a minimal Bor chain config for testing func newTestBorChainConfig() *params.ChainConfig { diff --git a/eth/handler_bor_test.go b/eth/handler_bor_test.go index 4ecc83b3c5..82070d70dd 100644 --- a/eth/handler_bor_test.go +++ b/eth/handler_bor_test.go @@ -65,6 +65,13 @@ func (m *mockHeimdall) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error return &ctypes.SyncInfo{CatchingUp: false}, nil } +func (m *mockHeimdall) GetBlockHeightByTime(context.Context, int64) (int64, error) { + return 0, nil +} +func (m *mockHeimdall) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} + func TestFetchWhitelistCheckpointAndMilestone(t *testing.T) { t.Parallel() diff --git a/go.mod b/go.mod index f618297e80..d73837ce34 100644 --- a/go.mod +++ b/go.mod @@ -351,6 +351,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. + github.com/0xPolygon/heimdall-v2 => ../heimdall-v2 // TODO marcello remove when heimdall-v2 is available. github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/params/config.go b/params/config.go index 9624c367af..6dadf6482e 100644 --- a/params/config.go +++ b/params/config.go @@ -939,19 +939,21 @@ type BorConfig struct { TargetBaseFee *uint64 `json:"-"` // Desired base fee in wei; target gas % adjusts around this value. Set via --miner.targetBaseFee BaseFeeBuffer *uint64 `json:"-"` // Buffer in wei; no adjustment when parentBaseFee is within ±buffer of TargetBaseFee. Set via --miner.baseFeeBuffer - JaipurBlock *big.Int `json:"jaipurBlock"` // Jaipur switch block (nil = no fork, 0 = already on jaipur) - DelhiBlock *big.Int `json:"delhiBlock"` // Delhi switch block (nil = no fork, 0 = already on delhi) - IndoreBlock *big.Int `json:"indoreBlock"` // Indore switch block (nil = no fork, 0 = already on indore) - StateSyncConfirmationDelay map[string]uint64 `json:"stateSyncConfirmationDelay"` // StateSync Confirmation Delay, in seconds, to calculate `to` - AhmedabadBlock *big.Int `json:"ahmedabadBlock"` // Ahmedabad switch block (nil = no fork, 0 = already on ahmedabad) - BhilaiBlock *big.Int `json:"bhilaiBlock"` // Bhilai switch block (nil = no fork, 0 = already on bhilai) - RioBlock *big.Int `json:"rioBlock"` // Rio switch block (nil = no fork, 0 = already on rio) - MadhugiriBlock *big.Int `json:"madhugiriBlock"` // Madhugiri switch block (nil = no fork, 0 = already on madhugiri) - MadhugiriProBlock *big.Int `json:"madhugiriProBlock"` // MadhugiriPro switch block (nil = no fork, 0 = already on madhugiriPro) - DandeliBlock *big.Int `json:"dandeliBlock"` // Dandeli switch block (nil = no fork, 0 = already on dandeli) - LisovoBlock *big.Int `json:"lisovoBlock"` // Lisovo switch block (nil = no fork, 0 = already on lisovo) - LisovoProBlock *big.Int `json:"lisovoProBlock"` // LisovoPro switch block (nil = no fork, 0 = already on lisovoPro) - GiuglianoBlock *big.Int `json:"giuglianoBlock"` // Giugliano switch block (nil = no fork, 0 = already on giugliano) + JaipurBlock *big.Int `json:"jaipurBlock"` // Jaipur switch block (nil = no fork, 0 = already on jaipur) + DelhiBlock *big.Int `json:"delhiBlock"` // Delhi switch block (nil = no fork, 0 = already on delhi) + IndoreBlock *big.Int `json:"indoreBlock"` // Indore switch block (nil = no fork, 0 = already on indore) + StateSyncConfirmationDelay map[string]uint64 `json:"stateSyncConfirmationDelay"` // StateSync Confirmation Delay, in seconds, to calculate `to` + AhmedabadBlock *big.Int `json:"ahmedabadBlock"` // Ahmedabad switch block (nil = no fork, 0 = already on ahmedabad) + BhilaiBlock *big.Int `json:"bhilaiBlock"` // Bhilai switch block (nil = no fork, 0 = already on bhilai) + RioBlock *big.Int `json:"rioBlock"` // Rio switch block (nil = no fork, 0 = already on rio) + MadhugiriBlock *big.Int `json:"madhugiriBlock"` // Madhugiri switch block (nil = no fork, 0 = already on madhugiri) + MadhugiriProBlock *big.Int `json:"madhugiriProBlock"` // MadhugiriPro switch block (nil = no fork, 0 = already on madhugiriPro) + DandeliBlock *big.Int `json:"dandeliBlock"` // Dandeli switch block (nil = no fork, 0 = already on dandeli) + LisovoBlock *big.Int `json:"lisovoBlock"` // Lisovo switch block (nil = no fork, 0 = already on lisovo) + LisovoProBlock *big.Int `json:"lisovoProBlock"` // LisovoPro switch block (nil = no fork, 0 = already on lisovoPro) + GiuglianoBlock *big.Int `json:"giuglianoBlock"` // Giugliano switch block (nil = no fork, 0 = already on giugliano) + // TODO marcello define block numbers + DeterministicStateSyncBlock *big.Int `json:"deterministicStateSyncBlock,omitempty"` // DeterministicStateSync switch block (nil = no fork, 0 = already active) } // String implements the stringer interface, returning the consensus engine details. @@ -1027,6 +1029,10 @@ func (c *BorConfig) IsGiugliano(number *big.Int) bool { return isBlockForked(c.GiuglianoBlock, number) } +func (c *BorConfig) IsDeterministicStateSync(number *big.Int) bool { + return isBlockForked(c.DeterministicStateSyncBlock, number) +} + // GetTargetGasPercentage returns the target gas percentage for gas limit calculation. // After Lisovo hard fork, this value can be configured via CLI flags (stored in BorConfig at runtime). // It validates the configured value and falls back to defaults if invalid or nil. diff --git a/params/config_test.go b/params/config_test.go index bdd6b18842..d7e1c77cf0 100644 --- a/params/config_test.go +++ b/params/config_test.go @@ -1329,3 +1329,47 @@ func TestGetDynamicTargetGasPercentage_InvalidMinMax(t *testing.T) { } }) } + +func TestIsDeterministicStateSync(t *testing.T) { + t.Parallel() + + config := &BorConfig{ + DeterministicStateSyncBlock: big.NewInt(100), + } + + t.Run("nil block is not active", func(t *testing.T) { + t.Parallel() + if config.IsDeterministicStateSync(nil) { + t.Error("expected IsDeterministicStateSync(nil) to return false") + } + }) + + t.Run("block before fork is not active", func(t *testing.T) { + t.Parallel() + if config.IsDeterministicStateSync(big.NewInt(99)) { + t.Error("expected IsDeterministicStateSync(99) to return false for fork at 100") + } + }) + + t.Run("block at fork is active", func(t *testing.T) { + t.Parallel() + if !config.IsDeterministicStateSync(big.NewInt(100)) { + t.Error("expected IsDeterministicStateSync(100) to return true for fork at 100") + } + }) + + t.Run("block after fork is active", func(t *testing.T) { + t.Parallel() + if !config.IsDeterministicStateSync(big.NewInt(200)) { + t.Error("expected IsDeterministicStateSync(200) to return true for fork at 100") + } + }) + + t.Run("nil fork block means not active", func(t *testing.T) { + t.Parallel() + noForkConfig := &BorConfig{DeterministicStateSyncBlock: nil} + if noForkConfig.IsDeterministicStateSync(big.NewInt(100)) { + t.Error("expected IsDeterministicStateSync to return false when fork block is nil") + } + }) +} diff --git a/tests/bor/mocks/IHeimdallClient.go b/tests/bor/mocks/IHeimdallClient.go index 3608823bd4..cd5fdf9c63 100644 --- a/tests/bor/mocks/IHeimdallClient.go +++ b/tests/bor/mocks/IHeimdallClient.go @@ -157,6 +157,36 @@ func (mr *MockIHeimdallClientMockRecorder) GetSpan(ctx, spanID interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSpan", reflect.TypeOf((*MockIHeimdallClient)(nil).GetSpan), ctx, spanID) } +// GetBlockHeightByTime mocks base method. +func (m *MockIHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetBlockHeightByTime", ctx, cutoffTime) + ret0, _ := ret[0].(int64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetBlockHeightByTime indicates an expected call of GetBlockHeightByTime. +func (mr *MockIHeimdallClientMockRecorder) GetBlockHeightByTime(ctx, cutoffTime interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockHeightByTime", reflect.TypeOf((*MockIHeimdallClient)(nil).GetBlockHeightByTime), ctx, cutoffTime) +} + +// StateSyncEventsAtHeight mocks base method. +func (m *MockIHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StateSyncEventsAtHeight", ctx, fromID, toTime, heimdallHeight) + ret0, _ := ret[0].([]*clerk.EventRecordWithTime) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StateSyncEventsAtHeight indicates an expected call of StateSyncEventsAtHeight. +func (mr *MockIHeimdallClientMockRecorder) StateSyncEventsAtHeight(ctx, fromID, toTime, heimdallHeight interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEventsAtHeight", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEventsAtHeight), ctx, fromID, toTime, heimdallHeight) +} + // StateSyncEvents mocks base method. func (m *MockIHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { m.ctrl.T.Helper() From fdb0082bb582fed858b87e317fa119b950c67907 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Thu, 26 Mar 2026 09:51:49 +0100 Subject: [PATCH 02/28] temp bump heimdall to committed version for testing purposes --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d73837ce34..6f47da3497 100644 --- a/go.mod +++ b/go.mod @@ -351,7 +351,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. - github.com/0xPolygon/heimdall-v2 => ../heimdall-v2 // TODO marcello remove when heimdall-v2 is available. + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa // POS-3441: deterministic state sync proto types github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index 07b3445317..1c61c7294e 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.6.0 h1:rA8RISMnns1w08PxTLvDBS5WiaTOFHJGSrhDWDJLtHc= -github.com/0xPolygon/heimdall-v2 v0.6.0/go.mod h1:fVkGiODG6cGLaDyrE3qxIrvz1rbUr4Zdrr3dOm2SPgg= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa h1:BK20c4Vlr/4C44W9n7a4cmxBfl5fdf57izlBxvkXcNk= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= From 3ae74d8aba0aff96b0fd31161fd13c0ff5407c39 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Thu, 26 Mar 2026 12:09:04 +0100 Subject: [PATCH 03/28] fix parsing --- consensus/bor/heimdall/client.go | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index 2aa20cf206..f412d52760 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -11,6 +11,7 @@ import ( "path" "reflect" "sort" + "strconv" "time" "github.com/0xPolygon/heimdall-v2/x/bor/types" @@ -272,8 +273,9 @@ func (h *HeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, err } // BlockHeightByTimeResponse is the response from the Heimdall clerk/block-height-by-time endpoint. +// Note: Cosmos SDK REST gateway serializes int64 fields as JSON strings. type BlockHeightByTimeResponse struct { - Height int64 `json:"height"` + Height string `json:"height"` } // GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff unix timestamp. @@ -290,7 +292,12 @@ func (h *HeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime in return 0, err } - return response.Height, nil + height, err := strconv.ParseInt(response.Height, 10, 64) + if err != nil { + return 0, fmt.Errorf("failed to parse height %q: %w", response.Height, err) + } + + return height, nil } // RecordListVisibleAtHeightResponse is the response from the Heimdall clerk/state-syncs-at-height endpoint. From 405e5e69ea1f4501f821ef208ff6af6438b09df4 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Thu, 26 Mar 2026 12:26:07 +0100 Subject: [PATCH 04/28] fix unmarshalling of RecordListVisibleAtHeightResponse --- consensus/bor/heimdall/client.go | 8 ++++---- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index f412d52760..bea3f5a08c 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -300,10 +300,10 @@ func (h *HeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime in return height, nil } -// RecordListVisibleAtHeightResponse is the response from the Heimdall clerk/state-syncs-at-height endpoint. -type RecordListVisibleAtHeightResponse struct { - EventRecords []clerkTypes.EventRecord `json:"event_records"` -} +// RecordListVisibleAtHeightResponse uses the proto-generated response type from heimdall-v2. +// This handles Cosmos SDK's string-encoded integers correctly via gogoproto JSON unmarshaling. +// Type alias added for readability. +type RecordListVisibleAtHeightResponse = clerkTypes.RecordListVisibleAtHeightResponse // StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, // using the new query endpoint that queries the latest state with immutable visibility_height indexes. diff --git a/go.mod b/go.mod index 6f47da3497..ca83beab4d 100644 --- a/go.mod +++ b/go.mod @@ -351,7 +351,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. - github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa // POS-3441: deterministic state sync proto types + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8 // POS-3441: deterministic state sync proto types github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index 1c61c7294e..e00bd69638 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa h1:BK20c4Vlr/4C44W9n7a4cmxBfl5fdf57izlBxvkXcNk= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326084433-ac4da0b53ffa/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8 h1:7ikiiN0R11iZ4WEdnWsoRDUbPlvzKF4DKSUhxdauY2A= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= From 18935576ba82d9b28144ed54887df13c519b826f Mon Sep 17 00:00:00 2001 From: marcello33 Date: Thu, 26 Mar 2026 17:29:00 +0100 Subject: [PATCH 05/28] change heimdall dep for testing --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ca83beab4d..fadc60dd27 100644 --- a/go.mod +++ b/go.mod @@ -351,7 +351,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. - github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8 // POS-3441: deterministic state sync proto types + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b // POS-3441: deterministic state sync proto types github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index e00bd69638..b121c1e463 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8 h1:7ikiiN0R11iZ4WEdnWsoRDUbPlvzKF4DKSUhxdauY2A= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326111030-b21e4809d0c8/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b h1:nCXkBXNeDQX2cyqiBZYXFVf8s+l0sYCu/XNja0KFKxg= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= From f3446c4faa1e35f028f87c73407039d6f64f7f42 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Fri, 27 Mar 2026 09:59:20 +0100 Subject: [PATCH 06/28] add DeterministicStateSyncBlock to GatherForks --- core/forkid/forkid.go | 1 + 1 file changed, 1 insertion(+) diff --git a/core/forkid/forkid.go b/core/forkid/forkid.go index fc287a71ee..cf03f4e5ad 100644 --- a/core/forkid/forkid.go +++ b/core/forkid/forkid.go @@ -335,6 +335,7 @@ func GatherForks(config *params.ChainConfig, genesisTime uint64) (heightForks [] config.Bor.LisovoBlock, config.Bor.LisovoProBlock, config.Bor.GiuglianoBlock, + config.Bor.DeterministicStateSyncBlock, } { if fork != nil { heightForks = append(heightForks, fork.Uint64()) From 47f48bce0203da8656c4e2eeeba2bbee90401e7b Mon Sep 17 00:00:00 2001 From: marcello33 Date: Tue, 31 Mar 2026 20:52:38 +0200 Subject: [PATCH 07/28] better comment on go.mod --- go.mod | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index fadc60dd27..224ef906f3 100644 --- a/go.mod +++ b/go.mod @@ -351,7 +351,8 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. - github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b // POS-3441: deterministic state sync proto types + // TODO marcello update to final version once released + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon From 13a51b83d9727425c63e2af707e566ea92ad6c35 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Tue, 31 Mar 2026 20:58:54 +0200 Subject: [PATCH 08/28] fix linter --- consensus/bor/bor_test.go | 27 ++++++++++++++------------- consensus/bor/heimdall/metrics.go | 4 ++-- params/config.go | 28 ++++++++++++++-------------- 3 files changed, 30 insertions(+), 29 deletions(-) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index b5bc835f1e..fbc4a9d772 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -4981,6 +4981,7 @@ func TestVerifyHeaderRejectsInvalidBlockNumber(t *testing.T) { t.Fatalf("expected ErrInvalidNumber for overflow, got %v", err) } } + // giuglianoBorConfig returns a BorConfig with Giugliano enabled at genesis. func giuglianoBorConfig() *params.BorConfig { return ¶ms.BorConfig{ @@ -5275,16 +5276,16 @@ func TestVerifyHeader_PreGiugliano_NoCheck(t *testing.T) { // It returns configurable results and tracks call counts for assertions. type trackingHeimdallClient struct { // Call counters - stateSyncEventsCalled int - getBlockHeightByTimeCalled int + stateSyncEventsCalled int + getBlockHeightByTimeCalled int stateSyncEventsAtHeightCalled int // Configurable return values - blockHeight int64 - blockHeightErr error - events []*clerk.EventRecordWithTime - eventsErr error - eventsAtHeight []*clerk.EventRecordWithTime + blockHeight int64 + blockHeightErr error + events []*clerk.EventRecordWithTime + eventsErr error + eventsAtHeight []*clerk.EventRecordWithTime eventsAtHeightErr error } @@ -5326,12 +5327,12 @@ func (t *trackingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (i // deterministicBorConfig returns a BorConfig with DeterministicStateSyncBlock set. func deterministicBorConfig(forkBlock int64) *params.BorConfig { return ¶ms.BorConfig{ - Sprint: map[string]uint64{"0": 16}, - Period: map[string]uint64{"0": 2}, - IndoreBlock: big.NewInt(0), - StateSyncConfirmationDelay: map[string]uint64{"0": 0}, - RioBlock: big.NewInt(1000000), - DeterministicStateSyncBlock: big.NewInt(forkBlock), + Sprint: map[string]uint64{"0": 16}, + Period: map[string]uint64{"0": 2}, + IndoreBlock: big.NewInt(0), + StateSyncConfirmationDelay: map[string]uint64{"0": 0}, + RioBlock: big.NewInt(1000000), + DeterministicStateSyncBlock: big.NewInt(forkBlock), } } diff --git a/consensus/bor/heimdall/metrics.go b/consensus/bor/heimdall/metrics.go index a4ab09151c..8c126aec60 100644 --- a/consensus/bor/heimdall/metrics.go +++ b/consensus/bor/heimdall/metrics.go @@ -29,8 +29,8 @@ const ( MilestoneLastNoAckRequest requestType = "milestone-last-no-ack" MilestoneIDRequest requestType = "milestone-id" StatusRequest requestType = "status" - BlockHeightByTimeRequest requestType = "block-height-by-time" - StateSyncAtHeightRequest requestType = "state-sync-at-height" + BlockHeightByTimeRequest requestType = "block-height-by-time" + StateSyncAtHeightRequest requestType = "state-sync-at-height" ) func WithRequestType(ctx context.Context, reqType requestType) context.Context { diff --git a/params/config.go b/params/config.go index c2be25e237..1bdd4a14bb 100644 --- a/params/config.go +++ b/params/config.go @@ -941,21 +941,21 @@ type BorConfig struct { TargetBaseFee *uint64 `json:"-"` // Desired base fee in wei; target gas % adjusts around this value. Set via --miner.targetBaseFee BaseFeeBuffer *uint64 `json:"-"` // Buffer in wei; no adjustment when parentBaseFee is within ±buffer of TargetBaseFee. Set via --miner.baseFeeBuffer - JaipurBlock *big.Int `json:"jaipurBlock"` // Jaipur switch block (nil = no fork, 0 = already on jaipur) - DelhiBlock *big.Int `json:"delhiBlock"` // Delhi switch block (nil = no fork, 0 = already on delhi) - IndoreBlock *big.Int `json:"indoreBlock"` // Indore switch block (nil = no fork, 0 = already on indore) - StateSyncConfirmationDelay map[string]uint64 `json:"stateSyncConfirmationDelay"` // StateSync Confirmation Delay, in seconds, to calculate `to` - AhmedabadBlock *big.Int `json:"ahmedabadBlock"` // Ahmedabad switch block (nil = no fork, 0 = already on ahmedabad) - BhilaiBlock *big.Int `json:"bhilaiBlock"` // Bhilai switch block (nil = no fork, 0 = already on bhilai) - RioBlock *big.Int `json:"rioBlock"` // Rio switch block (nil = no fork, 0 = already on rio) - MadhugiriBlock *big.Int `json:"madhugiriBlock"` // Madhugiri switch block (nil = no fork, 0 = already on madhugiri) - MadhugiriProBlock *big.Int `json:"madhugiriProBlock"` // MadhugiriPro switch block (nil = no fork, 0 = already on madhugiriPro) - DandeliBlock *big.Int `json:"dandeliBlock"` // Dandeli switch block (nil = no fork, 0 = already on dandeli) - LisovoBlock *big.Int `json:"lisovoBlock"` // Lisovo switch block (nil = no fork, 0 = already on lisovo) - LisovoProBlock *big.Int `json:"lisovoProBlock"` // LisovoPro switch block (nil = no fork, 0 = already on lisovoPro) - GiuglianoBlock *big.Int `json:"giuglianoBlock"` // Giugliano switch block (nil = no fork, 0 = already on giugliano) + JaipurBlock *big.Int `json:"jaipurBlock"` // Jaipur switch block (nil = no fork, 0 = already on jaipur) + DelhiBlock *big.Int `json:"delhiBlock"` // Delhi switch block (nil = no fork, 0 = already on delhi) + IndoreBlock *big.Int `json:"indoreBlock"` // Indore switch block (nil = no fork, 0 = already on indore) + StateSyncConfirmationDelay map[string]uint64 `json:"stateSyncConfirmationDelay"` // StateSync Confirmation Delay, in seconds, to calculate `to` + AhmedabadBlock *big.Int `json:"ahmedabadBlock"` // Ahmedabad switch block (nil = no fork, 0 = already on ahmedabad) + BhilaiBlock *big.Int `json:"bhilaiBlock"` // Bhilai switch block (nil = no fork, 0 = already on bhilai) + RioBlock *big.Int `json:"rioBlock"` // Rio switch block (nil = no fork, 0 = already on rio) + MadhugiriBlock *big.Int `json:"madhugiriBlock"` // Madhugiri switch block (nil = no fork, 0 = already on madhugiri) + MadhugiriProBlock *big.Int `json:"madhugiriProBlock"` // MadhugiriPro switch block (nil = no fork, 0 = already on madhugiriPro) + DandeliBlock *big.Int `json:"dandeliBlock"` // Dandeli switch block (nil = no fork, 0 = already on dandeli) + LisovoBlock *big.Int `json:"lisovoBlock"` // Lisovo switch block (nil = no fork, 0 = already on lisovo) + LisovoProBlock *big.Int `json:"lisovoProBlock"` // LisovoPro switch block (nil = no fork, 0 = already on lisovoPro) + GiuglianoBlock *big.Int `json:"giuglianoBlock"` // Giugliano switch block (nil = no fork, 0 = already on giugliano) // TODO marcello define block numbers - DeterministicStateSyncBlock *big.Int `json:"deterministicStateSyncBlock,omitempty"` // DeterministicStateSync switch block (nil = no fork, 0 = already active) + DeterministicStateSyncBlock *big.Int `json:"deterministicStateSyncBlock,omitempty"` // DeterministicStateSync switch block (nil = no fork, 0 = already active) } // String implements the stringer interface, returning the consensus engine details. From 96d8797766f30ad25600da021dc0b0a05beeef5c Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 11:06:47 +0200 Subject: [PATCH 09/28] address comments --- consensus/bor/bor.go | 18 +++++----------- consensus/bor/bor_test.go | 2 +- consensus/bor/heimdall/client.go | 36 ++++++++++++++++---------------- 3 files changed, 24 insertions(+), 32 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index d520fa39a7..c84ee63d38 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1760,30 +1760,22 @@ func (c *Bor) CommitStates( // Wait for heimdall to be synced before fetching state sync events c.spanStore.waitUntilHeimdallIsSynced(c.ctx) - queryCtx := c.ctx - if c.config.IsDeterministicStateSync(header.Number) { - heimdallHeight, err := c.HeimdallClient.GetBlockHeightByTime(queryCtx, to.Unix()) + heimdallHeight, err := c.HeimdallClient.GetBlockHeightByTime(c.ctx, to.Unix()) if err != nil { - log.Error("Failed to get Heimdall height for deterministic state sync", "to", to.Unix(), "err", err) - // TODO marcello double check: if we silently return “no state syncs” here, - // different validators could end up deriving different state sync sets from different Heimdall views, - // which is what we are trying to solve + // Post-fork: fail hard to preserve determinism across validators return nil, fmt.Errorf("deterministic state sync: failed to resolve Heimdall height: %w", err) } log.Info("Using deterministic state sync", "cutoff", to.Unix(), "heimdallHeight", heimdallHeight) - eventRecords, err = c.HeimdallClient.StateSyncEventsAtHeight(queryCtx, from, to.Unix(), heimdallHeight) + eventRecords, err = c.HeimdallClient.StateSyncEventsAtHeight(c.ctx, from, to.Unix(), heimdallHeight) if err != nil { - log.Error("Failed to fetch state sync events at height", "fromID", from, "to", to.Unix(), "heimdallHeight", heimdallHeight, "err", err) - // TODO marcello double check: if we silently return “no state syncs” here, - // different validators could end up deriving different state sync sets from different Heimdall views, - // which is what we are trying to solve + // Post-fork: fail hard to preserve determinism across validators return nil, fmt.Errorf("deterministic state sync: failed to fetch events at height %d: %w", heimdallHeight, err) } } else { - eventRecords, err = c.HeimdallClient.StateSyncEvents(queryCtx, from, to.Unix()) + eventRecords, err = c.HeimdallClient.StateSyncEvents(c.ctx, from, to.Unix()) if err != nil { // Pre-fork: preserve existing behavior (returning empty, no error) log.Error("Error occurred when fetching state sync events", "fromID", from, "to", to.Unix(), "err", err) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index fbc4a9d772..d16c4d07bb 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -5367,7 +5367,7 @@ func TestCommitStates_DeterministicForkSwitch(t *testing.T) { require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, "pre-fork should not call GetBlockHeightByTime") require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, "pre-fork should not call StateSyncEventsAtHeight") - // Post-fork: block 112 should use GetBlockHeightByTime + StateSyncEventsAtHeight (deterministi state sync) + // Post-fork: block 112 should use GetBlockHeightByTime + StateSyncEventsAtHeight (deterministic state sync) tracker2 := &trackingHeimdallClient{ blockHeight: 500, eventsAtHeight: []*clerk.EventRecordWithTime{}, diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index bea3f5a08c..f6a4b5f3fd 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -101,7 +101,6 @@ const ( fetchBlockHeightByTimeFormat = "cutoff_time=%d" fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" - fetchStateSyncsAtHeightFormat = "from_id=%d&heimdall_height=%d&to_time=%s&pagination.limit=%d" ) // StateSyncEvents fetches the state sync events from heimdall @@ -316,7 +315,7 @@ func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uin return nil, err } - log.Info("Fetching state sync events at height", "queryParams", u.RawQuery) + log.Debug("Fetching state sync events at height", "queryParams", u.RawQuery) ctx = WithRequestType(ctx, StateSyncAtHeightRequest) @@ -327,20 +326,18 @@ func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uin } for _, e := range response.EventRecords { - if e.Id >= fromID && e.RecordTime.Before(time.Unix(toTime, 0)) { - record := &clerk.EventRecordWithTime{ - EventRecord: clerk.EventRecord{ - ID: e.Id, - ChainID: e.BorChainId, - Contract: common.HexToAddress(e.Contract), - Data: e.Data, - LogIndex: e.LogIndex, - TxHash: common.HexToHash(e.TxHash), - }, - Time: e.RecordTime, - } - eventRecords = append(eventRecords, record) + record := &clerk.EventRecordWithTime{ + EventRecord: clerk.EventRecord{ + ID: e.Id, + ChainID: e.BorChainId, + Contract: common.HexToAddress(e.Contract), + Data: e.Data, + LogIndex: e.LogIndex, + TxHash: common.HexToHash(e.TxHash), + }, + Time: e.RecordTime, } + eventRecords = append(eventRecords, record) } if len(response.EventRecords) < stateFetchLimit { @@ -536,9 +533,12 @@ func blockHeightByTimeURL(urlString string, cutoffTime int64) (*url.URL, error) func visibleAtHeightURL(urlString string, fromID uint64, heimdallHeight int64, toTime int64) (*url.URL, error) { t := time.Unix(toTime, 0).UTC() - formattedTime := t.Format(time.RFC3339Nano) - queryParams := fmt.Sprintf(fetchStateSyncsAtHeightFormat, fromID, heimdallHeight, formattedTime, stateFetchLimit) - return makeURL(urlString, fetchStateSyncsAtHeightPath, queryParams) + params := url.Values{} + params.Set("from_id", fmt.Sprintf("%d", fromID)) + params.Set("heimdall_height", fmt.Sprintf("%d", heimdallHeight)) + params.Set("to_time", t.Format(time.RFC3339Nano)) + params.Set("pagination.limit", fmt.Sprintf("%d", stateFetchLimit)) + return makeURL(urlString, fetchStateSyncsAtHeightPath, params.Encode()) } func makeURL(urlString, rawPath, rawQuery string) (*url.URL, error) { From c6f0d5552af027fb1a79d064b5ad9f68b7afaac6 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 11:37:49 +0200 Subject: [PATCH 10/28] address comments --- consensus/bor/heimdall/client.go | 2 +- consensus/bor/heimdall/state_sync_url_test.go | 29 +++++-------------- consensus/bor/heimdallapp/state_sync.go | 5 ++++ consensus/bor/heimdallgrpc/state_sync.go | 5 ++++ 4 files changed, 19 insertions(+), 22 deletions(-) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index f6a4b5f3fd..4eae18cd18 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -100,7 +100,7 @@ const ( fetchBlockHeightByTimePath = "clerk/block-height-by-time" fetchBlockHeightByTimeFormat = "cutoff_time=%d" - fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" + fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" ) // StateSyncEvents fetches the state sync events from heimdall diff --git a/consensus/bor/heimdall/state_sync_url_test.go b/consensus/bor/heimdall/state_sync_url_test.go index c6dfbce71f..80c06493a3 100644 --- a/consensus/bor/heimdall/state_sync_url_test.go +++ b/consensus/bor/heimdall/state_sync_url_test.go @@ -23,28 +23,15 @@ func TestStateSyncsAtHeightURL_Format(t *testing.T) { require.True(t, strings.HasSuffix(u.Path, "clerk/state-syncs-at-height"), "expected path to end with clerk/state-syncs-at-height, got %s", u.Path) - // to_time must be RFC3339Nano, NOT raw unix seconds - expectedTime := time.Unix(toTime, 0).UTC().Format(time.RFC3339Nano) - require.Contains(t, u.RawQuery, fmt.Sprintf("to_time=%s", expectedTime), - "to_time should be RFC3339Nano formatted") - require.NotContains(t, u.RawQuery, fmt.Sprintf("to_time=%d", toTime), - "to_time should NOT be raw unix seconds") - - // from_id parameter - require.Contains(t, u.RawQuery, fmt.Sprintf("from_id=%d", fromID)) - - // heimdall_height parameter - require.Contains(t, u.RawQuery, fmt.Sprintf("heimdall_height=%d", heimdallHeight)) + // Validate individual query parameters using parsed values (not raw string) + q := u.Query() - // pagination.limit parameter - require.Contains(t, u.RawQuery, fmt.Sprintf("pagination.limit=%d", stateFetchLimit)) - - // Full URL sanity check - expected := fmt.Sprintf( - "http://bor0/clerk/state-syncs-at-height?from_id=%d&heimdall_height=%d&to_time=%s&pagination.limit=%d", - fromID, heimdallHeight, expectedTime, stateFetchLimit, - ) - require.Equal(t, expected, u.String()) + expectedTime := time.Unix(toTime, 0).UTC().Format(time.RFC3339Nano) + require.Equal(t, expectedTime, q.Get("to_time"), "to_time should be RFC3339Nano formatted") + require.NotEqual(t, fmt.Sprintf("%d", toTime), q.Get("to_time"), "to_time should NOT be raw unix seconds") + require.Equal(t, fmt.Sprintf("%d", fromID), q.Get("from_id")) + require.Equal(t, fmt.Sprintf("%d", heimdallHeight), q.Get("heimdall_height")) + require.Equal(t, fmt.Sprintf("%d", stateFetchLimit), q.Get("pagination.limit")) } func TestBlockHeightByTimeURL_Format(t *testing.T) { diff --git a/consensus/bor/heimdallapp/state_sync.go b/consensus/bor/heimdallapp/state_sync.go index 0df6fae35d..e7cc994192 100644 --- a/consensus/bor/heimdallapp/state_sync.go +++ b/consensus/bor/heimdallapp/state_sync.go @@ -2,6 +2,7 @@ package heimdallapp import ( "context" + "sort" "time" "github.com/0xPolygon/heimdall-v2/x/clerk/keeper" @@ -66,6 +67,10 @@ func (h *HeimdallAppClient) StateSyncEventsAtHeight(_ context.Context, fromID ui fromID += uint64(stateFetchLimit) } + sort.SliceStable(totalRecords, func(i, j int) bool { + return totalRecords[i].ID < totalRecords[j].ID + }) + return totalRecords, nil } diff --git a/consensus/bor/heimdallgrpc/state_sync.go b/consensus/bor/heimdallgrpc/state_sync.go index 13eaae8574..0726a82360 100644 --- a/consensus/bor/heimdallgrpc/state_sync.go +++ b/consensus/bor/heimdallgrpc/state_sync.go @@ -2,6 +2,7 @@ package heimdallgrpc import ( "context" + "sort" "time" "github.com/cosmos/cosmos-sdk/types/query" @@ -144,6 +145,10 @@ func (h *HeimdallGRPCClient) StateSyncEventsAtHeight(ctx context.Context, fromID fromID += uint64(stateFetchLimit) } + sort.SliceStable(eventRecords, func(i, j int) bool { + return eventRecords[i].ID < eventRecords[j].ID + }) + return eventRecords, nil } From a94f1a97ea1984cfc895a30e216c31bdc8312001 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 12:07:35 +0200 Subject: [PATCH 11/28] remove omitempty --- params/config.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/params/config.go b/params/config.go index 1bdd4a14bb..4f3057a508 100644 --- a/params/config.go +++ b/params/config.go @@ -955,7 +955,7 @@ type BorConfig struct { LisovoProBlock *big.Int `json:"lisovoProBlock"` // LisovoPro switch block (nil = no fork, 0 = already on lisovoPro) GiuglianoBlock *big.Int `json:"giuglianoBlock"` // Giugliano switch block (nil = no fork, 0 = already on giugliano) // TODO marcello define block numbers - DeterministicStateSyncBlock *big.Int `json:"deterministicStateSyncBlock,omitempty"` // DeterministicStateSync switch block (nil = no fork, 0 = already active) + DeterministicStateSyncBlock *big.Int `json:"deterministicStateSyncBlock"` // DeterministicStateSync switch block (nil = no fork, 0 = already active) } // String implements the stringer interface, returning the consensus engine details. From 46e17502f1cf5346beefc63972d45faf9e4247ab Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 15:17:58 +0200 Subject: [PATCH 12/28] update banner --- params/config.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/params/config.go b/params/config.go index 4f3057a508..33572561b6 100644 --- a/params/config.go +++ b/params/config.go @@ -1243,6 +1243,9 @@ func (c *ChainConfig) Description() string { if c.Bor.GiuglianoBlock != nil { banner += fmt.Sprintf(" - Giugliano: #%-8v\n", c.Bor.GiuglianoBlock) } + if c.Bor.DeterministicStateSyncBlock != nil { + banner += fmt.Sprintf(" - Deterministic State Sync: #%-8v\n", c.Bor.DeterministicStateSyncBlock) + } return banner } From 653d8b6d7b451ca8670ef2087bcc04f4231eb816 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 15:21:26 +0200 Subject: [PATCH 13/28] timeout for StateSyncEventsAtHeight --- consensus/bor/heimdall/client.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index 4eae18cd18..9542ebfcb8 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -307,6 +307,9 @@ type RecordListVisibleAtHeightResponse = clerkTypes.RecordListVisibleAtHeightRes // StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, // using the new query endpoint that queries the latest state with immutable visibility_height indexes. func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + eventRecords := make([]*clerk.EventRecordWithTime, 0) for { From bdb52b028438fc1589c3b36e9c7a1db1135b5891 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 16:05:21 +0200 Subject: [PATCH 14/28] address comments --- consensus/bor/heimdall/client.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index 9542ebfcb8..84aaea242b 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -310,6 +310,8 @@ func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uin ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() + ctx = WithRequestType(ctx, StateSyncAtHeightRequest) + eventRecords := make([]*clerk.EventRecordWithTime, 0) for { @@ -320,10 +322,7 @@ func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uin log.Debug("Fetching state sync events at height", "queryParams", u.RawQuery) - ctx = WithRequestType(ctx, StateSyncAtHeightRequest) - - request := &Request{client: h.client, url: u, start: time.Now()} - response, err := Fetch[RecordListVisibleAtHeightResponse](ctx, request) + response, err := FetchWithRetry[RecordListVisibleAtHeightResponse](ctx, h.client, u, h.closeCh) if err != nil { return nil, err } From cd46ca1ae0bbb8b82560027fda076c8780caed6e Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 17:05:42 +0200 Subject: [PATCH 15/28] address comments --- consensus/bor/bor.go | 3 +++ consensus/bor/heimdall/client.go | 3 --- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index c84ee63d38..873f15a20e 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1766,6 +1766,9 @@ func (c *Bor) CommitStates( // Post-fork: fail hard to preserve determinism across validators return nil, fmt.Errorf("deterministic state sync: failed to resolve Heimdall height: %w", err) } + if heimdallHeight <= 0 { + return nil, fmt.Errorf("deterministic state sync: invalid Heimdall height %d for cutoff %d", heimdallHeight, to.Unix()) + } log.Info("Using deterministic state sync", "cutoff", to.Unix(), "heimdallHeight", heimdallHeight) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index 84aaea242b..da53fd9272 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -307,9 +307,6 @@ type RecordListVisibleAtHeightResponse = clerkTypes.RecordListVisibleAtHeightRes // StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, // using the new query endpoint that queries the latest state with immutable visibility_height indexes. func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - ctx = WithRequestType(ctx, StateSyncAtHeightRequest) eventRecords := make([]*clerk.EventRecordWithTime, 0) From d508db413ce7a350bbabc95db30f98890346a9cd Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 17:42:58 +0200 Subject: [PATCH 16/28] added tests --- .../bor/heimdall/failover_client_test.go | 289 +++++++++++++++++- 1 file changed, 283 insertions(+), 6 deletions(-) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index dbd9282a08..92d4419309 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -31,9 +31,11 @@ type mockHeimdallClient struct { fetchCheckpointCntFn func(ctx context.Context) (int64, error) fetchMilestoneFn func(ctx context.Context) (*milestone.Milestone, error) fetchMilestoneCntFn func(ctx context.Context) (int64, error) - fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) - closeFn func() - hits atomic.Int32 + fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) + stateSyncEventsAtHeightFn func(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) + getBlockHeightByTimeFn func(ctx context.Context, cutoffTime int64) (int64, error) + closeFn func() + hits atomic.Int32 } func (m *mockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { @@ -122,13 +124,23 @@ func (m *mockHeimdallClient) Close() { } } -func (m *mockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { +func (m *mockHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { m.hits.Add(1) - return 0, nil + + if m.getBlockHeightByTimeFn != nil { + return m.getBlockHeightByTimeFn(ctx, cutoffTime) + } + + return 100, nil } -func (m *mockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { +func (m *mockHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { m.hits.Add(1) + + if m.stateSyncEventsAtHeightFn != nil { + return m.stateSyncEventsAtHeightFn(ctx, fromID, toTime, heimdallHeight) + } + return []*clerk.EventRecordWithTime{}, nil } @@ -1265,3 +1277,268 @@ func TestRegistry_InformedCascade_RespectsCooldown(t *testing.T) { // Should prefer tertiary (cooled) over primary (uncooled) assert.Equal(t, 2, fc.registry.Active(), "should prefer cooled tertiary over uncooled primary") } + +// --- StateSyncEventsAtHeight failover tests --- + +func TestMultiFailover_StateSyncEventsAtHeight_SwitchOnPrimaryDown(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil + }, + } + + fc := newInstantMulti(primary, secondary) + defer fc.Close() + + events, err := fc.StateSyncEventsAtHeight(context.Background(), 42, 100, 200) + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, uint64(42), events[0].ID) + + assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") + assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been called") +} + +func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnSuccess(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 5 * time.Second + fc.probeTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + fc.ensureHealthRegistry() + time.Sleep(50 * time.Millisecond) + + secondaryBefore := secondary.hits.Load() + + events, err := fc.StateSyncEventsAtHeight(context.Background(), 7, 50, 100) + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, uint64(7), events[0].ID) + assert.Equal(t, secondaryBefore, secondary.hits.Load(), "secondary should not be contacted when primary succeeds") +} + +func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnServiceUnavailable(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, ErrServiceUnavailable + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrServiceUnavailable)) + assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on 503") +} + +func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnShutdownDetected(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, ErrShutdownDetected + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrShutdownDetected)) + assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on shutdown") +} + +func TestMultiFailover_StateSyncEventsAtHeight_ThreeClients_CascadeToTertiary(t *testing.T) { + connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, connErr + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, connErr + }, + } + tertiary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil + }, + } + + fc := newInstantMulti(primary, secondary, tertiary) + defer fc.Close() + + events, err := fc.StateSyncEventsAtHeight(context.Background(), 10, 50, 100) + require.NoError(t, err) + require.Len(t, events, 1) + assert.Equal(t, uint64(10), events[0].ID) + + assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") + assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") + assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") +} + +// --- GetBlockHeightByTime failover tests --- + +func TestMultiFailover_GetBlockHeightByTime_SwitchOnPrimaryDown(t *testing.T) { + primary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 0, &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + }, + } + secondary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, cutoffTime int64) (int64, error) { + return 500, nil + }, + } + + fc := newInstantMulti(primary, secondary) + defer fc.Close() + + height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) + require.NoError(t, err) + assert.Equal(t, int64(500), height) + + assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") + assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been called") +} + +func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnSuccess(t *testing.T) { + primary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 999, nil + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 5 * time.Second + fc.probeTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + fc.ensureHealthRegistry() + time.Sleep(50 * time.Millisecond) + + secondaryBefore := secondary.hits.Load() + + height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) + require.NoError(t, err) + assert.Equal(t, int64(999), height) + assert.Equal(t, secondaryBefore, secondary.hits.Load(), "secondary should not be contacted when primary succeeds") +} + +func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnServiceUnavailable(t *testing.T) { + primary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 0, ErrServiceUnavailable + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrServiceUnavailable)) + assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on 503") +} + +func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnShutdownDetected(t *testing.T) { + primary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 0, ErrShutdownDetected + }, + } + secondary := &mockHeimdallClient{} + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrShutdownDetected)) + assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on shutdown") +} + +func TestMultiFailover_GetBlockHeightByTime_ThreeClients_CascadeToTertiary(t *testing.T) { + connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + + primary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 0, connErr + }, + } + secondary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 0, connErr + }, + } + tertiary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + return 750, nil + }, + } + + fc := newInstantMulti(primary, secondary, tertiary) + defer fc.Close() + + height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) + require.NoError(t, err) + assert.Equal(t, int64(750), height) + + assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") + assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") + assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") +} From 360e53302d0e127949512b86af261f85d71542e9 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Wed, 1 Apr 2026 18:07:08 +0200 Subject: [PATCH 17/28] address minor err shadowing and fix lint --- consensus/bor/bor.go | 3 ++- .../bor/heimdall/failover_client_test.go | 24 +++++++++---------- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 873f15a20e..66f8065606 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1761,7 +1761,8 @@ func (c *Bor) CommitStates( c.spanStore.waitUntilHeimdallIsSynced(c.ctx) if c.config.IsDeterministicStateSync(header.Number) { - heimdallHeight, err := c.HeimdallClient.GetBlockHeightByTime(c.ctx, to.Unix()) + var heimdallHeight int64 + heimdallHeight, err = c.HeimdallClient.GetBlockHeightByTime(c.ctx, to.Unix()) if err != nil { // Post-fork: fail hard to preserve determinism across validators return nil, fmt.Errorf("deterministic state sync: failed to resolve Heimdall height: %w", err) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 92d4419309..999bf3a628 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -24,18 +24,18 @@ import ( // mockHeimdallClient is a configurable mock implementing the Endpoint interface. type mockHeimdallClient struct { - getSpanFn func(ctx context.Context, spanID uint64) (*types.Span, error) - getLatestSpanFn func(ctx context.Context) (*types.Span, error) - stateSyncEventsFn func(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) - fetchCheckpointFn func(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) - fetchCheckpointCntFn func(ctx context.Context) (int64, error) - fetchMilestoneFn func(ctx context.Context) (*milestone.Milestone, error) - fetchMilestoneCntFn func(ctx context.Context) (int64, error) - fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) - stateSyncEventsAtHeightFn func(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) - getBlockHeightByTimeFn func(ctx context.Context, cutoffTime int64) (int64, error) - closeFn func() - hits atomic.Int32 + getSpanFn func(ctx context.Context, spanID uint64) (*types.Span, error) + getLatestSpanFn func(ctx context.Context) (*types.Span, error) + stateSyncEventsFn func(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) + fetchCheckpointFn func(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) + fetchCheckpointCntFn func(ctx context.Context) (int64, error) + fetchMilestoneFn func(ctx context.Context) (*milestone.Milestone, error) + fetchMilestoneCntFn func(ctx context.Context) (int64, error) + fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) + stateSyncEventsAtHeightFn func(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) + getBlockHeightByTimeFn func(ctx context.Context, cutoffTime int64) (int64, error) + closeFn func() + hits atomic.Int32 } func (m *mockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { From c1c1e3d27d86c68872419ce8672e686f21f825e5 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Fri, 3 Apr 2026 09:41:37 +0200 Subject: [PATCH 18/28] test single endpoint --- consensus/bor/bor.go | 22 ++---- consensus/bor/bor_test.go | 94 ++++++++++++++--------- consensus/bor/heimdall.go | 1 + consensus/bor/heimdall/client.go | 63 +++++++++++++++ consensus/bor/heimdall/failover_client.go | 7 ++ consensus/bor/heimdall/metrics.go | 8 ++ consensus/bor/heimdallapp/state_sync.go | 35 +++++++++ consensus/bor/heimdallgrpc/state_sync.go | 65 ++++++++++++++++ consensus/bor/span_store_test.go | 15 ++++ go.mod | 2 +- go.sum | 2 - tests/bor/mocks/IHeimdallClient.go | 15 ++++ 12 files changed, 275 insertions(+), 54 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index 66f8065606..be981c6290 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1761,22 +1761,16 @@ func (c *Bor) CommitStates( c.spanStore.waitUntilHeimdallIsSynced(c.ctx) if c.config.IsDeterministicStateSync(header.Number) { - var heimdallHeight int64 - heimdallHeight, err = c.HeimdallClient.GetBlockHeightByTime(c.ctx, to.Unix()) - if err != nil { - // Post-fork: fail hard to preserve determinism across validators - return nil, fmt.Errorf("deterministic state sync: failed to resolve Heimdall height: %w", err) - } - if heimdallHeight <= 0 { - return nil, fmt.Errorf("deterministic state sync: invalid Heimdall height %d for cutoff %d", heimdallHeight, to.Unix()) - } + log.Info("Using deterministic state sync", "cutoff", to.Unix()) - log.Info("Using deterministic state sync", "cutoff", to.Unix(), "heimdallHeight", heimdallHeight) - - eventRecords, err = c.HeimdallClient.StateSyncEventsAtHeight(c.ctx, from, to.Unix(), heimdallHeight) + eventRecords, err = c.HeimdallClient.StateSyncEventsByTime(c.ctx, from, to.Unix()) if err != nil { - // Post-fork: fail hard to preserve determinism across validators - return nil, fmt.Errorf("deterministic state sync: failed to fetch events at height %d: %w", heimdallHeight, err) + // Match pre-fork resilience: log and return empty on transient errors. + // Determinism is preserved because all validators independently skip + // the same sprint, and events will be picked up in the next sprint. + log.Error("Error fetching deterministic state sync events", "fromID", from, "to", to.Unix(), "err", err) + + return make([]*types.StateSyncData, 0), nil } } else { eventRecords, err = c.HeimdallClient.StateSyncEvents(c.ctx, from, to.Unix()) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index d16c4d07bb..10970ffa06 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -125,6 +125,9 @@ func (f *failingHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) func (f *failingHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, errors.New("state sync events at height failed") } +func (f *failingHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, errors.New("state sync events by time failed") +} // newStateDBForTest creates a fresh state database for testing. func newStateDBForTest(t *testing.T, root common.Hash) *state.StateDB { @@ -2986,6 +2989,9 @@ func (m *mockHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (i func (m *mockHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return m.events, nil +} func TestEncodeSigHeader_WithBaseFee(t *testing.T) { t.Parallel() h := &types.Header{ @@ -5276,17 +5282,20 @@ func TestVerifyHeader_PreGiugliano_NoCheck(t *testing.T) { // It returns configurable results and tracks call counts for assertions. type trackingHeimdallClient struct { // Call counters - stateSyncEventsCalled int - getBlockHeightByTimeCalled int + stateSyncEventsCalled int + getBlockHeightByTimeCalled int stateSyncEventsAtHeightCalled int + stateSyncEventsByTimeCalled int // Configurable return values - blockHeight int64 - blockHeightErr error - events []*clerk.EventRecordWithTime - eventsErr error - eventsAtHeight []*clerk.EventRecordWithTime - eventsAtHeightErr error + blockHeight int64 + blockHeightErr error + events []*clerk.EventRecordWithTime + eventsErr error + eventsAtHeight []*clerk.EventRecordWithTime + eventsAtHeightErr error + eventsByTime []*clerk.EventRecordWithTime + eventsByTimeErr error } func (t *trackingHeimdallClient) Close() {} @@ -5323,6 +5332,10 @@ func (t *trackingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (i t.getBlockHeightByTimeCalled++ return t.blockHeight, t.blockHeightErr } +func (t *trackingHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + t.stateSyncEventsByTimeCalled++ + return t.eventsByTime, t.eventsByTimeErr +} // deterministicBorConfig returns a BorConfig with DeterministicStateSyncBlock set. func deterministicBorConfig(forkBlock int64) *params.BorConfig { @@ -5366,11 +5379,11 @@ func TestCommitStates_DeterministicForkSwitch(t *testing.T) { require.Equal(t, 1, tracker.stateSyncEventsCalled, "pre-fork should call StateSyncEvents") require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, "pre-fork should not call GetBlockHeightByTime") require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, "pre-fork should not call StateSyncEventsAtHeight") + require.Equal(t, 0, tracker.stateSyncEventsByTimeCalled, "pre-fork should not call StateSyncEventsByTime") - // Post-fork: block 112 should use GetBlockHeightByTime + StateSyncEventsAtHeight (deterministic state sync) + // Post-fork: block 112 should use StateSyncEventsByTime (deterministic state sync) tracker2 := &trackingHeimdallClient{ - blockHeight: 500, - eventsAtHeight: []*clerk.EventRecordWithTime{}, + eventsByTime: []*clerk.EventRecordWithTime{}, } b.SetHeimdallClient(tracker2) @@ -5380,11 +5393,12 @@ func TestCommitStates_DeterministicForkSwitch(t *testing.T) { _, err = b.CommitStates(stateDb2, h2, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) require.NoError(t, err) require.Equal(t, 0, tracker2.stateSyncEventsCalled, "post-fork should not call StateSyncEvents") - require.Equal(t, 1, tracker2.getBlockHeightByTimeCalled, "post-fork should call GetBlockHeightByTime") - require.Equal(t, 1, tracker2.stateSyncEventsAtHeightCalled, "post-fork should call StateSyncEventsAtHeight") + require.Equal(t, 1, tracker2.stateSyncEventsByTimeCalled, "post-fork should call StateSyncEventsByTime") + require.Equal(t, 0, tracker2.getBlockHeightByTimeCalled, "post-fork should not call GetBlockHeightByTime") + require.Equal(t, 0, tracker2.stateSyncEventsAtHeightCalled, "post-fork should not call StateSyncEventsAtHeight") } -func TestCommitStates_FailLoudPostFork(t *testing.T) { +func TestCommitStates_ResilientPostFork(t *testing.T) { t.Parallel() addr1 := common.HexToAddress("0x1") @@ -5400,31 +5414,35 @@ func TestCommitStates_FailLoudPostFork(t *testing.T) { genesis := chain.HeaderChain().GetHeaderByNumber(0) now := time.Now() - // GetBlockHeightByTime returns an error + // StateSyncEventsByTime returns an error tracker := &trackingHeimdallClient{ - blockHeightErr: errors.New("heimdall height lookup failed"), + eventsByTimeErr: errors.New("heimdall state sync by time failed"), } b.SetHeimdallClient(tracker) stateDb := newStateDBForTest(t, genesis.Root) h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} - _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + result, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) - // Must return a non-nil error - require.Error(t, err, "post-fork should fail loudly when GetBlockHeightByTime errors") - require.Contains(t, err.Error(), "deterministic state sync") + // Post-fork errors are resilient: log + return empty, no error + require.NoError(t, err, "post-fork should not return error on StateSyncEventsByTime failure") + require.Empty(t, result, "post-fork should return empty on StateSyncEventsByTime failure") + // StateSyncEventsByTime should have been called + require.Equal(t, 1, tracker.stateSyncEventsByTimeCalled, + "StateSyncEventsByTime should have been called once") // Must not fallback to StateSyncEvents require.Equal(t, 0, tracker.stateSyncEventsCalled, - "post-fork should NOT fall back to StateSyncEvents on GetBlockHeightByTime error") - require.Equal(t, 1, tracker.getBlockHeightByTimeCalled, - "GetBlockHeightByTime should have been called once") + "post-fork should NOT fall back to StateSyncEvents on error") + // Old two-call pattern should not be used + require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, + "GetBlockHeightByTime should NOT be called") require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, - "StateSyncEventsAtHeight should NOT be called when GetBlockHeightByTime fails") + "StateSyncEventsAtHeight should NOT be called") } -func TestCommitStates_FailLoudPostFork_EventsAtHeightError(t *testing.T) { +func TestCommitStates_ResilientPostFork_ReturnsEmptyOnError(t *testing.T) { t.Parallel() addr1 := common.HexToAddress("0x1") @@ -5439,28 +5457,30 @@ func TestCommitStates_FailLoudPostFork_EventsAtHeightError(t *testing.T) { genesis := chain.HeaderChain().GetHeaderByNumber(0) now := time.Now() - // GetBlockHeightByTime succeeds, but StateSyncEventsAtHeight fails + // StateSyncEventsByTime fails with an HTTP error tracker := &trackingHeimdallClient{ - blockHeight: 500, - eventsAtHeightErr: errors.New("HTTP 503: service unavailable"), + eventsByTimeErr: errors.New("HTTP 503: service unavailable"), } b.SetHeimdallClient(tracker) stateDb := newStateDBForTest(t, genesis.Root) h := &types.Header{Number: big.NewInt(16), ParentHash: genesis.Hash(), Time: uint64(now.Unix())} - _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) + result, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) - // Must return a non-nil error (not silently return empty list) - require.Error(t, err, "post-fork should fail loudly when StateSyncEventsAtHeight errors") - require.Contains(t, err.Error(), "deterministic state sync") + // Post-fork is resilient: returns empty on error, does not propagate + require.NoError(t, err, "post-fork should not return error on StateSyncEventsByTime failure") + require.Empty(t, result, "post-fork should return empty on StateSyncEventsByTime failure") - // Both methods should have been called - require.Equal(t, 1, tracker.getBlockHeightByTimeCalled, - "GetBlockHeightByTime should have been called") - require.Equal(t, 1, tracker.stateSyncEventsAtHeightCalled, - "StateSyncEventsAtHeight should have been called") + // StateSyncEventsByTime should have been called + require.Equal(t, 1, tracker.stateSyncEventsByTimeCalled, + "StateSyncEventsByTime should have been called") // Old path should not have been called as fallback require.Equal(t, 0, tracker.stateSyncEventsCalled, "post-fork should not fall back to StateSyncEvents") + // Old two-call pattern should not be used + require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, + "GetBlockHeightByTime should NOT be called") + require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, + "StateSyncEventsAtHeight should NOT be called") } diff --git a/consensus/bor/heimdall.go b/consensus/bor/heimdall.go index edf4c9404f..87f64833b2 100644 --- a/consensus/bor/heimdall.go +++ b/consensus/bor/heimdall.go @@ -15,6 +15,7 @@ import ( type IHeimdallClient interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) + StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index da53fd9272..b40b7069db 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -101,6 +101,8 @@ const ( fetchBlockHeightByTimeFormat = "cutoff_time=%d" fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" + + fetchStateSyncsByTimePath = "clerk/state-syncs-by-time" ) // StateSyncEvents fetches the state sync events from heimdall @@ -353,6 +355,58 @@ func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uin return eventRecords, nil } +// StateSyncsByTimeResponse uses the proto-generated response type from heimdall-v2. +type StateSyncsByTimeResponse = clerkTypes.StateSyncsByTimeResponse + +// StateSyncEventsByTime fetches state sync events using the combined endpoint that +// resolves the Heimdall height from the cutoff time internally. +func (h *HeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + ctx = WithRequestType(ctx, StateSyncByTimeRequest) + + eventRecords := make([]*clerk.EventRecordWithTime, 0) + + for { + u, err := stateSyncsByTimeURL(h.urlString, fromID, toTime) + if err != nil { + return nil, err + } + + log.Debug("Fetching state sync events by time", "queryParams", u.RawQuery) + + response, err := FetchWithRetry[StateSyncsByTimeResponse](ctx, h.client, u, h.closeCh) + if err != nil { + return nil, err + } + + for _, e := range response.EventRecords { + record := &clerk.EventRecordWithTime{ + EventRecord: clerk.EventRecord{ + ID: e.Id, + ChainID: e.BorChainId, + Contract: common.HexToAddress(e.Contract), + Data: e.Data, + LogIndex: e.LogIndex, + TxHash: common.HexToHash(e.TxHash), + }, + Time: e.RecordTime, + } + eventRecords = append(eventRecords, record) + } + + if len(response.EventRecords) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + sort.SliceStable(eventRecords, func(i, j int) bool { + return eventRecords[i].ID < eventRecords[j].ID + }) + + return eventRecords, nil +} + func FetchOnce[T any](ctx context.Context, client http.Client, url *url.URL, closeCh chan struct{}) (*T, error) { request := &Request{client: client, url: url, start: time.Now()} return Fetch[T](ctx, request) @@ -540,6 +594,15 @@ func visibleAtHeightURL(urlString string, fromID uint64, heimdallHeight int64, t return makeURL(urlString, fetchStateSyncsAtHeightPath, params.Encode()) } +func stateSyncsByTimeURL(urlString string, fromID uint64, toTime int64) (*url.URL, error) { + t := time.Unix(toTime, 0).UTC() + params := url.Values{} + params.Set("from_id", fmt.Sprintf("%d", fromID)) + params.Set("to_time", t.Format(time.RFC3339Nano)) + params.Set("pagination.limit", fmt.Sprintf("%d", stateFetchLimit)) + return makeURL(urlString, fetchStateSyncsByTimePath, params.Encode()) +} + func makeURL(urlString, rawPath, rawQuery string) (*url.URL, error) { u, err := url.Parse(urlString) if err != nil { diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 98a8699dd0..96cb6a5367 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -30,6 +30,7 @@ const ( type Endpoint interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) + StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) FetchCheckpoint(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) @@ -117,6 +118,12 @@ func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromI }) } +func (f *MultiHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) { + return c.StateSyncEventsByTime(ctx, fromID, toTime) + }) +} + func (f *MultiHeimdallClient) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) { return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) (*types.Span, error) { return c.GetSpan(ctx, spanID) diff --git a/consensus/bor/heimdall/metrics.go b/consensus/bor/heimdall/metrics.go index 8c126aec60..990915e351 100644 --- a/consensus/bor/heimdall/metrics.go +++ b/consensus/bor/heimdall/metrics.go @@ -31,6 +31,7 @@ const ( StatusRequest requestType = "status" BlockHeightByTimeRequest requestType = "block-height-by-time" StateSyncAtHeightRequest requestType = "state-sync-at-height" + StateSyncByTimeRequest requestType = "state-sync-by-time" ) func WithRequestType(ctx context.Context, reqType requestType) context.Context { @@ -128,6 +129,13 @@ var ( }, timer: metrics.NewRegisteredTimer("client/requests/statesyncatheight/duration", nil), }, + StateSyncByTimeRequest: { + request: map[bool]*metrics.Meter{ + true: metrics.NewRegisteredMeter("client/requests/statesyncbytime/valid", nil), + false: metrics.NewRegisteredMeter("client/requests/statesyncbytime/invalid", nil), + }, + timer: metrics.NewRegisteredTimer("client/requests/statesyncbytime/duration", nil), + }, } ) diff --git a/consensus/bor/heimdallapp/state_sync.go b/consensus/bor/heimdallapp/state_sync.go index e7cc994192..d9942dfed3 100644 --- a/consensus/bor/heimdallapp/state_sync.go +++ b/consensus/bor/heimdallapp/state_sync.go @@ -79,6 +79,41 @@ func (h *HeimdallAppClient) GetBlockHeightByTime(_ context.Context, cutoffTime i return h.hApp.ClerkKeeper.GetBlockHeightByTime(h.NewContext(), cutoffTime) } +// StateSyncEventsByTime fetches state sync events using the combined endpoint that +// resolves the Heimdall height from the cutoff time internally. +func (h *HeimdallAppClient) StateSyncEventsByTime(_ context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + totalRecords := make([]*clerk.EventRecordWithTime, 0) + + queryServer := keeper.NewQueryServer(&h.hApp.ClerkKeeper) + + for { + req := &types.StateSyncsByTimeRequest{ + FromId: fromID, + ToTime: time.Unix(toTime, 0), + Pagination: query.PageRequest{Limit: stateFetchLimit}, + } + + res, err := queryServer.GetStateSyncsByTime(h.NewContext(), req) + if err != nil { + return nil, err + } + + totalRecords = append(totalRecords, toEvents(res.EventRecords)...) + + if len(res.EventRecords) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + sort.SliceStable(totalRecords, func(i, j int) bool { + return totalRecords[i].ID < totalRecords[j].ID + }) + + return totalRecords, nil +} + func toEvents(hdEvents []types.EventRecord) []*clerk.EventRecordWithTime { events := make([]*clerk.EventRecordWithTime, len(hdEvents)) diff --git a/consensus/bor/heimdallgrpc/state_sync.go b/consensus/bor/heimdallgrpc/state_sync.go index 0726a82360..5db523a0a2 100644 --- a/consensus/bor/heimdallgrpc/state_sync.go +++ b/consensus/bor/heimdallgrpc/state_sync.go @@ -152,6 +152,71 @@ func (h *HeimdallGRPCClient) StateSyncEventsAtHeight(ctx context.Context, fromID return eventRecords, nil } +// StateSyncEventsByTime fetches state sync events using the combined endpoint that +// resolves the Heimdall height from the cutoff time internally. +func (h *HeimdallGRPCClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + log.Info("Fetching state sync events by time (gRPC)", "fromID", fromID, "toTime", toTime) + + var err error + + globalCtx, cancel := context.WithTimeout(ctx, stateSyncTotalTimeout) + defer cancel() + + start := time.Now() + ctx = heimdall.WithRequestType(globalCtx, heimdall.StateSyncByTimeRequest) + + defer func() { + heimdall.SendMetrics(ctx, start, err == nil) + }() + + eventRecords := make([]*clerk.EventRecordWithTime, 0) + + for { + req := &types.StateSyncsByTimeRequest{ + FromId: fromID, + ToTime: time.Unix(toTime, 0), + Pagination: query.PageRequest{Limit: stateFetchLimit}, + } + + var res *types.StateSyncsByTimeResponse + pageCtx, pageCancel := context.WithTimeout(ctx, defaultTimeout) + res, err = h.clerkQueryClient.GetStateSyncsByTime(pageCtx, req) + pageCancel() + if err != nil { + return nil, err + } + + events := res.GetEventRecords() + + for _, event := range events { + eventRecord := &clerk.EventRecordWithTime{ + EventRecord: clerk.EventRecord{ + ID: event.Id, + Contract: common.HexToAddress(event.Contract), + Data: event.Data, + TxHash: common.HexToHash(event.TxHash), + LogIndex: event.LogIndex, + ChainID: event.BorChainId, + }, + Time: event.RecordTime, + } + eventRecords = append(eventRecords, eventRecord) + } + + if len(events) < stateFetchLimit { + break + } + + fromID += uint64(stateFetchLimit) + } + + sort.SliceStable(eventRecords, func(i, j int) bool { + return eventRecords[i].ID < eventRecords[j].ID + }) + + return eventRecords, nil +} + // GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff // unix timestamp using the native gRPC GetBlockHeightByTime endpoint. func (h *HeimdallGRPCClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { diff --git a/consensus/bor/span_store_test.go b/consensus/bor/span_store_test.go index bbea14b354..84fcc70f48 100644 --- a/consensus/bor/span_store_test.go +++ b/consensus/bor/span_store_test.go @@ -91,6 +91,9 @@ func (h *MockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64 func (h *MockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (h *MockHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_SpanById(t *testing.T) { spanStore := NewSpanStore(&MockHeimdallClient{}, nil, "1337") @@ -411,6 +414,9 @@ func (h *MockOverlappingHeimdallClient) GetBlockHeightByTime(context.Context, in func (h *MockOverlappingHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (h *MockOverlappingHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_SpanByBlockNumber_OverlappingSpans(t *testing.T) { spanStore := NewSpanStore(&MockOverlappingHeimdallClient{}, nil, "1337") @@ -978,6 +984,9 @@ func (d *dynamicHeimdallClient) GetBlockHeightByTime(context.Context, int64) (in func (d *dynamicHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (d *dynamicHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func makeTestSpan(id, start, end uint64, producerAddr string) *types.Span { producer := stakeTypes.Validator{ @@ -1101,6 +1110,9 @@ func (m *MockSyncStatusClient) GetBlockHeightByTime(context.Context, int64) (int func (m *MockSyncStatusClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (m *MockSyncStatusClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_WaitUntilHeimdallIsSynced(t *testing.T) { t.Run("heimdall already synced", func(t *testing.T) { @@ -1491,6 +1503,9 @@ func (h *TimeoutHeimdallClient) GetBlockHeightByTime(context.Context, int64) (in func (h *TimeoutHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (h *TimeoutHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestSpanStore_HeimdallDownTimeout(t *testing.T) { t.Run("heimdallStatus set to nil on FetchStatus error", func(t *testing.T) { diff --git a/go.mod b/go.mod index 224ef906f3..19167e1b80 100644 --- a/go.mod +++ b/go.mod @@ -352,7 +352,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. // TODO marcello update to final version once released - github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b + github.com/0xPolygon/heimdall-v2 => ../heimdall-v2 github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index b121c1e463..03666b1652 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,6 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b h1:nCXkBXNeDQX2cyqiBZYXFVf8s+l0sYCu/XNja0KFKxg= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260326162702-f1a836fca08b/go.mod h1:UF66PnswnmINRxlbPL4KzOUKGCdIJ6iTqAY1UuR2lEQ= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/tests/bor/mocks/IHeimdallClient.go b/tests/bor/mocks/IHeimdallClient.go index cd5fdf9c63..9cd0ec28c1 100644 --- a/tests/bor/mocks/IHeimdallClient.go +++ b/tests/bor/mocks/IHeimdallClient.go @@ -187,6 +187,21 @@ func (mr *MockIHeimdallClientMockRecorder) StateSyncEventsAtHeight(ctx, fromID, return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEventsAtHeight", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEventsAtHeight), ctx, fromID, toTime, heimdallHeight) } +// StateSyncEventsByTime mocks base method. +func (m *MockIHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "StateSyncEventsByTime", ctx, fromID, toTime) + ret0, _ := ret[0].([]*clerk.EventRecordWithTime) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// StateSyncEventsByTime indicates an expected call of StateSyncEventsByTime. +func (mr *MockIHeimdallClientMockRecorder) StateSyncEventsByTime(ctx, fromID, toTime interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEventsByTime", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEventsByTime), ctx, fromID, toTime) +} + // StateSyncEvents mocks base method. func (m *MockIHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { m.ctrl.T.Helper() From 9697c138acd0598c37006c6ff8fb4cdf27e27cf6 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Fri, 3 Apr 2026 09:45:04 +0200 Subject: [PATCH 19/28] update heimdall-v2 dependency to DSS-test branch --- go.mod | 2 +- go.sum | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 19167e1b80..5b442f3325 100644 --- a/go.mod +++ b/go.mod @@ -352,7 +352,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. // TODO marcello update to final version once released - github.com/0xPolygon/heimdall-v2 => ../heimdall-v2 + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876 github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index 03666b1652..928e15eb69 100644 --- a/go.sum +++ b/go.sum @@ -86,6 +86,8 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876 h1:miDtsRgAYA+loiUpPeYgr+RvnyxCr4ni5bWmZAYRQ2A= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876/go.mod h1:wap2yjeiFg1j6Sew0UtCxr9qx+bxi1u/NMH4oStEg5M= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= From 925c5e5e9024b05da532e64aa1a146381f667b91 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 11:43:51 +0200 Subject: [PATCH 20/28] address comments --- .../bor/heimdall/failover_client_test.go | 35 +++++++++++++++---- eth/ethconfig/config_test.go | 3 ++ eth/handler_bor_test.go | 3 ++ 3 files changed, 35 insertions(+), 6 deletions(-) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 999bf3a628..9474c8838a 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -144,6 +144,11 @@ func (m *mockHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID return []*clerk.EventRecordWithTime{}, nil } +func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + m.hits.Add(1) + return []*clerk.EventRecordWithTime{}, nil +} + // testConnErr is a reusable connection-refused error for tests. var testConnErr = &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} @@ -276,12 +281,18 @@ func TestFailover_NoSwitchOnContextCanceled(t *testing.T) { } func TestFailover_NoSwitchOnServiceUnavailable(t *testing.T) { + var secondaryCalled atomic.Bool primary := &mockHeimdallClient{ getSpanFn: func(_ context.Context, _ uint64) (*types.Span, error) { return nil, ErrServiceUnavailable }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + getSpanFn: func(_ context.Context, _ uint64) (*types.Span, error) { + secondaryCalled.Store(true) + return &types.Span{Id: 1}, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -295,7 +306,7 @@ func TestFailover_NoSwitchOnServiceUnavailable(t *testing.T) { _, err = fc.GetSpan(context.Background(), 1) require.Error(t, err) assert.True(t, errors.Is(err, ErrServiceUnavailable)) - assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on 503") + assert.False(t, secondaryCalled.Load(), "should not failover on 503") } func TestFailover_NoSwitchOnShutdownDetected(t *testing.T) { @@ -1467,12 +1478,18 @@ func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnSuccess(t *testing.T) { } func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnServiceUnavailable(t *testing.T) { + var secondaryCalled atomic.Bool primary := &mockHeimdallClient{ getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { return 0, ErrServiceUnavailable }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + secondaryCalled.Store(true) + return 500, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -1486,16 +1503,22 @@ func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnServiceUnavailable(t *test _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) require.Error(t, err) assert.True(t, errors.Is(err, ErrServiceUnavailable)) - assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on 503") + assert.False(t, secondaryCalled.Load(), "should not failover on 503") } func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnShutdownDetected(t *testing.T) { + var secondaryCalled atomic.Bool primary := &mockHeimdallClient{ getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { return 0, ErrShutdownDetected }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { + secondaryCalled.Store(true) + return 500, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -1509,7 +1532,7 @@ func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnShutdownDetected(t *testin _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) require.Error(t, err) assert.True(t, errors.Is(err, ErrShutdownDetected)) - assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on shutdown") + assert.False(t, secondaryCalled.Load(), "should not failover on shutdown") } func TestMultiFailover_GetBlockHeightByTime_ThreeClients_CascadeToTertiary(t *testing.T) { diff --git a/eth/ethconfig/config_test.go b/eth/ethconfig/config_test.go index 8b971292dc..60359b5771 100644 --- a/eth/ethconfig/config_test.go +++ b/eth/ethconfig/config_test.go @@ -55,6 +55,9 @@ func (m *mockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64 func (m *mockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} // newTestBorChainConfig creates a minimal Bor chain config for testing func newTestBorChainConfig() *params.ChainConfig { diff --git a/eth/handler_bor_test.go b/eth/handler_bor_test.go index 82070d70dd..94c9ceae21 100644 --- a/eth/handler_bor_test.go +++ b/eth/handler_bor_test.go @@ -71,6 +71,9 @@ func (m *mockHeimdall) GetBlockHeightByTime(context.Context, int64) (int64, erro func (m *mockHeimdall) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } +func (m *mockHeimdall) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, nil +} func TestFetchWhitelistCheckpointAndMilestone(t *testing.T) { t.Parallel() From 5dfe949c1575aabcbe95eb1219716a470a699779 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 11:51:30 +0200 Subject: [PATCH 21/28] solve lint issue --- consensus/bor/bor_test.go | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index 10970ffa06..fcac1c7ef1 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -5282,20 +5282,20 @@ func TestVerifyHeader_PreGiugliano_NoCheck(t *testing.T) { // It returns configurable results and tracks call counts for assertions. type trackingHeimdallClient struct { // Call counters - stateSyncEventsCalled int - getBlockHeightByTimeCalled int + stateSyncEventsCalled int + getBlockHeightByTimeCalled int stateSyncEventsAtHeightCalled int - stateSyncEventsByTimeCalled int + stateSyncEventsByTimeCalled int // Configurable return values - blockHeight int64 - blockHeightErr error - events []*clerk.EventRecordWithTime - eventsErr error - eventsAtHeight []*clerk.EventRecordWithTime - eventsAtHeightErr error - eventsByTime []*clerk.EventRecordWithTime - eventsByTimeErr error + blockHeight int64 + blockHeightErr error + events []*clerk.EventRecordWithTime + eventsErr error + eventsAtHeight []*clerk.EventRecordWithTime + eventsAtHeightErr error + eventsByTime []*clerk.EventRecordWithTime + eventsByTimeErr error } func (t *trackingHeimdallClient) Close() {} From 8d06151399f6613d18672889e1f7b8412f89cd5d Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 20:58:05 +0200 Subject: [PATCH 22/28] address comments --- consensus/bor/bor.go | 11 +- consensus/bor/heimdall/client.go | 5 + consensus/bor/heimdall/failover_client.go | 24 +++- .../bor/heimdall/failover_client_test.go | 133 +++++++++++++++++- consensus/bor/heimdall/state_sync_url_test.go | 25 ++++ 5 files changed, 188 insertions(+), 10 deletions(-) diff --git a/consensus/bor/bor.go b/consensus/bor/bor.go index be981c6290..d88415d66c 100644 --- a/consensus/bor/bor.go +++ b/consensus/bor/bor.go @@ -1765,9 +1765,14 @@ func (c *Bor) CommitStates( eventRecords, err = c.HeimdallClient.StateSyncEventsByTime(c.ctx, from, to.Unix()) if err != nil { - // Match pre-fork resilience: log and return empty on transient errors. - // Determinism is preserved because all validators independently skip - // the same sprint, and events will be picked up in the next sprint. + // Liveness-over-safety tradeoff (matches pre-fork behavior): + // FetchWithRetry already retries aggressively and MultiHeimdallClient + // provides failover, so errors reaching here are persistent. Returning + // empty lets the proposer build a block with 0 state syncs. If other + // validators succeed, they will derive a different state root and reject + // this block — the proposer misses a slot but no silent divergence + // occurs. Skipped events are retried at the next sprint since `from` + // is derived from the on-chain LastStateId. log.Error("Error fetching deterministic state sync events", "fromID", from, "to", to.Unix(), "err", err) return make([]*types.StateSyncData, 0), nil diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index b40b7069db..26175b291b 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -361,6 +361,11 @@ type StateSyncsByTimeResponse = clerkTypes.StateSyncsByTimeResponse // StateSyncEventsByTime fetches state sync events using the combined endpoint that // resolves the Heimdall height from the cutoff time internally. func (h *HeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + // Global timeout bounding the entire paginated fetch, matching the gRPC + // implementation's stateSyncTotalTimeout (1 minute). + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + ctx = WithRequestType(ctx, StateSyncByTimeRequest) eventRecords := make([]*clerk.EventRecordWithTime, 0) diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 96cb6a5367..556a68fe54 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -189,9 +189,19 @@ func callWithFailover[T any](f *MultiHeimdallClient, ctx context.Context, fn fun active := f.registry.Active() - subCtx, cancel := context.WithTimeout(ctx, f.attemptTimeout) + // Only apply attemptTimeout if the caller hasn't set a tighter deadline. + // Paginated methods (e.g. StateSyncEventsByTime) set their own global + // timeout before reaching here; capping them with a shorter attemptTimeout + // would silently truncate the pagination. + subCtx := ctx + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + subCtx, cancel = context.WithTimeout(ctx, f.attemptTimeout) + defer cancel() + } + result, err := fn(subCtx, f.clients[active]) - cancel() + if err == nil { return result, nil @@ -247,9 +257,15 @@ func cascadeClients[T any](f *MultiHeimdallClient, ctx context.Context, fn func( for _, candidates := range passes { for _, i := range candidates { - subCtx, cancel := context.WithTimeout(ctx, f.attemptTimeout) + subCtx := ctx + if _, hasDeadline := ctx.Deadline(); !hasDeadline { + var cancel context.CancelFunc + subCtx, cancel = context.WithTimeout(ctx, f.attemptTimeout) + defer cancel() + } + result, err := fn(subCtx, f.clients[i]) - cancel() + if err == nil { f.registry.SetActive(i) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 9474c8838a..b53f3ff850 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -33,6 +33,7 @@ type mockHeimdallClient struct { fetchMilestoneCntFn func(ctx context.Context) (int64, error) fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) stateSyncEventsAtHeightFn func(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) + stateSyncEventsByTimeFn func(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) getBlockHeightByTimeFn func(ctx context.Context, cutoffTime int64) (int64, error) closeFn func() hits atomic.Int32 @@ -144,8 +145,13 @@ func (m *mockHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID return []*clerk.EventRecordWithTime{}, nil } -func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { +func (m *mockHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { m.hits.Add(1) + + if m.stateSyncEventsByTimeFn != nil { + return m.stateSyncEventsByTimeFn(ctx, fromID, toTime) + } + return []*clerk.EventRecordWithTime{}, nil } @@ -780,13 +786,19 @@ func TestFailover_ThreeClients_ProbeBackToPrimary(t *testing.T) { // Active client returns non-failover error: should return directly, no cascade. func TestFailover_ActiveNonFailoverError(t *testing.T) { + var tertiaryCalled atomic.Bool primary := &mockHeimdallClient{} secondary := &mockHeimdallClient{ getSpanFn: func(_ context.Context, _ uint64) (*types.Span, error) { return nil, ErrShutdownDetected }, } - tertiary := &mockHeimdallClient{} + tertiary := &mockHeimdallClient{ + getSpanFn: func(_ context.Context, _ uint64) (*types.Span, error) { + tertiaryCalled.Store(true) + return &types.Span{Id: 1}, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary, tertiary) require.NoError(t, err) @@ -803,7 +815,7 @@ func TestFailover_ActiveNonFailoverError(t *testing.T) { _, err = fc.GetSpan(context.Background(), 1) require.Error(t, err) assert.True(t, errors.Is(err, ErrShutdownDetected)) - assert.Equal(t, int32(0), tertiary.hits.Load(), "should not cascade to tertiary on non-failover error") + assert.False(t, tertiaryCalled.Load(), "should not cascade to tertiary on non-failover error") } // Active client returns failover error: cascade should try by priority. @@ -1565,3 +1577,118 @@ func TestMultiFailover_GetBlockHeightByTime_ThreeClients_CascadeToTertiary(t *te assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") } + +// --- StateSyncEventsByTime failover tests --- + +func TestMultiFailover_StateSyncEventsByTime_SwitchOnPrimaryDown(t *testing.T) { + connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + expected := []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: 42}}} + + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, connErr + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return expected, nil + }, + } + + fc := newInstantMulti(primary, secondary) + defer fc.Close() + + result, err := fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.NoError(t, err) + assert.Equal(t, expected, result) +} + +func TestMultiFailover_StateSyncEventsByTime_NoSwitchOnServiceUnavailable(t *testing.T) { + var secondaryCalled atomic.Bool + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, ErrServiceUnavailable + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + secondaryCalled.Store(true) + return []*clerk.EventRecordWithTime{}, nil + }, + } + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrServiceUnavailable)) + assert.False(t, secondaryCalled.Load(), "should not failover on 503") +} + +func TestMultiFailover_StateSyncEventsByTime_NoSwitchOnShutdownDetected(t *testing.T) { + var secondaryCalled atomic.Bool + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, ErrShutdownDetected + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + secondaryCalled.Store(true) + return []*clerk.EventRecordWithTime{}, nil + }, + } + + fc, err := NewMultiHeimdallClient(primary, secondary) + require.NoError(t, err) + + fc.attemptTimeout = 100 * time.Millisecond + fc.registry.HealthCheckInterval = 1 * time.Hour + fc.registry.ConsecutiveThreshold = 1 + fc.registry.PromotionCooldown = 0 + defer fc.Close() + + _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.Error(t, err) + assert.True(t, errors.Is(err, ErrShutdownDetected)) + assert.False(t, secondaryCalled.Load(), "should not failover on shutdown") +} + +func TestMultiFailover_StateSyncEventsByTime_ThreeClients_CascadeToTertiary(t *testing.T) { + connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} + expected := []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: 99}}} + + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, connErr + }, + } + secondary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return nil, connErr + }, + } + tertiary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + return expected, nil + }, + } + + fc := newInstantMulti(primary, secondary, tertiary) + defer fc.Close() + + result, err := fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.NoError(t, err) + assert.Equal(t, expected, result) + + assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") + assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") + assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") +} diff --git a/consensus/bor/heimdall/state_sync_url_test.go b/consensus/bor/heimdall/state_sync_url_test.go index 80c06493a3..5c266ba1e4 100644 --- a/consensus/bor/heimdall/state_sync_url_test.go +++ b/consensus/bor/heimdall/state_sync_url_test.go @@ -54,6 +54,31 @@ func TestBlockHeightByTimeURL_Format(t *testing.T) { require.Equal(t, expected, u.String()) } +func TestStateSyncsByTimeURL_Format(t *testing.T) { + t.Parallel() + + fromID := uint64(42) + toTime := int64(1700000000) // 2023-11-14T22:13:20Z + + u, err := stateSyncsByTimeURL("http://bor0", fromID, toTime) + require.NoError(t, err) + + // Path must be clerk/state-syncs-by-time + require.True(t, strings.HasSuffix(u.Path, "clerk/state-syncs-by-time"), + "expected path to end with clerk/state-syncs-by-time, got %s", u.Path) + + // Validate query parameters + q := u.Query() + + expectedTime := time.Unix(toTime, 0).UTC().Format(time.RFC3339Nano) + require.Equal(t, expectedTime, q.Get("to_time"), "to_time should be RFC3339Nano formatted") + require.Equal(t, fmt.Sprintf("%d", fromID), q.Get("from_id")) + require.Equal(t, fmt.Sprintf("%d", stateFetchLimit), q.Get("pagination.limit")) + + // Should NOT have heimdall_height (resolved internally by heimdall) + require.Empty(t, q.Get("heimdall_height"), "combined endpoint should not send heimdall_height") +} + func TestStateSyncURL_ToTimeIsRFC3339(t *testing.T) { t.Parallel() From 3a9842199077214283f513d23d4edb820830fc25 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 21:15:27 +0200 Subject: [PATCH 23/28] address comments --- consensus/bor/heimdall/failover_client.go | 24 ++++------------------- 1 file changed, 4 insertions(+), 20 deletions(-) diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 556a68fe54..96cb6a5367 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -189,19 +189,9 @@ func callWithFailover[T any](f *MultiHeimdallClient, ctx context.Context, fn fun active := f.registry.Active() - // Only apply attemptTimeout if the caller hasn't set a tighter deadline. - // Paginated methods (e.g. StateSyncEventsByTime) set their own global - // timeout before reaching here; capping them with a shorter attemptTimeout - // would silently truncate the pagination. - subCtx := ctx - if _, hasDeadline := ctx.Deadline(); !hasDeadline { - var cancel context.CancelFunc - subCtx, cancel = context.WithTimeout(ctx, f.attemptTimeout) - defer cancel() - } - + subCtx, cancel := context.WithTimeout(ctx, f.attemptTimeout) result, err := fn(subCtx, f.clients[active]) - + cancel() if err == nil { return result, nil @@ -257,15 +247,9 @@ func cascadeClients[T any](f *MultiHeimdallClient, ctx context.Context, fn func( for _, candidates := range passes { for _, i := range candidates { - subCtx := ctx - if _, hasDeadline := ctx.Deadline(); !hasDeadline { - var cancel context.CancelFunc - subCtx, cancel = context.WithTimeout(ctx, f.attemptTimeout) - defer cancel() - } - + subCtx, cancel := context.WithTimeout(ctx, f.attemptTimeout) result, err := fn(subCtx, f.clients[i]) - + cancel() if err == nil { f.registry.SetActive(i) From a1fa6b9a992a8a0bef169ff59493bc2deb71fe42 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 22:06:56 +0200 Subject: [PATCH 24/28] address comments --- consensus/bor/heimdall/client.go | 5 ++ consensus/bor/heimdall/failover_client.go | 11 +++++ .../bor/heimdall/failover_client_test.go | 48 +++++++++++++++++++ 3 files changed, 64 insertions(+) diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index 26175b291b..f3188a8e5a 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -309,6 +309,11 @@ type RecordListVisibleAtHeightResponse = clerkTypes.RecordListVisibleAtHeightRes // StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, // using the new query endpoint that queries the latest state with immutable visibility_height indexes. func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + // Global timeout bounding the entire paginated fetch, matching the gRPC + // implementation's stateSyncTotalTimeout (1 minute). + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + ctx = WithRequestType(ctx, StateSyncAtHeightRequest) eventRecords := make([]*clerk.EventRecordWithTime, 0) diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 96cb6a5367..25e98c3dfc 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -113,12 +113,23 @@ func (f *MultiHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64 } func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { + // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover + // applies its per-attempt 30s cap. This way each attempt gets min(1min, 30s) = 30s, + // and multiple failover attempts share the 1-minute budget. + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) { return c.StateSyncEventsAtHeight(ctx, fromID, toTime, heimdallHeight) }) } func (f *MultiHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { + // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover + // applies its per-attempt 30s cap. + ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) + defer cancel() + return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) { return c.StateSyncEventsByTime(ctx, fromID, toTime) }) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index b53f3ff850..131a39e1fa 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -1692,3 +1692,51 @@ func TestMultiFailover_StateSyncEventsByTime_ThreeClients_CascadeToTertiary(t *t assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") } + +func TestMultiFailover_StateSyncEventsByTime_CapsAttemptTimeoutWithGlobalPaginationDeadline(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(ctx context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok, "paginated call should have a deadline") + + remaining := time.Until(deadline) + assert.LessOrEqual(t, remaining, time.Minute+2*time.Second, "global pagination deadline should cap per-attempt timeout") + assert.Greater(t, remaining, 55*time.Second, "global pagination deadline should be close to 1 minute") + + return []*clerk.EventRecordWithTime{}, nil + }, + } + + fc, err := NewMultiHeimdallClient(primary) + require.NoError(t, err) + defer fc.Close() + + fc.attemptTimeout = 2 * time.Minute + + _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.NoError(t, err) +} + +func TestMultiFailover_StateSyncEventsAtHeight_CapsAttemptTimeoutWithGlobalPaginationDeadline(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(ctx context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok, "paginated call should have a deadline") + + remaining := time.Until(deadline) + assert.LessOrEqual(t, remaining, time.Minute+2*time.Second, "global pagination deadline should cap per-attempt timeout") + assert.Greater(t, remaining, 55*time.Second, "global pagination deadline should be close to 1 minute") + + return []*clerk.EventRecordWithTime{}, nil + }, + } + + fc, err := NewMultiHeimdallClient(primary) + require.NoError(t, err) + defer fc.Close() + + fc.attemptTimeout = 2 * time.Minute + + _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 9999, 123) + require.NoError(t, err) +} From dce64833c4d100a1561cd6d17e9e7c74b53ab639 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Sat, 4 Apr 2026 22:25:04 +0200 Subject: [PATCH 25/28] address comments --- consensus/bor/heimdall/failover_client.go | 7 ++-- .../bor/heimdall/failover_client_test.go | 35 ++++++++++++++----- 2 files changed, 31 insertions(+), 11 deletions(-) diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 25e98c3dfc..24b19d80f5 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -114,8 +114,9 @@ func (f *MultiHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64 func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover - // applies its per-attempt 30s cap. This way each attempt gets min(1min, 30s) = 30s, - // and multiple failover attempts share the 1-minute budget. + // applies its per-attempt attemptTimeout cap. This way each attempt gets + // min(1 minute, attemptTimeout), and multiple failover attempts share the + // same 1-minute budget. ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() @@ -126,7 +127,7 @@ func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromI func (f *MultiHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover - // applies its per-attempt 30s cap. + // applies its per-attempt attemptTimeout cap. ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) defer cancel() diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 131a39e1fa..0328cc3f4e 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -1328,12 +1328,19 @@ func TestMultiFailover_StateSyncEventsAtHeight_SwitchOnPrimaryDown(t *testing.T) } func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnSuccess(t *testing.T) { + var secondaryCalled atomic.Bool + primary := &mockHeimdallClient{ stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + secondaryCalled.Store(true) + return []*clerk.EventRecordWithTime{}, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -1348,22 +1355,27 @@ func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnSuccess(t *testing.T) { fc.ensureHealthRegistry() time.Sleep(50 * time.Millisecond) - secondaryBefore := secondary.hits.Load() - events, err := fc.StateSyncEventsAtHeight(context.Background(), 7, 50, 100) require.NoError(t, err) require.Len(t, events, 1) assert.Equal(t, uint64(7), events[0].ID) - assert.Equal(t, secondaryBefore, secondary.hits.Load(), "secondary should not be contacted when primary succeeds") + assert.False(t, secondaryCalled.Load(), "secondary stateSyncEventsAtHeight should not be called when primary succeeds") } func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnServiceUnavailable(t *testing.T) { + var secondaryCalled atomic.Bool + primary := &mockHeimdallClient{ stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, ErrServiceUnavailable }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + secondaryCalled.Store(true) + return []*clerk.EventRecordWithTime{}, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -1377,16 +1389,23 @@ func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnServiceUnavailable(t *t _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) require.Error(t, err) assert.True(t, errors.Is(err, ErrServiceUnavailable)) - assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on 503") + assert.False(t, secondaryCalled.Load(), "should not failover on 503") } func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnShutdownDetected(t *testing.T) { + var secondaryCalled atomic.Bool + primary := &mockHeimdallClient{ stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, ErrShutdownDetected }, } - secondary := &mockHeimdallClient{} + secondary := &mockHeimdallClient{ + stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { + secondaryCalled.Store(true) + return []*clerk.EventRecordWithTime{}, nil + }, + } fc, err := NewMultiHeimdallClient(primary, secondary) require.NoError(t, err) @@ -1400,7 +1419,7 @@ func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnShutdownDetected(t *tes _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) require.Error(t, err) assert.True(t, errors.Is(err, ErrShutdownDetected)) - assert.Equal(t, int32(0), secondary.hits.Load(), "should not failover on shutdown") + assert.False(t, secondaryCalled.Load(), "should not failover on shutdown") } func TestMultiFailover_StateSyncEventsAtHeight_ThreeClients_CascadeToTertiary(t *testing.T) { From ca9ee85ddf0c533d9557aaf929cddbc3abe25c32 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Mon, 6 Apr 2026 11:12:18 +0200 Subject: [PATCH 26/28] unify endpoints --- consensus/bor/bor_test.go | 40 +-- consensus/bor/heimdall.go | 2 - consensus/bor/heimdall/client.go | 108 ------ consensus/bor/heimdall/failover_client.go | 21 -- .../bor/heimdall/failover_client_test.go | 320 ------------------ consensus/bor/heimdall/metrics.go | 16 - consensus/bor/heimdall/state_sync_url_test.go | 45 --- consensus/bor/heimdallapp/state_sync.go | 41 --- consensus/bor/heimdallgrpc/state_sync.go | 95 ------ go.mod | 2 +- go.sum | 4 +- tests/bor/mocks/IHeimdallClient.go | 53 +-- 12 files changed, 20 insertions(+), 727 deletions(-) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index fcac1c7ef1..7c9c47ef03 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -5282,20 +5282,14 @@ func TestVerifyHeader_PreGiugliano_NoCheck(t *testing.T) { // It returns configurable results and tracks call counts for assertions. type trackingHeimdallClient struct { // Call counters - stateSyncEventsCalled int - getBlockHeightByTimeCalled int - stateSyncEventsAtHeightCalled int - stateSyncEventsByTimeCalled int + stateSyncEventsCalled int + stateSyncEventsByTimeCalled int // Configurable return values - blockHeight int64 - blockHeightErr error - events []*clerk.EventRecordWithTime - eventsErr error - eventsAtHeight []*clerk.EventRecordWithTime - eventsAtHeightErr error - eventsByTime []*clerk.EventRecordWithTime - eventsByTimeErr error + events []*clerk.EventRecordWithTime + eventsErr error + eventsByTime []*clerk.EventRecordWithTime + eventsByTimeErr error } func (t *trackingHeimdallClient) Close() {} @@ -5303,10 +5297,6 @@ func (t *trackingHeimdallClient) StateSyncEvents(context.Context, uint64, int64) t.stateSyncEventsCalled++ return t.events, t.eventsErr } -func (t *trackingHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - t.stateSyncEventsAtHeightCalled++ - return t.eventsAtHeight, t.eventsAtHeightErr -} func (t *trackingHeimdallClient) GetSpan(context.Context, uint64) (*borTypes.Span, error) { return nil, nil } @@ -5328,10 +5318,6 @@ func (t *trackingHeimdallClient) FetchMilestoneCount(context.Context) (int64, er func (t *trackingHeimdallClient) FetchStatus(context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } -func (t *trackingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - t.getBlockHeightByTimeCalled++ - return t.blockHeight, t.blockHeightErr -} func (t *trackingHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { t.stateSyncEventsByTimeCalled++ return t.eventsByTime, t.eventsByTimeErr @@ -5377,8 +5363,6 @@ func TestCommitStates_DeterministicForkSwitch(t *testing.T) { _, err := b.CommitStates(stateDb, h, statefull.ChainContext{Chain: chain.HeaderChain(), Bor: b}) require.NoError(t, err) require.Equal(t, 1, tracker.stateSyncEventsCalled, "pre-fork should call StateSyncEvents") - require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, "pre-fork should not call GetBlockHeightByTime") - require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, "pre-fork should not call StateSyncEventsAtHeight") require.Equal(t, 0, tracker.stateSyncEventsByTimeCalled, "pre-fork should not call StateSyncEventsByTime") // Post-fork: block 112 should use StateSyncEventsByTime (deterministic state sync) @@ -5394,8 +5378,6 @@ func TestCommitStates_DeterministicForkSwitch(t *testing.T) { require.NoError(t, err) require.Equal(t, 0, tracker2.stateSyncEventsCalled, "post-fork should not call StateSyncEvents") require.Equal(t, 1, tracker2.stateSyncEventsByTimeCalled, "post-fork should call StateSyncEventsByTime") - require.Equal(t, 0, tracker2.getBlockHeightByTimeCalled, "post-fork should not call GetBlockHeightByTime") - require.Equal(t, 0, tracker2.stateSyncEventsAtHeightCalled, "post-fork should not call StateSyncEventsAtHeight") } func TestCommitStates_ResilientPostFork(t *testing.T) { @@ -5435,11 +5417,6 @@ func TestCommitStates_ResilientPostFork(t *testing.T) { // Must not fallback to StateSyncEvents require.Equal(t, 0, tracker.stateSyncEventsCalled, "post-fork should NOT fall back to StateSyncEvents on error") - // Old two-call pattern should not be used - require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, - "GetBlockHeightByTime should NOT be called") - require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, - "StateSyncEventsAtHeight should NOT be called") } func TestCommitStates_ResilientPostFork_ReturnsEmptyOnError(t *testing.T) { @@ -5478,9 +5455,4 @@ func TestCommitStates_ResilientPostFork_ReturnsEmptyOnError(t *testing.T) { // Old path should not have been called as fallback require.Equal(t, 0, tracker.stateSyncEventsCalled, "post-fork should not fall back to StateSyncEvents") - // Old two-call pattern should not be used - require.Equal(t, 0, tracker.getBlockHeightByTimeCalled, - "GetBlockHeightByTime should NOT be called") - require.Equal(t, 0, tracker.stateSyncEventsAtHeightCalled, - "StateSyncEventsAtHeight should NOT be called") } diff --git a/consensus/bor/heimdall.go b/consensus/bor/heimdall.go index 87f64833b2..a1b9f3a8bc 100644 --- a/consensus/bor/heimdall.go +++ b/consensus/bor/heimdall.go @@ -14,7 +14,6 @@ import ( //go:generate mockgen -source=heimdall.go -destination=../../tests/bor/mocks/IHeimdallClient.go -package=mocks type IHeimdallClient interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) - StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) @@ -23,7 +22,6 @@ type IHeimdallClient interface { FetchMilestone(ctx context.Context) (*milestone.Milestone, error) FetchMilestoneCount(ctx context.Context) (int64, error) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) - GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) Close() } diff --git a/consensus/bor/heimdall/client.go b/consensus/bor/heimdall/client.go index f3188a8e5a..2ec976db96 100644 --- a/consensus/bor/heimdall/client.go +++ b/consensus/bor/heimdall/client.go @@ -11,7 +11,6 @@ import ( "path" "reflect" "sort" - "strconv" "time" "github.com/0xPolygon/heimdall-v2/x/bor/types" @@ -97,11 +96,6 @@ const ( fetchStatus = "/status" - fetchBlockHeightByTimePath = "clerk/block-height-by-time" - fetchBlockHeightByTimeFormat = "cutoff_time=%d" - - fetchStateSyncsAtHeightPath = "clerk/state-syncs-at-height" - fetchStateSyncsByTimePath = "clerk/state-syncs-by-time" ) @@ -273,93 +267,6 @@ func (h *HeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, err return response, nil } -// BlockHeightByTimeResponse is the response from the Heimdall clerk/block-height-by-time endpoint. -// Note: Cosmos SDK REST gateway serializes int64 fields as JSON strings. -type BlockHeightByTimeResponse struct { - Height string `json:"height"` -} - -// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff unix timestamp. -func (h *HeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { - heightByTimeURL, err := blockHeightByTimeURL(h.urlString, cutoffTime) - if err != nil { - return 0, err - } - - ctx = WithRequestType(ctx, BlockHeightByTimeRequest) - - response, err := FetchWithRetry[BlockHeightByTimeResponse](ctx, h.client, heightByTimeURL, h.closeCh) - if err != nil { - return 0, err - } - - height, err := strconv.ParseInt(response.Height, 10, 64) - if err != nil { - return 0, fmt.Errorf("failed to parse height %q: %w", response.Height, err) - } - - return height, nil -} - -// RecordListVisibleAtHeightResponse uses the proto-generated response type from heimdall-v2. -// This handles Cosmos SDK's string-encoded integers correctly via gogoproto JSON unmarshaling. -// Type alias added for readability. -type RecordListVisibleAtHeightResponse = clerkTypes.RecordListVisibleAtHeightResponse - -// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height, -// using the new query endpoint that queries the latest state with immutable visibility_height indexes. -func (h *HeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - // Global timeout bounding the entire paginated fetch, matching the gRPC - // implementation's stateSyncTotalTimeout (1 minute). - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - - ctx = WithRequestType(ctx, StateSyncAtHeightRequest) - - eventRecords := make([]*clerk.EventRecordWithTime, 0) - - for { - u, err := visibleAtHeightURL(h.urlString, fromID, heimdallHeight, toTime) - if err != nil { - return nil, err - } - - log.Debug("Fetching state sync events at height", "queryParams", u.RawQuery) - - response, err := FetchWithRetry[RecordListVisibleAtHeightResponse](ctx, h.client, u, h.closeCh) - if err != nil { - return nil, err - } - - for _, e := range response.EventRecords { - record := &clerk.EventRecordWithTime{ - EventRecord: clerk.EventRecord{ - ID: e.Id, - ChainID: e.BorChainId, - Contract: common.HexToAddress(e.Contract), - Data: e.Data, - LogIndex: e.LogIndex, - TxHash: common.HexToHash(e.TxHash), - }, - Time: e.RecordTime, - } - eventRecords = append(eventRecords, record) - } - - if len(response.EventRecords) < stateFetchLimit { - break - } - - fromID += uint64(stateFetchLimit) - } - - sort.SliceStable(eventRecords, func(i, j int) bool { - return eventRecords[i].ID < eventRecords[j].ID - }) - - return eventRecords, nil -} - // StateSyncsByTimeResponse uses the proto-generated response type from heimdall-v2. type StateSyncsByTimeResponse = clerkTypes.StateSyncsByTimeResponse @@ -589,21 +496,6 @@ func statusURL(urlString string) (*url.URL, error) { return makeURL(urlString, fetchStatus, "") } -func blockHeightByTimeURL(urlString string, cutoffTime int64) (*url.URL, error) { - queryParams := fmt.Sprintf(fetchBlockHeightByTimeFormat, cutoffTime) - return makeURL(urlString, fetchBlockHeightByTimePath, queryParams) -} - -func visibleAtHeightURL(urlString string, fromID uint64, heimdallHeight int64, toTime int64) (*url.URL, error) { - t := time.Unix(toTime, 0).UTC() - params := url.Values{} - params.Set("from_id", fmt.Sprintf("%d", fromID)) - params.Set("heimdall_height", fmt.Sprintf("%d", heimdallHeight)) - params.Set("to_time", t.Format(time.RFC3339Nano)) - params.Set("pagination.limit", fmt.Sprintf("%d", stateFetchLimit)) - return makeURL(urlString, fetchStateSyncsAtHeightPath, params.Encode()) -} - func stateSyncsByTimeURL(urlString string, fromID uint64, toTime int64) (*url.URL, error) { t := time.Unix(toTime, 0).UTC() params := url.Values{} diff --git a/consensus/bor/heimdall/failover_client.go b/consensus/bor/heimdall/failover_client.go index 24b19d80f5..73a28080bc 100644 --- a/consensus/bor/heimdall/failover_client.go +++ b/consensus/bor/heimdall/failover_client.go @@ -29,7 +29,6 @@ const ( // running into Go's covariant-slice restriction. type Endpoint interface { StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) - StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) GetSpan(ctx context.Context, spanID uint64) (*types.Span, error) GetLatestSpan(ctx context.Context) (*types.Span, error) @@ -38,7 +37,6 @@ type Endpoint interface { FetchMilestone(ctx context.Context) (*milestone.Milestone, error) FetchMilestoneCount(ctx context.Context) (int64, error) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) - GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) Close() } @@ -112,19 +110,6 @@ func (f *MultiHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64 }) } -func (f *MultiHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover - // applies its per-attempt attemptTimeout cap. This way each attempt gets - // min(1 minute, attemptTimeout), and multiple failover attempts share the - // same 1-minute budget. - ctx, cancel := context.WithTimeout(ctx, 1*time.Minute) - defer cancel() - - return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) ([]*clerk.EventRecordWithTime, error) { - return c.StateSyncEventsAtHeight(ctx, fromID, toTime, heimdallHeight) - }) -} - func (f *MultiHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { // Set a 1-minute global timeout for the paginated fetch BEFORE callWithFailover // applies its per-attempt attemptTimeout cap. @@ -178,12 +163,6 @@ func (f *MultiHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo }) } -func (f *MultiHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { - return callWithFailover(f, ctx, func(ctx context.Context, c Endpoint) (int64, error) { - return c.GetBlockHeightByTime(ctx, cutoffTime) - }) -} - func (f *MultiHeimdallClient) Close() { f.probeCancel() // cancel in-flight probes first f.registry.Stop() diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index 0328cc3f4e..fad8973e19 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -1301,302 +1301,6 @@ func TestRegistry_InformedCascade_RespectsCooldown(t *testing.T) { assert.Equal(t, 2, fc.registry.Active(), "should prefer cooled tertiary over uncooled primary") } -// --- StateSyncEventsAtHeight failover tests --- - -func TestMultiFailover_StateSyncEventsAtHeight_SwitchOnPrimaryDown(t *testing.T) { - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} - }, - } - secondary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil - }, - } - - fc := newInstantMulti(primary, secondary) - defer fc.Close() - - events, err := fc.StateSyncEventsAtHeight(context.Background(), 42, 100, 200) - require.NoError(t, err) - require.Len(t, events, 1) - assert.Equal(t, uint64(42), events[0].ID) - - assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") - assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been called") -} - -func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnSuccess(t *testing.T) { - var secondaryCalled atomic.Bool - - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil - }, - } - secondary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - secondaryCalled.Store(true) - return []*clerk.EventRecordWithTime{}, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 5 * time.Second - fc.probeTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - fc.ensureHealthRegistry() - time.Sleep(50 * time.Millisecond) - - events, err := fc.StateSyncEventsAtHeight(context.Background(), 7, 50, 100) - require.NoError(t, err) - require.Len(t, events, 1) - assert.Equal(t, uint64(7), events[0].ID) - assert.False(t, secondaryCalled.Load(), "secondary stateSyncEventsAtHeight should not be called when primary succeeds") -} - -func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnServiceUnavailable(t *testing.T) { - var secondaryCalled atomic.Bool - - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, ErrServiceUnavailable - }, - } - secondary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - secondaryCalled.Store(true) - return []*clerk.EventRecordWithTime{}, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) - require.Error(t, err) - assert.True(t, errors.Is(err, ErrServiceUnavailable)) - assert.False(t, secondaryCalled.Load(), "should not failover on 503") -} - -func TestMultiFailover_StateSyncEventsAtHeight_NoSwitchOnShutdownDetected(t *testing.T) { - var secondaryCalled atomic.Bool - - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, ErrShutdownDetected - }, - } - secondary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - secondaryCalled.Store(true) - return []*clerk.EventRecordWithTime{}, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 50, 100) - require.Error(t, err) - assert.True(t, errors.Is(err, ErrShutdownDetected)) - assert.False(t, secondaryCalled.Load(), "should not failover on shutdown") -} - -func TestMultiFailover_StateSyncEventsAtHeight_ThreeClients_CascadeToTertiary(t *testing.T) { - connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} - - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, connErr - }, - } - secondary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, connErr - }, - } - tertiary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(_ context.Context, fromID uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return []*clerk.EventRecordWithTime{{EventRecord: clerk.EventRecord{ID: fromID}}}, nil - }, - } - - fc := newInstantMulti(primary, secondary, tertiary) - defer fc.Close() - - events, err := fc.StateSyncEventsAtHeight(context.Background(), 10, 50, 100) - require.NoError(t, err) - require.Len(t, events, 1) - assert.Equal(t, uint64(10), events[0].ID) - - assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") - assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") - assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") -} - -// --- GetBlockHeightByTime failover tests --- - -func TestMultiFailover_GetBlockHeightByTime_SwitchOnPrimaryDown(t *testing.T) { - primary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 0, &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} - }, - } - secondary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, cutoffTime int64) (int64, error) { - return 500, nil - }, - } - - fc := newInstantMulti(primary, secondary) - defer fc.Close() - - height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) - require.NoError(t, err) - assert.Equal(t, int64(500), height) - - assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") - assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been called") -} - -func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnSuccess(t *testing.T) { - primary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 999, nil - }, - } - secondary := &mockHeimdallClient{} - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 5 * time.Second - fc.probeTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - fc.ensureHealthRegistry() - time.Sleep(50 * time.Millisecond) - - secondaryBefore := secondary.hits.Load() - - height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) - require.NoError(t, err) - assert.Equal(t, int64(999), height) - assert.Equal(t, secondaryBefore, secondary.hits.Load(), "secondary should not be contacted when primary succeeds") -} - -func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnServiceUnavailable(t *testing.T) { - var secondaryCalled atomic.Bool - primary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 0, ErrServiceUnavailable - }, - } - secondary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - secondaryCalled.Store(true) - return 500, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) - require.Error(t, err) - assert.True(t, errors.Is(err, ErrServiceUnavailable)) - assert.False(t, secondaryCalled.Load(), "should not failover on 503") -} - -func TestMultiFailover_GetBlockHeightByTime_NoSwitchOnShutdownDetected(t *testing.T) { - var secondaryCalled atomic.Bool - primary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 0, ErrShutdownDetected - }, - } - secondary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - secondaryCalled.Store(true) - return 500, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary, secondary) - require.NoError(t, err) - - fc.attemptTimeout = 100 * time.Millisecond - fc.registry.HealthCheckInterval = 1 * time.Hour - fc.registry.ConsecutiveThreshold = 1 - fc.registry.PromotionCooldown = 0 - defer fc.Close() - - _, err = fc.GetBlockHeightByTime(context.Background(), 1234567890) - require.Error(t, err) - assert.True(t, errors.Is(err, ErrShutdownDetected)) - assert.False(t, secondaryCalled.Load(), "should not failover on shutdown") -} - -func TestMultiFailover_GetBlockHeightByTime_ThreeClients_CascadeToTertiary(t *testing.T) { - connErr := &net.OpError{Op: "dial", Net: "tcp", Err: errors.New("connection refused")} - - primary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 0, connErr - }, - } - secondary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 0, connErr - }, - } - tertiary := &mockHeimdallClient{ - getBlockHeightByTimeFn: func(_ context.Context, _ int64) (int64, error) { - return 750, nil - }, - } - - fc := newInstantMulti(primary, secondary, tertiary) - defer fc.Close() - - height, err := fc.GetBlockHeightByTime(context.Background(), 1234567890) - require.NoError(t, err) - assert.Equal(t, int64(750), height) - - assert.GreaterOrEqual(t, primary.hits.Load(), int32(1), "primary should have been tried") - assert.GreaterOrEqual(t, secondary.hits.Load(), int32(1), "secondary should have been tried") - assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") -} - // --- StateSyncEventsByTime failover tests --- func TestMultiFailover_StateSyncEventsByTime_SwitchOnPrimaryDown(t *testing.T) { @@ -1735,27 +1439,3 @@ func TestMultiFailover_StateSyncEventsByTime_CapsAttemptTimeoutWithGlobalPaginat _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) require.NoError(t, err) } - -func TestMultiFailover_StateSyncEventsAtHeight_CapsAttemptTimeoutWithGlobalPaginationDeadline(t *testing.T) { - primary := &mockHeimdallClient{ - stateSyncEventsAtHeightFn: func(ctx context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - deadline, ok := ctx.Deadline() - require.True(t, ok, "paginated call should have a deadline") - - remaining := time.Until(deadline) - assert.LessOrEqual(t, remaining, time.Minute+2*time.Second, "global pagination deadline should cap per-attempt timeout") - assert.Greater(t, remaining, 55*time.Second, "global pagination deadline should be close to 1 minute") - - return []*clerk.EventRecordWithTime{}, nil - }, - } - - fc, err := NewMultiHeimdallClient(primary) - require.NoError(t, err) - defer fc.Close() - - fc.attemptTimeout = 2 * time.Minute - - _, err = fc.StateSyncEventsAtHeight(context.Background(), 1, 9999, 123) - require.NoError(t, err) -} diff --git a/consensus/bor/heimdall/metrics.go b/consensus/bor/heimdall/metrics.go index 990915e351..05ac753d41 100644 --- a/consensus/bor/heimdall/metrics.go +++ b/consensus/bor/heimdall/metrics.go @@ -29,8 +29,6 @@ const ( MilestoneLastNoAckRequest requestType = "milestone-last-no-ack" MilestoneIDRequest requestType = "milestone-id" StatusRequest requestType = "status" - BlockHeightByTimeRequest requestType = "block-height-by-time" - StateSyncAtHeightRequest requestType = "state-sync-at-height" StateSyncByTimeRequest requestType = "state-sync-by-time" ) @@ -115,20 +113,6 @@ var ( }, timer: metrics.NewRegisteredTimer("client/requests/milestoneid/duration", nil), }, - BlockHeightByTimeRequest: { - request: map[bool]*metrics.Meter{ - true: metrics.NewRegisteredMeter("client/requests/blockheightbytime/valid", nil), - false: metrics.NewRegisteredMeter("client/requests/blockheightbytime/invalid", nil), - }, - timer: metrics.NewRegisteredTimer("client/requests/blockheightbytime/duration", nil), - }, - StateSyncAtHeightRequest: { - request: map[bool]*metrics.Meter{ - true: metrics.NewRegisteredMeter("client/requests/statesyncatheight/valid", nil), - false: metrics.NewRegisteredMeter("client/requests/statesyncatheight/invalid", nil), - }, - timer: metrics.NewRegisteredTimer("client/requests/statesyncatheight/duration", nil), - }, StateSyncByTimeRequest: { request: map[bool]*metrics.Meter{ true: metrics.NewRegisteredMeter("client/requests/statesyncbytime/valid", nil), diff --git a/consensus/bor/heimdall/state_sync_url_test.go b/consensus/bor/heimdall/state_sync_url_test.go index 5c266ba1e4..026cd5184a 100644 --- a/consensus/bor/heimdall/state_sync_url_test.go +++ b/consensus/bor/heimdall/state_sync_url_test.go @@ -9,51 +9,6 @@ import ( "github.com/stretchr/testify/require" ) -func TestStateSyncsAtHeightURL_Format(t *testing.T) { - t.Parallel() - - fromID := uint64(42) - heimdallHeight := int64(9999) - toTime := int64(1700000000) // 2023-11-14T22:13:20Z - - u, err := visibleAtHeightURL("http://bor0", fromID, heimdallHeight, toTime) - require.NoError(t, err) - - // Path must be clerk/state-syncs-at-height - require.True(t, strings.HasSuffix(u.Path, "clerk/state-syncs-at-height"), - "expected path to end with clerk/state-syncs-at-height, got %s", u.Path) - - // Validate individual query parameters using parsed values (not raw string) - q := u.Query() - - expectedTime := time.Unix(toTime, 0).UTC().Format(time.RFC3339Nano) - require.Equal(t, expectedTime, q.Get("to_time"), "to_time should be RFC3339Nano formatted") - require.NotEqual(t, fmt.Sprintf("%d", toTime), q.Get("to_time"), "to_time should NOT be raw unix seconds") - require.Equal(t, fmt.Sprintf("%d", fromID), q.Get("from_id")) - require.Equal(t, fmt.Sprintf("%d", heimdallHeight), q.Get("heimdall_height")) - require.Equal(t, fmt.Sprintf("%d", stateFetchLimit), q.Get("pagination.limit")) -} - -func TestBlockHeightByTimeURL_Format(t *testing.T) { - t.Parallel() - - cutoffTime := int64(1700000000) - - u, err := blockHeightByTimeURL("http://bor0", cutoffTime) - require.NoError(t, err) - - // Path must be clerk/block-height-by-time - require.True(t, strings.HasSuffix(u.Path, "clerk/block-height-by-time"), - "expected path to end with clerk/block-height-by-time, got %s", u.Path) - - // cutoff_time should be raw unix seconds (integer) - require.Contains(t, u.RawQuery, fmt.Sprintf("cutoff_time=%d", cutoffTime)) - - // Full URL sanity check - expected := fmt.Sprintf("http://bor0/clerk/block-height-by-time?cutoff_time=%d", cutoffTime) - require.Equal(t, expected, u.String()) -} - func TestStateSyncsByTimeURL_Format(t *testing.T) { t.Parallel() diff --git a/consensus/bor/heimdallapp/state_sync.go b/consensus/bor/heimdallapp/state_sync.go index d9942dfed3..76bfb9d4c6 100644 --- a/consensus/bor/heimdallapp/state_sync.go +++ b/consensus/bor/heimdallapp/state_sync.go @@ -38,47 +38,6 @@ func (h *HeimdallAppClient) StateSyncEvents(ctx context.Context, fromID uint64, return totalRecords, nil } -// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height. -// Uses the clerk query server to apply the same visibility_height filtering as gRPC/HTTP paths. -func (h *HeimdallAppClient) StateSyncEventsAtHeight(_ context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - totalRecords := make([]*clerk.EventRecordWithTime, 0) - - queryServer := keeper.NewQueryServer(&h.hApp.ClerkKeeper) - - for { - req := &types.RecordListVisibleAtHeightRequest{ - FromId: fromID, - HeimdallHeight: heimdallHeight, - ToTime: time.Unix(toTime, 0), - Pagination: query.PageRequest{Limit: stateFetchLimit}, - } - - res, err := queryServer.GetRecordListVisibleAtHeight(h.NewContext(), req) - if err != nil { - return nil, err - } - - totalRecords = append(totalRecords, toEvents(res.EventRecords)...) - - if len(res.EventRecords) < stateFetchLimit { - break - } - - fromID += uint64(stateFetchLimit) - } - - sort.SliceStable(totalRecords, func(i, j int) bool { - return totalRecords[i].ID < totalRecords[j].ID - }) - - return totalRecords, nil -} - -// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff unix timestamp. -func (h *HeimdallAppClient) GetBlockHeightByTime(_ context.Context, cutoffTime int64) (int64, error) { - return h.hApp.ClerkKeeper.GetBlockHeightByTime(h.NewContext(), cutoffTime) -} - // StateSyncEventsByTime fetches state sync events using the combined endpoint that // resolves the Heimdall height from the cutoff time internally. func (h *HeimdallAppClient) StateSyncEventsByTime(_ context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { diff --git a/consensus/bor/heimdallgrpc/state_sync.go b/consensus/bor/heimdallgrpc/state_sync.go index 5db523a0a2..fd1ec88eb0 100644 --- a/consensus/bor/heimdallgrpc/state_sync.go +++ b/consensus/bor/heimdallgrpc/state_sync.go @@ -86,72 +86,6 @@ func (h *HeimdallGRPCClient) StateSyncEvents(ctx context.Context, fromID uint64, return eventRecords, nil } -// StateSyncEventsAtHeight fetches state sync events visible at a specific Heimdall height -// using the native gRPC GetRecordListVisibleAtHeight endpoint. -func (h *HeimdallGRPCClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - log.Info("Fetching state sync events at height (gRPC)", "fromID", fromID, "toTime", toTime, "heimdallHeight", heimdallHeight) - - var err error - - globalCtx, cancel := context.WithTimeout(ctx, stateSyncTotalTimeout) - defer cancel() - - start := time.Now() - ctx = heimdall.WithRequestType(globalCtx, heimdall.StateSyncAtHeightRequest) - - defer func() { - heimdall.SendMetrics(ctx, start, err == nil) - }() - - eventRecords := make([]*clerk.EventRecordWithTime, 0) - - for { - req := &types.RecordListVisibleAtHeightRequest{ - FromId: fromID, - HeimdallHeight: heimdallHeight, - ToTime: time.Unix(toTime, 0), - Pagination: query.PageRequest{Limit: stateFetchLimit}, - } - - var res *types.RecordListVisibleAtHeightResponse - pageCtx, pageCancel := context.WithTimeout(ctx, defaultTimeout) - res, err = h.clerkQueryClient.GetRecordListVisibleAtHeight(pageCtx, req) - pageCancel() - if err != nil { - return nil, err - } - - events := res.GetEventRecords() - - for _, event := range events { - eventRecord := &clerk.EventRecordWithTime{ - EventRecord: clerk.EventRecord{ - ID: event.Id, - Contract: common.HexToAddress(event.Contract), - Data: event.Data, - TxHash: common.HexToHash(event.TxHash), - LogIndex: event.LogIndex, - ChainID: event.BorChainId, - }, - Time: event.RecordTime, - } - eventRecords = append(eventRecords, eventRecord) - } - - if len(events) < stateFetchLimit { - break - } - - fromID += uint64(stateFetchLimit) - } - - sort.SliceStable(eventRecords, func(i, j int) bool { - return eventRecords[i].ID < eventRecords[j].ID - }) - - return eventRecords, nil -} - // StateSyncEventsByTime fetches state sync events using the combined endpoint that // resolves the Heimdall height from the cutoff time internally. func (h *HeimdallGRPCClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { @@ -216,32 +150,3 @@ func (h *HeimdallGRPCClient) StateSyncEventsByTime(ctx context.Context, fromID u return eventRecords, nil } - -// GetBlockHeightByTime returns the Heimdall block height at or before the given cutoff -// unix timestamp using the native gRPC GetBlockHeightByTime endpoint. -func (h *HeimdallGRPCClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { - log.Info("Fetching block height by time (gRPC)", "cutoffTime", cutoffTime) - - var err error - - start := time.Now() - ctx = heimdall.WithRequestType(ctx, heimdall.BlockHeightByTimeRequest) - - defer func() { - heimdall.SendMetrics(ctx, start, err == nil) - }() - - reqCtx, cancel := context.WithTimeout(ctx, defaultTimeout) - defer cancel() - - req := &types.BlockHeightByTimeRequest{ - CutoffTime: cutoffTime, - } - - res, err := h.clerkQueryClient.GetBlockHeightByTime(reqCtx, req) - if err != nil { - return 0, err - } - - return res.Height, nil -} diff --git a/go.mod b/go.mod index 5b442f3325..35bd568904 100644 --- a/go.mod +++ b/go.mod @@ -352,7 +352,7 @@ require ( replace ( cosmossdk.io/client/v2 => github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 // Same as heimdall-v2. // TODO marcello update to final version once released - github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876 + github.com/0xPolygon/heimdall-v2 => github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260406091052-f1ea38254ac0 github.com/Masterminds/goutils => github.com/Masterminds/goutils v1.1.1 github.com/cometbft/cometbft => github.com/0xPolygon/cometbft v0.3.6-polygon github.com/cometbft/cometbft-db => github.com/0xPolygon/cometbft-db v0.14.1-polygon diff --git a/go.sum b/go.sum index 928e15eb69..3ca6b3536c 100644 --- a/go.sum +++ b/go.sum @@ -86,8 +86,8 @@ github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6 h1:+6AxZcMTWHaRHV0HILf/r github.com/0xPolygon/cosmos-sdk/client/v2 v2.0.0-beta.6/go.mod h1:4p0P6o0ro+FizakJUYS9SeM94RNbv0thLmkHRw5o5as= github.com/0xPolygon/crand v1.0.3 h1:BYYflmgLhmGPEgqtopG4muq6wV6DOkwD8uPymNz5WeQ= github.com/0xPolygon/crand v1.0.3/go.mod h1:km4366oC7EVFl1xNUCwzxUXNM10swZqd8LZ0E5SgbAE= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876 h1:miDtsRgAYA+loiUpPeYgr+RvnyxCr4ni5bWmZAYRQ2A= -github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260403074111-60332eecc876/go.mod h1:wap2yjeiFg1j6Sew0UtCxr9qx+bxi1u/NMH4oStEg5M= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260406091052-f1ea38254ac0 h1:rxebnDDy61MkxnrzQ3jvZ3xZqJfi+0wAC6J6kcYDVR0= +github.com/0xPolygon/heimdall-v2 v0.6.1-0.20260406091052-f1ea38254ac0/go.mod h1:wap2yjeiFg1j6Sew0UtCxr9qx+bxi1u/NMH4oStEg5M= github.com/0xPolygon/polyproto v0.0.7 h1:Ody+kFyCRK4QXRPXbsP5pdxKrDgwAAXtFB8NPgaIxRs= github.com/0xPolygon/polyproto v0.0.7/go.mod h1:2Iw93k2LismvckKKeXQITuhJH9vLbqOa212AMskH6no= github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 h1:/vQbFIOMbk2FiG/kXiLl8BRyzTWDw7gX/Hz7Dd5eDMs= diff --git a/tests/bor/mocks/IHeimdallClient.go b/tests/bor/mocks/IHeimdallClient.go index 9cd0ec28c1..11f510e45b 100644 --- a/tests/bor/mocks/IHeimdallClient.go +++ b/tests/bor/mocks/IHeimdallClient.go @@ -1,5 +1,5 @@ // Code generated by MockGen. DO NOT EDIT. -// Source: heimdall.go +// Source: consensus/bor/heimdall.go // Package mocks is a generated GoMock package. package mocks @@ -8,13 +8,12 @@ import ( context "context" reflect "reflect" - gomock "go.uber.org/mock/gomock" - types "github.com/0xPolygon/heimdall-v2/x/bor/types" - coretypes "github.com/cometbft/cometbft/rpc/core/types" + types0 "github.com/cometbft/cometbft/rpc/core/types" clerk "github.com/ethereum/go-ethereum/consensus/bor/clerk" checkpoint "github.com/ethereum/go-ethereum/consensus/bor/heimdall/checkpoint" milestone "github.com/ethereum/go-ethereum/consensus/bor/heimdall/milestone" + gomock "go.uber.org/mock/gomock" ) // MockIHeimdallClient is a mock of IHeimdallClient interface. @@ -113,10 +112,10 @@ func (mr *MockIHeimdallClientMockRecorder) FetchMilestoneCount(ctx interface{}) } // FetchStatus mocks base method. -func (m *MockIHeimdallClient) FetchStatus(ctx context.Context) (*coretypes.SyncInfo, error) { +func (m *MockIHeimdallClient) FetchStatus(ctx context.Context) (*types0.SyncInfo, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "FetchStatus", ctx) - ret0, _ := ret[0].(*coretypes.SyncInfo) + ret0, _ := ret[0].(*types0.SyncInfo) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -157,34 +156,19 @@ func (mr *MockIHeimdallClientMockRecorder) GetSpan(ctx, spanID interface{}) *gom return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSpan", reflect.TypeOf((*MockIHeimdallClient)(nil).GetSpan), ctx, spanID) } -// GetBlockHeightByTime mocks base method. -func (m *MockIHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetBlockHeightByTime", ctx, cutoffTime) - ret0, _ := ret[0].(int64) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// GetBlockHeightByTime indicates an expected call of GetBlockHeightByTime. -func (mr *MockIHeimdallClientMockRecorder) GetBlockHeightByTime(ctx, cutoffTime interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetBlockHeightByTime", reflect.TypeOf((*MockIHeimdallClient)(nil).GetBlockHeightByTime), ctx, cutoffTime) -} - -// StateSyncEventsAtHeight mocks base method. -func (m *MockIHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { +// StateSyncEvents mocks base method. +func (m *MockIHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StateSyncEventsAtHeight", ctx, fromID, toTime, heimdallHeight) + ret := m.ctrl.Call(m, "StateSyncEvents", ctx, fromID, to) ret0, _ := ret[0].([]*clerk.EventRecordWithTime) ret1, _ := ret[1].(error) return ret0, ret1 } -// StateSyncEventsAtHeight indicates an expected call of StateSyncEventsAtHeight. -func (mr *MockIHeimdallClientMockRecorder) StateSyncEventsAtHeight(ctx, fromID, toTime, heimdallHeight interface{}) *gomock.Call { +// StateSyncEvents indicates an expected call of StateSyncEvents. +func (mr *MockIHeimdallClientMockRecorder) StateSyncEvents(ctx, fromID, to interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEventsAtHeight", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEventsAtHeight), ctx, fromID, toTime, heimdallHeight) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEvents", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEvents), ctx, fromID, to) } // StateSyncEventsByTime mocks base method. @@ -202,21 +186,6 @@ func (mr *MockIHeimdallClientMockRecorder) StateSyncEventsByTime(ctx, fromID, to return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEventsByTime", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEventsByTime), ctx, fromID, toTime) } -// StateSyncEvents mocks base method. -func (m *MockIHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "StateSyncEvents", ctx, fromID, to) - ret0, _ := ret[0].([]*clerk.EventRecordWithTime) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// StateSyncEvents indicates an expected call of StateSyncEvents. -func (mr *MockIHeimdallClientMockRecorder) StateSyncEvents(ctx, fromID, to interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "StateSyncEvents", reflect.TypeOf((*MockIHeimdallClient)(nil).StateSyncEvents), ctx, fromID, to) -} - // MockIHeimdallWSClient is a mock of IHeimdallWSClient interface. type MockIHeimdallWSClient struct { ctrl *gomock.Controller From 85521b2acf07a0437b203e0591e3c6ca75e853b8 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Mon, 6 Apr 2026 18:05:42 +0200 Subject: [PATCH 27/28] address comments --- .../bor/heimdall/failover_client_test.go | 24 ++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index fad8973e19..a4df0d1964 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -1416,7 +1416,7 @@ func TestMultiFailover_StateSyncEventsByTime_ThreeClients_CascadeToTertiary(t *t assert.GreaterOrEqual(t, tertiary.hits.Load(), int32(1), "tertiary should have been called") } -func TestMultiFailover_StateSyncEventsByTime_CapsAttemptTimeoutWithGlobalPaginationDeadline(t *testing.T) { +func TestMultiFailover_StateSyncEventsByTime_UsesGlobalPaginationDeadlineWhenAttemptTimeoutExceedsIt(t *testing.T) { primary := &mockHeimdallClient{ stateSyncEventsByTimeFn: func(ctx context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { deadline, ok := ctx.Deadline() @@ -1439,3 +1439,25 @@ func TestMultiFailover_StateSyncEventsByTime_CapsAttemptTimeoutWithGlobalPaginat _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) require.NoError(t, err) } + +func TestMultiFailover_StateSyncEventsByTime_UsesDefaultPerAttemptTimeoutWhenSmallerThanGlobalDeadline(t *testing.T) { + primary := &mockHeimdallClient{ + stateSyncEventsByTimeFn: func(ctx context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { + deadline, ok := ctx.Deadline() + require.True(t, ok, "paginated call should have a deadline") + + remaining := time.Until(deadline) + assert.LessOrEqual(t, remaining, defaultAttemptTimeout+2*time.Second, "default per-attempt timeout should bound the call") + assert.Greater(t, remaining, 25*time.Second, "default per-attempt timeout should be close to 30 seconds") + + return []*clerk.EventRecordWithTime{}, nil + }, + } + + fc, err := NewMultiHeimdallClient(primary) + require.NoError(t, err) + defer fc.Close() + + _, err = fc.StateSyncEventsByTime(context.Background(), 1, 9999) + require.NoError(t, err) +} From 432627223096d277092ba69c338256a92657bdb4 Mon Sep 17 00:00:00 2001 From: marcello33 Date: Mon, 6 Apr 2026 20:50:59 +0200 Subject: [PATCH 28/28] remove dead code --- consensus/bor/bor_test.go | 12 ----- .../bor/heimdall/failover_client_test.go | 44 +++++-------------- consensus/bor/span_store_test.go | 32 -------------- eth/ethconfig/config_test.go | 6 --- eth/handler_bor_test.go | 7 --- eth/tracers/data.csv | 18 ++++---- 6 files changed, 20 insertions(+), 99 deletions(-) diff --git a/consensus/bor/bor_test.go b/consensus/bor/bor_test.go index 7c9c47ef03..b1983a141c 100644 --- a/consensus/bor/bor_test.go +++ b/consensus/bor/bor_test.go @@ -119,12 +119,6 @@ func (f *failingHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, func (f *failingHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return nil, errors.New("fetch status failed") } -func (f *failingHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) { - return 0, errors.New("get block height by time failed") -} -func (f *failingHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, errors.New("state sync events at height failed") -} func (f *failingHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, errors.New("state sync events by time failed") } @@ -2983,12 +2977,6 @@ func (m *mockHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, er func (m *mockHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } -func (m *mockHeimdallClient) GetBlockHeightByTime(_ context.Context, _ int64) (int64, error) { - return 0, nil -} -func (m *mockHeimdallClient) StateSyncEventsAtHeight(_ context.Context, _ uint64, _ int64, _ int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { return m.events, nil } diff --git a/consensus/bor/heimdall/failover_client_test.go b/consensus/bor/heimdall/failover_client_test.go index a4df0d1964..410a60ee3e 100644 --- a/consensus/bor/heimdall/failover_client_test.go +++ b/consensus/bor/heimdall/failover_client_test.go @@ -24,19 +24,17 @@ import ( // mockHeimdallClient is a configurable mock implementing the Endpoint interface. type mockHeimdallClient struct { - getSpanFn func(ctx context.Context, spanID uint64) (*types.Span, error) - getLatestSpanFn func(ctx context.Context) (*types.Span, error) - stateSyncEventsFn func(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) - fetchCheckpointFn func(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) - fetchCheckpointCntFn func(ctx context.Context) (int64, error) - fetchMilestoneFn func(ctx context.Context) (*milestone.Milestone, error) - fetchMilestoneCntFn func(ctx context.Context) (int64, error) - fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) - stateSyncEventsAtHeightFn func(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) - stateSyncEventsByTimeFn func(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) - getBlockHeightByTimeFn func(ctx context.Context, cutoffTime int64) (int64, error) - closeFn func() - hits atomic.Int32 + getSpanFn func(ctx context.Context, spanID uint64) (*types.Span, error) + getLatestSpanFn func(ctx context.Context) (*types.Span, error) + stateSyncEventsFn func(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) + fetchCheckpointFn func(ctx context.Context, number int64) (*checkpoint.Checkpoint, error) + fetchCheckpointCntFn func(ctx context.Context) (int64, error) + fetchMilestoneFn func(ctx context.Context) (*milestone.Milestone, error) + fetchMilestoneCntFn func(ctx context.Context) (int64, error) + fetchStatusFn func(ctx context.Context) (*ctypes.SyncInfo, error) + stateSyncEventsByTimeFn func(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) + closeFn func() + hits atomic.Int32 } func (m *mockHeimdallClient) StateSyncEvents(ctx context.Context, fromID uint64, to int64) ([]*clerk.EventRecordWithTime, error) { @@ -125,26 +123,6 @@ func (m *mockHeimdallClient) Close() { } } -func (m *mockHeimdallClient) GetBlockHeightByTime(ctx context.Context, cutoffTime int64) (int64, error) { - m.hits.Add(1) - - if m.getBlockHeightByTimeFn != nil { - return m.getBlockHeightByTimeFn(ctx, cutoffTime) - } - - return 100, nil -} - -func (m *mockHeimdallClient) StateSyncEventsAtHeight(ctx context.Context, fromID uint64, toTime int64, heimdallHeight int64) ([]*clerk.EventRecordWithTime, error) { - m.hits.Add(1) - - if m.stateSyncEventsAtHeightFn != nil { - return m.stateSyncEventsAtHeightFn(ctx, fromID, toTime, heimdallHeight) - } - - return []*clerk.EventRecordWithTime{}, nil -} - func (m *mockHeimdallClient) StateSyncEventsByTime(ctx context.Context, fromID uint64, toTime int64) ([]*clerk.EventRecordWithTime, error) { m.hits.Add(1) diff --git a/consensus/bor/span_store_test.go b/consensus/bor/span_store_test.go index 84fcc70f48..39713e1263 100644 --- a/consensus/bor/span_store_test.go +++ b/consensus/bor/span_store_test.go @@ -84,13 +84,6 @@ func (h *MockHeimdallClient) GetLatestSpan(ctx context.Context) (*types.Span, er func (h *MockHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } - -func (h *MockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (h *MockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (h *MockHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } @@ -407,13 +400,6 @@ func (h *MockOverlappingHeimdallClient) Close() { func (h *MockOverlappingHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } - -func (h *MockOverlappingHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (h *MockOverlappingHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (h *MockOverlappingHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } @@ -978,12 +964,6 @@ func (d *dynamicHeimdallClient) Close() {} func (d *dynamicHeimdallClient) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } -func (d *dynamicHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (d *dynamicHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (d *dynamicHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } @@ -1104,12 +1084,6 @@ func (m *MockSyncStatusClient) FetchMilestoneID(ctx context.Context, milestoneID panic("not implemented") } func (m *MockSyncStatusClient) Close() {} -func (m *MockSyncStatusClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (m *MockSyncStatusClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (m *MockSyncStatusClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } @@ -1497,12 +1471,6 @@ func (h *TimeoutHeimdallClient) FetchMilestoneCount(ctx context.Context) (int64, panic("not implemented") } func (h *TimeoutHeimdallClient) Close() {} -func (h *TimeoutHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (h *TimeoutHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (h *TimeoutHeimdallClient) StateSyncEventsByTime(context.Context, uint64, int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } diff --git a/eth/ethconfig/config_test.go b/eth/ethconfig/config_test.go index 60359b5771..10edfa75ee 100644 --- a/eth/ethconfig/config_test.go +++ b/eth/ethconfig/config_test.go @@ -49,12 +49,6 @@ func (m *mockHeimdallClient) FetchMilestoneCount(context.Context) (int64, error) func (m *mockHeimdallClient) FetchStatus(context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } -func (m *mockHeimdallClient) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (m *mockHeimdallClient) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (m *mockHeimdallClient) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } diff --git a/eth/handler_bor_test.go b/eth/handler_bor_test.go index 94c9ceae21..754bec368b 100644 --- a/eth/handler_bor_test.go +++ b/eth/handler_bor_test.go @@ -64,13 +64,6 @@ func (m *mockHeimdall) Close() {} func (m *mockHeimdall) FetchStatus(ctx context.Context) (*ctypes.SyncInfo, error) { return &ctypes.SyncInfo{CatchingUp: false}, nil } - -func (m *mockHeimdall) GetBlockHeightByTime(context.Context, int64) (int64, error) { - return 0, nil -} -func (m *mockHeimdall) StateSyncEventsAtHeight(context.Context, uint64, int64, int64) ([]*clerk.EventRecordWithTime, error) { - return nil, nil -} func (m *mockHeimdall) StateSyncEventsByTime(_ context.Context, _ uint64, _ int64) ([]*clerk.EventRecordWithTime, error) { return nil, nil } diff --git a/eth/tracers/data.csv b/eth/tracers/data.csv index ea81ea7ab5..c02b8981c7 100644 --- a/eth/tracers/data.csv +++ b/eth/tracers/data.csv @@ -1,28 +1,28 @@ TransactionIndex, Incarnation, VersionTxIdx, VersionInc, Path, Operation -0 , 0, -1 , -1, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read -0 , 0, -1 , -1, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read 0 , 0, -1 , -1, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Read 0 , 0, -1 , -1, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Read +0 , 0, -1 , -1, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read +0 , 0, -1 , -1, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read 0 , 0, -1 , -1, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write 0 , 0, -1 , -1, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 0 , 0, -1 , -1, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Write 0 , 0, -1 , -1, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write -1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read 1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Read 1 , 0, 0 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Read 1 , 0, 0 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read +1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read +1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write 1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write 1 , 0, 0 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 1 , 0, 0 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Write -1 , 0, 0 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write 2 , 0, 1 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Read 2 , 0, 1 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Read 2 , 0, 1 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read 2 , 0, 1 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read -2 , 0, 1 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write -2 , 0, 1 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 2 , 0, 1 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Write 2 , 0, 1 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write +2 , 0, 1 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write +2 , 0, 1 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 3 , 0, 2 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Read 3 , 0, 2 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Read 3 , 0, 2 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read @@ -31,11 +31,11 @@ TransactionIndex, Incarnation, VersionTxIdx, VersionInc, Path, Operation 3 , 0, 2 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write 3 , 0, 2 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 3 , 0, 2 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Write -4 , 0, 3 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read -4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read 4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Read 4 , 0, 3 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Read -4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write +4 , 0, 3 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Read +4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Read 4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000001, Write 4 , 0, 3 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000001, Write 4 , 0, 3 , 0, 000000000000000000000000000000000000dead00000000000000000000000000000000000000000000000000000000000000000103, Write +4 , 0, 3 , 0, 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000103, Write