diff --git a/cmd/util/cmd/verify_execution_result/cmd.go b/cmd/util/cmd/verify_execution_result/cmd.go index b226f0d8350..546be9c071a 100644 --- a/cmd/util/cmd/verify_execution_result/cmd.go +++ b/cmd/util/cmd/verify_execution_result/cmd.go @@ -88,6 +88,8 @@ func run(*cobra.Command, []string) { lg.Info().Msgf("look for 'could not verify' in the log for any mismatch, or try again with --stop_on_mismatch true to stop on first mismatch") } + var totalStats verifier.BlockVerificationStats + if flagFromTo != "" { from, to, err := parseFromTo(flagFromTo) if err != nil { @@ -95,20 +97,49 @@ func run(*cobra.Command, []string) { } lg.Info().Msgf("verifying range from %d to %d", from, to) - err = verifier.VerifyRange(lockManager, from, to, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledTransactionsEnabled) + totalStats, err = verifier.VerifyRange( + lockManager, + from, + to, + chainID, + flagDatadir, + flagChunkDataPackDir, + flagWorkerCount, + flagStopOnMismatch, + flagtransactionFeesDisabled, + flagScheduledTransactionsEnabled, + ) if err != nil { lg.Fatal().Err(err).Msgf("could not verify range from %d to %d", from, to) } - lg.Info().Msgf("finished verified range from %d to %d", from, to) + lg.Info().Msgf("finished verifying range from %d to %d", from, to) } else { lg.Info().Msgf("verifying last %d sealed blocks", flagLastK) - err := verifier.VerifyLastKHeight(lockManager, flagLastK, chainID, flagDatadir, flagChunkDataPackDir, flagWorkerCount, flagStopOnMismatch, flagtransactionFeesDisabled, flagScheduledTransactionsEnabled) + var err error + totalStats, err = verifier.VerifyLastKHeight( + lockManager, + flagLastK, + chainID, + flagDatadir, + flagChunkDataPackDir, + flagWorkerCount, + flagStopOnMismatch, + flagtransactionFeesDisabled, + flagScheduledTransactionsEnabled, + ) if err != nil { lg.Fatal().Err(err).Msg("could not verify last k height") } - lg.Info().Msgf("finished verified last %d sealed blocks", flagLastK) + lg.Info().Msgf("finished verifying last %d sealed blocks", flagLastK) } + + lg.Info().Msgf("matching chunks: %d/%d. matching transactions: %d/%d", + totalStats.MatchedChunkCount, + totalStats.MatchedChunkCount+totalStats.MismatchedChunkCount, + totalStats.MatchedTransactionCount, + totalStats.MatchedTransactionCount+totalStats.MismatchedTransactionCount, + ) } func parseFromTo(fromTo string) (from, to uint64, err error) { diff --git a/engine/verification/verifier/verifiers.go b/engine/verification/verifier/verifiers.go index d66be7c3134..d416d15a94e 100644 --- a/engine/verification/verifier/verifiers.go +++ b/engine/verification/verifier/verifiers.go @@ -41,10 +41,13 @@ func VerifyLastKHeight( stopOnMismatch bool, transactionFeesDisabled bool, scheduledTransactionsEnabled bool, -) (err error) { +) ( + totalStats BlockVerificationStats, + err error, +) { closer, storages, chunkDataPacks, state, verifier, err := initStorages(lockManager, chainID, protocolDataDir, chunkDataPackDir, transactionFeesDisabled, scheduledTransactionsEnabled) if err != nil { - return fmt.Errorf("could not init storages: %w", err) + return BlockVerificationStats{}, fmt.Errorf("could not init storages: %w", err) } defer func() { closerErr := closer() @@ -55,14 +58,18 @@ func VerifyLastKHeight( lastSealed, err := state.Sealed().Head() if err != nil { - return fmt.Errorf("could not get last sealed height: %w", err) + return BlockVerificationStats{}, fmt.Errorf("could not get last sealed height: %w", err) } root := state.Params().SealedRoot().Height // preventing overflow if k > lastSealed.Height+1 { - return fmt.Errorf("k is greater than the number of sealed blocks, k: %d, last sealed height: %d", k, lastSealed.Height) + return BlockVerificationStats{}, fmt.Errorf( + "k is greater than the number of sealed blocks, k: %d, last sealed height: %d", + k, + lastSealed.Height, + ) } from := lastSealed.Height - k + 1 @@ -78,12 +85,23 @@ func VerifyLastKHeight( log.Info().Msgf("verifying blocks from %d to %d", from, to) - err = verifyConcurrently(from, to, nWorker, stopOnMismatch, storages.Headers, chunkDataPacks, storages.Results, state, verifier, verifyHeight) + totalStats, err = verifyConcurrently( + from, + to, + nWorker, + stopOnMismatch, + storages.Headers, + chunkDataPacks, + storages.Results, + state, + verifier, + verifyHeight, + ) if err != nil { - return err + return totalStats, err } - return nil + return totalStats, nil } // VerifyRange verifies all chunks in the results of the blocks in the given range. @@ -97,10 +115,13 @@ func VerifyRange( stopOnMismatch bool, transactionFeesDisabled bool, scheduledTransactionsEnabled bool, -) (err error) { +) ( + totalStats BlockVerificationStats, + err error, +) { closer, storages, chunkDataPacks, state, verifier, err := initStorages(lockManager, chainID, protocolDataDir, chunkDataPackDir, transactionFeesDisabled, scheduledTransactionsEnabled) if err != nil { - return fmt.Errorf("could not init storages: %w", err) + return BlockVerificationStats{}, fmt.Errorf("could not init storages: %w", err) } defer func() { closerErr := closer() @@ -114,15 +135,30 @@ func VerifyRange( root := state.Params().SealedRoot().Height if from <= root { - return fmt.Errorf("cannot verify blocks before the root block, from: %d, root: %d", from, root) + return BlockVerificationStats{}, fmt.Errorf( + "cannot verify blocks before the root block, from: %d, root: %d", + from, + root, + ) } - err = verifyConcurrently(from, to, nWorker, stopOnMismatch, storages.Headers, chunkDataPacks, storages.Results, state, verifier, verifyHeight) + totalStats, err = verifyConcurrently( + from, + to, + nWorker, + stopOnMismatch, + storages.Headers, + chunkDataPacks, + storages.Results, + state, + verifier, + verifyHeight, + ) if err != nil { - return err + return totalStats, err } - return nil + return totalStats, nil } func verifyConcurrently( @@ -134,17 +170,29 @@ func verifyConcurrently( results storage.ExecutionResults, state protocol.State, verifier module.ChunkVerifier, - verifyHeight func(uint64, storage.Headers, storage.ChunkDataPacks, storage.ExecutionResults, protocol.State, module.ChunkVerifier, bool) error, -) error { + verifyHeight func( + height uint64, + headers storage.Headers, + chunkDataPacks storage.ChunkDataPacks, + results storage.ExecutionResults, + state protocol.State, + verifier module.ChunkVerifier, + stopOnMismatch bool, + ) (BlockVerificationStats, error), +) (BlockVerificationStats, error) { + tasks := make(chan uint64, int(nWorker)) ctx, cancel := context.WithCancel(context.Background()) defer cancel() // Ensure cancel is called to release resources - var lowestErr error - var lowestErrHeight = ^uint64(0) // Initialize to max value of uint64 - var mu sync.Mutex // To protect access to lowestErr and lowestErrHeight + var ( + lowestErr error + lowestErrHeight = ^uint64(0) // Initialize to max value of uint64 + totalStats BlockVerificationStats + mu sync.Mutex // To protect access to variables above and blocksStats + ) - lg := util.LogProgress( + logProgress := util.LogProgress( log.Logger, util.DefaultLogProgressConfig( fmt.Sprintf("verifying heights progress for [%v:%v]", from, to), @@ -162,8 +210,26 @@ func verifyConcurrently( if !ok { return // Exit if the tasks channel is closed } + log.Info().Uint64("height", height).Msg("verifying height") - err := verifyHeight(height, headers, chunkDataPacks, results, state, verifier, stopOnMismatch) + + blockStats, err := verifyHeight( + height, + headers, + chunkDataPacks, + results, + state, + verifier, + stopOnMismatch, + ) + + mu.Lock() + + totalStats.MatchedChunkCount += blockStats.MatchedChunkCount + totalStats.MismatchedChunkCount += blockStats.MismatchedChunkCount + totalStats.MatchedTransactionCount += blockStats.MatchedTransactionCount + totalStats.MismatchedTransactionCount += blockStats.MismatchedTransactionCount + if err != nil { log.Error().Uint64("height", height).Err(err).Msg("error encountered while verifying height") @@ -171,18 +237,18 @@ func verifyConcurrently( // error, so we need to first cancel the context to stop worker from processing further tasks // and wait until all workers are done, which will ensure all the heights before this height // that had error are processed. Then we can safely update the lowestErr and lowestErrHeight - mu.Lock() if height < lowestErrHeight { lowestErr = err lowestErrHeight = height cancel() // Cancel context to stop further task dispatch } - mu.Unlock() } else { log.Info().Uint64("height", height).Msg("verified height successfully") } - lg(1) // log progress + mu.Unlock() + + logProgress(1) } } } @@ -215,10 +281,10 @@ func verifyConcurrently( // Check if there was an error if lowestErr != nil { log.Error().Uint64("height", lowestErrHeight).Err(lowestErr).Msg("error encountered while verifying height") - return fmt.Errorf("could not verify height %d: %w", lowestErrHeight, lowestErr) + return totalStats, fmt.Errorf("could not verify height %d: %w", lowestErrHeight, lowestErr) } - return nil + return totalStats, nil } func initStorages( @@ -273,6 +339,13 @@ func initStorages( return closer, storages, chunkDataPacks, state, verifier, nil } +type BlockVerificationStats struct { + MatchedChunkCount uint64 + MismatchedChunkCount uint64 + MatchedTransactionCount uint64 + MismatchedTransactionCount uint64 +} + // verifyHeight verifies all chunks in the results of the block at the given height. // Note: it returns nil if the block is not executed. func verifyHeight( @@ -283,10 +356,13 @@ func verifyHeight( state protocol.State, verifier module.ChunkVerifier, stopOnMismatch bool, -) error { +) ( + stats BlockVerificationStats, + err error, +) { header, err := headers.ByHeight(height) if err != nil { - return fmt.Errorf("could not get block header by height %d: %w", height, err) + return BlockVerificationStats{}, fmt.Errorf("could not get block header by height %d: %w", height, err) } blockID := header.ID() @@ -295,34 +371,71 @@ func verifyHeight( if err != nil { if errors.Is(err, storage.ErrNotFound) { log.Warn().Uint64("height", height).Hex("block_id", blockID[:]).Msg("execution result not found") - return nil + return BlockVerificationStats{}, nil } - return fmt.Errorf("could not get execution result by block ID %s: %w", blockID, err) + return BlockVerificationStats{}, fmt.Errorf("could not get execution result by block ID %s: %w", blockID, err) } snapshot := state.AtBlockID(blockID) for i, chunk := range result.Chunks { chunkDataPack, err := chunkDataPacks.ByChunkID(chunk.ID()) if err != nil { - return fmt.Errorf("could not get chunk data pack by chunk ID %s: %w", chunk.ID(), err) + return BlockVerificationStats{}, fmt.Errorf("could not get chunk data pack by chunk ID %s: %w", chunk.ID(), err) } vcd, err := convert.FromChunkDataPack(chunk, chunkDataPack, header, snapshot, result) if err != nil { - return err + return BlockVerificationStats{}, err } + chunkTransactionCount := vcd.Chunk.NumberOfTransactions + _, err = verifier.Verify(vcd) if err != nil { + var collectionID flow.Identifier + if chunkDataPack.Collection != nil { + collectionID = chunkDataPack.Collection.ID() + } + if stopOnMismatch { - return fmt.Errorf("could not verify chunk (index: %v) at block %v (%v): %w", i, height, blockID, err) + return BlockVerificationStats{ + MismatchedChunkCount: 1, + MismatchedTransactionCount: chunkTransactionCount, + }, fmt.Errorf( + "could not verify chunk (index: %v, ID: %v) at block %v (%v): %w", + i, + collectionID, + height, + blockID, + err, + ) + } + + if vcd.IsSystemChunk { + log.Warn().Err(err).Msgf( + "could not verify system chunk (index: %v, ID: %v) at block %v (%v)", + i, collectionID, height, blockID, + ) + } else { + + log.Error().Err(err).Msgf( + "could not verify chunk (index: %v, ID: %v) at block %v (%v)", + i, collectionID, height, blockID, + ) } - log.Error().Err(err).Msgf("could not verify chunk (index: %v) at block %v (%v)", i, height, blockID) + stats.MismatchedChunkCount++ + stats.MismatchedTransactionCount += chunkTransactionCount + } else { + log.Info().Msgf("verified chunk (index: %v) at block %v (%v) successfully", i, height, blockID) + + stats.MatchedChunkCount++ + stats.MatchedTransactionCount += chunkTransactionCount } } - return nil + + return stats, nil } func makeVerifier( diff --git a/engine/verification/verifier/verifiers_test.go b/engine/verification/verifier/verifiers_test.go index 907d9b6915c..addc07b844f 100644 --- a/engine/verification/verifier/verifiers_test.go +++ b/engine/verification/verifier/verifiers_test.go @@ -2,19 +2,29 @@ package verifier import ( "errors" - "fmt" "testing" + "github.com/stretchr/testify/assert" + testifymock "github.com/stretchr/testify/mock" + "github.com/stretchr/testify/require" + + "github.com/onflow/flow-go/model/flow" "github.com/onflow/flow-go/module" mockmodule "github.com/onflow/flow-go/module/mock" "github.com/onflow/flow-go/state/protocol" + protocolmock "github.com/onflow/flow-go/state/protocol/mock" "github.com/onflow/flow-go/storage" - "github.com/onflow/flow-go/storage/mock" + storagemock "github.com/onflow/flow-go/storage/mock" + "github.com/onflow/flow-go/utils/unittest" unittestMocks "github.com/onflow/flow-go/utils/unittest/mocks" ) func TestVerifyConcurrently(t *testing.T) { + errMock := errors.New("mock error") + errTwo := errors.New("error 2") + errFour := errors.New("error 4") + tests := []struct { name string from uint64 @@ -32,20 +42,25 @@ func TestVerifyConcurrently(t *testing.T) { expectedErr: nil, }, { - name: "Single error at a height", - from: 1, - to: 5, - nWorker: 3, - errors: map[uint64]error{3: errors.New("mock error")}, - expectedErr: fmt.Errorf("mock error"), + name: "Single error at a height", + from: 1, + to: 5, + nWorker: 3, + errors: map[uint64]error{ + 3: errMock, + }, + expectedErr: errMock, }, { - name: "Multiple errors, lowest height returned", - from: 1, - to: 5, - nWorker: 3, - errors: map[uint64]error{2: errors.New("error 2"), 4: errors.New("error 4")}, - expectedErr: fmt.Errorf("error 2"), + name: "Multiple errors, lowest height returned", + from: 1, + to: 5, + nWorker: 3, + errors: map[uint64]error{ + 2: errTwo, + 4: errFour, + }, + expectedErr: errTwo, }, } @@ -60,22 +75,33 @@ func TestVerifyConcurrently(t *testing.T) { state protocol.State, verifier module.ChunkVerifier, stopOnMismatch bool, - ) error { + ) (BlockVerificationStats, error) { if err, ok := tt.errors[height]; ok { - return err + return BlockVerificationStats{}, err } - return nil + return BlockVerificationStats{}, nil } - mockHeaders := mock.NewHeaders(t) - mockChunkDataPacks := mock.NewChunkDataPacks(t) - mockResults := mock.NewExecutionResults(t) + mockHeaders := storagemock.NewHeaders(t) + mockChunkDataPacks := storagemock.NewChunkDataPacks(t) + mockResults := storagemock.NewExecutionResults(t) mockState := unittestMocks.NewProtocolState() mockVerifier := mockmodule.NewChunkVerifier(t) - err := verifyConcurrently(tt.from, tt.to, tt.nWorker, true, mockHeaders, mockChunkDataPacks, mockResults, mockState, mockVerifier, mockVerifyHeight) + _, err := verifyConcurrently( + tt.from, + tt.to, + tt.nWorker, + true, + mockHeaders, + mockChunkDataPacks, + mockResults, + mockState, + mockVerifier, + mockVerifyHeight, + ) if tt.expectedErr != nil { - if err == nil || errors.Is(err, tt.expectedErr) { + if err == nil || !errors.Is(err, tt.expectedErr) { t.Fatalf("expected error: %v, got: %v", tt.expectedErr, err) } } else if err != nil { @@ -84,3 +110,258 @@ func TestVerifyConcurrently(t *testing.T) { }) } } + +func TestVerifyConcurrentlyAggregatesStats(t *testing.T) { + // each height returns different stats, verify they are summed correctly + statsPerHeight := map[uint64]BlockVerificationStats{ + 1: {MatchedChunkCount: 2, MismatchedChunkCount: 0, MatchedTransactionCount: 10, MismatchedTransactionCount: 0}, + 2: {MatchedChunkCount: 1, MismatchedChunkCount: 1, MatchedTransactionCount: 5, MismatchedTransactionCount: 3}, + 3: {MatchedChunkCount: 0, MismatchedChunkCount: 2, MatchedTransactionCount: 0, MismatchedTransactionCount: 8}, + } + + mockVerifyHeight := func( + height uint64, + headers storage.Headers, + chunkDataPacks storage.ChunkDataPacks, + results storage.ExecutionResults, + state protocol.State, + verifier module.ChunkVerifier, + stopOnMismatch bool, + ) (BlockVerificationStats, error) { + return statsPerHeight[height], nil + } + + totalStats, err := verifyConcurrently( + 1, 3, 2, false, + storagemock.NewHeaders(t), + storagemock.NewChunkDataPacks(t), + storagemock.NewExecutionResults(t), + unittestMocks.NewProtocolState(), + mockmodule.NewChunkVerifier(t), + mockVerifyHeight, + ) + require.NoError(t, err) + + assert.Equal(t, uint64(3), totalStats.MatchedChunkCount) + assert.Equal(t, uint64(3), totalStats.MismatchedChunkCount) + assert.Equal(t, uint64(15), totalStats.MatchedTransactionCount) + assert.Equal(t, uint64(11), totalStats.MismatchedTransactionCount) +} + +// setupVerifyHeightMocks creates mocks for verifyHeight with the given number of chunks. +// Each chunk gets NumberOfTransactions set to txPerChunk. +// The verifier mock is returned unconfigured so the caller can set up per-chunk behavior. +func setupVerifyHeightMocks( + t *testing.T, + height uint64, + numChunks int, + txPerChunk uint64, +) ( + *storagemock.Headers, + *storagemock.ChunkDataPacks, + *storagemock.ExecutionResults, + *protocolmock.State, + *mockmodule.ChunkVerifier, + *flow.ExecutionResult, +) { + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) + blockID := header.ID() + + startState := unittest.StateCommitmentFixture() + chunks := unittest.ChunkListFixture(uint(numChunks), blockID, startState) + for _, chunk := range chunks { + chunk.NumberOfTransactions = txPerChunk + } + + result := unittest.ExecutionResultFixture() + result.BlockID = blockID + result.Chunks = chunks + + headers := storagemock.NewHeaders(t) + headers.On("ByHeight", height).Return(header, nil) + + results := storagemock.NewExecutionResults(t) + results.On("ByBlockID", blockID).Return(result, nil) + + chunkDataPacks := storagemock.NewChunkDataPacks(t) + for _, chunk := range chunks { + coll := unittest.CollectionFixture(1) + cdp := unittest.ChunkDataPackFixture(chunk.ID(), unittest.WithChunkDataPackCollection(&coll)) + chunkDataPacks.On("ByChunkID", chunk.ID()).Return(cdp, nil).Maybe() + } + + mockState := protocolmock.NewState(t) + snapshot := protocolmock.NewSnapshot(t) + mockState.On("AtBlockID", blockID).Return(snapshot) + + chunkVerifier := mockmodule.NewChunkVerifier(t) + + return headers, chunkDataPacks, results, mockState, chunkVerifier, result +} + +func TestVerifyHeight_AllChunksMatch(t *testing.T) { + height := uint64(100) + txPerChunk := uint64(10) + numChunks := 3 + + headers, chunkDataPacks, results, state, chunkVerifier, _ := setupVerifyHeightMocks(t, height, numChunks, txPerChunk) + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, nil) + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, false) + require.NoError(t, err) + + assert.Equal(t, uint64(numChunks), stats.MatchedChunkCount) + assert.Equal(t, uint64(0), stats.MismatchedChunkCount) + assert.Equal(t, uint64(numChunks)*txPerChunk, stats.MatchedTransactionCount) + assert.Equal(t, uint64(0), stats.MismatchedTransactionCount) +} + +func TestVerifyHeight_MismatchWithoutStop(t *testing.T) { + height := uint64(100) + txPerChunk := uint64(5) + numChunks := 3 + + headers, chunkDataPacks, results, state, chunkVerifier, _ := setupVerifyHeightMocks(t, height, numChunks, txPerChunk) + + verifyErr := errors.New("chunk mismatch") + // first call fails, subsequent calls succeed + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, verifyErr).Once() + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, nil) + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, false) + require.NoError(t, err) + + assert.Equal(t, uint64(2), stats.MatchedChunkCount) + assert.Equal(t, uint64(1), stats.MismatchedChunkCount) + assert.Equal(t, uint64(2)*txPerChunk, stats.MatchedTransactionCount) + assert.Equal(t, txPerChunk, stats.MismatchedTransactionCount) +} + +func TestVerifyHeight_MismatchWithStop(t *testing.T) { + height := uint64(100) + txPerChunk := uint64(5) + numChunks := 3 + + headers, chunkDataPacks, results, state, chunkVerifier, _ := setupVerifyHeightMocks(t, height, numChunks, txPerChunk) + + verifyErr := errors.New("chunk mismatch") + // first chunk fails - with stopOnMismatch=true, should return error immediately + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, verifyErr).Once() + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, true) + require.Error(t, err) + assert.ErrorIs(t, err, verifyErr) + assert.Equal(t, uint64(0), stats.MatchedChunkCount) + assert.Equal(t, uint64(1), stats.MismatchedChunkCount) + assert.Equal(t, uint64(0), stats.MatchedTransactionCount) + assert.Equal(t, txPerChunk, stats.MismatchedTransactionCount) +} + +func TestVerifyHeight_BlockNotExecuted(t *testing.T) { + height := uint64(100) + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) + blockID := header.ID() + + headers := storagemock.NewHeaders(t) + headers.On("ByHeight", height).Return(header, nil) + + results := storagemock.NewExecutionResults(t) + results.On("ByBlockID", blockID).Return(nil, storage.ErrNotFound) + + state := protocolmock.NewState(t) + chunkVerifier := mockmodule.NewChunkVerifier(t) + chunkDataPacks := storagemock.NewChunkDataPacks(t) + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, false) + require.NoError(t, err) + assert.Equal(t, BlockVerificationStats{}, stats) +} + +func TestVerifyHeight_AllChunksMismatchWithoutStop(t *testing.T) { + height := uint64(100) + txPerChunk := uint64(7) + numChunks := 3 + + headers, chunkDataPacks, results, state, chunkVerifier, _ := setupVerifyHeightMocks(t, height, numChunks, txPerChunk) + + verifyErr := errors.New("chunk mismatch") + // all chunks fail, but stopOnMismatch=false so we continue and count all mismatches + // this also exercises the system chunk path (last chunk) + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, verifyErr) + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, false) + require.NoError(t, err) + + assert.Equal(t, uint64(0), stats.MatchedChunkCount) + assert.Equal(t, uint64(numChunks), stats.MismatchedChunkCount) + assert.Equal(t, uint64(0), stats.MatchedTransactionCount) + assert.Equal(t, uint64(numChunks)*txPerChunk, stats.MismatchedTransactionCount) +} + +func TestVerifyHeight_VaryingTransactionCounts(t *testing.T) { + height := uint64(100) + header := unittest.BlockHeaderFixture(unittest.WithHeaderHeight(height)) + blockID := header.ID() + + startState := unittest.StateCommitmentFixture() + chunks := unittest.ChunkListFixture(3, blockID, startState) + // set different tx counts per chunk + chunks[0].NumberOfTransactions = 10 + chunks[1].NumberOfTransactions = 20 + chunks[2].NumberOfTransactions = 30 + + result := unittest.ExecutionResultFixture() + result.BlockID = blockID + result.Chunks = chunks + + headers := storagemock.NewHeaders(t) + headers.On("ByHeight", height).Return(header, nil) + + results := storagemock.NewExecutionResults(t) + results.On("ByBlockID", blockID).Return(result, nil) + + chunkDataPacks := storagemock.NewChunkDataPacks(t) + for _, chunk := range chunks { + coll := unittest.CollectionFixture(1) + cdp := unittest.ChunkDataPackFixture(chunk.ID(), unittest.WithChunkDataPackCollection(&coll)) + chunkDataPacks.On("ByChunkID", chunk.ID()).Return(cdp, nil).Maybe() + } + + state := protocolmock.NewState(t) + snapshot := protocolmock.NewSnapshot(t) + state.On("AtBlockID", blockID).Return(snapshot) + + verifyErr := errors.New("chunk mismatch") + chunkVerifier := mockmodule.NewChunkVerifier(t) + // chunk 0 (10 tx) matches, chunk 1 (20 tx) mismatches, chunk 2 (30 tx) matches + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, nil).Once() + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, verifyErr).Once() + chunkVerifier.On("Verify", testifymock.Anything).Return(nil, nil).Once() + + stats, err := verifyHeight(height, headers, chunkDataPacks, results, state, chunkVerifier, false) + require.NoError(t, err) + + assert.Equal(t, uint64(2), stats.MatchedChunkCount) + assert.Equal(t, uint64(1), stats.MismatchedChunkCount) + assert.Equal(t, uint64(40), stats.MatchedTransactionCount) // 10 + 30 + assert.Equal(t, uint64(20), stats.MismatchedTransactionCount) // 20 +} + +func TestVerifyHeight_HeaderNotFound(t *testing.T) { + height := uint64(100) + + headers := storagemock.NewHeaders(t) + headers.On("ByHeight", height).Return(nil, storage.ErrNotFound) + + stats, err := verifyHeight( + height, + headers, + storagemock.NewChunkDataPacks(t), + storagemock.NewExecutionResults(t), + protocolmock.NewState(t), + mockmodule.NewChunkVerifier(t), + false, + ) + require.Error(t, err) + assert.Equal(t, BlockVerificationStats{}, stats) +}