NOMT storage by default in TestRollup

.config/nextest.toml

@@ -29,6 +29,20 @@ filter = 'package(sov-sequencer) & test(/flaky_test_archival_state_is_immediatel
 priority = 95
 threads-required = 3
 
+#[[profile.default.overrides]]
+#filter = 'package(sov-sequencer) & test(/seq_behind_deferred_slots_count|test_no_crashes_on_resync|sequencer_back_pressure|heavy_blob_submission/)'
+#test-group = 'heavy-sequencer-tests'
+#threads-required = 4                                                 # Limit parallelism when these tests run
+#slow-timeout = { period = "60s", terminate-after = 10 }              # These tests need up to 10 minutes under load
+#
+## This test has a race condition in archival state queries under load - needs full isolation
+#[[profile.default.overrides]]
+#filter = 'package(sov-sequencer)'
+#threads-required = 'num-cpus'
+#
+[test-groups.heavy-sequencer-tests]
+max-threads = 1  # Run these tests one at a time to avoid resource contention
+
 [[profile.default.overrides]]
 filter = 'package(sov-demo-rollup) & test(/replica::/)'
 priority = 90                                                        # try to run it first, so it doesn't add up at the end

CHANGELOG.md

@@ -1,3 +1,5 @@
+# 2026-01-14
+- #2329 Changes default storage in sov-test-utils's TestRollup to be NOMT. No impact for regular customers
 # 2026-01-16 
 - #2342 EVM: Populate the `gas_limit` for pending block.
 

crates/full-node/sov-db/src/config.rs

@@ -63,29 +63,21 @@ impl RollupDbConfig {
         Self {
             path,
             state_cache_size: Some(1_000_000), // Use a 1MB state cache for tests
-            user_commit_concurrency: Some(4),
+            user_commit_concurrency: Some(2),
             user_hashtable_buckets: Some(if cfg!(debug_assertions) {
-                2_500 // 9.77MB
+                500
             } else {
-                15_000_000
+                1_000_000
             }),
-            user_preallocate_ht: if cfg!(debug_assertions) {
-                Some(false)
-            } else {
-                None
-            },
-            user_page_cache_size: None,
-            user_leaf_cache_size: None,
+            user_preallocate_ht: Some(false),
+            user_page_cache_size: Some(16),
+            user_leaf_cache_size: Some(16),
             kernel_commit_concurrency: Some(2),
             kernel_hashtable_buckets: None,
-            kernel_preallocate_ht: if cfg!(debug_assertions) {
-                Some(false)
-            } else {
-                None
-            },
-            kernel_page_cache_size: None,
-            kernel_leaf_cache_size: None,
-            pruner_block_interval: Some(100),
+            kernel_preallocate_ht: Some(false),
+            kernel_page_cache_size: Some(16),
+            kernel_leaf_cache_size: Some(16),
+            pruner_block_interval: None,
             pruner_versions_to_keep: Some(20),
             pruner_max_batch_size: None,
         }
@@ -102,13 +94,16 @@ impl RollupDbConfig {
             self.kernel_commit_concurrency
                 .expect("`kernel_commit_concurrency` concurrency must be set"),
         );
-        if cfg!(debug_assertions) {
+        if let Some(hashtable_buckets) = self.kernel_hashtable_buckets {
+            opts.hashtable_buckets(hashtable_buckets);
+        } else if cfg!(debug_assertions) {
             // 9.77MB
             opts.hashtable_buckets(2_500);
         } else {
             // 1000MB
             opts.hashtable_buckets(self.kernel_hashtable_buckets.unwrap_or(256_000));
         }
+
         if let Some(preallocate_ht) = self.kernel_preallocate_ht {
             opts.preallocate_ht(preallocate_ht);
         }
@@ -119,6 +114,7 @@ impl RollupDbConfig {
         if let Some(leaf_cache_size) = self.kernel_leaf_cache_size {
             opts.leaf_cache_size(leaf_cache_size);
         }
+
         opts.path(self.path.join("kernel_nomt_db"));
 
         opts
@@ -146,6 +142,7 @@ impl RollupDbConfig {
         if let Some(leaf_cache_size) = self.user_leaf_cache_size {
             opts.leaf_cache_size(leaf_cache_size);
         }
+
         opts.path(self.path.join("user_nomt_db"));
         opts
     }

crates/full-node/sov-sequencer/src/preferred/state_root_compute.rs

@@ -198,7 +198,7 @@ async fn compute_state_root<S: Spec, Rt: Runtime<S>>(
 
 impl<S: Spec> StateRootTask<S> {
     pub(super) fn create<Rt: Runtime<S>>(
-        mut block_excutors_shutdown_receiver: mpsc::Receiver<()>,
+        mut block_executors_shutdown_receiver: mpsc::Receiver<()>,
         check_state_roots: bool,
     ) -> (JoinHandle<()>, StateRootTask<S>) {
         let span = span!(Level::DEBUG, "state_root_compute_background_task");
@@ -222,7 +222,7 @@ impl<S: Spec> StateRootTask<S> {
                             }
                         }
                     }
-                   _ = block_excutors_shutdown_receiver.recv() => {
+                   _ = block_executors_shutdown_receiver.recv() => {
                         info!(
                             "Sequencer state root background task shutdown in response to signal",
                         );
@@ -356,7 +356,7 @@ mod tests {
         ForklessStorageManager, SimpleNomtStorageManager, SimpleStorageManager,
     };
     use sov_test_utils::{
-        generate_optimistic_runtime, TestHasher, TestNomtSpec, TestSpec, TestStorageSpec,
+        generate_optimistic_runtime, TestHasher, TestJmtSpec, TestSpec, TestStorageSpec,
     };
     use tokio::task::JoinHandle;
 
@@ -367,7 +367,7 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn test_jmt_new_rollup_height_state_root_on_stale_storage() {
         let storage_manager = SimpleStorageManager::<TestStorageSpec>::new();
-        new_rollup_height_state_root_on_stale_storage::<TestSpec, _, TestRuntime<TestSpec>>(
+        new_rollup_height_state_root_on_stale_storage::<TestJmtSpec, _, TestRuntime<TestJmtSpec>>(
             storage_manager,
         )
         .await;
@@ -376,7 +376,7 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn test_jmt_known_rollup_height_state_root_on_stale_storage() {
         let storage_manager = SimpleStorageManager::<TestStorageSpec>::new();
-        known_rollup_height_state_root_on_stale_storage::<TestSpec, _, TestRuntime<TestSpec>>(
+        known_rollup_height_state_root_on_stale_storage::<TestJmtSpec, _, TestRuntime<TestJmtSpec>>(
             storage_manager,
         )
         .await;
@@ -385,13 +385,19 @@ mod tests {
     #[tokio::test(flavor = "multi_thread")]
     async fn test_nomt_new_rollup_height_state_root_on_stale_storage() {
         let storage_manager = SimpleNomtStorageManager::<TestStorageSpec>::new();
-        new_rollup_height_state_root_on_stale_storage::<TestNomtSpec, _, TestRuntime<TestNomtSpec>>(storage_manager).await;
+        new_rollup_height_state_root_on_stale_storage::<TestSpec, _, TestRuntime<TestSpec>>(
+            storage_manager,
+        )
+        .await;
     }
 
     #[tokio::test(flavor = "multi_thread")]
     async fn test_nomt_known_rollup_height_state_root_on_stale_storage() {
         let storage_manager = SimpleNomtStorageManager::<TestStorageSpec>::new();
-        known_rollup_height_state_root_on_stale_storage::<TestNomtSpec, _, TestRuntime<TestNomtSpec>>(storage_manager).await;
+        known_rollup_height_state_root_on_stale_storage::<TestSpec, _, TestRuntime<TestSpec>>(
+            storage_manager,
+        )
+        .await;
     }
 
     // Helpers go below
@@ -625,9 +631,9 @@ mod tests {
         )
         .unwrap();
         known_rollup_height_state_root_on_stale_storage_with_deep_jumps::<
-            TestNomtSpec,
+            TestSpec,
             _,
-            TestRuntime<TestNomtSpec>,
+            TestRuntime<TestSpec>,
         >(storage_manager)
         .await;
     }
@@ -777,7 +783,7 @@ mod tests {
 
         // Now we'll run a loop of...
         // - Commit the next block info, making one more set of sequencer storages become stale.
-        // - Iterate over all of the seqeuncer storages and compute the state root for each one. Make sure that they get the correct value
+        // - Iterate over all of the sequencer storages and compute the state root for each one. Make sure that they get the correct value
         //
         // This checks that stale storage is handled correctly, even when it becomes *very* stale. (For example, the storage we created for block 1 should still compute the correct root after block 10 has been written to disk.)
         for i in 0..block_infos.len() {

crates/full-node/sov-sequencer/tests/integration/pinned_cache.rs

@@ -3,19 +3,13 @@ use crate::utils::tempdir_inside_codebase_dir;
 use base64::prelude::BASE64_STANDARD;
 use base64::Engine;
 use sov_api_spec::types as api_types;
-use sov_db::storage_manager::NomtStorageManager;
 use sov_mock_da::BlockProducingConfig;
-use sov_mock_da::MockHash;
 use sov_mock_zkvm::crypto::private_key::Ed25519PrivateKey;
-use sov_modules_api::CryptoSpec;
 use sov_modules_api::RawTx;
-use sov_modules_api::Spec;
 use sov_modules_api::{DispatchCall, HexHash, HexString};
 use sov_modules_stf_blueprint::Runtime;
 use sov_sequencer::SequencerKindConfig;
-use sov_state::nomt::prover_storage::NomtProverStorage;
 use sov_state::pinned_cache::PinnedCache;
-use sov_state::DefaultStorageSpec;
 use sov_test_modules::pinned_cache::CallMessage as PinnedCacheCallMessage;
 use sov_test_modules::pinned_cache::PinnedCacheTester;
 use sov_test_modules::pinned_cache::ValueRange;
@@ -26,25 +20,14 @@ use sov_test_utils::test_rollup::GenesisSource;
 use sov_test_utils::test_rollup::RollupBuilder;
 use sov_test_utils::test_rollup::StoragePath;
 use sov_test_utils::test_rollup::TestRollup;
-use sov_test_utils::MockDaSpec;
 use sov_test_utils::RtAgnosticBlueprint;
-use sov_test_utils::TestNomtSpec as TestSpec;
+use sov_test_utils::TestSpec;
+use sov_test_utils::TestStorageManager;
 use sov_test_utils::TestUser;
 use sov_test_utils::{default_test_signed_transaction, TEST_DEFAULT_MOCK_DA_BLOCK_TIME_MS};
 use tokio_stream::StreamExt;
 
-type TestNomtBlueprint = RtAgnosticBlueprint<
-    TestSpec,
-    TestRuntime<TestSpec>,
-    NomtStorageManager<
-        MockDaSpec,
-        <<TestSpec as Spec>::CryptoSpec as CryptoSpec>::Hasher,
-        NomtProverStorage<
-            DefaultStorageSpec<<<TestSpec as Spec>::CryptoSpec as CryptoSpec>::Hasher>,
-            MockHash,
-        >,
-    >,
->;
+type TestNomtBlueprint = RtAgnosticBlueprint<TestSpec, TestRuntime<TestSpec>, TestStorageManager>;
 
 const PINNED_ADDRESS: HexHash = HexString([1u8; 32]);
 

crates/full-node/sov-sequencer/tests/integration/preferred_end_to_end.rs

@@ -55,6 +55,7 @@ use tokio_stream::StreamExt;
 use tracing::{debug, info};
 
 const DELAYED_TX_DELAY_MS: u64 = 2500;
+const NOTIFICATION_TIMEOUT: Duration = Duration::from_secs(30);
 
 generate_optimistic_runtime_with_kernel!(
     TestRuntime <=
@@ -141,11 +142,19 @@ impl DaLayerWithSubscription {
         let subscription = self.slot_subscription.as_mut().unwrap();
         while self.back_slot_notifications > 1 {
             self.back_slot_notifications -= 1;
-            subscription.next().await.unwrap().unwrap();
+            tokio::time::timeout(NOTIFICATION_TIMEOUT, subscription.next())
+                .await
+                .expect("timeout waiting for slot notification")
+                .unwrap()
+                .unwrap();
         }
 
         self.back_slot_notifications -= 1;
-        subscription.next().await.unwrap().unwrap()
+        tokio::time::timeout(NOTIFICATION_TIMEOUT, subscription.next())
+            .await
+            .expect("timeout waiting for slot notification")
+            .unwrap()
+            .unwrap()
     }
 
     /// Gets the next state update notification, clearing any *known* updates from the queue first.
@@ -154,7 +163,11 @@ impl DaLayerWithSubscription {
     /// how many state update notifications we should ultimately be receiving.
     pub async fn next_state_update_notification(&mut self) -> StateUpdateNotification {
         let subscription = self.state_update_subscription.as_mut().unwrap();
-        subscription.next().await.unwrap().unwrap()
+        tokio::time::timeout(NOTIFICATION_TIMEOUT, subscription.next())
+            .await
+            .expect("timeout waiting for state update notification")
+            .unwrap()
+            .unwrap()
     }
 
     /// Produces a slot and waits for the state update and slot notifications.
@@ -2211,9 +2224,24 @@ async fn test_no_crashes_on_resync_with_transactions() {
 
     let rollup_storage_path = builder.storage_path();
     // Next, delete everything except the preferred sequencer DB. Resync again to verify that this
-    // doesn't interfere
-    for path in ["state", "accessory", "ledger", "blob_sender"] {
-        std::fs::remove_dir_all(rollup_storage_path.path().join(path)).unwrap();
+    // doesn't interfere.
+    // NOMT uses different directories than JMT:
+    // - user_nomt_db, kernel_nomt_db (NOMT state)
+    // - state-db, archival-state-db (FlatStateDb)
+    // - accessory, ledger, blob_sender (common to both)
+    for path in [
+        "user_nomt_db",
+        "kernel_nomt_db",
+        "state-db",
+        "archival-state-db",
+        "accessory",
+        "ledger",
+        "blob_sender",
+    ] {
+        let full_path = rollup_storage_path.path().join(path);
+        if full_path.exists() {
+            std::fs::remove_dir_all(full_path).unwrap();
+        }
     }
 
     let test_rollup = builder.start().await.unwrap();

crates/full-node/sov-sequencer/tests/integration/upgradability/mod.rs

@@ -78,7 +78,7 @@ async fn sequencer_stops_if_stop_at_height_too_small(finalization_blocks: u32) {
 
     let stop_at_height = RollupHeight::new(3);
 
-    let (test_rollup, _) = create_test_rollup(
+    let (test_rollup, admin) = create_test_rollup(
         0,
         TEST_MAX_BATCH_SIZE,
         TEST_BLOB_PROCESSING_TIMEOUT,
@@ -87,28 +87,40 @@ async fn sequencer_stops_if_stop_at_height_too_small(finalization_blocks: u32) {
     )
     .await;
 
-    // Produce a few blocks to DA blocks to make sure there's a finalized slot after genesis.
-    // This is for make rollup operational, so rollup will give out slot notifications.
-    test_rollup
-        .tenderly_produce_blocks((finalization_blocks + 1) as usize)
-        .await
-        .unwrap();
-
     let mut slot_subscription = test_rollup.client.client.subscribe_slots().await.unwrap();
-    let padding_to_shutdown = 20;
-    test_rollup
-        .tenderly_produce_blocks(padding_to_shutdown)
-        .await
-        .unwrap();
-    for _ in 0..padding_to_shutdown {
-        let _slot = slot_subscription.next().await.unwrap().unwrap();
+    let api_client = test_rollup.api_client().clone();
+
+    // Produce enough finalized DA blocks so the sequencer can start accepting transactions.
+    let da = test_rollup.da_service.clone();
+    let mut da_sub = da.subscribe_finalized_header().await.unwrap();
+    for _ in 0..finalization_blocks + 3 {
+        test_rollup.da_service.produce_block_now().await.unwrap();
+        tokio::time::sleep(Duration::from_millis(300)).await;
+    }
+    da_sub.next().await;
+
+    // Produce enough blocks with transactions to advance rollup height past stop_at_height.
+    // Transactions are required to trigger batch production and increment rollup height.
+    let target_height = stop_at_height.get() + finalization_blocks as u64 + 2;
+    let mut nonce = 0;
+    while test_rollup.height().await.get() < target_height {
+        send_tx(&admin, nonce, &api_client).await.unwrap();
+        nonce += 1;
+        test_rollup.da_service.produce_block_now().await.unwrap();
+        slot_subscription.next().await;
+        tokio::time::sleep(Duration::from_millis(100)).await;
     }
 
     let slot_height = test_rollup.height().await;
 
     // Assert the condition that triggers an early return.
     assert!(stop_at_height < slot_height);
 
+    // Ensure that state up to at least `stop_at_height` is finalized and persisted to disk.
+    // Without this, non-finalized blocks' state changes are lost on shutdown, causing the
+    // restart to see a lower height than expected.
+    test_rollup.produce_enough_finalized_slots().await;
+
     let Err(err) = test_rollup
         .restart_with_heights(None, Some(stop_at_height))
         .await

crates/full-node/sov-stf-runner/proptest-regressions/state_manager/tests.txt

@@ -8,3 +8,4 @@ cc 1d6a2dd54bc060b6e87ed820d43395d35ac6b7ccbf4570bffc4e8dd117f45183 # shrinks to
 cc e0bbfa807cefc6c34f175fa5aad5c25107050efebbe5e0ef69fd96bece9d232a # shrinks to finality = 0, loop_blocks = 1, batches = 2, reshuffle_after = 1, seed = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
 cc d37cc28df7615393e7842f3a881120ac2aa9027d1efff7562894d946f7dbc90b # shrinks to finality = 5, loop_blocks = 6, batches = 1, reshuffle_after = 1, seed = [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
 cc da5788a9f42ff779f8dc98c5a993738b5d4b20d1c0f41902ad5cab46b6e09aef # shrinks to finality = 5, loop_blocks = 8, batches = 2, reshuffle_after = 3, seed = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]
+cc 33519192141d90a6faa0b98080a71c47ddff97d43516fa18089dbd812b985214 # shrinks to finality = 0, loop_blocks = 18, batches = 0, reshuffle_after = 3, seed = [2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2, 2]