Move persistence from out of band to in-band

bloat-check/src/bin/bloat-check.rs

@@ -59,7 +59,8 @@ use rs_matter::crypto::backend::rustcrypto::RustCrypto;
 use rs_matter::crypto::{Crypto, RngCore, WeakTestOnlyRand};
 use rs_matter::dm::clusters::desc::{self, ClusterHandler as _, DescHandler};
 use rs_matter::dm::clusters::net_comm::{
-    NetCtl, NetCtlError, NetCtlStatus, NetworkScanInfo, NetworkType, Networks, WirelessCreds,
+    NetCtl, NetCtlError, NetCtlStatus, NetworkScanInfo, NetworkType, NetworksAccess,
+    SharedNetworks, WirelessCreds,
 };
 use rs_matter::dm::clusters::on_off::NoLevelControl;
 use rs_matter::dm::clusters::on_off::{self, test::TestOnOffDeviceLogic, OnOffHooks};
@@ -80,6 +81,7 @@ use rs_matter::dm::{Async, DataModel, Dataver, EmptyHandler, Endpoint, EpClMatch
 use rs_matter::error::Error;
 use rs_matter::pairing::qr::QrTextType;
 use rs_matter::pairing::DiscoveryCapabilities;
+use rs_matter::persist::{DummyKvBlobStore, SharedKvBlobStore};
 use rs_matter::respond::DefaultResponder;
 use rs_matter::sc::pase::MAX_COMM_WINDOW_TIMEOUT_SECS;
 use rs_matter::tlv::Nullable;
@@ -157,7 +159,7 @@ struct MatterStack<'a> {
     buffers: PooledBuffers<10, IMBuffer>,
     subscriptions: Subscriptions<{ DEFAULT_MAX_SUBSCRIPTIONS }>,
     events: Events<DEFAULT_BYTES_PER_BUF>,
-    networks: WifiNetworks<3>,
+    networks: SharedNetworks<WifiNetworks<3>>,
     net_ctl_state: NetCtlStateMutex,
     btp: Btp,
     wireless_mgr_buffer: MaybeUninit<[u8; MAX_CREDS_SIZE]>,
@@ -179,7 +181,7 @@ impl<'a> MatterStack<'a> {
             buffers <- PooledBuffers::init(0),
             subscriptions <- Subscriptions::init(),
             events <- Events::init(dummy_epoch),
-            networks <- WifiNetworks::init(),
+            networks <- SharedNetworks::init(WifiNetworks::init()),
             net_ctl_state <- NetCtlState::init_with_mutex(),
             btp <- Btp::init(),
             wireless_mgr_buffer: MaybeUninit::zeroed(),
@@ -191,23 +193,26 @@ impl<'a> MatterStack<'a> {
 // Fully spelled-out types for everything which is passed down as arguments to `embassy-executor` tasks
 // Necessary, because `embassy-executor` doesn't grok generics
 
+type AppNetworks = SharedNetworks<WifiNetworks<3>>;
 type AppNetCtl<'a> = NetCtlWithStatusImpl<'a, FakeWifi>;
-type AppWirelessMgr<'a> = WirelessMgr<'a, &'a WifiNetworks<3>, &'a AppNetCtl<'a>>;
+type AppWirelessMgr<'a> = WirelessMgr<'a, &'a AppNetworks, &'a AppNetCtl<'a>>;
 type AppTransport<'a> = ChainedNetwork<FakeUdp, &'a Btp, fn(&Address) -> bool>;
 type AppHandler<'a> = handler_chain_type!(
     EpClMatcher => on_off::HandlerAsyncAdaptor<on_off::OnOffHandler<'a, TestOnOffDeviceLogic, NoLevelControl>>,
     EpClMatcher => Async<desc::HandlerAdaptor<DescHandler<'a>>>
     | EmptyHandler
 );
 type AppCrypto = RustCrypto<'static, WeakTestOnlyRand>;
-type AppDmHandler<'a> = WifiHandler<'a, &'a AppNetCtl<'a>, SysHandler<'a, AppHandler<'a>>>;
+type AppDmHandler<'a> =
+    WifiHandler<'a, &'a AppNetworks, &'a AppNetCtl<'a>, SysHandler<'a, AppHandler<'a>>>;
 type AppDataModel<'a> = DataModel<
     'a,
     DEFAULT_MAX_SUBSCRIPTIONS,
     DEFAULT_BYTES_PER_BUF,
     &'a AppCrypto,
     PooledBuffers<10, IMBuffer>,
     (Node<'a>, &'a AppDmHandler<'a>),
+    SharedKvBlobStore<DummyKvBlobStore, &'static mut [u8]>,
 >;
 type AppResponder<'d, 'a> = DefaultResponder<
     'd,
@@ -217,6 +222,7 @@ type AppResponder<'d, 'a> = DefaultResponder<
     &'a AppCrypto,
     PooledBuffers<10, IMBuffer>,
     (Node<'a>, &'a AppDmHandler<'a>),
+    SharedKvBlobStore<DummyKvBlobStore, &'static mut [u8]>,
 >;
 
 #[cfg_attr(target_os = "none", main)]
@@ -324,6 +330,9 @@ fn main() -> ! {
 
     let mut rand = unwrap!(crypto.weak_rand());
 
+    let kv_buf = unsafe { stack.psm_buffer.assume_init_mut() }.as_mut_slice();
+    let kv = SharedKvBlobStore::new(DummyKvBlobStore, kv_buf);
+
     // A Wireless handler with a sample app cluster (on-off)
     let handler = mk_static!(
         AppDmHandler,
@@ -334,8 +343,8 @@ fn main() -> ! {
                 1,
                 TestOnOffDeviceLogic::new(true),
             ),
-            net_ctl,
             &stack.networks,
+            net_ctl,
         )
     );
 
@@ -351,6 +360,7 @@ fn main() -> ! {
             &stack.subscriptions,
             Some(&stack.events),
             (NODE, handler),
+            kv,
         )
     );
 
@@ -636,20 +646,21 @@ const NODE: Node<'static> = Node {
 
 /// The Data Model handler for our Matter device.
 /// The handler is the root endpoint 0 handler plus the on-off handler and its descriptor.
-fn dm_handler<'a, N>(
+fn dm_handler<'a, N, T>(
     mut rand: impl RngCore + Copy,
     on_off: on_off::OnOffHandler<'a, TestOnOffDeviceLogic, NoLevelControl>,
-    net_ctl: &'a N,
-    networks: &'a dyn Networks,
-) -> WifiHandler<'a, &'a N, SysHandler<'a, AppHandler<'a>>>
+    networks: N,
+    net_ctl: &'a T,
+) -> WifiHandler<'a, N, &'a T, SysHandler<'a, AppHandler<'a>>>
 where
-    N: NetCtl + NetCtlStatus + WifiDiag,
+    N: NetworksAccess,
+    T: NetCtl + NetCtlStatus + WifiDiag,
 {
     endpoints::with_wifi(
         &(),
         &(),
-        net_ctl,
         networks,
+        net_ctl,
         rand,
         endpoints::with_sys(
             &true,

examples/src/bin/bridge.rs

@@ -23,9 +23,10 @@ use core::pin::pin;
 
 use std::net::UdpSocket;
 
-use embassy_futures::select::{select, select4};
+use embassy_futures::select::select4;
 
 use rand::RngCore;
+
 use rs_matter::crypto::{default_crypto, Crypto};
 use rs_matter::dm::clusters::desc::{self, ClusterHandler as _};
 use rs_matter::dm::clusters::groups::{self, ClusterHandler as _};
@@ -45,7 +46,7 @@ use rs_matter::dm::{
 use rs_matter::error::Error;
 use rs_matter::pairing::qr::QrTextType;
 use rs_matter::pairing::DiscoveryCapabilities;
-use rs_matter::persist::{Psm, NO_NETWORKS};
+use rs_matter::persist::{DirKvBlobStore, SharedKvBlobStore};
 use rs_matter::respond::DefaultResponder;
 use rs_matter::sc::pase::MAX_COMM_WINDOW_TIMEOUT_SECS;
 use rs_matter::tlv::{TLVBuilderParent, Utf8StrBuilder};
@@ -67,11 +68,16 @@ fn main() -> Result<(), Error> {
     );
 
     // Create the Matter object
-    let matter = Matter::new_default(&TEST_DEV_DET, TEST_DEV_COMM, &TEST_DEV_ATT, MATTER_PORT);
+    let mut matter = Matter::new_default(&TEST_DEV_DET, TEST_DEV_COMM, &TEST_DEV_ATT, MATTER_PORT);
 
     // Need to call this once
     matter.initialize_transport_buffers()?;
 
+    // Persistence
+    let mut kv_buf = [0; 4096];
+    let mut kv = DirKvBlobStore::new_default();
+    futures_lite::future::block_on(matter.load_persist(&mut kv, &mut kv_buf))?;
+
     // Create the transport buffers
     let buffers = PooledBuffers::<10, _>::new(0);
 
@@ -106,6 +112,7 @@ fn main() -> Result<(), Error> {
         &subscriptions,
         Some(&events),
         dm_handler(rand, &on_off_handler_ep2, &on_off_handler_ep3),
+        SharedKvBlobStore::new(kv, kv_buf.as_mut_slice()),
     );
 
     // Create a default responder capable of handling up to 3 subscriptions
@@ -126,12 +133,6 @@ fn main() -> Result<(), Error> {
     let mut mdns = pin!(mdns::run_mdns(&matter, &crypto, dm.change_notify()));
     let mut transport = pin!(matter.run(&crypto, &socket, &socket, &socket));
 
-    // Create, load and run the persister
-    let mut psm: Psm<4096> = Psm::new();
-    let path = std::env::temp_dir().join("rs-matter");
-
-    psm.load(&path, &matter, NO_NETWORKS, Some(&events))?;
-
     if !matter.is_commissioned() {
         // If the device is not commissioned yet, print the QR text and code to the console
         // and enable basic commissioning
@@ -142,18 +143,11 @@ fn main() -> Result<(), Error> {
         matter.open_basic_comm_window(MAX_COMM_WINDOW_TIMEOUT_SECS, &crypto, dm.change_notify())?;
     }
 
-    let mut persist = pin!(psm.run(&path, &matter, NO_NETWORKS, Some(&events)));
-
     // Combine all async tasks in a single one
-    let all = select4(
-        &mut transport,
-        &mut mdns,
-        &mut persist,
-        select(&mut respond, &mut dm_job).coalesce(),
-    );
+    let all = select4(&mut transport, &mut mdns, &mut respond, &mut dm_job).coalesce();
 
     // Run with a simple `block_on`. Any local executor would do.
-    futures_lite::future::block_on(all.coalesce())
+    futures_lite::future::block_on(all)
 }
 
 /// The Node meta-data describing our Matter device.

examples/src/bin/chip_tool_tests.rs

@@ -21,11 +21,10 @@
 use core::pin::pin;
 
 use std::net::UdpSocket;
-use std::path::PathBuf;
 
 use async_signal::{Signal, Signals};
 
-use embassy_futures::select::{select3, select4};
+use embassy_futures::select::select3;
 
 use futures_lite::StreamExt;
 
@@ -58,7 +57,7 @@ use rs_matter::dm::{
 use rs_matter::error::Error;
 use rs_matter::pairing::qr::QrTextType;
 use rs_matter::pairing::DiscoveryCapabilities;
-use rs_matter::persist::{Psm, NO_NETWORKS};
+use rs_matter::persist::{FileKvBlobStore, SharedKvBlobStore};
 use rs_matter::respond::DefaultResponder;
 use rs_matter::sc::pase::MAX_COMM_WINDOW_TIMEOUT_SECS;
 use rs_matter::transport::MATTER_SOCKET_BIND_ADDR;
@@ -73,24 +72,14 @@ use static_cell::StaticCell;
 #[path = "../common/mdns.rs"]
 mod mdns;
 
-/// The `chip-tool` tests expect the persistent storage location
-/// to be `/tmp/chip_kvs`.
-///
-/// Moreover, this _must_ be a file rather than a directory.
-///
-/// While there seem to be some facilities to change that in some of the Python scripts,
-/// these facilities are simply not exposed at the top level test suite Python runner.
-/// TODO: Open a bug for that (and for the single-file expectation) in the `connectedhomeip` repo.
-const PERSIST_FILE_NAME: &str = "/tmp/chip_kvs";
-
 // Statically allocate in BSS the bigger objects
 // `rs-matter` supports efficient initialization of BSS objects (with `init`)
 // as well as just allocating the objects on-stack or on the heap.
 static MATTER: StaticCell<Matter> = StaticCell::new();
 static BUFFERS: StaticCell<PooledBuffers<10, rs_matter::dm::IMBuffer>> = StaticCell::new();
 static SUBSCRIPTIONS: StaticCell<DefaultSubscriptions> = StaticCell::new();
 static EVENTS: StaticCell<DefaultEvents> = StaticCell::new();
-static PSM: StaticCell<Psm<32768>> = StaticCell::new();
+static KV_BUF: StaticCell<[u8; 4096]> = StaticCell::new();
 static UNIT_TESTING_DATA: StaticCell<RefCell<UnitTestingHandlerData>> = StaticCell::new();
 
 fn main() -> Result<(), Error> {
@@ -126,6 +115,11 @@ fn main() -> Result<(), Error> {
     // Need to call this once
     matter.initialize_transport_buffers()?;
 
+    // Persistence
+    let kv_buf = KV_BUF.uninit().init_zeroed().as_mut_slice();
+    let mut kv = FileKvBlobStore::new_default();
+    futures_lite::future::block_on(matter.load_persist(&mut kv, kv_buf))?;
+
     // Create the transport buffers
     let buffers = BUFFERS.uninit().init_with(PooledBuffers::init(0));
 
@@ -176,6 +170,7 @@ fn main() -> Result<(), Error> {
             &on_off_handler_1,
             &on_off_handler_2,
         ),
+        SharedKvBlobStore::new(kv, kv_buf),
     );
 
     // Create a default responder capable of handling up to 3 subscriptions
@@ -207,18 +202,6 @@ fn main() -> Result<(), Error> {
     let mut mdns = pin!(mdns::run_mdns(matter, &crypto, dm.change_notify()));
     let mut transport = pin!(matter.run(&crypto, &socket, &socket, &socket));
 
-    // Create, load and run the persister
-    let psm = PSM.uninit().init_with(Psm::init());
-    let path = PathBuf::from(PERSIST_FILE_NAME);
-
-    info!(
-        "Persist memory: Persist (BSS)={}B, Persist fut (stack)={}B",
-        core::mem::size_of::<Psm<4096>>(),
-        core::mem::size_of_val(&psm.run(&path, matter, NO_NETWORKS, Some(events)))
-    );
-
-    psm.load(&path, matter, NO_NETWORKS, Some(events))?;
-
     // We need to always print the QR text, because the test runner expects it to be printed
     // even if the device is already commissioned
     matter.print_standard_qr_text(DiscoveryCapabilities::IP)?;
@@ -232,8 +215,6 @@ fn main() -> Result<(), Error> {
         matter.open_basic_comm_window(MAX_COMM_WINDOW_TIMEOUT_SECS, &crypto, dm.change_notify())?;
     }
 
-    let mut persist = pin!(psm.run(&path, matter, NO_NETWORKS, Some(events)));
-
     // Listen to SIGTERM because at the end of the test we'll receive it
     let mut term_signal = Signals::new([Signal::Term])?;
     let mut term = pin!(async {
@@ -242,10 +223,9 @@ fn main() -> Result<(), Error> {
     });
 
     // Combine all async tasks in a single one
-    let all = select4(
+    let all = select3(
         &mut transport,
         &mut mdns,
-        &mut persist,
         select3(&mut respond, &mut dm_job, &mut term).coalesce(),
     );