From c087079afbd8e358f0d865c1080efeecc1789c4d Mon Sep 17 00:00:00 2001 From: alanpoon Date: Tue, 31 Mar 2026 13:21:38 +0800 Subject: [PATCH 01/21] voip --- Cargo.lock | 588 ++++++++++++++++-- Cargo.toml | 2 +- src/app.rs | 88 ++- src/home/home_screen.rs | 37 +- src/home/navigation_tab_bar.rs | 14 +- src/lib.rs | 2 + src/voip/call_state.rs | 40 ++ src/voip/camera.rs | 197 ++++++ src/voip/livekit_client.rs | 154 +++++ src/voip/mod.rs | 102 ++++ src/voip/participants_list.rs | 118 ++++ src/voip/speaking.rs | 88 +++ src/voip/voip_screen.rs | 1049 ++++++++++++++++++++++++++++++++ 13 files changed, 2421 insertions(+), 58 deletions(-) create mode 100644 src/voip/call_state.rs create mode 100644 src/voip/camera.rs create mode 100644 src/voip/livekit_client.rs create mode 100644 src/voip/mod.rs create mode 100644 src/voip/participants_list.rs create mode 100644 src/voip/speaking.rs create mode 100644 src/voip/voip_screen.rs diff --git a/Cargo.lock b/Cargo.lock index 5e87a8380..e781ca1e2 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,7 +41,7 @@ version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures", ] @@ -466,7 +466,7 @@ version = "0.32.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ee74396bee4da70c2e27cf94762714c911725efe69d9e2672f998512a67a4ce4" dependencies = [ - "bindgen", + "bindgen 0.72.1", "cc", "cmake", "dunce", @@ -563,6 +563,29 @@ version = "1.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" +[[package]] +name = "bindgen" +version = "0.65.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cfdf7b466f9a4903edc73f95d6d2bcd5baf8ae620638762244d3f60143643cc5" +dependencies = [ + "bitflags 1.3.2", + "cexpr", + "clang-sys", + "lazy_static", + "lazycell", + "log", + "peeking_take_while", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash 1.1.0", + "shlex", + "syn 2.0.106", + "which", +] + [[package]] name = "bindgen" version = "0.72.1" @@ -598,6 +621,12 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5e764a1d40d510daf35e07be9eb06e75770908c27d411ee6c92109c9840eaaf7" +[[package]] +name = "bitflags" +version = "1.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" + [[package]] name = "bitflags" version = "2.10.0" @@ -645,10 +674,16 @@ dependencies = [ "arrayref", "arrayvec", "cc", - "cfg-if", + "cfg-if 1.0.4", "constant_time_eq", ] +[[package]] +name = "block" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0d8c1fef690941d3e7788d328517591fecc684c084084702d6ff1641e993699a" + [[package]] name = "block-buffer" version = "0.10.4" @@ -725,6 +760,12 @@ version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" +[[package]] +name = "bytemuck" +version = "1.25.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8efb64bd706a16a1bdde310ae86b351e4d21550d98d056f22f8a7f7a2183fec" + [[package]] name = "bytemuck" version = "1.25.0" @@ -741,6 +782,12 @@ name = "byteorder" version = "1.5.0" source = "git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements#461b05134a501b8e67f431b2706fb200d4bcf68a" +[[package]] +name = "byteorder-lite" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f1fe948ff07f4bd06c30984e69f5b4899c516a3ef74f34df92a2df2ab535495" + [[package]] name = "bytes" version = "1.11.1" @@ -789,6 +836,12 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-if" +version = "0.1.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4785bdd1c96b2a846b2bd7cc02e86b6b3dbf14e7e53446c4f54c92a361040822" + [[package]] name = "cfg-if" version = "1.0.4" @@ -807,7 +860,7 @@ version = "0.9.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c3613f74bd2eac03dad61bd53dbe620703d4371614fe0bc3b9f04dd36fe4e818" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cipher", "cpufeatures", ] @@ -937,6 +990,34 @@ dependencies = [ "cc", ] +[[package]] +name = "cocoa" +version = "0.20.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c49e86fc36d5704151f5996b7b3795385f50ce09e3be0f47a0cfde869681cf8" +dependencies = [ + "bitflags 1.3.2", + "block", + "core-foundation 0.7.0", + "core-graphics", + "foreign-types", + "libc", + "objc", +] + +[[package]] +name = "cocoa-foundation" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81411967c50ee9a1fc11365f8c585f863a22a9697c89239c452292c40ba79b0d" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "block", + "core-foundation 0.10.1", + "core-graphics-types", + "objc", +] + [[package]] name = "codespan-reporting" version = "0.12.0" @@ -1024,13 +1105,23 @@ dependencies = [ "unicode-segmentation 1.12.0 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "core-foundation" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57d24c7a13c43e870e37c1556b74555437870a04514f7685f5b354e090567171" +dependencies = [ + "core-foundation-sys 0.7.0", + "libc", +] + [[package]] name = "core-foundation" version = "0.9.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "91e195e091a93c46f7102ec7818a2aa394e1e1771c3ab4825963fa03e45afb8f" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -1040,16 +1131,70 @@ version = "0.10.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] +[[package]] +name = "core-foundation-sys" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3a71ab494c0b5b860bdc8407ae08978052417070c2ced38573a9157ad75b8ac" + [[package]] name = "core-foundation-sys" version = "0.8.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "773648b94d0e5d620f64f280777445740e61fe701025087ec8b57f45c791888b" +[[package]] +name = "core-graphics" +version = "0.19.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3889374e6ea6ab25dba90bb5d96202f61108058361f6dc72e8b03e6f8bbe923" +dependencies = [ + "bitflags 1.3.2", + "core-foundation 0.7.0", + "foreign-types", + "libc", +] + +[[package]] +name = "core-graphics-types" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d44a101f213f6c4cdc1853d4b78aef6db6bdfa3468798cc1d9912f4735013eb" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "core-foundation 0.10.1", + "libc", +] + +[[package]] +name = "core-media-sys" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "273bf3fc5bf51fd06a7766a84788c1540b6527130a0bce39e00567d6ab9f31f1" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "libc", +] + +[[package]] +name = "core-video-sys" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ecad23610ad9757664d644e369246edde1803fcb43ed72876565098a5d3828" +dependencies = [ + "cfg-if 0.1.10", + "core-foundation-sys 0.7.0", + "core-graphics", + "libc", + "metal", + "objc", +] + [[package]] name = "core2" version = "0.4.0" @@ -1089,7 +1234,7 @@ version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -1210,7 +1355,7 @@ version = "4.1.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "97fb8b7c4503de7d6ae7b42ab72a5a59857b4c937ec27a3d4539dba95b5ab2be" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "curve25519-dalek-derive", "digest 0.10.7", @@ -1459,7 +1604,7 @@ dependencies = [ "libc", "option-ext", "redox_users", - "windows-sys 0.60.2", + "windows-sys 0.61.1", ] [[package]] @@ -1589,7 +1734,7 @@ version = "0.8.35" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -1605,7 +1750,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.61.1", ] [[package]] @@ -1614,7 +1759,7 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "136d1b5283a1ab77bd9257427ffd09d8667ced0570b6f938942bc7568ed5b943" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "home", "windows-sys 0.48.0", ] @@ -1750,6 +1895,7 @@ checksum = "da0e4dd2a88388a1f4ccc7c9ce104604dab68d9f408dc34cd45823d5a9069095" dependencies = [ "futures-core", "futures-sink", + "nanorand", "spin", ] @@ -1948,7 +2094,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "605183a538e3e2a9c1038635cc5c2d194e2ee8fd0d1b66b8349fad7dbacce5a2" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "libc", "log", "rustversion", @@ -1972,7 +2118,7 @@ version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "wasi 0.11.1+wasi-snapshot-preview1", @@ -1985,7 +2131,7 @@ version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "26145e563e54f2cadc477553f1ec5ee650b00862f0a58bcd12cbdc5f0ea2d2f4" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "js-sys", "libc", "r-efi", @@ -2082,7 +2228,7 @@ version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "crunchy", "num-traits", "zerocopy", @@ -2402,7 +2548,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "iana-time-zone-haiku", "js-sys", "log", @@ -2532,6 +2678,18 @@ dependencies = [ "icu_properties", ] +[[package]] +name = "image" +version = "0.25.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85ab80394333c02fe689eaf900ab500fbd0c2213da414687ebf995a65d5a6104" +dependencies = [ + "bytemuck 1.25.0 (registry+https://github.com/rust-lang/crates.io-index)", + "byteorder-lite", + "moxcms", + "num-traits", +] + [[package]] name = "imbl" version = "6.1.0" @@ -2687,7 +2845,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" dependencies = [ "cesu8", - "cfg-if", + "cfg-if 1.0.4", "combine", "jni-sys", "log", @@ -2746,7 +2904,7 @@ version = "0.13.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6e3919bbaa2945715f0bb6d3934a173d1e9a59ac23767fbaaef277265a7411b" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "ecdsa", "elliptic-curve", "sha2 0.10.9", @@ -2787,6 +2945,12 @@ dependencies = [ "spin", ] +[[package]] +name = "lazycell" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" + [[package]] name = "libc" version = "0.2.176" @@ -2799,8 +2963,8 @@ version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "07033963ba89ebaf1584d767badaa2e8fcec21aedea6b8c0346d487d49c28667" dependencies = [ - "cfg-if", - "windows-targets 0.48.5", + "cfg-if 1.0.4", + "windows-targets 0.53.4", ] [[package]] @@ -2840,6 +3004,12 @@ dependencies = [ "memchr 2.7.6 (registry+https://github.com/rust-lang/crates.io-index)", ] +[[package]] +name = "linux-raw-sys" +version = "0.4.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d26c52dbd32dccf2d10cac7725f8eae5296885fb5703b261f7d0a0739ec807ab" + [[package]] name = "linux-raw-sys" version = "0.11.0" @@ -2874,7 +3044,7 @@ version = "0.7.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "419e0dc8046cb947daa77eb95ae174acfbddb7673b4151f56d1eed8e93fbfaca" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "generator", "scoped-tls", "tracing", @@ -3288,6 +3458,15 @@ dependencies = [ "makepad-zune-inflate", ] +[[package]] +name = "malloc_buf" +version = "0.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "62bb907fe88d54d8d9ce32a3cceab4218ed2f6b7d35617cafe9adf84e43919cb" +dependencies = [ + "libc", +] + [[package]] name = "maplit" version = "1.0.2" @@ -3370,7 +3549,7 @@ dependencies = [ "backon", "bytes", "bytesize", - "cfg-if", + "cfg-if 1.0.4", "event-listener", "eyeball", "eyeball-im", @@ -3474,7 +3653,7 @@ dependencies = [ "async-trait", "bs58", "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if", + "cfg-if 1.0.4", "ctr", "eyeball", "futures-core", @@ -3627,7 +3806,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "245ff6a224b4df7b0c90dda2dd5a6eb46112708d49e8bdd8b007fccb09fea8e4" dependencies = [ "accessory", - "cfg-if", + "cfg-if 1.0.4", "delegate-display", "derive_more 2.0.1", "fancy_constructor", @@ -3665,7 +3844,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d89e7ee0cfbedfc4da3340218492196241d89eefb6dab27de5df917a6d2e78cf" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "digest 0.10.7", ] @@ -3680,6 +3859,21 @@ name = "memchr" version = "2.7.6" source = "git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements#461b05134a501b8e67f431b2706fb200d4bcf68a" +[[package]] +name = "metal" +version = "0.18.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e198a0ee42bdbe9ef2c09d0b9426f3b2b47d90d93a4a9b0395c4cea605e92dc0" +dependencies = [ + "bitflags 1.3.2", + "block", + "cocoa", + "core-graphics", + "foreign-types", + "log", + "objc", +] + [[package]] name = "mime" version = "0.3.17" @@ -3718,6 +3912,41 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "moxcms" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb85c154ba489f01b25c0d36ae69a87e4a1c73a72631fc6c0eb6dde34a73e44b" +dependencies = [ + "num-traits", + "pxfm", +] + +[[package]] +name = "mozjpeg" +version = "0.10.13" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7891b80aaa86097d38d276eb98b3805d6280708c4e0a1e6f6aed9380c51fec9" +dependencies = [ + "arrayvec", + "bytemuck 1.25.0 (registry+https://github.com/rust-lang/crates.io-index)", + "libc", + "mozjpeg-sys", + "rgb", +] + +[[package]] +name = "mozjpeg-sys" +version = "2.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f0dc668bf9bf888c88e2fb1ab16a406d2c380f1d082b20d51dd540ab2aa70c1" +dependencies = [ + "cc", + "dunce", + "libc", + "nasm-rs", +] + [[package]] name = "multihash" version = "0.19.3" @@ -3737,7 +3966,7 @@ dependencies = [ "arrayvec", "bit-set", "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if", + "cfg-if 1.0.4", "cfg_aliases", "codespan-reporting", "half", @@ -3754,6 +3983,15 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "nanorand" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6a51313c5820b0b02bd422f4b44776fbf47961755c74ce64afc73bfad10226c3" +dependencies = [ + "getrandom 0.2.16", +] + [[package]] name = "napi-derive-backend-ohos" version = "0.0.7" @@ -3774,7 +4012,7 @@ version = "0.0.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8462d74a2d6c7a671bd610f99f9ba34c739aadd2da4d8dd9f109a7e666cc2ad2" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "convert_case", "napi-derive-backend-ohos", "proc-macro2", @@ -3803,6 +4041,16 @@ dependencies = [ "libloading", ] +[[package]] +name = "nasm-rs" +version = "0.3.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "706bf8a5e8c8ddb99128c3291d31bd21f4bcde17f0f4c20ec678d85c74faa149" +dependencies = [ + "jobserver", + "log", +] + [[package]] name = "native-tls" version = "0.2.14" @@ -3832,6 +4080,72 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nokhwa" +version = "0.10.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4cae50786bfa1214ed441f98addbea51ca1b9aaa9e4bf5369cda36654b3efaa" +dependencies = [ + "flume", + "image", + "nokhwa-bindings-linux", + "nokhwa-bindings-macos", + "nokhwa-bindings-windows", + "nokhwa-core", + "paste", + "thiserror 2.0.17", +] + +[[package]] +name = "nokhwa-bindings-linux" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4bd666aaa41d14357817bd9a981773a73c4d00b34d344cfc244e47ebd397b1ec" +dependencies = [ + "nokhwa-core", + "v4l", +] + +[[package]] +name = "nokhwa-bindings-macos" +version = "0.2.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "de78eb4a2d47a68f490899aa0516070d7a972f853ec2bb374ab53be0bd39b60f" +dependencies = [ + "block", + "cocoa-foundation", + "core-foundation 0.10.1", + "core-media-sys", + "core-video-sys", + "flume", + "nokhwa-core", + "objc", + "once_cell", +] + +[[package]] +name = "nokhwa-bindings-windows" +version = "0.4.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "899799275c93ef69bbe8cb888cf6f8249abe751cbc50be5299105022aec14a1c" +dependencies = [ + "nokhwa-core", + "once_cell", + "windows 0.62.1", +] + +[[package]] +name = "nokhwa-core" +version = "0.1.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109975552bbd690894f613bce3d408222911e317197c72b2e8b9a1912dc261ae" +dependencies = [ + "bytes", + "image", + "mozjpeg", + "thiserror 2.0.17", +] + [[package]] name = "nom" version = "7.1.3" @@ -3933,6 +4247,16 @@ dependencies = [ "url", ] +[[package]] +name = "objc" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "915b1b472bc21c53464d6c8461c9d3af805ba1ef837e1cac254428f4a77177b1" +dependencies = [ + "malloc_buf", + "objc_exception", +] + [[package]] name = "objc2" version = "0.6.2" @@ -4003,6 +4327,15 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc_exception" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ad970fb455818ad6cba4c122ad012fae53ae8b4795f86378bce65e4f6bab2ca4" +dependencies = [ + "cc", +] + [[package]] name = "ohos-sys" version = "0.2.2" @@ -4034,7 +4367,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", - "cfg-if", + "cfg-if 1.0.4", "foreign-types", "libc", "once_cell", @@ -4123,7 +4456,7 @@ version = "0.9.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bc838d2a56b5b1a6c25f55575dfc605fabb63bb2365f6c2353ef9159aa69e4a5" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "libc", "redox_syscall", "smallvec 1.15.1 (registry+https://github.com/rust-lang/crates.io-index)", @@ -4163,6 +4496,12 @@ dependencies = [ "hmac", ] +[[package]] +name = "peeking_take_while" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099" + [[package]] name = "pem-rfc7468" version = "0.7.0" @@ -4290,7 +4629,7 @@ version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9d1fe60d06143b2430aa532c94cfe9e29783047f06c0d7fd359a9a51b729fa25" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "opaque-debug", "universal-hash", @@ -4435,6 +4774,12 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "007d8adb5ddab6f8e3f491ac63566a7d5002cc7ed73901f72057943fa71ae1ae" +[[package]] +name = "pxfm" +version = "0.1.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5a041e753da8b807c9255f28de81879c78c876392ff2469cde94799b2896b9d" + [[package]] name = "quinn" version = "0.11.9" @@ -4722,6 +5067,15 @@ dependencies = [ "subtle", ] +[[package]] +name = "rgb" +version = "0.8.53" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b34b781b31e5d73e9fbc8689c70551fd1ade9a19e3e28cfec8580a79290cc4" +dependencies = [ + "bytemuck 1.25.0 (registry+https://github.com/rust-lang/crates.io-index)", +] + [[package]] name = "ring" version = "0.17.14" @@ -4729,7 +5083,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" dependencies = [ "cc", - "cfg-if", + "cfg-if 1.0.4", "getrandom 0.2.16", "libc", "untrusted 0.9.0", @@ -4785,7 +5139,7 @@ version = "0.2.0" source = "git+https://github.com/project-robius/robius#87ea5c1e155d618a5902cae477d9603abe3f64c4" dependencies = [ "android-build", - "cfg-if", + "cfg-if 1.0.4", "jni", "objc2", "objc2-core-location", @@ -4800,7 +5154,7 @@ version = "0.2.0" source = "git+https://github.com/project-robius/robius#87ea5c1e155d618a5902cae477d9603abe3f64c4" dependencies = [ "block2", - "cfg-if", + "cfg-if 1.0.4", "dispatch2", "jni", "objc2", @@ -4847,6 +5201,7 @@ dependencies = [ "matrix-sdk", "matrix-sdk-base", "matrix-sdk-ui", + "nokhwa", "percent-encoding", "quinn", "rand 0.8.5", @@ -5031,7 +5386,7 @@ version = "0.17.1" source = "git+https://github.com/project-robius/ruma.git?branch=tsp#7028b7fbf1351a092328ec250da86beed885a681" dependencies = [ "as_variant", - "cfg-if", + "cfg-if 1.0.4", "proc-macro-crate", "proc-macro2", "quote", @@ -5091,6 +5446,19 @@ dependencies = [ "semver", ] +[[package]] +name = "rustix" +version = "0.38.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fdb5bc1ae2baa591800df16c9ca78619bf65c0488b41b96ccec5d11220d8c154" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "errno", + "libc", + "linux-raw-sys 0.4.15", + "windows-sys 0.59.0", +] + [[package]] name = "rustix" version = "1.1.2" @@ -5100,8 +5468,8 @@ dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "errno", "libc", - "linux-raw-sys", - "windows-sys 0.52.0", + "linux-raw-sys 0.11.0", + "windows-sys 0.61.1", ] [[package]] @@ -5175,7 +5543,7 @@ version = "0.18.0" source = "git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements#461b05134a501b8e67f431b2706fb200d4bcf68a" dependencies = [ "bitflags 2.10.0 (git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements)", - "bytemuck", + "bytemuck 1.25.0 (git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements)", "makepad-error-log", "smallvec 1.15.1 (git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements)", "ttf-parser", @@ -5306,7 +5674,7 @@ checksum = "897b2245f0b511c87893af39b033e5ca9cce68824c4d7e7630b5a1d339658d02" dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.9.4", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "security-framework-sys", ] @@ -5319,7 +5687,7 @@ checksum = "cc198e42d9b7510827939c9a15f5062a0c913f3371d765977e586d2fe6c16f4a" dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "core-foundation 0.10.1", - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", "security-framework-sys", ] @@ -5330,7 +5698,7 @@ version = "2.15.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -5507,7 +5875,7 @@ version = "0.10.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "digest 0.10.7", ] @@ -5518,7 +5886,7 @@ version = "0.10.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a7507d819769d01a365ab707794a4084392c824f54a7a6a7862f8c3d0892b283" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "digest 0.10.7", ] @@ -5529,7 +5897,7 @@ version = "0.11.0-rc.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "19d43dc0354d88b791216bb5c1bfbb60c0814460cc653ae0ebd71f286d0bd927" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "cpufeatures", "digest 0.11.0-rc.4", ] @@ -5935,7 +6303,7 @@ version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e1d1b10ced5ca923a1fcb8d03e96b8d3268065d724548c0211415ff6ac6bac4" dependencies = [ - "core-foundation-sys", + "core-foundation-sys 0.8.7", "libc", ] @@ -5948,8 +6316,8 @@ dependencies = [ "fastrand", "getrandom 0.3.3", "once_cell", - "rustix", - "windows-sys 0.52.0", + "rustix 1.1.2", + "windows-sys 0.61.1", ] [[package]] @@ -6009,7 +6377,7 @@ version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", ] [[package]] @@ -6582,6 +6950,26 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "v4l" +version = "0.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8fbfea44a46799d62c55323f3c55d06df722fbe577851d848d328a1041c3403" +dependencies = [ + "bitflags 1.3.2", + "libc", + "v4l2-sys-mit", +] + +[[package]] +name = "v4l2-sys-mit" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6779878362b9bacadc7893eac76abe69612e8837ef746573c4a5239daf11990b" +dependencies = [ + "bindgen 0.65.1", +] + [[package]] name = "valuable" version = "0.1.1" @@ -6685,7 +7073,7 @@ version = "0.2.107" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1310980282a2842658e512a8bd683c962bbf9395e0544fa7bc0509343b8f7d10" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "once_cell", "rustversion", "wasm-bindgen-macro", @@ -6698,7 +7086,7 @@ version = "0.4.57" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "de050049980fd9bee908eebfcdc8fa78dddb59acdbe7cbcc5b523a93c9fe0a4e" dependencies = [ - "cfg-if", + "cfg-if 1.0.4", "futures-util", "js-sys", "once_cell", @@ -6861,6 +7249,18 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "which" +version = "4.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "87ba24419a2078cd2b0f2ede2691b6c66d8e47836da3b6db8265ebad47afbfc7" +dependencies = [ + "either", + "home", + "once_cell", + "rustix 0.38.44", +] + [[package]] name = "whoami" version = "1.6.1" @@ -6883,7 +7283,7 @@ version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.48.0", + "windows-sys 0.61.1", ] [[package]] @@ -6906,7 +7306,19 @@ dependencies = [ "windows-core 0.61.2", "windows-future 0.2.1", "windows-link 0.1.3", - "windows-numerics", + "windows-numerics 0.2.0", +] + +[[package]] +name = "windows" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "49e6c4a1f363c8210c6f77ba24f645c61c6fb941eccf013da691f7e09515b8ac" +dependencies = [ + "windows-collections 0.3.1", + "windows-core 0.62.1", + "windows-future 0.3.1", + "windows-numerics 0.3.0", ] [[package]] @@ -6928,6 +7340,15 @@ dependencies = [ "windows-core 0.61.2", ] +[[package]] +name = "windows-collections" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "123e712f464a8a60ce1a13f4c446d2d43ab06464cb5842ff68f5c71b6fb7852e" +dependencies = [ + "windows-core 0.62.1", +] + [[package]] name = "windows-collections" version = "0.3.2" @@ -6961,6 +7382,19 @@ dependencies = [ "windows-strings 0.4.2", ] +[[package]] +name = "windows-core" +version = "0.62.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6844ee5416b285084d3d3fffd743b925a6c9385455f64f6d4fa3031c4c2749a9" +dependencies = [ + "windows-implement 0.60.1", + "windows-interface 0.59.2", + "windows-link 0.2.0", + "windows-result 0.4.0", + "windows-strings 0.5.0", +] + [[package]] name = "windows-core" version = "0.62.2" @@ -6979,7 +7413,18 @@ checksum = "fc6a41e98427b19fe4b73c550f060b59fa592d7d686537eebf9385621bfbad8e" dependencies = [ "windows-core 0.61.2", "windows-link 0.1.3", - "windows-threading", + "windows-threading 0.1.0", +] + +[[package]] +name = "windows-future" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "68f3db6b24b120200d649cd4811b4947188ed3a8d2626f7075146c5d178a9a4a" +dependencies = [ + "windows-core 0.62.1", + "windows-link 0.2.0", + "windows-threading 0.2.0", ] [[package]] @@ -7061,6 +7506,16 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-numerics" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ce3498fe0aba81e62e477408383196b4b0363db5e0c27646f932676283b43d8" +dependencies = [ + "windows-core 0.62.1", + "windows-link 0.2.0", +] + [[package]] name = "windows-registry" version = "0.5.3" @@ -7090,6 +7545,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-result" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7084dcc306f89883455a206237404d3eaf961e5bd7e0f312f7c91f57eb44167f" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows-result" version = "0.4.1" @@ -7107,6 +7571,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-strings" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7218c655a553b0bed4426cf54b20d7ba363ef543b52d515b3e48d7fd55318dda" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows-strings" version = "0.5.1" @@ -7241,6 +7714,15 @@ dependencies = [ "windows-link 0.1.3", ] +[[package]] +name = "windows-threading" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ab47f085ad6932defa48855254c758cdd0e2f2d48e62a34118a268d8f345e118" +dependencies = [ + "windows-link 0.2.0", +] + [[package]] name = "windows_aarch64_gnullvm" version = "0.42.2" diff --git a/Cargo.toml b/Cargo.toml index 8bd24357a..fba955a8f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -77,7 +77,7 @@ tokio = { version = "1.43.1", features = ["macros", "rt-multi-thread"] } tracing-subscriber = "0.3.17" unicode-segmentation = "1.11.0" url = "2.5.0" - +nokhwa = { version = "0.10", features = ["input-native"] } ## Dependencies for TSP support. ## Commit "f0bc4625dcd729e07e4a36257df2f1d94c81cef4" is the most recent one without the invalid change to pin serde to 1.0.219. diff --git a/src/app.rs b/src/app.rs index b9a75f6ee..72e990aa4 100644 --- a/src/app.rs +++ b/src/app.rs @@ -2,10 +2,11 @@ //! //! See `handle_startup()` for the first code that runs on app startup. -use std::{cell::RefCell, collections::HashMap}; +use std::{cell::RefCell, collections::HashMap, sync::mpsc, io::{BufRead, BufReader}}; use makepad_widgets::*; use matrix_sdk::{RoomState, ruma::{OwnedEventId, OwnedRoomId, OwnedUserId, RoomId, UserId}}; use serde::{Deserialize, Serialize}; +use makepad_widgets::makepad_platform::permission::Permission; use crate::{ avatar_cache::clear_avatar_cache, home::{ event_source_modal::{EventSourceModalAction, EventSourceModalWidgetRefExt}, invite_modal::{InviteModalAction, InviteModalWidgetRefExt}, invite_screen::InviteScreenWidgetRefExt, main_desktop_ui::MainDesktopUiAction, navigation_tab_bar::{NavigationBarAction, SelectedTab}, new_message_context_menu::NewMessageContextMenuWidgetRefExt, room_context_menu::RoomContextMenuWidgetRefExt, room_screen::{InviteAction, MessageAction, RoomScreenWidgetRefExt, clear_timeline_states}, rooms_list::{RoomsListAction, RoomsListRef, RoomsListUpdate, clear_all_invited_rooms, enqueue_rooms_list_update}, space_lobby::SpaceLobbyScreenWidgetRefExt @@ -14,7 +15,8 @@ use crate::{ }, login::login_screen::LoginAction, logout::logout_confirm_modal::{LogoutAction, LogoutConfirmModalAction, LogoutConfirmModalWidgetRefExt}, persistence, profile::user_profile_cache::clear_user_profile_cache, room::BasicRoomDetails, shared::{confirmation_modal::{ConfirmationModalContent, ConfirmationModalWidgetRefExt}, image_viewer::{ImageViewerAction, LoadState}, popup_list::{PopupKind, enqueue_popup_notification}}, sliding_sync::{DirectMessageRoomAction, MatrixRequest, current_user_id, submit_async_request}, utils::RoomNameId, verification::VerificationAction, verification_modal::{ VerificationModalAction, VerificationModalWidgetRefExt, - } + }, + voip::VoipGlobalState, }; script_mod! { @@ -174,6 +176,10 @@ pub struct App { /// A stack of previously-selected rooms for mobile navigation. /// When a view is popped off the stack, the previous `selected_room` is restored from here. #[rust] mobile_room_nav_stack: Vec, + /// Stdin command receiver for switching to VoIP screen + #[rust] stdin_rx: Option>, + /// Timer for polling stdin + #[rust] stdin_poll_timer: Timer, } impl ScriptHook for App { @@ -237,6 +243,27 @@ impl MatchEvent for App { log!("App::Startup: initializing TSP (Trust Spanning Protocol) module."); crate::tsp::tsp_init(_tokio_rt_handle).unwrap(); } + + // Start stdin reader thread for VoIP screen commands + let (tx, rx) = mpsc::channel(); + self.stdin_rx = Some(rx); + std::thread::spawn(move || { + let stdin = std::io::stdin(); + let reader = BufReader::new(stdin); + for line in reader.lines() { + if let Ok(line) = line { + let _ = tx.send(line); + } + } + }); + + // Start timer to poll stdin (10 times per second) + self.stdin_poll_timer = cx.start_interval(0.1); + + // Initialize VoIP global state and request camera permissions/video inputs early + VoipGlobalState::initialize(cx); + + log!("App: stdin listener started. Type 'voip' to switch to VoIP screen."); } fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { @@ -740,6 +767,7 @@ impl AppMain for App { crate::join_leave_room_modal::script_mod(vm); crate::verification_modal::script_mod(vm); crate::profile::script_mod(vm); + crate::voip::voip_screen::script_mod(vm); crate::home::script_mod(vm); crate::login::script_mod(vm); crate::logout::script_mod(vm); @@ -780,6 +808,22 @@ impl AppMain for App { } } + // Handle VoIP-related events at app level (before VoipScreen is shown) + match event { + Event::PermissionResult(result) if result.permission == Permission::Camera => { + VoipGlobalState::handle_permission_result(cx, result.status); + } + Event::VideoInputs(ev) => { + VoipGlobalState::handle_video_inputs(cx, ev); + } + _ => {} + } + + // Poll stdin for VoIP commands + if self.stdin_poll_timer.is_event(event).is_some() { + self.poll_stdin(cx); + } + // Forward events to the MatchEvent trait implementation. self.match_event(cx, event); let scope = &mut Scope::with_data(&mut self.app_state); @@ -820,6 +864,46 @@ impl AppMain for App { } impl App { + /// Poll stdin for commands to switch screens + fn poll_stdin(&mut self, cx: &mut Cx) { + let commands: Vec = if let Some(rx) = &self.stdin_rx { + let mut cmds = Vec::new(); + while let Ok(line) = rx.try_recv() { + cmds.push(line); + } + cmds + } else { + Vec::new() + }; + + for line in commands { + let cmd = line.trim().to_lowercase(); + match cmd.as_str() { + "voip" => { + log!("Stdin command: switching to VoIP screen"); + cx.action(NavigationBarAction::GoToVoip); + self.ui.redraw(cx); + } + "home" => { + log!("Stdin command: switching to Home screen"); + cx.action(NavigationBarAction::GoToHome); + self.ui.redraw(cx); + } + "help" => { + log!("=== Stdin Commands ==="); + log!(" voip - Switch to VoIP call screen"); + log!(" home - Switch to Home screen"); + log!(" help - Show this help"); + log!("======================"); + } + _ if !cmd.is_empty() => { + log!("Unknown command: '{}'. Type 'help' for available commands.", cmd); + } + _ => {} + } + } + } + fn update_login_visibility(&self, cx: &mut Cx) { let show_login = !self.app_state.logged_in; if !show_login { diff --git a/src/home/home_screen.rs b/src/home/home_screen.rs index c4d34d2aa..86b388bc0 100644 --- a/src/home/home_screen.rs +++ b/src/home/home_screen.rs @@ -1,6 +1,6 @@ use makepad_widgets::*; -use crate::{app::AppState, home::navigation_tab_bar::{NavigationBarAction, SelectedTab}, settings::settings_screen::SettingsScreenWidgetRefExt}; +use crate::{app::AppState, home::navigation_tab_bar::{NavigationBarAction, SelectedTab}, voip::VoipScreenWidgetRefExt, settings::settings_screen::SettingsScreenWidgetRefExt}; script_mod! { use mod.prelude.widgets.* @@ -248,6 +248,16 @@ script_mod! { add_room_screen := mod.widgets.AddRoomScreen {} } } + + voip_page := SolidView { + width: Fill, height: Fill + show_bg: true, + draw_bg.color: (COLOR_PRIMARY) + + CachedWidget { + voip_screen := mod.widgets.VoipScreen {} + } + } } } @@ -297,6 +307,15 @@ script_mod! { add_room_screen := mod.widgets.AddRoomScreen {} } } + + voip_page := View { + width: Fill, height: Fill + padding: Inset{top: 20} + + CachedWidget { + voip_screen := mod.widgets.VoipScreen {} + } + } } // Show the SpacesBar right above the navigation tab bar. @@ -466,6 +485,21 @@ impl Widget for HomeScreen { self.view.redraw(cx); } } + Some(NavigationBarAction::GoToVoip) => { + if !matches!(app_state.selected_tab, SelectedTab::VoIP) { + self.previous_selection = app_state.selected_tab.clone(); + app_state.selected_tab = SelectedTab::VoIP; + cx.action(NavigationBarAction::TabSelected(app_state.selected_tab.clone())); + if let Some(voip_page) = self.update_active_page_from_selection(cx, app_state) { + voip_page + .voip_screen(cx, ids!(voip_screen)) + .initialize(cx); + self.view.redraw(cx); + } else { + error!("BUG: failed to set active page to show VoIP screen."); + } + } + } Some(NavigationBarAction::ToggleSpacesBar) => { self.is_spaces_bar_shown = !self.is_spaces_bar_shown; self.view.spaces_bar_wrapper(cx, ids!(spaces_bar_wrapper)) @@ -508,6 +542,7 @@ impl HomeScreen { | SelectedTab::Home => id!(home_page), SelectedTab::Settings => id!(settings_page), SelectedTab::AddRoom => id!(add_room_page), + SelectedTab::VoIP => id!(voip_page), }, ) } diff --git a/src/home/navigation_tab_bar.rs b/src/home/navigation_tab_bar.rs index 95cec1317..823eb9f40 100644 --- a/src/home/navigation_tab_bar.rs +++ b/src/home/navigation_tab_bar.rs @@ -460,6 +460,15 @@ impl Widget for NavigationTabBar { SelectedTab::Home => self.view.radio_button(cx, ids!(home_button)).select(cx, scope), SelectedTab::AddRoom => self.view.radio_button(cx, ids!(add_room_button)).select(cx, scope), SelectedTab::Settings => self.view.radio_button(cx, ids!(settings_button)).select(cx, scope), + SelectedTab::VoIP => { + // VoIP doesn't have a dedicated button in the tab bar, + // so just deselect all buttons + for rb in radio_button_set.iter() { + if let Some(mut rb_inner) = rb.borrow_mut() { + rb_inner.animator_play(cx, ids!(active.off)); + } + } + } SelectedTab::Space { .. } => { for rb in radio_button_set.iter() { if let Some(mut rb_inner) = rb.borrow_mut() { @@ -487,6 +496,7 @@ pub enum SelectedTab { Home, AddRoom, Settings, + VoIP, // AlertsInbox, Space { space_name_id: RoomNameId }, } @@ -532,11 +542,13 @@ pub enum NavigationBarAction { CloseSettings, /// Go the space screen for the given space. GoToSpace { space_name_id: RoomNameId }, + /// Go to the VoIP call screen. + GoToVoip, // TODO: add GoToAlertsInbox, once we add that button/screen /// The given tab was selected as the active top-level view. - /// This is needed to ensure that the proper tab is marked as selected. + /// This is needed to ensure that the proper tab is marked as selected. TabSelected(SelectedTab), /// Toggle whether the SpacesBar is shown, i.e., show/hide it. /// This is only applicable in the Mobile view mode, because the SpacesBar diff --git a/src/lib.rs b/src/lib.rs index 346c0314b..a0e172c31 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -60,6 +60,8 @@ pub mod shared; /// Generating text previews of timeline events/messages. mod event_preview; pub mod room; +/// VoIP call screen and related functionality. +pub mod voip; /// All content related to TSP (Trust Spanning Protocol) wallets/identities. diff --git a/src/voip/call_state.rs b/src/voip/call_state.rs new file mode 100644 index 000000000..af8748f82 --- /dev/null +++ b/src/voip/call_state.rs @@ -0,0 +1,40 @@ +//! Call state management + +use std::collections::HashMap; + +/// Connection state for a call +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum ConnectionState { + #[default] + Disconnected, + Connecting, + Connected, + Disconnecting, +} + +/// Type of call (voice or video) +#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)] +pub enum CallType { + Voice, + #[default] + Video, +} + +/// Represents a participant in the call +#[derive(Debug, Clone)] +pub struct CallParticipant { + pub user_id: String, +} + +/// Main call state structure +#[derive(Debug, Clone, Default)] +pub struct Call { + pub call_type: CallType, + pub connection_state: ConnectionState, + pub participants: HashMap, + pub local_audio_muted: bool, + pub local_video_muted: bool, + pub is_screen_sharing: bool, + pub livekit_url: Option, + pub livekit_token: Option, +} diff --git a/src/voip/camera.rs b/src/voip/camera.rs new file mode 100644 index 000000000..dce4f72b5 --- /dev/null +++ b/src/voip/camera.rs @@ -0,0 +1,197 @@ +//! Camera handling and video management + +use makepad_widgets::*; +use makepad_widgets::makepad_platform::video::{VideoInputId, VideoFormatId, VideoInputsEvent, VideoPixelFormat}; +use makepad_widgets::video::VideoCameraPreviewMode; + +/// Camera format choice +#[derive(Clone)] +pub struct CameraChoice { + pub input_id: VideoInputId, + pub format_id: VideoFormatId, + pub name: String, + pub width: usize, + pub height: usize, + pub pixel_format: VideoPixelFormat, +} + +/// Camera manager handles camera selection and video playback +pub struct CameraManager; + +impl CameraManager { + /// Pick the best camera format from available options + pub fn pick_camera_choice(ev: &VideoInputsEvent) -> Option { + let desc = ev.descs.first()?; + + log!("Camera: {} has {} formats", desc.name, desc.formats.len()); + for (i, fmt) in desc.formats.iter().enumerate() { + log!( + " Format {}: {}x{} {:?} fps={:?}", + i, fmt.width, fmt.height, fmt.pixel_format, fmt.frame_rate + ); + } + + fn pixel_rank(pixel_format: VideoPixelFormat) -> usize { + match pixel_format { + VideoPixelFormat::NV12 => 4, + VideoPixelFormat::YUY2 => 3, + VideoPixelFormat::YUV420 => 2, + VideoPixelFormat::RGB24 => 1, + _ => 0, + } + } + + fn better( + a: &makepad_widgets::makepad_platform::video::VideoFormat, + b: &makepad_widgets::makepad_platform::video::VideoFormat, + ) -> bool { + let a_rank = pixel_rank(a.pixel_format); + let b_rank = pixel_rank(b.pixel_format); + if a_rank != b_rank { + return a_rank > b_rank; + } + let a_pixels = a.width * a.height; + let b_pixels = b.width * b.height; + if a_pixels != b_pixels { + return a_pixels > b_pixels; + } + let a_fps = a.frame_rate.unwrap_or(0.0); + let b_fps = b.frame_rate.unwrap_or(0.0); + a_fps > b_fps + } + + let mut best: Option = None; + + // Pass 1: NV12 at <= 1080p (preferred) + for fmt in &desc.formats { + if fmt.pixel_format != VideoPixelFormat::NV12 { + continue; + } + if fmt.width > 1920 || fmt.height > 1080 { + continue; + } + if best.as_ref().map_or(true, |b| better(fmt, b)) { + best = Some(*fmt); + } + } + + // Pass 2: any NV12 + if best.is_none() { + for fmt in &desc.formats { + if fmt.pixel_format != VideoPixelFormat::NV12 { + continue; + } + if best.as_ref().map_or(true, |b| better(fmt, b)) { + best = Some(*fmt); + } + } + } + + // Pass 3: YUY2 or YUV420 + if best.is_none() { + for fmt in &desc.formats { + if !matches!(fmt.pixel_format, VideoPixelFormat::YUY2 | VideoPixelFormat::YUV420) { + continue; + } + if best.as_ref().map_or(true, |b| better(fmt, b)) { + best = Some(*fmt); + } + } + } + + // Pass 4: Any format (fallback) + if best.is_none() { + log!("No preferred format found, taking first available format..."); + best = desc.formats.first().copied(); + } + + let format = match best { + Some(f) => f, + None => { + log!("No camera format available!"); + return None; + } + }; + log!("Selected format: {}x{} {:?}", format.width, format.height, format.pixel_format); + + Some(CameraChoice { + input_id: desc.input_id, + format_id: format.format_id, + name: desc.name.clone(), + width: format.width, + height: format.height, + pixel_format: format.pixel_format, + }) + } + + /// Start camera for lobby preview + pub fn start_lobby_camera(ui: &View, cx: &mut Cx, choice: &CameraChoice) -> bool { + let video = ui.video(cx, &[live_id!(lobby_camera_video)]); + + if !video.is_unprepared() { + return false; + } + + log!("Starting lobby camera: {} ({}x{} {:?})", + choice.name, choice.width, choice.height, choice.pixel_format); + + ui.view(cx, ids!(lobby_video_host)).set_visible(cx, true); + ui.view(cx, ids!(lobby_camera_placeholder)).set_visible(cx, false); + + video.set_camera_preview_mode(cx, VideoCameraPreviewMode::Native); + video.set_source_camera(cx, choice.input_id, choice.format_id); + video.begin_playback(cx); + true + } + + /// Start camera for in-call video + pub fn start_call_camera(ui: &View, cx: &mut Cx, choice: &CameraChoice) -> bool { + let video = ui.video(cx, &[live_id!(local_camera_video)]); + + if !video.is_unprepared() { + return false; + } + + log!("Starting call camera..."); + + ui.view(cx, ids!(local_video_host)).set_visible(cx, true); + ui.view(cx, ids!(local_avatar_view)).set_visible(cx, false); + + video.set_camera_preview_mode(cx, VideoCameraPreviewMode::Native); + video.set_source_camera(cx, choice.input_id, choice.format_id); + video.begin_playback(cx); + true + } + + /// Stop lobby camera + pub fn stop_lobby_camera(ui: &View, cx: &mut Cx) { + let video = ui.video(cx, &[live_id!(lobby_camera_video)]); + if !video.is_unprepared() && !video.is_cleaning_up() { + video.stop_and_cleanup_resources(cx); + } + ui.view(cx, ids!(lobby_video_host)).set_visible(cx, false); + ui.view(cx, ids!(lobby_camera_placeholder)).set_visible(cx, true); + } + + /// Stop in-call camera + pub fn stop_call_camera(ui: &View, cx: &mut Cx) { + let video = ui.video(cx, &[live_id!(local_camera_video)]); + if !video.is_unprepared() && !video.is_cleaning_up() { + video.stop_and_cleanup_resources(cx); + } + ui.view(cx, ids!(local_video_host)).set_visible(cx, false); + ui.view(cx, ids!(local_avatar_view)).set_visible(cx, true); + } + + /// Show video view for lobby + pub fn show_lobby_video(ui: &View, cx: &mut Cx) { + ui.view(cx, ids!(lobby_video_host)).set_visible(cx, true); + ui.view(cx, ids!(lobby_camera_placeholder)).set_visible(cx, false); + } + + /// Show video view for call + pub fn show_call_video(ui: &View, cx: &mut Cx) { + ui.view(cx, ids!(local_video_host)).set_visible(cx, true); + ui.view(cx, ids!(local_avatar_view)).set_visible(cx, false); + } +} diff --git a/src/voip/livekit_client.rs b/src/voip/livekit_client.rs new file mode 100644 index 000000000..8988e672f --- /dev/null +++ b/src/voip/livekit_client.rs @@ -0,0 +1,154 @@ +//! LiveKit client integration for WebRTC + +use std::sync::{Arc, Mutex}; +use tokio::sync::mpsc; + +use super::call_state::CallParticipant; + +/// Video frame data for publishing +#[derive(Clone)] +pub struct VideoFrame { + pub data: Vec, + pub width: u32, + pub height: u32, + #[allow(dead_code)] + pub format: VideoFrameFormat, +} + +#[derive(Clone, Copy, Debug)] +#[allow(dead_code)] +pub enum VideoFrameFormat { + Rgb24, + Rgba32, +} + +/// Messages sent from LiveKit client to UI +#[derive(Debug, Clone)] +#[allow(dead_code)] +pub enum LiveKitMessage { + Connected, + Disconnected, + Error(String), + ParticipantJoined(CallParticipant), + ParticipantLeft(String), +} + +/// Commands sent from UI to LiveKit client +pub enum LiveKitCommand { + Connect { url: String }, + Disconnect, + SetMicrophoneMuted(bool), + SetCameraMuted(bool), + StartScreenShare, + StopScreenShare, + PublishVideoFrame(VideoFrame), +} + +/// LiveKit client state +pub struct LiveKitClient { + command_tx: Option>, + is_connected: Arc>, +} + +impl LiveKitClient { + pub fn new() -> Self { + Self { + command_tx: None, + is_connected: Arc::new(Mutex::new(false)), + } + } + + /// Start the LiveKit client with channels for communication + pub fn start(&mut self) -> mpsc::UnboundedReceiver { + let (cmd_tx, cmd_rx) = mpsc::unbounded_channel(); + let (msg_tx, msg_rx) = mpsc::unbounded_channel(); + + self.command_tx = Some(cmd_tx); + + let is_connected = self.is_connected.clone(); + + std::thread::spawn(move || { + let rt = tokio::runtime::Runtime::new().unwrap(); + rt.block_on(async { + Self::run_event_loop(cmd_rx, msg_tx, is_connected).await; + }); + }); + + msg_rx + } + + async fn run_event_loop( + mut cmd_rx: mpsc::UnboundedReceiver, + msg_tx: mpsc::UnboundedSender, + is_connected: Arc>, + ) { + while let Some(cmd) = cmd_rx.recv().await { + match cmd { + LiveKitCommand::Connect { url } => { + println!("Connecting to LiveKit: {}", url); + if let Ok(mut connected) = is_connected.lock() { + *connected = true; + } + let _ = msg_tx.send(LiveKitMessage::Connected); + } + LiveKitCommand::Disconnect => { + println!("Disconnecting from LiveKit"); + if let Ok(mut connected) = is_connected.lock() { + *connected = false; + } + let _ = msg_tx.send(LiveKitMessage::Disconnected); + } + LiveKitCommand::SetMicrophoneMuted(muted) => { + println!("Set microphone muted: {}", muted); + } + LiveKitCommand::SetCameraMuted(muted) => { + println!("Set camera muted: {}", muted); + } + LiveKitCommand::StartScreenShare => { + println!("Starting screen share"); + } + LiveKitCommand::StopScreenShare => { + println!("Stopping screen share"); + } + LiveKitCommand::PublishVideoFrame(frame) => { + println!( + "Publishing video frame: {}x{} ({} bytes)", + frame.width, frame.height, frame.data.len() + ); + } + } + } + } + + pub fn send_command(&self, cmd: LiveKitCommand) { + if let Some(tx) = &self.command_tx { + let _ = tx.send(cmd); + } + } + + pub fn connect(&self, url: String, _token: String) { + self.send_command(LiveKitCommand::Connect { url }); + } + + pub fn disconnect(&self) { + self.send_command(LiveKitCommand::Disconnect); + } + + pub fn set_microphone_muted(&self, muted: bool) { + self.send_command(LiveKitCommand::SetMicrophoneMuted(muted)); + } + + pub fn set_camera_muted(&self, muted: bool) { + self.send_command(LiveKitCommand::SetCameraMuted(muted)); + } + + pub fn publish_video_frame(&self, frame: VideoFrame) { + self.send_command(LiveKitCommand::PublishVideoFrame(frame)); + } +} + +impl Default for LiveKitClient { + fn default() -> Self { + Self::new() + } +} diff --git a/src/voip/mod.rs b/src/voip/mod.rs new file mode 100644 index 000000000..681665372 --- /dev/null +++ b/src/voip/mod.rs @@ -0,0 +1,102 @@ +//! VoIP screen module for voice/video calls +//! +//! This module provides VoIP functionality including: +//! - Call state management +//! - Camera handling +//! - LiveKit WebRTC integration +//! - Speaking detection +//! - Participants list + +use makepad_widgets::*; +use makepad_widgets::makepad_platform::video::VideoInputsEvent; +use makepad_widgets::makepad_platform::permission::PermissionStatus; + +pub mod call_state; +pub mod camera; +pub mod livekit_client; +pub mod speaking; +pub mod participants_list; +pub mod voip_screen; + +pub use voip_screen::VoipScreenWidgetRefExt; +pub use participants_list::{Participant, ParticipantsListWidgetRefExt}; +pub use camera::CameraChoice; + +/// Global VoIP state stored in Makepad's Cx context. +/// This allows camera permission and video inputs events to be captured +/// at app startup before VoipScreen is shown. +#[derive(Default)] +pub struct VoipGlobalState { + /// Camera permission status (captured at app level) + pub camera_permission: Option, + /// Selected camera choice from VideoInputsEvent + pub camera_choice: Option, + /// Whether video inputs have been requested + pub video_inputs_requested: bool, +} + +impl VoipGlobalState { + /// Initialize global VoIP state and request permissions/video inputs. + /// Call this in App::handle_startup. + pub fn initialize(cx: &mut Cx) { + // Set global state + cx.set_global(VoipGlobalState::default()); + + // Request camera permission + log!("VoipGlobalState: Requesting camera permission..."); + cx.request_permission(makepad_widgets::makepad_platform::permission::Permission::Camera); + + // Request video inputs enumeration - this triggers VideoInputsEvent + log!("VoipGlobalState: Requesting video inputs..."); + cx.video_input(0, |_buf| {}); + } + + /// Handle camera permission result. Call from App's event handler. + pub fn handle_permission_result(cx: &mut Cx, status: PermissionStatus) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: Camera permission result: {:?}", status); + state.camera_permission = Some(status); + } + } + + /// Handle video inputs event. Call from App's event handler. + pub fn handle_video_inputs(cx: &mut Cx, ev: &VideoInputsEvent) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: VideoInputs event with {} cameras", ev.descs.len()); + state.video_inputs_requested = true; + + if ev.descs.is_empty() { + log!("VoipGlobalState: No cameras found"); + state.camera_choice = None; + } else { + state.camera_choice = camera::CameraManager::pick_camera_choice(ev); + if let Some(ref choice) = state.camera_choice { + log!("VoipGlobalState: Selected camera: {} ({}x{} {:?})", + choice.name, choice.width, choice.height, choice.pixel_format); + } else { + log!("VoipGlobalState: No suitable camera format found"); + } + } + } + } + + /// Get camera permission from global state + pub fn get_camera_permission(cx: &mut Cx) -> Option { + if cx.has_global::() { + cx.get_global::().camera_permission + } else { + None + } + } + + /// Get camera choice from global state + pub fn get_camera_choice(cx: &mut Cx) -> Option { + if cx.has_global::() { + cx.get_global::().camera_choice.clone() + } else { + None + } + } +} diff --git a/src/voip/participants_list.rs b/src/voip/participants_list.rs new file mode 100644 index 000000000..634e28586 --- /dev/null +++ b/src/voip/participants_list.rs @@ -0,0 +1,118 @@ +//! Participants list widget for VoIP calls + +use makepad_widgets::*; + +#[derive(Clone, Debug)] +pub struct Participant { + pub id: String, + pub name: String, + pub avatar_letter: String, + pub is_muted: bool, + pub is_speaking: bool, +} + +impl Default for Participant { + fn default() -> Self { + Self { + id: String::new(), + name: String::from("Unknown"), + avatar_letter: String::from("?"), + is_muted: false, + is_speaking: false, + } + } +} + +#[derive(Script, ScriptHook, Widget)] +pub struct ParticipantsList { + #[deref] + view: View, + #[rust] + participants: Vec, +} + +impl Widget for ParticipantsList { + fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { + while let Some(item) = self.view.draw_walk(cx, scope, walk).step() { + if let Some(mut list) = item.as_flat_list().borrow_mut() { + for (i, participant) in self.participants.iter().enumerate() { + let item_id = LiveId::from_num(0, i as u64); + if let Some(widget) = list.item(cx, item_id, live_id!(ParticipantItem)) { + widget.label(cx, ids!(avatar_letter)).set_text(cx, &participant.avatar_letter); + widget.label(cx, ids!(name_label)).set_text(cx, &participant.name); + widget.label(cx, ids!(mute_icon)).set_text(cx, if participant.is_muted { "M" } else { "" }); + widget.label(cx, ids!(status_label)).set_text(cx, if participant.is_speaking { "Speaking..." } else { "" }); + widget.draw_all(cx, scope); + } + } + } + } + DrawStep::done() + } + + fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) { + self.view.handle_event(cx, event, scope); + } +} + +impl ParticipantsList { + pub fn add_participant(&mut self, cx: &mut Cx, participant: Participant) { + self.participants.push(participant); + self.redraw(cx); + } + + pub fn remove_participant(&mut self, cx: &mut Cx, id: &str) { + self.participants.retain(|p| p.id != id); + self.redraw(cx); + } + + pub fn update_participant(&mut self, cx: &mut Cx, id: &str, updater: impl FnOnce(&mut Participant)) { + if let Some(participant) = self.participants.iter_mut().find(|p| p.id == id) { + updater(participant); + self.redraw(cx); + } + } + + pub fn clear(&mut self, cx: &mut Cx) { + self.participants.clear(); + self.redraw(cx); + } + + pub fn participants(&self) -> &[Participant] { + &self.participants + } +} + +impl ParticipantsListRef { + pub fn add_participant(&self, cx: &mut Cx, participant: Participant) { + if let Some(mut inner) = self.borrow_mut() { + inner.add_participant(cx, participant); + } + } + + pub fn remove_participant(&self, cx: &mut Cx, id: &str) { + if let Some(mut inner) = self.borrow_mut() { + inner.remove_participant(cx, id); + } + } + + pub fn update_participant(&self, cx: &mut Cx, id: &str, updater: impl FnOnce(&mut Participant)) { + if let Some(mut inner) = self.borrow_mut() { + inner.update_participant(cx, id, updater); + } + } + + pub fn clear(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.clear(cx); + } + } + + pub fn get_participants(&self) -> Vec { + if let Some(inner) = self.borrow() { + inner.participants.clone() + } else { + Vec::new() + } + } +} diff --git a/src/voip/speaking.rs b/src/voip/speaking.rs new file mode 100644 index 000000000..072c849f9 --- /dev/null +++ b/src/voip/speaking.rs @@ -0,0 +1,88 @@ +//! Audio level detection and speaking indicator + +use makepad_widgets::*; +use makepad_widgets::View; +use makepad_widgets::makepad_platform::audio::{AudioDeviceId, AudioDevicesEvent}; +use std::sync::atomic::{AtomicU32, Ordering}; +use std::sync::Arc; + +/// Threshold for speaking detection (RMS level) +const SPEAKING_THRESHOLD: f32 = 0.01; + +/// Speaking detector handles audio level monitoring +pub struct SpeakingDetector { + pub audio_device: Option, + pub audio_level: Arc, + pub is_speaking: bool, +} + +impl Default for SpeakingDetector { + fn default() -> Self { + Self::new() + } +} + +impl SpeakingDetector { + pub fn new() -> Self { + Self { + audio_device: None, + audio_level: Arc::new(AtomicU32::new(0)), + is_speaking: false, + } + } + + /// Handle audio devices event and start monitoring + pub fn handle_audio_devices(&mut self, cx: &mut Cx, ev: &AudioDevicesEvent) { + log!("AudioDevices event: {} devices found", ev.descs.len()); + + let inputs = ev.default_input(); + if let Some(device_id) = inputs.first() { + log!("Using audio input device: {:?}", device_id); + self.audio_device = Some(*device_id); + cx.use_audio_inputs(&[*device_id]); + + let audio_level = self.audio_level.clone(); + cx.audio_input(0, move |_info, buffer| { + let rms = Self::calculate_rms(&buffer.data); + audio_level.store(rms.to_bits(), Ordering::Relaxed); + }); + } + } + + /// Calculate RMS (root mean square) of audio samples + fn calculate_rms(samples: &[f32]) -> f32 { + if samples.is_empty() { + return 0.0; + } + let mut sum = 0.0f32; + for sample in samples { + sum += sample * sample; + } + (sum / samples.len() as f32).sqrt() + } + + /// Get current audio level + pub fn get_level(&self) -> f32 { + let level_bits = self.audio_level.load(Ordering::Relaxed); + f32::from_bits(level_bits) + } + + /// Check if user is currently speaking (level above threshold) + pub fn check_speaking(&mut self, is_muted: bool) -> bool { + if is_muted { + self.is_speaking = false; + return false; + } + + let level = self.get_level(); + let was_speaking = self.is_speaking; + self.is_speaking = level > SPEAKING_THRESHOLD; + + was_speaking != self.is_speaking + } + + /// Update the speaking indicator in the UI + pub fn update_indicator(ui: &View, cx: &mut Cx, is_speaking: bool) { + ui.view(cx, ids!(local_speaking_border)).set_visible(cx, is_speaking); + } +} diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs new file mode 100644 index 000000000..639e44ff1 --- /dev/null +++ b/src/voip/voip_screen.rs @@ -0,0 +1,1049 @@ +//! VoIP Screen - Voice/Video call interface +//! +//! This module provides a VoIP screen widget that can be used for voice/video calls. +//! It uses the Matrix client from the room screen for authentication and signaling. + +use makepad_widgets::*; +use makepad_widgets::makepad_platform::permission::{Permission, PermissionStatus}; +use matrix_sdk::Client; +use ruma::OwnedRoomId; +use tokio::sync::mpsc; + +use crate::sliding_sync::get_client; +use super::VoipGlobalState; + +use super::call_state::{Call, CallType, ConnectionState}; +use super::camera::{CameraChoice, CameraManager}; +use super::livekit_client::{LiveKitClient, LiveKitCommand, LiveKitMessage}; +use super::speaking::SpeakingDetector; +use super::participants_list::{Participant, ParticipantsListWidgetExt}; + +script_mod! { + use mod.prelude.widgets.* + use mod.widgets.* + + // ParticipantsList widget definition + let ParticipantsListBase = #(super::participants_list::ParticipantsList::register_widget(vm)) + mod.widgets.VoipParticipantsList = set_type_default() do ParticipantsListBase { + width: Fill + height: Fill + + list := FlatList { + width: Fill + height: Fill + flow: Down + grab_key_focus: true + + ParticipantItem := RoundedView { + width: Fill + height: Fit + padding: 8 + margin: Inset{bottom: 4} + draw_bg.color: #3a3a5a + draw_bg.radius: 6.0 + flow: Right + spacing: 8 + align: Align{y: 0.5} + + avatar := RoundedView { + width: 32 + height: 32 + draw_bg.color: #a0d0a0 + draw_bg.radius: 16.0 + align: Center + + avatar_letter := Label { + text: "?" + draw_text.text_style.font_size: 14 + draw_text.color: #2a6a2a + } + } + + info_container := View { + width: Fill + height: Fit + flow: Down + spacing: 2 + + name_label := Label { + text: "Participant" + draw_text.text_style.font_size: 12 + draw_text.color: #fff + } + + status_label := Label { + text: "" + draw_text.text_style.font_size: 10 + draw_text.color: #888 + } + } + + mute_icon := Label { + text: "" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + } + } + } + + // VoIP Screen widget + mod.widgets.VoipScreen = #(VoipScreen::register_widget(vm)) { + width: Fill + height: Fill + flow: Overlay + + // Main call view + call_view := View { + width: Fill + height: Fill + flow: Down + show_bg: true + draw_bg.color: #1a1a2e + visible: false + + // Call header + call_header := View { + width: Fill + height: Fit + padding: 16 + flow: Right + spacing: 12 + + View { + width: Fit + height: Fit + flow: Down + spacing: 4 + + room_name := Label { + text: "Call Room" + draw_text.text_style.font_size: 18 + draw_text.color: #fff + } + + call_status := Label { + text: "Not connected" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + } + + View { width: Fill height: 1 } + + call_duration := Label { + text: "" + draw_text.text_style.font_size: 14 + draw_text.color: #888 + } + + participant_count := Label { + text: "0 participants" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + } + + // Participants grid + participants_grid := View { + width: Fill + height: Fill + flow: Right + spacing: 16 + padding: 16 + align: Center + + // Local user card wrapper + local_card_wrapper := View { + width: Fit + height: Fit + flow: Overlay + + // Speaking indicator border + local_speaking_border := RoundedView { + width: 286 + height: 216 + draw_bg.color: #4CAF50 + draw_bg.radius: 15.0 + visible: false + } + + // Main card + local_participant_card := RoundedView { + width: 280 + height: 210 + margin: 3 + draw_bg.color: #e8e8e8 + draw_bg.radius: 12.0 + flow: Overlay + + // Video container + local_video_container := View { + width: Fill + height: Fill + flow: Overlay + + // Avatar placeholder + local_avatar_view := View { + width: Fill + height: Fill + align: Center + + RoundedView { + width: 80 + height: 80 + draw_bg.color: #a0d0a0 + draw_bg.radius: 40.0 + align: Center + + local_avatar_letter := Label { + text: "Y" + draw_text.text_style.font_size: 32 + draw_text.color: #2a6a2a + } + } + } + + // Camera video + local_video_host := View { + width: Fill + height: Fill + visible: false + + local_camera_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false + } + } + } + + // Name badge + View { + width: Fill + height: Fit + align: Align{x: 0.0 y: 1.0} + padding: 8 + + RoundedView { + width: Fit + height: Fit + padding: Inset{left: 8 right: 8 top: 4 bottom: 4} + draw_bg.color: #fff + draw_bg.radius: 4.0 + flow: Right + spacing: 4 + + local_mute_icon := Label { + text: "" + draw_text.text_style.font_size: 12 + draw_text.color: #666 + } + + local_name_label := Label { + text: "You" + draw_text.text_style.font_size: 12 + draw_text.color: #333 + } + } + } + } + } + + // Remote participant card + remote_participant_card := RoundedView { + width: 280 + height: 210 + draw_bg.color: #e8e8e8 + draw_bg.radius: 12.0 + flow: Overlay + visible: false + + View { + width: Fill + height: Fill + align: Center + + RoundedView { + width: 80 + height: 80 + draw_bg.color: #d0a0d0 + draw_bg.radius: 40.0 + align: Center + + remote_avatar_letter := Label { + text: "R" + draw_text.text_style.font_size: 32 + draw_text.color: #6a2a6a + } + } + } + + View { + width: Fill + height: Fit + align: Align{x: 0.0 y: 1.0} + padding: 8 + + RoundedView { + width: Fit + height: Fit + padding: Inset{left: 8 right: 8 top: 4 bottom: 4} + draw_bg.color: #fff + draw_bg.radius: 4.0 + + remote_name_label := Label { + text: "Remote" + draw_text.text_style.font_size: 12 + draw_text.color: #333 + } + } + } + } + } + + // Call controls + call_controls := View { + width: Fill + height: Fit + padding: Inset{bottom: 20 top: 10} + align: Center + + RoundedView { + width: Fit + height: Fit + padding: 12 + draw_bg.color: #2a2a4a + draw_bg.radius: 24.0 + flow: Right + spacing: 8 + + mic_button := Button { text: "Mic" width: 60 } + camera_button := Button { text: "Cam" width: 60 } + screenshare_button := Button { text: "Share" width: 60 } + participants_button := Button { text: "Users" width: 60 } + hangup_button := Button { text: "End" width: 60 } + } + } + } + + // Participants sidebar + participants_sidebar := View { + width: 200 + height: Fill + margin: Inset{top: 60 bottom: 80} + padding: 12 + show_bg: true + draw_bg.color: #2a2a4a + flow: Down + spacing: 8 + visible: false + align: Align{x: 1.0 y: 0.0} + + Label { + text: "Participants" + draw_text.text_style.font_size: 14 + draw_text.color: #fff + } + + participants_list := mod.widgets.VoipParticipantsList {} + } + + // Lobby view + lobby_view := View { + width: Fill + height: Fill + flow: Down + spacing: 20 + padding: 40 + align: Center + show_bg: true + draw_bg.color: #1a1a2e + visible: true + + Label { + text: "Join Call" + draw_text.text_style.font_size: 24 + draw_text.color: #fff + } + + lobby_camera_container := View { + width: 320 + height: 240 + + lobby_camera_placeholder := RoundedView { + width: Fill + height: Fill + draw_bg.color: #2a2a4a + draw_bg.radius: 12.0 + align: Center + + Label { + text: "Camera Preview" + draw_text.text_style.font_size: 14 + draw_text.color: #666 + } + } + + lobby_video_host := View { + width: Fill + height: Fill + visible: false + + lobby_camera_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false + } + } + } + + View { + width: Fit + height: Fit + flow: Down + spacing: 12 + align: Center + + View { + width: Fit + height: Fit + flow: Right + spacing: 8 + align: Center + + Label { + text: "Microphone:" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + + lobby_mic_button := Button { + text: "Mic On" + width: 80 + draw_text.color: #333 + } + } + + View { + width: Fit + height: Fit + flow: Right + spacing: 8 + align: Center + + Label { + text: "Camera:" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + + lobby_camera_button := Button { + text: "Cam On" + width: 80 + draw_text.color: #333 + } + } + } + + View { + width: Fit + height: Fit + flow: Right + spacing: 12 + + video_call_button := Button { + text: "Video Call" + width: 120 + draw_text.color: #333 + } + + voice_call_button := Button { + text: "Voice Call" + width: 120 + draw_text.color: #333 + } + } + + lobby_status := Label { + text: "" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + } + } + + // Debug panel + debug_panel := View { + width: 300 + height: 200 + margin: Inset{right: 10 bottom: 10} + padding: 10 + align: Align{x: 1.0 y: 1.0} + show_bg: true + draw_bg.color: #000000 + visible: false + + message_log := Label { + width: Fill + height: Fill + text: "" + draw_text.text_style.font_size: 9 + draw_text.color: #0f0 + } + } + } +} + +/// VoIP Screen widget for voice/video calls +#[derive(Script, ScriptHook, Widget)] +pub struct VoipScreen { + #[deref] + view: View, + + // Call state + #[rust] call: Call, + #[rust] in_lobby: bool, + #[rust] lobby_mic_enabled: bool, + #[rust] lobby_camera_enabled: bool, + #[rust] show_participants: bool, + #[rust] show_debug: bool, + + // LiveKit client + #[rust] livekit_client: Option, + #[rust] livekit_rx: Option>, + + // Call timing + #[rust] call_start_time: Option, + + // Matrix auth (from room screen client) + #[rust] room_id: Option, + + // Camera + #[rust] camera_permission: Option, + #[rust] camera_choice: Option, + #[rust] camera_active: bool, + + // Speaking detection + #[rust] speaking_detector: SpeakingDetector, + #[rust] speaking_check_timer: Timer, + + // Video publish timer + #[rust] video_publish_timer: Timer, + + // Flag to start call camera after lobby camera releases + #[rust] pending_call_camera_start: bool, + + // Participant counter + #[rust] participant_counter: usize, +} + +impl Widget for VoipScreen { + fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) { + match event { + Event::PermissionResult(result) => { + if result.permission == Permission::Camera { + log!("VoipScreen: Camera permission result: {:?}", result.status); + // Sync from global state (App already updated it) + self.camera_permission = VoipGlobalState::get_camera_permission(cx); + self.try_start_camera(cx); + } + } + Event::VideoInputs(ev) => { + log!("VoipScreen: VideoInputs event received with {} cameras", ev.descs.len()); + // Sync from global state (App already updated it) + self.camera_choice = VoipGlobalState::get_camera_choice(cx); + if let Some(ref choice) = self.camera_choice { + log!("VoipScreen: Got camera from global: {} ({}x{} {:?})", + choice.name, choice.width, choice.height, choice.pixel_format); + } + self.try_start_camera(cx); + } + Event::AudioDevices(ev) => { + self.speaking_detector.handle_audio_devices(cx, ev); + } + Event::VideoPlaybackPrepared(ev) => { + log!("VideoPlaybackPrepared: {:?}", ev.video_id); + self.handle_video_prepared(cx); + } + Event::VideoTextureUpdated(ev) => { + log!("VideoTextureUpdated: {:?}", ev.video_id); + self.handle_video_texture_updated(cx); + } + Event::VideoPlaybackResourcesReleased(_) => { + self.handle_video_resources_released(cx); + } + Event::Actions(actions) => { + self.handle_actions(cx, actions); + } + _ => { + if self.speaking_check_timer.is_event(event).is_some() { + self.check_speaking_state(cx); + } + if self.video_publish_timer.is_event(event).is_some() { + // Video publishing handled here if needed + } + } + } + + // Poll LiveKit messages + if self.poll_livekit_messages(cx) { + self.update_ui(cx); + } + + self.view.handle_event(cx, event, scope); + } + + fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { + self.view.draw_walk(cx, scope, walk) + } +} + +impl VoipScreen { + /// Initialize the VoIP screen + pub fn initialize(&mut self, cx: &mut Cx) { + self.in_lobby = true; + self.lobby_mic_enabled = true; + self.lobby_camera_enabled = true; + self.call = Call::default(); + self.speaking_detector = SpeakingDetector::new(); + + log!("VoipScreen initialized"); + + // Initialize LiveKit client + let mut client = LiveKitClient::new(); + let rx = client.start(); + self.livekit_client = Some(client); + self.livekit_rx = Some(rx); + + // Timer for speaking detection + self.speaking_check_timer = cx.start_interval(0.1); + + // Timer for video frames (~30fps) + self.video_publish_timer = cx.start_interval(1.0 / 30.0); + + // Read camera permission and choice from global state (captured at app startup) + self.camera_permission = VoipGlobalState::get_camera_permission(cx); + self.camera_choice = VoipGlobalState::get_camera_choice(cx); + + log!("VoipScreen: Read from global state - permission={:?}, choice={:?}", + self.camera_permission, self.camera_choice.as_ref().map(|c| &c.name)); + + // Try to start camera if we already have permission and camera choice + self.try_start_camera(cx); + + // Set default room + let room_id: OwnedRoomId = "!rTeTgZzSYKoeJEVosH:matrix.org".try_into().unwrap(); + self.set_room(cx, room_id); + + self.update_ui(cx); + } + + /// Set the room for this VoIP call (uses Matrix client from room screen) + pub fn set_room(&mut self, cx: &mut Cx, room_id: OwnedRoomId) { + self.room_id = Some(room_id.clone()); + + // Get room name from client + if let Some(client) = get_client() { + if let Some(room) = client.get_room(&room_id) { + let room_name = room.name().unwrap_or_else(|| room_id.to_string()); + self.view.label(cx, ids!(room_name)).set_text(cx, &room_name); + } + } + + self.update_ui(cx); + } + + /// Get the Matrix client from the sliding_sync module + #[allow(dead_code)] + fn get_matrix_client(&self) -> Option { + get_client() + } + + /// Start a call of the given type + fn start_call(&mut self, cx: &mut Cx, call_type: CallType) { + self.call.call_type = call_type; + self.call.local_audio_muted = !self.lobby_mic_enabled; + self.call.local_video_muted = !self.lobby_camera_enabled; + self.call.connection_state = ConnectionState::Connecting; + + log!("Starting {:?} call...", call_type); + + // In a full implementation, this would send call member state via Matrix + // For now, we simulate connection + self.call.connection_state = ConnectionState::Connected; + self.in_lobby = false; + self.call_start_time = Some(Cx::time_now()); + + // Stop lobby camera + CameraManager::stop_lobby_camera(&self.view, cx); + self.pending_call_camera_start = true; + self.camera_active = false; + + self.update_ui(cx); + } + + /// Poll for LiveKit messages + fn poll_livekit_messages(&mut self, _cx: &mut Cx) -> bool { + let messages: Vec = if let Some(rx) = &mut self.livekit_rx { + let mut msgs = Vec::new(); + while let Ok(msg) = rx.try_recv() { + msgs.push(msg); + } + msgs + } else { + Vec::new() + }; + + let mut needs_update = false; + for msg in messages { + match msg { + LiveKitMessage::Connected => { + self.call.connection_state = ConnectionState::Connected; + self.in_lobby = false; + self.call_start_time = Some(Cx::time_now()); + log!("LiveKit connected"); + needs_update = true; + } + LiveKitMessage::Disconnected => { + self.call.connection_state = ConnectionState::Disconnected; + log!("LiveKit disconnected"); + needs_update = true; + } + LiveKitMessage::ParticipantJoined(p) => { + self.call.participants.insert(p.user_id.clone(), p); + log!("Participant joined"); + needs_update = true; + } + LiveKitMessage::ParticipantLeft(id) => { + self.call.participants.remove(&id); + log!("Participant left"); + needs_update = true; + } + LiveKitMessage::Error(e) => { + log!("LiveKit error: {}", e); + needs_update = true; + } + } + } + + needs_update + } + + /// Toggle microphone + fn toggle_microphone(&mut self) { + self.call.local_audio_muted = !self.call.local_audio_muted; + if let Some(client) = &self.livekit_client { + client.set_microphone_muted(self.call.local_audio_muted); + } + log!("Microphone {}", if self.call.local_audio_muted { "muted" } else { "unmuted" }); + } + + /// Toggle camera + fn toggle_camera(&mut self) { + self.call.local_video_muted = !self.call.local_video_muted; + if let Some(client) = &self.livekit_client { + client.set_camera_muted(self.call.local_video_muted); + } + log!("Camera {}", if self.call.local_video_muted { "off" } else { "on" }); + } + + /// Toggle screen sharing + fn toggle_screenshare(&mut self) { + self.call.is_screen_sharing = !self.call.is_screen_sharing; + if let Some(client) = &self.livekit_client { + if self.call.is_screen_sharing { + client.send_command(LiveKitCommand::StartScreenShare); + } else { + client.send_command(LiveKitCommand::StopScreenShare); + } + } + log!("Screen sharing {}", if self.call.is_screen_sharing { "started" } else { "stopped" }); + } + + /// Hangup the call + fn hangup(&mut self, cx: &mut Cx) { + self.call.connection_state = ConnectionState::Disconnecting; + if let Some(client) = &self.livekit_client { + client.disconnect(); + } + CameraManager::stop_call_camera(&self.view, cx); + self.camera_active = false; + + log!("Ending call..."); + + // Reset state + self.call.connection_state = ConnectionState::Disconnected; + self.in_lobby = true; + self.call_start_time = None; + self.try_start_camera(cx); + } + + /// Update UI to reflect current state + fn update_ui(&mut self, cx: &mut Cx) { + + self.view.view(cx, ids!(lobby_view)).set_visible(cx, self.in_lobby); + self.view.view(cx, ids!(call_view)).set_visible(cx, !self.in_lobby); + + let status = match self.call.connection_state { + ConnectionState::Disconnected => "Not connected", + ConnectionState::Connecting => "Connecting...", + ConnectionState::Connected => "Connected", + ConnectionState::Disconnecting => "Disconnecting...", + }; + self.view.label(cx, ids!(call_status)).set_text(cx, status); + + let count = self.call.participants.len() + 1; + self.view.label(cx, ids!(participant_count)) + .set_text(cx, &format!("{} participant{}", count, if count == 1 { "" } else { "s" })); + + let mic_text = if self.call.local_audio_muted { "Muted" } else { "Mic" }; + let cam_text = if self.call.local_video_muted { "Cam Off" } else { "Cam" }; + let screen_text = if self.call.is_screen_sharing { "Stop" } else { "Share" }; + + self.view.button(cx, ids!(mic_button)).set_text(cx, mic_text); + self.view.button(cx, ids!(camera_button)).set_text(cx, cam_text); + self.view.button(cx, ids!(screenshare_button)).set_text(cx, screen_text); + + let mute_icon = if self.call.local_audio_muted { "M" } else { "" }; + self.view.label(cx, ids!(local_mute_icon)).set_text(cx, mute_icon); + + self.view.view(cx, ids!(participants_sidebar)).set_visible(cx, self.show_participants); + self.view.view(cx, ids!(debug_panel)).set_visible(cx, self.show_debug); + + if let Some(start) = self.call_start_time { + let elapsed = (Cx::time_now() - start) as u64; + let mins = elapsed / 60; + let secs = elapsed % 60; + self.view.label(cx, ids!(call_duration)) + .set_text(cx, &format!("{:02}:{:02}", mins, secs)); + } + + // Update lobby buttons + let mic_text = if self.lobby_mic_enabled { "Mic On" } else { "Mic Off" }; + let cam_text = if self.lobby_camera_enabled { "Cam On" } else { "Cam Off" }; + self.view.button(cx, ids!(lobby_mic_button)).set_text(cx, mic_text); + self.view.button(cx, ids!(lobby_camera_button)).set_text(cx, cam_text); + } + + /// Try to start camera + fn try_start_camera(&mut self, cx: &mut Cx) { + log!("try_start_camera: permission={:?}, choice={:?}", + self.camera_permission, self.camera_choice.as_ref().map(|c| &c.name)); + + if !matches!(self.camera_permission, Some(PermissionStatus::Granted)) { + log!("Waiting for camera permission..."); + self.view.label(cx, ids!(lobby_status)).set_text(cx, "Waiting for camera permission..."); + return; + } + + let Some(choice) = self.camera_choice.clone() else { + log!("Waiting for camera device..."); + self.view.label(cx, ids!(lobby_status)).set_text(cx, "Waiting for camera device..."); + return; + }; + + if self.in_lobby { + log!("Starting lobby camera: {}", choice.name); + self.view.label(cx, ids!(lobby_status)) + .set_text(cx, &format!("Camera: {} ({}x{})", choice.name, choice.width, choice.height)); + if CameraManager::start_lobby_camera(&self.view, cx, &choice) { + log!("Lobby camera started successfully"); + self.camera_active = true; + } else { + log!("Failed to start lobby camera (already running?)"); + } + } else if CameraManager::start_call_camera(&self.view, cx, &choice) { + log!("Call camera started successfully"); + self.camera_active = true; + } + } + + /// Check speaking state + fn check_speaking_state(&mut self, cx: &mut Cx) { + if self.in_lobby { + if self.speaking_detector.is_speaking { + self.speaking_detector.is_speaking = false; + SpeakingDetector::update_indicator(&self.view, cx, false); + } + return; + } + + if self.speaking_detector.check_speaking(self.call.local_audio_muted) { + SpeakingDetector::update_indicator(&self.view, cx, self.speaking_detector.is_speaking); + } + } + + /// Handle camera permission result + pub fn handle_camera_permission(&mut self, cx: &mut Cx, status: PermissionStatus) { + self.camera_permission = Some(status); + match status { + PermissionStatus::Granted => { + log!("Camera permission granted"); + self.try_start_camera(cx); + } + PermissionStatus::DeniedPermanent => { + log!("Camera permission denied permanently"); + self.view.label(cx, ids!(lobby_status)).set_text(cx, "Camera permission denied"); + } + _ => { + log!("Camera permission: {:?}", status); + } + } + } + + /// Handle video playback prepared + fn handle_video_prepared(&mut self, cx: &mut Cx) { + if self.camera_active { + if self.in_lobby { + CameraManager::show_lobby_video(&self.view, cx); + } else { + CameraManager::show_call_video(&self.view, cx); + } + } + } + + /// Handle video texture updated + fn handle_video_texture_updated(&mut self, cx: &mut Cx) { + if self.camera_active { + if self.in_lobby { + CameraManager::show_lobby_video(&self.view, cx); + } else { + CameraManager::show_call_video(&self.view, cx); + } + } + } + + /// Handle video resources released + fn handle_video_resources_released(&mut self, cx: &mut Cx) { + log!("Video resources released"); + if self.pending_call_camera_start { + self.pending_call_camera_start = false; + if let Some(choice) = self.camera_choice.clone() { + if CameraManager::start_call_camera(&self.view, cx, &choice) { + self.camera_active = true; + } + } + } + } + + /// Handle UI actions + fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { + + // Lobby buttons + if self.view.button(cx, ids!(video_call_button)).clicked(actions) { + log!("Video call button clicked"); + self.start_call(cx, CallType::Video); + } + if self.view.button(cx, ids!(voice_call_button)).clicked(actions) { + self.start_call(cx, CallType::Voice); + } + if self.view.button(cx, ids!(lobby_mic_button)).clicked(actions) { + self.lobby_mic_enabled = !self.lobby_mic_enabled; + self.update_ui(cx); + } + if self.view.button(cx, ids!(lobby_camera_button)).clicked(actions) { + self.lobby_camera_enabled = !self.lobby_camera_enabled; + self.update_ui(cx); + } + + // Call controls + if self.view.button(cx, ids!(mic_button)).clicked(actions) { + self.toggle_microphone(); + self.update_ui(cx); + } + if self.view.button(cx, ids!(camera_button)).clicked(actions) { + self.toggle_camera(); + self.update_ui(cx); + } + if self.view.button(cx, ids!(screenshare_button)).clicked(actions) { + self.toggle_screenshare(); + self.update_ui(cx); + } + if self.view.button(cx, ids!(participants_button)).clicked(actions) { + self.show_participants = !self.show_participants; + self.update_ui(cx); + } + if self.view.button(cx, ids!(hangup_button)).clicked(actions) { + self.hangup(cx); + self.update_ui(cx); + } + } + + /// Add a test participant + pub fn add_participant(&mut self, cx: &mut Cx, name: &str) { + self.participant_counter += 1; + let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); + let participant = Participant { + id: format!("{}", self.participant_counter), + name: name.to_string(), + avatar_letter: letter, + is_muted: false, + is_speaking: false, + }; + log!("Adding participant: {} (id={})", name, self.participant_counter); + + let list = self.view.participants_list(cx, ids!(participants_list)); + list.add_participant(cx, participant); + } + + /// Remove a participant + pub fn remove_participant(&mut self, cx: &mut Cx, id: &str) { + log!("Removing participant with id={}", id); + let list = self.view.participants_list(cx, ids!(participants_list)); + list.remove_participant(cx, id); + } + + /// Clear all participants + pub fn clear_participants(&mut self, cx: &mut Cx) { + log!("Clearing all participants"); + let list = self.view.participants_list(cx, ids!(participants_list)); + list.clear(cx); + self.participant_counter = 0; + } +} + +impl VoipScreenRef { + /// Initialize the VoIP screen + pub fn initialize(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.initialize(cx); + } + } + + /// Set the room for this VoIP call + pub fn set_room(&self, cx: &mut Cx, room_id: OwnedRoomId) { + if let Some(mut inner) = self.borrow_mut() { + inner.set_room(cx, room_id); + } + } + + /// Handle camera permission + pub fn handle_camera_permission(&self, cx: &mut Cx, status: PermissionStatus) { + if let Some(mut inner) = self.borrow_mut() { + inner.handle_camera_permission(cx, status); + } + } + + /// Add a participant + pub fn add_participant(&self, cx: &mut Cx, name: &str) { + if let Some(mut inner) = self.borrow_mut() { + inner.add_participant(cx, name); + } + } + + /// Remove a participant + pub fn remove_participant(&self, cx: &mut Cx, id: &str) { + if let Some(mut inner) = self.borrow_mut() { + inner.remove_participant(cx, id); + } + } + + /// Clear all participants + pub fn clear_participants(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.clear_participants(cx); + } + } +} From c0c3dfd6cd29910464a79c4019a44ef6b26ae3f9 Mon Sep 17 00:00:00 2001 From: alanpoon Date: Fri, 3 Apr 2026 12:25:39 +0800 Subject: [PATCH 02/21] SelectedRoom Voip --- resources/icons/microphone.svg | 6 + resources/icons/video.svg | 4 + src/app.rs | 62 +++++- src/home/home_screen.rs | 66 ++++--- src/home/main_desktop_ui.rs | 42 +++- src/home/main_mobile_ui.rs | 8 + src/home/navigation_tab_bar.rs | 4 +- src/home/room_screen.rs | 242 ++++++++++++++++++++++- src/home/rooms_list.rs | 15 ++ src/home/rooms_list_entry.rs | 21 ++ src/shared/styles.rs | 2 + src/sliding_sync.rs | 15 ++ src/voip/mod.rs | 10 + src/voip/voip_screen.rs | 352 ++++++++++++++++++++++----------- 14 files changed, 698 insertions(+), 151 deletions(-) create mode 100644 resources/icons/microphone.svg create mode 100644 resources/icons/video.svg diff --git a/resources/icons/microphone.svg b/resources/icons/microphone.svg new file mode 100644 index 000000000..4f2957aa5 --- /dev/null +++ b/resources/icons/microphone.svg @@ -0,0 +1,6 @@ + + + + + + diff --git a/resources/icons/video.svg b/resources/icons/video.svg new file mode 100644 index 000000000..5ac9a5a63 --- /dev/null +++ b/resources/icons/video.svg @@ -0,0 +1,4 @@ + + + + diff --git a/src/app.rs b/src/app.rs index 72e990aa4..8760580d1 100644 --- a/src/app.rs +++ b/src/app.rs @@ -16,7 +16,7 @@ use crate::{ VerificationModalAction, VerificationModalWidgetRefExt, }, - voip::VoipGlobalState, + voip::{VoipGlobalState, VoipAction}, }; script_mod! { @@ -402,6 +402,16 @@ impl MatchEvent for App { _ => {} } + // Handle VoIP close action - reset the VoIP visibility on the mobile RoomScreen + // if let Some(VoipAction::Close(_room_id)) = action.downcast_ref() { + // log!("App: VoipAction::Close received, resetting VoIP visibility"); + // // Reset VoIP visibility on room_screen_0 (mobile path) + // let room_screen = self.ui.room_screen(cx, ids!(room_screen_0)); + // room_screen.set_voip_visible(cx, false, None); + // self.ui.redraw(cx); + // continue; + // } + // When a stack navigation pop is initiated (back button pressed), // pop the mobile nav stack so it stays in sync with StackNavigation. if let StackNavigationAction::Pop = action.as_widget_action().cast() { @@ -628,6 +638,8 @@ impl MatchEvent for App { continue; } + // Note: RtcCallAction::JoinCall is now handled via RoomsListAction::Selected(SelectedRoom::Voip) + // Handle a request to show the generic positive confirmation modal. if let Some(PositiveConfirmationModalAction::Show(content_opt)) = action.downcast_ref() { if let Some(content) = content_opt.borrow_mut().take() { @@ -880,9 +892,19 @@ impl App { let cmd = line.trim().to_lowercase(); match cmd.as_str() { "voip" => { - log!("Stdin command: switching to VoIP screen"); - cx.action(NavigationBarAction::GoToVoip); - self.ui.redraw(cx); + // Get the currently selected room and show VoIP for it + if let Some(selected) = &self.app_state.selected_room { + let room_name_id = selected.room_name().clone(); + log!("Stdin command: showing VoIP for room {}", room_name_id.room_id()); + // Use widget_action so it's handled by the widget action handlers + cx.widget_action( + WidgetUid(0), // Use a placeholder widget_uid + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + self.ui.redraw(cx); + } else { + log!("Stdin command: no room selected, cannot show VoIP"); + } } "home" => { log!("Stdin command: switching to Home screen"); @@ -1067,6 +1089,12 @@ impl App { .set_displayed_space(cx, space_name_id); id!(space_lobby_view) } + SelectedRoom::Voip { room_name_id } => { + // VoIP uses RoomScreen with VoIP as main content (no timeline) + let room_screen = self.ui.room_screen(cx, ids!(room_screen_0)); + room_screen.set_voip_visible(cx, true, Some(room_name_id.room_id().clone())); + id!(room_view_0) + } }; // Set the header title for the view being pushed. @@ -1112,6 +1140,9 @@ pub struct AppState { pub logged_in: bool, /// Local configuration and UI state for bot-assisted room binding. pub bot_settings: BotSettingsState, + /// The room ID for VoIP calls, set when navigating to VoIP screen from a call notification. + #[serde(skip)] + pub voip_room_id: Option, } /// Local bot integration settings persisted per Matrix account. @@ -1280,6 +1311,9 @@ pub enum SelectedRoom { Space { space_name_id: RoomNameId, }, + Voip { + room_name_id: RoomNameId, + }, } impl SelectedRoom { @@ -1289,6 +1323,8 @@ impl SelectedRoom { SelectedRoom::InvitedRoom { room_name_id } => room_name_id.room_id(), SelectedRoom::Space { space_name_id } => space_name_id.room_id(), SelectedRoom::Thread { room_name_id, .. } => room_name_id.room_id(), + SelectedRoom::Voip { room_name_id } => room_name_id.room_id(), + } } @@ -1298,6 +1334,7 @@ impl SelectedRoom { SelectedRoom::InvitedRoom { room_name_id } => room_name_id, SelectedRoom::Space { space_name_id } => space_name_id, SelectedRoom::Thread { room_name_id, .. } => room_name_id, + SelectedRoom::Voip { room_name_id } => room_name_id, } } @@ -1328,6 +1365,12 @@ impl SelectedRoom { &format!("{}##{}", room_name_id.room_id(), thread_root_event_id) ) } + SelectedRoom::Voip { room_name_id } => { + // VoIP tabs get a distinct ID to differentiate from normal room tabs + LiveId::from_str( + &format!("{}##voip", room_name_id.room_id()) + ) + } other => LiveId::from_str(other.room_id().as_str()), } } @@ -1339,6 +1382,7 @@ impl SelectedRoom { SelectedRoom::InvitedRoom { room_name_id } => room_name_id.to_string(), SelectedRoom::Space { space_name_id } => format!("[Space] {space_name_id}"), SelectedRoom::Thread { room_name_id, .. } => format!("[Thread] {room_name_id}"), + SelectedRoom::Voip { room_name_id } => format!("[VoIP] {room_name_id}"), } } } @@ -1346,6 +1390,7 @@ impl SelectedRoom { impl PartialEq for SelectedRoom { fn eq(&self, other: &Self) -> bool { match (self, other) { + // Threads are equal if room_id and thread_root_event_id match ( SelectedRoom::Thread { room_name_id: lhs_room_name_id, @@ -1359,7 +1404,16 @@ impl PartialEq for SelectedRoom { lhs_room_name_id.room_id() == rhs_room_name_id.room_id() && lhs_thread_root_event_id == rhs_thread_root_event_id } + // Thread is never equal to non-Thread (SelectedRoom::Thread { .. }, _) | (_, SelectedRoom::Thread { .. }) => false, + // VoIP rooms are equal only to other VoIP rooms with same room_id + ( + SelectedRoom::Voip { room_name_id: lhs }, + SelectedRoom::Voip { room_name_id: rhs }, + ) => lhs.room_id() == rhs.room_id(), + // VoIP is never equal to non-VoIP (even if same room_id) + (SelectedRoom::Voip { .. }, _) | (_, SelectedRoom::Voip { .. }) => false, + // All other variants compare by room_id only _ => self.room_id() == other.room_id(), } } diff --git a/src/home/home_screen.rs b/src/home/home_screen.rs index 86b388bc0..c0d3333d9 100644 --- a/src/home/home_screen.rs +++ b/src/home/home_screen.rs @@ -1,6 +1,6 @@ use makepad_widgets::*; -use crate::{app::AppState, home::navigation_tab_bar::{NavigationBarAction, SelectedTab}, voip::VoipScreenWidgetRefExt, settings::settings_screen::SettingsScreenWidgetRefExt}; +use crate::{app::AppState, home::navigation_tab_bar::{NavigationBarAction, SelectedTab}, settings::settings_screen::SettingsScreenWidgetRefExt}; script_mod! { use mod.prelude.widgets.* @@ -249,15 +249,15 @@ script_mod! { } } - voip_page := SolidView { - width: Fill, height: Fill - show_bg: true, - draw_bg.color: (COLOR_PRIMARY) + // voip_page := SolidView { + // width: Fill, height: Fill + // show_bg: true, + // draw_bg.color: (COLOR_PRIMARY) - CachedWidget { - voip_screen := mod.widgets.VoipScreen {} - } - } + // CachedWidget { + // voip_screen := mod.widgets.VoipScreen {} + // } + // } } } @@ -308,14 +308,14 @@ script_mod! { } } - voip_page := View { - width: Fill, height: Fill - padding: Inset{top: 20} + // voip_page := View { + // width: Fill, height: Fill + // padding: Inset{top: 20} - CachedWidget { - voip_screen := mod.widgets.VoipScreen {} - } - } + // CachedWidget { + // voip_screen := mod.widgets.VoipScreen {} + // } + // } } // Show the SpacesBar right above the navigation tab bar. @@ -485,21 +485,25 @@ impl Widget for HomeScreen { self.view.redraw(cx); } } - Some(NavigationBarAction::GoToVoip) => { - if !matches!(app_state.selected_tab, SelectedTab::VoIP) { - self.previous_selection = app_state.selected_tab.clone(); - app_state.selected_tab = SelectedTab::VoIP; - cx.action(NavigationBarAction::TabSelected(app_state.selected_tab.clone())); - if let Some(voip_page) = self.update_active_page_from_selection(cx, app_state) { - voip_page - .voip_screen(cx, ids!(voip_screen)) - .initialize(cx); - self.view.redraw(cx); - } else { - error!("BUG: failed to set active page to show VoIP screen."); - } - } - } + // Some(NavigationBarAction::GoToVoip) => { + // if !matches!(app_state.selected_tab, SelectedTab::VoIP) { + // self.previous_selection = app_state.selected_tab.clone(); + // app_state.selected_tab = SelectedTab::VoIP; + // cx.action(NavigationBarAction::TabSelected(app_state.selected_tab.clone())); + // if let Some(voip_page) = self.update_active_page_from_selection(cx, app_state) { + // let voip_screen = voip_page.voip_screen(cx, ids!(voip_screen)); + // voip_screen.initialize(cx); + // // If a room_id was set from a call notification, pass it to the VoIP screen + // if let Some(room_id) = app_state.voip_room_id.take() { + // log!("Setting VoIP room from call notification: {}", room_id); + // voip_screen.set_room(cx, room_id); + // } + // self.view.redraw(cx); + // } else { + // error!("BUG: failed to set active page to show VoIP screen."); + // } + // } + // } Some(NavigationBarAction::ToggleSpacesBar) => { self.is_spaces_bar_shown = !self.is_spaces_bar_shown; self.view.spaces_bar_wrapper(cx, ids!(spaces_bar_wrapper)) diff --git a/src/home/main_desktop_ui.rs b/src/home/main_desktop_ui.rs index 628d477bf..1b829d837 100644 --- a/src/home/main_desktop_ui.rs +++ b/src/home/main_desktop_ui.rs @@ -3,7 +3,7 @@ use ruma::OwnedRoomId; use tokio::sync::Notify; use std::{collections::HashMap, sync::Arc}; -use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, utils::RoomNameId}; +use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, utils::RoomNameId, voip::{voip_screen::VoipScreenWidgetRefExt, VoipAction}}; use super::{invite_screen::InviteScreenWidgetRefExt, room_screen::RoomScreenWidgetRefExt, rooms_list::RoomsListAction}; script_mod! { @@ -63,6 +63,7 @@ script_mod! { room_screen := mod.widgets.RoomScreen {} invite_screen := mod.widgets.InviteScreen {} space_lobby_screen := mod.widgets.SpaceLobbyScreen {} + voip_screen := mod.widgets.VoipScreen {} } } } @@ -160,6 +161,7 @@ impl MainDesktopUI { let kind = match &room { SelectedRoom::JoinedRoom { .. } | SelectedRoom::Thread { .. } => id!(room_screen), + SelectedRoom::Voip { .. } => id!(voip_screen), // VoIP uses standalone VoipScreen SelectedRoom::InvitedRoom { .. } => id!(invite_screen), SelectedRoom::Space { .. } => id!(space_lobby_screen), }; @@ -210,6 +212,11 @@ impl MainDesktopUI { space_name_id, ); } + SelectedRoom::Voip { room_name_id } => { + // VoIP uses standalone VoipScreen widget + log!("MainDesktopUI: Creating VoIP tab for room {}", room_name_id.room_id()); + new_widget.as_voip_screen().initialize(cx, room_name_id.room_id().clone()); + } } cx.action(MainDesktopUiAction::SaveDockIntoAppState); } else { @@ -222,8 +229,10 @@ impl MainDesktopUI { /// Closes a tab in the dock and focuses on the latest open room. fn close_tab(&mut self, cx: &mut Cx, tab_id: LiveId) { + log!("close_tab called for tab_id: {:?}", tab_id); let dock = self.view.dock(cx, ids!(dock)); if let Some(room_being_closed) = self.open_rooms.get(&tab_id) { + log!("Found room to close: {:?}", room_being_closed); self.room_order.retain(|sr| sr != room_being_closed); if self.open_rooms.len() > 1 { @@ -242,15 +251,20 @@ impl MainDesktopUI { } } else { // If there is no room to focus, notify app to reset the selected room in the app state + log!("No more rooms to focus, selecting home_tab"); cx.action(AppStateAction::FocusNone); dock.select_tab(cx, id!(home_tab)); self.most_recently_selected_room = None; } + } else { + log!("Room not found in open_rooms for tab_id: {:?}", tab_id); } + log!("Calling dock.close_tab for tab_id: {:?}", tab_id); dock.close_tab(cx, tab_id); self.tab_to_close = None; self.open_rooms.remove(&tab_id); + log!("Tab closed, open_rooms now has {} tabs", self.open_rooms.len()); } /// Closes all tabs @@ -389,6 +403,10 @@ impl MainDesktopUI { Some(thread_root_event_id.clone()), ); } + Some(SelectedRoom::Voip { room_name_id }) => { + // VoIP uses standalone VoipScreen widget + widget.as_voip_screen().initialize(cx, room_name_id.room_id().clone()); + } None => { } } } @@ -498,6 +516,7 @@ impl WidgetMatchEvent for MainDesktopUI { // Note that this cannot be performed within draw_walk() as the draw flow prevents from // performing actions that would trigger a redraw, and the Dock internally performs (and expects) // a redraw to be happening in order to draw the tab content. + log!("MainDesktopUI: RoomsListAction::Selected received for {:?}", selected_room); self.focus_or_create_tab(cx, selected_room.clone()); } RoomsListAction::InviteAccepted { room_name_id } => { @@ -507,6 +526,27 @@ impl WidgetMatchEvent for MainDesktopUI { RoomsListAction::None => { } } + // Handle VoIP actions + if let Some(VoipAction::Close(room_id)) = action.downcast_ref() { + log!("MainDesktopUI: VoipAction::Close received for room {:?}", room_id); + // Find the VoIP tab for this room and close it + let voip_tab_id = LiveId::from_str(&format!("{}##voip", room_id)); + log!("Looking for voip_tab_id: {:?}, open_rooms has {} tabs", voip_tab_id, self.open_rooms.len()); + if self.open_rooms.contains_key(&voip_tab_id) { + log!("Found VoIP tab, closing it"); + self.tab_to_close = Some(voip_tab_id); + self.close_tab(cx, voip_tab_id); + self.redraw(cx); + should_save_dock_action = true; + } else { + log!("VoIP tab NOT found in open_rooms! Trying to close anyway..."); + // Try closing the tab anyway via the dock + let dock = self.view.dock(cx, ids!(dock)); + dock.close_tab(cx, voip_tab_id); + self.redraw(cx); + } + } + // Handle our own actions related to dock updates that we have previously emitted. match action.downcast_ref() { Some(MainDesktopUiAction::LoadDockFromAppState) => { diff --git a/src/home/main_mobile_ui.rs b/src/home/main_mobile_ui.rs index f06118447..3ecc6842d 100644 --- a/src/home/main_mobile_ui.rs +++ b/src/home/main_mobile_ui.rs @@ -116,6 +116,14 @@ impl Widget for MainMobileUI { .room_screen(cx, ids!(room_screen)) .set_displayed_room(cx, room_name_id, Some(thread_root_event_id.clone())); } + Some(SelectedRoom::Voip { room_name_id }) => { + show_welcome = false; + show_room = true; // VoIP uses RoomScreen with VoIP as main content + show_invite = false; + show_space_lobby = false; + let room_screen = self.view.room_screen(cx, ids!(room_screen)); + room_screen.set_voip_visible(cx, true, Some(room_name_id.room_id().clone())); + } None => { show_welcome = true; show_room = false; diff --git a/src/home/navigation_tab_bar.rs b/src/home/navigation_tab_bar.rs index 823eb9f40..02d568ee4 100644 --- a/src/home/navigation_tab_bar.rs +++ b/src/home/navigation_tab_bar.rs @@ -542,8 +542,8 @@ pub enum NavigationBarAction { CloseSettings, /// Go the space screen for the given space. GoToSpace { space_name_id: RoomNameId }, - /// Go to the VoIP call screen. - GoToVoip, + // /// Go to the VoIP call screen. + // GoToVoip, // TODO: add GoToAlertsInbox, once we add that button/screen diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index a74781e9b..a3e6bd1b8 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -40,6 +40,7 @@ use crate::home::event_reaction_list::ReactionListWidgetRefExt; use crate::home::room_read_receipt::AvatarRowWidgetRefExt; use crate::room::room_input_bar::RoomInputBarWidgetExt; use crate::shared::mentionable_text_input::MentionableTextInputAction; +use crate::voip::voip_screen::VoipScreenWidgetExt; use rangemap::RangeSet; @@ -516,6 +517,86 @@ script_mod! { } + // The view used for RTC notification events (call invites, call notifications). + // Displays a call notification with a "Join Call" button. + mod.widgets.RtcNotificationEvent = View { + width: Fill, + height: Fit, + flow: Right, + margin: Inset{ top: 8.0, bottom: 8.0 } + padding: Inset{ top: 8.0, bottom: 8.0, left: 10.0, right: 10.0 } + spacing: 0.0 + cursor: MouseCursor.Default + + show_bg: true + draw_bg +: { + color: #f0e6ff + } + + body := View { + width: Fill, + height: Fit + flow: Right, + padding: Inset{ left: 7.0, top: 2.0, bottom: 2.0 } + spacing: 10.0 + align: Align{y: 0.5} + + left_container := View { + align: Align{x: 0.5, y: 0} + width: 70.0, + height: Fit + + timestamp := Timestamp { + margin: Inset{top: 3} + } + } + + avatar := Avatar { + width: 24., + height: 24., + margin: 0 + + text_view +: { + text +: { + draw_text +: { + text_style: TITLE_TEXT { font_size: 9.0 } + } + } + } + } + + content := Label { + width: Fit, + height: Fit + margin: Inset{top: 2.5} + draw_text +: { + text_style: SMALL_STATE_TEXT_STYLE {}, + color: #333 + } + text: "" + } + + View { width: Fill, height: 1 } + + join_call_button := RobrixPositiveIconButton { + margin: Inset{ top: -1.5, left: 2, right: 2} + padding: Inset{top: 6, bottom: 6, left: 12, right: 12} + draw_bg +: { + border_size: 0.75 + color: #7b1fa2 + } + draw_text +: { + color: #fff + text_style: SMALL_STATE_TEXT_STYLE {} + } + text: "Join Call" + } + + avatar_row := mod.widgets.AvatarRow {} + } + } + + // The view used for each day divider in a room's timeline. // The date text is centered between two horizontal lines. mod.widgets.DateDivider = View { @@ -789,6 +870,7 @@ script_mod! { ImageMessage := mod.widgets.ImageMessage {} CondensedImageMessage := mod.widgets.CondensedImageMessage {} SmallStateEvent := mod.widgets.SmallStateEvent {} + RtcNotificationEvent := mod.widgets.RtcNotificationEvent {} Empty := mod.widgets.Empty {} DateDivider := mod.widgets.DateDivider {} ReadMarker := mod.widgets.ReadMarker {} @@ -840,6 +922,31 @@ script_mod! { // The top space should be displayed as an overlay at the top of the timeline. top_space := mod.widgets.TopSpace { } + // Video call button - floating in top right corner + video_call_button_container := View { + width: Fill + height: Fit + align: Align{x: 1.0, y: 0.0} + padding: Inset{top: 10, right: 10} + + video_call_button := RobrixIconButton { + width: 40 + height: 40 + padding: 8 + draw_icon.svg: (ICON_VIDEO) + icon_walk: Walk{width: 20, height: 20} + draw_bg +: { + color: #fff + border_radius: 20.0 + border_size: 1.0 + border_color: #ddd + } + draw_icon +: { + color: #333 + } + } + } + // The user profile sliding pane should be displayed on top of other "static" subviews // (on top of all other views that are always visible). user_profile_sliding_pane := mod.widgets.UserProfileSlidingPane { } @@ -860,7 +967,6 @@ script_mod! { } } - /* * TODO: add the action bar back in as a series of floating buttons. * @@ -904,6 +1010,8 @@ pub struct RoomScreen { #[rust] all_rooms_loaded: bool, /// Whether the in-room app service quick actions card is currently visible. #[rust] show_app_service_actions: bool, + /// Whether the VoIP call screen is currently visible for this room. + #[rust] show_voip_screen: bool, } impl Drop for RoomScreen { @@ -1048,6 +1156,27 @@ impl Widget for RoomScreen { cx.action(InviteAction::ShowInviteConfirmationModal(RefCell::new(Some(content)))); } } + + // Handle the join_call_button (in an RtcNotificationEvent) being clicked. + if wr.button(cx, ids!(join_call_button)).clicked(actions) { + let Some(room_name_id) = self.room_name_id.clone() else { continue }; + log!("Join call button clicked for room: {}", room_name_id.room_id()); + cx.widget_action( + self.widget_uid(), + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + } + } + + // Handle the video_call_button (floating button in top-right) being clicked. + if self.view.button(cx, ids!(video_call_button)).clicked(actions) { + if let Some(room_name_id) = self.room_name_id.clone() { + log!("Video call button clicked for room: {}", room_name_id.room_id()); + cx.widget_action( + self.widget_uid(), + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + } } self.handle_message_actions(cx, actions, &portal_list, &loading_pane); @@ -1688,6 +1817,16 @@ impl Widget for RoomScreen { other, item_drawn_status, ), + TimelineItemContent::CallInvite | TimelineItemContent::RtcNotification => { + populate_rtc_notification_event( + cx, + list, + item_id, + &tl_state.kind, + event_tl_item, + item_drawn_status, + ) + }, unhandled => { let item = list.item(cx, item_id, id!(SmallStateEvent)); item.label(cx, ids!(content)).set_text(cx, &format!("[Unsupported] {:?}", unhandled)); @@ -3315,6 +3454,37 @@ impl RoomScreenRef { let Some(mut inner) = self.borrow_mut() else { return }; inner.set_displayed_room(cx, room_name_id, thread_root_event_id); } + + /// Shows or hides the VoIP call screen for this room. + /// When visible, hides the timeline and shows VoIP as the main content. + pub fn set_voip_visible(&self, cx: &mut Cx, visible: bool, room_id: Option) { + println!("set_voip_visible visible {:?}", visible); + let Some(mut inner) = self.borrow_mut() else { + // This can happen on mobile path when desktop is active + println!("Failed to borrow RoomScreen mutably"); + return; + }; + inner.show_voip_screen = visible; + + // Hide/show the timeline area (keyboard_view contains timeline, typing notice, input bar) + inner.view.view(cx, ids!(keyboard_view)).set_visible(cx, !visible); + + if visible { + // Initialize the VoIP screen if showing + let voip_screen = inner.view.voip_screen(cx, ids!(voip_screen)); + if let Some(room_id) = room_id { + log!("RoomScreen: Showing VoIP screen for room {}", room_id); + voip_screen.initialize(cx, room_id); + } + } else { + let voip_screen = inner.view.voip_screen(cx, ids!(voip_screen)); + println!("hangup"); + voip_screen.hangup(cx); + } + + // Force a redraw to ensure visibility changes take effect + inner.view.redraw(cx); + } } /// Immutable RoomScreen states passed via Scope props @@ -5202,6 +5372,64 @@ fn get_profile_display_name(event_tl_item: &EventTimelineItem) -> Option } } +/// Creates, populates, and adds an RtcNotificationEvent widget to the given `PortalList` +/// with the given `item_id`. +/// +/// This is used for CallInvite and RtcNotification timeline events. +fn populate_rtc_notification_event( + cx: &mut Cx, + list: &mut PortalList, + item_id: usize, + timeline_kind: &TimelineKind, + event_tl_item: &EventTimelineItem, + item_drawn_status: ItemDrawnStatus, +) -> (WidgetRef, ItemDrawnStatus) { + let mut new_drawn_status = item_drawn_status; + let (item, existed) = list.item_with_existed(cx, item_id, id!(RtcNotificationEvent)); + + let skip_redrawing_profile = existed && item_drawn_status.profile_drawn; + let skip_redrawing_content = skip_redrawing_profile && item_drawn_status.content_drawn; + + populate_read_receipts(&item, cx, timeline_kind, event_tl_item); + + if skip_redrawing_content { + return (item, new_drawn_status); + } + + // Get username for display + let username = if skip_redrawing_profile { + get_profile_display_name(event_tl_item) + .unwrap_or_else(|| event_tl_item.sender().to_string()) + } else { + let avatar_ref = item.avatar(cx, ids!(avatar)); + let (username, profile_drawn) = avatar_ref.set_avatar_and_get_username( + cx, + timeline_kind, + event_tl_item.sender(), + Some(event_tl_item.sender_profile()), + event_tl_item.event_id(), + true, + ); + // Draw the timestamp as part of the profile. + if let Some(dt) = unix_time_millis_to_datetime(event_tl_item.timestamp()) { + item.timestamp(cx, ids!(left_container.timestamp)).set_date_time(cx, dt); + } + new_drawn_status.profile_drawn = profile_drawn; + username + }; + + // Set the content text based on the event type + let content_text = match event_tl_item.content() { + TimelineItemContent::CallInvite => format!("{} started a call", username), + TimelineItemContent::RtcNotification => format!("{} is in a call", username), + _ => format!("{} - call notification", username), + }; + item.label(cx, ids!(content)).set_text(cx, &content_text); + + new_drawn_status.content_drawn = true; + (item, new_drawn_status) +} + /// Actions related to invites within a room. /// @@ -5216,6 +5444,18 @@ pub enum InviteAction { ShowInviteConfirmationModal(RefCell>), } +/// Actions related to RTC (Real-Time Communication) calls. +/// +/// These are NOT widget actions, just regular actions. +#[derive(Debug, Clone)] +pub enum RtcCallAction { + /// The user clicked the "Join Call" button on an RTC notification event. + /// Navigate to the VoIP screen with the given room ID. + JoinCall { + room_id: OwnedRoomId, + }, +} + /// The result of inviting a user to a room. /// #[derive(Debug)] diff --git a/src/home/rooms_list.rs b/src/home/rooms_list.rs index 73ce9375e..ff6327a03 100644 --- a/src/home/rooms_list.rs +++ b/src/home/rooms_list.rs @@ -203,6 +203,11 @@ pub enum RoomsListUpdate { TombstonedRoom { room_id: OwnedRoomId }, + /// Update whether the given room has an active call. + UpdateActiveCall { + room_id: OwnedRoomId, + has_active_call: bool, + }, /// Hide the given room from being displayed. /// /// This is useful for temporarily preventing a room from being shown, @@ -298,6 +303,8 @@ pub struct JoinedRoomInfo { pub is_direct: bool, /// Whether this room is tombstoned (shut down and replaced with a successor room). pub is_tombstoned: bool, + /// Whether this room has an active video/voice call. + pub has_active_call: bool, // TODO: we could store the parent chain(s) of this room, i.e., which spaces // they are children of. One room can be in multiple spaces. @@ -763,6 +770,14 @@ impl RoomsList { warning!("Warning: couldn't find room {room_id} to update the tombstone status"); } } + RoomsListUpdate::UpdateActiveCall { room_id, has_active_call } => { + if let Some(room) = self.all_joined_rooms.get_mut(&room_id) { + room.has_active_call = has_active_call; + log!("Updated room {} has_active_call to {}", room_id, has_active_call); + } else { + warning!("Warning: couldn't find room {room_id} to update active call status"); + } + } RoomsListUpdate::HideRoom { room_id } => { self.hidden_rooms.insert(room_id.clone()); // Hiding a regular room is the most common case (e.g., after its successor is joined), diff --git a/src/home/rooms_list_entry.rs b/src/home/rooms_list_entry.rs index d421a12ac..6e81e8ae1 100644 --- a/src/home/rooms_list_entry.rs +++ b/src/home/rooms_list_entry.rs @@ -30,6 +30,22 @@ script_mod! { } } + // A video call icon to be displayed when there's an active call in the room. + mod.widgets.ActiveCallIcon = View { + width: Fit, height: Fit, + visible: false, + + Icon { + width: 19, height: 19, + align: Align{x: 0.5, y: 0.5} + draw_icon +: { + svg: (ICON_VIDEO) + color: #4CAF50 // Green color for active call + } + icon_walk: Walk{ width: 15, height: 15 } + } + } + mod.widgets.RoomName = Label { width: Fill, height: Fit flow: Flow.Right{wrap: false}, @@ -150,6 +166,7 @@ script_mod! { avatar := Avatar {} unread_badge := UnreadBadge {} tombstone_icon := mod.widgets.TombstoneIcon {} + active_call_icon := mod.widgets.ActiveCallIcon {} } } IconAndName := mod.widgets.RoomsListEntryContent { @@ -159,6 +176,7 @@ script_mod! { room_name := mod.widgets.RoomName {} unread_badge := UnreadBadge {} tombstone_icon := mod.widgets.TombstoneIcon {} + active_call_icon := mod.widgets.ActiveCallIcon {} } FullPreview := mod.widgets.RoomsListEntryContent { padding: 10 @@ -184,6 +202,7 @@ script_mod! { View { width: Fit, height: Fit align: Align{ x: 1.0 } + active_call_icon := mod.widgets.ActiveCallIcon {} unread_badge := UnreadBadge {} tombstone_icon := mod.widgets.TombstoneIcon {} } @@ -333,6 +352,8 @@ impl RoomsListEntryContent { self.draw_common(cx, &room_info.room_avatar, room_info.is_selected); // Show tombstone icon if the room is tombstoned self.view.view(cx, ids!(tombstone_icon)).set_visible(cx, room_info.is_tombstoned); + // Show active call icon if there's an active call in this room + self.view.view(cx, ids!(active_call_icon)).set_visible(cx, room_info.has_active_call); } /// Populates this RoomsListEntry with info about an invited room. diff --git a/src/shared/styles.rs b/src/shared/styles.rs index a80fa55e5..c6b1299f9 100644 --- a/src/shared/styles.rs +++ b/src/shared/styles.rs @@ -6,6 +6,8 @@ script_mod! { use mod.widgets.* mod.widgets.ICON_ADD = crate_resource("self://resources/icons/add.svg") + mod.widgets.ICON_MICROPHONE = crate_resource("self://resources/icons/microphone.svg") + mod.widgets.ICON_VIDEO = crate_resource("self://resources/icons/video.svg") mod.widgets.ICON_ADD_REACTION = crate_resource("self://resources/icons/add_reaction.svg") mod.widgets.ICON_ADD_USER = crate_resource("self://resources/icons/add_user.svg") // TODO: FIX mod.widgets.ICON_ADD_WALLET = crate_resource("self://resources/icons/add_wallet.svg") diff --git a/src/sliding_sync.rs b/src/sliding_sync.rs index 131e6610f..8b8039b0c 100644 --- a/src/sliding_sync.rs +++ b/src/sliding_sync.rs @@ -2570,6 +2570,7 @@ struct RoomListServiceRoomInfo { is_direct: bool, is_marked_unread: bool, is_tombstoned: bool, + has_active_call: bool, tags: Option, user_power_levels: Option, latest_event_timestamp: Option, @@ -2601,6 +2602,7 @@ impl RoomListServiceRoomInfo { is_direct: is_direct.unwrap_or(false), is_marked_unread: room.is_marked_unread(), is_tombstoned: room.is_tombstoned(), + has_active_call: room.has_active_room_call(), tags: tags.ok().flatten(), user_power_levels, latest_event_timestamp: room.latest_event_timestamp(), @@ -3335,6 +3337,18 @@ async fn update_room( }); } + if old_room.has_active_call != new_room.has_active_call { + log!("Updating room {} has_active_call from {} to {}", + new_room_id, + old_room.has_active_call, + new_room.has_active_call, + ); + enqueue_rooms_list_update(RoomsListUpdate::UpdateActiveCall { + room_id: new_room_id.clone(), + has_active_call: new_room.has_active_call, + }); + } + let mut __timeline_update_sender_opt = None; let mut get_timeline_update_sender = |room_id| { if __timeline_update_sender_opt.is_none() { @@ -3535,6 +3549,7 @@ async fn add_new_room( is_selected: false, is_direct: new_room.is_direct, is_tombstoned: new_room.is_tombstoned, + has_active_call: new_room.has_active_call, })); Cx::post_action(AppStateAction::RoomLoadedSuccessfully { diff --git a/src/voip/mod.rs b/src/voip/mod.rs index 681665372..b131b867d 100644 --- a/src/voip/mod.rs +++ b/src/voip/mod.rs @@ -10,6 +10,7 @@ use makepad_widgets::*; use makepad_widgets::makepad_platform::video::VideoInputsEvent; use makepad_widgets::makepad_platform::permission::PermissionStatus; +use matrix_sdk::ruma::OwnedRoomId; pub mod call_state; pub mod camera; @@ -22,6 +23,15 @@ pub use voip_screen::VoipScreenWidgetRefExt; pub use participants_list::{Participant, ParticipantsListWidgetRefExt}; pub use camera::CameraChoice; +/// Actions emitted by VoIP screens +#[derive(Clone, Debug, Default)] +pub enum VoipAction { + /// Close the VoIP screen and return to the room + Close(OwnedRoomId), + #[default] + None, +} + /// Global VoIP state stored in Makepad's Cx context. /// This allows camera permission and video inputs events to be captured /// at app startup before VoipScreen is shown. diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs index 639e44ff1..d109b86fd 100644 --- a/src/voip/voip_screen.rs +++ b/src/voip/voip_screen.rs @@ -10,7 +10,7 @@ use ruma::OwnedRoomId; use tokio::sync::mpsc; use crate::sliding_sync::get_client; -use super::VoipGlobalState; +use super::{VoipGlobalState, VoipAction}; use super::call_state::{Call, CallType, ConnectionState}; use super::camera::{CameraChoice, CameraManager}; @@ -355,123 +355,186 @@ script_mod! { width: Fill height: Fill flow: Down - spacing: 20 - padding: 40 - align: Center - show_bg: true - draw_bg.color: #1a1a2e visible: true + show_bg: true + draw_bg.color: #2a2a4a - Label { - text: "Join Call" - draw_text.text_style.font_size: 24 - draw_text.color: #fff + // Top bar with close button + lobby_header := View { + width: Fill + height: Fit + flow: Right + padding: Inset{top: 12, right: 12, bottom: 0, left: 12} + align: Align{x: 1.0, y: 0.0} + + close_button := RobrixIconButton { + width: 40 + height: 40 + padding: 8 + draw_icon.svg: (ICON_CLOSE) + icon_walk: Walk{width: 20, height: 20} + draw_bg +: { + color: #ffffff20 + border_radius: 20.0 + } + draw_icon +: { + color: #fff + } + } } - lobby_camera_container := View { - width: 320 - height: 240 + // Main content area with camera preview - use flex to fill remaining space + lobby_content := View { + width: Fill + height: Fill + flow: Overlay + margin: 0 - lobby_camera_placeholder := RoundedView { + // Camera preview background + lobby_camera_container := View { width: Fill height: Fill - draw_bg.color: #2a2a4a - draw_bg.radius: 12.0 - align: Center - Label { - text: "Camera Preview" - draw_text.text_style.font_size: 14 - draw_text.color: #666 - } - } + lobby_camera_placeholder := View { + width: Fill + height: Fill + show_bg: true + draw_bg.color: #2a2a4a + align: Center - lobby_video_host := View { - width: Fill - height: Fill - visible: false + // Placeholder logo/icon + RoundedView { + width: 120 + height: 120 + draw_bg.color: #1a1a2e + draw_bg.radius: 60.0 + align: Center - lobby_camera_video := Video { + Label { + text: "VoIP" + draw_text.text_style.font_size: 24 + draw_text.color: #fff + } + } + } + + lobby_video_host := View { width: Fill height: Fill - autoplay: false - show_controls: false + visible: false + + lobby_camera_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false + } } } - } - View { - width: Fit - height: Fit - flow: Down - spacing: 12 - align: Center + // Join Call button overlay - centered vertically and horizontally + View { + width: Fill + height: Fill + align: Align{x: 0.5, y: 0.7} + + join_call_button := Button { + text: "Join call" + width: 160 + height: 48 + draw_bg +: { + color: #4CAF50 + border_radius: 24.0 + } + draw_text +: { + color: #fff + text_style.font_size: 16 + } + } + } + // Status label at bottom View { - width: Fit - height: Fit - flow: Right - spacing: 8 - align: Center + width: Fill + height: Fill + align: Align{x: 0.5, y: 0.85} - Label { - text: "Microphone:" + lobby_status := Label { + text: "" draw_text.text_style.font_size: 12 - draw_text.color: #888 - } - - lobby_mic_button := Button { - text: "Mic On" - width: 80 - draw_text.color: #333 + draw_text.color: #aaa } } + } + + // Bottom control bar with icons + lobby_controls := View { + width: Fill + height: Fit + padding: Inset{top: 16 bottom: 24} + align: Center + show_bg: true + draw_bg.color: #fff View { width: Fit height: Fit flow: Right - spacing: 8 + spacing: 16 align: Center - Label { - text: "Camera:" - draw_text.text_style.font_size: 12 - draw_text.color: #888 + lobby_mic_button := RobrixIconButton { + width: 48 + height: 48 + padding: Inset{top: 10, bottom: 10, left: 10, right: 10} + margin: 0, + draw_icon.svg: (ICON_MICROPHONE) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #fff + border_radius: 0.0 + border_size: 1.5 + border_color: #ccc + } } - lobby_camera_button := Button { - text: "Cam On" - width: 80 - draw_text.color: #333 + // Video icon button + lobby_camera_button := RobrixIconButton { + width: 48 + height: 48 + padding: Inset{top: 10, bottom: 10, left: 10, right: 10} + margin: 0, + draw_icon.svg: (ICON_VIDEO) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #fff + border_radius: 0.0 + border_size: 1.5 + border_color: #ccc + } } - } - } - - View { - width: Fit - height: Fit - flow: Right - spacing: 12 - - video_call_button := Button { - text: "Video Call" - width: 120 - draw_text.color: #333 - } - voice_call_button := Button { - text: "Voice Call" - width: 120 - draw_text.color: #333 + // Settings icon button + lobby_settings_button := RobrixIconButton { + width: 48 + height: 48 + padding: Inset{top: 10, bottom: 10, left: 10, right: 10} + margin: 0, + draw_icon.svg: (ICON_SETTINGS) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #000000 + border_radius: 0.0 + border_size: 1.5 + border_color: #ccc + } + } } } - lobby_status := Label { - text: "" - draw_text.text_style.font_size: 12 - draw_text.color: #888 - } + // Hidden buttons for compatibility + video_call_button := Button { visible: false text: "Video Call" } + voice_call_button := Button { visible: false text: "Voice Call" } } // Debug panel @@ -507,6 +570,8 @@ pub struct VoipScreen { #[rust] in_lobby: bool, #[rust] lobby_mic_enabled: bool, #[rust] lobby_camera_enabled: bool, + /// Whether the user navigated here from a call notification (to join an existing call) + #[rust] from_notification: bool, #[rust] show_participants: bool, #[rust] show_debug: bool, @@ -574,9 +639,6 @@ impl Widget for VoipScreen { Event::VideoPlaybackResourcesReleased(_) => { self.handle_video_resources_released(cx); } - Event::Actions(actions) => { - self.handle_actions(cx, actions); - } _ => { if self.speaking_check_timer.is_event(event).is_some() { self.check_speaking_state(cx); @@ -592,7 +654,13 @@ impl Widget for VoipScreen { self.update_ui(cx); } + // Let the view process events first (including button clicks) self.view.handle_event(cx, event, scope); + + // Then handle actions AFTER the view has processed them + if let Event::Actions(actions) = event { + self.handle_actions(cx, actions); + } } fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { @@ -602,15 +670,14 @@ impl Widget for VoipScreen { impl VoipScreen { /// Initialize the VoIP screen - pub fn initialize(&mut self, cx: &mut Cx) { + pub fn initialize(&mut self, cx: &mut Cx, room_id: OwnedRoomId) { + log!("VoipScreen: Initializing for room {}", room_id); self.in_lobby = true; self.lobby_mic_enabled = true; self.lobby_camera_enabled = true; self.call = Call::default(); self.speaking_detector = SpeakingDetector::new(); - log!("VoipScreen initialized"); - // Initialize LiveKit client let mut client = LiveKitClient::new(); let rx = client.start(); @@ -627,22 +694,20 @@ impl VoipScreen { self.camera_permission = VoipGlobalState::get_camera_permission(cx); self.camera_choice = VoipGlobalState::get_camera_choice(cx); - log!("VoipScreen: Read from global state - permission={:?}, choice={:?}", - self.camera_permission, self.camera_choice.as_ref().map(|c| &c.name)); - // Try to start camera if we already have permission and camera choice self.try_start_camera(cx); // Set default room - let room_id: OwnedRoomId = "!rTeTgZzSYKoeJEVosH:matrix.org".try_into().unwrap(); self.set_room(cx, room_id); self.update_ui(cx); } /// Set the room for this VoIP call (uses Matrix client from room screen) + /// When called from a call notification, this shows the "Join Call" button. pub fn set_room(&mut self, cx: &mut Cx, room_id: OwnedRoomId) { self.room_id = Some(room_id.clone()); + self.from_notification = true; // Show "Join Call" button // Get room name from client if let Some(client) = get_client() { @@ -777,12 +842,16 @@ impl VoipScreen { self.call.connection_state = ConnectionState::Disconnected; self.in_lobby = true; self.call_start_time = None; - self.try_start_camera(cx); + //self.try_start_camera(cx); + CameraManager::stop_lobby_camera(&self.view, cx); + self.pending_call_camera_start = true; + self.camera_active = false; + + self.update_ui(cx); } /// Update UI to reflect current state fn update_ui(&mut self, cx: &mut Cx) { - self.view.view(cx, ids!(lobby_view)).set_visible(cx, self.in_lobby); self.view.view(cx, ids!(call_view)).set_visible(cx, !self.in_lobby); @@ -820,26 +889,52 @@ impl VoipScreen { .set_text(cx, &format!("{:02}:{:02}", mins, secs)); } - // Update lobby buttons - let mic_text = if self.lobby_mic_enabled { "Mic On" } else { "Mic Off" }; - let cam_text = if self.lobby_camera_enabled { "Cam On" } else { "Cam Off" }; - self.view.button(cx, ids!(lobby_mic_button)).set_text(cx, mic_text); - self.view.button(cx, ids!(lobby_camera_button)).set_text(cx, cam_text); + // Update lobby icon button styles based on state + // When disabled, show different border color + if self.in_lobby { + let mut mic_btn = self.view.button(cx, ids!(lobby_mic_button)); + let mut cam_btn = self.view.button(cx, ids!(lobby_camera_button)); + + if self.lobby_mic_enabled { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { border_color: #ccc } + draw_icon +: { color: #333 } + }); + } else { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { border_color: #f00 } + draw_icon +: { color: #f00 } + }); + } + + if self.lobby_camera_enabled { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { border_color: #ccc } + draw_icon +: { color: #333 } + }); + } else { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { border_color: #f00 } + draw_icon +: { color: #f00 } + }); + } + } + + // Show "Join Call" button always in lobby (it's the main action button now) + self.view.button(cx, ids!(join_call_button)).set_visible(cx, self.in_lobby); + + // Force redraw to ensure all visibility changes take effect + self.view.redraw(cx); } /// Try to start camera fn try_start_camera(&mut self, cx: &mut Cx) { - log!("try_start_camera: permission={:?}, choice={:?}", - self.camera_permission, self.camera_choice.as_ref().map(|c| &c.name)); - if !matches!(self.camera_permission, Some(PermissionStatus::Granted)) { - log!("Waiting for camera permission..."); self.view.label(cx, ids!(lobby_status)).set_text(cx, "Waiting for camera permission..."); return; } let Some(choice) = self.camera_choice.clone() else { - log!("Waiting for camera device..."); self.view.label(cx, ids!(lobby_status)).set_text(cx, "Waiting for camera device..."); return; }; @@ -932,21 +1027,48 @@ impl VoipScreen { fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { // Lobby buttons - if self.view.button(cx, ids!(video_call_button)).clicked(actions) { - log!("Video call button clicked"); - self.start_call(cx, CallType::Video); + // Close button - exit VoIP screen + if self.view.button(cx, ids!(close_button)).clicked(actions) { + log!("Close button clicked, exiting VoIP screen"); + if let Some(room_id) = self.room_id.clone() { + log!("Emitting VoipAction::Close for room {}", room_id); + + self.hangup(cx); + cx.action(VoipAction::Close(room_id)); + } } - if self.view.button(cx, ids!(voice_call_button)).clicked(actions) { - self.start_call(cx, CallType::Voice); + // Join Call button - main action to start the call + if self.view.button(cx, ids!(join_call_button)).clicked(actions) { + log!("Join call button clicked for room: {:?}", self.room_id); + self.from_notification = false; + self.start_call(cx, CallType::Video); } + // Microphone toggle (icon button) if self.view.button(cx, ids!(lobby_mic_button)).clicked(actions) { self.lobby_mic_enabled = !self.lobby_mic_enabled; + log!("Lobby mic toggled: {}", self.lobby_mic_enabled); self.update_ui(cx); } + // Camera toggle (icon button) if self.view.button(cx, ids!(lobby_camera_button)).clicked(actions) { self.lobby_camera_enabled = !self.lobby_camera_enabled; + log!("Lobby camera toggled: {}", self.lobby_camera_enabled); + self.update_ui(cx); + } + // Settings button (icon button) + if self.view.button(cx, ids!(lobby_settings_button)).clicked(actions) { + log!("Settings button clicked"); + // Toggle debug panel for now + self.show_debug = !self.show_debug; self.update_ui(cx); } + // Legacy buttons (hidden, for compatibility) + if self.view.button(cx, ids!(video_call_button)).clicked(actions) { + self.start_call(cx, CallType::Video); + } + if self.view.button(cx, ids!(voice_call_button)).clicked(actions) { + self.start_call(cx, CallType::Voice); + } // Call controls if self.view.button(cx, ids!(mic_button)).clicked(actions) { @@ -1006,9 +1128,9 @@ impl VoipScreen { impl VoipScreenRef { /// Initialize the VoIP screen - pub fn initialize(&self, cx: &mut Cx) { + pub fn initialize(&self, cx: &mut Cx, room_id: OwnedRoomId) { if let Some(mut inner) = self.borrow_mut() { - inner.initialize(cx); + inner.initialize(cx, room_id); } } @@ -1046,4 +1168,10 @@ impl VoipScreenRef { inner.clear_participants(cx); } } + + pub fn hangup(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.hangup(cx); + } + } } From 9b8cd228733c7bada037714a4c27b2475ed9e30a Mon Sep 17 00:00:00 2001 From: alanpoon Date: Tue, 7 Apr 2026 14:20:25 +0800 Subject: [PATCH 03/21] voip --- .cargo/config.toml | 4 + Cargo.lock | 2 +- Cargo.toml | 6 +- makepad | 1 + src/app.rs | 111 +++++- src/home/rooms_list.rs | 39 ++ src/sliding_sync.rs | 325 ++++++++++++++- src/voip/call_state.rs | 5 + src/voip/camera.rs | 1 + src/voip/livekit_client.rs | 56 ++- src/voip/mod.rs | 57 +++ src/voip/participants_list.rs | 9 +- src/voip/remote_video_session.rs | 103 +++++ src/voip/voip_screen.rs | 653 ++++++++++++++++++++----------- 14 files changed, 1122 insertions(+), 250 deletions(-) create mode 160000 makepad create mode 100644 src/voip/remote_video_session.rs diff --git a/.cargo/config.toml b/.cargo/config.toml index 1e4c80007..68ce1fdb5 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,3 +3,7 @@ [target.'cfg(all())'] rustflags = ["--cfg", "ruma_identifiers_storage=\"Arc\""] +## LiveKit/libwebrtc requires -ObjC linker flag on macOS +[target.'cfg(target_os = "macos")'] +rustflags = ["-C", "link-args=-Wl,-ObjC"] + diff --git a/Cargo.lock b/Cargo.lock index e781ca1e2..7793e9e52 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2553,7 +2553,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.2", + "windows-core 0.62.1", ] [[package]] diff --git a/Cargo.toml b/Cargo.toml index fba955a8f..ca9bcd595 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,8 +14,8 @@ version = "0.0.1-pre-alpha-4" metadata.makepad-auto-version = "zqpv-Yj-K7WNVK2I8h5Okhho46Q=" [dependencies] -# makepad-widgets = { git = "https://github.com/makepad/makepad", branch = "dev", features = ["serde"] } -# makepad-code-editor = { git = "https://github.com/makepad/makepad", branch = "dev" } +#makepad-widgets = { path = "/Users/alanpoon/Documents/rust/makepad/widgets", features = ["serde"] } +#makepad-code-editor = { path = "/Users/alanpoon/Documents/rust/makepad/code_editor" } makepad-widgets = { git = "https://github.com/kevinaboos/makepad", branch = "stack_nav_improvements", features = ["serde"] } makepad-code-editor = { git = "https://github.com/kevinaboos/makepad", branch = "stack_nav_improvements" } @@ -66,6 +66,7 @@ matrix-sdk-ui = { git = "https://github.com/matrix-org/matrix-rust-sdk", branch ruma = { version = "0.14.1", features = [ "compat-optional", "compat-unset-avatar", + "unstable-msc3401", ] } rand = "0.8.5" rangemap = "1.5.0" @@ -74,6 +75,7 @@ serde = "1.0" serde_json = "1.0" thiserror = "2.0.16" tokio = { version = "1.43.1", features = ["macros", "rt-multi-thread"] } +# livekit = "0.7" # Requires native libwebrtc build - enable when ready tracing-subscriber = "0.3.17" unicode-segmentation = "1.11.0" url = "2.5.0" diff --git a/makepad b/makepad new file mode 160000 index 000000000..2cff94d01 --- /dev/null +++ b/makepad @@ -0,0 +1 @@ +Subproject commit 2cff94d01b842643699c0ee4a23c4d9462c43f8d diff --git a/src/app.rs b/src/app.rs index 8760580d1..f9c01d587 100644 --- a/src/app.rs +++ b/src/app.rs @@ -911,13 +911,118 @@ impl App { cx.action(NavigationBarAction::GoToHome); self.ui.redraw(cx); } + "joincall" => { + log!("Stdin command: triggering JoinCall action"); + cx.action(VoipAction::JoinCall); + self.ui.redraw(cx); + } + "testvoip" => { + // Get the first available joined room and show VoIP for it + let rooms_list = cx.get_global::(); + if let Some(room_name_id) = rooms_list.get_first_joined_room() { + log!("Stdin command: testvoip - using first room: {}", + room_name_id.room_id()); + cx.widget_action( + WidgetUid(0), + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + self.ui.redraw(cx); + } else { + log!("Stdin command: testvoip - no joined rooms available"); + } + } + "showp" | "showparticipants" => { + log!("Stdin command: toggling participants sidebar"); + cx.action(VoipAction::TestToggleParticipantsSidebar); + self.ui.redraw(cx); + } + "clearp" | "clearparticipants" => { + log!("Stdin command: clearing all participants"); + cx.action(VoipAction::TestClearParticipants); + self.ui.redraw(cx); + } + "rooms" => { + log!("=== Joined Rooms ==="); + let rooms_list = cx.get_global::(); + for (name, id) in rooms_list.list_rooms() { + log!(" {} ({})", name, id); + } + log!("===================="); + } "help" => { log!("=== Stdin Commands ==="); - log!(" voip - Switch to VoIP call screen"); - log!(" home - Switch to Home screen"); - log!(" help - Show this help"); + log!(" voip - Switch to VoIP call screen (requires selected room)"); + log!(" testvoip - Open VoIP with first available room (for testing)"); + log!(" join - Open VoIP and join call for room by name"); + log!(" joincall - Trigger Join Call button"); + log!(" rooms - List all joined rooms"); + log!(" home - Switch to Home screen"); + log!(" showp - Toggle participants sidebar"); + log!(" addp [video] - Add participant (with optional video)"); + log!(" togglev - Toggle participant video (id=1,2,3...)"); + log!(" removep - Remove participant by id"); + log!(" clearp - Clear all participants"); + log!(" help - Show this help"); log!("======================"); } + _ if cmd.starts_with("join ") => { + if let Some(room_name) = cmd.strip_prefix("join ").map(|s| s.trim()) { + log!("Stdin command: joining call in room '{}'", room_name); + let rooms_list = cx.get_global::(); + if let Some(room_name_id) = rooms_list.find_room_by_name(room_name) { + log!("Found room: {} ({})", room_name_id.display_name(), room_name_id.room_id()); + // Open VoIP screen + cx.widget_action( + WidgetUid(0), + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + self.ui.redraw(cx); + // Schedule JoinCall action after a short delay to let VoIP screen initialize + // We'll trigger it immediately - the VoIP screen should handle it + cx.action(VoipAction::JoinCall); + } else { + log!("Room '{}' not found. Use 'rooms' to list available rooms.", room_name); + } + } else { + log!("Usage: join "); + } + } + _ if cmd.starts_with("addp ") => { + let args: Vec<&str> = cmd.strip_prefix("addp ").unwrap().split_whitespace().collect(); + if let Some(name) = args.first() { + let is_video_on = args.get(1).map_or(false, |&v| v == "video" || v == "v" || v == "1"); + log!("Stdin command: adding participant '{}' with video={}", name, is_video_on); + cx.action(VoipAction::TestAddParticipant { + name: name.to_string(), + is_video_on, + }); + self.ui.redraw(cx); + } else { + log!("Usage: addp [video]"); + } + } + _ if cmd.starts_with("togglev ") => { + if let Some(id) = cmd.strip_prefix("togglev ").map(|s| s.trim()) { + log!("Stdin command: toggling video for participant id={}", id); + cx.action(VoipAction::TestToggleParticipantVideo { + id: id.to_string(), + }); + self.ui.redraw(cx); + } else { + log!("Usage: togglev "); + } + } + _ if cmd.starts_with("removep ") => { + if let Some(id) = cmd.strip_prefix("removep ").map(|s| s.trim()) { + log!("Stdin command: removing participant id={}", id); + cx.action(VoipAction::TestRemoveParticipant { + id: id.to_string(), + }); + self.ui.redraw(cx); + } else { + log!("Usage: removep "); + } + } _ if !cmd.is_empty() => { log!("Unknown command: '{}'. Type 'help' for available commands.", cmd); } diff --git a/src/home/rooms_list.rs b/src/home/rooms_list.rs index ff6327a03..43010eb53 100644 --- a/src/home/rooms_list.rs +++ b/src/home/rooms_list.rs @@ -1597,6 +1597,45 @@ impl RoomsListRef { .get(space_id) .map(|smv| smv.parent_chain.clone()) } + + /// Returns the RoomNameId of the first joined room, if any. + /// This is primarily for testing purposes. + pub fn get_first_joined_room(&self) -> Option { + self.borrow()? + .all_joined_rooms + .values() + .next() + .map(|jr| jr.room_name_id.clone()) + } + + /// Find a room by name (case-insensitive partial match). + /// Returns the RoomNameId if found. + pub fn find_room_by_name(&self, name: &str) -> Option { + let inner = self.borrow()?; + let name_lower = name.to_lowercase(); + inner.all_joined_rooms + .values() + .find(|jr| { + let display_name = jr.room_name_id.display_name().to_string().to_lowercase(); + display_name.contains(&name_lower) + }) + .map(|jr| jr.room_name_id.clone()) + } + + /// List all joined rooms (for debugging). + pub fn list_rooms(&self) -> Vec<(String, String)> { + self.borrow() + .map(|inner| { + inner.all_joined_rooms + .values() + .map(|jr| ( + jr.room_name_id.display_name().to_string(), + jr.room_name_id.room_id().to_string(), + )) + .collect() + }) + .unwrap_or_default() + } } pub struct RoomsListScopeProps { diff --git a/src/sliding_sync.rs b/src/sliding_sync.rs index 8b8039b0c..13c2f93e1 100644 --- a/src/sliding_sync.rs +++ b/src/sliding_sync.rs @@ -27,7 +27,19 @@ use matrix_sdk_ui::{ RoomListService, Timeline, encryption_sync_service, room_list_service::{RoomListItem, RoomListLoadingState, SyncIndicator, filters}, sync_service::{self, SyncService}, timeline::{LatestEventValue, RoomExt, TimelineEventItemId, TimelineFocus, TimelineItem, TimelineReadReceiptTracking, TimelineDetails} }; use robius_open::Uri; -use ruma::{OwnedRoomOrAliasId, RoomId, events::tag::Tags}; +use ruma::{ + OwnedRoomOrAliasId, RoomId, + api::client::account::request_openid_token, + events::{ + tag::Tags, + call::member::{ + CallMemberEventContent, CallMemberStateKey, + Application, CallApplicationContent, CallScope, + Focus, LivekitFocus, + ActiveFocus, ActiveLivekitFocus, FocusSelection, + }, + }, +}; use tokio::{ runtime::Handle, sync::{broadcast, mpsc::{Sender, UnboundedReceiver, UnboundedSender}, watch, Notify}, task::JoinHandle, time::error::Elapsed, @@ -919,6 +931,31 @@ pub enum MatrixRequest { destination: Arc>, update_sender: Option>, }, + /// Send a call member state event to join a call (MSC3401). + SendCallMemberState { + room_id: OwnedRoomId, + call_type: crate::voip::call_state::CallType, + }, + /// Send an empty call member state event to leave a call (MSC3401). + SendEndCallState { + room_id: OwnedRoomId, + }, + /// Fetch call member state events to get list of participants in the call (MSC3401). + GetCallMembers { + room_id: OwnedRoomId, + }, + /// Fetch OpenID token from Matrix homeserver (for LiveKit authentication). + FetchOpenIdToken { + room_id: OwnedRoomId, + }, + /// Fetch LiveKit JWT using OpenID token. + FetchLiveKitJwt { + room_id: OwnedRoomId, + access_token: String, + token_type: String, + matrix_server_name: String, + expires_in: u64, + }, } /// Submits a request to the worker thread to be executed asynchronously. @@ -2288,6 +2325,292 @@ async fn matrix_worker_task( SignalToUI::set_ui_signal(); }); } + + MatrixRequest::SendCallMemberState { room_id, call_type: _ } => { + let Some(client) = get_client() else { continue }; + let _send_call_member_task = Handle::current().spawn(async move { + log!("Sending call member state for room {room_id}..."); + + let Some(room) = client.get_room(&room_id) else { + error!("Room {room_id} not found for call member state"); + return; + }; + + let Some(session) = client.session_meta() else { + error!("No session metadata available for call member state"); + return; + }; + let user_id = session.user_id.clone(); + let device_id = session.device_id.clone(); + + // LiveKit service URL (Element Call uses this) + let livekit_service_url = "https://livekit-jwt.call.matrix.org"; + + // Build the call application content + let call_app = CallApplicationContent::new( + String::new(), // call_id: empty for room-scoped calls + CallScope::Room, + ); + + // Build the active focus (LiveKit with oldest_membership selection) + let mut active_livekit = ActiveLivekitFocus::new(); + active_livekit.focus_selection = FocusSelection::OldestMembership; + let focus_active = ActiveFocus::Livekit(active_livekit); + + // Build the preferred foci list + let foci_preferred = vec![ + Focus::Livekit(LivekitFocus::new( + room_id.to_string(), // alias + livekit_service_url.to_string(), // service_url + )) + ]; + + // Build the call member event content using typed API (MSC3401) + let content = CallMemberEventContent::new( + Application::Call(call_app), + device_id.clone(), + focus_active, + foci_preferred, + None, // created_ts: will be set by server + Some(Duration::from_secs(7200)), // expires: 2 hours + ); + + // Build the state key using typed API + let state_key = CallMemberStateKey::new( + user_id.clone(), + Some(format!("{}_m.call", device_id)), + true, // use underscore prefix + ); + + // Send the typed state event + match room.send_state_event_for_key(&state_key, content).await { + Ok(response) => { + log!("Successfully sent call member state, event_id: {}", response.event_id); + // Notify UI that call member state was sent + Cx::post_action(crate::voip::VoipAction::CallMemberStateSent { + room_id: room_id.clone(), + success: true, + }); + } + Err(e) => { + error!("Failed to send call member state: {e:?}"); + Cx::post_action(crate::voip::VoipAction::CallMemberStateSent { + room_id: room_id.clone(), + success: false, + }); + } + } + SignalToUI::set_ui_signal(); + }); + } + + MatrixRequest::SendEndCallState { room_id } => { + let Some(client) = get_client() else { continue }; + let _send_end_call_task = Handle::current().spawn(async move { + log!("Sending end call state for room {room_id}..."); + + let Some(room) = client.get_room(&room_id) else { + error!("Room {room_id} not found for end call state"); + return; + }; + + let Some(session) = client.session_meta() else { + error!("No session metadata available for end call state"); + return; + }; + let user_id = session.user_id.clone(); + let device_id = session.device_id.clone(); + + // Build empty content to leave the call + let content = CallMemberEventContent::new_empty(None); + + // Build the state key using typed API + let state_key = CallMemberStateKey::new( + user_id.clone(), + Some(format!("{}_m.call", device_id)), + true, // use underscore prefix + ); + + match room.send_state_event_for_key(&state_key, content).await { + Ok(response) => { + log!("Successfully sent end call state, event_id: {}", response.event_id); + } + Err(e) => { + error!("Failed to send end call state: {e:?}"); + } + } + SignalToUI::set_ui_signal(); + }); + } + + MatrixRequest::GetCallMembers { room_id } => { + let Some(client) = get_client() else { continue }; + let _get_call_members_task = Handle::current().spawn(async move { + log!("Fetching call members for room {room_id}..."); + + let Some(room) = client.get_room(&room_id) else { + error!("Room {room_id} not found for get call members"); + return; + }; + + // Use RoomInfo::active_room_call_participants() for simpler API + let participants = room.active_room_call_participants(); + log!("Found {} active room call participants", participants.len()); + + let mut members = Vec::new(); + for user_id in participants { + // Get display name for the user + let display_name = room.get_member_no_sync(&user_id) + .await + .ok() + .flatten() + .and_then(|m| m.display_name().map(|n| n.to_string())); + + log!("Active call participant: {}", user_id); + + members.push(crate::voip::CallMember { + user_id: user_id.to_string(), + device_id: String::new(), // Not available from this API + display_name, + }); + } + + log!("Total active call members: {}", members.len()); + Cx::post_action(crate::voip::VoipAction::CallMembersUpdated { + room_id: room_id.clone(), + members, + }); + SignalToUI::set_ui_signal(); + }); + } + + MatrixRequest::FetchOpenIdToken { room_id } => { + let Some(client) = get_client() else { continue }; + let _fetch_openid_task = Handle::current().spawn(async move { + log!("Fetching OpenID token for room {room_id}..."); + + // Get the user ID for the OpenID request + let user_id = client.user_id().expect("Client should have user_id").to_owned(); + + // Use the Matrix SDK to get an OpenID token via direct API call + // This corresponds to POST /_matrix/client/v3/user/{userId}/openid/request_token + match client.send(request_openid_token::v3::Request::new(user_id)).await { + Ok(response) => { + log!("OpenID token fetched successfully"); + log!(" matrix_server_name: {}", response.matrix_server_name); + log!(" expires_in: {:?}", response.expires_in); + + // Convert expires_in Duration to seconds + let expires_in_secs = response.expires_in.as_secs(); + + Cx::post_action(crate::voip::VoipAction::OpenIdTokenFetched { + room_id: room_id.clone(), + access_token: response.access_token, + token_type: "Bearer".to_string(), + matrix_server_name: response.matrix_server_name.to_string(), + expires_in: expires_in_secs, + }); + } + Err(e) => { + error!("Failed to fetch OpenID token: {e:?}"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: format!("Failed to fetch OpenID token: {e}"), + }); + } + } + SignalToUI::set_ui_signal(); + }); + } + + MatrixRequest::FetchLiveKitJwt { room_id, access_token, token_type, matrix_server_name, expires_in } => { + let _fetch_jwt_task = Handle::current().spawn(async move { + log!("Fetching LiveKit JWT for room {room_id}..."); + + // LiveKit JWT endpoint (external service, not Matrix API) + // POST https://livekit-jwt.call.matrix.org/sfu/get + let jwt_endpoint = "https://livekit-jwt.call.matrix.org/sfu/get"; + + // Get device_id from client + let device_id = get_client() + .and_then(|c| c.session_meta().map(|m| m.device_id.to_string())) + .unwrap_or_default(); + + // Build the request body matching webrtc example format + // JwtRequest { room, openid_token: OpenIdToken, device_id } + let request_body = serde_json::json!({ + "room": room_id.as_str(), + "openid_token": { + "access_token": access_token, + "token_type": token_type, + "matrix_server_name": matrix_server_name, + "expires_in": expires_in, + }, + "device_id": device_id, + }); + + log!("LiveKit JWT request body: {}", serde_json::to_string_pretty(&request_body).unwrap_or_default()); + + // Make HTTP request to LiveKit JWT endpoint + let http_client = reqwest::Client::new(); + match http_client + .post(jwt_endpoint) + .header("Content-Type", "application/json") + .json(&request_body) + .send() + .await + { + Ok(response) => { + if response.status().is_success() { + match response.json::().await { + Ok(json) => { + let url = json["url"].as_str().unwrap_or("").to_string(); + let jwt = json["jwt"].as_str().unwrap_or("").to_string(); + + if !url.is_empty() && !jwt.is_empty() { + log!("LiveKit JWT fetched successfully, url: {}", url); + Cx::post_action(crate::voip::VoipAction::LiveKitJwtFetched { + room_id: room_id.clone(), + url, + jwt, + }); + } else { + error!("LiveKit JWT response missing url or jwt"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: "Invalid JWT response".to_string(), + }); + } + } + Err(e) => { + error!("Failed to parse LiveKit JWT response: {e:?}"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: format!("Failed to parse JWT response: {e}"), + }); + } + } + } else { + let status = response.status(); + let error_text = response.text().await.unwrap_or_default(); + error!("LiveKit JWT request failed: {} - {}", status, error_text); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: format!("JWT request failed: {status}"), + }); + } + } + Err(e) => { + error!("Failed to fetch LiveKit JWT: {e:?}"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: format!("Network error: {e}"), + }); + } + } + SignalToUI::set_ui_signal(); + }); + } } } diff --git a/src/voip/call_state.rs b/src/voip/call_state.rs index af8748f82..3df9c2ab6 100644 --- a/src/voip/call_state.rs +++ b/src/voip/call_state.rs @@ -24,6 +24,11 @@ pub enum CallType { #[derive(Debug, Clone)] pub struct CallParticipant { pub user_id: String, + pub display_name: String, + pub is_muted: bool, + pub is_video_on: bool, + pub is_speaking: bool, + pub is_screen_sharing: bool, } /// Main call state structure diff --git a/src/voip/camera.rs b/src/voip/camera.rs index dce4f72b5..fecb3901f 100644 --- a/src/voip/camera.rs +++ b/src/voip/camera.rs @@ -171,6 +171,7 @@ impl CameraManager { } ui.view(cx, ids!(lobby_video_host)).set_visible(cx, false); ui.view(cx, ids!(lobby_camera_placeholder)).set_visible(cx, true); + ui.view(cx, ids!(join_call_button_view)).set_visible(cx, true); } /// Stop in-call camera diff --git a/src/voip/livekit_client.rs b/src/voip/livekit_client.rs index 8988e672f..f03acafeb 100644 --- a/src/voip/livekit_client.rs +++ b/src/voip/livekit_client.rs @@ -1,7 +1,11 @@ //! LiveKit client integration for WebRTC +//! +//! This is currently a stub implementation. When the livekit crate is enabled, +//! it will provide actual WebRTC connectivity. use std::sync::{Arc, Mutex}; use tokio::sync::mpsc; +use makepad_widgets::{SignalToUI, log}; use super::call_state::CallParticipant; @@ -31,11 +35,25 @@ pub enum LiveKitMessage { Error(String), ParticipantJoined(CallParticipant), ParticipantLeft(String), + /// I420 video frame received from remote participant + RemoteVideoFrame { + participant_id: String, + /// Y plane data + y: Vec, + /// U plane data + u: Vec, + /// V plane data + v: Vec, + width: u32, + height: u32, + /// Presentation timestamp in milliseconds + pts_ms: u64, + }, } /// Commands sent from UI to LiveKit client pub enum LiveKitCommand { - Connect { url: String }, + Connect { url: String, token: String }, Disconnect, SetMicrophoneMuted(bool), SetCameraMuted(bool), @@ -82,39 +100,49 @@ impl LiveKitClient { msg_tx: mpsc::UnboundedSender, is_connected: Arc>, ) { + // Stub implementation - simulates LiveKit behavior while let Some(cmd) = cmd_rx.recv().await { match cmd { - LiveKitCommand::Connect { url } => { - println!("Connecting to LiveKit: {}", url); + LiveKitCommand::Connect { url, token: _ } => { + log!("LiveKit (stub): Connecting to {}", url); + + // Simulate successful connection if let Ok(mut connected) = is_connected.lock() { *connected = true; } let _ = msg_tx.send(LiveKitMessage::Connected); + SignalToUI::set_ui_signal(); + + // Note: In a real implementation, we would: + // 1. Connect to LiveKit using Room::connect(&url, &token, RoomOptions::default()) + // 2. Listen for RoomEvent::ParticipantConnected, TrackSubscribed, etc. + // 3. Extract I420 frames from video tracks using: + // let i420 = frame.buffer.to_i420(); + // let (data_y, data_u, data_v) = i420.data(); + log!("LiveKit (stub): Connection simulated. Enable 'livekit' crate for real WebRTC."); } LiveKitCommand::Disconnect => { - println!("Disconnecting from LiveKit"); + log!("LiveKit (stub): Disconnecting"); if let Ok(mut connected) = is_connected.lock() { *connected = false; } let _ = msg_tx.send(LiveKitMessage::Disconnected); + SignalToUI::set_ui_signal(); } LiveKitCommand::SetMicrophoneMuted(muted) => { - println!("Set microphone muted: {}", muted); + log!("LiveKit (stub): Set microphone muted: {}", muted); } LiveKitCommand::SetCameraMuted(muted) => { - println!("Set camera muted: {}", muted); + log!("LiveKit (stub): Set camera muted: {}", muted); } LiveKitCommand::StartScreenShare => { - println!("Starting screen share"); + log!("LiveKit (stub): Starting screen share"); } LiveKitCommand::StopScreenShare => { - println!("Stopping screen share"); + log!("LiveKit (stub): Stopping screen share"); } LiveKitCommand::PublishVideoFrame(frame) => { - println!( - "Publishing video frame: {}x{} ({} bytes)", - frame.width, frame.height, frame.data.len() - ); + log!("LiveKit (stub): Publishing video frame: {}x{}", frame.width, frame.height); } } } @@ -126,8 +154,8 @@ impl LiveKitClient { } } - pub fn connect(&self, url: String, _token: String) { - self.send_command(LiveKitCommand::Connect { url }); + pub fn connect(&self, url: String, token: String) { + self.send_command(LiveKitCommand::Connect { url, token }); } pub fn disconnect(&self) { diff --git a/src/voip/mod.rs b/src/voip/mod.rs index b131b867d..4eb1762f4 100644 --- a/src/voip/mod.rs +++ b/src/voip/mod.rs @@ -15,6 +15,7 @@ use matrix_sdk::ruma::OwnedRoomId; pub mod call_state; pub mod camera; pub mod livekit_client; +pub mod remote_video_session; pub mod speaking; pub mod participants_list; pub mod voip_screen; @@ -23,11 +24,67 @@ pub use voip_screen::VoipScreenWidgetRefExt; pub use participants_list::{Participant, ParticipantsListWidgetRefExt}; pub use camera::CameraChoice; +/// Represents a call member from Matrix state events +#[derive(Clone, Debug)] +pub struct CallMember { + pub user_id: String, + pub device_id: String, + pub display_name: Option, +} + /// Actions emitted by VoIP screens #[derive(Clone, Debug, Default)] pub enum VoipAction { /// Close the VoIP screen and return to the room Close(OwnedRoomId), + /// Join the call (triggers the join_call_button click) + JoinCall, + /// Notification that call member state was sent (or failed) + CallMemberStateSent { + room_id: OwnedRoomId, + success: bool, + }, + /// Call members list updated from Matrix state events + CallMembersUpdated { + room_id: OwnedRoomId, + members: Vec, + }, + /// OpenID token fetched from Matrix + OpenIdTokenFetched { + room_id: OwnedRoomId, + access_token: String, + token_type: String, + matrix_server_name: String, + expires_in: u64, + }, + /// LiveKit JWT fetched + LiveKitJwtFetched { + room_id: OwnedRoomId, + url: String, + jwt: String, + }, + /// LiveKit connection failed + LiveKitConnectionFailed { + room_id: OwnedRoomId, + error: String, + }, + /// Test action: Add a participant + TestAddParticipant { + name: String, + is_video_on: bool, + }, + /// Test action: Toggle participant video + TestToggleParticipantVideo { + id: String, + }, + /// Test action: Remove a participant + TestRemoveParticipant { + id: String, + }, + /// Test action: Clear all participants + TestClearParticipants, + /// Test action: Toggle participants sidebar + TestToggleParticipantsSidebar, #[default] None, } diff --git a/src/voip/participants_list.rs b/src/voip/participants_list.rs index 634e28586..177daf8e1 100644 --- a/src/voip/participants_list.rs +++ b/src/voip/participants_list.rs @@ -9,6 +9,7 @@ pub struct Participant { pub avatar_letter: String, pub is_muted: bool, pub is_speaking: bool, + pub is_video_on: bool, } impl Default for Participant { @@ -19,6 +20,7 @@ impl Default for Participant { avatar_letter: String::from("?"), is_muted: false, is_speaking: false, + is_video_on: false, } } } @@ -41,7 +43,12 @@ impl Widget for ParticipantsList { widget.label(cx, ids!(avatar_letter)).set_text(cx, &participant.avatar_letter); widget.label(cx, ids!(name_label)).set_text(cx, &participant.name); widget.label(cx, ids!(mute_icon)).set_text(cx, if participant.is_muted { "M" } else { "" }); - widget.label(cx, ids!(status_label)).set_text(cx, if participant.is_speaking { "Speaking..." } else { "" }); + widget.label(cx, ids!(status_label)).set_text(cx, if participant.is_speaking { "Speaking" } else { "" }); + + // Toggle video/avatar visibility based on is_video_on + widget.view(cx, ids!(participant_video_host)).set_visible(cx, participant.is_video_on); + widget.view(cx, ids!(avatar_container)).set_visible(cx, !participant.is_video_on); + widget.draw_all(cx, scope); } } diff --git a/src/voip/remote_video_session.rs b/src/voip/remote_video_session.rs new file mode 100644 index 000000000..47b1a549a --- /dev/null +++ b/src/voip/remote_video_session.rs @@ -0,0 +1,103 @@ +//! Remote video session for displaying LiveKit participant video + +use std::collections::VecDeque; +use std::sync::{Arc, Mutex}; +use makepad_widgets::makepad_platform::{ + VideoFrameSession, VideoSessionState, + MseDecodedFrame, +}; +use makepad_widgets::makepad_platform::video_decode::yuv::{YuvPlaneData, YuvLayout, YuvColorMatrix}; + +/// A video session that receives I420 frames from LiveKit remote participants +pub struct RemoteVideoSession { + frames: Arc>>, + dimensions: Arc>>, + state: Arc>, +} + +impl RemoteVideoSession { + pub fn new() -> Self { + Self { + frames: Arc::new(Mutex::new(VecDeque::new())), + dimensions: Arc::new(Mutex::new(None)), + state: Arc::new(Mutex::new(VideoSessionState::Connecting)), + } + } + + /// Get a handle to push frames into this session + pub fn get_pusher(&self) -> RemoteVideoFramePusher { + RemoteVideoFramePusher { + frames: self.frames.clone(), + dimensions: self.dimensions.clone(), + state: self.state.clone(), + } + } +} + +impl VideoFrameSession for RemoteVideoSession { + fn take_frames(&mut self) -> Vec { + let mut frames = self.frames.lock().unwrap(); + frames.drain(..).collect() + } + + fn dimensions(&self) -> Option<(u32, u32)> { + *self.dimensions.lock().unwrap() + } + + fn state(&self) -> VideoSessionState { + self.state.lock().unwrap().clone() + } +} + +/// Handle for pushing frames into a RemoteVideoSession from another thread +#[derive(Clone)] +pub struct RemoteVideoFramePusher { + frames: Arc>>, + dimensions: Arc>>, + state: Arc>, +} + +impl RemoteVideoFramePusher { + /// Push an I420 frame into the session + pub fn push_i420_frame(&self, y: Vec, u: Vec, v: Vec, width: u32, height: u32, pts_ms: u64) { + // Update dimensions + *self.dimensions.lock().unwrap() = Some((width, height)); + + // Set state to active + *self.state.lock().unwrap() = VideoSessionState::Active; + + // Create the decoded frame + let frame = MseDecodedFrame { + track_id: 0, + pts_ms, + yuv: YuvPlaneData { + y, + u, + v, + width, + height, + layout: YuvLayout::I420, + matrix: YuvColorMatrix::BT709, + }, + }; + + // Push to queue (limit queue size to avoid memory buildup) + let mut frames = self.frames.lock().unwrap(); + if frames.len() >= 3 { + frames.pop_front(); + } + frames.push_back(frame); + } + + /// Mark the session as ended + #[allow(dead_code)] + pub fn set_ended(&self) { + *self.state.lock().unwrap() = VideoSessionState::Ended; + } + + /// Mark the session as errored + #[allow(dead_code)] + pub fn set_error(&self, error: String) { + *self.state.lock().unwrap() = VideoSessionState::Error(error); + } +} diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs index d109b86fd..e60df0a09 100644 --- a/src/voip/voip_screen.rs +++ b/src/voip/voip_screen.rs @@ -9,8 +9,8 @@ use matrix_sdk::Client; use ruma::OwnedRoomId; use tokio::sync::mpsc; -use crate::sliding_sync::get_client; -use super::{VoipGlobalState, VoipAction}; +use crate::sliding_sync::{get_client, submit_async_request, MatrixRequest}; +use super::{VoipGlobalState, VoipAction, CallMember}; use super::call_state::{Call, CallType, ConnectionState}; use super::camera::{CameraChoice, CameraManager}; @@ -36,52 +36,81 @@ script_mod! { ParticipantItem := RoundedView { width: Fill - height: Fit - padding: 8 - margin: Inset{bottom: 4} + height: 120 + margin: Inset{bottom: 8} draw_bg.color: #3a3a5a - draw_bg.radius: 6.0 - flow: Right - spacing: 8 - align: Align{y: 0.5} - - avatar := RoundedView { - width: 32 - height: 32 - draw_bg.color: #a0d0a0 - draw_bg.radius: 16.0 - align: Center + draw_bg.radius: 8.0 + flow: Overlay - avatar_letter := Label { - text: "?" - draw_text.text_style.font_size: 14 - draw_text.color: #2a6a2a + // Video view (shown when video is on) + participant_video_host := View { + width: Fill + height: Fill + visible: false + + participant_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false } } - info_container := View { + // Avatar view (shown when video is off) + avatar_container := View { width: Fill - height: Fit - flow: Down - spacing: 2 + height: Fill + align: Center - name_label := Label { - text: "Participant" - draw_text.text_style.font_size: 12 - draw_text.color: #fff - } + avatar := RoundedView { + width: 48 + height: 48 + draw_bg.color: #a0d0a0 + draw_bg.radius: 24.0 + align: Center - status_label := Label { - text: "" - draw_text.text_style.font_size: 10 - draw_text.color: #888 + avatar_letter := Label { + text: "?" + draw_text.text_style.font_size: 20 + draw_text.color: #2a6a2a + } } } - mute_icon := Label { - text: "" - draw_text.text_style.font_size: 12 - draw_text.color: #888 + // Info overlay at bottom + View { + width: Fill + height: Fill + align: Align{x: 0.0 y: 1.0} + padding: 8 + + RoundedView { + width: Fit + height: Fit + padding: Inset{left: 8 right: 8 top: 4 bottom: 4} + draw_bg.color: #000000aa + draw_bg.radius: 4.0 + flow: Right + spacing: 6 + + mute_icon := Label { + text: "" + draw_text.text_style.font_size: 10 + draw_text.color: #fff + } + + name_label := Label { + text: "Participant" + draw_text.text_style.font_size: 10 + draw_text.color: #fff + } + + status_label := Label { + text: "" + draw_text.text_style.font_size: 10 + draw_text.color: #4CAF50 + } + } } } } @@ -144,159 +173,111 @@ script_mod! { } } - // Participants grid - participants_grid := View { + // Main content area (participants + video) + call_content := View { width: Fill height: Fill flow: Right - spacing: 16 - padding: 16 - align: Center + spacing: 0 - // Local user card wrapper - local_card_wrapper := View { - width: Fit - height: Fit + // Participants list on the left + participants_panel := View { + width: 220 + height: Fill + padding: 12 + show_bg: true + draw_bg.color: #1e1e3a + flow: Down + spacing: 8 + + Label { + text: "Participants" + draw_text.text_style.font_size: 14 + draw_text.color: #fff + margin: Inset{bottom: 8} + } + + participants_list := mod.widgets.VoipParticipantsList {} + } + + // Local video container (takes remaining space) + local_video_container := View { + width: Fill + height: Fill flow: Overlay // Speaking indicator border local_speaking_border := RoundedView { - width: 286 - height: 216 + width: Fill + height: Fill draw_bg.color: #4CAF50 - draw_bg.radius: 15.0 + draw_bg.radius: 0.0 visible: false } - // Main card - local_participant_card := RoundedView { - width: 280 - height: 210 - margin: 3 - draw_bg.color: #e8e8e8 - draw_bg.radius: 12.0 - flow: Overlay - - // Video container - local_video_container := View { - width: Fill - height: Fill - flow: Overlay - - // Avatar placeholder - local_avatar_view := View { - width: Fill - height: Fill - align: Center - - RoundedView { - width: 80 - height: 80 - draw_bg.color: #a0d0a0 - draw_bg.radius: 40.0 - align: Center - - local_avatar_letter := Label { - text: "Y" - draw_text.text_style.font_size: 32 - draw_text.color: #2a6a2a - } - } - } - - // Camera video - local_video_host := View { - width: Fill - height: Fill - visible: false - - local_camera_video := Video { - width: Fill - height: Fill - autoplay: false - show_controls: false - } - } - } + // Avatar placeholder (shown when camera is off) + local_avatar_view := View { + width: Fill + height: Fill + align: Center + show_bg: true + draw_bg.color: #2a2a4a - // Name badge - View { - width: Fill - height: Fit - align: Align{x: 0.0 y: 1.0} - padding: 8 - - RoundedView { - width: Fit - height: Fit - padding: Inset{left: 8 right: 8 top: 4 bottom: 4} - draw_bg.color: #fff - draw_bg.radius: 4.0 - flow: Right - spacing: 4 - - local_mute_icon := Label { - text: "" - draw_text.text_style.font_size: 12 - draw_text.color: #666 - } + RoundedView { + width: 120 + height: 120 + draw_bg.color: #a0d0a0 + draw_bg.radius: 60.0 + align: Center - local_name_label := Label { - text: "You" - draw_text.text_style.font_size: 12 - draw_text.color: #333 - } + local_avatar_letter := Label { + text: "Y" + draw_text.text_style.font_size: 48 + draw_text.color: #2a6a2a } } } - } - // Remote participant card - remote_participant_card := RoundedView { - width: 280 - height: 210 - draw_bg.color: #e8e8e8 - draw_bg.radius: 12.0 - flow: Overlay - visible: false - - View { + // Camera video (shown when camera is on) + local_video_host := View { width: Fill height: Fill - align: Center - - RoundedView { - width: 80 - height: 80 - draw_bg.color: #d0a0d0 - draw_bg.radius: 40.0 - align: Center + visible: false - remote_avatar_letter := Label { - text: "R" - draw_text.text_style.font_size: 32 - draw_text.color: #6a2a6a - } + local_camera_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false } } + // Name badge overlay at bottom left View { width: Fill - height: Fit + height: Fill align: Align{x: 0.0 y: 1.0} - padding: 8 + padding: 16 RoundedView { width: Fit height: Fit - padding: Inset{left: 8 right: 8 top: 4 bottom: 4} - draw_bg.color: #fff - draw_bg.radius: 4.0 - - remote_name_label := Label { - text: "Remote" - draw_text.text_style.font_size: 12 - draw_text.color: #333 + padding: Inset{left: 12 right: 12 top: 6 bottom: 6} + draw_bg.color: #000000aa + draw_bg.radius: 6.0 + flow: Right + spacing: 6 + + local_mute_icon := Label { + text: "" + draw_text.text_style.font_size: 14 + draw_text.color: #fff + } + + local_name_label := Label { + text: "You" + draw_text.text_style.font_size: 14 + draw_text.color: #fff } } } @@ -322,34 +303,11 @@ script_mod! { mic_button := Button { text: "Mic" width: 60 } camera_button := Button { text: "Cam" width: 60 } screenshare_button := Button { text: "Share" width: 60 } - participants_button := Button { text: "Users" width: 60 } hangup_button := Button { text: "End" width: 60 } } } } - // Participants sidebar - participants_sidebar := View { - width: 200 - height: Fill - margin: Inset{top: 60 bottom: 80} - padding: 12 - show_bg: true - draw_bg.color: #2a2a4a - flow: Down - spacing: 8 - visible: false - align: Align{x: 1.0 y: 0.0} - - Label { - text: "Participants" - draw_text.text_style.font_size: 14 - draw_text.color: #fff - } - - participants_list := mod.widgets.VoipParticipantsList {} - } - // Lobby view lobby_view := View { width: Fill @@ -389,17 +347,18 @@ script_mod! { height: Fill flow: Overlay margin: 0 + show_bg: true + draw_bg.color: #2a2a4a // Camera preview background lobby_camera_container := View { width: Fill height: Fill + flow: Overlay lobby_camera_placeholder := View { width: Fill height: Fill - show_bg: true - draw_bg.color: #2a2a4a align: Center // Placeholder logo/icon @@ -432,27 +391,6 @@ script_mod! { } } - // Join Call button overlay - centered vertically and horizontally - View { - width: Fill - height: Fill - align: Align{x: 0.5, y: 0.7} - - join_call_button := Button { - text: "Join call" - width: 160 - height: 48 - draw_bg +: { - color: #4CAF50 - border_radius: 24.0 - } - draw_text +: { - color: #fff - text_style.font_size: 16 - } - } - } - // Status label at bottom View { width: Fill @@ -466,7 +404,25 @@ script_mod! { } } } + join_call_button_view := View { + width: Fill + height: Fit + align: Align{x: 0.5, y: 0.7} + join_call_button := Button { + text: "Join call" + width: 100 + height: 48 + draw_bg +: { + color: #4CAF50 + border_radius: 20.0 + } + draw_text +: { + color: #fff + text_style.font_size: 16 + } + } + } // Bottom control bar with icons lobby_controls := View { width: Fill @@ -602,6 +558,9 @@ pub struct VoipScreen { // Participant counter #[rust] participant_counter: usize, + + // Timer for refreshing call members from Matrix + #[rust] call_members_refresh_timer: Timer, } impl Widget for VoipScreen { @@ -646,6 +605,14 @@ impl Widget for VoipScreen { if self.video_publish_timer.is_event(event).is_some() { // Video publishing handled here if needed } + if self.call_members_refresh_timer.is_event(event).is_some() { + // Refresh call members from Matrix (only when in a call) + if !self.in_lobby { + if let Some(room_id) = self.room_id.clone() { + submit_async_request(MatrixRequest::GetCallMembers { room_id }); + } + } + } } } @@ -660,6 +627,113 @@ impl Widget for VoipScreen { // Then handle actions AFTER the view has processed them if let Event::Actions(actions) = event { self.handle_actions(cx, actions); + + // Handle VoipActions + for action in actions { + if let Some(voip_action) = action.downcast_ref::() { + match voip_action { + VoipAction::JoinCall => { + if self.in_lobby { + log!("VoipScreen: Received VoipAction::JoinCall, triggering join call"); + self.start_call(cx, super::call_state::CallType::Video); + } else { + log!("VoipScreen: VoipAction::JoinCall ignored - not in lobby"); + } + } + VoipAction::CallMemberStateSent { room_id, success } => { + if self.room_id.as_ref() == Some(room_id) { + if *success { + log!("VoipScreen: Call member state sent successfully"); + self.call.connection_state = ConnectionState::Connecting; + self.in_lobby = false; + self.call_start_time = Some(Cx::time_now()); + + // Stop lobby camera and prepare for call camera + CameraManager::stop_lobby_camera(&self.view, cx); + self.pending_call_camera_start = true; + self.camera_active = false; + + // Fetch call members immediately after joining + submit_async_request(MatrixRequest::GetCallMembers { room_id: room_id.clone() }); + + // Start LiveKit connection flow: fetch OpenID token + log!("VoipScreen: Fetching OpenID token for LiveKit auth"); + submit_async_request(MatrixRequest::FetchOpenIdToken { room_id: room_id.clone() }); + } else { + log!("VoipScreen: Failed to send call member state"); + self.call.connection_state = ConnectionState::Disconnected; + } + self.update_ui(cx); + } + } + VoipAction::OpenIdTokenFetched { room_id, access_token, token_type, matrix_server_name, expires_in } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: OpenID token fetched, now fetching LiveKit JWT"); + log!(" server_name: {}", matrix_server_name); + log!(" expires_in: {} seconds", expires_in); + + // Next step: fetch LiveKit JWT from SFU + // POST https://livekit-jwt.call.matrix.org/sfu/get + submit_async_request(MatrixRequest::FetchLiveKitJwt { + room_id: room_id.clone(), + access_token: access_token.clone(), + token_type: token_type.clone(), + matrix_server_name: matrix_server_name.clone(), + expires_in: *expires_in, + }); + } + } + VoipAction::LiveKitJwtFetched { room_id, url, jwt } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: LiveKit JWT fetched, connecting to LiveKit"); + log!(" url: {}", url); + self.connect_livekit(cx, url, jwt); + } + } + VoipAction::LiveKitConnectionFailed { room_id, error } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: LiveKit connection failed: {}", error); + self.call.connection_state = ConnectionState::Disconnected; + self.update_ui(cx); + } + } + VoipAction::CallMembersUpdated { room_id, members } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: CallMembersUpdated - {} members", members.len()); + self.update_participants_from_call_members(cx, members); + self.update_ui(cx); + self.redraw(cx); + } + } + VoipAction::TestAddParticipant { name, is_video_on } => { + log!("VoipScreen: TestAddParticipant - name={}, video={}", name, is_video_on); + self.add_participant(cx, name, *is_video_on); + self.update_ui(cx); + } + VoipAction::TestToggleParticipantVideo { id } => { + log!("VoipScreen: TestToggleParticipantVideo - id={}", id); + self.toggle_participant_video(cx, id); + self.update_ui(cx); + } + VoipAction::TestRemoveParticipant { id } => { + log!("VoipScreen: TestRemoveParticipant - id={}", id); + self.remove_participant(cx, id); + self.update_ui(cx); + } + VoipAction::TestClearParticipants => { + log!("VoipScreen: TestClearParticipants"); + self.clear_participants(cx); + self.update_ui(cx); + } + VoipAction::TestToggleParticipantsSidebar => { + log!("VoipScreen: TestToggleParticipantsSidebar"); + self.show_participants = !self.show_participants; + self.update_ui(cx); + } + _ => {} + } + } + } } } @@ -690,6 +764,9 @@ impl VoipScreen { // Timer for video frames (~30fps) self.video_publish_timer = cx.start_interval(1.0 / 30.0); + // Timer for refreshing call members (every 5 seconds) + self.call_members_refresh_timer = cx.start_interval(5.0); + // Read camera permission and choice from global state (captured at app startup) self.camera_permission = VoipGlobalState::get_camera_permission(cx); self.camera_choice = VoipGlobalState::get_camera_choice(cx); @@ -698,7 +775,10 @@ impl VoipScreen { self.try_start_camera(cx); // Set default room - self.set_room(cx, room_id); + self.set_room(cx, room_id.clone()); + + // Fetch initial call members + submit_async_request(MatrixRequest::GetCallMembers { room_id }); self.update_ui(cx); } @@ -735,16 +815,17 @@ impl VoipScreen { log!("Starting {:?} call...", call_type); - // In a full implementation, this would send call member state via Matrix - // For now, we simulate connection - self.call.connection_state = ConnectionState::Connected; - self.in_lobby = false; - self.call_start_time = Some(Cx::time_now()); - - // Stop lobby camera - CameraManager::stop_lobby_camera(&self.view, cx); - self.pending_call_camera_start = true; - self.camera_active = false; + // Send call member state event via Matrix (MSC3401) + if let Some(room_id) = self.room_id.clone() { + log!("Submitting SendCallMemberState request for room {}", room_id); + submit_async_request(MatrixRequest::SendCallMemberState { + room_id, + call_type, + }); + } else { + log!("Error: No room_id set, cannot start call"); + self.call.connection_state = ConnectionState::Disconnected; + } self.update_ui(cx); } @@ -790,6 +871,14 @@ impl VoipScreen { log!("LiveKit error: {}", e); needs_update = true; } + LiveKitMessage::RemoteVideoFrame { participant_id, y, u, v, width, height, pts_ms } => { + // TODO: Update participant's video texture with the I420 frame data + // This would use RemoteVideoSession to push frames to a Video widget + log!("Remote video frame from {}: {}x{} (Y:{} U:{} V:{} bytes) pts={}ms", + participant_id, width, height, y.len(), u.len(), v.len(), pts_ms); + // For now, just mark needs_update to trigger UI refresh + needs_update = true; + } } } @@ -806,12 +895,26 @@ impl VoipScreen { } /// Toggle camera - fn toggle_camera(&mut self) { + fn toggle_camera(&mut self, cx: &mut Cx) { self.call.local_video_muted = !self.call.local_video_muted; if let Some(client) = &self.livekit_client { client.set_camera_muted(self.call.local_video_muted); } - log!("Camera {}", if self.call.local_video_muted { "off" } else { "on" }); + + if self.call.local_video_muted { + // Camera off - stop the call camera + log!("Camera off - stopping call camera"); + CameraManager::stop_call_camera(&self.view, cx); + self.camera_active = false; + } else { + // Camera on - start the call camera + log!("Camera on - starting call camera"); + if let Some(choice) = self.camera_choice.clone() { + if CameraManager::start_call_camera(&self.view, cx, &choice) { + self.camera_active = true; + } + } + } } /// Toggle screen sharing @@ -838,11 +941,16 @@ impl VoipScreen { log!("Ending call..."); + // Send end call state event via Matrix (MSC3401) + if let Some(room_id) = self.room_id.clone() { + log!("Submitting SendEndCallState request for room {}", room_id); + submit_async_request(MatrixRequest::SendEndCallState { room_id }); + } + // Reset state self.call.connection_state = ConnectionState::Disconnected; self.in_lobby = true; self.call_start_time = None; - //self.try_start_camera(cx); CameraManager::stop_lobby_camera(&self.view, cx); self.pending_call_camera_start = true; self.camera_active = false; @@ -878,7 +986,7 @@ impl VoipScreen { let mute_icon = if self.call.local_audio_muted { "M" } else { "" }; self.view.label(cx, ids!(local_mute_icon)).set_text(cx, mute_icon); - self.view.view(cx, ids!(participants_sidebar)).set_visible(cx, self.show_participants); + // Participants panel is now always visible on the left (no toggle needed) self.view.view(cx, ids!(debug_panel)).set_visible(cx, self.show_debug); if let Some(start) = self.call_start_time { @@ -921,6 +1029,7 @@ impl VoipScreen { } // Show "Join Call" button always in lobby (it's the main action button now) + self.view.view(cx, ids!(join_call_button_view)).set_visible(cx, self.in_lobby); self.view.button(cx, ids!(join_call_button)).set_visible(cx, self.in_lobby); // Force redraw to ensure all visibility changes take effect @@ -993,6 +1102,7 @@ impl VoipScreen { if self.camera_active { if self.in_lobby { CameraManager::show_lobby_video(&self.view, cx); + self.view.view(cx, ids!(join_call_button_view)).set_visible(cx, true); } else { CameraManager::show_call_video(&self.view, cx); } @@ -1010,16 +1120,19 @@ impl VoipScreen { } } - /// Handle video resources released + /// Handle video resources released (camera handoff from lobby to call) fn handle_video_resources_released(&mut self, cx: &mut Cx) { - log!("Video resources released"); if self.pending_call_camera_start { + log!("Lobby camera released, starting call camera..."); self.pending_call_camera_start = false; if let Some(choice) = self.camera_choice.clone() { if CameraManager::start_call_camera(&self.view, cx, &choice) { + log!("Call camera started successfully"); self.camera_active = true; } } + } else { + log!("Video resources released"); } } @@ -1076,7 +1189,7 @@ impl VoipScreen { self.update_ui(cx); } if self.view.button(cx, ids!(camera_button)).clicked(actions) { - self.toggle_camera(); + self.toggle_camera(cx); self.update_ui(cx); } if self.view.button(cx, ids!(screenshare_button)).clicked(actions) { @@ -1093,8 +1206,8 @@ impl VoipScreen { } } - /// Add a test participant - pub fn add_participant(&mut self, cx: &mut Cx, name: &str) { + /// Add a test participant (with optional video on) + pub fn add_participant(&mut self, cx: &mut Cx, name: &str, is_video_on: bool) { self.participant_counter += 1; let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); let participant = Participant { @@ -1103,13 +1216,24 @@ impl VoipScreen { avatar_letter: letter, is_muted: false, is_speaking: false, + is_video_on, }; - log!("Adding participant: {} (id={})", name, self.participant_counter); + log!("Adding participant: {} (id={}, video={})", name, self.participant_counter, is_video_on); let list = self.view.participants_list(cx, ids!(participants_list)); list.add_participant(cx, participant); } + /// Toggle participant video state + pub fn toggle_participant_video(&mut self, cx: &mut Cx, id: &str) { + log!("Toggling video for participant id={}", id); + let list = self.view.participants_list(cx, ids!(participants_list)); + list.update_participant(cx, id, |p| { + p.is_video_on = !p.is_video_on; + log!("Participant {} video is now {}", p.name, if p.is_video_on { "on" } else { "off" }); + }); + } + /// Remove a participant pub fn remove_participant(&mut self, cx: &mut Cx, id: &str) { log!("Removing participant with id={}", id); @@ -1124,6 +1248,72 @@ impl VoipScreen { list.clear(cx); self.participant_counter = 0; } + + /// Update participants list from Matrix call member state events + fn update_participants_from_call_members(&mut self, cx: &mut Cx, members: &[CallMember]) { + log!("update_participants_from_call_members: received {} members", members.len()); + + // Clear existing participants and rebuild from call members + let list = self.view.participants_list(cx, ids!(participants_list)); + list.clear(cx); + self.participant_counter = 0; + + // Get current user to exclude self from participants list + let current_user_id = get_client() + .and_then(|c| c.session_meta().map(|m| m.user_id.to_string())); + log!("Current user ID: {:?}", current_user_id); + + for member in members { + // Skip self + if current_user_id.as_ref() == Some(&member.user_id) { + continue; + } + + self.participant_counter += 1; + let name = member.display_name.clone() + .unwrap_or_else(|| member.user_id.clone()); + let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); + + let participant = Participant { + id: format!("{}_{}", member.user_id, member.device_id), + name, + avatar_letter: letter, + is_muted: false, // We don't have this info from state events + is_speaking: false, + is_video_on: false, // We don't have this info from state events + }; + + log!("Adding call member: {} (user={}, device={})", + participant.name, member.user_id, member.device_id); + list.add_participant(cx, participant); + } + + // Update participant count display + let count = members.len(); + self.view.label(cx, ids!(participant_count)) + .set_text(cx, &format!("{} participant{}", count, if count == 1 { "" } else { "s" })); + + log!("Updated participants panel with {} other participants", self.participant_counter); + } + + /// Connect to LiveKit with the given URL and JWT token + fn connect_livekit(&mut self, cx: &mut Cx, url: &str, jwt: &str) { + log!("connect_livekit: url={}", url); + + if let Some(client) = &self.livekit_client { + // Connect to LiveKit + client.connect(url.to_string(), jwt.to_string()); + + // Update connection state + self.call.connection_state = ConnectionState::Connected; + log!("LiveKit connection initiated"); + } else { + log!("Error: LiveKit client not initialized"); + self.call.connection_state = ConnectionState::Disconnected; + } + + self.update_ui(cx); + } } impl VoipScreenRef { @@ -1149,9 +1339,16 @@ impl VoipScreenRef { } /// Add a participant - pub fn add_participant(&self, cx: &mut Cx, name: &str) { + pub fn add_participant(&self, cx: &mut Cx, name: &str, is_video_on: bool) { + if let Some(mut inner) = self.borrow_mut() { + inner.add_participant(cx, name, is_video_on); + } + } + + /// Toggle participant video state + pub fn toggle_participant_video(&self, cx: &mut Cx, id: &str) { if let Some(mut inner) = self.borrow_mut() { - inner.add_participant(cx, name); + inner.toggle_participant_video(cx, id); } } From 80c37789e3dff18bcc4ddcdb8e61dce34211430f Mon Sep 17 00:00:00 2001 From: alanpoon Date: Thu, 9 Apr 2026 22:00:05 +0800 Subject: [PATCH 04/21] change to webrtc_video shared --- Cargo.lock | 1186 ++++++++++++++++++++++++++++++--- Cargo.toml | 9 +- src/app.rs | 9 + src/shared/mod.rs | 3 +- src/shared/webrtc_video.rs | 322 +++++++++ src/voip/livekit_client.rs | 316 ++++++++- src/voip/mod.rs | 110 +++ src/voip/participants_list.rs | 210 +++++- src/voip/token_cache.rs | 284 ++++++++ src/voip/voip_screen.rs | 658 +++++++++++++++--- 10 files changed, 2902 insertions(+), 205 deletions(-) create mode 100644 src/shared/webrtc_video.rs create mode 100644 src/voip/token_cache.rs diff --git a/Cargo.lock b/Cargo.lock index 7793e9e52..9072824bb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -190,7 +190,7 @@ dependencies = [ "base64ct", "blake2 0.10.6", "cpufeatures", - "password-hash", + "password-hash 0.5.0", ] [[package]] @@ -249,7 +249,7 @@ dependencies = [ "aes", "aes-gcm", "argon2", - "base64", + "base64 0.22.1", "blake2 0.10.6", "block-modes", "bls12_381", @@ -285,7 +285,7 @@ source = "git+https://github.com/openwallet-foundation/askar.git#49086e28d199596 dependencies = [ "aead", "argon2", - "base64", + "base64 0.22.1", "blake2 0.10.6", "chacha20", "chacha20poly1305", @@ -484,7 +484,7 @@ dependencies = [ "bytes", "form_urlencoded", "futures-util", - "http", + "http 1.3.1", "http-body", "http-body-util", "hyper", @@ -516,7 +516,7 @@ checksum = "68464cd0412f486726fb3373129ef5d2993f90c34bc2bc1c1e9943b2f4fc7ca6" dependencies = [ "bytes", "futures-core", - "http", + "http 1.3.1", "http-body", "http-body-util", "mime", @@ -551,6 +551,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6107fe1be6682a68940da878d9e9f5e90ca5745b3dec9fd1bb393c8777d4f581" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -675,7 +681,7 @@ dependencies = [ "arrayvec", "cc", "cfg-if 1.0.4", - "constant_time_eq", + "constant_time_eq 0.3.1", ] [[package]] @@ -745,6 +751,16 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e79769241dcd44edf79a732545e8b5cec84c247ac060f5252cd51885d093a8fc" +[[package]] +name = "bmrng" +version = "0.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d54df9073108f1558f90ae6c5bf5ab9c917c4185f5527b280c87a993cbead0ac" +dependencies = [ + "futures-core", + "tokio", +] + [[package]] name = "bs58" version = "0.5.1" @@ -800,6 +816,26 @@ version = "2.3.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6bd91ee7b2422bcb158d90ef4d14f75ef67f340943fc4149891dcce8f8b972a3" +[[package]] +name = "bzip2" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bdb116a6ef3f6c3698828873ad02c3014b3c85cadb88496095628e3ef1e347f8" +dependencies = [ + "bzip2-sys", + "libc", +] + +[[package]] +name = "bzip2-sys" +version = "0.1.13+1.0.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +dependencies = [ + "cc", + "pkg-config", +] + [[package]] name = "cbc" version = "0.1.2" @@ -836,6 +872,16 @@ dependencies = [ "nom", ] +[[package]] +name = "cfg-expr" +version = "0.20.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3c6b04e07d8080154ed4ac03546d9a2b303cc2fe1901ba0b35b301516e289368" +dependencies = [ + "smallvec 1.15.1 (registry+https://github.com/rust-lang/crates.io-index)", + "target-lexicon", +] + [[package]] name = "cfg-if" version = "0.1.10" @@ -960,7 +1006,7 @@ dependencies = [ "anstream", "anstyle", "clap_lex", - "strsim", + "strsim 0.11.1", ] [[package]] @@ -969,7 +1015,7 @@ version = "4.5.47" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbfd7eae0b0f1a6e63d4b13c9c478de77c2eb546fba158ad50b4203dc24b9f9c" dependencies = [ - "heck", + "heck 0.5.0", "proc-macro2", "quote", "syn 2.0.106", @@ -1027,6 +1073,17 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "codespan-reporting" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "af491d569909a7e4dee0ad7db7f5341fef5c614d5b8ec8cf765732aba3cff681" +dependencies = [ + "serde", + "termcolor", + "unicode-width", +] + [[package]] name = "colorchoice" version = "1.0.4" @@ -1090,6 +1147,12 @@ dependencies = [ "typewit", ] +[[package]] +name = "constant_time_eq" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc" + [[package]] name = "constant_time_eq" version = "0.3.1" @@ -1377,14 +1440,100 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "cxx" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "747d8437319e3a2f43d93b341c137927ca70c0f5dabeea7a005a73665e247c7e" +dependencies = [ + "cc", + "cxx-build", + "cxxbridge-cmd", + "cxxbridge-flags", + "cxxbridge-macro", + "foldhash 0.2.0", + "link-cplusplus", +] + +[[package]] +name = "cxx-build" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0f4697d190a142477b16aef7da8a99bfdc41e7e8b1687583c0d23a79c7afc1e" +dependencies = [ + "cc", + "codespan-reporting 0.13.1", + "indexmap 2.13.0", + "proc-macro2", + "quote", + "scratch", + "syn 2.0.106", +] + +[[package]] +name = "cxxbridge-cmd" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0956799fa8678d4c50eed028f2de1c0552ae183c76e976cf7ca8c4e36a7c328" +dependencies = [ + "clap", + "codespan-reporting 0.13.1", + "indexmap 2.13.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "cxxbridge-flags" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23384a836ab4f0ad98ace7e3955ad2de39de42378ab487dc28d3990392cb283a" + +[[package]] +name = "cxxbridge-macro" +version = "1.0.194" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6acc6b5822b9526adfb4fc377b67128fdd60aac757cc4a741a6278603f763cf" +dependencies = [ + "indexmap 2.13.0", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "darling" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b750cb3417fd1b327431a470f388520309479ab0bf5e323505daf0290cd3850" +dependencies = [ + "darling_core 0.14.4", + "darling_macro 0.14.4", +] + [[package]] name = "darling" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9cdf337090841a411e2a7f3deb9187445851f91b309c0c0a29e05f74a00a48c0" dependencies = [ - "darling_core", - "darling_macro", + "darling_core 0.21.3", + "darling_macro 0.21.3", +] + +[[package]] +name = "darling_core" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "109c1ca6e6b7f82cc233a97004ea8ed7ca123a9af07a8230878fcfda9b158bf0" +dependencies = [ + "fnv", + "ident_case", + "proc-macro2", + "quote", + "strsim 0.10.0", + "syn 1.0.109", ] [[package]] @@ -1397,17 +1546,28 @@ dependencies = [ "ident_case", "proc-macro2", "quote", - "strsim", + "strsim 0.11.1", "syn 2.0.106", ] +[[package]] +name = "darling_macro" +version = "0.14.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4aab4dbc9f7611d8b55048a3a16d2d010c2c8334e46304b40ac1cc14bf3b48e" +dependencies = [ + "darling_core 0.14.4", + "quote", + "syn 1.0.109", +] + [[package]] name = "darling_macro" version = "0.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d38308df82d1080de0afee5d069fa14b0326a88c14f15c5ccda35b4a6c414c81" dependencies = [ - "darling_core", + "darling_core 0.21.3", "quote", "syn 2.0.106", ] @@ -1663,6 +1823,7 @@ dependencies = [ "elliptic-curve", "rfc6979", "signature", + "spki", ] [[package]] @@ -1713,6 +1874,8 @@ dependencies = [ "generic-array", "group", "hkdf", + "pem-rfc7468", + "pkcs8", "rand_core 0.6.4", "sec1", "subtle", @@ -1737,6 +1900,29 @@ dependencies = [ "cfg-if 1.0.4", ] +[[package]] +name = "env_filter" +version = "1.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" +dependencies = [ + "log", + "regex", +] + +[[package]] +name = "env_logger" +version = "0.11.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" +dependencies = [ + "anstream", + "anstyle", + "env_filter", + "jiff", + "log", +] + [[package]] name = "equivalent" version = "1.0.2" @@ -1877,6 +2063,12 @@ version = "0.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1ced73b1dacfc750a6db6c0a0c3a3853c8b41997e2e2c563dc90804ae6867959" +[[package]] +name = "fixedbitset" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ce7134b9999ecaf8bcd65542e436736ef32ddca1b3e06094cb6ec5755203b80" + [[package]] name = "flate2" version = "1.1.2" @@ -1941,6 +2133,37 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "from_variants" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e859c8f2057687618905dbe99fc76e836e0a69738865ef90e46fc214a41bbf2" +dependencies = [ + "from_variants_impl", +] + +[[package]] +name = "from_variants_impl" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55a5e644a80e6d96b2b4910fa7993301d7b7926c045b475b62202b20a36ce69e" +dependencies = [ + "darling 0.14.4", + "proc-macro2", + "quote", + "syn 1.0.109", +] + +[[package]] +name = "fs2" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213" +dependencies = [ + "libc", + "winapi", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -2149,6 +2372,63 @@ dependencies = [ "polyval", ] +[[package]] +name = "gio-sys" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0071fe88dba8e40086c8ff9bbb62622999f49628344b1d1bf490a48a29d80f22" +dependencies = [ + "glib-sys", + "gobject-sys", + "libc", + "system-deps", + "windows-sys 0.61.1", +] + +[[package]] +name = "glib" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "16de123c2e6c90ce3b573b7330de19be649080ec612033d397d72da265f1bd8b" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "futures-channel", + "futures-core", + "futures-executor", + "futures-task", + "futures-util", + "gio-sys", + "glib-macros", + "glib-sys", + "gobject-sys", + "libc", + "memchr 2.7.6 (registry+https://github.com/rust-lang/crates.io-index)", + "smallvec 1.15.1 (registry+https://github.com/rust-lang/crates.io-index)", +] + +[[package]] +name = "glib-macros" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf59b675301228a696fe01c3073974643365080a76cc3ed5bc2cbc466ad87f17" +dependencies = [ + "heck 0.5.0", + "proc-macro-crate", + "proc-macro2", + "quote", + "syn 2.0.106", +] + +[[package]] +name = "glib-sys" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d95e1a3a19ae464a7286e14af9a90683c64d70c02532d88d87ce95056af3e6c" +dependencies = [ + "libc", + "system-deps", +] + [[package]] name = "glob" version = "0.3.3" @@ -2180,6 +2460,17 @@ dependencies = [ "web-sys", ] +[[package]] +name = "gobject-sys" +version = "0.21.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2dca35da0d19a18f4575f3cb99fe1c9e029a2941af5662f326f738a21edaf294" +dependencies = [ + "glib-sys", + "libc", + "system-deps", +] + [[package]] name = "group" version = "0.13.0" @@ -2214,7 +2505,7 @@ dependencies = [ "fnv", "futures-core", "futures-sink", - "http", + "http 1.3.1", "indexmap 2.13.0", "slab", "tokio", @@ -2277,10 +2568,10 @@ version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b3314d5adb5d94bcdf56771f2e50dbbc80bb4bdf88967526706205ac9eff24eb" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "headers-core", - "http", + "http 1.3.1", "httpdate", "mime", "sha1", @@ -2292,9 +2583,15 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "54b4a22553d4242c49fddb9ba998a99962b5cc6f22cb5a3482bec22522403ce4" dependencies = [ - "http", + "http 1.3.1", ] +[[package]] +name = "heck" +version = "0.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "95505c38b4572b2d910cecb0281560f54b440a19336cbbcb27bf6ce6adc6f5a8" + [[package]] name = "heck" version = "0.5.0" @@ -2394,6 +2691,17 @@ dependencies = [ "pastey", ] +[[package]] +name = "http" +version = "0.2.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "601cbb57e577e2f5ef5be8e7b83f0f63994f25aa94d673e54a92d5c516d101f1" +dependencies = [ + "bytes", + "fnv", + "itoa", +] + [[package]] name = "http" version = "1.3.1" @@ -2421,7 +2729,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1efedce1fb8e6913f23e0c92de8e62cd5b772a67e7b3946df930a62566c93184" dependencies = [ "bytes", - "http", + "http 1.3.1", ] [[package]] @@ -2432,7 +2740,7 @@ checksum = "b021d93e26becf5dc7e1b75b1bed1fd93124b374ceb73f43d4d4eafec896a64a" dependencies = [ "bytes", "futures-core", - "http", + "http 1.3.1", "http-body", "pin-project-lite", ] @@ -2469,7 +2777,7 @@ dependencies = [ "futures-channel", "futures-core", "h2", - "http", + "http 1.3.1", "http-body", "httparse", "httpdate", @@ -2487,14 +2795,14 @@ version = "0.27.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e3c93eb611681b207e1fe55d5a71ecf91572ec8a6705cdb6857f7d8d5242cf58" dependencies = [ - "http", + "http 1.3.1", "hyper", "hyper-util", - "rustls", - "rustls-native-certs", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", + "tokio-rustls 0.26.3", "tower-service", "webpki-roots", ] @@ -2521,12 +2829,12 @@ version = "0.1.17" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c6995591a8f1380fcb4ba966a252a4b29188d51d2b89e3a252f5305be65aea8" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "futures-channel", "futures-core", "futures-util", - "http", + "http 1.3.1", "http-body", "hyper", "ipnet", @@ -2814,6 +3122,15 @@ dependencies = [ "either", ] +[[package]] +name = "itertools" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b1c173a5686ce8bfa551b3563d0c2170bf24ca44da99c7ca4bfdab5418c3fe57" +dependencies = [ + "either", +] + [[package]] name = "itertools" version = "0.13.0" @@ -2838,6 +3155,30 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" +[[package]] +name = "jiff" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" +dependencies = [ + "jiff-static", + "log", + "portable-atomic", + "portable-atomic-util", + "serde_core", +] + +[[package]] +name = "jiff-static" +version = "0.2.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.106", +] + [[package]] name = "jni" version = "0.21.1" @@ -2898,6 +3239,27 @@ dependencies = [ "serde_core", ] +[[package]] +name = "jsonwebtoken" +version = "10.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0529410abe238729a60b108898784df8984c87f6054c9c4fcacc47e4803c1ce1" +dependencies = [ + "base64 0.22.1", + "ed25519-dalek", + "getrandom 0.2.16", + "hmac", + "js-sys", + "p256", + "p384", + "rand 0.8.5", + "rsa", + "serde", + "serde_json", + "sha2 0.10.9", + "signature", +] + [[package]] name = "k256" version = "0.13.4" @@ -2995,6 +3357,41 @@ dependencies = [ "vcpkg", ] +[[package]] +name = "libwebrtc" +version = "0.3.29" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e6e6738afd128524d26e4896f59574f52118883b7fb6c3146d4acdb4e0dec590" +dependencies = [ + "cxx", + "glib", + "jni", + "js-sys", + "lazy_static", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "rtrb", + "serde", + "serde_json", + "thiserror 2.0.17", + "tokio", + "wasm-bindgen", + "wasm-bindgen-futures", + "web-sys", + "webrtc-sys", +] + +[[package]] +name = "link-cplusplus" +version = "1.0.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7f78c730aaa7d0b9336a299029ea49f9ee53b0ed06e9202e8cb7db9bae7b8c82" +dependencies = [ + "cc", +] + [[package]] name = "linkify" version = "0.10.0" @@ -3022,6 +3419,112 @@ version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "241eaef5fd12c88705a01fc1066c48c4b36e0dd4377dcdc7ec3942cea7a69956" +[[package]] +name = "livekit" +version = "0.7.36" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dd316178ce26fb2ee1ccd453ac2d78d4a3b027ac56cc02f4538b06b54989afcf" +dependencies = [ + "base64 0.22.1", + "bmrng", + "bytes", + "chrono", + "futures-util", + "lazy_static", + "libloading", + "libwebrtc", + "livekit-api", + "livekit-datatrack", + "livekit-protocol", + "livekit-runtime", + "log", + "parking_lot", + "prost 0.12.6", + "semver", + "serde", + "serde_json", + "thiserror 1.0.69", + "tokio", +] + +[[package]] +name = "livekit-api" +version = "0.4.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e2247e3127fc52f9cc30f2763726f0508120d0424bd5159464d731cb58e2bea1" +dependencies = [ + "base64 0.21.7", + "futures-util", + "http 1.3.1", + "jsonwebtoken", + "livekit-protocol", + "livekit-runtime", + "log", + "os_info", + "parking_lot", + "pbjson-types", + "prost 0.12.6", + "rand 0.9.2", + "reqwest", + "rustls-native-certs 0.6.3", + "scopeguard", + "serde", + "serde_json", + "sha2 0.10.9", + "thiserror 2.0.17", + "tokio", + "tokio-rustls 0.24.1", + "tokio-tungstenite 0.20.1", + "url", +] + +[[package]] +name = "livekit-datatrack" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "27259f6ba14232601345a1118f3c020d1cfb3b5356e7a79275ed192fdea2ed29" +dependencies = [ + "anyhow", + "bytes", + "from_variants", + "futures-core", + "futures-util", + "livekit-protocol", + "livekit-runtime", + "log", + "rand 0.9.2", + "thiserror 2.0.17", + "tokio", + "tokio-stream", +] + +[[package]] +name = "livekit-protocol" +version = "0.7.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d080ff1e4806c4b57427db696dc093e1a6817de9500a262167259cf7bab646e" +dependencies = [ + "futures-util", + "livekit-runtime", + "parking_lot", + "pbjson", + "pbjson-types", + "prost 0.12.6", + "serde", + "thiserror 2.0.17", + "tokio", +] + +[[package]] +name = "livekit-runtime" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "532e84c6cdc5fe774f2b5d9912597b5f3bea561927a48296d03e24549d21c3f6" +dependencies = [ + "tokio", + "tokio-stream", +] + [[package]] name = "lock_api" version = "0.4.13" @@ -3556,7 +4059,7 @@ dependencies = [ "futures-core", "futures-util", "gloo-timers", - "http", + "http 1.3.1", "imbl", "indexmap 2.13.0", "itertools 0.14.0", @@ -3663,7 +4166,7 @@ dependencies = [ "itertools 0.14.0", "js_option", "matrix-sdk-common", - "pbkdf2", + "pbkdf2 0.12.2", "rand 0.8.5", "rmp-serde", "ruma", @@ -3688,7 +4191,7 @@ version = "0.16.0" source = "git+https://github.com/matrix-org/matrix-rust-sdk?branch=main#d64c9906587a686f477042a5129d4ca9e6f0d1a2" dependencies = [ "async-trait", - "base64", + "base64 0.22.1", "futures-util", "getrandom 0.2.16", "gloo-utils", @@ -3745,12 +4248,12 @@ name = "matrix-sdk-store-encryption" version = "0.16.0" source = "git+https://github.com/matrix-org/matrix-rust-sdk?branch=main#d64c9906587a686f477042a5129d4ca9e6f0d1a2" dependencies = [ - "base64", + "base64 0.22.1", "blake3", "chacha20poly1305", "getrandom 0.2.16", "hmac", - "pbkdf2", + "pbkdf2 0.12.2", "rand 0.8.5", "rmp-serde", "serde", @@ -3957,6 +4460,12 @@ dependencies = [ "unsigned-varint", ] +[[package]] +name = "multimap" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d87ecb2933e8aeadb3e3a02b828fed80a7528047e68b4f424523a0981a3a084" + [[package]] name = "naga" version = "27.0.3" @@ -3968,7 +4477,7 @@ dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "cfg-if 1.0.4", "cfg_aliases", - "codespan-reporting", + "codespan-reporting 0.12.0", "half", "hashbrown 0.16.1", "hexf-parse", @@ -4080,6 +4589,18 @@ version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "650eef8c711430f1a879fdd01d4745a7deea475becfb90269c06775983bbf086" +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "cfg-if 1.0.4", + "cfg_aliases", + "libc", +] + [[package]] name = "nokhwa" version = "0.10.10" @@ -4233,10 +4754,10 @@ version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "51e219e79014df21a225b1860a479e2dcd7cbd9130f4defd4bd0e191ea31d67d" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "getrandom 0.2.16", - "http", + "http 1.3.1", "rand 0.8.5", "reqwest", "serde", @@ -4277,6 +4798,27 @@ dependencies = [ "objc2-foundation", ] +[[package]] +name = "objc2-cloud-kit" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "17614fdcd9b411e6ff1117dfb1d0150f908ba83a7df81b1f118005fe0a8ea15d" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "objc2", + "objc2-foundation", +] + +[[package]] +name = "objc2-core-data" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "291fbbf7d29287518e8686417cf7239c74700fd4b607623140a7d4a3c834329d" +dependencies = [ + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-core-foundation" version = "0.3.1" @@ -4288,6 +4830,29 @@ dependencies = [ "objc2", ] +[[package]] +name = "objc2-core-graphics" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "989c6c68c13021b5c2d6b71456ebb0f9dc78d752e86a98da7c716f4f9470f5a4" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "dispatch2", + "objc2", + "objc2-core-foundation", + "objc2-io-surface", +] + +[[package]] +name = "objc2-core-image" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "79b3dc0cc4386b6ccf21c157591b34a7f44c8e75b064f85502901ab2188c007e" +dependencies = [ + "objc2", + "objc2-foundation", +] + [[package]] name = "objc2-core-location" version = "0.3.1" @@ -4311,10 +4876,35 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "900831247d2fe1a09a683278e5384cfb8c80c79fe6b166f9d14bfdde0ea1b03c" dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "block2", + "libc", "objc2", "objc2-core-foundation", ] +[[package]] +name = "objc2-io-surface" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7282e9ac92529fa3457ce90ebb15f4ecbc383e8338060960760fa2cf75420c3c" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "objc2", + "objc2-core-foundation", +] + +[[package]] +name = "objc2-quartz-core" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "90ffb6a0cd5f182dc964334388560b12a57f7b74b3e2dec5e2722aa2dfb2ccd5" +dependencies = [ + "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", + "objc2", + "objc2-core-foundation", + "objc2-foundation", +] + [[package]] name = "objc2-ui-kit" version = "0.3.1" @@ -4323,6 +4913,24 @@ checksum = "25b1312ad7bc8a0e92adae17aa10f90aae1fb618832f9b993b022b591027daed" dependencies = [ "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "block2", + "objc2", + "objc2-cloud-kit", + "objc2-core-data", + "objc2-core-foundation", + "objc2-core-graphics", + "objc2-core-image", + "objc2-core-location", + "objc2-foundation", + "objc2-quartz-core", + "objc2-user-notifications", +] + +[[package]] +name = "objc2-user-notifications" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a3f5ec77a81d9e0c5a0b32159b0cb143d7086165e79708351e02bf37dfc65cd" +dependencies = [ "objc2", "objc2-foundation", ] @@ -4410,6 +5018,22 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "04744f49eae99ab78e0d5c0b603ab218f515ea8cfe5a456d7629ad883a3b6e7d" +[[package]] +name = "os_info" +version = "3.14.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e4022a17595a00d6a369236fdae483f0de7f0a339960a53118b818238e132224" +dependencies = [ + "android_system_properties", + "log", + "nix", + "objc2", + "objc2-foundation", + "objc2-ui-kit", + "serde", + "windows-sys 0.61.1", +] + [[package]] name = "p256" version = "0.13.2" @@ -4464,27 +5088,87 @@ dependencies = [ ] [[package]] -name = "password-hash" -version = "0.5.0" +name = "password-hash" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7676374caaee8a325c9e7a2ae557f216c5563a171d6997b0ef8a65af35147700" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "password-hash" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +dependencies = [ + "base64ct", + "rand_core 0.6.4", + "subtle", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pastey" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" + +[[package]] +name = "pbjson" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1030c719b0ec2a2d25a5df729d6cff1acf3cc230bf766f4f97833591f7577b90" +dependencies = [ + "base64 0.21.7", + "serde", +] + +[[package]] +name = "pbjson-build" +version = "0.6.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "346f04948ba92c43e8469c1ee6736c7563d71012b17d40745260fe106aac2166" +checksum = "2580e33f2292d34be285c5bc3dba5259542b083cfad6037b6d70345f24dcb735" dependencies = [ - "base64ct", - "rand_core 0.6.4", - "subtle", + "heck 0.4.1", + "itertools 0.11.0", + "prost 0.12.6", + "prost-types", ] [[package]] -name = "paste" -version = "1.0.15" +name = "pbjson-types" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "18f596653ba4ac51bdecbb4ef6773bc7f56042dc13927910de1684ad3d32aa12" +dependencies = [ + "bytes", + "chrono", + "pbjson", + "pbjson-build", + "prost 0.12.6", + "prost-build", + "serde", +] [[package]] -name = "pastey" -version = "0.1.1" +name = "pbkdf2" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "35fb2e5f958ec131621fdd531e9fc186ed768cbe395337403ae56c17a74c68ec" +checksum = "83a0692ec44e4cf1ef28ca317f14f8f07da2d95ec3fa01f86e4467b725e60917" +dependencies = [ + "digest 0.10.7", + "hmac", + "password-hash 0.4.2", + "sha2 0.10.9", +] [[package]] name = "pbkdf2" @@ -4517,6 +5201,16 @@ version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "petgraph" +version = "0.6.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4c5cc86750666a3ed20bdaf5ca2a0344f9c67674cae0515bec2da16fbaa47db" +dependencies = [ + "fixedbitset", + "indexmap 2.13.0", +] + [[package]] name = "phf" version = "0.11.3" @@ -4635,6 +5329,21 @@ dependencies = [ "universal-hash", ] +[[package]] +name = "portable-atomic" +version = "1.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" + +[[package]] +name = "portable-atomic-util" +version = "0.2.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" +dependencies = [ + "portable-atomic", +] + [[package]] name = "potential_utf" version = "0.1.3" @@ -4723,6 +5432,16 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "prost" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "deb1435c188b76130da55f17a466d252ff7b1418b2ad3e037d127b94e3411f29" +dependencies = [ + "bytes", + "prost-derive 0.12.6", +] + [[package]] name = "prost" version = "0.13.5" @@ -4730,7 +5449,41 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2796faa41db3ec313a31f7624d9286acf277b52de526150b7e69f3debf891ee5" dependencies = [ "bytes", - "prost-derive", + "prost-derive 0.13.5", +] + +[[package]] +name = "prost-build" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22505a5c94da8e3b7c2996394d1c933236c4d743e81a410bcca4e6989fc066a4" +dependencies = [ + "bytes", + "heck 0.5.0", + "itertools 0.10.5", + "log", + "multimap", + "once_cell", + "petgraph", + "prettyplease", + "prost 0.12.6", + "prost-types", + "regex", + "syn 2.0.106", + "tempfile", +] + +[[package]] +name = "prost-derive" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "81bddcdb20abf9501610992b6759a4c888aef7d1a7247ef75e2404275ac24af1" +dependencies = [ + "anyhow", + "itertools 0.10.5", + "proc-macro2", + "quote", + "syn 2.0.106", ] [[package]] @@ -4746,6 +5499,15 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "prost-types" +version = "0.12.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9091c90b0a32608e984ff2fa4091273cbdd755d54935c51d520887f4a1dbd5b0" +dependencies = [ + "prost 0.12.6", +] + [[package]] name = "pulldown-cmark" version = "0.12.2" @@ -4792,7 +5554,7 @@ dependencies = [ "quinn-proto", "quinn-udp", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.32", "socket2", "thiserror 2.0.17", "tokio", @@ -4813,7 +5575,7 @@ dependencies = [ "rand 0.9.2", "ring", "rustc-hash 2.1.1", - "rustls", + "rustls 0.23.32", "rustls-pki-types", "slab", "thiserror 2.0.17", @@ -5015,13 +5777,14 @@ version = "0.12.28" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "eddd3ca559203180a307f12d114c268abf583f59b03cb906fd0b3ff8646c1147" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "encoding_rs", + "futures-channel", "futures-core", "futures-util", "h2", - "http", + "http 1.3.1", "http-body", "http-body-util", "hyper", @@ -5035,8 +5798,8 @@ dependencies = [ "percent-encoding", "pin-project-lite", "quinn", - "rustls", - "rustls-native-certs", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "serde", "serde_json", @@ -5044,7 +5807,7 @@ dependencies = [ "sync_wrapper", "tokio", "tokio-native-tls", - "tokio-rustls", + "tokio-rustls 0.26.3", "tokio-util", "tower", "tower-http", @@ -5187,6 +5950,7 @@ dependencies = [ "clap", "crossbeam-channel", "crossbeam-queue", + "env_logger", "eyeball", "eyeball-im", "futures-util", @@ -5196,6 +5960,8 @@ dependencies = [ "imghdr", "indexmap 2.13.0", "linkify", + "livekit", + "log", "makepad-code-editor", "makepad-widgets", "matrix-sdk", @@ -5243,6 +6009,12 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rtrb" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7204ed6420f698836b76d4d5c2ec5dec7585fd5c3a788fd1cde855d1de598239" + [[package]] name = "ruma" version = "0.14.1" @@ -5268,7 +6040,7 @@ dependencies = [ "assign", "bytes", "date_header", - "http", + "http 1.3.1", "js_int", "js_option", "maplit", @@ -5288,11 +6060,11 @@ version = "0.17.1" source = "git+https://github.com/project-robius/ruma.git?branch=tsp#7028b7fbf1351a092328ec250da86beed885a681" dependencies = [ "as_variant", - "base64", + "base64 0.22.1", "bytes", "form_urlencoded", "getrandom 0.2.16", - "http", + "http 1.3.1", "indexmap 2.13.0", "js-sys", "js_int", @@ -5347,7 +6119,7 @@ version = "0.13.1" source = "git+https://github.com/project-robius/ruma.git?branch=tsp#7028b7fbf1351a092328ec250da86beed885a681" dependencies = [ "headers", - "http", + "http 1.3.1", "http-auth", "js_int", "mime", @@ -5401,7 +6173,7 @@ name = "ruma-signatures" version = "0.19.0" source = "git+https://github.com/project-robius/ruma.git?branch=tsp#7028b7fbf1351a092328ec250da86beed885a681" dependencies = [ - "base64", + "base64 0.22.1", "ed25519-dalek", "pkcs8", "rand 0.8.5", @@ -5472,6 +6244,18 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "rustls" +version = "0.21.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3f56a14d1f48b391359b22f731fd4bd7e43c97f3c50eee276f3aa09c94784d3e" +dependencies = [ + "log", + "ring", + "rustls-webpki 0.101.7", + "sct", +] + [[package]] name = "rustls" version = "0.23.32" @@ -5483,11 +6267,23 @@ dependencies = [ "once_cell", "ring", "rustls-pki-types", - "rustls-webpki", + "rustls-webpki 0.103.6", "subtle", "zeroize", ] +[[package]] +name = "rustls-native-certs" +version = "0.6.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9aace74cb666635c918e9c12bc0d348266037aa8eb599b5cba565709a8dff00" +dependencies = [ + "openssl-probe", + "rustls-pemfile 1.0.4", + "schannel", + "security-framework 2.11.1", +] + [[package]] name = "rustls-native-certs" version = "0.8.1" @@ -5500,6 +6296,15 @@ dependencies = [ "security-framework 3.5.0", ] +[[package]] +name = "rustls-pemfile" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1c74cae0a4cf6ccbbf5f359f08efdf8ee7e1dc532573bf0db71968cb56b1448c" +dependencies = [ + "base64 0.21.7", +] + [[package]] name = "rustls-pemfile" version = "2.2.0" @@ -5519,6 +6324,16 @@ dependencies = [ "zeroize", ] +[[package]] +name = "rustls-webpki" +version = "0.101.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8b6275d1ee7a1cd780b64aca7726599a1dbc893b1e64144529e55c3c2f745765" +dependencies = [ + "ring", + "untrusted 0.9.0", +] + [[package]] name = "rustls-webpki" version = "0.103.6" @@ -5637,6 +6452,22 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scratch" +version = "1.0.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d68f2ec51b097e4c1a75b681a8bec621909b5e91f15bb7b840c4f2f7b01148b2" + +[[package]] +name = "sct" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "da046153aa2352493d6cb7da4b6e5c0c057d8a1d0a9aa8560baffdd945acd414" +dependencies = [ + "ring", + "untrusted 0.9.0", +] + [[package]] name = "sdfer" version = "0.2.1" @@ -5662,6 +6493,7 @@ dependencies = [ "base16ct", "der", "generic-array", + "pkcs8", "subtle", "zeroize", ] @@ -5843,7 +6675,7 @@ version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c522100790450cf78eeac1507263d0a350d4d5b30df0c8e1fe051a10c22b376e" dependencies = [ - "base64", + "base64 0.22.1", "chrono", "hex", "indexmap 1.9.3", @@ -5863,7 +6695,7 @@ version = "3.14.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "327ada00f7d64abaac1e55a6911e90cf665aa051b9a561c7006c157f4633135e" dependencies = [ - "darling", + "darling 0.21.3", "proc-macro2", "quote", "syn 2.0.106", @@ -6022,7 +6854,7 @@ name = "sqlx-core" version = "0.8.6" source = "git+https://github.com/project-robius/sqlx.git?branch=update_libsqlite3-sys_version#345c0446a0eee02acec01ec4635e3945fdf4693b" dependencies = [ - "base64", + "base64 0.22.1", "bytes", "chrono", "crc", @@ -6070,7 +6902,7 @@ source = "git+https://github.com/project-robius/sqlx.git?branch=update_libsqlite dependencies = [ "dotenvy", "either", - "heck", + "heck 0.5.0", "hex", "once_cell", "proc-macro2", @@ -6093,7 +6925,7 @@ version = "0.8.6" source = "git+https://github.com/project-robius/sqlx.git?branch=update_libsqlite3-sys_version#345c0446a0eee02acec01ec4635e3945fdf4693b" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "bytes", @@ -6135,7 +6967,7 @@ version = "0.8.6" source = "git+https://github.com/project-robius/sqlx.git?branch=update_libsqlite3-sys_version#345c0446a0eee02acec01ec4635e3945fdf4693b" dependencies = [ "atoi", - "base64", + "base64 0.22.1", "bitflags 2.10.0 (registry+https://github.com/rust-lang/crates.io-index)", "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", "chrono", @@ -6232,6 +7064,12 @@ dependencies = [ "unicode-properties 0.1.3", ] +[[package]] +name = "strsim" +version = "0.10.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623" + [[package]] name = "strsim" version = "0.11.1" @@ -6307,6 +7145,25 @@ dependencies = [ "libc", ] +[[package]] +name = "system-deps" +version = "7.0.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "48c8f33736f986f16d69b6cb8b03f55ddcad5c41acc4ccc39dd88e84aa805e7f" +dependencies = [ + "cfg-expr", + "heck 0.5.0", + "pkg-config", + "toml", + "version-compare", +] + +[[package]] +name = "target-lexicon" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" + [[package]] name = "tempfile" version = "3.23.0" @@ -6331,6 +7188,15 @@ dependencies = [ "utf-8", ] +[[package]] +name = "termcolor" +version = "1.4.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06794f8f6c5c898b3275aebefa6b8a1cb24cd2c6c79397ab15774837a0bc5755" +dependencies = [ + "winapi-util", +] + [[package]] name = "thiserror" version = "1.0.69" @@ -6473,13 +7339,23 @@ dependencies = [ "tokio", ] +[[package]] +name = "tokio-rustls" +version = "0.24.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c28327cf380ac148141087fbfb9de9d7bd4e84ab5d2c28fbc911d753de8a7081" +dependencies = [ + "rustls 0.21.12", + "tokio", +] + [[package]] name = "tokio-rustls" version = "0.26.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f63835928ca123f1bef57abbcd23bb2ba0ac9ae1235f1e65bda0d06e7786bd" dependencies = [ - "rustls", + "rustls 0.23.32", "tokio", ] @@ -6495,6 +7371,21 @@ dependencies = [ "tokio-util", ] +[[package]] +name = "tokio-tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "212d5dcb2a1ce06d81107c3d0ffa3121fe974b73f068c8282cb1c32328113b6c" +dependencies = [ + "futures-util", + "log", + "rustls 0.21.12", + "rustls-native-certs 0.6.3", + "tokio", + "tokio-rustls 0.24.1", + "tungstenite 0.20.1", +] + [[package]] name = "tokio-tungstenite" version = "0.26.2" @@ -6503,12 +7394,12 @@ checksum = "7a9daff607c6d2bf6c16fd681ccb7eecc83e4e2cdc1ca067ffaadfca5de7f084" dependencies = [ "futures-util", "log", - "rustls", - "rustls-native-certs", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", "rustls-pki-types", "tokio", - "tokio-rustls", - "tungstenite", + "tokio-rustls 0.26.3", + "tungstenite 0.26.2", ] [[package]] @@ -6530,10 +7421,12 @@ version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ + "indexmap 2.13.0", "serde_core", "serde_spanned", "toml_datetime", "toml_parser", + "toml_writer", "winnow", ] @@ -6567,6 +7460,12 @@ dependencies = [ "winnow", ] +[[package]] +name = "toml_writer" +version = "1.1.1+spec-1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "756daf9b1013ebe47a8776667b466417e2d4c5679d441c26230efd9ef78692db" + [[package]] name = "tower" version = "0.5.2" @@ -6594,7 +7493,7 @@ dependencies = [ "bytes", "futures-core", "futures-util", - "http", + "http 1.3.1", "http-body", "http-body-util", "iri-string", @@ -6709,9 +7608,9 @@ dependencies = [ "rand 0.8.5", "rand_core 0.6.4", "reqwest", - "rustls", - "rustls-native-certs", - "rustls-pemfile", + "rustls 0.23.32", + "rustls-native-certs 0.8.1", + "rustls-pemfile 2.2.0", "rustls-pki-types", "serde", "serde_json", @@ -6719,8 +7618,8 @@ dependencies = [ "sha2 0.11.0-rc.3", "thiserror 2.0.17", "tokio", - "tokio-rustls", - "tokio-tungstenite", + "tokio-rustls 0.26.3", + "tokio-tungstenite 0.26.2", "tokio-util", "tracing", "typenum", @@ -6733,6 +7632,26 @@ name = "ttf-parser" version = "0.24.1" source = "git+https://github.com/kevinaboos/makepad?branch=stack_nav_improvements#461b05134a501b8e67f431b2706fb200d4bcf68a" +[[package]] +name = "tungstenite" +version = "0.20.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e3dac10fd62eaf6617d3a904ae222845979aec67c615d1c842b4002c7666fb9" +dependencies = [ + "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bytes", + "data-encoding", + "http 0.2.12", + "httparse", + "log", + "rand 0.8.5", + "rustls 0.21.12", + "sha1", + "thiserror 1.0.69", + "url", + "utf-8", +] + [[package]] name = "tungstenite" version = "0.26.2" @@ -6741,11 +7660,11 @@ checksum = "4793cb5e56680ecbb1d843515b23b6de9a75eb04b66643e256a396d43be33c13" dependencies = [ "bytes", "data-encoding", - "http", + "http 1.3.1", "httparse", "log", "rand 0.9.2", - "rustls", + "rustls 0.23.32", "rustls-pki-types", "sha1", "thiserror 2.0.17", @@ -6982,6 +7901,12 @@ version = "0.2.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "accd4ea62f7bb7a82fe23066fb0957d48ef677f6eeb8215f372f52e48bb32426" +[[package]] +name = "version-compare" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03c2856837ef78f57382f06b2b8563a2f512f7185d732608fd9176cb3b8edf0e" + [[package]] name = "version_check" version = "0.9.5" @@ -6996,7 +7921,7 @@ checksum = "c022a277687e4e8685d72b95a7ca3ccfec907daa946678e715f8badaa650883d" dependencies = [ "aes", "arrayvec", - "base64", + "base64 0.22.1", "base64ct", "cbc", "chacha20poly1305", @@ -7006,7 +7931,7 @@ dependencies = [ "hkdf", "hmac", "matrix-pickle", - "prost", + "prost 0.13.5", "rand 0.8.5", "serde", "serde_bytes", @@ -7249,6 +8174,36 @@ dependencies = [ "rustls-pki-types", ] +[[package]] +name = "webrtc-sys" +version = "0.3.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "63493a8d3b0cfe1ac4cc947d7c7ebba8f2d5f9cc890ee4f807be8c27a7482764" +dependencies = [ + "cc", + "cxx", + "cxx-build", + "glob", + "log", + "pkg-config", + "webrtc-sys-build", +] + +[[package]] +name = "webrtc-sys-build" +version = "0.3.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f9a618b192cc7c7824b0f3bb931876fa7df8ba518ea163bd9f6e6e2d4b485e8" +dependencies = [ + "anyhow", + "fs2", + "regex", + "reqwest", + "scratch", + "semver", + "zip", +] + [[package]] name = "which" version = "4.4.2" @@ -7277,6 +8232,22 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "39b7d07a236abaef6607536ccfaf19b396dbe3f5110ddb73d39f4562902ed382" +[[package]] +name = "winapi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419" +dependencies = [ + "winapi-i686-pc-windows-gnu", + "winapi-x86_64-pc-windows-gnu", +] + +[[package]] +name = "winapi-i686-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" + [[package]] name = "winapi-util" version = "0.1.11" @@ -7286,6 +8257,12 @@ dependencies = [ "windows-sys 0.61.1", ] +[[package]] +name = "winapi-x86_64-pc-windows-gnu" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" + [[package]] name = "windows" version = "0.56.0" @@ -8060,8 +9037,57 @@ dependencies = [ "syn 2.0.106", ] +[[package]] +name = "zip" +version = "0.6.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "760394e246e4c28189f19d488c058bf16f564016aefac5d32bb1f3b51d5e9261" +dependencies = [ + "aes", + "byteorder 1.5.0 (registry+https://github.com/rust-lang/crates.io-index)", + "bzip2", + "constant_time_eq 0.1.5", + "crc32fast", + "crossbeam-utils", + "flate2", + "hmac", + "pbkdf2 0.11.0", + "sha1", + "time", + "zstd", +] + [[package]] name = "zmij" version = "1.0.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bd8f3f50b848df28f887acb68e41201b5aea6bc8a8dacc00fb40635ff9a72fea" + +[[package]] +name = "zstd" +version = "0.11.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "20cc960326ece64f010d2d2107537f26dc589a6573a316bd5b1dba685fa5fde4" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "5.0.2+zstd.1.5.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d2a5585e04f9eea4b2a3d1eca508c4dee9592a89ef6f450c11719da0726f4db" +dependencies = [ + "libc", + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.16+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e19ebc2adc8f83e43039e79776e3fda8ca919132d68a1fed6a5faca2683748" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index ca9bcd595..75bd276cb 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -75,7 +75,12 @@ serde = "1.0" serde_json = "1.0" thiserror = "2.0.16" tokio = { version = "1.43.1", features = ["macros", "rt-multi-thread"] } -# livekit = "0.7" # Requires native libwebrtc build - enable when ready +## LiveKit WebRTC SDK - requires native libwebrtc build +## Enable the "livekit" feature to use real WebRTC functionality +## rustls-tls-native-roots is required for WSS connections +livekit = { version = "0.7", optional = true, features = ["rustls-tls-native-roots"] } +env_logger = { version = "0.11", optional = true } +log = { version = "0.4", optional = true } tracing-subscriber = "0.3.17" unicode-segmentation = "1.11.0" url = "2.5.0" @@ -110,6 +115,8 @@ reqwest = { version = "0.12", default-features = false, features = [ default = [] ## Enables experimental support for using TSP wallets. tsp = ["dep:tsp_sdk", "dep:quinn", "dep:aws-lc-rs", "dep:percent-encoding"] +## Enables real LiveKit WebRTC functionality. Requires native libwebrtc build. +livekit = ["dep:livekit", "dep:env_logger", "dep:log"] ## Hides the command prompt console on Windows. hide_windows_console = [] diff --git a/src/app.rs b/src/app.rs index f9c01d587..26a5a8b54 100644 --- a/src/app.rs +++ b/src/app.rs @@ -447,6 +447,10 @@ impl MatchEvent for App { let logged_in_actual = self.app_state.logged_in; self.app_state = app_state.clone(); self.app_state.logged_in = logged_in_actual; + + // Restore VoIP token state to global state for caching + VoipGlobalState::restore_token_state(cx, self.app_state.voip_tokens.clone()); + cx.action(MainDesktopUiAction::LoadDockFromAppState); continue; } @@ -794,6 +798,8 @@ impl AppMain for App { error!("Failed to save window state. Error: {e}"); } if let Some(user_id) = current_user_id() { + // Get the latest VoIP token state from global state before saving + self.app_state.voip_tokens = VoipGlobalState::get_token_state(cx); let app_state = self.app_state.clone(); if let Err(e) = persistence::save_app_state(app_state, user_id) { error!("Failed to save app state. Error: {e}"); @@ -1248,6 +1254,9 @@ pub struct AppState { /// The room ID for VoIP calls, set when navigating to VoIP screen from a call notification. #[serde(skip)] pub voip_room_id: Option, + /// Cached VoIP tokens (OpenID and LiveKit JWT) for faster reconnection. + #[serde(default)] + pub voip_tokens: crate::voip::VoipTokenState, } /// Local bot integration settings persisted per Matrix account. diff --git a/src/shared/mod.rs b/src/shared/mod.rs index a92a81fd9..3a3b9492d 100644 --- a/src/shared/mod.rs +++ b/src/shared/mod.rs @@ -20,7 +20,7 @@ pub mod unread_badge; pub mod verification_badge; pub mod restore_status_view; pub mod image_viewer; - +pub mod webrtc_video; pub fn script_mod(vm: &mut ScriptVm) { // Order matters here, as some widget definitions depend on others. @@ -44,4 +44,5 @@ pub fn script_mod(vm: &mut ScriptVm) { restore_status_view::script_mod(vm); confirmation_modal::script_mod(vm); image_viewer::script_mod(vm); + webrtc_video::script_mod(vm); } diff --git a/src/shared/webrtc_video.rs b/src/shared/webrtc_video.rs new file mode 100644 index 000000000..52713075e --- /dev/null +++ b/src/shared/webrtc_video.rs @@ -0,0 +1,322 @@ +//! WebRTC video streaming widget for displaying video frames from WebRTC sources. +//! +//! This widget is designed to receive video frames as RGBA data from WebRTC +//! streams (e.g., LiveKit) and display them efficiently using GPU textures. + +use makepad_widgets::*; + +script_mod! { + use mod.prelude.widgets_internal.* + + mod.widgets.WebRtcVideoBase = #(WebRtcVideo::register_widget(vm)) + + mod.widgets.WebRtcVideo = set_type_default() do mod.widgets.WebRtcVideoBase{ + width: 320 + height: 240 + + draw_bg +: { + video_texture: texture_2d(float) + opacity: instance(1.0) + + pixel: fn() { + let color = self.video_texture.sample_as_bgra(self.pos) + return Pal.premul(vec4(color.xyz, color.w * self.opacity)) + } + } + } +} + +/// Video frame data structure for receiving frames from WebRTC sources. +#[derive(Clone)] +pub struct WebRtcVideoFrame { + /// RGBA pixel data (4 bytes per pixel: R, G, B, A) + pub data: Vec, + /// Frame width in pixels + pub width: u32, + /// Frame height in pixels + pub height: u32, + /// Optional participant identifier + pub participant_id: Option, +} + +impl std::fmt::Debug for WebRtcVideoFrame { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("WebRtcVideoFrame") + .field("width", &self.width) + .field("height", &self.height) + .field("participant_id", &self.participant_id) + .field("data_len", &self.data.len()) + .finish() + } +} + +/// Actions emitted by the WebRtcVideo widget. +#[derive(Clone, Debug, Default)] +pub enum WebRtcVideoAction { + #[default] + None, + /// A new frame has been displayed + FrameUpdated { + width: u32, + height: u32, + }, + /// The video stream has started (first frame received) + StreamStarted, + /// The video stream has stopped + StreamStopped, +} + +/// WebRTC video streaming widget. +/// +/// This widget displays video frames from WebRTC sources such as LiveKit. +/// Frames are provided as RGBA data and rendered using GPU textures. +/// +/// # Example +/// +/// ```ignore +/// // In your UI definition: +/// webrtc_video := WebRtcVideo { +/// width: 320 +/// height: 240 +/// } +/// +/// // In your code: +/// let frame = WebRtcVideoFrame { +/// data: rgba_data, +/// width: 640, +/// height: 480, +/// participant_id: Some("user123".to_string()), +/// }; +/// self.ui.webrtc_video(cx, ids!(webrtc_video)).set_frame(cx, frame); +/// ``` +#[derive(Script, ScriptHook, Widget)] +pub struct WebRtcVideo { + #[uid] + uid: WidgetUid, + #[source] + source: ScriptObjectRef, + + #[redraw] + #[live] + draw_bg: DrawColor, + + #[walk] + walk: Walk, + + #[live] + layout: Layout, + + #[visible] + #[live(true)] + visible: bool, + + /// The current video texture + #[rust] + texture: Option, + + /// Current frame dimensions + #[rust] + frame_width: u32, + #[rust] + frame_height: u32, + + /// Whether the stream is active (has received at least one frame) + #[rust] + stream_active: bool, + + /// Participant ID of the current stream source + #[rust] + participant_id: Option, +} + +impl Widget for WebRtcVideo { + fn draw_walk(&mut self, cx: &mut Cx2d, _scope: &mut Scope, walk: Walk) -> DrawStep { + if !self.visible { + return DrawStep::done(); + } + + // Set texture if available + if let Some(texture) = &self.texture { + self.draw_bg.draw_vars.set_texture(0, texture); + } else { + self.draw_bg.draw_vars.empty_texture(0); + } + + self.draw_bg.draw_walk(cx, walk); + DrawStep::done() + } + + fn handle_event(&mut self, _cx: &mut Cx, _event: &Event, _scope: &mut Scope) { + // No special event handling needed for basic video display + } +} + +impl WebRtcVideo { + /// Sets a video frame to be displayed. + /// + /// The frame data should be in RGBA format (4 bytes per pixel). + /// This method handles texture creation and updates efficiently. + pub fn set_frame(&mut self, cx: &mut Cx, frame: WebRtcVideoFrame) { + let width = frame.width as usize; + let height = frame.height as usize; + let pixel_count = width * height; + + // Convert RGBA bytes to packed u32 for VecBGRAu8_32 texture format + // Input: RGBA bytes [R, G, B, A] + // Output: 0xAARRGGBB format (BGRA in memory on little-endian) + let mut data_u32: Vec = Vec::with_capacity(pixel_count); + let rgba = &frame.data; + + for i in 0..pixel_count { + let idx = i * 4; + if idx + 3 < rgba.len() { + let r = rgba[idx] as u32; + let g = rgba[idx + 1] as u32; + let b = rgba[idx + 2] as u32; + let a = rgba[idx + 3] as u32; + // Pack as 0xAARRGGBB + data_u32.push((a << 24) | (r << 16) | (g << 8) | b); + } + } + + // Check if we need to create a new texture (dimension change or first frame) + let needs_new_texture = match &self.texture { + Some(texture) => { + texture.get_format(cx).vec_width_height() != Some((width, height)) + } + None => true, + }; + + let was_inactive = !self.stream_active; + + if needs_new_texture { + // Create new texture with the frame data + let texture = Texture::new_with_format( + cx, + TextureFormat::VecBGRAu8_32 { + width, + height, + data: Some(data_u32), + updated: TextureUpdated::Full, + }, + ); + self.texture = Some(texture); + self.frame_width = frame.width; + self.frame_height = frame.height; + } else if let Some(texture) = &self.texture { + // Reuse existing texture, just update the data + texture.set_data_u32(cx, width, height, data_u32); + } + + // Update state + self.stream_active = true; + self.participant_id = frame.participant_id; + + // Emit action for first frame + if was_inactive { + cx.widget_action(self.uid, WebRtcVideoAction::StreamStarted); + } + + cx.widget_action( + self.uid, + WebRtcVideoAction::FrameUpdated { + width: frame.width, + height: frame.height, + }, + ); + + // Trigger redraw + self.redraw(cx); + } + + /// Clears the current video frame and marks the stream as inactive. + pub fn clear_frame(&mut self, cx: &mut Cx) { + self.texture = None; + self.frame_width = 0; + self.frame_height = 0; + self.participant_id = None; + + if self.stream_active { + self.stream_active = false; + cx.widget_action(self.uid, WebRtcVideoAction::StreamStopped); + } + + self.redraw(cx); + } + + /// Returns the current frame dimensions, or None if no frame has been set. + pub fn frame_size(&self) -> Option<(u32, u32)> { + if self.stream_active { + Some((self.frame_width, self.frame_height)) + } else { + None + } + } + + /// Returns true if the stream is currently active (has received frames). + pub fn is_active(&self) -> bool { + self.stream_active + } + + /// Returns the participant ID of the current stream source. + pub fn participant_id(&self) -> Option<&str> { + self.participant_id.as_deref() + } + + /// Sets the texture directly (advanced usage). + pub fn set_texture(&mut self, cx: &mut Cx, texture: Option) { + self.texture = texture; + self.redraw(cx); + } +} + +/// Reference to a WebRtcVideo widget. +impl WebRtcVideoRef { + /// Sets a video frame to be displayed. + pub fn set_frame(&self, cx: &mut Cx, frame: WebRtcVideoFrame) { + if let Some(mut inner) = self.borrow_mut() { + inner.set_frame(cx, frame); + } + } + + /// Clears the current video frame. + pub fn clear_frame(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.clear_frame(cx); + } + } + + /// Returns the current frame dimensions. + pub fn frame_size(&self) -> Option<(u32, u32)> { + if let Some(inner) = self.borrow() { + inner.frame_size() + } else { + None + } + } + + /// Returns true if the stream is currently active. + pub fn is_active(&self) -> bool { + if let Some(inner) = self.borrow() { + inner.is_active() + } else { + false + } + } + + /// Returns the participant ID of the current stream source. + pub fn participant_id(&self) -> Option { + if let Some(inner) = self.borrow() { + inner.participant_id().map(|s| s.to_string()) + } else { + None + } + } + + /// Sets the texture directly. + pub fn set_texture(&self, cx: &mut Cx, texture: Option) { + if let Some(mut inner) = self.borrow_mut() { + inner.set_texture(cx, texture); + } + } +} diff --git a/src/voip/livekit_client.rs b/src/voip/livekit_client.rs index f03acafeb..cd7c64d68 100644 --- a/src/voip/livekit_client.rs +++ b/src/voip/livekit_client.rs @@ -1,7 +1,8 @@ //! LiveKit client integration for WebRTC //! -//! This is currently a stub implementation. When the livekit crate is enabled, -//! it will provide actual WebRTC connectivity. +//! This module provides LiveKit WebRTC connectivity for VoIP calls. +//! When the `livekit` feature is enabled, it uses the real LiveKit SDK. +//! Otherwise, it provides a stub implementation for development. use std::sync::{Arc, Mutex}; use tokio::sync::mpsc; @@ -49,6 +50,10 @@ pub enum LiveKitMessage { /// Presentation timestamp in milliseconds pts_ms: u64, }, + /// Remote participant's video track subscribed + VideoTrackSubscribed { participant_id: String }, + /// Remote participant's video track unsubscribed + VideoTrackUnsubscribed { participant_id: String }, } /// Commands sent from UI to LiveKit client @@ -60,6 +65,7 @@ pub enum LiveKitCommand { StartScreenShare, StopScreenShare, PublishVideoFrame(VideoFrame), + PublishData { payload: Vec, reliable: bool }, } /// LiveKit client state @@ -86,6 +92,12 @@ impl LiveKitClient { let is_connected = self.is_connected.clone(); std::thread::spawn(move || { + #[cfg(feature = "livekit")] + { + // Initialize env_logger for log::info!, log::error!, etc. + let _ = env_logger::try_init(); + } + let rt = tokio::runtime::Runtime::new().unwrap(); rt.block_on(async { Self::run_event_loop(cmd_rx, msg_tx, is_connected).await; @@ -95,6 +107,291 @@ impl LiveKitClient { msg_rx } + #[cfg(feature = "livekit")] + async fn run_event_loop( + mut cmd_rx: mpsc::UnboundedReceiver, + msg_tx: mpsc::UnboundedSender, + is_connected: Arc>, + ) { + use livekit::prelude::*; + use livekit::RoomOptions; + use livekit::webrtc::video_stream::native::NativeVideoStream; + use livekit::webrtc::prelude::VideoBuffer; + use futures_util::StreamExt; + + let mut room: Option> = None; + + while let Some(cmd) = cmd_rx.recv().await { + match cmd { + LiveKitCommand::Connect { url, token } => { + log!("LiveKit: Connecting to {}", url); + log!("LiveKit: Token length: {}, starts with: {}", + token.len(), + if token.len() > 20 { &token[..20] } else { &token }); + + // Validate token is not empty and looks like a JWT (three dot-separated parts) + if token.is_empty() { + log!("LiveKit: ERROR - Token is empty!"); + let _ = msg_tx.send(LiveKitMessage::Error("Token is empty".to_string())); + SignalToUI::set_ui_signal(); + continue; + } + + let jwt_parts: Vec<&str> = token.split('.').collect(); + if jwt_parts.len() != 3 { + log!("LiveKit: WARNING - Token doesn't look like a JWT (expected 3 parts, got {})", jwt_parts.len()); + } else { + log!("LiveKit: Token appears to be valid JWT format"); + } + + // First validate the RTC connection using Authorization header + let validate_url = format!( + "{}/rtc/validate?auto_subscribe=1&sdk=rust&version=0.7.36&protocol=16&adaptive_stream=1", + url.replace("wss://", "https://").replace("ws://", "http://"), + ); + + log!("LiveKit: Validating RTC connection: {}", validate_url); + + let client = reqwest::Client::new(); + match client.get(&validate_url) + .header("Authorization", format!("Bearer {}", token)) + .send() + .await + { + Ok(response) => { + let status = response.status(); + let body = response.text().await.unwrap_or_default(); + log!("LiveKit: RTC validation response: {} - {}", status, body); + + if !status.is_success() { + log!("LiveKit: RTC validation failed: {} - {}", status, body); + let _ = msg_tx.send(LiveKitMessage::Error(format!("RTC validation failed: {} - {}", status, body))); + SignalToUI::set_ui_signal(); + continue; + } + } + Err(e) => { + log!("LiveKit: RTC validation request failed: {}", e); + // Continue anyway - validation might not be required + } + } + + log!("LiveKit: Calling Room::connect with url={} and token length={}", url, token.len()); + match Room::connect(&url, &token, RoomOptions::default()).await { + Ok((r, mut room_events)) => { + let room_name = r.name().to_string(); + let room_sid = String::from(r.sid().await); + + log!("LiveKit: Connected to room: {} - {}", room_name, room_sid); + + let r = Arc::new(r); + room = Some(r.clone()); + + if let Ok(mut connected) = is_connected.lock() { + *connected = true; + } + let _ = msg_tx.send(LiveKitMessage::Connected); + SignalToUI::set_ui_signal(); + + // Spawn task to handle room events + let msg_tx_clone = msg_tx.clone(); + tokio::spawn(async move { + while let Some(event) = room_events.recv().await { + log!("LiveKit: Room event: {:?}", event); + match event { + RoomEvent::ParticipantConnected(participant) => { + log!("LiveKit: Participant connected: {}", participant.identity()); + let name = participant.name(); + let display_name = if name.is_empty() { + participant.identity().to_string() + } else { + name.to_string() + }; + let participant_info = CallParticipant { + user_id: participant.identity().to_string(), + display_name, + is_muted: false, + is_video_on: false, + is_speaking: false, + is_screen_sharing: false, + }; + let _ = msg_tx_clone.send(LiveKitMessage::ParticipantJoined(participant_info)); + SignalToUI::set_ui_signal(); + } + RoomEvent::ParticipantDisconnected(participant) => { + log!("LiveKit: Participant disconnected: {}", participant.identity()); + let _ = msg_tx_clone.send(LiveKitMessage::ParticipantLeft( + participant.identity().to_string() + )); + SignalToUI::set_ui_signal(); + } + RoomEvent::TrackSubscribed { track, publication: _, participant } => { + let participant_id = participant.identity().to_string(); + log!("LiveKit: Track subscribed from {}: kind={:?}", participant_id, track.kind()); + + if let RemoteTrack::Video(video_track) = track { + log!("LiveKit: Video track subscribed from {}", participant_id); + let _ = msg_tx_clone.send(LiveKitMessage::VideoTrackSubscribed { + participant_id: participant_id.clone(), + }); + SignalToUI::set_ui_signal(); + + // Start receiving video frames + let msg_tx_video = msg_tx_clone.clone(); + let participant_id_clone = participant_id.clone(); + let rtc_track = video_track.rtc_track(); + + tokio::spawn(async move { + let mut video_stream = NativeVideoStream::new(rtc_track); + let mut frame_count = 0u64; + let mut pts_counter = 0u64; + + log!("LiveKit: Starting video frame reception for {}", participant_id_clone); + + while let Some(frame) = video_stream.next().await { + frame_count += 1; + + // Convert to I420 buffer + let buffer = frame.buffer.to_i420(); + let width = buffer.width(); + let height = buffer.height(); + + // Log first frame and then periodically + if frame_count == 1 { + log!("LiveKit: Received first video frame from {}: {}x{}", participant_id_clone, width, height); + } else if frame_count % 60 == 0 { + log!("LiveKit: Video frame #{} from {}: {}x{}", frame_count, participant_id_clone, width, height); + } + + // Get I420 plane data + let (data_y, data_u, data_v) = buffer.data(); + + // Calculate presentation timestamp (30fps assumed) + pts_counter += 33; // ~33ms per frame at 30fps + + // Send I420 frame directly (let the UI handle conversion) + let _ = msg_tx_video.send(LiveKitMessage::RemoteVideoFrame { + participant_id: participant_id_clone.clone(), + y: data_y.to_vec(), + u: data_u.to_vec(), + v: data_v.to_vec(), + width, + height, + pts_ms: pts_counter, + }); + SignalToUI::set_ui_signal(); + } + log!("LiveKit: Video stream ended for {} after {} frames", participant_id_clone, frame_count); + }); + } + } + RoomEvent::TrackUnsubscribed { track, publication: _, participant } => { + let participant_id = participant.identity().to_string(); + log!("LiveKit: Track unsubscribed from {}: kind={:?}", participant_id, track.kind()); + + if matches!(track, RemoteTrack::Video(_)) { + let _ = msg_tx_clone.send(LiveKitMessage::VideoTrackUnsubscribed { + participant_id, + }); + SignalToUI::set_ui_signal(); + } + } + RoomEvent::Disconnected { reason } => { + log!("LiveKit: Room disconnected: {:?}", reason); + let _ = msg_tx_clone.send(LiveKitMessage::Disconnected); + SignalToUI::set_ui_signal(); + break; + } + _ => {} + } + } + }); + } + Err(e) => { + log!("LiveKit: Failed to connect: {}", e); + let _ = msg_tx.send(LiveKitMessage::Error(e.to_string())); + SignalToUI::set_ui_signal(); + } + } + } + LiveKitCommand::Disconnect => { + log!("LiveKit: Disconnecting"); + if let Some(r) = room.take() { + r.close().await.ok(); + } + if let Ok(mut connected) = is_connected.lock() { + *connected = false; + } + let _ = msg_tx.send(LiveKitMessage::Disconnected); + SignalToUI::set_ui_signal(); + } + LiveKitCommand::SetMicrophoneMuted(muted) => { + log!("LiveKit: Set microphone muted: {}", muted); + // Note: Muting requires publishing/unpublishing tracks or using LocalAudioTrack::set_enabled + // For now, just log the request. Full implementation requires track management. + if let Some(r) = &room { + let local = r.local_participant(); + for (_, publication) in local.track_publications().iter() { + if matches!(publication.kind(), TrackKind::Audio) { + log!("LiveKit: Audio track found, muted state: {}", publication.is_muted()); + // publication.mute() is async in newer versions + } + } + } + } + LiveKitCommand::SetCameraMuted(muted) => { + log!("LiveKit: Set camera muted: {}", muted); + // Note: Muting requires publishing/unpublishing tracks or using LocalVideoTrack::set_enabled + // For now, just log the request. Full implementation requires track management. + if let Some(r) = &room { + let local = r.local_participant(); + for (_, publication) in local.track_publications().iter() { + if matches!(publication.kind(), TrackKind::Video) { + log!("LiveKit: Video track found, muted state: {}", publication.is_muted()); + // publication.mute() is async in newer versions + } + } + } + } + LiveKitCommand::StartScreenShare => { + log!("LiveKit: Starting screen share"); + // Screen sharing requires platform-specific implementation + // For now, just log the request + } + LiveKitCommand::StopScreenShare => { + log!("LiveKit: Stopping screen share"); + } + LiveKitCommand::PublishVideoFrame(frame) => { + log!("LiveKit: Publishing video frame: {}x{}, format: {:?}, data_len: {}", + frame.width, frame.height, frame.format, frame.data.len()); + // Video frame publishing requires creating a video track source + // For now, this is a placeholder for future implementation + } + LiveKitCommand::PublishData { payload, reliable } => { + if let Some(r) = &room { + let data_packet = livekit::DataPacket { + payload: payload.into(), + reliable, + ..Default::default() + }; + + match r.local_participant().publish_data(data_packet).await { + Ok(_) => { + log!("LiveKit: Published data packet"); + } + Err(e) => { + log!("LiveKit: Failed to publish data: {}", e); + } + } + } else { + log!("LiveKit: Cannot publish data: not connected to room"); + } + } + } + } + } + + #[cfg(not(feature = "livekit"))] async fn run_event_loop( mut cmd_rx: mpsc::UnboundedReceiver, msg_tx: mpsc::UnboundedSender, @@ -113,13 +410,7 @@ impl LiveKitClient { let _ = msg_tx.send(LiveKitMessage::Connected); SignalToUI::set_ui_signal(); - // Note: In a real implementation, we would: - // 1. Connect to LiveKit using Room::connect(&url, &token, RoomOptions::default()) - // 2. Listen for RoomEvent::ParticipantConnected, TrackSubscribed, etc. - // 3. Extract I420 frames from video tracks using: - // let i420 = frame.buffer.to_i420(); - // let (data_y, data_u, data_v) = i420.data(); - log!("LiveKit (stub): Connection simulated. Enable 'livekit' crate for real WebRTC."); + log!("LiveKit (stub): Connection simulated. Enable 'livekit' feature for real WebRTC."); } LiveKitCommand::Disconnect => { log!("LiveKit (stub): Disconnecting"); @@ -144,6 +435,9 @@ impl LiveKitClient { LiveKitCommand::PublishVideoFrame(frame) => { log!("LiveKit (stub): Publishing video frame: {}x{}", frame.width, frame.height); } + LiveKitCommand::PublishData { payload, reliable } => { + log!("LiveKit (stub): Publishing data: {} bytes, reliable: {}", payload.len(), reliable); + } } } } @@ -173,6 +467,10 @@ impl LiveKitClient { pub fn publish_video_frame(&self, frame: VideoFrame) { self.send_command(LiveKitCommand::PublishVideoFrame(frame)); } + + pub fn publish_data(&self, payload: Vec, reliable: bool) { + self.send_command(LiveKitCommand::PublishData { payload, reliable }); + } } impl Default for LiveKitClient { diff --git a/src/voip/mod.rs b/src/voip/mod.rs index 4eb1762f4..d1b51bfd2 100644 --- a/src/voip/mod.rs +++ b/src/voip/mod.rs @@ -6,6 +6,7 @@ //! - LiveKit WebRTC integration //! - Speaking detection //! - Participants list +//! - Token caching for OpenID and LiveKit JWT use makepad_widgets::*; use makepad_widgets::makepad_platform::video::VideoInputsEvent; @@ -18,11 +19,13 @@ pub mod livekit_client; pub mod remote_video_session; pub mod speaking; pub mod participants_list; +pub mod token_cache; pub mod voip_screen; pub use voip_screen::VoipScreenWidgetRefExt; pub use participants_list::{Participant, ParticipantsListWidgetRefExt}; pub use camera::CameraChoice; +pub use token_cache::{CachedOpenIdToken, CachedLiveKitJwt, VoipTokenState}; /// Represents a call member from Matrix state events #[derive(Clone, Debug)] @@ -85,6 +88,16 @@ pub enum VoipAction { TestClearParticipants, /// Test action: Toggle participants sidebar TestToggleParticipantsSidebar, + /// Test action: Push a test video frame to a participant + TestPushVideoFrame { + participant_id: String, + }, + /// Test action: Start continuous test video frames to a participant + TestStartVideoStream { + participant_id: String, + }, + /// Test action: Stop continuous test video frames + TestStopVideoStream, #[default] None, } @@ -92,6 +105,7 @@ pub enum VoipAction { /// Global VoIP state stored in Makepad's Cx context. /// This allows camera permission and video inputs events to be captured /// at app startup before VoipScreen is shown. +/// Also stores cached tokens for OpenID and LiveKit JWT. #[derive(Default)] pub struct VoipGlobalState { /// Camera permission status (captured at app level) @@ -100,6 +114,10 @@ pub struct VoipGlobalState { pub camera_choice: Option, /// Whether video inputs have been requested pub video_inputs_requested: bool, + /// Cached OpenID token (valid for any room, tied to user session) + pub cached_openid_token: Option, + /// Cached LiveKit JWTs (per-room, since JWTs are room-specific) + pub cached_livekit_jwts: Vec, } impl VoipGlobalState { @@ -166,4 +184,96 @@ impl VoipGlobalState { None } } + + /// Store a cached OpenID token in global state + pub fn store_openid_token(cx: &mut Cx, token: CachedOpenIdToken) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: Storing OpenID token (expires in {} seconds)", token.remaining_seconds()); + state.cached_openid_token = Some(token); + } + } + + /// Get a valid cached OpenID token from global state + pub fn get_valid_openid_token(cx: &mut Cx) -> Option { + if cx.has_global::() { + let state = cx.get_global::(); + if let Some(ref token) = state.cached_openid_token { + if token.is_valid() { + log!("VoipGlobalState: Using cached OpenID token ({} seconds remaining)", token.remaining_seconds()); + return Some(token.clone()); + } else { + log!("VoipGlobalState: Cached OpenID token expired, clearing"); + state.cached_openid_token = None; + } + } + } + None + } + + /// Store a cached LiveKit JWT in global state + pub fn store_livekit_jwt(cx: &mut Cx, jwt: CachedLiveKitJwt) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: Storing LiveKit JWT for room {} (expires in {} seconds)", + jwt.room_id, jwt.remaining_seconds()); + // Remove any existing JWT for this room + state.cached_livekit_jwts.retain(|j| j.room_id != jwt.room_id); + // Add the new JWT + state.cached_livekit_jwts.push(jwt); + // Clean up expired JWTs + state.cached_livekit_jwts.retain(|j| j.is_valid()); + } + } + + /// Get a valid cached LiveKit JWT for the given room from global state + pub fn get_valid_livekit_jwt(cx: &mut Cx, room_id: &OwnedRoomId) -> Option { + if cx.has_global::() { + let state = cx.get_global::(); + // Clean up expired JWTs first + state.cached_livekit_jwts.retain(|j| j.is_valid()); + // Find a valid JWT for this room + if let Some(jwt) = state.cached_livekit_jwts.iter().find(|j| j.is_valid_for_room(room_id)) { + log!("VoipGlobalState: Using cached LiveKit JWT for room {} ({} seconds remaining)", + room_id, jwt.remaining_seconds()); + return Some(jwt.clone()); + } + } + None + } + + /// Get the token state for persistence (to be called when saving app state) + pub fn get_token_state(cx: &mut Cx) -> VoipTokenState { + if cx.has_global::() { + let state = cx.get_global::(); + VoipTokenState { + cached_openid_token: state.cached_openid_token.clone(), + cached_livekit_jwts: state.cached_livekit_jwts.clone(), + } + } else { + VoipTokenState::default() + } + } + + /// Restore token state from persistence (to be called when loading app state) + pub fn restore_token_state(cx: &mut Cx, token_state: VoipTokenState) { + if cx.has_global::() { + let state = cx.get_global::(); + // Only restore valid tokens + if let Some(ref token) = token_state.cached_openid_token { + if token.is_valid() { + log!("VoipGlobalState: Restoring cached OpenID token ({} seconds remaining)", token.remaining_seconds()); + state.cached_openid_token = Some(token.clone()); + } + } + // Restore valid JWTs + for jwt in token_state.cached_livekit_jwts { + if jwt.is_valid() { + log!("VoipGlobalState: Restoring cached LiveKit JWT for room {} ({} seconds remaining)", + jwt.room_id, jwt.remaining_seconds()); + state.cached_livekit_jwts.push(jwt); + } + } + } + } } diff --git a/src/voip/participants_list.rs b/src/voip/participants_list.rs index 177daf8e1..61558d5ea 100644 --- a/src/voip/participants_list.rs +++ b/src/voip/participants_list.rs @@ -1,7 +1,13 @@ //! Participants list widget for VoIP calls +//! +//! This module provides a participants list widget that displays remote +//! participants in a VoIP call, including their video feeds rendered as textures. +use std::collections::HashMap; use makepad_widgets::*; +use crate::shared::webrtc_video::{WebRtcVideoWidgetRefExt, WebRtcVideoFrame}; +/// Represents a remote participant in a VoIP call #[derive(Clone, Debug)] pub struct Participant { pub id: String, @@ -25,12 +31,23 @@ impl Default for Participant { } } +/// Internal state for a participant's video frame data +struct ParticipantVideoFrame { + /// RGBA pixel data + data: Vec, + width: u32, + height: u32, +} + #[derive(Script, ScriptHook, Widget)] pub struct ParticipantsList { #[deref] view: View, #[rust] participants: Vec, + /// Video frames for each participant, keyed by participant ID + #[rust] + video_frames: HashMap, } impl Widget for ParticipantsList { @@ -42,12 +59,41 @@ impl Widget for ParticipantsList { if let Some(widget) = list.item(cx, item_id, live_id!(ParticipantItem)) { widget.label(cx, ids!(avatar_letter)).set_text(cx, &participant.avatar_letter); widget.label(cx, ids!(name_label)).set_text(cx, &participant.name); - widget.label(cx, ids!(mute_icon)).set_text(cx, if participant.is_muted { "M" } else { "" }); + + // Update mute icon color based on mute status + let mut mute_btn = widget.button(cx, ids!(mute_icon)); + if participant.is_muted { + script_apply_eval!(cx, mute_btn, { + draw_icon +: { color: #e53935 } + }); + } else { + script_apply_eval!(cx, mute_btn, { + draw_icon +: { color: #aaa } + }); + } + widget.label(cx, ids!(status_label)).set_text(cx, if participant.is_speaking { "Speaking" } else { "" }); // Toggle video/avatar visibility based on is_video_on - widget.view(cx, ids!(participant_video_host)).set_visible(cx, participant.is_video_on); - widget.view(cx, ids!(avatar_container)).set_visible(cx, !participant.is_video_on); + let has_frame = self.video_frames.contains_key(&participant.id); + let has_video = participant.is_video_on && has_frame; + + let video_widget = widget.web_rtc_video(cx, ids!(participant_video)); + video_widget.set_visible(cx, has_video); + widget.view(cx, ids!(avatar_container)).set_visible(cx, !has_video); + + // If video is on and we have frame data, set it on the WebRtcVideo widget + if has_video { + if let Some(video_frame) = self.video_frames.get(&participant.id) { + let webrtc_frame = WebRtcVideoFrame { + data: video_frame.data.clone(), + width: video_frame.width, + height: video_frame.height, + participant_id: Some(participant.id.clone()), + }; + video_widget.set_frame(cx, webrtc_frame); + } + } widget.draw_all(cx, scope); } @@ -63,16 +109,20 @@ impl Widget for ParticipantsList { } impl ParticipantsList { + /// Add a new participant to the list pub fn add_participant(&mut self, cx: &mut Cx, participant: Participant) { self.participants.push(participant); self.redraw(cx); } + /// Remove a participant from the list pub fn remove_participant(&mut self, cx: &mut Cx, id: &str) { self.participants.retain(|p| p.id != id); + self.video_frames.remove(id); self.redraw(cx); } + /// Update a participant's properties pub fn update_participant(&mut self, cx: &mut Cx, id: &str, updater: impl FnOnce(&mut Participant)) { if let Some(participant) = self.participants.iter_mut().find(|p| p.id == id) { updater(participant); @@ -80,14 +130,136 @@ impl ParticipantsList { } } + /// Clear all participants from the list pub fn clear(&mut self, cx: &mut Cx) { self.participants.clear(); + // Don't clear video_frames - they will be reused when participants are re-added + // This preserves video streams across participant list refreshes self.redraw(cx); } + /// Clear all participants and their video frames + pub fn clear_all(&mut self, cx: &mut Cx) { + self.participants.clear(); + self.video_frames.clear(); + self.redraw(cx); + } + + /// Get a reference to the participants list pub fn participants(&self) -> &[Participant] { &self.participants } + + /// Push an I420 video frame to a participant's video + /// + /// This converts the I420 YUV data to RGBA and stores it for rendering. + /// + /// # Arguments + /// * `livekit_participant_id` - The LiveKit identity (may include session suffix) + /// * `y` - Y plane data + /// * `u` - U plane data + /// * `v` - V plane data + /// * `width` - Frame width in pixels + /// * `height` - Frame height in pixels + /// * `_pts_ms` - Presentation timestamp in milliseconds (currently unused) + pub fn push_video_frame( + &mut self, + cx: &mut Cx, + livekit_participant_id: &str, + y: Vec, + u: Vec, + v: Vec, + width: u32, + height: u32, + _pts_ms: u64, + ) { + // LiveKit identity format: "@user:server.tld:session_id" + // Matrix user_id format: "@user:server.tld" + // We need to match by user_id prefix since LiveKit adds session suffix + + // Find matching participant - try exact match first, then prefix match + let storage_key = self.participants.iter() + .find(|p| p.id == livekit_participant_id) + .or_else(|| self.participants.iter().find(|p| livekit_participant_id.starts_with(&p.id))) + .map(|p| p.id.clone()); + + let storage_key = match storage_key { + Some(key) => key, + None => { + // No matching participant found - store under LiveKit ID anyway + // (participant might be added later) + livekit_participant_id.to_string() + } + }; + + // Convert I420 YUV to RGBA + let rgba_data = i420_to_rgba(&y, &u, &v, width, height); + + // Store the RGBA frame data + self.video_frames.insert( + storage_key.clone(), + ParticipantVideoFrame { + data: rgba_data, + width, + height, + }, + ); + + // Mark participant as having video + if let Some(participant) = self.participants.iter_mut().find(|p| p.id == storage_key) { + participant.is_video_on = true; + } + + self.redraw(cx); + } + + /// Check if a participant has an active video frame + /// Checks both exact match and prefix match (for LiveKit session IDs) + pub fn has_video_frame(&self, participant_id: &str) -> bool { + // Exact match + if self.video_frames.contains_key(participant_id) { + return true; + } + // Check if any frame key starts with this participant_id + // (for when frame was stored under LiveKit ID before participant was matched) + self.video_frames.keys().any(|k| k.starts_with(participant_id)) + } + + /// Check if a participant has an active video texture (alias for has_video_frame) + pub fn has_video_texture(&self, participant_id: &str) -> bool { + self.has_video_frame(participant_id) + } +} + +/// Convert I420 YUV to RGBA +fn i420_to_rgba(y: &[u8], u: &[u8], v: &[u8], width: u32, height: u32) -> Vec { + let width = width as usize; + let height = height as usize; + let mut rgba = vec![0u8; width * height * 4]; + + for j in 0..height { + for i in 0..width { + let y_idx = j * width + i; + let uv_idx = (j / 2) * (width / 2) + (i / 2); + + let y_val = y[y_idx] as f32; + let u_val = u[uv_idx] as f32 - 128.0; + let v_val = v[uv_idx] as f32 - 128.0; + + // BT.601 YUV to RGB conversion + let r = (y_val + 1.402 * v_val).clamp(0.0, 255.0) as u8; + let g = (y_val - 0.344 * u_val - 0.714 * v_val).clamp(0.0, 255.0) as u8; + let b = (y_val + 1.772 * u_val).clamp(0.0, 255.0) as u8; + + let rgba_idx = (j * width + i) * 4; + rgba[rgba_idx] = r; + rgba[rgba_idx + 1] = g; + rgba[rgba_idx + 2] = b; + rgba[rgba_idx + 3] = 255; // Alpha + } + } + + rgba } impl ParticipantsListRef { @@ -115,6 +287,12 @@ impl ParticipantsListRef { } } + pub fn clear_all(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.clear_all(cx); + } + } + pub fn get_participants(&self) -> Vec { if let Some(inner) = self.borrow() { inner.participants.clone() @@ -122,4 +300,30 @@ impl ParticipantsListRef { Vec::new() } } + + /// Check if a participant has an active video texture + pub fn has_video_texture(&self, participant_id: &str) -> bool { + if let Some(inner) = self.borrow() { + inner.has_video_texture(participant_id) + } else { + false + } + } + + /// Push an I420 video frame to a participant's video texture + pub fn push_video_frame( + &self, + cx: &mut Cx, + participant_id: &str, + y: Vec, + u: Vec, + v: Vec, + width: u32, + height: u32, + pts_ms: u64, + ) { + if let Some(mut inner) = self.borrow_mut() { + inner.push_video_frame(cx, participant_id, y, u, v, width, height, pts_ms); + } + } } diff --git a/src/voip/token_cache.rs b/src/voip/token_cache.rs new file mode 100644 index 000000000..db16ec7ee --- /dev/null +++ b/src/voip/token_cache.rs @@ -0,0 +1,284 @@ +//! Token caching for VoIP authentication (OpenID and LiveKit JWT) +//! +//! This module provides caching mechanisms for OpenID tokens and LiveKit JWTs +//! to avoid unnecessary network requests. Tokens are cached with their expiration +//! times and validated before use. + +use serde::{Deserialize, Serialize}; +use matrix_sdk::ruma::OwnedRoomId; + +/// Cached OpenID token with fetch timestamp for reuse. +/// This structure is serializable for persistence across app restarts. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CachedOpenIdToken { + pub access_token: String, + pub token_type: String, + pub matrix_server_name: String, + /// Unix timestamp (seconds) when the token was fetched + pub fetched_at: u64, + /// Expiry duration in seconds (copied from response for convenience) + pub expires_in: u64, +} + +impl CachedOpenIdToken { + /// Create a new cached token with the current timestamp + pub fn new( + access_token: String, + token_type: String, + matrix_server_name: String, + expires_in: u64, + ) -> Self { + let fetched_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + Self { + access_token, + token_type, + matrix_server_name, + fetched_at, + expires_in, + } + } + + /// Check if the token is still valid with a safety margin (60 seconds) + pub fn is_valid(&self) -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + // Token is valid if current time is less than (fetched_at + expires_in - margin) + // Use a 60-second safety margin to avoid edge cases + let safety_margin = 60; + let expiry_time = self.fetched_at.saturating_add(self.expires_in); + + now < expiry_time.saturating_sub(safety_margin) + } + + /// Get the remaining validity time in seconds + pub fn remaining_seconds(&self) -> u64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + let expiry_time = self.fetched_at.saturating_add(self.expires_in); + expiry_time.saturating_sub(now) + } +} + +/// Cached LiveKit JWT with fetch timestamp for reuse. +/// JWT tokens typically expire after some time, we cache them to avoid unnecessary requests. +#[derive(Clone, Debug, Serialize, Deserialize)] +pub struct CachedLiveKitJwt { + pub jwt: String, + pub url: String, + /// The room ID this JWT is valid for + pub room_id: OwnedRoomId, + /// Unix timestamp (seconds) when the JWT was fetched + pub fetched_at: u64, + /// Expiration timestamp extracted from JWT (seconds since epoch) + pub expires_at: u64, +} + +impl CachedLiveKitJwt { + /// Create a new cached JWT with the current timestamp. + /// Attempts to extract expiration from JWT payload, falls back to 1 hour default. + pub fn new(jwt: String, url: String, room_id: OwnedRoomId) -> Self { + let fetched_at = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + // Try to extract expiration from JWT + let expires_at = Self::extract_jwt_expiration(&jwt) + .unwrap_or(fetched_at + 3600); // Default to 1 hour if extraction fails + + Self { + jwt, + url, + room_id, + fetched_at, + expires_at, + } + } + + /// Extract expiration timestamp from JWT payload. + /// JWT format: header.payload.signature (all base64 encoded) + fn extract_jwt_expiration(jwt: &str) -> Option { + let parts: Vec<&str> = jwt.split('.').collect(); + if parts.len() != 3 { + return None; + } + + // Decode payload (second part) - JWT uses base64url encoding + let payload_b64 = parts[1]; + // Add padding if needed + let padding = (4 - payload_b64.len() % 4) % 4; + let padded = format!("{}{}", payload_b64, "=".repeat(padding)); + // Replace URL-safe chars + let standard_b64: String = padded + .chars() + .map(|c| match c { + '-' => '+', + '_' => '/', + c => c, + }) + .collect(); + + // Decode base64 + let decoded = base64_decode(&standard_b64)?; + let payload_str = String::from_utf8(decoded).ok()?; + + // Parse JSON and extract "exp" field + // Simple parsing without full JSON library + if let Some(exp_start) = payload_str.find("\"exp\":") { + let after_exp = &payload_str[exp_start + 6..]; + let exp_str: String = after_exp + .chars() + .take_while(|c| c.is_ascii_digit()) + .collect(); + return exp_str.parse().ok(); + } + + None + } + + /// Check if the JWT is still valid with a safety margin (60 seconds) + pub fn is_valid(&self) -> bool { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + // Token is valid if current time is less than (expires_at - margin) + let safety_margin = 60; + now < self.expires_at.saturating_sub(safety_margin) + } + + /// Check if this JWT is valid for the given room + pub fn is_valid_for_room(&self, room_id: &OwnedRoomId) -> bool { + self.is_valid() && &self.room_id == room_id + } + + /// Get the remaining validity time in seconds + pub fn remaining_seconds(&self) -> u64 { + let now = std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + + self.expires_at.saturating_sub(now) + } +} + +/// Simple base64 decoder (no external dependency) +fn base64_decode(input: &str) -> Option> { + const CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + + fn char_to_val(c: u8) -> Option { + CHARS.iter().position(|&x| x == c).map(|p| p as u8) + } + + let input: Vec = input.bytes().filter(|&b| b != b'=').collect(); + let mut output = Vec::new(); + + for chunk in input.chunks(4) { + let vals: Vec = chunk.iter().filter_map(|&c| char_to_val(c)).collect(); + if vals.len() < 2 { + continue; + } + + output.push((vals[0] << 2) | (vals[1] >> 4)); + if vals.len() > 2 { + output.push((vals[1] << 4) | (vals[2] >> 2)); + } + if vals.len() > 3 { + output.push((vals[2] << 6) | vals[3]); + } + } + + Some(output) +} + +/// Persistent VoIP token state, stored in AppState for persistence across app restarts. +#[derive(Clone, Debug, Default, Serialize, Deserialize)] +pub struct VoipTokenState { + /// Cached OpenID token (valid for any room, tied to user session) + pub cached_openid_token: Option, + /// Cached LiveKit JWTs (per-room, since JWTs are room-specific) + pub cached_livekit_jwts: Vec, +} + +impl VoipTokenState { + /// Get a valid cached LiveKit JWT for the given room, if available + pub fn get_valid_jwt_for_room(&self, room_id: &OwnedRoomId) -> Option<&CachedLiveKitJwt> { + self.cached_livekit_jwts + .iter() + .find(|jwt| jwt.is_valid_for_room(room_id)) + } + + /// Store a new LiveKit JWT, replacing any existing one for the same room + pub fn store_jwt(&mut self, jwt: CachedLiveKitJwt) { + // Remove any existing JWT for this room + self.cached_livekit_jwts + .retain(|j| j.room_id != jwt.room_id); + // Add the new JWT + self.cached_livekit_jwts.push(jwt); + // Clean up expired JWTs + self.cleanup_expired(); + } + + /// Store a new OpenID token + pub fn store_openid_token(&mut self, token: CachedOpenIdToken) { + self.cached_openid_token = Some(token); + } + + /// Get a valid cached OpenID token, if available + pub fn get_valid_openid_token(&self) -> Option<&CachedOpenIdToken> { + self.cached_openid_token + .as_ref() + .filter(|t| t.is_valid()) + } + + /// Clean up expired tokens + pub fn cleanup_expired(&mut self) { + // Remove expired JWTs + self.cached_livekit_jwts.retain(|jwt| jwt.is_valid()); + // Clear expired OpenID token + if let Some(ref token) = self.cached_openid_token { + if !token.is_valid() { + self.cached_openid_token = None; + } + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_openid_token_validity() { + let token = CachedOpenIdToken::new( + "test_token".to_string(), + "Bearer".to_string(), + "matrix.org".to_string(), + 3600, // 1 hour + ); + assert!(token.is_valid()); + assert!(token.remaining_seconds() > 3500); + } + + #[test] + fn test_jwt_expiration_extraction() { + // This is a test JWT with exp claim + // Header: {"alg":"HS256","typ":"JWT"} + // Payload: {"exp":9999999999,"sub":"test"} + // Note: This is not a real JWT, just for testing the extraction + let test_jwt = "eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJleHAiOjk5OTk5OTk5OTksInN1YiI6InRlc3QifQ.signature"; + let exp = CachedLiveKitJwt::extract_jwt_expiration(test_jwt); + assert_eq!(exp, Some(9999999999)); + } +} diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs index e60df0a09..030d23a44 100644 --- a/src/voip/voip_screen.rs +++ b/src/voip/voip_screen.rs @@ -36,24 +36,17 @@ script_mod! { ParticipantItem := RoundedView { width: Fill - height: 120 - margin: Inset{bottom: 8} + height: 100 + margin: Inset{bottom: 4} draw_bg.color: #3a3a5a - draw_bg.radius: 8.0 + draw_bg.radius: 6.0 flow: Overlay // Video view (shown when video is on) - participant_video_host := View { + participant_video := WebRtcVideo { width: Fill height: Fill visible: false - - participant_video := Video { - width: Fill - height: Fill - autoplay: false - show_controls: false - } } // Avatar view (shown when video is off) @@ -63,15 +56,15 @@ script_mod! { align: Center avatar := RoundedView { - width: 48 - height: 48 + width: 40 + height: 40 draw_bg.color: #a0d0a0 - draw_bg.radius: 24.0 + draw_bg.radius: 20.0 align: Center avatar_letter := Label { text: "?" - draw_text.text_style.font_size: 20 + draw_text.text_style.font_size: 18 draw_text.color: #2a6a2a } } @@ -81,34 +74,45 @@ script_mod! { View { width: Fill height: Fill - align: Align{x: 0.0 y: 1.0} - padding: 8 + align: Align{x: 0.5 y: 1.0} + padding: 6 RoundedView { width: Fit height: Fit - padding: Inset{left: 8 right: 8 top: 4 bottom: 4} - draw_bg.color: #000000aa - draw_bg.radius: 4.0 + padding: Inset{left: 8 right: 10 top: 4 bottom: 4} + draw_bg.color: #1a1a3a + draw_bg.radius: 12.0 flow: Right - spacing: 6 + spacing: 4 + align: Center - mute_icon := Label { - text: "" - draw_text.text_style.font_size: 10 - draw_text.color: #fff + mute_icon := RobrixIconButton { + width: 14 + height: 14 + padding: 0 + draw_icon.svg: (ICON_MICROPHONE) + icon_walk: Walk{width: 12, height: 12} + draw_bg +: { + color: #00000000 + border_radius: 0.0 + } + draw_icon +: { + color: #aaa + } } name_label := Label { text: "Participant" draw_text.text_style.font_size: 10 - draw_text.color: #fff + draw_text.color: #ddd } status_label := Label { text: "" draw_text.text_style.font_size: 10 draw_text.color: #4CAF50 + visible: false } } } @@ -137,25 +141,20 @@ script_mod! { height: Fit padding: 16 flow: Right - spacing: 12 - - View { - width: Fit - height: Fit - flow: Down - spacing: 4 + spacing: 8 + align: Center - room_name := Label { - text: "Call Room" - draw_text.text_style.font_size: 18 - draw_text.color: #fff - } + room_name := Label { + text: "Call Room" + draw_text.text_style.font_size: 18 + draw_text.color: #ddd + } - call_status := Label { - text: "Not connected" - draw_text.text_style.font_size: 12 - draw_text.color: #888 - } + call_status := Label { + text: "Not connected" + draw_text.text_style.font_size: 12 + draw_text.color: #888 + margin: Inset{left: 4} } View { width: Fill height: 1 } @@ -164,6 +163,7 @@ script_mod! { text: "" draw_text.text_style.font_size: 14 draw_text.color: #888 + margin: Inset{right: 8} } participant_count := Label { @@ -182,19 +182,19 @@ script_mod! { // Participants list on the left participants_panel := View { - width: 220 + width: 200 height: Fill - padding: 12 + padding: 8 show_bg: true draw_bg.color: #1e1e3a flow: Down - spacing: 8 + spacing: 4 Label { text: "Participants" - draw_text.text_style.font_size: 14 - draw_text.color: #fff - margin: Inset{bottom: 8} + draw_text.text_style.font_size: 13 + draw_text.color: #aaa + margin: Inset{bottom: 4} } participants_list := mod.widgets.VoipParticipantsList {} @@ -252,32 +252,42 @@ script_mod! { } } - // Name badge overlay at bottom left + // Name badge overlay at bottom center (always visible) View { width: Fill height: Fill - align: Align{x: 0.0 y: 1.0} - padding: 16 + align: Align{x: 0.5 y: 1.0} + padding: 12 RoundedView { width: Fit height: Fit - padding: Inset{left: 12 right: 12 top: 6 bottom: 6} - draw_bg.color: #000000aa - draw_bg.radius: 6.0 + padding: Inset{left: 10 right: 14 top: 6 bottom: 6} + draw_bg.color: #1a1a3a + draw_bg.radius: 14.0 flow: Right spacing: 6 + align: Center - local_mute_icon := Label { - text: "" - draw_text.text_style.font_size: 14 - draw_text.color: #fff + local_mute_icon := RobrixIconButton { + width: 16 + height: 16 + padding: 0 + draw_icon.svg: (ICON_MICROPHONE) + icon_walk: Walk{width: 14, height: 14} + draw_bg +: { + color: #00000000 + border_radius: 0.0 + } + draw_icon +: { + color: #aaa + } } local_name_label := Label { text: "You" - draw_text.text_style.font_size: 14 - draw_text.color: #fff + draw_text.text_style.font_size: 12 + draw_text.color: #ddd } } } @@ -298,12 +308,82 @@ script_mod! { draw_bg.color: #2a2a4a draw_bg.radius: 24.0 flow: Right - spacing: 8 + spacing: 12 + + mic_button := RobrixIconButton { + width: 48 + height: 48 + padding: 10 + draw_icon.svg: (ICON_MICROPHONE) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #3a3a5a + border_radius: 24.0 + } + draw_icon +: { + color: #fff + } + } + + camera_button := RobrixIconButton { + width: 48 + height: 48 + padding: 10 + draw_icon.svg: (ICON_VIDEO) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #3a3a5a + border_radius: 24.0 + } + draw_icon +: { + color: #fff + } + } + + screenshare_button := RobrixIconButton { + width: 48 + height: 48 + padding: 10 + draw_icon.svg: (ICON_SQUARES) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #3a3a5a + border_radius: 24.0 + } + draw_icon +: { + color: #fff + } + } - mic_button := Button { text: "Mic" width: 60 } - camera_button := Button { text: "Cam" width: 60 } - screenshare_button := Button { text: "Share" width: 60 } - hangup_button := Button { text: "End" width: 60 } + participants_button := RobrixIconButton { + width: 48 + height: 48 + padding: 10 + draw_icon.svg: (ICON_ADD_USER) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #3a3a5a + border_radius: 24.0 + } + draw_icon +: { + color: #fff + } + } + + hangup_button := RobrixIconButton { + width: 48 + height: 48 + padding: 10 + draw_icon.svg: (ICON_CLOSE) + icon_walk: Walk{width: 24, height: 24} + draw_bg +: { + color: #e53935 + border_radius: 24.0 + } + draw_icon +: { + color: #fff + } + } } } } @@ -410,12 +490,13 @@ script_mod! { align: Align{x: 0.5, y: 0.7} join_call_button := Button { + padding: Inset{left: 24, right: 24, top: 12, bottom: 12} text: "Join call" - width: 100 - height: 48 + width: Fit + height: Fit draw_bg +: { color: #4CAF50 - border_radius: 20.0 + border_radius: 24.0 } draw_text +: { color: #fff @@ -561,6 +642,14 @@ pub struct VoipScreen { // Timer for refreshing call members from Matrix #[rust] call_members_refresh_timer: Timer, + + // Timer for updating call duration display + #[rust] call_duration_timer: Timer, + + // Test mode: timer for pushing test video frames + #[rust] test_video_frame_timer: Timer, + // Test mode: participant ID to push frames to + #[rust] test_video_participant_id: Option, } impl Widget for VoipScreen { @@ -605,6 +694,10 @@ impl Widget for VoipScreen { if self.video_publish_timer.is_event(event).is_some() { // Video publishing handled here if needed } + if self.call_duration_timer.is_event(event).is_some() { + // Update call duration display + self.update_call_duration(cx); + } if self.call_members_refresh_timer.is_event(event).is_some() { // Refresh call members from Matrix (only when in a call) if !self.in_lobby { @@ -613,6 +706,12 @@ impl Widget for VoipScreen { } } } + // Test mode: push video frames at ~30fps + if self.test_video_frame_timer.is_event(event).is_some() { + if let Some(ref participant_id) = self.test_video_participant_id.clone() { + self.push_test_video_frame(cx, &participant_id); + } + } } } @@ -648,6 +747,9 @@ impl Widget for VoipScreen { self.in_lobby = false; self.call_start_time = Some(Cx::time_now()); + // Start call duration timer (updates every second) + self.call_duration_timer = cx.start_interval(1.0); + // Stop lobby camera and prepare for call camera CameraManager::stop_lobby_camera(&self.view, cx); self.pending_call_camera_start = true; @@ -656,9 +758,26 @@ impl Widget for VoipScreen { // Fetch call members immediately after joining submit_async_request(MatrixRequest::GetCallMembers { room_id: room_id.clone() }); - // Start LiveKit connection flow: fetch OpenID token - log!("VoipScreen: Fetching OpenID token for LiveKit auth"); - submit_async_request(MatrixRequest::FetchOpenIdToken { room_id: room_id.clone() }); + // Start LiveKit connection flow with token caching + // Check if we have a valid cached LiveKit JWT for this room + if let Some(cached_jwt) = VoipGlobalState::get_valid_livekit_jwt(cx, room_id) { + log!("VoipScreen: Using cached LiveKit JWT ({} seconds remaining)", cached_jwt.remaining_seconds()); + self.connect_livekit(cx, &cached_jwt.url, &cached_jwt.jwt); + } else if let Some(cached_openid) = VoipGlobalState::get_valid_openid_token(cx) { + // Have valid OpenID token, skip to JWT fetch + log!("VoipScreen: Using cached OpenID token ({} seconds remaining), fetching LiveKit JWT", cached_openid.remaining_seconds()); + submit_async_request(MatrixRequest::FetchLiveKitJwt { + room_id: room_id.clone(), + access_token: cached_openid.access_token.clone(), + token_type: cached_openid.token_type.clone(), + matrix_server_name: cached_openid.matrix_server_name.clone(), + expires_in: cached_openid.expires_in, + }); + } else { + // No cached tokens, start fresh + log!("VoipScreen: No cached tokens, fetching OpenID token for LiveKit auth"); + submit_async_request(MatrixRequest::FetchOpenIdToken { room_id: room_id.clone() }); + } } else { log!("VoipScreen: Failed to send call member state"); self.call.connection_state = ConnectionState::Disconnected; @@ -672,6 +791,15 @@ impl Widget for VoipScreen { log!(" server_name: {}", matrix_server_name); log!(" expires_in: {} seconds", expires_in); + // Cache the OpenID token for future use + let cached_token = super::CachedOpenIdToken::new( + access_token.clone(), + token_type.clone(), + matrix_server_name.clone(), + *expires_in, + ); + VoipGlobalState::store_openid_token(cx, cached_token); + // Next step: fetch LiveKit JWT from SFU // POST https://livekit-jwt.call.matrix.org/sfu/get submit_async_request(MatrixRequest::FetchLiveKitJwt { @@ -687,6 +815,15 @@ impl Widget for VoipScreen { if self.room_id.as_ref() == Some(room_id) { log!("VoipScreen: LiveKit JWT fetched, connecting to LiveKit"); log!(" url: {}", url); + + // Cache the LiveKit JWT for future use + let cached_jwt = super::CachedLiveKitJwt::new( + jwt.clone(), + url.clone(), + room_id.clone(), + ); + VoipGlobalState::store_livekit_jwt(cx, cached_jwt); + self.connect_livekit(cx, url, jwt); } } @@ -730,6 +867,19 @@ impl Widget for VoipScreen { self.show_participants = !self.show_participants; self.update_ui(cx); } + VoipAction::TestPushVideoFrame { participant_id } => { + log!("VoipScreen: TestPushVideoFrame - participant_id={}", participant_id); + self.push_test_video_frame(cx, participant_id); + self.update_ui(cx); + } + VoipAction::TestStartVideoStream { participant_id } => { + log!("VoipScreen: TestStartVideoStream - participant_id={}", participant_id); + self.start_test_video_stream(cx, participant_id); + } + VoipAction::TestStopVideoStream => { + log!("VoipScreen: TestStopVideoStream"); + self.stop_test_video_stream(cx); + } _ => {} } } @@ -749,6 +899,7 @@ impl VoipScreen { self.in_lobby = true; self.lobby_mic_enabled = true; self.lobby_camera_enabled = true; + self.show_participants = true; self.call = Call::default(); self.speaking_detector = SpeakingDetector::new(); @@ -789,12 +940,28 @@ impl VoipScreen { self.room_id = Some(room_id.clone()); self.from_notification = true; // Show "Join Call" button - // Get room name from client + // Get room name and user display name from client if let Some(client) = get_client() { if let Some(room) = client.get_room(&room_id) { let room_name = room.name().unwrap_or_else(|| room_id.to_string()); self.view.label(cx, ids!(room_name)).set_text(cx, &room_name); } + + // Set local user's display name + if let Some(session) = client.session_meta() { + let user_id = session.user_id.to_string(); + // Extract display name from user_id (remove @ prefix and domain) + let display_name = user_id + .strip_prefix('@') + .and_then(|s| s.split(':').next()) + .unwrap_or(&user_id); + // Set name on local user badge + self.view.label(cx, ids!(local_name_label)).set_text(cx, display_name); + + // Also set avatar letter + let letter = display_name.chars().next().unwrap_or('?').to_uppercase().to_string(); + self.view.label(cx, ids!(local_avatar_letter)).set_text(cx, &letter); + } } self.update_ui(cx); @@ -831,7 +998,7 @@ impl VoipScreen { } /// Poll for LiveKit messages - fn poll_livekit_messages(&mut self, _cx: &mut Cx) -> bool { + fn poll_livekit_messages(&mut self, cx: &mut Cx) -> bool { let messages: Vec = if let Some(rx) = &mut self.livekit_rx { let mut msgs = Vec::new(); while let Ok(msg) = rx.try_recv() { @@ -849,6 +1016,8 @@ impl VoipScreen { self.call.connection_state = ConnectionState::Connected; self.in_lobby = false; self.call_start_time = Some(Cx::time_now()); + // Start call duration timer (updates every second) + self.call_duration_timer = cx.start_interval(1.0); log!("LiveKit connected"); needs_update = true; } @@ -858,25 +1027,93 @@ impl VoipScreen { needs_update = true; } LiveKitMessage::ParticipantJoined(p) => { + log!("Participant joined: {}", p.user_id); + + // Add to participants list UI + let name = if p.display_name.is_empty() { + p.user_id.clone() + } else { + p.display_name.clone() + }; + let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); + + let participant = Participant { + id: p.user_id.clone(), + name, + avatar_letter: letter, + is_muted: p.is_muted, + is_speaking: p.is_speaking, + is_video_on: p.is_video_on, + }; + + let list = self.view.participants_list(cx, ids!(participants_list)); + list.add_participant(cx, participant); + self.call.participants.insert(p.user_id.clone(), p); - log!("Participant joined"); needs_update = true; } LiveKitMessage::ParticipantLeft(id) => { + log!("Participant left: {}", id); + + // Remove from participants list UI + let list = self.view.participants_list(cx, ids!(participants_list)); + list.remove_participant(cx, &id); + self.call.participants.remove(&id); - log!("Participant left"); needs_update = true; } LiveKitMessage::Error(e) => { log!("LiveKit error: {}", e); + self.call.connection_state = ConnectionState::Disconnected; + needs_update = true; + } + LiveKitMessage::VideoTrackSubscribed { participant_id } => { + log!("Video track subscribed for participant: {}", participant_id); + + // Update participant's video state + let list = self.view.participants_list(cx, ids!(participants_list)); + list.update_participant(cx, &participant_id, |p| { + p.is_video_on = true; + }); + + if let Some(p) = self.call.participants.get_mut(&participant_id) { + p.is_video_on = true; + } + needs_update = true; + } + LiveKitMessage::VideoTrackUnsubscribed { participant_id } => { + log!("Video track unsubscribed for participant: {}", participant_id); + + // Update participant's video state + let list = self.view.participants_list(cx, ids!(participants_list)); + list.update_participant(cx, &participant_id, |p| { + p.is_video_on = false; + }); + + if let Some(p) = self.call.participants.get_mut(&participant_id) { + p.is_video_on = false; + } needs_update = true; } LiveKitMessage::RemoteVideoFrame { participant_id, y, u, v, width, height, pts_ms } => { - // TODO: Update participant's video texture with the I420 frame data - // This would use RemoteVideoSession to push frames to a Video widget - log!("Remote video frame from {}: {}x{} (Y:{} U:{} V:{} bytes) pts={}ms", - participant_id, width, height, y.len(), u.len(), v.len(), pts_ms); - // For now, just mark needs_update to trigger UI refresh + // Push the I420 frame to the participant's video session + // Only log periodically to avoid spam + if pts_ms % 1000 < 33 { + log!("Remote video frame from {}: {}x{} pts={}ms", participant_id, width, height, pts_ms); + } + + // Get the participants list and push the frame + let participants_list = self.view.participants_list(cx, ids!(participants_list)); + participants_list.push_video_frame( + cx, + &participant_id, + y, + u, + v, + width, + height, + pts_ms, + ); needs_update = true; } } @@ -951,6 +1188,8 @@ impl VoipScreen { self.call.connection_state = ConnectionState::Disconnected; self.in_lobby = true; self.call_start_time = None; + // Stop call duration timer + self.call_duration_timer = Timer::default(); CameraManager::stop_lobby_camera(&self.view, cx); self.pending_call_camera_start = true; self.camera_active = false; @@ -958,6 +1197,18 @@ impl VoipScreen { self.update_ui(cx); } + /// Update call duration display + fn update_call_duration(&mut self, cx: &mut Cx) { + if let Some(start) = self.call_start_time { + let elapsed = (Cx::time_now() - start) as u64; + let mins = elapsed / 60; + let secs = elapsed % 60; + self.view.label(cx, ids!(call_duration)) + .set_text(cx, &format!("{:02}:{:02}", mins, secs)); + self.redraw(cx); + } + } + /// Update UI to reflect current state fn update_ui(&mut self, cx: &mut Cx) { self.view.view(cx, ids!(lobby_view)).set_visible(cx, self.in_lobby); @@ -975,28 +1226,75 @@ impl VoipScreen { self.view.label(cx, ids!(participant_count)) .set_text(cx, &format!("{} participant{}", count, if count == 1 { "" } else { "s" })); - let mic_text = if self.call.local_audio_muted { "Muted" } else { "Mic" }; - let cam_text = if self.call.local_video_muted { "Cam Off" } else { "Cam" }; - let screen_text = if self.call.is_screen_sharing { "Stop" } else { "Share" }; + // Update call control icon button styles based on state + let mut mic_btn = self.view.button(cx, ids!(mic_button)); + let mut cam_btn = self.view.button(cx, ids!(camera_button)); + let mut screen_btn = self.view.button(cx, ids!(screenshare_button)); + let mut users_btn = self.view.button(cx, ids!(participants_button)); + + // Mic button - red when muted + if self.call.local_audio_muted { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { color: #e53935 } + }); + } else { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { color: #3a3a5a } + }); + } - self.view.button(cx, ids!(mic_button)).set_text(cx, mic_text); - self.view.button(cx, ids!(camera_button)).set_text(cx, cam_text); - self.view.button(cx, ids!(screenshare_button)).set_text(cx, screen_text); + // Camera button - red when off + if self.call.local_video_muted { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { color: #e53935 } + }); + } else { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { color: #3a3a5a } + }); + } - let mute_icon = if self.call.local_audio_muted { "M" } else { "" }; - self.view.label(cx, ids!(local_mute_icon)).set_text(cx, mute_icon); + // Screen share button - green when sharing + if self.call.is_screen_sharing { + script_apply_eval!(cx, screen_btn, { + draw_bg +: { color: #4CAF50 } + }); + } else { + script_apply_eval!(cx, screen_btn, { + draw_bg +: { color: #3a3a5a } + }); + } - // Participants panel is now always visible on the left (no toggle needed) - self.view.view(cx, ids!(debug_panel)).set_visible(cx, self.show_debug); + // Participants button - highlighted when panel is visible + if self.show_participants { + script_apply_eval!(cx, users_btn, { + draw_bg +: { color: #4a4a6a } + }); + } else { + script_apply_eval!(cx, users_btn, { + draw_bg +: { color: #3a3a5a } + }); + } - if let Some(start) = self.call_start_time { - let elapsed = (Cx::time_now() - start) as u64; - let mins = elapsed / 60; - let secs = elapsed % 60; - self.view.label(cx, ids!(call_duration)) - .set_text(cx, &format!("{:02}:{:02}", mins, secs)); + // Update local mute icon - red when muted, gray when unmuted + let mut local_mute_btn = self.view.button(cx, ids!(local_mute_icon)); + if self.call.local_audio_muted { + script_apply_eval!(cx, local_mute_btn, { + draw_icon +: { color: #e53935 } + }); + } else { + script_apply_eval!(cx, local_mute_btn, { + draw_icon +: { color: #aaa } + }); } + // Toggle participants panel visibility + self.view.view(cx, ids!(participants_panel)).set_visible(cx, self.show_participants); + self.view.view(cx, ids!(debug_panel)).set_visible(cx, self.show_debug); + + // Update call duration display (timer handles continuous updates) + self.update_call_duration(cx); + // Update lobby icon button styles based on state // When disabled, show different border color if self.in_lobby { @@ -1207,21 +1505,25 @@ impl VoipScreen { } /// Add a test participant (with optional video on) - pub fn add_participant(&mut self, cx: &mut Cx, name: &str, is_video_on: bool) { + /// Returns the participant ID for use with push_test_video_frame + pub fn add_participant(&mut self, cx: &mut Cx, name: &str, is_video_on: bool) -> String { self.participant_counter += 1; let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); + // Use a predictable ID format: "test_" for easy testing + let participant_id = format!("test_{}", name.to_lowercase().replace(' ', "_")); let participant = Participant { - id: format!("{}", self.participant_counter), + id: participant_id.clone(), name: name.to_string(), avatar_letter: letter, is_muted: false, is_speaking: false, is_video_on, }; - log!("Adding participant: {} (id={}, video={})", name, self.participant_counter, is_video_on); + log!("Adding participant: {} (id={}, video={})", name, participant_id, is_video_on); let list = self.view.participants_list(cx, ids!(participants_list)); list.add_participant(cx, participant); + participant_id } /// Toggle participant video state @@ -1241,20 +1543,103 @@ impl VoipScreen { list.remove_participant(cx, id); } - /// Clear all participants + /// Clear all participants and their video textures pub fn clear_participants(&mut self, cx: &mut Cx) { log!("Clearing all participants"); let list = self.view.participants_list(cx, ids!(participants_list)); - list.clear(cx); + list.clear_all(cx); // Use clear_all to also remove video textures self.participant_counter = 0; } + /// Start continuous test video frames to a participant (~30fps) + pub fn start_test_video_stream(&mut self, cx: &mut Cx, participant_id: &str) { + log!("Starting test video stream for participant: {}", participant_id); + self.test_video_participant_id = Some(participant_id.to_string()); + self.test_video_frame_timer = cx.start_interval(1.0 / 30.0); // ~30fps + + // Also ensure the participant has video enabled + let list = self.view.participants_list(cx, ids!(participants_list)); + list.update_participant(cx, participant_id, |p| { + p.is_video_on = true; + }); + } + + /// Stop continuous test video frames + pub fn stop_test_video_stream(&mut self, cx: &mut Cx) { + log!("Stopping test video stream"); + self.test_video_frame_timer = Timer::default(); // Stop the timer + self.test_video_participant_id = None; + self.redraw(cx); + } + + /// Push a test video frame to a participant for debugging + /// Generates a colored gradient pattern in I420 format + pub fn push_test_video_frame(&mut self, cx: &mut Cx, participant_id: &str) { + let width: u32 = 320; + let height: u32 = 240; + + // Generate I420 test pattern (colored gradient) + let y_size = (width * height) as usize; + let uv_size = ((width / 2) * (height / 2)) as usize; + + let mut y_plane = vec![0u8; y_size]; + let mut u_plane = vec![128u8; uv_size]; // Neutral U + let mut v_plane = vec![128u8; uv_size]; // Neutral V + + // Create a gradient pattern - Y varies horizontally, U/V create color + // Use time-based offset to animate the pattern + let time_offset = (Cx::time_now() * 100.0) as u32 % 256; + + for j in 0..height { + for i in 0..width { + let y_idx = (j * width + i) as usize; + // Luminance gradient (bright in center, dark at edges) + let cx_dist = ((i as i32 - width as i32 / 2).abs() as f32) / (width as f32 / 2.0); + let cy_dist = ((j as i32 - height as i32 / 2).abs() as f32) / (height as f32 / 2.0); + let dist = (cx_dist * cx_dist + cy_dist * cy_dist).sqrt().min(1.0); + let luma = ((1.0 - dist * 0.5) * 200.0 + time_offset as f32) as u8; + y_plane[y_idx] = luma.wrapping_add(((i + j) % 32) as u8); + } + } + + // Create color pattern in UV planes (blue-ish tint that shifts over time) + for j in 0..(height / 2) { + for i in 0..(width / 2) { + let uv_idx = (j * (width / 2) + i) as usize; + // U controls blue-yellow, V controls red-cyan + u_plane[uv_idx] = (128u8).wrapping_add((time_offset / 2) as u8).wrapping_add((i * 2) as u8); + v_plane[uv_idx] = (128u8).wrapping_sub((time_offset / 3) as u8).wrapping_add((j * 2) as u8); + } + } + + let pts_ms = (Cx::time_now() * 1000.0) as u64; + + log!("Pushing test video frame to participant {}: {}x{} pts={}ms", + participant_id, width, height, pts_ms); + + let list = self.view.participants_list(cx, ids!(participants_list)); + list.push_video_frame( + cx, + participant_id, + y_plane, + u_plane, + v_plane, + width, + height, + pts_ms, + ); + } + /// Update participants list from Matrix call member state events fn update_participants_from_call_members(&mut self, cx: &mut Cx, members: &[CallMember]) { log!("update_participants_from_call_members: received {} members", members.len()); - // Clear existing participants and rebuild from call members + // Get the participants list reference let list = self.view.participants_list(cx, ids!(participants_list)); + + // Clear existing participants but preserve video textures + // Video textures are keyed by participant ID (user_id) and will be matched + // when participants are re-added with the same IDs list.clear(cx); self.participant_counter = 0; @@ -1263,33 +1648,50 @@ impl VoipScreen { .and_then(|c| c.session_meta().map(|m| m.user_id.to_string())); log!("Current user ID: {:?}", current_user_id); + // Track added user_ids to avoid duplicates (multiple devices same user) + let mut added_user_ids = std::collections::HashSet::new(); + for member in members { // Skip self if current_user_id.as_ref() == Some(&member.user_id) { continue; } + // Skip if we already added this user (multiple devices) + if added_user_ids.contains(&member.user_id) { + log!("Skipping duplicate user: {} (device={})", member.user_id, member.device_id); + continue; + } + added_user_ids.insert(member.user_id.clone()); + self.participant_counter += 1; let name = member.display_name.clone() .unwrap_or_else(|| member.user_id.clone()); let letter = name.chars().next().unwrap_or('?').to_uppercase().to_string(); + // Use just user_id as the participant ID to match LiveKit identity format + // LiveKit identity is set from the JWT which uses the Matrix user_id + let participant_id = member.user_id.clone(); + + // Check if this participant already has video texture (from LiveKit video frames) + let has_video = list.has_video_texture(&participant_id); + let participant = Participant { - id: format!("{}_{}", member.user_id, member.device_id), + id: participant_id.clone(), name, avatar_letter: letter, is_muted: false, // We don't have this info from state events is_speaking: false, - is_video_on: false, // We don't have this info from state events + is_video_on: has_video, // Preserve video state from LiveKit }; - log!("Adding call member: {} (user={}, device={})", - participant.name, member.user_id, member.device_id); + log!("Adding call member: {} (id={}, video={})", + participant.name, participant_id, has_video); list.add_participant(cx, participant); } // Update participant count display - let count = members.len(); + let count = self.participant_counter; self.view.label(cx, ids!(participant_count)) .set_text(cx, &format!("{} participant{}", count, if count == 1 { "" } else { "s" })); @@ -1299,6 +1701,17 @@ impl VoipScreen { /// Connect to LiveKit with the given URL and JWT token fn connect_livekit(&mut self, cx: &mut Cx, url: &str, jwt: &str) { log!("connect_livekit: url={}", url); + log!("connect_livekit: jwt length={}, empty={}", jwt.len(), jwt.is_empty()); + if jwt.len() > 20 { + log!("connect_livekit: jwt starts with: {}", &jwt[..20]); + } + + if jwt.is_empty() { + log!("ERROR: JWT token is empty, cannot connect to LiveKit"); + self.call.connection_state = ConnectionState::Disconnected; + self.update_ui(cx); + return; + } if let Some(client) = &self.livekit_client { // Connect to LiveKit @@ -1338,10 +1751,12 @@ impl VoipScreenRef { } } - /// Add a participant - pub fn add_participant(&self, cx: &mut Cx, name: &str, is_video_on: bool) { + /// Add a participant, returns the participant ID + pub fn add_participant(&self, cx: &mut Cx, name: &str, is_video_on: bool) -> Option { if let Some(mut inner) = self.borrow_mut() { - inner.add_participant(cx, name, is_video_on); + Some(inner.add_participant(cx, name, is_video_on)) + } else { + None } } @@ -1371,4 +1786,25 @@ impl VoipScreenRef { inner.hangup(cx); } } + + /// Push a test video frame to a participant for debugging + pub fn push_test_video_frame(&self, cx: &mut Cx, participant_id: &str) { + if let Some(mut inner) = self.borrow_mut() { + inner.push_test_video_frame(cx, participant_id); + } + } + + /// Start continuous test video frames to a participant + pub fn start_test_video_stream(&self, cx: &mut Cx, participant_id: &str) { + if let Some(mut inner) = self.borrow_mut() { + inner.start_test_video_stream(cx, participant_id); + } + } + + /// Stop continuous test video frames + pub fn stop_test_video_stream(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.stop_test_video_stream(cx); + } + } } From 22ad6cd458ade214e52fa9bf4fecd5b2c00b67a5 Mon Sep 17 00:00:00 2001 From: alanpoon Date: Thu, 9 Apr 2026 22:35:39 +0800 Subject: [PATCH 05/21] added hangup when closing tab. --- src/home/main_desktop_ui.rs | 7 +++++++ src/voip/voip_screen.rs | 8 ++++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/src/home/main_desktop_ui.rs b/src/home/main_desktop_ui.rs index 1b829d837..0828ab631 100644 --- a/src/home/main_desktop_ui.rs +++ b/src/home/main_desktop_ui.rs @@ -474,6 +474,13 @@ impl WidgetMatchEvent for MainDesktopUI { should_save_dock_action = true; } DockAction::TabCloseWasPressed(tab_id) => { + // If closing a VoIP tab, call hangup on the VoipScreen first + if let Some(SelectedRoom::Voip { .. }) = self.open_rooms.get(&tab_id) { + log!("MainDesktopUI: Closing VoIP tab via dock X button, calling hangup"); + let dock = self.view.dock(cx, ids!(dock)); + let widget = dock.item(tab_id); + widget.as_voip_screen().hangup(cx); + } self.tab_to_close = Some(tab_id); self.close_tab(cx, tab_id); self.redraw(cx); diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs index 030d23a44..02bd4aa02 100644 --- a/src/voip/voip_screen.rs +++ b/src/voip/voip_screen.rs @@ -147,7 +147,7 @@ script_mod! { room_name := Label { text: "Call Room" draw_text.text_style.font_size: 18 - draw_text.color: #ddd + draw_text.color: #888 } call_status := Label { @@ -184,16 +184,16 @@ script_mod! { participants_panel := View { width: 200 height: Fill - padding: 8 + padding: 0 show_bg: true draw_bg.color: #1e1e3a flow: Down - spacing: 4 + spacing: 0 Label { text: "Participants" draw_text.text_style.font_size: 13 - draw_text.color: #aaa + draw_text.color: #888 margin: Inset{bottom: 4} } From 32f92b930a48e6cfed1d85724b9071ad749f7e31 Mon Sep 17 00:00:00 2001 From: alanpoon Date: Thu, 9 Apr 2026 23:36:22 +0800 Subject: [PATCH 06/21] add pip --- src/app.rs | 50 +++- src/home/main_desktop_ui.rs | 27 +- src/voip/mod.rs | 86 ++++++ src/voip/pip_overlay.rs | 515 ++++++++++++++++++++++++++++++++++++ src/voip/voip_screen.rs | 211 ++++++++++++++- 5 files changed, 869 insertions(+), 20 deletions(-) create mode 100644 src/voip/pip_overlay.rs diff --git a/src/app.rs b/src/app.rs index 26a5a8b54..55ca2182c 100644 --- a/src/app.rs +++ b/src/app.rs @@ -16,7 +16,7 @@ use crate::{ VerificationModalAction, VerificationModalWidgetRefExt, }, - voip::{VoipGlobalState, VoipAction}, + voip::{VoipGlobalState, VoipAction, PipVoipOverlayWidgetRefExt}, }; script_mod! { @@ -150,6 +150,9 @@ script_mod! { } } + // PiP overlay for VoIP calls (shown when switching away from active call) + pip_voip_overlay := PipVoipOverlay {} + PopupList {} // Tooltips must be shown in front of all other UI elements, @@ -402,15 +405,41 @@ impl MatchEvent for App { _ => {} } - // Handle VoIP close action - reset the VoIP visibility on the mobile RoomScreen - // if let Some(VoipAction::Close(_room_id)) = action.downcast_ref() { - // log!("App: VoipAction::Close received, resetting VoIP visibility"); - // // Reset VoIP visibility on room_screen_0 (mobile path) - // let room_screen = self.ui.room_screen(cx, ids!(room_screen_0)); - // room_screen.set_voip_visible(cx, false, None); - // self.ui.redraw(cx); - // continue; - // } + // Handle VoIP PiP overlay actions + match action.downcast_ref() { + Some(VoipAction::ShowPip { room_id }) => { + log!("App: VoipAction::ShowPip received for room {}", room_id); + self.ui.pip_voip_overlay(cx, ids!(pip_voip_overlay)).show(cx, room_id.clone()); + continue; + } + Some(VoipAction::HidePip) => { + log!("App: VoipAction::HidePip received"); + self.ui.pip_voip_overlay(cx, ids!(pip_voip_overlay)).hide(cx); + continue; + } + Some(VoipAction::ReturnToVoipTab { room_id }) => { + log!("App: VoipAction::ReturnToVoipTab received for room {}", room_id); + // Hide the PiP overlay + self.ui.pip_voip_overlay(cx, ids!(pip_voip_overlay)).hide(cx); + // Navigate back to the VoIP tab by emitting a RoomsListAction::Selected + // We need to look up the room name from RoomsList + if let Some(room_name_id) = cx.get_global::().get_room_name(room_id) { + cx.widget_action( + self.ui.widget_uid(), + RoomsListAction::Selected(SelectedRoom::Voip { room_name_id }), + ); + } + self.ui.redraw(cx); + continue; + } + Some(VoipAction::PipHangup { room_id }) => { + log!("App: VoipAction::PipHangup received for room {}", room_id); + // Hide the PiP overlay - the VoipScreen will handle the actual hangup + self.ui.pip_voip_overlay(cx, ids!(pip_voip_overlay)).hide(cx); + // The action will continue to propagate to VoipScreen + } + _ => {} + } // When a stack navigation pop is initiated (back button pressed), // pop the mobile nav stack so it stays in sync with StackNavigation. @@ -784,6 +813,7 @@ impl AppMain for App { crate::verification_modal::script_mod(vm); crate::profile::script_mod(vm); crate::voip::voip_screen::script_mod(vm); + crate::voip::pip_overlay::script_mod(vm); crate::home::script_mod(vm); crate::login::script_mod(vm); crate::logout::script_mod(vm); diff --git a/src/home/main_desktop_ui.rs b/src/home/main_desktop_ui.rs index 0828ab631..ff6294257 100644 --- a/src/home/main_desktop_ui.rs +++ b/src/home/main_desktop_ui.rs @@ -3,7 +3,7 @@ use ruma::OwnedRoomId; use tokio::sync::Notify; use std::{collections::HashMap, sync::Arc}; -use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, utils::RoomNameId, voip::{voip_screen::VoipScreenWidgetRefExt, VoipAction}}; +use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, utils::RoomNameId, voip::{voip_screen::VoipScreenWidgetRefExt, VoipAction, VoipGlobalState}}; use super::{invite_screen::InviteScreenWidgetRefExt, room_screen::RoomScreenWidgetRefExt, rooms_list::RoomsListAction}; script_mod! { @@ -463,6 +463,25 @@ impl WidgetMatchEvent for MainDesktopUI { match widget_action.cast() { // Whenever a tab (except for the home_tab) is pressed, notify the app state. DockAction::TabWasPressed(tab_id) => { + // Check if we're switching FROM a VoIP tab with active call -> show PiP + // Or if we're switching TO a VoIP tab -> hide PiP + let prev_room = self.most_recently_selected_room.clone(); + let new_room = self.open_rooms.get(&tab_id).cloned(); + + // Detect switch TO VoIP tab -> hide PiP + if let Some(SelectedRoom::Voip { .. }) = &new_room { + log!("MainDesktopUI: Switching TO VoIP tab, hiding PiP"); + cx.action(VoipAction::HidePip); + } + // Detect switch FROM VoIP tab with active call -> show PiP + else if let Some(SelectedRoom::Voip { room_name_id }) = &prev_room { + let room_id = room_name_id.room_id(); + if VoipGlobalState::is_call_active(cx, room_id) { + log!("MainDesktopUI: Switching FROM VoIP tab with active call, showing PiP"); + cx.action(VoipAction::ShowPip { room_id: room_id.clone() }); + } + } + if tab_id == id!(home_tab) { cx.action(AppStateAction::FocusNone); self.most_recently_selected_room = None; @@ -474,12 +493,14 @@ impl WidgetMatchEvent for MainDesktopUI { should_save_dock_action = true; } DockAction::TabCloseWasPressed(tab_id) => { - // If closing a VoIP tab, call hangup on the VoipScreen first + // If closing a VoIP tab, call hangup on the VoipScreen first and hide PiP if let Some(SelectedRoom::Voip { .. }) = self.open_rooms.get(&tab_id) { - log!("MainDesktopUI: Closing VoIP tab via dock X button, calling hangup"); + log!("MainDesktopUI: Closing VoIP tab via dock X button, calling hangup and hiding PiP"); let dock = self.view.dock(cx, ids!(dock)); let widget = dock.item(tab_id); widget.as_voip_screen().hangup(cx); + // Hide PiP when VoIP tab is closed + cx.action(VoipAction::HidePip); } self.tab_to_close = Some(tab_id); self.close_tab(cx, tab_id); diff --git a/src/voip/mod.rs b/src/voip/mod.rs index d1b51bfd2..8be52ce01 100644 --- a/src/voip/mod.rs +++ b/src/voip/mod.rs @@ -16,6 +16,7 @@ use matrix_sdk::ruma::OwnedRoomId; pub mod call_state; pub mod camera; pub mod livekit_client; +pub mod pip_overlay; pub mod remote_video_session; pub mod speaking; pub mod participants_list; @@ -26,6 +27,7 @@ pub use voip_screen::VoipScreenWidgetRefExt; pub use participants_list::{Participant, ParticipantsListWidgetRefExt}; pub use camera::CameraChoice; pub use token_cache::{CachedOpenIdToken, CachedLiveKitJwt, VoipTokenState}; +pub use pip_overlay::PipVoipOverlayWidgetRefExt; /// Represents a call member from Matrix state events #[derive(Clone, Debug)] @@ -35,6 +37,35 @@ pub struct CallMember { pub display_name: Option, } +/// Information about a participant in an active call (for PiP display) +#[derive(Clone, Debug, Default)] +pub struct ParticipantInfo { + pub user_id: String, + pub display_name: String, + pub avatar_letter: String, +} + +/// State of an active VoIP call (stored in VoipGlobalState for PiP access) +#[derive(Clone, Debug, Default)] +pub struct ActiveCallState { + /// The room ID where the call is happening + pub room_id: Option, + /// Connection state description + pub status_text: String, + /// Whether we are in lobby or in an active call + pub in_call: bool, + /// Whether the local microphone is muted + pub mic_muted: bool, + /// Whether the local camera is off + pub camera_muted: bool, + /// Whether screen sharing is active + pub screen_sharing: bool, + /// Information about the local participant + pub local_participant: ParticipantInfo, + /// Number of remote participants + pub participant_count: usize, +} + /// Actions emitted by VoIP screens #[derive(Clone, Debug, Default)] pub enum VoipAction { @@ -98,6 +129,20 @@ pub enum VoipAction { }, /// Test action: Stop continuous test video frames TestStopVideoStream, + /// Show the PiP overlay for an active call + ShowPip { room_id: OwnedRoomId }, + /// Hide the PiP overlay + HidePip, + /// Toggle microphone from PiP + PipMicToggle { room_id: OwnedRoomId }, + /// Toggle camera from PiP + PipCameraToggle { room_id: OwnedRoomId }, + /// Toggle screen share from PiP + PipScreenShareToggle { room_id: OwnedRoomId }, + /// Hangup from PiP + PipHangup { room_id: OwnedRoomId }, + /// Return to the VoIP tab from clicking on PiP + ReturnToVoipTab { room_id: OwnedRoomId }, #[default] None, } @@ -118,6 +163,8 @@ pub struct VoipGlobalState { pub cached_openid_token: Option, /// Cached LiveKit JWTs (per-room, since JWTs are room-specific) pub cached_livekit_jwts: Vec, + /// Active call state for PiP overlay display + pub active_call: Option, } impl VoipGlobalState { @@ -276,4 +323,43 @@ impl VoipGlobalState { } } } + + /// Check if there is an active call for the given room + pub fn is_call_active(cx: &mut Cx, room_id: &OwnedRoomId) -> bool { + if cx.has_global::() { + let state = cx.get_global::(); + if let Some(ref active) = state.active_call { + return active.in_call && active.room_id.as_ref() == Some(room_id); + } + } + false + } + + /// Update the active call state + pub fn update_active_call(cx: &mut Cx, call: ActiveCallState) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: Updating active call state for room {:?}, in_call={}", + call.room_id, call.in_call); + state.active_call = Some(call); + } + } + + /// Clear the active call state + pub fn clear_active_call(cx: &mut Cx) { + if cx.has_global::() { + let state = cx.get_global::(); + log!("VoipGlobalState: Clearing active call state"); + state.active_call = None; + } + } + + /// Get the active call state + pub fn get_active_call(cx: &mut Cx) -> Option { + if cx.has_global::() { + let state = cx.get_global::(); + return state.active_call.clone(); + } + None + } } diff --git a/src/voip/pip_overlay.rs b/src/voip/pip_overlay.rs new file mode 100644 index 000000000..cc355ca16 --- /dev/null +++ b/src/voip/pip_overlay.rs @@ -0,0 +1,515 @@ +//! Picture-in-Picture (PiP) overlay for VoIP calls +//! +//! This module provides a floating PiP window that appears when the user switches +//! to a different room tab during an active VoIP call. It shows participant info, +//! call status, and control buttons. + +use makepad_widgets::*; +use makepad_widgets::video::VideoCameraPreviewMode; +use matrix_sdk::ruma::OwnedRoomId; +use super::{VoipGlobalState, VoipAction, CameraChoice}; + +script_mod! { + use mod.prelude.widgets.* + use mod.widgets.* + + mod.widgets.PipVoipOverlay = #(PipVoipOverlay::register_widget(vm)) { + width: Fit + height: Fit + flow: Overlay + visible: false + + // Position in top-right corner + margin: Inset { top: 60, right: 16, left: 0, bottom: 0 } + align: Align { x: 1.0, y: 0.0 } + + // Main container with click area + pip_container := RoundedView { + width: 280 + height: Fit + padding: 0 + draw_bg.color: #2a2a4a + draw_bg.radius: 12.0 + flow: Down + spacing: 0 + + // Video preview area + pip_video_container := View { + width: Fill + height: 160 + flow: Overlay + + // Avatar placeholder (shown when camera is off) + pip_avatar_view := View { + width: Fill + height: Fill + align: Center + show_bg: true + draw_bg.color: #1a1a2e + + RoundedView { + width: 60 + height: 60 + draw_bg.color: #a0d0a0 + draw_bg.radius: 30.0 + align: Center + + pip_avatar_letter := Label { + text: "?" + draw_text.text_style.font_size: 24 + draw_text.color: #2a6a2a + } + } + } + + // Camera video (shown when camera is on) + pip_video_host := View { + width: Fill + height: Fill + visible: false + + pip_camera_video := Video { + width: Fill + height: Fill + autoplay: false + show_controls: false + } + } + + // Back button overlay at top-left + View { + width: Fill + height: Fill + align: Align { x: 0.0, y: 0.0 } + padding: 8 + + pip_back_button := RobrixIconButton { + width: 32 + height: 32 + padding: 6 + draw_icon.svg: (ICON_JUMP) + icon_walk: Walk { width: 16, height: 16 } + draw_bg +: { + color: #1a1a3aCC + border_radius: 16.0 + } + draw_icon +: { + color: #fff + } + } + } + + // Name badge overlay at bottom + View { + width: Fill + height: Fill + align: Align { x: 0.5, y: 1.0 } + padding: 8 + + RoundedView { + width: Fit + height: Fit + padding: Inset { left: 8, right: 10, top: 4, bottom: 4 } + draw_bg.color: #1a1a3a + draw_bg.radius: 10.0 + flow: Right + spacing: 4 + align: Center + + participant_name := Label { + text: "User" + draw_text.text_style.font_size: 11 + draw_text.color: #ddd + } + + status_label := Label { + text: "" + draw_text.text_style.font_size: 10 + draw_text.color: #aaa + margin: Inset { left: 4 } + } + } + } + } + + // Control buttons row + controls_row := View { + width: Fill + height: Fit + flow: Right + spacing: 8 + align: Center + padding: 10 + + pip_mic_button := RobrixIconButton { + width: 36 + height: 36 + padding: 6 + draw_icon.svg: (ICON_MICROPHONE) + icon_walk: Walk { width: 18, height: 18 } + draw_bg +: { + color: #3a3a5a + border_radius: 18.0 + } + draw_icon +: { + color: #fff + } + } + + pip_camera_button := RobrixIconButton { + width: 36 + height: 36 + padding: 6 + draw_icon.svg: (ICON_VIDEO) + icon_walk: Walk { width: 18, height: 18 } + draw_bg +: { + color: #3a3a5a + border_radius: 18.0 + } + draw_icon +: { + color: #fff + } + } + + pip_screenshare_button := RobrixIconButton { + width: 36 + height: 36 + padding: 6 + draw_icon.svg: (ICON_SQUARES) + icon_walk: Walk { width: 18, height: 18 } + draw_bg +: { + color: #3a3a5a + border_radius: 18.0 + } + draw_icon +: { + color: #fff + } + } + + pip_hangup_button := RobrixIconButton { + width: 36 + height: 36 + padding: 6 + draw_icon.svg: (ICON_CLOSE) + icon_walk: Walk { width: 18, height: 18 } + draw_bg +: { + color: #e53935 + border_radius: 18.0 + } + draw_icon +: { + color: #fff + } + } + } + } + } +} + +/// PiP overlay widget for VoIP calls +#[derive(Script, ScriptHook, Widget)] +pub struct PipVoipOverlay { + #[deref] + view: View, + + /// The room ID of the active call being displayed + #[rust] + room_id: Option, + + /// Whether the PiP is currently visible + #[rust] + is_visible: bool, + + /// Whether the camera is active in PiP + #[rust] + camera_active: bool, + + /// Stored camera choice for starting camera + #[rust] + camera_choice: Option, +} + +impl Widget for PipVoipOverlay { + fn handle_event(&mut self, cx: &mut Cx, event: &Event, scope: &mut Scope) { + self.view.handle_event(cx, event, scope); + + // Handle video events + match event { + Event::VideoPlaybackPrepared(_) => { + if self.is_visible && self.camera_active { + self.show_video(cx); + } + } + Event::VideoTextureUpdated(_) => { + if self.is_visible && self.camera_active { + self.show_video(cx); + } + } + _ => {} + } + + if !self.is_visible { + return; + } + + // Handle button clicks + if let Event::Actions(actions) = event { + self.handle_actions(cx, actions); + } + + // Handle click anywhere on the PiP to return to VoIP tab + if let Hit::FingerUp(fe) = event.hits(cx, self.view.area()) { + if fe.was_tap() { + // Check if the click was NOT on a button (buttons handle their own clicks) + let back_area = self.view.button(cx, ids!(pip_back_button)).area(); + let mic_area = self.view.button(cx, ids!(pip_mic_button)).area(); + let cam_area = self.view.button(cx, ids!(pip_camera_button)).area(); + let share_area = self.view.button(cx, ids!(pip_screenshare_button)).area(); + let hangup_area = self.view.button(cx, ids!(pip_hangup_button)).area(); + + let click_pos = fe.abs; + let on_button = back_area.rect(cx).contains(click_pos) + || mic_area.rect(cx).contains(click_pos) + || cam_area.rect(cx).contains(click_pos) + || share_area.rect(cx).contains(click_pos) + || hangup_area.rect(cx).contains(click_pos); + + if !on_button { + if let Some(room_id) = self.room_id.clone() { + log!("PipVoipOverlay: Clicked on PiP, returning to VoIP tab"); + cx.action(VoipAction::ReturnToVoipTab { room_id }); + } + } + } + } + } + + fn draw_walk(&mut self, cx: &mut Cx2d, scope: &mut Scope, walk: Walk) -> DrawStep { + if self.is_visible { + // Update the display from global state before drawing + self.update_from_global_state(cx); + } + self.view.draw_walk(cx, scope, walk) + } +} + +impl PipVoipOverlay { + /// Show the PiP overlay for the given room + pub fn show(&mut self, cx: &mut Cx, room_id: OwnedRoomId) { + log!("PipVoipOverlay: Showing for room {}", room_id); + self.room_id = Some(room_id); + self.is_visible = true; + self.view.set_visible(cx, true); + self.update_from_global_state(cx); + + // Start camera if available and not muted + if let Some(active_call) = VoipGlobalState::get_active_call(cx) { + if !active_call.camera_muted { + self.start_camera(cx); + } + } + + self.redraw(cx); + } + + /// Hide the PiP overlay + pub fn hide(&mut self, cx: &mut Cx) { + log!("PipVoipOverlay: Hiding"); + self.is_visible = false; + + // Stop camera before hiding + self.stop_camera(cx); + + self.view.set_visible(cx, false); + self.redraw(cx); + } + + /// Start the camera in PiP + fn start_camera(&mut self, cx: &mut Cx) { + // Get camera choice from global state + let Some(choice) = VoipGlobalState::get_camera_choice(cx) else { + log!("PipVoipOverlay: No camera choice available"); + return; + }; + + let video = self.view.video(cx, &[live_id!(pip_camera_video)]); + + if !video.is_unprepared() { + log!("PipVoipOverlay: Camera already running or preparing"); + return; + } + + log!("PipVoipOverlay: Starting camera: {} ({}x{} {:?})", + choice.name, choice.width, choice.height, choice.pixel_format); + + self.camera_choice = Some(choice.clone()); + self.camera_active = true; + + video.set_camera_preview_mode(cx, VideoCameraPreviewMode::Native); + video.set_source_camera(cx, choice.input_id, choice.format_id); + video.begin_playback(cx); + } + + /// Stop the camera in PiP + fn stop_camera(&mut self, cx: &mut Cx) { + if !self.camera_active { + return; + } + + log!("PipVoipOverlay: Stopping camera"); + let video = self.view.video(cx, &[live_id!(pip_camera_video)]); + if !video.is_unprepared() && !video.is_cleaning_up() { + video.stop_and_cleanup_resources(cx); + } + + self.view.view(cx, ids!(pip_video_host)).set_visible(cx, false); + self.view.view(cx, ids!(pip_avatar_view)).set_visible(cx, true); + self.camera_active = false; + } + + /// Show video view + fn show_video(&mut self, cx: &mut Cx) { + self.view.view(cx, ids!(pip_video_host)).set_visible(cx, true); + self.view.view(cx, ids!(pip_avatar_view)).set_visible(cx, false); + } + + /// Update the display from global VoIP state + fn update_from_global_state(&mut self, cx: &mut Cx) { + if let Some(active_call) = VoipGlobalState::get_active_call(cx) { + // Update participant info + self.view.label(cx, ids!(pip_avatar_letter)) + .set_text(cx, &active_call.local_participant.avatar_letter); + self.view.label(cx, ids!(participant_name)) + .set_text(cx, &active_call.local_participant.display_name); + self.view.label(cx, ids!(status_label)) + .set_text(cx, &format!(" - {}", active_call.status_text)); + + // Update button styles based on state + let mut mic_btn = self.view.button(cx, ids!(pip_mic_button)); + let mut cam_btn = self.view.button(cx, ids!(pip_camera_button)); + let mut share_btn = self.view.button(cx, ids!(pip_screenshare_button)); + + // Mic button - red when muted + if active_call.mic_muted { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { color: #e53935 } + }); + } else { + script_apply_eval!(cx, mic_btn, { + draw_bg +: { color: #3a3a5a } + }); + } + + // Camera button - red when off + if active_call.camera_muted { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { color: #e53935 } + }); + // Hide video, show avatar when camera is muted + if self.camera_active { + self.stop_camera(cx); + } + } else { + script_apply_eval!(cx, cam_btn, { + draw_bg +: { color: #3a3a5a } + }); + // Start camera if not already active + if !self.camera_active && self.is_visible { + self.start_camera(cx); + } + } + + // Screen share button - green when sharing + if active_call.screen_sharing { + script_apply_eval!(cx, share_btn, { + draw_bg +: { color: #4CAF50 } + }); + } else { + script_apply_eval!(cx, share_btn, { + draw_bg +: { color: #3a3a5a } + }); + } + + // If the call ended, hide the PiP + if !active_call.in_call { + self.hide(cx); + } + } else { + // No active call, hide PiP + self.hide(cx); + } + } + + /// Handle UI actions (button clicks) + fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { + let Some(room_id) = self.room_id.clone() else { + return; + }; + + // Back button - return to VoIP tab + if self.view.button(cx, ids!(pip_back_button)).clicked(actions) { + log!("PipVoipOverlay: Back button clicked, returning to VoIP tab"); + cx.action(VoipAction::ReturnToVoipTab { room_id: room_id.clone() }); + return; + } + + // Mic button + if self.view.button(cx, ids!(pip_mic_button)).clicked(actions) { + log!("PipVoipOverlay: Mic button clicked"); + cx.action(VoipAction::PipMicToggle { room_id: room_id.clone() }); + } + + // Camera button + if self.view.button(cx, ids!(pip_camera_button)).clicked(actions) { + log!("PipVoipOverlay: Camera button clicked"); + cx.action(VoipAction::PipCameraToggle { room_id: room_id.clone() }); + } + + // Screen share button + if self.view.button(cx, ids!(pip_screenshare_button)).clicked(actions) { + log!("PipVoipOverlay: Screen share button clicked"); + cx.action(VoipAction::PipScreenShareToggle { room_id: room_id.clone() }); + } + + // Hangup button + if self.view.button(cx, ids!(pip_hangup_button)).clicked(actions) { + log!("PipVoipOverlay: Hangup button clicked"); + cx.action(VoipAction::PipHangup { room_id: room_id.clone() }); + // Hide PiP after hangup + self.hide(cx); + } + } +} + +impl PipVoipOverlayRef { + /// Show the PiP overlay for the given room + pub fn show(&self, cx: &mut Cx, room_id: OwnedRoomId) { + if let Some(mut inner) = self.borrow_mut() { + inner.show(cx, room_id); + } + } + + /// Hide the PiP overlay + pub fn hide(&self, cx: &mut Cx) { + if let Some(mut inner) = self.borrow_mut() { + inner.hide(cx); + } + } + + /// Check if the PiP is currently visible + pub fn is_visible(&self) -> bool { + if let Some(inner) = self.borrow() { + inner.is_visible + } else { + false + } + } + + /// Get the room ID of the active call being displayed + pub fn get_room_id(&self) -> Option { + if let Some(inner) = self.borrow() { + inner.room_id.clone() + } else { + None + } + } +} diff --git a/src/voip/voip_screen.rs b/src/voip/voip_screen.rs index 02bd4aa02..36b616596 100644 --- a/src/voip/voip_screen.rs +++ b/src/voip/voip_screen.rs @@ -10,7 +10,7 @@ use ruma::OwnedRoomId; use tokio::sync::mpsc; use crate::sliding_sync::{get_client, submit_async_request, MatrixRequest}; -use super::{VoipGlobalState, VoipAction, CallMember}; +use super::{VoipGlobalState, VoipAction, CallMember, ActiveCallState, ParticipantInfo}; use super::call_state::{Call, CallType, ConnectionState}; use super::camera::{CameraChoice, CameraManager}; @@ -880,6 +880,121 @@ impl Widget for VoipScreen { log!("VoipScreen: TestStopVideoStream"); self.stop_test_video_stream(cx); } + VoipAction::PipMicToggle { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: PipMicToggle from PiP"); + self.toggle_microphone(); + self.update_ui(cx); + } + } + VoipAction::PipCameraToggle { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: PipCameraToggle from PiP"); + self.toggle_camera(cx); + self.update_ui(cx); + } + } + VoipAction::PipScreenShareToggle { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: PipScreenShareToggle from PiP"); + self.toggle_screenshare(); + self.update_ui(cx); + } + } + VoipAction::PipHangup { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: PipHangup from PiP"); + self.hangup(cx); + } + } + VoipAction::ShowPip { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + log!("VoipScreen: ShowPip - stopping local camera for PiP"); + // Stop the call camera so PiP can use it + if !self.call.local_video_muted { + CameraManager::stop_call_camera(&self.view, cx); + self.camera_active = false; + } + } + } + VoipAction::HidePip => { + // When PiP is hidden (user returned to VoIP tab), restart camera + if !self.in_lobby && !self.call.local_video_muted { + log!("VoipScreen: HidePip - attempting to restart camera"); + + if let Some(choice) = self.camera_choice.clone() { + let video = self.view.video(cx, &[live_id!(local_camera_video)]); + let is_unprepared = video.is_unprepared(); + let is_cleaning_up = video.is_cleaning_up(); + log!("VoipScreen: local_camera_video state - unprepared={}, cleaning_up={}", + is_unprepared, is_cleaning_up); + + if is_unprepared { + // Camera is ready to start + log!("VoipScreen: Starting camera immediately"); + if CameraManager::start_call_camera(&self.view, cx, &choice) { + log!("VoipScreen: Call camera started successfully"); + self.camera_active = true; + } else { + log!("VoipScreen: Failed to start camera, will retry on release event"); + self.pending_call_camera_start = true; + self.camera_active = false; + } + } else if is_cleaning_up { + // Camera is still cleaning up, wait for release event + log!("VoipScreen: Camera is cleaning up, will wait for release"); + self.pending_call_camera_start = true; + self.camera_active = false; + } else { + // Video is in some other state - force stop and restart + log!("VoipScreen: Camera in unexpected state, forcing stop and restart"); + CameraManager::stop_call_camera(&self.view, cx); + self.pending_call_camera_start = true; + self.camera_active = false; + } + } + } + } + VoipAction::ReturnToVoipTab { room_id } => { + if self.room_id.as_ref() == Some(room_id) { + // When returning to VoIP tab from PiP, restart camera + if !self.in_lobby && !self.call.local_video_muted { + log!("VoipScreen: ReturnToVoipTab - attempting to restart camera"); + + if let Some(choice) = self.camera_choice.clone() { + let video = self.view.video(cx, &[live_id!(local_camera_video)]); + let is_unprepared = video.is_unprepared(); + let is_cleaning_up = video.is_cleaning_up(); + log!("VoipScreen: local_camera_video state - unprepared={}, cleaning_up={}", + is_unprepared, is_cleaning_up); + + if is_unprepared { + // Camera is ready to start + log!("VoipScreen: Starting camera immediately"); + if CameraManager::start_call_camera(&self.view, cx, &choice) { + log!("VoipScreen: Call camera started successfully"); + self.camera_active = true; + } else { + log!("VoipScreen: Failed to start camera, will retry on release event"); + self.pending_call_camera_start = true; + self.camera_active = false; + } + } else if is_cleaning_up { + // Camera is still cleaning up, wait for release event + log!("VoipScreen: Camera is cleaning up, will wait for release"); + self.pending_call_camera_start = true; + self.camera_active = false; + } else { + // Video is in some other state - force stop and restart + log!("VoipScreen: Camera in unexpected state, forcing stop and restart"); + CameraManager::stop_call_camera(&self.view, cx); + self.pending_call_camera_start = true; + self.camera_active = false; + } + } + } + } + } _ => {} } } @@ -1194,6 +1309,9 @@ impl VoipScreen { self.pending_call_camera_start = true; self.camera_active = false; + // Clear global state for PiP + VoipGlobalState::clear_active_call(cx); + self.update_ui(cx); } @@ -1332,6 +1450,64 @@ impl VoipScreen { // Force redraw to ensure all visibility changes take effect self.view.redraw(cx); + + // Sync state to global for PiP display + self.sync_to_global_state(cx); + } + + /// Sync current call state to global state for PiP overlay access + fn sync_to_global_state(&mut self, cx: &mut Cx) { + // Only sync if we have a room and are in a call (not lobby) + if self.in_lobby { + // Clear global state when in lobby + VoipGlobalState::clear_active_call(cx); + return; + } + + let status_text = match self.call.connection_state { + ConnectionState::Disconnected => "Not connected".to_string(), + ConnectionState::Connecting => "Connecting...".to_string(), + ConnectionState::Connected => "In call".to_string(), + ConnectionState::Disconnecting => "Disconnecting...".to_string(), + }; + + // Get local participant info + let local_participant = if let Some(client) = get_client() { + if let Some(session) = client.session_meta() { + let user_id = session.user_id.to_string(); + let display_name = user_id + .strip_prefix('@') + .and_then(|s| s.split(':').next()) + .unwrap_or(&user_id) + .to_string(); + let avatar_letter = display_name.chars().next() + .unwrap_or('?') + .to_uppercase() + .to_string(); + ParticipantInfo { + user_id, + display_name, + avatar_letter, + } + } else { + ParticipantInfo::default() + } + } else { + ParticipantInfo::default() + }; + + let active_call = ActiveCallState { + room_id: self.room_id.clone(), + status_text, + in_call: !self.in_lobby && self.call.connection_state != ConnectionState::Disconnected, + mic_muted: self.call.local_audio_muted, + camera_muted: self.call.local_video_muted, + screen_sharing: self.call.is_screen_sharing, + local_participant, + participant_count: self.call.participants.len() + 1, + }; + + VoipGlobalState::update_active_call(cx, active_call); } /// Try to start camera @@ -1418,19 +1594,40 @@ impl VoipScreen { } } - /// Handle video resources released (camera handoff from lobby to call) + /// Handle video resources released (camera handoff from lobby/PiP to call) fn handle_video_resources_released(&mut self, cx: &mut Cx) { if self.pending_call_camera_start { - log!("Lobby camera released, starting call camera..."); + log!("VoipScreen: Camera resources released, attempting to start call camera..."); self.pending_call_camera_start = false; if let Some(choice) = self.camera_choice.clone() { - if CameraManager::start_call_camera(&self.view, cx, &choice) { - log!("Call camera started successfully"); - self.camera_active = true; + let video = self.view.video(cx, &[live_id!(local_camera_video)]); + let is_unprepared = video.is_unprepared(); + let is_cleaning_up = video.is_cleaning_up(); + log!("VoipScreen: local_camera_video state - unprepared={}, cleaning_up={}", + is_unprepared, is_cleaning_up); + + if is_unprepared { + // Camera is ready to start + if CameraManager::start_call_camera(&self.view, cx, &choice) { + log!("VoipScreen: Call camera started successfully"); + self.camera_active = true; + } else { + log!("VoipScreen: Failed to start camera even though unprepared"); + self.pending_call_camera_start = true; + } + } else if is_cleaning_up { + // Still cleaning up, wait for next release event + log!("VoipScreen: Camera still cleaning up, will retry on next release"); + self.pending_call_camera_start = true; + } else { + // Unexpected state - force stop and retry + log!("VoipScreen: Camera in unexpected state, forcing stop"); + CameraManager::stop_call_camera(&self.view, cx); + self.pending_call_camera_start = true; } } } else { - log!("Video resources released"); + log!("VoipScreen: Video resources released (no pending start)"); } } From 7ed60d5603c440babe2321c3950d82e506644b7d Mon Sep 17 00:00:00 2001 From: alanpoon Date: Fri, 10 Apr 2026 10:30:14 +0800 Subject: [PATCH 07/21] fix compilation error --- src/app.rs | 24 +++++++++++---- src/home/main_desktop_ui.rs | 6 +++- src/home/rooms_list.rs | 1 + src/sliding_sync.rs | 58 ++++++++++++++++++++++--------------- 4 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/app.rs b/src/app.rs index 41f2bd85a..9c6038b64 100644 --- a/src/app.rs +++ b/src/app.rs @@ -1547,11 +1547,6 @@ impl AppMain for App { _ => {} } - // Poll stdin for VoIP commands - if self.stdin_poll_timer.is_event(event).is_some() { - self.poll_stdin(cx); - } - // Forward events to the MatchEvent trait implementation. self.match_event(cx, event); let scope = &mut Scope::with_data(&mut self.app_state); @@ -2409,6 +2404,25 @@ impl SelectedRoom { } } + /// Returns the `TimelineKind` for this room, if applicable. + /// Returns `None` for invited rooms, spaces, and VoIP rooms which don't have timelines. + pub fn timeline_kind(&self) -> Option { + match self { + SelectedRoom::JoinedRoom { room_name_id } => { + Some(TimelineKind::MainRoom { room_id: room_name_id.room_id().clone() }) + } + SelectedRoom::Thread { room_name_id, thread_root_event_id } => { + Some(TimelineKind::Thread { + room_id: room_name_id.room_id().clone(), + thread_root_event_id: thread_root_event_id.clone(), + }) + } + SelectedRoom::InvitedRoom { .. } => None, + SelectedRoom::Space { .. } => None, + SelectedRoom::Voip { .. } => None, + } + } + /// Upgrades this room from an invite to a joined room /// if its `room_id` matches the given `room_id`. /// diff --git a/src/home/main_desktop_ui.rs b/src/home/main_desktop_ui.rs index 96c266299..354e2f944 100644 --- a/src/home/main_desktop_ui.rs +++ b/src/home/main_desktop_ui.rs @@ -3,7 +3,7 @@ use ruma::OwnedRoomId; use tokio::sync::Notify; use std::{collections::HashMap, sync::Arc}; -use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, logout::logout_confirm_modal::LogoutAction, sliding_sync::AccountSwitchAction, utils::RoomNameId}; +use crate::{app::{AppState, AppStateAction, SavedDockState, SelectedRoom}, home::{navigation_tab_bar::{NavigationBarAction, SelectedTab}, rooms_list::RoomsListRef, space_lobby::SpaceLobbyScreenWidgetRefExt}, logout::logout_confirm_modal::LogoutAction, sliding_sync::AccountSwitchAction, utils::RoomNameId, voip::{VoipAction, VoipGlobalState, VoipScreenWidgetRefExt}}; use super::{invite_screen::InviteScreenWidgetRefExt, room_screen::RoomScreenWidgetRefExt, rooms_list::RoomsListAction}; script_mod! { @@ -177,6 +177,10 @@ impl MainDesktopUI { space_name_id, ); } + SelectedRoom::Voip { room_name_id } => { + // VoIP tabs use VoipScreen directly, not RoomScreen + widget.as_voip_screen().initialize(cx, room_name_id.room_id().clone()); + } } } diff --git a/src/home/rooms_list.rs b/src/home/rooms_list.rs index 618cc37c4..914730af4 100644 --- a/src/home/rooms_list.rs +++ b/src/home/rooms_list.rs @@ -585,6 +585,7 @@ impl RoomsList { is_selected: false, is_direct: false, is_tombstoned: false, + has_active_call: false, }); } } diff --git a/src/sliding_sync.rs b/src/sliding_sync.rs index 13edc9f83..6174ab1a5 100644 --- a/src/sliding_sync.rs +++ b/src/sliding_sync.rs @@ -3338,47 +3338,59 @@ async fn matrix_worker_task( log!("LiveKit JWT request body: {}", serde_json::to_string_pretty(&request_body).unwrap_or_default()); // Make HTTP request to LiveKit JWT endpoint - let http_client = reqwest::Client::new(); + let http_client = matrix_sdk::reqwest::Client::new(); + let request_body_str = serde_json::to_string(&request_body).unwrap_or_default(); match http_client .post(jwt_endpoint) .header("Content-Type", "application/json") - .json(&request_body) + .body(request_body_str) .send() .await { Ok(response) => { if response.status().is_success() { - match response.json::().await { - Ok(json) => { - let url = json["url"].as_str().unwrap_or("").to_string(); - let jwt = json["jwt"].as_str().unwrap_or("").to_string(); - - if !url.is_empty() && !jwt.is_empty() { - log!("LiveKit JWT fetched successfully, url: {}", url); - Cx::post_action(crate::voip::VoipAction::LiveKitJwtFetched { - room_id: room_id.clone(), - url, - jwt, - }); - } else { - error!("LiveKit JWT response missing url or jwt"); - Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { - room_id: room_id.clone(), - error: "Invalid JWT response".to_string(), - }); + match response.text().await { + Ok(text) => { + match serde_json::from_str::(&text) { + Ok(json) => { + let url = json["url"].as_str().unwrap_or("").to_string(); + let jwt = json["jwt"].as_str().unwrap_or("").to_string(); + + if !url.is_empty() && !jwt.is_empty() { + log!("LiveKit JWT fetched successfully, url: {}", url); + Cx::post_action(crate::voip::VoipAction::LiveKitJwtFetched { + room_id: room_id.clone(), + url, + jwt, + }); + } else { + error!("LiveKit JWT response missing url or jwt"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: "Invalid JWT response".to_string(), + }); + } + } + Err(e) => { + error!("Failed to parse LiveKit JWT response: {e:?}"); + Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { + room_id: room_id.clone(), + error: format!("Failed to parse JWT response: {e}"), + }); + } } } Err(e) => { - error!("Failed to parse LiveKit JWT response: {e:?}"); + error!("Failed to read LiveKit JWT response: {e:?}"); Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { room_id: room_id.clone(), - error: format!("Failed to parse JWT response: {e}"), + error: format!("Failed to read response: {e}"), }); } } } else { let status = response.status(); - let error_text = response.text().await.unwrap_or_default(); + let error_text = response.text().await.unwrap_or_else(|_| String::new()); error!("LiveKit JWT request failed: {} - {}", status, error_text); Cx::post_action(crate::voip::VoipAction::LiveKitConnectionFailed { room_id: room_id.clone(), From 294290f15a6451bd61a41474dea9288a72401a3f Mon Sep 17 00:00:00 2001 From: alanpoon Date: Fri, 10 Apr 2026 15:09:12 +0800 Subject: [PATCH 08/21] added voip --- Cargo.lock | 64 ------- Cargo.toml | 7 +- src/app.rs | 3 + src/home/room_screen.rs | 81 +++++++- src/shared/mod.rs | 2 + src/shared/webcam_capture.rs | 352 +++++++++++++++++++++++++++++++++++ src/sliding_sync.rs | 1 - src/voip/livekit_client.rs | 90 --------- src/voip/voip_screen.rs | 2 +- 9 files changed, 434 insertions(+), 168 deletions(-) create mode 100644 src/shared/webcam_capture.rs diff --git a/Cargo.lock b/Cargo.lock index dbb5d26d3..fbbee68d5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2240,29 +2240,6 @@ dependencies = [ "syn 2.0.106", ] -[[package]] -name = "env_filter" -version = "1.0.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a1c3cc8e57274ec99de65301228b537f1e4eedc1b8e0f9411c6caac8ae7308f" -dependencies = [ - "log", - "regex", -] - -[[package]] -name = "env_logger" -version = "0.11.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b2daee4ea451f429a58296525ddf28b45a3b64f1acf6587e2067437bb11e218d" -dependencies = [ - "anstream", - "anstyle", - "env_filter", - "jiff", - "log", -] - [[package]] name = "equator" version = "0.4.2" @@ -3585,30 +3562,6 @@ version = "1.0.15" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" -[[package]] -name = "jiff" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a3546dc96b6d42c5f24902af9e2538e82e39ad350b0c766eb3fbf2d8f3d8359" -dependencies = [ - "jiff-static", - "log", - "portable-atomic", - "portable-atomic-util", - "serde_core", -] - -[[package]] -name = "jiff-static" -version = "0.2.23" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2a8c8b344124222efd714b73bb41f8b5120b27a7cc1c75593a6ff768d9d05aa4" -dependencies = [ - "proc-macro2", - "quote", - "syn 2.0.106", -] - [[package]] name = "jni" version = "0.21.1" @@ -5906,21 +5859,6 @@ dependencies = [ "universal-hash", ] -[[package]] -name = "portable-atomic" -version = "1.13.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c33a9471896f1c69cecef8d20cbe2f7accd12527ce60845ff44c153bb2a21b49" - -[[package]] -name = "portable-atomic-util" -version = "0.2.6" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "091397be61a01d4be58e7841595bd4bfedb15f1cd54977d79b8271e94ed799a3" -dependencies = [ - "portable-atomic", -] - [[package]] name = "potential_utf" version = "0.1.3" @@ -6731,7 +6669,6 @@ dependencies = [ "clap", "crossbeam-channel", "crossbeam-queue", - "env_logger", "eyeball", "eyeball-im", "futures-util", @@ -6743,7 +6680,6 @@ dependencies = [ "indexmap 2.13.0", "linkify", "livekit", - "log", "makepad-code-editor", "makepad-widgets", "matrix-sdk", diff --git a/Cargo.toml b/Cargo.toml index 7ecc23575..51e8c296d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -79,11 +79,8 @@ serde_json = "1.0" thiserror = "2.0.16" tokio = { version = "1.43.1", features = ["macros", "rt-multi-thread"] } ## LiveKit WebRTC SDK - requires native libwebrtc build -## Enable the "livekit" feature to use real WebRTC functionality ## rustls-tls-native-roots is required for WSS connections -livekit = { version = "0.7", optional = true, features = ["rustls-tls-native-roots"] } -env_logger = { version = "0.11", optional = true } -log = { version = "0.4", optional = true } +livekit = { version = "0.7", features = ["rustls-tls-native-roots"] } tracing-subscriber = "0.3.17" unicode-segmentation = "1.11.0" url = "2.5.0" @@ -121,8 +118,6 @@ reqwest = { version = "0.12", default-features = false, optional = true, feature default = [] ## Enables experimental support for using TSP wallets. tsp = ["dep:tsp_sdk", "dep:quinn", "dep:aws-lc-rs", "dep:percent-encoding"] -## Enables real LiveKit WebRTC functionality. Requires native libwebrtc build. -livekit = ["dep:livekit", "dep:env_logger", "dep:log"] ## Hides the command prompt console on Windows. hide_windows_console = [] diff --git a/src/app.rs b/src/app.rs index 9c6038b64..9745d3500 100644 --- a/src/app.rs +++ b/src/app.rs @@ -635,6 +635,9 @@ impl MatchEvent for App { log!("App::Startup: initializing TSP (Trust Spanning Protocol) module."); crate::tsp::tsp_init(_tokio_rt_handle).unwrap(); } + + // Initialize VoIP global state (camera permissions, video inputs) + VoipGlobalState::initialize(cx); } fn handle_actions(&mut self, cx: &mut Cx, actions: &Actions) { diff --git a/src/home/room_screen.rs b/src/home/room_screen.rs index 81d6b87f8..9100451a5 100644 --- a/src/home/room_screen.rs +++ b/src/home/room_screen.rs @@ -910,8 +910,7 @@ script_mod! { View { width: Fill, height: 1 } join_call_button := RobrixPositiveIconButton { - margin: Inset{ top: -1.5, left: 2, right: 2} - padding: Inset{top: 6, bottom: 6, left: 12, right: 12} + padding: 6.0 draw_bg +: { border_size: 0.75 color: #7b1fa2 @@ -2075,6 +2074,51 @@ script_mod! { threads_sliding_pane := mod.widgets.ThreadsSlidingPane { } room_info_sliding_pane := mod.widgets.RoomInfoSlidingPane { } + // Active call banner - shown when there's an ongoing call in the room + active_call_banner := View { + width: Fill + height: Fit + visible: false + padding: Inset{top: 8, bottom: 8, left: 16, right: 16} + align: Align{x: 0.5, y: 0.0} + show_bg: true + draw_bg +: { + color: #7b1fa2 + } + + { + width: Fit + height: Fit + flow: Right + spacing: 12 + align: Align{y: 0.5} + + { + icon_path: dep("crate://self/resources/icons/video.svg") + draw_icon: { color: #fff } + icon_size: vec2(20.0, 20.0) + } + +