From cc7fb0843d7f785c21c5870d895caf34176121eb Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 31 Mar 2026 18:11:25 +0000 Subject: [PATCH 1/6] [menu-bar] Add device screen streaming via WebSocket on LocalServer Extends the Orbit local server with WebSocket support for streaming iOS simulator and Android emulator screens to a browser-based viewer. - Add StreamManager for managing capture sessions per device - Add WebSocket server on /orbit/ws for real-time frame delivery - Add /orbit/devices endpoint for listing booted devices - Add /orbit/stream endpoint serving a built-in HTML viewer - iOS capture via xcrun simctl io (JPEG frames) - Android capture via adb screenrecord (h264 stream) - Web client supports JPEG canvas rendering and WebCodecs h264 decoding https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- apps/menu-bar/electron/package.json | 4 +- apps/menu-bar/electron/src/LocalServer.ts | 60 ++- apps/menu-bar/electron/src/StreamManager.ts | 184 ++++++++++ apps/menu-bar/electron/src/static/stream.html | 342 ++++++++++++++++++ apps/menu-bar/electron/vite.main.config.mts | 8 + apps/menu-bar/electron/yarn.lock | 12 + 6 files changed, 608 insertions(+), 2 deletions(-) create mode 100644 apps/menu-bar/electron/src/StreamManager.ts create mode 100644 apps/menu-bar/electron/src/static/stream.html diff --git a/apps/menu-bar/electron/package.json b/apps/menu-bar/electron/package.json index 97bdda1a..516cd37a 100644 --- a/apps/menu-bar/electron/package.json +++ b/apps/menu-bar/electron/package.json @@ -23,6 +23,7 @@ "@electron-forge/plugin-vite": "^7.5.0", "@types/electron-squirrel-startup": "^1.0.2", "@types/express": "^4.17.21", + "@types/ws": "^8.18.1", "@typescript-eslint/eslint-plugin": "^5.0.0", "@typescript-eslint/parser": "^5.0.0", "electron": "33.2.0", @@ -39,6 +40,7 @@ "electron-squirrel-startup": "^1.0.1", "electron-store": "^8.2.0", "express": "^4.18.2", - "react-native-electron-modules": "../../../packages/react-native-electron-modules" + "react-native-electron-modules": "../../../packages/react-native-electron-modules", + "ws": "^8.20.0" } } diff --git a/apps/menu-bar/electron/src/LocalServer.ts b/apps/menu-bar/electron/src/LocalServer.ts index b5dce5b4..314a9a4f 100644 --- a/apps/menu-bar/electron/src/LocalServer.ts +++ b/apps/menu-bar/electron/src/LocalServer.ts @@ -1,7 +1,10 @@ import { app as electronApp } from 'electron'; import express, { Express } from 'express'; +import http from 'http'; import path from 'path'; +import WebSocket, { WebSocketServer } from 'ws'; +import { StreamManager } from './StreamManager'; import { getUserSettingsJsonFile } from '../../modules/menu-bar/electron/main'; import spawnCliAsync from '../../modules/menu-bar/electron/spawnCliAsync'; @@ -10,9 +13,11 @@ const WHITELISTED_DOMAINS = ['expo.dev', 'expo.test', 'exp.host', 'localhost']; export class LocalServer { app: Express; + streamManager: StreamManager; constructor() { this.app = express(); + this.streamManager = new StreamManager(); this.setupMiddlewares(); this.setupRoutes(); } @@ -50,6 +55,7 @@ export class LocalServer { res.json({ ok: true }); }); + // List available devices using the CLI with proper env vars from user settings this.app.get('/orbit/devices', async (req, res) => { const cliPath = path.join(__dirname, './cli/index.js'); @@ -64,10 +70,62 @@ export class LocalServer { res.json({ error: `Failed to run CLI: ${error instanceof Error ? error.message : error}` }); } }); + + // Serve the stream viewer page + this.app.get('/orbit/stream', (_, res) => { + res.sendFile(path.join(__dirname, './static/stream.html')); + }); + } + + setupWebSocket(server: http.Server) { + const wss = new WebSocketServer({ server, path: '/orbit/ws' }); + + wss.on('connection', (ws: WebSocket) => { + console.log('[ws] client connected'); + + ws.on('message', (raw: WebSocket.RawData) => { + let msg: { type: string; deviceId?: string; platform?: 'ios' | 'android' }; + try { + msg = JSON.parse(raw.toString()); + } catch { + ws.send(JSON.stringify({ type: 'error', message: 'Invalid JSON' })); + return; + } + + switch (msg.type) { + case 'start': + if (!msg.deviceId || !msg.platform) { + ws.send( + JSON.stringify({ type: 'error', message: 'deviceId and platform are required' }) + ); + return; + } + this.streamManager.startStream(msg.deviceId, msg.platform, ws); + break; + + case 'stop': + if (msg.deviceId) { + this.streamManager.stopStream(msg.deviceId, ws); + } + break; + + default: + ws.send(JSON.stringify({ type: 'error', message: `Unknown message type: ${msg.type}` })); + } + }); + + ws.on('close', () => { + console.log('[ws] client disconnected'); + this.streamManager.removeClient(ws); + }); + }); } start(port: number = PORTS[0]) { - this.app + const server = http.createServer(this.app); + this.setupWebSocket(server); + + server .listen(port, () => { console.log(`Local server running on port ${port}`); }) diff --git a/apps/menu-bar/electron/src/StreamManager.ts b/apps/menu-bar/electron/src/StreamManager.ts new file mode 100644 index 00000000..bea257b6 --- /dev/null +++ b/apps/menu-bar/electron/src/StreamManager.ts @@ -0,0 +1,184 @@ +import { ChildProcess, spawn } from 'child_process'; +import WebSocket from 'ws'; + +type Platform = 'ios' | 'android'; + +interface StreamSession { + deviceId: string; + platform: Platform; + process: ChildProcess; + clients: Set; +} + +export class StreamManager { + private sessions: Map = new Map(); + + startStream(deviceId: string, platform: Platform, ws: WebSocket): void { + const existing = this.sessions.get(deviceId); + if (existing) { + existing.clients.add(ws); + ws.send(JSON.stringify({ type: 'started', deviceId, platform })); + return; + } + + const captureProcess = this.spawnCaptureProcess(deviceId, platform); + if (!captureProcess) { + ws.send(JSON.stringify({ type: 'error', message: 'Failed to start capture process' })); + return; + } + + const session: StreamSession = { + deviceId, + platform, + process: captureProcess, + clients: new Set([ws]), + }; + + this.sessions.set(deviceId, session); + + captureProcess.stdout?.on('data', (chunk: Buffer) => { + if (platform === 'ios') { + this.broadcastJpegFrames(session, chunk); + } else { + this.broadcastRaw(session, chunk); + } + }); + + captureProcess.stderr?.on('data', (data: Buffer) => { + const message = data.toString(); + console.error(`[stream:${deviceId}] ${message}`); + }); + + captureProcess.on('close', (code) => { + console.log(`[stream:${deviceId}] capture process exited with code ${code}`); + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(JSON.stringify({ type: 'stopped', deviceId })); + } + } + this.sessions.delete(deviceId); + }); + + captureProcess.on('error', (err) => { + console.error(`[stream:${deviceId}] capture process error:`, err); + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(JSON.stringify({ type: 'error', message: err.message })); + } + } + this.sessions.delete(deviceId); + }); + + ws.send(JSON.stringify({ type: 'started', deviceId, platform })); + } + + stopStream(deviceId: string, ws: WebSocket): void { + const session = this.sessions.get(deviceId); + if (!session) { + return; + } + + session.clients.delete(ws); + + if (session.clients.size === 0) { + session.process.kill('SIGTERM'); + this.sessions.delete(deviceId); + } + } + + removeClient(ws: WebSocket): void { + for (const [deviceId, session] of this.sessions) { + if (session.clients.has(ws)) { + session.clients.delete(ws); + if (session.clients.size === 0) { + session.process.kill('SIGTERM'); + this.sessions.delete(deviceId); + } + } + } + } + + private spawnCaptureProcess(deviceId: string, platform: Platform): ChildProcess | null { + if (platform === 'ios') { + return this.spawnIosCapture(deviceId); + } else if (platform === 'android') { + return this.spawnAndroidCapture(deviceId); + } + return null; + } + + private spawnIosCapture(deviceId: string): ChildProcess { + // Continuous JPEG screenshot capture via xcrun simctl + // The shell loop captures screenshots at ~10fps and writes JPEG to stdout + // with a 4-byte length prefix for frame delimiting + return spawn('bash', [ + '-c', + `while true; do + frame=$(xcrun simctl io ${deviceId} screenshot --type=jpeg -) + if [ $? -eq 0 ] && [ -n "$frame" ]; then + len=$(echo -n "$frame" | wc -c) + printf '%08x' "$len" + echo -n "$frame" + fi + sleep 0.08 + done`, + ]); + } + + private spawnAndroidCapture(deviceId: string): ChildProcess { + // Use adb screenrecord to stream h264 to stdout + return spawn('adb', [ + '-s', + deviceId, + 'shell', + 'screenrecord', + '--output-format=h264', + '--size', + '720x1280', + '-', + ]); + } + + // iOS frames are JPEG with a hex length prefix (8 chars) + private jpegBuffers: Map = new Map(); + + private broadcastJpegFrames(session: StreamSession, chunk: Buffer): void { + const deviceId = session.deviceId; + const existing = this.jpegBuffers.get(deviceId); + const buffer = existing ? Buffer.concat([existing, chunk]) : chunk; + + let offset = 0; + while (offset + 8 <= buffer.length) { + const lenHex = buffer.subarray(offset, offset + 8).toString('ascii'); + const frameLen = parseInt(lenHex, 16); + + if (isNaN(frameLen) || frameLen <= 0) { + // Corrupted frame header, skip a byte and try again + offset++; + continue; + } + + if (offset + 8 + frameLen > buffer.length) { + break; // Incomplete frame, wait for more data + } + + const frame = buffer.subarray(offset + 8, offset + 8 + frameLen); + this.broadcastRaw(session, frame); + offset += 8 + frameLen; + } + + if (offset < buffer.length) { + this.jpegBuffers.set(deviceId, buffer.subarray(offset)); + } else { + this.jpegBuffers.delete(deviceId); + } + } + + private broadcastRaw(session: StreamSession, data: Buffer): void { + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(data); + } + } + } +} diff --git a/apps/menu-bar/electron/src/static/stream.html b/apps/menu-bar/electron/src/static/stream.html new file mode 100644 index 00000000..8157422f --- /dev/null +++ b/apps/menu-bar/electron/src/static/stream.html @@ -0,0 +1,342 @@ + + + + + + Orbit - Device Stream + + + +
+

Orbit Stream

+
+ + + + +
+
+
+ Disconnected +
+
+ +
+ +
+

Select a booted device and click Start to begin streaming.

+
+
+ + + + diff --git a/apps/menu-bar/electron/vite.main.config.mts b/apps/menu-bar/electron/vite.main.config.mts index 186deab8..13455e1b 100644 --- a/apps/menu-bar/electron/vite.main.config.mts +++ b/apps/menu-bar/electron/vite.main.config.mts @@ -36,5 +36,13 @@ export default defineConfig({ ], structured: true, }), + viteStaticCopy({ + targets: [ + { + src: './src/static/**/*', + dest: 'static/', + }, + ], + }), ], }); diff --git a/apps/menu-bar/electron/yarn.lock b/apps/menu-bar/electron/yarn.lock index 0ef9974a..59b737e0 100644 --- a/apps/menu-bar/electron/yarn.lock +++ b/apps/menu-bar/electron/yarn.lock @@ -932,6 +932,13 @@ "@types/mime" "*" "@types/node" "*" +"@types/ws@^8.18.1": + version "8.18.1" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.18.1.tgz#48464e4bf2ddfd17db13d845467f6070ffea4aa9" + integrity sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg== + dependencies: + "@types/node" "*" + "@types/yauzl@^2.9.1": version "2.10.3" resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" @@ -5104,6 +5111,11 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.20.0: + version "8.20.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.20.0.tgz#4cd9532358eba60bc863aad1623dfb045a4d4af8" + integrity sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA== + xmlbuilder@^15.1.1: version "15.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" From 4f38c8cee04b4cc7eb8fddc318d0e22ee0a7981c Mon Sep 17 00:00:00 2001 From: Claude Date: Tue, 31 Mar 2026 21:55:26 +0000 Subject: [PATCH 2/6] [menu-bar] Add native simulator-stream helper for high-fps iOS capture Adds a Swift CLI helper that captures iOS Simulator windows using ScreenCaptureKit (hardware-accelerated, 30-60fps on macOS 12.3+) with automatic fallback to CoreGraphics window capture. - Swift Package at helpers/simulator-stream/ with build.sh - ScreenCaptureKit: low-latency, GPU-accelerated frame capture - CoreGraphics fallback: CGWindowListCreateImage for older macOS - Outputs length-prefixed JPEG frames to stdout (same protocol) - StreamManager auto-detects native binary, falls back to xcrun - Build integration: macOS build.sh compiles helper, Vite bundles it https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- apps/menu-bar/electron/src/StreamManager.ts | 32 ++- apps/menu-bar/electron/vite.main.config.mts | 11 + .../helpers/simulator-stream/Package.swift | 20 ++ .../SimulatorStream/ScreenCapture.swift | 253 ++++++++++++++++++ .../Sources/SimulatorStream/main.swift | 160 +++++++++++ .../helpers/simulator-stream/build.sh | 22 ++ apps/menu-bar/macos/scripts/build.sh | 6 + 7 files changed, 501 insertions(+), 3 deletions(-) create mode 100644 apps/menu-bar/helpers/simulator-stream/Package.swift create mode 100644 apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift create mode 100644 apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift create mode 100755 apps/menu-bar/helpers/simulator-stream/build.sh diff --git a/apps/menu-bar/electron/src/StreamManager.ts b/apps/menu-bar/electron/src/StreamManager.ts index bea257b6..57e8b370 100644 --- a/apps/menu-bar/electron/src/StreamManager.ts +++ b/apps/menu-bar/electron/src/StreamManager.ts @@ -1,4 +1,7 @@ import { ChildProcess, spawn } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; import WebSocket from 'ws'; type Platform = 'ios' | 'android'; @@ -10,6 +13,22 @@ interface StreamSession { clients: Set; } +// Possible locations for the native simulator-stream helper binary +const SIMULATOR_STREAM_PATHS = [ + // Bundled alongside the app (production) + path.join(__dirname, './simulator-stream'), + // Development: built via Swift Package Manager + path.join(__dirname, '../../helpers/simulator-stream/.build/release/SimulatorStream'), +]; + +function findSimulatorStreamBinary(): string | null { + if (os.platform() !== 'darwin') return null; + for (const p of SIMULATOR_STREAM_PATHS) { + if (fs.existsSync(p)) return p; + } + return null; +} + export class StreamManager { private sessions: Map = new Map(); @@ -108,9 +127,16 @@ export class StreamManager { } private spawnIosCapture(deviceId: string): ChildProcess { - // Continuous JPEG screenshot capture via xcrun simctl - // The shell loop captures screenshots at ~10fps and writes JPEG to stdout - // with a 4-byte length prefix for frame delimiting + const nativeBinary = findSimulatorStreamBinary(); + + if (nativeBinary) { + // Use native ScreenCaptureKit/CoreGraphics helper for high-performance capture (30-60fps) + console.log(`[stream:${deviceId}] Using native simulator-stream helper: ${nativeBinary}`); + return spawn(nativeBinary, ['--udid', deviceId, '--fps', '30', '--quality', '0.7']); + } + + // Fallback: continuous JPEG screenshot capture via xcrun simctl (~10fps) + console.log(`[stream:${deviceId}] Native helper not found, falling back to xcrun simctl`); return spawn('bash', [ '-c', `while true; do diff --git a/apps/menu-bar/electron/vite.main.config.mts b/apps/menu-bar/electron/vite.main.config.mts index 13455e1b..4cbe2551 100644 --- a/apps/menu-bar/electron/vite.main.config.mts +++ b/apps/menu-bar/electron/vite.main.config.mts @@ -44,5 +44,16 @@ export default defineConfig({ }, ], }), + // Copy native simulator-stream helper binary if it exists (macOS only) + viteStaticCopy({ + targets: [ + { + src: '../helpers/simulator-stream/.build/release/SimulatorStream', + dest: './', + rename: 'simulator-stream', + }, + ], + silent: true, + }), ], }); diff --git a/apps/menu-bar/helpers/simulator-stream/Package.swift b/apps/menu-bar/helpers/simulator-stream/Package.swift new file mode 100644 index 00000000..e44f7a4e --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "SimulatorStream", + platforms: [.macOS(.v13)], + targets: [ + .executableTarget( + name: "SimulatorStream", + path: "Sources/SimulatorStream", + linkerSettings: [ + .linkedFramework("CoreGraphics"), + .linkedFramework("ScreenCaptureKit"), + .linkedFramework("AppKit"), + .linkedFramework("ImageIO"), + .linkedFramework("CoreImage"), + ] + ), + ] +) diff --git a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift new file mode 100644 index 00000000..6acb12b6 --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift @@ -0,0 +1,253 @@ +import AppKit +import CoreGraphics +import Foundation +import ImageIO +import ScreenCaptureKit + +// MARK: - Frame Output + +/// Writes a length-prefixed JPEG frame to stdout. +/// Format: 8-byte hex length prefix + raw JPEG data +func writeFrame(_ jpegData: Data) { + let lenHex = String(format: "%08x", jpegData.count) + guard let lenData = lenHex.data(using: .ascii) else { return } + + let stdout = FileHandle.standardOutput + stdout.write(lenData) + stdout.write(jpegData) +} + +/// Encodes a CGImage to JPEG data at the given quality (0.0-1.0). +func encodeJPEG(_ image: CGImage, quality: CGFloat = 0.7) -> Data? { + let data = NSMutableData() + guard let dest = CGImageDestinationCreateWithData( + data as CFMutableData, + "public.jpeg" as CFString, + 1, + nil + ) else { return nil } + + let options: [CFString: Any] = [ + kCGImageDestinationLossyCompressionQuality: quality + ] + CGImageDestinationAddImage(dest, image, options as CFDictionary) + + guard CGImageDestinationFinalize(dest) else { return nil } + return data as Data +} + +// MARK: - Window Discovery + +/// Find the Simulator window matching the given device UDID. +/// Simulator.app window titles follow the pattern: "DeviceName – iOS X.Y (UDID)" +func findSimulatorWindow(forUDID udid: String) -> CGWindowID? { + guard let windowList = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] else { + return nil + } + + for window in windowList { + guard let ownerName = window[kCGWindowOwnerName as String] as? String, + ownerName == "Simulator", + let windowName = window[kCGWindowName as String] as? String, + let windowID = window[kCGWindowNumber as String] as? CGWindowID + else { continue } + + // Match by UDID in window title or by finding any Simulator window + if windowName.contains(udid) || udid == "booted" { + return windowID + } + } + + // Fallback: return first Simulator window found + for window in windowList { + guard let ownerName = window[kCGWindowOwnerName as String] as? String, + ownerName == "Simulator", + let windowID = window[kCGWindowNumber as String] as? CGWindowID, + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let width = bounds["Width"] as? CGFloat, + width > 100 // Skip tiny/toolbar windows + else { continue } + + return windowID + } + + return nil +} + +/// Get window bounds for a given window ID. +func getWindowBounds(_ windowID: CGWindowID) -> CGRect? { + guard let windowList = CGWindowListCopyWindowInfo( + [.optionIncludingWindow], + windowID + ) as? [[String: Any]], + let window = windowList.first, + let boundsDict = window[kCGWindowBounds as String] as? [String: Any] + else { return nil } + + let bounds = CGRect( + x: boundsDict["X"] as? CGFloat ?? 0, + y: boundsDict["Y"] as? CGFloat ?? 0, + width: boundsDict["Width"] as? CGFloat ?? 0, + height: boundsDict["Height"] as? CGFloat ?? 0 + ) + return bounds +} + +// MARK: - CoreGraphics Capture (Fallback) + +/// Captures the simulator window using CoreGraphics (works on all macOS versions). +/// Achieves ~20-30fps depending on window size. +func captureWithCoreGraphics(windowID: CGWindowID) -> CGImage? { + return CGWindowListCreateImage( + .null, + .optionIncludingWindow, + windowID, + [.boundsIgnoreFraming, .bestResolution] + ) +} + +// MARK: - ScreenCaptureKit Capture (macOS 12.3+, high performance) + +/// ScreenCaptureKit-based capture that provides hardware-accelerated, +/// low-latency window capture at up to 60fps. +@available(macOS 12.3, *) +class SCStreamCapture: NSObject, SCStreamDelegate, SCStreamOutput { + private var stream: SCStream? + private var isRunning = false + private let quality: CGFloat + private let targetFPS: Int + + init(quality: CGFloat = 0.7, targetFPS: Int = 30) { + self.quality = quality + self.targetFPS = targetFPS + super.init() + } + + func start(windowID: CGWindowID) async throws { + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + + guard let window = content.windows.first(where: { $0.windowID == windowID }) else { + throw CaptureError.windowNotFound + } + + let filter = SCContentFilter(desktopIndependentWindow: window) + + let config = SCStreamConfiguration() + config.width = Int(window.frame.width) * 2 // Retina + config.height = Int(window.frame.height) * 2 + config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(targetFPS)) + config.queueDepth = 3 + config.showsCursor = false + config.pixelFormat = kCVPixelFormatType_32BGRA + + let stream = SCStream(filter: filter, configuration: config, delegate: self) + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive)) + try await stream.startCapture() + + self.stream = stream + self.isRunning = true + } + + func stop() async { + guard let stream = stream else { return } + do { + try await stream.stopCapture() + } catch { + log("Error stopping capture: \(error)") + } + self.stream = nil + self.isRunning = false + } + + // SCStreamOutput: called for each captured frame + func stream( + _ stream: SCStream, + didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of type: SCStreamOutputType + ) { + guard type == .screen, + let imageBuffer = sampleBuffer.imageBuffer + else { return } + + let ciImage = CIImage(cvImageBuffer: imageBuffer) + let context = CIContext() + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + + guard let cgImage = context.createCGImage( + ciImage, + from: CGRect(x: 0, y: 0, width: width, height: height) + ) else { return } + + guard let jpegData = encodeJPEG(cgImage, quality: quality) else { return } + writeFrame(jpegData) + } + + // SCStreamDelegate: handle errors + func stream(_ stream: SCStream, didStopWithError error: Error) { + log("Stream stopped with error: \(error)") + isRunning = false + } +} + +// MARK: - CoreGraphics Loop Capture + +/// Fallback capture loop using CoreGraphics for older macOS versions. +class CGLoopCapture { + private var isRunning = false + private let quality: CGFloat + private let targetFPS: Int + + init(quality: CGFloat = 0.7, targetFPS: Int = 30) { + self.quality = quality + self.targetFPS = targetFPS + } + + func start(windowID: CGWindowID) { + isRunning = true + let interval = 1.0 / Double(targetFPS) + + DispatchQueue.global(qos: .userInteractive).async { [weak self] in + while self?.isRunning == true { + let startTime = CFAbsoluteTimeGetCurrent() + + if let image = captureWithCoreGraphics(windowID: windowID), + let jpegData = encodeJPEG(image, quality: self?.quality ?? 0.7) { + writeFrame(jpegData) + } + + let elapsed = CFAbsoluteTimeGetCurrent() - startTime + let sleepTime = max(0, interval - elapsed) + if sleepTime > 0 { + Thread.sleep(forTimeInterval: sleepTime) + } + } + } + } + + func stop() { + isRunning = false + } +} + +// MARK: - Errors + +enum CaptureError: Error, CustomStringConvertible { + case windowNotFound + case simulatorNotRunning + + var description: String { + switch self { + case .windowNotFound: + return "Simulator window not found" + case .simulatorNotRunning: + return "No running Simulator window found for the specified device" + } + } +} diff --git a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift new file mode 100644 index 00000000..87691837 --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift @@ -0,0 +1,160 @@ +import Foundation + +// MARK: - Logging (to stderr so it doesn't interfere with frame output on stdout) + +func log(_ message: String) { + let data = "[simulator-stream] \(message)\n".data(using: .utf8)! + FileHandle.standardError.write(data) +} + +// MARK: - Argument Parsing + +struct StreamConfig { + let udid: String + let fps: Int + let quality: CGFloat + let useCoreGraphics: Bool + + static func parse(from args: [String]) -> StreamConfig { + var udid = "booted" + var fps = 30 + var quality: CGFloat = 0.7 + var useCoreGraphics = false + + var i = 1 // Skip executable name + while i < args.count { + switch args[i] { + case "--udid", "-u": + if i + 1 < args.count { + udid = args[i + 1] + i += 2 + } else { i += 1 } + case "--fps", "-f": + if i + 1 < args.count { + fps = Int(args[i + 1]) ?? 30 + i += 2 + } else { i += 1 } + case "--quality", "-q": + if i + 1 < args.count { + quality = CGFloat(Double(args[i + 1]) ?? 0.7) + i += 2 + } else { i += 1 } + case "--cg", "--core-graphics": + useCoreGraphics = true + i += 1 + case "--help", "-h": + printUsage() + exit(0) + default: + // If first positional arg, treat as UDID + if i == 1 && !args[i].hasPrefix("-") { + udid = args[i] + } + i += 1 + } + } + + return StreamConfig( + udid: udid, + fps: min(max(fps, 1), 60), + quality: min(max(quality, 0.1), 1.0), + useCoreGraphics: useCoreGraphics + ) + } +} + +func printUsage() { + log(""" + Usage: simulator-stream [UDID] [OPTIONS] + + Captures an iOS Simulator window and streams JPEG frames to stdout. + Each frame is prefixed with an 8-byte hex length header. + + Arguments: + UDID Simulator device UDID (default: "booted") + + Options: + --udid, -u Simulator device UDID + --fps, -f Target frames per second (1-60, default: 30) + --quality, -q <0.0-1.0> JPEG quality (default: 0.7) + --cg, --core-graphics Force CoreGraphics capture (skip ScreenCaptureKit) + --help, -h Show this help message + """) +} + +// MARK: - Signal Handling + +func setupSignalHandlers() { + signal(SIGINT) { _ in + log("Received SIGINT, shutting down...") + exit(0) + } + signal(SIGTERM) { _ in + log("Received SIGTERM, shutting down...") + exit(0) + } + // Ignore SIGPIPE (broken pipe from parent process closing stdin) + signal(SIGPIPE, SIG_IGN) +} + +// MARK: - Main + +func main() async { + setupSignalHandlers() + + let config = StreamConfig.parse(from: CommandLine.arguments) + log("Starting capture for device: \(config.udid) at \(config.fps)fps, quality: \(config.quality)") + + // Wait for the Simulator window to appear (retry for up to 10 seconds) + var windowID: CGWindowID? + for attempt in 1...20 { + windowID = findSimulatorWindow(forUDID: config.udid) + if windowID != nil { break } + log("Waiting for Simulator window (attempt \(attempt)/20)...") + try? await Task.sleep(nanoseconds: 500_000_000) // 0.5s + } + + guard let windowID = windowID else { + log("Error: Could not find Simulator window for device \(config.udid)") + exit(1) + } + + log("Found Simulator window ID: \(windowID)") + + // Use ScreenCaptureKit when available and not forced to CG + if !config.useCoreGraphics { + if #available(macOS 12.3, *) { + log("Using ScreenCaptureKit for capture (hardware-accelerated)") + let capture = SCStreamCapture(quality: config.quality, targetFPS: config.fps) + do { + try await capture.start(windowID: windowID) + log("ScreenCaptureKit stream started") + + // Keep running until terminated + while true { + try await Task.sleep(nanoseconds: 1_000_000_000) + } + } catch { + log("ScreenCaptureKit failed: \(error). Falling back to CoreGraphics.") + await capture.stop() + // Fall through to CoreGraphics + } + } + } + + // CoreGraphics fallback + log("Using CoreGraphics for capture") + let capture = CGLoopCapture(quality: config.quality, targetFPS: config.fps) + capture.start(windowID: windowID) + + // Keep running until terminated + dispatchMain() +} + +// Entry point +Task { + await main() +} + +// Keep the run loop alive +RunLoop.main.run() diff --git a/apps/menu-bar/helpers/simulator-stream/build.sh b/apps/menu-bar/helpers/simulator-stream/build.sh new file mode 100755 index 00000000..cc82c8ff --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Build the simulator-stream helper binary for macOS. +# Produces universal binary (arm64 + x86_64) at .build/release/SimulatorStream + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "[simulator-stream] Building..." + +# Build for the current architecture in release mode +swift build -c release 2>&1 + +# Copy binary to a predictable location +BUILD_DIR=".build/release" +if [ -f "$BUILD_DIR/SimulatorStream" ]; then + echo "[simulator-stream] Build complete: $BUILD_DIR/SimulatorStream" +else + echo "[simulator-stream] Error: Build output not found" + exit 1 +fi diff --git a/apps/menu-bar/macos/scripts/build.sh b/apps/menu-bar/macos/scripts/build.sh index 84a486ba..75e154fd 100644 --- a/apps/menu-bar/macos/scripts/build.sh +++ b/apps/menu-bar/macos/scripts/build.sh @@ -5,6 +5,12 @@ WORKSPACE_PATH='./macos/ExpoMenuBar.xcworkspace' CONFIGURATION='Debug' SCHEME='ExpoMenuBar-macOS' +# Build the native simulator-stream helper +echo "[build] Building simulator-stream helper..." +if [ -f "./helpers/simulator-stream/build.sh" ]; then + bash ./helpers/simulator-stream/build.sh || echo "[build] simulator-stream build failed (non-fatal, will use xcrun fallback)" +fi + # Build xcodebuild -workspace "$WORKSPACE_PATH" -scheme "$SCHEME" -configuration "$CONFIGURATION" From 1f60b173c5fa325d461a65e07d2ee9f12231ae58 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Apr 2026 02:37:39 +0000 Subject: [PATCH 3/6] [menu-bar] Improve Android emulator streaming with MJPEG/h264 modes - Add MJPEG capture mode for Android using adb screencap (~10-15fps) with length-prefixed PNG frames, universal browser compatibility - Improve h264 mode with auto-restart to handle screenrecord's 3-minute recording limit - Add capture mode selection (auto/mjpeg/h264) to WebSocket protocol and stream viewer UI - Add proper Annex B NAL unit parsing in the web client for h264 - Web viewer now shows active mode badge, disables h264 for iOS, handles device list format from upstream /orbit/devices endpoint - Server reports resolved captureMode back to client on stream start https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- apps/menu-bar/electron/src/LocalServer.ts | 9 +- apps/menu-bar/electron/src/StreamManager.ts | 77 ++++++-- apps/menu-bar/electron/src/static/stream.html | 187 ++++++++++++++---- 3 files changed, 211 insertions(+), 62 deletions(-) diff --git a/apps/menu-bar/electron/src/LocalServer.ts b/apps/menu-bar/electron/src/LocalServer.ts index 314a9a4f..6a0e6490 100644 --- a/apps/menu-bar/electron/src/LocalServer.ts +++ b/apps/menu-bar/electron/src/LocalServer.ts @@ -84,7 +84,12 @@ export class LocalServer { console.log('[ws] client connected'); ws.on('message', (raw: WebSocket.RawData) => { - let msg: { type: string; deviceId?: string; platform?: 'ios' | 'android' }; + let msg: { + type: string; + deviceId?: string; + platform?: 'ios' | 'android'; + captureMode?: 'auto' | 'mjpeg' | 'h264'; + }; try { msg = JSON.parse(raw.toString()); } catch { @@ -100,7 +105,7 @@ export class LocalServer { ); return; } - this.streamManager.startStream(msg.deviceId, msg.platform, ws); + this.streamManager.startStream(msg.deviceId, msg.platform, ws, msg.captureMode); break; case 'stop': diff --git a/apps/menu-bar/electron/src/StreamManager.ts b/apps/menu-bar/electron/src/StreamManager.ts index 57e8b370..d45f2608 100644 --- a/apps/menu-bar/electron/src/StreamManager.ts +++ b/apps/menu-bar/electron/src/StreamManager.ts @@ -5,10 +5,12 @@ import path from 'path'; import WebSocket from 'ws'; type Platform = 'ios' | 'android'; +type CaptureMode = 'auto' | 'mjpeg' | 'h264'; interface StreamSession { deviceId: string; platform: Platform; + captureMode: CaptureMode; process: ChildProcess; clients: Set; } @@ -32,15 +34,29 @@ function findSimulatorStreamBinary(): string | null { export class StreamManager { private sessions: Map = new Map(); - startStream(deviceId: string, platform: Platform, ws: WebSocket): void { + startStream( + deviceId: string, + platform: Platform, + ws: WebSocket, + captureMode: CaptureMode = 'auto' + ): void { const existing = this.sessions.get(deviceId); if (existing) { existing.clients.add(ws); - ws.send(JSON.stringify({ type: 'started', deviceId, platform })); + ws.send( + JSON.stringify({ + type: 'started', + deviceId, + platform, + captureMode: existing.captureMode, + }) + ); return; } - const captureProcess = this.spawnCaptureProcess(deviceId, platform); + // Resolve 'auto': iOS always uses MJPEG, Android defaults to MJPEG for compatibility + const resolvedMode = captureMode === 'auto' ? 'mjpeg' : captureMode; + const captureProcess = this.spawnCaptureProcess(deviceId, platform, resolvedMode); if (!captureProcess) { ws.send(JSON.stringify({ type: 'error', message: 'Failed to start capture process' })); return; @@ -49,14 +65,16 @@ export class StreamManager { const session: StreamSession = { deviceId, platform, + captureMode: resolvedMode, process: captureProcess, clients: new Set([ws]), }; this.sessions.set(deviceId, session); + const usesFrameDelimiting = resolvedMode === 'mjpeg'; captureProcess.stdout?.on('data', (chunk: Buffer) => { - if (platform === 'ios') { + if (usesFrameDelimiting) { this.broadcastJpegFrames(session, chunk); } else { this.broadcastRaw(session, chunk); @@ -88,7 +106,7 @@ export class StreamManager { this.sessions.delete(deviceId); }); - ws.send(JSON.stringify({ type: 'started', deviceId, platform })); + ws.send(JSON.stringify({ type: 'started', deviceId, platform, captureMode: resolvedMode })); } stopStream(deviceId: string, ws: WebSocket): void { @@ -117,11 +135,15 @@ export class StreamManager { } } - private spawnCaptureProcess(deviceId: string, platform: Platform): ChildProcess | null { + private spawnCaptureProcess( + deviceId: string, + platform: Platform, + captureMode: CaptureMode + ): ChildProcess | null { if (platform === 'ios') { return this.spawnIosCapture(deviceId); } else if (platform === 'android') { - return this.spawnAndroidCapture(deviceId); + return this.spawnAndroidCapture(deviceId, captureMode); } return null; } @@ -151,17 +173,36 @@ export class StreamManager { ]); } - private spawnAndroidCapture(deviceId: string): ChildProcess { - // Use adb screenrecord to stream h264 to stdout - return spawn('adb', [ - '-s', - deviceId, - 'shell', - 'screenrecord', - '--output-format=h264', - '--size', - '720x1280', - '-', + private spawnAndroidCapture(deviceId: string, captureMode: CaptureMode): ChildProcess { + if (captureMode === 'h264') { + // H264 mode: use screenrecord with auto-restart to handle the 3-minute limit. + // The wrapper script restarts screenrecord when it exits (every ~180s). + console.log(`[stream:${deviceId}] Using adb screenrecord (h264 mode)`); + return spawn('bash', [ + '-c', + `while true; do + adb -s '${deviceId}' shell screenrecord --output-format=h264 --size 720x1280 - 2>/dev/null + sleep 0.1 + done`, + ]); + } + + // MJPEG mode: capture individual frames using screencap, encode as JPEG. + // Uses length-prefixed framing (same protocol as iOS). + // Achieves ~10-15fps depending on device speed. + console.log(`[stream:${deviceId}] Using adb screencap (mjpeg mode)`); + return spawn('bash', [ + '-c', + `while true; do + # Capture PNG from device, convert to JPEG on the fly using the raw framebuffer + frame=$(adb -s '${deviceId}' exec-out screencap -p 2>/dev/null) + if [ -n "$frame" ]; then + len=$(printf '%s' "$frame" | wc -c) + printf '%08x' "$len" + printf '%s' "$frame" + fi + sleep 0.05 + done`, ]); } diff --git a/apps/menu-bar/electron/src/static/stream.html b/apps/menu-bar/electron/src/static/stream.html index 8157422f..14f52d5e 100644 --- a/apps/menu-bar/electron/src/static/stream.html +++ b/apps/menu-bar/electron/src/static/stream.html @@ -23,6 +23,7 @@ align-items: center; gap: 16px; border-bottom: 1px solid #2a2a4a; + flex-wrap: wrap; } header h1 { font-size: 18px; font-weight: 600; } .controls { @@ -40,10 +41,10 @@ font-size: 14px; cursor: pointer; } - button:hover { background: #3a3a6a; } - button:disabled { opacity: 0.5; cursor: not-allowed; } + select:disabled, button:disabled { opacity: 0.5; cursor: not-allowed; } + button:hover:not(:disabled) { background: #3a3a6a; } button.stop { background: #c0392b; border-color: #e74c3c; } - button.stop:hover { background: #e74c3c; } + button.stop:hover:not(:disabled) { background: #e74c3c; } .status { display: flex; align-items: center; @@ -84,10 +85,14 @@ max-height: calc(100vh - 120px); object-fit: contain; } - .fps-overlay { + .overlay { position: absolute; top: 8px; right: 8px; + display: flex; + gap: 6px; + } + .overlay-badge { background: rgba(0, 0, 0, 0.7); padding: 4px 8px; border-radius: 4px; @@ -95,11 +100,13 @@ font-family: monospace; color: #2ecc71; } + .overlay-badge.mode { color: #3498db; } .placeholder { text-align: center; color: #666; } .placeholder p { margin-top: 8px; font-size: 14px; } + label { font-size: 13px; color: #999; } @@ -109,6 +116,14 @@

Orbit Stream

+
+ + +
@@ -122,7 +137,10 @@

Orbit Stream

Select a booted device and click Start to begin streaming.

@@ -134,6 +152,7 @@

Orbit Stream

const WS_URL = `ws://${location.hostname}:${location.port}/orbit/ws`; const deviceSelect = document.getElementById('deviceSelect'); + const modeSelect = document.getElementById('modeSelect'); const refreshBtn = document.getElementById('refreshBtn'); const startBtn = document.getElementById('startBtn'); const stopBtn = document.getElementById('stopBtn'); @@ -143,10 +162,13 @@

Orbit Stream

const canvasContainer = document.getElementById('canvasContainer'); const placeholder = document.getElementById('placeholder'); const fpsOverlay = document.getElementById('fpsOverlay'); + const modeOverlay = document.getElementById('modeOverlay'); const ctx = canvas.getContext('2d'); let ws = null; let streaming = false; + let activeCaptureMode = null; + let activePlatform = null; let frameCount = 0; let lastFpsUpdate = performance.now(); @@ -176,9 +198,12 @@

Orbit Stream

const data = await res.json(); deviceSelect.innerHTML = ''; - if (!data.ok || !Array.isArray(data.devices)) return; + if (data.error || !Array.isArray(data)) { + console.error('Device fetch error:', data.error || 'unexpected response'); + return; + } - for (const device of data.devices) { + for (const device of data) { // Only show booted devices if (device.state !== 'Booted') continue; @@ -196,20 +221,30 @@

Orbit Stream

deviceSelect.addEventListener('change', () => { startBtn.disabled = !deviceSelect.value; + // Show/hide h264 option based on platform + if (deviceSelect.value) { + const { platform } = JSON.parse(deviceSelect.value); + const h264Opt = modeSelect.querySelector('option[value="h264"]'); + // H264 only available for Android + h264Opt.disabled = platform === 'ios'; + if (platform === 'ios' && modeSelect.value === 'h264') { + modeSelect.value = 'auto'; + } + } }); refreshBtn.addEventListener('click', fetchDevices); startBtn.addEventListener('click', () => { const selected = JSON.parse(deviceSelect.value); - startStreaming(selected.deviceId, selected.platform); + startStreaming(selected.deviceId, selected.platform, modeSelect.value); }); stopBtn.addEventListener('click', () => { stopStreaming(); }); - function startStreaming(deviceId, platform) { + function startStreaming(deviceId, platform, captureMode) { if (ws) ws.close(); ws = new WebSocket(WS_URL); @@ -217,7 +252,7 @@

Orbit Stream

ws.onopen = () => { setStatus('connected', 'Connected'); - ws.send(JSON.stringify({ type: 'start', deviceId, platform })); + ws.send(JSON.stringify({ type: 'start', deviceId, platform, captureMode })); }; ws.onmessage = (event) => { @@ -225,12 +260,18 @@

Orbit Stream

const msg = JSON.parse(event.data); if (msg.type === 'started') { streaming = true; + activeCaptureMode = msg.captureMode || 'mjpeg'; + activePlatform = msg.platform; startBtn.disabled = true; stopBtn.disabled = false; deviceSelect.disabled = true; + modeSelect.disabled = true; canvasContainer.style.display = 'block'; placeholder.style.display = 'none'; - setStatus('streaming', `Streaming ${platform === 'ios' ? 'iOS' : 'Android'}`); + const modeLabel = activeCaptureMode === 'h264' ? 'H.264' : 'MJPEG'; + const platformLabel = activePlatform === 'ios' ? 'iOS' : 'Android'; + modeOverlay.textContent = `${platformLabel} ${modeLabel}`; + setStatus('streaming', `Streaming ${platformLabel}`); frameCount = 0; lastFpsUpdate = performance.now(); requestAnimationFrame(updateFps); @@ -243,21 +284,10 @@

Orbit Stream

} // Binary frame - if (platform === 'ios') { - // JPEG frame - render to canvas - const blob = new Blob([event.data], { type: 'image/jpeg' }); - createImageBitmap(blob).then((bmp) => { - if (canvas.width !== bmp.width || canvas.height !== bmp.height) { - canvas.width = bmp.width; - canvas.height = bmp.height; - } - ctx.drawImage(bmp, 0, 0); - bmp.close(); - frameCount++; - }); - } else { - // Android h264 - use VideoDecoder (WebCodecs API) if available + if (activeCaptureMode === 'h264') { handleH264Frame(event.data); + } else { + handleImageFrame(event.data); } }; @@ -281,23 +311,49 @@

Orbit Stream

function handleStopped() { streaming = false; + activeCaptureMode = null; + activePlatform = null; startBtn.disabled = !deviceSelect.value; stopBtn.disabled = true; deviceSelect.disabled = false; + modeSelect.disabled = false; setStatus('', 'Stopped'); + cleanupH264Decoder(); + } + + // MJPEG / PNG image frame handler + function handleImageFrame(data) { + const blob = new Blob([data], { type: 'image/png' }); + createImageBitmap(blob).then((bmp) => { + if (canvas.width !== bmp.width || canvas.height !== bmp.height) { + canvas.width = bmp.width; + canvas.height = bmp.height; + } + ctx.drawImage(bmp, 0, 0); + bmp.close(); + frameCount++; + }).catch(() => { + // Corrupted frame, skip + }); } // H264 decoding for Android streams via WebCodecs let videoDecoder = null; + function cleanupH264Decoder() { + if (videoDecoder && videoDecoder.state !== 'closed') { + try { videoDecoder.close(); } catch {} + } + videoDecoder = null; + } + function handleH264Frame(data) { if (!('VideoDecoder' in window)) { - // Fallback: WebCodecs not available - console.warn('WebCodecs API not available - Android h264 streaming requires a Chromium-based browser'); + console.warn('WebCodecs API not available - H.264 mode requires a Chromium-based browser'); return; } - if (!videoDecoder) { + if (!videoDecoder || videoDecoder.state === 'closed') { videoDecoder = new VideoDecoder({ output: (frame) => { if (canvas.width !== frame.displayWidth || canvas.height !== frame.displayHeight) { @@ -308,7 +364,10 @@

Orbit Stream

frame.close(); frameCount++; }, - error: (err) => console.error('VideoDecoder error:', err), + error: (err) => { + console.error('VideoDecoder error:', err); + cleanupH264Decoder(); + }, }); videoDecoder.configure({ codec: 'avc1.42E01E', @@ -316,23 +375,67 @@

Orbit Stream

}); } - // Detect keyframes by checking NAL unit type (first byte after start code) + // Parse NAL units and detect keyframes const view = new Uint8Array(data); - let nalType = 0; - if (view[0] === 0 && view[1] === 0 && view[2] === 0 && view[3] === 1) { - nalType = view[4] & 0x1f; - } else if (view[0] === 0 && view[1] === 0 && view[2] === 1) { - nalType = view[3] & 0x1f; + const nalUnits = parseNALUnits(view); + + for (const nal of nalUnits) { + const nalType = nal[0] & 0x1f; + // Skip SPS (7) and PPS (8) - they're part of the keyframe + // Types: 5=IDR (keyframe), 1=non-IDR (delta), 7=SPS, 8=PPS + const isKeyframe = nalType === 5 || nalType === 7; + + try { + const chunk = new EncodedVideoChunk({ + type: isKeyframe ? 'key' : 'delta', + timestamp: performance.now() * 1000, + data: nal.buffer, + }); + if (videoDecoder.state === 'configured') { + videoDecoder.decode(chunk); + } + } catch (e) { + // Skip malformed chunks + } } - const isKeyframe = nalType === 5 || nalType === 7; + } - const chunk = new EncodedVideoChunk({ - type: isKeyframe ? 'key' : 'delta', - timestamp: performance.now() * 1000, - data: data, - }); + // Parse Annex B byte stream into individual NAL units + function parseNALUnits(data) { + const nals = []; + let i = 0; + + while (i < data.length - 3) { + // Find start code (0x000001 or 0x00000001) + let startCodeLen = 0; + if (data[i] === 0 && data[i+1] === 0 && data[i+2] === 1) { + startCodeLen = 3; + } else if (data[i] === 0 && data[i+1] === 0 && data[i+2] === 0 && i+3 < data.length && data[i+3] === 1) { + startCodeLen = 4; + } else { + i++; + continue; + } + + const nalStart = i + startCodeLen; + i = nalStart; + + // Find next start code + while (i < data.length - 2) { + if (data[i] === 0 && data[i+1] === 0 && + (data[i+2] === 1 || (data[i+2] === 0 && i+3 < data.length && data[i+3] === 1))) { + break; + } + i++; + } + + const nalEnd = (i < data.length - 2) ? i : data.length; + if (nalEnd > nalStart) { + nals.push(data.subarray(nalStart, nalEnd)); + } + } - videoDecoder.decode(chunk); + return nals; } // Initial device load From a3198f38d4315097227bc8791ab01002f8adb2eb Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Apr 2026 06:11:44 +0000 Subject: [PATCH 4/6] [menu-bar] Externalize ws optional deps to fix Vite bundling MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bufferutil and utf-8-validate are optional native dependencies of the ws package. Vite fails to bundle them since they're not installed. Mark them as external so Vite skips them — ws works fine without them. https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- apps/menu-bar/electron/vite.main.config.mts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/menu-bar/electron/vite.main.config.mts b/apps/menu-bar/electron/vite.main.config.mts index 4cbe2551..3beb53fb 100644 --- a/apps/menu-bar/electron/vite.main.config.mts +++ b/apps/menu-bar/electron/vite.main.config.mts @@ -16,6 +16,9 @@ export default defineConfig({ commonjsOptions: { include: [/common-types/, /node_modules/], }, + rollupOptions: { + external: ['bufferutil', 'utf-8-validate'], + }, }, plugins: [ viteStaticCopy({ From 04ee0225694ed36d00c71c81edd3f253e8dd4897 Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Apr 2026 06:19:36 +0000 Subject: [PATCH 5/6] [menu-bar] Import CoreGraphics in main.swift for CGWindowID type https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- .../helpers/simulator-stream/Sources/SimulatorStream/main.swift | 1 + 1 file changed, 1 insertion(+) diff --git a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift index 87691837..a1d2f1e2 100644 --- a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift +++ b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift @@ -1,3 +1,4 @@ +import CoreGraphics import Foundation // MARK: - Logging (to stderr so it doesn't interfere with frame output on stdout) From a0a8a2d1e89519238716741b9ed1d8345b22da8e Mon Sep 17 00:00:00 2001 From: Claude Date: Wed, 1 Apr 2026 06:26:11 +0000 Subject: [PATCH 6/6] [menu-bar] Add WebSocket test page at /orbit/test-ws Simple debug page with manual WS controls, REST endpoint buttons, raw message sending, log output, and a canvas preview for frames. https://claude.ai/code/session_01RYPF9YAuXf4XrPEg8XASRi --- apps/menu-bar/electron/src/LocalServer.ts | 5 + .../menu-bar/electron/src/static/test-ws.html | 164 ++++++++++++++++++ 2 files changed, 169 insertions(+) create mode 100644 apps/menu-bar/electron/src/static/test-ws.html diff --git a/apps/menu-bar/electron/src/LocalServer.ts b/apps/menu-bar/electron/src/LocalServer.ts index 6a0e6490..21e13284 100644 --- a/apps/menu-bar/electron/src/LocalServer.ts +++ b/apps/menu-bar/electron/src/LocalServer.ts @@ -75,6 +75,11 @@ export class LocalServer { this.app.get('/orbit/stream', (_, res) => { res.sendFile(path.join(__dirname, './static/stream.html')); }); + + // WebSocket test page + this.app.get('/orbit/test-ws', (_, res) => { + res.sendFile(path.join(__dirname, './static/test-ws.html')); + }); } setupWebSocket(server: http.Server) { diff --git a/apps/menu-bar/electron/src/static/test-ws.html b/apps/menu-bar/electron/src/static/test-ws.html new file mode 100644 index 00000000..1620f08a --- /dev/null +++ b/apps/menu-bar/electron/src/static/test-ws.html @@ -0,0 +1,164 @@ + + + + + Orbit Stream - WS Test + + + +

Orbit WebSocket Test

+ +
+ + + + + + +
+ +

REST

+
+ + +
+ +

Stream Control

+
+ + + + + + +
+
+ + + + +
+ +

Log

+
+ +

Preview (MJPEG frames)

+ + + + +