diff --git a/apps/menu-bar/electron/package.json b/apps/menu-bar/electron/package.json index 97bdda1a..516cd37a 100644 --- a/apps/menu-bar/electron/package.json +++ b/apps/menu-bar/electron/package.json @@ -23,6 +23,7 @@ "@electron-forge/plugin-vite": "^7.5.0", "@types/electron-squirrel-startup": "^1.0.2", "@types/express": "^4.17.21", + "@types/ws": "^8.18.1", "@typescript-eslint/eslint-plugin": "^5.0.0", "@typescript-eslint/parser": "^5.0.0", "electron": "33.2.0", @@ -39,6 +40,7 @@ "electron-squirrel-startup": "^1.0.1", "electron-store": "^8.2.0", "express": "^4.18.2", - "react-native-electron-modules": "../../../packages/react-native-electron-modules" + "react-native-electron-modules": "../../../packages/react-native-electron-modules", + "ws": "^8.20.0" } } diff --git a/apps/menu-bar/electron/src/LocalServer.ts b/apps/menu-bar/electron/src/LocalServer.ts index b5dce5b4..21e13284 100644 --- a/apps/menu-bar/electron/src/LocalServer.ts +++ b/apps/menu-bar/electron/src/LocalServer.ts @@ -1,7 +1,10 @@ import { app as electronApp } from 'electron'; import express, { Express } from 'express'; +import http from 'http'; import path from 'path'; +import WebSocket, { WebSocketServer } from 'ws'; +import { StreamManager } from './StreamManager'; import { getUserSettingsJsonFile } from '../../modules/menu-bar/electron/main'; import spawnCliAsync from '../../modules/menu-bar/electron/spawnCliAsync'; @@ -10,9 +13,11 @@ const WHITELISTED_DOMAINS = ['expo.dev', 'expo.test', 'exp.host', 'localhost']; export class LocalServer { app: Express; + streamManager: StreamManager; constructor() { this.app = express(); + this.streamManager = new StreamManager(); this.setupMiddlewares(); this.setupRoutes(); } @@ -50,6 +55,7 @@ export class LocalServer { res.json({ ok: true }); }); + // List available devices using the CLI with proper env vars from user settings this.app.get('/orbit/devices', async (req, res) => { const cliPath = path.join(__dirname, './cli/index.js'); @@ -64,10 +70,72 @@ export class LocalServer { res.json({ error: `Failed to run CLI: ${error instanceof Error ? error.message : error}` }); } }); + + // Serve the stream viewer page + this.app.get('/orbit/stream', (_, res) => { + res.sendFile(path.join(__dirname, './static/stream.html')); + }); + + // WebSocket test page + this.app.get('/orbit/test-ws', (_, res) => { + res.sendFile(path.join(__dirname, './static/test-ws.html')); + }); + } + + setupWebSocket(server: http.Server) { + const wss = new WebSocketServer({ server, path: '/orbit/ws' }); + + wss.on('connection', (ws: WebSocket) => { + console.log('[ws] client connected'); + + ws.on('message', (raw: WebSocket.RawData) => { + let msg: { + type: string; + deviceId?: string; + platform?: 'ios' | 'android'; + captureMode?: 'auto' | 'mjpeg' | 'h264'; + }; + try { + msg = JSON.parse(raw.toString()); + } catch { + ws.send(JSON.stringify({ type: 'error', message: 'Invalid JSON' })); + return; + } + + switch (msg.type) { + case 'start': + if (!msg.deviceId || !msg.platform) { + ws.send( + JSON.stringify({ type: 'error', message: 'deviceId and platform are required' }) + ); + return; + } + this.streamManager.startStream(msg.deviceId, msg.platform, ws, msg.captureMode); + break; + + case 'stop': + if (msg.deviceId) { + this.streamManager.stopStream(msg.deviceId, ws); + } + break; + + default: + ws.send(JSON.stringify({ type: 'error', message: `Unknown message type: ${msg.type}` })); + } + }); + + ws.on('close', () => { + console.log('[ws] client disconnected'); + this.streamManager.removeClient(ws); + }); + }); } start(port: number = PORTS[0]) { - this.app + const server = http.createServer(this.app); + this.setupWebSocket(server); + + server .listen(port, () => { console.log(`Local server running on port ${port}`); }) diff --git a/apps/menu-bar/electron/src/StreamManager.ts b/apps/menu-bar/electron/src/StreamManager.ts new file mode 100644 index 00000000..d45f2608 --- /dev/null +++ b/apps/menu-bar/electron/src/StreamManager.ts @@ -0,0 +1,251 @@ +import { ChildProcess, spawn } from 'child_process'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import WebSocket from 'ws'; + +type Platform = 'ios' | 'android'; +type CaptureMode = 'auto' | 'mjpeg' | 'h264'; + +interface StreamSession { + deviceId: string; + platform: Platform; + captureMode: CaptureMode; + process: ChildProcess; + clients: Set; +} + +// Possible locations for the native simulator-stream helper binary +const SIMULATOR_STREAM_PATHS = [ + // Bundled alongside the app (production) + path.join(__dirname, './simulator-stream'), + // Development: built via Swift Package Manager + path.join(__dirname, '../../helpers/simulator-stream/.build/release/SimulatorStream'), +]; + +function findSimulatorStreamBinary(): string | null { + if (os.platform() !== 'darwin') return null; + for (const p of SIMULATOR_STREAM_PATHS) { + if (fs.existsSync(p)) return p; + } + return null; +} + +export class StreamManager { + private sessions: Map = new Map(); + + startStream( + deviceId: string, + platform: Platform, + ws: WebSocket, + captureMode: CaptureMode = 'auto' + ): void { + const existing = this.sessions.get(deviceId); + if (existing) { + existing.clients.add(ws); + ws.send( + JSON.stringify({ + type: 'started', + deviceId, + platform, + captureMode: existing.captureMode, + }) + ); + return; + } + + // Resolve 'auto': iOS always uses MJPEG, Android defaults to MJPEG for compatibility + const resolvedMode = captureMode === 'auto' ? 'mjpeg' : captureMode; + const captureProcess = this.spawnCaptureProcess(deviceId, platform, resolvedMode); + if (!captureProcess) { + ws.send(JSON.stringify({ type: 'error', message: 'Failed to start capture process' })); + return; + } + + const session: StreamSession = { + deviceId, + platform, + captureMode: resolvedMode, + process: captureProcess, + clients: new Set([ws]), + }; + + this.sessions.set(deviceId, session); + + const usesFrameDelimiting = resolvedMode === 'mjpeg'; + captureProcess.stdout?.on('data', (chunk: Buffer) => { + if (usesFrameDelimiting) { + this.broadcastJpegFrames(session, chunk); + } else { + this.broadcastRaw(session, chunk); + } + }); + + captureProcess.stderr?.on('data', (data: Buffer) => { + const message = data.toString(); + console.error(`[stream:${deviceId}] ${message}`); + }); + + captureProcess.on('close', (code) => { + console.log(`[stream:${deviceId}] capture process exited with code ${code}`); + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(JSON.stringify({ type: 'stopped', deviceId })); + } + } + this.sessions.delete(deviceId); + }); + + captureProcess.on('error', (err) => { + console.error(`[stream:${deviceId}] capture process error:`, err); + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(JSON.stringify({ type: 'error', message: err.message })); + } + } + this.sessions.delete(deviceId); + }); + + ws.send(JSON.stringify({ type: 'started', deviceId, platform, captureMode: resolvedMode })); + } + + stopStream(deviceId: string, ws: WebSocket): void { + const session = this.sessions.get(deviceId); + if (!session) { + return; + } + + session.clients.delete(ws); + + if (session.clients.size === 0) { + session.process.kill('SIGTERM'); + this.sessions.delete(deviceId); + } + } + + removeClient(ws: WebSocket): void { + for (const [deviceId, session] of this.sessions) { + if (session.clients.has(ws)) { + session.clients.delete(ws); + if (session.clients.size === 0) { + session.process.kill('SIGTERM'); + this.sessions.delete(deviceId); + } + } + } + } + + private spawnCaptureProcess( + deviceId: string, + platform: Platform, + captureMode: CaptureMode + ): ChildProcess | null { + if (platform === 'ios') { + return this.spawnIosCapture(deviceId); + } else if (platform === 'android') { + return this.spawnAndroidCapture(deviceId, captureMode); + } + return null; + } + + private spawnIosCapture(deviceId: string): ChildProcess { + const nativeBinary = findSimulatorStreamBinary(); + + if (nativeBinary) { + // Use native ScreenCaptureKit/CoreGraphics helper for high-performance capture (30-60fps) + console.log(`[stream:${deviceId}] Using native simulator-stream helper: ${nativeBinary}`); + return spawn(nativeBinary, ['--udid', deviceId, '--fps', '30', '--quality', '0.7']); + } + + // Fallback: continuous JPEG screenshot capture via xcrun simctl (~10fps) + console.log(`[stream:${deviceId}] Native helper not found, falling back to xcrun simctl`); + return spawn('bash', [ + '-c', + `while true; do + frame=$(xcrun simctl io ${deviceId} screenshot --type=jpeg -) + if [ $? -eq 0 ] && [ -n "$frame" ]; then + len=$(echo -n "$frame" | wc -c) + printf '%08x' "$len" + echo -n "$frame" + fi + sleep 0.08 + done`, + ]); + } + + private spawnAndroidCapture(deviceId: string, captureMode: CaptureMode): ChildProcess { + if (captureMode === 'h264') { + // H264 mode: use screenrecord with auto-restart to handle the 3-minute limit. + // The wrapper script restarts screenrecord when it exits (every ~180s). + console.log(`[stream:${deviceId}] Using adb screenrecord (h264 mode)`); + return spawn('bash', [ + '-c', + `while true; do + adb -s '${deviceId}' shell screenrecord --output-format=h264 --size 720x1280 - 2>/dev/null + sleep 0.1 + done`, + ]); + } + + // MJPEG mode: capture individual frames using screencap, encode as JPEG. + // Uses length-prefixed framing (same protocol as iOS). + // Achieves ~10-15fps depending on device speed. + console.log(`[stream:${deviceId}] Using adb screencap (mjpeg mode)`); + return spawn('bash', [ + '-c', + `while true; do + # Capture PNG from device, convert to JPEG on the fly using the raw framebuffer + frame=$(adb -s '${deviceId}' exec-out screencap -p 2>/dev/null) + if [ -n "$frame" ]; then + len=$(printf '%s' "$frame" | wc -c) + printf '%08x' "$len" + printf '%s' "$frame" + fi + sleep 0.05 + done`, + ]); + } + + // iOS frames are JPEG with a hex length prefix (8 chars) + private jpegBuffers: Map = new Map(); + + private broadcastJpegFrames(session: StreamSession, chunk: Buffer): void { + const deviceId = session.deviceId; + const existing = this.jpegBuffers.get(deviceId); + const buffer = existing ? Buffer.concat([existing, chunk]) : chunk; + + let offset = 0; + while (offset + 8 <= buffer.length) { + const lenHex = buffer.subarray(offset, offset + 8).toString('ascii'); + const frameLen = parseInt(lenHex, 16); + + if (isNaN(frameLen) || frameLen <= 0) { + // Corrupted frame header, skip a byte and try again + offset++; + continue; + } + + if (offset + 8 + frameLen > buffer.length) { + break; // Incomplete frame, wait for more data + } + + const frame = buffer.subarray(offset + 8, offset + 8 + frameLen); + this.broadcastRaw(session, frame); + offset += 8 + frameLen; + } + + if (offset < buffer.length) { + this.jpegBuffers.set(deviceId, buffer.subarray(offset)); + } else { + this.jpegBuffers.delete(deviceId); + } + } + + private broadcastRaw(session: StreamSession, data: Buffer): void { + for (const client of session.clients) { + if (client.readyState === WebSocket.OPEN) { + client.send(data); + } + } + } +} diff --git a/apps/menu-bar/electron/src/static/stream.html b/apps/menu-bar/electron/src/static/stream.html new file mode 100644 index 00000000..14f52d5e --- /dev/null +++ b/apps/menu-bar/electron/src/static/stream.html @@ -0,0 +1,445 @@ + + + + + + Orbit - Device Stream + + + +
+

Orbit Stream

+
+ +
+ + +
+ + + +
+
+
+ Disconnected +
+
+ +
+ +
+

Select a booted device and click Start to begin streaming.

+
+
+ + + + diff --git a/apps/menu-bar/electron/src/static/test-ws.html b/apps/menu-bar/electron/src/static/test-ws.html new file mode 100644 index 00000000..1620f08a --- /dev/null +++ b/apps/menu-bar/electron/src/static/test-ws.html @@ -0,0 +1,164 @@ + + + + + Orbit Stream - WS Test + + + +

Orbit WebSocket Test

+ +
+ + + + + + +
+ +

REST

+
+ + +
+ +

Stream Control

+
+ + + + + + +
+
+ + + + +
+ +

Log

+
+ +

Preview (MJPEG frames)

+ + + + + diff --git a/apps/menu-bar/electron/vite.main.config.mts b/apps/menu-bar/electron/vite.main.config.mts index 186deab8..3beb53fb 100644 --- a/apps/menu-bar/electron/vite.main.config.mts +++ b/apps/menu-bar/electron/vite.main.config.mts @@ -16,6 +16,9 @@ export default defineConfig({ commonjsOptions: { include: [/common-types/, /node_modules/], }, + rollupOptions: { + external: ['bufferutil', 'utf-8-validate'], + }, }, plugins: [ viteStaticCopy({ @@ -36,5 +39,24 @@ export default defineConfig({ ], structured: true, }), + viteStaticCopy({ + targets: [ + { + src: './src/static/**/*', + dest: 'static/', + }, + ], + }), + // Copy native simulator-stream helper binary if it exists (macOS only) + viteStaticCopy({ + targets: [ + { + src: '../helpers/simulator-stream/.build/release/SimulatorStream', + dest: './', + rename: 'simulator-stream', + }, + ], + silent: true, + }), ], }); diff --git a/apps/menu-bar/electron/yarn.lock b/apps/menu-bar/electron/yarn.lock index 0ef9974a..59b737e0 100644 --- a/apps/menu-bar/electron/yarn.lock +++ b/apps/menu-bar/electron/yarn.lock @@ -932,6 +932,13 @@ "@types/mime" "*" "@types/node" "*" +"@types/ws@^8.18.1": + version "8.18.1" + resolved "https://registry.yarnpkg.com/@types/ws/-/ws-8.18.1.tgz#48464e4bf2ddfd17db13d845467f6070ffea4aa9" + integrity sha512-ThVF6DCVhA8kUGy+aazFQ4kXQ7E1Ty7A3ypFOe0IcJV8O/M511G99AW24irKrW56Wt44yG9+ij8FaqoBGkuBXg== + dependencies: + "@types/node" "*" + "@types/yauzl@^2.9.1": version "2.10.3" resolved "https://registry.yarnpkg.com/@types/yauzl/-/yauzl-2.10.3.tgz#e9b2808b4f109504a03cda958259876f61017999" @@ -5104,6 +5111,11 @@ ws@^7.4.6: resolved "https://registry.yarnpkg.com/ws/-/ws-7.5.9.tgz#54fa7db29f4c7cec68b1ddd3a89de099942bb591" integrity sha512-F+P9Jil7UiSKSkppIiD94dN07AwvFixvLIj1Og1Rl9GGMuNipJnV9JzjD6XuqmAeiswGvUmNLjr5cFuXwNS77Q== +ws@^8.20.0: + version "8.20.0" + resolved "https://registry.yarnpkg.com/ws/-/ws-8.20.0.tgz#4cd9532358eba60bc863aad1623dfb045a4d4af8" + integrity sha512-sAt8BhgNbzCtgGbt2OxmpuryO63ZoDk/sqaB/znQm94T4fCEsy/yV+7CdC1kJhOU9lboAEU7R3kquuycDoibVA== + xmlbuilder@^15.1.1: version "15.1.1" resolved "https://registry.yarnpkg.com/xmlbuilder/-/xmlbuilder-15.1.1.tgz#9dcdce49eea66d8d10b42cae94a79c3c8d0c2ec5" diff --git a/apps/menu-bar/helpers/simulator-stream/Package.swift b/apps/menu-bar/helpers/simulator-stream/Package.swift new file mode 100644 index 00000000..e44f7a4e --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Package.swift @@ -0,0 +1,20 @@ +// swift-tools-version: 5.9 +import PackageDescription + +let package = Package( + name: "SimulatorStream", + platforms: [.macOS(.v13)], + targets: [ + .executableTarget( + name: "SimulatorStream", + path: "Sources/SimulatorStream", + linkerSettings: [ + .linkedFramework("CoreGraphics"), + .linkedFramework("ScreenCaptureKit"), + .linkedFramework("AppKit"), + .linkedFramework("ImageIO"), + .linkedFramework("CoreImage"), + ] + ), + ] +) diff --git a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift new file mode 100644 index 00000000..6acb12b6 --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/ScreenCapture.swift @@ -0,0 +1,253 @@ +import AppKit +import CoreGraphics +import Foundation +import ImageIO +import ScreenCaptureKit + +// MARK: - Frame Output + +/// Writes a length-prefixed JPEG frame to stdout. +/// Format: 8-byte hex length prefix + raw JPEG data +func writeFrame(_ jpegData: Data) { + let lenHex = String(format: "%08x", jpegData.count) + guard let lenData = lenHex.data(using: .ascii) else { return } + + let stdout = FileHandle.standardOutput + stdout.write(lenData) + stdout.write(jpegData) +} + +/// Encodes a CGImage to JPEG data at the given quality (0.0-1.0). +func encodeJPEG(_ image: CGImage, quality: CGFloat = 0.7) -> Data? { + let data = NSMutableData() + guard let dest = CGImageDestinationCreateWithData( + data as CFMutableData, + "public.jpeg" as CFString, + 1, + nil + ) else { return nil } + + let options: [CFString: Any] = [ + kCGImageDestinationLossyCompressionQuality: quality + ] + CGImageDestinationAddImage(dest, image, options as CFDictionary) + + guard CGImageDestinationFinalize(dest) else { return nil } + return data as Data +} + +// MARK: - Window Discovery + +/// Find the Simulator window matching the given device UDID. +/// Simulator.app window titles follow the pattern: "DeviceName – iOS X.Y (UDID)" +func findSimulatorWindow(forUDID udid: String) -> CGWindowID? { + guard let windowList = CGWindowListCopyWindowInfo( + [.optionOnScreenOnly, .excludeDesktopElements], + kCGNullWindowID + ) as? [[String: Any]] else { + return nil + } + + for window in windowList { + guard let ownerName = window[kCGWindowOwnerName as String] as? String, + ownerName == "Simulator", + let windowName = window[kCGWindowName as String] as? String, + let windowID = window[kCGWindowNumber as String] as? CGWindowID + else { continue } + + // Match by UDID in window title or by finding any Simulator window + if windowName.contains(udid) || udid == "booted" { + return windowID + } + } + + // Fallback: return first Simulator window found + for window in windowList { + guard let ownerName = window[kCGWindowOwnerName as String] as? String, + ownerName == "Simulator", + let windowID = window[kCGWindowNumber as String] as? CGWindowID, + let bounds = window[kCGWindowBounds as String] as? [String: Any], + let width = bounds["Width"] as? CGFloat, + width > 100 // Skip tiny/toolbar windows + else { continue } + + return windowID + } + + return nil +} + +/// Get window bounds for a given window ID. +func getWindowBounds(_ windowID: CGWindowID) -> CGRect? { + guard let windowList = CGWindowListCopyWindowInfo( + [.optionIncludingWindow], + windowID + ) as? [[String: Any]], + let window = windowList.first, + let boundsDict = window[kCGWindowBounds as String] as? [String: Any] + else { return nil } + + let bounds = CGRect( + x: boundsDict["X"] as? CGFloat ?? 0, + y: boundsDict["Y"] as? CGFloat ?? 0, + width: boundsDict["Width"] as? CGFloat ?? 0, + height: boundsDict["Height"] as? CGFloat ?? 0 + ) + return bounds +} + +// MARK: - CoreGraphics Capture (Fallback) + +/// Captures the simulator window using CoreGraphics (works on all macOS versions). +/// Achieves ~20-30fps depending on window size. +func captureWithCoreGraphics(windowID: CGWindowID) -> CGImage? { + return CGWindowListCreateImage( + .null, + .optionIncludingWindow, + windowID, + [.boundsIgnoreFraming, .bestResolution] + ) +} + +// MARK: - ScreenCaptureKit Capture (macOS 12.3+, high performance) + +/// ScreenCaptureKit-based capture that provides hardware-accelerated, +/// low-latency window capture at up to 60fps. +@available(macOS 12.3, *) +class SCStreamCapture: NSObject, SCStreamDelegate, SCStreamOutput { + private var stream: SCStream? + private var isRunning = false + private let quality: CGFloat + private let targetFPS: Int + + init(quality: CGFloat = 0.7, targetFPS: Int = 30) { + self.quality = quality + self.targetFPS = targetFPS + super.init() + } + + func start(windowID: CGWindowID) async throws { + let content = try await SCShareableContent.excludingDesktopWindows( + false, + onScreenWindowsOnly: true + ) + + guard let window = content.windows.first(where: { $0.windowID == windowID }) else { + throw CaptureError.windowNotFound + } + + let filter = SCContentFilter(desktopIndependentWindow: window) + + let config = SCStreamConfiguration() + config.width = Int(window.frame.width) * 2 // Retina + config.height = Int(window.frame.height) * 2 + config.minimumFrameInterval = CMTime(value: 1, timescale: CMTimeScale(targetFPS)) + config.queueDepth = 3 + config.showsCursor = false + config.pixelFormat = kCVPixelFormatType_32BGRA + + let stream = SCStream(filter: filter, configuration: config, delegate: self) + try stream.addStreamOutput(self, type: .screen, sampleHandlerQueue: .global(qos: .userInteractive)) + try await stream.startCapture() + + self.stream = stream + self.isRunning = true + } + + func stop() async { + guard let stream = stream else { return } + do { + try await stream.stopCapture() + } catch { + log("Error stopping capture: \(error)") + } + self.stream = nil + self.isRunning = false + } + + // SCStreamOutput: called for each captured frame + func stream( + _ stream: SCStream, + didOutputSampleBuffer sampleBuffer: CMSampleBuffer, + of type: SCStreamOutputType + ) { + guard type == .screen, + let imageBuffer = sampleBuffer.imageBuffer + else { return } + + let ciImage = CIImage(cvImageBuffer: imageBuffer) + let context = CIContext() + let width = CVPixelBufferGetWidth(imageBuffer) + let height = CVPixelBufferGetHeight(imageBuffer) + + guard let cgImage = context.createCGImage( + ciImage, + from: CGRect(x: 0, y: 0, width: width, height: height) + ) else { return } + + guard let jpegData = encodeJPEG(cgImage, quality: quality) else { return } + writeFrame(jpegData) + } + + // SCStreamDelegate: handle errors + func stream(_ stream: SCStream, didStopWithError error: Error) { + log("Stream stopped with error: \(error)") + isRunning = false + } +} + +// MARK: - CoreGraphics Loop Capture + +/// Fallback capture loop using CoreGraphics for older macOS versions. +class CGLoopCapture { + private var isRunning = false + private let quality: CGFloat + private let targetFPS: Int + + init(quality: CGFloat = 0.7, targetFPS: Int = 30) { + self.quality = quality + self.targetFPS = targetFPS + } + + func start(windowID: CGWindowID) { + isRunning = true + let interval = 1.0 / Double(targetFPS) + + DispatchQueue.global(qos: .userInteractive).async { [weak self] in + while self?.isRunning == true { + let startTime = CFAbsoluteTimeGetCurrent() + + if let image = captureWithCoreGraphics(windowID: windowID), + let jpegData = encodeJPEG(image, quality: self?.quality ?? 0.7) { + writeFrame(jpegData) + } + + let elapsed = CFAbsoluteTimeGetCurrent() - startTime + let sleepTime = max(0, interval - elapsed) + if sleepTime > 0 { + Thread.sleep(forTimeInterval: sleepTime) + } + } + } + } + + func stop() { + isRunning = false + } +} + +// MARK: - Errors + +enum CaptureError: Error, CustomStringConvertible { + case windowNotFound + case simulatorNotRunning + + var description: String { + switch self { + case .windowNotFound: + return "Simulator window not found" + case .simulatorNotRunning: + return "No running Simulator window found for the specified device" + } + } +} diff --git a/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift new file mode 100644 index 00000000..a1d2f1e2 --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/Sources/SimulatorStream/main.swift @@ -0,0 +1,161 @@ +import CoreGraphics +import Foundation + +// MARK: - Logging (to stderr so it doesn't interfere with frame output on stdout) + +func log(_ message: String) { + let data = "[simulator-stream] \(message)\n".data(using: .utf8)! + FileHandle.standardError.write(data) +} + +// MARK: - Argument Parsing + +struct StreamConfig { + let udid: String + let fps: Int + let quality: CGFloat + let useCoreGraphics: Bool + + static func parse(from args: [String]) -> StreamConfig { + var udid = "booted" + var fps = 30 + var quality: CGFloat = 0.7 + var useCoreGraphics = false + + var i = 1 // Skip executable name + while i < args.count { + switch args[i] { + case "--udid", "-u": + if i + 1 < args.count { + udid = args[i + 1] + i += 2 + } else { i += 1 } + case "--fps", "-f": + if i + 1 < args.count { + fps = Int(args[i + 1]) ?? 30 + i += 2 + } else { i += 1 } + case "--quality", "-q": + if i + 1 < args.count { + quality = CGFloat(Double(args[i + 1]) ?? 0.7) + i += 2 + } else { i += 1 } + case "--cg", "--core-graphics": + useCoreGraphics = true + i += 1 + case "--help", "-h": + printUsage() + exit(0) + default: + // If first positional arg, treat as UDID + if i == 1 && !args[i].hasPrefix("-") { + udid = args[i] + } + i += 1 + } + } + + return StreamConfig( + udid: udid, + fps: min(max(fps, 1), 60), + quality: min(max(quality, 0.1), 1.0), + useCoreGraphics: useCoreGraphics + ) + } +} + +func printUsage() { + log(""" + Usage: simulator-stream [UDID] [OPTIONS] + + Captures an iOS Simulator window and streams JPEG frames to stdout. + Each frame is prefixed with an 8-byte hex length header. + + Arguments: + UDID Simulator device UDID (default: "booted") + + Options: + --udid, -u Simulator device UDID + --fps, -f Target frames per second (1-60, default: 30) + --quality, -q <0.0-1.0> JPEG quality (default: 0.7) + --cg, --core-graphics Force CoreGraphics capture (skip ScreenCaptureKit) + --help, -h Show this help message + """) +} + +// MARK: - Signal Handling + +func setupSignalHandlers() { + signal(SIGINT) { _ in + log("Received SIGINT, shutting down...") + exit(0) + } + signal(SIGTERM) { _ in + log("Received SIGTERM, shutting down...") + exit(0) + } + // Ignore SIGPIPE (broken pipe from parent process closing stdin) + signal(SIGPIPE, SIG_IGN) +} + +// MARK: - Main + +func main() async { + setupSignalHandlers() + + let config = StreamConfig.parse(from: CommandLine.arguments) + log("Starting capture for device: \(config.udid) at \(config.fps)fps, quality: \(config.quality)") + + // Wait for the Simulator window to appear (retry for up to 10 seconds) + var windowID: CGWindowID? + for attempt in 1...20 { + windowID = findSimulatorWindow(forUDID: config.udid) + if windowID != nil { break } + log("Waiting for Simulator window (attempt \(attempt)/20)...") + try? await Task.sleep(nanoseconds: 500_000_000) // 0.5s + } + + guard let windowID = windowID else { + log("Error: Could not find Simulator window for device \(config.udid)") + exit(1) + } + + log("Found Simulator window ID: \(windowID)") + + // Use ScreenCaptureKit when available and not forced to CG + if !config.useCoreGraphics { + if #available(macOS 12.3, *) { + log("Using ScreenCaptureKit for capture (hardware-accelerated)") + let capture = SCStreamCapture(quality: config.quality, targetFPS: config.fps) + do { + try await capture.start(windowID: windowID) + log("ScreenCaptureKit stream started") + + // Keep running until terminated + while true { + try await Task.sleep(nanoseconds: 1_000_000_000) + } + } catch { + log("ScreenCaptureKit failed: \(error). Falling back to CoreGraphics.") + await capture.stop() + // Fall through to CoreGraphics + } + } + } + + // CoreGraphics fallback + log("Using CoreGraphics for capture") + let capture = CGLoopCapture(quality: config.quality, targetFPS: config.fps) + capture.start(windowID: windowID) + + // Keep running until terminated + dispatchMain() +} + +// Entry point +Task { + await main() +} + +// Keep the run loop alive +RunLoop.main.run() diff --git a/apps/menu-bar/helpers/simulator-stream/build.sh b/apps/menu-bar/helpers/simulator-stream/build.sh new file mode 100755 index 00000000..cc82c8ff --- /dev/null +++ b/apps/menu-bar/helpers/simulator-stream/build.sh @@ -0,0 +1,22 @@ +#!/bin/bash +# Build the simulator-stream helper binary for macOS. +# Produces universal binary (arm64 + x86_64) at .build/release/SimulatorStream + +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +cd "$SCRIPT_DIR" + +echo "[simulator-stream] Building..." + +# Build for the current architecture in release mode +swift build -c release 2>&1 + +# Copy binary to a predictable location +BUILD_DIR=".build/release" +if [ -f "$BUILD_DIR/SimulatorStream" ]; then + echo "[simulator-stream] Build complete: $BUILD_DIR/SimulatorStream" +else + echo "[simulator-stream] Error: Build output not found" + exit 1 +fi diff --git a/apps/menu-bar/macos/scripts/build.sh b/apps/menu-bar/macos/scripts/build.sh index 84a486ba..75e154fd 100644 --- a/apps/menu-bar/macos/scripts/build.sh +++ b/apps/menu-bar/macos/scripts/build.sh @@ -5,6 +5,12 @@ WORKSPACE_PATH='./macos/ExpoMenuBar.xcworkspace' CONFIGURATION='Debug' SCHEME='ExpoMenuBar-macOS' +# Build the native simulator-stream helper +echo "[build] Building simulator-stream helper..." +if [ -f "./helpers/simulator-stream/build.sh" ]; then + bash ./helpers/simulator-stream/build.sh || echo "[build] simulator-stream build failed (non-fatal, will use xcrun fallback)" +fi + # Build xcodebuild -workspace "$WORKSPACE_PATH" -scheme "$SCHEME" -configuration "$CONFIGURATION"