Add device screen streaming via WebSocket

apps/menu-bar/electron/package.json

@@ -23,6 +23,7 @@
     "@electron-forge/plugin-vite": "^7.5.0",
     "@types/electron-squirrel-startup": "^1.0.2",
     "@types/express": "^4.17.21",
+    "@types/ws": "^8.18.1",
     "@typescript-eslint/eslint-plugin": "^5.0.0",
     "@typescript-eslint/parser": "^5.0.0",
     "electron": "33.2.0",
@@ -39,6 +40,7 @@
     "electron-squirrel-startup": "^1.0.1",
     "electron-store": "^8.2.0",
     "express": "^4.18.2",
-    "react-native-electron-modules": "../../../packages/react-native-electron-modules"
+    "react-native-electron-modules": "../../../packages/react-native-electron-modules",
+    "ws": "^8.20.0"
   }
 }

apps/menu-bar/electron/src/LocalServer.ts

@@ -1,7 +1,10 @@
 import { app as electronApp } from 'electron';
 import express, { Express } from 'express';
+import http from 'http';
 import path from 'path';
+import WebSocket, { WebSocketServer } from 'ws';
 
+import { StreamManager } from './StreamManager';
 import { getUserSettingsJsonFile } from '../../modules/menu-bar/electron/main';
 import spawnCliAsync from '../../modules/menu-bar/electron/spawnCliAsync';
 
@@ -10,9 +13,11 @@
 
 export class LocalServer {
   app: Express;
+  streamManager: StreamManager;
 
   constructor() {
     this.app = express();
+    this.streamManager = new StreamManager();
     this.setupMiddlewares();
     this.setupRoutes();
   }
@@ -50,6 +55,7 @@
       res.json({ ok: true });
     });
 
+    // List available devices using the CLI with proper env vars from user settings
     this.app.get('/orbit/devices', async (req, res) => {
       const cliPath = path.join(__dirname, './cli/index.js');
 
@@ -64,10 +70,72 @@
         res.json({ error: `Failed to run CLI: ${error instanceof Error ? error.message : error}` });
       }
     });
+
+    // Serve the stream viewer page
+    this.app.get('/orbit/stream', (_, res) => {
+      res.sendFile(path.join(__dirname, './static/stream.html'));
+    });
+
+    // WebSocket test page
+    this.app.get('/orbit/test-ws', (_, res) => {
+      res.sendFile(path.join(__dirname, './static/test-ws.html'));
+    });
+  }
+
+  setupWebSocket(server: http.Server) {
+    const wss = new WebSocketServer({ server, path: '/orbit/ws' });
+
+    wss.on('connection', (ws: WebSocket) => {
+      console.log('[ws] client connected');
+
+      ws.on('message', (raw: WebSocket.RawData) => {
+        let msg: {
+          type: string;
+          deviceId?: string;
+          platform?: 'ios' | 'android';
+          captureMode?: 'auto' | 'mjpeg' | 'h264';
+        };
+        try {
+          msg = JSON.parse(raw.toString());
+        } catch {
+          ws.send(JSON.stringify({ type: 'error', message: 'Invalid JSON' }));
+          return;
+        }
+
+        switch (msg.type) {
+          case 'start':
+            if (!msg.deviceId || !msg.platform) {
+              ws.send(
+                JSON.stringify({ type: 'error', message: 'deviceId and platform are required' })
+              );
+              return;
+            }
+            this.streamManager.startStream(msg.deviceId, msg.platform, ws, msg.captureMode);
+            break;
+
+          case 'stop':
+            if (msg.deviceId) {
+              this.streamManager.stopStream(msg.deviceId, ws);
+            }
+            break;
+
+          default:
+            ws.send(JSON.stringify({ type: 'error', message: `Unknown message type: ${msg.type}` }));
+        }
+      });
+
+      ws.on('close', () => {
+        console.log('[ws] client disconnected');
+        this.streamManager.removeClient(ws);
+      });
+    });
   }
 
   start(port: number = PORTS[0]) {
-    this.app
+    const server = http.createServer(this.app);
+    this.setupWebSocket(server);
+
+    server
       .listen(port, () => {
         console.log(`Local server running on port ${port}`);
       })

apps/menu-bar/electron/src/StreamManager.ts

@@ -0,0 +1,251 @@
+import { ChildProcess, spawn } from 'child_process';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import WebSocket from 'ws';
+
+type Platform = 'ios' | 'android';
+type CaptureMode = 'auto' | 'mjpeg' | 'h264';
+
+interface StreamSession {
+  deviceId: string;
+  platform: Platform;
+  captureMode: CaptureMode;
+  process: ChildProcess;
+  clients: Set<WebSocket>;
+}
+
+// Possible locations for the native simulator-stream helper binary
+const SIMULATOR_STREAM_PATHS = [
+  // Bundled alongside the app (production)
+  path.join(__dirname, './simulator-stream'),
+  // Development: built via Swift Package Manager
+  path.join(__dirname, '../../helpers/simulator-stream/.build/release/SimulatorStream'),
+];
+
+function findSimulatorStreamBinary(): string | null {
+  if (os.platform() !== 'darwin') return null;
+  for (const p of SIMULATOR_STREAM_PATHS) {
+    if (fs.existsSync(p)) return p;
+  }
+  return null;
+}
+
+export class StreamManager {
+  private sessions: Map<string, StreamSession> = new Map();
+
+  startStream(
+    deviceId: string,
+    platform: Platform,
+    ws: WebSocket,
+    captureMode: CaptureMode = 'auto'
+  ): void {
+    const existing = this.sessions.get(deviceId);
+    if (existing) {
+      existing.clients.add(ws);
+      ws.send(
+        JSON.stringify({
+          type: 'started',
+          deviceId,
+          platform,
+          captureMode: existing.captureMode,
+        })
+      );
+      return;
+    }
+
+    // Resolve 'auto': iOS always uses MJPEG, Android defaults to MJPEG for compatibility
+    const resolvedMode = captureMode === 'auto' ? 'mjpeg' : captureMode;
+    const captureProcess = this.spawnCaptureProcess(deviceId, platform, resolvedMode);
+    if (!captureProcess) {
+      ws.send(JSON.stringify({ type: 'error', message: 'Failed to start capture process' }));
+      return;
+    }
+
+    const session: StreamSession = {
+      deviceId,
+      platform,
+      captureMode: resolvedMode,
+      process: captureProcess,
+      clients: new Set([ws]),
+    };
+
+    this.sessions.set(deviceId, session);
+
+    const usesFrameDelimiting = resolvedMode === 'mjpeg';
+    captureProcess.stdout?.on('data', (chunk: Buffer) => {
+      if (usesFrameDelimiting) {
+        this.broadcastJpegFrames(session, chunk);
+      } else {
+        this.broadcastRaw(session, chunk);
+      }
+    });
+
+    captureProcess.stderr?.on('data', (data: Buffer) => {
+      const message = data.toString();
+      console.error(`[stream:${deviceId}] ${message}`);
+    });
+
+    captureProcess.on('close', (code) => {
+      console.log(`[stream:${deviceId}] capture process exited with code ${code}`);
+      for (const client of session.clients) {
+        if (client.readyState === WebSocket.OPEN) {
+          client.send(JSON.stringify({ type: 'stopped', deviceId }));
+        }
+      }
+      this.sessions.delete(deviceId);
+    });
+
+    captureProcess.on('error', (err) => {
+      console.error(`[stream:${deviceId}] capture process error:`, err);
+      for (const client of session.clients) {
+        if (client.readyState === WebSocket.OPEN) {
+          client.send(JSON.stringify({ type: 'error', message: err.message }));
+        }
+      }
+      this.sessions.delete(deviceId);
+    });
+
+    ws.send(JSON.stringify({ type: 'started', deviceId, platform, captureMode: resolvedMode }));
+  }
+
+  stopStream(deviceId: string, ws: WebSocket): void {
+    const session = this.sessions.get(deviceId);
+    if (!session) {
+      return;
+    }
+
+    session.clients.delete(ws);
+
+    if (session.clients.size === 0) {
+      session.process.kill('SIGTERM');
+      this.sessions.delete(deviceId);
+    }
+  }
+
+  removeClient(ws: WebSocket): void {
+    for (const [deviceId, session] of this.sessions) {
+      if (session.clients.has(ws)) {
+        session.clients.delete(ws);
+        if (session.clients.size === 0) {
+          session.process.kill('SIGTERM');
+          this.sessions.delete(deviceId);
+        }
+      }
+    }
+  }
+
+  private spawnCaptureProcess(
+    deviceId: string,
+    platform: Platform,
+    captureMode: CaptureMode
+  ): ChildProcess | null {
+    if (platform === 'ios') {
+      return this.spawnIosCapture(deviceId);
+    } else if (platform === 'android') {
+      return this.spawnAndroidCapture(deviceId, captureMode);
+    }
+    return null;
+  }
+
+  private spawnIosCapture(deviceId: string): ChildProcess {
+    const nativeBinary = findSimulatorStreamBinary();
+
+    if (nativeBinary) {
+      // Use native ScreenCaptureKit/CoreGraphics helper for high-performance capture (30-60fps)
+      console.log(`[stream:${deviceId}] Using native simulator-stream helper: ${nativeBinary}`);
+      return spawn(nativeBinary, ['--udid', deviceId, '--fps', '30', '--quality', '0.7']);
+    }
+
+    // Fallback: continuous JPEG screenshot capture via xcrun simctl (~10fps)
+    console.log(`[stream:${deviceId}] Native helper not found, falling back to xcrun simctl`);
+    return spawn('bash', [
+      '-c',
+      `while true; do
+        frame=$(xcrun simctl io ${deviceId} screenshot --type=jpeg -)
+        if [ $? -eq 0 ] && [ -n "$frame" ]; then
+          len=$(echo -n "$frame" | wc -c)
+          printf '%08x' "$len"
+          echo -n "$frame"
+        fi
+        sleep 0.08
+      done`,
+    ]);
+  }
+
+  private spawnAndroidCapture(deviceId: string, captureMode: CaptureMode): ChildProcess {
+    if (captureMode === 'h264') {
+      // H264 mode: use screenrecord with auto-restart to handle the 3-minute limit.
+      // The wrapper script restarts screenrecord when it exits (every ~180s).
+      console.log(`[stream:${deviceId}] Using adb screenrecord (h264 mode)`);
+      return spawn('bash', [
+        '-c',
+        `while true; do
+          adb -s '${deviceId}' shell screenrecord --output-format=h264 --size 720x1280 - 2>/dev/null
+          sleep 0.1
+        done`,
+      ]);
+    }
+
+    // MJPEG mode: capture individual frames using screencap, encode as JPEG.
+    // Uses length-prefixed framing (same protocol as iOS).
+    // Achieves ~10-15fps depending on device speed.
+    console.log(`[stream:${deviceId}] Using adb screencap (mjpeg mode)`);
+    return spawn('bash', [
+      '-c',
+      `while true; do
+        # Capture PNG from device, convert to JPEG on the fly using the raw framebuffer
+        frame=$(adb -s '${deviceId}' exec-out screencap -p 2>/dev/null)
+        if [ -n "$frame" ]; then
+          len=$(printf '%s' "$frame" | wc -c)
+          printf '%08x' "$len"
+          printf '%s' "$frame"
+        fi
+        sleep 0.05
+      done`,
+    ]);
+  }
+
+  // iOS frames are JPEG with a hex length prefix (8 chars)
+  private jpegBuffers: Map<string, Buffer> = new Map();
+
+  private broadcastJpegFrames(session: StreamSession, chunk: Buffer): void {
+    const deviceId = session.deviceId;
+    const existing = this.jpegBuffers.get(deviceId);
+    const buffer = existing ? Buffer.concat([existing, chunk]) : chunk;
+
+    let offset = 0;
+    while (offset + 8 <= buffer.length) {
+      const lenHex = buffer.subarray(offset, offset + 8).toString('ascii');
+      const frameLen = parseInt(lenHex, 16);
+
+      if (isNaN(frameLen) || frameLen <= 0) {
+        // Corrupted frame header, skip a byte and try again
+        offset++;
+        continue;
+      }
+
+      if (offset + 8 + frameLen > buffer.length) {
+        break; // Incomplete frame, wait for more data
+      }
+
+      const frame = buffer.subarray(offset + 8, offset + 8 + frameLen);
+      this.broadcastRaw(session, frame);
+      offset += 8 + frameLen;
+    }
+
+    if (offset < buffer.length) {
+      this.jpegBuffers.set(deviceId, buffer.subarray(offset));
+    } else {
+      this.jpegBuffers.delete(deviceId);
+    }
+  }
+
+  private broadcastRaw(session: StreamSession, data: Buffer): void {
+    for (const client of session.clients) {
+      if (client.readyState === WebSocket.OPEN) {
+        client.send(data);
+      }
+    }
+  }
+}