(null)
// Effect for height monitoring
useEffect(() => {
const cleanup = window.electronAPI.onResetView(() => {
console.log("Received 'reset-view' message from main process.")
queryClient.invalidateQueries(["screenshots"])
queryClient.invalidateQueries(["problem_statement"])
queryClient.invalidateQueries(["solution"])
queryClient.invalidateQueries(["new_solution"])
setView("queue")
})
return () => {
cleanup()
}
}, [])
useEffect(() => {
if (!containerRef.current) return
const updateHeight = () => {
if (!containerRef.current) return
const height = containerRef.current.scrollHeight
const width = containerRef.current.scrollWidth
window.electronAPI?.updateContentDimensions({ width, height })
}
const resizeObserver = new ResizeObserver(() => {
updateHeight()
})
// Initial height update
updateHeight()
// Observe for changes
resizeObserver.observe(containerRef.current)
// Also update height when view changes
const mutationObserver = new MutationObserver(() => {
updateHeight()
})
mutationObserver.observe(containerRef.current, {
childList: true,
subtree: true,
attributes: true,
characterData: true
})
return () => {
resizeObserver.disconnect()
mutationObserver.disconnect()
}
}, [view]) // Re-run when view changes
useEffect(() => {
const cleanupFunctions = [
window.electronAPI.onSolutionStart(() => {
setView("solutions")
console.log("starting processing")
}),
window.electronAPI.onUnauthorized(() => {
queryClient.removeQueries(["screenshots"])
queryClient.removeQueries(["solution"])
queryClient.removeQueries(["problem_statement"])
setView("queue")
console.log("Unauthorized")
}),
// Update this reset handler
window.electronAPI.onResetView(() => {
console.log("Received 'reset-view' message from main process")
queryClient.removeQueries(["screenshots"])
queryClient.removeQueries(["solution"])
queryClient.removeQueries(["problem_statement"])
setView("queue")
console.log("View reset to 'queue' via Command+R shortcut")
}),
window.electronAPI.onProblemExtracted((data: any) => {
if (view === "queue") {
console.log("Problem extracted successfully")
queryClient.invalidateQueries(["problem_statement"])
queryClient.setQueryData(["problem_statement"], data)
}
})
]
return () => cleanupFunctions.forEach((cleanup) => cleanup())
}, [])
return (
{view === "queue" ? (
) : view === "solutions" ? (
) : (
<>
)}
)
}
export default App
```
### frontend/electron/main.ts
```typescript
import { app, BrowserWindow, Tray, Menu, nativeImage } from "electron"
import { initializeIpcHandlers } from "./ipcHandlers"
import { WindowHelper } from "./WindowHelper"
import { ScreenshotHelper } from "./ScreenshotHelper"
import { ShortcutsHelper } from "./shortcuts"
import { ProcessingHelper } from "./ProcessingHelper"
import { ApiServerHelper } from "./ApiServerHelper"
export class AppState {
private static instance: AppState | null = null
private windowHelper: WindowHelper
private screenshotHelper: ScreenshotHelper
public shortcutsHelper: ShortcutsHelper
public processingHelper: ProcessingHelper
public apiServerHelper: ApiServerHelper
private tray: Tray | null = null
// View management
private view: "queue" | "solutions" = "queue"
private problemInfo: {
problem_statement: string
input_format: Record
output_format: Record
constraints: Array>
test_cases: Array>
} | null = null // Allow null
private hasDebugged: boolean = false
// Processing events
public readonly PROCESSING_EVENTS = {
//global states
UNAUTHORIZED: "procesing-unauthorized",
NO_SCREENSHOTS: "processing-no-screenshots",
//states for generating the initial solution
INITIAL_START: "initial-start",
PROBLEM_EXTRACTED: "problem-extracted",
SOLUTION_SUCCESS: "solution-success",
INITIAL_SOLUTION_ERROR: "solution-error",
//states for processing the debugging
DEBUG_START: "debug-start",
DEBUG_SUCCESS: "debug-success",
DEBUG_ERROR: "debug-error"
} as const
constructor() {
// Initialize WindowHelper with this
this.windowHelper = new WindowHelper(this)
// Initialize ScreenshotHelper
this.screenshotHelper = new ScreenshotHelper(this.view)
// Initialize ProcessingHelper
this.processingHelper = new ProcessingHelper(this)
// Initialize ShortcutsHelper
this.shortcutsHelper = new ShortcutsHelper(this)
// Initialize ApiServerHelper
this.apiServerHelper = new ApiServerHelper(this)
}
public static getInstance(): AppState {
if (!AppState.instance) {
AppState.instance = new AppState()
}
return AppState.instance
}
// Getters and Setters
public getMainWindow(): BrowserWindow | null {
return this.windowHelper.getMainWindow()
}
public getView(): "queue" | "solutions" {
return this.view
}
public setView(view: "queue" | "solutions"): void {
this.view = view
this.screenshotHelper.setView(view)
}
public isVisible(): boolean {
return this.windowHelper.isVisible()
}
public getScreenshotHelper(): ScreenshotHelper {
return this.screenshotHelper
}
public getProblemInfo(): any {
return this.problemInfo
}
public setProblemInfo(problemInfo: any): void {
this.problemInfo = problemInfo
}
public getScreenshotQueue(): string[] {
return this.screenshotHelper.getScreenshotQueue()
}
public getExtraScreenshotQueue(): string[] {
return this.screenshotHelper.getExtraScreenshotQueue()
}
// Window management methods
public createWindow(): void {
this.windowHelper.createWindow()
}
public hideMainWindow(): void {
this.windowHelper.hideMainWindow()
}
public showMainWindow(): void {
this.windowHelper.showMainWindow()
}
public toggleMainWindow(): void {
console.log(
"Screenshots: ",
this.screenshotHelper.getScreenshotQueue().length,
"Extra screenshots: ",
this.screenshotHelper.getExtraScreenshotQueue().length
)
this.windowHelper.toggleMainWindow()
}
public setWindowDimensions(width: number, height: number): void {
this.windowHelper.setWindowDimensions(width, height)
}
public clearQueues(): void {
this.screenshotHelper.clearQueues()
// Clear problem info
this.problemInfo = null
// Reset view to initial state
this.setView("queue")
}
// Screenshot management methods
public async takeScreenshot(): Promise {
if (!this.getMainWindow()) throw new Error("No main window available")
const screenshotPath = await this.screenshotHelper.takeScreenshot(
() => this.hideMainWindow(),
() => this.showMainWindow()
)
return screenshotPath
}
public async getImagePreview(filepath: string): Promise {
return this.screenshotHelper.getImagePreview(filepath)
}
public async deleteScreenshot(
path: string
): Promise<{ success: boolean; error?: string }> {
return this.screenshotHelper.deleteScreenshot(path)
}
// New methods to move the window
public moveWindowLeft(): void {
this.windowHelper.moveWindowLeft()
}
public moveWindowRight(): void {
this.windowHelper.moveWindowRight()
}
public moveWindowDown(): void {
this.windowHelper.moveWindowDown()
}
public moveWindowUp(): void {
this.windowHelper.moveWindowUp()
}
public centerAndShowWindow(): void {
this.windowHelper.centerAndShowWindow()
}
public createTray(): void {
// Create a simple tray icon
const image = nativeImage.createEmpty()
// Try to use a system template image for better integration
let trayImage = image
try {
// Create a minimal icon - just use an empty image and set the title
trayImage = nativeImage.createFromBuffer(Buffer.alloc(0))
} catch (error) {
console.log("Using empty tray image")
trayImage = nativeImage.createEmpty()
}
this.tray = new Tray(trayImage)
const contextMenu = Menu.buildFromTemplate([
{
label: 'Show Interview Coder',
click: () => {
this.centerAndShowWindow()
}
},
{
label: 'Toggle Window',
click: () => {
this.toggleMainWindow()
}
},
{
type: 'separator'
},
{
label: 'Take Screenshot (Cmd+H)',
click: async () => {
try {
const screenshotPath = await th
[truncated — 2194 more characters]
```
### backend/main.py
```python
"""HTTP bridge between Agent S and the UI server."""
from __future__ import annotations
import base64
import io
import logging
import os
import subprocess
import tempfile
import time
import wave
from dataclasses import dataclass
from http import HTTPStatus
from pathlib import Path
from typing import Any, Dict, Optional, Tuple
from uuid import uuid4
import requests
from flask import Flask, Response, jsonify, request
from dotenv import load_dotenv
import socketio
import audio
# Global flag to track audio playback state
_audio_playing = False
ENV_PATH = Path(__file__).resolve().parent.parent / ".env"
load_dotenv(dotenv_path=ENV_PATH)
def _env_bool(name: str, default: bool = False) -> bool:
raw = os.getenv(name)
return default if raw is None else raw.lower() in {"1", "true", "yes", "on"}
LOG_LEVEL = os.getenv("BACKEND_LOG_LEVEL", "INFO").upper()
logging.basicConfig(
level=LOG_LEVEL, format="%(asctime)s %(levelname)s %(name)s: %(message)s"
)
app = Flask(__name__)
# Socket.IO client for connecting to call.py service
sio = socketio.Client(logger=True, engineio_logger=False)
# Call service configuration
CALL_SERVICE_HOST = os.getenv("CALL_SERVICE_HOST", "localhost")
CALL_SERVICE_PORT = os.getenv("CALL_SERVICE_PORT", "5002")
CALL_SERVICE_URL = f"http://{CALL_SERVICE_HOST}:{CALL_SERVICE_PORT}"
# Audio format configuration (must mirror backend/call.py defaults)
AUDIO_SAMPLE_RATE = int(os.getenv("CALL_AUDIO_SAMPLE_RATE", "48000"))
AUDIO_CHANNELS = int(os.getenv("CALL_AUDIO_CHANNELS", "2"))
AUDIO_SAMPLE_WIDTH = 2 # bytes (int16 PCM)
def _pcm_to_wav_bytes(pcm_data: bytes) -> bytes:
"""Wrap raw PCM int16 audio data into a WAV container."""
if not pcm_data:
raise ValueError("PCM data must not be empty when converting to WAV")
buffer = io.BytesIO()
with wave.open(buffer, "wb") as wav_file:
wav_file.setnchannels(AUDIO_CHANNELS)
wav_file.setsampwidth(AUDIO_SAMPLE_WIDTH)
wav_file.setframerate(AUDIO_SAMPLE_RATE)
wav_file.writeframes(pcm_data)
return buffer.getvalue()
class CallManager:
"""Manages WebSocket connection and audio output to call.py service."""
def __init__(self):
self.connected = False
self.call_active = False
self.playback_in_progress = False
def connect_to_call_service(self):
"""Establish WebSocket connection to call.py service."""
if not self.connected:
try:
logging.info(f"Connecting to call service at {CALL_SERVICE_URL}")
sio.connect(CALL_SERVICE_URL, namespaces=["/"])
self.connected = True
logging.info("Connected to call service successfully")
except Exception as e:
logging.error(f"Failed to connect to call service: {e}")
raise
def disconnect_from_call_service(self):
"""Disconnect from call.py service."""
if self.connected:
try:
sio.disconnect()
self.connected = False
self.call_active = False
logging.info("Disconnected from call service")
except Exception as e:
logging.error(f"Error disconnecting from call service: {e}")
def start_call(self):
"""Start a FaceTime call session."""
if not self.connected:
self.connect_to_call_service()
if self.connected:
# Start recording from the virtual audio device
sio.emit("start_recording")
self.call_active = True
logging.info("Call started, recording initiated")
return True
return False
def end_call(self):
"""End the FaceTime call session."""
if self.connected and self.call_active:
# Stop recording
sio.emit("stop_recording")
self.call_active = False
logging.info("Call ended, recording stopped")
# Keep connection alive for potential future calls
return True
return False
def send_audio_to_output(self, audio_bytes: bytes):
"""Send audio to be played through the FaceTime output device."""
if not self.connected:
logging.error("Not connected to call service, cannot send audio")
return False
if not self.call_active:
logging.warning("No active call, cannot send audio")
return False
try:
# Encode audio as base64 for transmission
audio_b64 = base64.b64encode(audio_bytes).decode("utf-8")
sio.emit("audio_input", {"audio": audio_b64})
self.playback_in_progress = True
logging.debug(f"Sent {len(audio_bytes)} bytes of audio to FaceTime output")
return True
except Exception as e:
logging.error(f"Failed to send audio: {e}")
return False
# Initialize call manager
call_manager = CallManager()
# Socket.IO event handlers
@sio.on("connect")
def on_connect():
logging.info("Connected to call service via WebSocket")
call_manager.connected = True
@sio.on("disconnect")
def on_disconnect():
logging.info("Disconnected from call service")
call_manager.connected = False
call_manager.call_active = False
# Audio streaming buffer for accumulating chunks
current_utterance_chunks = []
utterance_start_time = None
@sio.on("utterance_start")
def on_utterance_start(data):
"""Handle start of user utterance."""
global current_utterance_chunks, utterance_start_time
current_utterance_chunks = []
utterance_start_time = data.get("timestamp", time.time())
logging.info("🎤 UTTERANCE START - Beginning audio stream from call.py")
@sio.on("audio_chunk")
def on_audio_chunk(data):
"""Handle incoming audio chunk during utterance."""
global current_utterance_chunks
try:
audio_b64 = data.get("audio")
if not audio_b64:
return
# Decod
[truncated — 27526 more characters]
```
[84 more indexed source files omitted to keep this export small. The full file list is in the Codebase structure section above.]