/* * Atoll (DynamicIsland) * Copyright (C) 2024-2026 Atoll Contributors * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by * the Free Software Foundation, either version 3 of the License, or * (at your option) any later version. * * This program is distributed in the hope that it will be useful, * but WITHOUT ANY WARRANTY; without even the implied warranty of * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * GNU General Public License for more details. * * You should have received a copy of the GNU General Public License * along with this program. If not, see . */ import AppKit import SwiftUI import AVFoundation import Defaults import Foundation // Chat message model struct ChatMessage: Identifiable, Codable { let id = UUID() let content: String let isFromUser: Bool let timestamp: Date let attachedFiles: [ScreenAssistantFile]? init(content: String, isFromUser: Bool, attachedFiles: [ScreenAssistantFile]? = nil) { self.content = content self.isFromUser = isFromUser self.timestamp = Date() self.attachedFiles = attachedFiles } } // Screen Assistant item data structure struct ScreenAssistantFile: Identifiable, Codable { let id = UUID() let name: String let type: FileType let timestamp: Date let fileURL: String? // For local files let audioFileName: String? // For audio recordings enum FileType: String, CaseIterable, Codable { case document = "document" case image = "image" case audio = "audio" case video = "video" case other = "other" var iconName: String { switch self { case .document: return "doc.text" case .image: return "photo" case .audio: return "waveform" case .video: return "video" case .other: return "doc" } } var displayName: String { switch self { case .document: return "Document" case .image: return "Image" case .audio: return "Audio" case .video: return "Video" case .other: return "File" } } } init(fileURL: URL) { // Defensive initialization with nil coalescing self.name = fileURL.lastPathComponent.isEmpty ? "Unknown File" : fileURL.lastPathComponent self.fileURL = fileURL.absoluteString self.audioFileName = nil self.timestamp = Date() // Safe file extension extraction let fileExtension = fileURL.pathExtension.lowercased() switch fileExtension { case "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "heic": self.type = .image case "mp3", "wav", "m4a", "aac", "flac": self.type = .audio case "mp4", "mov", "avi", "mkv": self.type = .video case "txt", "md", "pdf", "doc", "docx", "rtf": self.type = .document default: self.type = .other } print("āœ… ScreenAssistantFile: Created file entry - name: \(self.name), type: \(self.type), url: \(self.fileURL ?? "nil")") } init(audioFileName: String, name: String) { self.name = name self.type = .audio self.fileURL = nil self.audioFileName = audioFileName self.timestamp = Date() } } class ScreenAssistantManager: NSObject, ObservableObject { static let shared = ScreenAssistantManager() @Published var attachedFiles: [ScreenAssistantFile] = [] @Published var isRecording: Bool = false @Published var recordingDuration: TimeInterval = 0 @Published var chatMessages: [ChatMessage] = [] @Published var isLoading: Bool = false private var audioRecorder: AVAudioRecorder? private var recordingTimer: Timer? private var activeRequest: URLSessionTask? // Panel management private var chatMessagesPanel: ChatMessagesPanel? private var chatInputPanel: ChatInputPanel? // Directory for storing audio recordings static let audioDataDirectory: URL = { let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first! let audioDir = documentsPath.appendingPathComponent("ScreenAssistantAudio") // Create directory if it doesn't exist try? FileManager.default.createDirectory(at: audioDir, withIntermediateDirectories: true) return audioDir }() // Directory for storing screenshots static let screenshotDataDirectory: URL = { let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first! let screenshotDir = documentsPath.appendingPathComponent("ScreenAssistantScreenshots") // Create directory if it doesn't exist try? FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) return screenshotDir }() private override init() { super.init() loadFilesFromDefaults() } deinit { stopRecording() closePanels() } // MARK: - Panel Management func showPanels() { // Close existing panels first closePanels() // Create and show chat messages panel (left side) chatMessagesPanel = ChatMessagesPanel() chatMessagesPanel?.positionOnLeftSide() chatMessagesPanel?.makeKeyAndOrderFront(nil) // Create and show input panel (center) chatInputPanel = ChatInputPanel() chatInputPanel?.positionInCenter() chatInputPanel?.makeKeyAndOrderFront(nil) // Focus on input panel for immediate typing DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { self.chatInputPanel?.makeKey() } } func closePanels() { chatMessagesPanel?.close() chatInputPanel?.close() chatMessagesPanel = nil chatInputPanel = nil } func arePanelsVisible() -> Bool { return chatMessagesPanel?.isVisible == true || chatInputPanel?.isVisible == true } // MARK: - File Management func addFiles(_ urls: [URL]) { guard !urls.isEmpty else { print("āš ļø ScreenAssistant: No URLs provided to addFiles") return } print("šŸ“ ScreenAssistant: Adding \(urls.count) files") let newFiles = urls.compactMap { url -> ScreenAssistantFile? in // Wrap in autoreleasepool to manage memory return autoreleasepool { do { // Verify file exists guard FileManager.default.fileExists(atPath: url.path) else { print("āŒ ScreenAssistant: File does not exist at \(url.path)") return nil } // Verify file is readable guard FileManager.default.isReadableFile(atPath: url.path) else { print("āŒ ScreenAssistant: File is not readable at \(url.path)") return nil } // Create file entry with error handling let file = ScreenAssistantFile(fileURL: url) print("āœ… ScreenAssistant: Created file entry for \(file.name)") return file } catch { print("āŒ ScreenAssistant: Error creating file entry - \(error)") return nil } } } guard !newFiles.isEmpty else { print("āš ļø ScreenAssistant: No valid files to add") return } // Ensure we're on the main thread for @Published property updates DispatchQueue.main.async { [weak self] in guard let self = self else { print("āŒ ScreenAssistant: Self deallocated during addFiles") return } self.attachedFiles.append(contentsOf: newFiles) print("šŸ“ ScreenAssistant: Total attached files: \(self.attachedFiles.count)") // Save to defaults with error handling do { self.saveFilesToDefaults() } catch { print("āŒ ScreenAssistant: Failed to save files after adding - \(error)") } } } func removeFile(_ file: ScreenAssistantFile) { attachedFiles.removeAll { $0.id == file.id } // Clean up audio file if it exists if let audioFileName = file.audioFileName { let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(audioFileName) try? FileManager.default.removeItem(at: audioURL) } saveFilesToDefaults() } func clearAllFiles() { // Clean up all audio files for file in attachedFiles { if let audioFileName = file.audioFileName { let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(audioFileName) try? FileManager.default.removeItem(at: audioURL) } } attachedFiles.removeAll() saveFilesToDefaults() } // MARK: - Audio Recording func toggleRecording() { if isRecording { stopRecording() } else { startRecording() } } private func startRecording() { guard !isRecording else { return } let fileName = "recording_\(Date().timeIntervalSince1970).m4a" let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(fileName) let settings: [String: Any] = [ AVFormatIDKey: Int(kAudioFormatMPEG4AAC), AVSampleRateKey: 44100, AVNumberOfChannelsKey: 2, AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue ] do { audioRecorder = try AVAudioRecorder(url: audioURL, settings: settings) audioRecorder?.delegate = self audioRecorder?.isMeteringEnabled = true audioRecorder?.prepareToRecord() audioRecorder?.record() isRecording = true recordingDuration = 0 // Start timer for recording duration recordingTimer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in self?.updateRecordingDuration() } print("Started recording: \(fileName)") } catch { print("Failed to start recording: \(error)") } } private func stopRecording() { guard isRecording else { return } audioRecorder?.stop() recordingTimer?.invalidate() recordingTimer = nil isRecording = false print("Stopped recording") } private func updateRecordingDuration() { guard let recorder = audioRecorder, recorder.isRecording else { return } recordingDuration = recorder.currentTime } // MARK: - Persistence private func saveFilesToDefaults() { do { let encoded = try JSONEncoder().encode(attachedFiles) UserDefaults.standard.set(encoded, forKey: "ScreenAssistantFiles") print("āœ… ScreenAssistant: Saved \(attachedFiles.count) files to UserDefaults") } catch { print("āŒ ScreenAssistant: Failed to save files to UserDefaults - \(error)") // Don't throw - this is a non-critical operation } } private func loadFilesFromDefaults() { guard let data = UserDefaults.standard.data(forKey: "ScreenAssistantFiles"), let decoded = try? JSONDecoder().decode([ScreenAssistantFile].self, from: data) else { return } attachedFiles = decoded } // MARK: - Chat Management func sendMessage(_ message: String) { print("šŸ“¤ ScreenAssistant: Sending message - '\(message)'") print("šŸ“ ScreenAssistant: Attached files count: \(attachedFiles.count)") // Add user message to chat let userMessage = ChatMessage(content: message, isFromUser: true, attachedFiles: attachedFiles.isEmpty ? nil : attachedFiles) chatMessages.append(userMessage) // Print attached files details for (index, file) in attachedFiles.enumerated() { print("šŸ“Ž ScreenAssistant: File \(index + 1): \(file.name) (\(file.type.displayName))") } // Clear input and files after sending let currentFiles = attachedFiles clearAllFiles() // Send to appropriate AI API based on selected provider let provider = Defaults[.selectedAIProvider] sendToAI(message: message, files: currentFiles, provider: provider) } private func sendToAI(message: String, files: [ScreenAssistantFile], provider: AIModelProvider) { print("šŸš€ ScreenAssistant: Making API request to \(provider.displayName)") isLoading = true switch provider { case .gemini: sendToGeminiAPI(message: message, files: files) case .openai: sendToOpenAIAPI(message: message, files: files) case .claude: sendToClaudeAPI(message: message, files: files) case .local: sendToLocalAPI(message: message, files: files) } } private func sendToGeminiAPI(message: String, files: [ScreenAssistantFile]) { let apiKey = Defaults[.geminiApiKey] guard !apiKey.isEmpty else { print("āŒ ScreenAssistant: No Gemini API key configured") addAssistantMessage("Error: No Gemini API key configured. Please set your API key in model settings.") isLoading = false return } // Get selected model or default to gemini-2.5-flash let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", supportsThinking: true) let modelId = selectedModel.id guard let url = URL(string: "https://generativelanguage.googleapis.com/v1beta/models/\(modelId):generateContent?key=\(apiKey)") else { print("āŒ ScreenAssistant: Invalid Gemini API URL") addAssistantMessage("Error: Invalid API URL") isLoading = false return } performAPIRequest(url: url, requestBody: buildGeminiRequestBody(message: message, files: files), provider: .gemini) } private func sendToOpenAIAPI(message: String, files: [ScreenAssistantFile]) { let apiKey = Defaults[.openaiApiKey] guard !apiKey.isEmpty else { print("āŒ ScreenAssistant: No OpenAI API key configured") addAssistantMessage("Error: No OpenAI API key configured. Please set your API key in model settings.") isLoading = false return } // Get selected model or default to gpt-4o let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "gpt-4o", name: "GPT-4o", supportsThinking: false) let modelId = selectedModel.id guard let url = URL(string: "https://api.openai.com/v1/chat/completions") else { print("āŒ ScreenAssistant: Invalid OpenAI API URL") addAssistantMessage("Error: Invalid API URL") isLoading = false return } performOpenAIRequest(url: url, requestBody: buildOpenAIRequestBody(message: message, files: files, model: modelId), apiKey: apiKey) } private func sendToClaudeAPI(message: String, files: [ScreenAssistantFile]) { let apiKey = Defaults[.claudeApiKey] guard !apiKey.isEmpty else { print("āŒ ScreenAssistant: No Claude API key configured") addAssistantMessage("Error: No Claude API key configured. Please set your API key in model settings.") isLoading = false return } // Get selected model or default to claude-3-5-sonnet let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet", supportsThinking: false) let modelId = selectedModel.id guard let url = URL(string: "https://api.anthropic.com/v1/messages") else { print("āŒ ScreenAssistant: Invalid Claude API URL") addAssistantMessage("Error: Invalid API URL") isLoading = false return } performClaudeRequest(url: url, requestBody: buildClaudeRequestBody(message: message, files: files, model: modelId), apiKey: apiKey) } private func sendToLocalAPI(message: String, files: [ScreenAssistantFile]) { let endpoint = Defaults[.localModelEndpoint] guard !endpoint.isEmpty else { print("āŒ ScreenAssistant: No local endpoint configured") addAssistantMessage("Error: No local endpoint configured. Please set your endpoint in model settings.") isLoading = false return } guard let url = URL(string: "\(endpoint)/api/chat") else { print("āŒ ScreenAssistant: Invalid local API URL") addAssistantMessage("Error: Invalid local API URL") isLoading = false return } performAPIRequest(url: url, requestBody: buildOllamaRequestBody(message: message, files: files), provider: .local) } // MARK: - API Request Builders private func buildGeminiRequestBody(message: String, files: [ScreenAssistantFile]) -> [String: Any] { var contents: [[String: Any]] = [] // Add previous conversation messages (last 10 for context) let recentMessages = Array(chatMessages.suffix(10)) for chatMessage in recentMessages { if chatMessage.id != chatMessages.last?.id { // Don't include the message we just added let role = chatMessage.isFromUser ? "user" : "model" contents.append([ "role": role, "parts": [["text": chatMessage.content]] ]) } } // Build current message parts var parts: [[String: Any]] = [] // Add text part let contextualMessage = buildContextualMessage(message: message, files: files) parts.append(["text": contextualMessage]) // Add file content for supported types using proper Gemini 2.5 APIs for file in files { if let filePart = createGeminiFilePart(for: file) { parts.append(filePart) print("šŸ“Ž ScreenAssistant: Added file part for \(file.name)") } } // Add current message contents.append([ "role": "user", "parts": parts ]) var requestBody: [String: Any] = [ "contents": contents, "generationConfig": [ "temperature": 0.7, "topP": 0.8, "topK": 40, "maxOutputTokens": 2048, "responseMimeType": "text/plain" ], "safetySettings": [ [ "category": "HARM_CATEGORY_HARASSMENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" ], [ "category": "HARM_CATEGORY_HATE_SPEECH", "threshold": "BLOCK_MEDIUM_AND_ABOVE" ], [ "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" ], [ "category": "HARM_CATEGORY_DANGEROUS_CONTENT", "threshold": "BLOCK_MEDIUM_AND_ABOVE" ] ] ] // Add thinking configuration if enabled and model supports it let selectedModel = Defaults[.selectedAIModel] if selectedModel?.supportsThinking == true && Defaults[.enableThinkingMode] { requestBody["generationConfig"] = (requestBody["generationConfig"] as! [String: Any]).merging([ "thinkingConfig": [ "thinkingBudget": 0 // 0 means unlimited thinking ] ]) { (_, new) in new } } return requestBody } private func buildOpenAIRequestBody(message: String, files: [ScreenAssistantFile], model: String) -> [String: Any] { var messages: [[String: Any]] = [] // Add previous conversation messages (last 10 for context) let recentMessages = Array(chatMessages.suffix(10)) for chatMessage in recentMessages { if chatMessage.id != chatMessages.last?.id { let role = chatMessage.isFromUser ? "user" : "assistant" messages.append([ "role": role, "content": chatMessage.content ]) } } // Add current message let contextualMessage = buildContextualMessage(message: message, files: files) messages.append([ "role": "user", "content": contextualMessage ]) return [ "model": model, "messages": messages, "temperature": 0.7, "max_tokens": 2048 ] } private func buildClaudeRequestBody(message: String, files: [ScreenAssistantFile], model: String) -> [String: Any] { var messages: [[String: Any]] = [] // Add previous conversation messages (last 10 for context) let recentMessages = Array(chatMessages.suffix(10)) for chatMessage in recentMessages { if chatMessage.id != chatMessages.last?.id { let role = chatMessage.isFromUser ? "user" : "assistant" messages.append([ "role": role, "content": chatMessage.content ]) } } // Add current message let contextualMessage = buildContextualMessage(message: message, files: files) messages.append([ "role": "user", "content": contextualMessage ]) return [ "model": model, "max_tokens": 2048, "messages": messages ] } private func buildOllamaRequestBody(message: String, files: [ScreenAssistantFile]) -> [String: Any] { let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "llama3.2", name: "Llama 3.2", supportsThinking: false) let contextualMessage = buildContextualMessage(message: message, files: files) return [ "model": selectedModel.id, "messages": [ [ "role": "user", "content": contextualMessage ] ], "stream": false ] } // MARK: - API Request Performers private func performAPIRequest(url: URL, requestBody: [String: Any], provider: AIModelProvider) { var request = URLRequest(url: url) request.httpMethod = "POST" request.addValue("application/json", forHTTPHeaderField: "Content-Type") do { let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) request.httpBody = jsonData print("šŸ“‹ ScreenAssistant: Request body size: \(jsonData.count) bytes") } catch { print("āŒ ScreenAssistant: Failed to encode request - \(error)") addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") isLoading = false return } let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in DispatchQueue.main.async { guard let self = self else { return } self.handleResponse(data: data, response: response, error: error, provider: provider) } } activeRequest = task task.resume() } private func performOpenAIRequest(url: URL, requestBody: [String: Any], apiKey: String) { var request = URLRequest(url: url) request.httpMethod = "POST" request.addValue("application/json", forHTTPHeaderField: "Content-Type") request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") do { let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) request.httpBody = jsonData } catch { print("āŒ ScreenAssistant: Failed to encode OpenAI request - \(error)") addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") isLoading = false return } let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in DispatchQueue.main.async { guard let self = self else { return } self.handleResponse(data: data, response: response, error: error, provider: .openai) } } activeRequest = task task.resume() } private func performClaudeRequest(url: URL, requestBody: [String: Any], apiKey: String) { var request = URLRequest(url: url) request.httpMethod = "POST" request.addValue("application/json", forHTTPHeaderField: "Content-Type") request.addValue(apiKey, forHTTPHeaderField: "x-api-key") request.addValue("2023-06-01", forHTTPHeaderField: "anthropic-version") do { let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) request.httpBody = jsonData } catch { print("āŒ ScreenAssistant: Failed to encode Claude request - \(error)") addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") isLoading = false return } let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in DispatchQueue.main.async { guard let self = self else { return } self.handleResponse(data: data, response: response, error: error, provider: .claude) } } activeRequest = task task.resume() } // MARK: - Response Handlers private func handleResponse(data: Data?, response: URLResponse?, error: Error?, provider: AIModelProvider) { isLoading = false activeRequest = nil // Check if the request was cancelled (e.g., by resetConversationContext) if let error = error as? NSError, error.code == NSURLErrorCancelled { print("ā„¹ļø ScreenAssistant: Request was cancelled") return } if let error = error { print("āŒ ScreenAssistant: Network error - \(error)") addAssistantMessage("Error: \(error.localizedDescription)") return } if let httpResponse = response as? HTTPURLResponse { print("šŸ“Š ScreenAssistant: HTTP Status: \(httpResponse.statusCode)") if httpResponse.statusCode != 200 { handleAPIError(statusCode: httpResponse.statusCode, provider: provider) return } } guard let data = data else { print("āŒ ScreenAssistant: No response data") addAssistantMessage("Error: No response data") return } print("šŸ“Ø ScreenAssistant: Response data size: \(data.count) bytes") // Parse response based on provider switch provider { case .gemini: parseGeminiResponse(data: data) case .openai: parseOpenAIResponse(data: data) case .claude: parseClaudeResponse(data: data) case .local: parseOllamaResponse(data: data) } } private func parseGeminiResponse(data: Data) { do { if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { print("āœ… ScreenAssistant: Successfully parsed Gemini JSON response") if let candidates = json["candidates"] as? [[String: Any]], let firstCandidate = candidates.first, let content = firstCandidate["content"] as? [String: Any], let parts = content["parts"] as? [[String: Any]], let firstPart = parts.first, let text = firstPart["text"] as? String { print("āœ… ScreenAssistant: Got Gemini response text: \(text.prefix(100))...") addAssistantMessage(text) } else { if let error = json["error"] as? [String: Any] { handleAPIError(error: error) } else { print("āŒ ScreenAssistant: Unexpected Gemini response format") addAssistantMessage("Error: Unexpected response format from Gemini") } } } } catch { print("āŒ ScreenAssistant: Gemini JSON parsing error - \(error)") addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") } } private func parseOpenAIResponse(data: Data) { do { if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { print("āœ… ScreenAssistant: Successfully parsed OpenAI JSON response") if let choices = json["choices"] as? [[String: Any]], let firstChoice = choices.first, let message = firstChoice["message"] as? [String: Any], let content = message["content"] as? String { print("āœ… ScreenAssistant: Got OpenAI response text: \(content.prefix(100))...") addAssistantMessage(content) } else { if let error = json["error"] as? [String: Any] { handleOpenAIError(error: error) } else { print("āŒ ScreenAssistant: Unexpected OpenAI response format") addAssistantMessage("Error: Unexpected response format from OpenAI") } } } } catch { print("āŒ ScreenAssistant: OpenAI JSON parsing error - \(error)") addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") } } private func parseClaudeResponse(data: Data) { do { if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { print("āœ… ScreenAssistant: Successfully parsed Claude JSON response") if let content = json["content"] as? [[String: Any]], let firstContent = content.first, let text = firstContent["text"] as? String { print("āœ… ScreenAssistant: Got Claude response text: \(text.prefix(100))...") addAssistantMessage(text) } else { if let error = json["error"] as? [String: Any] { handleClaudeError(error: error) } else { print("āŒ ScreenAssistant: Unexpected Claude response format") addAssistantMessage("Error: Unexpected response format from Claude") } } } } catch { print("āŒ ScreenAssistant: Claude JSON parsing error - \(error)") addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") } } private func parseOllamaResponse(data: Data) { do { if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { print("āœ… ScreenAssistant: Successfully parsed Ollama JSON response") if let message = json["message"] as? [String: Any], let content = message["content"] as? String { print("āœ… ScreenAssistant: Got Ollama response text: \(content.prefix(100))...") addAssistantMessage(content) } else { print("āŒ ScreenAssistant: Unexpected Ollama response format") addAssistantMessage("Error: Unexpected response format from local model") } } } catch { print("āŒ ScreenAssistant: Ollama JSON parsing error - \(error)") addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") } } private func buildContextualMessage(message: String, files: [ScreenAssistantFile]) -> String { var contextualMessage = message // Add file context with specific instructions for different types if !files.isEmpty { contextualMessage += "\n\nI have attached the following files for your analysis:" var hasImages = false var hasDocuments = false var hasAudio = false var hasVideo = false for file in files { contextualMessage += "\n- \(file.name) (\(file.type.displayName))" switch file.type { case .image: hasImages = true case .document: hasDocuments = true case .audio: hasAudio = true case .video: hasVideo = true case .other: break } } // Add specific instructions based on file types contextualMessage += "\n\nPlease analyze these files in the context of my question. Specifically:" if hasImages { contextualMessage += "\n- For images: Describe what you see, identify objects, text, or patterns, and relate them to my question." } if hasDocuments { contextualMessage += "\n- For documents: Read and understand the content, extract key information, and provide insights relevant to my question." } if hasAudio { contextualMessage += "\n- For audio: Listen to and transcribe the audio content, identify speakers, topics, or sounds as relevant." } if hasVideo { contextualMessage += "\n- For video: Analyze both visual and audio content, describe actions, scenes, or dialogue as applicable." } contextualMessage += "\n\nProvide comprehensive insights that combine information from all attached files with your response to my question." } return contextualMessage } private func createGeminiFilePart(for file: ScreenAssistantFile) -> [String: Any]? { print("šŸ“Ž ScreenAssistant: Processing file for Gemini 2.5: \(file.name) (\(file.type.displayName))") guard let fileURL = file.fileURL, let url = URL(string: fileURL) else { print("āŒ ScreenAssistant: No valid URL for file \(file.name)") return ["text": "File: \(file.name) (no valid URL)"] } switch file.type { case .image: return createGeminiImagePart(for: url, fileName: file.name) case .document: return createGeminiDocumentPart(for: url, fileName: file.name) case .audio: return createGeminiAudioPart(for: url, fileName: file.name) case .video: return createGeminiVideoPart(for: url, fileName: file.name) case .other: return createGeminiTextPart(for: url, fileName: file.name) } } private func createGeminiImagePart(for url: URL, fileName: String) -> [String: Any]? { print("šŸ–¼ļø ScreenAssistant: Processing image file: \(fileName)") do { let imageData = try Data(contentsOf: url) let base64String = imageData.base64EncodedString() // Determine MIME type let mimeType: String let pathExtension = url.pathExtension.lowercased() switch pathExtension { case "jpg", "jpeg": mimeType = "image/jpeg" case "png": mimeType = "image/png" case "gif": mimeType = "image/gif" case "webp": mimeType = "image/webp" case "heic": mimeType = "image/heic" default: mimeType = "image/jpeg" } print("šŸ“Ž ScreenAssistant: Image encoded - \(base64String.count) bytes, MIME: \(mimeType)") return [ "inline_data": [ "mime_type": mimeType, "data": base64String ] ] } catch { print("āŒ ScreenAssistant: Failed to encode image \(fileName): \(error)") return ["text": "Image file: \(fileName) (failed to encode: \(error.localizedDescription))"] } } private func createGeminiDocumentPart(for url: URL, fileName: String) -> [String: Any]? { print("šŸ“„ ScreenAssistant: Processing document file: \(fileName)") let pathExtension = url.pathExtension.lowercased() if pathExtension == "pdf" { // Handle PDF files using base64 encoding for Gemini 2.5 do { let pdfData = try Data(contentsOf: url) let base64String = pdfData.base64EncodedString() print("šŸ“Ž ScreenAssistant: PDF encoded - \(base64String.count) bytes") return [ "inline_data": [ "mime_type": "application/pdf", "data": base64String ] ] } catch { print("āŒ ScreenAssistant: Failed to encode PDF \(fileName): \(error)") return ["text": "PDF file: \(fileName) (failed to encode: \(error.localizedDescription))"] } } else { // Handle text-based documents do { let content = try String(contentsOf: url) print("šŸ“„ ScreenAssistant: Read document content (\(content.count) characters)") return ["text": "File content of \(fileName):\n\(content)"] } catch { print("āŒ ScreenAssistant: Failed to read document \(fileName): \(error)") return ["text": "Document file: \(fileName) (could not read content: \(error.localizedDescription))"] } } } private func createGeminiAudioPart(for url: URL, fileName: String) -> [String: Any]? { print("šŸŽµ ScreenAssistant: Processing audio file: \(fileName)") do { let audioData = try Data(contentsOf: url) let base64String = audioData.base64EncodedString() // Determine MIME type let mimeType: String let pathExtension = url.pathExtension.lowercased() switch pathExtension { case "mp3": mimeType = "audio/mpeg" case "wav": mimeType = "audio/wav" case "m4a": mimeType = "audio/mp4" case "aac": mimeType = "audio/aac" case "flac": mimeType = "audio/flac" default: mimeType = "audio/mpeg" } print("ļæ½ ScreenAssistant: Audio encoded - \(base64String.count) bytes, MIME: \(mimeType)") return [ "inline_data": [ "mime_type": mimeType, "data": base64String ] ] } catch { print("āŒ ScreenAssistant: Failed to encode audio \(fileName): \(error)") return ["text": "Audio file: \(fileName) (failed to encode: \(error.localizedDescription))"] } } private func createGeminiVideoPart(for url: URL, fileName: String) -> [String: Any]? { print("ļæ½ ScreenAssistant: Processing video file: \(fileName)") do { let videoData = try Data(contentsOf: url) let base64String = videoData.base64EncodedString() // Determine MIME type let mimeType: String let pathExtension = url.pathExtension.lowercased() switch pathExtension { case "mp4": mimeType = "video/mp4" case "mov": mimeType = "video/quicktime" case "avi": mimeType = "video/x-msvideo" case "mkv": mimeType = "video/x-matroska" default: mimeType = "video/mp4" } print("šŸ“Ž ScreenAssistant: Video encoded - \(base64String.count) bytes, MIME: \(mimeType)") return [ "inline_data": [ "mime_type": mimeType, "data": base64String ] ] } catch { print("āŒ ScreenAssistant: Failed to encode video \(fileName): \(error)") return ["text": "Video file: \(fileName) (failed to encode: \(error.localizedDescription))"] } } private func createGeminiTextPart(for url: URL, fileName: String) -> [String: Any]? { print("šŸ“ ScreenAssistant: Processing text file: \(fileName)") do { let content = try String(contentsOf: url) print("šŸ“„ ScreenAssistant: Read text content (\(content.count) characters)") return ["text": "File content of \(fileName):\n\(content)"] } catch { print("āŒ ScreenAssistant: Failed to read text file \(fileName): \(error)") return ["text": "File: \(fileName) (could not read content: \(error.localizedDescription))"] } } func clearChat() { resetConversationContext() } func resetConversationContext() { // Cancel any in-flight request activeRequest?.cancel() activeRequest = nil isLoading = false chatMessages.removeAll() clearAllFiles() } private func addAssistantMessage(_ content: String) { print("šŸ’¬ ScreenAssistant: Adding assistant message: \(content.prefix(100))...") let assistantMessage = ChatMessage(content: content, isFromUser: false) chatMessages.append(assistantMessage) } private func handleAPIError(statusCode: Int, provider: AIModelProvider) { let userFriendlyMessage: String switch statusCode { case 429: userFriendlyMessage = "🚫 **Rate Limited**\n\n\(provider.displayName) is currently rate limiting requests. Please wait a moment and try again." case 400: userFriendlyMessage = "āŒ **Invalid Request**\n\nThere was an issue with your request to \(provider.displayName). Please check your message and attached files." case 401: userFriendlyMessage = "šŸ”‘ **Authentication Error**\n\nYour \(provider.displayName) API key appears to be invalid. Please check your API key in model settings." case 403: userFriendlyMessage = "🚫 **Access Denied**\n\nYour \(provider.displayName) API key doesn't have permission for this request." case 404: userFriendlyMessage = "šŸ” **Model Not Found**\n\nThe requested model is not available on \(provider.displayName)." case 500, 502, 503: userFriendlyMessage = "āš ļø **Server Error**\n\n\(provider.displayName) servers are experiencing issues. Please try again in a few minutes." default: userFriendlyMessage = "āŒ **API Error (\(statusCode))**\n\n\(provider.displayName) returned an error. Please try again." } addAssistantMessage(userFriendlyMessage) } private func handleAPIError(error: [String: Any]) { guard let code = error["code"] as? Int, let message = error["message"] as? String else { print("āŒ ScreenAssistant: Unknown API Error") addAssistantMessage("An unknown error occurred. Please try again.") return } print("āŒ ScreenAssistant: API Error \(code) - \(message)") let userFriendlyMessage: String switch code { case 429: // Quota exceeded if message.contains("quota") || message.contains("exceeded") { userFriendlyMessage = "🚫 **API Quota Exceeded**\n\nYou've reached your API usage limit. This usually happens when:\n\n• Too many requests in a short time\n• Daily/monthly quota exceeded\n• Free tier limits reached\n\n**What you can do:**\n• Wait a few minutes and try again\n• Check your API billing\n• Consider upgrading your plan\n\n*The system will work again once the quota resets.*" } else { userFriendlyMessage = "ā° **Rate Limited**\n\nToo many requests. Please wait a moment and try again." } case 400: userFriendlyMessage = "āŒ **Invalid Request**\n\nThere was an issue with your request. Please check your message and attached files." case 401: userFriendlyMessage = "šŸ”‘ **Authentication Error**\n\nYour API key appears to be invalid. Please check your API key in settings." case 403: userFriendlyMessage = "🚫 **Access Denied**\n\nYour API key doesn't have permission for this request. Please check your API key settings." case 404: userFriendlyMessage = "šŸ” **Model Not Found**\n\nThe requested AI model is not available. Please try again later." case 500, 502, 503: userFriendlyMessage = "āš ļø **Server Error**\n\nThe AI service is experiencing issues. Please try again in a few minutes." default: userFriendlyMessage = "āŒ **API Error (\(code))**\n\n\(message.components(separatedBy: ".").first ?? message)" } addAssistantMessage(userFriendlyMessage) } private func handleOpenAIError(error: [String: Any]) { if let message = error["message"] as? String { let userFriendlyMessage = "āŒ **OpenAI Error**\n\n\(message)" addAssistantMessage(userFriendlyMessage) } else { addAssistantMessage("āŒ **OpenAI Error**\n\nAn unknown error occurred with OpenAI.") } } private func handleClaudeError(error: [String: Any]) { if let message = error["message"] as? String { let userFriendlyMessage = "āŒ **Claude Error**\n\n\(message)" addAssistantMessage(userFriendlyMessage) } else { addAssistantMessage("āŒ **Claude Error**\n\nAn unknown error occurred with Claude.") } } } // MARK: - AVAudioRecorderDelegate extension ScreenAssistantManager: AVAudioRecorderDelegate { func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully flag: Bool) { isRecording = false recordingTimer?.invalidate() recordingTimer = nil if flag { let fileName = recorder.url.lastPathComponent let displayName = "Recording \(DateFormatter.shortTime.string(from: Date()))" let audioFile = ScreenAssistantFile(audioFileName: fileName, name: displayName) attachedFiles.append(audioFile) saveFilesToDefaults() print("Recording saved: \(fileName)") } else { print("Recording failed") } } func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: Error?) { print("Recording encode error: \(error?.localizedDescription ?? "Unknown error")") isRecording = false recordingTimer?.invalidate() recordingTimer = nil } } // MARK: - DateFormatter Extension extension DateFormatter { static let shortTime: DateFormatter = { let formatter = DateFormatter() formatter.timeStyle = .short return formatter }() }