From 2f7821f4fd2ebf504a7c92437884ad98c60720b4 Mon Sep 17 00:00:00 2001 From: Hariharan Mudaliar Date: Mon, 22 Sep 2025 00:39:40 +0530 Subject: [PATCH 1/3] Working Screen Assistant with snipping --- .gitignore | 5 +- DynamicIsland.xcodeproj/project.pbxproj | 2 + DynamicIsland/DynamicIsland.entitlements | 4 + DynamicIsland/DynamicIslandApp.swift | 32 +- DynamicIsland/Info.plist | 6 + DynamicIsland/Localizable.xcstrings | 144 ++ .../Shortcuts/ShortcutConstants.swift | 1 + .../components/Clipboard/ClipboardPanel.swift | 28 + .../ColorPicker/ColorPickerPanel.swift | 27 + .../ScreenAssistant/ChatPanels.swift | 672 +++++++++ .../ScreenAssistant/ModelSelectionPanel.swift | 535 +++++++ .../ScreenAssistantPanel.swift | 272 ++++ .../ScreenshotSnippingTool.swift | 142 ++ .../ScreenshotSnippingTool_old.swift | 500 +++++++ .../components/Settings/SettingsView.swift | 199 +++ .../Settings/SettingsWindowController.swift | 27 + .../components/Stats/StatsPanel.swift | 28 + .../managers/ScreenAssistantManager.swift | 1224 +++++++++++++++++ .../ScreenAssistantPanelManager.swift | 57 + DynamicIsland/models/Constants.swift | 104 ++ 20 files changed, 4007 insertions(+), 2 deletions(-) create mode 100644 DynamicIsland/components/ScreenAssistant/ChatPanels.swift create mode 100644 DynamicIsland/components/ScreenAssistant/ModelSelectionPanel.swift create mode 100644 DynamicIsland/components/ScreenAssistant/ScreenAssistantPanel.swift create mode 100644 DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift create mode 100644 DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift create mode 100644 DynamicIsland/managers/ScreenAssistantManager.swift create mode 100644 DynamicIsland/managers/ScreenAssistantPanelManager.swift diff --git a/.gitignore b/.gitignore index e488eb9..4a248a2 100644 --- a/.gitignore +++ b/.gitignore @@ -68,4 +68,7 @@ SettingsView.swift.md logic.md TheBoringWorker-HUD boringnotch + meteo + airpods + traduzione modificabile -segmentedhud.swift.ref \ No newline at end of file +segmentedhud.swift.ref +gemini-apireference.md + +ScreenshotApp-main \ No newline at end of file diff --git a/DynamicIsland.xcodeproj/project.pbxproj b/DynamicIsland.xcodeproj/project.pbxproj index c730875..6e97e69 100644 --- a/DynamicIsland.xcodeproj/project.pbxproj +++ b/DynamicIsland.xcodeproj/project.pbxproj @@ -370,6 +370,7 @@ "$(PROJECT_DIR)/mediaremote-adapter", ); GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = DynamicIsland/Info.plist; INFOPLIST_KEY_NSAppleEventsUsageDescription = "Need for Spotify and Apple Music"; INFOPLIST_KEY_NSCalendarsUsageDescription = "Need for Calendar Mode"; INFOPLIST_KEY_NSCameraUsageDescription = "We need your camera for Mirror mode"; @@ -411,6 +412,7 @@ "$(PROJECT_DIR)/mediaremote-adapter", ); GENERATE_INFOPLIST_FILE = YES; + INFOPLIST_FILE = DynamicIsland/Info.plist; INFOPLIST_KEY_NSAppleEventsUsageDescription = "Need for Spotify and Apple Music"; INFOPLIST_KEY_NSCalendarsUsageDescription = "Need for Calendar Mode"; INFOPLIST_KEY_NSCameraUsageDescription = "We need your camera for Mirror mode"; diff --git a/DynamicIsland/DynamicIsland.entitlements b/DynamicIsland/DynamicIsland.entitlements index 2e86356..ab90afd 100644 --- a/DynamicIsland/DynamicIsland.entitlements +++ b/DynamicIsland/DynamicIsland.entitlements @@ -4,6 +4,10 @@ com.apple.security.automation.apple-events + com.apple.security.device.camera + + com.apple.security.device.audio-input + com.apple.security.temporary-exception.apple-events com.spotify.client diff --git a/DynamicIsland/DynamicIslandApp.swift b/DynamicIsland/DynamicIslandApp.swift index 04c6ac4..444a14d 100644 --- a/DynamicIsland/DynamicIslandApp.swift +++ b/DynamicIsland/DynamicIslandApp.swift @@ -2,7 +2,7 @@ // DynamicIslandApp.swift // DynamicIslandApp // -// Created by Harsh Vardhan Goswami on 02/08/24. +// Modified by Hariharan Mudaliar on 20/09/25. // import AVFoundation @@ -581,6 +581,36 @@ class AppDelegate: NSObject, NSApplicationDelegate { ColorPickerPanelManager.shared.toggleColorPickerPanel() } + KeyboardShortcuts.onKeyDown(for: .screenAssistantPanel) { [weak self] in + guard let self = self else { return } + + // Only execute if shortcuts are enabled + guard Defaults[.enableShortcuts] else { return } + + // Only open screen assistant if the feature is enabled + guard Defaults[.enableScreenAssistant] else { return } + + // Handle keyboard shortcut based on display mode + switch Defaults[.screenAssistantDisplayMode] { + case .panel: + ScreenAssistantPanelManager.shared.toggleScreenAssistantPanel() + case .popover: + // For popover mode, first ensure notch is open, then toggle popover + + // If notch is closed, open it first + if self.vm.notchState == .closed { + self.vm.open() + // Wait a moment for the notch to open, then show popover + DispatchQueue.main.asyncAfter(deadline: .now() + 0.3) { + NotificationCenter.default.post(name: NSNotification.Name("ToggleScreenAssistantPopover"), object: nil) + } + } else { + // Notch is already open, toggle popover immediately + NotificationCenter.default.post(name: NSNotification.Name("ToggleScreenAssistantPopover"), object: nil) + } + } + } + KeyboardShortcuts.onKeyDown(for: .statsPanel) { [weak self] in guard let self = self else { return } diff --git a/DynamicIsland/Info.plist b/DynamicIsland/Info.plist index c7a13de..6560d9a 100644 --- a/DynamicIsland/Info.plist +++ b/DynamicIsland/Info.plist @@ -11,6 +11,12 @@ Dynamic Island uses AppleScripts to control Spotify and Apple Music NSCalendarsFullAccessUsageDescription Show calender events inside the notch + NSScreenCaptureUsageDescription + Dynamic Island needs screen recording permission to capture screenshots for the AI assistant feature. + NSDesktopFolderUsageDescription + Dynamic Island needs access to save screenshots to your desktop. + NSDocumentsFolderUsageDescription + Dynamic Island needs access to save and read AI assistant files. SUEnableDownloaderService SUEnableInstallerLauncherService diff --git a/DynamicIsland/Localizable.xcstrings b/DynamicIsland/Localizable.xcstrings index 570f136..9bde979 100644 --- a/DynamicIsland/Localizable.xcstrings +++ b/DynamicIsland/Localizable.xcstrings @@ -28,6 +28,9 @@ }, "%@" : { + }, + "%@ Models" : { + }, "%d%%" : { @@ -52,9 +55,18 @@ }, "• New feature 1" : { + }, + "• Thinking" : { + }, "• Updated %@" : { + }, + "••••••••" : { + + }, + "+%lld more" : { + }, "3 items" : { @@ -121,6 +133,9 @@ }, "Add" : { + }, + "Add files" : { + }, "Add manually" : { @@ -130,6 +145,21 @@ }, "Additional features" : { + }, + "AI Assistant" : { + + }, + "AI is thinking..." : { + + }, + "AI Model Selection" : { + + }, + "AI Provider" : { + + }, + "AI-powered assistant that can analyze files, images, and provide conversational help. Use Cmd+Shift+A to quickly access the assistant." : { + }, "AirDrop" : { @@ -148,6 +178,12 @@ }, "Always show tabs" : { + }, + "API Configuration" : { + + }, + "API Key Required" : { + }, "App icon" : { @@ -157,6 +193,12 @@ }, "Are you sure you want to delete this color from your history?" : { + }, + "Ask me anything..." : { + + }, + "Attached Files" : { + }, "Automatically check for updates" : { "localizations" : { @@ -223,6 +265,9 @@ }, "Cancel" : { + }, + "Change" : { + }, "Charging" : { "localizations" : { @@ -245,15 +290,27 @@ }, "Choose a Music Source" : { + }, + "Choose AI model" : { + }, "Choose File" : { + }, + "Choose your preferred AI model and configuration" : { + }, "Circle" : { }, "Clear" : { + }, + "Clear All Files" : { + + }, + "Clear all files removes all attached files and audio recordings. This action is permanent." : { + }, "Clear Clipboard History" : { @@ -293,6 +350,9 @@ }, "Close" : { + }, + "Close assistant" : { + }, "Color Details" : { @@ -327,6 +387,9 @@ } } } + }, + "Configuration" : { + }, "Continue" : { @@ -534,6 +597,9 @@ }, "Enable system stats monitoring in Settings to view detailed performance data." : { + }, + "Enable Thinking Mode" : { + }, "Enable timer feature in Settings to use this tab" : { @@ -546,6 +612,9 @@ }, "Enjoy your free time!" : { + }, + "Enter your Gemini API Key" : { + }, "Error" : { @@ -568,6 +637,9 @@ }, "Finish" : { + }, + "Gemini API Key" : { + }, "General" : { "localizations" : { @@ -587,6 +659,9 @@ }, "Get started" : { + }, + "Get your free API key from Google AI Studio" : { + }, "GitHub" : { "localizations" : { @@ -609,6 +684,9 @@ }, "Graph Visibility" : { + }, + "Hide" : { + }, "Hide DynamicIsland Options" : { @@ -878,6 +956,9 @@ }, "Not Now" : { + }, + "Not Set" : { + }, "Notch behavior" : { @@ -916,12 +997,21 @@ } } } + }, + "Open Google AI Studio" : { + + }, + "Open Model Settings" : { + }, "Open Settings" : { }, "Open System Settings" : { + }, + "Opens the AI assistant panel for file analysis and conversation. Default is Cmd+Shift+A. Only works when screen assistant feature is enabled." : { + }, "Opens the clipboard history panel. Default is Cmd+Shift+V (similar to Windows+V on PC). Only works when clipboard feature is enabled." : { @@ -937,6 +1027,9 @@ }, "Panel mode shows color picker in a floating window. Popover mode shows color picker as a dropdown attached to the color picker button." : { + }, + "Panel mode shows the assistant in a floating window near the notch. Popover mode shows the assistant as a dropdown attached to the AI button." : { + }, "Pause" : { @@ -977,6 +1070,9 @@ }, "Pinned Items" : { + }, + "Please configure your API key for the selected AI provider in model settings." : { + }, "Plugged In" : { @@ -986,6 +1082,9 @@ }, "Popover mode shows color picker as a dropdown attached to the color picker button. Panel mode shows color picker in a floating window." : { + }, + "Popover mode shows the assistant as a dropdown attached to the AI button. Panel mode shows the assistant in a floating window near the notch." : { + }, "Progress bar" : { "localizations" : { @@ -1029,9 +1128,18 @@ }, "Ready" : { + }, + "Reasoning Mode" : { + }, "Recent Colors" : { + }, + "Recording" : { + + }, + "Recording Status" : { + }, "Release name" : { "localizations" : { @@ -1063,6 +1171,21 @@ }, "Running" : { + }, + "Save" : { + + }, + "Save Configuration" : { + + }, + "Screen Assistant" : { + + }, + "Screen Assistant feature is disabled" : { + + }, + "Screen Assistant:" : { + }, "Search clipboard..." : { @@ -1107,6 +1230,9 @@ }, "Sensitivity" : { + }, + "Set" : { + }, "Set a custom duration for the timer. This will be used when you press the 'Custom' button in the timer interface." : { @@ -1185,6 +1311,9 @@ }, "Show shuffle and repeat buttons" : { + }, + "Shows the model's reasoning process before the final answer" : { + }, "Slider color" : { @@ -1213,6 +1342,9 @@ }, "Start" : { + }, + "Start a conversation to see your chat history here" : { + }, "Start Color Picking" : { @@ -1225,6 +1357,9 @@ }, "Start picking colors to build your history" : { + }, + "Start recording" : { + }, "Starts a 5-minute demo timer to test the timer live activity feature. Only works when timer feature is enabled." : { @@ -1258,6 +1393,9 @@ }, "Stop Monitoring" : { + }, + "Stop recording" : { + }, "Stopped" : { @@ -1272,6 +1410,9 @@ } } } + }, + "Supports reasoning mode" : { + }, "System features" : { @@ -1382,6 +1523,9 @@ }, "With %lld graphs enabled, the Dynamic Island will expand horizontally to accommodate all graphs in a single row." : { + }, + "You" : { + }, "You can now enjoy the app. If you want to tweak things further, you can always visit the settings." : { diff --git a/DynamicIsland/Shortcuts/ShortcutConstants.swift b/DynamicIsland/Shortcuts/ShortcutConstants.swift index dc65e93..122100e 100644 --- a/DynamicIsland/Shortcuts/ShortcutConstants.swift +++ b/DynamicIsland/Shortcuts/ShortcutConstants.swift @@ -11,6 +11,7 @@ import SwiftUI extension KeyboardShortcuts.Name { static let clipboardHistoryPanel = Self("clipboardHistoryPanel", default: .init(.c, modifiers: [.shift, .command])) static let colorPickerPanel = Self("colorPickerPanel", default: .init(.p, modifiers: [.shift, .command])) + static let screenAssistantPanel = Self("screenAssistantPanel", default: .init(.a, modifiers: [.shift, .command])) static let statsPanel = Self("statsPanel", default: .init(.s, modifiers: [.shift, .command])) static let toggleMicrophone = Self("toggleMicrophone", default: .init(.f5, modifiers: [.function])) static let decreaseBacklight = Self("decreaseBacklight", default: .init(.f1, modifiers: [.command])) diff --git a/DynamicIsland/components/Clipboard/ClipboardPanel.swift b/DynamicIsland/components/Clipboard/ClipboardPanel.swift index 2c66817..dec51fb 100644 --- a/DynamicIsland/components/Clipboard/ClipboardPanel.swift +++ b/DynamicIsland/components/Clipboard/ClipboardPanel.swift @@ -7,6 +7,7 @@ import AppKit import SwiftUI +import Defaults class ClipboardPanel: NSPanel { @@ -51,6 +52,10 @@ class ClipboardPanel: NSPanel { .fullScreenAuxiliary // Float above full-screen apps ] + // Apply screenshot protection + updateScreenshotProtection() + setupScreenshotProtectionObserver() + // Accept mouse moved events for proper hover behavior acceptsMouseMovedEvents = true } @@ -134,6 +139,29 @@ class ClipboardPanel: NSPanel { setFrameOrigin(NSPoint(x: xPosition, y: yPosition)) } + + private func setupScreenshotProtectionObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenshotProtection() + } + } + } + + private func updateScreenshotProtection() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Exclude from screenshots and screen recordings + self.sharingType = .none + print("🙈 ClipboardPanel: Protected from screenshots and screen recordings") + } else { + // Allow normal screenshot inclusion + self.sharingType = .readOnly + print("👁️ ClipboardPanel: Visible in screenshots and screen recordings") + } + } } struct ClipboardPanelView: View { diff --git a/DynamicIsland/components/ColorPicker/ColorPickerPanel.swift b/DynamicIsland/components/ColorPicker/ColorPickerPanel.swift index 0798746..ad39a0c 100644 --- a/DynamicIsland/components/ColorPicker/ColorPickerPanel.swift +++ b/DynamicIsland/components/ColorPicker/ColorPickerPanel.swift @@ -93,6 +93,10 @@ class ColorPickerPanel: NSPanel { .fullScreenAuxiliary // Float above full-screen apps ] + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + // Accept mouse moved events for proper hover behavior acceptsMouseMovedEvents = true @@ -156,6 +160,29 @@ class ColorPickerPanel: NSPanel { // Save position whenever it changes (user dragging) saveCurrentPosition() } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 ColorPickerPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ ColorPickerPanel: Visible in screen capture and recordings") + } + } } struct ColorPickerPanelView: View { diff --git a/DynamicIsland/components/ScreenAssistant/ChatPanels.swift b/DynamicIsland/components/ScreenAssistant/ChatPanels.swift new file mode 100644 index 0000000..588b555 --- /dev/null +++ b/DynamicIsland/components/ScreenAssistant/ChatPanels.swift @@ -0,0 +1,672 @@ +// +// ChatPanels.swift +// DynamicIsland +// +// Created by Hariharan Mudaliar + +import AppKit +import SwiftUI +import Defaults + +// MARK: - Chat Messages Panel (Left Side) +class ChatMessagesPanel: NSPanel { + + init() { + super.init( + contentRect: .zero, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: true + ) + + setupWindow() + setupContentView() + } + + override var canBecomeKey: Bool { + return false // Don't steal focus from input panel + } + + override var canBecomeMain: Bool { + return false + } + + private func setupWindow() { + backgroundColor = .clear + isOpaque = false + hasShadow = true + level = .floating + isMovableByWindowBackground = false // Fixed position + titlebarAppearsTransparent = true + titleVisibility = .hidden + isFloatingPanel = true + + collectionBehavior = [ + .canJoinAllSpaces, + .stationary, + .fullScreenAuxiliary + ] + + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + + acceptsMouseMovedEvents = true + } + + private func setupContentView() { + let contentView = ChatMessagesView() + let hostingView = NSHostingView(rootView: contentView) + self.contentView = hostingView + + // Set size for chat messages panel (wider and taller) + let preferredSize = CGSize(width: 600, height: 500) + hostingView.setFrameSize(preferredSize) + setContentSize(preferredSize) + } + + func positionOnLeftSide() { + guard let screen = NSScreen.main else { return } + + let screenFrame = screen.visibleFrame + let panelFrame = frame + + // Position on the left side of the screen + let xPosition = screenFrame.minX + 50 // 50pt from left edge + let yPosition = screenFrame.maxY - panelFrame.height - 100 // 100pt from top + + setFrameOrigin(NSPoint(x: xPosition, y: yPosition)) + } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 ChatMessagesPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ ChatMessagesPanel: Visible in screen capture and recordings") + } + } +} + +// MARK: - Chat Input Panel (Center) +class ChatInputPanel: NSPanel { + + init() { + super.init( + contentRect: .zero, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: true + ) + + setupWindow() + setupContentView() + } + + override var canBecomeKey: Bool { + return true // Can receive focus for text input + } + + override var canBecomeMain: Bool { + return true + } + + // Handle ESC key globally for the panel + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 { // ESC key + ScreenAssistantManager.shared.closePanels() + } else { + super.keyDown(with: event) + } + } + + private func setupWindow() { + backgroundColor = .clear + isOpaque = false + hasShadow = true + level = .floating + isMovableByWindowBackground = true // Enable dragging + titlebarAppearsTransparent = true + titleVisibility = .hidden + isFloatingPanel = true + + styleMask.insert(.fullSizeContentView) + + collectionBehavior = [ + .canJoinAllSpaces, + .stationary, + .fullScreenAuxiliary + ] + + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + + acceptsMouseMovedEvents = true + } + + private func setupContentView() { + let contentView = ChatInputView() + let hostingView = NSHostingView(rootView: contentView) + self.contentView = hostingView + + // Set compact size for single-line input panel + let preferredSize = CGSize(width: 500, height: 60) + hostingView.setFrameSize(preferredSize) + setContentSize(preferredSize) + } + + func positionInCenter() { + guard let screen = NSScreen.main else { return } + + let screenFrame = screen.visibleFrame + let panelFrame = frame + + // Position in the center-bottom of the screen (like a search bar) + let xPosition = (screenFrame.width - panelFrame.width) / 2 + screenFrame.minX + let yPosition = screenFrame.minY + 100 // 100pt from bottom + + setFrameOrigin(NSPoint(x: xPosition, y: yPosition)) + } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 ChatInputPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ ChatInputPanel: Visible in screen capture and recordings") + } + } +} + +// MARK: - Chat Messages View (Redesigned for standalone panel) +struct ChatMessagesView: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + + var body: some View { + VStack(spacing: 0) { + // Header with close button + HStack { + Text("AI Assistant") + .font(.headline) + .fontWeight(.semibold) + .foregroundColor(.primary) + + Spacer() + + Button(action: { + screenAssistantManager.closePanels() + }) { + Image(systemName: "xmark.circle.fill") + .font(.title2) + .foregroundColor(.secondary) + } + .buttonStyle(PlainButtonStyle()) + .help("Close assistant") + } + .padding(.horizontal, 20) + .padding(.vertical, 12) + .background(Color.gray.opacity(0.05)) + + Divider() + + // Chat content + ScrollViewReader { proxy in + ScrollView { + LazyVStack(alignment: .leading, spacing: 16) { + if screenAssistantManager.chatMessages.isEmpty { + VStack(spacing: 20) { + Image(systemName: "brain.head.profile") + .font(.system(size: 60)) + .foregroundColor(.blue.opacity(0.6)) + + VStack(spacing: 8) { + Text("AI Assistant") + .font(.title) + .fontWeight(.bold) + .foregroundColor(.primary) + + Text("Start a conversation to see your chat history here") + .font(.body) + .foregroundColor(.secondary) + .multilineTextAlignment(.center) + } + } + .frame(maxWidth: .infinity, maxHeight: .infinity) + .padding(.top, 80) + } else { + ForEach(screenAssistantManager.chatMessages) { message in + StreamingChatMessageBubble(message: message) + .id(message.id) + } + + if screenAssistantManager.isLoading { + HStack(spacing: 12) { + ProgressView() + .scaleEffect(0.8) + Text("AI is thinking...") + .font(.body) + .foregroundColor(.secondary) + Spacer() + } + .padding(.horizontal, 20) + .padding(.vertical, 12) + } + } + } + .padding(.vertical, 20) + } + .onChange(of: screenAssistantManager.chatMessages.count) { _, _ in + if let lastMessage = screenAssistantManager.chatMessages.last { + withAnimation(.easeOut(duration: 0.5)) { + proxy.scrollTo(lastMessage.id, anchor: .bottom) + } + } + } + .onChange(of: screenAssistantManager.isLoading) { _, _ in + if screenAssistantManager.isLoading { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.2) { + withAnimation(.easeOut(duration: 0.5)) { + if let lastMessage = screenAssistantManager.chatMessages.last { + proxy.scrollTo(lastMessage.id, anchor: .bottom) + } + } + } + } + } + } + } + .background(ChatPanelsVisualEffectView(material: .hudWindow, blendingMode: .behindWindow)) + .cornerRadius(16) + .shadow(color: .black.opacity(0.3), radius: 15, x: 0, y: 8) + } +} + +// MARK: - Chat Input View (Single Line Panel) +struct ChatInputView: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + @State private var messageText = "" + @State private var isDraggingFiles = false + @State private var showingApiKeyAlert = false + @FocusState private var isTextFieldFocused: Bool + + // Current model information + private var currentProvider: AIModelProvider { + Defaults[.selectedAIProvider] + } + + private var currentModel: AIModel? { + Defaults[.selectedAIModel] + } + + var body: some View { + VStack(spacing: 0) { + // Current model indicator + HStack { + HStack(spacing: 6) { + Image(systemName: iconForProvider(currentProvider)) + .font(.caption) + .foregroundColor(.blue) + + Text(currentModel?.name ?? currentProvider.displayName) + .font(.caption) + .foregroundColor(.secondary) + + if currentModel?.supportsThinking == true && Defaults[.enableThinkingMode] { + Text("• Thinking") + .font(.caption2) + .foregroundColor(.purple) + } + } + + Spacer() + + Button("Change", action: openModelSelection) + .font(.caption) + .foregroundColor(.blue) + } + .padding(.horizontal, 12) + .padding(.vertical, 6) + .background(Color.gray.opacity(0.05)) + + // File attachments row (if any) + if !screenAssistantManager.attachedFiles.isEmpty { + ScrollView(.horizontal, showsIndicators: false) { + HStack(spacing: 8) { + ForEach(screenAssistantManager.attachedFiles) { file in + AttachedFileChip(file: file) { + screenAssistantManager.removeFile(file) + } + } + } + .padding(.horizontal, 12) + } + .padding(.vertical, 8) + .background(Color.gray.opacity(0.05)) + + Divider() + } + + // Single line input row + HStack(spacing: 12) { + // Add files button + AddFilesButton() + + // Screenshot snipping button + ScreenshotButton() + + // Text input - SINGLE LINE + TextField("Ask me anything...", text: $messageText) + .textFieldStyle(.plain) + .font(.system(size: 14)) + .focused($isTextFieldFocused) + .padding(.horizontal, 12) + .padding(.vertical, 8) + .background(Color.gray.opacity(0.1)) + .cornerRadius(8) + .onSubmit { + sendMessage() + } + + // Model selection button + Button(action: openModelSelection) { + Image(systemName: "brain.head.profile.fill") + .foregroundColor(.purple) + .font(.system(size: 20)) + } + .buttonStyle(PlainButtonStyle()) + .help("Choose AI model") + + // Recording button + RecordingButton() + + // Send button + Button(action: sendMessage) { + Image(systemName: "paperplane.fill") + .foregroundColor(.white) + .font(.system(size: 12)) + .padding(8) + .background(canSend ? Color.blue : Color.gray) + .cornerRadius(6) + } + .buttonStyle(PlainButtonStyle()) + .disabled(!canSend) + } + .padding(12) + } + .background(ChatPanelsVisualEffectView(material: .hudWindow, blendingMode: .behindWindow)) + .cornerRadius(16) + .shadow(color: .black.opacity(0.3), radius: 15, x: 0, y: 8) + .onDrop(of: [.fileURL], isTargeted: $isDraggingFiles) { providers in + handleFilesDrop(providers) + } + .alert("API Key Required", isPresented: $showingApiKeyAlert) { + Button("Open Model Settings") { + openModelSelection() + } + Button("Cancel", role: .cancel) {} + } message: { + Text("Please configure your API key for the selected AI provider in model settings.") + } + .onAppear { + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { + isTextFieldFocused = true + } + } + } + + private func iconForProvider(_ provider: AIModelProvider) -> String { + switch provider { + case .gemini: return "sparkles" + case .openai: return "brain.head.profile" + case .claude: return "doc.text" + case .local: return "server.rack" + } + } + + private var canSend: Bool { + !messageText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty || !screenAssistantManager.attachedFiles.isEmpty + } + + private func sendMessage() { + // Check if API key is configured for the selected provider + let provider = Defaults[.selectedAIProvider] + var apiKey = "" + + switch provider { + case .gemini: + apiKey = Defaults[.geminiApiKey] + case .openai: + apiKey = Defaults[.openaiApiKey] + case .claude: + apiKey = Defaults[.claudeApiKey] + case .local: + // Local models don't need API keys + apiKey = "local" + } + + if apiKey.isEmpty { + showingApiKeyAlert = true + return + } + + // Prepare the message + let userMessage = messageText.trimmingCharacters(in: .whitespacesAndNewlines) + if userMessage.isEmpty && screenAssistantManager.attachedFiles.isEmpty { + return + } + + // Send message through manager + screenAssistantManager.sendMessage(userMessage) + messageText = "" + } + + private func openModelSelection() { + let panel = ModelSelectionPanel() + panel.positionInCenter() + panel.makeKeyAndOrderFront(nil) + panel.orderFrontRegardless() + + // Activate the app to ensure proper focus handling + NSApp.activate(ignoringOtherApps: true) + } + + private func handleFilesDrop(_ providers: [NSItemProvider]) -> Bool { + for provider in providers { + _ = provider.loadObject(ofClass: URL.self) { url, error in + if let url = url { + DispatchQueue.main.async { + screenAssistantManager.addFiles([url]) + } + } + } + } + return true + } +} + +// MARK: - Enhanced Chat Message Bubble (No Auto-Streaming) +struct StreamingChatMessageBubble: View { + let message: ChatMessage + + var body: some View { + HStack(alignment: .top, spacing: 12) { + if message.isFromUser { + Spacer() + } + + // Avatar + if !message.isFromUser { + Image(systemName: "brain.head.profile") + .font(.title2) + .foregroundColor(.blue) + .frame(width: 32, height: 32) + .background(Color.blue.opacity(0.1)) + .clipShape(Circle()) + } + + VStack(alignment: message.isFromUser ? .trailing : .leading, spacing: 8) { + // Header with name and timestamp + HStack { + Text(message.isFromUser ? "You" : "AI Assistant") + .font(.caption) + .fontWeight(.semibold) + .foregroundColor(message.isFromUser ? .blue : .green) + + Spacer() + + Text(message.timestamp, style: .time) + .font(.caption2) + .foregroundColor(.secondary) + } + + // File attachments (if any) + if let files = message.attachedFiles, !files.isEmpty { + HStack(spacing: 6) { + ForEach(files.prefix(3)) { file in + HStack(spacing: 4) { + Image(systemName: file.type.iconName) + .font(.caption2) + Text(file.name) + .font(.caption2) + .lineLimit(1) + } + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(Color.blue.opacity(0.1)) + .cornerRadius(8) + } + + if files.count > 3 { + Text("+\(files.count - 3) more") + .font(.caption2) + .foregroundColor(.secondary) + } + } + } + + // Message content - NO AUTO STREAMING + MarkdownText(content: message.content) + .padding(.horizontal, 16) + .padding(.vertical, 12) + .background( + RoundedRectangle(cornerRadius: 16) + .fill(message.isFromUser ? Color.blue : Color.gray.opacity(0.15)) + ) + .foregroundColor(message.isFromUser ? .white : .primary) + } + .frame(maxWidth: 400, alignment: message.isFromUser ? .trailing : .leading) + + // User avatar + if message.isFromUser { + Image(systemName: "person.circle.fill") + .font(.title2) + .foregroundColor(.blue) + .frame(width: 32, height: 32) + } else { + Spacer() + } + } + .padding(.horizontal, 20) + } +} + +// MARK: - Note: Shared Components (MarkdownText, AttachedFileChip, AddFilesButton, RecordingButton, ApiKeyAlertView) +// are defined in ScreenAssistantPanel.swift to avoid redeclaration conflicts. + +// MARK: - Screenshot Button Component +struct ScreenshotButton: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + @StateObject private var screenshotTool = ScreenshotSnippingTool.shared + + var body: some View { + Button(action: startScreenshot) { + Image(systemName: getIconName()) + .foregroundColor(getIconColor()) + .font(.system(size: 20)) + } + .buttonStyle(PlainButtonStyle()) + .help(getHelpText()) + .disabled(screenshotTool.isSnipping) + .scaleEffect(screenshotTool.isSnipping ? 1.1 : 1.0) + .animation(.easeInOut(duration: 0.2), value: screenshotTool.isSnipping) + } + + private func getIconName() -> String { + if screenshotTool.isSnipping { + return "camera.viewfinder" + } else { + return "camera.aperture" + } + } + + private func getIconColor() -> Color { + if screenshotTool.isSnipping { + return .red + } else { + return .green + } + } + + private func getHelpText() -> String { + return screenshotTool.isSnipping ? "Taking screenshot..." : "Take screenshot" + } + + private func startScreenshot() { + // Start snipping with direct callback (ScreenshotApp-based approach) + screenshotTool.startSnipping { [weak screenAssistantManager] screenshotURL in + guard let manager = screenAssistantManager else { + print("❌ ScreenshotTool: ScreenAssistantManager deallocated during callback") + return + } + + print("📁 ScreenshotTool: Adding screenshot to chat: \(screenshotURL.lastPathComponent)") + manager.addFiles([screenshotURL]) + print("📸 Screenshot captured and added to chat successfully") + } + } +} + +// MARK: - Visual Effect View for Chat Panels (to avoid conflicts) +struct ChatPanelsVisualEffectView: NSViewRepresentable { + let material: NSVisualEffectView.Material + let blendingMode: NSVisualEffectView.BlendingMode + + func makeNSView(context: Context) -> NSVisualEffectView { + let view = NSVisualEffectView() + view.material = material + view.blendingMode = blendingMode + view.state = .active + return view + } + + func updateNSView(_ nsView: NSVisualEffectView, context: Context) {} +} \ No newline at end of file diff --git a/DynamicIsland/components/ScreenAssistant/ModelSelectionPanel.swift b/DynamicIsland/components/ScreenAssistant/ModelSelectionPanel.swift new file mode 100644 index 0000000..65e82ff --- /dev/null +++ b/DynamicIsland/components/ScreenAssistant/ModelSelectionPanel.swift @@ -0,0 +1,535 @@ +// +// ModelSelectionPanel.swift +// DynamicIsland +// +// Created by Hariharan Mudaliar + +import AppKit +import SwiftUI +import Defaults + +// MARK: - Model Selection Panel +class ModelSelectionPanel: NSPanel { + + init() { + super.init( + contentRect: .zero, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: true + ) + + setupWindow() + setupContentView() + } + + override var canBecomeKey: Bool { + return true // Can receive focus for interaction + } + + override var canBecomeMain: Bool { + return true + } + + // Handle ESC key globally for the panel + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 { // ESC key + close() + } else { + super.keyDown(with: event) + } + } + + private func setupWindow() { + backgroundColor = .clear + isOpaque = false + hasShadow = true + level = .floating + isMovableByWindowBackground = true // Enable dragging + titlebarAppearsTransparent = true + titleVisibility = .hidden + isFloatingPanel = true + + styleMask.insert(.fullSizeContentView) + + collectionBehavior = [ + .canJoinAllSpaces, + .stationary, + .fullScreenAuxiliary + ] + + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + + acceptsMouseMovedEvents = true + } + + private func setupContentView() { + let contentView = ModelSelectionView() + let hostingView = NSHostingView(rootView: contentView) + self.contentView = hostingView + + // Set size for model selection panel + let preferredSize = CGSize(width: 450, height: 600) + hostingView.setFrameSize(preferredSize) + setContentSize(preferredSize) + } + + func positionInCenter() { + guard let screen = NSScreen.main else { return } + + let screenFrame = screen.visibleFrame + let panelFrame = frame + + // Position in the center of the screen + let xPosition = (screenFrame.width - panelFrame.width) / 2 + screenFrame.minX + let yPosition = (screenFrame.height - panelFrame.height) / 2 + screenFrame.minY + + setFrameOrigin(NSPoint(x: xPosition, y: yPosition)) + } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 ModelSelectionPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ ModelSelectionPanel: Visible in screen capture and recordings") + } + } +} + +// MARK: - Model Selection View +struct ModelSelectionView: View { + @State private var selectedProvider: AIModelProvider = Defaults[.selectedAIProvider] + @State private var selectedModel: AIModel? = Defaults[.selectedAIModel] + @State private var enableThinking: Bool = Defaults[.enableThinkingMode] + + // API Keys + @State private var geminiApiKey: String = Defaults[.geminiApiKey] + @State private var openaiApiKey: String = Defaults[.openaiApiKey] + @State private var claudeApiKey: String = Defaults[.claudeApiKey] + @State private var localEndpoint: String = Defaults[.localModelEndpoint] + + @State private var showingApiKeyAlert = false + + var body: some View { + VStack(spacing: 0) { + // Header + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("AI Model Selection") + .font(.title2) + .fontWeight(.bold) + .foregroundColor(.primary) + + Text("Choose your preferred AI model and configuration") + .font(.caption) + .foregroundColor(.secondary) + } + + Spacer() + + Button(action: closePanel) { + Image(systemName: "xmark.circle.fill") + .font(.title2) + .foregroundColor(.secondary) + } + .buttonStyle(PlainButtonStyle()) + .help("Close") + } + .padding(.horizontal, 24) + .padding(.vertical, 16) + .background(Color.gray.opacity(0.05)) + + Divider() + + // Content + ScrollView { + VStack(spacing: 24) { + // Provider Selection + VStack(alignment: .leading, spacing: 12) { + Text("AI Provider") + .font(.headline) + .foregroundColor(.primary) + + LazyVGrid(columns: Array(repeating: GridItem(.flexible()), count: 2), spacing: 12) { + ForEach(AIModelProvider.allCases) { provider in + ProviderCard( + provider: provider, + isSelected: selectedProvider == provider, + onSelect: { selectedProvider = provider } + ) + } + } + } + + Divider() + + // Model Selection + if !selectedProvider.supportedModels.isEmpty { + VStack(alignment: .leading, spacing: 12) { + Text("\(selectedProvider.displayName) Models") + .font(.headline) + .foregroundColor(.primary) + + VStack(spacing: 8) { + ForEach(selectedProvider.supportedModels) { model in + ModelRow( + model: model, + isSelected: selectedModel?.id == model.id, + onSelect: { selectedModel = model } + ) + } + } + } + } + + Divider() + + // Thinking Mode Toggle + if selectedModel?.supportsThinking == true { + VStack(alignment: .leading, spacing: 12) { + Text("Reasoning Mode") + .font(.headline) + .foregroundColor(.primary) + + HStack { + VStack(alignment: .leading, spacing: 4) { + Text("Enable Thinking Mode") + .font(.body) + .foregroundColor(.primary) + + Text("Shows the model's reasoning process before the final answer") + .font(.caption) + .foregroundColor(.secondary) + } + + Spacer() + + Toggle("", isOn: $enableThinking) + .toggleStyle(SwitchToggleStyle()) + } + .padding(16) + .background(Color.gray.opacity(0.1)) + .cornerRadius(12) + } + } + + Divider() + + // API Configuration + VStack(alignment: .leading, spacing: 12) { + Text("API Configuration") + .font(.headline) + .foregroundColor(.primary) + + ApiConfigurationSection( + provider: selectedProvider, + geminiApiKey: $geminiApiKey, + openaiApiKey: $openaiApiKey, + claudeApiKey: $claudeApiKey, + localEndpoint: $localEndpoint + ) + } + } + .padding(.horizontal, 24) + .padding(.vertical, 20) + } + + Divider() + + // Footer with Save/Cancel buttons + HStack { + Button("Cancel") { + closePanel() + } + .buttonStyle(PlainButtonStyle()) + + Spacer() + + Button("Save Configuration") { + saveConfiguration() + } + .buttonStyle(.borderedProminent) + .disabled(!isConfigurationValid) + } + .padding(.horizontal, 24) + .padding(.vertical, 16) + } + .background(ModelSelectionVisualEffectView(material: .hudWindow, blendingMode: .behindWindow)) + .cornerRadius(16) + .shadow(color: .black.opacity(0.3), radius: 20, x: 0, y: 10) + .onAppear { + loadCurrentConfiguration() + } + } + + private var isConfigurationValid: Bool { + switch selectedProvider { + case .gemini: + return !geminiApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + case .openai: + return !openaiApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + case .claude: + return !claudeApiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + case .local: + return !localEndpoint.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty + } + } + + private func loadCurrentConfiguration() { + selectedProvider = Defaults[.selectedAIProvider] + // Default to the latest Gemini 2.5 Flash if no model is selected + selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", supportsThinking: true) + enableThinking = Defaults[.enableThinkingMode] + + geminiApiKey = Defaults[.geminiApiKey] + openaiApiKey = Defaults[.openaiApiKey] + claudeApiKey = Defaults[.claudeApiKey] + localEndpoint = Defaults[.localModelEndpoint] + } + + private func saveConfiguration() { + Defaults[.selectedAIProvider] = selectedProvider + Defaults[.selectedAIModel] = selectedModel + Defaults[.enableThinkingMode] = enableThinking + + Defaults[.geminiApiKey] = geminiApiKey + Defaults[.openaiApiKey] = openaiApiKey + Defaults[.claudeApiKey] = claudeApiKey + Defaults[.localModelEndpoint] = localEndpoint + + closePanel() + + // Notify that configuration changed + NotificationCenter.default.post(name: .aiModelConfigurationChanged, object: nil) + } + + private func closePanel() { + if let window = NSApp.windows.first(where: { $0 is ModelSelectionPanel }) { + window.close() + } + } +} + +// MARK: - Provider Card +struct ProviderCard: View { + let provider: AIModelProvider + let isSelected: Bool + let onSelect: () -> Void + + var body: some View { + VStack(spacing: 12) { + // Icon + ZStack { + Circle() + .fill(isSelected ? Color.blue : Color.gray.opacity(0.2)) + .frame(width: 50, height: 50) + + Image(systemName: iconForProvider(provider)) + .font(.title2) + .foregroundColor(isSelected ? .white : .primary) + } + + // Name and description + VStack(spacing: 4) { + Text(provider.displayName) + .font(.headline) + .foregroundColor(.primary) + + Text(provider.description) + .font(.caption) + .foregroundColor(.secondary) + .multilineTextAlignment(.center) + .lineLimit(2) + } + } + .padding(16) + .background( + RoundedRectangle(cornerRadius: 12) + .fill(isSelected ? Color.blue.opacity(0.1) : Color.gray.opacity(0.05)) + .overlay( + RoundedRectangle(cornerRadius: 12) + .stroke(isSelected ? Color.blue : Color.clear, lineWidth: 2) + ) + ) + .onTapGesture { + onSelect() + } + .animation(.easeInOut(duration: 0.2), value: isSelected) + } + + private func iconForProvider(_ provider: AIModelProvider) -> String { + switch provider { + case .gemini: return "sparkles" + case .openai: return "brain.head.profile" + case .claude: return "doc.text" + case .local: return "server.rack" + } + } +} + +// MARK: - Model Row +struct ModelRow: View { + let model: AIModel + let isSelected: Bool + let onSelect: () -> Void + + var body: some View { + HStack { + VStack(alignment: .leading, spacing: 4) { + Text(model.name) + .font(.body) + .foregroundColor(.primary) + + if model.supportsThinking { + Text("Supports reasoning mode") + .font(.caption) + .foregroundColor(.blue) + .padding(.horizontal, 8) + .padding(.vertical, 2) + .background(Color.blue.opacity(0.1)) + .cornerRadius(4) + } + } + + Spacer() + + if isSelected { + Image(systemName: "checkmark.circle.fill") + .foregroundColor(.blue) + .font(.title3) + } else { + Circle() + .stroke(Color.gray.opacity(0.5), lineWidth: 2) + .frame(width: 20, height: 20) + } + } + .padding(12) + .background( + RoundedRectangle(cornerRadius: 8) + .fill(isSelected ? Color.blue.opacity(0.1) : Color.clear) + ) + .onTapGesture { + onSelect() + } + .animation(.easeInOut(duration: 0.2), value: isSelected) + } +} + +// MARK: - API Configuration Section +struct ApiConfigurationSection: View { + let provider: AIModelProvider + @Binding var geminiApiKey: String + @Binding var openaiApiKey: String + @Binding var claudeApiKey: String + @Binding var localEndpoint: String + + var body: some View { + VStack(spacing: 12) { + switch provider { + case .gemini: + ApiKeyField( + title: "Gemini API Key", + placeholder: "Enter your Gemini API key", + value: $geminiApiKey, + helpText: "Get your API key from Google AI Studio" + ) + + case .openai: + ApiKeyField( + title: "OpenAI API Key", + placeholder: "Enter your OpenAI API key", + value: $openaiApiKey, + helpText: "Get your API key from OpenAI Platform" + ) + + case .claude: + ApiKeyField( + title: "Claude API Key", + placeholder: "Enter your Claude API key", + value: $claudeApiKey, + helpText: "Get your API key from Anthropic Console" + ) + + case .local: + ApiKeyField( + title: "Local Endpoint", + placeholder: "http://localhost:11434", + value: $localEndpoint, + helpText: "Ollama or compatible API endpoint", + isSecure: false + ) + } + } + .padding(16) + .background(Color.gray.opacity(0.05)) + .cornerRadius(12) + } +} + +// MARK: - API Key Field +struct ApiKeyField: View { + let title: String + let placeholder: String + @Binding var value: String + let helpText: String + var isSecure: Bool = true + + var body: some View { + VStack(alignment: .leading, spacing: 8) { + Text(title) + .font(.subheadline) + .fontWeight(.medium) + .foregroundColor(.primary) + + if isSecure { + SecureField(placeholder, text: $value) + .textFieldStyle(.roundedBorder) + } else { + TextField(placeholder, text: $value) + .textFieldStyle(.roundedBorder) + } + + Text(helpText) + .font(.caption) + .foregroundColor(.secondary) + } + } +} + +// MARK: - Visual Effect View +struct ModelSelectionVisualEffectView: NSViewRepresentable { + let material: NSVisualEffectView.Material + let blendingMode: NSVisualEffectView.BlendingMode + + func makeNSView(context: Context) -> NSVisualEffectView { + let view = NSVisualEffectView() + view.material = material + view.blendingMode = blendingMode + view.state = .active + return view + } + + func updateNSView(_ nsView: NSVisualEffectView, context: Context) {} +} + +// MARK: - Notification Extension +extension Notification.Name { + static let aiModelConfigurationChanged = Notification.Name("aiModelConfigurationChanged") +} \ No newline at end of file diff --git a/DynamicIsland/components/ScreenAssistant/ScreenAssistantPanel.swift b/DynamicIsland/components/ScreenAssistant/ScreenAssistantPanel.swift new file mode 100644 index 0000000..5c36558 --- /dev/null +++ b/DynamicIsland/components/ScreenAssistant/ScreenAssistantPanel.swift @@ -0,0 +1,272 @@ +// +// ScreenAssistantPanel.swift +// DynamicIsland +// +// Created by Hariharan Mudaliar + +import AppKit +import SwiftUI +import Defaults +import UniformTypeIdentifiers + +// MARK: - Backward Compatibility Wrapper +class ScreenAssistantPanel: NSPanel { + + init() { + super.init( + contentRect: .zero, + styleMask: [.borderless, .nonactivatingPanel], + backing: .buffered, + defer: true + ) + + // This is now a compatibility wrapper + // The actual functionality is handled by the new ChatPanels + setupWindow() + } + + override var canBecomeKey: Bool { + return false + } + + override var canBecomeMain: Bool { + return false + } + + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 { // ESC key + close() + } else { + super.keyDown(with: event) + } + } + + private func setupWindow() { + backgroundColor = .clear + isOpaque = false + hasShadow = false + level = .floating + isMovableByWindowBackground = false + titlebarAppearsTransparent = true + titleVisibility = .hidden + isFloatingPanel = true + + collectionBehavior = [ + .canJoinAllSpaces, + .stationary, + .fullScreenAuxiliary + ] + + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + + // Set minimal size since this is just a wrapper + setContentSize(CGSize(width: 1, height: 1)) + setFrameOrigin(CGPoint(x: -1000, y: -1000)) // Move offscreen + } + + // Trigger the new panel system + override func makeKeyAndOrderFront(_ sender: Any?) { + ScreenAssistantManager.shared.showPanels() + // Don't actually show this wrapper panel + } + + override func close() { + ScreenAssistantManager.shared.closePanels() + super.close() + } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 ScreenAssistantPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ ScreenAssistantPanel: Visible in screen capture and recordings") + } + } +} + +// Keep the old panel view for any existing references, but make it redirect +struct ScreenAssistantPanelView: View { + let onClose: () -> Void + + var body: some View { + VStack { + Text("Screen Assistant") + .onAppear { + // Redirect to new panel system + ScreenAssistantManager.shared.showPanels() + onClose() + } + } + .frame(width: 1, height: 1) + } +} + +// MARK: - Shared Components (moved to ChatPanels.swift for the new implementation) +// These are kept here for backward compatibility only + +struct MarkdownText: View { + let content: String + + var body: some View { + // Simple markdown parsing for now + Text(parseMarkdown(content)) + .font(.system(size: 14)) + .textSelection(.enabled) + } + + private func parseMarkdown(_ text: String) -> AttributedString { + var attributedString = AttributedString(text) + + // Simple bold parsing (**text**) + let boldPattern = #"\*\*(.*?)\*\*"# + if let regex = try? NSRegularExpression(pattern: boldPattern) { + let matches = regex.matches(in: text, range: NSRange(text.startIndex..., in: text)) + for match in matches.reversed() { + if let range = Range(match.range, in: text) { + let boldText = String(text[range]).replacingOccurrences(of: "**", with: "") + if let attrRange = Range(match.range, in: attributedString) { + var boldAttributedText = AttributedString(boldText) + boldAttributedText.font = .system(size: 14, weight: .bold) + attributedString.replaceSubrange(attrRange, with: boldAttributedText) + } + } + } + } + + return attributedString + } +} + +struct AttachedFileChip: View { + let file: ScreenAssistantFile + let onRemove: () -> Void + + var body: some View { + HStack(spacing: 6) { + Image(systemName: file.type.iconName) + .foregroundColor(.blue) + .font(.system(size: 12)) + + Text(file.name) + .font(.system(size: 11)) + .lineLimit(1) + .truncationMode(.middle) + + Button(action: onRemove) { + Image(systemName: "xmark.circle.fill") + .foregroundColor(.gray) + .font(.system(size: 10)) + } + .buttonStyle(PlainButtonStyle()) + } + .padding(.horizontal, 8) + .padding(.vertical, 4) + .background(Color.blue.opacity(0.1)) + .cornerRadius(6) + .frame(maxWidth: 120) + } +} + +struct AddFilesButton: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + + var body: some View { + Button(action: selectFiles) { + Image(systemName: "plus.circle.fill") + .foregroundColor(.blue) + .font(.system(size: 20)) + } + .buttonStyle(PlainButtonStyle()) + .help("Add files") + } + + private func selectFiles() { + let panel = NSOpenPanel() + panel.allowsMultipleSelection = true + panel.canChooseDirectories = false + panel.canChooseFiles = true + panel.allowedContentTypes = [.data, .image, .movie, .audio, .text, .pdf] + + if let window = NSApp.keyWindow ?? NSApp.mainWindow { + panel.beginSheetModal(for: window) { response in + if response == .OK { + screenAssistantManager.addFiles(panel.urls) + } + } + } else { + if panel.runModal() == .OK { + screenAssistantManager.addFiles(panel.urls) + } + } + } +} + +struct RecordingButton: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + + var body: some View { + Button(action: { + screenAssistantManager.toggleRecording() + }) { + ZStack { + Circle() + .fill(screenAssistantManager.isRecording ? Color.red : Color.blue.opacity(0.2)) + .frame(width: 32, height: 32) + + Image(systemName: screenAssistantManager.isRecording ? "stop.fill" : "mic.fill") + .foregroundColor(screenAssistantManager.isRecording ? .white : .blue) + .font(.system(size: 14)) + } + } + .buttonStyle(PlainButtonStyle()) + .help(screenAssistantManager.isRecording ? "Stop recording" : "Start recording") + .scaleEffect(screenAssistantManager.isRecording ? 1.1 : 1.0) + .animation(.easeInOut(duration: 0.2), value: screenAssistantManager.isRecording) + } +} + +struct ApiKeyAlertView: View { + @State private var apiKey = "" + + var body: some View { + VStack(spacing: 12) { + TextField("Enter your Gemini API Key", text: $apiKey) + .textFieldStyle(.roundedBorder) + + HStack { + Button("Cancel") { + // Dialog will close automatically + } + + Button("Save") { + Defaults[.geminiApiKey] = apiKey + } + .disabled(apiKey.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + } + .frame(width: 300) + } +} + +#Preview { + ScreenAssistantPanelView { + print("Close panel") + } +} \ No newline at end of file diff --git a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift new file mode 100644 index 0000000..02c0a67 --- /dev/null +++ b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift @@ -0,0 +1,142 @@ +// +// ScreenshotSnippingTool.swift +// DynamicIsland +// +// Created by Assistant based on ScreenshotApp-main research + +import AppKit +import SwiftUI +import Foundation + +// MARK: - Simplified Screenshot Tool (Based on ScreenshotApp Research) +class ScreenshotSnippingTool: NSObject, ObservableObject { + static let shared = ScreenshotSnippingTool() + + @Published var isSnipping = false + private var completion: ((URL) -> Void)? + + enum ScreenshotError: Error { + case captureFailed + case noImageInPasteboard + case saveFailed + } + + override init() { + super.init() + } + + // MARK: - Simple API (Based on ScreenshotApp Implementation) + func startSnipping(completion: @escaping (URL) -> Void) { + guard !isSnipping else { return } + + print("🖼️ ScreenshotTool: Starting area screenshot using screencapture tool") + self.completion = completion + isSnipping = true + + // Use the same approach as ScreenshotApp - direct screencapture command + takeAreaScreenshot() + } + + // MARK: - ScreenshotApp-Style Implementation + private func takeAreaScreenshot() { + let task = Process() + task.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture") + task.arguments = ["-cs"] // -c = clipboard, -s = area selection + + do { + print("📸 ScreenshotTool: Running screencapture -cs command") + try task.run() + task.waitUntilExit() + + // Process completed - check if successful + if task.terminationStatus == 0 { + print("✅ ScreenshotTool: screencapture completed successfully") + getImageFromPasteboard() + } else { + print("❌ ScreenshotTool: screencapture failed with status: \(task.terminationStatus)") + finishSnipping() + } + + } catch { + print("❌ ScreenshotTool: Failed to run screencapture: \(error)") + finishSnipping() + } + } + + // MARK: - Pasteboard Integration (ScreenshotApp Pattern) + private func getImageFromPasteboard() { + print("� ScreenshotTool: Checking pasteboard for screenshot") + + guard NSPasteboard.general.canReadItem(withDataConformingToTypes: NSImage.imageTypes) else { + print("❌ ScreenshotTool: No image data in pasteboard") + finishSnipping() + return + } + + guard let image = NSImage(pasteboard: NSPasteboard.general) else { + print("❌ ScreenshotTool: Failed to create NSImage from pasteboard") + finishSnipping() + return + } + + print("✅ ScreenshotTool: Got image from pasteboard: \(image.size)") + saveImageAndComplete(image: image) + } + + // MARK: - Image Saving + private func saveImageAndComplete(image: NSImage) { + let filename = "screenshot_\(Int(Date().timeIntervalSince1970)).png" + let screenshotDir = ScreenAssistantManager.screenshotDataDirectory + + // Ensure directory exists + if !FileManager.default.fileExists(atPath: screenshotDir.path) { + try? FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) + } + + let screenshotURL = screenshotDir.appendingPathComponent(filename) + + // Convert NSImage to PNG data + guard let imageData = image.tiffRepresentation, + let bitmapRep = NSBitmapImageRep(data: imageData), + let pngData = bitmapRep.representation(using: NSBitmapImageRep.FileType.png, properties: [:]) else { + print("❌ ScreenshotTool: Failed to convert image to PNG") + finishSnipping() + return + } + + do { + try pngData.write(to: screenshotURL) + print("✅ ScreenshotTool: Screenshot saved to: \(screenshotURL.path)") + + // Execute completion callback + let callback = self.completion + self.completion = nil + finishSnipping() + + // Call completion on main thread + DispatchQueue.main.async { + callback?(screenshotURL) + } + + } catch { + print("❌ ScreenshotTool: Failed to save image: \(error)") + finishSnipping() + } + } + + // MARK: - State Management + private func finishSnipping() { + print("🔄 ScreenshotTool: Finishing snipping process") + + DispatchQueue.main.async { + self.isSnipping = false + self.completion = nil + print("✅ ScreenshotTool: Snipping process completed") + } + } + + func cancelSnipping() { + print("❌ ScreenshotTool: Snipping cancelled") + finishSnipping() + } +} \ No newline at end of file diff --git a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift new file mode 100644 index 0000000..b06c1a8 --- /dev/null +++ b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift @@ -0,0 +1,500 @@ +// This file has been replaced - keeping for reference only + +// MARK: - Screenshot Snipping Tool +class ScreenshotSnippingTool: NSObject, ObservableObject { + static let shared = ScreenshotSnippingTool() + + @Published var isSnipping = false + @Published var hasPermissions = false + @Published var permissionStatus: PermissionStatus = .unknown + private var snippingWindow: SnippingWindow? + + enum PermissionStatus { + case unknown + case granted + case denied + case restricted + } + + override init() { + super.init() + checkPermissionsAsync() + } + + private func checkPermissionsAsync() { + Task { + await checkPermissions() + } + } + + @MainActor + func checkPermissions() async { + print("🔍 ScreenshotTool: Checking permissions for bundle: \(Bundle.main.bundleIdentifier ?? "unknown")") + + do { + // First try a simple content check to see if we can access ScreenCaptureKit + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) + + // If we get here without error, permissions are granted + self.hasPermissions = true + self.permissionStatus = .granted + print("✅ ScreenshotTool: Permissions granted for \(content.displays.count) displays") + + } catch let error as NSError { + print("❌ ScreenshotTool: Permission error - Code: \(error.code), Domain: \(error.domain)") + + // Handle specific error codes + switch error.code { + case -3801: // User declined TCC permission + self.hasPermissions = false + self.permissionStatus = .denied + print("❌ ScreenshotTool: Permission denied by user - TCC database entry missing or denied") + + case -3802: // App is not authorized for screen capture + self.hasPermissions = false + self.permissionStatus = .restricted + print("❌ ScreenshotTool: App not authorized for screen capture - System restriction") + + default: + self.hasPermissions = false + self.permissionStatus = .unknown + print("❌ ScreenshotTool: Unknown permission error - \(error.localizedDescription)") + } + + // Provide user guidance + DispatchQueue.main.async { + self.showPermissionGuidance() + } + } + } + + private func showPermissionGuidance() { + let alert = NSAlert() + alert.messageText = "Screen Recording Permission Required" + alert.informativeText = """ + Dynamic Island needs screen recording permission to capture screenshots. + + To enable this: + 1. Open System Settings (or System Preferences) + 2. Go to Privacy & Security → Screen Recording + 3. Enable 'Dynamic Island' + 4. Restart the app if needed + """ + alert.alertStyle = .warning + alert.addButton(withTitle: "Open System Settings") + alert.addButton(withTitle: "Cancel") + + let response = alert.runModal() + if response == .alertFirstButtonReturn { + // Open System Settings to Screen Recording + if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") { + NSWorkspace.shared.open(url) + } + } + } + + func requestPermissions() async { + // Only request if we don't already have them + guard !hasPermissions else { return } + await checkPermissions() + } + + func startSnipping() { + guard !isSnipping else { return } + + // Check permissions first + if !hasPermissions { + print("❌ ScreenshotTool: No permissions - checking status") + Task { + await requestPermissions() + } + return + } + + print("🖼️ ScreenshotTool: Starting screen snipping") + isSnipping = true + + // Hide all app windows temporarily + hideAppWindows() + + // Create snipping overlay + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { + self.createSnippingOverlay() + } + } + + private func hideAppWindows() { + for window in NSApp.windows { + window.orderOut(nil) + } + } + + private func restoreAppWindows() { + for window in NSApp.windows { + if window != snippingWindow { + window.orderFront(nil) + } + } + } + + private func createSnippingOverlay() { + guard let screen = NSScreen.main else { + print("❌ ScreenshotTool: No main screen available") + finishSnipping() + return + } + + print("🎯 ScreenshotTool: Creating snipping overlay") + + snippingWindow = SnippingWindow(screen: screen) { [weak self] rect in + guard let self = self else { + print("❌ ScreenshotTool: Self deallocated during area selection") + return + } + self.captureScreenArea(rect) + } + + guard let window = snippingWindow else { + print("❌ ScreenshotTool: Failed to create snipping window") + finishSnipping() + return + } + + // Just make it key and order front, don't try to make it main + window.makeKeyAndOrderFront(nil) + window.orderFrontRegardless() + + // Activate the app to ensure proper focus + NSApp.activate(ignoringOtherApps: true) + + print("✅ ScreenshotTool: Snipping overlay created and activated") + } + + private func captureScreenArea(_ rect: CGRect) { + print("🖼️ ScreenshotTool: Capturing area: \(rect)") + + Task { + do { + let screenshot = try await captureScreenshot(rect: rect) + + DispatchQueue.main.async { [weak self] in + guard let self = self else { + print("❌ ScreenshotTool: Self deallocated during screenshot callback") + return + } + + print("✅ ScreenshotTool: Screenshot saved successfully") + + // Capture screenshot URL before cleanup + let screenshotURL = screenshot + + // Finish snipping first to clean up window state + self.finishSnipping() + + // Post notification after cleanup with captured URL + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { + print("📸 ScreenshotTool: Posting screenshot notification") + NotificationCenter.default.post( + name: Notification.Name("screenshotCaptured"), + object: screenshotURL + ) + print("✅ ScreenshotTool: Notification posted successfully") + } + } + } catch { + print("❌ ScreenshotTool: Failed to capture screenshot - \(error)") + DispatchQueue.main.async { [weak self] in + guard let self = self else { return } + + // Show error to user + self.showScreenshotError(error) + self.finishSnipping() + } + } + } + } + + private func showScreenshotError(_ error: Error) { + let alert = NSAlert() + alert.messageText = "Screenshot Failed" + alert.informativeText = "Failed to capture screenshot: \(error.localizedDescription)" + alert.alertStyle = .warning + alert.addButton(withTitle: "OK") + alert.runModal() + } + + private func captureScreenshot(rect: CGRect) async throws -> URL { + // Use ScreenCaptureKit for modern screenshot capture + let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) + + guard let display = content.displays.first else { + throw ScreenshotError.noDisplayFound + } + + let config = SCStreamConfiguration() + config.width = Int(rect.width) + config.height = Int(rect.height) + config.sourceRect = rect + config.pixelFormat = kCVPixelFormatType_32BGRA + config.showsCursor = false + + let filter = SCContentFilter(display: display, excludingWindows: []) + let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) + + // Ensure screenshot directory exists + let screenshotDir = ScreenAssistantManager.screenshotDataDirectory + if !FileManager.default.fileExists(atPath: screenshotDir.path) { + try FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) + } + + // Save to temporary file + let filename = "screenshot_\(Date().timeIntervalSince1970).png" + let screenshotURL = screenshotDir.appendingPathComponent(filename) + + // Create NSImage from CGImage and convert to PNG + let nsImage = NSImage(cgImage: image, size: NSSize(width: image.width, height: image.height)) + + guard let imageData = nsImage.tiffRepresentation, + let bitmapRep = NSBitmapImageRep(data: imageData), + let pngData = bitmapRep.representation(using: NSBitmapImageRep.FileType.png, properties: [:]) else { + throw ScreenshotError.imageConversionFailed + } + + do { + try pngData.write(to: screenshotURL) + print("✅ ScreenshotTool: Saved screenshot to: \(screenshotURL.path)") + return screenshotURL + } catch { + print("❌ ScreenshotTool: Failed to write screenshot file - \(error)") + throw ScreenshotError.fileWriteError + } + } + + private func finishSnipping() { + guard isSnipping else { + print("⚠️ ScreenshotTool: finishSnipping called but not currently snipping") + return + } + + print("🔄 ScreenshotTool: Finishing snipping process") + + isSnipping = false + + // Close and cleanup snipping window safely + if let window = snippingWindow { + window.orderOut(nil) + window.close() + snippingWindow = nil + print("🪟 ScreenshotTool: Snipping window closed and cleared") + } + + // Restore app windows with delay to prevent conflicts + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { + self.restoreAppWindows() + print("🏠 ScreenshotTool: App windows restored") + } + + print("✅ ScreenshotTool: Snipping process completed") + } + + func cancelSnipping() { + finishSnipping() + } +} + +// MARK: - Snipping Window +class SnippingWindow: NSWindow { + private let onAreaSelected: (CGRect) -> Void + private var selectionOverlay: SelectionOverlayView? + + init(screen: NSScreen, onAreaSelected: @escaping (CGRect) -> Void) { + self.onAreaSelected = onAreaSelected + + super.init( + contentRect: screen.frame, + styleMask: [.borderless, .fullSizeContentView], + backing: .buffered, + defer: false + ) + + setupWindow(screen: screen) + setupContent() + } + + override var canBecomeKey: Bool { + return true + } + + override var canBecomeMain: Bool { + return true + } + + private func setupWindow(screen: NSScreen) { + backgroundColor = NSColor.black.withAlphaComponent(0.3) + isOpaque = false + level = .screenSaver + ignoresMouseEvents = false + acceptsMouseMovedEvents = true + isMovableByWindowBackground = false + titleVisibility = .hidden + titlebarAppearsTransparent = true + + collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] + setFrame(screen.frame, display: true) + } + + private func setupContent() { + selectionOverlay = SelectionOverlayView { [weak self] rect in + self?.onAreaSelected(rect) + } + + if let overlay = selectionOverlay { + let hostingView = NSHostingView(rootView: overlay) + contentView = hostingView + } + } + + override func keyDown(with event: NSEvent) { + if event.keyCode == 53 { // ESC key + close() + ScreenshotSnippingTool.shared.cancelSnipping() + } else { + super.keyDown(with: event) + } + } +} + +// MARK: - Selection Overlay View +struct SelectionOverlayView: NSViewRepresentable { + let onAreaSelected: (CGRect) -> Void + + func makeNSView(context: Context) -> SelectionView { + let view = SelectionView() + view.onAreaSelected = onAreaSelected + return view + } + + func updateNSView(_ nsView: SelectionView, context: Context) {} +} + +// MARK: - Selection View +class SelectionView: NSView { + var onAreaSelected: ((CGRect) -> Void)? + private var startPoint: CGPoint? + private var currentRect: CGRect = .zero + private var isDragging = false + + override func awakeFromNib() { + super.awakeFromNib() + wantsLayer = true + layer?.backgroundColor = NSColor.clear.cgColor + } + + override func mouseDown(with event: NSEvent) { + startPoint = convert(event.locationInWindow, from: nil) + isDragging = true + currentRect = .zero + needsDisplay = true + } + + override func mouseDragged(with event: NSEvent) { + guard let start = startPoint, isDragging else { return } + + let currentPoint = convert(event.locationInWindow, from: nil) + + let x = min(start.x, currentPoint.x) + let y = min(start.y, currentPoint.y) + let width = abs(currentPoint.x - start.x) + let height = abs(currentPoint.y - start.y) + + currentRect = CGRect(x: x, y: y, width: width, height: height) + needsDisplay = true + } + + override func mouseUp(with event: NSEvent) { + guard isDragging, !currentRect.isEmpty else { + isDragging = false + return + } + + isDragging = false + + // Convert to screen coordinates + if let window = self.window { + let screenRect = window.convertToScreen(currentRect) + // Flip Y coordinate for screen capture + let flippedRect = CGRect( + x: screenRect.origin.x, + y: NSScreen.main!.frame.height - screenRect.origin.y - screenRect.height, + width: screenRect.width, + height: screenRect.height + ) + onAreaSelected?(flippedRect) + } + } + + override func draw(_ dirtyRect: NSRect) { + super.draw(dirtyRect) + + guard isDragging, !currentRect.isEmpty else { return } + + // Draw selection rectangle + NSColor.systemBlue.withAlphaComponent(0.3).setFill() + currentRect.fill() + + NSColor.systemBlue.setStroke() + let path = NSBezierPath(rect: currentRect) + path.lineWidth = 2.0 + path.stroke() + + // Draw selection info + let infoText = String(format: "%.0f × %.0f", currentRect.width, currentRect.height) + let attributes: [NSAttributedString.Key: Any] = [ + .font: NSFont.systemFont(ofSize: 12), + .foregroundColor: NSColor.white, + .backgroundColor: NSColor.black.withAlphaComponent(0.8) + ] + + let attributedString = NSAttributedString(string: infoText, attributes: attributes) + let textRect = CGRect( + x: currentRect.maxX - 80, + y: currentRect.minY - 20, + width: 80, + height: 20 + ) + + attributedString.draw(in: textRect) + } +} + +// MARK: - Screenshot Error Types +enum ScreenshotError: Error, LocalizedError { + case noDisplayFound + case imageConversionFailed + case fileWriteError + + var errorDescription: String? { + switch self { + case .noDisplayFound: + return "No display found for screenshot capture" + case .imageConversionFailed: + return "Failed to convert screenshot image" + case .fileWriteError: + return "Failed to save screenshot file" + } + } +} + +// MARK: - ScreenAssistantManager Extension for Screenshot Directory +extension ScreenAssistantManager { + static var screenshotDataDirectory: URL { + let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! + let appDir = appSupport.appendingPathComponent(bundleIdentifier) + let screenshotDir = appDir.appendingPathComponent("Screenshots") + + // Create directory if it doesn't exist + try? FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) + + return screenshotDir + } +} \ No newline at end of file diff --git a/DynamicIsland/components/Settings/SettingsView.swift b/DynamicIsland/components/Settings/SettingsView.swift index e6d33c1..0650c7a 100644 --- a/DynamicIsland/components/Settings/SettingsView.swift +++ b/DynamicIsland/components/Settings/SettingsView.swift @@ -56,6 +56,9 @@ struct SettingsView: View { NavigationLink(value: "Clipboard") { Label("Clipboard", systemImage: "clipboard") } + NavigationLink(value: "ScreenAssistant") { + Label("Screen Assistant", systemImage: "brain.head.profile") + } NavigationLink(value: "ColorPicker") { Label("Color Picker", systemImage: "eyedropper") } @@ -102,6 +105,8 @@ struct SettingsView: View { StatsSettings() case "Clipboard": ClipboardSettings() + case "ScreenAssistant": + ScreenAssistantSettings() case "ColorPicker": ColorPickerSettings() case "Downloads": @@ -183,6 +188,7 @@ struct GeneralSettings: View { NotificationCenter.default.post(name: Notification.Name.automaticallySwitchDisplayChanged, object: nil) } .disabled(showOnAllDisplays) + Defaults.Toggle("Hide panels from screenshots & screen recordings", key: .hidePanelsFromScreenCapture) } header: { Text("System features") } @@ -1260,6 +1266,29 @@ struct Shortcuts: View { .font(.caption) } + Section { + HStack { + VStack(alignment: .leading) { + KeyboardShortcuts.Recorder("Screen Assistant:", name: .screenAssistantPanel) + .disabled(!enableShortcuts || !Defaults[.enableScreenAssistant]) + if !Defaults[.enableScreenAssistant] { + Text("Screen Assistant feature is disabled") + .font(.caption) + .foregroundStyle(.secondary) + .padding(.top, 2) + } + } + Spacer() + } + } header: { + Text("AI Assistant") + } footer: { + Text("Opens the AI assistant panel for file analysis and conversation. Default is Cmd+Shift+A. Only works when screen assistant feature is enabled.") + .multilineTextAlignment(.trailing) + .foregroundStyle(.secondary) + .font(.caption) + } + Section { HStack { VStack(alignment: .leading) { @@ -1877,6 +1906,176 @@ struct ClipboardSettings: View { } } +struct ScreenAssistantSettings: View { + @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared + @Default(.enableScreenAssistant) var enableScreenAssistant + @Default(.screenAssistantDisplayMode) var screenAssistantDisplayMode + @Default(.geminiApiKey) var geminiApiKey + @State private var apiKeyText = "" + @State private var showingApiKey = false + + var body: some View { + Form { + Section { + Defaults.Toggle("Enable Screen Assistant", key: .enableScreenAssistant) + } header: { + Text("AI Assistant") + } footer: { + Text("AI-powered assistant that can analyze files, images, and provide conversational help. Use Cmd+Shift+A to quickly access the assistant.") + } + + if enableScreenAssistant { + Section { + HStack { + Text("Gemini API Key") + Spacer() + if geminiApiKey.isEmpty { + Text("Not Set") + .foregroundColor(.red) + } else { + Text("••••••••") + .foregroundColor(.green) + } + + Button(showingApiKey ? "Hide" : (geminiApiKey.isEmpty ? "Set" : "Change")) { + if showingApiKey { + showingApiKey = false + if !apiKeyText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty { + Defaults[.geminiApiKey] = apiKeyText + } + apiKeyText = "" + } else { + showingApiKey = true + apiKeyText = geminiApiKey + } + } + } + + if showingApiKey { + VStack(alignment: .leading, spacing: 8) { + SecureField("Enter your Gemini API Key", text: $apiKeyText) + .textFieldStyle(.roundedBorder) + + Text("Get your free API key from Google AI Studio") + .font(.caption) + .foregroundColor(.secondary) + + HStack { + Button("Open Google AI Studio") { + NSWorkspace.shared.open(URL(string: "https://aistudio.google.com/app/apikey")!) + } + .buttonStyle(.link) + + Spacer() + + Button("Save") { + Defaults[.geminiApiKey] = apiKeyText + showingApiKey = false + apiKeyText = "" + } + .disabled(apiKeyText.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) + } + } + } + + HStack { + Text("Display Mode") + Spacer() + Picker("Display Mode", selection: $screenAssistantDisplayMode) { + ForEach(ScreenAssistantDisplayMode.allCases, id: \.self) { mode in + Text(mode.displayName).tag(mode) + } + } + .pickerStyle(.menu) + .frame(width: 100) + } + + HStack { + Text("Attached Files") + Spacer() + Text("\(screenAssistantManager.attachedFiles.count)") + .foregroundColor(.secondary) + } + + HStack { + Text("Recording Status") + Spacer() + Text(screenAssistantManager.isRecording ? "Recording" : "Ready") + .foregroundColor(screenAssistantManager.isRecording ? .red : .secondary) + } + } header: { + Text("Configuration") + } footer: { + switch screenAssistantDisplayMode { + case .popover: + Text("Popover mode shows the assistant as a dropdown attached to the AI button. Panel mode shows the assistant in a floating window near the notch.") + case .panel: + Text("Panel mode shows the assistant in a floating window near the notch. Popover mode shows the assistant as a dropdown attached to the AI button.") + } + } + + Section { + Button("Clear All Files") { + screenAssistantManager.clearAllFiles() + } + .foregroundColor(.red) + .disabled(screenAssistantManager.attachedFiles.isEmpty) + } header: { + Text("Actions") + } footer: { + Text("Clear all files removes all attached files and audio recordings. This action is permanent.") + } + + if !screenAssistantManager.attachedFiles.isEmpty { + Section { + ForEach(screenAssistantManager.attachedFiles) { file in + VStack(alignment: .leading, spacing: 4) { + HStack { + Image(systemName: file.type.iconName) + .foregroundColor(.blue) + .frame(width: 16) + Text(file.type.displayName) + .font(.caption) + .foregroundColor(.secondary) + Spacer() + Text(timeAgoString(from: file.timestamp)) + .font(.caption2) + .foregroundColor(.secondary) + } + Text(file.name) + .font(.system(.body, design: .monospaced)) + .lineLimit(2) + } + .padding(.vertical, 2) + } + } header: { + Text("Attached Files") + } + } + } + } + .formStyle(.grouped) + .navigationTitle("Screen Assistant") + } + + private func timeAgoString(from date: Date) -> String { + let interval = Date().timeIntervalSince(date) + + if interval < 60 { + return "Just now" + } else if interval < 3600 { + let minutes = Int(interval / 60) + return "\(minutes)m ago" + } else if interval < 86400 { + let hours = Int(interval / 3600) + return "\(hours)h ago" + } else { + let days = Int(interval / 86400) + return "\(days)d ago" + } + } +} + struct ColorPickerSettings: View { @ObservedObject var colorPickerManager = ColorPickerManager.shared @Default(.enableColorPickerFeature) var enableColorPickerFeature diff --git a/DynamicIsland/components/Settings/SettingsWindowController.swift b/DynamicIsland/components/Settings/SettingsWindowController.swift index 1dbf35b..2aa66ae 100644 --- a/DynamicIsland/components/Settings/SettingsWindowController.swift +++ b/DynamicIsland/components/Settings/SettingsWindowController.swift @@ -64,6 +64,10 @@ class SettingsWindowController: NSWindowController { // Handle window closing window.delegate = self + + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() } func showWindow() { @@ -110,6 +114,29 @@ class SettingsWindowController: NSWindowController { // Set app back to accessory mode immediately NSApp.setActivationPolicy(.accessory) } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + window?.sharingType = .none + print("🙈 SettingsWindow: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + window?.sharingType = .readOnly + print("👁️ SettingsWindow: Visible in screen capture and recordings") + } + } } extension SettingsWindowController: NSWindowDelegate { diff --git a/DynamicIsland/components/Stats/StatsPanel.swift b/DynamicIsland/components/Stats/StatsPanel.swift index d1335b0..ee986c5 100644 --- a/DynamicIsland/components/Stats/StatsPanel.swift +++ b/DynamicIsland/components/Stats/StatsPanel.swift @@ -7,6 +7,7 @@ import AppKit import SwiftUI +import Defaults class StatsPanel: NSPanel { @@ -51,6 +52,10 @@ class StatsPanel: NSPanel { .fullScreenAuxiliary // Float above full-screen apps ] + // Apply screen capture hiding setting + updateScreenCaptureVisibility() + setupScreenCaptureObserver() + // Accept mouse moved events for proper hover behavior acceptsMouseMovedEvents = true } @@ -99,4 +104,27 @@ class StatsPanel: NSPanel { setFrameOrigin(NSPoint(x: xPosition, y: yPosition)) } + + private func setupScreenCaptureObserver() { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak self] change in + DispatchQueue.main.async { + self?.updateScreenCaptureVisibility() + } + } + } + + private func updateScreenCaptureVisibility() { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + self.sharingType = .none + print("🙈 StatsPanel: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + self.sharingType = .readOnly + print("👁️ StatsPanel: Visible in screen capture and recordings") + } + } } diff --git a/DynamicIsland/managers/ScreenAssistantManager.swift b/DynamicIsland/managers/ScreenAssistantManager.swift new file mode 100644 index 0000000..72c9c73 --- /dev/null +++ b/DynamicIsland/managers/ScreenAssistantManager.swift @@ -0,0 +1,1224 @@ +// +// ScreenAssistantManager.swift +// DynamicIsland +// +// Created by Hariharan Mudaliar + +import AppKit +import SwiftUI +import AVFoundation +import Defaults +import Foundation + +// Chat message model +struct ChatMessage: Identifiable, Codable { + let id = UUID() + let content: String + let isFromUser: Bool + let timestamp: Date + let attachedFiles: [ScreenAssistantFile]? + + init(content: String, isFromUser: Bool, attachedFiles: [ScreenAssistantFile]? = nil) { + self.content = content + self.isFromUser = isFromUser + self.timestamp = Date() + self.attachedFiles = attachedFiles + } +} + +// Screen Assistant item data structure +struct ScreenAssistantFile: Identifiable, Codable { + let id = UUID() + let name: String + let type: FileType + let timestamp: Date + let fileURL: String? // For local files + let audioFileName: String? // For audio recordings + + enum FileType: String, CaseIterable, Codable { + case document = "document" + case image = "image" + case audio = "audio" + case video = "video" + case other = "other" + + var iconName: String { + switch self { + case .document: return "doc.text" + case .image: return "photo" + case .audio: return "waveform" + case .video: return "video" + case .other: return "doc" + } + } + + var displayName: String { + switch self { + case .document: return "Document" + case .image: return "Image" + case .audio: return "Audio" + case .video: return "Video" + case .other: return "File" + } + } + } + + init(fileURL: URL) { + // Defensive initialization with nil coalescing + self.name = fileURL.lastPathComponent.isEmpty ? "Unknown File" : fileURL.lastPathComponent + self.fileURL = fileURL.absoluteString + self.audioFileName = nil + self.timestamp = Date() + + // Safe file extension extraction + let fileExtension = fileURL.pathExtension.lowercased() + switch fileExtension { + case "jpg", "jpeg", "png", "gif", "bmp", "tiff", "webp", "heic": + self.type = .image + case "mp3", "wav", "m4a", "aac", "flac": + self.type = .audio + case "mp4", "mov", "avi", "mkv": + self.type = .video + case "txt", "md", "pdf", "doc", "docx", "rtf": + self.type = .document + default: + self.type = .other + } + + print("✅ ScreenAssistantFile: Created file entry - name: \(self.name), type: \(self.type), url: \(self.fileURL ?? "nil")") + } + + init(audioFileName: String, name: String) { + self.name = name + self.type = .audio + self.fileURL = nil + self.audioFileName = audioFileName + self.timestamp = Date() + } +} + +class ScreenAssistantManager: NSObject, ObservableObject { + static let shared = ScreenAssistantManager() + + @Published var attachedFiles: [ScreenAssistantFile] = [] + @Published var isRecording: Bool = false + @Published var recordingDuration: TimeInterval = 0 + @Published var chatMessages: [ChatMessage] = [] + @Published var isLoading: Bool = false + + private var audioRecorder: AVAudioRecorder? + private var recordingTimer: Timer? + + // Panel management + private var chatMessagesPanel: ChatMessagesPanel? + private var chatInputPanel: ChatInputPanel? + + // Directory for storing audio recordings + static let audioDataDirectory: URL = { + let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first! + let audioDir = documentsPath.appendingPathComponent("ScreenAssistantAudio") + + // Create directory if it doesn't exist + try? FileManager.default.createDirectory(at: audioDir, withIntermediateDirectories: true) + + return audioDir + }() + + // Directory for storing screenshots + static let screenshotDataDirectory: URL = { + let documentsPath = FileManager.default.urls(for: .documentDirectory, in: .userDomainMask).first! + let screenshotDir = documentsPath.appendingPathComponent("ScreenAssistantScreenshots") + + // Create directory if it doesn't exist + try? FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) + + return screenshotDir + }() + + private override init() { + super.init() + loadFilesFromDefaults() + } + + deinit { + stopRecording() + closePanels() + } + + // MARK: - Panel Management + + func showPanels() { + // Close existing panels first + closePanels() + + // Create and show chat messages panel (left side) + chatMessagesPanel = ChatMessagesPanel() + chatMessagesPanel?.positionOnLeftSide() + chatMessagesPanel?.makeKeyAndOrderFront(nil) + + // Create and show input panel (center) + chatInputPanel = ChatInputPanel() + chatInputPanel?.positionInCenter() + chatInputPanel?.makeKeyAndOrderFront(nil) + + // Focus on input panel for immediate typing + DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { + self.chatInputPanel?.makeKey() + } + } + + func closePanels() { + chatMessagesPanel?.close() + chatInputPanel?.close() + chatMessagesPanel = nil + chatInputPanel = nil + } + + func arePanelsVisible() -> Bool { + return chatMessagesPanel?.isVisible == true || chatInputPanel?.isVisible == true + } + + // MARK: - File Management + + func addFiles(_ urls: [URL]) { + guard !urls.isEmpty else { + print("⚠️ ScreenAssistant: No URLs provided to addFiles") + return + } + + print("📁 ScreenAssistant: Adding \(urls.count) files") + + let newFiles = urls.compactMap { url -> ScreenAssistantFile? in + // Wrap in autoreleasepool to manage memory + return autoreleasepool { + do { + // Verify file exists + guard FileManager.default.fileExists(atPath: url.path) else { + print("❌ ScreenAssistant: File does not exist at \(url.path)") + return nil + } + + // Verify file is readable + guard FileManager.default.isReadableFile(atPath: url.path) else { + print("❌ ScreenAssistant: File is not readable at \(url.path)") + return nil + } + + // Create file entry with error handling + let file = ScreenAssistantFile(fileURL: url) + print("✅ ScreenAssistant: Created file entry for \(file.name)") + return file + + } catch { + print("❌ ScreenAssistant: Error creating file entry - \(error)") + return nil + } + } + } + + guard !newFiles.isEmpty else { + print("⚠️ ScreenAssistant: No valid files to add") + return + } + + // Ensure we're on the main thread for @Published property updates + DispatchQueue.main.async { [weak self] in + guard let self = self else { + print("❌ ScreenAssistant: Self deallocated during addFiles") + return + } + + self.attachedFiles.append(contentsOf: newFiles) + print("📁 ScreenAssistant: Total attached files: \(self.attachedFiles.count)") + + // Save to defaults with error handling + do { + self.saveFilesToDefaults() + } catch { + print("❌ ScreenAssistant: Failed to save files after adding - \(error)") + } + } + } + + func removeFile(_ file: ScreenAssistantFile) { + attachedFiles.removeAll { $0.id == file.id } + + // Clean up audio file if it exists + if let audioFileName = file.audioFileName { + let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(audioFileName) + try? FileManager.default.removeItem(at: audioURL) + } + + saveFilesToDefaults() + } + + func clearAllFiles() { + // Clean up all audio files + for file in attachedFiles { + if let audioFileName = file.audioFileName { + let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(audioFileName) + try? FileManager.default.removeItem(at: audioURL) + } + } + + attachedFiles.removeAll() + saveFilesToDefaults() + } + + // MARK: - Audio Recording + + func toggleRecording() { + if isRecording { + stopRecording() + } else { + startRecording() + } + } + + private func startRecording() { + guard !isRecording else { return } + + let fileName = "recording_\(Date().timeIntervalSince1970).m4a" + let audioURL = ScreenAssistantManager.audioDataDirectory.appendingPathComponent(fileName) + + let settings: [String: Any] = [ + AVFormatIDKey: Int(kAudioFormatMPEG4AAC), + AVSampleRateKey: 44100, + AVNumberOfChannelsKey: 2, + AVEncoderAudioQualityKey: AVAudioQuality.high.rawValue + ] + + do { + audioRecorder = try AVAudioRecorder(url: audioURL, settings: settings) + audioRecorder?.delegate = self + audioRecorder?.isMeteringEnabled = true + audioRecorder?.prepareToRecord() + audioRecorder?.record() + + isRecording = true + recordingDuration = 0 + + // Start timer for recording duration + recordingTimer = Timer.scheduledTimer(withTimeInterval: 0.1, repeats: true) { [weak self] _ in + self?.updateRecordingDuration() + } + + print("Started recording: \(fileName)") + } catch { + print("Failed to start recording: \(error)") + } + } + + private func stopRecording() { + guard isRecording else { return } + + audioRecorder?.stop() + recordingTimer?.invalidate() + recordingTimer = nil + isRecording = false + + print("Stopped recording") + } + + private func updateRecordingDuration() { + guard let recorder = audioRecorder, recorder.isRecording else { return } + recordingDuration = recorder.currentTime + } + + // MARK: - Persistence + + private func saveFilesToDefaults() { + do { + let encoded = try JSONEncoder().encode(attachedFiles) + UserDefaults.standard.set(encoded, forKey: "ScreenAssistantFiles") + print("✅ ScreenAssistant: Saved \(attachedFiles.count) files to UserDefaults") + } catch { + print("❌ ScreenAssistant: Failed to save files to UserDefaults - \(error)") + // Don't throw - this is a non-critical operation + } + } + + private func loadFilesFromDefaults() { + guard let data = UserDefaults.standard.data(forKey: "ScreenAssistantFiles"), + let decoded = try? JSONDecoder().decode([ScreenAssistantFile].self, from: data) else { + return + } + + attachedFiles = decoded + } + + // MARK: - Chat Management + + func sendMessage(_ message: String) { + print("📤 ScreenAssistant: Sending message - '\(message)'") + print("📁 ScreenAssistant: Attached files count: \(attachedFiles.count)") + + // Add user message to chat + let userMessage = ChatMessage(content: message, isFromUser: true, attachedFiles: attachedFiles.isEmpty ? nil : attachedFiles) + chatMessages.append(userMessage) + + // Print attached files details + for (index, file) in attachedFiles.enumerated() { + print("📎 ScreenAssistant: File \(index + 1): \(file.name) (\(file.type.displayName))") + } + + // Clear input and files after sending + let currentFiles = attachedFiles + clearAllFiles() + + // Send to appropriate AI API based on selected provider + let provider = Defaults[.selectedAIProvider] + sendToAI(message: message, files: currentFiles, provider: provider) + } + + private func sendToAI(message: String, files: [ScreenAssistantFile], provider: AIModelProvider) { + print("🚀 ScreenAssistant: Making API request to \(provider.displayName)") + isLoading = true + + switch provider { + case .gemini: + sendToGeminiAPI(message: message, files: files) + case .openai: + sendToOpenAIAPI(message: message, files: files) + case .claude: + sendToClaudeAPI(message: message, files: files) + case .local: + sendToLocalAPI(message: message, files: files) + } + } + + private func sendToGeminiAPI(message: String, files: [ScreenAssistantFile]) { + let apiKey = Defaults[.geminiApiKey] + guard !apiKey.isEmpty else { + print("❌ ScreenAssistant: No Gemini API key configured") + addAssistantMessage("Error: No Gemini API key configured. Please set your API key in model settings.") + isLoading = false + return + } + + // Get selected model or default to gemini-2.5-flash + let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", supportsThinking: true) + let modelId = selectedModel.id + + guard let url = URL(string: "https://generativelanguage.googleapis.com/v1beta/models/\(modelId):generateContent?key=\(apiKey)") else { + print("❌ ScreenAssistant: Invalid Gemini API URL") + addAssistantMessage("Error: Invalid API URL") + isLoading = false + return + } + + performAPIRequest(url: url, requestBody: buildGeminiRequestBody(message: message, files: files), provider: .gemini) + } + + private func sendToOpenAIAPI(message: String, files: [ScreenAssistantFile]) { + let apiKey = Defaults[.openaiApiKey] + guard !apiKey.isEmpty else { + print("❌ ScreenAssistant: No OpenAI API key configured") + addAssistantMessage("Error: No OpenAI API key configured. Please set your API key in model settings.") + isLoading = false + return + } + + // Get selected model or default to gpt-4o + let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "gpt-4o", name: "GPT-4o", supportsThinking: false) + let modelId = selectedModel.id + + guard let url = URL(string: "https://api.openai.com/v1/chat/completions") else { + print("❌ ScreenAssistant: Invalid OpenAI API URL") + addAssistantMessage("Error: Invalid API URL") + isLoading = false + return + } + + performOpenAIRequest(url: url, requestBody: buildOpenAIRequestBody(message: message, files: files, model: modelId), apiKey: apiKey) + } + + private func sendToClaudeAPI(message: String, files: [ScreenAssistantFile]) { + let apiKey = Defaults[.claudeApiKey] + guard !apiKey.isEmpty else { + print("❌ ScreenAssistant: No Claude API key configured") + addAssistantMessage("Error: No Claude API key configured. Please set your API key in model settings.") + isLoading = false + return + } + + // Get selected model or default to claude-3-5-sonnet + let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "claude-3-5-sonnet-20241022", name: "Claude 3.5 Sonnet", supportsThinking: false) + let modelId = selectedModel.id + + guard let url = URL(string: "https://api.anthropic.com/v1/messages") else { + print("❌ ScreenAssistant: Invalid Claude API URL") + addAssistantMessage("Error: Invalid API URL") + isLoading = false + return + } + + performClaudeRequest(url: url, requestBody: buildClaudeRequestBody(message: message, files: files, model: modelId), apiKey: apiKey) + } + + private func sendToLocalAPI(message: String, files: [ScreenAssistantFile]) { + let endpoint = Defaults[.localModelEndpoint] + guard !endpoint.isEmpty else { + print("❌ ScreenAssistant: No local endpoint configured") + addAssistantMessage("Error: No local endpoint configured. Please set your endpoint in model settings.") + isLoading = false + return + } + + guard let url = URL(string: "\(endpoint)/api/chat") else { + print("❌ ScreenAssistant: Invalid local API URL") + addAssistantMessage("Error: Invalid local API URL") + isLoading = false + return + } + + performAPIRequest(url: url, requestBody: buildOllamaRequestBody(message: message, files: files), provider: .local) + } + + // MARK: - API Request Builders + + private func buildGeminiRequestBody(message: String, files: [ScreenAssistantFile]) -> [String: Any] { + var contents: [[String: Any]] = [] + + // Add previous conversation messages (last 10 for context) + let recentMessages = Array(chatMessages.suffix(10)) + for chatMessage in recentMessages { + if chatMessage.id != chatMessages.last?.id { // Don't include the message we just added + let role = chatMessage.isFromUser ? "user" : "model" + contents.append([ + "role": role, + "parts": [["text": chatMessage.content]] + ]) + } + } + + // Build current message parts + var parts: [[String: Any]] = [] + + // Add text part + let contextualMessage = buildContextualMessage(message: message, files: files) + parts.append(["text": contextualMessage]) + + // Add file content for supported types using proper Gemini 2.5 APIs + for file in files { + if let filePart = createGeminiFilePart(for: file) { + parts.append(filePart) + print("📎 ScreenAssistant: Added file part for \(file.name)") + } + } + + // Add current message + contents.append([ + "role": "user", + "parts": parts + ]) + + var requestBody: [String: Any] = [ + "contents": contents, + "generationConfig": [ + "temperature": 0.7, + "topP": 0.8, + "topK": 40, + "maxOutputTokens": 2048, + "responseMimeType": "text/plain" + ], + "safetySettings": [ + [ + "category": "HARM_CATEGORY_HARASSMENT", + "threshold": "BLOCK_MEDIUM_AND_ABOVE" + ], + [ + "category": "HARM_CATEGORY_HATE_SPEECH", + "threshold": "BLOCK_MEDIUM_AND_ABOVE" + ], + [ + "category": "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "threshold": "BLOCK_MEDIUM_AND_ABOVE" + ], + [ + "category": "HARM_CATEGORY_DANGEROUS_CONTENT", + "threshold": "BLOCK_MEDIUM_AND_ABOVE" + ] + ] + ] + + // Add thinking configuration if enabled and model supports it + let selectedModel = Defaults[.selectedAIModel] + if selectedModel?.supportsThinking == true && Defaults[.enableThinkingMode] { + requestBody["generationConfig"] = (requestBody["generationConfig"] as! [String: Any]).merging([ + "thinkingConfig": [ + "thinkingBudget": 0 // 0 means unlimited thinking + ] + ]) { (_, new) in new } + } + + return requestBody + } + + private func buildOpenAIRequestBody(message: String, files: [ScreenAssistantFile], model: String) -> [String: Any] { + var messages: [[String: Any]] = [] + + // Add previous conversation messages (last 10 for context) + let recentMessages = Array(chatMessages.suffix(10)) + for chatMessage in recentMessages { + if chatMessage.id != chatMessages.last?.id { + let role = chatMessage.isFromUser ? "user" : "assistant" + messages.append([ + "role": role, + "content": chatMessage.content + ]) + } + } + + // Add current message + let contextualMessage = buildContextualMessage(message: message, files: files) + messages.append([ + "role": "user", + "content": contextualMessage + ]) + + return [ + "model": model, + "messages": messages, + "temperature": 0.7, + "max_tokens": 2048 + ] + } + + private func buildClaudeRequestBody(message: String, files: [ScreenAssistantFile], model: String) -> [String: Any] { + var messages: [[String: Any]] = [] + + // Add previous conversation messages (last 10 for context) + let recentMessages = Array(chatMessages.suffix(10)) + for chatMessage in recentMessages { + if chatMessage.id != chatMessages.last?.id { + let role = chatMessage.isFromUser ? "user" : "assistant" + messages.append([ + "role": role, + "content": chatMessage.content + ]) + } + } + + // Add current message + let contextualMessage = buildContextualMessage(message: message, files: files) + messages.append([ + "role": "user", + "content": contextualMessage + ]) + + return [ + "model": model, + "max_tokens": 2048, + "messages": messages + ] + } + + private func buildOllamaRequestBody(message: String, files: [ScreenAssistantFile]) -> [String: Any] { + let selectedModel = Defaults[.selectedAIModel] ?? AIModel(id: "llama3.2", name: "Llama 3.2", supportsThinking: false) + let contextualMessage = buildContextualMessage(message: message, files: files) + + return [ + "model": selectedModel.id, + "messages": [ + [ + "role": "user", + "content": contextualMessage + ] + ], + "stream": false + ] + } + + // MARK: - API Request Performers + + private func performAPIRequest(url: URL, requestBody: [String: Any], provider: AIModelProvider) { + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + + do { + let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) + request.httpBody = jsonData + + print("📋 ScreenAssistant: Request body size: \(jsonData.count) bytes") + } catch { + print("❌ ScreenAssistant: Failed to encode request - \(error)") + addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") + isLoading = false + return + } + + let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in + DispatchQueue.main.async { + guard let self = self else { return } + self.handleResponse(data: data, response: response, error: error, provider: provider) + } + } + + task.resume() + } + + private func performOpenAIRequest(url: URL, requestBody: [String: Any], apiKey: String) { + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue("Bearer \(apiKey)", forHTTPHeaderField: "Authorization") + + do { + let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) + request.httpBody = jsonData + } catch { + print("❌ ScreenAssistant: Failed to encode OpenAI request - \(error)") + addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") + isLoading = false + return + } + + let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in + DispatchQueue.main.async { + guard let self = self else { return } + self.handleResponse(data: data, response: response, error: error, provider: .openai) + } + } + + task.resume() + } + + private func performClaudeRequest(url: URL, requestBody: [String: Any], apiKey: String) { + var request = URLRequest(url: url) + request.httpMethod = "POST" + request.addValue("application/json", forHTTPHeaderField: "Content-Type") + request.addValue(apiKey, forHTTPHeaderField: "x-api-key") + request.addValue("2023-06-01", forHTTPHeaderField: "anthropic-version") + + do { + let jsonData = try JSONSerialization.data(withJSONObject: requestBody, options: .prettyPrinted) + request.httpBody = jsonData + } catch { + print("❌ ScreenAssistant: Failed to encode Claude request - \(error)") + addAssistantMessage("Error: Failed to encode request - \(error.localizedDescription)") + isLoading = false + return + } + + let task = URLSession.shared.dataTask(with: request) { [weak self] data, response, error in + DispatchQueue.main.async { + guard let self = self else { return } + self.handleResponse(data: data, response: response, error: error, provider: .claude) + } + } + + task.resume() + } + + // MARK: - Response Handlers + + private func handleResponse(data: Data?, response: URLResponse?, error: Error?, provider: AIModelProvider) { + isLoading = false + + if let error = error { + print("❌ ScreenAssistant: Network error - \(error)") + addAssistantMessage("Error: \(error.localizedDescription)") + return + } + + if let httpResponse = response as? HTTPURLResponse { + print("📊 ScreenAssistant: HTTP Status: \(httpResponse.statusCode)") + if httpResponse.statusCode != 200 { + handleAPIError(statusCode: httpResponse.statusCode, provider: provider) + return + } + } + + guard let data = data else { + print("❌ ScreenAssistant: No response data") + addAssistantMessage("Error: No response data") + return + } + + print("📨 ScreenAssistant: Response data size: \(data.count) bytes") + + // Parse response based on provider + switch provider { + case .gemini: + parseGeminiResponse(data: data) + case .openai: + parseOpenAIResponse(data: data) + case .claude: + parseClaudeResponse(data: data) + case .local: + parseOllamaResponse(data: data) + } + } + + private func parseGeminiResponse(data: Data) { + do { + if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + print("✅ ScreenAssistant: Successfully parsed Gemini JSON response") + + if let candidates = json["candidates"] as? [[String: Any]], + let firstCandidate = candidates.first, + let content = firstCandidate["content"] as? [String: Any], + let parts = content["parts"] as? [[String: Any]], + let firstPart = parts.first, + let text = firstPart["text"] as? String { + + print("✅ ScreenAssistant: Got Gemini response text: \(text.prefix(100))...") + addAssistantMessage(text) + } else { + if let error = json["error"] as? [String: Any] { + handleAPIError(error: error) + } else { + print("❌ ScreenAssistant: Unexpected Gemini response format") + addAssistantMessage("Error: Unexpected response format from Gemini") + } + } + } + } catch { + print("❌ ScreenAssistant: Gemini JSON parsing error - \(error)") + addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") + } + } + + private func parseOpenAIResponse(data: Data) { + do { + if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + print("✅ ScreenAssistant: Successfully parsed OpenAI JSON response") + + if let choices = json["choices"] as? [[String: Any]], + let firstChoice = choices.first, + let message = firstChoice["message"] as? [String: Any], + let content = message["content"] as? String { + + print("✅ ScreenAssistant: Got OpenAI response text: \(content.prefix(100))...") + addAssistantMessage(content) + } else { + if let error = json["error"] as? [String: Any] { + handleOpenAIError(error: error) + } else { + print("❌ ScreenAssistant: Unexpected OpenAI response format") + addAssistantMessage("Error: Unexpected response format from OpenAI") + } + } + } + } catch { + print("❌ ScreenAssistant: OpenAI JSON parsing error - \(error)") + addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") + } + } + + private func parseClaudeResponse(data: Data) { + do { + if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + print("✅ ScreenAssistant: Successfully parsed Claude JSON response") + + if let content = json["content"] as? [[String: Any]], + let firstContent = content.first, + let text = firstContent["text"] as? String { + + print("✅ ScreenAssistant: Got Claude response text: \(text.prefix(100))...") + addAssistantMessage(text) + } else { + if let error = json["error"] as? [String: Any] { + handleClaudeError(error: error) + } else { + print("❌ ScreenAssistant: Unexpected Claude response format") + addAssistantMessage("Error: Unexpected response format from Claude") + } + } + } + } catch { + print("❌ ScreenAssistant: Claude JSON parsing error - \(error)") + addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") + } + } + + private func parseOllamaResponse(data: Data) { + do { + if let json = try JSONSerialization.jsonObject(with: data) as? [String: Any] { + print("✅ ScreenAssistant: Successfully parsed Ollama JSON response") + + if let message = json["message"] as? [String: Any], + let content = message["content"] as? String { + + print("✅ ScreenAssistant: Got Ollama response text: \(content.prefix(100))...") + addAssistantMessage(content) + } else { + print("❌ ScreenAssistant: Unexpected Ollama response format") + addAssistantMessage("Error: Unexpected response format from local model") + } + } + } catch { + print("❌ ScreenAssistant: Ollama JSON parsing error - \(error)") + addAssistantMessage("Error: Failed to parse response - \(error.localizedDescription)") + } + } + + private func buildContextualMessage(message: String, files: [ScreenAssistantFile]) -> String { + var contextualMessage = message + + // Add file context with specific instructions for different types + if !files.isEmpty { + contextualMessage += "\n\nI have attached the following files for your analysis:" + + var hasImages = false + var hasDocuments = false + var hasAudio = false + var hasVideo = false + + for file in files { + contextualMessage += "\n- \(file.name) (\(file.type.displayName))" + + switch file.type { + case .image: hasImages = true + case .document: hasDocuments = true + case .audio: hasAudio = true + case .video: hasVideo = true + case .other: break + } + } + + // Add specific instructions based on file types + contextualMessage += "\n\nPlease analyze these files in the context of my question. Specifically:" + + if hasImages { + contextualMessage += "\n- For images: Describe what you see, identify objects, text, or patterns, and relate them to my question." + } + + if hasDocuments { + contextualMessage += "\n- For documents: Read and understand the content, extract key information, and provide insights relevant to my question." + } + + if hasAudio { + contextualMessage += "\n- For audio: Listen to and transcribe the audio content, identify speakers, topics, or sounds as relevant." + } + + if hasVideo { + contextualMessage += "\n- For video: Analyze both visual and audio content, describe actions, scenes, or dialogue as applicable." + } + + contextualMessage += "\n\nProvide comprehensive insights that combine information from all attached files with your response to my question." + } + + return contextualMessage + } + + private func createGeminiFilePart(for file: ScreenAssistantFile) -> [String: Any]? { + print("📎 ScreenAssistant: Processing file for Gemini 2.5: \(file.name) (\(file.type.displayName))") + + guard let fileURL = file.fileURL, let url = URL(string: fileURL) else { + print("❌ ScreenAssistant: No valid URL for file \(file.name)") + return ["text": "File: \(file.name) (no valid URL)"] + } + + switch file.type { + case .image: + return createGeminiImagePart(for: url, fileName: file.name) + case .document: + return createGeminiDocumentPart(for: url, fileName: file.name) + case .audio: + return createGeminiAudioPart(for: url, fileName: file.name) + case .video: + return createGeminiVideoPart(for: url, fileName: file.name) + case .other: + return createGeminiTextPart(for: url, fileName: file.name) + } + } + + private func createGeminiImagePart(for url: URL, fileName: String) -> [String: Any]? { + print("🖼️ ScreenAssistant: Processing image file: \(fileName)") + + do { + let imageData = try Data(contentsOf: url) + let base64String = imageData.base64EncodedString() + + // Determine MIME type + let mimeType: String + let pathExtension = url.pathExtension.lowercased() + switch pathExtension { + case "jpg", "jpeg": + mimeType = "image/jpeg" + case "png": + mimeType = "image/png" + case "gif": + mimeType = "image/gif" + case "webp": + mimeType = "image/webp" + case "heic": + mimeType = "image/heic" + default: + mimeType = "image/jpeg" + } + + print("📎 ScreenAssistant: Image encoded - \(base64String.count) bytes, MIME: \(mimeType)") + + return [ + "inline_data": [ + "mime_type": mimeType, + "data": base64String + ] + ] + } catch { + print("❌ ScreenAssistant: Failed to encode image \(fileName): \(error)") + return ["text": "Image file: \(fileName) (failed to encode: \(error.localizedDescription))"] + } + } + + private func createGeminiDocumentPart(for url: URL, fileName: String) -> [String: Any]? { + print("📄 ScreenAssistant: Processing document file: \(fileName)") + + let pathExtension = url.pathExtension.lowercased() + + if pathExtension == "pdf" { + // Handle PDF files using base64 encoding for Gemini 2.5 + do { + let pdfData = try Data(contentsOf: url) + let base64String = pdfData.base64EncodedString() + + print("📎 ScreenAssistant: PDF encoded - \(base64String.count) bytes") + + return [ + "inline_data": [ + "mime_type": "application/pdf", + "data": base64String + ] + ] + } catch { + print("❌ ScreenAssistant: Failed to encode PDF \(fileName): \(error)") + return ["text": "PDF file: \(fileName) (failed to encode: \(error.localizedDescription))"] + } + } else { + // Handle text-based documents + do { + let content = try String(contentsOf: url) + print("📄 ScreenAssistant: Read document content (\(content.count) characters)") + return ["text": "File content of \(fileName):\n\(content)"] + } catch { + print("❌ ScreenAssistant: Failed to read document \(fileName): \(error)") + return ["text": "Document file: \(fileName) (could not read content: \(error.localizedDescription))"] + } + } + } + + private func createGeminiAudioPart(for url: URL, fileName: String) -> [String: Any]? { + print("🎵 ScreenAssistant: Processing audio file: \(fileName)") + + do { + let audioData = try Data(contentsOf: url) + let base64String = audioData.base64EncodedString() + + // Determine MIME type + let mimeType: String + let pathExtension = url.pathExtension.lowercased() + switch pathExtension { + case "mp3": + mimeType = "audio/mpeg" + case "wav": + mimeType = "audio/wav" + case "m4a": + mimeType = "audio/mp4" + case "aac": + mimeType = "audio/aac" + case "flac": + mimeType = "audio/flac" + default: + mimeType = "audio/mpeg" + } + + print("� ScreenAssistant: Audio encoded - \(base64String.count) bytes, MIME: \(mimeType)") + + return [ + "inline_data": [ + "mime_type": mimeType, + "data": base64String + ] + ] + } catch { + print("❌ ScreenAssistant: Failed to encode audio \(fileName): \(error)") + return ["text": "Audio file: \(fileName) (failed to encode: \(error.localizedDescription))"] + } + } + + private func createGeminiVideoPart(for url: URL, fileName: String) -> [String: Any]? { + print("� ScreenAssistant: Processing video file: \(fileName)") + + do { + let videoData = try Data(contentsOf: url) + let base64String = videoData.base64EncodedString() + + // Determine MIME type + let mimeType: String + let pathExtension = url.pathExtension.lowercased() + switch pathExtension { + case "mp4": + mimeType = "video/mp4" + case "mov": + mimeType = "video/quicktime" + case "avi": + mimeType = "video/x-msvideo" + case "mkv": + mimeType = "video/x-matroska" + default: + mimeType = "video/mp4" + } + + print("📎 ScreenAssistant: Video encoded - \(base64String.count) bytes, MIME: \(mimeType)") + + return [ + "inline_data": [ + "mime_type": mimeType, + "data": base64String + ] + ] + } catch { + print("❌ ScreenAssistant: Failed to encode video \(fileName): \(error)") + return ["text": "Video file: \(fileName) (failed to encode: \(error.localizedDescription))"] + } + } + + private func createGeminiTextPart(for url: URL, fileName: String) -> [String: Any]? { + print("📝 ScreenAssistant: Processing text file: \(fileName)") + + do { + let content = try String(contentsOf: url) + print("📄 ScreenAssistant: Read text content (\(content.count) characters)") + return ["text": "File content of \(fileName):\n\(content)"] + } catch { + print("❌ ScreenAssistant: Failed to read text file \(fileName): \(error)") + return ["text": "File: \(fileName) (could not read content: \(error.localizedDescription))"] + } + } + + func clearChat() { + chatMessages.removeAll() + } + + private func addAssistantMessage(_ content: String) { + print("💬 ScreenAssistant: Adding assistant message: \(content.prefix(100))...") + let assistantMessage = ChatMessage(content: content, isFromUser: false) + chatMessages.append(assistantMessage) + } + + private func handleAPIError(statusCode: Int, provider: AIModelProvider) { + let userFriendlyMessage: String + + switch statusCode { + case 429: + userFriendlyMessage = "🚫 **Rate Limited**\n\n\(provider.displayName) is currently rate limiting requests. Please wait a moment and try again." + case 400: + userFriendlyMessage = "❌ **Invalid Request**\n\nThere was an issue with your request to \(provider.displayName). Please check your message and attached files." + case 401: + userFriendlyMessage = "🔑 **Authentication Error**\n\nYour \(provider.displayName) API key appears to be invalid. Please check your API key in model settings." + case 403: + userFriendlyMessage = "🚫 **Access Denied**\n\nYour \(provider.displayName) API key doesn't have permission for this request." + case 404: + userFriendlyMessage = "🔍 **Model Not Found**\n\nThe requested model is not available on \(provider.displayName)." + case 500, 502, 503: + userFriendlyMessage = "⚠️ **Server Error**\n\n\(provider.displayName) servers are experiencing issues. Please try again in a few minutes." + default: + userFriendlyMessage = "❌ **API Error (\(statusCode))**\n\n\(provider.displayName) returned an error. Please try again." + } + + addAssistantMessage(userFriendlyMessage) + } + + private func handleAPIError(error: [String: Any]) { + guard let code = error["code"] as? Int, + let message = error["message"] as? String else { + print("❌ ScreenAssistant: Unknown API Error") + addAssistantMessage("An unknown error occurred. Please try again.") + return + } + + print("❌ ScreenAssistant: API Error \(code) - \(message)") + + let userFriendlyMessage: String + + switch code { + case 429: + // Quota exceeded + if message.contains("quota") || message.contains("exceeded") { + userFriendlyMessage = "🚫 **API Quota Exceeded**\n\nYou've reached your API usage limit. This usually happens when:\n\n• Too many requests in a short time\n• Daily/monthly quota exceeded\n• Free tier limits reached\n\n**What you can do:**\n• Wait a few minutes and try again\n• Check your API billing\n• Consider upgrading your plan\n\n*The system will work again once the quota resets.*" + } else { + userFriendlyMessage = "⏰ **Rate Limited**\n\nToo many requests. Please wait a moment and try again." + } + + case 400: + userFriendlyMessage = "❌ **Invalid Request**\n\nThere was an issue with your request. Please check your message and attached files." + + case 401: + userFriendlyMessage = "🔑 **Authentication Error**\n\nYour API key appears to be invalid. Please check your API key in settings." + + case 403: + userFriendlyMessage = "🚫 **Access Denied**\n\nYour API key doesn't have permission for this request. Please check your API key settings." + + case 404: + userFriendlyMessage = "🔍 **Model Not Found**\n\nThe requested AI model is not available. Please try again later." + + case 500, 502, 503: + userFriendlyMessage = "⚠️ **Server Error**\n\nThe AI service is experiencing issues. Please try again in a few minutes." + + default: + userFriendlyMessage = "❌ **API Error (\(code))**\n\n\(message.components(separatedBy: ".").first ?? message)" + } + + addAssistantMessage(userFriendlyMessage) + } + + private func handleOpenAIError(error: [String: Any]) { + if let message = error["message"] as? String { + let userFriendlyMessage = "❌ **OpenAI Error**\n\n\(message)" + addAssistantMessage(userFriendlyMessage) + } else { + addAssistantMessage("❌ **OpenAI Error**\n\nAn unknown error occurred with OpenAI.") + } + } + + private func handleClaudeError(error: [String: Any]) { + if let message = error["message"] as? String { + let userFriendlyMessage = "❌ **Claude Error**\n\n\(message)" + addAssistantMessage(userFriendlyMessage) + } else { + addAssistantMessage("❌ **Claude Error**\n\nAn unknown error occurred with Claude.") + } + } +} + +// MARK: - AVAudioRecorderDelegate + +extension ScreenAssistantManager: AVAudioRecorderDelegate { + func audioRecorderDidFinishRecording(_ recorder: AVAudioRecorder, successfully flag: Bool) { + isRecording = false + recordingTimer?.invalidate() + recordingTimer = nil + + if flag { + let fileName = recorder.url.lastPathComponent + let displayName = "Recording \(DateFormatter.shortTime.string(from: Date()))" + let audioFile = ScreenAssistantFile(audioFileName: fileName, name: displayName) + attachedFiles.append(audioFile) + saveFilesToDefaults() + print("Recording saved: \(fileName)") + } else { + print("Recording failed") + } + } + + func audioRecorderEncodeErrorDidOccur(_ recorder: AVAudioRecorder, error: Error?) { + print("Recording encode error: \(error?.localizedDescription ?? "Unknown error")") + isRecording = false + recordingTimer?.invalidate() + recordingTimer = nil + } +} + +// MARK: - DateFormatter Extension + +extension DateFormatter { + static let shortTime: DateFormatter = { + let formatter = DateFormatter() + formatter.timeStyle = .short + return formatter + }() +} diff --git a/DynamicIsland/managers/ScreenAssistantPanelManager.swift b/DynamicIsland/managers/ScreenAssistantPanelManager.swift new file mode 100644 index 0000000..a48b2af --- /dev/null +++ b/DynamicIsland/managers/ScreenAssistantPanelManager.swift @@ -0,0 +1,57 @@ +// +// ScreenAssistantPanelManager.swift +// DynamicIsland +// +// Created by Hariharan Mudaliar + +import AppKit +import SwiftUI +import Defaults + +class ScreenAssistantPanelManager: ObservableObject { + static let shared = ScreenAssistantPanelManager() + + // Backward compatibility wrapper - delegates to the new ScreenAssistantManager + private var screenAssistantPanel: ScreenAssistantPanel? + + private init() {} + + func showScreenAssistantPanel() { + hideScreenAssistantPanel() // Close any existing panel + + // Use the new dual-panel system + ScreenAssistantManager.shared.showPanels() + + // Create a dummy panel for backward compatibility + let panel = ScreenAssistantPanel() + self.screenAssistantPanel = panel + + // The actual panels are managed by ScreenAssistantManager + // This is just for compatibility with existing code + + print("ScreenAssistant: New dual-panel system activated") + } + + func hideScreenAssistantPanel() { + // Close the new panel system + ScreenAssistantManager.shared.closePanels() + + // Clean up compatibility panel + screenAssistantPanel?.close() + screenAssistantPanel = nil + + print("ScreenAssistant: Panels hidden") + } + + func toggleScreenAssistantPanel() { + if ScreenAssistantManager.shared.arePanelsVisible() { + hideScreenAssistantPanel() + } else { + showScreenAssistantPanel() + } + } + + var isPanelVisible: Bool { + return ScreenAssistantManager.shared.arePanelsVisible() + } +} \ No newline at end of file diff --git a/DynamicIsland/models/Constants.swift b/DynamicIsland/models/Constants.swift index b52e256..b865bde 100644 --- a/DynamicIsland/models/Constants.swift +++ b/DynamicIsland/models/Constants.swift @@ -49,6 +49,25 @@ enum ClipboardDisplayMode: String, CaseIterable, Codable, Defaults.Serializable } } +enum ScreenAssistantDisplayMode: String, CaseIterable, Codable, Defaults.Serializable { + case popover = "popover" // Traditional popover attached to button + case panel = "panel" // Floating panel near notch + + var displayName: String { + switch self { + case .popover: return "Popover" + case .panel: return "Panel" + } + } + + var description: String { + switch self { + case .popover: return "Shows screen assistant as a dropdown attached to the AI button" + case .panel: return "Shows screen assistant in a floating panel near the notch" + } + } +} + enum ColorPickerDisplayMode: String, CaseIterable, Codable, Defaults.Serializable { case popover = "popover" // Traditional popover attached to button case panel = "panel" // Floating panel near notch @@ -105,12 +124,86 @@ enum ProgressBarStyle: String, CaseIterable, Identifiable, Defaults.Serializable var id: String { self.rawValue } } +// AI Model types for screen assistant +enum AIModelProvider: String, CaseIterable, Identifiable, Defaults.Serializable { + case gemini = "Gemini" + case openai = "OpenAI GPT" + case claude = "Claude" + case local = "Local Model" + + var id: String { self.rawValue } + + var displayName: String { + return self.rawValue + } + + var description: String { + switch self { + case .gemini: return "Google's Gemini AI with multimodal capabilities" + case .openai: return "OpenAI's GPT models with advanced reasoning" + case .claude: return "Anthropic's Claude with strong analytical skills" + case .local: return "Local AI model (Ollama or similar)" + } + } + + var supportedModels: [AIModel] { + switch self { + case .gemini: + return [ + // Gemini 2.5 Models (Latest) + AIModel(id: "gemini-2.5-pro", name: "Gemini 2.5 Pro", supportsThinking: true), + AIModel(id: "gemini-2.5-flash", name: "Gemini 2.5 Flash", supportsThinking: true), + AIModel(id: "gemini-2.5-flash-lite", name: "Gemini 2.5 Flash-Lite", supportsThinking: false), + AIModel(id: "gemini-2.5-flash-live", name: "Gemini 2.5 Flash Live", supportsThinking: false), + AIModel(id: "gemini-2.5-flash-native-audio", name: "Gemini 2.5 Flash Native Audio", supportsThinking: true), + + // Gemini 2.0 Models + AIModel(id: "gemini-2.0-flash", name: "Gemini 2.0 Flash", supportsThinking: false), + AIModel(id: "gemini-2.0-flash-lite", name: "Gemini 2.0 Flash-Lite", supportsThinking: false), + AIModel(id: "gemini-2.0-flash-live", name: "Gemini 2.0 Flash Live", supportsThinking: false), + + // Legacy 1.5 Models (for compatibility) + AIModel(id: "gemini-1.5-pro", name: "Gemini 1.5 Pro", supportsThinking: false), + AIModel(id: "gemini-1.5-flash", name: "Gemini 1.5 Flash", supportsThinking: false) + ] + case .openai: + return [ + AIModel(id: "gpt-4o", name: "GPT-4o", supportsThinking: false), + AIModel(id: "gpt-4o-mini", name: "GPT-4o Mini", supportsThinking: false), + AIModel(id: "o1-preview", name: "o1 Preview", supportsThinking: true), + AIModel(id: "o1-mini", name: "o1 Mini", supportsThinking: true) + ] + case .claude: + return [ + AIModel(id: "claude-3-5-sonnet", name: "Claude 3.5 Sonnet", supportsThinking: false), + AIModel(id: "claude-3-haiku", name: "Claude 3 Haiku", supportsThinking: false) + ] + case .local: + return [ + AIModel(id: "llama3.2", name: "Llama 3.2", supportsThinking: false), + AIModel(id: "qwen2.5", name: "Qwen 2.5", supportsThinking: false) + ] + } + } +} + +struct AIModel: Codable, Identifiable, Defaults.Serializable { + let id: String + let name: String + let supportsThinking: Bool + + var displayName: String { + return name + (supportsThinking ? " (Thinking)" : "") + } +} + extension Defaults.Keys { // MARK: General static let menubarIcon = Key("menubarIcon", default: true) static let showOnAllDisplays = Key("showOnAllDisplays", default: false) static let automaticallySwitchDisplay = Key("automaticallySwitchDisplay", default: true) static let releaseName = Key("releaseName", default: "alpha 0.0.1") + static let hidePanelsFromScreenCapture = Key("hideFromScreenCapture", default: false) // MARK: Behavior static let minimumHoverDuration = Key("minimumHoverDuration", default: 0.3) @@ -228,6 +321,17 @@ extension Defaults.Keys { static let showClipboardIcon = Key("showClipboardIcon", default: true) static let clipboardDisplayMode = Key("clipboardDisplayMode", default: .panel) + // MARK: Screen Assistant Feature + static let enableScreenAssistant = Key("enableScreenAssistant", default: true) + static let screenAssistantDisplayMode = Key("screenAssistantDisplayMode", default: .panel) + static let geminiApiKey = Key("geminiApiKey", default: "") + static let openaiApiKey = Key("openaiApiKey", default: "") + static let claudeApiKey = Key("claudeApiKey", default: "") + static let selectedAIProvider = Key("selectedAIProvider", default: .gemini) + static let selectedAIModel = Key("selectedAIModel", default: nil) + static let enableThinkingMode = Key("enableThinkingMode", default: false) + static let localModelEndpoint = Key("localModelEndpoint", default: "http://localhost:11434") + // MARK: Keyboard Shortcuts static let enableShortcuts = Key("enableShortcuts", default: true) From fb4f7a92c432494bf0f9bb7a82145eb7c91a98de Mon Sep 17 00:00:00 2001 From: Hariharan Mudaliar Date: Mon, 22 Sep 2025 00:59:03 +0530 Subject: [PATCH 2/3] Working and Hidden - with gemini api --- DynamicIsland/Localizable.xcstrings | 9 + .../ScreenAssistant/ChatPanels.swift | 188 ++++++- .../ScreenshotSnippingTool.swift | 61 ++- .../ScreenshotSnippingTool_old.swift | 500 ------------------ 4 files changed, 236 insertions(+), 522 deletions(-) delete mode 100644 DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift diff --git a/DynamicIsland/Localizable.xcstrings b/DynamicIsland/Localizable.xcstrings index 9bde979..91c901f 100644 --- a/DynamicIsland/Localizable.xcstrings +++ b/DynamicIsland/Localizable.xcstrings @@ -1186,6 +1186,12 @@ }, "Screen Assistant:" : { + }, + "Screenshot options" : { + + }, + "Screenshot Type" : { + }, "Search clipboard..." : { @@ -1419,6 +1425,9 @@ }, "System Performance Monitor" : { + }, + "Take area screenshot" : { + }, "TheDynamicIsland" : { diff --git a/DynamicIsland/components/ScreenAssistant/ChatPanels.swift b/DynamicIsland/components/ScreenAssistant/ChatPanels.swift index 588b555..3932db5 100644 --- a/DynamicIsland/components/ScreenAssistant/ChatPanels.swift +++ b/DynamicIsland/components/ScreenAssistant/ChatPanels.swift @@ -606,18 +606,38 @@ struct StreamingChatMessageBubble: View { struct ScreenshotButton: View { @ObservedObject var screenAssistantManager = ScreenAssistantManager.shared @StateObject private var screenshotTool = ScreenshotSnippingTool.shared + @State private var showingScreenshotOptions = false var body: some View { - Button(action: startScreenshot) { - Image(systemName: getIconName()) - .foregroundColor(getIconColor()) - .font(.system(size: 20)) + HStack(spacing: 4) { + // Main screenshot button + Button(action: startQuickScreenshot) { + Image(systemName: getIconName()) + .foregroundColor(getIconColor()) + .font(.system(size: 20)) + } + .buttonStyle(PlainButtonStyle()) + .help("Take area screenshot") + .disabled(screenshotTool.isSnipping) + .scaleEffect(screenshotTool.isSnipping ? 1.1 : 1.0) + .animation(.easeInOut(duration: 0.2), value: screenshotTool.isSnipping) + + // Options dropdown button + Button(action: { showingScreenshotOptions.toggle() }) { + Image(systemName: "chevron.down") + .foregroundColor(.secondary) + .font(.system(size: 12)) + } + .buttonStyle(PlainButtonStyle()) + .help("Screenshot options") + .disabled(screenshotTool.isSnipping) + .popover(isPresented: $showingScreenshotOptions) { + ScreenshotOptionsPopover { type in + startScreenshot(type: type) + showingScreenshotOptions = false + } + } } - .buttonStyle(PlainButtonStyle()) - .help(getHelpText()) - .disabled(screenshotTool.isSnipping) - .scaleEffect(screenshotTool.isSnipping ? 1.1 : 1.0) - .animation(.easeInOut(duration: 0.2), value: screenshotTool.isSnipping) } private func getIconName() -> String { @@ -636,21 +656,22 @@ struct ScreenshotButton: View { } } - private func getHelpText() -> String { - return screenshotTool.isSnipping ? "Taking screenshot..." : "Take screenshot" + private func startQuickScreenshot() { + // Default to area screenshot for quick action + startScreenshot(type: .area) } - private func startScreenshot() { + private func startScreenshot(type: ScreenshotSnippingTool.ScreenshotType) { // Start snipping with direct callback (ScreenshotApp-based approach) - screenshotTool.startSnipping { [weak screenAssistantManager] screenshotURL in + screenshotTool.startSnipping(type: type) { [weak screenAssistantManager] screenshotURL in guard let manager = screenAssistantManager else { print("❌ ScreenshotTool: ScreenAssistantManager deallocated during callback") return } - print("📁 ScreenshotTool: Adding screenshot to chat: \(screenshotURL.lastPathComponent)") + print("📁 ScreenshotTool: Adding \(type.displayName.lowercased()) screenshot to chat: \(screenshotURL.lastPathComponent)") manager.addFiles([screenshotURL]) - print("📸 Screenshot captured and added to chat successfully") + print("📸 \(type.displayName) screenshot captured and added to chat successfully") } } } @@ -669,4 +690,141 @@ struct ChatPanelsVisualEffectView: NSViewRepresentable { } func updateNSView(_ nsView: NSVisualEffectView, context: Context) {} +} + +// MARK: - Screenshot Options Popover (Hidden from Screen Recording) +struct ScreenshotOptionsPopover: View { + let onOptionSelected: (ScreenshotSnippingTool.ScreenshotType) -> Void + + var body: some View { + VStack(spacing: 8) { + Text("Screenshot Type") + .font(.headline) + .padding(.top, 8) + + VStack(spacing: 4) { + ScreenshotOptionButton( + type: .area, + description: "Select an area", + onTap: onOptionSelected + ) + + ScreenshotOptionButton( + type: .window, + description: "Select a window", + onTap: onOptionSelected + ) + + ScreenshotOptionButton( + type: .full, + description: "Capture full screen", + onTap: onOptionSelected + ) + } + .padding(.bottom, 8) + } + .padding(.horizontal, 12) + .frame(width: 200) + .background( + ScreenshotPopoverBackground() + ) + } +} + +// MARK: - Screenshot Option Button +struct ScreenshotOptionButton: View { + let type: ScreenshotSnippingTool.ScreenshotType + let description: String + let onTap: (ScreenshotSnippingTool.ScreenshotType) -> Void + + var body: some View { + Button(action: { onTap(type) }) { + HStack(spacing: 12) { + Image(systemName: type.iconName) + .foregroundColor(.blue) + .font(.system(size: 16)) + .frame(width: 20, alignment: .center) + + VStack(alignment: .leading, spacing: 2) { + Text(type.displayName) + .font(.system(size: 14, weight: .medium)) + .foregroundColor(.primary) + + Text(description) + .font(.system(size: 12)) + .foregroundColor(.secondary) + } + + Spacer() + } + .padding(.horizontal, 8) + .padding(.vertical, 6) + .background( + RoundedRectangle(cornerRadius: 6) + .fill(Color.clear) + .contentShape(Rectangle()) + ) + .overlay( + RoundedRectangle(cornerRadius: 6) + .stroke(Color.clear, lineWidth: 1) + ) + } + .buttonStyle(PlainButtonStyle()) + .background( + RoundedRectangle(cornerRadius: 6) + .fill(Color(NSColor.controlBackgroundColor)) + .opacity(0.5) + ) + .onHover { isHovered in + // Add subtle hover effect if needed + } + } +} + +// MARK: - Screenshot Popover Background (Hidden from Screen Recording) +struct ScreenshotPopoverBackground: NSViewRepresentable { + func makeNSView(context: Context) -> NSView { + let view = NSView() + + // Apply screen capture protection like our other panels + DispatchQueue.main.async { + if let window = view.window { + self.updateScreenCaptureVisibility(window: window) + self.setupScreenCaptureObserver(window: window) + } + } + + return view + } + + func updateNSView(_ nsView: NSView, context: Context) { + // Update screen capture protection when view updates + if let window = nsView.window { + updateScreenCaptureVisibility(window: window) + } + } + + private func updateScreenCaptureVisibility(window: NSWindow) { + let shouldHide = Defaults[.hidePanelsFromScreenCapture] + + if shouldHide { + // Hide from screen capture and recording + window.sharingType = .none + print("🙈 ScreenshotPopover: Hidden from screen capture and recordings") + } else { + // Allow normal screen capture + window.sharingType = .readOnly + print("👁️ ScreenshotPopover: Visible in screen capture and recordings") + } + } + + private func setupScreenCaptureObserver(window: NSWindow) { + // Observe changes to hidePanelsFromScreenCapture setting + Defaults.observe(.hidePanelsFromScreenCapture) { [weak window] change in + DispatchQueue.main.async { + guard let window = window else { return } + self.updateScreenCaptureVisibility(window: window) + } + } + } } \ No newline at end of file diff --git a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift index 02c0a67..f62dc5f 100644 --- a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift +++ b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift @@ -15,6 +15,40 @@ class ScreenshotSnippingTool: NSObject, ObservableObject { @Published var isSnipping = false private var completion: ((URL) -> Void)? + // MARK: - Screenshot Types (Based on ScreenshotApp) + enum ScreenshotType { + case full + case window + case area + + var processArguments: [String] { + switch self { + case .full: + return ["-c"] // -c = clipboard + case .window: + return ["-cw"] // -c = clipboard, -w = window selection + case .area: + return ["-cs"] // -c = clipboard, -s = area selection + } + } + + var displayName: String { + switch self { + case .full: return "Full Screen" + case .window: return "Window" + case .area: return "Area" + } + } + + var iconName: String { + switch self { + case .full: return "rectangle.dashed" + case .window: return "macwindow" + case .area: return "viewfinder.rectangular" + } + } + } + enum ScreenshotError: Error { case captureFailed case noImageInPasteboard @@ -25,26 +59,39 @@ class ScreenshotSnippingTool: NSObject, ObservableObject { super.init() } - // MARK: - Simple API (Based on ScreenshotApp Implementation) - func startSnipping(completion: @escaping (URL) -> Void) { + // MARK: - Enhanced API (Based on ScreenshotApp Implementation) + func startSnipping(type: ScreenshotType = .area, completion: @escaping (URL) -> Void) { guard !isSnipping else { return } - print("🖼️ ScreenshotTool: Starting area screenshot using screencapture tool") + print("🖼️ ScreenshotTool: Starting \(type.displayName.lowercased()) screenshot using screencapture tool") self.completion = completion isSnipping = true // Use the same approach as ScreenshotApp - direct screencapture command - takeAreaScreenshot() + takeScreenshot(type: type) + } + + // MARK: - Convenience Methods for Different Types + func startAreaScreenshot(completion: @escaping (URL) -> Void) { + startSnipping(type: .area, completion: completion) + } + + func startFullScreenshot(completion: @escaping (URL) -> Void) { + startSnipping(type: .full, completion: completion) + } + + func startWindowScreenshot(completion: @escaping (URL) -> Void) { + startSnipping(type: .window, completion: completion) } // MARK: - ScreenshotApp-Style Implementation - private func takeAreaScreenshot() { + private func takeScreenshot(type: ScreenshotType) { let task = Process() task.executableURL = URL(fileURLWithPath: "/usr/sbin/screencapture") - task.arguments = ["-cs"] // -c = clipboard, -s = area selection + task.arguments = type.processArguments do { - print("📸 ScreenshotTool: Running screencapture -cs command") + print("📸 ScreenshotTool: Running screencapture \(type.processArguments.joined(separator: " ")) command") try task.run() task.waitUntilExit() diff --git a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift deleted file mode 100644 index b06c1a8..0000000 --- a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool_old.swift +++ /dev/null @@ -1,500 +0,0 @@ -// This file has been replaced - keeping for reference only - -// MARK: - Screenshot Snipping Tool -class ScreenshotSnippingTool: NSObject, ObservableObject { - static let shared = ScreenshotSnippingTool() - - @Published var isSnipping = false - @Published var hasPermissions = false - @Published var permissionStatus: PermissionStatus = .unknown - private var snippingWindow: SnippingWindow? - - enum PermissionStatus { - case unknown - case granted - case denied - case restricted - } - - override init() { - super.init() - checkPermissionsAsync() - } - - private func checkPermissionsAsync() { - Task { - await checkPermissions() - } - } - - @MainActor - func checkPermissions() async { - print("🔍 ScreenshotTool: Checking permissions for bundle: \(Bundle.main.bundleIdentifier ?? "unknown")") - - do { - // First try a simple content check to see if we can access ScreenCaptureKit - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: false) - - // If we get here without error, permissions are granted - self.hasPermissions = true - self.permissionStatus = .granted - print("✅ ScreenshotTool: Permissions granted for \(content.displays.count) displays") - - } catch let error as NSError { - print("❌ ScreenshotTool: Permission error - Code: \(error.code), Domain: \(error.domain)") - - // Handle specific error codes - switch error.code { - case -3801: // User declined TCC permission - self.hasPermissions = false - self.permissionStatus = .denied - print("❌ ScreenshotTool: Permission denied by user - TCC database entry missing or denied") - - case -3802: // App is not authorized for screen capture - self.hasPermissions = false - self.permissionStatus = .restricted - print("❌ ScreenshotTool: App not authorized for screen capture - System restriction") - - default: - self.hasPermissions = false - self.permissionStatus = .unknown - print("❌ ScreenshotTool: Unknown permission error - \(error.localizedDescription)") - } - - // Provide user guidance - DispatchQueue.main.async { - self.showPermissionGuidance() - } - } - } - - private func showPermissionGuidance() { - let alert = NSAlert() - alert.messageText = "Screen Recording Permission Required" - alert.informativeText = """ - Dynamic Island needs screen recording permission to capture screenshots. - - To enable this: - 1. Open System Settings (or System Preferences) - 2. Go to Privacy & Security → Screen Recording - 3. Enable 'Dynamic Island' - 4. Restart the app if needed - """ - alert.alertStyle = .warning - alert.addButton(withTitle: "Open System Settings") - alert.addButton(withTitle: "Cancel") - - let response = alert.runModal() - if response == .alertFirstButtonReturn { - // Open System Settings to Screen Recording - if let url = URL(string: "x-apple.systempreferences:com.apple.preference.security?Privacy_ScreenCapture") { - NSWorkspace.shared.open(url) - } - } - } - - func requestPermissions() async { - // Only request if we don't already have them - guard !hasPermissions else { return } - await checkPermissions() - } - - func startSnipping() { - guard !isSnipping else { return } - - // Check permissions first - if !hasPermissions { - print("❌ ScreenshotTool: No permissions - checking status") - Task { - await requestPermissions() - } - return - } - - print("🖼️ ScreenshotTool: Starting screen snipping") - isSnipping = true - - // Hide all app windows temporarily - hideAppWindows() - - // Create snipping overlay - DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { - self.createSnippingOverlay() - } - } - - private func hideAppWindows() { - for window in NSApp.windows { - window.orderOut(nil) - } - } - - private func restoreAppWindows() { - for window in NSApp.windows { - if window != snippingWindow { - window.orderFront(nil) - } - } - } - - private func createSnippingOverlay() { - guard let screen = NSScreen.main else { - print("❌ ScreenshotTool: No main screen available") - finishSnipping() - return - } - - print("🎯 ScreenshotTool: Creating snipping overlay") - - snippingWindow = SnippingWindow(screen: screen) { [weak self] rect in - guard let self = self else { - print("❌ ScreenshotTool: Self deallocated during area selection") - return - } - self.captureScreenArea(rect) - } - - guard let window = snippingWindow else { - print("❌ ScreenshotTool: Failed to create snipping window") - finishSnipping() - return - } - - // Just make it key and order front, don't try to make it main - window.makeKeyAndOrderFront(nil) - window.orderFrontRegardless() - - // Activate the app to ensure proper focus - NSApp.activate(ignoringOtherApps: true) - - print("✅ ScreenshotTool: Snipping overlay created and activated") - } - - private func captureScreenArea(_ rect: CGRect) { - print("🖼️ ScreenshotTool: Capturing area: \(rect)") - - Task { - do { - let screenshot = try await captureScreenshot(rect: rect) - - DispatchQueue.main.async { [weak self] in - guard let self = self else { - print("❌ ScreenshotTool: Self deallocated during screenshot callback") - return - } - - print("✅ ScreenshotTool: Screenshot saved successfully") - - // Capture screenshot URL before cleanup - let screenshotURL = screenshot - - // Finish snipping first to clean up window state - self.finishSnipping() - - // Post notification after cleanup with captured URL - DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { - print("📸 ScreenshotTool: Posting screenshot notification") - NotificationCenter.default.post( - name: Notification.Name("screenshotCaptured"), - object: screenshotURL - ) - print("✅ ScreenshotTool: Notification posted successfully") - } - } - } catch { - print("❌ ScreenshotTool: Failed to capture screenshot - \(error)") - DispatchQueue.main.async { [weak self] in - guard let self = self else { return } - - // Show error to user - self.showScreenshotError(error) - self.finishSnipping() - } - } - } - } - - private func showScreenshotError(_ error: Error) { - let alert = NSAlert() - alert.messageText = "Screenshot Failed" - alert.informativeText = "Failed to capture screenshot: \(error.localizedDescription)" - alert.alertStyle = .warning - alert.addButton(withTitle: "OK") - alert.runModal() - } - - private func captureScreenshot(rect: CGRect) async throws -> URL { - // Use ScreenCaptureKit for modern screenshot capture - let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: true) - - guard let display = content.displays.first else { - throw ScreenshotError.noDisplayFound - } - - let config = SCStreamConfiguration() - config.width = Int(rect.width) - config.height = Int(rect.height) - config.sourceRect = rect - config.pixelFormat = kCVPixelFormatType_32BGRA - config.showsCursor = false - - let filter = SCContentFilter(display: display, excludingWindows: []) - let image = try await SCScreenshotManager.captureImage(contentFilter: filter, configuration: config) - - // Ensure screenshot directory exists - let screenshotDir = ScreenAssistantManager.screenshotDataDirectory - if !FileManager.default.fileExists(atPath: screenshotDir.path) { - try FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) - } - - // Save to temporary file - let filename = "screenshot_\(Date().timeIntervalSince1970).png" - let screenshotURL = screenshotDir.appendingPathComponent(filename) - - // Create NSImage from CGImage and convert to PNG - let nsImage = NSImage(cgImage: image, size: NSSize(width: image.width, height: image.height)) - - guard let imageData = nsImage.tiffRepresentation, - let bitmapRep = NSBitmapImageRep(data: imageData), - let pngData = bitmapRep.representation(using: NSBitmapImageRep.FileType.png, properties: [:]) else { - throw ScreenshotError.imageConversionFailed - } - - do { - try pngData.write(to: screenshotURL) - print("✅ ScreenshotTool: Saved screenshot to: \(screenshotURL.path)") - return screenshotURL - } catch { - print("❌ ScreenshotTool: Failed to write screenshot file - \(error)") - throw ScreenshotError.fileWriteError - } - } - - private func finishSnipping() { - guard isSnipping else { - print("⚠️ ScreenshotTool: finishSnipping called but not currently snipping") - return - } - - print("🔄 ScreenshotTool: Finishing snipping process") - - isSnipping = false - - // Close and cleanup snipping window safely - if let window = snippingWindow { - window.orderOut(nil) - window.close() - snippingWindow = nil - print("🪟 ScreenshotTool: Snipping window closed and cleared") - } - - // Restore app windows with delay to prevent conflicts - DispatchQueue.main.asyncAfter(deadline: .now() + 0.1) { - self.restoreAppWindows() - print("🏠 ScreenshotTool: App windows restored") - } - - print("✅ ScreenshotTool: Snipping process completed") - } - - func cancelSnipping() { - finishSnipping() - } -} - -// MARK: - Snipping Window -class SnippingWindow: NSWindow { - private let onAreaSelected: (CGRect) -> Void - private var selectionOverlay: SelectionOverlayView? - - init(screen: NSScreen, onAreaSelected: @escaping (CGRect) -> Void) { - self.onAreaSelected = onAreaSelected - - super.init( - contentRect: screen.frame, - styleMask: [.borderless, .fullSizeContentView], - backing: .buffered, - defer: false - ) - - setupWindow(screen: screen) - setupContent() - } - - override var canBecomeKey: Bool { - return true - } - - override var canBecomeMain: Bool { - return true - } - - private func setupWindow(screen: NSScreen) { - backgroundColor = NSColor.black.withAlphaComponent(0.3) - isOpaque = false - level = .screenSaver - ignoresMouseEvents = false - acceptsMouseMovedEvents = true - isMovableByWindowBackground = false - titleVisibility = .hidden - titlebarAppearsTransparent = true - - collectionBehavior = [.canJoinAllSpaces, .fullScreenAuxiliary] - setFrame(screen.frame, display: true) - } - - private func setupContent() { - selectionOverlay = SelectionOverlayView { [weak self] rect in - self?.onAreaSelected(rect) - } - - if let overlay = selectionOverlay { - let hostingView = NSHostingView(rootView: overlay) - contentView = hostingView - } - } - - override func keyDown(with event: NSEvent) { - if event.keyCode == 53 { // ESC key - close() - ScreenshotSnippingTool.shared.cancelSnipping() - } else { - super.keyDown(with: event) - } - } -} - -// MARK: - Selection Overlay View -struct SelectionOverlayView: NSViewRepresentable { - let onAreaSelected: (CGRect) -> Void - - func makeNSView(context: Context) -> SelectionView { - let view = SelectionView() - view.onAreaSelected = onAreaSelected - return view - } - - func updateNSView(_ nsView: SelectionView, context: Context) {} -} - -// MARK: - Selection View -class SelectionView: NSView { - var onAreaSelected: ((CGRect) -> Void)? - private var startPoint: CGPoint? - private var currentRect: CGRect = .zero - private var isDragging = false - - override func awakeFromNib() { - super.awakeFromNib() - wantsLayer = true - layer?.backgroundColor = NSColor.clear.cgColor - } - - override func mouseDown(with event: NSEvent) { - startPoint = convert(event.locationInWindow, from: nil) - isDragging = true - currentRect = .zero - needsDisplay = true - } - - override func mouseDragged(with event: NSEvent) { - guard let start = startPoint, isDragging else { return } - - let currentPoint = convert(event.locationInWindow, from: nil) - - let x = min(start.x, currentPoint.x) - let y = min(start.y, currentPoint.y) - let width = abs(currentPoint.x - start.x) - let height = abs(currentPoint.y - start.y) - - currentRect = CGRect(x: x, y: y, width: width, height: height) - needsDisplay = true - } - - override func mouseUp(with event: NSEvent) { - guard isDragging, !currentRect.isEmpty else { - isDragging = false - return - } - - isDragging = false - - // Convert to screen coordinates - if let window = self.window { - let screenRect = window.convertToScreen(currentRect) - // Flip Y coordinate for screen capture - let flippedRect = CGRect( - x: screenRect.origin.x, - y: NSScreen.main!.frame.height - screenRect.origin.y - screenRect.height, - width: screenRect.width, - height: screenRect.height - ) - onAreaSelected?(flippedRect) - } - } - - override func draw(_ dirtyRect: NSRect) { - super.draw(dirtyRect) - - guard isDragging, !currentRect.isEmpty else { return } - - // Draw selection rectangle - NSColor.systemBlue.withAlphaComponent(0.3).setFill() - currentRect.fill() - - NSColor.systemBlue.setStroke() - let path = NSBezierPath(rect: currentRect) - path.lineWidth = 2.0 - path.stroke() - - // Draw selection info - let infoText = String(format: "%.0f × %.0f", currentRect.width, currentRect.height) - let attributes: [NSAttributedString.Key: Any] = [ - .font: NSFont.systemFont(ofSize: 12), - .foregroundColor: NSColor.white, - .backgroundColor: NSColor.black.withAlphaComponent(0.8) - ] - - let attributedString = NSAttributedString(string: infoText, attributes: attributes) - let textRect = CGRect( - x: currentRect.maxX - 80, - y: currentRect.minY - 20, - width: 80, - height: 20 - ) - - attributedString.draw(in: textRect) - } -} - -// MARK: - Screenshot Error Types -enum ScreenshotError: Error, LocalizedError { - case noDisplayFound - case imageConversionFailed - case fileWriteError - - var errorDescription: String? { - switch self { - case .noDisplayFound: - return "No display found for screenshot capture" - case .imageConversionFailed: - return "Failed to convert screenshot image" - case .fileWriteError: - return "Failed to save screenshot file" - } - } -} - -// MARK: - ScreenAssistantManager Extension for Screenshot Directory -extension ScreenAssistantManager { - static var screenshotDataDirectory: URL { - let appSupport = FileManager.default.urls(for: .applicationSupportDirectory, in: .userDomainMask).first! - let appDir = appSupport.appendingPathComponent(bundleIdentifier) - let screenshotDir = appDir.appendingPathComponent("Screenshots") - - // Create directory if it doesn't exist - try? FileManager.default.createDirectory(at: screenshotDir, withIntermediateDirectories: true) - - return screenshotDir - } -} \ No newline at end of file From 648e25e45ca40bbe29f4e56d3c238443183ed39a Mon Sep 17 00:00:00 2001 From: Hariharan Mudaliar Date: Sat, 27 Sep 2025 16:32:18 +0530 Subject: [PATCH 3/3] BugFix: ScreenshotSnipping --- .../components/ScreenAssistant/ScreenshotSnippingTool.swift | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift index f62dc5f..c70facd 100644 --- a/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift +++ b/DynamicIsland/components/ScreenAssistant/ScreenshotSnippingTool.swift @@ -186,4 +186,5 @@ class ScreenshotSnippingTool: NSObject, ObservableObject { print("❌ ScreenshotTool: Snipping cancelled") finishSnipping() } -} \ No newline at end of file +} +