Skip to content

Commit 8260313

Browse files
jasonjmcgheejasonjmcghee
authored
Merge pull request #74 from jasonjmcghee/fix-external-monitor-bugs
fix external monitor bugs, mainly by calling CGDisplayCreateImage without passing the display frame
2 parents 6f1d981 + e0b3afc commit 8260313

File tree

6 files changed

+114
-12
lines changed

6 files changed

+114
-12
lines changed

‎README.md

+2-1
Original file line numberDiff line numberDiff line change
@@ -41,12 +41,13 @@ Also, that means there is no tracking / analytics of any kind, which means I don
4141
- [x] Search everything you've viewed with keyword search (and filter by application)
4242
- [x] Easily grab recent context for use with LLMs
4343
- [x] First [Intel build](https://github.com/jasonjmcghee/rem/releases/download/v0.1.11/rem-0.1.11-intel.dmg) (please help test!)
44+
- [x] It "works" with external / multiple monitors connected
4445
- [ ] Natural language search / agent interaction via updating local vector embedding
4546
- [I've also been exploring novel approaches to vector dbs](https://github.com/jasonjmcghee/portable-hnsw)
4647
- [ ] Novel search experiences like spatial / similar images
4748
- [ ] More search filters (by time, etc.)
4849
- [ ] Fine-grained purging / trimming / selecting recording
49-
- [ ] Multi-monitor support
50+
- [ ] Better / First-class multi-monitor support
5051

5152
## Getting Started
5253

‎rem.xcodeproj/project.pbxproj

+4
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88

99
/* Begin PBXBuildFile section */
1010
102CA4C82B3E240C00C3DA2E /* SQLite.framework in Frameworks */ = {isa = PBXBuildFile; fileRef = 96E66BC32B2F5745006E1E97 /* SQLite.framework */; };
11+
960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */ = {isa = PBXBuildFile; fileRef = 960AC0232B58D0590050C62A /* ImageResizer.swift */; };
1112
961C95DA2B2E19B30093F228 /* remApp.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95D92B2E19B30093F228 /* remApp.swift */; };
1213
961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */ = {isa = PBXBuildFile; fileRef = 961C95DB2B2E19B30093F228 /* ContentView.swift */; };
1314
961C95E12B2E19B40093F228 /* Preview Assets.xcassets in Resources */ = {isa = PBXBuildFile; fileRef = 961C95E02B2E19B40093F228 /* Preview Assets.xcassets */; };
@@ -193,6 +194,7 @@
193194
/* End PBXCopyFilesBuildPhase section */
194195

195196
/* Begin PBXFileReference section */
197+
960AC0232B58D0590050C62A /* ImageResizer.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ImageResizer.swift; sourceTree = "<group>"; };
196198
961C95D62B2E19B30093F228 /* rem.app */ = {isa = PBXFileReference; explicitFileType = wrapper.application; includeInIndex = 0; path = rem.app; sourceTree = BUILT_PRODUCTS_DIR; };
197199
961C95D92B2E19B30093F228 /* remApp.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = remApp.swift; sourceTree = "<group>"; };
198200
961C95DB2B2E19B30093F228 /* ContentView.swift */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.swift; path = ContentView.swift; sourceTree = "<group>"; };
@@ -289,6 +291,7 @@
289291
969F3F092B3B7F760085787B /* Info.plist */,
290292
961C96122B2EB7DB0093F228 /* TimelineView.swift */,
291293
BF5FEBFA2B44B26800744FC2 /* ImageHelper.swift */,
294+
960AC0232B58D0590050C62A /* ImageResizer.swift */,
292295
961C95D92B2E19B30093F228 /* remApp.swift */,
293296
961C95DB2B2E19B30093F228 /* ContentView.swift */,
294297
961C95DD2B2E19B40093F228 /* Assets.xcassets */,
@@ -656,6 +659,7 @@
656659
isa = PBXSourcesBuildPhase;
657660
buildActionMask = 2147483647;
658661
files = (
662+
960AC0242B58D0590050C62A /* ImageResizer.swift in Sources */,
659663
961C95DC2B2E19B30093F228 /* ContentView.swift in Sources */,
660664
961C96152B2EBEE50093F228 /* DB.swift in Sources */,
661665
96B0DA3A2B3A08280030E8AE /* TextMerger.swift in Sources */,

‎rem.xcodeproj/xcuserdata/jason.xcuserdatad/xcschemes/xcschememanagement.plist

+47-5
Original file line numberDiff line numberDiff line change
@@ -9,31 +9,73 @@
99
<key>isShown</key>
1010
<false/>
1111
<key>orderHint</key>
12-
<integer>6</integer>
12+
<integer>7</integer>
1313
</dict>
1414
<key>SQLite (Playground) 2.xcscheme</key>
1515
<dict>
1616
<key>isShown</key>
1717
<false/>
1818
<key>orderHint</key>
19-
<integer>7</integer>
19+
<integer>8</integer>
20+
</dict>
21+
<key>SQLite (Playground) 3.xcscheme</key>
22+
<dict>
23+
<key>isShown</key>
24+
<false/>
25+
<key>orderHint</key>
26+
<integer>9</integer>
27+
</dict>
28+
<key>SQLite (Playground) 4.xcscheme</key>
29+
<dict>
30+
<key>isShown</key>
31+
<false/>
32+
<key>orderHint</key>
33+
<integer>10</integer>
34+
</dict>
35+
<key>SQLite (Playground) 5.xcscheme</key>
36+
<dict>
37+
<key>isShown</key>
38+
<false/>
39+
<key>orderHint</key>
40+
<integer>11</integer>
41+
</dict>
42+
<key>SQLite (Playground) 6.xcscheme</key>
43+
<dict>
44+
<key>isShown</key>
45+
<false/>
46+
<key>orderHint</key>
47+
<integer>12</integer>
48+
</dict>
49+
<key>SQLite (Playground) 7.xcscheme</key>
50+
<dict>
51+
<key>isShown</key>
52+
<false/>
53+
<key>orderHint</key>
54+
<integer>13</integer>
55+
</dict>
56+
<key>SQLite (Playground) 8.xcscheme</key>
57+
<dict>
58+
<key>isShown</key>
59+
<false/>
60+
<key>orderHint</key>
61+
<integer>14</integer>
2062
</dict>
2163
<key>SQLite (Playground).xcscheme</key>
2264
<dict>
2365
<key>isShown</key>
2466
<false/>
2567
<key>orderHint</key>
26-
<integer>5</integer>
68+
<integer>6</integer>
2769
</dict>
2870
<key>ffmpegX.xcscheme_^#shared#^_</key>
2971
<dict>
3072
<key>orderHint</key>
31-
<integer>5</integer>
73+
<integer>4</integer>
3274
</dict>
3375
<key>rem.xcscheme_^#shared#^_</key>
3476
<dict>
3577
<key>orderHint</key>
36-
<integer>4</integer>
78+
<integer>5</integer>
3779
</dict>
3880
</dict>
3981
</dict>

‎rem/ImageResizer.swift

+42
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,42 @@
1+
//
2+
// ImageResizer.swift
3+
// rem
4+
//
5+
// Created by Jason McGhee on 1/17/24.
6+
//
7+
8+
import Foundation
9+
import Cocoa
10+
11+
class ImageResizer {
12+
private var context: CGContext
13+
private let targetWidth: CGFloat
14+
private let targetHeight: CGFloat
15+
16+
init(targetWidth: Int, targetHeight: Int) {
17+
self.targetWidth = CGFloat(targetWidth)
18+
self.targetHeight = CGFloat(targetHeight)
19+
20+
let colorSpace = CGColorSpaceCreateDeviceRGB()
21+
let bitmapInfo = CGImageAlphaInfo.premultipliedLast.rawValue
22+
context = CGContext(data: nil, width: targetWidth, height: targetHeight, bitsPerComponent: 8, bytesPerRow: 0, space: colorSpace, bitmapInfo: bitmapInfo)!
23+
}
24+
25+
func resizeAndPad(image: CGImage) -> CGImage? {
26+
let widthScaleRatio = targetWidth / CGFloat(image.width)
27+
let heightScaleRatio = targetHeight / CGFloat(image.height)
28+
let scaleFactor = min(widthScaleRatio, heightScaleRatio)
29+
30+
let scaledWidth = CGFloat(image.width) * scaleFactor
31+
let scaledHeight = CGFloat(image.height) * scaleFactor
32+
let imageRect = CGRect(x: (targetWidth - scaledWidth) / 2, y: (targetHeight - scaledHeight) / 2, width: scaledWidth, height: scaledHeight)
33+
34+
context.clear(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
35+
context.setFillColor(NSColor.black.cgColor)
36+
context.fill(CGRect(x: 0, y: 0, width: targetWidth, height: targetHeight))
37+
context.interpolationQuality = .high
38+
context.draw(image, in: imageRect)
39+
40+
return context.makeImage()
41+
}
42+
}

‎rem/TimelineView.swift

+1-1
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ class CustomHostingView: NSHostingView<AnyView> {
118118
private func configureImageView(with image: NSImage, in frame: NSRect) {
119119
imageView.image = image
120120

121-
imageView.imageScaling = .scaleAxesIndependently
121+
imageView.imageScaling = .scaleProportionallyUpOrDown
122122

123123
// Configuring frame to account for the offset and scaling
124124
imageView.frame = CGRect(x: 0, y: 0, width: frame.width, height: frame.height)

‎rem/remApp.swift

+18-5
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,8 @@ class AppDelegate: NSObject, NSApplicationDelegate {
8989

9090
private var lastImageData: Data? = nil
9191
private var lastActiveApplication: String? = nil
92+
93+
private var imageResizer = ImageResizer(targetWidth: Int(NSScreen.main!.frame.width), targetHeight: Int(NSScreen.main!.frame.height))
9294

9395
func applicationDidFinishLaunching(_ notification: Notification) {
9496
let _ = DatabaseManager.shared
@@ -349,6 +351,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
349351
var displayID: CGDirectDisplayID? = nil
350352
if let screenID = NSScreen.main?.deviceDescription[NSDeviceDescriptionKey("NSScreenNumber")] as? NSNumber {
351353
displayID = CGDirectDisplayID(screenID.uint32Value)
354+
logger.debug("Display ID: \(displayID ?? 999)")
352355
}
353356
guard displayID != nil else { return }
354357

@@ -358,10 +361,20 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
358361
logger.debug("Active Application: \(activeApplicationName ?? "<undefined>")")
359362

360363
// Do we want to record the timeline being searched?
361-
guard let image = CGDisplayCreateImage(display.displayID, rect: display.frame) else { return }
364+
guard let image = CGDisplayCreateImage(display.displayID) else {
365+
logger.error("Failed to create a screenshot for the display!")
366+
return
367+
}
368+
guard let resizedImage = imageResizer.resizeAndPad(image: image) else {
369+
logger.error("Failed to resize the image!")
370+
return
371+
}
362372

363-
let bitmapRep = NSBitmapImageRep(cgImage: image)
364-
guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else { return }
373+
let bitmapRep = NSBitmapImageRep(cgImage: resizedImage)
374+
guard let imageData = bitmapRep.representation(using: .png, properties: [:]) else {
375+
logger.error("Failed to create a PNG from the screenshot!")
376+
return
377+
}
365378

366379
// Might as well only check if the applications are the same, otherwise obviously different
367380
if activeApplicationName != lastActiveApplication || displayImageChangedFromLast(imageData: imageData) {
@@ -371,7 +384,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
371384
let frameId = DatabaseManager.shared.insertFrame(activeApplicationName: activeApplicationName)
372385

373386
if settingsManager.settings.onlyOCRFrontmostWindow {
374-
// User wants to perform OCR on only active window.
387+
// default: User wants to perform OCR on only active window.
375388

376389
// We need to determine the scale factor for cropping. CGImage is
377390
// measured in pixels, display sizes are measured in points.
@@ -384,7 +397,7 @@ func drawStatusBarIcon(rect: CGRect) -> Bool {
384397
self.performOCR(frameId: frameId, on: cropped)
385398
}
386399
} else {
387-
// default: User wants to perform OCR on full display.
400+
// User wants to perform OCR on full display.
388401
self.performOCR(frameId: frameId, on: image)
389402
}
390403

0 commit comments

Comments
 (0)