tenant.openagents/omega

1// Read the text a person actually sees in a captured window.
2//
3// Vision returns one observation per recognised line together with its
4// normalised bounding box, so a caller can ask not only whether a phrase is on
5// screen but where it sits relative to another phrase. That ordering is the
6// thing an accessibility-tree read cannot give when the application publishes
7// no tree at all.
8//
9// Output is one JSON object: {"lines":[{"text":..,"confidence":..,"x":..,
10// "y":..,"w":..,"h":..}]}. `y` is measured from the TOP of the image, so a
11// smaller `y` means higher on screen.
12
13import Foundation
14import Vision
15import CoreGraphics
16import ImageIO
17
18struct Line: Codable {
19    let text: String
20    let confidence: Double
21    let x: Double
22    let y: Double
23    let w: Double
24    let h: Double
25}
26
27guard CommandLine.arguments.count == 2 else {
28    FileHandle.standardError.write("usage: ocr <image>\n".data(using: .utf8)!)
29    exit(2)
30}
31
32let path = CommandLine.arguments[1]
33guard let source = CGImageSourceCreateWithURL(URL(fileURLWithPath: path) as CFURL, nil),
34      let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else {
35    FileHandle.standardError.write("cannot read image\n".data(using: .utf8)!)
36    exit(3)
37}
38
39let request = VNRecognizeTextRequest()
40request.recognitionLevel = .accurate
41request.usesLanguageCorrection = false
42
43let handler = VNImageRequestHandler(cgImage: image, options: [:])
44do {
45    try handler.perform([request])
46} catch {
47    FileHandle.standardError.write("vision failed: \(error)\n".data(using: .utf8)!)
48    exit(4)
49}
50
51var lines: [Line] = []
52for observation in (request.results ?? []) {
53    guard let candidate = observation.topCandidates(1).first else { continue }
54    let box = observation.boundingBox
55    // Vision measures y from the bottom. Flip it so the caller can reason in
56    // reading order without having to remember which way is up.
57    lines.append(
58        Line(
59            text: candidate.string,
60            confidence: Double(candidate.confidence),
61            x: Double(box.minX),
62            y: Double(1.0 - box.maxY),
63            w: Double(box.width),
64            h: Double(box.height)
65        )
66    )
67}
68lines.sort { $0.y < $1.y }
69
70let encoder = JSONEncoder()
71encoder.outputFormatting = [.prettyPrinted, .sortedKeys]
72let data = try encoder.encode(["lines": lines])
73FileHandle.standardOutput.write(data)
74FileHandle.standardOutput.write("\n".data(using: .utf8)!)
75