OpenAgents

Commit

feat(coder): support dropped image file inputs with [Image #N] references

db20404264eb · AtlantisPleb · · parent 989df27770bf 

feat(coder): support dropped image file inputs with [Image #N] references

Deploy story

What this commit did to the running system — joined from the forge receipt chain, the part a commit page elsewhere cannot show.

Not deployed through the forge lane

No push, promotion, build, or deploy receipt references this commit (receipts are scanned over a bounded recent window). Changes shipped by full node replacement carry their proof in the release gate receipt instead.

Changed files

  • added packages/openagents-cli/src/coder-image.ts
  • modified packages/openagents-cli/src/coder-paste.ts
  • modified packages/openagents-cli/src/coder-ui.ts
  • added packages/openagents-cli/test/coder-image.test.ts

Diff

4 files changed, +418 -17

packages/openagents-cli/src/coder-image.ts added +199

@@ -0,0 +1,199 @@

1 
import * as fs from "node:fs";
2 
import * as path from "node:path";
3 
import * as crypto from "node:crypto";
4
5 
export const IMAGE_EXTENSION_REGEX = /\.(png|jpe?g|gif|webp)$/i;
6
7 
export interface PastedImageContent {
8 
  readonly id: number;
9 
  readonly path: string;
10 
  readonly filename: string;
11 
  readonly mime: string;
12 
  readonly sizeBytes: number;
13 
  readonly base64?: string;
14 
}
15
16 
/**
17 
 * Format image reference token: `[Image #1]`.
18 
 */
19 
export function formatImageRef(id: number): string {
20 
  return `[Image #${id}]`;
21 
}
22
23 
export const IMAGE_REF_REGEX = /\[Image #(\d+)\]/g;
24
25 
/**
26 
 * Remove outer single or double quotes from a path.
27 
 */
28 
export function removeOuterQuotes(text: string): string {
29 
  const trimmed = text.trim();
30 
  if (
31 
    (trimmed.startsWith('"') && trimmed.endsWith('"')) ||
32 
    (trimmed.startsWith("'") && trimmed.endsWith("'"))
33 
  ) {
34 
    return trimmed.slice(1, -1);
35 
  }
36 
  return trimmed;
37 
}
38
39 
/**
40 
 * Remove shell escape backslashes from a path.
41 
 */
42 
export function stripBackslashEscapes(filePath: string): string {
43 
  if (process.platform === "win32") {
44 
    return filePath;
45 
  }
46 
  const salt = crypto.randomBytes(8).toString("hex");
47 
  const placeholder = `__DBL_BS_${salt}__`;
48 
  const withPlaceholder = filePath.replace(/\\\\/g, placeholder);
49 
  const withoutEscapes = withPlaceholder.replace(/\\(.)/g, "$1");
50 
  return withoutEscapes.replace(new RegExp(placeholder, "g"), "\\");
51 
}
52
53 
/**
54 
 * Normalize and test if text represents an image file path.
55 
 */
56 
export function asImageFilePath(text: string): string | null {
57 
  const cleaned = removeOuterQuotes(text);
58 
  const unescaped = stripBackslashEscapes(cleaned);
59 
  if (IMAGE_EXTENSION_REGEX.test(unescaped)) {
60 
    return unescaped;
61 
  }
62 
  return null;
63 
}
64
65 
/**
66 
 * Split dropped text into path segments, handling quotes, spaces, and newlines.
67 
 */
68 
export function splitDroppedPaths(text: string): string[] {
69 
  const trimmed = text.trim();
70 
  if (trimmed.length === 0) return [];
71
72 
  const tokens: string[] = [];
73 
  let current = "";
74 
  let inDoubleQuote = false;
75 
  let inSingleQuote = false;
76 
  let escaped = false;
77
78 
  for (let i = 0; i < trimmed.length; i++) {
79 
    const char = trimmed[i];
80
81 
    if (escaped) {
82 
      current += char;
83 
      escaped = false;
84 
      continue;
85 
    }
86
87 
    if (char === "\\") {
88 
      escaped = true;
89 
      current += char;
90 
      continue;
91 
    }
92
93 
    if (char === '"' && !inSingleQuote) {
94 
      inDoubleQuote = !inDoubleQuote;
95 
      current += char;
96 
      continue;
97 
    }
98
99 
    if (char === "'" && !inDoubleQuote) {
100 
      inSingleQuote = !inSingleQuote;
101 
      current += char;
102 
      continue;
103 
    }
104
105 
    if ((char === " " || char === "\n" || char === "\r" || char === "\t") && !inDoubleQuote && !inSingleQuote) {
106 
      if (current.trim().length > 0) {
107 
        tokens.push(current.trim());
108 
        current = "";
109 
      }
110 
      continue;
111 
    }
112
113 
    current += char;
114 
  }
115
116 
  if (current.trim().length > 0) {
117 
    tokens.push(current.trim());
118 
  }
119
120 
  return tokens;
121 
}
122
123 
/**
124 
 * Check whether a string consists of one or more dropped image file paths.
125 
 * Returns array of parsed and existing file paths, or empty array if not images.
126 
 */
127 
export function parseDroppedImagePaths(text: string): string[] {
128 
  const rawParts = splitDroppedPaths(text);
129 
  if (rawParts.length === 0) return [];
130
131 
  const parsedPaths: string[] = [];
132 
  for (const part of rawParts) {
133 
    const candidate = asImageFilePath(part);
134 
    if (candidate !== null && fs.existsSync(candidate)) {
135 
      try {
136 
        const stat = fs.statSync(candidate);
137 
        if (stat.isFile()) {
138 
          parsedPaths.push(candidate);
139 
        }
140 
      } catch {
141 
        // ignore invalid files
142 
      }
143 
    }
144 
  }
145 
  return parsedPaths;
146 
}
147
148 
/**
149 
 * Detect MIME type from extension.
150 
 */
151 
export function mimeTypeForImage(filePath: string): string {
152 
  const ext = path.extname(filePath).toLowerCase();
153 
  switch (ext) {
154 
    case ".jpg":
155 
    case ".jpeg":
156 
      return "image/jpeg";
157 
    case ".png":
158 
      return "image/png";
159 
    case ".gif":
160 
      return "image/gif";
161 
    case ".webp":
162 
      return "image/webp";
163 
    default:
164 
      return "application/octet-stream";
165 
  }
166 
}
167
168 
/**
169 
 * Replace `[Image #N]` references with Markdown image tags or text descriptions.
170 
 */
171 
export function expandImageRefsForModel(
172 
  input: string,
173 
  images: ReadonlyMap<number, PastedImageContent> | Record<number, PastedImageContent>,
174 
): string {
175 
  const isMap = images instanceof Map;
176 
  const getEntry = (id: number): PastedImageContent | undefined => {
177 
    if (isMap) {
178 
      return (images as ReadonlyMap<number, PastedImageContent>).get(id);
179 
    }
180 
    return (images as Record<number, PastedImageContent>)[id];
181 
  };
182
183 
  const matches = [...input.matchAll(IMAGE_REF_REGEX)];
184 
  let expanded = input;
185 
  for (let i = matches.length - 1; i >= 0; i--) {
186 
    const match = matches[i];
187 
    if (match === undefined || match.index === undefined) continue;
188 
    const id = parseInt(match[1] ?? "0", 10);
189 
    const entry = getEntry(id);
190 
    if (entry !== undefined) {
191 
      const replacement = `![${entry.filename}](${entry.path})`;
192 
      expanded =
193 
        expanded.slice(0, match.index) +
194 
        replacement +
195 
        expanded.slice(match.index + match[0].length);
196 
    }
197 
  }
198 
  return expanded;
199 
}
packages/openagents-cli/src/coder-paste.ts modified +77 -4

@@ -2,6 +2,16 @@

2 2 
 * Types and utilities for compact paste placeholders in the coder composer.
3 3 
 */
4 4
5 
import {
6 
  expandImageRefsForModel,
7 
  formatImageRef,
8 
  mimeTypeForImage,
9 
  parseDroppedImagePaths,
10 
  type PastedImageContent,
11 
} from "./coder-image.js";
12 
import * as path from "node:path";
13 
import * as fs from "node:fs";
14
5 15 
export const PASTE_TEXT_THRESHOLD = 800;
6 16 
export const PASTE_MAX_LINES = 1;
7 17

@@ -65,6 +75,18 @@ export function expandPastedTextRefs(

65 75 
  return expanded;
66 76 
}
67 77
78 
/**
79 
 * Replace both text and image references for model submission.
80 
 */
81 
export function expandComposerPrompt(
82 
  input: string,
83 
  pastedText: ReadonlyMap<number, PastedTextContent> | Record<number, PastedTextContent>,
84 
  pastedImages: ReadonlyMap<number, PastedImageContent> | Record<number, PastedImageContent>,
85 
): string {
86 
  const withText = expandPastedTextRefs(input, pastedText);
87 
  return expandImageRefsForModel(withText, pastedImages);
88 
}
89
68 90 
/**
69 91 
 * Check if a pasted string qualifies for compact placeholder representation.
70 92 
 */

@@ -74,12 +96,63 @@ export function shouldCollapsePaste(text: string, maxLines = PASTE_MAX_LINES): b

74 96 
}
75 97
76 98 
/**
77 
 * Remove trailing reference token if the backspace lands on it.
99 
 * Process a pasted raw string chunk from bracketed paste or normal paste.
100 
 * If the paste consists of dropped image file paths, creates image entries and returns token placeholders.
101 
 */
102 
export function handleIncomingPasteChunk(
103 
  text: string,
104 
  state: {
105 
    nextTextId: number;
106 
    nextImageId: number;
107 
    pastedText: Map<number, PastedTextContent>;
108 
    pastedImages: Map<number, PastedImageContent>;
109 
  },
110 
): string {
111 
  const imagePaths = parseDroppedImagePaths(text);
112 
  if (imagePaths.length > 0) {
113 
    const tokens: string[] = [];
114 
    for (const imgPath of imagePaths) {
115 
      const id = state.nextImageId++;
116 
      let sizeBytes = 0;
117 
      try {
118 
        sizeBytes = fs.statSync(imgPath).size;
119 
      } catch {
120 
        // ignore
121 
      }
122 
      const entry: PastedImageContent = {
123 
        id,
124 
        path: imgPath,
125 
        filename: path.basename(imgPath),
126 
        mime: mimeTypeForImage(imgPath),
127 
        sizeBytes,
128 
      };
129 
      state.pastedImages.set(id, entry);
130 
      tokens.push(formatImageRef(id));
131 
    }
132 
    return tokens.join(" ");
133 
  }
134
135 
  if (shouldCollapsePaste(text)) {
136 
    const pasteId = state.nextTextId++;
137 
    state.pastedText.set(pasteId, { id: pasteId, content: text });
138 
    const numLines = getPastedTextRefNumLines(text);
139 
    return formatPastedTextRef(pasteId, numLines);
140 
  }
141
142 
  return text;
143 
}
144
145 
/**
146 
 * Remove trailing reference token (text or image) if the backspace lands on it.
78 147 
 */
79 148 
export function backspaceComposer(composer: string): string {
80 
  const trailingToken = /\[Pasted text #\d+(?: \+\d+ lines?)?\]$/.exec(composer);
81 
  if (trailingToken !== null) {
82 
    return composer.slice(0, -trailingToken[0].length);
149 
  const trailingPastedToken = /\[Pasted text #\d+(?: \+\d+ lines?)?\]$/.exec(composer);
150 
  if (trailingPastedToken !== null) {
151 
    return composer.slice(0, -trailingPastedToken[0].length);
152 
  }
153 
  const trailingImageToken = /\[Image #\d+\]$/.exec(composer);
154 
  if (trailingImageToken !== null) {
155 
    return composer.slice(0, -trailingImageToken[0].length);
83 156 
  }
84 157 
  return composer.slice(0, -1);
85 158 
}
packages/openagents-cli/src/coder-ui.ts modified +18 -13

@@ -45,12 +45,11 @@ import { coderTierLabel } from "./coder-tiers.js";

45 45 
import { RELOAD_EXIT_CODE, sourceCheckout } from "./coder-reload.js";
46 46 
import {
47 47 
  backspaceComposer,
48 
  expandPastedTextRefs,
49 
  formatPastedTextRef,
50 
  getPastedTextRefNumLines,
48 
  expandComposerPrompt,
49 
  handleIncomingPasteChunk,
51 50 
  type PastedTextContent,
52 
  shouldCollapsePaste,
53 51 
} from "./coder-paste.js";
52 
import type { PastedImageContent } from "./coder-image.js";
54 53 
import type { SkillSelection } from "./coder-skills.js";
55 54
56 55 
const ALT_SCREEN_ON = "\x1b[?1049h";

@@ -416,7 +415,9 @@ export function runCoderUi(session: CoderSession, options: CoderUiOptions): Prom

416 415 
  /** A paste whose end has not arrived yet. */
417 416 
  let pendingPaste = "";
418 417 
  const pastedContents = new Map<number, PastedTextContent>();
418 
  const pastedImages = new Map<number, PastedImageContent>();
419 419 
  let nextPasteId = 1;
420 
  let nextImageId = 1;
420 421 
  let escapeTimer: NodeJS.Timeout | undefined;
421 422
422 423 
  const write = (text: string) => {

@@ -1065,9 +1066,10 @@ export function runCoderUi(session: CoderSession, options: CoderUiOptions): Prom

1065 1066 
    };
1066 1067
1067 1068 
    const submit = (mode: "steer" | "queue" = "steer") => {
1068 
      const prompt = expandPastedTextRefs(composer, pastedContents);
1069 
      const prompt = expandComposerPrompt(composer, pastedContents, pastedImages);
1069 1070 
      composer = "";
1070 1071 
      pastedContents.clear();
1072 
      pastedImages.clear();
1071 1073 
      anchor = undefined;
1072 1074 
      // Only when a turn actually begins. Resetting on every submission made
1073 1075 
      // `/export` mid-turn put the elapsed clock back to zero, which reads as

@@ -1194,6 +1196,7 @@ export function runCoderUi(session: CoderSession, options: CoderUiOptions): Prom

1194 1196 
      else {
1195 1197 
        composer = "";
1196 1198 
        pastedContents.clear();
1199 
        pastedImages.clear();
1197 1200 
      }
1198 1201 
      render();
1199 1202 
    };

@@ -1241,14 +1244,15 @@ export function runCoderUi(session: CoderSession, options: CoderUiOptions): Prom

1241 1244 
            break;
1242 1245 
          }
1243 1246 
          const pastedText = text.slice(from, to);
1244 
          if (shouldCollapsePaste(pastedText)) {
1245 
            const pasteId = nextPasteId++;
1246 
            pastedContents.set(pasteId, { id: pasteId, content: pastedText });
1247 
            const numLines = getPastedTextRefNumLines(pastedText);
1248 
            composer += formatPastedTextRef(pasteId, numLines);
1249 
          } else {
1250 
            composer += pastedText;
1251 
          }
1247 
          const state = {
1248 
            nextTextId: nextPasteId,
1249 
            nextImageId,
1250 
            pastedText: pastedContents,
1251 
            pastedImages,
1252 
          };
1253 
          composer += handleIncomingPasteChunk(pastedText, state);
1254 
          nextPasteId = state.nextTextId;
1255 
          nextImageId = state.nextImageId;
1252 1256 
          index = to + PASTE_END.length;
1253 1257 
          dirty = true;
1254 1258 
          continue;

@@ -1537,6 +1541,7 @@ export function runCoderUi(session: CoderSession, options: CoderUiOptions): Prom

1537 1541 
        if (char === "\x15") {
1538 1542 
          composer = "";
1539 1543 
          pastedContents.clear();
1544 
          pastedImages.clear();
1540 1545 
          dirty = true;
1541 1546 
          index += 1;
1542 1547 
          continue;
packages/openagents-cli/test/coder-image.test.ts added +124

@@ -0,0 +1,124 @@

1 
import * as fs from "node:fs";
2 
import * as path from "node:path";
3 
import * as os from "node:os";
4 
import { describe, expect, it } from "vitest";
5 
import {
6 
  asImageFilePath,
7 
  expandImageRefsForModel,
8 
  formatImageRef,
9 
  mimeTypeForImage,
10 
  parseDroppedImagePaths,
11 
  removeOuterQuotes,
12 
  stripBackslashEscapes,
13 
} from "../src/coder-image.js";
14 
import { handleIncomingPasteChunk, expandComposerPrompt, backspaceComposer } from "../src/coder-paste.js";
15
16 
describe("coder-image utilities", () => {
17 
  it("removes outer quotes", () => {
18 
    expect(removeOuterQuotes('"image.png"')).toBe("image.png");
19 
    expect(removeOuterQuotes("'image.png'")).toBe("image.png");
20 
    expect(removeOuterQuotes("image.png")).toBe("image.png");
21 
  });
22
23 
  it("strips shell escape backslashes on non-windows platforms", () => {
24 
    if (process.platform !== "win32") {
25 
      expect(stripBackslashEscapes("/path/to/my\\ screenshot\\ (1).png")).toBe(
26 
        "/path/to/my screenshot (1).png",
27 
      );
28 
    }
29 
  });
30
31 
  it("recognizes valid image file paths", () => {
32 
    expect(asImageFilePath("/path/to/test.PNG")).toBe("/path/to/test.PNG");
33 
    expect(asImageFilePath("/path/to/test.jpeg")).toBe("/path/to/test.jpeg");
34 
    expect(asImageFilePath("/path/to/test.webp")).toBe("/path/to/test.webp");
35 
    expect(asImageFilePath("/path/to/test.txt")).toBe(null);
36 
  });
37
38 
  it("infers mime types", () => {
39 
    expect(mimeTypeForImage("test.jpg")).toBe("image/jpeg");
40 
    expect(mimeTypeForImage("test.jpeg")).toBe("image/jpeg");
41 
    expect(mimeTypeForImage("test.png")).toBe("image/png");
42 
    expect(mimeTypeForImage("test.webp")).toBe("image/webp");
43 
    expect(mimeTypeForImage("test.gif")).toBe("image/gif");
44 
  });
45
46 
  it("parses single and multiple dropped image paths that exist on disk", () => {
47 
    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openagents-img-test-"));
48 
    const img1 = path.join(tmpDir, "screen1.png");
49 
    const img2 = path.join(tmpDir, "screen2.jpg");
50 
    fs.writeFileSync(img1, "fake png content");
51 
    fs.writeFileSync(img2, "fake jpg content");
52
53 
    try {
54 
      const single = parseDroppedImagePaths(img1);
55 
      expect(single).toEqual([img1]);
56
57 
      const multiple = parseDroppedImagePaths(`${img1} ${img2}`);
58 
      expect(multiple).toEqual([img1, img2]);
59
60 
      const withQuotes = parseDroppedImagePaths(`"${img1}" "${img2}"`);
61 
      expect(withQuotes).toEqual([img1, img2]);
62 
    } finally {
63 
      fs.rmSync(tmpDir, { recursive: true, force: true });
64 
    }
65 
  });
66
67 
  it("handles incoming paste chunk for dropped images", () => {
68 
    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openagents-img-test-"));
69 
    const img = path.join(tmpDir, "diagram.png");
70 
    fs.writeFileSync(img, "fake content");
71
72 
    try {
73 
      const state = {
74 
        nextTextId: 1,
75 
        nextImageId: 1,
76 
        pastedText: new Map(),
77 
        pastedImages: new Map(),
78 
      };
79
80 
      const result = handleIncomingPasteChunk(img, state);
81 
      expect(result).toBe("[Image #1]");
82 
      expect(state.pastedImages.has(1)).toBe(true);
83 
      expect(state.pastedImages.get(1)?.filename).toBe("diagram.png");
84 
    } finally {
85 
      fs.rmSync(tmpDir, { recursive: true, force: true });
86 
    }
87 
  });
88
89 
  it("expands image references and prompt text for model submission", () => {
90 
    const tmpDir = fs.mkdtempSync(path.join(os.tmpdir(), "openagents-img-test-"));
91 
    const img = path.join(tmpDir, "chart.png");
92 
    fs.writeFileSync(img, "fake chart");
93
94 
    try {
95 
      const pastedText = new Map([[1, { id: 1, content: "pasted multi\nline text" }]]);
96 
      const pastedImages = new Map([
97 
        [
98 
          1,
99 
          {
100 
            id: 1,
101 
            path: img,
102 
            filename: "chart.png",
103 
            mime: "image/png",
104 
            sizeBytes: 10,
105 
          },
106 
        ],
107 
      ]);
108
109 
      const composer = "Please analyze [Image #1] and read [Pasted text #1 +1 line].";
110 
      const expanded = expandComposerPrompt(composer, pastedText, pastedImages);
111
112 
      expect(expanded).toBe(
113 
        `Please analyze ![chart.png](${img}) and read pasted multi\nline text.`,
114 
      );
115 
    } finally {
116 
      fs.rmSync(tmpDir, { recursive: true, force: true });
117 
    }
118 
  });
119
120 
  it("removes [Image #N] atomically on backspace", () => {
121 
    const composer = "Look at this [Image #1]";
122 
    expect(backspaceComposer(composer)).toBe("Look at this ");
123 
  });
124 
});

This page updates live while a promote is in flight · changelog