From c52245fc0aeef229e52de2410ca4c18b87a60da2 Mon Sep 17 00:00:00 2001 From: KRATOS <84986124+gnanam1990@users.noreply.github.com> Date: Sat, 4 Apr 2026 11:40:26 +0530 Subject: [PATCH] fix: restore image paste and image tool-result handling (#308) --- bun.lock | 59 +++++++++++++++++++++ package.json | 3 +- scripts/build.ts | 3 +- src/hooks/usePasteHandler.test.ts | 22 ++++++++ src/hooks/usePasteHandler.ts | 33 ++++++++++-- src/services/api/openaiShim.test.ts | 82 +++++++++++++++++++++++++++++ src/services/api/openaiShim.ts | 37 +++++++++++-- 7 files changed, 228 insertions(+), 11 deletions(-) create mode 100644 src/hooks/usePasteHandler.test.ts diff --git a/bun.lock b/bun.lock index ab1a66ee..92841d5d 100644 --- a/bun.lock +++ b/bun.lock @@ -61,6 +61,7 @@ "react-compiler-runtime": "1.0.0", "react-reconciler": "0.33.0", "semver": "7.7.4", + "sharp": "^0.34.5", "shell-quote": "1.8.3", "signal-exit": "4.1.0", "stack-utils": "2.0.6", @@ -177,6 +178,8 @@ "@commander-js/extra-typings": ["@commander-js/extra-typings@12.1.0", "", { "peerDependencies": { "commander": "~12.1.0" } }, "sha512-wf/lwQvWAA0goIghcb91dQYpkLBcyhOhQNqG/VgWhnKzgt+UOMvra7EX/2fv70arm5RW+PUHoQHHDa6/p77Eqg=="], + "@emnapi/runtime": ["@emnapi/runtime@1.9.2", "", { "dependencies": { "tslib": "^2.4.0" } }, "sha512-3U4+MIWHImeyu1wnmVygh5WlgfYDtyf0k8AbLhMFxOipihf6nrWC4syIm/SwEeec0mNSafiiNnMJwbza/Is6Lw=="], + "@growthbook/growthbook": ["@growthbook/growthbook@1.6.5", "", { "dependencies": { "dom-mutator": "^0.6.0" } }, "sha512-mUaMsgeUTpRIUOTn33EUXHRK6j7pxBjwqH4WpQyq+pukjd1AIzWlEa6w7i6bInJUcweGgP2beXZmaP6b6UPn7A=="], "@grpc/grpc-js": ["@grpc/grpc-js@1.14.3", "", { "dependencies": { "@grpc/proto-loader": "^0.8.0", "@js-sdsl/ordered-map": "^4.4.2" } }, "sha512-Iq8QQQ/7X3Sac15oB6p0FmUg/klxQvXLeileoqrTRGJYLV+/9tubbr9ipz0GKHjmXVsgFPo/+W+2cA8eNcR+XA=="], @@ -185,6 +188,56 @@ "@hono/node-server": ["@hono/node-server@1.19.12", "", { "peerDependencies": { "hono": "^4" } }, "sha512-txsUW4SQ1iilgE0l9/e9VQWmELXifEFvmdA1j6WFh/aFPj99hIntrSsq/if0UWyGVkmrRPKA1wCeP+UCr1B9Uw=="], + "@img/colour": ["@img/colour@1.1.0", "", {}, "sha512-Td76q7j57o/tLVdgS746cYARfSyxk8iEfRxewL9h4OMzYhbW4TAcppl0mT4eyqXddh6L/jwoM75mo7ixa/pCeQ=="], + + "@img/sharp-darwin-arm64": ["@img/sharp-darwin-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-arm64": "1.2.4" }, "os": "darwin", "cpu": "arm64" }, "sha512-imtQ3WMJXbMY4fxb/Ndp6HBTNVtWCUI0WdobyheGf5+ad6xX8VIDO8u2xE4qc/fr08CKG/7dDseFtn6M6g/r3w=="], + + "@img/sharp-darwin-x64": ["@img/sharp-darwin-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-darwin-x64": "1.2.4" }, "os": "darwin", "cpu": "x64" }, "sha512-YNEFAF/4KQ/PeW0N+r+aVVsoIY0/qxxikF2SWdp+NRkmMB7y9LBZAVqQ4yhGCm/H3H270OSykqmQMKLBhBJDEw=="], + + "@img/sharp-libvips-darwin-arm64": ["@img/sharp-libvips-darwin-arm64@1.2.4", "", { "os": "darwin", "cpu": "arm64" }, "sha512-zqjjo7RatFfFoP0MkQ51jfuFZBnVE2pRiaydKJ1G/rHZvnsrHAOcQALIi9sA5co5xenQdTugCvtb1cuf78Vf4g=="], + + "@img/sharp-libvips-darwin-x64": ["@img/sharp-libvips-darwin-x64@1.2.4", "", { "os": "darwin", "cpu": "x64" }, "sha512-1IOd5xfVhlGwX+zXv2N93k0yMONvUlANylbJw1eTah8K/Jtpi15KC+WSiaX/nBmbm2HxRM1gZ0nSdjSsrZbGKg=="], + + "@img/sharp-libvips-linux-arm": ["@img/sharp-libvips-linux-arm@1.2.4", "", { "os": "linux", "cpu": "arm" }, "sha512-bFI7xcKFELdiNCVov8e44Ia4u2byA+l3XtsAj+Q8tfCwO6BQ8iDojYdvoPMqsKDkuoOo+X6HZA0s0q11ANMQ8A=="], + + "@img/sharp-libvips-linux-arm64": ["@img/sharp-libvips-linux-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-excjX8DfsIcJ10x1Kzr4RcWe1edC9PquDRRPx3YVCvQv+U5p7Yin2s32ftzikXojb1PIFc/9Mt28/y+iRklkrw=="], + + "@img/sharp-libvips-linux-ppc64": ["@img/sharp-libvips-linux-ppc64@1.2.4", "", { "os": "linux", "cpu": "ppc64" }, "sha512-FMuvGijLDYG6lW+b/UvyilUWu5Ayu+3r2d1S8notiGCIyYU/76eig1UfMmkZ7vwgOrzKzlQbFSuQfgm7GYUPpA=="], + + "@img/sharp-libvips-linux-riscv64": ["@img/sharp-libvips-linux-riscv64@1.2.4", "", { "os": "linux", "cpu": "none" }, "sha512-oVDbcR4zUC0ce82teubSm+x6ETixtKZBh/qbREIOcI3cULzDyb18Sr/Wcyx7NRQeQzOiHTNbZFF1UwPS2scyGA=="], + + "@img/sharp-libvips-linux-s390x": ["@img/sharp-libvips-linux-s390x@1.2.4", "", { "os": "linux", "cpu": "s390x" }, "sha512-qmp9VrzgPgMoGZyPvrQHqk02uyjA0/QrTO26Tqk6l4ZV0MPWIW6LTkqOIov+J1yEu7MbFQaDpwdwJKhbJvuRxQ=="], + + "@img/sharp-libvips-linux-x64": ["@img/sharp-libvips-linux-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-tJxiiLsmHc9Ax1bz3oaOYBURTXGIRDODBqhveVHonrHJ9/+k89qbLl0bcJns+e4t4rvaNBxaEZsFtSfAdquPrw=="], + + "@img/sharp-libvips-linuxmusl-arm64": ["@img/sharp-libvips-linuxmusl-arm64@1.2.4", "", { "os": "linux", "cpu": "arm64" }, "sha512-FVQHuwx1IIuNow9QAbYUzJ+En8KcVm9Lk5+uGUQJHaZmMECZmOlix9HnH7n1TRkXMS0pGxIJokIVB9SuqZGGXw=="], + + "@img/sharp-libvips-linuxmusl-x64": ["@img/sharp-libvips-linuxmusl-x64@1.2.4", "", { "os": "linux", "cpu": "x64" }, "sha512-+LpyBk7L44ZIXwz/VYfglaX/okxezESc6UxDSoyo2Ks6Jxc4Y7sGjpgU9s4PMgqgjj1gZCylTieNamqA1MF7Dg=="], + + "@img/sharp-linux-arm": ["@img/sharp-linux-arm@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm": "1.2.4" }, "os": "linux", "cpu": "arm" }, "sha512-9dLqsvwtg1uuXBGZKsxem9595+ujv0sJ6Vi8wcTANSFpwV/GONat5eCkzQo/1O6zRIkh0m/8+5BjrRr7jDUSZw=="], + + "@img/sharp-linux-arm64": ["@img/sharp-linux-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-bKQzaJRY/bkPOXyKx5EVup7qkaojECG6NLYswgktOZjaXecSAeCWiZwwiFf3/Y+O1HrauiE3FVsGxFg8c24rZg=="], + + "@img/sharp-linux-ppc64": ["@img/sharp-linux-ppc64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-ppc64": "1.2.4" }, "os": "linux", "cpu": "ppc64" }, "sha512-7zznwNaqW6YtsfrGGDA6BRkISKAAE1Jo0QdpNYXNMHu2+0dTrPflTLNkpc8l7MUP5M16ZJcUvysVWWrMefZquA=="], + + "@img/sharp-linux-riscv64": ["@img/sharp-linux-riscv64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-riscv64": "1.2.4" }, "os": "linux", "cpu": "none" }, "sha512-51gJuLPTKa7piYPaVs8GmByo7/U7/7TZOq+cnXJIHZKavIRHAP77e3N2HEl3dgiqdD/w0yUfiJnII77PuDDFdw=="], + + "@img/sharp-linux-s390x": ["@img/sharp-linux-s390x@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-s390x": "1.2.4" }, "os": "linux", "cpu": "s390x" }, "sha512-nQtCk0PdKfho3eC5MrbQoigJ2gd1CgddUMkabUj+rBevs8tZ2cULOx46E7oyX+04WGfABgIwmMC0VqieTiR4jg=="], + + "@img/sharp-linux-x64": ["@img/sharp-linux-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linux-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-MEzd8HPKxVxVenwAa+JRPwEC7QFjoPWuS5NZnBt6B3pu7EG2Ge0id1oLHZpPJdn3OQK+BQDiw9zStiHBTJQQQQ=="], + + "@img/sharp-linuxmusl-arm64": ["@img/sharp-linuxmusl-arm64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-arm64": "1.2.4" }, "os": "linux", "cpu": "arm64" }, "sha512-fprJR6GtRsMt6Kyfq44IsChVZeGN97gTD331weR1ex1c1rypDEABN6Tm2xa1wE6lYb5DdEnk03NZPqA7Id21yg=="], + + "@img/sharp-linuxmusl-x64": ["@img/sharp-linuxmusl-x64@0.34.5", "", { "optionalDependencies": { "@img/sharp-libvips-linuxmusl-x64": "1.2.4" }, "os": "linux", "cpu": "x64" }, "sha512-Jg8wNT1MUzIvhBFxViqrEhWDGzqymo3sV7z7ZsaWbZNDLXRJZoRGrjulp60YYtV4wfY8VIKcWidjojlLcWrd8Q=="], + + "@img/sharp-wasm32": ["@img/sharp-wasm32@0.34.5", "", { "dependencies": { "@emnapi/runtime": "^1.7.0" }, "cpu": "none" }, "sha512-OdWTEiVkY2PHwqkbBI8frFxQQFekHaSSkUIJkwzclWZe64O1X4UlUjqqqLaPbUpMOQk6FBu/HtlGXNblIs0huw=="], + + "@img/sharp-win32-arm64": ["@img/sharp-win32-arm64@0.34.5", "", { "os": "win32", "cpu": "arm64" }, "sha512-WQ3AgWCWYSb2yt+IG8mnC6Jdk9Whs7O0gxphblsLvdhSpSTtmu69ZG1Gkb6NuvxsNACwiPV6cNSZNzt0KPsw7g=="], + + "@img/sharp-win32-ia32": ["@img/sharp-win32-ia32@0.34.5", "", { "os": "win32", "cpu": "ia32" }, "sha512-FV9m/7NmeCmSHDD5j4+4pNI8Cp3aW+JvLoXcTUo0IqyjSfAZJ8dIUmijx1qaJsIiU+Hosw6xM5KijAWRJCSgNg=="], + + "@img/sharp-win32-x64": ["@img/sharp-win32-x64@0.34.5", "", { "os": "win32", "cpu": "x64" }, "sha512-+29YMsqY2/9eFEiW93eqWnuLcWcufowXewwSNIT6UwZdUUCrM3oFjMWH/Z6/TMmb4hlFenmfAVbpWeup2jryCw=="], + "@js-sdsl/ordered-map": ["@js-sdsl/ordered-map@4.4.2", "", {}, "sha512-iUKgm52T8HOE/makSxjqoWhe95ZJA1/G1sYsGev2JDKUSS14KAgg1LHb+Ba+IPow0xflbnSkOsZcO08C7w1gYw=="], "@mendable/firecrawl-js": ["@mendable/firecrawl-js@4.18.1", "", { "dependencies": { "axios": "1.14.0", "firecrawl": "4.16.0", "typescript-event-target": "^1.1.1", "zod": "^3.23.8", "zod-to-json-schema": "^3.23.0" } }, "sha512-NfmJv+xcHoZthj8I3NP/8KAgO8EWcvOcTvCAvszxqs7/6sCs1CRss6Tum6RycZNSwJkr5RzQossN89IlixRfng=="], @@ -437,6 +490,8 @@ "depd": ["depd@2.0.0", "", {}, "sha512-g7nH6P6dyDioJogAAGprGpCtVImJhpPk/roCzdb3fIh61/s/nPsfR6onyMwkCAR/OlC3yBC0lESvUoQEAssIrw=="], + "detect-libc": ["detect-libc@2.1.2", "", {}, "sha512-Btj2BOOO83o3WyH59e8MgXsxEQVcarkUOpEYrubB0urwnN10yQ364rsiByU11nZlqWYZm05i/of7io4mzihBtQ=="], + "diff": ["diff@8.0.3", "", {}, "sha512-qejHi7bcSD4hQAZE0tNAawRK1ZtafHDmMTMkrrIGgSLl7hTnQHmKCeB45xAcbfTqK2zowkM3j3bHt/4b/ARbYQ=="], "dijkstrajs": ["dijkstrajs@1.0.3", "", {}, "sha512-qiSlmBq9+BCdCA/L46dw8Uy93mloxsPSbwnm5yrKn2vMPiy8KyAskTF6zuV/j5BMsmOGZDPs7KjU+mjb670kfA=="], @@ -727,6 +782,8 @@ "setprototypeof": ["setprototypeof@1.2.0", "", {}, "sha512-E5LDX7Wrp85Kil5bhZv46j8jOeboKq5JMmYM3gVGdGH8xFpPWXUMsNrlODCrkoxMEeNi/XZIwuRvY4XNwYMJpw=="], + "sharp": ["sharp@0.34.5", "", { "dependencies": { "@img/colour": "^1.0.0", "detect-libc": "^2.1.2", "semver": "^7.7.3" }, "optionalDependencies": { "@img/sharp-darwin-arm64": "0.34.5", "@img/sharp-darwin-x64": "0.34.5", "@img/sharp-libvips-darwin-arm64": "1.2.4", "@img/sharp-libvips-darwin-x64": "1.2.4", "@img/sharp-libvips-linux-arm": "1.2.4", "@img/sharp-libvips-linux-arm64": "1.2.4", "@img/sharp-libvips-linux-ppc64": "1.2.4", "@img/sharp-libvips-linux-riscv64": "1.2.4", "@img/sharp-libvips-linux-s390x": "1.2.4", "@img/sharp-libvips-linux-x64": "1.2.4", "@img/sharp-libvips-linuxmusl-arm64": "1.2.4", "@img/sharp-libvips-linuxmusl-x64": "1.2.4", "@img/sharp-linux-arm": "0.34.5", "@img/sharp-linux-arm64": "0.34.5", "@img/sharp-linux-ppc64": "0.34.5", "@img/sharp-linux-riscv64": "0.34.5", "@img/sharp-linux-s390x": "0.34.5", "@img/sharp-linux-x64": "0.34.5", "@img/sharp-linuxmusl-arm64": "0.34.5", "@img/sharp-linuxmusl-x64": "0.34.5", "@img/sharp-wasm32": "0.34.5", "@img/sharp-win32-arm64": "0.34.5", "@img/sharp-win32-ia32": "0.34.5", "@img/sharp-win32-x64": "0.34.5" } }, "sha512-Ou9I5Ft9WNcCbXrU9cMgPBcCK8LiwLqcbywW3t4oDV37n1pzpuNLsYiAV8eODnjbtQlSDwZ2cUEeQz4E54Hltg=="], + "shebang-command": ["shebang-command@2.0.0", "", { "dependencies": { "shebang-regex": "^3.0.0" } }, "sha512-kHxr2zZpYtdmrN1qDjrrX/Z1rR1kG8Dx+gkpK1G4eXmvXswmcE1hTWBWYUzlraYw1/yZp6YuDY77YtvbN0dmDA=="], "shebang-regex": ["shebang-regex@3.0.0", "", {}, "sha512-7++dFhtcx3353uBaq8DDR4NuxBetBzC7ZQOhmTQInHEd6bSrXdiEyzCvG07Z44UYdLShWUyXt5M/yhz8ekcb1A=="], @@ -1025,6 +1082,8 @@ "@aws-sdk/xml-builder/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "@emnapi/runtime/tslib": ["tslib@2.8.1", "", {}, "sha512-oJFu94HQb+KVduSUQL7wnpmqnfmLsOA/nAh6b6EH0wCEoK0/mPeXU6c3wKDV83MkOuHPRHtSXKKU99IBazS/2w=="], + "@grpc/proto-loader/yargs": ["yargs@17.7.2", "", { "dependencies": { "cliui": "^8.0.1", "escalade": "^3.1.1", "get-caller-file": "^2.0.5", "require-directory": "^2.1.1", "string-width": "^4.2.3", "y18n": "^5.0.5", "yargs-parser": "^21.1.1" } }, "sha512-7dSzzRQ++CKnNI/krKnYRV7JKKPUXMEh61soaHKg9mrWEhzFWhFnxPxGl+69cD1Ou63C13NUPCnmIcrvqCuM6w=="], "@opentelemetry/exporter-trace-otlp-grpc/@opentelemetry/core": ["@opentelemetry/core@1.30.1", "", { "dependencies": { "@opentelemetry/semantic-conventions": "1.28.0" }, "peerDependencies": { "@opentelemetry/api": ">=1.0.0 <1.10.0" } }, "sha512-OOCM2C/QIURhJMuKaekP3TRBxBKxG/TWWA0TL2J6nXUtDnuCtccy49LUJF8xPFXMX+0LMcxFpCo8M9cGY1W6rQ=="], diff --git a/package.json b/package.json index 6fa9bbdc..c05ca16d 100644 --- a/package.json +++ b/package.json @@ -64,7 +64,6 @@ "@opentelemetry/sdk-trace-base": "2.6.1", "@opentelemetry/sdk-trace-node": "2.6.1", "@opentelemetry/semantic-conventions": "1.40.0", - "duck-duck-scrape": "^2.2.7", "ajv": "8.18.0", "auto-bind": "5.0.1", "axios": "1.14.0", @@ -76,6 +75,7 @@ "code-excerpt": "4.0.0", "commander": "12.1.0", "diff": "8.0.3", + "duck-duck-scrape": "^2.2.7", "emoji-regex": "10.6.0", "env-paths": "3.0.0", "execa": "9.6.1", @@ -99,6 +99,7 @@ "react-compiler-runtime": "1.0.0", "react-reconciler": "0.33.0", "semver": "7.7.4", + "sharp": "^0.34.5", "shell-quote": "1.8.3", "signal-exit": "4.1.0", "stack-utils": "2.0.6", diff --git a/scripts/build.ts b/scripts/build.ts index 137fadb6..53e051dd 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -158,7 +158,6 @@ export async function handleBgFlag() { throw new Error("Background sessions are 'modifiers-napi', 'url-handler-napi', 'color-diff-napi', - 'sharp', '@anthropic-ai/mcpb', '@ant/claude-for-chrome-mcp', '@anthropic-ai/sandbox-runtime', @@ -275,6 +274,8 @@ export const SeverityNumber = {}; '@opentelemetry/sdk-logs', '@opentelemetry/sdk-metrics', '@opentelemetry/semantic-conventions', + // Native image processing + 'sharp', // Cloud provider SDKs '@aws-sdk/client-bedrock', '@aws-sdk/client-bedrock-runtime', diff --git a/src/hooks/usePasteHandler.test.ts b/src/hooks/usePasteHandler.test.ts new file mode 100644 index 00000000..22207e2a --- /dev/null +++ b/src/hooks/usePasteHandler.test.ts @@ -0,0 +1,22 @@ +import { expect, test } from 'bun:test' +import { supportsClipboardImageFallback } from './usePasteHandler.ts' + +test('supports clipboard image fallback on Windows', () => { + expect(supportsClipboardImageFallback('windows')).toBe(true) +}) + +test('supports clipboard image fallback on macOS', () => { + expect(supportsClipboardImageFallback('macos')).toBe(true) +}) + +test('supports clipboard image fallback on Linux', () => { + expect(supportsClipboardImageFallback('linux')).toBe(true) +}) + +test('does not support clipboard image fallback on WSL', () => { + expect(supportsClipboardImageFallback('wsl')).toBe(false) +}) + +test('does not support clipboard image fallback on unknown platforms', () => { + expect(supportsClipboardImageFallback('unknown')).toBe(false) +}) diff --git a/src/hooks/usePasteHandler.ts b/src/hooks/usePasteHandler.ts index d6257b9a..a5fc6a96 100644 --- a/src/hooks/usePasteHandler.ts +++ b/src/hooks/usePasteHandler.ts @@ -15,6 +15,14 @@ import { getPlatform } from '../utils/platform.js' const CLIPBOARD_CHECK_DEBOUNCE_MS = 50 const PASTE_COMPLETION_TIMEOUT_MS = 100 +export function supportsClipboardImageFallback( + platform: ReturnType, +): boolean { + return ( + platform === 'macos' || platform === 'windows' || platform === 'linux' + ) +} + type PasteHandlerProps = { onPaste?: (text: string) => void onInput: (input: string, key: Key) => void @@ -52,7 +60,9 @@ export function usePasteHandler({ // that key is Enter, it submits the old input and the paste is lost. const pastePendingRef = React.useRef(false) - const isMacOS = React.useMemo(() => getPlatform() === 'macos', []) + const platform = React.useMemo(() => getPlatform(), []) + const isMacOS = platform === 'macos' + const canFallbackToClipboardImage = supportsClipboardImageFallback(platform) React.useEffect(() => { return () => { @@ -178,7 +188,11 @@ export function usePasteHandler({ // If paste is empty (common when trying to paste images with Cmd+V), // check if clipboard has an image (macOS only) - if (isMacOS && onImagePaste && pastedText.length === 0) { + if ( + canFallbackToClipboardImage && + onImagePaste && + pastedText.length === 0 + ) { checkClipboardForImage() return { chunks: [], timeoutId: null } } @@ -202,7 +216,13 @@ export function usePasteHandler({ pastePendingRef, ) }, - [checkClipboardForImage, isMacOS, onImagePaste, onPaste], + [ + checkClipboardForImage, + canFallbackToClipboardImage, + isMacOS, + onImagePaste, + onPaste, + ], ) // Paste detection is now done via the InputEvent's keypress.isPasted flag, @@ -242,7 +262,12 @@ export function usePasteHandler({ // When the user pastes an image with Cmd+V, the terminal sends an empty // bracketed paste sequence. The keypress parser emits this as isPasted=true // with empty input. - if (isFromPaste && input.length === 0 && isMacOS && onImagePaste) { + if ( + isFromPaste && + input.length === 0 && + canFallbackToClipboardImage && + onImagePaste + ) { checkClipboardForImage() // Reset isPasting since there's no text content to process setIsPasting(false) diff --git a/src/services/api/openaiShim.test.ts b/src/services/api/openaiShim.test.ts index 101c2ae5..681755e3 100644 --- a/src/services/api/openaiShim.test.ts +++ b/src/services/api/openaiShim.test.ts @@ -226,6 +226,88 @@ test('preserves Gemini tool call extra_content in follow-up requests', async () }) }) +test('preserves image tool results as placeholders in follow-up requests', async () => { + let requestBody: Record | undefined + + globalThis.fetch = (async (_input, init) => { + requestBody = JSON.parse(String(init?.body)) + + return new Response( + JSON.stringify({ + id: 'chatcmpl-1', + model: 'qwen/qwen3.6-plus', + choices: [ + { + message: { + role: 'assistant', + content: 'done', + }, + finish_reason: 'stop', + }, + ], + usage: { + prompt_tokens: 12, + completion_tokens: 4, + total_tokens: 16, + }, + }), + { + headers: { + 'Content-Type': 'application/json', + }, + }, + ) + }) as FetchType + + const client = createOpenAIShimClient({}) as OpenAIShimClient + + await client.beta.messages.create({ + model: 'qwen/qwen3.6-plus', + system: 'test system', + messages: [ + { role: 'user', content: 'Read this screenshot' }, + { + role: 'assistant', + content: [ + { + type: 'tool_use', + id: 'call_image_1', + name: 'Read', + input: { file_path: 'C:\\temp\\screenshot.png' }, + }, + ], + }, + { + role: 'user', + content: [ + { + type: 'tool_result', + tool_use_id: 'call_image_1', + content: [ + { + type: 'image', + source: { + type: 'base64', + media_type: 'image/png', + data: 'ZmFrZQ==', + }, + }, + ], + }, + ], + }, + ], + max_tokens: 64, + stream: false, + }) + + const toolMessage = (requestBody?.messages as Array>).find( + message => message.role === 'tool', + ) as { content?: string } | undefined + + expect(toolMessage?.content).toContain('[image:image/png]') +}) + test('preserves Gemini tool call extra_content from streaming chunks', async () => { globalThis.fetch = (async (_input, _init) => { const chunks = makeStreamChunks([ diff --git a/src/services/api/openaiShim.ts b/src/services/api/openaiShim.ts index f1e6ed82..128b075d 100644 --- a/src/services/api/openaiShim.ts +++ b/src/services/api/openaiShim.ts @@ -113,6 +113,37 @@ function convertSystemPrompt( return String(system) } +function convertToolResultContent(content: unknown): string { + if (typeof content === 'string') return content + if (!Array.isArray(content)) return JSON.stringify(content ?? '') + + const chunks: string[] = [] + for (const block of content) { + if (block?.type === 'text' && typeof block.text === 'string') { + chunks.push(block.text) + continue + } + + if (block?.type === 'image') { + const source = block.source + if (source?.type === 'url' && source.url) { + chunks.push(`[Image](${source.url})`) + } else if (source?.type === 'base64') { + chunks.push(`[image:${source.media_type ?? 'unknown'}]`) + } else { + chunks.push('[image]') + } + continue + } + + if (typeof block?.text === 'string') { + chunks.push(block.text) + } + } + + return chunks.join('\n') +} + function convertContentBlocks( content: unknown, ): string | Array<{ type: string; text?: string; image_url?: { url: string } }> { @@ -189,11 +220,7 @@ function convertMessages( // Emit tool results as tool messages for (const tr of toolResults) { - const trContent = Array.isArray(tr.content) - ? tr.content.map((c: { text?: string }) => c.text ?? '').join('\n') - : typeof tr.content === 'string' - ? tr.content - : JSON.stringify(tr.content ?? '') + const trContent = convertToolResultContent(tr.content) result.push({ role: 'tool', tool_call_id: tr.tool_use_id ?? 'unknown',