From 094ff0da802d2eeadeeccb2da9ee663d76edcbab Mon Sep 17 00:00:00 2001 From: Benjamin Liu Date: Fri, 1 May 2026 20:03:01 +0800 Subject: [PATCH] feat(deepseek): add vision mode support MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat(deepseek): add vision mode support DeepSeek added a third model "识图模式" (Vision Mode) that accepts image uploads for visual understanding. Add vision to the --model choices, update selectModel to use explicit index mapping for all three models, skip the search toggle in vision mode (not available), and extend waitForFilePreview to detect image thumbnails via send button state since vision mode shows a preview image instead of a filename label. Also catch "Not allowed" errors from setFileInput (Cloudflare may block CDP file operations) so the DataTransfer fallback can run. Closes #1215 * fix(deepseek): harden vision upload mode --------- Co-authored-by: jackwener --- cli-manifest.json | 5 +- clis/deepseek/ask.js | 21 +++++-- clis/deepseek/ask.test.js | 46 +++++++++++++++ clis/deepseek/utils.js | 24 +++++--- clis/deepseek/utils.test.js | 98 +++++++++++++++++++++++++++++++ docs/adapters/browser/deepseek.md | 8 ++- 6 files changed, 187 insertions(+), 15 deletions(-) diff --git a/cli-manifest.json b/cli-manifest.json index 6c4015b7..a548acc7 100644 --- a/cli-manifest.json +++ b/cli-manifest.json @@ -4380,10 +4380,11 @@ "type": "str", "default": "instant", "required": false, - "help": "Model to use: instant or expert", + "help": "Model to use: instant, expert, or vision", "choices": [ "instant", - "expert" + "expert", + "vision" ] }, { diff --git a/clis/deepseek/ask.js b/clis/deepseek/ask.js index f49c5ebd..c79dab14 100644 --- a/clis/deepseek/ask.js +++ b/clis/deepseek/ask.js @@ -18,7 +18,7 @@ export const askCommand = cli({ { name: 'prompt', positional: true, required: true, help: 'Prompt to send' }, { name: 'timeout', type: 'int', default: 120, help: 'Max seconds to wait for response' }, { name: 'new', type: 'boolean', default: false, help: 'Start a new chat before sending' }, - { name: 'model', default: 'instant', choices: ['instant', 'expert'], help: 'Model to use: instant or expert' }, + { name: 'model', default: 'instant', choices: ['instant', 'expert', 'vision'], help: 'Model to use: instant, expert, or vision' }, { name: 'think', type: 'boolean', default: false, help: 'Enable DeepThink mode' }, { name: 'search', type: 'boolean', default: false, help: 'Enable web search' }, { name: 'file', help: 'Attach a file (PDF, image, text) with the prompt' }, @@ -78,9 +78,22 @@ export const askCommand = cli({ throw new CommandExecutionError('Could not enable DeepThink'); } - const searchResult = await withRetry(() => setFeature(page, 'Search', wantSearch)); - if (!searchResult?.ok && wantSearch) { - throw new CommandExecutionError('Could not enable Search'); + if (wantModel === 'vision' && wantSearch) { + throw new CliError( + 'ARGUMENT', + 'DeepSeek vision mode does not support --search.', + 'Run without --search, or use --model instant/expert for web search.', + EXIT_CODES.USAGE_ERROR, + ); + } + + // Vision mode does not have the search toggle. + let searchResult; + if (wantModel !== 'vision') { + searchResult = await withRetry(() => setFeature(page, 'Search', wantSearch)); + if (!searchResult?.ok && wantSearch) { + throw new CommandExecutionError('Could not enable Search'); + } } if (thinkResult?.toggled || searchResult?.toggled) await page.wait(0.5); diff --git a/clis/deepseek/ask.test.js b/clis/deepseek/ask.test.js index 4f13d6c3..6a246455 100644 --- a/clis/deepseek/ask.test.js +++ b/clis/deepseek/ask.test.js @@ -263,4 +263,50 @@ describe('deepseek ask conversation resume', () => { expect(rows).toEqual([{ response: 'follow-up reply' }]); expect(mockSelectModel).toHaveBeenCalled(); }); + + it('skips search toggle in vision mode when search is not requested', async () => { + mockEnsureOnDeepSeek.mockResolvedValue(false); + mockSelectModel.mockResolvedValue({ ok: true, toggled: false }); + mockSetFeature.mockResolvedValue({ ok: true, toggled: false }); + mockSendMessage.mockResolvedValue({ ok: true }); + mockGetBubbleCount.mockResolvedValue(0); + mockWaitForResponse.mockResolvedValue('vision reply'); + page.evaluate.mockResolvedValue('https://chat.deepseek.com/'); + + const rows = await askCommand.func(page, { + prompt: 'describe', + timeout: 120, + new: false, + model: 'vision', + think: false, + search: false, + }); + + expect(rows).toEqual([{ response: 'vision reply' }]); + expect(mockSetFeature).toHaveBeenCalledTimes(1); + expect(mockSetFeature).toHaveBeenCalledWith(expect.anything(), 'DeepThink', false); + }); + + it('fails fast instead of silently ignoring --search in vision mode', async () => { + mockEnsureOnDeepSeek.mockResolvedValue(false); + mockSelectModel.mockResolvedValue({ ok: true, toggled: false }); + page.evaluate.mockResolvedValue('https://chat.deepseek.com/'); + + await expect(askCommand.func(page, { + prompt: 'describe', + timeout: 120, + new: false, + model: 'vision', + think: false, + search: true, + })).rejects.toMatchObject(new CliError( + 'ARGUMENT', + 'DeepSeek vision mode does not support --search.', + 'Run without --search, or use --model instant/expert for web search.', + EXIT_CODES.USAGE_ERROR, + )); + + expect(mockSendMessage).not.toHaveBeenCalled(); + expect(mockSendWithFile).not.toHaveBeenCalled(); + }); }); diff --git a/clis/deepseek/utils.js b/clis/deepseek/utils.js index b00ad5c4..efd1bf0d 100644 --- a/clis/deepseek/utils.js +++ b/clis/deepseek/utils.js @@ -40,9 +40,10 @@ export async function selectModel(page, modelName) { return page.evaluate(`(() => { var radios = document.querySelectorAll('div[role="radio"]'); if (radios.length === 0) return { ok: false }; - var isFirst = '${modelName}'.toLowerCase() === 'instant'; - if (!isFirst && radios.length < 2) return { ok: false }; - var target = isFirst ? radios[0] : radios[radios.length - 1]; + var name = '${modelName}'.toLowerCase(); + var index = name === 'instant' ? 0 : name === 'expert' ? 1 : name === 'vision' ? 2 : -1; + if (index < 0 || index >= radios.length) return { ok: false }; + var target = radios[index]; var alreadySelected = target.getAttribute('aria-checked') === 'true'; if (!alreadySelected) target.click(); return { ok: true, toggled: !alreadySelected }; @@ -277,9 +278,18 @@ async function waitForFilePreview(page, fileName) { for (let attempt = 0; attempt < 8; attempt++) { await page.wait(2); const ready = await page.evaluate(`(() => { - const name = ${JSON.stringify(fileName)}; - return Array.from(document.querySelectorAll('div')) - .some((el) => el.children.length === 0 && (el.textContent || '').trim() === name); + var name = ${JSON.stringify(fileName)}; + var hasFileName = Array.from(document.querySelectorAll('div')) + .some(function(el) { return el.children.length === 0 && (el.textContent || '').trim() === name; }); + if (hasFileName) return true; + // Vision mode shows an image thumbnail, not filename text. Require + // a preview-like node here; send-button readiness is checked later. + var box = document.querySelector('${TEXTAREA_SELECTOR}'); + if (!box) return false; + var c = box.parentElement; + while (c && !c.querySelector('div[role="button"]')) c = c.parentElement; + if (!c) return false; + return !!c.querySelector('img[src], canvas, video, [style*="background-image"], [class*="preview"], [class*="upload"]'); })()`); if (ready) return true; } @@ -318,7 +328,7 @@ export async function sendWithFile(page, filePath, prompt) { uploaded = true; } catch (err) { const msg = String(err?.message || err); - if (!msg.includes('Unknown action') && !msg.includes('not supported')) { + if (!msg.includes('Unknown action') && !msg.includes('not supported') && !msg.includes('Not allowed')) { throw err; } } diff --git a/clis/deepseek/utils.test.js b/clis/deepseek/utils.test.js index dfd41ea2..623c1268 100644 --- a/clis/deepseek/utils.test.js +++ b/clis/deepseek/utils.test.js @@ -163,4 +163,102 @@ describe('deepseek selectModel', () => { expect(result).toEqual({ ok: false }); expect(instantRadio.click).not.toHaveBeenCalled(); }); + + it('selects the correct radio for each model', async () => { + const radios = [0, 1, 2].map(() => ({ + getAttribute: vi.fn(() => 'false'), + click: vi.fn(), + })); + global.document = { + querySelectorAll: vi.fn(() => radios), + }; + const page = { + evaluate: vi.fn(async (script) => eval(script)), + }; + + await selectModel(page, 'instant'); + expect(radios[0].click).toHaveBeenCalled(); + expect(radios[1].click).not.toHaveBeenCalled(); + expect(radios[2].click).not.toHaveBeenCalled(); + + radios.forEach(r => r.click.mockClear()); + await selectModel(page, 'expert'); + expect(radios[1].click).toHaveBeenCalled(); + + radios.forEach(r => r.click.mockClear()); + await selectModel(page, 'vision'); + expect(radios[2].click).toHaveBeenCalled(); + }); + + it('rejects unknown model names', async () => { + const radios = [0, 1, 2].map(() => ({ + getAttribute: vi.fn(() => 'false'), + click: vi.fn(), + })); + global.document = { + querySelectorAll: vi.fn(() => radios), + }; + const page = { + evaluate: vi.fn(async (script) => eval(script)), + }; + + const result = await selectModel(page, 'turbo'); + expect(result).toEqual({ ok: false }); + }); +}); + +describe('deepseek sendWithFile Not allowed fallback', () => { + const tempDirs = []; + + afterEach(() => { + vi.restoreAllMocks(); + while (tempDirs.length) { + fs.rmSync(tempDirs.pop(), { recursive: true, force: true }); + } + }); + + it('falls back to DataTransfer when setFileInput throws Not allowed', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-deepseek-')); + tempDirs.push(dir); + const filePath = path.join(dir, 'image.png'); + fs.writeFileSync(filePath, 'fake-png'); + + const page = { + setFileInput: vi.fn().mockRejectedValue(new Error('Not allowed')), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn() + .mockResolvedValueOnce(undefined) // sidebar collapse + .mockResolvedValueOnce({ ok: true }) // DataTransfer fallback + .mockResolvedValueOnce(true) // waitForFilePreview + .mockResolvedValueOnce(true) // send button enabled + .mockResolvedValueOnce({ ok: true }),// sendMessage + }; + + const result = await sendWithFile(page, filePath, 'describe'); + + expect(page.setFileInput).toHaveBeenCalled(); + expect(page.evaluate).toHaveBeenCalledTimes(5); + expect(result).toEqual({ ok: true }); + }); + + it('does not treat send-button enablement alone as image upload proof', async () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'opencli-deepseek-')); + tempDirs.push(dir); + const filePath = path.join(dir, 'image.png'); + fs.writeFileSync(filePath, 'fake-png'); + + const page = { + setFileInput: vi.fn().mockResolvedValue(undefined), + wait: vi.fn().mockResolvedValue(undefined), + evaluate: vi.fn() + .mockResolvedValueOnce(undefined) // sidebar collapse + .mockResolvedValue(false), // no filename / thumbnail preview + }; + + const result = await sendWithFile(page, filePath, 'describe'); + + expect(result).toEqual({ ok: false, reason: 'file preview did not appear' }); + expect(page.evaluate.mock.calls[1][0]).toContain('img[src], canvas, video'); + expect(page.evaluate.mock.calls[1][0]).not.toContain("aria-disabled') === 'false'"); + }); }); diff --git a/docs/adapters/browser/deepseek.md b/docs/adapters/browser/deepseek.md index 02a744b7..95acd6cb 100644 --- a/docs/adapters/browser/deepseek.md +++ b/docs/adapters/browser/deepseek.md @@ -24,6 +24,9 @@ opencli deepseek ask "hello" --new # Use Expert model instead of Instant opencli deepseek ask "prove that sqrt(2) is irrational" --model expert +# Use Vision model with an image +opencli deepseek ask "describe this image" --model vision --file ./image.png + # Enable DeepThink mode opencli deepseek ask "prove that sqrt(2) is irrational" --think @@ -62,7 +65,7 @@ opencli deepseek history --limit 10 | `` | The message to send (required, positional) | | `--timeout` | Wait timeout in seconds (default: 120) | | `--new` | Start a new chat before sending (default: false) | -| `--model` | Model to use: `instant` or `expert` (default: instant) | +| `--model` | Model to use: `instant`, `expert`, or `vision` (default: instant) | | `--think` | Enable DeepThink mode (default: false) | | `--search` | Enable web search (default: false) | | `--file` | Attach a file (PDF, image, text) with the prompt (max 100 MB) | @@ -76,5 +79,6 @@ opencli deepseek history --limit 10 - This adapter drives the DeepSeek web UI in the browser, not an API - Default mode is Instant with DeepThink and Search disabled; each flag (`--model`, `--think`, `--search`) is synced on every invocation so omitting a flag resets it +- Vision mode does not support `--search`; use `--model instant` or `--model expert` for web search - Long responses (code, essays) may need a higher `--timeout` -- File upload reads the file into memory and passes it via base64 to the browser; files over 100 MB are rejected +- File upload prefers the browser file-input path, falls back to base64 injection when needed, and rejects files over 100 MB