Compare commits
15 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ae4929e73d | |||
| 92408e7ee9 | |||
| 257e9ec4f8 | |||
| 6f629260e7 | |||
| 1e0b83bb84 | |||
| 1ff184e24d | |||
| 8e84145ccc | |||
| c77c8a8e3a | |||
| 9e024d4e46 | |||
| 34a9bff2b3 | |||
| b0c51ddf19 | |||
| 141c2cf7d5 | |||
| 7ee0c9c96b | |||
| 76eefed83d | |||
| 08ae9c3fed |
@@ -0,0 +1,26 @@
|
||||
name: Setup Chrome + xvfb
|
||||
description: Install real Chrome and xvfb virtual display for headed browser testing
|
||||
|
||||
outputs:
|
||||
chrome-path:
|
||||
description: Path to the installed Chrome binary
|
||||
value: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install real Chrome (stable)
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
id: setup-chrome
|
||||
with:
|
||||
chrome-version: stable
|
||||
|
||||
- name: Verify Chrome installation
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Chrome path: ${{ steps.setup-chrome.outputs.chrome-path }}"
|
||||
${{ steps.setup-chrome.outputs.chrome-path }} --version
|
||||
|
||||
- name: Install xvfb for headed mode
|
||||
shell: bash
|
||||
run: sudo apt-get install -y xvfb
|
||||
@@ -2,12 +2,16 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [main, dev]
|
||||
schedule:
|
||||
- cron: '0 8 * * 1' # Weekly Monday 08:00 UTC — smoke tests
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check:
|
||||
# ── Fast gate: typecheck + build ──
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -15,6 +19,7 @@ jobs:
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
@@ -24,3 +29,54 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── Unit tests (vitest shard) ──
|
||||
unit-test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run unit tests (shard ${{ matrix.shard }}/2)
|
||||
run: npx vitest run src/ --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
|
||||
# ── Smoke tests (scheduled / manual only) ──
|
||||
smoke-test:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Setup Chrome + xvfb
|
||||
uses: ./.github/actions/setup-chrome
|
||||
id: setup-chrome
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Run smoke tests
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/smoke/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
timeout-minutes: 15
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
name: E2E Headed Chrome
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
branches: [main, dev]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
e2e-headed:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Setup Chrome + xvfb
|
||||
uses: ./.github/actions/setup-chrome
|
||||
id: setup-chrome
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Run E2E tests (headed Chrome + xvfb)
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/e2e/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
@@ -2,3 +2,4 @@ node_modules/
|
||||
dist/
|
||||
*.tsbuildinfo
|
||||
.opencli/
|
||||
.mcp.json
|
||||
|
||||
@@ -9,7 +9,7 @@
|
||||
[](https://nodejs.org)
|
||||
[](./LICENSE)
|
||||
|
||||
A CLI tool that turns **any website** into a command-line interface. **57 commands** across **17 sites** — bilibili, zhihu, xiaohongshu, twitter, reddit, xueqiu, github, v2ex, hackernews, bbc, weibo, boss, yahoo-finance, reuters, smzdm, ctrip, youtube — powered by browser session reuse and AI-native discovery.
|
||||
A CLI tool that turns **any website** into a command-line interface. **59 commands** across **18 sites** — bilibili, zhihu, xiaohongshu, twitter, reddit, xueqiu, github, v2ex, hackernews, bbc, weibo, boss, yahoo-finance, reuters, smzdm, ctrip, youtube, coupang — powered by browser session reuse and AI-native discovery.
|
||||
|
||||
---
|
||||
|
||||
@@ -21,6 +21,7 @@ A CLI tool that turns **any website** into a command-line interface. **57 comman
|
||||
- [Built-in Commands](#built-in-commands)
|
||||
- [Output Formats](#output-formats)
|
||||
- [For AI Agents (Developer Guide)](#for-ai-agents-developer-guide)
|
||||
- [Testing](#testing)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Releasing New Versions](#releasing-new-versions)
|
||||
- [License](#license)
|
||||
@@ -72,6 +73,12 @@ And, so that `opencli` commands can use it directly in the terminal, export it i
|
||||
export PLAYWRIGHT_MCP_EXTENSION_TOKEN="<your-token-here>"
|
||||
```
|
||||
|
||||
After configuring, run `opencli doctor` to verify your token is correctly set up across all locations:
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Install via npm (recommended)
|
||||
@@ -120,13 +127,14 @@ npm install -g @jackwener/opencli@latest
|
||||
| **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 Browser |
|
||||
| **weibo** | `hot` | 🔐 Browser |
|
||||
| **boss** | `search` | 🔐 Browser |
|
||||
| **coupang** | `search` `add-to-cart` | 🔐 Browser |
|
||||
| **youtube** | `search` | 🔐 Browser |
|
||||
| **yahoo-finance** | `quote` | 🔐 Browser |
|
||||
| **reuters** | `search` | 🔐 Browser |
|
||||
| **smzdm** | `search` | 🔐 Browser |
|
||||
| **ctrip** | `search` | 🔐 Browser |
|
||||
| **github** | `search` | 🌐 Public |
|
||||
| **v2ex** | `hot` `latest` `topic` | 🌐 Public |
|
||||
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 Public / 🔐 Browser |
|
||||
| **hackernews** | `top` | 🌐 Public |
|
||||
| **bbc** | `news` | 🌐 Public |
|
||||
|
||||
@@ -169,6 +177,24 @@ opencli cascade https://api.example.com/data
|
||||
|
||||
Explore outputs to `.opencli/explore/<site>/` (manifest.json, endpoints.json, capabilities.json, auth.json).
|
||||
|
||||
## Testing
|
||||
|
||||
See **[TESTING.md](./TESTING.md)** for the full testing guide, including:
|
||||
|
||||
- Current test coverage (unit + ~52 E2E tests across all 18 sites)
|
||||
- How to run tests locally
|
||||
- How to add tests when creating new adapters
|
||||
- CI/CD pipeline with sharding
|
||||
- Headless browser mode (`OPENCLI_HEADLESS=1`)
|
||||
|
||||
```bash
|
||||
# Quick start
|
||||
npm run build
|
||||
npx vitest run # All tests
|
||||
npx vitest run src/ # Unit tests only
|
||||
npx vitest run tests/e2e/ # E2E tests
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **"Failed to connect to Playwright MCP Bridge"**
|
||||
|
||||
+10
-3
@@ -9,7 +9,7 @@
|
||||
[](https://nodejs.org)
|
||||
[](./LICENSE)
|
||||
|
||||
OpenCLI 将任何网站变成命令行工具。**57 个命令**覆盖 **17 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube — 复用浏览器登录态,AI 驱动探索。
|
||||
OpenCLI 将任何网站变成命令行工具。**59 个命令**覆盖 **18 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube、Coupang — 复用浏览器登录态,AI 驱动探索。
|
||||
|
||||
---
|
||||
|
||||
@@ -29,7 +29,7 @@ OpenCLI 将任何网站变成命令行工具。**57 个命令**覆盖 **17 个
|
||||
|
||||
## 亮点
|
||||
|
||||
- **57 个命令,17 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球(xueqiu)、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube
|
||||
- **59 个命令,18 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球(xueqiu)、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube、Coupang
|
||||
- **零风控** — 复用 Chrome 登录态,无需存储任何凭证
|
||||
- **AI 原生** — `explore` 自动发现 API,`synthesize` 生成适配器,`cascade` 探测认证策略
|
||||
- **动态加载引擎** — 声明式的 `.yaml` 或者底层定制的 `.ts` 适配器,放入 `clis/` 文件夹即可自动注册生效
|
||||
@@ -72,6 +72,12 @@ OpenCLI 通过 Playwright MCP Bridge 扩展与你的浏览器通信。
|
||||
export PLAYWRIGHT_MCP_EXTENSION_TOKEN="<你的-token>"
|
||||
```
|
||||
|
||||
配置完成后,运行 `opencli doctor` 检测你的 Token 是否在所有位置都正确配置:
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
### npm 全局安装(推荐)
|
||||
@@ -120,13 +126,14 @@ npm install -g @jackwener/opencli@latest
|
||||
| **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 浏览器 |
|
||||
| **weibo** | `hot` | 🔐 浏览器 |
|
||||
| **boss** | `search` | 🔐 浏览器 |
|
||||
| **coupang** | `search` `add-to-cart` | 🔐 浏览器 |
|
||||
| **youtube** | `search` | 🔐 浏览器 |
|
||||
| **yahoo-finance** | `quote` | 🔐 浏览器 |
|
||||
| **reuters** | `search` | 🔐 浏览器 |
|
||||
| **smzdm** | `search` | 🔐 浏览器 |
|
||||
| **ctrip** | `search` | 🔐 浏览器 |
|
||||
| **github** | `search` | 🌐 公共 API |
|
||||
| **v2ex** | `hot` `latest` `topic` | 🌐 公共 API |
|
||||
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 公共 API / 🔐 浏览器 |
|
||||
| **hackernews** | `top` | 🌐 公共 API |
|
||||
| **bbc** | `news` | 🌐 公共 API |
|
||||
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
---
|
||||
name: opencli
|
||||
description: "OpenCLI — Make any website your CLI. Zero risk, AI-powered, reuse Chrome login."
|
||||
version: 0.4.6
|
||||
version: 0.5.1
|
||||
author: jackwener
|
||||
tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, reddit, xiaohongshu, xueqiu, AI, agent]
|
||||
---
|
||||
@@ -95,10 +95,13 @@ opencli reddit frontpage --limit 10 # 首页
|
||||
opencli reddit search --keyword "AI" # 搜索
|
||||
opencli reddit subreddit --name rust # 子版块浏览
|
||||
|
||||
# V2EX (public)
|
||||
# V2EX (public + browser)
|
||||
opencli v2ex hot --limit 10 # 热门话题
|
||||
opencli v2ex latest --limit 10 # 最新话题
|
||||
opencli v2ex topic --id 1024 # 主题详情
|
||||
opencli v2ex daily # 每日签到 (browser)
|
||||
opencli v2ex me # 我的信息 (browser)
|
||||
opencli v2ex notifications --limit 10 # 通知 (browser)
|
||||
|
||||
# Hacker News (public)
|
||||
opencli hackernews top --limit 10 # Top stories
|
||||
@@ -156,8 +159,8 @@ opencli cascade <api-url>
|
||||
# Explore with interactive fuzzing (click buttons to trigger lazy APIs)
|
||||
opencli explore <url> --auto --click "字幕,CC,评论"
|
||||
|
||||
# Verify: smoke-test a generated adapter
|
||||
opencli verify <site/name> --smoke
|
||||
# Verify: validate adapter definitions
|
||||
opencli verify
|
||||
```
|
||||
|
||||
## Output Formats
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
# Testing Guide
|
||||
|
||||
> 面向开发者和 AI Agent 的测试参考手册。
|
||||
|
||||
## 目录
|
||||
|
||||
- [测试架构](#测试架构)
|
||||
- [当前覆盖范围](#当前覆盖范围)
|
||||
- [本地运行测试](#本地运行测试)
|
||||
- [如何添加新测试](#如何添加新测试)
|
||||
- [CI/CD 流水线](#cicd-流水线)
|
||||
- [浏览器模式](#浏览器模式)
|
||||
- [站点兼容性](#站点兼容性)
|
||||
|
||||
---
|
||||
|
||||
## 测试架构
|
||||
|
||||
测试分为三层,全部使用 **vitest** 运行:
|
||||
|
||||
```
|
||||
tests/
|
||||
├── e2e/ # E2E 集成测试(子进程运行真实 CLI)
|
||||
│ ├── helpers.ts # runCli() 共享工具
|
||||
│ ├── public-commands.test.ts # 公开 API 命令(无需浏览器)
|
||||
│ ├── browser-public.test.ts # 浏览器命令(公开数据)
|
||||
│ ├── browser-auth.test.ts # 需登录命令(graceful failure 测试)
|
||||
│ ├── management.test.ts # 管理命令(list, validate, verify, help)
|
||||
│ └── output-formats.test.ts # 输出格式(json/yaml/csv/md)
|
||||
├── smoke/ # 烟雾测试(仅定时 / 手动触发)
|
||||
│ └── api-health.test.ts # 外部 API 可用性检测
|
||||
src/
|
||||
├── *.test.ts # 单元测试(已有 8 个)
|
||||
```
|
||||
|
||||
| 层 | 位置 | 运行方式 | 用途 |
|
||||
|---|---|---|---|
|
||||
| 单元测试 | `src/**/*.test.ts` | `npx vitest run src/` | 内部模块逻辑 |
|
||||
| E2E 测试 | `tests/e2e/*.test.ts` | `npx vitest run tests/e2e/` | 真实 CLI 命令执行 |
|
||||
| 烟雾测试 | `tests/smoke/*.test.ts` | `npx vitest run tests/smoke/` | 外部 API 健康 |
|
||||
|
||||
---
|
||||
|
||||
## 当前覆盖范围
|
||||
|
||||
### 单元测试(8 个文件)
|
||||
|
||||
| 文件 | 覆盖内容 |
|
||||
|---|---|
|
||||
| `browser.test.ts` | JSON-RPC、tab 管理、extension/standalone 模式切换 |
|
||||
| `engine.test.ts` | 命令发现与执行 |
|
||||
| `registry.test.ts` | 命令注册与策略分配 |
|
||||
| `output.test.ts` | 输出格式渲染 |
|
||||
| `doctor.test.ts` | Token 诊断 |
|
||||
| `coupang.test.ts` | 数据归一化 |
|
||||
| `pipeline/template.test.ts` | 模板表达式求值 |
|
||||
| `pipeline/transform.test.ts` | 数据变换步骤 |
|
||||
|
||||
### E2E 测试(~52 个用例)
|
||||
|
||||
| 文件 | 覆盖站点/功能 | 测试数 |
|
||||
|---|---|---|
|
||||
| `public-commands.test.ts` | hackernews/top, v2ex/hot, v2ex/latest, v2ex/topic | 5 |
|
||||
| `browser-public.test.ts` | bbc, bilibili×3, weibo, zhihu×2, reddit×2, twitter, xueqiu×2, reuters, youtube, smzdm, boss, ctrip, coupang, xiaohongshu, yahoo-finance, v2ex/daily | 21 |
|
||||
| `browser-auth.test.ts` | bilibili/me,dynamic,favorite,history,following + twitter/bookmarks,timeline,notifications + v2ex/me,notifications + xueqiu/feed,watchlist + xiaohongshu/feed,notifications | 14 |
|
||||
| `management.test.ts` | list×5 格式, validate×3 级别, verify, --version, --help, unknown cmd | 12 |
|
||||
| `output-formats.test.ts` | json, yaml, csv, md 格式验证 | 5 |
|
||||
|
||||
### 烟雾测试
|
||||
|
||||
公开 API 可用性(hackernews, v2ex×2, v2ex/topic)+ 全站点注册完整性检查。
|
||||
|
||||
---
|
||||
|
||||
## 本地运行测试
|
||||
|
||||
### 前置条件
|
||||
|
||||
```bash
|
||||
npm ci # 安装依赖
|
||||
npm run build # 编译(E2E 测试需要 dist/main.js)
|
||||
```
|
||||
|
||||
### 运行命令
|
||||
|
||||
```bash
|
||||
# 全部单元测试
|
||||
npx vitest run src/
|
||||
|
||||
# 全部 E2E 测试(会真实调用外部 API)
|
||||
npx vitest run tests/e2e/
|
||||
|
||||
# 单个测试文件
|
||||
npx vitest run tests/e2e/management.test.ts
|
||||
|
||||
# 全部测试(单元 + E2E)
|
||||
npx vitest run
|
||||
|
||||
# 烟雾测试
|
||||
npx vitest run tests/smoke/
|
||||
|
||||
# watch 模式(开发时推荐)
|
||||
npx vitest src/
|
||||
```
|
||||
|
||||
### 浏览器命令本地测试须知
|
||||
|
||||
- 无 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 时,opencli 自动启动一个独立浏览器实例
|
||||
- `browser-public.test.ts` 使用 `tryBrowserCommand()`,站点反爬导致空数据时 warn + pass
|
||||
- `browser-auth.test.ts` 验证 **graceful failure**(不 crash 不 hang 即通过)
|
||||
- 如需测试完整登录态,保持 Chrome 登录态 + 设置 `PLAYWRIGHT_MCP_EXTENSION_TOKEN`,手动跑对应测试
|
||||
|
||||
---
|
||||
|
||||
## 如何添加新测试
|
||||
|
||||
### 新增 YAML Adapter(如 `src/clis/producthunt/trending.yaml`)
|
||||
|
||||
1. **无需额外操作**:`validate` 测试会自动覆盖 YAML 结构验证
|
||||
2. 根据 adapter 类型,在对应文件加一个 `it()` block:
|
||||
|
||||
```typescript
|
||||
// 如果 browser: false(公开 API)→ tests/e2e/public-commands.test.ts
|
||||
it('producthunt trending returns data', async () => {
|
||||
const { stdout, code } = await runCli(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
```
|
||||
|
||||
```typescript
|
||||
// 如果 browser: true 但可公开访问 → tests/e2e/browser-public.test.ts
|
||||
it('producthunt trending returns data', async () => {
|
||||
const data = await tryBrowserCommand(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'producthunt trending');
|
||||
}, 60_000);
|
||||
```
|
||||
|
||||
```typescript
|
||||
// 如果 browser: true 且需登录 → tests/e2e/browser-auth.test.ts
|
||||
it('producthunt me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['producthunt', 'me', '-f', 'json'], 'producthunt me');
|
||||
}, 60_000);
|
||||
```
|
||||
|
||||
### 新增管理命令(如 `opencli export`)
|
||||
|
||||
在 `tests/e2e/management.test.ts` 添加测试。
|
||||
|
||||
### 新增内部模块
|
||||
|
||||
在 `src/` 下对应位置创建 `*.test.ts`。
|
||||
|
||||
### 决策流程图
|
||||
|
||||
```
|
||||
新增功能 → 是内部模块? → 是 → src/ 下加 *.test.ts
|
||||
↓ 否
|
||||
是 CLI 命令? → browser: false? → tests/e2e/public-commands.test.ts
|
||||
↓ true
|
||||
公开数据? → tests/e2e/browser-public.test.ts
|
||||
↓ 需登录
|
||||
tests/e2e/browser-auth.test.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CI/CD 流水线
|
||||
|
||||
### ci.yml(主流水线)
|
||||
|
||||
| Job | 触发条件 | 内容 |
|
||||
|---|---|---|
|
||||
| **build** | push/PR to main,dev | typecheck + build |
|
||||
| **unit-test** | push/PR to main,dev | 单元测试,2 shard 并行 |
|
||||
| **smoke-test** | 每周一 08:00 UTC / 手动 | xvfb + real Chrome,外部 API 健康检查 |
|
||||
|
||||
### e2e-headed.yml(E2E 测试)
|
||||
|
||||
| Job | 触发条件 | 内容 |
|
||||
|---|---|---|
|
||||
| **e2e-headed** | push/PR to main,dev | xvfb + real Chrome,全部 E2E 测试 |
|
||||
|
||||
E2E 使用 `browser-actions/setup-chrome` 安装真实 Chrome,配合 `xvfb-run` 提供虚拟显示器,以 headed 模式运行浏览器。
|
||||
|
||||
### Sharding
|
||||
|
||||
单元测试使用 vitest 内置 shard:
|
||||
|
||||
```yaml
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- run: npx vitest run src/ --shard=${{ matrix.shard }}/2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 浏览器模式
|
||||
|
||||
opencli 根据 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 环境变量自动选择模式:
|
||||
|
||||
| 条件 | 模式 | MCP 参数 | 使用场景 |
|
||||
|---|---|---|---|
|
||||
| Token 已设置 | Extension 模式 | `--extension` | 本地用户,连接已登录的 Chrome |
|
||||
| Token 未设置 | Standalone 模式 | (无特殊 flag) | CI 或无扩展环境,自启浏览器 |
|
||||
|
||||
CI 中使用 `OPENCLI_BROWSER_EXECUTABLE_PATH` 指定真实 Chrome 路径:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 站点兼容性
|
||||
|
||||
在 GitHub Actions 美国 runner 上,部分站点因地域限制或登录要求返回空数据。E2E 测试对这些站点使用 warn + pass 策略,不影响 CI 绿灯。
|
||||
|
||||
| 站点 | CI 状态 | 限制原因 |
|
||||
|---|---|---|
|
||||
| hackernews, bbc, v2ex | ✅ 返回数据 | 无限制 |
|
||||
| yahoo-finance | ✅ 返回数据 | 无限制 |
|
||||
| bilibili, zhihu, weibo, xiaohongshu | ⚠️ 空数据 | 地域限制(中国站点) |
|
||||
| reddit, twitter, youtube | ⚠️ 空数据 | 需登录或 cookie |
|
||||
| smzdm, boss, ctrip, coupang, xueqiu | ⚠️ 空数据 | 地域限制 / 需登录 |
|
||||
|
||||
> 使用 self-hosted runner(国内服务器)可解决地域限制问题。
|
||||
Generated
+5
-2
@@ -1,12 +1,12 @@
|
||||
{
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.2",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"chalk": "^5.3.0",
|
||||
@@ -24,6 +24,9 @@
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.8.2",
|
||||
"vitest": "^4.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@colors/colors": {
|
||||
|
||||
+1
-1
@@ -1,6 +1,6 @@
|
||||
{
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.5.0",
|
||||
"version": "0.5.2",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
|
||||
+69
-1
@@ -20,6 +20,17 @@ describe('browser helpers', () => {
|
||||
]);
|
||||
});
|
||||
|
||||
it('extracts tab entries from MCP markdown format', () => {
|
||||
const entries = __test__.extractTabEntries(
|
||||
'- 0: (current) [Playwright MCP extension](chrome-extension://abc/connect.html)\n- 1: [知乎 - 首页](https://www.zhihu.com/)'
|
||||
);
|
||||
|
||||
expect(entries).toEqual([
|
||||
{ index: 0, identity: '(current) [Playwright MCP extension](chrome-extension://abc/connect.html)' },
|
||||
{ index: 1, identity: '[知乎 - 首页](https://www.zhihu.com/)' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('closes only tabs that were opened during the session', () => {
|
||||
const tabsToClose = __test__.diffTabIndexes(
|
||||
['https://example.com', 'Chrome Extension'],
|
||||
@@ -38,8 +49,65 @@ describe('browser helpers', () => {
|
||||
expect(__test__.appendLimited('12345', '67890', 8)).toBe('34567890');
|
||||
});
|
||||
|
||||
it('builds extension MCP args in local mode (no CI)', () => {
|
||||
const savedCI = process.env.CI;
|
||||
delete process.env.CI;
|
||||
try {
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
executablePath: '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--extension',
|
||||
'--executable-path',
|
||||
'/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
|
||||
]);
|
||||
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--extension',
|
||||
]);
|
||||
} finally {
|
||||
if (savedCI !== undefined) {
|
||||
process.env.CI = savedCI;
|
||||
} else {
|
||||
delete process.env.CI;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('builds standalone MCP args in CI mode', () => {
|
||||
const savedCI = process.env.CI;
|
||||
process.env.CI = 'true';
|
||||
try {
|
||||
// CI mode: no --extension — browser launches in standalone headed mode
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
]);
|
||||
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
executablePath: '/usr/bin/chromium',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--executable-path',
|
||||
'/usr/bin/chromium',
|
||||
]);
|
||||
} finally {
|
||||
if (savedCI !== undefined) {
|
||||
process.env.CI = savedCI;
|
||||
} else {
|
||||
delete process.env.CI;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('times out slow promises', async () => {
|
||||
await expect(__test__.withTimeout(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
|
||||
await expect(__test__.withTimeoutMs(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
|
||||
});
|
||||
});
|
||||
|
||||
|
||||
+51
-94
@@ -10,10 +10,10 @@ import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { formatSnapshot } from './snapshotFormatter.js';
|
||||
|
||||
// Read version from package.json (single source of truth)
|
||||
const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
|
||||
import { PKG_VERSION } from './version.js';
|
||||
import { normalizeEvaluateSource } from './pipeline/template.js';
|
||||
import { generateInterceptorJs, generateReadInterceptedJs } from './interceptor.js';
|
||||
import { withTimeoutMs } from './runtime.js';
|
||||
|
||||
const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
|
||||
const STDERR_BUFFER_LIMIT = 16 * 1024;
|
||||
@@ -158,23 +158,10 @@ export class Page implements IPage {
|
||||
|
||||
async evaluate(js: string): Promise<any> {
|
||||
// Normalize IIFE format to function format expected by MCP browser_evaluate
|
||||
const normalized = this.normalizeEval(js);
|
||||
const normalized = normalizeEvaluateSource(js);
|
||||
return this.call('tools/call', { name: 'browser_evaluate', arguments: { function: normalized } });
|
||||
}
|
||||
|
||||
private normalizeEval(source: string): string {
|
||||
const s = source.trim();
|
||||
if (!s) return '() => undefined';
|
||||
// IIFE: (async () => {...})() → wrap as () => (...)
|
||||
if (s.startsWith('(') && s.endsWith(')()')) return `() => (${s})`;
|
||||
// Already a function/arrow
|
||||
if (/^(async\s+)?\([^)]*\)\s*=>/.test(s)) return s;
|
||||
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(s)) return s;
|
||||
if (s.startsWith('function ') || s.startsWith('async function ')) return s;
|
||||
// Raw expression → wrap
|
||||
return `() => (${s})`;
|
||||
}
|
||||
|
||||
async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
|
||||
const raw = await this.call('tools/call', { name: 'browser_snapshot', arguments: {} });
|
||||
if (opts.raw) return raw;
|
||||
@@ -263,57 +250,15 @@ export class Page implements IPage {
|
||||
}
|
||||
|
||||
async installInterceptor(pattern: string): Promise<void> {
|
||||
const js = `
|
||||
() => {
|
||||
window.__opencli_xhr = window.__opencli_xhr || [];
|
||||
window.__opencli_patterns = window.__opencli_patterns || [];
|
||||
if (!window.__opencli_patterns.includes('${pattern}')) {
|
||||
window.__opencli_patterns.push('${pattern}');
|
||||
}
|
||||
|
||||
if (!window.__patched_xhr) {
|
||||
const checkMatch = (url) => window.__opencli_patterns.some(p => url.includes(p));
|
||||
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url) {
|
||||
this._url = url;
|
||||
return open.call(this, method, url, ...Array.prototype.slice.call(arguments, 2));
|
||||
};
|
||||
XHR.send = function() {
|
||||
this.addEventListener('load', function() {
|
||||
if (checkMatch(this._url)) {
|
||||
try { window.__opencli_xhr.push({url: this._url, data: JSON.parse(this.responseText)}); } catch(e){}
|
||||
}
|
||||
});
|
||||
return send.apply(this, arguments);
|
||||
};
|
||||
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const res = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (checkMatch(u)) {
|
||||
const clone = res.clone();
|
||||
const j = await clone.json();
|
||||
window.__opencli_xhr.push({url: u, data: j});
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return res;
|
||||
};
|
||||
window.__patched_xhr = true;
|
||||
}
|
||||
}
|
||||
`;
|
||||
await this.evaluate(js);
|
||||
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
|
||||
arrayName: '__opencli_xhr',
|
||||
patchGuard: '__opencli_interceptor_patched',
|
||||
}));
|
||||
}
|
||||
|
||||
async getInterceptedRequests(): Promise<any[]> {
|
||||
return (await this.evaluate('() => window.__opencli_xhr')) || [];
|
||||
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
|
||||
return result || [];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -404,6 +349,7 @@ export class PlaywrightMCP {
|
||||
return new Promise<Page>((resolve, reject) => {
|
||||
const isDebug = process.env.DEBUG?.includes('opencli:mcp');
|
||||
const debugLog = (msg: string) => isDebug && console.error(`[opencli:mcp] ${msg}`);
|
||||
const useExtension = !!process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
|
||||
const extensionToken = process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
|
||||
const tokenFingerprint = getTokenFingerprint(extensionToken);
|
||||
let stderrBuffer = '';
|
||||
@@ -442,12 +388,13 @@ export class PlaywrightMCP {
|
||||
}));
|
||||
}, timeout * 1000);
|
||||
|
||||
const mcpArgs: string[] = [mcpPath, '--extension'];
|
||||
const mcpArgs = buildMcpArgs({
|
||||
mcpPath,
|
||||
executablePath: process.env.OPENCLI_BROWSER_EXECUTABLE_PATH,
|
||||
});
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error(`[opencli] Extension token: ${extensionToken ? `configured (fingerprint ${tokenFingerprint})` : 'missing'}`);
|
||||
}
|
||||
if (process.env.OPENCLI_BROWSER_EXECUTABLE_PATH) {
|
||||
mcpArgs.push('--executablePath', process.env.OPENCLI_BROWSER_EXECUTABLE_PATH);
|
||||
console.error(`[opencli] Mode: ${useExtension ? 'extension' : 'standalone'}`);
|
||||
if (useExtension) console.error(`[opencli] Extension token: fingerprint ${tokenFingerprint}`);
|
||||
}
|
||||
debugLog(`Spawning node ${mcpArgs.join(' ')}`);
|
||||
|
||||
@@ -530,7 +477,7 @@ export class PlaywrightMCP {
|
||||
|
||||
// Use tabs as a readiness probe and for tab cleanup bookkeeping.
|
||||
debugLog('Fetching initial tabs count...');
|
||||
withTimeout(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
|
||||
withTimeoutMs(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
|
||||
debugLog(`Tabs response: ${typeof tabs === 'string' ? tabs : JSON.stringify(tabs)}`);
|
||||
this._initialTabIdentities = extractTabIdentities(tabs);
|
||||
settleSuccess(page);
|
||||
@@ -555,7 +502,7 @@ export class PlaywrightMCP {
|
||||
// Extension mode opens bridge/session tabs that we can clean up best-effort.
|
||||
if (this._page && this._proc && !this._proc.killed) {
|
||||
try {
|
||||
const tabs = await withTimeout(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
|
||||
const tabs = await withTimeoutMs(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
|
||||
const tabEntries = extractTabEntries(tabs);
|
||||
const tabsToClose = diffTabIndexes(this._initialTabIdentities, tabEntries);
|
||||
for (const index of tabsToClose) {
|
||||
@@ -610,12 +557,23 @@ function extractTabEntries(raw: any): Array<{ index: number; identity: string }>
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean)
|
||||
.map(line => {
|
||||
const match = line.match(/Tab\s+(\d+)\s*(.*)$/);
|
||||
if (!match) return null;
|
||||
return {
|
||||
index: parseInt(match[1], 10),
|
||||
identity: match[2].trim() || `tab-${match[1]}`,
|
||||
};
|
||||
// Match actual Playwright MCP format: "- 0: (current) [title](url)" or "- 1: [title](url)"
|
||||
const mcpMatch = line.match(/^-\s+(\d+):\s*(.*)$/);
|
||||
if (mcpMatch) {
|
||||
return {
|
||||
index: parseInt(mcpMatch[1], 10),
|
||||
identity: mcpMatch[2].trim() || `tab-${mcpMatch[1]}`,
|
||||
};
|
||||
}
|
||||
// Legacy format: "Tab 0 ..."
|
||||
const legacyMatch = line.match(/Tab\s+(\d+)\s*(.*)$/);
|
||||
if (legacyMatch) {
|
||||
return {
|
||||
index: parseInt(legacyMatch[1], 10),
|
||||
identity: legacyMatch[2].trim() || `tab-${legacyMatch[1]}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter((entry): entry is { index: number; identity: string } => entry !== null);
|
||||
}
|
||||
@@ -653,20 +611,18 @@ function appendLimited(current: string, chunk: string, limit: number): string {
|
||||
return next.slice(-limit);
|
||||
}
|
||||
|
||||
function withTimeout<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||
promise.then(
|
||||
(value) => {
|
||||
clearTimeout(timer);
|
||||
resolve(value);
|
||||
},
|
||||
(error) => {
|
||||
clearTimeout(timer);
|
||||
reject(error);
|
||||
},
|
||||
);
|
||||
});
|
||||
function buildMcpArgs(input: { mcpPath: string; executablePath?: string | null }): string[] {
|
||||
const args = [input.mcpPath];
|
||||
if (!process.env.CI) {
|
||||
// Local: always connect to user's running Chrome via MCP Bridge extension
|
||||
args.push('--extension');
|
||||
}
|
||||
// CI: standalone mode — @playwright/mcp launches its own browser (headed by default).
|
||||
// xvfb provides a virtual display for headed mode in GitHub Actions.
|
||||
if (input.executablePath) {
|
||||
args.push('--executable-path', input.executablePath);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
@@ -674,7 +630,8 @@ export const __test__ = {
|
||||
extractTabEntries,
|
||||
diffTabIndexes,
|
||||
appendLimited,
|
||||
withTimeout,
|
||||
buildMcpArgs,
|
||||
withTimeoutMs,
|
||||
};
|
||||
|
||||
function findMcpServerPath(): string | null {
|
||||
|
||||
+47
-75
@@ -37,6 +37,49 @@ interface CascadeResult {
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the JavaScript source for a fetch probe.
|
||||
* Shared logic for PUBLIC, COOKIE, and HEADER strategies.
|
||||
*/
|
||||
function buildFetchProbeJs(url: string, opts: {
|
||||
credentials?: boolean;
|
||||
extractCsrf?: boolean;
|
||||
}): string {
|
||||
const credentialsLine = opts.credentials ? `credentials: 'include',` : '';
|
||||
const headerSetup = opts.extractCsrf
|
||||
? `
|
||||
const cookies = document.cookie.split(';').map(c => c.trim());
|
||||
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
|
||||
const headers = {};
|
||||
if (csrf) { headers['X-Csrf-Token'] = csrf; headers['X-XSRF-Token'] = csrf; }
|
||||
`
|
||||
: 'const headers = {};';
|
||||
|
||||
return `
|
||||
async () => {
|
||||
try {
|
||||
${headerSetup}
|
||||
const resp = await fetch(${JSON.stringify(url)}, {
|
||||
${credentialsLine}
|
||||
headers
|
||||
});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
// Check for API-level error codes (common in Chinese sites)
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe an endpoint with a specific strategy.
|
||||
* Returns whether the probe succeeded and basic response info.
|
||||
@@ -45,32 +88,14 @@ export async function probeEndpoint(
|
||||
page: IPage,
|
||||
url: string,
|
||||
strategy: Strategy,
|
||||
opts: { timeout?: number } = {},
|
||||
_opts: { timeout?: number } = {},
|
||||
): Promise<ProbeResult> {
|
||||
const result: ProbeResult = { strategy, success: false };
|
||||
|
||||
try {
|
||||
switch (strategy) {
|
||||
case Strategy.PUBLIC: {
|
||||
// Try direct fetch without browser (no credentials)
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, {}));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -79,27 +104,7 @@ export async function probeEndpoint(
|
||||
}
|
||||
|
||||
case Strategy.COOKIE: {
|
||||
// Fetch with credentials: 'include' (uses browser cookies)
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
// Check for API-level error codes (common in Chinese sites)
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true }));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -108,39 +113,7 @@ export async function probeEndpoint(
|
||||
}
|
||||
|
||||
case Strategy.HEADER: {
|
||||
// Fetch with credentials + try to extract common auth headers
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
// Try to extract CSRF tokens from cookies
|
||||
const cookies = document.cookie.split(';').map(c => c.trim());
|
||||
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
|
||||
|
||||
const headers = {};
|
||||
if (csrf) {
|
||||
headers['X-Csrf-Token'] = csrf;
|
||||
headers['X-XSRF-Token'] = csrf;
|
||||
}
|
||||
|
||||
const resp = await fetch(${JSON.stringify(url)}, {
|
||||
credentials: 'include',
|
||||
headers
|
||||
});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true, extractCsrf: true }));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -151,7 +124,6 @@ export async function probeEndpoint(
|
||||
case Strategy.INTERCEPT:
|
||||
case Strategy.UI:
|
||||
// These require specific implementation per-site
|
||||
// Mark as needing manual implementation
|
||||
result.success = false;
|
||||
result.error = `Strategy ${strategy} requires site-specific implementation`;
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { canonicalizeProductUrl, normalizeProductId } from '../../coupang.js';
|
||||
|
||||
function escapeJsString(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function buildAddToCartEvaluate(expectedProductId: string): string {
|
||||
return `
|
||||
(async () => {
|
||||
const expectedProductId = ${escapeJsString(expectedProductId)};
|
||||
const text = document.body.innerText || '';
|
||||
const loginHints = {
|
||||
hasLoginLink: Boolean(document.querySelector('a[href*="login"], a[title*="로그인"]')),
|
||||
hasMyCoupang: /마이쿠팡/.test(text),
|
||||
};
|
||||
|
||||
const pathMatch = location.pathname.match(/\\/vp\\/products\\/(\\d+)/);
|
||||
const currentProductId = pathMatch?.[1] || '';
|
||||
if (expectedProductId && currentProductId && expectedProductId !== currentProductId) {
|
||||
return { ok: false, reason: 'PRODUCT_MISMATCH', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
const optionSelectors = [
|
||||
'select',
|
||||
'[role="listbox"]',
|
||||
'.prod-option, .product-option, .option-select, .option-dropdown',
|
||||
];
|
||||
const hasRequiredOption = optionSelectors.some((selector) => {
|
||||
try {
|
||||
const nodes = Array.from(document.querySelectorAll(selector));
|
||||
return nodes.some((node) => {
|
||||
const label = (node.textContent || '') + ' ' + (node.getAttribute?.('aria-label') || '');
|
||||
return /옵션|색상|사이즈|용량|선택/i.test(label);
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
if (hasRequiredOption) {
|
||||
return { ok: false, reason: 'OPTION_REQUIRED', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
const clickCandidate = (elements) => {
|
||||
for (const element of elements) {
|
||||
if (!(element instanceof HTMLElement)) continue;
|
||||
const label = ((element.innerText || '') + ' ' + (element.getAttribute('aria-label') || '')).trim();
|
||||
if (/장바구니|카트|cart/i.test(label) && !/sold out|품절/i.test(label)) {
|
||||
element.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const beforeCount = (() => {
|
||||
const node = document.querySelector('[class*="cart"] .count, #headerCartCount, .cart-count');
|
||||
const text = node?.textContent || '';
|
||||
const num = Number(text.replace(/[^\\d]/g, ''));
|
||||
return Number.isFinite(num) ? num : null;
|
||||
})();
|
||||
|
||||
const buttons = Array.from(document.querySelectorAll('button, a[role="button"], input[type="button"]'));
|
||||
const clicked = clickCandidate(buttons);
|
||||
if (!clicked) {
|
||||
return { ok: false, reason: 'ADD_TO_CART_BUTTON_NOT_FOUND', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
|
||||
const afterText = document.body.innerText || '';
|
||||
const successMessage = /장바구니에 담|장바구니 담기 완료|added to cart/i.test(afterText);
|
||||
const afterCount = (() => {
|
||||
const node = document.querySelector('[class*="cart"] .count, #headerCartCount, .cart-count');
|
||||
const text = node?.textContent || '';
|
||||
const num = Number(text.replace(/[^\\d]/g, ''));
|
||||
return Number.isFinite(num) ? num : null;
|
||||
})();
|
||||
const countIncreased =
|
||||
beforeCount != null &&
|
||||
afterCount != null &&
|
||||
afterCount >= beforeCount &&
|
||||
(afterCount > beforeCount || beforeCount === 0);
|
||||
|
||||
return {
|
||||
ok: successMessage || countIncreased,
|
||||
reason: successMessage || countIncreased ? 'SUCCESS' : 'UNKNOWN',
|
||||
currentProductId,
|
||||
beforeCount,
|
||||
afterCount,
|
||||
loginHints,
|
||||
};
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'coupang',
|
||||
name: 'add-to-cart',
|
||||
description: 'Add a Coupang product to cart using logged-in browser session',
|
||||
domain: 'www.coupang.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'productId', required: false, help: 'Coupang product ID' },
|
||||
{ name: 'url', required: false, help: 'Canonical product URL' },
|
||||
],
|
||||
columns: ['ok', 'product_id', 'url', 'message'],
|
||||
func: async (page, kwargs) => {
|
||||
const rawProductId = kwargs.productId ?? kwargs.product_id;
|
||||
const productId = normalizeProductId(rawProductId);
|
||||
const targetUrl = canonicalizeProductUrl(kwargs.url, productId);
|
||||
|
||||
if (!productId && !targetUrl) {
|
||||
throw new Error('Either --product-id or --url is required');
|
||||
}
|
||||
|
||||
const finalUrl = targetUrl || canonicalizeProductUrl('', productId);
|
||||
await page.goto(finalUrl);
|
||||
await page.wait(3);
|
||||
|
||||
const result = await page.evaluate(buildAddToCartEvaluate(productId));
|
||||
const loginHints = result?.loginHints ?? {};
|
||||
if (loginHints.hasLoginLink && !loginHints.hasMyCoupang) {
|
||||
throw new Error('Coupang login required. Please log into Coupang in Chrome and retry.');
|
||||
}
|
||||
|
||||
const actualProductId = normalizeProductId(result?.currentProductId || productId);
|
||||
if (result?.reason === 'PRODUCT_MISMATCH') {
|
||||
throw new Error(`Product mismatch: expected ${productId}, got ${actualProductId || 'unknown'}`);
|
||||
}
|
||||
if (result?.reason === 'OPTION_REQUIRED') {
|
||||
throw new Error('This product requires option selection and is not supported in v1.');
|
||||
}
|
||||
if (result?.reason === 'ADD_TO_CART_BUTTON_NOT_FOUND') {
|
||||
throw new Error('Could not find an add-to-cart button on the product page.');
|
||||
}
|
||||
if (!result?.ok) {
|
||||
throw new Error('Failed to confirm add-to-cart success.');
|
||||
}
|
||||
|
||||
return [{
|
||||
ok: true,
|
||||
product_id: actualProductId || productId,
|
||||
url: finalUrl,
|
||||
message: 'Added to cart',
|
||||
}];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,466 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { mergeSearchItems, normalizeSearchItem, sanitizeSearchItems } from '../../coupang.js';
|
||||
|
||||
function escapeJsString(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function buildApplyFilterEvaluate(filter: string): string {
|
||||
return `
|
||||
() => {
|
||||
const filter = ${escapeJsString(filter)};
|
||||
const labels = Array.from(document.querySelectorAll('label'));
|
||||
const normalize = (value) => (value == null ? '' : String(value).trim().toLowerCase());
|
||||
const target = labels.find((label) => {
|
||||
const component = normalize(label.getAttribute('data-component-name'));
|
||||
const imgAlt = normalize(label.querySelector('img')?.getAttribute('alt'));
|
||||
const text = normalize(label.textContent);
|
||||
|
||||
if (filter === 'rocket') {
|
||||
return (
|
||||
component.includes('deliveryfilteroption-rocket_luxury,rocket_wow,coupang_global') ||
|
||||
imgAlt.includes('rocket_luxury,rocket_wow,coupang_global') ||
|
||||
imgAlt.includes('rocket-all') ||
|
||||
text.includes('로켓')
|
||||
);
|
||||
}
|
||||
|
||||
return component.includes(filter) || imgAlt.includes(filter) || text.includes(filter);
|
||||
});
|
||||
|
||||
if (!target) {
|
||||
return { ok: false, reason: 'FILTER_NOT_FOUND' };
|
||||
}
|
||||
|
||||
target.click();
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
reason: 'FILTER_CLICKED',
|
||||
component: target.getAttribute('data-component-name') || '',
|
||||
text: (target.textContent || '').trim(),
|
||||
alt: target.querySelector('img')?.getAttribute('alt') || '',
|
||||
};
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
function buildCurrentLocationEvaluate(): string {
|
||||
return `
|
||||
() => ({
|
||||
href: location.href
|
||||
})
|
||||
`;
|
||||
}
|
||||
|
||||
function buildSearchEvaluate(query: string, limit: number, pageNumber: number): string {
|
||||
return `
|
||||
(async () => {
|
||||
const query = ${escapeJsString(query)};
|
||||
const limit = ${limit};
|
||||
const pageNumber = ${pageNumber};
|
||||
|
||||
const normalizeText = (value) => (value == null ? '' : String(value).trim());
|
||||
const parseNum = (value) => {
|
||||
const text = normalizeText(value).replace(/[^\\d.]/g, '');
|
||||
if (!text) return null;
|
||||
const num = Number(text);
|
||||
return Number.isFinite(num) ? num : null;
|
||||
};
|
||||
const extractPriceFromText = (text) => {
|
||||
const matches = normalizeText(text).match(/\\d{1,3}(?:,\\d{3})*원/g) || [];
|
||||
if (!matches.length) return '';
|
||||
if (matches.length >= 2) return matches[matches.length - 2];
|
||||
return matches[0];
|
||||
};
|
||||
const extractPriceInfo = (root) => {
|
||||
const priceArea =
|
||||
root.querySelector('.PriceArea_priceArea__NntJz, [class*="PriceArea_priceArea"], [class*="priceArea"]') ||
|
||||
root;
|
||||
const priceAreaText = normalizeText(priceArea.textContent || '');
|
||||
const originalPrice = normalizeText(
|
||||
priceArea.querySelector(
|
||||
'del, .base-price, .origin-price, .original-price, .strike-price, [class*="base-price"], [class*="origin-price"], [class*="line-through"]'
|
||||
)?.textContent || ''
|
||||
);
|
||||
const originalPriceNum = parseNum(originalPrice);
|
||||
const unitPrice =
|
||||
normalizeText(
|
||||
priceArea.querySelector('.unit-price, [class*="unit-price"], [class*="unitPrice"]')?.textContent || ''
|
||||
) ||
|
||||
priceAreaText.match(/\\([^)]*당\\s*[^)]*원[^)]*\\)/)?.[0] ||
|
||||
'';
|
||||
|
||||
const candidates = Array.from(priceArea.querySelectorAll('span, strong, div'))
|
||||
.map((node) => {
|
||||
const text = normalizeText(node.textContent || '');
|
||||
if (!text || !/\\d/.test(text)) return null;
|
||||
if (/\\d{1,2}:\\d{2}:\\d{2}/.test(text)) return null;
|
||||
if (/당\\s*\\d/.test(text)) return null;
|
||||
if (/^\\d+%$/.test(text)) return null;
|
||||
|
||||
const num = parseNum(text);
|
||||
if (num == null) return null;
|
||||
|
||||
const className = normalizeText(node.getAttribute('class') || '').toLowerCase();
|
||||
let score = 0;
|
||||
if (/price|sale|selling|final/.test(className)) score += 6;
|
||||
if (/red/.test(className)) score += 5;
|
||||
if (/font-bold|bold/.test(className)) score += 3;
|
||||
if (/line-through/.test(className)) score -= 12;
|
||||
if (text.includes('원')) score += 2;
|
||||
if (originalPriceNum != null && num === originalPriceNum) score -= 10;
|
||||
if (num < 100) score -= 10;
|
||||
|
||||
return { text, num, score };
|
||||
})
|
||||
.filter(Boolean)
|
||||
.sort((a, b) => {
|
||||
if (b.score !== a.score) return b.score - a.score;
|
||||
if (originalPriceNum != null) {
|
||||
const aPrefer = a.num !== originalPriceNum ? 1 : 0;
|
||||
const bPrefer = b.num !== originalPriceNum ? 1 : 0;
|
||||
if (bPrefer !== aPrefer) return bPrefer - aPrefer;
|
||||
}
|
||||
return b.num - a.num;
|
||||
});
|
||||
|
||||
const currentPrice =
|
||||
normalizeText(candidates.find((candidate) => candidate.num !== originalPriceNum)?.text || '') ||
|
||||
normalizeText(candidates[0]?.text || '') ||
|
||||
extractPriceFromText(priceAreaText) ||
|
||||
'';
|
||||
|
||||
return {
|
||||
price: currentPrice,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
};
|
||||
};
|
||||
const canonicalUrl = (url, productId) => {
|
||||
if (url) {
|
||||
try {
|
||||
const parsed = new URL(url, 'https://www.coupang.com');
|
||||
const match = parsed.pathname.match(/\\/vp\\/products\\/(\\d+)/);
|
||||
return 'https://www.coupang.com/vp/products/' + (match?.[1] || productId || '');
|
||||
} catch {}
|
||||
}
|
||||
return productId ? 'https://www.coupang.com/vp/products/' + productId : '';
|
||||
};
|
||||
const normalize = (raw) => {
|
||||
const rawText = normalizeText(raw.text || raw.badgeText || raw.deliveryText || raw.summary);
|
||||
const productId = normalizeText(
|
||||
raw.productId || raw.product_id || raw.id || raw.productNo ||
|
||||
raw?.product?.productId || raw?.item?.id
|
||||
).match(/(\\d{6,})/)?.[1] || '';
|
||||
const title = normalizeText(
|
||||
raw.title || raw.name || raw.productName || raw.productTitle || raw.itemName
|
||||
);
|
||||
const price = parseNum(raw.price || raw.salePrice || raw.finalPrice || raw.sellingPrice);
|
||||
const originalPrice = parseNum(raw.originalPrice || raw.basePrice || raw.listPrice || raw.originPrice);
|
||||
const unitPrice = normalizeText(raw.unitPrice || raw.unit_price || raw.unitPriceText);
|
||||
const rating = parseNum(raw.rating || raw.star || raw.reviewRating);
|
||||
const reviewCount = parseNum(raw.reviewCount || raw.ratingCount || raw.reviewCnt || raw.reviews);
|
||||
const badge = Array.isArray(raw.badges) ? raw.badges.map(normalizeText).filter(Boolean).join(', ') : normalizeText(raw.badge || raw.labels);
|
||||
const seller = normalizeText(raw.seller || raw.sellerName || raw.vendorName || raw.merchantName);
|
||||
const category = normalizeText(raw.category || raw.categoryName || raw.categoryPath);
|
||||
const discountRate = parseNum(raw.discountRate || raw.discount || raw.discountPercent);
|
||||
const url = canonicalUrl(raw.url || raw.productUrl || raw.link, productId);
|
||||
return {
|
||||
productId,
|
||||
title,
|
||||
price,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
discountRate,
|
||||
rating,
|
||||
reviewCount,
|
||||
rocket: normalizeText(raw.rocket || raw.rocketType),
|
||||
deliveryType: normalizeText(raw.deliveryType || raw.deliveryBadge || raw.shippingType || raw.shippingBadge),
|
||||
deliveryPromise: normalizeText(raw.deliveryPromise || raw.promise || raw.arrivalText || raw.arrivalBadge),
|
||||
seller,
|
||||
badge,
|
||||
category,
|
||||
url,
|
||||
};
|
||||
};
|
||||
|
||||
const byApi = async () => {
|
||||
const candidates = [
|
||||
'/np/search?q=' + encodeURIComponent(query) + '&component=&channel=user&page=' + pageNumber,
|
||||
'/np/search?component=&q=' + encodeURIComponent(query) + '&channel=user&page=' + pageNumber,
|
||||
];
|
||||
|
||||
for (const path of candidates) {
|
||||
try {
|
||||
const resp = await fetch(path, { credentials: 'include' });
|
||||
if (!resp.ok) continue;
|
||||
const text = await resp.text();
|
||||
const data = text.trim().startsWith('<') ? null : JSON.parse(text);
|
||||
const maybeItems =
|
||||
data?.data?.products ||
|
||||
data?.data?.productList ||
|
||||
data?.products ||
|
||||
data?.productList ||
|
||||
data?.items;
|
||||
if (Array.isArray(maybeItems) && maybeItems.length) {
|
||||
return maybeItems.slice(0, limit).map(normalize);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byBootstrap = () => {
|
||||
const isProductLike = (item) => {
|
||||
if (!item || typeof item !== 'object') return false;
|
||||
const values = [item.productId, item.product_id, item.id, item.productNo, item.url, item.productUrl, item.link, item.title, item.productName];
|
||||
return values.some((value) => /\\/vp\\/products\\/|\\d{6,}/.test(normalizeText(value)));
|
||||
};
|
||||
|
||||
const collectProducts = (node) => {
|
||||
const queue = [node];
|
||||
while (queue.length) {
|
||||
const current = queue.shift();
|
||||
if (!current || typeof current !== 'object') continue;
|
||||
if (Array.isArray(current)) {
|
||||
const productish = current.filter(isProductLike);
|
||||
if (productish.length >= 3) return productish.slice(0, limit).map(normalize);
|
||||
queue.push(...current.slice(0, 50));
|
||||
continue;
|
||||
}
|
||||
for (const value of Object.values(current)) queue.push(value);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const scriptNodes = Array.from(document.scripts);
|
||||
for (const script of scriptNodes) {
|
||||
const text = script.textContent || '';
|
||||
if (!text || !/product|search/i.test(text)) continue;
|
||||
const arrayMatches = [
|
||||
...text.matchAll(/"products?"\\s*:\\s*(\\[[\\s\\S]{100,}?\\])/g),
|
||||
...text.matchAll(/"itemList"\\s*:\\s*(\\[[\\s\\S]{100,}?\\])/g),
|
||||
];
|
||||
for (const match of arrayMatches) {
|
||||
try {
|
||||
const products = JSON.parse(match[1]);
|
||||
if (Array.isArray(products) && products.length) {
|
||||
return products.slice(0, limit).map(normalize);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const globals = [
|
||||
window.__NEXT_DATA__,
|
||||
window.__APOLLO_STATE__,
|
||||
window.__INITIAL_STATE__,
|
||||
window.__STATE__,
|
||||
window.__PRELOADED_STATE__,
|
||||
];
|
||||
for (const candidate of globals) {
|
||||
if (!candidate || typeof candidate !== 'object') continue;
|
||||
const found = collectProducts(candidate);
|
||||
if (found.length) return found;
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byJsonLd = () => {
|
||||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||
for (const script of scripts) {
|
||||
const text = script.textContent || '';
|
||||
if (!text) continue;
|
||||
try {
|
||||
const payload = JSON.parse(text);
|
||||
const docs = Array.isArray(payload) ? payload : [payload];
|
||||
for (const doc of docs) {
|
||||
const items =
|
||||
doc?.itemListElement ||
|
||||
doc?.about?.itemListElement ||
|
||||
doc?.mainEntity?.itemListElement ||
|
||||
[];
|
||||
if (!Array.isArray(items) || !items.length) continue;
|
||||
const mapped = items.map((entry) => {
|
||||
const item = entry?.item || entry;
|
||||
return normalize({
|
||||
productId: item?.url || item?.sku || item?.productID,
|
||||
title: item?.name,
|
||||
price: item?.offers?.price,
|
||||
originalPrice: item?.offers?.highPrice,
|
||||
rating: item?.aggregateRating?.ratingValue,
|
||||
reviewCount: item?.aggregateRating?.reviewCount,
|
||||
seller: item?.offers?.seller?.name,
|
||||
badge: item?.offers?.availability,
|
||||
category: item?.category,
|
||||
url: item?.url,
|
||||
});
|
||||
}).filter((item) => item.productId || item.url || item.title);
|
||||
if (mapped.length) return mapped.slice(0, limit);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byDom = () => {
|
||||
const domScanLimit = Math.max(limit * 6, 60);
|
||||
const cards = Array.from(new Set([
|
||||
...document.querySelectorAll('li.search-product'),
|
||||
...document.querySelectorAll('li[class*="search-product"], div[class*="search-product"], article[class*="search-product"]'),
|
||||
...document.querySelectorAll('li[class*="ProductUnit_productUnit"], [class*="ProductUnit_productUnit"]'),
|
||||
...document.querySelectorAll('.impression-logged, [class*="promotion-item"], [class*="product-item"]'),
|
||||
...document.querySelectorAll('[data-product-id]'),
|
||||
...document.querySelectorAll('[data-id]'),
|
||||
...document.querySelectorAll('a[href*="/vp/products/"]'),
|
||||
])).slice(0, domScanLimit);
|
||||
const items = [];
|
||||
for (const el of cards) {
|
||||
const root = el.closest('li, div, article, section') || el;
|
||||
const html = root.innerHTML || '';
|
||||
const priceInfo = extractPriceInfo(root);
|
||||
const badgeImages = Array.from(root.querySelectorAll('img[data-badge-id]'));
|
||||
const badgeIds = badgeImages
|
||||
.map((node) => node.getAttribute('data-badge-id') || '')
|
||||
.filter(Boolean);
|
||||
const badgeSrcText = badgeImages
|
||||
.map((node) => (node.getAttribute('data-badge-id') || '') + ' ' + (node.getAttribute('src') || ''))
|
||||
.join(' ');
|
||||
const productId =
|
||||
root.getAttribute('data-product-id') ||
|
||||
el.getAttribute('data-product-id') ||
|
||||
root.querySelector('a[href*="/vp/products/"]')?.getAttribute('data-product-id') ||
|
||||
root.querySelector('a[href*="/vp/products/"]')?.getAttribute('href')?.match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
html.match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
(el.getAttribute('href') || '').match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
'';
|
||||
const title =
|
||||
root.querySelector('.name, .title, .product-name, .search-product-title, .item-title, .ProductUnit_productNameV2__cV9cw, [class*="ProductUnit_productName"], [class*="productName"], [class*="product-name"], [class*="title"]')?.textContent ||
|
||||
root.querySelector('img[alt]')?.getAttribute('alt') ||
|
||||
html.match(/alt="([^"]+)"/)?.[1] ||
|
||||
(root.textContent || '').replace(/\\s+/g, ' ').trim().match(/^(.+?)(\\d{1,3},\\d{3}원|무료배송|내일\\(|오늘\\(|새벽)/)?.[1] ||
|
||||
el.getAttribute('title') ||
|
||||
'';
|
||||
const price = priceInfo.price || '';
|
||||
const originalPrice = priceInfo.originalPrice || '';
|
||||
const unitPrice = priceInfo.unitPrice || '';
|
||||
const rating =
|
||||
root.querySelector('.rating, .star em, [class*="rating"], [class*="star"], [class*="ProductRating"] [aria-label], [aria-label][class*="ProductRating"]')?.getAttribute?.('aria-label') ||
|
||||
root.querySelector('.rating, .star em, [class*="rating"], [class*="star"], [class*="ProductRating"]')?.textContent ||
|
||||
'';
|
||||
const reviewCount =
|
||||
root.querySelector('.rating-total-count, .count, .review-count, .promotion-item-review-count, [class*="review"], [class*="count"], [class*="ProductRating"] span, [class*="ProductRating"] [class*="fw-text"]')?.textContent ||
|
||||
'';
|
||||
const seller =
|
||||
root.querySelector('.seller, .vendor, .search-product-wrap .vendor-name, [class*="vendor"], [class*="seller"]')?.textContent ||
|
||||
'';
|
||||
const category =
|
||||
root.getAttribute('data-category') ||
|
||||
root.querySelector('[class*="category"]')?.textContent ||
|
||||
'';
|
||||
const text = (root.textContent || '').replace(/\\s+/g, ' ').trim();
|
||||
const badgeNodes = Array.from(root.querySelectorAll('.badge, .delivery, .tag, .icon-service, .pdd-text, .delivery-text, [class*="badge"], [class*="delivery"]'));
|
||||
const hrefNode = root.querySelector('a[href*="/vp/products/"]');
|
||||
items.push(normalize({
|
||||
productId,
|
||||
title,
|
||||
price,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
rating,
|
||||
reviewCount,
|
||||
seller,
|
||||
badges: [...badgeIds, ...badgeNodes.map((node) => node.textContent || '').filter(Boolean)],
|
||||
rocket: badgeSrcText + ' ' + badgeNodes.map((node) => node.textContent || '').join(' '),
|
||||
deliveryType: badgeNodes.map((node) => node.textContent || '').join(' ') + ' ' + text,
|
||||
deliveryPromise: badgeNodes.map((node) => node.textContent || '').join(' ') + ' ' + text,
|
||||
category,
|
||||
text,
|
||||
url: hrefNode?.getAttribute('href') || '',
|
||||
}));
|
||||
}
|
||||
return items.slice(0, domScanLimit);
|
||||
};
|
||||
|
||||
let items = await byApi();
|
||||
if (!items.length) items = byJsonLd();
|
||||
if (!items.length) items = byBootstrap();
|
||||
const domItems = byDom();
|
||||
if (!items.length) items = domItems;
|
||||
|
||||
return {
|
||||
loginHints: {
|
||||
hasLoginLink: Boolean(document.querySelector('a[href*="login"], a[title*="로그인"]')),
|
||||
hasMyCoupang: /마이쿠팡/.test(document.body.innerText),
|
||||
},
|
||||
items,
|
||||
domItems,
|
||||
};
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'coupang',
|
||||
name: 'search',
|
||||
description: 'Search Coupang products with logged-in browser session',
|
||||
domain: 'www.coupang.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', required: true, help: 'Search keyword' },
|
||||
{ name: 'page', type: 'int', default: 1, help: 'Search result page number' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
|
||||
{ name: 'filter', required: false, help: 'Optional search filter (currently supports: rocket)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'price', 'unit_price', 'rating', 'review_count', 'rocket', 'delivery_type', 'delivery_promise', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query || '').trim();
|
||||
const pageNumber = Math.max(Number(kwargs.page || 1), 1);
|
||||
const limit = Math.min(Math.max(Number(kwargs.limit || 20), 1), 50);
|
||||
const filter = String(kwargs.filter || '').trim().toLowerCase();
|
||||
if (!query) throw new Error('Query is required');
|
||||
|
||||
const initialPage = filter ? 1 : pageNumber;
|
||||
const url = `https://www.coupang.com/np/search?q=${encodeURIComponent(query)}&channel=user&page=${initialPage}`;
|
||||
await page.goto(url);
|
||||
await page.wait(3);
|
||||
if (filter) {
|
||||
const filterResult = await page.evaluate(buildApplyFilterEvaluate(filter));
|
||||
if (!filterResult?.ok) {
|
||||
throw new Error(`Unsupported or unavailable filter: ${filter}`);
|
||||
}
|
||||
await page.wait(3);
|
||||
if (pageNumber > 1) {
|
||||
const locationInfo = await page.evaluate(buildCurrentLocationEvaluate());
|
||||
const filteredUrl = new URL(locationInfo?.href || url);
|
||||
filteredUrl.searchParams.set('page', String(pageNumber));
|
||||
await page.goto(filteredUrl.toString());
|
||||
await page.wait(3);
|
||||
}
|
||||
}
|
||||
await page.autoScroll({ times: filter ? 3 : 2, delayMs: 1500 });
|
||||
|
||||
const raw = await page.evaluate(buildSearchEvaluate(query, limit, pageNumber));
|
||||
const loginHints = raw?.loginHints ?? {};
|
||||
const items = Array.isArray(raw?.items) ? raw.items : [];
|
||||
const domItems = Array.isArray(raw?.domItems) ? raw.domItems : [];
|
||||
const normalizedBase = sanitizeSearchItems(
|
||||
items.map((item: Record<string, unknown>, index: number) => normalizeSearchItem(item, index)),
|
||||
limit
|
||||
);
|
||||
const normalizedDom = sanitizeSearchItems(
|
||||
domItems.map((item: Record<string, unknown>, index: number) => normalizeSearchItem(item, index)),
|
||||
Math.max(limit * 6, 60)
|
||||
);
|
||||
const normalized = filter
|
||||
? sanitizeSearchItems(normalizedDom, limit)
|
||||
: mergeSearchItems(normalizedBase, normalizedDom, limit);
|
||||
|
||||
if (!normalized.length && loginHints.hasLoginLink && !loginHints.hasMyCoupang) {
|
||||
throw new Error('Coupang login required. Please log into Coupang in Chrome and retry.');
|
||||
}
|
||||
return normalized;
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Shared constants used across explore, synthesize, and pipeline modules.
|
||||
*/
|
||||
|
||||
/** URL query params that are volatile/ephemeral and should be stripped from patterns */
|
||||
export const VOLATILE_PARAMS = new Set([
|
||||
'w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign',
|
||||
]);
|
||||
|
||||
/** Search-related query parameter names */
|
||||
export const SEARCH_PARAMS = new Set([
|
||||
'q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w',
|
||||
]);
|
||||
|
||||
/** Pagination-related query parameter names */
|
||||
export const PAGINATION_PARAMS = new Set([
|
||||
'page', 'pn', 'offset', 'cursor', 'next', 'page_num',
|
||||
]);
|
||||
|
||||
/** Limit/page-size query parameter names */
|
||||
export const LIMIT_PARAMS = new Set([
|
||||
'limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num',
|
||||
]);
|
||||
|
||||
/** Field role → common API field names mapping */
|
||||
export const FIELD_ROLES: Record<string, string[]> = {
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
};
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
canonicalizeProductUrl,
|
||||
dedupeSearchItems,
|
||||
normalizeProductId,
|
||||
normalizeSearchItem,
|
||||
sanitizeSearchItems,
|
||||
} from './coupang.js';
|
||||
|
||||
describe('normalizeProductId', () => {
|
||||
it('extracts product id from canonical path', () => {
|
||||
expect(normalizeProductId('https://www.coupang.com/vp/products/123456789')).toBe('123456789');
|
||||
});
|
||||
|
||||
it('preserves numeric ids', () => {
|
||||
expect(normalizeProductId('987654321')).toBe('987654321');
|
||||
});
|
||||
});
|
||||
|
||||
describe('canonicalizeProductUrl', () => {
|
||||
it('normalizes relative Coupang paths', () => {
|
||||
expect(canonicalizeProductUrl('/vp/products/123456789?itemId=1', '')).toBe(
|
||||
'https://www.coupang.com/vp/products/123456789'
|
||||
);
|
||||
});
|
||||
|
||||
it('builds url from product id', () => {
|
||||
expect(canonicalizeProductUrl('', '123456789')).toBe('https://www.coupang.com/vp/products/123456789');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeSearchItem', () => {
|
||||
it('maps raw fields into compare-ready shape', () => {
|
||||
const item = normalizeSearchItem({
|
||||
productId: '123456789',
|
||||
productName: '무선 마우스',
|
||||
salePrice: '29,900원',
|
||||
originalPrice: '39,900원',
|
||||
rating: '4.8',
|
||||
reviewCount: '1,234',
|
||||
sellerName: '쿠팡',
|
||||
badge: ['ROCKET', 'TOMORROW', '무료배송'],
|
||||
categoryName: 'PC',
|
||||
url: '/vp/products/123456789?itemId=1',
|
||||
}, 0);
|
||||
|
||||
expect(item).toMatchObject({
|
||||
rank: 1,
|
||||
product_id: '123456789',
|
||||
title: '무선 마우스',
|
||||
price: 29900,
|
||||
original_price: 39900,
|
||||
rating: 4.8,
|
||||
review_count: 1234,
|
||||
rocket: '로켓배송',
|
||||
delivery_type: '무료배송',
|
||||
delivery_promise: '내일도착',
|
||||
seller: '쿠팡',
|
||||
category: 'PC',
|
||||
url: 'https://www.coupang.com/vp/products/123456789',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeSearchItems', () => {
|
||||
it('drops duplicates and invalid rows', () => {
|
||||
const rows = [
|
||||
normalizeSearchItem({ productId: '1', productName: 'A', price: '1000', url: '/vp/products/1' }, 0),
|
||||
normalizeSearchItem({ productId: '1', productName: 'A', price: '1000', url: '/vp/products/1' }, 1),
|
||||
normalizeSearchItem({ productId: '', productName: '', price: '1000' }, 2),
|
||||
normalizeSearchItem({ productId: '2', productName: 'B', price: '2000', url: '/vp/products/2' }, 3),
|
||||
];
|
||||
|
||||
expect(dedupeSearchItems(rows)).toHaveLength(3);
|
||||
expect(sanitizeSearchItems(rows, 10)).toHaveLength(2);
|
||||
expect(sanitizeSearchItems(rows, 10).map(item => item.rank)).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
export interface CoupangSearchItem {
|
||||
rank: number;
|
||||
product_id: string;
|
||||
title: string;
|
||||
price: number | null;
|
||||
original_price: number | null;
|
||||
unit_price: string;
|
||||
discount_rate: number | null;
|
||||
rating: number | null;
|
||||
review_count: number | null;
|
||||
rocket: string;
|
||||
delivery_type: string;
|
||||
delivery_promise: string;
|
||||
seller: string;
|
||||
badge: string;
|
||||
category: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function itemKey(item: CoupangSearchItem): string {
|
||||
return item.url || item.product_id || `${item.title}:${item.price ?? ''}`;
|
||||
}
|
||||
|
||||
const ROCKET_PATTERNS = ['판매자로켓', '로켓프레시', '로켓와우', '로켓배송', '로켓직구'] as const;
|
||||
const DELIVERY_TYPE_PATTERNS = ['무료배송', '일반배송'] as const;
|
||||
const DELIVERY_PROMISE_PATTERNS = ['오늘도착', '내일도착', '새벽도착', '오늘출발'] as const;
|
||||
|
||||
const BADGE_ID_TO_ROCKET: Record<string, string> = {
|
||||
ROCKET: '로켓배송',
|
||||
ROCKET_MERCHANT: '판매자로켓',
|
||||
ROCKET_WOW: '로켓와우',
|
||||
WOW: '로켓와우',
|
||||
ROCKET_FRESH: '로켓프레시',
|
||||
FRESH: '로켓프레시',
|
||||
SELLER_ROCKET: '판매자로켓',
|
||||
ROCKET_JIKGU: '로켓직구',
|
||||
JIKGU: '로켓직구',
|
||||
COUPANG_GLOBAL: '로켓직구',
|
||||
};
|
||||
|
||||
const BADGE_ID_TO_PROMISE: Record<string, string> = {
|
||||
DAWN: '새벽도착',
|
||||
EARLY_DAWN: '새벽도착',
|
||||
TOMORROW: '내일도착',
|
||||
TODAY: '오늘도착',
|
||||
SAME_DAY: '오늘도착',
|
||||
TODAY_SHIP: '오늘출발',
|
||||
TODAY_DISPATCH: '오늘출발',
|
||||
};
|
||||
|
||||
function asString(value: unknown): string {
|
||||
if (value == null) return '';
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function toNumber(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
const text = asString(value).replace(/[^\d.]/g, '');
|
||||
if (!text) return null;
|
||||
const num = Number(text);
|
||||
return Number.isFinite(num) ? num : null;
|
||||
}
|
||||
|
||||
function pickFirst(obj: Record<string, unknown>, paths: string[]): unknown {
|
||||
for (const path of paths) {
|
||||
const parts = path.split('.');
|
||||
let current: unknown = obj;
|
||||
let ok = true;
|
||||
for (const part of parts) {
|
||||
if (!current || typeof current !== 'object' || !(part in (current as Record<string, unknown>))) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
current = (current as Record<string, unknown>)[part];
|
||||
}
|
||||
if (ok && current != null && asString(current) !== '') return current;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function normalizeProductId(raw: unknown): string {
|
||||
const text = asString(raw);
|
||||
if (!text) return '';
|
||||
const match = text.match(/\/vp\/products\/(\d+)/) || text.match(/\b(\d{6,})\b/);
|
||||
return match?.[1] ?? text;
|
||||
}
|
||||
|
||||
export function canonicalizeProductUrl(rawUrl: unknown, productId?: unknown): string {
|
||||
const raw = asString(rawUrl);
|
||||
if (raw) {
|
||||
try {
|
||||
const url = new URL(raw.startsWith('http') ? raw : `https://www.coupang.com${raw}`);
|
||||
if (!url.hostname.includes('coupang.com')) return '';
|
||||
const id = normalizeProductId(url.pathname) || normalizeProductId(productId);
|
||||
if (!id) return url.toString();
|
||||
return `https://www.coupang.com/vp/products/${id}`;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
const id = normalizeProductId(productId);
|
||||
return id ? `https://www.coupang.com/vp/products/${id}` : '';
|
||||
}
|
||||
|
||||
function extractTokens(values: unknown[]): string[] {
|
||||
return values
|
||||
.flatMap((value) => {
|
||||
const text = asString(value);
|
||||
if (!text) return [];
|
||||
return text.split(/[,\s|]+/);
|
||||
})
|
||||
.map((token) => token.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function normalizeJoinedText(...values: unknown[]): string {
|
||||
return values
|
||||
.map(asString)
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.replace(/schema\.org\/[A-Za-z]+/gi, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeRocket(...values: unknown[]): string {
|
||||
const tokens = extractTokens(values);
|
||||
for (const token of tokens) {
|
||||
if (BADGE_ID_TO_ROCKET[token]) return BADGE_ID_TO_ROCKET[token];
|
||||
}
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/판매자\s*로켓/.test(text)) return '판매자로켓';
|
||||
if (/로켓\s*프레시|새벽\s*도착\s*보장/.test(text)) return '로켓프레시';
|
||||
if (/로켓\s*와우/.test(text)) return '로켓와우';
|
||||
if (/로켓\s*직구|직구/.test(text)) return '로켓직구';
|
||||
if (/로켓\s*배송/.test(text)) return '로켓배송';
|
||||
return ROCKET_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeDeliveryType(...values: unknown[]): string {
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/무료\s*배송/.test(text)) return '무료배송';
|
||||
if (/일반\s*배송/.test(text)) return '일반배송';
|
||||
return DELIVERY_TYPE_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeDeliveryPromise(...values: unknown[]): string {
|
||||
const tokens = extractTokens(values);
|
||||
for (const token of tokens) {
|
||||
if (BADGE_ID_TO_PROMISE[token]) return BADGE_ID_TO_PROMISE[token];
|
||||
}
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/오늘\s*출발/.test(text)) return '오늘출발';
|
||||
if (/오늘.*도착/.test(text)) return '오늘도착';
|
||||
if (/새벽.*도착/.test(text)) return '새벽도착';
|
||||
if (/내일.*도착/.test(text)) return '내일도착';
|
||||
return DELIVERY_PROMISE_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeBadge(value: unknown): string {
|
||||
const normalizeOne = (entry: unknown): string => {
|
||||
const text = asString(entry);
|
||||
if (!text) return '';
|
||||
if (/schema\.org\//i.test(text)) {
|
||||
return text.split('/').pop() ?? '';
|
||||
}
|
||||
return text;
|
||||
};
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(normalizeOne).filter(Boolean).join(', ');
|
||||
}
|
||||
return normalizeOne(value);
|
||||
}
|
||||
|
||||
export function normalizeSearchItem(raw: Record<string, unknown>, index: number): CoupangSearchItem {
|
||||
const productId = normalizeProductId(
|
||||
pickFirst(raw, ['productId', 'product_id', 'id', 'productNo', 'item.id', 'product.productId', 'url'])
|
||||
);
|
||||
const title = asString(
|
||||
pickFirst(raw, ['title', 'name', 'productName', 'productTitle', 'itemName', 'item.title'])
|
||||
);
|
||||
const price = toNumber(
|
||||
pickFirst(raw, ['price', 'salePrice', 'finalPrice', 'sellingPrice', 'discountPrice', 'item.price'])
|
||||
);
|
||||
const originalPrice = toNumber(
|
||||
pickFirst(raw, ['originalPrice', 'basePrice', 'listPrice', 'originPrice', 'strikePrice'])
|
||||
);
|
||||
const unitPrice = asString(
|
||||
pickFirst(raw, ['unitPrice', 'unit_price', 'unitPriceText'])
|
||||
);
|
||||
const rating = toNumber(
|
||||
pickFirst(raw, ['rating', 'star', 'reviewRating', 'review.rating', 'item.rating'])
|
||||
);
|
||||
const reviewCount = toNumber(
|
||||
pickFirst(raw, ['reviewCount', 'ratingCount', 'reviews', 'reviewCnt', 'item.reviewCount'])
|
||||
);
|
||||
const deliveryHintValues = [
|
||||
pickFirst(raw, ['deliveryType', 'deliveryBadge', 'badgeLabel', 'shippingType', 'shippingBadge']),
|
||||
pickFirst(raw, ['badge', 'badges', 'labels', 'benefitBadge', 'promotionBadge']),
|
||||
pickFirst(raw, ['text', 'summary']),
|
||||
pickFirst(raw, ['deliveryPromise', 'promise', 'arrivalText', 'arrivalBadge']),
|
||||
pickFirst(raw, ['rocket', 'rocketType']),
|
||||
];
|
||||
const deliveryType = normalizeDeliveryType(...deliveryHintValues);
|
||||
const deliveryPromise = normalizeDeliveryPromise(...deliveryHintValues);
|
||||
const rocket = normalizeRocket(...deliveryHintValues);
|
||||
const badge = normalizeBadge(
|
||||
pickFirst(raw, ['badge', 'badges', 'labels', 'benefitBadge', 'promotionBadge'])
|
||||
);
|
||||
const category = asString(
|
||||
pickFirst(raw, ['category', 'categoryName', 'categoryPath', 'item.category'])
|
||||
);
|
||||
const seller = asString(
|
||||
pickFirst(raw, ['seller', 'sellerName', 'vendorName', 'merchantName', 'item.seller'])
|
||||
);
|
||||
const url = canonicalizeProductUrl(
|
||||
pickFirst(raw, ['url', 'productUrl', 'link', 'item.url']),
|
||||
productId
|
||||
);
|
||||
const discountRate = toNumber(
|
||||
pickFirst(raw, ['discountRate', 'discount', 'discountPercent', 'discount_rate'])
|
||||
);
|
||||
|
||||
return {
|
||||
rank: index + 1,
|
||||
product_id: productId,
|
||||
title,
|
||||
price,
|
||||
original_price: originalPrice,
|
||||
unit_price: unitPrice,
|
||||
discount_rate: discountRate,
|
||||
rating,
|
||||
review_count: reviewCount,
|
||||
rocket,
|
||||
delivery_type: deliveryType,
|
||||
delivery_promise: deliveryPromise,
|
||||
seller,
|
||||
badge,
|
||||
category,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
export function dedupeSearchItems(items: CoupangSearchItem[]): CoupangSearchItem[] {
|
||||
const seen = new Set<string>();
|
||||
const out: CoupangSearchItem[] = [];
|
||||
for (const item of items) {
|
||||
const key = itemKey(item);
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push({ ...item, rank: out.length + 1 });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function sanitizeSearchItems(items: CoupangSearchItem[], limit: number): CoupangSearchItem[] {
|
||||
return dedupeSearchItems(
|
||||
items.filter(item => Boolean(item.title && (item.product_id || item.url)))
|
||||
).slice(0, limit);
|
||||
}
|
||||
|
||||
export function mergeSearchItems(base: CoupangSearchItem[], extra: CoupangSearchItem[], limit: number): CoupangSearchItem[] {
|
||||
const extraMap = new Map<string, CoupangSearchItem>();
|
||||
for (const item of extra) {
|
||||
const key = itemKey(item);
|
||||
if (key) extraMap.set(key, item);
|
||||
}
|
||||
|
||||
const merged = base.map((item) => {
|
||||
const key = itemKey(item);
|
||||
const patch = key ? extraMap.get(key) : null;
|
||||
if (!patch) return item;
|
||||
return {
|
||||
...item,
|
||||
price: patch.price ?? item.price,
|
||||
original_price: patch.original_price ?? item.original_price,
|
||||
unit_price: patch.unit_price || item.unit_price,
|
||||
discount_rate: patch.discount_rate ?? item.discount_rate,
|
||||
rating: patch.rating ?? item.rating,
|
||||
review_count: patch.review_count ?? item.review_count,
|
||||
rocket: patch.rocket || item.rocket,
|
||||
delivery_type: patch.delivery_type || item.delivery_type,
|
||||
delivery_promise: patch.delivery_promise || item.delivery_promise,
|
||||
seller: patch.seller || item.seller,
|
||||
badge: patch.badge || item.badge,
|
||||
category: patch.category || item.category,
|
||||
url: patch.url || item.url,
|
||||
};
|
||||
});
|
||||
|
||||
const mergedKeys = new Set(merged.map(item => itemKey(item)).filter(Boolean));
|
||||
const appended = extra.filter(item => {
|
||||
const key = itemKey(item);
|
||||
return key && !mergedKeys.has(key);
|
||||
});
|
||||
|
||||
return sanitizeSearchItems([...merged, ...appended], limit);
|
||||
}
|
||||
@@ -90,6 +90,8 @@ describe('doctor report rendering', () => {
|
||||
const text = renderBrowserDoctorReport({
|
||||
envToken: 'abc123',
|
||||
envFingerprint: 'fp1',
|
||||
extensionToken: 'abc123',
|
||||
extensionFingerprint: 'fp1',
|
||||
shellFiles: [{ path: '/tmp/.zshrc', exists: true, token: 'abc123', fingerprint: 'fp1' }],
|
||||
configs: [{ path: '/tmp/mcp.json', exists: true, format: 'json', token: 'abc123', fingerprint: 'fp1', writable: true }],
|
||||
recommendedToken: 'abc123',
|
||||
@@ -106,6 +108,8 @@ describe('doctor report rendering', () => {
|
||||
const text = renderBrowserDoctorReport({
|
||||
envToken: 'abc123',
|
||||
envFingerprint: 'fp1',
|
||||
extensionToken: null,
|
||||
extensionFingerprint: null,
|
||||
shellFiles: [{ path: '/tmp/.zshrc', exists: true, token: 'def456', fingerprint: 'fp2' }],
|
||||
configs: [{ path: '/tmp/mcp.json', exists: true, format: 'json', token: 'abc123', fingerprint: 'fp1', writable: true }],
|
||||
recommendedToken: 'abc123',
|
||||
|
||||
+157
-1
@@ -1,6 +1,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import type { IPage } from './types.js';
|
||||
@@ -43,6 +44,8 @@ export type DoctorReport = {
|
||||
cliVersion?: string;
|
||||
envToken: string | null;
|
||||
envFingerprint: string | null;
|
||||
extensionToken: string | null;
|
||||
extensionFingerprint: string | null;
|
||||
shellFiles: ShellFileStatus[];
|
||||
configs: McpConfigStatus[];
|
||||
recommendedToken: string | null;
|
||||
@@ -79,12 +82,16 @@ export function getDefaultMcpConfigPaths(cwd: string = process.cwd()): string[]
|
||||
path.join(home, '.codex', 'config.toml'),
|
||||
path.join(home, '.codex', 'mcp.json'),
|
||||
path.join(home, '.cursor', 'mcp.json'),
|
||||
path.join(home, '.claude.json'),
|
||||
path.join(home, '.gemini', 'settings.json'),
|
||||
path.join(home, '.gemini', 'antigravity', 'mcp_config.json'),
|
||||
path.join(home, '.config', 'opencode', 'opencode.json'),
|
||||
path.join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json'),
|
||||
path.join(home, '.config', 'Claude', 'claude_desktop_config.json'),
|
||||
path.join(cwd, '.cursor', 'mcp.json'),
|
||||
path.join(cwd, '.vscode', 'mcp.json'),
|
||||
path.join(cwd, '.opencode', 'opencode.json'),
|
||||
path.join(cwd, '.mcp.json'),
|
||||
];
|
||||
return [...new Set(candidates)];
|
||||
}
|
||||
@@ -220,6 +227,145 @@ function readConfigStatus(filePath: string): McpConfigStatus {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the auth token stored by the Playwright MCP Bridge extension
|
||||
* by scanning Chrome's LevelDB localStorage files directly.
|
||||
*
|
||||
* Uses `strings` + `grep` for fast binary scanning on macOS/Linux,
|
||||
* with a pure-Node fallback on Windows.
|
||||
*/
|
||||
export function discoverExtensionToken(): string | null {
|
||||
const home = os.homedir();
|
||||
const platform = os.platform();
|
||||
const bases: string[] = [];
|
||||
|
||||
if (platform === 'darwin') {
|
||||
bases.push(
|
||||
path.join(home, 'Library', 'Application Support', 'Google', 'Chrome'),
|
||||
path.join(home, 'Library', 'Application Support', 'Google', 'Chrome Canary'),
|
||||
path.join(home, 'Library', 'Application Support', 'Chromium'),
|
||||
path.join(home, 'Library', 'Application Support', 'Microsoft Edge'),
|
||||
);
|
||||
} else if (platform === 'linux') {
|
||||
bases.push(
|
||||
path.join(home, '.config', 'google-chrome'),
|
||||
path.join(home, '.config', 'chromium'),
|
||||
path.join(home, '.config', 'microsoft-edge'),
|
||||
);
|
||||
} else if (platform === 'win32') {
|
||||
const appData = process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local');
|
||||
bases.push(
|
||||
path.join(appData, 'Google', 'Chrome', 'User Data'),
|
||||
path.join(appData, 'Microsoft', 'Edge', 'User Data'),
|
||||
);
|
||||
}
|
||||
|
||||
const profiles = ['Default', 'Profile 1', 'Profile 2', 'Profile 3'];
|
||||
// Token is 43 chars of base64url (from 32 random bytes)
|
||||
const tokenRe = /([A-Za-z0-9_-]{40,50})/;
|
||||
|
||||
for (const base of bases) {
|
||||
for (const profile of profiles) {
|
||||
const dir = path.join(base, profile, 'Local Storage', 'leveldb');
|
||||
if (!fileExists(dir)) continue;
|
||||
|
||||
// Fast path: use strings + grep to find candidate files and extract token
|
||||
if (platform !== 'win32') {
|
||||
const token = extractTokenViaStrings(dir, tokenRe);
|
||||
if (token) return token;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Slow path (Windows): read binary files directly
|
||||
const token = extractTokenViaBinaryRead(dir, tokenRe);
|
||||
if (token) return token;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractTokenViaStrings(dir: string, tokenRe: RegExp): string | null {
|
||||
try {
|
||||
// Single shell pipeline: for each LevelDB file, extract strings, find lines
|
||||
// after the extension ID, and filter for base64url token pattern.
|
||||
//
|
||||
// LevelDB `strings` output for the extension's auth-token entry:
|
||||
// auth-token ← key name
|
||||
// 4,mmlmfjhmonkocbjadbfplnigmagldckm.7 ← LevelDB internal key
|
||||
// hqI86ncsD1QpcVcj-k9CyzTF-ieCQd_4KreZ_wy1WHA ← token value
|
||||
//
|
||||
// We get the line immediately after any EXTENSION_ID mention and check
|
||||
// if it looks like a base64url token (40-50 chars, [A-Za-z0-9_-]).
|
||||
const shellDir = dir.replace(/'/g, "'\\''");
|
||||
const cmd = `for f in '${shellDir}'/*.ldb '${shellDir}'/*.log; do ` +
|
||||
`[ -f "$f" ] && strings "$f" 2>/dev/null | ` +
|
||||
`grep -A1 '${PLAYWRIGHT_EXTENSION_ID}' | ` +
|
||||
`grep -v '${PLAYWRIGHT_EXTENSION_ID}' | ` +
|
||||
`grep -E '^[A-Za-z0-9_-]{40,50}$' | head -1; ` +
|
||||
`done 2>/dev/null`;
|
||||
const result = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim();
|
||||
|
||||
// Take the first non-empty line
|
||||
for (const line of result.split('\n')) {
|
||||
const token = line.trim();
|
||||
if (token && validateBase64urlToken(token)) return token;
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractTokenViaBinaryRead(dir: string, tokenRe: RegExp): string | null {
|
||||
const extIdBuf = Buffer.from(PLAYWRIGHT_EXTENSION_ID);
|
||||
const keyBuf = Buffer.from('auth-token');
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(dir)
|
||||
.filter(f => f.endsWith('.ldb') || f.endsWith('.log'))
|
||||
.map(f => path.join(dir, f));
|
||||
} catch { return null; }
|
||||
|
||||
// Sort by mtime descending
|
||||
files.sort((a, b) => {
|
||||
try { return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs; } catch { return 0; }
|
||||
});
|
||||
|
||||
for (const file of files) {
|
||||
let data: Buffer;
|
||||
try { data = fs.readFileSync(file); } catch { continue; }
|
||||
|
||||
// Quick check: does file contain both the extension ID and auth-token key?
|
||||
const extPos = data.indexOf(extIdBuf);
|
||||
if (extPos === -1) continue;
|
||||
const keyPos = data.indexOf(keyBuf, Math.max(0, extPos - 500));
|
||||
if (keyPos === -1) continue;
|
||||
|
||||
// Scan for token value after auth-token key
|
||||
let idx = 0;
|
||||
while (true) {
|
||||
const kp = data.indexOf(keyBuf, idx);
|
||||
if (kp === -1) break;
|
||||
|
||||
const contextStart = Math.max(0, kp - 500);
|
||||
if (data.indexOf(extIdBuf, contextStart) !== -1 && data.indexOf(extIdBuf, contextStart) < kp) {
|
||||
const after = data.subarray(kp + keyBuf.length, kp + keyBuf.length + 200).toString('latin1');
|
||||
const m = after.match(tokenRe);
|
||||
if (m && validateBase64urlToken(m[1])) return m[1];
|
||||
}
|
||||
idx = kp + 1;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function validateBase64urlToken(token: string): boolean {
|
||||
try {
|
||||
const b64 = token.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const decoded = Buffer.from(b64, 'base64');
|
||||
return decoded.length >= 28 && decoded.length <= 36;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
|
||||
export async function runBrowserDoctor(opts: DoctorOptions = {}): Promise<DoctorReport> {
|
||||
@@ -234,19 +380,25 @@ export async function runBrowserDoctor(opts: DoctorOptions = {}): Promise<Doctor
|
||||
const configPaths = opts.configPaths?.length ? opts.configPaths : getDefaultMcpConfigPaths();
|
||||
const configs = configPaths.map(readConfigStatus);
|
||||
|
||||
// Try to discover the token directly from the Chrome extension's localStorage
|
||||
const extensionToken = discoverExtensionToken();
|
||||
|
||||
const allTokens = [
|
||||
opts.token ?? null,
|
||||
extensionToken,
|
||||
envToken,
|
||||
...shellFiles.map(s => s.token),
|
||||
...configs.map(c => c.token),
|
||||
].filter((v): v is string => !!v);
|
||||
const uniqueTokens = [...new Set(allTokens)];
|
||||
const recommendedToken = opts.token ?? envToken ?? (uniqueTokens.length === 1 ? uniqueTokens[0] : null) ?? null;
|
||||
const recommendedToken = opts.token ?? extensionToken ?? envToken ?? (uniqueTokens.length === 1 ? uniqueTokens[0] : null) ?? null;
|
||||
|
||||
const report: DoctorReport = {
|
||||
cliVersion: opts.cliVersion,
|
||||
envToken,
|
||||
envFingerprint: getTokenFingerprint(envToken ?? undefined),
|
||||
extensionToken,
|
||||
extensionFingerprint: getTokenFingerprint(extensionToken ?? undefined),
|
||||
shellFiles,
|
||||
configs,
|
||||
recommendedToken,
|
||||
@@ -270,6 +422,7 @@ export async function runBrowserDoctor(opts: DoctorOptions = {}): Promise<Doctor
|
||||
|
||||
export function renderBrowserDoctorReport(report: DoctorReport): string {
|
||||
const tokenFingerprints = [
|
||||
report.extensionFingerprint,
|
||||
report.envFingerprint,
|
||||
...report.shellFiles.map(shell => shell.fingerprint),
|
||||
...report.configs.filter(config => config.exists).map(config => config.fingerprint),
|
||||
@@ -278,6 +431,9 @@ export function renderBrowserDoctorReport(report: DoctorReport): string {
|
||||
const hasMismatch = uniqueFingerprints.length > 1;
|
||||
const lines = [`opencli v${report.cliVersion ?? 'unknown'} doctor`, ''];
|
||||
|
||||
const extStatus: ReportStatus = !report.extensionToken ? 'MISSING' : hasMismatch ? 'MISMATCH' : 'OK';
|
||||
lines.push(statusLine(extStatus, `Extension token (Chrome LevelDB): ${tokenSummary(report.extensionToken, report.extensionFingerprint)}`));
|
||||
|
||||
const envStatus: ReportStatus = !report.envToken ? 'MISSING' : hasMismatch ? 'MISMATCH' : 'OK';
|
||||
lines.push(statusLine(envStatus, `Environment token: ${tokenSummary(report.envToken, report.envFingerprint)}`));
|
||||
|
||||
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Tests for engine.ts: CLI discovery and command execution.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { discoverClis, executeCommand } from './engine.js';
|
||||
import { getRegistry, cli, Strategy } from './registry.js';
|
||||
|
||||
describe('discoverClis', () => {
|
||||
it('handles non-existent directories gracefully', async () => {
|
||||
// Should not throw for missing directories
|
||||
await expect(discoverClis('/tmp/nonexistent-opencli-test-dir')).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('executeCommand', () => {
|
||||
it('executes a command with func', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'func-test',
|
||||
description: 'test command with func',
|
||||
browser: false,
|
||||
strategy: Strategy.PUBLIC,
|
||||
func: async (_page, kwargs) => {
|
||||
return [{ title: kwargs.query ?? 'default' }];
|
||||
},
|
||||
});
|
||||
|
||||
const result = await executeCommand(cmd, null, { query: 'hello' });
|
||||
expect(result).toEqual([{ title: 'hello' }]);
|
||||
});
|
||||
|
||||
it('executes a command with pipeline', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'pipe-test',
|
||||
description: 'test command with pipeline',
|
||||
browser: false,
|
||||
strategy: Strategy.PUBLIC,
|
||||
pipeline: [
|
||||
{ evaluate: '() => [{ n: 1 }, { n: 2 }, { n: 3 }]' },
|
||||
{ limit: '2' },
|
||||
],
|
||||
});
|
||||
|
||||
// Pipeline commands require page for evaluate step, so we'll test the error path
|
||||
await expect(executeCommand(cmd, null, {})).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('throws for command with no func or pipeline', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'empty-test',
|
||||
description: 'empty command',
|
||||
browser: false,
|
||||
});
|
||||
|
||||
await expect(executeCommand(cmd, null, {})).rejects.toThrow('has no func or pipeline');
|
||||
});
|
||||
|
||||
it('passes debug flag to func', async () => {
|
||||
let receivedDebug = false;
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'debug-test',
|
||||
description: 'debug test',
|
||||
browser: false,
|
||||
func: async (_page, _kwargs, debug) => {
|
||||
receivedDebug = debug ?? false;
|
||||
return [];
|
||||
},
|
||||
});
|
||||
|
||||
await executeCommand(cmd, null, {}, true);
|
||||
expect(receivedDebug).toBe(true);
|
||||
});
|
||||
});
|
||||
+5
-5
@@ -11,7 +11,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { type CliCommand, type Arg, Strategy, registerCommand } from './registry.js';
|
||||
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
|
||||
import type { IPage } from './types.js';
|
||||
import { executePipeline } from './pipeline.js';
|
||||
|
||||
@@ -66,7 +66,7 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
|
||||
// The actual module is loaded lazily on first executeCommand().
|
||||
const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
|
||||
const modulePath = path.resolve(clisDir, entry.modulePath);
|
||||
const cmd: CliCommand = {
|
||||
const cmd: InternalCliCommand = {
|
||||
site: entry.site,
|
||||
name: entry.name,
|
||||
description: entry.description ?? '',
|
||||
@@ -77,7 +77,6 @@ function loadFromManifest(manifestPath: string, clisDir: string): void {
|
||||
columns: entry.columns,
|
||||
timeoutSeconds: entry.timeout,
|
||||
source: modulePath,
|
||||
// Mark as lazy — executeCommand will load the module before running
|
||||
_lazy: true,
|
||||
_modulePath: modulePath,
|
||||
};
|
||||
@@ -170,8 +169,9 @@ export async function executeCommand(
|
||||
debug: boolean = false,
|
||||
): Promise<any> {
|
||||
// Lazy-load TS module on first execution
|
||||
if ((cmd as any)._lazy && (cmd as any)._modulePath) {
|
||||
const modulePath = (cmd as any)._modulePath;
|
||||
const internal = cmd as InternalCliCommand;
|
||||
if (internal._lazy && internal._modulePath) {
|
||||
const modulePath = internal._modulePath;
|
||||
if (!_loadedModules.has(modulePath)) {
|
||||
try {
|
||||
await import(`file://${modulePath}`);
|
||||
|
||||
+2
-15
@@ -9,6 +9,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
||||
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
|
||||
|
||||
// ── Site name detection ────────────────────────────────────────────────────
|
||||
|
||||
@@ -43,21 +44,7 @@ export function slugify(value: string): string {
|
||||
|
||||
// ── Field & capability inference ───────────────────────────────────────────
|
||||
|
||||
const FIELD_ROLES: Record<string, string[]> = {
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
};
|
||||
|
||||
const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']);
|
||||
const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']);
|
||||
const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']);
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']);
|
||||
// (constants now imported from constants.ts)
|
||||
|
||||
// ── Network analysis ───────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Shared XHR/Fetch interceptor JavaScript generators.
|
||||
*
|
||||
* Provides a single source of truth for monkey-patching browser
|
||||
* fetch() and XMLHttpRequest to capture API responses matching
|
||||
* a URL pattern. Used by:
|
||||
* - Page.installInterceptor() (browser.ts)
|
||||
* - stepIntercept (pipeline/steps/intercept.ts)
|
||||
* - stepTap (pipeline/steps/tap.ts)
|
||||
*/
|
||||
|
||||
/**
|
||||
* Generate JavaScript source that installs a fetch/XHR interceptor.
|
||||
* Captured responses are pushed to `window.__opencli_intercepted`.
|
||||
*
|
||||
* @param patternExpr - JS expression resolving to a URL substring to match (e.g. a JSON.stringify'd string)
|
||||
* @param opts.arrayName - Global array name for captured data (default: '__opencli_intercepted')
|
||||
* @param opts.patchGuard - Global boolean name to prevent double-patching (default: '__opencli_interceptor_patched')
|
||||
*/
|
||||
export function generateInterceptorJs(
|
||||
patternExpr: string,
|
||||
opts: { arrayName?: string; patchGuard?: string } = {},
|
||||
): string {
|
||||
const arr = opts.arrayName ?? '__opencli_intercepted';
|
||||
const guard = opts.patchGuard ?? '__opencli_interceptor_patched';
|
||||
|
||||
return `
|
||||
() => {
|
||||
window.${arr} = window.${arr} || [];
|
||||
const __pattern = ${patternExpr};
|
||||
|
||||
if (!window.${guard}) {
|
||||
const __checkMatch = (url) => __pattern && url.includes(__pattern);
|
||||
|
||||
// ── Patch fetch ──
|
||||
const __origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
const reqUrl = typeof args[0] === 'string' ? args[0]
|
||||
: (args[0] && args[0].url) || '';
|
||||
const response = await __origFetch.apply(this, args);
|
||||
if (__checkMatch(reqUrl)) {
|
||||
try {
|
||||
const clone = response.clone();
|
||||
const json = await clone.json();
|
||||
window.${arr}.push(json);
|
||||
} catch(e) {}
|
||||
}
|
||||
return response;
|
||||
};
|
||||
|
||||
// ── Patch XMLHttpRequest ──
|
||||
const __XHR = XMLHttpRequest.prototype;
|
||||
const __origOpen = __XHR.open;
|
||||
const __origSend = __XHR.send;
|
||||
__XHR.open = function(method, url) {
|
||||
this.__opencli_url = String(url);
|
||||
return __origOpen.apply(this, arguments);
|
||||
};
|
||||
__XHR.send = function() {
|
||||
if (__checkMatch(this.__opencli_url)) {
|
||||
this.addEventListener('load', function() {
|
||||
try {
|
||||
window.${arr}.push(JSON.parse(this.responseText));
|
||||
} catch(e) {}
|
||||
});
|
||||
}
|
||||
return __origSend.apply(this, arguments);
|
||||
};
|
||||
|
||||
window.${guard} = true;
|
||||
}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JavaScript source to read and clear intercepted data.
|
||||
*/
|
||||
export function generateReadInterceptedJs(arrayName: string = '__opencli_intercepted'): string {
|
||||
return `
|
||||
() => {
|
||||
const data = window.${arrayName} || [];
|
||||
window.${arrayName} = [];
|
||||
return data;
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a self-contained tap interceptor for store-action bridge.
|
||||
* Unlike the global interceptor, this one:
|
||||
* - Installs temporarily, restores originals in finally block
|
||||
* - Resolves a promise on first capture (for immediate await)
|
||||
* - Returns captured data directly
|
||||
*/
|
||||
export function generateTapInterceptorJs(patternExpr: string): {
|
||||
setupVar: string;
|
||||
capturedVar: string;
|
||||
promiseVar: string;
|
||||
resolveVar: string;
|
||||
fetchPatch: string;
|
||||
xhrPatch: string;
|
||||
restorePatch: string;
|
||||
} {
|
||||
return {
|
||||
setupVar: `
|
||||
let captured = null;
|
||||
let captureResolve;
|
||||
const capturePromise = new Promise(r => { captureResolve = r; });
|
||||
const capturePattern = ${patternExpr};
|
||||
`,
|
||||
capturedVar: 'captured',
|
||||
promiseVar: 'capturePromise',
|
||||
resolveVar: 'captureResolve',
|
||||
fetchPatch: `
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...fetchArgs) {
|
||||
const resp = await origFetch.apply(this, fetchArgs);
|
||||
try {
|
||||
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
||||
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
||||
if (capturePattern && url.includes(capturePattern) && !captured) {
|
||||
try { captured = await resp.clone().json(); captureResolve(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
return resp;
|
||||
};
|
||||
`,
|
||||
xhrPatch: `
|
||||
const origXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const origXhrSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, url) {
|
||||
this.__tapUrl = String(url);
|
||||
return origXhrOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(body) {
|
||||
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
||||
this.addEventListener('load', function() {
|
||||
if (!captured) {
|
||||
try { captured = JSON.parse(this.responseText); captureResolve(); } catch {}
|
||||
}
|
||||
});
|
||||
}
|
||||
return origXhrSend.apply(this, arguments);
|
||||
};
|
||||
`,
|
||||
restorePatch: `
|
||||
window.fetch = origFetch;
|
||||
XMLHttpRequest.prototype.open = origXhrOpen;
|
||||
XMLHttpRequest.prototype.send = origXhrSend;
|
||||
`,
|
||||
};
|
||||
}
|
||||
+9
-5
@@ -3,7 +3,6 @@
|
||||
* opencli — Make any website your CLI. AI-powered.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -14,16 +13,13 @@ import { type CliCommand, fullName, getRegistry, strategyLabel } from './registr
|
||||
import { render as renderOutput } from './output.js';
|
||||
import { PlaywrightMCP } from './browser.js';
|
||||
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
||||
import { PKG_VERSION } from './version.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
|
||||
const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
||||
|
||||
// Read version from package.json (single source of truth)
|
||||
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
||||
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
||||
|
||||
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
||||
|
||||
const program = new Command();
|
||||
@@ -115,6 +111,14 @@ program.command('doctor')
|
||||
}
|
||||
});
|
||||
|
||||
program.command('setup')
|
||||
.description('Interactive setup: configure Playwright MCP token across all detected tools')
|
||||
.option('--token <token>', 'Provide token directly instead of auto-detecting')
|
||||
.action(async (opts) => {
|
||||
const { runSetup } = await import('./setup.js');
|
||||
await runSetup({ cliVersion: PKG_VERSION, token: opts.token });
|
||||
});
|
||||
|
||||
// ── Dynamic site commands ──────────────────────────────────────────────────
|
||||
|
||||
const registry = getRegistry();
|
||||
|
||||
@@ -40,10 +40,6 @@ function renderTable(data: any, opts: RenderOptions): void {
|
||||
style: { head: [], border: [] },
|
||||
wordWrap: true,
|
||||
wrapOnWordBoundary: true,
|
||||
colWidths: columns.map((_c, i) => {
|
||||
if (i === 0) return 6;
|
||||
return null as any;
|
||||
}).filter(() => true),
|
||||
});
|
||||
|
||||
for (const row of rows) {
|
||||
|
||||
+15
-15
@@ -15,26 +15,26 @@ export interface PipelineContext {
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
/** Step handler signature */
|
||||
/** Step handler: all steps conform to (page, params, data, args) => Promise<any> */
|
||||
type StepHandler = (page: IPage | null, params: any, data: any, args: Record<string, any>) => Promise<any>;
|
||||
|
||||
/** Registry of all available step handlers */
|
||||
const STEP_HANDLERS: Record<string, StepHandler> = {
|
||||
navigate: stepNavigate as StepHandler,
|
||||
navigate: stepNavigate,
|
||||
fetch: stepFetch,
|
||||
select: stepSelect as StepHandler,
|
||||
evaluate: stepEvaluate as StepHandler,
|
||||
snapshot: stepSnapshot as StepHandler,
|
||||
click: stepClick as StepHandler,
|
||||
type: stepType as StepHandler,
|
||||
wait: stepWait as StepHandler,
|
||||
press: stepPress as StepHandler,
|
||||
map: stepMap as StepHandler,
|
||||
filter: stepFilter as StepHandler,
|
||||
sort: stepSort as StepHandler,
|
||||
limit: stepLimit as StepHandler,
|
||||
intercept: stepIntercept as StepHandler,
|
||||
tap: stepTap as StepHandler,
|
||||
select: stepSelect,
|
||||
evaluate: stepEvaluate,
|
||||
snapshot: stepSnapshot,
|
||||
click: stepClick,
|
||||
type: stepType,
|
||||
wait: stepWait,
|
||||
press: stepPress,
|
||||
map: stepMap,
|
||||
filter: stepFilter,
|
||||
sort: stepSort,
|
||||
limit: stepLimit,
|
||||
intercept: stepIntercept,
|
||||
tap: stepTap,
|
||||
};
|
||||
|
||||
export async function executePipeline(
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import type { IPage } from '../../types.js';
|
||||
import { render } from '../template.js';
|
||||
import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
|
||||
|
||||
export async function stepIntercept(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
|
||||
const cfg = typeof params === 'object' ? params : {};
|
||||
@@ -15,52 +16,7 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
|
||||
if (!capturePattern) return data;
|
||||
|
||||
// Step 1: Inject fetch/XHR interceptor BEFORE trigger
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__opencli_intercepted = window.__opencli_intercepted || [];
|
||||
const pattern = ${JSON.stringify(capturePattern)};
|
||||
|
||||
if (!window.__opencli_fetch_patched) {
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
const reqUrl = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const response = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (reqUrl.includes(pattern)) {
|
||||
const clone = response.clone();
|
||||
const json = await clone.json();
|
||||
window.__opencli_intercepted.push(json);
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return response;
|
||||
};
|
||||
window.__opencli_fetch_patched = true;
|
||||
}
|
||||
|
||||
if (!window.__opencli_xhr_patched) {
|
||||
const XHR = XMLHttpRequest.prototype;
|
||||
const open = XHR.open;
|
||||
const send = XHR.send;
|
||||
XHR.open = function(method, url, ...args) {
|
||||
this._reqUrl = url;
|
||||
return open.call(this, method, url, ...args);
|
||||
};
|
||||
XHR.send = function(...args) {
|
||||
this.addEventListener('load', function() {
|
||||
try {
|
||||
if (this._reqUrl && this._reqUrl.includes(pattern)) {
|
||||
window.__opencli_intercepted.push(JSON.parse(this.responseText));
|
||||
}
|
||||
} catch(e) {}
|
||||
});
|
||||
return send.apply(this, args);
|
||||
};
|
||||
window.__opencli_xhr_patched = true;
|
||||
}
|
||||
}
|
||||
`);
|
||||
await page.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
|
||||
|
||||
// Step 2: Execute the trigger action
|
||||
if (trigger.startsWith('navigate:')) {
|
||||
@@ -81,16 +37,9 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
|
||||
await page.wait(Math.min(timeout, 3));
|
||||
|
||||
// Step 4: Retrieve captured data
|
||||
const matchingResponses = await page.evaluate(`
|
||||
() => {
|
||||
const data = window.__opencli_intercepted || [];
|
||||
window.__opencli_intercepted = []; // clear after reading
|
||||
return data;
|
||||
}
|
||||
`);
|
||||
const matchingResponses = await page.evaluate(generateReadInterceptedJs());
|
||||
|
||||
|
||||
// Step 4: Select from response if specified
|
||||
// Step 5: Select from response if specified
|
||||
let result = matchingResponses.length === 1 ? matchingResponses[0] :
|
||||
matchingResponses.length > 1 ? matchingResponses : data;
|
||||
|
||||
|
||||
+12
-51
@@ -11,6 +11,7 @@
|
||||
|
||||
import type { IPage } from '../../types.js';
|
||||
import { render } from '../template.js';
|
||||
import { generateTapInterceptorJs } from '../../interceptor.js';
|
||||
|
||||
export async function stepTap(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
|
||||
const cfg = typeof params === 'object' ? params : {};
|
||||
@@ -38,53 +39,15 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
|
||||
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
|
||||
: `store[${JSON.stringify(actionName)}]()`;
|
||||
|
||||
// Use shared interceptor generator for fetch/XHR patching
|
||||
const tap = generateTapInterceptorJs(JSON.stringify(capturePattern));
|
||||
|
||||
const js = `
|
||||
async () => {
|
||||
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
|
||||
let captured = null;
|
||||
let captureResolve;
|
||||
const capturePromise = new Promise(r => { captureResolve = r; });
|
||||
const capturePattern = ${JSON.stringify(capturePattern)};
|
||||
|
||||
// Intercept fetch API
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...fetchArgs) {
|
||||
const resp = await origFetch.apply(this, fetchArgs);
|
||||
try {
|
||||
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
||||
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
||||
if (capturePattern && url.includes(capturePattern) && !captured) {
|
||||
try { captured = await resp.clone().json(); captureResolve(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
return resp;
|
||||
};
|
||||
|
||||
// Intercept XMLHttpRequest
|
||||
const origXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const origXhrSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, url) {
|
||||
this.__tapUrl = String(url);
|
||||
return origXhrOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(body) {
|
||||
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
||||
const xhr = this;
|
||||
const origHandler = xhr.onreadystatechange;
|
||||
xhr.onreadystatechange = function() {
|
||||
if (xhr.readyState === 4 && !captured) {
|
||||
try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {}
|
||||
}
|
||||
if (origHandler) origHandler.apply(this, arguments);
|
||||
};
|
||||
const origOnload = xhr.onload;
|
||||
xhr.onload = function() {
|
||||
if (!captured) { try { captured = JSON.parse(xhr.responseText); captureResolve(); } catch {} }
|
||||
if (origOnload) origOnload.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
return origXhrSend.apply(this, arguments);
|
||||
};
|
||||
${tap.setupVar}
|
||||
${tap.fetchPatch}
|
||||
${tap.xhrPatch}
|
||||
|
||||
try {
|
||||
// ── 2. Find store ──
|
||||
@@ -119,19 +82,17 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
|
||||
await ${actionCall};
|
||||
|
||||
// ── 4. Wait for network response ──
|
||||
if (!captured) {
|
||||
if (!${tap.capturedVar}) {
|
||||
const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000));
|
||||
await Promise.race([capturePromise, timeoutPromise]);
|
||||
await Promise.race([${tap.promiseVar}, timeoutPromise]);
|
||||
}
|
||||
} finally {
|
||||
// ── 5. Always restore originals ──
|
||||
window.fetch = origFetch;
|
||||
XMLHttpRequest.prototype.open = origXhrOpen;
|
||||
XMLHttpRequest.prototype.send = origXhrSend;
|
||||
${tap.restorePatch}
|
||||
}
|
||||
|
||||
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
||||
return captured${selectChain} ?? captured;
|
||||
if (!${tap.capturedVar}) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
||||
return ${tap.capturedVar}${selectChain} ?? ${tap.capturedVar};
|
||||
}
|
||||
`;
|
||||
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Tests for registry.ts: Strategy enum, cli() registration, helpers.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { cli, getRegistry, fullName, strategyLabel, registerCommand, Strategy, type CliCommand } from './registry.js';
|
||||
|
||||
describe('cli() registration', () => {
|
||||
it('registers a command and returns it', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'hello',
|
||||
description: 'A test command',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: false,
|
||||
});
|
||||
|
||||
expect(cmd.site).toBe('test-registry');
|
||||
expect(cmd.name).toBe('hello');
|
||||
expect(cmd.strategy).toBe(Strategy.PUBLIC);
|
||||
expect(cmd.browser).toBe(false);
|
||||
expect(cmd.args).toEqual([]);
|
||||
});
|
||||
|
||||
it('puts registered command in the registry', () => {
|
||||
cli({
|
||||
site: 'test-registry',
|
||||
name: 'registered',
|
||||
description: 'test',
|
||||
});
|
||||
|
||||
const registry = getRegistry();
|
||||
expect(registry.has('test-registry/registered')).toBe(true);
|
||||
});
|
||||
|
||||
it('defaults strategy to COOKIE when browser is true', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'default-strategy',
|
||||
});
|
||||
|
||||
expect(cmd.strategy).toBe(Strategy.COOKIE);
|
||||
expect(cmd.browser).toBe(true);
|
||||
});
|
||||
|
||||
it('defaults strategy to PUBLIC when browser is false', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'no-browser',
|
||||
browser: false,
|
||||
});
|
||||
|
||||
expect(cmd.strategy).toBe(Strategy.PUBLIC);
|
||||
});
|
||||
|
||||
it('overwrites existing command on re-registration', () => {
|
||||
cli({ site: 'test-registry', name: 'overwrite', description: 'v1' });
|
||||
cli({ site: 'test-registry', name: 'overwrite', description: 'v2' });
|
||||
|
||||
const reg = getRegistry();
|
||||
expect(reg.get('test-registry/overwrite')?.description).toBe('v2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('fullName', () => {
|
||||
it('returns site/name', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'bilibili', name: 'hot', description: '', args: [],
|
||||
};
|
||||
expect(fullName(cmd)).toBe('bilibili/hot');
|
||||
});
|
||||
});
|
||||
|
||||
describe('strategyLabel', () => {
|
||||
it('returns strategy string', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test', name: 'test', description: '', args: [],
|
||||
strategy: Strategy.INTERCEPT,
|
||||
};
|
||||
expect(strategyLabel(cmd)).toBe('intercept');
|
||||
});
|
||||
|
||||
it('returns public when no strategy set', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test', name: 'test', description: '', args: [],
|
||||
};
|
||||
expect(strategyLabel(cmd)).toBe('public');
|
||||
});
|
||||
});
|
||||
|
||||
describe('registerCommand', () => {
|
||||
it('registers a pre-built command', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test-registry',
|
||||
name: 'direct-reg',
|
||||
description: 'directly registered',
|
||||
args: [],
|
||||
strategy: Strategy.HEADER,
|
||||
browser: true,
|
||||
};
|
||||
registerCommand(cmd);
|
||||
|
||||
const reg = getRegistry();
|
||||
expect(reg.get('test-registry/direct-reg')?.strategy).toBe(Strategy.HEADER);
|
||||
});
|
||||
});
|
||||
+4
-1
@@ -34,7 +34,10 @@ export interface CliCommand {
|
||||
pipeline?: any[];
|
||||
timeoutSeconds?: number;
|
||||
source?: string;
|
||||
/** Internal: lazy-loaded TS module support */
|
||||
}
|
||||
|
||||
/** Internal extension for lazy-loaded TS modules (not exposed in public API) */
|
||||
export interface InternalCliCommand extends CliCommand {
|
||||
_lazy?: boolean;
|
||||
_modulePath?: string;
|
||||
}
|
||||
|
||||
+22
-8
@@ -9,23 +9,37 @@ export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROW
|
||||
export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
|
||||
export const DEFAULT_BROWSER_SMOKE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_SMOKE_TIMEOUT ?? '60', 10);
|
||||
|
||||
/**
|
||||
* Timeout with seconds unit. Used for high-level command timeouts.
|
||||
*/
|
||||
export async function runWithTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
opts: { timeout: number; label?: string },
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
reject(new Error(`${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`));
|
||||
}, opts.timeout * 1000);
|
||||
return withTimeoutMs(promise, opts.timeout * 1000, `${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`);
|
||||
}
|
||||
|
||||
promise
|
||||
.then((result) => { clearTimeout(timer); resolve(result); })
|
||||
.catch((err) => { clearTimeout(timer); reject(err); });
|
||||
/**
|
||||
* Timeout with milliseconds unit. Used for low-level internal timeouts.
|
||||
*/
|
||||
export function withTimeoutMs<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||
promise.then(
|
||||
(value) => { clearTimeout(timer); resolve(value); },
|
||||
(error) => { clearTimeout(timer); reject(error); },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Interface for browser factory (PlaywrightMCP or test mocks) */
|
||||
export interface IBrowserFactory {
|
||||
connect(opts?: { timeout?: number }): Promise<IPage>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export async function browserSession<T>(
|
||||
BrowserFactory: new () => any,
|
||||
BrowserFactory: new () => IBrowserFactory,
|
||||
fn: (page: IPage) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const mcp = new BrowserFactory();
|
||||
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* setup.ts — Interactive Playwright MCP token setup
|
||||
*
|
||||
* Discovers the extension token, shows an interactive checkbox
|
||||
* for selecting which config files to update, and applies changes.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import chalk from 'chalk';
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import {
|
||||
type DoctorReport,
|
||||
discoverExtensionToken,
|
||||
getDefaultShellRcPath,
|
||||
runBrowserDoctor,
|
||||
upsertJsonConfigToken,
|
||||
upsertShellToken,
|
||||
upsertTomlConfigToken,
|
||||
} from './doctor.js';
|
||||
import { getTokenFingerprint } from './browser.js';
|
||||
import { type CheckboxItem, checkboxPrompt } from './tui.js';
|
||||
|
||||
const PLAYWRIGHT_TOKEN_ENV = 'PLAYWRIGHT_MCP_EXTENSION_TOKEN';
|
||||
|
||||
function fileExists(p: string): boolean {
|
||||
try { return fs.statSync(p).isFile() || fs.statSync(p).isDirectory(); } catch { return false; }
|
||||
}
|
||||
|
||||
function writeFileWithMkdir(filePath: string, content: string) {
|
||||
const dir = filePath.substring(0, filePath.lastIndexOf('/'));
|
||||
if (dir && !fileExists(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
}
|
||||
|
||||
function shortenPath(p: string): string {
|
||||
const home = process.env.HOME || process.env.USERPROFILE || '';
|
||||
return home && p.startsWith(home) ? '~' + p.slice(home.length) : p;
|
||||
}
|
||||
|
||||
function toolName(p: string): string {
|
||||
if (p.includes('.codex/')) return 'Codex';
|
||||
if (p.includes('.cursor/')) return 'Cursor';
|
||||
if (p.includes('.claude.json')) return 'Claude Code';
|
||||
if (p.includes('antigravity')) return 'Antigravity';
|
||||
if (p.includes('.gemini/settings')) return 'Gemini CLI';
|
||||
if (p.includes('opencode')) return 'OpenCode';
|
||||
if (p.includes('Claude/claude_desktop')) return 'Claude Desktop';
|
||||
if (p.includes('.vscode/')) return 'VS Code';
|
||||
if (p.includes('.mcp.json')) return 'Project MCP';
|
||||
if (p.includes('.zshrc') || p.includes('.bashrc') || p.includes('.profile')) return 'Shell';
|
||||
return '';
|
||||
}
|
||||
|
||||
export async function runSetup(opts: { cliVersion?: string; token?: string } = {}) {
|
||||
console.log();
|
||||
console.log(chalk.bold(' opencli setup') + chalk.dim(' — Playwright MCP token configuration'));
|
||||
console.log();
|
||||
|
||||
// Step 1: Discover token
|
||||
let token = opts.token ?? null;
|
||||
|
||||
if (!token) {
|
||||
const extensionToken = discoverExtensionToken();
|
||||
const envToken = process.env[PLAYWRIGHT_TOKEN_ENV] ?? null;
|
||||
|
||||
if (extensionToken && envToken && extensionToken === envToken) {
|
||||
token = extensionToken;
|
||||
console.log(` ${chalk.green('✓')} Token auto-discovered from Chrome extension`);
|
||||
console.log(` Fingerprint: ${chalk.bold(getTokenFingerprint(token) ?? 'unknown')}`);
|
||||
} else if (extensionToken) {
|
||||
token = extensionToken;
|
||||
console.log(` ${chalk.green('✓')} Token discovered from Chrome extension ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
if (envToken && envToken !== extensionToken) {
|
||||
console.log(` ${chalk.yellow('!')} Environment has different token ` +
|
||||
chalk.dim(`(${getTokenFingerprint(envToken)})`));
|
||||
}
|
||||
} else if (envToken) {
|
||||
token = envToken;
|
||||
console.log(` ${chalk.green('✓')} Token from environment variable ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
}
|
||||
} else {
|
||||
console.log(` ${chalk.green('✓')} Using provided token ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
}
|
||||
|
||||
if (!token) {
|
||||
console.log(` ${chalk.yellow('!')} No token found. Please enter it manually.`);
|
||||
console.log(chalk.dim(' (Find it in the Playwright MCP Bridge extension → Status page)'));
|
||||
console.log();
|
||||
const rl = createInterface({ input, output });
|
||||
const answer = await rl.question(' Token: ');
|
||||
rl.close();
|
||||
token = answer.trim();
|
||||
if (!token) {
|
||||
console.log(chalk.red('\n No token provided. Aborting.\n'));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fingerprint = getTokenFingerprint(token) ?? 'unknown';
|
||||
console.log();
|
||||
|
||||
// Step 2: Scan all config locations
|
||||
const report = await runBrowserDoctor({ token, cliVersion: opts.cliVersion });
|
||||
|
||||
// Step 3: Build checkbox items
|
||||
const items: CheckboxItem[] = [];
|
||||
|
||||
// Shell file
|
||||
const shellPath = report.shellFiles[0]?.path ?? getDefaultShellRcPath();
|
||||
const shellStatus = report.shellFiles[0];
|
||||
const shellFp = shellStatus?.fingerprint;
|
||||
const shellOk = shellFp === fingerprint;
|
||||
items.push({
|
||||
label: padRight(`${shortenPath(shellPath)}`, 50) + chalk.dim(` [${toolName(shellPath) || 'Shell'}]`),
|
||||
value: `shell:${shellPath}`,
|
||||
checked: !shellOk,
|
||||
status: shellOk ? `configured (${shellFp})` : shellFp ? `mismatch (${shellFp})` : 'missing',
|
||||
statusColor: shellOk ? 'green' : shellFp ? 'yellow' : 'red',
|
||||
});
|
||||
|
||||
// Config files
|
||||
for (const config of report.configs) {
|
||||
const fp = config.fingerprint;
|
||||
const ok = fp === fingerprint;
|
||||
const tool = toolName(config.path);
|
||||
items.push({
|
||||
label: padRight(`${shortenPath(config.path)}`, 50) + chalk.dim(tool ? ` [${tool}]` : ''),
|
||||
value: `config:${config.path}`,
|
||||
checked: !ok,
|
||||
status: ok ? `configured (${fp})` : !config.exists ? 'will create' : fp ? `mismatch (${fp})` : 'missing',
|
||||
statusColor: ok ? 'green' : 'yellow',
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Show interactive checkbox
|
||||
const selected = await checkboxPrompt(items, {
|
||||
title: ` Select files to update with token ${chalk.cyan(fingerprint)}:`,
|
||||
});
|
||||
|
||||
if (selected.length === 0) {
|
||||
console.log(chalk.dim(' No changes made.\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: Apply changes
|
||||
const written: string[] = [];
|
||||
|
||||
for (const sel of selected) {
|
||||
if (sel.startsWith('shell:')) {
|
||||
const path = sel.slice('shell:'.length);
|
||||
const before = fileExists(path) ? fs.readFileSync(path, 'utf-8') : '';
|
||||
writeFileWithMkdir(path, upsertShellToken(before, token));
|
||||
written.push(path);
|
||||
} else if (sel.startsWith('config:')) {
|
||||
const path = sel.slice('config:'.length);
|
||||
const config = report.configs.find(c => c.path === path);
|
||||
if (config && config.parseError) continue;
|
||||
const before = fileExists(path) ? fs.readFileSync(path, 'utf-8') : '';
|
||||
const format = config?.format ?? (path.endsWith('.toml') ? 'toml' : 'json');
|
||||
const next = format === 'toml' ? upsertTomlConfigToken(before, token) : upsertJsonConfigToken(before, token);
|
||||
writeFileWithMkdir(path, next);
|
||||
written.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
process.env[PLAYWRIGHT_TOKEN_ENV] = token;
|
||||
|
||||
// Step 6: Summary
|
||||
if (written.length > 0) {
|
||||
console.log(chalk.green.bold(` ✓ Updated ${written.length} file(s):`));
|
||||
for (const p of written) {
|
||||
console.log(` ${chalk.dim('•')} ${shortenPath(p)}`);
|
||||
}
|
||||
} else {
|
||||
console.log(chalk.yellow(' No files were changed.'));
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
function padRight(s: string, n: number): string {
|
||||
// Account for ANSI escape codes in length calculation
|
||||
const visible = s.replace(/\x1b\[[0-9;]*m/g, '');
|
||||
return visible.length >= n ? s : s + ' '.repeat(n - visible.length);
|
||||
}
|
||||
+5
-5
@@ -6,12 +6,12 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { VOLATILE_PARAMS, SEARCH_PARAMS, LIMIT_PARAMS, PAGINATION_PARAMS } from './constants.js';
|
||||
|
||||
/** Volatile params to strip from generated URLs */
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
|
||||
const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
|
||||
const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
|
||||
const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
|
||||
/** Renamed aliases for backward compatibility with local references */
|
||||
const SEARCH_PARAM_NAMES = SEARCH_PARAMS;
|
||||
const LIMIT_PARAM_NAMES = LIMIT_PARAMS;
|
||||
const PAGE_PARAM_NAMES = PAGINATION_PARAMS;
|
||||
|
||||
export function synthesizeFromExplore(
|
||||
target: string,
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* tui.ts — Zero-dependency interactive TUI components
|
||||
*
|
||||
* Uses raw stdin mode + ANSI escape codes for interactive prompts.
|
||||
*/
|
||||
import chalk from 'chalk';
|
||||
|
||||
export interface CheckboxItem {
|
||||
label: string;
|
||||
value: string;
|
||||
checked: boolean;
|
||||
/** Optional status to display after the label */
|
||||
status?: string;
|
||||
statusColor?: 'green' | 'yellow' | 'red' | 'dim';
|
||||
}
|
||||
|
||||
/**
|
||||
* Interactive multi-select checkbox prompt.
|
||||
*
|
||||
* Controls:
|
||||
* ↑/↓ or j/k — navigate
|
||||
* Space — toggle selection
|
||||
* a — toggle all
|
||||
* Enter — confirm
|
||||
* q/Esc — cancel (returns empty)
|
||||
*/
|
||||
export async function checkboxPrompt(
|
||||
items: CheckboxItem[],
|
||||
opts: { title?: string; hint?: string } = {},
|
||||
): Promise<string[]> {
|
||||
if (items.length === 0) return [];
|
||||
|
||||
const { stdin, stdout } = process;
|
||||
if (!stdin.isTTY) {
|
||||
// Non-interactive: return all checked items
|
||||
return items.filter(i => i.checked).map(i => i.value);
|
||||
}
|
||||
|
||||
let cursor = 0;
|
||||
const state = items.map(i => ({ ...i }));
|
||||
|
||||
function colorStatus(status: string | undefined, color: CheckboxItem['statusColor']): string {
|
||||
if (!status) return '';
|
||||
switch (color) {
|
||||
case 'green': return chalk.green(status);
|
||||
case 'yellow': return chalk.yellow(status);
|
||||
case 'red': return chalk.red(status);
|
||||
case 'dim': return chalk.dim(status);
|
||||
default: return chalk.dim(status);
|
||||
}
|
||||
}
|
||||
|
||||
function render() {
|
||||
// Move cursor to start and clear
|
||||
let out = '';
|
||||
|
||||
if (opts.title) {
|
||||
out += `\n${chalk.bold(opts.title)}\n\n`;
|
||||
}
|
||||
|
||||
for (let i = 0; i < state.length; i++) {
|
||||
const item = state[i];
|
||||
const pointer = i === cursor ? chalk.cyan('❯') : ' ';
|
||||
const checkbox = item.checked ? chalk.green('◉') : chalk.dim('○');
|
||||
const label = i === cursor ? chalk.bold(item.label) : item.label;
|
||||
const status = colorStatus(item.status, item.statusColor);
|
||||
out += ` ${pointer} ${checkbox} ${label}${status ? ` ${status}` : ''}\n`;
|
||||
}
|
||||
|
||||
out += `\n ${chalk.dim('↑↓ navigate · Space toggle · a all · Enter confirm · q cancel')}\n`;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
return new Promise<string[]>((resolve) => {
|
||||
const wasRaw = stdin.isRaw;
|
||||
stdin.setRawMode(true);
|
||||
stdin.resume();
|
||||
|
||||
let rendered = '';
|
||||
|
||||
function draw() {
|
||||
// Clear previous render
|
||||
if (rendered) {
|
||||
const lines = rendered.split('\n').length;
|
||||
stdout.write(`\x1b[${lines}A\x1b[J`);
|
||||
}
|
||||
rendered = render();
|
||||
stdout.write(rendered);
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
stdin.setRawMode(wasRaw ?? false);
|
||||
stdin.pause();
|
||||
stdin.removeListener('data', onData);
|
||||
// Clear the TUI
|
||||
if (rendered) {
|
||||
const lines = rendered.split('\n').length;
|
||||
stdout.write(`\x1b[${lines}A\x1b[J`);
|
||||
}
|
||||
}
|
||||
|
||||
function onData(data: Buffer) {
|
||||
const key = data.toString();
|
||||
|
||||
// Arrow up / k
|
||||
if (key === '\x1b[A' || key === 'k') {
|
||||
cursor = (cursor - 1 + state.length) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Arrow down / j
|
||||
if (key === '\x1b[B' || key === 'j') {
|
||||
cursor = (cursor + 1) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Space — toggle
|
||||
if (key === ' ') {
|
||||
state[cursor].checked = !state[cursor].checked;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Tab — toggle and move down
|
||||
if (key === '\t') {
|
||||
state[cursor].checked = !state[cursor].checked;
|
||||
cursor = (cursor + 1) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// 'a' — toggle all
|
||||
if (key === 'a') {
|
||||
const allChecked = state.every(i => i.checked);
|
||||
for (const item of state) item.checked = !allChecked;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Enter — confirm
|
||||
if (key === '\r' || key === '\n') {
|
||||
cleanup();
|
||||
const selected = state.filter(i => i.checked).map(i => i.value);
|
||||
// Show summary
|
||||
stdout.write(` ${chalk.green('✓')} ${chalk.bold(`${selected.length} file(s) selected`)}\n\n`);
|
||||
resolve(selected);
|
||||
return;
|
||||
}
|
||||
|
||||
// q / Esc / Ctrl+C — cancel
|
||||
if (key === 'q' || key === '\x1b' || key === '\x03') {
|
||||
cleanup();
|
||||
stdout.write(` ${chalk.yellow('✗')} ${chalk.dim('Cancelled')}\n\n`);
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
stdin.on('data', onData);
|
||||
draw();
|
||||
});
|
||||
}
|
||||
@@ -3,6 +3,14 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
/** All recognized pipeline step names */
|
||||
const KNOWN_STEP_NAMES = new Set([
|
||||
'navigate', 'click', 'type', 'wait', 'press', 'snapshot', 'scroll',
|
||||
'fetch', 'evaluate',
|
||||
'select', 'map', 'filter', 'sort', 'limit',
|
||||
'intercept', 'tap',
|
||||
]);
|
||||
|
||||
export function validateClisWithTarget(dirs: string[], target?: string): any {
|
||||
const results: any[] = [];
|
||||
let errors = 0; let warnings = 0; let files = 0;
|
||||
@@ -38,6 +46,20 @@ function validateYamlFile(filePath: string): any {
|
||||
if (def.pipeline && !Array.isArray(def.pipeline)) errors.push('"pipeline" must be an array');
|
||||
if (def.columns && !Array.isArray(def.columns)) errors.push('"columns" must be an array');
|
||||
if (def.args && typeof def.args !== 'object') errors.push('"args" must be an object');
|
||||
// Validate pipeline step names (catch typos like 'navaigate')
|
||||
if (Array.isArray(def.pipeline)) {
|
||||
for (let i = 0; i < def.pipeline.length; i++) {
|
||||
const step = def.pipeline[i];
|
||||
if (step && typeof step === 'object') {
|
||||
const stepKeys = Object.keys(step);
|
||||
for (const key of stepKeys) {
|
||||
if (!KNOWN_STEP_NAMES.has(key)) {
|
||||
warnings.push(`Pipeline step ${i}: unknown step name "${key}" (did you mean one of: ${[...KNOWN_STEP_NAMES].join(', ')}?)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: any) { errors.push(`YAML parse error: ${e.message}`); }
|
||||
return { path: filePath, errors, warnings };
|
||||
}
|
||||
|
||||
+10
-1
@@ -1,9 +1,18 @@
|
||||
/** Verification: validate + smoke. */
|
||||
/**
|
||||
* Verification: runs validation and optional smoke test.
|
||||
*
|
||||
* The smoke test is intentionally kept as a stub — full browser-based
|
||||
* smoke testing requires a running browser session and is better suited
|
||||
* to the `opencli test` command or CI pipelines.
|
||||
*/
|
||||
|
||||
import { validateClisWithTarget, renderValidationReport } from './validate.js';
|
||||
|
||||
export async function verifyClis(opts: any): Promise<any> {
|
||||
const report = validateClisWithTarget([opts.builtinClis, opts.userClis], opts.target);
|
||||
return { ok: report.ok, validation: report, smoke: null };
|
||||
}
|
||||
|
||||
export function renderVerifyReport(report: any): string {
|
||||
return renderValidationReport(report.validation);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Single source of truth for package version.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
||||
|
||||
export const PKG_VERSION: string = (() => {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version;
|
||||
} catch {
|
||||
return '0.0.0';
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* E2E tests for login-required browser commands.
|
||||
* These commands REQUIRE authentication (cookie/session).
|
||||
* In CI (headless, no login), they should fail gracefully — NOT crash.
|
||||
*
|
||||
* These tests verify the error handling path, not the data extraction.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli } from './helpers.js';
|
||||
|
||||
/**
|
||||
* Verify a login-required command fails gracefully (no crash, no hang).
|
||||
* Acceptable outcomes: exit code 1 with error message, OR timeout handled.
|
||||
*/
|
||||
async function expectGracefulAuthFailure(args: string[], label: string) {
|
||||
const { stdout, stderr, code } = await runCli(args, { timeout: 60_000 });
|
||||
// Should either fail with exit code 1 (error message) or succeed with empty data
|
||||
// The key assertion: it should NOT hang forever or crash with unhandled exception
|
||||
if (code !== 0) {
|
||||
// Verify stderr has a meaningful error, not an unhandled crash
|
||||
const output = stderr + stdout;
|
||||
expect(output.length).toBeGreaterThan(0);
|
||||
}
|
||||
// If it somehow succeeds (e.g., partial public data), that's fine too
|
||||
}
|
||||
|
||||
describe('login-required commands — graceful failure', () => {
|
||||
|
||||
// ── bilibili (requires cookie session) ──
|
||||
it('bilibili me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'me', '-f', 'json'], 'bilibili me');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili dynamic fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'dynamic', '--limit', '3', '-f', 'json'], 'bilibili dynamic');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili favorite fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'favorite', '--limit', '3', '-f', 'json'], 'bilibili favorite');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili history fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'history', '--limit', '3', '-f', 'json'], 'bilibili history');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili following fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'following', '--limit', '3', '-f', 'json'], 'bilibili following');
|
||||
}, 60_000);
|
||||
|
||||
// ── twitter (requires login) ──
|
||||
it('twitter bookmarks fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'bookmarks', '--limit', '3', '-f', 'json'], 'twitter bookmarks');
|
||||
}, 60_000);
|
||||
|
||||
it('twitter timeline fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'timeline', '--limit', '3', '-f', 'json'], 'twitter timeline');
|
||||
}, 60_000);
|
||||
|
||||
it('twitter notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'notifications', '--limit', '3', '-f', 'json'], 'twitter notifications');
|
||||
}, 60_000);
|
||||
|
||||
// ── v2ex (requires login) ──
|
||||
it('v2ex me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['v2ex', 'me', '-f', 'json'], 'v2ex me');
|
||||
}, 60_000);
|
||||
|
||||
it('v2ex notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['v2ex', 'notifications', '--limit', '3', '-f', 'json'], 'v2ex notifications');
|
||||
}, 60_000);
|
||||
|
||||
// ── xueqiu (requires login) ──
|
||||
it('xueqiu feed fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xueqiu', 'feed', '--limit', '3', '-f', 'json'], 'xueqiu feed');
|
||||
}, 60_000);
|
||||
|
||||
it('xueqiu watchlist fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xueqiu', 'watchlist', '-f', 'json'], 'xueqiu watchlist');
|
||||
}, 60_000);
|
||||
|
||||
// ── xiaohongshu (requires login) ──
|
||||
it('xiaohongshu feed fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xiaohongshu', 'feed', '--limit', '3', '-f', 'json'], 'xiaohongshu feed');
|
||||
}, 60_000);
|
||||
|
||||
it('xiaohongshu notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xiaohongshu', 'notifications', '--limit', '3', '-f', 'json'], 'xiaohongshu notifications');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* E2E tests for browser commands that access PUBLIC data (no login required).
|
||||
* These use OPENCLI_HEADLESS=1 to launch a headless Chromium.
|
||||
*
|
||||
* NOTE: Some sites may block headless browsers with bot detection.
|
||||
* Tests are wrapped with tryBrowserCommand() which allows graceful failure.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
/**
|
||||
* Run a browser command — returns parsed data or null on failure.
|
||||
*/
|
||||
async function tryBrowserCommand(args: string[]): Promise<any[] | null> {
|
||||
const { stdout, code } = await runCli(args, { timeout: 60_000 });
|
||||
if (code !== 0) return null;
|
||||
try {
|
||||
const data = parseJsonOutput(stdout);
|
||||
return Array.isArray(data) ? data : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert browser command returns data OR log a warning if blocked.
|
||||
* Empty results (bot detection, geo-blocking) are treated as a warning, not a failure.
|
||||
*/
|
||||
function expectDataOrSkip(data: any[] | null, label: string) {
|
||||
if (data === null || data.length === 0) {
|
||||
console.warn(`${label}: skipped — no data returned (likely bot detection or geo-blocking)`);
|
||||
return;
|
||||
}
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}
|
||||
|
||||
describe('browser public-data commands E2E', () => {
|
||||
|
||||
// ── bbc (browser: true, strategy: public) ──
|
||||
it('bbc news returns headlines', async () => {
|
||||
const data = await tryBrowserCommand(['bbc', 'news', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bbc news');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// ── v2ex daily (browser: true) ──
|
||||
it('v2ex daily returns topics', async () => {
|
||||
const data = await tryBrowserCommand(['v2ex', 'daily', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'v2ex daily');
|
||||
}, 60_000);
|
||||
|
||||
// ── bilibili (browser: true, cookie strategy) ──
|
||||
it('bilibili hot returns trending videos', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili hot');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili ranking returns ranked videos', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'ranking', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili ranking');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili search returns results', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'search', 'typescript', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili search');
|
||||
}, 60_000);
|
||||
|
||||
// ── weibo (browser: true, cookie strategy) ──
|
||||
it('weibo hot returns trending topics', async () => {
|
||||
const data = await tryBrowserCommand(['weibo', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'weibo hot');
|
||||
}, 60_000);
|
||||
|
||||
// ── zhihu (browser: true, cookie strategy) ──
|
||||
it('zhihu hot returns trending questions', async () => {
|
||||
const data = await tryBrowserCommand(['zhihu', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'zhihu hot');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it('zhihu search returns results', async () => {
|
||||
const data = await tryBrowserCommand(['zhihu', 'search', 'playwright', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'zhihu search');
|
||||
}, 60_000);
|
||||
|
||||
// ── reddit (browser: true, cookie strategy) ──
|
||||
it('reddit hot returns posts', async () => {
|
||||
const data = await tryBrowserCommand(['reddit', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reddit hot');
|
||||
}, 60_000);
|
||||
|
||||
it('reddit frontpage returns posts', async () => {
|
||||
const data = await tryBrowserCommand(['reddit', 'frontpage', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reddit frontpage');
|
||||
}, 60_000);
|
||||
|
||||
// ── twitter (browser: true) ──
|
||||
it('twitter trending returns trends', async () => {
|
||||
const data = await tryBrowserCommand(['twitter', 'trending', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'twitter trending');
|
||||
}, 60_000);
|
||||
|
||||
// ── xueqiu (browser: true, cookie strategy) ──
|
||||
it('xueqiu hot returns hot posts', async () => {
|
||||
const data = await tryBrowserCommand(['xueqiu', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xueqiu hot');
|
||||
}, 60_000);
|
||||
|
||||
it('xueqiu hot-stock returns stocks', async () => {
|
||||
const data = await tryBrowserCommand(['xueqiu', 'hot-stock', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xueqiu hot-stock');
|
||||
}, 60_000);
|
||||
|
||||
// ── reuters (browser: true) ──
|
||||
it('reuters search returns articles', async () => {
|
||||
const data = await tryBrowserCommand(['reuters', 'search', 'technology', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reuters search');
|
||||
}, 60_000);
|
||||
|
||||
// ── youtube (browser: true) ──
|
||||
it('youtube search returns videos', async () => {
|
||||
const data = await tryBrowserCommand(['youtube', 'search', 'typescript tutorial', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'youtube search');
|
||||
}, 60_000);
|
||||
|
||||
// ── smzdm (browser: true) ──
|
||||
it('smzdm search returns deals', async () => {
|
||||
const data = await tryBrowserCommand(['smzdm', 'search', '键盘', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'smzdm search');
|
||||
}, 60_000);
|
||||
|
||||
// ── boss (browser: true) ──
|
||||
it('boss search returns jobs', async () => {
|
||||
const data = await tryBrowserCommand(['boss', 'search', 'golang', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'boss search');
|
||||
}, 60_000);
|
||||
|
||||
// ── ctrip (browser: true) ──
|
||||
it('ctrip search returns flights', async () => {
|
||||
const data = await tryBrowserCommand(['ctrip', 'search', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'ctrip search');
|
||||
}, 60_000);
|
||||
|
||||
// ── coupang (browser: true) ──
|
||||
it('coupang search returns products', async () => {
|
||||
const data = await tryBrowserCommand(['coupang', 'search', 'laptop', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'coupang search');
|
||||
}, 60_000);
|
||||
|
||||
// ── xiaohongshu (browser: true) ──
|
||||
it('xiaohongshu search returns notes', async () => {
|
||||
const data = await tryBrowserCommand(['xiaohongshu', 'search', '美食', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xiaohongshu search');
|
||||
}, 60_000);
|
||||
|
||||
// ── yahoo-finance (browser: true) ──
|
||||
it('yahoo-finance quote returns stock data', async () => {
|
||||
const data = await tryBrowserCommand(['yahoo-finance', 'quote', '--symbol', 'AAPL', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'yahoo-finance quote');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Shared helpers for E2E tests.
|
||||
* Runs the built opencli binary as a subprocess.
|
||||
*/
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const MAIN = path.join(ROOT, 'dist', 'main.js');
|
||||
|
||||
export interface CliResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `opencli` as a child process with the given arguments.
|
||||
* Without PLAYWRIGHT_MCP_EXTENSION_TOKEN, opencli auto-launches its own browser.
|
||||
*/
|
||||
export async function runCli(
|
||||
args: string[],
|
||||
opts: { timeout?: number; env?: Record<string, string> } = {},
|
||||
): Promise<CliResult> {
|
||||
const timeout = opts.timeout ?? 30_000;
|
||||
try {
|
||||
const { stdout, stderr } = await exec('node', [MAIN, ...args], {
|
||||
cwd: ROOT,
|
||||
timeout,
|
||||
env: {
|
||||
...process.env,
|
||||
// Prevent chalk colors from polluting test assertions
|
||||
FORCE_COLOR: '0',
|
||||
NO_COLOR: '1',
|
||||
...opts.env,
|
||||
},
|
||||
});
|
||||
return { stdout, stderr, code: 0 };
|
||||
} catch (err: any) {
|
||||
return {
|
||||
stdout: err.stdout ?? '',
|
||||
stderr: err.stderr ?? '',
|
||||
code: err.code ?? 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JSON output from a CLI command.
|
||||
* Throws a descriptive error if parsing fails.
|
||||
*/
|
||||
export function parseJsonOutput(stdout: string): any {
|
||||
try {
|
||||
return JSON.parse(stdout.trim());
|
||||
} catch {
|
||||
throw new Error(`Failed to parse CLI JSON output:\n${stdout.slice(0, 500)}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* E2E tests for management/built-in commands.
|
||||
* These commands require no external network access (except verify --smoke).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
describe('management commands E2E', () => {
|
||||
|
||||
// ── list ──
|
||||
it('list shows all registered commands', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
// Should have 50+ commands across 18 sites
|
||||
expect(data.length).toBeGreaterThan(50);
|
||||
// Each entry should have the standard fields
|
||||
expect(data[0]).toHaveProperty('command');
|
||||
expect(data[0]).toHaveProperty('site');
|
||||
expect(data[0]).toHaveProperty('name');
|
||||
expect(data[0]).toHaveProperty('strategy');
|
||||
expect(data[0]).toHaveProperty('browser');
|
||||
});
|
||||
|
||||
it('list default table format renders sites', async () => {
|
||||
const { stdout, code } = await runCli(['list']);
|
||||
expect(code).toBe(0);
|
||||
// Should contain site names
|
||||
expect(stdout).toContain('hackernews');
|
||||
expect(stdout).toContain('bilibili');
|
||||
expect(stdout).toContain('twitter');
|
||||
expect(stdout).toContain('commands across');
|
||||
});
|
||||
|
||||
it('list -f yaml produces valid yaml', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'yaml']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('command:');
|
||||
expect(stdout).toContain('site:');
|
||||
});
|
||||
|
||||
it('list -f csv produces valid csv', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'csv']);
|
||||
expect(code).toBe(0);
|
||||
const lines = stdout.trim().split('\n');
|
||||
expect(lines.length).toBeGreaterThan(50);
|
||||
});
|
||||
|
||||
it('list -f md produces markdown table', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'md']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('|');
|
||||
expect(stdout).toContain('command');
|
||||
});
|
||||
|
||||
// ── validate ──
|
||||
it('validate passes for all built-in adapters', async () => {
|
||||
const { stdout, code } = await runCli(['validate']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
expect(stdout).not.toContain('❌');
|
||||
});
|
||||
|
||||
it('validate works for specific site', async () => {
|
||||
const { stdout, code } = await runCli(['validate', 'hackernews']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
it('validate works for specific command', async () => {
|
||||
const { stdout, code } = await runCli(['validate', 'hackernews/top']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── verify ──
|
||||
it('verify runs validation without smoke tests', async () => {
|
||||
const { stdout, code } = await runCli(['verify']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── version ──
|
||||
it('--version shows version number', async () => {
|
||||
const { stdout, code } = await runCli(['--version']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout.trim()).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
});
|
||||
|
||||
// ── help ──
|
||||
it('--help shows usage', async () => {
|
||||
const { stdout, code } = await runCli(['--help']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('opencli');
|
||||
expect(stdout).toContain('list');
|
||||
expect(stdout).toContain('validate');
|
||||
});
|
||||
|
||||
// ── unknown command ──
|
||||
it('unknown command shows error', async () => {
|
||||
const { stderr, code } = await runCli(['nonexistent-command-xyz']);
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* E2E tests for output format rendering.
|
||||
* Uses hackernews (public, fast) as a stable data source.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
const FORMATS = ['json', 'yaml', 'csv', 'md'] as const;
|
||||
|
||||
describe('output formats E2E', () => {
|
||||
for (const fmt of FORMATS) {
|
||||
it(`hackernews top -f ${fmt} produces valid output`, async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '2', '-f', fmt]);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout.trim().length).toBeGreaterThan(0);
|
||||
|
||||
if (fmt === 'json') {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBe(2);
|
||||
}
|
||||
|
||||
if (fmt === 'yaml') {
|
||||
expect(stdout).toContain('title:');
|
||||
}
|
||||
|
||||
if (fmt === 'csv') {
|
||||
// CSV should have a header row + data rows
|
||||
const lines = stdout.trim().split('\n');
|
||||
expect(lines.length).toBeGreaterThanOrEqual(2);
|
||||
}
|
||||
|
||||
if (fmt === 'md') {
|
||||
// Markdown table should have pipe characters
|
||||
expect(stdout).toContain('|');
|
||||
}
|
||||
}, 30_000);
|
||||
}
|
||||
|
||||
it('list -f csv produces valid csv', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'csv']);
|
||||
expect(code).toBe(0);
|
||||
const lines = stdout.trim().split('\n');
|
||||
// Header + many data lines
|
||||
expect(lines.length).toBeGreaterThan(50);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* E2E tests for public API commands (browser: false).
|
||||
* These commands use Node.js fetch directly — no browser needed.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
describe('public commands E2E', () => {
|
||||
// ── hackernews ──
|
||||
it('hackernews top returns structured data', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBe(3);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
expect(data[0]).toHaveProperty('score');
|
||||
expect(data[0]).toHaveProperty('rank');
|
||||
}, 30_000);
|
||||
|
||||
it('hackernews top respects --limit', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '1', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBe(1);
|
||||
}, 30_000);
|
||||
|
||||
// ── v2ex (public API, browser: false) ──
|
||||
it('v2ex hot returns topics', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'hot', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex latest returns topics', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'latest', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex topic returns topic detail', async () => {
|
||||
// Topic 1000001 is a well-known V2EX topic
|
||||
const { stdout, code } = await runCli(['v2ex', 'topic', '--id', '1000001', '-f', 'json']);
|
||||
// May fail if V2EX rate-limits, but should return structured data
|
||||
if (code === 0) {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Smoke tests for external API health.
|
||||
* Only run on schedule or manual dispatch — NOT on every push/PR.
|
||||
* These verify that external APIs haven't changed their structure.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from '../e2e/helpers.js';
|
||||
|
||||
describe('API health smoke tests', () => {
|
||||
|
||||
// ── Public API commands (should always work) ──
|
||||
it('hackernews API is responsive and returns expected structure', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '5', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBe(5);
|
||||
for (const item of data) {
|
||||
expect(item).toHaveProperty('title');
|
||||
expect(item).toHaveProperty('score');
|
||||
expect(item).toHaveProperty('author');
|
||||
expect(item).toHaveProperty('rank');
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex hot API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'hot', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex latest API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'latest', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex topic API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'topic', '--id', '1000001', '-f', 'json']);
|
||||
if (code === 0) {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
// ── Validate all adapters ──
|
||||
it('all adapter definitions are valid', async () => {
|
||||
const { stdout, code } = await runCli(['validate']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── Command registry integrity ──
|
||||
it('all expected sites are registered', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
const sites = new Set(data.map((d: any) => d.site));
|
||||
// Verify all 17 sites are present
|
||||
for (const expected of [
|
||||
'hackernews', 'bbc', 'bilibili', 'v2ex', 'weibo', 'zhihu',
|
||||
'twitter', 'reddit', 'xueqiu', 'reuters', 'youtube',
|
||||
'smzdm', 'boss', 'ctrip', 'coupang', 'xiaohongshu',
|
||||
'yahoo-finance',
|
||||
]) {
|
||||
expect(sites.has(expected)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user