Compare commits
72 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| ae4929e73d | |||
| 92408e7ee9 | |||
| 257e9ec4f8 | |||
| 6f629260e7 | |||
| 1e0b83bb84 | |||
| 1ff184e24d | |||
| 8e84145ccc | |||
| c77c8a8e3a | |||
| 9e024d4e46 | |||
| 34a9bff2b3 | |||
| b0c51ddf19 | |||
| 141c2cf7d5 | |||
| 7ee0c9c96b | |||
| 76eefed83d | |||
| 08ae9c3fed | |||
| 5a1581e67a | |||
| 7a24e8d74b | |||
| 95fb841fc7 | |||
| 67d50191af | |||
| 61a62f80c4 | |||
| f8491f01ae | |||
| 4ec454530a | |||
| e2192a2e8d | |||
| db8194337b | |||
| 8e5a61458d | |||
| dca908db90 | |||
| 5a676b9f4b | |||
| 797a392250 | |||
| 59a02d3a5d | |||
| cef50dc8bf | |||
| 1aaafed2db | |||
| ee6669f45d | |||
| eb9050fadd | |||
| 53cd006a23 | |||
| c9710a656b | |||
| 4f570fa75e | |||
| 730285c0ca | |||
| c5915d9ee3 | |||
| b7c8067bc9 | |||
| c28ccd4b23 | |||
| 00f0ac9133 | |||
| 64d655ae0d | |||
| 9c78ffdd39 | |||
| a6e9c3b611 | |||
| da155e96f0 | |||
| c20eaee393 | |||
| 15d8c45b95 | |||
| 3ac5aa3023 | |||
| 9a6e157af6 | |||
| 9a45988197 | |||
| 2d638e3a0a | |||
| b3850d9c53 | |||
| 91d7892516 | |||
| 3f859b3238 | |||
| 2fdab085a7 | |||
| 7ec04c66cb | |||
| f8ee50b776 | |||
| 53b28637bb | |||
| ad9e9605fc | |||
| 0e1cea84b6 | |||
| ee2e0a3bfc | |||
| 4fc4c4b92f | |||
| a47d687513 | |||
| 2dacba8244 | |||
| e64fc65554 | |||
| 4e6d6bb81d | |||
| 790c94d644 | |||
| aef110a8f8 | |||
| 71f2f3d6e6 | |||
| 829172fc59 | |||
| 20f7d58dd6 | |||
| 92c23e8f9b |
@@ -0,0 +1,26 @@
|
||||
name: Setup Chrome + xvfb
|
||||
description: Install real Chrome and xvfb virtual display for headed browser testing
|
||||
|
||||
outputs:
|
||||
chrome-path:
|
||||
description: Path to the installed Chrome binary
|
||||
value: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
|
||||
runs:
|
||||
using: composite
|
||||
steps:
|
||||
- name: Install real Chrome (stable)
|
||||
uses: browser-actions/setup-chrome@v1
|
||||
id: setup-chrome
|
||||
with:
|
||||
chrome-version: stable
|
||||
|
||||
- name: Verify Chrome installation
|
||||
shell: bash
|
||||
run: |
|
||||
echo "Chrome path: ${{ steps.setup-chrome.outputs.chrome-path }}"
|
||||
${{ steps.setup-chrome.outputs.chrome-path }} --version
|
||||
|
||||
- name: Install xvfb for headed mode
|
||||
shell: bash
|
||||
run: sudo apt-get install -y xvfb
|
||||
@@ -2,12 +2,16 @@ name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main]
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
branches: [main]
|
||||
branches: [main, dev]
|
||||
schedule:
|
||||
- cron: '0 8 * * 1' # Weekly Monday 08:00 UTC — smoke tests
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
check:
|
||||
# ── Fast gate: typecheck + build ──
|
||||
build:
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@@ -15,6 +19,7 @@ jobs:
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
@@ -24,3 +29,54 @@ jobs:
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# ── Unit tests (vitest shard) ──
|
||||
unit-test:
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Run unit tests (shard ${{ matrix.shard }}/2)
|
||||
run: npx vitest run src/ --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
|
||||
# ── Smoke tests (scheduled / manual only) ──
|
||||
smoke-test:
|
||||
if: github.event_name == 'schedule' || github.event_name == 'workflow_dispatch'
|
||||
needs: build
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Setup Chrome + xvfb
|
||||
uses: ./.github/actions/setup-chrome
|
||||
id: setup-chrome
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Run smoke tests
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/smoke/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
timeout-minutes: 15
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
name: E2E Headed Chrome
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [main, dev]
|
||||
pull_request:
|
||||
branches: [main, dev]
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
e2e-headed:
|
||||
runs-on: ubuntu-latest
|
||||
timeout-minutes: 20
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-node@v4
|
||||
with:
|
||||
node-version: '22'
|
||||
cache: 'npm'
|
||||
|
||||
- name: Install dependencies
|
||||
run: npm ci
|
||||
|
||||
- name: Setup Chrome + xvfb
|
||||
uses: ./.github/actions/setup-chrome
|
||||
id: setup-chrome
|
||||
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
- name: Run E2E tests (headed Chrome + xvfb)
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/e2e/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
@@ -2,3 +2,4 @@ node_modules/
|
||||
dist/
|
||||
*.tsbuildinfo
|
||||
.opencli/
|
||||
.mcp.json
|
||||
|
||||
+245
-116
@@ -1,8 +1,70 @@
|
||||
# CLI-CREATOR — 适配器开发完全指南
|
||||
# CLI-EXPLORER — 适配器探索式开发完全指南
|
||||
|
||||
> 本文档教你(或 AI Agent)如何为 OpenCLI 添加一个新网站的命令。
|
||||
> 从零到发布,覆盖 API 发现、方案选择、适配器编写、测试验证全流程。
|
||||
|
||||
> [!TIP]
|
||||
> **只想为一个具体页面快速生成一个命令?** 看 [CLI-ONESHOT.md](./CLI-ONESHOT.md)(~150 行,4 步搞定)。
|
||||
> 本文档适合从零探索一个新站点的完整流程。
|
||||
|
||||
---
|
||||
|
||||
## AI Agent 开发者必读:用 Playwright MCP Bridge 探索
|
||||
|
||||
> [!CAUTION]
|
||||
> **你(AI Agent)必须通过 Playwright MCP Bridge 打开浏览器去访问目标网站!**
|
||||
> 不要只靠 `opencli explore` 命令或静态分析来发现 API。
|
||||
> 你拥有 Playwright MCP 工具,必须主动用它们浏览网页、观察网络请求、模拟用户交互。
|
||||
|
||||
### 为什么?
|
||||
|
||||
很多 API 是**懒加载**的(用户必须点击某个按钮/标签才会触发网络请求)。字幕、评论、关注列表等深层数据不会在页面首次加载时出现在 Network 面板中。**如果你不主动去浏览和交互页面,你永远发现不了这些 API。**
|
||||
|
||||
### AI Agent 探索工作流(必须遵循)
|
||||
|
||||
| 步骤 | 工具 | 做什么 |
|
||||
|------|------|--------|
|
||||
| 0. 打开浏览器 | `browser_navigate` | 导航到目标页面 |
|
||||
| 1. 观察页面 | `browser_snapshot` | 观察可交互元素(按钮/标签/链接) |
|
||||
| 2. 首次抓包 | `browser_network_requests` | 筛选 JSON API 端点,记录 URL pattern |
|
||||
| 3. 模拟交互 | `browser_click` + `browser_wait_for` | 点击"字幕""评论""关注"等按钮 |
|
||||
| 4. 二次抓包 | `browser_network_requests` | 对比步骤 2,找出新触发的 API |
|
||||
| 5. 验证 API | `browser_evaluate` | `fetch(url, {credentials:'include'})` 测试返回结构 |
|
||||
| 6. 写代码 | — | 基于确认的 API 写适配器 |
|
||||
|
||||
### 常犯错误
|
||||
|
||||
| ❌ 错误做法 | ✅ 正确做法 |
|
||||
|------------|------------|
|
||||
| 只用 `opencli explore` 命令,等结果自动出来 | 用 MCP Bridge 打开浏览器,主动浏览页面 |
|
||||
| 直接在代码里 `fetch(url)`,不看浏览器实际请求 | 先在浏览器中确认 API 可用,再写代码 |
|
||||
| 页面打开后直接抓包,期望所有 API 都出现 | 模拟点击交互(展开评论/切换标签/加载更多) |
|
||||
| 遇到 HTTP 200 但空数据就放弃 | 检查是否需要 Wbi 签名或 Cookie 鉴权 |
|
||||
| 完全依赖 `__INITIAL_STATE__` 拿所有数据 | `__INITIAL_STATE__` 只有首屏数据,深层数据要调 API |
|
||||
|
||||
### 实战成功案例:5 分钟实现「关注列表」适配器
|
||||
|
||||
以下是用上述工作流实际发现 Bilibili 关注列表 API 的完整过程:
|
||||
|
||||
```
|
||||
1. browser_navigate → https://space.bilibili.com/{uid}/fans/follow
|
||||
2. browser_network_requests → 发现:
|
||||
GET /x/relation/followings?vmid={uid}&pn=1&ps=24 → [200]
|
||||
GET /x/relation/stat?vmid={uid} → [200]
|
||||
3. browser_evaluate → 验证 API:
|
||||
fetch('/x/relation/followings?vmid=137702077&pn=1&ps=5', {credentials:'include'})
|
||||
→ { code: 0, data: { total: 1342, list: [{mid, uname, sign, ...}] } }
|
||||
4. 结论:标准 Cookie API,无需 Wbi 签名
|
||||
5. 写 following.ts → 一次构建通过
|
||||
```
|
||||
|
||||
**关键决策点**:
|
||||
- 直接访问 `fans/follow` 页面(不是首页),页面加载就会触发 following API
|
||||
- 看到 URL 里没有 `/wbi/` → 不需要签名 → 直接用 `fetchJson` 而非 `apiGet`
|
||||
- API 返回 `code: 0` + 非空 `list` → Tier 2 Cookie 策略确认
|
||||
|
||||
---
|
||||
|
||||
## 核心流程
|
||||
|
||||
```
|
||||
@@ -51,7 +113,17 @@ opencli bilibili hot -v # 查看已有命令的 pipeline 每步数据流
|
||||
- **Request Headers**: Cookie? Bearer? 自定义签名头(X-s、X-t)?
|
||||
- **Response Body**: JSON 结构,特别是数据在哪个路径(`data.items`、`data.list`)
|
||||
|
||||
### 1c. 框架检测
|
||||
### 1c. 高阶 API 发现捷径法则 (Heuristics)
|
||||
|
||||
在开始死磕复杂的抓包拦截之前,按照以下优先级进行尝试:
|
||||
|
||||
1. **后缀爆破法 (`.json`)**: 像 Reddit 这样复杂的网站,只要在其 URL 后加上 `.json`(例如 `/r/all.json`),就能在带 Cookie 的情况下直接利用 `fetch` 拿到极其干净的 REST 数据(Tier 2 Cookie 策略极速秒杀)。另外如功能完备的**雪球 (xueqiu)** 也可以走这种纯 API 的方式极简获取,成为你构建简单 YAML 的黄金标杆。
|
||||
2. **全局状态查找法 (`__INITIAL_STATE__`)**: 许多服务端渲染 (SSR) 的网站(如小红书、Bilibili)会将首页或详情页的完整数据挂载到全局 window 对象上。与其去拦截网络请求,不如直接 `page.evaluate('() => window.__INITIAL_STATE__')` 获取整个数据树。
|
||||
3. **主动交互触发法 (Active Interaction)**: 很多深层 API(如视频字幕、评论下的回复)是懒加载的。在静态抓包找不到数据时,尝试在 `evaluate` 步骤或手动打断点时,主动去**点击(Click)页面上的对应按钮**(如"CC"、"展开全部"),从而诱发隐藏的 Network Fetch。
|
||||
4. **框架探测与 Store Action 截断**: 如果站点使用 Vue + Pinia,可以使用 `tap` 步骤调用 action,让前端框架代替你完成复杂的鉴权签名封装。
|
||||
5. **底层 XHR/Fetch 拦截**: 最后手段,当上述都不行时,使用 TypeScript 适配器进行无侵入式的请求抓取。
|
||||
|
||||
### 1d. 框架检测
|
||||
|
||||
Explore 自动检测前端框架。如果需要手动确认:
|
||||
|
||||
@@ -104,15 +176,54 @@ opencli cascade https://api.example.com/hot
|
||||
|
||||
---
|
||||
|
||||
## Step 2.5: 准备工作(写代码之前)
|
||||
|
||||
### 先找模板:从最相似的现有适配器开始
|
||||
|
||||
**不要从零开始写**。先看看同站点已有哪些适配器:
|
||||
|
||||
```bash
|
||||
ls src/clis/<site>/ # 看看已有什么
|
||||
cat src/clis/<site>/feed.ts # 读最相似的那个
|
||||
```
|
||||
|
||||
最高效的方式是 **复制最相似的适配器,然后改 3 个地方**:
|
||||
1. `name` → 新命令名
|
||||
2. API URL → 你在 Step 1 发现的端点
|
||||
3. 字段映射 → 对应新 API 的字段
|
||||
|
||||
### 平台 SDK 速查表
|
||||
|
||||
写 TS 适配器之前,先看看你的目标站点有没有**现成的 helper 函数**可以复用:
|
||||
|
||||
#### Bilibili (`src/bilibili.ts`)
|
||||
|
||||
| 函数 | 用途 | 何时使用 |
|
||||
|------|------|----------|
|
||||
| `fetchJson(page, url)` | 带 Cookie 的 fetch + JSON 解析 | 普通 Cookie-tier API |
|
||||
| `apiGet(page, path, {signed, params})` | 带 Wbi 签名的 API 调用 | URL 含 `/wbi/` 的接口 |
|
||||
| `getSelfUid(page)` | 获取当前登录用户的 UID | "我的xxx" 类命令 |
|
||||
| `resolveUid(page, input)` | 解析用户输入的 UID(支持数字/URL) | `--uid` 参数处理 |
|
||||
| `wbiSign(page, params)` | 底层 Wbi 签名生成 | 通常不直接用,`apiGet` 已封装 |
|
||||
| `stripHtml(s)` | 去除 HTML 标签 | 清理富文本字段 |
|
||||
|
||||
**如何判断需不需要 `apiGet`**?看 Network 请求 URL:
|
||||
- 含 `/wbi/` 或 `w_rid=` → 必须用 `apiGet(..., { signed: true })`
|
||||
- 不含 → 直接用 `fetchJson`
|
||||
|
||||
> 其他站点(Twitter、小红书等)暂无专用 SDK,直接用 `page.evaluate` + `fetch` 即可。
|
||||
|
||||
---
|
||||
|
||||
## Step 3: 编写适配器
|
||||
|
||||
### YAML vs TS?先看决策树
|
||||
|
||||
```
|
||||
你的 pipeline 里有 evaluate 步骤(内嵌 JS 代码)?
|
||||
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),需在 index.ts 注册
|
||||
→ ✅ 用 TypeScript (src/clis/<site>/<name>.ts),保存即自动动态注册
|
||||
→ ❌ 纯声明式(navigate + tap + map + limit)?
|
||||
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),放入即自动注册
|
||||
→ ✅ 用 YAML (src/clis/<site>/<name>.yaml),保存即自动注册
|
||||
```
|
||||
|
||||
| 场景 | 选择 | 示例 |
|
||||
@@ -126,6 +237,27 @@ opencli cascade https://api.example.com/hot
|
||||
|
||||
> **经验法则**:如果你发现 YAML 里嵌了超过 10 行 JS,改用 TS 更可维护。
|
||||
|
||||
### 通用模式:分页 API
|
||||
|
||||
很多 API 使用 `pn`(页码)+ `ps`(每页数量)分页。标准处理模式:
|
||||
|
||||
```typescript
|
||||
args: [
|
||||
{ name: 'page', type: 'int', required: false, default: 1, help: '页码' },
|
||||
{ name: 'limit', type: 'int', required: false, default: 50, help: '每页数量 (最大 50)' },
|
||||
],
|
||||
func: async (page, kwargs) => {
|
||||
const pn = kwargs.page ?? 1;
|
||||
const ps = Math.min(kwargs.limit ?? 50, 50); // 尊重 API 的 ps 上限
|
||||
const payload = await fetchJson(page,
|
||||
`https://api.example.com/list?pn=${pn}&ps=${ps}`
|
||||
);
|
||||
return payload.data?.list || [];
|
||||
},
|
||||
```
|
||||
|
||||
> 大多数站点的 `ps` 上限是 20~50。超过会被静默截断或返回错误。
|
||||
|
||||
### 方式 A: YAML Pipeline(声明式,推荐)
|
||||
|
||||
文件路径: `src/clis/<site>/<name>.yaml`,放入即自动注册。
|
||||
@@ -310,7 +442,7 @@ pipeline:
|
||||
|
||||
适用于需要嵌入 JS 代码读取 Pinia state、XHR 拦截、GraphQL、分页、复杂数据转换等场景。
|
||||
|
||||
文件路径: `src/clis/<site>/<name>.ts`,还需要在 `src/clis/index.ts` 中 import 注册。
|
||||
文件路径: `src/clis/<site>/<name>.ts`。文件将会在运行时被动态扫描并注册(切勿在 `index.ts` 中手动 `import`)。
|
||||
|
||||
#### Tier 3 — Header 认证(Twitter)
|
||||
|
||||
@@ -353,90 +485,62 @@ cli({
|
||||
});
|
||||
```
|
||||
|
||||
#### Tier 4 — Store Action + XHR 拦截(小红书)
|
||||
#### Tier 4 — XHR/Fetch 双重拦截 (Twitter/小红书 通用模式)
|
||||
|
||||
```typescript
|
||||
// src/clis/xiaohongshu/search.ts
|
||||
// src/clis/xiaohongshu/user.ts
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'xiaohongshu',
|
||||
name: 'search',
|
||||
description: '搜索小红书笔记',
|
||||
strategy: Strategy.COOKIE, // 实际是 intercept 模式
|
||||
args: [{ name: 'keyword', required: true }],
|
||||
columns: ['rank', 'title', 'author', 'likes', 'type'],
|
||||
name: 'user',
|
||||
description: '获取用户笔记',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
args: [{ name: 'id', required: true }],
|
||||
columns: ['rank', 'title', 'likes', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.xiaohongshu.com');
|
||||
await page.wait(2);
|
||||
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
|
||||
await page.wait(5);
|
||||
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const app = document.querySelector('#app')?.__vue_app__;
|
||||
const pinia = app?.config?.globalProperties?.$pinia;
|
||||
if (!pinia?._s) return { error: 'Page not ready' };
|
||||
// XHR/Fetch 底层拦截:捕获所有包含 'v1/user/posted' 的请求
|
||||
await page.installInterceptor('v1/user/posted');
|
||||
|
||||
const searchStore = pinia._s.get('search');
|
||||
if (!searchStore) return { error: 'Search store not found' };
|
||||
// 触发后端 API:模拟人类用户向底部滚动2次
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// XHR 拦截:捕获 store action 发出的请求
|
||||
let captured = null;
|
||||
const origOpen = XMLHttpRequest.prototype.open;
|
||||
const origSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(m, u) {
|
||||
this.__url = u;
|
||||
return origOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(b) {
|
||||
if (this.__url?.includes('search/notes')) {
|
||||
const x = this;
|
||||
const orig = x.onreadystatechange;
|
||||
x.onreadystatechange = function() {
|
||||
if (x.readyState === 4 && !captured) {
|
||||
try { captured = JSON.parse(x.responseText); } catch {}
|
||||
}
|
||||
if (orig) orig.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
return origSend.apply(this, arguments);
|
||||
};
|
||||
// 提取所有被拦截捕获的 JSON 响应体
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
try {
|
||||
// 触发 Store Action,让网站自己签名发请求
|
||||
searchStore.mutateSearchValue('${kwargs.keyword}');
|
||||
await searchStore.loadMore();
|
||||
await new Promise(r => setTimeout(r, 800));
|
||||
} finally {
|
||||
// 恢复原始 XHR
|
||||
XMLHttpRequest.prototype.open = origOpen;
|
||||
XMLHttpRequest.prototype.send = origSend;
|
||||
let results = [];
|
||||
for (const req of requests) {
|
||||
if (req.data?.data?.notes) {
|
||||
for (const note of req.data.data.notes) {
|
||||
results.push({
|
||||
title: note.display_title || '',
|
||||
likes: note.interact_info?.liked_count || '0',
|
||||
url: `https://explore/${note.note_id || note.id}`
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (!captured?.success) return { error: captured?.msg || 'Search failed' };
|
||||
return (captured.data?.items || []).map(i => ({
|
||||
title: i.note_card?.display_title || '',
|
||||
author: i.note_card?.user?.nickname || '',
|
||||
likes: i.note_card?.interact_info?.liked_count || '0',
|
||||
type: i.note_card?.type || '',
|
||||
}));
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(data)) return [];
|
||||
return data.slice(0, kwargs.limit || 20).map((item, i) => ({
|
||||
return results.slice(0, 20).map((item, i) => ({
|
||||
rank: i + 1, ...item,
|
||||
}));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
> **XHR 拦截核心思路**:不自己构造签名,而是劫持网站自己的 `XMLHttpRequest`,让网站的 Store Action 发出正确签名的请求,我们只是"窃听"响应。用完后必须恢复原始方法。
|
||||
> **拦截核心思路**:不自己构造签名,而是利用 `installInterceptor` 劫持网站自己的 `XMLHttpRequest` 和 `fetch`,让网站发请求,我们直接在底层取出解析好的 `response.json()`。
|
||||
|
||||
> **级联请求**(如 BVID→CID→字幕)的完整模板和要点见下方[进阶模式: 级联请求](#进阶模式-级联请求-cascading-requests)章节。
|
||||
|
||||
---
|
||||
|
||||
## Step 4: 测试
|
||||
|
||||
> **⚠️ 构建通过 ≠ 功能正常**。`npm run build` 只验证 TypeScript / YAML 语法,不验证运行时行为。
|
||||
> **构建通过 ≠ 功能正常**。`npm run build` 只验证 TypeScript / YAML 语法,不验证运行时行为。
|
||||
> 每个新命令 **必须实际运行** 并确认输出正确后才算完成。
|
||||
|
||||
### 必做清单
|
||||
@@ -455,7 +559,7 @@ opencli mysite hot --limit 3 -f json # JSON 输出确认字段完整
|
||||
|
||||
### tap 步骤调试(intercept 策略专用)
|
||||
|
||||
> **⚠️ 不要猜 store name / action name**。先用 evaluate 探索,再写 YAML。
|
||||
> **不要猜 store name / action name**。先用 evaluate 探索,再写 YAML。
|
||||
|
||||
#### Step 1: 列出所有 Pinia store
|
||||
|
||||
@@ -495,68 +599,90 @@ opencli evaluate "(() => {
|
||||
└──────────────┘ └──────────────┘ └──────────────┘ └────────┘
|
||||
```
|
||||
|
||||
### Verbose 模式
|
||||
### Verbose 模式 & 输出验证
|
||||
|
||||
```bash
|
||||
# 查看 pipeline 每步的输入输出
|
||||
opencli bilibili hot --limit 1 -v
|
||||
```
|
||||
|
||||
输出示例:
|
||||
```
|
||||
[1/4] navigate → https://www.bilibili.com
|
||||
→ (no data)
|
||||
[2/4] evaluate → (async () => { const res = await fetch(…
|
||||
→ [{title: "…", author: "…", play: 230835}]
|
||||
[3/4] map (rank, title, author, play, danmaku)
|
||||
→ [{rank: 1, title: "…", author: "…"}]
|
||||
[4/4] limit → 1
|
||||
→ [{rank: 1, title: "…"}]
|
||||
```
|
||||
|
||||
### 输出格式验证
|
||||
|
||||
```bash
|
||||
# 确认表格渲染正确
|
||||
opencli mysite hot -f table
|
||||
|
||||
# 确认 JSON 可被 jq 解析
|
||||
opencli mysite hot -f json | jq '.[0]'
|
||||
|
||||
# 确认 CSV 可被导入
|
||||
opencli mysite hot -f csv > data.csv
|
||||
opencli bilibili hot --limit 1 -v # 查看 pipeline 每步数据流
|
||||
opencli mysite hot -f json | jq '.[0]' # 确认 JSON 可被解析
|
||||
opencli mysite hot -f csv > data.csv # 确认 CSV 可导入
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## Step 5: 注册 & 发布
|
||||
## Step 5: 提交发布
|
||||
|
||||
### YAML 适配器
|
||||
文件放入 `src/clis/<site>/` 即自动注册(YAML 或 TS 无需手动 import),然后:
|
||||
|
||||
放入 `src/clis/<site>/<name>.yaml` 即自动注册,无需额外操作。
|
||||
```bash
|
||||
opencli list | grep mysite # 确认注册
|
||||
git add src/clis/mysite/ && git commit -m "feat(mysite): add hot" && git push
|
||||
```
|
||||
|
||||
### TS 适配器
|
||||
> **架构理念**:OpenCLI 内建 **Zero-Dependency jq** 数据流 — 所有解析在 `evaluate` 的原生 JS 内完成,外层 YAML 用 `select`/`map` 提取,无需依赖系统 `jq` 二进制。
|
||||
|
||||
在 `src/clis/index.ts` 添加 import:
|
||||
---
|
||||
|
||||
## 进阶模式: 级联请求 (Cascading Requests)
|
||||
|
||||
当目标数据需要多步 API 链式获取时(如 `BVID → CID → 字幕列表 → 字幕内容`),必须使用 **TS 适配器**。YAML 无法处理这种多步逻辑。
|
||||
|
||||
### 模板代码
|
||||
|
||||
```typescript
|
||||
import './mysite/search.js';
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
import { apiGet } from '../../bilibili.js'; // 复用平台 SDK
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
name: 'subtitle',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [{ name: 'bvid', required: true }],
|
||||
columns: ['index', 'from', 'to', 'content'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
// Step 1: 建立 Session
|
||||
await page.goto(`https://www.bilibili.com/video/${kwargs.bvid}/`);
|
||||
|
||||
// Step 2: 从页面提取中间 ID (__INITIAL_STATE__)
|
||||
const cid = await page.evaluate(`(async () => {
|
||||
return window.__INITIAL_STATE__?.videoData?.cid;
|
||||
})()`);
|
||||
if (!cid) throw new Error('无法提取 CID');
|
||||
|
||||
// Step 3: 用中间 ID 调用下一级 API (自动 Wbi 签名)
|
||||
const payload = await apiGet(page, '/x/player/wbi/v2', {
|
||||
params: { bvid: kwargs.bvid, cid },
|
||||
signed: true, // ← 自动生成 w_rid
|
||||
});
|
||||
|
||||
// Step 4: 检测风控降级 (空值断言)
|
||||
const subtitles = payload.data?.subtitle?.subtitles || [];
|
||||
const url = subtitles[0]?.subtitle_url;
|
||||
if (!url) throw new Error('subtitle_url 为空,疑似风控降级');
|
||||
|
||||
// Step 5: 拉取最终数据 (CDN JSON)
|
||||
const items = await page.evaluate(`(async () => {
|
||||
const res = await fetch(${JSON.stringify('https:' + url)});
|
||||
const json = await res.json();
|
||||
return { data: json.body || json };
|
||||
})()`);
|
||||
|
||||
return items.data.map((item, idx) => ({ ... }));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### 验证注册
|
||||
### 关键要点
|
||||
|
||||
```bash
|
||||
opencli list # 确认新命令出现
|
||||
opencli validate mysite # 校验定义完整性
|
||||
```
|
||||
|
||||
### 提交
|
||||
|
||||
```bash
|
||||
git add src/clis/mysite/
|
||||
git commit -m "feat(mysite): add hot and search adapters"
|
||||
git push
|
||||
```
|
||||
| 步骤 | 注意事项 |
|
||||
|------|----------|
|
||||
| 提取中间 ID | 优先从 `__INITIAL_STATE__` 拿,避免额外 API 调用 |
|
||||
| Wbi 签名 | B 站 `/wbi/` 接口**强制校验** `w_rid`,纯 `fetch` 会被 403 |
|
||||
| 空值断言 | 即使 HTTP 200,核心字段可能为空串(风控降级) |
|
||||
| CDN URL | 常以 `//` 开头,记得补 `https:` |
|
||||
| `JSON.stringify` | 拼接 URL 到 evaluate 时必须用它转义,避免注入 |
|
||||
|
||||
---
|
||||
|
||||
@@ -574,6 +700,8 @@ git push
|
||||
| TS evaluate 格式 | `() => {}` 报 `result is not a function` | TS 中 `page.evaluate()` 必须用 IIFE:`(async () => { ... })()` |
|
||||
| 页面异步加载 | evaluate 拿到空数据(store state 还没更新) | 在 evaluate 内用 polling 等待数据出现,或增加 `wait` 时间 |
|
||||
| YAML 内嵌大段 JS | 调试困难,字符串转义问题 | 超过 10 行 JS 的命令改用 TS adapter |
|
||||
| **风控被拦截(伪200)** | 获取到的 JSON 里核心数据是 `""` (空串) | 极易被误判。必须添加断言!无核心数据立刻要求升级鉴权 Tier 并重新配置 Cookie |
|
||||
| **API 没找见** | `explore` 工具打分出来的都拿不到深层数据 | 点击页面按钮诱发懒加载数据,再结合 `getInterceptedRequests` 获取 |
|
||||
|
||||
---
|
||||
|
||||
@@ -586,9 +714,10 @@ git push
|
||||
opencli generate https://www.example.com --goal "hot"
|
||||
|
||||
# 或分步执行:
|
||||
opencli explore https://www.example.com --site mysite # 发现 API
|
||||
opencli synthesize mysite # 生成候选 YAML
|
||||
opencli verify mysite/hot --smoke # 冒烟测试
|
||||
opencli explore https://www.example.com --site mysite # 发现 API
|
||||
opencli explore https://www.example.com --auto --click "字幕,CC" # 模拟点击触发懒加载 API
|
||||
opencli synthesize mysite # 生成候选 YAML
|
||||
opencli verify mysite/hot --smoke # 冒烟测试
|
||||
```
|
||||
|
||||
生成的候选 YAML 保存在 `.opencli/explore/mysite/candidates/`,可直接复制到 `src/clis/mysite/` 并微调。
|
||||
+216
@@ -0,0 +1,216 @@
|
||||
# CLI-ONESHOT — 单点快速 CLI 生成
|
||||
|
||||
> 给一个 URL + 一句话描述,4 步生成一个 CLI 命令。
|
||||
> 完整探索式开发请看 [CLI-EXPLORER.md](./CLI-EXPLORER.md)。
|
||||
|
||||
---
|
||||
|
||||
## 输入
|
||||
|
||||
| 项目 | 示例 |
|
||||
|------|------|
|
||||
| **URL** | `https://x.com/jakevin7/lists` |
|
||||
| **Goal** | 获取我的 Twitter Lists |
|
||||
|
||||
---
|
||||
|
||||
## 流程
|
||||
|
||||
### Step 1: 打开页面 + 抓包
|
||||
|
||||
```
|
||||
1. browser_navigate → 打开目标 URL
|
||||
2. 等待 3-5 秒(让页面加载完、API 请求触发)
|
||||
3. browser_network_requests → 筛选 JSON API
|
||||
```
|
||||
|
||||
**关键**:只关注返回 `application/json` 的请求,忽略静态资源。
|
||||
如果没有自动触发 API,手动点击目标按钮/标签再抓一次。
|
||||
|
||||
### Step 2: 锁定一个接口
|
||||
|
||||
从抓包结果中找到**那个**目标 API。看这几个字段:
|
||||
|
||||
| 字段 | 关注什么 |
|
||||
|------|----------|
|
||||
| URL | API 路径 pattern(如 `/i/api/graphql/xxx/ListsManagePinTimeline`) |
|
||||
| Method | GET / POST |
|
||||
| Headers | 有 Cookie? Bearer? CSRF? 自定义签名? |
|
||||
| Response | 数据在哪个路径(如 `data.list.lists`) |
|
||||
|
||||
### Step 3: 验证接口能复现
|
||||
|
||||
在 `browser_evaluate` 中用 `fetch` 复现请求:
|
||||
|
||||
```javascript
|
||||
// Tier 2 (Cookie): 大多数情况
|
||||
fetch('/api/endpoint', { credentials: 'include' }).then(r => r.json())
|
||||
|
||||
// Tier 3 (Header): 如 Twitter 需要额外 header
|
||||
const ct0 = document.cookie.match(/ct0=([^;]+)/)?.[1];
|
||||
fetch('/api/endpoint', {
|
||||
headers: { 'Authorization': 'Bearer ...', 'X-Csrf-Token': ct0 },
|
||||
credentials: 'include'
|
||||
}).then(r => r.json())
|
||||
```
|
||||
|
||||
如果 fetch 能拿到数据 → 用 YAML 或简单 TS adapter。
|
||||
如果 fetch 拿不到(签名/风控)→ 用 intercept 策略。
|
||||
|
||||
### Step 4: 套模板,生成 adapter
|
||||
|
||||
根据 Step 3 判定的策略,选一个模板生成文件。
|
||||
|
||||
---
|
||||
|
||||
## 认证速查
|
||||
|
||||
```
|
||||
fetch(url) 直接能拿到? → Tier 1: public (YAML, browser: false)
|
||||
fetch(url, {credentials:'include'})? → Tier 2: cookie (YAML)
|
||||
加 Bearer/CSRF header 后拿到? → Tier 3: header (TS)
|
||||
都不行,但页面自己能请求成功? → Tier 4: intercept (TS, installInterceptor)
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 模板
|
||||
|
||||
### YAML — Cookie/Public(最简)
|
||||
|
||||
```yaml
|
||||
# src/clis/<site>/<name>.yaml
|
||||
site: mysite
|
||||
name: mycommand
|
||||
description: "一句话描述"
|
||||
domain: www.example.com
|
||||
strategy: cookie # 或 public (加 browser: false)
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.example.com/target-page
|
||||
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const res = await fetch('/api/target', { credentials: 'include' });
|
||||
const d = await res.json();
|
||||
return (d.data?.items || []).map(item => ({
|
||||
title: item.title,
|
||||
value: item.value,
|
||||
}));
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ index + 1 }}
|
||||
title: ${{ item.title }}
|
||||
value: ${{ item.value }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, title, value]
|
||||
```
|
||||
|
||||
### TS — Intercept(抓包模式)
|
||||
|
||||
```typescript
|
||||
// src/clis/<site>/<name>.ts
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'mysite',
|
||||
name: 'mycommand',
|
||||
description: '一句话描述',
|
||||
domain: 'www.example.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['rank', 'title', 'value'],
|
||||
func: async (page, kwargs) => {
|
||||
// 1. 导航
|
||||
await page.goto('https://www.example.com/target-page');
|
||||
await page.wait(3);
|
||||
|
||||
// 2. 注入拦截器(URL 子串匹配)
|
||||
await page.installInterceptor('target-api-keyword');
|
||||
|
||||
// 3. 触发 API(滚动/点击)
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// 4. 读取拦截的响应
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests?.length) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
const items = req.data?.data?.items || [];
|
||||
results.push(...items);
|
||||
}
|
||||
|
||||
return results.slice(0, kwargs.limit).map((item, i) => ({
|
||||
rank: i + 1,
|
||||
title: item.title || '',
|
||||
value: item.value || '',
|
||||
}));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
### TS — Header(如 Twitter GraphQL)
|
||||
|
||||
```typescript
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'mycommand',
|
||||
description: '一句话描述',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.HEADER,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['rank', 'name', 'value'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://x.com');
|
||||
const data = await page.evaluate(`(async () => {
|
||||
const ct0 = document.cookie.match(/ct0=([^;]+)/)?.[1];
|
||||
if (!ct0) return { error: 'Not logged in' };
|
||||
const bearer = 'AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D...';
|
||||
const res = await fetch('/i/api/graphql/QUERY_ID/Endpoint', {
|
||||
headers: {
|
||||
'Authorization': 'Bearer ' + decodeURIComponent(bearer),
|
||||
'X-Csrf-Token': ct0,
|
||||
'X-Twitter-Auth-Type': 'OAuth2Session',
|
||||
},
|
||||
credentials: 'include',
|
||||
});
|
||||
return res.json();
|
||||
})()`);
|
||||
// 解析 data...
|
||||
return [];
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 测试(必做)
|
||||
|
||||
```bash
|
||||
npm run build # 语法检查
|
||||
opencli list | grep mysite # 确认注册
|
||||
opencli mysite mycommand --limit 3 -v # 实际运行
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 就这样,没了
|
||||
|
||||
写完文件 → build → run → 提交。有问题再看 [CLI-EXPLORER.md](./CLI-EXPLORER.md)。
|
||||
@@ -0,0 +1,28 @@
|
||||
BSD 3-Clause License
|
||||
|
||||
Copyright (c) 2025, jackwener
|
||||
|
||||
Redistribution and use in source and binary forms, with or without
|
||||
modification, are permitted provided that the following conditions are met:
|
||||
|
||||
1. Redistributions of source code must retain the above copyright notice, this
|
||||
list of conditions and the following disclaimer.
|
||||
|
||||
2. Redistributions in binary form must reproduce the above copyright notice,
|
||||
this list of conditions and the following disclaimer in the documentation
|
||||
and/or other materials provided with the distribution.
|
||||
|
||||
3. Neither the name of the copyright holder nor the names of its
|
||||
contributors may be used to endorse or promote products derived from
|
||||
this software without specific prior written permission.
|
||||
|
||||
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
|
||||
AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
|
||||
IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
|
||||
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
|
||||
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
|
||||
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
|
||||
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
|
||||
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
|
||||
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
|
||||
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
|
||||
@@ -5,18 +5,81 @@
|
||||
|
||||
[中文文档](./README.zh-CN.md)
|
||||
|
||||
[](https://www.npmjs.com/package/@jackwener/opencli)
|
||||
[](https://www.npmjs.com/package/@jackwener/opencli)
|
||||
[](https://nodejs.org)
|
||||
[](./LICENSE)
|
||||
|
||||
A CLI tool that turns **any website** into a command-line interface. **28+ commands** across **16 sites** — bilibili, zhihu, xiaohongshu, twitter, reddit, github, v2ex, hackernews, bbc, weibo, boss, yahoo-finance, reuters, smzdm, ctrip, youtube — powered by browser session reuse and AI-native discovery.
|
||||
A CLI tool that turns **any website** into a command-line interface. **59 commands** across **18 sites** — bilibili, zhihu, xiaohongshu, twitter, reddit, xueqiu, github, v2ex, hackernews, bbc, weibo, boss, yahoo-finance, reuters, smzdm, ctrip, youtube, coupang — powered by browser session reuse and AI-native discovery.
|
||||
|
||||
## ✨ Highlights
|
||||
---
|
||||
|
||||
- 🔐 **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser
|
||||
- 🤖 **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies
|
||||
- 📝 **Declarative YAML** — Most adapters are ~30 lines of YAML pipeline
|
||||
- 🔌 **TypeScript escape hatch** — Complex adapters (XHR interception, GraphQL) in TS
|
||||
## Table of Contents
|
||||
|
||||
## 🚀 Quick Start
|
||||
- [Highlights](#highlights)
|
||||
- [Prerequisites](#prerequisites)
|
||||
- [Quick Start](#quick-start)
|
||||
- [Built-in Commands](#built-in-commands)
|
||||
- [Output Formats](#output-formats)
|
||||
- [For AI Agents (Developer Guide)](#for-ai-agents-developer-guide)
|
||||
- [Testing](#testing)
|
||||
- [Troubleshooting](#troubleshooting)
|
||||
- [Releasing New Versions](#releasing-new-versions)
|
||||
- [License](#license)
|
||||
|
||||
---
|
||||
|
||||
## Highlights
|
||||
|
||||
- **Account-safe** — Reuses Chrome's logged-in state; your credentials never leave the browser.
|
||||
- **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies.
|
||||
- **Dynamic Loader** — Simply drop `.ts` or `.yaml` adapters into the `clis/` folder for auto-registration.
|
||||
- **Dual-Engine Architecture** — Supports both YAML declarative data pipelines and robust browser runtime typescript injections.
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Node.js**: >= 18.0.0
|
||||
- **Chrome** running **and logged into the target site** (e.g. bilibili.com, zhihu.com, xiaohongshu.com).
|
||||
|
||||
> **⚠️ Important**: Browser commands reuse your Chrome login session. You must be logged into the target website in Chrome before running commands. If you get empty data or errors, check your login status first.
|
||||
|
||||
OpenCLI connects to your browser through the Playwright MCP Bridge extension.
|
||||
|
||||
### Playwright MCP Bridge Extension Setup
|
||||
|
||||
1. Install **[Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm)** extension in Chrome.
|
||||
2. Obtain your token by clicking the extension icon in the browser toolbar or from the extension settings page.
|
||||
|
||||
**You must configure this token in BOTH your MCP configuration AND system environment variables.**
|
||||
|
||||
First, add it to your MCP client config (e.g. Claude/Cursor):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"playwright": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@playwright/mcp@latest", "--extension"],
|
||||
"env": {
|
||||
"PLAYWRIGHT_MCP_EXTENSION_TOKEN": "<your-token-here>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
And, so that `opencli` commands can use it directly in the terminal, export it in your shell environment (e.g. `~/.zshrc`):
|
||||
|
||||
```bash
|
||||
export PLAYWRIGHT_MCP_EXTENSION_TOKEN="<your-token-here>"
|
||||
```
|
||||
|
||||
After configuring, run `opencli doctor` to verify your token is correctly set up across all locations:
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
```
|
||||
|
||||
## Quick Start
|
||||
|
||||
### Install via npm (recommended)
|
||||
|
||||
@@ -28,86 +91,75 @@ Then use directly:
|
||||
|
||||
```bash
|
||||
opencli list # See all commands
|
||||
opencli list -f yaml # List commands as YAML
|
||||
opencli hackernews top --limit 5 # Public API, no browser
|
||||
opencli bilibili hot --limit 5 # Browser command
|
||||
opencli zhihu hot -f json # JSON output
|
||||
opencli zhihu hot -f yaml # YAML output
|
||||
```
|
||||
|
||||
### Install from source
|
||||
### Install from source (for developers)
|
||||
|
||||
```bash
|
||||
git clone git@github.com:jackwener/opencli.git
|
||||
cd opencli && npm install
|
||||
npx tsx src/main.ts list
|
||||
cd opencli
|
||||
npm install
|
||||
npm run build
|
||||
npm link # Link binary globally
|
||||
opencli list # Now you can use it anywhere!
|
||||
```
|
||||
|
||||
### Update
|
||||
|
||||
```bash
|
||||
# npm global
|
||||
npm update -g @jackwener/opencli
|
||||
|
||||
# Or reinstall to latest
|
||||
npm install -g @jackwener/opencli@latest
|
||||
```
|
||||
|
||||
## 📋 Prerequisites
|
||||
|
||||
Browser commands need:
|
||||
1. **Chrome** running **and logged into the target site** (e.g. bilibili.com, zhihu.com, xiaohongshu.com)
|
||||
2. **[Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm)** extension installed
|
||||
3. Configure `PLAYWRIGHT_MCP_EXTENSION_TOKEN` (from the extension settings page) in your MCP config:
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"playwright": {
|
||||
"command": "npx",
|
||||
"args": ["@playwright/mcp@latest", "--extension"],
|
||||
"env": {
|
||||
"PLAYWRIGHT_MCP_EXTENSION_TOKEN": "<your-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
Public API commands (`hackernews`, `github search`, `v2ex`) need no browser at all.
|
||||
|
||||
> **⚠️ Important**: Browser commands reuse your Chrome login session. You must be logged into the target website in Chrome before running commands. If you get empty data or errors, check your login status first.
|
||||
|
||||
## 📦 Built-in Commands
|
||||
## Built-in Commands
|
||||
|
||||
| Site | Commands | Mode |
|
||||
|------|----------|------|
|
||||
| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `user-videos` | 🔐 Browser |
|
||||
| **bilibili** | `hot` `search` `me` `favorite` ... (11 commands) | 🔐 Browser |
|
||||
| **zhihu** | `hot` `search` `question` | 🔐 Browser |
|
||||
| **xiaohongshu** | `search` `notifications` `feed` | 🔐 Browser |
|
||||
| **twitter** | `trending` | 🔐 Browser |
|
||||
| **reddit** | `hot` | 🔐 Browser |
|
||||
| **xiaohongshu** | `search` `notifications` `feed` `me` `user` | 🔐 Browser |
|
||||
| **xueqiu** | `feed` `hot-stock` `hot` `search` `stock` `watchlist` | 🔐 Browser |
|
||||
| **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `following` `followers` `notifications` `post` `reply` `delete` `like` | 🔐 Browser |
|
||||
| **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 Browser |
|
||||
| **weibo** | `hot` | 🔐 Browser |
|
||||
| **boss** | `search` | 🔐 Browser |
|
||||
| **coupang** | `search` `add-to-cart` | 🔐 Browser |
|
||||
| **youtube** | `search` | 🔐 Browser |
|
||||
| **yahoo-finance** | `quote` | 🔐 Browser |
|
||||
| **reuters** | `search` | 🔐 Browser |
|
||||
| **smzdm** | `search` | 🔐 Browser |
|
||||
| **ctrip** | `search` | 🔐 Browser |
|
||||
| **github** | `trending` `search` | 🔐 / 🌐 |
|
||||
| **v2ex** | `hot` `latest` `topic` | 🌐 Public |
|
||||
| **github** | `search` | 🌐 Public |
|
||||
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 Public / 🔐 Browser |
|
||||
| **hackernews** | `top` | 🌐 Public |
|
||||
| **bbc** | `news` | 🌐 Public |
|
||||
|
||||
## 🎨 Output Formats
|
||||
## Output Formats
|
||||
|
||||
All built-in commands support `--format` / `-f` with `table`, `json`, `yaml`, `md`, and `csv`.
|
||||
The `list` command supports the same format options, and keeps `--json` for backward compatibility.
|
||||
|
||||
```bash
|
||||
opencli bilibili hot -f table # Default: rich table
|
||||
opencli bilibili hot -f json # JSON (pipe to jq, feed to AI)
|
||||
opencli list -f yaml # Command registry as YAML
|
||||
opencli bilibili hot -f table # Default: rich terminal table
|
||||
opencli bilibili hot -f json # JSON (pipe to jq or LLMs)
|
||||
opencli bilibili hot -f yaml # YAML (human-readable structured output)
|
||||
opencli bilibili hot -f md # Markdown
|
||||
opencli bilibili hot -f csv # CSV
|
||||
opencli bilibili hot -v # Verbose: show pipeline steps
|
||||
opencli bilibili hot -v # Verbose: show pipeline debug steps
|
||||
```
|
||||
|
||||
## 🧠 AI Agent Workflow
|
||||
## For AI Agents (Developer Guide)
|
||||
|
||||
If you are an AI assistant tasked with creating a new command adapter for `opencli`, please follow the AI Agent workflow below:
|
||||
|
||||
> **Quick mode**: To generate a single command for a specific page URL, see [CLI-ONESHOT.md](./CLI-ONESHOT.md) — just a URL + one-line goal, 4 steps done.
|
||||
|
||||
> **Full mode**: Before writing any adapter code, read [CLI-EXPLORER.md](./CLI-EXPLORER.md). It contains the complete browser exploration workflow, the 5-tier authentication strategy decision tree, and debugging guide.
|
||||
|
||||
```bash
|
||||
# 1. Deep Explore — discover APIs, infer capabilities, detect framework
|
||||
@@ -123,30 +175,46 @@ opencli generate https://example.com --goal "hot"
|
||||
opencli cascade https://api.example.com/data
|
||||
```
|
||||
|
||||
Explore outputs to `.opencli/explore/<site>/`:
|
||||
- `manifest.json` — site metadata, framework detection
|
||||
- `endpoints.json` — scored API endpoints with response schemas
|
||||
- `capabilities.json` — inferred capabilities with confidence scores
|
||||
- `auth.json` — authentication strategy recommendations
|
||||
Explore outputs to `.opencli/explore/<site>/` (manifest.json, endpoints.json, capabilities.json, auth.json).
|
||||
|
||||
## 🔧 Create New Commands
|
||||
## Testing
|
||||
|
||||
See **[SKILL.md](./SKILL.md)** for the full adapter guide (YAML pipeline + TypeScript).
|
||||
See **[TESTING.md](./TESTING.md)** for the full testing guide, including:
|
||||
|
||||
- Current test coverage (unit + ~52 E2E tests across all 18 sites)
|
||||
- How to run tests locally
|
||||
- How to add tests when creating new adapters
|
||||
- CI/CD pipeline with sharding
|
||||
- Headless browser mode (`OPENCLI_HEADLESS=1`)
|
||||
|
||||
```bash
|
||||
# Quick start
|
||||
npm run build
|
||||
npx vitest run # All tests
|
||||
npx vitest run src/ # Unit tests only
|
||||
npx vitest run tests/e2e/ # E2E tests
|
||||
```
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **"Failed to connect to Playwright MCP Bridge"**
|
||||
- Ensure the Playwright MCP extension is installed and **enabled** in your running Chrome.
|
||||
- Restart the Chrome browser if you just installed the extension.
|
||||
- **Empty data returns or 'Unauthorized' error**
|
||||
- Your login session in Chrome might have expired. Open a normal Chrome tab, navigate to the target site, and log in or refresh the page to prove you are human.
|
||||
- **Node API errors**
|
||||
- Make sure you are using Node.js >= 18. Some dependencies require modern Node APIs.
|
||||
|
||||
## Releasing New Versions
|
||||
|
||||
```bash
|
||||
# Bump version
|
||||
npm version patch # 0.1.0 → 0.1.1
|
||||
npm version minor # 0.1.0 → 0.2.0
|
||||
npm version major # 0.1.0 → 1.0.0
|
||||
|
||||
# Push tag to trigger GitHub Actions auto-release
|
||||
git push --follow-tags
|
||||
```
|
||||
|
||||
The CI will automatically build, create a GitHub release, and publish to npm.
|
||||
|
||||
## 📄 License
|
||||
## License
|
||||
|
||||
MIT
|
||||
[BSD-3-Clause](./LICENSE)
|
||||
|
||||
+115
-65
@@ -5,19 +5,80 @@
|
||||
|
||||
[English](./README.md)
|
||||
|
||||
[](https://www.npmjs.com/package/@jackwener/opencli)
|
||||
[](https://www.npmjs.com/package/@jackwener/opencli)
|
||||
[](https://nodejs.org)
|
||||
[](./LICENSE)
|
||||
|
||||
OpenCLI 通过 Chrome 浏览器 + [Playwright MCP Bridge](https://github.com/nichochar/playwright-mcp) 扩展,将任何网站变成命令行工具。不存密码、不泄 token,直接复用浏览器已登录状态。
|
||||
OpenCLI 将任何网站变成命令行工具。**59 个命令**覆盖 **18 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube、Coupang — 复用浏览器登录态,AI 驱动探索。
|
||||
|
||||
## ✨ 亮点
|
||||
---
|
||||
|
||||
- 🌐 **28+ 命令,16 个站点** — B站、知乎、小红书、Twitter、Reddit、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube
|
||||
- 🔐 **零风控** — 复用 Chrome 登录态,无需存储任何凭证
|
||||
- 🤖 **AI 原生** — `explore` 自动发现 API,`synthesize` 生成适配器,`cascade` 探测认证策略
|
||||
- 📝 **声明式 YAML** — 大部分适配器只需 ~30 行 YAML
|
||||
- 🔌 **TypeScript 扩展** — 复杂场景(XHR 拦截、GraphQL)可用 TS 编写
|
||||
## 目录
|
||||
|
||||
## 🚀 快速开始
|
||||
- [亮点](#亮点)
|
||||
- [前置要求](#前置要求)
|
||||
- [快速开始](#快速开始)
|
||||
- [内置命令](#内置命令)
|
||||
- [输出格式](#输出格式)
|
||||
- [致 AI Agent(开发者指南)](#致-ai-agent开发者指南)
|
||||
- [常见问题排查](#常见问题排查)
|
||||
- [版本发布](#版本发布)
|
||||
- [License](#license)
|
||||
|
||||
---
|
||||
|
||||
## 亮点
|
||||
|
||||
- **59 个命令,18 个站点** — B站、知乎、小红书、Twitter、Reddit、雪球(xueqiu)、GitHub、V2EX、Hacker News、BBC、微博、BOSS直聘、Yahoo Finance、路透社、什么值得买、携程、YouTube、Coupang
|
||||
- **零风控** — 复用 Chrome 登录态,无需存储任何凭证
|
||||
- **AI 原生** — `explore` 自动发现 API,`synthesize` 生成适配器,`cascade` 探测认证策略
|
||||
- **动态加载引擎** — 声明式的 `.yaml` 或者底层定制的 `.ts` 适配器,放入 `clis/` 文件夹即可自动注册生效
|
||||
|
||||
## 前置要求
|
||||
|
||||
- **Node.js**: >= 18.0.0
|
||||
- **Chrome** 浏览器正在运行,且**已登录目标网站**(如 bilibili.com、zhihu.com、xiaohongshu.com)
|
||||
|
||||
> **⚠️ 重要**:大多数命令复用你的 Chrome 登录状态。运行命令前,你必须已在 Chrome 中打开目标网站并完成登录。如果获取到空数据或报错,请先检查你的浏览器登录状态。
|
||||
|
||||
OpenCLI 通过 Playwright MCP Bridge 扩展与你的浏览器通信。
|
||||
|
||||
### Playwright MCP Bridge 扩展配置
|
||||
|
||||
1. 安装 **[Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm)** 扩展
|
||||
2. 在浏览器插件栏点击该插件,或者在插件设置页获取你的 Extension Token。
|
||||
|
||||
**你必须将这个 Token 同时配置到你的 MCP 配置文件 AND 环境变量中。**
|
||||
|
||||
首先,配置你的 MCP 客户端(如 Claude/Cursor 等):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"playwright": {
|
||||
"command": "npx",
|
||||
"args": ["-y", "@playwright/mcp@latest", "--extension"],
|
||||
"env": {
|
||||
"PLAYWRIGHT_MCP_EXTENSION_TOKEN": "<你的-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
并且,为了让 `opencli` 命令行也能直接使用它,你必须在你的终端系统环境变量中导出它(建议写进 `~/.zshrc` 或 `~/.bashrc`):
|
||||
|
||||
```bash
|
||||
export PLAYWRIGHT_MCP_EXTENSION_TOKEN="<你的-token>"
|
||||
```
|
||||
|
||||
配置完成后,运行 `opencli doctor` 检测你的 Token 是否在所有位置都正确配置:
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
```
|
||||
|
||||
## 快速开始
|
||||
|
||||
### npm 全局安装(推荐)
|
||||
|
||||
@@ -29,86 +90,75 @@ npm install -g @jackwener/opencli
|
||||
|
||||
```bash
|
||||
opencli list # 查看所有命令
|
||||
opencli list -f yaml # 以 YAML 列出所有命令
|
||||
opencli hackernews top --limit 5 # 公共 API,无需浏览器
|
||||
opencli bilibili hot --limit 5 # 浏览器命令
|
||||
opencli zhihu hot -f json # JSON 输出
|
||||
opencli zhihu hot -f yaml # YAML 输出
|
||||
```
|
||||
|
||||
### 从源码安装
|
||||
### 从源码安装(面向开发者)
|
||||
|
||||
```bash
|
||||
git clone git@github.com:jackwener/opencli.git
|
||||
cd opencli && npm install
|
||||
npx tsx src/main.ts list
|
||||
cd opencli
|
||||
npm install
|
||||
npm run build
|
||||
npm link # 链接到全局环境
|
||||
opencli list # 可以在任何地方使用了!
|
||||
```
|
||||
|
||||
### 更新
|
||||
|
||||
```bash
|
||||
# npm 全局更新
|
||||
npm update -g @jackwener/opencli
|
||||
|
||||
# 或直接安装最新版
|
||||
npm install -g @jackwener/opencli@latest
|
||||
```
|
||||
|
||||
## 📋 前置要求
|
||||
|
||||
浏览器命令需要:
|
||||
1. **Chrome** 浏览器正在运行,且**已登录目标网站**(如 bilibili.com、zhihu.com、xiaohongshu.com)
|
||||
2. 安装 **[Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm)** 扩展
|
||||
3. 在 MCP 配置中设置 `PLAYWRIGHT_MCP_EXTENSION_TOKEN`(从扩展设置页获取):
|
||||
|
||||
```json
|
||||
{
|
||||
"mcpServers": {
|
||||
"playwright": {
|
||||
"command": "npx",
|
||||
"args": ["@playwright/mcp@latest", "--extension"],
|
||||
"env": {
|
||||
"PLAYWRIGHT_MCP_EXTENSION_TOKEN": "<your-token>"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
公共 API 命令(`hackernews`、`github search`、`v2ex`)无需浏览器。
|
||||
|
||||
> **⚠️ 重要**:浏览器命令复用你的 Chrome 登录状态。运行命令前,你必须已在 Chrome 中登录目标网站。如果获取到空数据或报错,请先检查登录状态。
|
||||
|
||||
## 📦 内置命令
|
||||
## 内置命令
|
||||
|
||||
| 站点 | 命令 | 模式 |
|
||||
|------|------|------|
|
||||
| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `user-videos` | 🔐 浏览器 |
|
||||
| **bilibili** | `hot` `search` `me` `favorite` ...(共11个) | 🔐 浏览器 |
|
||||
| **zhihu** | `hot` `search` `question` | 🔐 浏览器 |
|
||||
| **xiaohongshu** | `search` `notifications` `feed` | 🔐 浏览器 |
|
||||
| **twitter** | `trending` | 🔐 浏览器 |
|
||||
| **reddit** | `hot` | 🔐 浏览器 |
|
||||
| **xiaohongshu** | `search` `notifications` `feed` `me` `user` | 🔐 浏览器 |
|
||||
| **xueqiu** | `feed` `hot-stock` `hot` `search` `stock` `watchlist` | 🔐 浏览器 |
|
||||
| **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `following` `followers` `notifications` `post` `reply` `delete` `like` | 🔐 浏览器 |
|
||||
| **reddit** | `hot` `frontpage` `search` `subreddit` | 🔐 浏览器 |
|
||||
| **weibo** | `hot` | 🔐 浏览器 |
|
||||
| **boss** | `search` | 🔐 浏览器 |
|
||||
| **coupang** | `search` `add-to-cart` | 🔐 浏览器 |
|
||||
| **youtube** | `search` | 🔐 浏览器 |
|
||||
| **yahoo-finance** | `quote` | 🔐 浏览器 |
|
||||
| **reuters** | `search` | 🔐 浏览器 |
|
||||
| **smzdm** | `search` | 🔐 浏览器 |
|
||||
| **ctrip** | `search` | 🔐 浏览器 |
|
||||
| **github** | `trending` `search` | 🔐 / 🌐 |
|
||||
| **v2ex** | `hot` `latest` `topic` | 🌐 公共 API |
|
||||
| **github** | `search` | 🌐 公共 API |
|
||||
| **v2ex** | `hot` `latest` `topic` `daily` `me` `notifications` | 🌐 公共 API / 🔐 浏览器 |
|
||||
| **hackernews** | `top` | 🌐 公共 API |
|
||||
| **bbc** | `news` | 🌐 公共 API |
|
||||
|
||||
## 🎨 输出格式
|
||||
## 输出格式
|
||||
|
||||
所有内置命令都支持 `--format` / `-f`,可选值为 `table`、`json`、`yaml`、`md`、`csv`。
|
||||
`list` 命令也支持同样的格式参数,同时继续兼容 `--json`。
|
||||
|
||||
```bash
|
||||
opencli bilibili hot -f table # 默认:表格
|
||||
opencli bilibili hot -f json # JSON(可 pipe 给 jq 或 AI agent)
|
||||
opencli list -f yaml # 用 YAML 列出命令注册表
|
||||
opencli bilibili hot -f table # 默认:富文本表格
|
||||
opencli bilibili hot -f json # JSON(适合传给 jq 或者各类 AI Agent)
|
||||
opencli bilibili hot -f yaml # YAML(更适合人类直接阅读)
|
||||
opencli bilibili hot -f md # Markdown
|
||||
opencli bilibili hot -f csv # CSV
|
||||
opencli bilibili hot -v # 详细模式:展示 pipeline 每步数据
|
||||
opencli bilibili hot -v # 详细模式:展示管线执行步骤调试信息
|
||||
```
|
||||
|
||||
## 🧠 AI Agent 工作流
|
||||
## 致 AI Agent(开发者指南)
|
||||
|
||||
如果你是一个被要求查阅代码并编写新 `opencli` 适配器的 AI,请遵守以下工作流。
|
||||
|
||||
> **快速模式**:只想为某个页面快速生成一个命令?看 [CLI-ONESHOT.md](./CLI-ONESHOT.md) — 给一个 URL + 一句话描述,4 步搞定。
|
||||
|
||||
> **完整模式**:在编写任何新代码前,先阅读 [CLI-EXPLORER.md](./CLI-EXPLORER.md)。它包含完整的适配器探索开发指南、API 探测流程、5级认证策略以及常见陷阱。
|
||||
|
||||
```bash
|
||||
# 1. Deep Explore — 网络拦截 → 响应分析 → 能力推理 → 框架检测
|
||||
@@ -124,28 +174,28 @@ opencli generate https://example.com --goal "hot"
|
||||
opencli cascade https://api.example.com/data
|
||||
```
|
||||
|
||||
探索结果输出到 `.opencli/explore/<site>/`:
|
||||
- `manifest.json` — 站点元数据、框架检测结果
|
||||
- `endpoints.json` — 评分排序的 API 端点,含响应 schema
|
||||
- `capabilities.json` — 推理出的能力及置信度
|
||||
- `auth.json` — 认证策略建议
|
||||
探索结果输出到 `.opencli/explore/<site>/`。
|
||||
|
||||
## 🔧 创建新命令
|
||||
## 常见问题排查
|
||||
|
||||
查看 **[SKILL.md](./SKILL.md)** 了解完整的适配器开发指南(YAML pipeline + TypeScript)。
|
||||
- **"Failed to connect to Playwright MCP Bridge"** 报错
|
||||
- 确保你当前的 Chrome 已安装且**开启了** Playwright MCP Bridge 浏览器插件。
|
||||
- 如果是刚装完插件,需要重启 Chrome 浏览器。
|
||||
- **返回空数据,或者报错 "Unauthorized"**
|
||||
- Chrome 里的登录态可能已经过期(甚至被要求过滑动验证码)。请打开当前 Chrome 页面,在新标签页重新手工登录或刷新该页面。
|
||||
- **Node API 错误 (如 parseArgs, fs 等)**
|
||||
- 确保 Node.js 版本 `>= 18`。旧版不支持我们使用的现代核心库 API。
|
||||
|
||||
## 版本发布
|
||||
|
||||
```bash
|
||||
# 升级版本号
|
||||
npm version patch # 0.1.0 → 0.1.1
|
||||
npm version minor # 0.1.0 → 0.2.0
|
||||
npm version major # 0.1.0 → 1.0.0
|
||||
|
||||
# 推送 tag,GitHub Actions 自动发 release 并发布到 npm
|
||||
# 推送 tag,GitHub Actions 将自动执行发版和 npm 发布
|
||||
git push --follow-tags
|
||||
```
|
||||
|
||||
## 📄 License
|
||||
## License
|
||||
|
||||
MIT
|
||||
[BSD-3-Clause](./LICENSE)
|
||||
|
||||
@@ -1,15 +1,20 @@
|
||||
---
|
||||
name: opencli
|
||||
description: "OpenCLI — Make any website your CLI. Zero risk, AI-powered, reuse Chrome login."
|
||||
version: 0.1.0
|
||||
version: 0.5.1
|
||||
author: jackwener
|
||||
tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, reddit, xiaohongshu, AI, agent]
|
||||
tags: [cli, browser, web, mcp, playwright, bilibili, zhihu, twitter, github, v2ex, hackernews, reddit, xiaohongshu, xueqiu, AI, agent]
|
||||
---
|
||||
|
||||
# OpenCLI
|
||||
|
||||
> Make any website your CLI. Reuse Chrome login, zero risk, AI-powered discovery.
|
||||
|
||||
> [!CAUTION]
|
||||
> **AI Agent 必读:创建或修改任何适配器之前,你必须先阅读 [CLI-EXPLORER.md](./CLI-EXPLORER.md)!**
|
||||
> 该文档包含完整的 API 发现工作流(必须使用 Playwright MCP Bridge 浏览器探索)、5 级认证策略决策树、平台 SDK 速查表、`tap` 步骤调试流程、分页 API 模板、级联请求模式、以及常见陷阱。
|
||||
> **本文件(SKILL.md)仅提供命令参考和简化模板,不足以正确开发适配器。**
|
||||
|
||||
## Install & Run
|
||||
|
||||
```bash
|
||||
@@ -29,8 +34,7 @@ npm update -g @jackwener/opencli
|
||||
|
||||
Browser commands require:
|
||||
1. Chrome browser running **(logged into target sites)**
|
||||
2. [Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm) extension
|
||||
3. Configure `PLAYWRIGHT_MCP_EXTENSION_TOKEN` (from extension settings) in your MCP config
|
||||
2. [Playwright MCP Bridge](https://chromewebstore.google.com/detail/playwright-mcp-bridge/mmlmfjhmonkocbjadbfplnigmagldckm) extension installed and configured
|
||||
|
||||
> **Note**: You must be logged into the target website in Chrome before running commands. Tabs opened during command execution are auto-closed afterwards.
|
||||
|
||||
@@ -49,6 +53,10 @@ opencli bilibili favorite # 我的收藏
|
||||
opencli bilibili history --limit 20 # 观看历史
|
||||
opencli bilibili feed --limit 10 # 动态时间线
|
||||
opencli bilibili user-videos --uid 12345 # 用户投稿
|
||||
opencli bilibili subtitle --bvid BV1xxx # 获取视频字幕 (支持 --lang zh-CN)
|
||||
opencli bilibili dynamic --limit 10 # 动态
|
||||
opencli bilibili ranking --limit 10 # 排行榜
|
||||
opencli bilibili following --limit 20 # 我的关注列表 (支持 --uid 查看他人)
|
||||
|
||||
# 知乎 (browser)
|
||||
opencli zhihu hot --limit 10 # 知乎热榜
|
||||
@@ -59,22 +67,41 @@ opencli zhihu question --id 34816524 # 问题详情和回答
|
||||
opencli xiaohongshu search --keyword "美食" # 搜索笔记
|
||||
opencli xiaohongshu notifications # 通知(mentions/likes/connections)
|
||||
opencli xiaohongshu feed --limit 10 # 推荐 Feed
|
||||
opencli xiaohongshu me # 我的信息
|
||||
opencli xiaohongshu user --uid xxx # 用户主页
|
||||
|
||||
# GitHub (trending=browser, search=public)
|
||||
opencli github trending --limit 10 # GitHub Trending
|
||||
# 雪球 Xueqiu (browser)
|
||||
opencli xueqiu hot-stock --limit 10 # 雪球热门股票榜
|
||||
opencli xueqiu stock --symbol SH600519 # 查看股票实时行情
|
||||
opencli xueqiu watchlist # 获取自选股/持仓列表
|
||||
opencli xueqiu feed # 我的关注 timeline
|
||||
opencli xueqiu hot --limit 10 # 雪球热榜
|
||||
opencli xueqiu search --keyword "特斯拉" # 搜索
|
||||
|
||||
# GitHub (public)
|
||||
opencli github search --keyword "cli" # 搜索仓库
|
||||
|
||||
# Twitter/X (browser)
|
||||
opencli twitter trending --limit 10 # 热门话题
|
||||
opencli twitter bookmarks --limit 20 # 获取收藏的书签推文
|
||||
opencli twitter search --keyword "AI" # 搜索推文
|
||||
opencli twitter profile --username elonmusk # 用户资料
|
||||
opencli twitter timeline --limit 20 # 时间线
|
||||
|
||||
# Reddit (browser)
|
||||
opencli reddit hot --limit 10 # 热门帖子
|
||||
opencli reddit hot --subreddit programming # 指定子版块
|
||||
opencli reddit frontpage --limit 10 # 首页
|
||||
opencli reddit search --keyword "AI" # 搜索
|
||||
opencli reddit subreddit --name rust # 子版块浏览
|
||||
|
||||
# V2EX (public)
|
||||
# V2EX (public + browser)
|
||||
opencli v2ex hot --limit 10 # 热门话题
|
||||
opencli v2ex latest --limit 10 # 最新话题
|
||||
opencli v2ex topic --id 1024 # 主题详情
|
||||
opencli v2ex daily # 每日签到 (browser)
|
||||
opencli v2ex me # 我的信息 (browser)
|
||||
opencli v2ex notifications --limit 10 # 通知 (browser)
|
||||
|
||||
# Hacker News (public)
|
||||
opencli hackernews top --limit 10 # Top stories
|
||||
@@ -109,6 +136,7 @@ opencli ctrip search --query "三亚" # 搜索目的地
|
||||
```bash
|
||||
opencli list # List all commands
|
||||
opencli list --json # JSON output
|
||||
opencli list -f yaml # YAML output
|
||||
opencli validate # Validate all CLI definitions
|
||||
opencli validate bilibili # Validate specific site
|
||||
```
|
||||
@@ -128,17 +156,23 @@ opencli generate <url> --goal "hot"
|
||||
# Strategy Cascade: auto-probe PUBLIC → COOKIE → HEADER
|
||||
opencli cascade <api-url>
|
||||
|
||||
# Verify: smoke-test a generated adapter
|
||||
opencli verify <site/name> --smoke
|
||||
# Explore with interactive fuzzing (click buttons to trigger lazy APIs)
|
||||
opencli explore <url> --auto --click "字幕,CC,评论"
|
||||
|
||||
# Verify: validate adapter definitions
|
||||
opencli verify
|
||||
```
|
||||
|
||||
## Output Formats
|
||||
|
||||
All commands support `--format` / `-f`:
|
||||
All built-in commands support `--format` / `-f` with `table`, `json`, `yaml`, `md`, and `csv`.
|
||||
The `list` command supports the same formats and also keeps `--json` as a compatibility alias.
|
||||
|
||||
```bash
|
||||
opencli list -f yaml # YAML command registry
|
||||
opencli bilibili hot -f table # Default: rich table
|
||||
opencli bilibili hot -f json # JSON (pipe to jq, feed to AI agent)
|
||||
opencli bilibili hot -f yaml # YAML (readable structured output)
|
||||
opencli bilibili hot -f md # Markdown
|
||||
opencli bilibili hot -f csv # CSV
|
||||
```
|
||||
@@ -151,6 +185,15 @@ opencli bilibili hot -v # Show each pipeline step and data flow
|
||||
|
||||
## Creating Adapters
|
||||
|
||||
> [!TIP]
|
||||
> **快速模式**:如果你只想为一个具体页面生成一个命令,直接看 [CLI-ONESHOT.md](./CLI-ONESHOT.md)。
|
||||
> 只需要一个 URL + 一句话描述,4 步搞定。
|
||||
|
||||
> [!IMPORTANT]
|
||||
> **完整模式 — 在写任何代码之前,先阅读 [CLI-EXPLORER.md](./CLI-EXPLORER.md)。**
|
||||
> 它包含:① AI Agent 浏览器探索工作流(必须用 Playwright MCP 抓包验证 API)② 认证策略决策树 ③ 平台 SDK(如 Bilibili 的 `apiGet`/`fetchJson`)④ YAML vs TS 选择指南 ⑤ `tap` 步骤调试方法 ⑥ 级联请求模板 ⑦ 常见陷阱表。
|
||||
> **下方仅为简化模板参考,直接使用极易踩坑。**
|
||||
|
||||
### YAML Pipeline (declarative, recommended)
|
||||
|
||||
Create `src/clis/<site>/<name>.yaml`:
|
||||
@@ -209,7 +252,7 @@ pipeline:
|
||||
|
||||
### TypeScript Adapter (programmatic)
|
||||
|
||||
Create `src/clis/<site>/<name>.ts` and import in `clis/index.ts`:
|
||||
Create `src/clis/<site>/<name>.ts`. It will be automatically dynamically loaded (DO NOT manually import it in `index.ts`):
|
||||
|
||||
```typescript
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
@@ -217,27 +260,33 @@ import { cli, Strategy } from '../../registry.js';
|
||||
cli({
|
||||
site: 'mysite',
|
||||
name: 'search',
|
||||
strategy: Strategy.COOKIE,
|
||||
strategy: Strategy.INTERCEPT, // Or COOKIE
|
||||
args: [{ name: 'keyword', required: true }],
|
||||
columns: ['rank', 'title', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.mysite.com');
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const res = await fetch('/api/search?q=${kwargs.keyword}', {
|
||||
credentials: 'include'
|
||||
});
|
||||
return await res.json();
|
||||
})()
|
||||
`);
|
||||
return data.items.map((item, i) => ({
|
||||
await page.goto('https://www.mysite.com/search');
|
||||
|
||||
// Inject native XHR/Fetch interceptor hook
|
||||
await page.installInterceptor('/api/search');
|
||||
|
||||
// Auto scroll down to trigger lazy loading
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// Retrieve intercepted JSON payloads
|
||||
const requests = await page.getInterceptedRequests();
|
||||
|
||||
let results = [];
|
||||
for (const req of requests) {
|
||||
results.push(...req.data.items);
|
||||
}
|
||||
return results.map((item, i) => ({
|
||||
rank: i + 1, title: item.title, url: item.url,
|
||||
}));
|
||||
},
|
||||
});
|
||||
```
|
||||
|
||||
**When to use TS**: XHR interception (小红书), cookie extraction (Twitter ct0), Wbi signing (Bilibili), auto-pagination, complex data transforms.
|
||||
**When to use TS**: XHR interception (`page.installInterceptor`), infinite scrolling (`page.autoScroll`), cookie extraction, complex data transforms (like GraphQL unwrapping).
|
||||
|
||||
## Pipeline Steps
|
||||
|
||||
@@ -292,7 +341,6 @@ ${{ index + 1 }}
|
||||
| `OPENCLI_BROWSER_CONNECT_TIMEOUT` | 30 | Browser connection timeout (sec) |
|
||||
| `OPENCLI_BROWSER_COMMAND_TIMEOUT` | 45 | Command execution timeout (sec) |
|
||||
| `OPENCLI_BROWSER_EXPLORE_TIMEOUT` | 120 | Explore timeout (sec) |
|
||||
| `OPENCLI_EXTENSION_LOCK_TIMEOUT` | 120 | Extension lock timeout (sec) |
|
||||
| `PLAYWRIGHT_MCP_EXTENSION_TOKEN` | — | Auto-approve extension connection |
|
||||
|
||||
## Troubleshooting
|
||||
@@ -300,7 +348,6 @@ ${{ index + 1 }}
|
||||
| Issue | Solution |
|
||||
|-------|----------|
|
||||
| `npx not found` | Install Node.js: `brew install node` |
|
||||
| `Timed out connecting to browser` | 1) Chrome must be open 2) Install MCP Bridge extension 3) Click to approve |
|
||||
| `Extension lock timed out` | Another opencli command is running; browser commands run serially |
|
||||
| `Timed out connecting to browser` | 1) Chrome must be open 2) Install MCP Bridge extension and configure token |
|
||||
| `Target page context` error | Add `navigate:` step before `evaluate:` in YAML |
|
||||
| Empty table data | Check if evaluate returns JSON string (MCP parsing) or data path is wrong |
|
||||
|
||||
+233
@@ -0,0 +1,233 @@
|
||||
# Testing Guide
|
||||
|
||||
> 面向开发者和 AI Agent 的测试参考手册。
|
||||
|
||||
## 目录
|
||||
|
||||
- [测试架构](#测试架构)
|
||||
- [当前覆盖范围](#当前覆盖范围)
|
||||
- [本地运行测试](#本地运行测试)
|
||||
- [如何添加新测试](#如何添加新测试)
|
||||
- [CI/CD 流水线](#cicd-流水线)
|
||||
- [浏览器模式](#浏览器模式)
|
||||
- [站点兼容性](#站点兼容性)
|
||||
|
||||
---
|
||||
|
||||
## 测试架构
|
||||
|
||||
测试分为三层,全部使用 **vitest** 运行:
|
||||
|
||||
```
|
||||
tests/
|
||||
├── e2e/ # E2E 集成测试(子进程运行真实 CLI)
|
||||
│ ├── helpers.ts # runCli() 共享工具
|
||||
│ ├── public-commands.test.ts # 公开 API 命令(无需浏览器)
|
||||
│ ├── browser-public.test.ts # 浏览器命令(公开数据)
|
||||
│ ├── browser-auth.test.ts # 需登录命令(graceful failure 测试)
|
||||
│ ├── management.test.ts # 管理命令(list, validate, verify, help)
|
||||
│ └── output-formats.test.ts # 输出格式(json/yaml/csv/md)
|
||||
├── smoke/ # 烟雾测试(仅定时 / 手动触发)
|
||||
│ └── api-health.test.ts # 外部 API 可用性检测
|
||||
src/
|
||||
├── *.test.ts # 单元测试(已有 8 个)
|
||||
```
|
||||
|
||||
| 层 | 位置 | 运行方式 | 用途 |
|
||||
|---|---|---|---|
|
||||
| 单元测试 | `src/**/*.test.ts` | `npx vitest run src/` | 内部模块逻辑 |
|
||||
| E2E 测试 | `tests/e2e/*.test.ts` | `npx vitest run tests/e2e/` | 真实 CLI 命令执行 |
|
||||
| 烟雾测试 | `tests/smoke/*.test.ts` | `npx vitest run tests/smoke/` | 外部 API 健康 |
|
||||
|
||||
---
|
||||
|
||||
## 当前覆盖范围
|
||||
|
||||
### 单元测试(8 个文件)
|
||||
|
||||
| 文件 | 覆盖内容 |
|
||||
|---|---|
|
||||
| `browser.test.ts` | JSON-RPC、tab 管理、extension/standalone 模式切换 |
|
||||
| `engine.test.ts` | 命令发现与执行 |
|
||||
| `registry.test.ts` | 命令注册与策略分配 |
|
||||
| `output.test.ts` | 输出格式渲染 |
|
||||
| `doctor.test.ts` | Token 诊断 |
|
||||
| `coupang.test.ts` | 数据归一化 |
|
||||
| `pipeline/template.test.ts` | 模板表达式求值 |
|
||||
| `pipeline/transform.test.ts` | 数据变换步骤 |
|
||||
|
||||
### E2E 测试(~52 个用例)
|
||||
|
||||
| 文件 | 覆盖站点/功能 | 测试数 |
|
||||
|---|---|---|
|
||||
| `public-commands.test.ts` | hackernews/top, v2ex/hot, v2ex/latest, v2ex/topic | 5 |
|
||||
| `browser-public.test.ts` | bbc, bilibili×3, weibo, zhihu×2, reddit×2, twitter, xueqiu×2, reuters, youtube, smzdm, boss, ctrip, coupang, xiaohongshu, yahoo-finance, v2ex/daily | 21 |
|
||||
| `browser-auth.test.ts` | bilibili/me,dynamic,favorite,history,following + twitter/bookmarks,timeline,notifications + v2ex/me,notifications + xueqiu/feed,watchlist + xiaohongshu/feed,notifications | 14 |
|
||||
| `management.test.ts` | list×5 格式, validate×3 级别, verify, --version, --help, unknown cmd | 12 |
|
||||
| `output-formats.test.ts` | json, yaml, csv, md 格式验证 | 5 |
|
||||
|
||||
### 烟雾测试
|
||||
|
||||
公开 API 可用性(hackernews, v2ex×2, v2ex/topic)+ 全站点注册完整性检查。
|
||||
|
||||
---
|
||||
|
||||
## 本地运行测试
|
||||
|
||||
### 前置条件
|
||||
|
||||
```bash
|
||||
npm ci # 安装依赖
|
||||
npm run build # 编译(E2E 测试需要 dist/main.js)
|
||||
```
|
||||
|
||||
### 运行命令
|
||||
|
||||
```bash
|
||||
# 全部单元测试
|
||||
npx vitest run src/
|
||||
|
||||
# 全部 E2E 测试(会真实调用外部 API)
|
||||
npx vitest run tests/e2e/
|
||||
|
||||
# 单个测试文件
|
||||
npx vitest run tests/e2e/management.test.ts
|
||||
|
||||
# 全部测试(单元 + E2E)
|
||||
npx vitest run
|
||||
|
||||
# 烟雾测试
|
||||
npx vitest run tests/smoke/
|
||||
|
||||
# watch 模式(开发时推荐)
|
||||
npx vitest src/
|
||||
```
|
||||
|
||||
### 浏览器命令本地测试须知
|
||||
|
||||
- 无 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 时,opencli 自动启动一个独立浏览器实例
|
||||
- `browser-public.test.ts` 使用 `tryBrowserCommand()`,站点反爬导致空数据时 warn + pass
|
||||
- `browser-auth.test.ts` 验证 **graceful failure**(不 crash 不 hang 即通过)
|
||||
- 如需测试完整登录态,保持 Chrome 登录态 + 设置 `PLAYWRIGHT_MCP_EXTENSION_TOKEN`,手动跑对应测试
|
||||
|
||||
---
|
||||
|
||||
## 如何添加新测试
|
||||
|
||||
### 新增 YAML Adapter(如 `src/clis/producthunt/trending.yaml`)
|
||||
|
||||
1. **无需额外操作**:`validate` 测试会自动覆盖 YAML 结构验证
|
||||
2. 根据 adapter 类型,在对应文件加一个 `it()` block:
|
||||
|
||||
```typescript
|
||||
// 如果 browser: false(公开 API)→ tests/e2e/public-commands.test.ts
|
||||
it('producthunt trending returns data', async () => {
|
||||
const { stdout, code } = await runCli(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
```
|
||||
|
||||
```typescript
|
||||
// 如果 browser: true 但可公开访问 → tests/e2e/browser-public.test.ts
|
||||
it('producthunt trending returns data', async () => {
|
||||
const data = await tryBrowserCommand(['producthunt', 'trending', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'producthunt trending');
|
||||
}, 60_000);
|
||||
```
|
||||
|
||||
```typescript
|
||||
// 如果 browser: true 且需登录 → tests/e2e/browser-auth.test.ts
|
||||
it('producthunt me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['producthunt', 'me', '-f', 'json'], 'producthunt me');
|
||||
}, 60_000);
|
||||
```
|
||||
|
||||
### 新增管理命令(如 `opencli export`)
|
||||
|
||||
在 `tests/e2e/management.test.ts` 添加测试。
|
||||
|
||||
### 新增内部模块
|
||||
|
||||
在 `src/` 下对应位置创建 `*.test.ts`。
|
||||
|
||||
### 决策流程图
|
||||
|
||||
```
|
||||
新增功能 → 是内部模块? → 是 → src/ 下加 *.test.ts
|
||||
↓ 否
|
||||
是 CLI 命令? → browser: false? → tests/e2e/public-commands.test.ts
|
||||
↓ true
|
||||
公开数据? → tests/e2e/browser-public.test.ts
|
||||
↓ 需登录
|
||||
tests/e2e/browser-auth.test.ts
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## CI/CD 流水线
|
||||
|
||||
### ci.yml(主流水线)
|
||||
|
||||
| Job | 触发条件 | 内容 |
|
||||
|---|---|---|
|
||||
| **build** | push/PR to main,dev | typecheck + build |
|
||||
| **unit-test** | push/PR to main,dev | 单元测试,2 shard 并行 |
|
||||
| **smoke-test** | 每周一 08:00 UTC / 手动 | xvfb + real Chrome,外部 API 健康检查 |
|
||||
|
||||
### e2e-headed.yml(E2E 测试)
|
||||
|
||||
| Job | 触发条件 | 内容 |
|
||||
|---|---|---|
|
||||
| **e2e-headed** | push/PR to main,dev | xvfb + real Chrome,全部 E2E 测试 |
|
||||
|
||||
E2E 使用 `browser-actions/setup-chrome` 安装真实 Chrome,配合 `xvfb-run` 提供虚拟显示器,以 headed 模式运行浏览器。
|
||||
|
||||
### Sharding
|
||||
|
||||
单元测试使用 vitest 内置 shard:
|
||||
|
||||
```yaml
|
||||
strategy:
|
||||
matrix:
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- run: npx vitest run src/ --shard=${{ matrix.shard }}/2
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 浏览器模式
|
||||
|
||||
opencli 根据 `PLAYWRIGHT_MCP_EXTENSION_TOKEN` 环境变量自动选择模式:
|
||||
|
||||
| 条件 | 模式 | MCP 参数 | 使用场景 |
|
||||
|---|---|---|---|
|
||||
| Token 已设置 | Extension 模式 | `--extension` | 本地用户,连接已登录的 Chrome |
|
||||
| Token 未设置 | Standalone 模式 | (无特殊 flag) | CI 或无扩展环境,自启浏览器 |
|
||||
|
||||
CI 中使用 `OPENCLI_BROWSER_EXECUTABLE_PATH` 指定真实 Chrome 路径:
|
||||
|
||||
```yaml
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 站点兼容性
|
||||
|
||||
在 GitHub Actions 美国 runner 上,部分站点因地域限制或登录要求返回空数据。E2E 测试对这些站点使用 warn + pass 策略,不影响 CI 绿灯。
|
||||
|
||||
| 站点 | CI 状态 | 限制原因 |
|
||||
|---|---|---|
|
||||
| hackernews, bbc, v2ex | ✅ 返回数据 | 无限制 |
|
||||
| yahoo-finance | ✅ 返回数据 | 无限制 |
|
||||
| bilibili, zhihu, weibo, xiaohongshu | ⚠️ 空数据 | 地域限制(中国站点) |
|
||||
| reddit, twitter, youtube | ⚠️ 空数据 | 需登录或 cookie |
|
||||
| smzdm, boss, ctrip, coupang, xueqiu | ⚠️ 空数据 | 地域限制 / 需登录 |
|
||||
|
||||
> 使用 self-hosted runner(国内服务器)可解决地域限制问题。
|
||||
Generated
+6
-3
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.2.0",
|
||||
"version": "0.5.2",
|
||||
"lockfileVersion": 3,
|
||||
"requires": true,
|
||||
"packages": {
|
||||
"": {
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.2.0",
|
||||
"license": "MIT",
|
||||
"version": "0.5.2",
|
||||
"license": "BSD-3-Clause",
|
||||
"dependencies": {
|
||||
"chalk": "^5.3.0",
|
||||
"cli-table3": "^0.6.5",
|
||||
@@ -24,6 +24,9 @@
|
||||
"tsx": "^4.19.3",
|
||||
"typescript": "^5.8.2",
|
||||
"vitest": "^4.1.0"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
}
|
||||
},
|
||||
"node_modules/@colors/colors": {
|
||||
|
||||
+7
-3
@@ -1,10 +1,13 @@
|
||||
{
|
||||
"name": "@jackwener/opencli",
|
||||
"version": "0.2.0",
|
||||
"version": "0.5.2",
|
||||
"publishConfig": {
|
||||
"access": "public"
|
||||
},
|
||||
"description": "Make any website your CLI. AI-powered.",
|
||||
"engines": {
|
||||
"node": ">=18.0.0"
|
||||
},
|
||||
"type": "module",
|
||||
"main": "dist/main.js",
|
||||
"bin": {
|
||||
@@ -12,7 +15,8 @@
|
||||
},
|
||||
"scripts": {
|
||||
"dev": "tsx src/main.ts",
|
||||
"build": "tsc && npm run clean-yaml && npm run copy-yaml",
|
||||
"build": "tsc && npm run clean-yaml && npm run copy-yaml && npm run build-manifest",
|
||||
"build-manifest": "node dist/build-manifest.js || true",
|
||||
"clean-yaml": "find dist/clis -name '*.yaml' -o -name '*.yml' 2>/dev/null | xargs rm -f",
|
||||
"copy-yaml": "find src/clis -name '*.yaml' -o -name '*.yml' | while read f; do d=\"dist/${f#src/}\"; mkdir -p \"$(dirname \"$d\")\"; cp \"$f\" \"$d\"; done",
|
||||
"start": "node dist/main.js",
|
||||
@@ -30,7 +34,7 @@
|
||||
"playwright"
|
||||
],
|
||||
"author": "jackwener",
|
||||
"license": "MIT",
|
||||
"license": "BSD-3-Clause",
|
||||
"repository": {
|
||||
"type": "git",
|
||||
"url": "git+https://github.com/jackwener/opencli.git"
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { PlaywrightMCP, __test__ } from './browser.js';
|
||||
|
||||
describe('browser helpers', () => {
|
||||
it('creates JSON-RPC requests with unique ids', () => {
|
||||
const first = __test__.createJsonRpcRequest('tools/call', { name: 'browser_tabs' });
|
||||
const second = __test__.createJsonRpcRequest('tools/call', { name: 'browser_snapshot' });
|
||||
|
||||
expect(second.id).toBe(first.id + 1);
|
||||
expect(first.message).toContain(`"id":${first.id}`);
|
||||
expect(second.message).toContain(`"id":${second.id}`);
|
||||
});
|
||||
|
||||
it('extracts tab entries from string snapshots', () => {
|
||||
const entries = __test__.extractTabEntries('Tab 0 https://example.com\nTab 1 Chrome Extension');
|
||||
|
||||
expect(entries).toEqual([
|
||||
{ index: 0, identity: 'https://example.com' },
|
||||
{ index: 1, identity: 'Chrome Extension' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('extracts tab entries from MCP markdown format', () => {
|
||||
const entries = __test__.extractTabEntries(
|
||||
'- 0: (current) [Playwright MCP extension](chrome-extension://abc/connect.html)\n- 1: [知乎 - 首页](https://www.zhihu.com/)'
|
||||
);
|
||||
|
||||
expect(entries).toEqual([
|
||||
{ index: 0, identity: '(current) [Playwright MCP extension](chrome-extension://abc/connect.html)' },
|
||||
{ index: 1, identity: '[知乎 - 首页](https://www.zhihu.com/)' },
|
||||
]);
|
||||
});
|
||||
|
||||
it('closes only tabs that were opened during the session', () => {
|
||||
const tabsToClose = __test__.diffTabIndexes(
|
||||
['https://example.com', 'Chrome Extension'],
|
||||
[
|
||||
{ index: 0, identity: 'https://example.com' },
|
||||
{ index: 1, identity: 'Chrome Extension' },
|
||||
{ index: 2, identity: 'https://target.example/page' },
|
||||
{ index: 3, identity: 'chrome-extension://bridge' },
|
||||
],
|
||||
);
|
||||
|
||||
expect(tabsToClose).toEqual([3, 2]);
|
||||
});
|
||||
|
||||
it('keeps only the tail of stderr buffers', () => {
|
||||
expect(__test__.appendLimited('12345', '67890', 8)).toBe('34567890');
|
||||
});
|
||||
|
||||
it('builds extension MCP args in local mode (no CI)', () => {
|
||||
const savedCI = process.env.CI;
|
||||
delete process.env.CI;
|
||||
try {
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
executablePath: '/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--extension',
|
||||
'--executable-path',
|
||||
'/mnt/c/Program Files/Google/Chrome/Application/chrome.exe',
|
||||
]);
|
||||
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--extension',
|
||||
]);
|
||||
} finally {
|
||||
if (savedCI !== undefined) {
|
||||
process.env.CI = savedCI;
|
||||
} else {
|
||||
delete process.env.CI;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('builds standalone MCP args in CI mode', () => {
|
||||
const savedCI = process.env.CI;
|
||||
process.env.CI = 'true';
|
||||
try {
|
||||
// CI mode: no --extension — browser launches in standalone headed mode
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
]);
|
||||
|
||||
expect(__test__.buildMcpArgs({
|
||||
mcpPath: '/tmp/cli.js',
|
||||
executablePath: '/usr/bin/chromium',
|
||||
})).toEqual([
|
||||
'/tmp/cli.js',
|
||||
'--executable-path',
|
||||
'/usr/bin/chromium',
|
||||
]);
|
||||
} finally {
|
||||
if (savedCI !== undefined) {
|
||||
process.env.CI = savedCI;
|
||||
} else {
|
||||
delete process.env.CI;
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
it('times out slow promises', async () => {
|
||||
await expect(__test__.withTimeoutMs(new Promise(() => {}), 10, 'timeout')).rejects.toThrow('timeout');
|
||||
});
|
||||
});
|
||||
|
||||
describe('PlaywrightMCP state', () => {
|
||||
it('transitions to closed after close()', async () => {
|
||||
const mcp = new PlaywrightMCP();
|
||||
|
||||
expect(mcp.state).toBe('idle');
|
||||
|
||||
await mcp.close();
|
||||
|
||||
expect(mcp.state).toBe('closed');
|
||||
});
|
||||
|
||||
it('rejects connect() after the session has been closed', async () => {
|
||||
const mcp = new PlaywrightMCP();
|
||||
await mcp.close();
|
||||
|
||||
await expect(mcp.connect()).rejects.toThrow('Playwright MCP session is closed');
|
||||
});
|
||||
|
||||
it('rejects connect() while already connecting', async () => {
|
||||
const mcp = new PlaywrightMCP();
|
||||
(mcp as any)._state = 'connecting';
|
||||
|
||||
await expect(mcp.connect()).rejects.toThrow('Playwright MCP is already connecting');
|
||||
});
|
||||
|
||||
it('rejects connect() while closing', async () => {
|
||||
const mcp = new PlaywrightMCP();
|
||||
(mcp as any)._state = 'closing';
|
||||
|
||||
await expect(mcp.connect()).rejects.toThrow('Playwright MCP is closing');
|
||||
});
|
||||
|
||||
|
||||
});
|
||||
+506
-119
@@ -1,28 +1,119 @@
|
||||
/**
|
||||
* Browser interaction via Playwright MCP Bridge extension.
|
||||
* Connects to an existing Chrome browser through the extension's stdio JSON-RPC.
|
||||
* Connects to an existing Chrome browser through the extension.
|
||||
*/
|
||||
|
||||
import { spawn, execSync, type ChildProcess } from 'node:child_process';
|
||||
import { createHash } from 'node:crypto';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { formatSnapshot } from './snapshotFormatter.js';
|
||||
import { PKG_VERSION } from './version.js';
|
||||
import { normalizeEvaluateSource } from './pipeline/template.js';
|
||||
import { generateInterceptorJs, generateReadInterceptedJs } from './interceptor.js';
|
||||
import { withTimeoutMs } from './runtime.js';
|
||||
|
||||
// Read version from package.json (single source of truth)
|
||||
const __browser_dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const PKG_VERSION = (() => { try { return JSON.parse(fs.readFileSync(path.resolve(__browser_dirname, '..', 'package.json'), 'utf-8')).version; } catch { return '0.0.0'; } })();
|
||||
|
||||
const EXTENSION_LOCK_TIMEOUT = parseInt(process.env.OPENCLI_EXTENSION_LOCK_TIMEOUT ?? '120', 10);
|
||||
const EXTENSION_LOCK_POLL = parseInt(process.env.OPENCLI_EXTENSION_LOCK_POLL_INTERVAL ?? '1', 10);
|
||||
const CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
|
||||
const LOCK_DIR = path.join(os.tmpdir(), 'opencli-mcp-lock');
|
||||
const STDERR_BUFFER_LIMIT = 16 * 1024;
|
||||
const INITIAL_TABS_TIMEOUT_MS = 1500;
|
||||
const TAB_CLEANUP_TIMEOUT_MS = 2000;
|
||||
let _cachedMcpServerPath: string | null | undefined;
|
||||
|
||||
type ConnectFailureKind = 'missing-token' | 'extension-timeout' | 'extension-not-installed' | 'mcp-init' | 'process-exit' | 'unknown';
|
||||
type PlaywrightMCPState = 'idle' | 'connecting' | 'connected' | 'closing' | 'closed';
|
||||
|
||||
type ConnectFailureInput = {
|
||||
kind: ConnectFailureKind;
|
||||
timeout: number;
|
||||
hasExtensionToken: boolean;
|
||||
tokenFingerprint?: string | null;
|
||||
stderr?: string;
|
||||
exitCode?: number | null;
|
||||
rawMessage?: string;
|
||||
};
|
||||
|
||||
export function getTokenFingerprint(token: string | undefined): string | null {
|
||||
if (!token) return null;
|
||||
return createHash('sha256').update(token).digest('hex').slice(0, 8);
|
||||
}
|
||||
|
||||
export function formatBrowserConnectError(input: ConnectFailureInput): Error {
|
||||
const stderr = input.stderr?.trim();
|
||||
const suffix = stderr ? `\n\nMCP stderr:\n${stderr}` : '';
|
||||
const tokenHint = input.tokenFingerprint ? ` Token fingerprint: ${input.tokenFingerprint}.` : '';
|
||||
|
||||
if (input.kind === 'missing-token') {
|
||||
return new Error(
|
||||
'Failed to connect to Playwright MCP Bridge: PLAYWRIGHT_MCP_EXTENSION_TOKEN is not set.\n\n' +
|
||||
'Without this token, Chrome will show a manual approval dialog for every new MCP connection. ' +
|
||||
'Copy the token from the Playwright MCP Bridge extension and set it in BOTH your shell environment and MCP client config.' +
|
||||
suffix,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.kind === 'extension-not-installed') {
|
||||
return new Error(
|
||||
'Failed to connect to Playwright MCP Bridge: the browser extension did not attach.\n\n' +
|
||||
'Make sure Chrome is running and the "Playwright MCP Bridge" extension is installed and enabled. ' +
|
||||
'If Chrome shows an approval dialog, click Allow.' +
|
||||
suffix,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.kind === 'extension-timeout') {
|
||||
const likelyCause = input.hasExtensionToken
|
||||
? `The most likely cause is that PLAYWRIGHT_MCP_EXTENSION_TOKEN does not match the token currently shown by the browser extension.${tokenHint} Re-copy the token from the extension and update BOTH your shell environment and MCP client config.`
|
||||
: 'PLAYWRIGHT_MCP_EXTENSION_TOKEN is not configured, so the extension may be waiting for manual approval.';
|
||||
return new Error(
|
||||
`Timed out connecting to Playwright MCP Bridge (${input.timeout}s).\n\n` +
|
||||
`${likelyCause} If a browser prompt is visible, click Allow.` +
|
||||
suffix,
|
||||
);
|
||||
}
|
||||
|
||||
if (input.kind === 'mcp-init') {
|
||||
return new Error(`Failed to initialize Playwright MCP: ${input.rawMessage ?? 'unknown error'}${suffix}`);
|
||||
}
|
||||
|
||||
if (input.kind === 'process-exit') {
|
||||
return new Error(
|
||||
`Playwright MCP process exited before the browser connection was established${input.exitCode == null ? '' : ` (code ${input.exitCode})`}.` +
|
||||
suffix,
|
||||
);
|
||||
}
|
||||
|
||||
return new Error(input.rawMessage ?? 'Failed to connect to browser');
|
||||
}
|
||||
|
||||
function inferConnectFailureKind(args: {
|
||||
hasExtensionToken: boolean;
|
||||
stderr: string;
|
||||
rawMessage?: string;
|
||||
exited?: boolean;
|
||||
}): ConnectFailureKind {
|
||||
const haystack = `${args.rawMessage ?? ''}\n${args.stderr}`.toLowerCase();
|
||||
|
||||
if (!args.hasExtensionToken)
|
||||
return 'missing-token';
|
||||
if (haystack.includes('extension connection timeout') || haystack.includes('playwright mcp bridge'))
|
||||
return 'extension-not-installed';
|
||||
if (args.rawMessage?.startsWith('MCP init failed:'))
|
||||
return 'mcp-init';
|
||||
if (args.exited)
|
||||
return 'process-exit';
|
||||
return 'extension-timeout';
|
||||
}
|
||||
|
||||
// JSON-RPC helpers
|
||||
let _nextId = 1;
|
||||
function jsonRpcRequest(method: string, params: Record<string, any> = {}): string {
|
||||
return JSON.stringify({ jsonrpc: '2.0', id: _nextId++, method, params }) + '\n';
|
||||
function createJsonRpcRequest(method: string, params: Record<string, any> = {}): { id: number; message: string } {
|
||||
const id = _nextId++;
|
||||
return {
|
||||
id,
|
||||
message: JSON.stringify({ jsonrpc: '2.0', id, method, params }) + '\n',
|
||||
};
|
||||
}
|
||||
|
||||
import type { IPage } from './types.js';
|
||||
@@ -31,11 +122,10 @@ import type { IPage } from './types.js';
|
||||
* Page abstraction wrapping JSON-RPC calls to Playwright MCP.
|
||||
*/
|
||||
export class Page implements IPage {
|
||||
constructor(private _send: (msg: string) => void, private _recv: () => Promise<any>) {}
|
||||
constructor(private _request: (method: string, params?: Record<string, any>) => Promise<any>) {}
|
||||
|
||||
async call(method: string, params: Record<string, any> = {}): Promise<any> {
|
||||
this._send(jsonRpcRequest(method, params));
|
||||
const resp = await this._recv();
|
||||
const resp = await this._request(method, params);
|
||||
if (resp.error) throw new Error(`page.${method}: ${resp.error.message ?? JSON.stringify(resp.error)}`);
|
||||
// Extract text content from MCP result
|
||||
const result = resp.result;
|
||||
@@ -68,23 +158,10 @@ export class Page implements IPage {
|
||||
|
||||
async evaluate(js: string): Promise<any> {
|
||||
// Normalize IIFE format to function format expected by MCP browser_evaluate
|
||||
const normalized = this.normalizeEval(js);
|
||||
const normalized = normalizeEvaluateSource(js);
|
||||
return this.call('tools/call', { name: 'browser_evaluate', arguments: { function: normalized } });
|
||||
}
|
||||
|
||||
private normalizeEval(source: string): string {
|
||||
const s = source.trim();
|
||||
if (!s) return '() => undefined';
|
||||
// IIFE: (async () => {...})() → wrap as () => (...)
|
||||
if (s.startsWith('(') && s.endsWith(')()')) return `() => (${s})`;
|
||||
// Already a function/arrow
|
||||
if (/^(async\s+)?\([^)]*\)\s*=>/.test(s)) return s;
|
||||
if (/^(async\s+)?[A-Za-z_][A-Za-z0-9_]*\s*=>/.test(s)) return s;
|
||||
if (s.startsWith('function ') || s.startsWith('async function ')) return s;
|
||||
// Raw expression → wrap
|
||||
return `() => (${s})`;
|
||||
}
|
||||
|
||||
async snapshot(opts: { interactive?: boolean; compact?: boolean; maxDepth?: number; raw?: boolean } = {}): Promise<any> {
|
||||
const raw = await this.call('tools/call', { name: 'browser_snapshot', arguments: {} });
|
||||
if (opts.raw) return raw;
|
||||
@@ -104,8 +181,13 @@ export class Page implements IPage {
|
||||
await this.call('tools/call', { name: 'browser_press_key', arguments: { key } });
|
||||
}
|
||||
|
||||
async wait(seconds: number): Promise<void> {
|
||||
await this.call('tools/call', { name: 'browser_wait_for', arguments: { time: seconds } });
|
||||
async wait(options: number | { text?: string; time?: number; timeout?: number }): Promise<void> {
|
||||
if (typeof options === 'number') {
|
||||
await this.call('tools/call', { name: 'browser_wait_for', arguments: { time: options } });
|
||||
} else {
|
||||
// Pass directly to native wait_for, which supports natively awaiting text strings without heavy DOM polling
|
||||
await this.call('tools/call', { name: 'browser_wait_for', arguments: options });
|
||||
}
|
||||
}
|
||||
|
||||
async tabs(): Promise<any> {
|
||||
@@ -135,42 +217,197 @@ export class Page implements IPage {
|
||||
async scroll(direction: string = 'down', amount: number = 500): Promise<void> {
|
||||
await this.call('tools/call', { name: 'browser_press_key', arguments: { key: direction === 'down' ? 'PageDown' : 'PageUp' } });
|
||||
}
|
||||
|
||||
async autoScroll(options: { times?: number; delayMs?: number } = {}): Promise<void> {
|
||||
const times = options.times ?? 3;
|
||||
const delayMs = options.delayMs ?? 2000;
|
||||
const js = `
|
||||
async () => {
|
||||
const maxTimes = ${times};
|
||||
const maxWaitMs = ${delayMs};
|
||||
for (let i = 0; i < maxTimes; i++) {
|
||||
const lastHeight = document.body.scrollHeight;
|
||||
window.scrollTo(0, lastHeight);
|
||||
await new Promise(resolve => {
|
||||
let timeoutId;
|
||||
const observer = new MutationObserver(() => {
|
||||
if (document.body.scrollHeight > lastHeight) {
|
||||
clearTimeout(timeoutId);
|
||||
observer.disconnect();
|
||||
setTimeout(resolve, 100); // Small debounce for rendering
|
||||
}
|
||||
});
|
||||
observer.observe(document.body, { childList: true, subtree: true });
|
||||
timeoutId = setTimeout(() => {
|
||||
observer.disconnect();
|
||||
resolve(null);
|
||||
}, maxWaitMs);
|
||||
});
|
||||
}
|
||||
}
|
||||
`;
|
||||
await this.evaluate(js);
|
||||
}
|
||||
|
||||
async installInterceptor(pattern: string): Promise<void> {
|
||||
await this.evaluate(generateInterceptorJs(JSON.stringify(pattern), {
|
||||
arrayName: '__opencli_xhr',
|
||||
patchGuard: '__opencli_interceptor_patched',
|
||||
}));
|
||||
}
|
||||
|
||||
async getInterceptedRequests(): Promise<any[]> {
|
||||
const result = await this.evaluate(generateReadInterceptedJs('__opencli_xhr'));
|
||||
return result || [];
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Playwright MCP process manager.
|
||||
*/
|
||||
export class PlaywrightMCP {
|
||||
private static _activeInsts: Set<PlaywrightMCP> = new Set();
|
||||
private static _cleanupRegistered = false;
|
||||
|
||||
private static _registerGlobalCleanup() {
|
||||
if (this._cleanupRegistered) return;
|
||||
this._cleanupRegistered = true;
|
||||
const cleanup = () => {
|
||||
for (const inst of this._activeInsts) {
|
||||
if (inst._proc && !inst._proc.killed) {
|
||||
try { inst._proc.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
}
|
||||
};
|
||||
process.on('exit', cleanup);
|
||||
process.on('SIGINT', () => { cleanup(); process.exit(130); });
|
||||
process.on('SIGTERM', () => { cleanup(); process.exit(143); });
|
||||
}
|
||||
|
||||
private _proc: ChildProcess | null = null;
|
||||
private _buffer = '';
|
||||
private _waiters: Array<(data: any) => void> = [];
|
||||
private _lockAcquired = false;
|
||||
private _initialTabCount = 0;
|
||||
private _pending = new Map<number, { resolve: (data: any) => void; reject: (error: Error) => void }>();
|
||||
private _initialTabIdentities: string[] = [];
|
||||
private _closingPromise: Promise<void> | null = null;
|
||||
private _state: PlaywrightMCPState = 'idle';
|
||||
|
||||
private _page: Page | null = null;
|
||||
|
||||
get state(): PlaywrightMCPState {
|
||||
return this._state;
|
||||
}
|
||||
|
||||
private _sendRequest(method: string, params: Record<string, any> = {}): Promise<any> {
|
||||
return new Promise<any>((resolve, reject) => {
|
||||
if (!this._proc?.stdin?.writable) {
|
||||
reject(new Error('Playwright MCP process is not writable'));
|
||||
return;
|
||||
}
|
||||
const { id, message } = createJsonRpcRequest(method, params);
|
||||
this._pending.set(id, { resolve, reject });
|
||||
this._proc.stdin.write(message, (err) => {
|
||||
if (!err) return;
|
||||
this._pending.delete(id);
|
||||
reject(err);
|
||||
});
|
||||
});
|
||||
}
|
||||
|
||||
private _rejectPendingRequests(error: Error): void {
|
||||
const pending = [...this._pending.values()];
|
||||
this._pending.clear();
|
||||
for (const waiter of pending) waiter.reject(error);
|
||||
}
|
||||
|
||||
private _resetAfterFailedConnect(): void {
|
||||
const proc = this._proc;
|
||||
this._page = null;
|
||||
this._proc = null;
|
||||
this._buffer = '';
|
||||
this._initialTabIdentities = [];
|
||||
this._rejectPendingRequests(new Error('Playwright MCP connect failed'));
|
||||
PlaywrightMCP._activeInsts.delete(this);
|
||||
if (proc && !proc.killed) {
|
||||
try { proc.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
}
|
||||
|
||||
async connect(opts: { timeout?: number } = {}): Promise<Page> {
|
||||
await this._acquireLock();
|
||||
const timeout = opts.timeout ?? CONNECT_TIMEOUT;
|
||||
if (this._state === 'connected' && this._page) return this._page;
|
||||
if (this._state === 'connecting') throw new Error('Playwright MCP is already connecting');
|
||||
if (this._state === 'closing') throw new Error('Playwright MCP is closing');
|
||||
if (this._state === 'closed') throw new Error('Playwright MCP session is closed');
|
||||
|
||||
const mcpPath = findMcpServerPath();
|
||||
if (!mcpPath) throw new Error('Playwright MCP server not found. Install: npm install -D @playwright/mcp');
|
||||
|
||||
return new Promise<Page>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(`Timed out connecting to browser (${timeout}s)`)), timeout * 1000);
|
||||
PlaywrightMCP._registerGlobalCleanup();
|
||||
PlaywrightMCP._activeInsts.add(this);
|
||||
this._state = 'connecting';
|
||||
const timeout = opts.timeout ?? CONNECT_TIMEOUT;
|
||||
|
||||
this._proc = spawn('node', [mcpPath, '--extension'], {
|
||||
return new Promise<Page>((resolve, reject) => {
|
||||
const isDebug = process.env.DEBUG?.includes('opencli:mcp');
|
||||
const debugLog = (msg: string) => isDebug && console.error(`[opencli:mcp] ${msg}`);
|
||||
const useExtension = !!process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
|
||||
const extensionToken = process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN;
|
||||
const tokenFingerprint = getTokenFingerprint(extensionToken);
|
||||
let stderrBuffer = '';
|
||||
let settled = false;
|
||||
|
||||
const settleError = (kind: ConnectFailureKind, extra: { rawMessage?: string; exitCode?: number | null } = {}) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
this._state = 'idle';
|
||||
clearTimeout(timer);
|
||||
this._resetAfterFailedConnect();
|
||||
reject(formatBrowserConnectError({
|
||||
kind,
|
||||
timeout,
|
||||
hasExtensionToken: !!extensionToken,
|
||||
tokenFingerprint,
|
||||
stderr: stderrBuffer,
|
||||
exitCode: extra.exitCode,
|
||||
rawMessage: extra.rawMessage,
|
||||
}));
|
||||
};
|
||||
|
||||
const settleSuccess = (pageToResolve: Page) => {
|
||||
if (settled) return;
|
||||
settled = true;
|
||||
this._state = 'connected';
|
||||
clearTimeout(timer);
|
||||
resolve(pageToResolve);
|
||||
};
|
||||
|
||||
const timer = setTimeout(() => {
|
||||
debugLog('Connection timed out');
|
||||
settleError(inferConnectFailureKind({
|
||||
hasExtensionToken: !!extensionToken,
|
||||
stderr: stderrBuffer,
|
||||
}));
|
||||
}, timeout * 1000);
|
||||
|
||||
const mcpArgs = buildMcpArgs({
|
||||
mcpPath,
|
||||
executablePath: process.env.OPENCLI_BROWSER_EXECUTABLE_PATH,
|
||||
});
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error(`[opencli] Mode: ${useExtension ? 'extension' : 'standalone'}`);
|
||||
if (useExtension) console.error(`[opencli] Extension token: fingerprint ${tokenFingerprint}`);
|
||||
}
|
||||
debugLog(`Spawning node ${mcpArgs.join(' ')}`);
|
||||
|
||||
this._proc = spawn('node', mcpArgs, {
|
||||
stdio: ['pipe', 'pipe', 'pipe'],
|
||||
env: { ...process.env, ...(process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN ? { PLAYWRIGHT_MCP_EXTENSION_TOKEN: process.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN } : {}) },
|
||||
env: { ...process.env },
|
||||
});
|
||||
|
||||
// Increase max listeners to avoid warnings
|
||||
this._proc.setMaxListeners(20);
|
||||
if (this._proc.stdout) this._proc.stdout.setMaxListeners(20);
|
||||
|
||||
const page = new Page(
|
||||
(msg) => { if (this._proc?.stdin?.writable) this._proc.stdin.write(msg); },
|
||||
() => new Promise<any>((res) => { this._waiters.push(res); }),
|
||||
);
|
||||
const page = new Page((method, params = {}) => this._sendRequest(method, params));
|
||||
this._page = page;
|
||||
|
||||
this._proc.stdout?.on('data', (chunk: Buffer) => {
|
||||
@@ -179,107 +416,247 @@ export class PlaywrightMCP {
|
||||
this._buffer = lines.pop() ?? '';
|
||||
for (const line of lines) {
|
||||
if (!line.trim()) continue;
|
||||
debugLog(`RECV: ${line}`);
|
||||
try {
|
||||
const parsed = JSON.parse(line);
|
||||
const waiter = this._waiters.shift();
|
||||
if (waiter) waiter(parsed);
|
||||
} catch {}
|
||||
if (typeof parsed?.id === 'number') {
|
||||
const waiter = this._pending.get(parsed.id);
|
||||
if (waiter) {
|
||||
this._pending.delete(parsed.id);
|
||||
waiter.resolve(parsed);
|
||||
}
|
||||
}
|
||||
} catch (e) {
|
||||
debugLog(`Parse error: ${e}`);
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
this._proc.stderr?.on('data', () => {});
|
||||
this._proc.on('error', (err) => { clearTimeout(timer); reject(err); });
|
||||
this._proc.stderr?.on('data', (chunk: Buffer) => {
|
||||
const text = chunk.toString();
|
||||
stderrBuffer = appendLimited(stderrBuffer, text, STDERR_BUFFER_LIMIT);
|
||||
debugLog(`STDERR: ${text}`);
|
||||
});
|
||||
this._proc.on('error', (err) => {
|
||||
debugLog(`Subprocess error: ${err.message}`);
|
||||
this._rejectPendingRequests(new Error(`Playwright MCP process error: ${err.message}`));
|
||||
settleError('process-exit', { rawMessage: err.message });
|
||||
});
|
||||
this._proc.on('close', (code) => {
|
||||
debugLog(`Subprocess closed with code ${code}`);
|
||||
this._rejectPendingRequests(new Error(`Playwright MCP process exited before response${code == null ? '' : ` (code ${code})`}`));
|
||||
if (!settled) {
|
||||
settleError(inferConnectFailureKind({
|
||||
hasExtensionToken: !!extensionToken,
|
||||
stderr: stderrBuffer,
|
||||
exited: true,
|
||||
}), { exitCode: code });
|
||||
}
|
||||
});
|
||||
|
||||
// Initialize: send initialize request
|
||||
const initMsg = jsonRpcRequest('initialize', {
|
||||
debugLog('Waiting for initialize response...');
|
||||
this._sendRequest('initialize', {
|
||||
protocolVersion: '2024-11-05',
|
||||
capabilities: {},
|
||||
clientInfo: { name: 'opencli', version: PKG_VERSION },
|
||||
}).then((resp) => {
|
||||
debugLog('Got initialize response');
|
||||
if (resp.error) {
|
||||
settleError(inferConnectFailureKind({
|
||||
hasExtensionToken: !!extensionToken,
|
||||
stderr: stderrBuffer,
|
||||
rawMessage: `MCP init failed: ${resp.error.message}`,
|
||||
}), { rawMessage: resp.error.message });
|
||||
return;
|
||||
}
|
||||
|
||||
const initializedMsg = JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n';
|
||||
debugLog(`SEND: ${initializedMsg.trim()}`);
|
||||
this._proc?.stdin?.write(initializedMsg);
|
||||
|
||||
// Use tabs as a readiness probe and for tab cleanup bookkeeping.
|
||||
debugLog('Fetching initial tabs count...');
|
||||
withTimeoutMs(page.tabs(), INITIAL_TABS_TIMEOUT_MS, 'Timed out fetching initial tabs').then((tabs: any) => {
|
||||
debugLog(`Tabs response: ${typeof tabs === 'string' ? tabs : JSON.stringify(tabs)}`);
|
||||
this._initialTabIdentities = extractTabIdentities(tabs);
|
||||
settleSuccess(page);
|
||||
}).catch((err) => {
|
||||
debugLog(`Tabs fetch error: ${err.message}`);
|
||||
settleSuccess(page);
|
||||
});
|
||||
}).catch((err) => {
|
||||
debugLog(`Init promise rejected: ${err.message}`);
|
||||
settleError('mcp-init', { rawMessage: err.message });
|
||||
});
|
||||
this._proc.stdin?.write(initMsg);
|
||||
|
||||
// Wait for initialize response, then send initialized notification
|
||||
const origRecv = () => new Promise<any>((res) => { this._waiters.push(res); });
|
||||
origRecv().then((resp) => {
|
||||
if (resp.error) { clearTimeout(timer); reject(new Error(`MCP init failed: ${resp.error.message}`)); return; }
|
||||
this._proc?.stdin?.write(JSON.stringify({ jsonrpc: '2.0', method: 'notifications/initialized' }) + '\n');
|
||||
|
||||
// Get initial tab count for cleanup
|
||||
page.tabs().then((tabs: any) => {
|
||||
if (typeof tabs === 'string') {
|
||||
this._initialTabCount = (tabs.match(/Tab \d+/g) || []).length;
|
||||
} else if (Array.isArray(tabs)) {
|
||||
this._initialTabCount = tabs.length;
|
||||
}
|
||||
clearTimeout(timer);
|
||||
resolve(page);
|
||||
}).catch(() => { clearTimeout(timer); resolve(page); });
|
||||
}).catch((err) => { clearTimeout(timer); reject(err); });
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
async close(): Promise<void> {
|
||||
try {
|
||||
// Close tabs opened during this session (site tabs + extension tabs)
|
||||
if (this._page && this._proc && !this._proc.killed) {
|
||||
try {
|
||||
const tabs = await this._page.tabs();
|
||||
const tabStr = typeof tabs === 'string' ? tabs : JSON.stringify(tabs);
|
||||
const allTabs = tabStr.match(/Tab (\d+)/g) || [];
|
||||
const currentTabCount = allTabs.length;
|
||||
|
||||
// Close tabs in reverse order to avoid index shifting issues
|
||||
// Keep the original tabs that existed before the command started
|
||||
if (currentTabCount > this._initialTabCount && this._initialTabCount > 0) {
|
||||
for (let i = currentTabCount - 1; i >= this._initialTabCount; i--) {
|
||||
try { await this._page.closeTab(i); } catch {}
|
||||
if (this._closingPromise) return this._closingPromise;
|
||||
if (this._state === 'closed') return;
|
||||
this._state = 'closing';
|
||||
this._closingPromise = (async () => {
|
||||
try {
|
||||
// Extension mode opens bridge/session tabs that we can clean up best-effort.
|
||||
if (this._page && this._proc && !this._proc.killed) {
|
||||
try {
|
||||
const tabs = await withTimeoutMs(this._page.tabs(), TAB_CLEANUP_TIMEOUT_MS, 'Timed out fetching tabs during cleanup');
|
||||
const tabEntries = extractTabEntries(tabs);
|
||||
const tabsToClose = diffTabIndexes(this._initialTabIdentities, tabEntries);
|
||||
for (const index of tabsToClose) {
|
||||
try { await this._page.closeTab(index); } catch {}
|
||||
}
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
if (this._proc && !this._proc.killed) {
|
||||
this._proc.kill('SIGTERM');
|
||||
await new Promise<void>((res) => { this._proc?.on('exit', () => res()); setTimeout(res, 3000); });
|
||||
}
|
||||
} finally {
|
||||
this._page = null;
|
||||
this._releaseLock();
|
||||
}
|
||||
}
|
||||
|
||||
private async _acquireLock(): Promise<void> {
|
||||
const start = Date.now();
|
||||
while (true) {
|
||||
try { fs.mkdirSync(LOCK_DIR, { recursive: false }); this._lockAcquired = true; return; }
|
||||
catch (e: any) {
|
||||
if (e.code !== 'EEXIST') throw e;
|
||||
if ((Date.now() - start) / 1000 > EXTENSION_LOCK_TIMEOUT) {
|
||||
// Force remove stale lock
|
||||
try { fs.rmdirSync(LOCK_DIR); } catch {}
|
||||
continue;
|
||||
} catch {}
|
||||
}
|
||||
await new Promise(r => setTimeout(r, EXTENSION_LOCK_POLL * 1000));
|
||||
if (this._proc && !this._proc.killed) {
|
||||
this._proc.kill('SIGTERM');
|
||||
const exited = await new Promise<boolean>((res) => {
|
||||
let done = false;
|
||||
const finish = (value: boolean) => {
|
||||
if (done) return;
|
||||
done = true;
|
||||
res(value);
|
||||
};
|
||||
this._proc?.once('exit', () => finish(true));
|
||||
setTimeout(() => finish(false), 3000);
|
||||
});
|
||||
if (!exited && this._proc && !this._proc.killed) {
|
||||
try { this._proc.kill('SIGKILL'); } catch {}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
this._rejectPendingRequests(new Error('Playwright MCP session closed'));
|
||||
this._page = null;
|
||||
this._proc = null;
|
||||
this._state = 'closed';
|
||||
PlaywrightMCP._activeInsts.delete(this);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private _releaseLock(): void {
|
||||
if (this._lockAcquired) {
|
||||
try { fs.rmdirSync(LOCK_DIR); } catch {}
|
||||
this._lockAcquired = false;
|
||||
}
|
||||
})();
|
||||
return this._closingPromise;
|
||||
}
|
||||
}
|
||||
|
||||
function extractTabEntries(raw: any): Array<{ index: number; identity: string }> {
|
||||
if (Array.isArray(raw)) {
|
||||
return raw.map((tab: any, index: number) => ({
|
||||
index,
|
||||
identity: [
|
||||
tab?.id ?? '',
|
||||
tab?.url ?? '',
|
||||
tab?.title ?? '',
|
||||
tab?.name ?? '',
|
||||
].join('|'),
|
||||
}));
|
||||
}
|
||||
|
||||
if (typeof raw === 'string') {
|
||||
return raw
|
||||
.split('\n')
|
||||
.map(line => line.trim())
|
||||
.filter(Boolean)
|
||||
.map(line => {
|
||||
// Match actual Playwright MCP format: "- 0: (current) [title](url)" or "- 1: [title](url)"
|
||||
const mcpMatch = line.match(/^-\s+(\d+):\s*(.*)$/);
|
||||
if (mcpMatch) {
|
||||
return {
|
||||
index: parseInt(mcpMatch[1], 10),
|
||||
identity: mcpMatch[2].trim() || `tab-${mcpMatch[1]}`,
|
||||
};
|
||||
}
|
||||
// Legacy format: "Tab 0 ..."
|
||||
const legacyMatch = line.match(/Tab\s+(\d+)\s*(.*)$/);
|
||||
if (legacyMatch) {
|
||||
return {
|
||||
index: parseInt(legacyMatch[1], 10),
|
||||
identity: legacyMatch[2].trim() || `tab-${legacyMatch[1]}`,
|
||||
};
|
||||
}
|
||||
return null;
|
||||
})
|
||||
.filter((entry): entry is { index: number; identity: string } => entry !== null);
|
||||
}
|
||||
|
||||
return [];
|
||||
}
|
||||
|
||||
function extractTabIdentities(raw: any): string[] {
|
||||
return extractTabEntries(raw).map(tab => tab.identity);
|
||||
}
|
||||
|
||||
function diffTabIndexes(initialIdentities: string[], currentTabs: Array<{ index: number; identity: string }>): number[] {
|
||||
if (initialIdentities.length === 0 || currentTabs.length === 0) return [];
|
||||
const remaining = new Map<string, number>();
|
||||
for (const identity of initialIdentities) {
|
||||
remaining.set(identity, (remaining.get(identity) ?? 0) + 1);
|
||||
}
|
||||
|
||||
const tabsToClose: number[] = [];
|
||||
for (const tab of currentTabs) {
|
||||
const count = remaining.get(tab.identity) ?? 0;
|
||||
if (count > 0) {
|
||||
remaining.set(tab.identity, count - 1);
|
||||
continue;
|
||||
}
|
||||
tabsToClose.push(tab.index);
|
||||
}
|
||||
|
||||
return tabsToClose.sort((a, b) => b - a);
|
||||
}
|
||||
|
||||
function appendLimited(current: string, chunk: string, limit: number): string {
|
||||
const next = current + chunk;
|
||||
if (next.length <= limit) return next;
|
||||
return next.slice(-limit);
|
||||
}
|
||||
|
||||
function buildMcpArgs(input: { mcpPath: string; executablePath?: string | null }): string[] {
|
||||
const args = [input.mcpPath];
|
||||
if (!process.env.CI) {
|
||||
// Local: always connect to user's running Chrome via MCP Bridge extension
|
||||
args.push('--extension');
|
||||
}
|
||||
// CI: standalone mode — @playwright/mcp launches its own browser (headed by default).
|
||||
// xvfb provides a virtual display for headed mode in GitHub Actions.
|
||||
if (input.executablePath) {
|
||||
args.push('--executable-path', input.executablePath);
|
||||
}
|
||||
return args;
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
createJsonRpcRequest,
|
||||
extractTabEntries,
|
||||
diffTabIndexes,
|
||||
appendLimited,
|
||||
buildMcpArgs,
|
||||
withTimeoutMs,
|
||||
};
|
||||
|
||||
function findMcpServerPath(): string | null {
|
||||
if (_cachedMcpServerPath !== undefined) return _cachedMcpServerPath;
|
||||
|
||||
const envMcp = process.env.OPENCLI_MCP_SERVER_PATH;
|
||||
if (envMcp && fs.existsSync(envMcp)) {
|
||||
_cachedMcpServerPath = envMcp;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
|
||||
// Check local node_modules first (@playwright/mcp is the modern package)
|
||||
const localMcp = path.resolve('node_modules', '@playwright', 'mcp', 'cli.js');
|
||||
if (fs.existsSync(localMcp)) return localMcp;
|
||||
if (fs.existsSync(localMcp)) {
|
||||
_cachedMcpServerPath = localMcp;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
|
||||
// Check project-relative path
|
||||
const __dirname2 = path.dirname(fileURLToPath(import.meta.url));
|
||||
const projectMcp = path.resolve(__dirname2, '..', 'node_modules', '@playwright', 'mcp', 'cli.js');
|
||||
if (fs.existsSync(projectMcp)) return projectMcp;
|
||||
if (fs.existsSync(projectMcp)) {
|
||||
_cachedMcpServerPath = projectMcp;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
|
||||
// Check common locations
|
||||
const candidates = [
|
||||
@@ -291,13 +668,19 @@ function findMcpServerPath(): string | null {
|
||||
// Try npx resolution (legacy package name)
|
||||
try {
|
||||
const result = execSync('npx -y --package=@playwright/mcp which mcp-server-playwright 2>/dev/null', { encoding: 'utf-8', timeout: 10000 }).trim();
|
||||
if (result && fs.existsSync(result)) return result;
|
||||
if (result && fs.existsSync(result)) {
|
||||
_cachedMcpServerPath = result;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Try which
|
||||
try {
|
||||
const result = execSync('which mcp-server-playwright 2>/dev/null', { encoding: 'utf-8', timeout: 5000 }).trim();
|
||||
if (result && fs.existsSync(result)) return result;
|
||||
if (result && fs.existsSync(result)) {
|
||||
_cachedMcpServerPath = result;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
} catch {}
|
||||
|
||||
// Search in common npx cache
|
||||
@@ -305,9 +688,13 @@ function findMcpServerPath(): string | null {
|
||||
if (!fs.existsSync(base)) continue;
|
||||
try {
|
||||
const found = execSync(`find "${base}" -name "cli.js" -path "*playwright*mcp*" 2>/dev/null | head -1`, { encoding: 'utf-8', timeout: 5000 }).trim();
|
||||
if (found) return found;
|
||||
if (found) {
|
||||
_cachedMcpServerPath = found;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
|
||||
return null;
|
||||
_cachedMcpServerPath = null;
|
||||
return _cachedMcpServerPath;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,198 @@
|
||||
#!/usr/bin/env node
|
||||
/**
|
||||
* Build-time CLI manifest compiler.
|
||||
*
|
||||
* Scans all YAML/TS CLI definitions and pre-compiles them into a single
|
||||
* manifest.json for instant cold-start registration (no runtime YAML parsing).
|
||||
*
|
||||
* Usage: npx tsx src/build-manifest.ts
|
||||
* Output: dist/cli-manifest.json
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const CLIS_DIR = path.resolve(__dirname, 'clis');
|
||||
const OUTPUT = path.resolve(__dirname, '..', 'dist', 'cli-manifest.json');
|
||||
|
||||
interface ManifestEntry {
|
||||
site: string;
|
||||
name: string;
|
||||
description: string;
|
||||
domain?: string;
|
||||
strategy: string;
|
||||
browser: boolean;
|
||||
args: Array<{
|
||||
name: string;
|
||||
type?: string;
|
||||
default?: any;
|
||||
required?: boolean;
|
||||
help?: string;
|
||||
choices?: string[];
|
||||
}>;
|
||||
columns?: string[];
|
||||
pipeline?: any[];
|
||||
timeout?: number;
|
||||
/** 'yaml' or 'ts' — determines how executeCommand loads the handler */
|
||||
type: 'yaml' | 'ts';
|
||||
/** Relative path from clis/ dir, e.g. 'bilibili/hot.yaml' or 'bilibili/search.js' */
|
||||
modulePath?: string;
|
||||
}
|
||||
|
||||
function scanYaml(filePath: string, site: string): ManifestEntry | null {
|
||||
try {
|
||||
const raw = fs.readFileSync(filePath, 'utf-8');
|
||||
const def = yaml.load(raw) as any;
|
||||
if (!def || typeof def !== 'object') return null;
|
||||
|
||||
const strategyStr = def.strategy ?? (def.browser === false ? 'public' : 'cookie');
|
||||
const strategy = strategyStr.toUpperCase();
|
||||
const browser = def.browser ?? (strategy !== 'PUBLIC');
|
||||
|
||||
const args: ManifestEntry['args'] = [];
|
||||
if (def.args && typeof def.args === 'object') {
|
||||
for (const [argName, argDef] of Object.entries(def.args as Record<string, any>)) {
|
||||
args.push({
|
||||
name: argName,
|
||||
type: argDef?.type ?? 'str',
|
||||
default: argDef?.default,
|
||||
required: argDef?.required ?? false,
|
||||
help: argDef?.description ?? argDef?.help ?? '',
|
||||
choices: argDef?.choices,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
site: def.site ?? site,
|
||||
name: def.name ?? path.basename(filePath, path.extname(filePath)),
|
||||
description: def.description ?? '',
|
||||
domain: def.domain,
|
||||
strategy: strategy.toLowerCase(),
|
||||
browser,
|
||||
args,
|
||||
columns: def.columns,
|
||||
pipeline: def.pipeline,
|
||||
timeout: def.timeout,
|
||||
type: 'yaml',
|
||||
};
|
||||
} catch (err: any) {
|
||||
process.stderr.write(`Warning: failed to parse ${filePath}: ${err.message}\n`);
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function scanTs(filePath: string, site: string): ManifestEntry {
|
||||
// TS adapters self-register via cli() at import time.
|
||||
// We statically parse the source to extract metadata for the manifest stub.
|
||||
const baseName = path.basename(filePath, path.extname(filePath));
|
||||
const relativePath = `${site}/${baseName}.js`;
|
||||
|
||||
const entry: ManifestEntry = {
|
||||
site,
|
||||
name: baseName,
|
||||
description: '',
|
||||
strategy: 'cookie',
|
||||
browser: true,
|
||||
args: [],
|
||||
type: 'ts',
|
||||
modulePath: relativePath,
|
||||
};
|
||||
|
||||
try {
|
||||
const src = fs.readFileSync(filePath, 'utf-8');
|
||||
|
||||
// Extract description
|
||||
const descMatch = src.match(/description\s*:\s*['"`]([^'"`]*)['"`]/);
|
||||
if (descMatch) entry.description = descMatch[1];
|
||||
|
||||
// Extract domain
|
||||
const domainMatch = src.match(/domain\s*:\s*['"`]([^'"`]*)['"`]/);
|
||||
if (domainMatch) entry.domain = domainMatch[1];
|
||||
|
||||
// Extract strategy
|
||||
const stratMatch = src.match(/strategy\s*:\s*Strategy\.(\w+)/);
|
||||
if (stratMatch) entry.strategy = stratMatch[1].toLowerCase();
|
||||
|
||||
// Extract browser: false (some adapters bypass browser entirely)
|
||||
const browserMatch = src.match(/browser\s*:\s*(true|false)/);
|
||||
if (browserMatch) entry.browser = browserMatch[1] === 'true';
|
||||
|
||||
// Extract columns
|
||||
const colMatch = src.match(/columns\s*:\s*\[([^\]]*)\]/);
|
||||
if (colMatch) {
|
||||
entry.columns = colMatch[1].split(',').map(s => s.trim().replace(/^['"`]|['"`]$/g, '')).filter(Boolean);
|
||||
}
|
||||
|
||||
// Extract args array items: { name: '...', ... }
|
||||
const argsBlockMatch = src.match(/args\s*:\s*\[([\s\S]*?)\]\s*,/);
|
||||
if (argsBlockMatch) {
|
||||
const argsBlock = argsBlockMatch[1];
|
||||
const argRegex = /\{\s*name\s*:\s*['"`](\w+)['"`]([^}]*)\}/g;
|
||||
let m;
|
||||
while ((m = argRegex.exec(argsBlock)) !== null) {
|
||||
const argName = m[1];
|
||||
const body = m[2];
|
||||
const typeMatch = body.match(/type\s*:\s*['"`](\w+)['"`]/);
|
||||
const defaultMatch = body.match(/default\s*:\s*([^,}]+)/);
|
||||
const requiredMatch = body.match(/required\s*:\s*(true|false)/);
|
||||
const helpMatch = body.match(/help\s*:\s*['"`]([^'"`]*)['"`]/);
|
||||
|
||||
let defaultVal: any = undefined;
|
||||
if (defaultMatch) {
|
||||
const raw = defaultMatch[1].trim();
|
||||
if (raw === 'true') defaultVal = true;
|
||||
else if (raw === 'false') defaultVal = false;
|
||||
else if (/^\d+$/.test(raw)) defaultVal = parseInt(raw, 10);
|
||||
else if (/^\d+\.\d+$/.test(raw)) defaultVal = parseFloat(raw);
|
||||
else defaultVal = raw.replace(/^['"`]|['"`]$/g, '');
|
||||
}
|
||||
|
||||
entry.args.push({
|
||||
name: argName,
|
||||
type: typeMatch?.[1] ?? 'str',
|
||||
default: defaultVal,
|
||||
required: requiredMatch?.[1] === 'true',
|
||||
help: helpMatch?.[1] ?? '',
|
||||
});
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// If parsing fails, fall back to empty metadata — module will self-register at runtime
|
||||
}
|
||||
|
||||
return entry;
|
||||
}
|
||||
|
||||
// Main
|
||||
const manifest: ManifestEntry[] = [];
|
||||
|
||||
if (fs.existsSync(CLIS_DIR)) {
|
||||
for (const site of fs.readdirSync(CLIS_DIR)) {
|
||||
const siteDir = path.join(CLIS_DIR, site);
|
||||
if (!fs.statSync(siteDir).isDirectory()) continue;
|
||||
for (const file of fs.readdirSync(siteDir)) {
|
||||
const filePath = path.join(siteDir, file);
|
||||
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
||||
const entry = scanYaml(filePath, site);
|
||||
if (entry) manifest.push(entry);
|
||||
} else if (
|
||||
(file.endsWith('.ts') && !file.endsWith('.d.ts') && file !== 'index.ts') ||
|
||||
(file.endsWith('.js') && !file.endsWith('.d.js') && file !== 'index.js')
|
||||
) {
|
||||
manifest.push(scanTs(filePath, site));
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Ensure output directory exists
|
||||
fs.mkdirSync(path.dirname(OUTPUT), { recursive: true });
|
||||
fs.writeFileSync(OUTPUT, JSON.stringify(manifest, null, 2));
|
||||
|
||||
const yamlCount = manifest.filter(e => e.type === 'yaml').length;
|
||||
const tsCount = manifest.filter(e => e.type === 'ts').length;
|
||||
console.log(`✅ Manifest compiled: ${manifest.length} entries (${yamlCount} YAML, ${tsCount} TS) → ${OUTPUT}`);
|
||||
+47
-75
@@ -37,6 +37,49 @@ interface CascadeResult {
|
||||
confidence: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the JavaScript source for a fetch probe.
|
||||
* Shared logic for PUBLIC, COOKIE, and HEADER strategies.
|
||||
*/
|
||||
function buildFetchProbeJs(url: string, opts: {
|
||||
credentials?: boolean;
|
||||
extractCsrf?: boolean;
|
||||
}): string {
|
||||
const credentialsLine = opts.credentials ? `credentials: 'include',` : '';
|
||||
const headerSetup = opts.extractCsrf
|
||||
? `
|
||||
const cookies = document.cookie.split(';').map(c => c.trim());
|
||||
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
|
||||
const headers = {};
|
||||
if (csrf) { headers['X-Csrf-Token'] = csrf; headers['X-XSRF-Token'] = csrf; }
|
||||
`
|
||||
: 'const headers = {};';
|
||||
|
||||
return `
|
||||
async () => {
|
||||
try {
|
||||
${headerSetup}
|
||||
const resp = await fetch(${JSON.stringify(url)}, {
|
||||
${credentialsLine}
|
||||
headers
|
||||
});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
// Check for API-level error codes (common in Chinese sites)
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Probe an endpoint with a specific strategy.
|
||||
* Returns whether the probe succeeded and basic response info.
|
||||
@@ -45,32 +88,14 @@ export async function probeEndpoint(
|
||||
page: IPage,
|
||||
url: string,
|
||||
strategy: Strategy,
|
||||
opts: { timeout?: number } = {},
|
||||
_opts: { timeout?: number } = {},
|
||||
): Promise<ProbeResult> {
|
||||
const result: ProbeResult = { strategy, success: false };
|
||||
|
||||
try {
|
||||
switch (strategy) {
|
||||
case Strategy.PUBLIC: {
|
||||
// Try direct fetch without browser (no credentials)
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, {}));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -79,27 +104,7 @@ export async function probeEndpoint(
|
||||
}
|
||||
|
||||
case Strategy.COOKIE: {
|
||||
// Fetch with credentials: 'include' (uses browser cookies)
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
// Check for API-level error codes (common in Chinese sites)
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true }));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -108,39 +113,7 @@ export async function probeEndpoint(
|
||||
}
|
||||
|
||||
case Strategy.HEADER: {
|
||||
// Fetch with credentials + try to extract common auth headers
|
||||
const js = `
|
||||
async () => {
|
||||
try {
|
||||
// Try to extract CSRF tokens from cookies
|
||||
const cookies = document.cookie.split(';').map(c => c.trim());
|
||||
const csrf = cookies.find(c => c.startsWith('ct0=') || c.startsWith('csrf_token=') || c.startsWith('_csrf='))?.split('=').slice(1).join('=');
|
||||
|
||||
const headers = {};
|
||||
if (csrf) {
|
||||
headers['X-Csrf-Token'] = csrf;
|
||||
headers['X-XSRF-Token'] = csrf;
|
||||
}
|
||||
|
||||
const resp = await fetch(${JSON.stringify(url)}, {
|
||||
credentials: 'include',
|
||||
headers
|
||||
});
|
||||
const status = resp.status;
|
||||
if (!resp.ok) return { status, ok: false };
|
||||
const text = await resp.text();
|
||||
let hasData = false;
|
||||
try {
|
||||
const json = JSON.parse(text);
|
||||
hasData = !!json && (Array.isArray(json) ? json.length > 0 :
|
||||
typeof json === 'object' && Object.keys(json).length > 0);
|
||||
if (json.code !== undefined && json.code !== 0) hasData = false;
|
||||
} catch {}
|
||||
return { status, ok: true, hasData, preview: text.slice(0, 200) };
|
||||
} catch (e) { return { ok: false, error: e.message }; }
|
||||
}
|
||||
`;
|
||||
const resp = await page.evaluate(js);
|
||||
const resp = await page.evaluate(buildFetchProbeJs(url, { credentials: true, extractCsrf: true }));
|
||||
result.statusCode = resp?.status;
|
||||
result.success = resp?.ok && resp?.hasData;
|
||||
result.hasData = resp?.hasData;
|
||||
@@ -151,7 +124,6 @@ export async function probeEndpoint(
|
||||
case Strategy.INTERCEPT:
|
||||
case Strategy.UI:
|
||||
// These require specific implementation per-site
|
||||
// Mark as needing manual implementation
|
||||
result.success = false;
|
||||
result.error = `Strategy ${strategy} requires site-specific implementation`;
|
||||
break;
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { apiGet } from '../../bilibili.js';
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
name: 'dynamic',
|
||||
description: 'Get Bilibili user dynamic feed',
|
||||
domain: 'www.bilibili.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 15 },
|
||||
],
|
||||
columns: ['id', 'author', 'text', 'likes', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
const payload = await apiGet(page, '/x/polymer/web-dynamic/v1/feed/all', { params: {}, signed: false });
|
||||
const results: any[] = payload?.data?.items ?? [];
|
||||
return results.slice(0, Number(kwargs.limit)).map((item: any) => {
|
||||
let text = '';
|
||||
if (item.modules?.module_dynamic?.desc?.text) {
|
||||
text = item.modules.module_dynamic.desc.text;
|
||||
} else if (item.modules?.module_dynamic?.major?.archive?.title) {
|
||||
text = item.modules.module_dynamic.major.archive.title;
|
||||
}
|
||||
|
||||
return {
|
||||
id: item.id_str ?? '',
|
||||
author: item.modules?.module_author?.name ?? '',
|
||||
text: text,
|
||||
likes: item.modules?.module_stat?.like?.count ?? 0,
|
||||
url: item.id_str ? `https://t.bilibili.com/${item.id_str}` : ''
|
||||
};
|
||||
});
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
import { fetchJson, getSelfUid, resolveUid } from '../../bilibili.js';
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
name: 'following',
|
||||
description: '获取 Bilibili 用户的关注列表',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'uid', required: false, help: '目标用户 ID(默认为当前登录用户)' },
|
||||
{ name: 'page', type: 'int', required: false, default: 1, help: '页码' },
|
||||
{ name: 'limit', type: 'int', required: false, default: 50, help: '每页数量 (最大 50)' },
|
||||
],
|
||||
columns: ['mid', 'name', 'sign', 'following', 'fans'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
// 1. Resolve UID (default to self)
|
||||
const uid = kwargs.uid
|
||||
? await resolveUid(page, kwargs.uid)
|
||||
: await getSelfUid(page);
|
||||
|
||||
const pn = kwargs.page ?? 1;
|
||||
const ps = Math.min(kwargs.limit ?? 50, 50);
|
||||
|
||||
// 2. Fetch following list (standard Cookie API, no Wbi signing needed)
|
||||
const payload = await fetchJson(page,
|
||||
`https://api.bilibili.com/x/relation/followings?vmid=${uid}&pn=${pn}&ps=${ps}&order=desc`
|
||||
);
|
||||
|
||||
if (payload.code !== 0) {
|
||||
throw new Error(`获取关注列表失败: ${payload.message} (${payload.code})`);
|
||||
}
|
||||
|
||||
const list = payload.data?.list || [];
|
||||
if (list.length === 0) {
|
||||
return [{ mid: '-', name: `共 ${payload.data?.total ?? 0} 人关注,当前页无数据`, sign: '', following: '', fans: '' }];
|
||||
}
|
||||
|
||||
// 3. Map to output
|
||||
return list.map((u: any) => ({
|
||||
mid: u.mid,
|
||||
name: u.uname,
|
||||
sign: (u.sign || '').slice(0, 40),
|
||||
following: u.attribute === 6 ? '互相关注' : '已关注',
|
||||
fans: u.official_verify?.desc || '',
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,25 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { apiGet } from '../../bilibili.js';
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
name: 'ranking',
|
||||
description: 'Get Bilibili video ranking board',
|
||||
domain: 'www.bilibili.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['rank', 'title', 'author', 'score', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
const payload = await apiGet(page, '/x/web-interface/ranking/v2', { params: { rid: 0, type: 'all' }, signed: false });
|
||||
const results: any[] = payload?.data?.list ?? [];
|
||||
return results.slice(0, Number(kwargs.limit)).map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
title: item.title ?? '',
|
||||
author: item.owner?.name ?? '',
|
||||
score: item.stat?.view ?? 0,
|
||||
url: item.bvid ? `https://www.bilibili.com/video/${item.bvid}` : ''
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,100 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
import { apiGet } from '../../bilibili.js';
|
||||
|
||||
cli({
|
||||
site: 'bilibili',
|
||||
name: 'subtitle',
|
||||
description: '获取 Bilibili 视频的字幕',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'bvid', required: true },
|
||||
{ name: 'lang', required: false, help: '字幕语言代码 (如 zh-CN, en-US, ai-zh),默认取第一个' },
|
||||
],
|
||||
columns: ['index', 'from', 'to', 'content'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
// 1. 先前往视频详情页 (建立有鉴权的 Session,且这里不需要加载完整个视频)
|
||||
await page.goto(`https://www.bilibili.com/video/${kwargs.bvid}/`);
|
||||
|
||||
// 2. 利用 __INITIAL_STATE__ 获取基础信息,拿 CID
|
||||
const cid = await page.evaluate(`(async () => {
|
||||
const state = window.__INITIAL_STATE__ || {};
|
||||
return state?.videoData?.cid;
|
||||
})()`);
|
||||
|
||||
if (!cid) {
|
||||
throw new Error('无法在页面中提取到当前视频的 CID,请检查页面是否正常加载。');
|
||||
}
|
||||
|
||||
// 3. 在 Node 端使用 apiGet 获取带 Wbi 签名的字幕列表
|
||||
// 之前纯靠 evaluate 里的 fetch 会失败,因为 B 站 /wbi/ 开头的接口强校验 w_rid,未签名直接被风控返回 403 HTML
|
||||
const payload = await apiGet(page, '/x/player/wbi/v2', {
|
||||
params: { bvid: kwargs.bvid, cid },
|
||||
signed: true, // 开启 wbi_sign 自动签名
|
||||
});
|
||||
|
||||
if (payload.code !== 0) {
|
||||
throw new Error(`获取视频播放信息失败: ${payload.message} (${payload.code})`);
|
||||
}
|
||||
|
||||
const subtitles = payload.data?.subtitle?.subtitles || [];
|
||||
if (subtitles.length === 0) {
|
||||
throw new Error('此视频没有发现外挂或智能字幕。');
|
||||
}
|
||||
|
||||
// 4. 选择目标字幕语言
|
||||
const target = kwargs.lang
|
||||
? subtitles.find((s: any) => s.lan === kwargs.lang) || subtitles[0]
|
||||
: subtitles[0];
|
||||
|
||||
const targetSubUrl = target.subtitle_url;
|
||||
if (!targetSubUrl || targetSubUrl === '') {
|
||||
throw new Error('[风控拦截/未登录] 获取到的 subtitle_url 为空!请确保 CLI 已成功登录且风控未封锁此账号。');
|
||||
}
|
||||
|
||||
const finalUrl = targetSubUrl.startsWith('//') ? 'https:' + targetSubUrl : targetSubUrl;
|
||||
|
||||
|
||||
// 5. 解析并拉取 CDN 的 JSON 文件
|
||||
const fetchJs = `
|
||||
(async () => {
|
||||
const url = ${JSON.stringify(finalUrl)};
|
||||
const res = await fetch(url);
|
||||
const text = await res.text();
|
||||
|
||||
if (text.startsWith('<!DOCTYPE') || text.startsWith('<html')) {
|
||||
return { error: 'HTML', text: text.substring(0, 100), url };
|
||||
}
|
||||
|
||||
try {
|
||||
const subJson = JSON.parse(text);
|
||||
// B站真实返回格式是 { font_size: 0.4, font_color: "#FFFFFF", background_alpha: 0.5, background_color: "#9C27B0", Stroke: "none", type: "json" , body: [{from: 0, to: 0, content: ""}] }
|
||||
if (Array.isArray(subJson?.body)) return { success: true, data: subJson.body };
|
||||
if (Array.isArray(subJson)) return { success: true, data: subJson };
|
||||
return { error: 'UNKNOWN_JSON', data: subJson };
|
||||
} catch (e) {
|
||||
return { error: 'PARSE_FAILED', text: text.substring(0, 100) };
|
||||
}
|
||||
})()
|
||||
`;
|
||||
const items = await page.evaluate(fetchJs);
|
||||
|
||||
if (items?.error) {
|
||||
throw new Error(`字幕获取失败: ${items.error}${items.text ? ' — ' + items.text : ''}`);
|
||||
}
|
||||
|
||||
const finalItems = items?.data || [];
|
||||
if (!Array.isArray(finalItems)) {
|
||||
throw new Error('解析到的字幕列表对象不符合数组格式');
|
||||
}
|
||||
|
||||
// 6. 数据映射
|
||||
return finalItems.map((item: any, idx: number) => ({
|
||||
index: idx + 1,
|
||||
from: Number(item.from || 0).toFixed(2) + 's',
|
||||
to: Number(item.to || 0).toFixed(2) + 's',
|
||||
content: item.content
|
||||
}));
|
||||
},
|
||||
});
|
||||
+196
-29
@@ -1,8 +1,67 @@
|
||||
/**
|
||||
* BOSS直聘 job search — browser cookie API.
|
||||
* Source: bb-sites/boss/search.js
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
/** City name → BOSS Zhipin city code mapping */
|
||||
const CITY_CODES: Record<string, string> = {
|
||||
'全国': '100010000', '北京': '101010100', '上海': '101020100',
|
||||
'广州': '101280100', '深圳': '101280600', '杭州': '101210100',
|
||||
'成都': '101270100', '南京': '101190100', '武汉': '101200100',
|
||||
'西安': '101110100', '苏州': '101190400', '长沙': '101250100',
|
||||
'天津': '101030100', '重庆': '101040100', '郑州': '101180100',
|
||||
'东莞': '101281600', '青岛': '101120200', '合肥': '101220100',
|
||||
'佛山': '101280800', '宁波': '101210400', '厦门': '101230200',
|
||||
'大连': '101070200', '珠海': '101280700', '无锡': '101190200',
|
||||
'济南': '101120100', '福州': '101230100', '昆明': '101290100',
|
||||
'哈尔滨': '101050100', '沈阳': '101070100', '石家庄': '101090100',
|
||||
'贵阳': '101260100', '南宁': '101300100', '太原': '101100100',
|
||||
'海口': '101310100', '兰州': '101160100', '乌鲁木齐': '101130100',
|
||||
'长春': '101060100', '南昌': '101240100', '常州': '101191100',
|
||||
'温州': '101210700', '嘉兴': '101210300', '徐州': '101190800',
|
||||
'香港': '101320100',
|
||||
};
|
||||
|
||||
const EXP_MAP: Record<string, string> = {
|
||||
'不限': '0', '在校/应届': '108', '应届': '108', '1年以内': '101',
|
||||
'1-3年': '102', '3-5年': '103', '5-10年': '104', '10年以上': '105',
|
||||
};
|
||||
|
||||
const DEGREE_MAP: Record<string, string> = {
|
||||
'不限': '0', '初中及以下': '209', '中专/中技': '208', '高中': '206',
|
||||
'大专': '202', '本科': '203', '硕士': '204', '博士': '205',
|
||||
};
|
||||
|
||||
const SALARY_MAP: Record<string, string> = {
|
||||
'不限': '0', '3K以下': '401', '3-5K': '402', '5-10K': '403',
|
||||
'10-15K': '404', '15-20K': '405', '20-30K': '406', '30-50K': '407', '50K以上': '408',
|
||||
};
|
||||
|
||||
const INDUSTRY_MAP: Record<string, string> = {
|
||||
'不限': '0', '互联网': '100020', '电子商务': '100021', '游戏': '100024',
|
||||
'人工智能': '100901', '大数据': '100902', '金融': '100101',
|
||||
'教育培训': '100200', '医疗健康': '100300',
|
||||
};
|
||||
|
||||
function resolveCity(input: string): string {
|
||||
if (!input) return '101010100';
|
||||
if (/^\d+$/.test(input)) return input;
|
||||
if (CITY_CODES[input]) return CITY_CODES[input];
|
||||
for (const [name, code] of Object.entries(CITY_CODES)) {
|
||||
if (name.includes(input)) return code;
|
||||
}
|
||||
return '101010100';
|
||||
}
|
||||
|
||||
function resolveMap(input: string | undefined, map: Record<string, string>): string {
|
||||
if (!input) return '';
|
||||
if (map[input] !== undefined) return map[input];
|
||||
for (const [key, val] of Object.entries(map)) {
|
||||
if (key.includes(input)) return val;
|
||||
}
|
||||
return input;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'boss',
|
||||
@@ -10,38 +69,146 @@ cli({
|
||||
description: 'BOSS直聘搜索职位',
|
||||
domain: 'www.zhipin.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', required: true, help: 'Search keyword (e.g. AI agent, 前端)' },
|
||||
{ name: 'city', default: '101010100', help: 'City code (101010100=北京, 101020100=上海, 101210100=杭州, 101280100=广州)' },
|
||||
{ name: 'city', default: '北京', help: 'City name or code (e.g. 杭州, 上海, 101010100)' },
|
||||
{ name: 'experience', default: '', help: 'Experience: 应届/1年以内/1-3年/3-5年/5-10年/10年以上' },
|
||||
{ name: 'degree', default: '', help: 'Degree: 大专/本科/硕士/博士' },
|
||||
{ name: 'salary', default: '', help: 'Salary: 3K以下/3-5K/5-10K/10-15K/15-20K/20-30K/30-50K/50K以上' },
|
||||
{ name: 'industry', default: '', help: 'Industry code or name (e.g. 100020, 互联网)' },
|
||||
{ name: 'page', type: 'int', default: 1, help: 'Page number' },
|
||||
{ name: 'limit', type: 'int', default: 15, help: 'Number of results' },
|
||||
],
|
||||
columns: ['name', 'salary', 'company', 'city', 'experience', 'degree', 'boss', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.zhipin.com');
|
||||
await page.wait(2);
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const params = new URLSearchParams({
|
||||
scene: '1', query: '${kwargs.query.replace(/'/g, "\\'")}',
|
||||
city: '${kwargs.city || '101010100'}', page: '1', pageSize: '15',
|
||||
experience: '', degree: '', payType: '', partTime: '',
|
||||
industry: '', scale: '', stage: '', position: '',
|
||||
jobType: '', salary: '', multiBusinessDistrict: '', multiSubway: ''
|
||||
});
|
||||
const resp = await fetch('/wapi/zpgeek/search/joblist.json?' + params.toString(), {credentials: 'include'});
|
||||
if (!resp.ok) return {error: 'HTTP ' + resp.status};
|
||||
const d = await resp.json();
|
||||
if (d.code !== 0) return {error: d.message || 'API error'};
|
||||
const zpData = d.zpData || {};
|
||||
return (zpData.jobList || []).map(j => ({
|
||||
name: j.jobName, salary: j.salaryDesc, company: j.brandName,
|
||||
city: j.cityName, experience: j.jobExperience, degree: j.jobDegree,
|
||||
columns: ['name', 'salary', 'company', 'area', 'experience', 'degree', 'skills', 'boss', 'url'],
|
||||
func: async (page: IPage | null, kwargs) => {
|
||||
if (!page) throw new Error('Browser page required');
|
||||
|
||||
const cityCode = resolveCity(kwargs.city);
|
||||
|
||||
if (process.env.OPENCLI_VERBOSE || process.env.DEBUG?.includes('opencli')) {
|
||||
console.error(`[opencli:boss] Navigating to set referrer context...`);
|
||||
}
|
||||
// Navigate to the Web search view first to establish proper referrer context
|
||||
// This is a lesson learned from boss-cli: referrer is important
|
||||
await page.goto(`https://www.zhipin.com/web/geek/job?query=${encodeURIComponent(kwargs.query)}&city=${cityCode}`);
|
||||
|
||||
// Give the page a tiny bit of time to settle to avoid immediate 403s
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
const expVal = resolveMap(kwargs.experience, EXP_MAP);
|
||||
const degreeVal = resolveMap(kwargs.degree, DEGREE_MAP);
|
||||
const salaryVal = resolveMap(kwargs.salary, SALARY_MAP);
|
||||
const industryVal = resolveMap(kwargs.industry, INDUSTRY_MAP);
|
||||
|
||||
const limit = kwargs.limit || 15;
|
||||
let currentPage = kwargs.page || 1;
|
||||
let allJobs: any[] = [];
|
||||
const seenIds = new Set<string>();
|
||||
|
||||
while (allJobs.length < limit) {
|
||||
if (allJobs.length > 0) {
|
||||
// Human-like pause between page fetches (1-3 seconds)
|
||||
await new Promise(r => setTimeout(r, 1000 + Math.random() * 2000));
|
||||
}
|
||||
|
||||
const qs = new URLSearchParams({
|
||||
scene: '1',
|
||||
query: kwargs.query,
|
||||
city: cityCode,
|
||||
page: String(currentPage),
|
||||
pageSize: '15',
|
||||
});
|
||||
if (expVal) qs.set('experience', expVal);
|
||||
if (degreeVal) qs.set('degree', degreeVal);
|
||||
if (salaryVal) qs.set('salary', salaryVal);
|
||||
if (industryVal) qs.set('industry', industryVal);
|
||||
|
||||
const targetUrl = `https://www.zhipin.com/wapi/zpgeek/search/joblist.json?${qs.toString()}`;
|
||||
|
||||
if (process.env.OPENCLI_VERBOSE || process.env.DEBUG?.includes('opencli')) {
|
||||
console.error(`[opencli:boss] Fetching page ${currentPage}... (current jobs: ${allJobs.length})`);
|
||||
}
|
||||
|
||||
const evaluateScript = `
|
||||
async () => {
|
||||
return new Promise((resolve, reject) => {
|
||||
const xhr = new window.XMLHttpRequest();
|
||||
xhr.open('GET', '${targetUrl}', true);
|
||||
xhr.withCredentials = true;
|
||||
xhr.timeout = 15000; // 15s timeout
|
||||
xhr.setRequestHeader('Accept', 'application/json, text/plain, */*');
|
||||
xhr.onload = () => {
|
||||
if (xhr.status >= 200 && xhr.status < 300) {
|
||||
try {
|
||||
resolve(JSON.parse(xhr.responseText));
|
||||
} catch (e) {
|
||||
reject(new Error('Failed to parse JSON. Raw (200 chars): ' + xhr.responseText.substring(0, 200)));
|
||||
}
|
||||
} else {
|
||||
reject(new Error('XHR HTTP Status: ' + xhr.status));
|
||||
}
|
||||
};
|
||||
xhr.onerror = () => reject(new Error('XHR Network Error'));
|
||||
xhr.ontimeout = () => reject(new Error('XHR Timeout'));
|
||||
xhr.send();
|
||||
});
|
||||
}
|
||||
`;
|
||||
|
||||
let data: any;
|
||||
try {
|
||||
data = await page.evaluate(evaluateScript);
|
||||
} catch (e: any) {
|
||||
throw new Error('API evaluate failed: ' + e.message);
|
||||
}
|
||||
|
||||
if (data.code !== 0) {
|
||||
if (data.code === 37) {
|
||||
throw new Error('Cookie 已过期!请在当前 Chrome 浏览器中重新登录 BOSS 直聘。');
|
||||
}
|
||||
throw new Error(`BOSS API error: ${data.message || 'Unknown'} (code=${data.code})\nRaw data: ${JSON.stringify(data)}`);
|
||||
}
|
||||
|
||||
const zpData = data.zpData || {};
|
||||
const batch = zpData.jobList || [];
|
||||
if (batch.length === 0) {
|
||||
break; // No more results
|
||||
}
|
||||
|
||||
let addedInBatch = 0;
|
||||
for (const j of batch) {
|
||||
if (!j.encryptJobId || seenIds.has(j.encryptJobId)) continue;
|
||||
seenIds.add(j.encryptJobId);
|
||||
|
||||
allJobs.push({
|
||||
name: j.jobName,
|
||||
salary: j.salaryDesc,
|
||||
company: j.brandName,
|
||||
area: [j.cityName, j.areaDistrict, j.businessDistrict].filter(Boolean).join('·'),
|
||||
experience: j.jobExperience,
|
||||
degree: j.jobDegree,
|
||||
skills: (j.skills || []).join(','),
|
||||
boss: j.bossName + ' · ' + j.bossTitle,
|
||||
url: j.encryptJobId ? 'https://www.zhipin.com/job_detail/' + j.encryptJobId + '.html' : ''
|
||||
}));
|
||||
})()
|
||||
`);
|
||||
if (!Array.isArray(data)) return [];
|
||||
return data.slice(0, kwargs.limit || 15);
|
||||
url: 'https://www.zhipin.com/job_detail/' + j.encryptJobId + '.html',
|
||||
});
|
||||
addedInBatch++;
|
||||
if (allJobs.length >= limit) break;
|
||||
}
|
||||
|
||||
if (addedInBatch === 0) {
|
||||
// Boss API is repeating identical pages, we've hit the pagination limit
|
||||
if (process.env.OPENCLI_VERBOSE) console.error(`[opencli:boss] API returned duplicate page, stopping pagination at ${allJobs.length} items`);
|
||||
break;
|
||||
}
|
||||
|
||||
if (!zpData.hasMore) {
|
||||
break; // API says no more pages
|
||||
}
|
||||
currentPage++;
|
||||
}
|
||||
|
||||
return allJobs;
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,149 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { canonicalizeProductUrl, normalizeProductId } from '../../coupang.js';
|
||||
|
||||
function escapeJsString(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function buildAddToCartEvaluate(expectedProductId: string): string {
|
||||
return `
|
||||
(async () => {
|
||||
const expectedProductId = ${escapeJsString(expectedProductId)};
|
||||
const text = document.body.innerText || '';
|
||||
const loginHints = {
|
||||
hasLoginLink: Boolean(document.querySelector('a[href*="login"], a[title*="로그인"]')),
|
||||
hasMyCoupang: /마이쿠팡/.test(text),
|
||||
};
|
||||
|
||||
const pathMatch = location.pathname.match(/\\/vp\\/products\\/(\\d+)/);
|
||||
const currentProductId = pathMatch?.[1] || '';
|
||||
if (expectedProductId && currentProductId && expectedProductId !== currentProductId) {
|
||||
return { ok: false, reason: 'PRODUCT_MISMATCH', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
const optionSelectors = [
|
||||
'select',
|
||||
'[role="listbox"]',
|
||||
'.prod-option, .product-option, .option-select, .option-dropdown',
|
||||
];
|
||||
const hasRequiredOption = optionSelectors.some((selector) => {
|
||||
try {
|
||||
const nodes = Array.from(document.querySelectorAll(selector));
|
||||
return nodes.some((node) => {
|
||||
const label = (node.textContent || '') + ' ' + (node.getAttribute?.('aria-label') || '');
|
||||
return /옵션|색상|사이즈|용량|선택/i.test(label);
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
if (hasRequiredOption) {
|
||||
return { ok: false, reason: 'OPTION_REQUIRED', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
const clickCandidate = (elements) => {
|
||||
for (const element of elements) {
|
||||
if (!(element instanceof HTMLElement)) continue;
|
||||
const label = ((element.innerText || '') + ' ' + (element.getAttribute('aria-label') || '')).trim();
|
||||
if (/장바구니|카트|cart/i.test(label) && !/sold out|품절/i.test(label)) {
|
||||
element.click();
|
||||
return true;
|
||||
}
|
||||
}
|
||||
return false;
|
||||
};
|
||||
|
||||
const beforeCount = (() => {
|
||||
const node = document.querySelector('[class*="cart"] .count, #headerCartCount, .cart-count');
|
||||
const text = node?.textContent || '';
|
||||
const num = Number(text.replace(/[^\\d]/g, ''));
|
||||
return Number.isFinite(num) ? num : null;
|
||||
})();
|
||||
|
||||
const buttons = Array.from(document.querySelectorAll('button, a[role="button"], input[type="button"]'));
|
||||
const clicked = clickCandidate(buttons);
|
||||
if (!clicked) {
|
||||
return { ok: false, reason: 'ADD_TO_CART_BUTTON_NOT_FOUND', currentProductId, loginHints };
|
||||
}
|
||||
|
||||
await new Promise((resolve) => setTimeout(resolve, 2500));
|
||||
|
||||
const afterText = document.body.innerText || '';
|
||||
const successMessage = /장바구니에 담|장바구니 담기 완료|added to cart/i.test(afterText);
|
||||
const afterCount = (() => {
|
||||
const node = document.querySelector('[class*="cart"] .count, #headerCartCount, .cart-count');
|
||||
const text = node?.textContent || '';
|
||||
const num = Number(text.replace(/[^\\d]/g, ''));
|
||||
return Number.isFinite(num) ? num : null;
|
||||
})();
|
||||
const countIncreased =
|
||||
beforeCount != null &&
|
||||
afterCount != null &&
|
||||
afterCount >= beforeCount &&
|
||||
(afterCount > beforeCount || beforeCount === 0);
|
||||
|
||||
return {
|
||||
ok: successMessage || countIncreased,
|
||||
reason: successMessage || countIncreased ? 'SUCCESS' : 'UNKNOWN',
|
||||
currentProductId,
|
||||
beforeCount,
|
||||
afterCount,
|
||||
loginHints,
|
||||
};
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'coupang',
|
||||
name: 'add-to-cart',
|
||||
description: 'Add a Coupang product to cart using logged-in browser session',
|
||||
domain: 'www.coupang.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'productId', required: false, help: 'Coupang product ID' },
|
||||
{ name: 'url', required: false, help: 'Canonical product URL' },
|
||||
],
|
||||
columns: ['ok', 'product_id', 'url', 'message'],
|
||||
func: async (page, kwargs) => {
|
||||
const rawProductId = kwargs.productId ?? kwargs.product_id;
|
||||
const productId = normalizeProductId(rawProductId);
|
||||
const targetUrl = canonicalizeProductUrl(kwargs.url, productId);
|
||||
|
||||
if (!productId && !targetUrl) {
|
||||
throw new Error('Either --product-id or --url is required');
|
||||
}
|
||||
|
||||
const finalUrl = targetUrl || canonicalizeProductUrl('', productId);
|
||||
await page.goto(finalUrl);
|
||||
await page.wait(3);
|
||||
|
||||
const result = await page.evaluate(buildAddToCartEvaluate(productId));
|
||||
const loginHints = result?.loginHints ?? {};
|
||||
if (loginHints.hasLoginLink && !loginHints.hasMyCoupang) {
|
||||
throw new Error('Coupang login required. Please log into Coupang in Chrome and retry.');
|
||||
}
|
||||
|
||||
const actualProductId = normalizeProductId(result?.currentProductId || productId);
|
||||
if (result?.reason === 'PRODUCT_MISMATCH') {
|
||||
throw new Error(`Product mismatch: expected ${productId}, got ${actualProductId || 'unknown'}`);
|
||||
}
|
||||
if (result?.reason === 'OPTION_REQUIRED') {
|
||||
throw new Error('This product requires option selection and is not supported in v1.');
|
||||
}
|
||||
if (result?.reason === 'ADD_TO_CART_BUTTON_NOT_FOUND') {
|
||||
throw new Error('Could not find an add-to-cart button on the product page.');
|
||||
}
|
||||
if (!result?.ok) {
|
||||
throw new Error('Failed to confirm add-to-cart success.');
|
||||
}
|
||||
|
||||
return [{
|
||||
ok: true,
|
||||
product_id: actualProductId || productId,
|
||||
url: finalUrl,
|
||||
message: 'Added to cart',
|
||||
}];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,466 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import { mergeSearchItems, normalizeSearchItem, sanitizeSearchItems } from '../../coupang.js';
|
||||
|
||||
function escapeJsString(value: string): string {
|
||||
return JSON.stringify(value);
|
||||
}
|
||||
|
||||
function buildApplyFilterEvaluate(filter: string): string {
|
||||
return `
|
||||
() => {
|
||||
const filter = ${escapeJsString(filter)};
|
||||
const labels = Array.from(document.querySelectorAll('label'));
|
||||
const normalize = (value) => (value == null ? '' : String(value).trim().toLowerCase());
|
||||
const target = labels.find((label) => {
|
||||
const component = normalize(label.getAttribute('data-component-name'));
|
||||
const imgAlt = normalize(label.querySelector('img')?.getAttribute('alt'));
|
||||
const text = normalize(label.textContent);
|
||||
|
||||
if (filter === 'rocket') {
|
||||
return (
|
||||
component.includes('deliveryfilteroption-rocket_luxury,rocket_wow,coupang_global') ||
|
||||
imgAlt.includes('rocket_luxury,rocket_wow,coupang_global') ||
|
||||
imgAlt.includes('rocket-all') ||
|
||||
text.includes('로켓')
|
||||
);
|
||||
}
|
||||
|
||||
return component.includes(filter) || imgAlt.includes(filter) || text.includes(filter);
|
||||
});
|
||||
|
||||
if (!target) {
|
||||
return { ok: false, reason: 'FILTER_NOT_FOUND' };
|
||||
}
|
||||
|
||||
target.click();
|
||||
|
||||
return {
|
||||
ok: true,
|
||||
reason: 'FILTER_CLICKED',
|
||||
component: target.getAttribute('data-component-name') || '',
|
||||
text: (target.textContent || '').trim(),
|
||||
alt: target.querySelector('img')?.getAttribute('alt') || '',
|
||||
};
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
function buildCurrentLocationEvaluate(): string {
|
||||
return `
|
||||
() => ({
|
||||
href: location.href
|
||||
})
|
||||
`;
|
||||
}
|
||||
|
||||
function buildSearchEvaluate(query: string, limit: number, pageNumber: number): string {
|
||||
return `
|
||||
(async () => {
|
||||
const query = ${escapeJsString(query)};
|
||||
const limit = ${limit};
|
||||
const pageNumber = ${pageNumber};
|
||||
|
||||
const normalizeText = (value) => (value == null ? '' : String(value).trim());
|
||||
const parseNum = (value) => {
|
||||
const text = normalizeText(value).replace(/[^\\d.]/g, '');
|
||||
if (!text) return null;
|
||||
const num = Number(text);
|
||||
return Number.isFinite(num) ? num : null;
|
||||
};
|
||||
const extractPriceFromText = (text) => {
|
||||
const matches = normalizeText(text).match(/\\d{1,3}(?:,\\d{3})*원/g) || [];
|
||||
if (!matches.length) return '';
|
||||
if (matches.length >= 2) return matches[matches.length - 2];
|
||||
return matches[0];
|
||||
};
|
||||
const extractPriceInfo = (root) => {
|
||||
const priceArea =
|
||||
root.querySelector('.PriceArea_priceArea__NntJz, [class*="PriceArea_priceArea"], [class*="priceArea"]') ||
|
||||
root;
|
||||
const priceAreaText = normalizeText(priceArea.textContent || '');
|
||||
const originalPrice = normalizeText(
|
||||
priceArea.querySelector(
|
||||
'del, .base-price, .origin-price, .original-price, .strike-price, [class*="base-price"], [class*="origin-price"], [class*="line-through"]'
|
||||
)?.textContent || ''
|
||||
);
|
||||
const originalPriceNum = parseNum(originalPrice);
|
||||
const unitPrice =
|
||||
normalizeText(
|
||||
priceArea.querySelector('.unit-price, [class*="unit-price"], [class*="unitPrice"]')?.textContent || ''
|
||||
) ||
|
||||
priceAreaText.match(/\\([^)]*당\\s*[^)]*원[^)]*\\)/)?.[0] ||
|
||||
'';
|
||||
|
||||
const candidates = Array.from(priceArea.querySelectorAll('span, strong, div'))
|
||||
.map((node) => {
|
||||
const text = normalizeText(node.textContent || '');
|
||||
if (!text || !/\\d/.test(text)) return null;
|
||||
if (/\\d{1,2}:\\d{2}:\\d{2}/.test(text)) return null;
|
||||
if (/당\\s*\\d/.test(text)) return null;
|
||||
if (/^\\d+%$/.test(text)) return null;
|
||||
|
||||
const num = parseNum(text);
|
||||
if (num == null) return null;
|
||||
|
||||
const className = normalizeText(node.getAttribute('class') || '').toLowerCase();
|
||||
let score = 0;
|
||||
if (/price|sale|selling|final/.test(className)) score += 6;
|
||||
if (/red/.test(className)) score += 5;
|
||||
if (/font-bold|bold/.test(className)) score += 3;
|
||||
if (/line-through/.test(className)) score -= 12;
|
||||
if (text.includes('원')) score += 2;
|
||||
if (originalPriceNum != null && num === originalPriceNum) score -= 10;
|
||||
if (num < 100) score -= 10;
|
||||
|
||||
return { text, num, score };
|
||||
})
|
||||
.filter(Boolean)
|
||||
.sort((a, b) => {
|
||||
if (b.score !== a.score) return b.score - a.score;
|
||||
if (originalPriceNum != null) {
|
||||
const aPrefer = a.num !== originalPriceNum ? 1 : 0;
|
||||
const bPrefer = b.num !== originalPriceNum ? 1 : 0;
|
||||
if (bPrefer !== aPrefer) return bPrefer - aPrefer;
|
||||
}
|
||||
return b.num - a.num;
|
||||
});
|
||||
|
||||
const currentPrice =
|
||||
normalizeText(candidates.find((candidate) => candidate.num !== originalPriceNum)?.text || '') ||
|
||||
normalizeText(candidates[0]?.text || '') ||
|
||||
extractPriceFromText(priceAreaText) ||
|
||||
'';
|
||||
|
||||
return {
|
||||
price: currentPrice,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
};
|
||||
};
|
||||
const canonicalUrl = (url, productId) => {
|
||||
if (url) {
|
||||
try {
|
||||
const parsed = new URL(url, 'https://www.coupang.com');
|
||||
const match = parsed.pathname.match(/\\/vp\\/products\\/(\\d+)/);
|
||||
return 'https://www.coupang.com/vp/products/' + (match?.[1] || productId || '');
|
||||
} catch {}
|
||||
}
|
||||
return productId ? 'https://www.coupang.com/vp/products/' + productId : '';
|
||||
};
|
||||
const normalize = (raw) => {
|
||||
const rawText = normalizeText(raw.text || raw.badgeText || raw.deliveryText || raw.summary);
|
||||
const productId = normalizeText(
|
||||
raw.productId || raw.product_id || raw.id || raw.productNo ||
|
||||
raw?.product?.productId || raw?.item?.id
|
||||
).match(/(\\d{6,})/)?.[1] || '';
|
||||
const title = normalizeText(
|
||||
raw.title || raw.name || raw.productName || raw.productTitle || raw.itemName
|
||||
);
|
||||
const price = parseNum(raw.price || raw.salePrice || raw.finalPrice || raw.sellingPrice);
|
||||
const originalPrice = parseNum(raw.originalPrice || raw.basePrice || raw.listPrice || raw.originPrice);
|
||||
const unitPrice = normalizeText(raw.unitPrice || raw.unit_price || raw.unitPriceText);
|
||||
const rating = parseNum(raw.rating || raw.star || raw.reviewRating);
|
||||
const reviewCount = parseNum(raw.reviewCount || raw.ratingCount || raw.reviewCnt || raw.reviews);
|
||||
const badge = Array.isArray(raw.badges) ? raw.badges.map(normalizeText).filter(Boolean).join(', ') : normalizeText(raw.badge || raw.labels);
|
||||
const seller = normalizeText(raw.seller || raw.sellerName || raw.vendorName || raw.merchantName);
|
||||
const category = normalizeText(raw.category || raw.categoryName || raw.categoryPath);
|
||||
const discountRate = parseNum(raw.discountRate || raw.discount || raw.discountPercent);
|
||||
const url = canonicalUrl(raw.url || raw.productUrl || raw.link, productId);
|
||||
return {
|
||||
productId,
|
||||
title,
|
||||
price,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
discountRate,
|
||||
rating,
|
||||
reviewCount,
|
||||
rocket: normalizeText(raw.rocket || raw.rocketType),
|
||||
deliveryType: normalizeText(raw.deliveryType || raw.deliveryBadge || raw.shippingType || raw.shippingBadge),
|
||||
deliveryPromise: normalizeText(raw.deliveryPromise || raw.promise || raw.arrivalText || raw.arrivalBadge),
|
||||
seller,
|
||||
badge,
|
||||
category,
|
||||
url,
|
||||
};
|
||||
};
|
||||
|
||||
const byApi = async () => {
|
||||
const candidates = [
|
||||
'/np/search?q=' + encodeURIComponent(query) + '&component=&channel=user&page=' + pageNumber,
|
||||
'/np/search?component=&q=' + encodeURIComponent(query) + '&channel=user&page=' + pageNumber,
|
||||
];
|
||||
|
||||
for (const path of candidates) {
|
||||
try {
|
||||
const resp = await fetch(path, { credentials: 'include' });
|
||||
if (!resp.ok) continue;
|
||||
const text = await resp.text();
|
||||
const data = text.trim().startsWith('<') ? null : JSON.parse(text);
|
||||
const maybeItems =
|
||||
data?.data?.products ||
|
||||
data?.data?.productList ||
|
||||
data?.products ||
|
||||
data?.productList ||
|
||||
data?.items;
|
||||
if (Array.isArray(maybeItems) && maybeItems.length) {
|
||||
return maybeItems.slice(0, limit).map(normalize);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byBootstrap = () => {
|
||||
const isProductLike = (item) => {
|
||||
if (!item || typeof item !== 'object') return false;
|
||||
const values = [item.productId, item.product_id, item.id, item.productNo, item.url, item.productUrl, item.link, item.title, item.productName];
|
||||
return values.some((value) => /\\/vp\\/products\\/|\\d{6,}/.test(normalizeText(value)));
|
||||
};
|
||||
|
||||
const collectProducts = (node) => {
|
||||
const queue = [node];
|
||||
while (queue.length) {
|
||||
const current = queue.shift();
|
||||
if (!current || typeof current !== 'object') continue;
|
||||
if (Array.isArray(current)) {
|
||||
const productish = current.filter(isProductLike);
|
||||
if (productish.length >= 3) return productish.slice(0, limit).map(normalize);
|
||||
queue.push(...current.slice(0, 50));
|
||||
continue;
|
||||
}
|
||||
for (const value of Object.values(current)) queue.push(value);
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const scriptNodes = Array.from(document.scripts);
|
||||
for (const script of scriptNodes) {
|
||||
const text = script.textContent || '';
|
||||
if (!text || !/product|search/i.test(text)) continue;
|
||||
const arrayMatches = [
|
||||
...text.matchAll(/"products?"\\s*:\\s*(\\[[\\s\\S]{100,}?\\])/g),
|
||||
...text.matchAll(/"itemList"\\s*:\\s*(\\[[\\s\\S]{100,}?\\])/g),
|
||||
];
|
||||
for (const match of arrayMatches) {
|
||||
try {
|
||||
const products = JSON.parse(match[1]);
|
||||
if (Array.isArray(products) && products.length) {
|
||||
return products.slice(0, limit).map(normalize);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
|
||||
const globals = [
|
||||
window.__NEXT_DATA__,
|
||||
window.__APOLLO_STATE__,
|
||||
window.__INITIAL_STATE__,
|
||||
window.__STATE__,
|
||||
window.__PRELOADED_STATE__,
|
||||
];
|
||||
for (const candidate of globals) {
|
||||
if (!candidate || typeof candidate !== 'object') continue;
|
||||
const found = collectProducts(candidate);
|
||||
if (found.length) return found;
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byJsonLd = () => {
|
||||
const scripts = Array.from(document.querySelectorAll('script[type="application/ld+json"]'));
|
||||
for (const script of scripts) {
|
||||
const text = script.textContent || '';
|
||||
if (!text) continue;
|
||||
try {
|
||||
const payload = JSON.parse(text);
|
||||
const docs = Array.isArray(payload) ? payload : [payload];
|
||||
for (const doc of docs) {
|
||||
const items =
|
||||
doc?.itemListElement ||
|
||||
doc?.about?.itemListElement ||
|
||||
doc?.mainEntity?.itemListElement ||
|
||||
[];
|
||||
if (!Array.isArray(items) || !items.length) continue;
|
||||
const mapped = items.map((entry) => {
|
||||
const item = entry?.item || entry;
|
||||
return normalize({
|
||||
productId: item?.url || item?.sku || item?.productID,
|
||||
title: item?.name,
|
||||
price: item?.offers?.price,
|
||||
originalPrice: item?.offers?.highPrice,
|
||||
rating: item?.aggregateRating?.ratingValue,
|
||||
reviewCount: item?.aggregateRating?.reviewCount,
|
||||
seller: item?.offers?.seller?.name,
|
||||
badge: item?.offers?.availability,
|
||||
category: item?.category,
|
||||
url: item?.url,
|
||||
});
|
||||
}).filter((item) => item.productId || item.url || item.title);
|
||||
if (mapped.length) return mapped.slice(0, limit);
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return [];
|
||||
};
|
||||
|
||||
const byDom = () => {
|
||||
const domScanLimit = Math.max(limit * 6, 60);
|
||||
const cards = Array.from(new Set([
|
||||
...document.querySelectorAll('li.search-product'),
|
||||
...document.querySelectorAll('li[class*="search-product"], div[class*="search-product"], article[class*="search-product"]'),
|
||||
...document.querySelectorAll('li[class*="ProductUnit_productUnit"], [class*="ProductUnit_productUnit"]'),
|
||||
...document.querySelectorAll('.impression-logged, [class*="promotion-item"], [class*="product-item"]'),
|
||||
...document.querySelectorAll('[data-product-id]'),
|
||||
...document.querySelectorAll('[data-id]'),
|
||||
...document.querySelectorAll('a[href*="/vp/products/"]'),
|
||||
])).slice(0, domScanLimit);
|
||||
const items = [];
|
||||
for (const el of cards) {
|
||||
const root = el.closest('li, div, article, section') || el;
|
||||
const html = root.innerHTML || '';
|
||||
const priceInfo = extractPriceInfo(root);
|
||||
const badgeImages = Array.from(root.querySelectorAll('img[data-badge-id]'));
|
||||
const badgeIds = badgeImages
|
||||
.map((node) => node.getAttribute('data-badge-id') || '')
|
||||
.filter(Boolean);
|
||||
const badgeSrcText = badgeImages
|
||||
.map((node) => (node.getAttribute('data-badge-id') || '') + ' ' + (node.getAttribute('src') || ''))
|
||||
.join(' ');
|
||||
const productId =
|
||||
root.getAttribute('data-product-id') ||
|
||||
el.getAttribute('data-product-id') ||
|
||||
root.querySelector('a[href*="/vp/products/"]')?.getAttribute('data-product-id') ||
|
||||
root.querySelector('a[href*="/vp/products/"]')?.getAttribute('href')?.match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
html.match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
(el.getAttribute('href') || '').match(/\\/vp\\/products\\/(\\d+)/)?.[1] ||
|
||||
'';
|
||||
const title =
|
||||
root.querySelector('.name, .title, .product-name, .search-product-title, .item-title, .ProductUnit_productNameV2__cV9cw, [class*="ProductUnit_productName"], [class*="productName"], [class*="product-name"], [class*="title"]')?.textContent ||
|
||||
root.querySelector('img[alt]')?.getAttribute('alt') ||
|
||||
html.match(/alt="([^"]+)"/)?.[1] ||
|
||||
(root.textContent || '').replace(/\\s+/g, ' ').trim().match(/^(.+?)(\\d{1,3},\\d{3}원|무료배송|내일\\(|오늘\\(|새벽)/)?.[1] ||
|
||||
el.getAttribute('title') ||
|
||||
'';
|
||||
const price = priceInfo.price || '';
|
||||
const originalPrice = priceInfo.originalPrice || '';
|
||||
const unitPrice = priceInfo.unitPrice || '';
|
||||
const rating =
|
||||
root.querySelector('.rating, .star em, [class*="rating"], [class*="star"], [class*="ProductRating"] [aria-label], [aria-label][class*="ProductRating"]')?.getAttribute?.('aria-label') ||
|
||||
root.querySelector('.rating, .star em, [class*="rating"], [class*="star"], [class*="ProductRating"]')?.textContent ||
|
||||
'';
|
||||
const reviewCount =
|
||||
root.querySelector('.rating-total-count, .count, .review-count, .promotion-item-review-count, [class*="review"], [class*="count"], [class*="ProductRating"] span, [class*="ProductRating"] [class*="fw-text"]')?.textContent ||
|
||||
'';
|
||||
const seller =
|
||||
root.querySelector('.seller, .vendor, .search-product-wrap .vendor-name, [class*="vendor"], [class*="seller"]')?.textContent ||
|
||||
'';
|
||||
const category =
|
||||
root.getAttribute('data-category') ||
|
||||
root.querySelector('[class*="category"]')?.textContent ||
|
||||
'';
|
||||
const text = (root.textContent || '').replace(/\\s+/g, ' ').trim();
|
||||
const badgeNodes = Array.from(root.querySelectorAll('.badge, .delivery, .tag, .icon-service, .pdd-text, .delivery-text, [class*="badge"], [class*="delivery"]'));
|
||||
const hrefNode = root.querySelector('a[href*="/vp/products/"]');
|
||||
items.push(normalize({
|
||||
productId,
|
||||
title,
|
||||
price,
|
||||
originalPrice,
|
||||
unitPrice,
|
||||
rating,
|
||||
reviewCount,
|
||||
seller,
|
||||
badges: [...badgeIds, ...badgeNodes.map((node) => node.textContent || '').filter(Boolean)],
|
||||
rocket: badgeSrcText + ' ' + badgeNodes.map((node) => node.textContent || '').join(' '),
|
||||
deliveryType: badgeNodes.map((node) => node.textContent || '').join(' ') + ' ' + text,
|
||||
deliveryPromise: badgeNodes.map((node) => node.textContent || '').join(' ') + ' ' + text,
|
||||
category,
|
||||
text,
|
||||
url: hrefNode?.getAttribute('href') || '',
|
||||
}));
|
||||
}
|
||||
return items.slice(0, domScanLimit);
|
||||
};
|
||||
|
||||
let items = await byApi();
|
||||
if (!items.length) items = byJsonLd();
|
||||
if (!items.length) items = byBootstrap();
|
||||
const domItems = byDom();
|
||||
if (!items.length) items = domItems;
|
||||
|
||||
return {
|
||||
loginHints: {
|
||||
hasLoginLink: Boolean(document.querySelector('a[href*="login"], a[title*="로그인"]')),
|
||||
hasMyCoupang: /마이쿠팡/.test(document.body.innerText),
|
||||
},
|
||||
items,
|
||||
domItems,
|
||||
};
|
||||
})()
|
||||
`;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'coupang',
|
||||
name: 'search',
|
||||
description: 'Search Coupang products with logged-in browser session',
|
||||
domain: 'www.coupang.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', required: true, help: 'Search keyword' },
|
||||
{ name: 'page', type: 'int', default: 1, help: 'Search result page number' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Max results (max 50)' },
|
||||
{ name: 'filter', required: false, help: 'Optional search filter (currently supports: rocket)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'price', 'unit_price', 'rating', 'review_count', 'rocket', 'delivery_type', 'delivery_promise', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query || '').trim();
|
||||
const pageNumber = Math.max(Number(kwargs.page || 1), 1);
|
||||
const limit = Math.min(Math.max(Number(kwargs.limit || 20), 1), 50);
|
||||
const filter = String(kwargs.filter || '').trim().toLowerCase();
|
||||
if (!query) throw new Error('Query is required');
|
||||
|
||||
const initialPage = filter ? 1 : pageNumber;
|
||||
const url = `https://www.coupang.com/np/search?q=${encodeURIComponent(query)}&channel=user&page=${initialPage}`;
|
||||
await page.goto(url);
|
||||
await page.wait(3);
|
||||
if (filter) {
|
||||
const filterResult = await page.evaluate(buildApplyFilterEvaluate(filter));
|
||||
if (!filterResult?.ok) {
|
||||
throw new Error(`Unsupported or unavailable filter: ${filter}`);
|
||||
}
|
||||
await page.wait(3);
|
||||
if (pageNumber > 1) {
|
||||
const locationInfo = await page.evaluate(buildCurrentLocationEvaluate());
|
||||
const filteredUrl = new URL(locationInfo?.href || url);
|
||||
filteredUrl.searchParams.set('page', String(pageNumber));
|
||||
await page.goto(filteredUrl.toString());
|
||||
await page.wait(3);
|
||||
}
|
||||
}
|
||||
await page.autoScroll({ times: filter ? 3 : 2, delayMs: 1500 });
|
||||
|
||||
const raw = await page.evaluate(buildSearchEvaluate(query, limit, pageNumber));
|
||||
const loginHints = raw?.loginHints ?? {};
|
||||
const items = Array.isArray(raw?.items) ? raw.items : [];
|
||||
const domItems = Array.isArray(raw?.domItems) ? raw.domItems : [];
|
||||
const normalizedBase = sanitizeSearchItems(
|
||||
items.map((item: Record<string, unknown>, index: number) => normalizeSearchItem(item, index)),
|
||||
limit
|
||||
);
|
||||
const normalizedDom = sanitizeSearchItems(
|
||||
domItems.map((item: Record<string, unknown>, index: number) => normalizeSearchItem(item, index)),
|
||||
Math.max(limit * 6, 60)
|
||||
);
|
||||
const normalized = filter
|
||||
? sanitizeSearchItems(normalizedDom, limit)
|
||||
: mergeSearchItems(normalizedBase, normalizedDom, limit);
|
||||
|
||||
if (!normalized.length && loginHints.hasLoginLink && !loginHints.hasMyCoupang) {
|
||||
throw new Error('Coupang login required. Please log into Coupang in Chrome and retry.');
|
||||
}
|
||||
return normalized;
|
||||
},
|
||||
});
|
||||
@@ -1,46 +0,0 @@
|
||||
/**
|
||||
* Import all TypeScript CLI adapters so they self-register.
|
||||
*
|
||||
* Each TS adapter calls cli() on import, which adds itself to the global registry.
|
||||
*/
|
||||
|
||||
// bilibili
|
||||
import './bilibili/search.js';
|
||||
import './bilibili/me.js';
|
||||
import './bilibili/favorite.js';
|
||||
import './bilibili/history.js';
|
||||
import './bilibili/feed.js';
|
||||
import './bilibili/user-videos.js';
|
||||
|
||||
// github
|
||||
import './github/search.js';
|
||||
|
||||
// zhihu
|
||||
import './zhihu/question.js';
|
||||
|
||||
// xiaohongshu
|
||||
import './xiaohongshu/search.js';
|
||||
|
||||
// bbc
|
||||
import './bbc/news.js';
|
||||
|
||||
// weibo
|
||||
import './weibo/hot.js';
|
||||
|
||||
// boss
|
||||
import './boss/search.js';
|
||||
|
||||
// yahoo-finance
|
||||
import './yahoo-finance/quote.js';
|
||||
|
||||
// reuters
|
||||
import './reuters/search.js';
|
||||
|
||||
// smzdm
|
||||
import './smzdm/search.js';
|
||||
|
||||
// ctrip
|
||||
import './ctrip/search.js';
|
||||
|
||||
// youtube
|
||||
import './youtube/search.js';
|
||||
@@ -0,0 +1,30 @@
|
||||
site: reddit
|
||||
name: frontpage
|
||||
description: Reddit Frontpage / r/all
|
||||
domain: reddit.com
|
||||
strategy: cookie
|
||||
browser: true
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 15
|
||||
|
||||
columns: [title, subreddit, author, upvotes, comments, url]
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.reddit.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const res = await fetch('/r/all.json?limit=${{ args.limit }}', { credentials: 'include' });
|
||||
const j = await res.json();
|
||||
return j?.data?.children || [];
|
||||
})()
|
||||
- map:
|
||||
title: ${{ item.data.title }}
|
||||
subreddit: ${{ item.data.subreddit_name_prefixed }}
|
||||
author: ${{ item.data.author }}
|
||||
upvotes: ${{ item.data.score }}
|
||||
comments: ${{ item.data.num_comments }}
|
||||
url: https://www.reddit.com${{ item.data.permalink }}
|
||||
- limit: ${{ args.limit }}
|
||||
@@ -18,9 +18,10 @@ pipeline:
|
||||
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const sub = '${{ args.subreddit }}';
|
||||
const sub = ${{ args.subreddit | json }};
|
||||
const path = sub ? '/r/' + sub + '/hot.json' : '/hot.json';
|
||||
const res = await fetch(path + '?limit=${{ args.limit }}&raw_json=1', {
|
||||
const limit = ${{ args.limit }};
|
||||
const res = await fetch(path + '?limit=' + limit + '&raw_json=1', {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
site: reddit
|
||||
name: search
|
||||
description: Search Reddit Posts
|
||||
domain: reddit.com
|
||||
strategy: cookie
|
||||
browser: true
|
||||
|
||||
args:
|
||||
query:
|
||||
type: string
|
||||
required: true
|
||||
limit:
|
||||
type: int
|
||||
default: 15
|
||||
|
||||
columns: [title, subreddit, author, upvotes, comments, url]
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.reddit.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const q = encodeURIComponent('${{ args.query }}');
|
||||
const res = await fetch('/search.json?q=' + q + '&limit=${{ args.limit }}', { credentials: 'include' });
|
||||
const j = await res.json();
|
||||
return j?.data?.children || [];
|
||||
})()
|
||||
- map:
|
||||
title: ${{ item.data.title }}
|
||||
subreddit: ${{ item.data.subreddit_name_prefixed }}
|
||||
author: ${{ item.data.author }}
|
||||
upvotes: ${{ item.data.score }}
|
||||
comments: ${{ item.data.num_comments }}
|
||||
url: https://www.reddit.com${{ item.data.permalink }}
|
||||
- limit: ${{ args.limit }}
|
||||
@@ -0,0 +1,39 @@
|
||||
site: reddit
|
||||
name: subreddit
|
||||
description: Get posts from a specific Subreddit
|
||||
domain: reddit.com
|
||||
strategy: cookie
|
||||
browser: true
|
||||
|
||||
args:
|
||||
name:
|
||||
type: string
|
||||
required: true
|
||||
sort:
|
||||
type: string
|
||||
default: hot
|
||||
description: "Sorting method: hot, new, top, rising"
|
||||
limit:
|
||||
type: int
|
||||
default: 15
|
||||
|
||||
columns: [title, author, upvotes, comments, url]
|
||||
|
||||
pipeline:
|
||||
- navigate: https://www.reddit.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
let sub = '${{ args.name }}';
|
||||
if (sub.startsWith('r/')) sub = sub.slice(2);
|
||||
const sort = '${{ args.sort }}';
|
||||
const res = await fetch('/r/' + sub + '/' + sort + '.json?limit=${{ args.limit }}', { credentials: 'include' });
|
||||
const j = await res.json();
|
||||
return j?.data?.children || [];
|
||||
})()
|
||||
- map:
|
||||
title: ${{ item.data.title }}
|
||||
author: ${{ item.data.author }}
|
||||
upvotes: ${{ item.data.score }}
|
||||
comments: ${{ item.data.num_comments }}
|
||||
url: https://www.reddit.com${{ item.data.permalink }}
|
||||
- limit: ${{ args.limit }}
|
||||
@@ -0,0 +1,85 @@
|
||||
site: twitter
|
||||
name: bookmarks
|
||||
description: 获取 Twitter 书签列表
|
||||
domain: x.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: Number of bookmarks to return (default 20)
|
||||
|
||||
pipeline:
|
||||
- navigate: https://x.com/i/bookmarks
|
||||
- wait: 2
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const ct0 = document.cookie.split(';').map(c=>c.trim()).find(c=>c.startsWith('ct0='))?.split('=')[1];
|
||||
if (!ct0) throw new Error('No ct0 cookie. Hint: Not logged into x.com.');
|
||||
const bearer = decodeURIComponent('AAAAAAAAAAAAAAAAAAAAANRILgAAAAAAnNwIzUejRCOuH5E6I8xnZz4puTs%3D1Zv7ttfk8LF81IUq16cHjhLTvJu4FA33AGWWjCpTnA');
|
||||
const _h = {'Authorization':'Bearer '+bearer, 'X-Csrf-Token':ct0, 'X-Twitter-Auth-Type':'OAuth2Session', 'X-Twitter-Active-User':'yes'};
|
||||
|
||||
const count = Math.min(${{ args.limit }}, 100);
|
||||
const variables = JSON.stringify({count, includePromotedContent: false});
|
||||
const features = JSON.stringify({
|
||||
rweb_video_screen_enabled: false, profile_label_improvements_pcf_label_in_post_enabled: true,
|
||||
responsive_web_profile_redirect_enabled: false, rweb_tipjar_consumption_enabled: false,
|
||||
verified_phone_label_enabled: false, creator_subscriptions_tweet_preview_api_enabled: true,
|
||||
responsive_web_graphql_timeline_navigation_enabled: true,
|
||||
responsive_web_graphql_skip_user_profile_image_extensions_enabled: false,
|
||||
premium_content_api_read_enabled: false, communities_web_enable_tweet_community_results_fetch: true,
|
||||
c9s_tweet_anatomy_moderator_badge_enabled: true,
|
||||
articles_preview_enabled: true, responsive_web_edit_tweet_api_enabled: true,
|
||||
graphql_is_translatable_rweb_tweet_is_translatable_enabled: true,
|
||||
view_counts_everywhere_api_enabled: true, longform_notetweets_consumption_enabled: true,
|
||||
responsive_web_twitter_article_tweet_consumption_enabled: true,
|
||||
tweet_awards_web_tipping_enabled: false,
|
||||
content_disclosure_indicator_enabled: true, content_disclosure_ai_generated_indicator_enabled: true,
|
||||
freedom_of_speech_not_reach_fetch_enabled: true, standardized_nudges_misinfo: true,
|
||||
tweet_with_visibility_results_prefer_gql_limited_actions_policy_enabled: true,
|
||||
longform_notetweets_rich_text_read_enabled: true, longform_notetweets_inline_media_enabled: false,
|
||||
responsive_web_enhance_cards_enabled: false
|
||||
});
|
||||
const url = '/i/api/graphql/Fy0QMy4q_aZCpkO0PnyLYw/Bookmarks?variables=' + encodeURIComponent(variables) + '&features=' + encodeURIComponent(features);
|
||||
const resp = await fetch(url, {headers: _h, credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + '. Hint: queryId may have changed.');
|
||||
const d = await resp.json();
|
||||
|
||||
const instructions = d.data?.bookmark_timeline_v2?.timeline?.instructions || d.data?.bookmark_timeline?.timeline?.instructions || [];
|
||||
let tweets = [], seen = new Set();
|
||||
for (const inst of instructions) {
|
||||
for (const entry of (inst.entries || [])) {
|
||||
const r = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (!r) continue;
|
||||
const tw = r.tweet || r;
|
||||
const l = tw.legacy || {};
|
||||
if (!tw.rest_id || seen.has(tw.rest_id)) continue;
|
||||
seen.add(tw.rest_id);
|
||||
const u = tw.core?.user_results?.result;
|
||||
const nt = tw.note_tweet?.note_tweet_results?.result?.text;
|
||||
const screenName = u?.legacy?.screen_name || u?.core?.screen_name;
|
||||
tweets.push({
|
||||
id: tw.rest_id,
|
||||
author: screenName,
|
||||
name: u?.legacy?.name || u?.core?.name,
|
||||
url: 'https://x.com/' + (screenName || '_') + '/status/' + tw.rest_id,
|
||||
text: nt || l.full_text || '',
|
||||
likes: l.favorite_count,
|
||||
retweets: l.retweet_count,
|
||||
created_at: l.created_at
|
||||
});
|
||||
}
|
||||
}
|
||||
return tweets;
|
||||
})()
|
||||
|
||||
- map:
|
||||
author: ${{ item.author }}
|
||||
text: ${{ item.text }}
|
||||
likes: ${{ item.likes }}
|
||||
url: ${{ item.url }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [author, text, likes, url]
|
||||
@@ -0,0 +1,78 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'delete',
|
||||
description: 'Delete a specific tweet by URL',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.UI, // Utilizes internal DOM flows for interaction
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'url', type: 'string', required: true, help: 'The URL of the tweet to delete' },
|
||||
],
|
||||
columns: ['status', 'message'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
console.log(`Navigating to tweet: ${kwargs.url}`);
|
||||
await page.goto(kwargs.url);
|
||||
await page.wait(5); // Wait for tweet to load completely
|
||||
|
||||
const result = await page.evaluate(`(async () => {
|
||||
try {
|
||||
// Wait for caret button (which has 'More' aria-label) within the main tweet body
|
||||
// Getting the first 'More' usually corresponds to the main displayed tweet of the URL
|
||||
const moreMenu = document.querySelector('[aria-label="More"]');
|
||||
if (!moreMenu) {
|
||||
return { ok: false, message: 'Could not find the "More" context menu on this tweet. Are you sure you are logged in and looking at a valid tweet?' };
|
||||
}
|
||||
|
||||
// Click the 'More' 3 dots button to open the dropdown menu
|
||||
moreMenu.click();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Wait for dropdown pop-out to appear and look for the 'Delete' option
|
||||
const items = document.querySelectorAll('[role="menuitem"]');
|
||||
let deleteBtn = null;
|
||||
for (const item of items) {
|
||||
if (item.textContent.includes('Delete') && !item.textContent.includes('List')) {
|
||||
deleteBtn = item;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
if (!deleteBtn) {
|
||||
// If there's no Delete button, it's not our tweet OR localization is not English.
|
||||
// Assuming English default for now.
|
||||
return { ok: false, message: 'This tweet does not seem to belong to you, or the Delete option is missing (not your tweet).' };
|
||||
}
|
||||
|
||||
// Click Delete
|
||||
deleteBtn.click();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Find and click the confirmation 'Delete' prompt inside the modal
|
||||
const confirmBtn = document.querySelector('[data-testid="confirmationSheetConfirm"]');
|
||||
if (confirmBtn) {
|
||||
confirmBtn.click();
|
||||
return { ok: true, message: 'Tweet successfully deleted.' };
|
||||
} else {
|
||||
return { ok: false, message: 'Delete confirmation dialog did not appear.' };
|
||||
}
|
||||
} catch (e) {
|
||||
return { ok: false, message: e.toString() };
|
||||
}
|
||||
})()`);
|
||||
|
||||
if (result.ok) {
|
||||
// Wait for the deletion request to be processed
|
||||
await page.wait(2);
|
||||
}
|
||||
|
||||
return [{
|
||||
status: result.ok ? 'success' : 'failed',
|
||||
message: result.message
|
||||
}];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'followers',
|
||||
description: 'Get accounts following a Twitter/X user',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'user', type: 'string', required: false },
|
||||
{ name: 'limit', type: 'int', default: 50 },
|
||||
],
|
||||
columns: ['screen_name', 'name', 'bio', 'followers'],
|
||||
func: async (page, kwargs) => {
|
||||
let targetUser = kwargs.user;
|
||||
|
||||
// If no user is specified, we must figure out the logged-in user's handle
|
||||
if (!targetUser) {
|
||||
await page.goto('https://x.com/home');
|
||||
// wait for home page navigation
|
||||
await page.wait(5);
|
||||
|
||||
const href = await page.evaluate(`() => {
|
||||
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
|
||||
return link ? link.getAttribute('href') : null;
|
||||
}`);
|
||||
|
||||
if (!href) {
|
||||
throw new Error('Could not find logged-in user profile link. Are you logged in?');
|
||||
}
|
||||
targetUser = href.replace('/', '');
|
||||
}
|
||||
|
||||
// 1. Navigate to user profile page
|
||||
await page.goto(`https://x.com/${targetUser}`);
|
||||
await page.wait(3);
|
||||
|
||||
// 2. Inject interceptor for Followers GraphQL API (or user_flow.json)
|
||||
await page.installInterceptor('graphql');
|
||||
|
||||
// 3. Click the followers link inside the profile page
|
||||
await page.evaluate(`() => {
|
||||
const target = '${targetUser}';
|
||||
const link = document.querySelector('a[href="/' + target + '/followers"]');
|
||||
if (link) link.click();
|
||||
}`);
|
||||
await page.wait(3);
|
||||
|
||||
// 4. Trigger API by scrolling
|
||||
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data from opencli's registered interceptors
|
||||
const allRequests = await page.getInterceptedRequests();
|
||||
|
||||
// Debug: Force dump all intercepted XHRs that match followers
|
||||
if (!allRequests || allRequests.length === 0) {
|
||||
console.log('No GraphQL requests captured by the interceptor backend.');
|
||||
return [];
|
||||
}
|
||||
|
||||
console.log('Intercepted keys:', allRequests.map((r: any) => {
|
||||
try {
|
||||
const u = new URL(r.url); return u.pathname;
|
||||
} catch (e) {
|
||||
return r.url;
|
||||
}
|
||||
}));
|
||||
|
||||
const requests = allRequests.filter((r: any) => r.url.includes('Followers'));
|
||||
if (!requests || requests.length === 0) {
|
||||
console.log('No specific Followers requests captured. Check keys printed above.');
|
||||
return [];
|
||||
}
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
try {
|
||||
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
|
||||
if (!instructions) continue;
|
||||
|
||||
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
if (!addEntries) {
|
||||
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
|
||||
}
|
||||
|
||||
if (!addEntries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('user-')) continue;
|
||||
|
||||
const item = entry.content?.itemContent?.user_results?.result;
|
||||
if (!item || item.__typename !== 'User') continue;
|
||||
|
||||
// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
|
||||
const core = item.core || {};
|
||||
const legacy = item.legacy || {};
|
||||
|
||||
results.push({
|
||||
screen_name: core.screen_name || legacy.screen_name || 'unknown',
|
||||
name: core.name || legacy.name || 'unknown',
|
||||
bio: legacy.description || item.profile_bio?.description || '',
|
||||
followers: legacy.followers_count || legacy.normal_followers_count || 0
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parsing errors for individual payloads
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate by screen_name in case multiple scrolls caught the same
|
||||
const unique = new Map();
|
||||
results.forEach(r => unique.set(r.screen_name, r));
|
||||
const deduplicatedResults = Array.from(unique.values());
|
||||
|
||||
return deduplicatedResults.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'following',
|
||||
description: 'Get accounts a Twitter/X user is following',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'user', type: 'string', required: false },
|
||||
{ name: 'limit', type: 'int', default: 50 },
|
||||
],
|
||||
columns: ['screen_name', 'name', 'bio', 'followers'],
|
||||
func: async (page, kwargs) => {
|
||||
let targetUser = kwargs.user;
|
||||
|
||||
// If no user is specified, we must figure out the logged-in user's handle
|
||||
if (!targetUser) {
|
||||
await page.goto('https://x.com/home');
|
||||
// wait for home page navigation
|
||||
await page.wait(5);
|
||||
|
||||
const href = await page.evaluate(`() => {
|
||||
const link = document.querySelector('a[data-testid="AppTabBar_Profile_Link"]');
|
||||
return link ? link.getAttribute('href') : null;
|
||||
}`);
|
||||
|
||||
if (!href) {
|
||||
throw new Error('Could not find logged-in user profile link. Are you logged in?');
|
||||
}
|
||||
targetUser = href.replace('/', '');
|
||||
}
|
||||
|
||||
// 1. Navigate to user profile page
|
||||
await page.goto(`https://x.com/${targetUser}`);
|
||||
await page.wait(3);
|
||||
|
||||
// 2. Inject interceptor for Following GraphQL API
|
||||
await page.installInterceptor('Following');
|
||||
|
||||
// 3. Click the following link inside the profile page
|
||||
await page.evaluate(`() => {
|
||||
const target = '${targetUser}';
|
||||
const link = document.querySelector('a[href="/' + target + '/following"]');
|
||||
if (link) link.click();
|
||||
}`);
|
||||
await page.wait(3);
|
||||
|
||||
// 4. Trigger API by scrolling
|
||||
await page.autoScroll({ times: Math.ceil(kwargs.limit / 20), delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data from opencli's registered interceptors
|
||||
const requests = await page.getInterceptedRequests();
|
||||
|
||||
// Debug: Force dump all intercepted XHRs that match following
|
||||
if (!requests || requests.length === 0) {
|
||||
console.log('No Following requests captured by the interceptor backend.');
|
||||
return [];
|
||||
}
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
try {
|
||||
let instructions = req.data?.data?.user?.result?.timeline?.timeline?.instructions;
|
||||
if (!instructions) continue;
|
||||
|
||||
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
if (!addEntries) {
|
||||
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
|
||||
}
|
||||
|
||||
if (!addEntries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('user-')) continue;
|
||||
|
||||
const item = entry.content?.itemContent?.user_results?.result;
|
||||
if (!item || item.__typename !== 'User') continue;
|
||||
|
||||
// Twitter GraphQL sometimes nests `core` differently depending on the endpoint profile state
|
||||
const core = item.core || {};
|
||||
const legacy = item.legacy || {};
|
||||
|
||||
results.push({
|
||||
screen_name: core.screen_name || legacy.screen_name || 'unknown',
|
||||
name: core.name || legacy.name || 'unknown',
|
||||
bio: legacy.description || item.profile_bio?.description || '',
|
||||
followers: legacy.followers_count || legacy.normal_followers_count || 0
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parsing errors for individual payloads
|
||||
}
|
||||
}
|
||||
|
||||
// Deduplicate by screen_name in case multiple scrolls caught the same
|
||||
const unique = new Map();
|
||||
results.forEach(r => unique.set(r.screen_name, r));
|
||||
const deduplicatedResults = Array.from(unique.values());
|
||||
|
||||
return deduplicatedResults.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,74 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'like',
|
||||
description: 'Like a specific tweet',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.UI, // Utilizes internal DOM flows for interaction
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'url', type: 'string', required: true, help: 'The URL of the tweet to like' },
|
||||
],
|
||||
columns: ['status', 'message'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
console.log(`Navigating to tweet: ${kwargs.url}`);
|
||||
await page.goto(kwargs.url);
|
||||
await page.wait(5); // Wait for tweet to load completely
|
||||
|
||||
const result = await page.evaluate(`(async () => {
|
||||
try {
|
||||
// Poll for the tweet to render
|
||||
let attempts = 0;
|
||||
let likeBtn = null;
|
||||
let unlikeBtn = null;
|
||||
|
||||
while (attempts < 20) {
|
||||
unlikeBtn = document.querySelector('[data-testid="unlike"]');
|
||||
likeBtn = document.querySelector('[data-testid="like"]');
|
||||
|
||||
if (unlikeBtn || likeBtn) break;
|
||||
|
||||
await new Promise(r => setTimeout(r, 500));
|
||||
attempts++;
|
||||
}
|
||||
|
||||
// Check if it's already liked
|
||||
if (unlikeBtn) {
|
||||
return { ok: true, message: 'Tweet is already liked.' };
|
||||
}
|
||||
|
||||
if (!likeBtn) {
|
||||
return { ok: false, message: 'Could not find the Like button on this tweet after waiting 10 seconds. Are you logged in?' };
|
||||
}
|
||||
|
||||
// Click Like
|
||||
likeBtn.click();
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Verify success by checking if the 'unlike' button appeared
|
||||
const verifyBtn = document.querySelector('[data-testid="unlike"]');
|
||||
if (verifyBtn) {
|
||||
return { ok: true, message: 'Tweet successfully liked.' };
|
||||
} else {
|
||||
return { ok: false, message: 'Like action was initiated but UI did not update as expected.' };
|
||||
}
|
||||
} catch (e) {
|
||||
return { ok: false, message: e.toString() };
|
||||
}
|
||||
})()`);
|
||||
|
||||
if (result.ok) {
|
||||
// Wait for the like network request to be processed
|
||||
await page.wait(2);
|
||||
}
|
||||
|
||||
return [{
|
||||
status: result.ok ? 'success' : 'failed',
|
||||
message: result.message
|
||||
}];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,119 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import * as fs from 'fs';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'notifications',
|
||||
description: 'Get Twitter/X notifications',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['id', 'action', 'author', 'text', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// 1. Navigate to notifications
|
||||
await page.goto('https://x.com/notifications');
|
||||
await page.wait(5);
|
||||
|
||||
// 2. Inject interceptor
|
||||
await page.installInterceptor('NotificationsTimeline');
|
||||
|
||||
// 3. Trigger API by scrolling (if we need to load more)
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
try {
|
||||
let instructions = [];
|
||||
if (req.data?.data?.viewer?.timeline_response?.timeline?.instructions) {
|
||||
instructions = req.data.data.viewer.timeline_response.timeline.instructions;
|
||||
} else if (req.data?.data?.viewer_v2?.user_results?.result?.notification_timeline?.timeline?.instructions) {
|
||||
instructions = req.data.data.viewer_v2.user_results.result.notification_timeline.timeline.instructions;
|
||||
} else if (req.data?.data?.timeline?.instructions) {
|
||||
instructions = req.data.data.timeline.instructions;
|
||||
}
|
||||
|
||||
let addEntries = instructions.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
|
||||
// Sometimes it's the first object without a 'type' field but has 'entries'
|
||||
if (!addEntries) {
|
||||
addEntries = instructions.find((i: any) => i.entries && Array.isArray(i.entries));
|
||||
}
|
||||
|
||||
if (!addEntries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('notification-')) {
|
||||
if (entry.content?.items) {
|
||||
for (const subItem of entry.content.items) {
|
||||
processNotificationItem(subItem.item?.itemContent, subItem.entryId);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
processNotificationItem(entry.content?.itemContent, entry.entryId);
|
||||
}
|
||||
|
||||
function processNotificationItem(itemContent: any, entryId: string) {
|
||||
if (!itemContent) return;
|
||||
|
||||
// Twitter wraps standard notifications
|
||||
let item = itemContent?.notification_results?.result || itemContent?.tweet_results?.result || itemContent;
|
||||
|
||||
let actionText = 'Notification';
|
||||
let author = 'unknown';
|
||||
let text = '';
|
||||
let urlStr = '';
|
||||
|
||||
if (item.__typename === 'TimelineNotification') {
|
||||
// Greet likes, retweet, mentions
|
||||
text = item.rich_message?.text || item.message?.text || '';
|
||||
author = item.template?.from_users?.[0]?.user_results?.result?.core?.screen_name || 'unknown';
|
||||
urlStr = item.notification_url?.url || '';
|
||||
actionText = item.notification_icon || 'Activity';
|
||||
|
||||
// If there's an attached tweet
|
||||
const targetTweet = item.template?.target_objects?.[0]?.tweet_results?.result;
|
||||
if (targetTweet) {
|
||||
text += ' | ' + (targetTweet.legacy?.full_text || '');
|
||||
if (!urlStr) {
|
||||
urlStr = `https://x.com/i/status/${targetTweet.rest_id}`;
|
||||
}
|
||||
}
|
||||
} else if (item.__typename === 'TweetNotification') {
|
||||
// Direct mention/reply
|
||||
const tweet = item.tweet_result?.result;
|
||||
author = tweet?.core?.user_results?.result?.legacy?.screen_name || 'unknown';
|
||||
text = tweet?.legacy?.full_text || item.message?.text || '';
|
||||
actionText = 'Mention/Reply';
|
||||
urlStr = `https://x.com/i/status/${tweet?.rest_id}`;
|
||||
} else if (item.__typename === 'Tweet') {
|
||||
author = item.core?.user_results?.result?.legacy?.screen_name || 'unknown';
|
||||
text = item.legacy?.full_text || '';
|
||||
actionText = 'Mention';
|
||||
urlStr = `https://x.com/i/status/${item.rest_id}`;
|
||||
}
|
||||
|
||||
results.push({
|
||||
id: item.id || item.rest_id || entryId,
|
||||
action: actionText,
|
||||
author: author,
|
||||
text: text,
|
||||
url: urlStr || `https://x.com/notifications`
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parsing errors for individual payloads
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,68 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'post',
|
||||
description: 'Post a new tweet/thread',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'text', type: 'string', required: true, help: 'The text content of the tweet' },
|
||||
],
|
||||
columns: ['status', 'message', 'text'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
// 1. Navigate directly to the compose tweet modal
|
||||
await page.goto('https://x.com/compose/tweet');
|
||||
await page.wait(3); // Wait for the modal and React app to hydrate
|
||||
|
||||
// 2. Automate typing and clicking
|
||||
const result = await page.evaluate(`(async () => {
|
||||
try {
|
||||
// Find the active text area
|
||||
const box = document.querySelector('[data-testid="tweetTextarea_0"]');
|
||||
if (box) {
|
||||
box.focus();
|
||||
// insertText is the most reliable way to trigger React's onChange events
|
||||
document.execCommand('insertText', false, ${JSON.stringify(kwargs.text)});
|
||||
} else {
|
||||
return { ok: false, message: 'Could not find the tweet composer text area.' };
|
||||
}
|
||||
|
||||
// Wait a brief moment for the button state to update
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Click the post button
|
||||
const btn = document.querySelector('[data-testid="tweetButton"]');
|
||||
if (btn && !btn.disabled) {
|
||||
btn.click();
|
||||
return { ok: true, message: 'Tweet posted successfully.' };
|
||||
} else {
|
||||
// Sometimes it's rendered inline depending on the viewport
|
||||
const inlineBtn = document.querySelector('[data-testid="tweetButtonInline"]');
|
||||
if (inlineBtn && !inlineBtn.disabled) {
|
||||
inlineBtn.click();
|
||||
return { ok: true, message: 'Tweet posted successfully.' };
|
||||
}
|
||||
return { ok: false, message: 'Tweet button is disabled or not found.' };
|
||||
}
|
||||
} catch (e) {
|
||||
return { ok: false, message: e.toString() };
|
||||
}
|
||||
})()`);
|
||||
|
||||
// 3. Wait a few seconds for the network request to finish sending
|
||||
if (result.ok) {
|
||||
await page.wait(3);
|
||||
}
|
||||
|
||||
return [{
|
||||
status: result.ok ? 'success' : 'failed',
|
||||
message: result.message,
|
||||
text: kwargs.text
|
||||
}];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,61 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'profile',
|
||||
description: 'Fetch tweets from a user profile',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'username', type: 'string', required: true },
|
||||
{ name: 'limit', type: 'int', default: 15 },
|
||||
],
|
||||
columns: ['id', 'text', 'likes', 'views', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// Navigate to user profile via search for reliability
|
||||
await page.goto(`https://x.com/search?q=from:${kwargs.username}&f=live`);
|
||||
await page.wait(5);
|
||||
|
||||
// Inject XHR interceptor
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// Trigger API by scrolling
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// Retrieve data
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
try {
|
||||
const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
|
||||
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
if (!addEntries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('tweet-')) continue;
|
||||
|
||||
let tweet = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (!tweet) continue;
|
||||
|
||||
if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
|
||||
tweet = tweet.tweet;
|
||||
}
|
||||
|
||||
results.push({
|
||||
id: tweet.rest_id,
|
||||
text: tweet.legacy?.full_text || '',
|
||||
likes: tweet.legacy?.favorite_count || 0,
|
||||
views: tweet.views?.count || '0',
|
||||
url: `https://x.com/i/status/${tweet.rest_id}`
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,62 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'reply',
|
||||
description: 'Reply to a specific tweet',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.UI, // Uses the UI directly to input and click post
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'url', type: 'string', required: true, help: 'The URL of the tweet to reply to' },
|
||||
{ name: 'text', type: 'string', required: true, help: 'The text content of your reply' },
|
||||
],
|
||||
columns: ['status', 'message', 'text'],
|
||||
func: async (page: IPage | null, kwargs: any) => {
|
||||
if (!page) throw new Error('Requires browser');
|
||||
|
||||
// 1. Navigate to the tweet page
|
||||
await page.goto(kwargs.url);
|
||||
await page.wait(5); // Wait for the react application to hydrate
|
||||
|
||||
// 2. Automate typing the reply and clicking reply
|
||||
const result = await page.evaluate(`(async () => {
|
||||
try {
|
||||
// Find the reply text area on the tweet page.
|
||||
// The placeholder is usually "Post your reply"
|
||||
const box = document.querySelector('[data-testid="tweetTextarea_0"]');
|
||||
if (box) {
|
||||
box.focus();
|
||||
document.execCommand('insertText', false, ${JSON.stringify(kwargs.text)});
|
||||
} else {
|
||||
return { ok: false, message: 'Could not find the reply text area. Are you logged in?' };
|
||||
}
|
||||
|
||||
// Wait for React state to register the input and enable the button
|
||||
await new Promise(r => setTimeout(r, 1000));
|
||||
|
||||
// Find the Reply button. It usually shares the same test id tweetButtonInline in this context
|
||||
const btn = document.querySelector('[data-testid="tweetButtonInline"]');
|
||||
if (btn && !btn.disabled) {
|
||||
btn.click();
|
||||
return { ok: true, message: 'Reply posted successfully.' };
|
||||
} else {
|
||||
return { ok: false, message: 'Reply button is disabled or not found.' };
|
||||
}
|
||||
} catch (e) {
|
||||
return { ok: false, message: e.toString() };
|
||||
}
|
||||
})()`);
|
||||
|
||||
if (result.ok) {
|
||||
await page.wait(3); // Wait for network submission to complete
|
||||
}
|
||||
|
||||
return [{
|
||||
status: result.ok ? 'success' : 'failed',
|
||||
message: result.message,
|
||||
text: kwargs.text
|
||||
}];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,65 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'search',
|
||||
description: 'Search Twitter/X for tweets',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.INTERCEPT, // Use intercept strategy
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', type: 'string', required: true },
|
||||
{ name: 'limit', type: 'int', default: 15 },
|
||||
],
|
||||
columns: ['id', 'author', 'text', 'likes', 'views', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
// 1. Navigate to the search page
|
||||
const q = encodeURIComponent(kwargs.query);
|
||||
await page.goto(`https://x.com/search?q=${q}&f=top`);
|
||||
await page.wait(5);
|
||||
|
||||
// 2. Inject XHR interceptor
|
||||
await page.installInterceptor('SearchTimeline');
|
||||
|
||||
// 3. Trigger API by scrolling
|
||||
await page.autoScroll({ times: 3, delayMs: 2000 });
|
||||
|
||||
// 4. Retrieve data
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
try {
|
||||
const insts = req.data.data.search_by_raw_query.search_timeline.timeline.instructions;
|
||||
const addEntries = insts.find((i: any) => i.type === 'TimelineAddEntries');
|
||||
if (!addEntries) continue;
|
||||
|
||||
for (const entry of addEntries.entries) {
|
||||
if (!entry.entryId.startsWith('tweet-')) continue;
|
||||
|
||||
let tweet = entry.content?.itemContent?.tweet_results?.result;
|
||||
if (!tweet) continue;
|
||||
|
||||
// Handle retweet wrapping
|
||||
if (tweet.__typename === 'TweetWithVisibilityResults' && tweet.tweet) {
|
||||
tweet = tweet.tweet;
|
||||
}
|
||||
|
||||
results.push({
|
||||
id: tweet.rest_id,
|
||||
author: tweet.core?.user_results?.result?.legacy?.screen_name || 'unknown',
|
||||
text: tweet.legacy?.full_text || '',
|
||||
likes: tweet.legacy?.favorite_count || 0,
|
||||
views: tweet.views?.count || '0',
|
||||
url: `https://x.com/i/status/${tweet.rest_id}`
|
||||
});
|
||||
}
|
||||
} catch (e) {
|
||||
// ignore parsing errors for individual payloads
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,50 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'twitter',
|
||||
name: 'timeline',
|
||||
description: 'Twitter Home Timeline',
|
||||
domain: 'x.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20 },
|
||||
],
|
||||
columns: ['responseType', 'first'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://x.com/home');
|
||||
await page.wait(5);
|
||||
// Inject the fetch interceptor manually to see exactly what happens
|
||||
await page.evaluate(`
|
||||
() => {
|
||||
window.__intercept_data = [];
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
let u = typeof args[0] === 'string' ? args[0] : (args[0] && args[0].url) || '';
|
||||
const res = await origFetch.apply(this, args);
|
||||
setTimeout(async () => {
|
||||
try {
|
||||
if (u.includes('HomeTimeline')) {
|
||||
const clone = res.clone();
|
||||
const j = await clone.json();
|
||||
window.__intercept_data.push(j);
|
||||
}
|
||||
} catch(e) {}
|
||||
}, 0);
|
||||
return res;
|
||||
};
|
||||
}
|
||||
`);
|
||||
|
||||
// trigger scroll
|
||||
for(let i=0; i<3; i++) {
|
||||
await page.evaluate('() => window.scrollTo(0, document.body.scrollHeight)');
|
||||
await page.wait(2);
|
||||
}
|
||||
|
||||
// extract
|
||||
const data = await page.evaluate('() => window.__intercept_data');
|
||||
if (!data || data.length === 0) return [{responseType: 'no data captured'}];
|
||||
|
||||
return [{responseType: `captured ${data.length} responses`, first: JSON.stringify(data[0]).substring(0,300)}];
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* V2EX Daily Check-in adapter.
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'v2ex',
|
||||
name: 'daily',
|
||||
description: 'V2EX 每日签到并领取铜币',
|
||||
domain: 'www.v2ex.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
|
||||
args: [],
|
||||
columns: ['status', 'message'],
|
||||
func: async (page: IPage | null) => {
|
||||
if (!page) throw new Error('Browser page required');
|
||||
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error('[opencli:v2ex] Navigating to /mission/daily');
|
||||
}
|
||||
await page.goto('https://www.v2ex.com/mission/daily');
|
||||
|
||||
// Cloudflare challenge bypass wait
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
const title = await page.evaluate(`() => document.title`);
|
||||
if (!title?.includes('Just a moment')) break;
|
||||
if (process.env.OPENCLI_VERBOSE) console.error('[opencli:v2ex] Waiting for Cloudflare...');
|
||||
}
|
||||
|
||||
// Evaluate DOM to find if we need to check in
|
||||
const checkResult = await page.evaluate(`
|
||||
async () => {
|
||||
const btn = document.querySelector('input.super.normal.button');
|
||||
if (!btn || !btn.value.includes('领取')) {
|
||||
return { claimed: true, message: '今日奖励已发/无需领取' };
|
||||
}
|
||||
|
||||
const onclick = btn.getAttribute('onclick');
|
||||
if (onclick) {
|
||||
const match = onclick.match(/once=(\\d+)/);
|
||||
if (match) {
|
||||
return { claimed: false, once: match[1], message: btn.value };
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
claimed: false,
|
||||
error: '找到了按钮,但未能提取 once token',
|
||||
debug_title: document.title,
|
||||
debug_body: document.body.innerText.substring(0, 200).replace(/\\n/g, ' ')
|
||||
};
|
||||
}
|
||||
`);
|
||||
|
||||
if (checkResult.error) {
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error(`[opencli:v2ex:debug] Page Title: ${checkResult.debug_title}`);
|
||||
console.error(`[opencli:v2ex:debug] Page Body: ${checkResult.debug_body}`);
|
||||
}
|
||||
throw new Error(checkResult.error);
|
||||
}
|
||||
|
||||
if (checkResult.claimed) {
|
||||
return [{ status: '✅ 已签到', message: checkResult.message }];
|
||||
}
|
||||
|
||||
// Perform check in
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error(`[opencli:v2ex] Found check-in token: once=${checkResult.once}. Checking in...`);
|
||||
}
|
||||
|
||||
await page.goto(`https://www.v2ex.com/mission/daily/redeem?once=${checkResult.once}`);
|
||||
await new Promise(resolve => setTimeout(resolve, 3000)); // wait longer for redirect
|
||||
|
||||
// Verify result
|
||||
const verifyResult = await page.evaluate(`
|
||||
async () => {
|
||||
const btn = document.querySelector('input.super.normal.button');
|
||||
if (!btn || !btn.value.includes('领取')) {
|
||||
// fetch balance to show user
|
||||
let balance = '';
|
||||
const balanceLink = document.querySelector('a.balance_area');
|
||||
if (balanceLink) {
|
||||
balance = Array.from(balanceLink.childNodes)
|
||||
.filter(n => n.nodeType === 3)
|
||||
.map(n => n.textContent?.trim())
|
||||
.join(' ')
|
||||
.trim();
|
||||
}
|
||||
return { success: true, balance };
|
||||
}
|
||||
return { success: false };
|
||||
}
|
||||
`);
|
||||
|
||||
if (verifyResult.success) {
|
||||
return [{ status: '🎉 签到成功', message: `当前余额: ${verifyResult.balance || '未知'}` }];
|
||||
} else {
|
||||
return [{ status: '❌ 签到失败', message: '未能确认签到结果,请手动检查' }];
|
||||
}
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,103 @@
|
||||
/**
|
||||
* V2EX Me (Profile/Balance) adapter.
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'v2ex',
|
||||
name: 'me',
|
||||
description: 'V2EX 获取个人资料 (余额/未读提醒)',
|
||||
domain: 'www.v2ex.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
|
||||
args: [],
|
||||
columns: ['username', 'balance', 'unread_notifications', 'daily_reward_ready'],
|
||||
func: async (page: IPage | null) => {
|
||||
if (!page) throw new Error('Browser page required');
|
||||
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error('[opencli:v2ex] Navigating to /');
|
||||
}
|
||||
await page.goto('https://www.v2ex.com/');
|
||||
|
||||
// Cloudflare challenge bypass wait
|
||||
for (let i = 0; i < 5; i++) {
|
||||
await new Promise(r => setTimeout(r, 1500));
|
||||
const title = await page.evaluate(`() => document.title`);
|
||||
if (!title?.includes('Just a moment')) break;
|
||||
if (process.env.OPENCLI_VERBOSE) console.error('[opencli:v2ex] Waiting for Cloudflare...');
|
||||
}
|
||||
|
||||
// Evaluate DOM to extract user profile
|
||||
const data = await page.evaluate(`
|
||||
async () => {
|
||||
let username = 'Unknown';
|
||||
const navLinks = Array.from(document.querySelectorAll('a.top')).map(a => a.textContent?.trim());
|
||||
if (navLinks.length > 1 && navLinks[0] === '首页') {
|
||||
username = navLinks[1] || 'Unknown';
|
||||
}
|
||||
|
||||
if (username === 'Unknown') {
|
||||
// Fallback check just in case
|
||||
const profileEl = document.querySelector('a[href^="/member/"]');
|
||||
if (profileEl && profileEl.textContent && profileEl.textContent.trim().length > 0) {
|
||||
username = profileEl.textContent.trim();
|
||||
}
|
||||
}
|
||||
|
||||
let balance = '0';
|
||||
const balanceLink = document.querySelector('a.balance_area');
|
||||
if (balanceLink) {
|
||||
balance = Array.from(balanceLink.childNodes)
|
||||
.filter(n => n.nodeType === 3)
|
||||
.map(n => n.textContent?.trim())
|
||||
.join(' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
let unread_notifications = '0';
|
||||
const notesEl = document.querySelector('a[href="/notifications"]');
|
||||
if (notesEl) {
|
||||
const text = notesEl.textContent?.trim() || '';
|
||||
const match = text.match(/(\\d+)\\s*未读提醒/);
|
||||
if (match) {
|
||||
unread_notifications = match[1];
|
||||
}
|
||||
}
|
||||
|
||||
let daily_reward_ready = false;
|
||||
const dailyEl = document.querySelector('a[href^="/mission/daily"]');
|
||||
if (dailyEl && dailyEl.textContent?.includes('领取今日的登录奖励')) {
|
||||
daily_reward_ready = true;
|
||||
}
|
||||
|
||||
if (username === 'Unknown') {
|
||||
return {
|
||||
error: '请先登录 V2EX(可能是 Cookie 未配置或已失效)',
|
||||
debug_title: document.title,
|
||||
debug_body: document.body.innerText.substring(0, 200).replace(/\\n/g, ' ')
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
username,
|
||||
balance,
|
||||
unread_notifications,
|
||||
daily_reward_ready: daily_reward_ready ? '是' : '否'
|
||||
};
|
||||
}
|
||||
`);
|
||||
|
||||
if (data.error) {
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error(`[opencli:v2ex:debug] Page Title: ${data.debug_title}`);
|
||||
console.error(`[opencli:v2ex:debug] Page Body: ${data.debug_body}`);
|
||||
}
|
||||
throw new Error(data.error);
|
||||
}
|
||||
|
||||
return [data];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* V2EX Notifications adapter.
|
||||
*/
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
import type { IPage } from '../../types.js';
|
||||
|
||||
cli({
|
||||
site: 'v2ex',
|
||||
name: 'notifications',
|
||||
description: 'V2EX 获取提醒 (回复/由于)',
|
||||
domain: 'www.v2ex.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of notifications' }
|
||||
],
|
||||
columns: ['type', 'content', 'time'],
|
||||
func: async (page: IPage | null, kwargs) => {
|
||||
if (!page) throw new Error('Browser page required');
|
||||
|
||||
if (process.env.OPENCLI_VERBOSE) {
|
||||
console.error('[opencli:v2ex] Navigating to /notifications');
|
||||
}
|
||||
await page.goto('https://www.v2ex.com/notifications');
|
||||
await new Promise(r => setTimeout(r, 1500)); // waitForLoadState doesn't always work robustly
|
||||
|
||||
// Evaluate DOM to extract notifications
|
||||
const data = await page.evaluate(`
|
||||
async () => {
|
||||
const items = Array.from(document.querySelectorAll('#Main .box .cell[id^="n_"]'));
|
||||
return items.map(item => {
|
||||
let type = '通知';
|
||||
let time = '';
|
||||
|
||||
// determine type based on text content
|
||||
const text = item.textContent || '';
|
||||
if (text.includes('回复了你')) type = '回复';
|
||||
else if (text.includes('感谢了你')) type = '感谢';
|
||||
else if (text.includes('收藏了你')) type = '收藏';
|
||||
else if (text.includes('提及你')) type = '提及';
|
||||
|
||||
const timeEl = item.querySelector('.snow');
|
||||
if (timeEl) {
|
||||
time = timeEl.textContent?.trim() || '';
|
||||
}
|
||||
|
||||
// payload contains the actual reply text if any
|
||||
let payload = '';
|
||||
const payloadEl = item.querySelector('.payload');
|
||||
if (payloadEl) {
|
||||
payload = payloadEl.textContent?.trim() || '';
|
||||
}
|
||||
|
||||
// fallback to full text cleaning if no payload (e.g. for favorites/thanks)
|
||||
let content = payload;
|
||||
if (!content) {
|
||||
content = text.replace(/\\s+/g, ' ').trim();
|
||||
// strip out time from content if present
|
||||
if (time && content.includes(time)) {
|
||||
content = content.replace(time, '').trim();
|
||||
}
|
||||
}
|
||||
|
||||
return { type, content, time };
|
||||
});
|
||||
}
|
||||
`);
|
||||
|
||||
if (!Array.isArray(data)) {
|
||||
throw new Error('Failed to parse notifications data');
|
||||
}
|
||||
|
||||
const limit = kwargs.limit || 20;
|
||||
return data.slice(0, limit);
|
||||
},
|
||||
});
|
||||
@@ -1,6 +1,9 @@
|
||||
/**
|
||||
* Xiaohongshu search — trigger search via Pinia store + XHR interception.
|
||||
* Inspired by bb-sites/xiaohongshu/search.js but adapted for opencli pipeline.
|
||||
* Xiaohongshu search — DOM-based extraction from search results page.
|
||||
* The previous Pinia store + XHR interception approach broke because
|
||||
* the API now returns empty items. This version navigates directly to
|
||||
* the search results page and extracts data from rendered DOM elements.
|
||||
* Ref: https://github.com/jackwener/opencli/issues/10
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
@@ -15,57 +18,51 @@ cli({
|
||||
{ name: 'keyword', required: true, help: 'Search keyword' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of results' },
|
||||
],
|
||||
columns: ['rank', 'title', 'author', 'likes', 'type'],
|
||||
columns: ['rank', 'title', 'author', 'likes'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto('https://www.xiaohongshu.com');
|
||||
await page.wait(2);
|
||||
const keyword = encodeURIComponent(kwargs.keyword);
|
||||
await page.goto(
|
||||
`https://www.xiaohongshu.com/search_result?keyword=${keyword}&source=web_search_result_notes`
|
||||
);
|
||||
await page.wait(3);
|
||||
|
||||
// Scroll a couple of times to load more results
|
||||
await page.autoScroll({ times: 2 });
|
||||
|
||||
const data = await page.evaluate(`
|
||||
(async () => {
|
||||
const app = document.querySelector('#app')?.__vue_app__;
|
||||
const pinia = app?.config?.globalProperties?.$pinia;
|
||||
if (!pinia?._s) return {error: 'Page not ready'};
|
||||
(() => {
|
||||
const notes = document.querySelectorAll('section.note-item');
|
||||
const results = [];
|
||||
notes.forEach(el => {
|
||||
// Skip "related searches" sections
|
||||
if (el.classList.contains('query-note-item')) return;
|
||||
|
||||
const searchStore = pinia._s.get('search');
|
||||
if (!searchStore) return {error: 'Search store not found'};
|
||||
const titleEl = el.querySelector('.title, .note-title, a.title');
|
||||
const nameEl = el.querySelector('.name, .author-name, .nick-name');
|
||||
const likesEl = el.querySelector('.count, .like-count, .like-wrapper .count');
|
||||
const linkEl = el.querySelector('a[href*="/explore/"], a[href*="/search_result/"], a[href*="/note/"]');
|
||||
|
||||
let captured = null;
|
||||
const origOpen = XMLHttpRequest.prototype.open;
|
||||
const origSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(m, u) { this.__url = u; return origOpen.apply(this, arguments); };
|
||||
XMLHttpRequest.prototype.send = function(b) {
|
||||
if (this.__url?.includes('search/notes')) {
|
||||
const x = this;
|
||||
const orig = x.onreadystatechange;
|
||||
x.onreadystatechange = function() { if (x.readyState === 4 && !captured) { try { captured = JSON.parse(x.responseText); } catch {} } if (orig) orig.apply(this, arguments); };
|
||||
}
|
||||
return origSend.apply(this, arguments);
|
||||
};
|
||||
const href = linkEl?.getAttribute('href') || '';
|
||||
const noteId = href.match(/\\/(?:explore|note)\\/([a-f0-9]+)/)?.[1] || '';
|
||||
|
||||
try {
|
||||
searchStore.mutateSearchValue('${kwargs.keyword}');
|
||||
await searchStore.loadMore();
|
||||
await new Promise(r => setTimeout(r, 800));
|
||||
} finally {
|
||||
XMLHttpRequest.prototype.open = origOpen;
|
||||
XMLHttpRequest.prototype.send = origSend;
|
||||
}
|
||||
|
||||
if (!captured?.success) return {error: captured?.msg || 'Search failed'};
|
||||
return (captured.data?.items || []).map(i => ({
|
||||
title: i.note_card?.display_title || '',
|
||||
type: i.note_card?.type || '',
|
||||
url: 'https://www.xiaohongshu.com/explore/' + i.id,
|
||||
author: i.note_card?.user?.nickname || '',
|
||||
likes: i.note_card?.interact_info?.liked_count || '0',
|
||||
}));
|
||||
results.push({
|
||||
title: (titleEl?.textContent || '').trim(),
|
||||
author: (nameEl?.textContent || '').trim(),
|
||||
likes: (likesEl?.textContent || '0').trim(),
|
||||
url: noteId ? 'https://www.xiaohongshu.com/explore/' + noteId : '',
|
||||
});
|
||||
});
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
|
||||
if (!Array.isArray(data)) return [];
|
||||
return data.slice(0, kwargs.limit).map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
...item,
|
||||
}));
|
||||
return data
|
||||
.filter((item: any) => item.title)
|
||||
.slice(0, kwargs.limit)
|
||||
.map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
...item,
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { cli, Strategy } from '../../registry.js';
|
||||
|
||||
cli({
|
||||
site: 'xiaohongshu',
|
||||
name: 'user',
|
||||
description: 'Get user notes from Xiaohongshu',
|
||||
domain: 'xiaohongshu.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'id', type: 'string', required: true },
|
||||
{ name: 'limit', type: 'int', default: 15 },
|
||||
],
|
||||
columns: ['id', 'title', 'type', 'likes', 'url'],
|
||||
func: async (page, kwargs) => {
|
||||
await page.goto(`https://www.xiaohongshu.com/user/profile/${kwargs.id}`);
|
||||
await page.wait(5);
|
||||
|
||||
await page.installInterceptor('v1/user/posted');
|
||||
|
||||
// Trigger API by scrolling
|
||||
await page.autoScroll({ times: 2, delayMs: 2000 });
|
||||
|
||||
// Retrieve data
|
||||
const requests = await page.getInterceptedRequests();
|
||||
if (!requests || requests.length === 0) return [];
|
||||
|
||||
let results: any[] = [];
|
||||
for (const req of requests) {
|
||||
if (req.data && req.data.data && req.data.data.notes) {
|
||||
for (const note of req.data.data.notes) {
|
||||
results.push({
|
||||
id: note.note_id || note.id,
|
||||
title: note.display_title || '',
|
||||
type: note.type || '',
|
||||
likes: note.interact_info?.liked_count || '0',
|
||||
url: `https://www.xiaohongshu.com/explore/${note.note_id || note.id}`
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return results.slice(0, kwargs.limit);
|
||||
}
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
site: xueqiu
|
||||
name: feed
|
||||
description: 获取雪球首页时间线(关注用户的动态)
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
page:
|
||||
type: int
|
||||
default: 1
|
||||
description: 页码,默认 1
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: 每页数量,默认 20
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const page = ${{ args.page }};
|
||||
const count = ${{ args.limit }};
|
||||
const resp = await fetch(`https://xueqiu.com/v4/statuses/home_timeline.json?page=${page}&count=${count}`, {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
|
||||
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').trim();
|
||||
const list = d.home_timeline || d.list || [];
|
||||
return list.map(item => {
|
||||
const user = item.user || {};
|
||||
return {
|
||||
id: item.id,
|
||||
text: strip(item.description).substring(0, 200),
|
||||
url: 'https://xueqiu.com/' + user.id + '/' + item.id,
|
||||
author: user.screen_name,
|
||||
likes: item.fav_count,
|
||||
retweets: item.retweet_count,
|
||||
replies: item.reply_count,
|
||||
created_at: item.created_at ? new Date(item.created_at).toISOString() : null
|
||||
};
|
||||
});
|
||||
})()
|
||||
|
||||
- map:
|
||||
author: ${{ item.author }}
|
||||
text: ${{ item.text }}
|
||||
likes: ${{ item.likes }}
|
||||
replies: ${{ item.replies }}
|
||||
url: ${{ item.url }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [author, text, likes, replies, url]
|
||||
@@ -0,0 +1,49 @@
|
||||
site: xueqiu
|
||||
name: hot-stock
|
||||
description: 获取雪球热门股票榜
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: 返回数量,默认 20,最大 50
|
||||
type:
|
||||
type: str
|
||||
default: "10"
|
||||
description: 榜单类型 10=人气榜(默认) 12=关注榜
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const count = ${{ args.limit }};
|
||||
const type = ${{ args.type | json }};
|
||||
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/hot_stock/list.json?size=${count}&type=${type}`, {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
if (!d.data || !d.data.items) throw new Error('获取失败');
|
||||
return d.data.items.map((s, i) => ({
|
||||
rank: i + 1,
|
||||
symbol: s.symbol,
|
||||
name: s.name,
|
||||
price: s.current,
|
||||
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
||||
heat: s.value,
|
||||
rank_change: s.rank_change,
|
||||
url: 'https://xueqiu.com/S/' + s.symbol
|
||||
}));
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ item.rank }}
|
||||
symbol: ${{ item.symbol }}
|
||||
name: ${{ item.name }}
|
||||
price: ${{ item.price }}
|
||||
changePercent: ${{ item.changePercent }}
|
||||
heat: ${{ item.heat }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, symbol, name, price, changePercent, heat]
|
||||
@@ -0,0 +1,46 @@
|
||||
site: xueqiu
|
||||
name: hot
|
||||
description: 获取雪球热门动态
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
limit:
|
||||
type: int
|
||||
default: 20
|
||||
description: 返回数量,默认 20,最大 50
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const resp = await fetch('https://xueqiu.com/statuses/hot/listV3.json?source=hot&page=1', {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
const list = d.list || [];
|
||||
|
||||
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').trim();
|
||||
return list.map((item, i) => {
|
||||
const user = item.user || {};
|
||||
return {
|
||||
rank: i + 1,
|
||||
text: strip(item.description).substring(0, 200),
|
||||
url: 'https://xueqiu.com/' + user.id + '/' + item.id,
|
||||
author: user.screen_name,
|
||||
likes: item.fav_count,
|
||||
retweets: item.retweet_count,
|
||||
replies: item.reply_count
|
||||
};
|
||||
});
|
||||
})()
|
||||
|
||||
- map:
|
||||
rank: ${{ item.rank }}
|
||||
author: ${{ item.author }}
|
||||
text: ${{ item.text }}
|
||||
likes: ${{ item.likes }}
|
||||
url: ${{ item.url }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [rank, author, text, likes, url]
|
||||
@@ -0,0 +1,53 @@
|
||||
site: xueqiu
|
||||
name: search
|
||||
description: 搜索雪球股票(代码或名称)
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
query:
|
||||
type: str
|
||||
description: 搜索关键词,如 茅台、AAPL、腾讯
|
||||
limit:
|
||||
type: int
|
||||
default: 10
|
||||
description: 返回数量,默认 10
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const query = ${{ args.query | json }};
|
||||
const count = ${{ args.limit }};
|
||||
if (!query) throw new Error('Missing argument: query');
|
||||
const resp = await fetch(`https://xueqiu.com/stock/search.json?code=${encodeURIComponent(query)}&size=${count}`, {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
return (d.stocks || []).map(s => {
|
||||
let symbol = '';
|
||||
if (s.exchange === 'SH' || s.exchange === 'SZ' || s.exchange === 'BJ') {
|
||||
symbol = s.code.startsWith(s.exchange) ? s.code : s.exchange + s.code;
|
||||
} else {
|
||||
symbol = s.code;
|
||||
}
|
||||
return {
|
||||
symbol: symbol,
|
||||
name: s.name,
|
||||
exchange: s.exchange,
|
||||
price: s.current,
|
||||
changePercent: s.percentage != null ? s.percentage.toFixed(2) + '%' : null,
|
||||
url: 'https://xueqiu.com/S/' + symbol
|
||||
};
|
||||
});
|
||||
})()
|
||||
|
||||
- map:
|
||||
symbol: ${{ item.symbol }}
|
||||
name: ${{ item.name }}
|
||||
exchange: ${{ item.exchange }}
|
||||
price: ${{ item.price }}
|
||||
changePercent: ${{ item.changePercent }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [symbol, name, exchange, price, changePercent]
|
||||
@@ -0,0 +1,67 @@
|
||||
site: xueqiu
|
||||
name: stock
|
||||
description: 获取雪球股票实时行情
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
symbol:
|
||||
type: str
|
||||
description: 股票代码,如 SH600519、SZ000858、AAPL、00700
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const symbol = (${{ args.symbol | json }} || '').toUpperCase();
|
||||
if (!symbol) throw new Error('Missing argument: symbol');
|
||||
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/batch/quote.json?symbol=${encodeURIComponent(symbol)}`, {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
if (!d.data || !d.data.items || d.data.items.length === 0) throw new Error('未找到股票: ' + symbol);
|
||||
|
||||
function fmtAmount(v) {
|
||||
if (v == null) return null;
|
||||
if (Math.abs(v) >= 1e12) return (v / 1e12).toFixed(2) + '万亿';
|
||||
if (Math.abs(v) >= 1e8) return (v / 1e8).toFixed(2) + '亿';
|
||||
if (Math.abs(v) >= 1e4) return (v / 1e4).toFixed(2) + '万';
|
||||
return v.toString();
|
||||
}
|
||||
|
||||
const item = d.data.items[0];
|
||||
const q = item.quote || {};
|
||||
const m = item.market || {};
|
||||
|
||||
return [{
|
||||
name: q.name,
|
||||
symbol: q.symbol,
|
||||
exchange: q.exchange,
|
||||
currency: q.currency,
|
||||
price: q.current,
|
||||
change: q.chg,
|
||||
changePercent: q.percent != null ? q.percent.toFixed(2) + '%' : null,
|
||||
open: q.open,
|
||||
high: q.high,
|
||||
low: q.low,
|
||||
prevClose: q.last_close,
|
||||
amplitude: q.amplitude != null ? q.amplitude.toFixed(2) + '%' : null,
|
||||
volume: q.volume,
|
||||
amount: fmtAmount(q.amount),
|
||||
turnover_rate: q.turnover_rate != null ? q.turnover_rate.toFixed(2) + '%' : null,
|
||||
marketCap: fmtAmount(q.market_capital),
|
||||
floatMarketCap: fmtAmount(q.float_market_capital),
|
||||
ytdPercent: q.current_year_percent != null ? q.current_year_percent.toFixed(2) + '%' : null,
|
||||
market_status: m.status || null,
|
||||
time: q.timestamp ? new Date(q.timestamp).toISOString() : null,
|
||||
url: 'https://xueqiu.com/S/' + q.symbol
|
||||
}];
|
||||
})()
|
||||
|
||||
- map:
|
||||
name: ${{ item.name }}
|
||||
symbol: ${{ item.symbol }}
|
||||
price: ${{ item.price }}
|
||||
changePercent: ${{ item.changePercent }}
|
||||
marketCap: ${{ item.marketCap }}
|
||||
|
||||
columns: [name, symbol, price, changePercent, marketCap]
|
||||
@@ -0,0 +1,46 @@
|
||||
site: xueqiu
|
||||
name: watchlist
|
||||
description: 获取雪球自选股列表
|
||||
domain: xueqiu.com
|
||||
browser: true
|
||||
|
||||
args:
|
||||
category:
|
||||
type: str # using str to prevent parsing issues like 01
|
||||
default: "1"
|
||||
description: "分类:1=自选(默认) 2=持仓 3=关注"
|
||||
limit:
|
||||
type: int
|
||||
default: 100
|
||||
description: 默认 100
|
||||
|
||||
pipeline:
|
||||
- navigate: https://xueqiu.com
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const category = parseInt(${{ args.category | json }}) || 1;
|
||||
const resp = await fetch(`https://stock.xueqiu.com/v5/stock/portfolio/stock/list.json?size=100&category=${category}&pid=-1`, {credentials: 'include'});
|
||||
if (!resp.ok) throw new Error('HTTP ' + resp.status + ' Hint: Not logged in?');
|
||||
const d = await resp.json();
|
||||
if (!d.data || !d.data.stocks) throw new Error('获取失败,可能未登录');
|
||||
|
||||
return d.data.stocks.map(s => ({
|
||||
symbol: s.symbol,
|
||||
name: s.name,
|
||||
price: s.current,
|
||||
change: s.chg,
|
||||
changePercent: s.percent != null ? s.percent.toFixed(2) + '%' : null,
|
||||
volume: s.volume,
|
||||
url: 'https://xueqiu.com/S/' + s.symbol
|
||||
}));
|
||||
})()
|
||||
|
||||
- map:
|
||||
symbol: ${{ item.symbol }}
|
||||
name: ${{ item.name }}
|
||||
price: ${{ item.price }}
|
||||
changePercent: ${{ item.changePercent }}
|
||||
|
||||
- limit: ${{ args.limit }}
|
||||
|
||||
columns: [symbol, name, price, changePercent]
|
||||
@@ -17,12 +17,16 @@ pipeline:
|
||||
const res = await fetch('https://www.zhihu.com/api/v3/feed/topstory/hot-lists/total?limit=50', {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
const text = await res.text();
|
||||
const d = JSON.parse(
|
||||
text.replace(/("id"\s*:\s*)(\d{16,})/g, '$1"$2"')
|
||||
);
|
||||
return (d?.data || []).map((item) => {
|
||||
const t = item.target || {};
|
||||
const questionId = t.id == null ? '' : String(t.id);
|
||||
return {
|
||||
title: t.title,
|
||||
url: 'https://www.zhihu.com/question/' + t.id,
|
||||
url: 'https://www.zhihu.com/question/' + questionId,
|
||||
answer_count: t.answer_count,
|
||||
follower_count: t.follower_count,
|
||||
heat: item.detail_text || '',
|
||||
|
||||
@@ -19,7 +19,9 @@ pipeline:
|
||||
- evaluate: |
|
||||
(async () => {
|
||||
const strip = (html) => (html || '').replace(/<[^>]+>/g, '').replace(/ /g, ' ').replace(/</g, '<').replace(/>/g, '>').replace(/&/g, '&').replace(/<em>/g, '').replace(/<\/em>/g, '').trim();
|
||||
const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent('${{ args.keyword }}') + '&t=general&offset=0&limit=${{ args.limit }}', {
|
||||
const keyword = ${{ args.keyword | json }};
|
||||
const limit = ${{ args.limit }};
|
||||
const res = await fetch('https://www.zhihu.com/api/v4/search_v3?q=' + encodeURIComponent(keyword) + '&t=general&offset=0&limit=' + limit, {
|
||||
credentials: 'include'
|
||||
});
|
||||
const d = await res.json();
|
||||
|
||||
@@ -0,0 +1,35 @@
|
||||
/**
|
||||
* Shared constants used across explore, synthesize, and pipeline modules.
|
||||
*/
|
||||
|
||||
/** URL query params that are volatile/ephemeral and should be stripped from patterns */
|
||||
export const VOLATILE_PARAMS = new Set([
|
||||
'w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign',
|
||||
]);
|
||||
|
||||
/** Search-related query parameter names */
|
||||
export const SEARCH_PARAMS = new Set([
|
||||
'q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w',
|
||||
]);
|
||||
|
||||
/** Pagination-related query parameter names */
|
||||
export const PAGINATION_PARAMS = new Set([
|
||||
'page', 'pn', 'offset', 'cursor', 'next', 'page_num',
|
||||
]);
|
||||
|
||||
/** Limit/page-size query parameter names */
|
||||
export const LIMIT_PARAMS = new Set([
|
||||
'limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num',
|
||||
]);
|
||||
|
||||
/** Field role → common API field names mapping */
|
||||
export const FIELD_ROLES: Record<string, string[]> = {
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
};
|
||||
@@ -0,0 +1,78 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
canonicalizeProductUrl,
|
||||
dedupeSearchItems,
|
||||
normalizeProductId,
|
||||
normalizeSearchItem,
|
||||
sanitizeSearchItems,
|
||||
} from './coupang.js';
|
||||
|
||||
describe('normalizeProductId', () => {
|
||||
it('extracts product id from canonical path', () => {
|
||||
expect(normalizeProductId('https://www.coupang.com/vp/products/123456789')).toBe('123456789');
|
||||
});
|
||||
|
||||
it('preserves numeric ids', () => {
|
||||
expect(normalizeProductId('987654321')).toBe('987654321');
|
||||
});
|
||||
});
|
||||
|
||||
describe('canonicalizeProductUrl', () => {
|
||||
it('normalizes relative Coupang paths', () => {
|
||||
expect(canonicalizeProductUrl('/vp/products/123456789?itemId=1', '')).toBe(
|
||||
'https://www.coupang.com/vp/products/123456789'
|
||||
);
|
||||
});
|
||||
|
||||
it('builds url from product id', () => {
|
||||
expect(canonicalizeProductUrl('', '123456789')).toBe('https://www.coupang.com/vp/products/123456789');
|
||||
});
|
||||
});
|
||||
|
||||
describe('normalizeSearchItem', () => {
|
||||
it('maps raw fields into compare-ready shape', () => {
|
||||
const item = normalizeSearchItem({
|
||||
productId: '123456789',
|
||||
productName: '무선 마우스',
|
||||
salePrice: '29,900원',
|
||||
originalPrice: '39,900원',
|
||||
rating: '4.8',
|
||||
reviewCount: '1,234',
|
||||
sellerName: '쿠팡',
|
||||
badge: ['ROCKET', 'TOMORROW', '무료배송'],
|
||||
categoryName: 'PC',
|
||||
url: '/vp/products/123456789?itemId=1',
|
||||
}, 0);
|
||||
|
||||
expect(item).toMatchObject({
|
||||
rank: 1,
|
||||
product_id: '123456789',
|
||||
title: '무선 마우스',
|
||||
price: 29900,
|
||||
original_price: 39900,
|
||||
rating: 4.8,
|
||||
review_count: 1234,
|
||||
rocket: '로켓배송',
|
||||
delivery_type: '무료배송',
|
||||
delivery_promise: '내일도착',
|
||||
seller: '쿠팡',
|
||||
category: 'PC',
|
||||
url: 'https://www.coupang.com/vp/products/123456789',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('sanitizeSearchItems', () => {
|
||||
it('drops duplicates and invalid rows', () => {
|
||||
const rows = [
|
||||
normalizeSearchItem({ productId: '1', productName: 'A', price: '1000', url: '/vp/products/1' }, 0),
|
||||
normalizeSearchItem({ productId: '1', productName: 'A', price: '1000', url: '/vp/products/1' }, 1),
|
||||
normalizeSearchItem({ productId: '', productName: '', price: '1000' }, 2),
|
||||
normalizeSearchItem({ productId: '2', productName: 'B', price: '2000', url: '/vp/products/2' }, 3),
|
||||
];
|
||||
|
||||
expect(dedupeSearchItems(rows)).toHaveLength(3);
|
||||
expect(sanitizeSearchItems(rows, 10)).toHaveLength(2);
|
||||
expect(sanitizeSearchItems(rows, 10).map(item => item.rank)).toEqual([1, 2]);
|
||||
});
|
||||
});
|
||||
+302
@@ -0,0 +1,302 @@
|
||||
export interface CoupangSearchItem {
|
||||
rank: number;
|
||||
product_id: string;
|
||||
title: string;
|
||||
price: number | null;
|
||||
original_price: number | null;
|
||||
unit_price: string;
|
||||
discount_rate: number | null;
|
||||
rating: number | null;
|
||||
review_count: number | null;
|
||||
rocket: string;
|
||||
delivery_type: string;
|
||||
delivery_promise: string;
|
||||
seller: string;
|
||||
badge: string;
|
||||
category: string;
|
||||
url: string;
|
||||
}
|
||||
|
||||
function itemKey(item: CoupangSearchItem): string {
|
||||
return item.url || item.product_id || `${item.title}:${item.price ?? ''}`;
|
||||
}
|
||||
|
||||
const ROCKET_PATTERNS = ['판매자로켓', '로켓프레시', '로켓와우', '로켓배송', '로켓직구'] as const;
|
||||
const DELIVERY_TYPE_PATTERNS = ['무료배송', '일반배송'] as const;
|
||||
const DELIVERY_PROMISE_PATTERNS = ['오늘도착', '내일도착', '새벽도착', '오늘출발'] as const;
|
||||
|
||||
const BADGE_ID_TO_ROCKET: Record<string, string> = {
|
||||
ROCKET: '로켓배송',
|
||||
ROCKET_MERCHANT: '판매자로켓',
|
||||
ROCKET_WOW: '로켓와우',
|
||||
WOW: '로켓와우',
|
||||
ROCKET_FRESH: '로켓프레시',
|
||||
FRESH: '로켓프레시',
|
||||
SELLER_ROCKET: '판매자로켓',
|
||||
ROCKET_JIKGU: '로켓직구',
|
||||
JIKGU: '로켓직구',
|
||||
COUPANG_GLOBAL: '로켓직구',
|
||||
};
|
||||
|
||||
const BADGE_ID_TO_PROMISE: Record<string, string> = {
|
||||
DAWN: '새벽도착',
|
||||
EARLY_DAWN: '새벽도착',
|
||||
TOMORROW: '내일도착',
|
||||
TODAY: '오늘도착',
|
||||
SAME_DAY: '오늘도착',
|
||||
TODAY_SHIP: '오늘출발',
|
||||
TODAY_DISPATCH: '오늘출발',
|
||||
};
|
||||
|
||||
function asString(value: unknown): string {
|
||||
if (value == null) return '';
|
||||
return String(value).trim();
|
||||
}
|
||||
|
||||
function toNumber(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) return value;
|
||||
const text = asString(value).replace(/[^\d.]/g, '');
|
||||
if (!text) return null;
|
||||
const num = Number(text);
|
||||
return Number.isFinite(num) ? num : null;
|
||||
}
|
||||
|
||||
function pickFirst(obj: Record<string, unknown>, paths: string[]): unknown {
|
||||
for (const path of paths) {
|
||||
const parts = path.split('.');
|
||||
let current: unknown = obj;
|
||||
let ok = true;
|
||||
for (const part of parts) {
|
||||
if (!current || typeof current !== 'object' || !(part in (current as Record<string, unknown>))) {
|
||||
ok = false;
|
||||
break;
|
||||
}
|
||||
current = (current as Record<string, unknown>)[part];
|
||||
}
|
||||
if (ok && current != null && asString(current) !== '') return current;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function normalizeProductId(raw: unknown): string {
|
||||
const text = asString(raw);
|
||||
if (!text) return '';
|
||||
const match = text.match(/\/vp\/products\/(\d+)/) || text.match(/\b(\d{6,})\b/);
|
||||
return match?.[1] ?? text;
|
||||
}
|
||||
|
||||
export function canonicalizeProductUrl(rawUrl: unknown, productId?: unknown): string {
|
||||
const raw = asString(rawUrl);
|
||||
if (raw) {
|
||||
try {
|
||||
const url = new URL(raw.startsWith('http') ? raw : `https://www.coupang.com${raw}`);
|
||||
if (!url.hostname.includes('coupang.com')) return '';
|
||||
const id = normalizeProductId(url.pathname) || normalizeProductId(productId);
|
||||
if (!id) return url.toString();
|
||||
return `https://www.coupang.com/vp/products/${id}`;
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
const id = normalizeProductId(productId);
|
||||
return id ? `https://www.coupang.com/vp/products/${id}` : '';
|
||||
}
|
||||
|
||||
function extractTokens(values: unknown[]): string[] {
|
||||
return values
|
||||
.flatMap((value) => {
|
||||
const text = asString(value);
|
||||
if (!text) return [];
|
||||
return text.split(/[,\s|]+/);
|
||||
})
|
||||
.map((token) => token.trim().toUpperCase())
|
||||
.filter(Boolean);
|
||||
}
|
||||
|
||||
function normalizeJoinedText(...values: unknown[]): string {
|
||||
return values
|
||||
.map(asString)
|
||||
.filter(Boolean)
|
||||
.join(' ')
|
||||
.replace(/schema\.org\/[A-Za-z]+/gi, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function normalizeRocket(...values: unknown[]): string {
|
||||
const tokens = extractTokens(values);
|
||||
for (const token of tokens) {
|
||||
if (BADGE_ID_TO_ROCKET[token]) return BADGE_ID_TO_ROCKET[token];
|
||||
}
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/판매자\s*로켓/.test(text)) return '판매자로켓';
|
||||
if (/로켓\s*프레시|새벽\s*도착\s*보장/.test(text)) return '로켓프레시';
|
||||
if (/로켓\s*와우/.test(text)) return '로켓와우';
|
||||
if (/로켓\s*직구|직구/.test(text)) return '로켓직구';
|
||||
if (/로켓\s*배송/.test(text)) return '로켓배송';
|
||||
return ROCKET_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeDeliveryType(...values: unknown[]): string {
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/무료\s*배송/.test(text)) return '무료배송';
|
||||
if (/일반\s*배송/.test(text)) return '일반배송';
|
||||
return DELIVERY_TYPE_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeDeliveryPromise(...values: unknown[]): string {
|
||||
const tokens = extractTokens(values);
|
||||
for (const token of tokens) {
|
||||
if (BADGE_ID_TO_PROMISE[token]) return BADGE_ID_TO_PROMISE[token];
|
||||
}
|
||||
const text = normalizeJoinedText(...values);
|
||||
if (!text) return '';
|
||||
if (/오늘\s*출발/.test(text)) return '오늘출발';
|
||||
if (/오늘.*도착/.test(text)) return '오늘도착';
|
||||
if (/새벽.*도착/.test(text)) return '새벽도착';
|
||||
if (/내일.*도착/.test(text)) return '내일도착';
|
||||
return DELIVERY_PROMISE_PATTERNS.find(pattern => text.includes(pattern)) ?? '';
|
||||
}
|
||||
|
||||
function normalizeBadge(value: unknown): string {
|
||||
const normalizeOne = (entry: unknown): string => {
|
||||
const text = asString(entry);
|
||||
if (!text) return '';
|
||||
if (/schema\.org\//i.test(text)) {
|
||||
return text.split('/').pop() ?? '';
|
||||
}
|
||||
return text;
|
||||
};
|
||||
if (Array.isArray(value)) {
|
||||
return value.map(normalizeOne).filter(Boolean).join(', ');
|
||||
}
|
||||
return normalizeOne(value);
|
||||
}
|
||||
|
||||
export function normalizeSearchItem(raw: Record<string, unknown>, index: number): CoupangSearchItem {
|
||||
const productId = normalizeProductId(
|
||||
pickFirst(raw, ['productId', 'product_id', 'id', 'productNo', 'item.id', 'product.productId', 'url'])
|
||||
);
|
||||
const title = asString(
|
||||
pickFirst(raw, ['title', 'name', 'productName', 'productTitle', 'itemName', 'item.title'])
|
||||
);
|
||||
const price = toNumber(
|
||||
pickFirst(raw, ['price', 'salePrice', 'finalPrice', 'sellingPrice', 'discountPrice', 'item.price'])
|
||||
);
|
||||
const originalPrice = toNumber(
|
||||
pickFirst(raw, ['originalPrice', 'basePrice', 'listPrice', 'originPrice', 'strikePrice'])
|
||||
);
|
||||
const unitPrice = asString(
|
||||
pickFirst(raw, ['unitPrice', 'unit_price', 'unitPriceText'])
|
||||
);
|
||||
const rating = toNumber(
|
||||
pickFirst(raw, ['rating', 'star', 'reviewRating', 'review.rating', 'item.rating'])
|
||||
);
|
||||
const reviewCount = toNumber(
|
||||
pickFirst(raw, ['reviewCount', 'ratingCount', 'reviews', 'reviewCnt', 'item.reviewCount'])
|
||||
);
|
||||
const deliveryHintValues = [
|
||||
pickFirst(raw, ['deliveryType', 'deliveryBadge', 'badgeLabel', 'shippingType', 'shippingBadge']),
|
||||
pickFirst(raw, ['badge', 'badges', 'labels', 'benefitBadge', 'promotionBadge']),
|
||||
pickFirst(raw, ['text', 'summary']),
|
||||
pickFirst(raw, ['deliveryPromise', 'promise', 'arrivalText', 'arrivalBadge']),
|
||||
pickFirst(raw, ['rocket', 'rocketType']),
|
||||
];
|
||||
const deliveryType = normalizeDeliveryType(...deliveryHintValues);
|
||||
const deliveryPromise = normalizeDeliveryPromise(...deliveryHintValues);
|
||||
const rocket = normalizeRocket(...deliveryHintValues);
|
||||
const badge = normalizeBadge(
|
||||
pickFirst(raw, ['badge', 'badges', 'labels', 'benefitBadge', 'promotionBadge'])
|
||||
);
|
||||
const category = asString(
|
||||
pickFirst(raw, ['category', 'categoryName', 'categoryPath', 'item.category'])
|
||||
);
|
||||
const seller = asString(
|
||||
pickFirst(raw, ['seller', 'sellerName', 'vendorName', 'merchantName', 'item.seller'])
|
||||
);
|
||||
const url = canonicalizeProductUrl(
|
||||
pickFirst(raw, ['url', 'productUrl', 'link', 'item.url']),
|
||||
productId
|
||||
);
|
||||
const discountRate = toNumber(
|
||||
pickFirst(raw, ['discountRate', 'discount', 'discountPercent', 'discount_rate'])
|
||||
);
|
||||
|
||||
return {
|
||||
rank: index + 1,
|
||||
product_id: productId,
|
||||
title,
|
||||
price,
|
||||
original_price: originalPrice,
|
||||
unit_price: unitPrice,
|
||||
discount_rate: discountRate,
|
||||
rating,
|
||||
review_count: reviewCount,
|
||||
rocket,
|
||||
delivery_type: deliveryType,
|
||||
delivery_promise: deliveryPromise,
|
||||
seller,
|
||||
badge,
|
||||
category,
|
||||
url,
|
||||
};
|
||||
}
|
||||
|
||||
export function dedupeSearchItems(items: CoupangSearchItem[]): CoupangSearchItem[] {
|
||||
const seen = new Set<string>();
|
||||
const out: CoupangSearchItem[] = [];
|
||||
for (const item of items) {
|
||||
const key = itemKey(item);
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
out.push({ ...item, rank: out.length + 1 });
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
export function sanitizeSearchItems(items: CoupangSearchItem[], limit: number): CoupangSearchItem[] {
|
||||
return dedupeSearchItems(
|
||||
items.filter(item => Boolean(item.title && (item.product_id || item.url)))
|
||||
).slice(0, limit);
|
||||
}
|
||||
|
||||
export function mergeSearchItems(base: CoupangSearchItem[], extra: CoupangSearchItem[], limit: number): CoupangSearchItem[] {
|
||||
const extraMap = new Map<string, CoupangSearchItem>();
|
||||
for (const item of extra) {
|
||||
const key = itemKey(item);
|
||||
if (key) extraMap.set(key, item);
|
||||
}
|
||||
|
||||
const merged = base.map((item) => {
|
||||
const key = itemKey(item);
|
||||
const patch = key ? extraMap.get(key) : null;
|
||||
if (!patch) return item;
|
||||
return {
|
||||
...item,
|
||||
price: patch.price ?? item.price,
|
||||
original_price: patch.original_price ?? item.original_price,
|
||||
unit_price: patch.unit_price || item.unit_price,
|
||||
discount_rate: patch.discount_rate ?? item.discount_rate,
|
||||
rating: patch.rating ?? item.rating,
|
||||
review_count: patch.review_count ?? item.review_count,
|
||||
rocket: patch.rocket || item.rocket,
|
||||
delivery_type: patch.delivery_type || item.delivery_type,
|
||||
delivery_promise: patch.delivery_promise || item.delivery_promise,
|
||||
seller: patch.seller || item.seller,
|
||||
badge: patch.badge || item.badge,
|
||||
category: patch.category || item.category,
|
||||
url: patch.url || item.url,
|
||||
};
|
||||
});
|
||||
|
||||
const mergedKeys = new Set(merged.map(item => itemKey(item)).filter(Boolean));
|
||||
const appended = extra.filter(item => {
|
||||
const key = itemKey(item);
|
||||
return key && !mergedKeys.has(key);
|
||||
});
|
||||
|
||||
return sanitizeSearchItems([...merged, ...appended], limit);
|
||||
}
|
||||
@@ -0,0 +1,125 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import {
|
||||
readTokenFromShellContent,
|
||||
renderBrowserDoctorReport,
|
||||
upsertShellToken,
|
||||
readTomlConfigToken,
|
||||
upsertTomlConfigToken,
|
||||
upsertJsonConfigToken,
|
||||
} from './doctor.js';
|
||||
|
||||
describe('shell token helpers', () => {
|
||||
it('reads token from shell export', () => {
|
||||
expect(readTokenFromShellContent('export PLAYWRIGHT_MCP_EXTENSION_TOKEN="abc123"\n')).toBe('abc123');
|
||||
});
|
||||
|
||||
it('appends token export when missing', () => {
|
||||
const next = upsertShellToken('export PATH="/usr/bin"\n', 'abc123');
|
||||
expect(next).toContain('export PLAYWRIGHT_MCP_EXTENSION_TOKEN="abc123"');
|
||||
});
|
||||
|
||||
it('replaces token export when present', () => {
|
||||
const next = upsertShellToken('export PLAYWRIGHT_MCP_EXTENSION_TOKEN="old"\n', 'new');
|
||||
expect(next).toContain('export PLAYWRIGHT_MCP_EXTENSION_TOKEN="new"');
|
||||
expect(next).not.toContain('"old"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('toml token helpers', () => {
|
||||
it('reads token from playwright env section', () => {
|
||||
const content = `
|
||||
[mcp_servers.playwright.env]
|
||||
PLAYWRIGHT_MCP_EXTENSION_TOKEN = "abc123"
|
||||
`;
|
||||
expect(readTomlConfigToken(content)).toBe('abc123');
|
||||
});
|
||||
|
||||
it('updates token inside existing env section', () => {
|
||||
const content = `
|
||||
[mcp_servers.playwright.env]
|
||||
PLAYWRIGHT_MCP_EXTENSION_TOKEN = "old"
|
||||
`;
|
||||
const next = upsertTomlConfigToken(content, 'new');
|
||||
expect(next).toContain('PLAYWRIGHT_MCP_EXTENSION_TOKEN = "new"');
|
||||
expect(next).not.toContain('"old"');
|
||||
});
|
||||
|
||||
it('creates env section when missing', () => {
|
||||
const content = `
|
||||
[mcp_servers.playwright]
|
||||
type = "stdio"
|
||||
`;
|
||||
const next = upsertTomlConfigToken(content, 'abc123');
|
||||
expect(next).toContain('[mcp_servers.playwright.env]');
|
||||
expect(next).toContain('PLAYWRIGHT_MCP_EXTENSION_TOKEN = "abc123"');
|
||||
});
|
||||
});
|
||||
|
||||
describe('json token helpers', () => {
|
||||
it('writes token into standard mcpServers config', () => {
|
||||
const next = upsertJsonConfigToken(JSON.stringify({
|
||||
mcpServers: {
|
||||
playwright: {
|
||||
command: 'npx',
|
||||
args: ['-y', '@playwright/mcp@latest', '--extension'],
|
||||
},
|
||||
},
|
||||
}), 'abc123');
|
||||
const parsed = JSON.parse(next);
|
||||
expect(parsed.mcpServers.playwright.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN).toBe('abc123');
|
||||
});
|
||||
|
||||
it('writes token into opencode mcp config', () => {
|
||||
const next = upsertJsonConfigToken(JSON.stringify({
|
||||
$schema: 'https://opencode.ai/config.json',
|
||||
mcp: {
|
||||
playwright: {
|
||||
command: ['npx', '-y', '@playwright/mcp@latest', '--extension'],
|
||||
enabled: true,
|
||||
type: 'local',
|
||||
},
|
||||
},
|
||||
}), 'abc123');
|
||||
const parsed = JSON.parse(next);
|
||||
expect(parsed.mcp.playwright.env.PLAYWRIGHT_MCP_EXTENSION_TOKEN).toBe('abc123');
|
||||
});
|
||||
});
|
||||
|
||||
describe('doctor report rendering', () => {
|
||||
it('renders OK-style report when tokens match', () => {
|
||||
const text = renderBrowserDoctorReport({
|
||||
envToken: 'abc123',
|
||||
envFingerprint: 'fp1',
|
||||
extensionToken: 'abc123',
|
||||
extensionFingerprint: 'fp1',
|
||||
shellFiles: [{ path: '/tmp/.zshrc', exists: true, token: 'abc123', fingerprint: 'fp1' }],
|
||||
configs: [{ path: '/tmp/mcp.json', exists: true, format: 'json', token: 'abc123', fingerprint: 'fp1', writable: true }],
|
||||
recommendedToken: 'abc123',
|
||||
recommendedFingerprint: 'fp1',
|
||||
warnings: [],
|
||||
issues: [],
|
||||
});
|
||||
|
||||
expect(text).toContain('[OK] Environment token: configured (fp1)');
|
||||
expect(text).toContain('[OK] MCP config /tmp/mcp.json: configured (fp1)');
|
||||
});
|
||||
|
||||
it('renders MISMATCH-style report when fingerprints differ', () => {
|
||||
const text = renderBrowserDoctorReport({
|
||||
envToken: 'abc123',
|
||||
envFingerprint: 'fp1',
|
||||
extensionToken: null,
|
||||
extensionFingerprint: null,
|
||||
shellFiles: [{ path: '/tmp/.zshrc', exists: true, token: 'def456', fingerprint: 'fp2' }],
|
||||
configs: [{ path: '/tmp/mcp.json', exists: true, format: 'json', token: 'abc123', fingerprint: 'fp1', writable: true }],
|
||||
recommendedToken: 'abc123',
|
||||
recommendedFingerprint: 'fp1',
|
||||
warnings: [],
|
||||
issues: ['Detected inconsistent Playwright MCP tokens across env/config files.'],
|
||||
});
|
||||
|
||||
expect(text).toContain('[MISMATCH] Environment token: configured (fp1)');
|
||||
expect(text).toContain('[MISMATCH] Shell file /tmp/.zshrc: configured (fp2)');
|
||||
expect(text).toContain('[MISMATCH] Recommended token fingerprint: fp1');
|
||||
});
|
||||
});
|
||||
+525
@@ -0,0 +1,525 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { execSync } from 'node:child_process';
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import type { IPage } from './types.js';
|
||||
import { PlaywrightMCP, getTokenFingerprint } from './browser.js';
|
||||
import { browserSession } from './runtime.js';
|
||||
|
||||
const PLAYWRIGHT_SERVER_NAME = 'playwright';
|
||||
const PLAYWRIGHT_TOKEN_ENV = 'PLAYWRIGHT_MCP_EXTENSION_TOKEN';
|
||||
const PLAYWRIGHT_EXTENSION_ID = 'mmlmfjhmonkocbjadbfplnigmagldckm';
|
||||
const TOKEN_LINE_RE = /^(\s*export\s+PLAYWRIGHT_MCP_EXTENSION_TOKEN=)(['"]?)([^'"\\\n]+)\2\s*$/m;
|
||||
export type DoctorOptions = {
|
||||
fix?: boolean;
|
||||
yes?: boolean;
|
||||
shellRc?: string;
|
||||
configPaths?: string[];
|
||||
token?: string;
|
||||
cliVersion?: string;
|
||||
};
|
||||
|
||||
export type ShellFileStatus = {
|
||||
path: string;
|
||||
exists: boolean;
|
||||
token: string | null;
|
||||
fingerprint: string | null;
|
||||
};
|
||||
|
||||
export type McpConfigFormat = 'json' | 'toml';
|
||||
|
||||
export type McpConfigStatus = {
|
||||
path: string;
|
||||
exists: boolean;
|
||||
format: McpConfigFormat;
|
||||
token: string | null;
|
||||
fingerprint: string | null;
|
||||
writable: boolean;
|
||||
parseError?: string;
|
||||
};
|
||||
|
||||
export type DoctorReport = {
|
||||
cliVersion?: string;
|
||||
envToken: string | null;
|
||||
envFingerprint: string | null;
|
||||
extensionToken: string | null;
|
||||
extensionFingerprint: string | null;
|
||||
shellFiles: ShellFileStatus[];
|
||||
configs: McpConfigStatus[];
|
||||
recommendedToken: string | null;
|
||||
recommendedFingerprint: string | null;
|
||||
warnings: string[];
|
||||
issues: string[];
|
||||
};
|
||||
|
||||
type ReportStatus = 'OK' | 'MISSING' | 'MISMATCH' | 'WARN';
|
||||
|
||||
function label(status: ReportStatus): string {
|
||||
return `[${status}]`;
|
||||
}
|
||||
|
||||
function statusLine(status: ReportStatus, text: string): string {
|
||||
return `${label(status)} ${text}`;
|
||||
}
|
||||
|
||||
function tokenSummary(token: string | null, fingerprint: string | null): string {
|
||||
if (!token) return 'missing';
|
||||
return `configured (${fingerprint})`;
|
||||
}
|
||||
|
||||
export function getDefaultShellRcPath(): string {
|
||||
const shell = process.env.SHELL ?? '';
|
||||
if (shell.endsWith('/bash')) return path.join(os.homedir(), '.bashrc');
|
||||
if (shell.endsWith('/fish')) return path.join(os.homedir(), '.config', 'fish', 'config.fish');
|
||||
return path.join(os.homedir(), '.zshrc');
|
||||
}
|
||||
|
||||
export function getDefaultMcpConfigPaths(cwd: string = process.cwd()): string[] {
|
||||
const home = os.homedir();
|
||||
const candidates = [
|
||||
path.join(home, '.codex', 'config.toml'),
|
||||
path.join(home, '.codex', 'mcp.json'),
|
||||
path.join(home, '.cursor', 'mcp.json'),
|
||||
path.join(home, '.claude.json'),
|
||||
path.join(home, '.gemini', 'settings.json'),
|
||||
path.join(home, '.gemini', 'antigravity', 'mcp_config.json'),
|
||||
path.join(home, '.config', 'opencode', 'opencode.json'),
|
||||
path.join(home, 'Library', 'Application Support', 'Claude', 'claude_desktop_config.json'),
|
||||
path.join(home, '.config', 'Claude', 'claude_desktop_config.json'),
|
||||
path.join(cwd, '.cursor', 'mcp.json'),
|
||||
path.join(cwd, '.vscode', 'mcp.json'),
|
||||
path.join(cwd, '.opencode', 'opencode.json'),
|
||||
path.join(cwd, '.mcp.json'),
|
||||
];
|
||||
return [...new Set(candidates)];
|
||||
}
|
||||
|
||||
export function readTokenFromShellContent(content: string): string | null {
|
||||
const m = content.match(TOKEN_LINE_RE);
|
||||
return m?.[3] ?? null;
|
||||
}
|
||||
|
||||
export function upsertShellToken(content: string, token: string): string {
|
||||
const nextLine = `export ${PLAYWRIGHT_TOKEN_ENV}="${token}"`;
|
||||
if (!content.trim()) return `${nextLine}\n`;
|
||||
if (TOKEN_LINE_RE.test(content)) return content.replace(TOKEN_LINE_RE, `$1"${
|
||||
token
|
||||
}"`);
|
||||
return `${content.replace(/\s*$/, '')}\n${nextLine}\n`;
|
||||
}
|
||||
|
||||
function readJsonConfigToken(content: string): string | null {
|
||||
try {
|
||||
const parsed = JSON.parse(content);
|
||||
return readTokenFromJsonObject(parsed);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function readTokenFromJsonObject(parsed: any): string | null {
|
||||
const direct = parsed?.mcpServers?.[PLAYWRIGHT_SERVER_NAME]?.env?.[PLAYWRIGHT_TOKEN_ENV];
|
||||
if (typeof direct === 'string' && direct) return direct;
|
||||
const opencode = parsed?.mcp?.[PLAYWRIGHT_SERVER_NAME]?.env?.[PLAYWRIGHT_TOKEN_ENV];
|
||||
if (typeof opencode === 'string' && opencode) return opencode;
|
||||
return null;
|
||||
}
|
||||
|
||||
export function upsertJsonConfigToken(content: string, token: string): string {
|
||||
const parsed = content.trim() ? JSON.parse(content) : {};
|
||||
if (parsed?.mcpServers) {
|
||||
parsed.mcpServers[PLAYWRIGHT_SERVER_NAME] = parsed.mcpServers[PLAYWRIGHT_SERVER_NAME] ?? {
|
||||
command: 'npx',
|
||||
args: ['-y', '@playwright/mcp@latest', '--extension'],
|
||||
};
|
||||
parsed.mcpServers[PLAYWRIGHT_SERVER_NAME].env = parsed.mcpServers[PLAYWRIGHT_SERVER_NAME].env ?? {};
|
||||
parsed.mcpServers[PLAYWRIGHT_SERVER_NAME].env[PLAYWRIGHT_TOKEN_ENV] = token;
|
||||
} else {
|
||||
parsed.mcp = parsed.mcp ?? {};
|
||||
parsed.mcp[PLAYWRIGHT_SERVER_NAME] = parsed.mcp[PLAYWRIGHT_SERVER_NAME] ?? {
|
||||
command: ['npx', '-y', '@playwright/mcp@latest', '--extension'],
|
||||
enabled: true,
|
||||
type: 'local',
|
||||
};
|
||||
parsed.mcp[PLAYWRIGHT_SERVER_NAME].env = parsed.mcp[PLAYWRIGHT_SERVER_NAME].env ?? {};
|
||||
parsed.mcp[PLAYWRIGHT_SERVER_NAME].env[PLAYWRIGHT_TOKEN_ENV] = token;
|
||||
}
|
||||
return `${JSON.stringify(parsed, null, 2)}\n`;
|
||||
}
|
||||
|
||||
export function readTomlConfigToken(content: string): string | null {
|
||||
const sectionMatch = content.match(/\[mcp_servers\.playwright\.env\][\s\S]*?(?=\n\[|$)/);
|
||||
if (!sectionMatch) return null;
|
||||
const tokenMatch = sectionMatch[0].match(/^\s*PLAYWRIGHT_MCP_EXTENSION_TOKEN\s*=\s*"([^"\n]+)"/m);
|
||||
return tokenMatch?.[1] ?? null;
|
||||
}
|
||||
|
||||
export function upsertTomlConfigToken(content: string, token: string): string {
|
||||
const envSectionRe = /(\[mcp_servers\.playwright\.env\][\s\S]*?)(?=\n\[|$)/;
|
||||
const tokenLine = `PLAYWRIGHT_MCP_EXTENSION_TOKEN = "${token}"`;
|
||||
if (envSectionRe.test(content)) {
|
||||
return content.replace(envSectionRe, (section) => {
|
||||
if (/^\s*PLAYWRIGHT_MCP_EXTENSION_TOKEN\s*=/m.test(section)) {
|
||||
return section.replace(/^\s*PLAYWRIGHT_MCP_EXTENSION_TOKEN\s*=.*$/m, tokenLine);
|
||||
}
|
||||
return `${section.replace(/\s*$/, '')}\n${tokenLine}\n`;
|
||||
});
|
||||
}
|
||||
|
||||
const baseSectionRe = /(\[mcp_servers\.playwright\][\s\S]*?)(?=\n\[|$)/;
|
||||
if (baseSectionRe.test(content)) {
|
||||
return content.replace(baseSectionRe, (section) => `${section.replace(/\s*$/, '')}\n\n[mcp_servers.playwright.env]\n${tokenLine}\n`);
|
||||
}
|
||||
|
||||
const prefix = content.trim() ? `${content.replace(/\s*$/, '')}\n\n` : '';
|
||||
return `${prefix}[mcp_servers.playwright]\ntype = "stdio"\ncommand = "npx"\nargs = ["-y", "@playwright/mcp@latest", "--extension"]\n\n[mcp_servers.playwright.env]\n${tokenLine}\n`;
|
||||
}
|
||||
|
||||
function fileExists(filePath: string): boolean {
|
||||
try {
|
||||
return fs.existsSync(filePath);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function canWrite(filePath: string): boolean {
|
||||
try {
|
||||
if (fileExists(filePath)) {
|
||||
fs.accessSync(filePath, fs.constants.W_OK);
|
||||
return true;
|
||||
}
|
||||
fs.accessSync(path.dirname(filePath), fs.constants.W_OK);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function readConfigStatus(filePath: string): McpConfigStatus {
|
||||
const format: McpConfigFormat = filePath.endsWith('.toml') ? 'toml' : 'json';
|
||||
if (!fileExists(filePath)) {
|
||||
return { path: filePath, exists: false, format, token: null, fingerprint: null, writable: canWrite(filePath) };
|
||||
}
|
||||
try {
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const token = format === 'toml' ? readTomlConfigToken(content) : readJsonConfigToken(content);
|
||||
return {
|
||||
path: filePath,
|
||||
exists: true,
|
||||
format,
|
||||
token,
|
||||
fingerprint: getTokenFingerprint(token ?? undefined),
|
||||
writable: canWrite(filePath),
|
||||
};
|
||||
} catch (error: any) {
|
||||
return {
|
||||
path: filePath,
|
||||
exists: true,
|
||||
format,
|
||||
token: null,
|
||||
fingerprint: null,
|
||||
writable: canWrite(filePath),
|
||||
parseError: error?.message ?? String(error),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Discover the auth token stored by the Playwright MCP Bridge extension
|
||||
* by scanning Chrome's LevelDB localStorage files directly.
|
||||
*
|
||||
* Uses `strings` + `grep` for fast binary scanning on macOS/Linux,
|
||||
* with a pure-Node fallback on Windows.
|
||||
*/
|
||||
export function discoverExtensionToken(): string | null {
|
||||
const home = os.homedir();
|
||||
const platform = os.platform();
|
||||
const bases: string[] = [];
|
||||
|
||||
if (platform === 'darwin') {
|
||||
bases.push(
|
||||
path.join(home, 'Library', 'Application Support', 'Google', 'Chrome'),
|
||||
path.join(home, 'Library', 'Application Support', 'Google', 'Chrome Canary'),
|
||||
path.join(home, 'Library', 'Application Support', 'Chromium'),
|
||||
path.join(home, 'Library', 'Application Support', 'Microsoft Edge'),
|
||||
);
|
||||
} else if (platform === 'linux') {
|
||||
bases.push(
|
||||
path.join(home, '.config', 'google-chrome'),
|
||||
path.join(home, '.config', 'chromium'),
|
||||
path.join(home, '.config', 'microsoft-edge'),
|
||||
);
|
||||
} else if (platform === 'win32') {
|
||||
const appData = process.env.LOCALAPPDATA || path.join(home, 'AppData', 'Local');
|
||||
bases.push(
|
||||
path.join(appData, 'Google', 'Chrome', 'User Data'),
|
||||
path.join(appData, 'Microsoft', 'Edge', 'User Data'),
|
||||
);
|
||||
}
|
||||
|
||||
const profiles = ['Default', 'Profile 1', 'Profile 2', 'Profile 3'];
|
||||
// Token is 43 chars of base64url (from 32 random bytes)
|
||||
const tokenRe = /([A-Za-z0-9_-]{40,50})/;
|
||||
|
||||
for (const base of bases) {
|
||||
for (const profile of profiles) {
|
||||
const dir = path.join(base, profile, 'Local Storage', 'leveldb');
|
||||
if (!fileExists(dir)) continue;
|
||||
|
||||
// Fast path: use strings + grep to find candidate files and extract token
|
||||
if (platform !== 'win32') {
|
||||
const token = extractTokenViaStrings(dir, tokenRe);
|
||||
if (token) return token;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Slow path (Windows): read binary files directly
|
||||
const token = extractTokenViaBinaryRead(dir, tokenRe);
|
||||
if (token) return token;
|
||||
}
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractTokenViaStrings(dir: string, tokenRe: RegExp): string | null {
|
||||
try {
|
||||
// Single shell pipeline: for each LevelDB file, extract strings, find lines
|
||||
// after the extension ID, and filter for base64url token pattern.
|
||||
//
|
||||
// LevelDB `strings` output for the extension's auth-token entry:
|
||||
// auth-token ← key name
|
||||
// 4,mmlmfjhmonkocbjadbfplnigmagldckm.7 ← LevelDB internal key
|
||||
// hqI86ncsD1QpcVcj-k9CyzTF-ieCQd_4KreZ_wy1WHA ← token value
|
||||
//
|
||||
// We get the line immediately after any EXTENSION_ID mention and check
|
||||
// if it looks like a base64url token (40-50 chars, [A-Za-z0-9_-]).
|
||||
const shellDir = dir.replace(/'/g, "'\\''");
|
||||
const cmd = `for f in '${shellDir}'/*.ldb '${shellDir}'/*.log; do ` +
|
||||
`[ -f "$f" ] && strings "$f" 2>/dev/null | ` +
|
||||
`grep -A1 '${PLAYWRIGHT_EXTENSION_ID}' | ` +
|
||||
`grep -v '${PLAYWRIGHT_EXTENSION_ID}' | ` +
|
||||
`grep -E '^[A-Za-z0-9_-]{40,50}$' | head -1; ` +
|
||||
`done 2>/dev/null`;
|
||||
const result = execSync(cmd, { encoding: 'utf-8', timeout: 10000 }).trim();
|
||||
|
||||
// Take the first non-empty line
|
||||
for (const line of result.split('\n')) {
|
||||
const token = line.trim();
|
||||
if (token && validateBase64urlToken(token)) return token;
|
||||
}
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractTokenViaBinaryRead(dir: string, tokenRe: RegExp): string | null {
|
||||
const extIdBuf = Buffer.from(PLAYWRIGHT_EXTENSION_ID);
|
||||
const keyBuf = Buffer.from('auth-token');
|
||||
|
||||
let files: string[];
|
||||
try {
|
||||
files = fs.readdirSync(dir)
|
||||
.filter(f => f.endsWith('.ldb') || f.endsWith('.log'))
|
||||
.map(f => path.join(dir, f));
|
||||
} catch { return null; }
|
||||
|
||||
// Sort by mtime descending
|
||||
files.sort((a, b) => {
|
||||
try { return fs.statSync(b).mtimeMs - fs.statSync(a).mtimeMs; } catch { return 0; }
|
||||
});
|
||||
|
||||
for (const file of files) {
|
||||
let data: Buffer;
|
||||
try { data = fs.readFileSync(file); } catch { continue; }
|
||||
|
||||
// Quick check: does file contain both the extension ID and auth-token key?
|
||||
const extPos = data.indexOf(extIdBuf);
|
||||
if (extPos === -1) continue;
|
||||
const keyPos = data.indexOf(keyBuf, Math.max(0, extPos - 500));
|
||||
if (keyPos === -1) continue;
|
||||
|
||||
// Scan for token value after auth-token key
|
||||
let idx = 0;
|
||||
while (true) {
|
||||
const kp = data.indexOf(keyBuf, idx);
|
||||
if (kp === -1) break;
|
||||
|
||||
const contextStart = Math.max(0, kp - 500);
|
||||
if (data.indexOf(extIdBuf, contextStart) !== -1 && data.indexOf(extIdBuf, contextStart) < kp) {
|
||||
const after = data.subarray(kp + keyBuf.length, kp + keyBuf.length + 200).toString('latin1');
|
||||
const m = after.match(tokenRe);
|
||||
if (m && validateBase64urlToken(m[1])) return m[1];
|
||||
}
|
||||
idx = kp + 1;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function validateBase64urlToken(token: string): boolean {
|
||||
try {
|
||||
const b64 = token.replace(/-/g, '+').replace(/_/g, '/');
|
||||
const decoded = Buffer.from(b64, 'base64');
|
||||
return decoded.length >= 28 && decoded.length <= 36;
|
||||
} catch { return false; }
|
||||
}
|
||||
|
||||
|
||||
export async function runBrowserDoctor(opts: DoctorOptions = {}): Promise<DoctorReport> {
|
||||
const envToken = process.env[PLAYWRIGHT_TOKEN_ENV] ?? null;
|
||||
const shellPath = opts.shellRc ?? getDefaultShellRcPath();
|
||||
const shellFiles: ShellFileStatus[] = [shellPath].map((filePath) => {
|
||||
if (!fileExists(filePath)) return { path: filePath, exists: false, token: null, fingerprint: null };
|
||||
const content = fs.readFileSync(filePath, 'utf-8');
|
||||
const token = readTokenFromShellContent(content);
|
||||
return { path: filePath, exists: true, token, fingerprint: getTokenFingerprint(token ?? undefined) };
|
||||
});
|
||||
const configPaths = opts.configPaths?.length ? opts.configPaths : getDefaultMcpConfigPaths();
|
||||
const configs = configPaths.map(readConfigStatus);
|
||||
|
||||
// Try to discover the token directly from the Chrome extension's localStorage
|
||||
const extensionToken = discoverExtensionToken();
|
||||
|
||||
const allTokens = [
|
||||
opts.token ?? null,
|
||||
extensionToken,
|
||||
envToken,
|
||||
...shellFiles.map(s => s.token),
|
||||
...configs.map(c => c.token),
|
||||
].filter((v): v is string => !!v);
|
||||
const uniqueTokens = [...new Set(allTokens)];
|
||||
const recommendedToken = opts.token ?? extensionToken ?? envToken ?? (uniqueTokens.length === 1 ? uniqueTokens[0] : null) ?? null;
|
||||
|
||||
const report: DoctorReport = {
|
||||
cliVersion: opts.cliVersion,
|
||||
envToken,
|
||||
envFingerprint: getTokenFingerprint(envToken ?? undefined),
|
||||
extensionToken,
|
||||
extensionFingerprint: getTokenFingerprint(extensionToken ?? undefined),
|
||||
shellFiles,
|
||||
configs,
|
||||
recommendedToken,
|
||||
recommendedFingerprint: getTokenFingerprint(recommendedToken ?? undefined),
|
||||
warnings: [],
|
||||
issues: [],
|
||||
};
|
||||
|
||||
if (!envToken) report.issues.push(`Current environment is missing ${PLAYWRIGHT_TOKEN_ENV}.`);
|
||||
if (!shellFiles.some(s => s.token)) report.issues.push('Shell startup file does not export PLAYWRIGHT_MCP_EXTENSION_TOKEN.');
|
||||
if (!configs.some(c => c.token)) report.issues.push('No scanned MCP config currently contains a Playwright extension token.');
|
||||
if (uniqueTokens.length > 1) report.issues.push('Detected inconsistent Playwright MCP tokens across env/config files.');
|
||||
for (const config of configs) {
|
||||
if (config.parseError) report.warnings.push(`Could not parse ${config.path}: ${config.parseError}`);
|
||||
}
|
||||
if (!recommendedToken) {
|
||||
report.warnings.push('No token source found.');
|
||||
}
|
||||
return report;
|
||||
}
|
||||
|
||||
export function renderBrowserDoctorReport(report: DoctorReport): string {
|
||||
const tokenFingerprints = [
|
||||
report.extensionFingerprint,
|
||||
report.envFingerprint,
|
||||
...report.shellFiles.map(shell => shell.fingerprint),
|
||||
...report.configs.filter(config => config.exists).map(config => config.fingerprint),
|
||||
].filter((value): value is string => !!value);
|
||||
const uniqueFingerprints = [...new Set(tokenFingerprints)];
|
||||
const hasMismatch = uniqueFingerprints.length > 1;
|
||||
const lines = [`opencli v${report.cliVersion ?? 'unknown'} doctor`, ''];
|
||||
|
||||
const extStatus: ReportStatus = !report.extensionToken ? 'MISSING' : hasMismatch ? 'MISMATCH' : 'OK';
|
||||
lines.push(statusLine(extStatus, `Extension token (Chrome LevelDB): ${tokenSummary(report.extensionToken, report.extensionFingerprint)}`));
|
||||
|
||||
const envStatus: ReportStatus = !report.envToken ? 'MISSING' : hasMismatch ? 'MISMATCH' : 'OK';
|
||||
lines.push(statusLine(envStatus, `Environment token: ${tokenSummary(report.envToken, report.envFingerprint)}`));
|
||||
|
||||
for (const shell of report.shellFiles) {
|
||||
const shellStatus: ReportStatus = !shell.token ? 'MISSING' : hasMismatch ? 'MISMATCH' : 'OK';
|
||||
lines.push(statusLine(shellStatus, `Shell file ${shell.path}: ${tokenSummary(shell.token, shell.fingerprint)}`));
|
||||
}
|
||||
const existingConfigs = report.configs.filter(config => config.exists);
|
||||
const missingConfigCount = report.configs.length - existingConfigs.length;
|
||||
if (existingConfigs.length > 0) {
|
||||
for (const config of existingConfigs) {
|
||||
const parseSuffix = config.parseError ? ` (parse error: ${config.parseError})` : '';
|
||||
const configStatus: ReportStatus = config.parseError
|
||||
? 'WARN'
|
||||
: !config.token
|
||||
? 'MISSING'
|
||||
: hasMismatch
|
||||
? 'MISMATCH'
|
||||
: 'OK';
|
||||
lines.push(statusLine(configStatus, `MCP config ${config.path}: ${tokenSummary(config.token, config.fingerprint)}${parseSuffix}`));
|
||||
}
|
||||
} else {
|
||||
lines.push(statusLine('MISSING', 'MCP config: no existing config files found in scanned locations'));
|
||||
}
|
||||
if (missingConfigCount > 0) lines.push(` Other scanned config locations not present: ${missingConfigCount}`);
|
||||
lines.push('');
|
||||
lines.push(statusLine(
|
||||
hasMismatch ? 'MISMATCH' : report.recommendedToken ? 'OK' : 'WARN',
|
||||
`Recommended token fingerprint: ${report.recommendedFingerprint ?? 'unavailable'}`,
|
||||
));
|
||||
if (report.issues.length) {
|
||||
lines.push('', 'Issues:');
|
||||
for (const issue of report.issues) lines.push(`- ${issue}`);
|
||||
}
|
||||
if (report.warnings.length) {
|
||||
lines.push('', 'Warnings:');
|
||||
for (const warning of report.warnings) lines.push(`- ${warning}`);
|
||||
}
|
||||
return lines.join('\n');
|
||||
}
|
||||
|
||||
async function confirmPrompt(question: string): Promise<boolean> {
|
||||
const rl = createInterface({ input, output });
|
||||
try {
|
||||
const answer = (await rl.question(`${question} [y/N] `)).trim().toLowerCase();
|
||||
return answer === 'y' || answer === 'yes';
|
||||
} finally {
|
||||
rl.close();
|
||||
}
|
||||
}
|
||||
|
||||
function writeFileWithMkdir(filePath: string, content: string): void {
|
||||
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
}
|
||||
|
||||
export async function applyBrowserDoctorFix(report: DoctorReport, opts: DoctorOptions = {}): Promise<string[]> {
|
||||
const token = opts.token ?? report.recommendedToken;
|
||||
if (!token) throw new Error('No Playwright MCP token is available to write. Provide --token first.');
|
||||
|
||||
const plannedWrites: string[] = [];
|
||||
const shellPath = opts.shellRc ?? report.shellFiles[0]?.path ?? getDefaultShellRcPath();
|
||||
plannedWrites.push(shellPath);
|
||||
for (const config of report.configs) {
|
||||
if (!config.writable) continue;
|
||||
plannedWrites.push(config.path);
|
||||
}
|
||||
|
||||
if (!opts.yes) {
|
||||
const ok = await confirmPrompt(`Update ${plannedWrites.length} file(s) with Playwright MCP token fingerprint ${getTokenFingerprint(token)}?`);
|
||||
if (!ok) return [];
|
||||
}
|
||||
|
||||
const written: string[] = [];
|
||||
const shellBefore = fileExists(shellPath) ? fs.readFileSync(shellPath, 'utf-8') : '';
|
||||
writeFileWithMkdir(shellPath, upsertShellToken(shellBefore, token));
|
||||
written.push(shellPath);
|
||||
|
||||
for (const config of report.configs) {
|
||||
if (!config.writable || config.parseError) continue;
|
||||
const before = fileExists(config.path) ? fs.readFileSync(config.path, 'utf-8') : '';
|
||||
const next = config.format === 'toml' ? upsertTomlConfigToken(before, token) : upsertJsonConfigToken(before, token);
|
||||
writeFileWithMkdir(config.path, next);
|
||||
written.push(config.path);
|
||||
}
|
||||
|
||||
process.env[PLAYWRIGHT_TOKEN_ENV] = token;
|
||||
return written;
|
||||
}
|
||||
@@ -0,0 +1,77 @@
|
||||
/**
|
||||
* Tests for engine.ts: CLI discovery and command execution.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, vi, beforeEach } from 'vitest';
|
||||
import { discoverClis, executeCommand } from './engine.js';
|
||||
import { getRegistry, cli, Strategy } from './registry.js';
|
||||
|
||||
describe('discoverClis', () => {
|
||||
it('handles non-existent directories gracefully', async () => {
|
||||
// Should not throw for missing directories
|
||||
await expect(discoverClis('/tmp/nonexistent-opencli-test-dir')).resolves.not.toThrow();
|
||||
});
|
||||
});
|
||||
|
||||
describe('executeCommand', () => {
|
||||
it('executes a command with func', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'func-test',
|
||||
description: 'test command with func',
|
||||
browser: false,
|
||||
strategy: Strategy.PUBLIC,
|
||||
func: async (_page, kwargs) => {
|
||||
return [{ title: kwargs.query ?? 'default' }];
|
||||
},
|
||||
});
|
||||
|
||||
const result = await executeCommand(cmd, null, { query: 'hello' });
|
||||
expect(result).toEqual([{ title: 'hello' }]);
|
||||
});
|
||||
|
||||
it('executes a command with pipeline', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'pipe-test',
|
||||
description: 'test command with pipeline',
|
||||
browser: false,
|
||||
strategy: Strategy.PUBLIC,
|
||||
pipeline: [
|
||||
{ evaluate: '() => [{ n: 1 }, { n: 2 }, { n: 3 }]' },
|
||||
{ limit: '2' },
|
||||
],
|
||||
});
|
||||
|
||||
// Pipeline commands require page for evaluate step, so we'll test the error path
|
||||
await expect(executeCommand(cmd, null, {})).rejects.toThrow();
|
||||
});
|
||||
|
||||
it('throws for command with no func or pipeline', async () => {
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'empty-test',
|
||||
description: 'empty command',
|
||||
browser: false,
|
||||
});
|
||||
|
||||
await expect(executeCommand(cmd, null, {})).rejects.toThrow('has no func or pipeline');
|
||||
});
|
||||
|
||||
it('passes debug flag to func', async () => {
|
||||
let receivedDebug = false;
|
||||
const cmd = cli({
|
||||
site: 'test-engine',
|
||||
name: 'debug-test',
|
||||
description: 'debug test',
|
||||
browser: false,
|
||||
func: async (_page, _kwargs, debug) => {
|
||||
receivedDebug = debug ?? false;
|
||||
return [];
|
||||
},
|
||||
});
|
||||
|
||||
await executeCommand(cmd, null, {}, true);
|
||||
expect(receivedDebug).toBe(true);
|
||||
});
|
||||
});
|
||||
+126
-11
@@ -1,28 +1,116 @@
|
||||
/**
|
||||
* CLI discovery: finds YAML/TS CLI definitions and registers them.
|
||||
*
|
||||
* Supports two modes:
|
||||
* 1. FAST PATH (manifest): If a pre-compiled cli-manifest.json exists,
|
||||
* registers all YAML commands instantly without runtime YAML parsing.
|
||||
* TS modules are loaded lazily only when their command is executed.
|
||||
* 2. FALLBACK (filesystem scan): Traditional runtime discovery for development.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { type CliCommand, type Arg, Strategy, registerCommand } from './registry.js';
|
||||
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
|
||||
import type { IPage } from './types.js';
|
||||
import { executePipeline } from './pipeline.js';
|
||||
|
||||
export function discoverClis(...dirs: string[]): void {
|
||||
/** Set of TS module paths that have been loaded */
|
||||
const _loadedModules = new Set<string>();
|
||||
|
||||
/**
|
||||
* Discover and register CLI commands.
|
||||
* Uses pre-compiled manifest when available for instant startup.
|
||||
*/
|
||||
export async function discoverClis(...dirs: string[]): Promise<void> {
|
||||
// Fast path: try manifest first (production / post-build)
|
||||
for (const dir of dirs) {
|
||||
if (!fs.existsSync(dir)) continue;
|
||||
for (const site of fs.readdirSync(dir)) {
|
||||
const siteDir = path.join(dir, site);
|
||||
if (!fs.statSync(siteDir).isDirectory()) continue;
|
||||
for (const file of fs.readdirSync(siteDir)) {
|
||||
const filePath = path.join(siteDir, file);
|
||||
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
||||
registerYamlCli(filePath, site);
|
||||
}
|
||||
const manifestPath = path.resolve(dir, '..', 'cli-manifest.json');
|
||||
if (fs.existsSync(manifestPath)) {
|
||||
loadFromManifest(manifestPath, dir);
|
||||
continue; // Skip filesystem scan for this directory
|
||||
}
|
||||
// Fallback: runtime filesystem scan (development)
|
||||
await discoverClisFromFs(dir);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fast-path: register commands from pre-compiled manifest.
|
||||
* YAML pipelines are inlined — zero YAML parsing at runtime.
|
||||
* TS modules are deferred — loaded lazily on first execution.
|
||||
*/
|
||||
function loadFromManifest(manifestPath: string, clisDir: string): void {
|
||||
try {
|
||||
const manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf-8')) as any[];
|
||||
for (const entry of manifest) {
|
||||
if (entry.type === 'yaml') {
|
||||
// YAML pipelines fully inlined in manifest — register directly
|
||||
const strategy = (Strategy as any)[entry.strategy.toUpperCase()] ?? Strategy.COOKIE;
|
||||
const cmd: CliCommand = {
|
||||
site: entry.site,
|
||||
name: entry.name,
|
||||
description: entry.description ?? '',
|
||||
domain: entry.domain,
|
||||
strategy,
|
||||
browser: entry.browser,
|
||||
args: entry.args ?? [],
|
||||
columns: entry.columns,
|
||||
pipeline: entry.pipeline,
|
||||
timeoutSeconds: entry.timeout,
|
||||
source: `manifest:${entry.site}/${entry.name}`,
|
||||
};
|
||||
registerCommand(cmd);
|
||||
} else if (entry.type === 'ts' && entry.modulePath) {
|
||||
// TS adapters: register a lightweight stub.
|
||||
// The actual module is loaded lazily on first executeCommand().
|
||||
const strategy = (Strategy as any)[(entry.strategy ?? 'cookie').toUpperCase()] ?? Strategy.COOKIE;
|
||||
const modulePath = path.resolve(clisDir, entry.modulePath);
|
||||
const cmd: InternalCliCommand = {
|
||||
site: entry.site,
|
||||
name: entry.name,
|
||||
description: entry.description ?? '',
|
||||
domain: entry.domain,
|
||||
strategy,
|
||||
browser: entry.browser ?? true,
|
||||
args: entry.args ?? [],
|
||||
columns: entry.columns,
|
||||
timeoutSeconds: entry.timeout,
|
||||
source: modulePath,
|
||||
_lazy: true,
|
||||
_modulePath: modulePath,
|
||||
};
|
||||
registerCommand(cmd);
|
||||
}
|
||||
}
|
||||
} catch (err: any) {
|
||||
process.stderr.write(`Warning: failed to load manifest ${manifestPath}: ${err.message}\n`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Fallback: traditional filesystem scan (used during development with tsx).
|
||||
*/
|
||||
async function discoverClisFromFs(dir: string): Promise<void> {
|
||||
if (!fs.existsSync(dir)) return;
|
||||
const promises: Promise<any>[] = [];
|
||||
for (const site of fs.readdirSync(dir)) {
|
||||
const siteDir = path.join(dir, site);
|
||||
if (!fs.statSync(siteDir).isDirectory()) continue;
|
||||
for (const file of fs.readdirSync(siteDir)) {
|
||||
const filePath = path.join(siteDir, file);
|
||||
if (file.endsWith('.yaml') || file.endsWith('.yml')) {
|
||||
registerYamlCli(filePath, site);
|
||||
} else if (file.endsWith('.js') && !file.endsWith('.d.js')) {
|
||||
promises.push(
|
||||
import(`file://${filePath}`).catch((err: any) => {
|
||||
process.stderr.write(`Warning: failed to load module ${filePath}: ${err.message}\n`);
|
||||
})
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
await Promise.all(promises);
|
||||
}
|
||||
|
||||
function registerYamlCli(filePath: string, defaultSite: string): void {
|
||||
@@ -71,12 +159,39 @@ function registerYamlCli(filePath: string, defaultSite: string): void {
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Execute a CLI command. Handles lazy-loading of TS modules.
|
||||
*/
|
||||
export async function executeCommand(
|
||||
cmd: CliCommand,
|
||||
page: IPage | null,
|
||||
kwargs: Record<string, any>,
|
||||
debug: boolean = false,
|
||||
): Promise<any> {
|
||||
// Lazy-load TS module on first execution
|
||||
const internal = cmd as InternalCliCommand;
|
||||
if (internal._lazy && internal._modulePath) {
|
||||
const modulePath = internal._modulePath;
|
||||
if (!_loadedModules.has(modulePath)) {
|
||||
try {
|
||||
await import(`file://${modulePath}`);
|
||||
_loadedModules.add(modulePath);
|
||||
} catch (err: any) {
|
||||
throw new Error(`Failed to load adapter module ${modulePath}: ${err.message}`);
|
||||
}
|
||||
}
|
||||
// After loading, the module's cli() call will have updated the registry
|
||||
// with the real func/pipeline. Re-fetch the command.
|
||||
const { getRegistry, fullName } = await import('./registry.js');
|
||||
const updated = getRegistry().get(fullName(cmd));
|
||||
if (updated && updated.func) {
|
||||
return updated.func(page, kwargs, debug);
|
||||
}
|
||||
if (updated && updated.pipeline) {
|
||||
return executePipeline(page, updated.pipeline, { args: kwargs, debug });
|
||||
}
|
||||
}
|
||||
|
||||
if (cmd.func) {
|
||||
return cmd.func(page, kwargs, debug);
|
||||
}
|
||||
|
||||
+53
-15
@@ -9,6 +9,7 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { DEFAULT_BROWSER_EXPLORE_TIMEOUT, browserSession, runWithTimeout } from './runtime.js';
|
||||
import { VOLATILE_PARAMS, SEARCH_PARAMS, PAGINATION_PARAMS, LIMIT_PARAMS, FIELD_ROLES } from './constants.js';
|
||||
|
||||
// ── Site name detection ────────────────────────────────────────────────────
|
||||
|
||||
@@ -43,21 +44,7 @@ export function slugify(value: string): string {
|
||||
|
||||
// ── Field & capability inference ───────────────────────────────────────────
|
||||
|
||||
const FIELD_ROLES: Record<string, string[]> = {
|
||||
title: ['title', 'name', 'text', 'content', 'desc', 'description', 'headline', 'subject'],
|
||||
url: ['url', 'uri', 'link', 'href', 'permalink', 'jump_url', 'web_url', 'share_url'],
|
||||
author: ['author', 'username', 'user_name', 'nickname', 'nick', 'owner', 'creator', 'up_name', 'uname'],
|
||||
score: ['score', 'hot', 'heat', 'likes', 'like_count', 'view_count', 'views', 'play', 'favorite_count', 'reply_count'],
|
||||
time: ['time', 'created_at', 'publish_time', 'pub_time', 'date', 'ctime', 'mtime', 'pubdate', 'created'],
|
||||
id: ['id', 'aid', 'bvid', 'mid', 'uid', 'oid', 'note_id', 'item_id'],
|
||||
cover: ['cover', 'pic', 'image', 'thumbnail', 'poster', 'avatar'],
|
||||
category: ['category', 'tag', 'type', 'tname', 'channel', 'section'],
|
||||
};
|
||||
|
||||
const SEARCH_PARAMS = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'search_query', 'w']);
|
||||
const PAGINATION_PARAMS = new Set(['page', 'pn', 'offset', 'cursor', 'next', 'page_num']);
|
||||
const LIMIT_PARAMS = new Set(['limit', 'count', 'size', 'per_page', 'page_size', 'ps', 'num']);
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', '_', 'callback', 'timestamp', 't', 'nonce', 'sign']);
|
||||
// (constants now imported from constants.ts)
|
||||
|
||||
// ── Network analysis ───────────────────────────────────────────────────────
|
||||
|
||||
@@ -184,6 +171,8 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: any; pattern
|
||||
if (ep.hasPaginationParam) s += 2;
|
||||
if (ep.hasLimitParam) s += 2;
|
||||
if (ep.status === 200) s += 2;
|
||||
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
|
||||
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3;
|
||||
return s;
|
||||
}
|
||||
|
||||
@@ -277,6 +266,30 @@ export interface DiscoveredStore {
|
||||
stateKeys: string[];
|
||||
}
|
||||
|
||||
// ── Auto-Interaction (Fuzzing) ─────────────────────────────────────────────
|
||||
|
||||
const INTERACT_FUZZ_JS = `
|
||||
async () => {
|
||||
const sleep = ms => new Promise(r => setTimeout(r, ms));
|
||||
const clickables = Array.from(document.querySelectorAll(
|
||||
'button, [role="button"], [role="tab"], .tab, .btn, a[href="javascript:void(0)"], a[href="#"]'
|
||||
)).slice(0, 15); // limit to 15 to avoid endless loops
|
||||
|
||||
let clicked = 0;
|
||||
for (const el of clickables) {
|
||||
try {
|
||||
const rect = el.getBoundingClientRect();
|
||||
if (rect.width > 0 && rect.height > 0) {
|
||||
el.dispatchEvent(new MouseEvent('click', { bubbles: true, cancelable: true, view: window }));
|
||||
clicked++;
|
||||
await sleep(300); // give it time to trigger network
|
||||
}
|
||||
} catch {}
|
||||
}
|
||||
return clicked;
|
||||
}
|
||||
`;
|
||||
|
||||
// ── Main explore function ──────────────────────────────────────────────────
|
||||
|
||||
export async function exploreUrl(
|
||||
@@ -300,6 +313,31 @@ export async function exploreUrl(
|
||||
// Step 2: Auto-scroll to trigger lazy loading (use keyboard since page.scroll may not exist)
|
||||
for (let i = 0; i < 3; i++) { try { await page.pressKey('End'); } catch {} await page.wait(1); }
|
||||
|
||||
// Step 2.5: Interactive Fuzzing (if requested)
|
||||
if (opts.auto) {
|
||||
try {
|
||||
// First: targeted clicks by label (e.g. "字幕", "CC", "评论")
|
||||
if (opts.clickLabels?.length) {
|
||||
for (const label of opts.clickLabels) {
|
||||
const safeLabel = label.replace(/'/g, "\\'");
|
||||
await page.evaluate(`
|
||||
(() => {
|
||||
const el = [...document.querySelectorAll('button, [role="button"], [role="tab"], a, span')]
|
||||
.find(e => e.textContent && e.textContent.trim().includes('${safeLabel}'));
|
||||
if (el) el.click();
|
||||
})()
|
||||
`);
|
||||
await page.wait(1);
|
||||
}
|
||||
}
|
||||
// Then: blind fuzzing on generic interactive elements
|
||||
const clicks = await page.evaluate(INTERACT_FUZZ_JS);
|
||||
await page.wait(2); // wait for XHRs to settle
|
||||
} catch (e) {
|
||||
// fuzzing is best-effort, don't fail the whole explore
|
||||
}
|
||||
}
|
||||
|
||||
// Step 3: Read page metadata
|
||||
const metadata = await readPageMetadata(page);
|
||||
|
||||
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Shared XHR/Fetch interceptor JavaScript generators.
|
||||
*
|
||||
* Provides a single source of truth for monkey-patching browser
|
||||
* fetch() and XMLHttpRequest to capture API responses matching
|
||||
* a URL pattern. Used by:
|
||||
* - Page.installInterceptor() (browser.ts)
|
||||
* - stepIntercept (pipeline/steps/intercept.ts)
|
||||
* - stepTap (pipeline/steps/tap.ts)
|
||||
*/
|
||||
|
||||
/**
|
||||
* Generate JavaScript source that installs a fetch/XHR interceptor.
|
||||
* Captured responses are pushed to `window.__opencli_intercepted`.
|
||||
*
|
||||
* @param patternExpr - JS expression resolving to a URL substring to match (e.g. a JSON.stringify'd string)
|
||||
* @param opts.arrayName - Global array name for captured data (default: '__opencli_intercepted')
|
||||
* @param opts.patchGuard - Global boolean name to prevent double-patching (default: '__opencli_interceptor_patched')
|
||||
*/
|
||||
export function generateInterceptorJs(
|
||||
patternExpr: string,
|
||||
opts: { arrayName?: string; patchGuard?: string } = {},
|
||||
): string {
|
||||
const arr = opts.arrayName ?? '__opencli_intercepted';
|
||||
const guard = opts.patchGuard ?? '__opencli_interceptor_patched';
|
||||
|
||||
return `
|
||||
() => {
|
||||
window.${arr} = window.${arr} || [];
|
||||
const __pattern = ${patternExpr};
|
||||
|
||||
if (!window.${guard}) {
|
||||
const __checkMatch = (url) => __pattern && url.includes(__pattern);
|
||||
|
||||
// ── Patch fetch ──
|
||||
const __origFetch = window.fetch;
|
||||
window.fetch = async function(...args) {
|
||||
const reqUrl = typeof args[0] === 'string' ? args[0]
|
||||
: (args[0] && args[0].url) || '';
|
||||
const response = await __origFetch.apply(this, args);
|
||||
if (__checkMatch(reqUrl)) {
|
||||
try {
|
||||
const clone = response.clone();
|
||||
const json = await clone.json();
|
||||
window.${arr}.push(json);
|
||||
} catch(e) {}
|
||||
}
|
||||
return response;
|
||||
};
|
||||
|
||||
// ── Patch XMLHttpRequest ──
|
||||
const __XHR = XMLHttpRequest.prototype;
|
||||
const __origOpen = __XHR.open;
|
||||
const __origSend = __XHR.send;
|
||||
__XHR.open = function(method, url) {
|
||||
this.__opencli_url = String(url);
|
||||
return __origOpen.apply(this, arguments);
|
||||
};
|
||||
__XHR.send = function() {
|
||||
if (__checkMatch(this.__opencli_url)) {
|
||||
this.addEventListener('load', function() {
|
||||
try {
|
||||
window.${arr}.push(JSON.parse(this.responseText));
|
||||
} catch(e) {}
|
||||
});
|
||||
}
|
||||
return __origSend.apply(this, arguments);
|
||||
};
|
||||
|
||||
window.${guard} = true;
|
||||
}
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate JavaScript source to read and clear intercepted data.
|
||||
*/
|
||||
export function generateReadInterceptedJs(arrayName: string = '__opencli_intercepted'): string {
|
||||
return `
|
||||
() => {
|
||||
const data = window.${arrayName} || [];
|
||||
window.${arrayName} = [];
|
||||
return data;
|
||||
}
|
||||
`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Generate a self-contained tap interceptor for store-action bridge.
|
||||
* Unlike the global interceptor, this one:
|
||||
* - Installs temporarily, restores originals in finally block
|
||||
* - Resolves a promise on first capture (for immediate await)
|
||||
* - Returns captured data directly
|
||||
*/
|
||||
export function generateTapInterceptorJs(patternExpr: string): {
|
||||
setupVar: string;
|
||||
capturedVar: string;
|
||||
promiseVar: string;
|
||||
resolveVar: string;
|
||||
fetchPatch: string;
|
||||
xhrPatch: string;
|
||||
restorePatch: string;
|
||||
} {
|
||||
return {
|
||||
setupVar: `
|
||||
let captured = null;
|
||||
let captureResolve;
|
||||
const capturePromise = new Promise(r => { captureResolve = r; });
|
||||
const capturePattern = ${patternExpr};
|
||||
`,
|
||||
capturedVar: 'captured',
|
||||
promiseVar: 'capturePromise',
|
||||
resolveVar: 'captureResolve',
|
||||
fetchPatch: `
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...fetchArgs) {
|
||||
const resp = await origFetch.apply(this, fetchArgs);
|
||||
try {
|
||||
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
||||
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
||||
if (capturePattern && url.includes(capturePattern) && !captured) {
|
||||
try { captured = await resp.clone().json(); captureResolve(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
return resp;
|
||||
};
|
||||
`,
|
||||
xhrPatch: `
|
||||
const origXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const origXhrSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, url) {
|
||||
this.__tapUrl = String(url);
|
||||
return origXhrOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(body) {
|
||||
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
||||
this.addEventListener('load', function() {
|
||||
if (!captured) {
|
||||
try { captured = JSON.parse(this.responseText); captureResolve(); } catch {}
|
||||
}
|
||||
});
|
||||
}
|
||||
return origXhrSend.apply(this, arguments);
|
||||
};
|
||||
`,
|
||||
restorePatch: `
|
||||
window.fetch = origFetch;
|
||||
XMLHttpRequest.prototype.open = origXhrOpen;
|
||||
XMLHttpRequest.prototype.send = origXhrSend;
|
||||
`,
|
||||
};
|
||||
}
|
||||
+66
-13
@@ -3,7 +3,6 @@
|
||||
* opencli — Make any website your CLI. AI-powered.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as os from 'node:os';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
@@ -12,31 +11,45 @@ import chalk from 'chalk';
|
||||
import { discoverClis, executeCommand } from './engine.js';
|
||||
import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
|
||||
import { render as renderOutput } from './output.js';
|
||||
import './clis/index.js';
|
||||
import { PlaywrightMCP } from './browser.js';
|
||||
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
|
||||
import { PKG_VERSION } from './version.js';
|
||||
|
||||
const __filename = fileURLToPath(import.meta.url);
|
||||
const __dirname = path.dirname(__filename);
|
||||
const BUILTIN_CLIS = path.resolve(__dirname, 'clis');
|
||||
const USER_CLIS = path.join(os.homedir(), '.opencli', 'clis');
|
||||
|
||||
// Read version from package.json (single source of truth)
|
||||
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
||||
const PKG_VERSION = JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version ?? '0.0.0';
|
||||
|
||||
discoverClis(BUILTIN_CLIS, USER_CLIS);
|
||||
await discoverClis(BUILTIN_CLIS, USER_CLIS);
|
||||
|
||||
const program = new Command();
|
||||
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
|
||||
|
||||
// ── Built-in commands ──────────────────────────────────────────────────────
|
||||
|
||||
program.command('list').description('List all available CLI commands').option('--json', 'JSON output')
|
||||
program.command('list').description('List all available CLI commands').option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)')
|
||||
.action((opts) => {
|
||||
const registry = getRegistry();
|
||||
const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
|
||||
if (opts.json) { console.log(JSON.stringify(commands.map(c => ({ command: fullName(c), site: c.site, name: c.name, description: c.description, strategy: strategyLabel(c), browser: c.browser, args: c.args.map(a => a.name) })), null, 2)); return; }
|
||||
const rows = commands.map(c => ({
|
||||
command: fullName(c),
|
||||
site: c.site,
|
||||
name: c.name,
|
||||
description: c.description,
|
||||
strategy: strategyLabel(c),
|
||||
browser: c.browser,
|
||||
args: c.args.map(a => a.name).join(', '),
|
||||
}));
|
||||
const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format;
|
||||
if (fmt !== 'table') {
|
||||
renderOutput(rows, {
|
||||
fmt,
|
||||
columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args'],
|
||||
title: 'opencli/list',
|
||||
source: 'opencli list',
|
||||
});
|
||||
return;
|
||||
}
|
||||
const sites = new Map<string, CliCommand[]>();
|
||||
for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); }
|
||||
console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log();
|
||||
@@ -54,8 +67,8 @@ program.command('validate').description('Validate CLI definitions').argument('[t
|
||||
program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
|
||||
.action(async (target, opts) => { const { verifyClis, renderVerifyReport } = await import('./verify.js'); const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke }); console.log(renderVerifyReport(r)); process.exitCode = r.ok ? 0 : 1; });
|
||||
|
||||
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3')
|
||||
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait) }))); });
|
||||
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
|
||||
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: PlaywrightMCP, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels }))); });
|
||||
|
||||
program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
|
||||
.action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
|
||||
@@ -74,6 +87,38 @@ program.command('cascade').description('Strategy cascade: find simplest working
|
||||
console.log(renderCascadeResult(result));
|
||||
});
|
||||
|
||||
program.command('doctor')
|
||||
.description('Diagnose Playwright MCP Bridge, token consistency, and Chrome remote debugging')
|
||||
.option('--fix', 'Apply suggested fixes to shell rc and detected MCP configs', false)
|
||||
.option('-y, --yes', 'Skip confirmation prompts when applying fixes', false)
|
||||
.option('--token <token>', 'Override token to write instead of auto-detecting')
|
||||
.option('--shell-rc <path>', 'Shell startup file to update')
|
||||
.option('--mcp-config <paths>', 'Comma-separated MCP config paths to scan/update')
|
||||
.action(async (opts) => {
|
||||
const { runBrowserDoctor, renderBrowserDoctorReport, applyBrowserDoctorFix } = await import('./doctor.js');
|
||||
const configPaths = opts.mcpConfig ? String(opts.mcpConfig).split(',').map((s: string) => s.trim()).filter(Boolean) : undefined;
|
||||
const report = await runBrowserDoctor({ token: opts.token, shellRc: opts.shellRc, configPaths, cliVersion: PKG_VERSION });
|
||||
console.log(renderBrowserDoctorReport(report));
|
||||
if (opts.fix) {
|
||||
const written = await applyBrowserDoctorFix(report, { fix: true, yes: opts.yes, token: opts.token, shellRc: opts.shellRc, configPaths });
|
||||
console.log();
|
||||
if (written.length > 0) {
|
||||
console.log(chalk.green('Updated files:'));
|
||||
for (const filePath of written) console.log(`- ${filePath}`);
|
||||
} else {
|
||||
console.log(chalk.yellow('No files were changed.'));
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
program.command('setup')
|
||||
.description('Interactive setup: configure Playwright MCP token across all detected tools')
|
||||
.option('--token <token>', 'Provide token directly instead of auto-detecting')
|
||||
.action(async (opts) => {
|
||||
const { runSetup } = await import('./setup.js');
|
||||
await runSetup({ cliVersion: PKG_VERSION, token: opts.token });
|
||||
});
|
||||
|
||||
// ── Dynamic site commands ──────────────────────────────────────────────────
|
||||
|
||||
const registry = getRegistry();
|
||||
@@ -90,7 +135,7 @@ for (const [, cmd] of registry) {
|
||||
else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
|
||||
else subCmd.option(flag, arg.help ?? '');
|
||||
}
|
||||
subCmd.option('-f, --format <fmt>', 'Output format: table, json, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
|
||||
subCmd.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
|
||||
|
||||
subCmd.action(async (actionOpts) => {
|
||||
const startTime = Date.now();
|
||||
@@ -100,12 +145,20 @@ for (const [, cmd] of registry) {
|
||||
else if (arg.default != null) kwargs[arg.name] = arg.default;
|
||||
}
|
||||
try {
|
||||
if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1';
|
||||
let result: any;
|
||||
if (cmd.browser) {
|
||||
result = await browserSession(PlaywrightMCP, async (page) => runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) }));
|
||||
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
|
||||
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
|
||||
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
|
||||
}
|
||||
renderOutput(result, { fmt: actionOpts.format, columns: cmd.columns, title: `${cmd.site}/${cmd.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(cmd) });
|
||||
} catch (err: any) { console.error(chalk.red(`Error: ${err.message ?? err}`)); process.exitCode = 1; }
|
||||
} catch (err: any) {
|
||||
if (actionOpts.verbose && err.stack) { console.error(chalk.red(err.stack)); }
|
||||
else { console.error(chalk.red(`Error: ${err.message ?? err}`)); }
|
||||
process.exitCode = 1;
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
import { render } from './output.js';
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('render', () => {
|
||||
it('renders YAML output', () => {
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
|
||||
render([{ title: 'Hello', rank: 1 }], { fmt: 'yaml' });
|
||||
|
||||
expect(log).toHaveBeenCalledOnce();
|
||||
expect(log.mock.calls[0]?.[0]).toContain('- title: Hello');
|
||||
expect(log.mock.calls[0]?.[0]).toContain('rank: 1');
|
||||
});
|
||||
|
||||
it('renders yml alias as YAML output', () => {
|
||||
const log = vi.spyOn(console, 'log').mockImplementation(() => {});
|
||||
|
||||
render({ title: 'Hello' }, { fmt: 'yml' });
|
||||
|
||||
expect(log).toHaveBeenCalledOnce();
|
||||
expect(log.mock.calls[0]?.[0]).toContain('title: Hello');
|
||||
});
|
||||
});
|
||||
@@ -40,10 +40,6 @@ function renderTable(data: any, opts: RenderOptions): void {
|
||||
style: { head: [], border: [] },
|
||||
wordWrap: true,
|
||||
wrapOnWordBoundary: true,
|
||||
colWidths: columns.map((_c, i) => {
|
||||
if (i === 0) return 6;
|
||||
return null as any;
|
||||
}).filter(() => true),
|
||||
});
|
||||
|
||||
for (const row of rows) {
|
||||
|
||||
+15
-15
@@ -15,26 +15,26 @@ export interface PipelineContext {
|
||||
debug?: boolean;
|
||||
}
|
||||
|
||||
/** Step handler signature */
|
||||
/** Step handler: all steps conform to (page, params, data, args) => Promise<any> */
|
||||
type StepHandler = (page: IPage | null, params: any, data: any, args: Record<string, any>) => Promise<any>;
|
||||
|
||||
/** Registry of all available step handlers */
|
||||
const STEP_HANDLERS: Record<string, StepHandler> = {
|
||||
navigate: stepNavigate as StepHandler,
|
||||
navigate: stepNavigate,
|
||||
fetch: stepFetch,
|
||||
select: stepSelect as StepHandler,
|
||||
evaluate: stepEvaluate as StepHandler,
|
||||
snapshot: stepSnapshot as StepHandler,
|
||||
click: stepClick as StepHandler,
|
||||
type: stepType as StepHandler,
|
||||
wait: stepWait as StepHandler,
|
||||
press: stepPress as StepHandler,
|
||||
map: stepMap as StepHandler,
|
||||
filter: stepFilter as StepHandler,
|
||||
sort: stepSort as StepHandler,
|
||||
limit: stepLimit as StepHandler,
|
||||
intercept: stepIntercept as StepHandler,
|
||||
tap: stepTap as StepHandler,
|
||||
select: stepSelect,
|
||||
evaluate: stepEvaluate,
|
||||
snapshot: stepSnapshot,
|
||||
click: stepClick,
|
||||
type: stepType,
|
||||
wait: stepWait,
|
||||
press: stepPress,
|
||||
map: stepMap,
|
||||
filter: stepFilter,
|
||||
sort: stepSort,
|
||||
limit: stepLimit,
|
||||
intercept: stepIntercept,
|
||||
tap: stepTap,
|
||||
};
|
||||
|
||||
export async function executePipeline(
|
||||
|
||||
@@ -31,13 +31,10 @@ export async function stepWait(page: IPage, params: any, data: any, args: Record
|
||||
if (typeof params === 'number') await page.wait(params);
|
||||
else if (typeof params === 'object' && params) {
|
||||
if ('text' in params) {
|
||||
const timeout = params.timeout ?? 10;
|
||||
const start = Date.now();
|
||||
while ((Date.now() - start) / 1000 < timeout) {
|
||||
const snap = await page.snapshot({ raw: true });
|
||||
if (typeof snap === 'string' && snap.includes(params.text)) break;
|
||||
await page.wait(0.5);
|
||||
}
|
||||
await page.wait({
|
||||
text: String(render(params.text, { args, data })),
|
||||
timeout: params.timeout
|
||||
});
|
||||
} else if ('time' in params) await page.wait(Number(params.time));
|
||||
} else if (typeof params === 'string') await page.wait(Number(render(params, { args, data })));
|
||||
return data;
|
||||
|
||||
@@ -5,6 +5,23 @@
|
||||
import type { IPage } from '../../types.js';
|
||||
import { render } from '../template.js';
|
||||
|
||||
/** Simple async concurrency limiter */
|
||||
async function mapConcurrent<T, R>(items: T[], limit: number, fn: (item: T, index: number) => Promise<R>): Promise<R[]> {
|
||||
const results: R[] = new Array(items.length);
|
||||
let index = 0;
|
||||
|
||||
async function worker() {
|
||||
while (index < items.length) {
|
||||
const i = index++;
|
||||
results[i] = await fn(items[i], i);
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(limit, items.length) }, () => worker());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
|
||||
/** Single URL fetch helper */
|
||||
async function fetchSingle(
|
||||
page: IPage | null, url: string, method: string,
|
||||
@@ -39,6 +56,46 @@ async function fetchSingle(
|
||||
`);
|
||||
}
|
||||
|
||||
/**
|
||||
* Batch fetch: send all URLs into the browser as a single evaluate() call.
|
||||
* This eliminates N-1 cross-process IPC round trips, performing all fetches
|
||||
* inside the V8 engine and returning results as one JSON array.
|
||||
*/
|
||||
async function fetchBatchInBrowser(
|
||||
page: IPage, urls: string[], method: string,
|
||||
headers: Record<string, string>, concurrency: number,
|
||||
): Promise<any[]> {
|
||||
const headersJs = JSON.stringify(headers);
|
||||
const urlsJs = JSON.stringify(urls);
|
||||
return page.evaluate(`
|
||||
async () => {
|
||||
const urls = ${urlsJs};
|
||||
const method = "${method}";
|
||||
const headers = ${headersJs};
|
||||
const concurrency = ${concurrency};
|
||||
|
||||
const results = new Array(urls.length);
|
||||
let idx = 0;
|
||||
|
||||
async function worker() {
|
||||
while (idx < urls.length) {
|
||||
const i = idx++;
|
||||
try {
|
||||
const resp = await fetch(urls[i], { method, headers, credentials: "include" });
|
||||
results[i] = await resp.json();
|
||||
} catch (e) {
|
||||
results[i] = { error: e.message };
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const workers = Array.from({ length: Math.min(concurrency, urls.length) }, () => worker());
|
||||
await Promise.all(workers);
|
||||
return results;
|
||||
}
|
||||
`);
|
||||
}
|
||||
|
||||
export async function stepFetch(page: IPage | null, params: any, data: any, args: Record<string, any>): Promise<any> {
|
||||
const urlOrObj = typeof params === 'string' ? params : (params?.url ?? '');
|
||||
const method = params?.method ?? 'GET';
|
||||
@@ -48,12 +105,33 @@ export async function stepFetch(page: IPage | null, params: any, data: any, args
|
||||
|
||||
// Per-item fetch when data is array and URL references item
|
||||
if (Array.isArray(data) && urlTemplate.includes('item')) {
|
||||
const results: any[] = [];
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const itemUrl = String(render(urlTemplate, { args, data, item: data[i], index: i }));
|
||||
results.push(await fetchSingle(page, itemUrl, method, queryParams, headers, args, data));
|
||||
const concurrency = typeof params?.concurrency === 'number' ? params.concurrency : 5;
|
||||
|
||||
// Render all URLs upfront
|
||||
const renderedHeaders: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(headers)) renderedHeaders[k] = String(render(v, { args, data }));
|
||||
const renderedParams: Record<string, string> = {};
|
||||
for (const [k, v] of Object.entries(queryParams)) renderedParams[k] = String(render(v, { args, data }));
|
||||
|
||||
const urls = data.map((item: any, index: number) => {
|
||||
let url = String(render(urlTemplate, { args, data, item, index }));
|
||||
if (Object.keys(renderedParams).length > 0) {
|
||||
const qs = new URLSearchParams(renderedParams).toString();
|
||||
url = `${url}${url.includes('?') ? '&' : '?'}${qs}`;
|
||||
}
|
||||
return url;
|
||||
});
|
||||
|
||||
// BATCH IPC: if browser is available, batch all fetches into a single evaluate() call
|
||||
if (page !== null) {
|
||||
return fetchBatchInBrowser(page, urls, method.toUpperCase(), renderedHeaders, concurrency);
|
||||
}
|
||||
return results;
|
||||
|
||||
// Non-browser: use concurrent pool (already optimized)
|
||||
return mapConcurrent(data, concurrency, async (item, index) => {
|
||||
const itemUrl = String(render(urlTemplate, { args, data, item, index }));
|
||||
return fetchSingle(null, itemUrl, method, queryParams, headers, args, data);
|
||||
});
|
||||
}
|
||||
const url = render(urlOrObj, { args, data });
|
||||
return fetchSingle(page, String(url), method, queryParams, headers, args, data);
|
||||
|
||||
@@ -4,6 +4,7 @@
|
||||
|
||||
import type { IPage } from '../../types.js';
|
||||
import { render } from '../template.js';
|
||||
import { generateInterceptorJs, generateReadInterceptedJs } from '../../interceptor.js';
|
||||
|
||||
export async function stepIntercept(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
|
||||
const cfg = typeof params === 'object' ? params : {};
|
||||
@@ -14,7 +15,10 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
|
||||
|
||||
if (!capturePattern) return data;
|
||||
|
||||
// Step 1: Execute the trigger action
|
||||
// Step 1: Inject fetch/XHR interceptor BEFORE trigger
|
||||
await page.evaluate(generateInterceptorJs(JSON.stringify(capturePattern)));
|
||||
|
||||
// Step 2: Execute the trigger action
|
||||
if (trigger.startsWith('navigate:')) {
|
||||
const url = render(trigger.slice('navigate:'.length), { args, data });
|
||||
await page.goto(String(url));
|
||||
@@ -29,38 +33,13 @@ export async function stepIntercept(page: IPage, params: any, data: any, args: R
|
||||
await page.scroll('down');
|
||||
}
|
||||
|
||||
// Step 2: Wait a bit for network requests to fire
|
||||
// Step 3: Wait a bit for network requests to fire
|
||||
await page.wait(Math.min(timeout, 3));
|
||||
|
||||
// Step 3: Get network requests and find matching ones
|
||||
const rawNetwork = await page.networkRequests(false);
|
||||
const matchingResponses: any[] = [];
|
||||
// Step 4: Retrieve captured data
|
||||
const matchingResponses = await page.evaluate(generateReadInterceptedJs());
|
||||
|
||||
if (typeof rawNetwork === 'string') {
|
||||
const lines = rawNetwork.split('\n');
|
||||
for (const line of lines) {
|
||||
const match = line.match(/\[?(GET|POST)\]?\s+(\S+)\s*(?:=>|→)\s*\[?(\d+)\]?/i);
|
||||
if (match) {
|
||||
const [, , url, status] = match;
|
||||
if (url.includes(capturePattern) && status === '200') {
|
||||
try {
|
||||
const body = await page.evaluate(`
|
||||
async () => {
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)}, { credentials: 'include' });
|
||||
if (!resp.ok) return null;
|
||||
return await resp.json();
|
||||
} catch { return null; }
|
||||
}
|
||||
`);
|
||||
if (body) matchingResponses.push(body);
|
||||
} catch {}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Step 4: Select from response if specified
|
||||
// Step 5: Select from response if specified
|
||||
let result = matchingResponses.length === 1 ? matchingResponses[0] :
|
||||
matchingResponses.length > 1 ? matchingResponses : data;
|
||||
|
||||
|
||||
+13
-50
@@ -11,6 +11,7 @@
|
||||
|
||||
import type { IPage } from '../../types.js';
|
||||
import { render } from '../template.js';
|
||||
import { generateTapInterceptorJs } from '../../interceptor.js';
|
||||
|
||||
export async function stepTap(page: IPage, params: any, data: any, args: Record<string, any>): Promise<any> {
|
||||
const cfg = typeof params === 'object' ? params : {};
|
||||
@@ -38,51 +39,15 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
|
||||
? `store[${JSON.stringify(actionName)}](${actionArgsRendered.join(', ')})`
|
||||
: `store[${JSON.stringify(actionName)}]()`;
|
||||
|
||||
// Use shared interceptor generator for fetch/XHR patching
|
||||
const tap = generateTapInterceptorJs(JSON.stringify(capturePattern));
|
||||
|
||||
const js = `
|
||||
async () => {
|
||||
// ── 1. Setup capture proxy (fetch + XHR dual interception) ──
|
||||
let captured = null;
|
||||
const capturePattern = ${JSON.stringify(capturePattern)};
|
||||
|
||||
// Intercept fetch API
|
||||
const origFetch = window.fetch;
|
||||
window.fetch = async function(...fetchArgs) {
|
||||
const resp = await origFetch.apply(this, fetchArgs);
|
||||
try {
|
||||
const url = typeof fetchArgs[0] === 'string' ? fetchArgs[0]
|
||||
: fetchArgs[0] instanceof Request ? fetchArgs[0].url : String(fetchArgs[0]);
|
||||
if (capturePattern && url.includes(capturePattern) && !captured) {
|
||||
try { captured = await resp.clone().json(); } catch {}
|
||||
}
|
||||
} catch {}
|
||||
return resp;
|
||||
};
|
||||
|
||||
// Intercept XMLHttpRequest
|
||||
const origXhrOpen = XMLHttpRequest.prototype.open;
|
||||
const origXhrSend = XMLHttpRequest.prototype.send;
|
||||
XMLHttpRequest.prototype.open = function(method, url) {
|
||||
this.__tapUrl = String(url);
|
||||
return origXhrOpen.apply(this, arguments);
|
||||
};
|
||||
XMLHttpRequest.prototype.send = function(body) {
|
||||
if (capturePattern && this.__tapUrl?.includes(capturePattern)) {
|
||||
const xhr = this;
|
||||
const origHandler = xhr.onreadystatechange;
|
||||
xhr.onreadystatechange = function() {
|
||||
if (xhr.readyState === 4 && !captured) {
|
||||
try { captured = JSON.parse(xhr.responseText); } catch {}
|
||||
}
|
||||
if (origHandler) origHandler.apply(this, arguments);
|
||||
};
|
||||
const origOnload = xhr.onload;
|
||||
xhr.onload = function() {
|
||||
if (!captured) { try { captured = JSON.parse(xhr.responseText); } catch {} }
|
||||
if (origOnload) origOnload.apply(this, arguments);
|
||||
};
|
||||
}
|
||||
return origXhrSend.apply(this, arguments);
|
||||
};
|
||||
${tap.setupVar}
|
||||
${tap.fetchPatch}
|
||||
${tap.xhrPatch}
|
||||
|
||||
try {
|
||||
// ── 2. Find store ──
|
||||
@@ -117,19 +82,17 @@ export async function stepTap(page: IPage, params: any, data: any, args: Record<
|
||||
await ${actionCall};
|
||||
|
||||
// ── 4. Wait for network response ──
|
||||
const deadline = Date.now() + ${timeout} * 1000;
|
||||
while (!captured && Date.now() < deadline) {
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
if (!${tap.capturedVar}) {
|
||||
const timeoutPromise = new Promise(r => setTimeout(r, ${timeout} * 1000));
|
||||
await Promise.race([${tap.promiseVar}, timeoutPromise]);
|
||||
}
|
||||
} finally {
|
||||
// ── 5. Always restore originals ──
|
||||
window.fetch = origFetch;
|
||||
XMLHttpRequest.prototype.open = origXhrOpen;
|
||||
XMLHttpRequest.prototype.send = origXhrSend;
|
||||
${tap.restorePatch}
|
||||
}
|
||||
|
||||
if (!captured) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
||||
return captured${selectChain} ?? captured;
|
||||
if (!${tap.capturedVar}) return { error: 'No matching response captured for pattern: ' + capturePattern };
|
||||
return ${tap.capturedVar}${selectChain} ?? ${tap.capturedVar};
|
||||
}
|
||||
`;
|
||||
|
||||
|
||||
@@ -75,6 +75,12 @@ describe('evalExpr', () => {
|
||||
it('applies length filter', () => {
|
||||
expect(evalExpr('item.items | length', { item: { items: [1, 2, 3] } })).toBe(3);
|
||||
});
|
||||
it('applies json filter to strings with quotes', () => {
|
||||
expect(evalExpr('args.keyword | json', { args: { keyword: "O'Reilly" } })).toBe('"O\'Reilly"');
|
||||
});
|
||||
it('applies json filter to nullish values', () => {
|
||||
expect(evalExpr('args.keyword | json', { args: {} })).toBe('null');
|
||||
});
|
||||
});
|
||||
|
||||
describe('render', () => {
|
||||
|
||||
@@ -75,7 +75,7 @@ export function evalExpr(expr: string, ctx: RenderContext): any {
|
||||
* Apply a named filter to a value.
|
||||
* Supported filters:
|
||||
* default(val), join(sep), upper, lower, truncate(n), trim,
|
||||
* replace(old,new), keys, length, first, last
|
||||
* replace(old,new), keys, length, first, last, json
|
||||
*/
|
||||
function applyFilter(filterExpr: string, value: any): any {
|
||||
const match = filterExpr.match(/^(\w+)(?:\((.+)\))?$/);
|
||||
@@ -117,6 +117,8 @@ function applyFilter(filterExpr: string, value: any): any {
|
||||
return Array.isArray(value) ? value[0] : value;
|
||||
case 'last':
|
||||
return Array.isArray(value) ? value[value.length - 1] : value;
|
||||
case 'json':
|
||||
return JSON.stringify(value ?? null);
|
||||
default:
|
||||
return value;
|
||||
}
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* Tests for registry.ts: Strategy enum, cli() registration, helpers.
|
||||
*/
|
||||
|
||||
import { describe, it, expect, beforeEach } from 'vitest';
|
||||
import { cli, getRegistry, fullName, strategyLabel, registerCommand, Strategy, type CliCommand } from './registry.js';
|
||||
|
||||
describe('cli() registration', () => {
|
||||
it('registers a command and returns it', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'hello',
|
||||
description: 'A test command',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: false,
|
||||
});
|
||||
|
||||
expect(cmd.site).toBe('test-registry');
|
||||
expect(cmd.name).toBe('hello');
|
||||
expect(cmd.strategy).toBe(Strategy.PUBLIC);
|
||||
expect(cmd.browser).toBe(false);
|
||||
expect(cmd.args).toEqual([]);
|
||||
});
|
||||
|
||||
it('puts registered command in the registry', () => {
|
||||
cli({
|
||||
site: 'test-registry',
|
||||
name: 'registered',
|
||||
description: 'test',
|
||||
});
|
||||
|
||||
const registry = getRegistry();
|
||||
expect(registry.has('test-registry/registered')).toBe(true);
|
||||
});
|
||||
|
||||
it('defaults strategy to COOKIE when browser is true', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'default-strategy',
|
||||
});
|
||||
|
||||
expect(cmd.strategy).toBe(Strategy.COOKIE);
|
||||
expect(cmd.browser).toBe(true);
|
||||
});
|
||||
|
||||
it('defaults strategy to PUBLIC when browser is false', () => {
|
||||
const cmd = cli({
|
||||
site: 'test-registry',
|
||||
name: 'no-browser',
|
||||
browser: false,
|
||||
});
|
||||
|
||||
expect(cmd.strategy).toBe(Strategy.PUBLIC);
|
||||
});
|
||||
|
||||
it('overwrites existing command on re-registration', () => {
|
||||
cli({ site: 'test-registry', name: 'overwrite', description: 'v1' });
|
||||
cli({ site: 'test-registry', name: 'overwrite', description: 'v2' });
|
||||
|
||||
const reg = getRegistry();
|
||||
expect(reg.get('test-registry/overwrite')?.description).toBe('v2');
|
||||
});
|
||||
});
|
||||
|
||||
describe('fullName', () => {
|
||||
it('returns site/name', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'bilibili', name: 'hot', description: '', args: [],
|
||||
};
|
||||
expect(fullName(cmd)).toBe('bilibili/hot');
|
||||
});
|
||||
});
|
||||
|
||||
describe('strategyLabel', () => {
|
||||
it('returns strategy string', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test', name: 'test', description: '', args: [],
|
||||
strategy: Strategy.INTERCEPT,
|
||||
};
|
||||
expect(strategyLabel(cmd)).toBe('intercept');
|
||||
});
|
||||
|
||||
it('returns public when no strategy set', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test', name: 'test', description: '', args: [],
|
||||
};
|
||||
expect(strategyLabel(cmd)).toBe('public');
|
||||
});
|
||||
});
|
||||
|
||||
describe('registerCommand', () => {
|
||||
it('registers a pre-built command', () => {
|
||||
const cmd: CliCommand = {
|
||||
site: 'test-registry',
|
||||
name: 'direct-reg',
|
||||
description: 'directly registered',
|
||||
args: [],
|
||||
strategy: Strategy.HEADER,
|
||||
browser: true,
|
||||
};
|
||||
registerCommand(cmd);
|
||||
|
||||
const reg = getRegistry();
|
||||
expect(reg.get('test-registry/direct-reg')?.strategy).toBe(Strategy.HEADER);
|
||||
});
|
||||
});
|
||||
+5
-1
@@ -36,6 +36,11 @@ export interface CliCommand {
|
||||
source?: string;
|
||||
}
|
||||
|
||||
/** Internal extension for lazy-loaded TS modules (not exposed in public API) */
|
||||
export interface InternalCliCommand extends CliCommand {
|
||||
_lazy?: boolean;
|
||||
_modulePath?: string;
|
||||
}
|
||||
export interface CliOptions {
|
||||
site: string;
|
||||
name: string;
|
||||
@@ -49,7 +54,6 @@ export interface CliOptions {
|
||||
pipeline?: any[];
|
||||
timeoutSeconds?: number;
|
||||
}
|
||||
|
||||
const _registry = new Map<string, CliCommand>();
|
||||
|
||||
export function cli(opts: CliOptions): CliCommand {
|
||||
|
||||
+22
-8
@@ -9,23 +9,37 @@ export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROW
|
||||
export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
|
||||
export const DEFAULT_BROWSER_SMOKE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_SMOKE_TIMEOUT ?? '60', 10);
|
||||
|
||||
/**
|
||||
* Timeout with seconds unit. Used for high-level command timeouts.
|
||||
*/
|
||||
export async function runWithTimeout<T>(
|
||||
promise: Promise<T>,
|
||||
opts: { timeout: number; label?: string },
|
||||
): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => {
|
||||
reject(new Error(`${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`));
|
||||
}, opts.timeout * 1000);
|
||||
return withTimeoutMs(promise, opts.timeout * 1000, `${opts.label ?? 'Operation'} timed out after ${opts.timeout}s`);
|
||||
}
|
||||
|
||||
promise
|
||||
.then((result) => { clearTimeout(timer); resolve(result); })
|
||||
.catch((err) => { clearTimeout(timer); reject(err); });
|
||||
/**
|
||||
* Timeout with milliseconds unit. Used for low-level internal timeouts.
|
||||
*/
|
||||
export function withTimeoutMs<T>(promise: Promise<T>, timeoutMs: number, message: string): Promise<T> {
|
||||
return new Promise<T>((resolve, reject) => {
|
||||
const timer = setTimeout(() => reject(new Error(message)), timeoutMs);
|
||||
promise.then(
|
||||
(value) => { clearTimeout(timer); resolve(value); },
|
||||
(error) => { clearTimeout(timer); reject(error); },
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
/** Interface for browser factory (PlaywrightMCP or test mocks) */
|
||||
export interface IBrowserFactory {
|
||||
connect(opts?: { timeout?: number }): Promise<IPage>;
|
||||
close(): Promise<void>;
|
||||
}
|
||||
|
||||
export async function browserSession<T>(
|
||||
BrowserFactory: new () => any,
|
||||
BrowserFactory: new () => IBrowserFactory,
|
||||
fn: (page: IPage) => Promise<T>,
|
||||
): Promise<T> {
|
||||
const mcp = new BrowserFactory();
|
||||
|
||||
+187
@@ -0,0 +1,187 @@
|
||||
/**
|
||||
* setup.ts — Interactive Playwright MCP token setup
|
||||
*
|
||||
* Discovers the extension token, shows an interactive checkbox
|
||||
* for selecting which config files to update, and applies changes.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import chalk from 'chalk';
|
||||
import { createInterface } from 'node:readline/promises';
|
||||
import { stdin as input, stdout as output } from 'node:process';
|
||||
import {
|
||||
type DoctorReport,
|
||||
discoverExtensionToken,
|
||||
getDefaultShellRcPath,
|
||||
runBrowserDoctor,
|
||||
upsertJsonConfigToken,
|
||||
upsertShellToken,
|
||||
upsertTomlConfigToken,
|
||||
} from './doctor.js';
|
||||
import { getTokenFingerprint } from './browser.js';
|
||||
import { type CheckboxItem, checkboxPrompt } from './tui.js';
|
||||
|
||||
const PLAYWRIGHT_TOKEN_ENV = 'PLAYWRIGHT_MCP_EXTENSION_TOKEN';
|
||||
|
||||
function fileExists(p: string): boolean {
|
||||
try { return fs.statSync(p).isFile() || fs.statSync(p).isDirectory(); } catch { return false; }
|
||||
}
|
||||
|
||||
function writeFileWithMkdir(filePath: string, content: string) {
|
||||
const dir = filePath.substring(0, filePath.lastIndexOf('/'));
|
||||
if (dir && !fileExists(dir)) fs.mkdirSync(dir, { recursive: true });
|
||||
fs.writeFileSync(filePath, content, 'utf-8');
|
||||
}
|
||||
|
||||
function shortenPath(p: string): string {
|
||||
const home = process.env.HOME || process.env.USERPROFILE || '';
|
||||
return home && p.startsWith(home) ? '~' + p.slice(home.length) : p;
|
||||
}
|
||||
|
||||
function toolName(p: string): string {
|
||||
if (p.includes('.codex/')) return 'Codex';
|
||||
if (p.includes('.cursor/')) return 'Cursor';
|
||||
if (p.includes('.claude.json')) return 'Claude Code';
|
||||
if (p.includes('antigravity')) return 'Antigravity';
|
||||
if (p.includes('.gemini/settings')) return 'Gemini CLI';
|
||||
if (p.includes('opencode')) return 'OpenCode';
|
||||
if (p.includes('Claude/claude_desktop')) return 'Claude Desktop';
|
||||
if (p.includes('.vscode/')) return 'VS Code';
|
||||
if (p.includes('.mcp.json')) return 'Project MCP';
|
||||
if (p.includes('.zshrc') || p.includes('.bashrc') || p.includes('.profile')) return 'Shell';
|
||||
return '';
|
||||
}
|
||||
|
||||
export async function runSetup(opts: { cliVersion?: string; token?: string } = {}) {
|
||||
console.log();
|
||||
console.log(chalk.bold(' opencli setup') + chalk.dim(' — Playwright MCP token configuration'));
|
||||
console.log();
|
||||
|
||||
// Step 1: Discover token
|
||||
let token = opts.token ?? null;
|
||||
|
||||
if (!token) {
|
||||
const extensionToken = discoverExtensionToken();
|
||||
const envToken = process.env[PLAYWRIGHT_TOKEN_ENV] ?? null;
|
||||
|
||||
if (extensionToken && envToken && extensionToken === envToken) {
|
||||
token = extensionToken;
|
||||
console.log(` ${chalk.green('✓')} Token auto-discovered from Chrome extension`);
|
||||
console.log(` Fingerprint: ${chalk.bold(getTokenFingerprint(token) ?? 'unknown')}`);
|
||||
} else if (extensionToken) {
|
||||
token = extensionToken;
|
||||
console.log(` ${chalk.green('✓')} Token discovered from Chrome extension ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
if (envToken && envToken !== extensionToken) {
|
||||
console.log(` ${chalk.yellow('!')} Environment has different token ` +
|
||||
chalk.dim(`(${getTokenFingerprint(envToken)})`));
|
||||
}
|
||||
} else if (envToken) {
|
||||
token = envToken;
|
||||
console.log(` ${chalk.green('✓')} Token from environment variable ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
}
|
||||
} else {
|
||||
console.log(` ${chalk.green('✓')} Using provided token ` +
|
||||
chalk.dim(`(${getTokenFingerprint(token)})`));
|
||||
}
|
||||
|
||||
if (!token) {
|
||||
console.log(` ${chalk.yellow('!')} No token found. Please enter it manually.`);
|
||||
console.log(chalk.dim(' (Find it in the Playwright MCP Bridge extension → Status page)'));
|
||||
console.log();
|
||||
const rl = createInterface({ input, output });
|
||||
const answer = await rl.question(' Token: ');
|
||||
rl.close();
|
||||
token = answer.trim();
|
||||
if (!token) {
|
||||
console.log(chalk.red('\n No token provided. Aborting.\n'));
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
const fingerprint = getTokenFingerprint(token) ?? 'unknown';
|
||||
console.log();
|
||||
|
||||
// Step 2: Scan all config locations
|
||||
const report = await runBrowserDoctor({ token, cliVersion: opts.cliVersion });
|
||||
|
||||
// Step 3: Build checkbox items
|
||||
const items: CheckboxItem[] = [];
|
||||
|
||||
// Shell file
|
||||
const shellPath = report.shellFiles[0]?.path ?? getDefaultShellRcPath();
|
||||
const shellStatus = report.shellFiles[0];
|
||||
const shellFp = shellStatus?.fingerprint;
|
||||
const shellOk = shellFp === fingerprint;
|
||||
items.push({
|
||||
label: padRight(`${shortenPath(shellPath)}`, 50) + chalk.dim(` [${toolName(shellPath) || 'Shell'}]`),
|
||||
value: `shell:${shellPath}`,
|
||||
checked: !shellOk,
|
||||
status: shellOk ? `configured (${shellFp})` : shellFp ? `mismatch (${shellFp})` : 'missing',
|
||||
statusColor: shellOk ? 'green' : shellFp ? 'yellow' : 'red',
|
||||
});
|
||||
|
||||
// Config files
|
||||
for (const config of report.configs) {
|
||||
const fp = config.fingerprint;
|
||||
const ok = fp === fingerprint;
|
||||
const tool = toolName(config.path);
|
||||
items.push({
|
||||
label: padRight(`${shortenPath(config.path)}`, 50) + chalk.dim(tool ? ` [${tool}]` : ''),
|
||||
value: `config:${config.path}`,
|
||||
checked: !ok,
|
||||
status: ok ? `configured (${fp})` : !config.exists ? 'will create' : fp ? `mismatch (${fp})` : 'missing',
|
||||
statusColor: ok ? 'green' : 'yellow',
|
||||
});
|
||||
}
|
||||
|
||||
// Step 4: Show interactive checkbox
|
||||
const selected = await checkboxPrompt(items, {
|
||||
title: ` Select files to update with token ${chalk.cyan(fingerprint)}:`,
|
||||
});
|
||||
|
||||
if (selected.length === 0) {
|
||||
console.log(chalk.dim(' No changes made.\n'));
|
||||
return;
|
||||
}
|
||||
|
||||
// Step 5: Apply changes
|
||||
const written: string[] = [];
|
||||
|
||||
for (const sel of selected) {
|
||||
if (sel.startsWith('shell:')) {
|
||||
const path = sel.slice('shell:'.length);
|
||||
const before = fileExists(path) ? fs.readFileSync(path, 'utf-8') : '';
|
||||
writeFileWithMkdir(path, upsertShellToken(before, token));
|
||||
written.push(path);
|
||||
} else if (sel.startsWith('config:')) {
|
||||
const path = sel.slice('config:'.length);
|
||||
const config = report.configs.find(c => c.path === path);
|
||||
if (config && config.parseError) continue;
|
||||
const before = fileExists(path) ? fs.readFileSync(path, 'utf-8') : '';
|
||||
const format = config?.format ?? (path.endsWith('.toml') ? 'toml' : 'json');
|
||||
const next = format === 'toml' ? upsertTomlConfigToken(before, token) : upsertJsonConfigToken(before, token);
|
||||
writeFileWithMkdir(path, next);
|
||||
written.push(path);
|
||||
}
|
||||
}
|
||||
|
||||
process.env[PLAYWRIGHT_TOKEN_ENV] = token;
|
||||
|
||||
// Step 6: Summary
|
||||
if (written.length > 0) {
|
||||
console.log(chalk.green.bold(` ✓ Updated ${written.length} file(s):`));
|
||||
for (const p of written) {
|
||||
console.log(` ${chalk.dim('•')} ${shortenPath(p)}`);
|
||||
}
|
||||
} else {
|
||||
console.log(chalk.yellow(' No files were changed.'));
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
function padRight(s: string, n: number): string {
|
||||
// Account for ANSI escape codes in length calculation
|
||||
const visible = s.replace(/\x1b\[[0-9;]*m/g, '');
|
||||
return visible.length >= n ? s : s + ' '.repeat(n - visible.length);
|
||||
}
|
||||
+5
-5
@@ -6,12 +6,12 @@
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
import { VOLATILE_PARAMS, SEARCH_PARAMS, LIMIT_PARAMS, PAGINATION_PARAMS } from './constants.js';
|
||||
|
||||
/** Volatile params to strip from generated URLs */
|
||||
const VOLATILE_PARAMS = new Set(['w_rid', 'wts', 'callback', '_', 'timestamp', 't', 'nonce', 'sign']);
|
||||
const SEARCH_PARAM_NAMES = new Set(['q', 'query', 'keyword', 'search', 'wd', 'kw', 'w', 'search_query']);
|
||||
const LIMIT_PARAM_NAMES = new Set(['ps', 'page_size', 'limit', 'count', 'per_page', 'size', 'num']);
|
||||
const PAGE_PARAM_NAMES = new Set(['pn', 'page', 'page_num', 'offset', 'cursor']);
|
||||
/** Renamed aliases for backward compatibility with local references */
|
||||
const SEARCH_PARAM_NAMES = SEARCH_PARAMS;
|
||||
const LIMIT_PARAM_NAMES = LIMIT_PARAMS;
|
||||
const PAGE_PARAM_NAMES = PAGINATION_PARAMS;
|
||||
|
||||
export function synthesizeFromExplore(
|
||||
target: string,
|
||||
|
||||
+165
@@ -0,0 +1,165 @@
|
||||
/**
|
||||
* tui.ts — Zero-dependency interactive TUI components
|
||||
*
|
||||
* Uses raw stdin mode + ANSI escape codes for interactive prompts.
|
||||
*/
|
||||
import chalk from 'chalk';
|
||||
|
||||
export interface CheckboxItem {
|
||||
label: string;
|
||||
value: string;
|
||||
checked: boolean;
|
||||
/** Optional status to display after the label */
|
||||
status?: string;
|
||||
statusColor?: 'green' | 'yellow' | 'red' | 'dim';
|
||||
}
|
||||
|
||||
/**
|
||||
* Interactive multi-select checkbox prompt.
|
||||
*
|
||||
* Controls:
|
||||
* ↑/↓ or j/k — navigate
|
||||
* Space — toggle selection
|
||||
* a — toggle all
|
||||
* Enter — confirm
|
||||
* q/Esc — cancel (returns empty)
|
||||
*/
|
||||
export async function checkboxPrompt(
|
||||
items: CheckboxItem[],
|
||||
opts: { title?: string; hint?: string } = {},
|
||||
): Promise<string[]> {
|
||||
if (items.length === 0) return [];
|
||||
|
||||
const { stdin, stdout } = process;
|
||||
if (!stdin.isTTY) {
|
||||
// Non-interactive: return all checked items
|
||||
return items.filter(i => i.checked).map(i => i.value);
|
||||
}
|
||||
|
||||
let cursor = 0;
|
||||
const state = items.map(i => ({ ...i }));
|
||||
|
||||
function colorStatus(status: string | undefined, color: CheckboxItem['statusColor']): string {
|
||||
if (!status) return '';
|
||||
switch (color) {
|
||||
case 'green': return chalk.green(status);
|
||||
case 'yellow': return chalk.yellow(status);
|
||||
case 'red': return chalk.red(status);
|
||||
case 'dim': return chalk.dim(status);
|
||||
default: return chalk.dim(status);
|
||||
}
|
||||
}
|
||||
|
||||
function render() {
|
||||
// Move cursor to start and clear
|
||||
let out = '';
|
||||
|
||||
if (opts.title) {
|
||||
out += `\n${chalk.bold(opts.title)}\n\n`;
|
||||
}
|
||||
|
||||
for (let i = 0; i < state.length; i++) {
|
||||
const item = state[i];
|
||||
const pointer = i === cursor ? chalk.cyan('❯') : ' ';
|
||||
const checkbox = item.checked ? chalk.green('◉') : chalk.dim('○');
|
||||
const label = i === cursor ? chalk.bold(item.label) : item.label;
|
||||
const status = colorStatus(item.status, item.statusColor);
|
||||
out += ` ${pointer} ${checkbox} ${label}${status ? ` ${status}` : ''}\n`;
|
||||
}
|
||||
|
||||
out += `\n ${chalk.dim('↑↓ navigate · Space toggle · a all · Enter confirm · q cancel')}\n`;
|
||||
|
||||
return out;
|
||||
}
|
||||
|
||||
return new Promise<string[]>((resolve) => {
|
||||
const wasRaw = stdin.isRaw;
|
||||
stdin.setRawMode(true);
|
||||
stdin.resume();
|
||||
|
||||
let rendered = '';
|
||||
|
||||
function draw() {
|
||||
// Clear previous render
|
||||
if (rendered) {
|
||||
const lines = rendered.split('\n').length;
|
||||
stdout.write(`\x1b[${lines}A\x1b[J`);
|
||||
}
|
||||
rendered = render();
|
||||
stdout.write(rendered);
|
||||
}
|
||||
|
||||
function cleanup() {
|
||||
stdin.setRawMode(wasRaw ?? false);
|
||||
stdin.pause();
|
||||
stdin.removeListener('data', onData);
|
||||
// Clear the TUI
|
||||
if (rendered) {
|
||||
const lines = rendered.split('\n').length;
|
||||
stdout.write(`\x1b[${lines}A\x1b[J`);
|
||||
}
|
||||
}
|
||||
|
||||
function onData(data: Buffer) {
|
||||
const key = data.toString();
|
||||
|
||||
// Arrow up / k
|
||||
if (key === '\x1b[A' || key === 'k') {
|
||||
cursor = (cursor - 1 + state.length) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Arrow down / j
|
||||
if (key === '\x1b[B' || key === 'j') {
|
||||
cursor = (cursor + 1) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Space — toggle
|
||||
if (key === ' ') {
|
||||
state[cursor].checked = !state[cursor].checked;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Tab — toggle and move down
|
||||
if (key === '\t') {
|
||||
state[cursor].checked = !state[cursor].checked;
|
||||
cursor = (cursor + 1) % state.length;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// 'a' — toggle all
|
||||
if (key === 'a') {
|
||||
const allChecked = state.every(i => i.checked);
|
||||
for (const item of state) item.checked = !allChecked;
|
||||
draw();
|
||||
return;
|
||||
}
|
||||
|
||||
// Enter — confirm
|
||||
if (key === '\r' || key === '\n') {
|
||||
cleanup();
|
||||
const selected = state.filter(i => i.checked).map(i => i.value);
|
||||
// Show summary
|
||||
stdout.write(` ${chalk.green('✓')} ${chalk.bold(`${selected.length} file(s) selected`)}\n\n`);
|
||||
resolve(selected);
|
||||
return;
|
||||
}
|
||||
|
||||
// q / Esc / Ctrl+C — cancel
|
||||
if (key === 'q' || key === '\x1b' || key === '\x03') {
|
||||
cleanup();
|
||||
stdout.write(` ${chalk.yellow('✗')} ${chalk.dim('Cancelled')}\n\n`);
|
||||
resolve([]);
|
||||
return;
|
||||
}
|
||||
}
|
||||
|
||||
stdin.on('data', onData);
|
||||
draw();
|
||||
});
|
||||
}
|
||||
+4
-1
@@ -12,7 +12,7 @@ export interface IPage {
|
||||
click(ref: string): Promise<void>;
|
||||
typeText(ref: string, text: string): Promise<void>;
|
||||
pressKey(key: string): Promise<void>;
|
||||
wait(seconds: number): Promise<void>;
|
||||
wait(options: number | { text?: string; time?: number; timeout?: number }): Promise<void>;
|
||||
tabs(): Promise<any>;
|
||||
closeTab(index?: number): Promise<void>;
|
||||
newTab(): Promise<void>;
|
||||
@@ -20,4 +20,7 @@ export interface IPage {
|
||||
networkRequests(includeStatic?: boolean): Promise<any>;
|
||||
consoleMessages(level?: string): Promise<any>;
|
||||
scroll(direction?: string, amount?: number): Promise<void>;
|
||||
autoScroll(options?: { times?: number; delayMs?: number }): Promise<void>;
|
||||
installInterceptor(pattern: string): Promise<void>;
|
||||
getInterceptedRequests(): Promise<any[]>;
|
||||
}
|
||||
|
||||
@@ -3,6 +3,14 @@ import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import yaml from 'js-yaml';
|
||||
|
||||
/** All recognized pipeline step names */
|
||||
const KNOWN_STEP_NAMES = new Set([
|
||||
'navigate', 'click', 'type', 'wait', 'press', 'snapshot', 'scroll',
|
||||
'fetch', 'evaluate',
|
||||
'select', 'map', 'filter', 'sort', 'limit',
|
||||
'intercept', 'tap',
|
||||
]);
|
||||
|
||||
export function validateClisWithTarget(dirs: string[], target?: string): any {
|
||||
const results: any[] = [];
|
||||
let errors = 0; let warnings = 0; let files = 0;
|
||||
@@ -38,6 +46,20 @@ function validateYamlFile(filePath: string): any {
|
||||
if (def.pipeline && !Array.isArray(def.pipeline)) errors.push('"pipeline" must be an array');
|
||||
if (def.columns && !Array.isArray(def.columns)) errors.push('"columns" must be an array');
|
||||
if (def.args && typeof def.args !== 'object') errors.push('"args" must be an object');
|
||||
// Validate pipeline step names (catch typos like 'navaigate')
|
||||
if (Array.isArray(def.pipeline)) {
|
||||
for (let i = 0; i < def.pipeline.length; i++) {
|
||||
const step = def.pipeline[i];
|
||||
if (step && typeof step === 'object') {
|
||||
const stepKeys = Object.keys(step);
|
||||
for (const key of stepKeys) {
|
||||
if (!KNOWN_STEP_NAMES.has(key)) {
|
||||
warnings.push(`Pipeline step ${i}: unknown step name "${key}" (did you mean one of: ${[...KNOWN_STEP_NAMES].join(', ')}?)`);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} catch (e: any) { errors.push(`YAML parse error: ${e.message}`); }
|
||||
return { path: filePath, errors, warnings };
|
||||
}
|
||||
|
||||
+10
-1
@@ -1,9 +1,18 @@
|
||||
/** Verification: validate + smoke. */
|
||||
/**
|
||||
* Verification: runs validation and optional smoke test.
|
||||
*
|
||||
* The smoke test is intentionally kept as a stub — full browser-based
|
||||
* smoke testing requires a running browser session and is better suited
|
||||
* to the `opencli test` command or CI pipelines.
|
||||
*/
|
||||
|
||||
import { validateClisWithTarget, renderValidationReport } from './validate.js';
|
||||
|
||||
export async function verifyClis(opts: any): Promise<any> {
|
||||
const report = validateClisWithTarget([opts.builtinClis, opts.userClis], opts.target);
|
||||
return { ok: report.ok, validation: report, smoke: null };
|
||||
}
|
||||
|
||||
export function renderVerifyReport(report: any): string {
|
||||
return renderValidationReport(report.validation);
|
||||
}
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
/**
|
||||
* Single source of truth for package version.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const pkgJsonPath = path.resolve(__dirname, '..', 'package.json');
|
||||
|
||||
export const PKG_VERSION: string = (() => {
|
||||
try {
|
||||
return JSON.parse(fs.readFileSync(pkgJsonPath, 'utf-8')).version;
|
||||
} catch {
|
||||
return '0.0.0';
|
||||
}
|
||||
})();
|
||||
@@ -0,0 +1,90 @@
|
||||
/**
|
||||
* E2E tests for login-required browser commands.
|
||||
* These commands REQUIRE authentication (cookie/session).
|
||||
* In CI (headless, no login), they should fail gracefully — NOT crash.
|
||||
*
|
||||
* These tests verify the error handling path, not the data extraction.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli } from './helpers.js';
|
||||
|
||||
/**
|
||||
* Verify a login-required command fails gracefully (no crash, no hang).
|
||||
* Acceptable outcomes: exit code 1 with error message, OR timeout handled.
|
||||
*/
|
||||
async function expectGracefulAuthFailure(args: string[], label: string) {
|
||||
const { stdout, stderr, code } = await runCli(args, { timeout: 60_000 });
|
||||
// Should either fail with exit code 1 (error message) or succeed with empty data
|
||||
// The key assertion: it should NOT hang forever or crash with unhandled exception
|
||||
if (code !== 0) {
|
||||
// Verify stderr has a meaningful error, not an unhandled crash
|
||||
const output = stderr + stdout;
|
||||
expect(output.length).toBeGreaterThan(0);
|
||||
}
|
||||
// If it somehow succeeds (e.g., partial public data), that's fine too
|
||||
}
|
||||
|
||||
describe('login-required commands — graceful failure', () => {
|
||||
|
||||
// ── bilibili (requires cookie session) ──
|
||||
it('bilibili me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'me', '-f', 'json'], 'bilibili me');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili dynamic fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'dynamic', '--limit', '3', '-f', 'json'], 'bilibili dynamic');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili favorite fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'favorite', '--limit', '3', '-f', 'json'], 'bilibili favorite');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili history fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'history', '--limit', '3', '-f', 'json'], 'bilibili history');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili following fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['bilibili', 'following', '--limit', '3', '-f', 'json'], 'bilibili following');
|
||||
}, 60_000);
|
||||
|
||||
// ── twitter (requires login) ──
|
||||
it('twitter bookmarks fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'bookmarks', '--limit', '3', '-f', 'json'], 'twitter bookmarks');
|
||||
}, 60_000);
|
||||
|
||||
it('twitter timeline fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'timeline', '--limit', '3', '-f', 'json'], 'twitter timeline');
|
||||
}, 60_000);
|
||||
|
||||
it('twitter notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['twitter', 'notifications', '--limit', '3', '-f', 'json'], 'twitter notifications');
|
||||
}, 60_000);
|
||||
|
||||
// ── v2ex (requires login) ──
|
||||
it('v2ex me fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['v2ex', 'me', '-f', 'json'], 'v2ex me');
|
||||
}, 60_000);
|
||||
|
||||
it('v2ex notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['v2ex', 'notifications', '--limit', '3', '-f', 'json'], 'v2ex notifications');
|
||||
}, 60_000);
|
||||
|
||||
// ── xueqiu (requires login) ──
|
||||
it('xueqiu feed fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xueqiu', 'feed', '--limit', '3', '-f', 'json'], 'xueqiu feed');
|
||||
}, 60_000);
|
||||
|
||||
it('xueqiu watchlist fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xueqiu', 'watchlist', '-f', 'json'], 'xueqiu watchlist');
|
||||
}, 60_000);
|
||||
|
||||
// ── xiaohongshu (requires login) ──
|
||||
it('xiaohongshu feed fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xiaohongshu', 'feed', '--limit', '3', '-f', 'json'], 'xiaohongshu feed');
|
||||
}, 60_000);
|
||||
|
||||
it('xiaohongshu notifications fails gracefully without login', async () => {
|
||||
await expectGracefulAuthFailure(['xiaohongshu', 'notifications', '--limit', '3', '-f', 'json'], 'xiaohongshu notifications');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* E2E tests for browser commands that access PUBLIC data (no login required).
|
||||
* These use OPENCLI_HEADLESS=1 to launch a headless Chromium.
|
||||
*
|
||||
* NOTE: Some sites may block headless browsers with bot detection.
|
||||
* Tests are wrapped with tryBrowserCommand() which allows graceful failure.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
/**
|
||||
* Run a browser command — returns parsed data or null on failure.
|
||||
*/
|
||||
async function tryBrowserCommand(args: string[]): Promise<any[] | null> {
|
||||
const { stdout, code } = await runCli(args, { timeout: 60_000 });
|
||||
if (code !== 0) return null;
|
||||
try {
|
||||
const data = parseJsonOutput(stdout);
|
||||
return Array.isArray(data) ? data : null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Assert browser command returns data OR log a warning if blocked.
|
||||
* Empty results (bot detection, geo-blocking) are treated as a warning, not a failure.
|
||||
*/
|
||||
function expectDataOrSkip(data: any[] | null, label: string) {
|
||||
if (data === null || data.length === 0) {
|
||||
console.warn(`${label}: skipped — no data returned (likely bot detection or geo-blocking)`);
|
||||
return;
|
||||
}
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}
|
||||
|
||||
describe('browser public-data commands E2E', () => {
|
||||
|
||||
// ── bbc (browser: true, strategy: public) ──
|
||||
it('bbc news returns headlines', async () => {
|
||||
const data = await tryBrowserCommand(['bbc', 'news', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bbc news');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
// ── v2ex daily (browser: true) ──
|
||||
it('v2ex daily returns topics', async () => {
|
||||
const data = await tryBrowserCommand(['v2ex', 'daily', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'v2ex daily');
|
||||
}, 60_000);
|
||||
|
||||
// ── bilibili (browser: true, cookie strategy) ──
|
||||
it('bilibili hot returns trending videos', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili hot');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili ranking returns ranked videos', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'ranking', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili ranking');
|
||||
}, 60_000);
|
||||
|
||||
it('bilibili search returns results', async () => {
|
||||
const data = await tryBrowserCommand(['bilibili', 'search', 'typescript', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'bilibili search');
|
||||
}, 60_000);
|
||||
|
||||
// ── weibo (browser: true, cookie strategy) ──
|
||||
it('weibo hot returns trending topics', async () => {
|
||||
const data = await tryBrowserCommand(['weibo', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'weibo hot');
|
||||
}, 60_000);
|
||||
|
||||
// ── zhihu (browser: true, cookie strategy) ──
|
||||
it('zhihu hot returns trending questions', async () => {
|
||||
const data = await tryBrowserCommand(['zhihu', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'zhihu hot');
|
||||
if (data) {
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}
|
||||
}, 60_000);
|
||||
|
||||
it('zhihu search returns results', async () => {
|
||||
const data = await tryBrowserCommand(['zhihu', 'search', 'playwright', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'zhihu search');
|
||||
}, 60_000);
|
||||
|
||||
// ── reddit (browser: true, cookie strategy) ──
|
||||
it('reddit hot returns posts', async () => {
|
||||
const data = await tryBrowserCommand(['reddit', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reddit hot');
|
||||
}, 60_000);
|
||||
|
||||
it('reddit frontpage returns posts', async () => {
|
||||
const data = await tryBrowserCommand(['reddit', 'frontpage', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reddit frontpage');
|
||||
}, 60_000);
|
||||
|
||||
// ── twitter (browser: true) ──
|
||||
it('twitter trending returns trends', async () => {
|
||||
const data = await tryBrowserCommand(['twitter', 'trending', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'twitter trending');
|
||||
}, 60_000);
|
||||
|
||||
// ── xueqiu (browser: true, cookie strategy) ──
|
||||
it('xueqiu hot returns hot posts', async () => {
|
||||
const data = await tryBrowserCommand(['xueqiu', 'hot', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xueqiu hot');
|
||||
}, 60_000);
|
||||
|
||||
it('xueqiu hot-stock returns stocks', async () => {
|
||||
const data = await tryBrowserCommand(['xueqiu', 'hot-stock', '--limit', '5', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xueqiu hot-stock');
|
||||
}, 60_000);
|
||||
|
||||
// ── reuters (browser: true) ──
|
||||
it('reuters search returns articles', async () => {
|
||||
const data = await tryBrowserCommand(['reuters', 'search', 'technology', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'reuters search');
|
||||
}, 60_000);
|
||||
|
||||
// ── youtube (browser: true) ──
|
||||
it('youtube search returns videos', async () => {
|
||||
const data = await tryBrowserCommand(['youtube', 'search', 'typescript tutorial', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'youtube search');
|
||||
}, 60_000);
|
||||
|
||||
// ── smzdm (browser: true) ──
|
||||
it('smzdm search returns deals', async () => {
|
||||
const data = await tryBrowserCommand(['smzdm', 'search', '键盘', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'smzdm search');
|
||||
}, 60_000);
|
||||
|
||||
// ── boss (browser: true) ──
|
||||
it('boss search returns jobs', async () => {
|
||||
const data = await tryBrowserCommand(['boss', 'search', 'golang', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'boss search');
|
||||
}, 60_000);
|
||||
|
||||
// ── ctrip (browser: true) ──
|
||||
it('ctrip search returns flights', async () => {
|
||||
const data = await tryBrowserCommand(['ctrip', 'search', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'ctrip search');
|
||||
}, 60_000);
|
||||
|
||||
// ── coupang (browser: true) ──
|
||||
it('coupang search returns products', async () => {
|
||||
const data = await tryBrowserCommand(['coupang', 'search', 'laptop', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'coupang search');
|
||||
}, 60_000);
|
||||
|
||||
// ── xiaohongshu (browser: true) ──
|
||||
it('xiaohongshu search returns notes', async () => {
|
||||
const data = await tryBrowserCommand(['xiaohongshu', 'search', '美食', '--limit', '3', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'xiaohongshu search');
|
||||
}, 60_000);
|
||||
|
||||
// ── yahoo-finance (browser: true) ──
|
||||
it('yahoo-finance quote returns stock data', async () => {
|
||||
const data = await tryBrowserCommand(['yahoo-finance', 'quote', '--symbol', 'AAPL', '-f', 'json']);
|
||||
expectDataOrSkip(data, 'yahoo-finance quote');
|
||||
}, 60_000);
|
||||
});
|
||||
@@ -0,0 +1,63 @@
|
||||
/**
|
||||
* Shared helpers for E2E tests.
|
||||
* Runs the built opencli binary as a subprocess.
|
||||
*/
|
||||
|
||||
import { execFile } from 'node:child_process';
|
||||
import { promisify } from 'node:util';
|
||||
import * as path from 'node:path';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
|
||||
const exec = promisify(execFile);
|
||||
const __dirname = path.dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = path.resolve(__dirname, '../..');
|
||||
const MAIN = path.join(ROOT, 'dist', 'main.js');
|
||||
|
||||
export interface CliResult {
|
||||
stdout: string;
|
||||
stderr: string;
|
||||
code: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run `opencli` as a child process with the given arguments.
|
||||
* Without PLAYWRIGHT_MCP_EXTENSION_TOKEN, opencli auto-launches its own browser.
|
||||
*/
|
||||
export async function runCli(
|
||||
args: string[],
|
||||
opts: { timeout?: number; env?: Record<string, string> } = {},
|
||||
): Promise<CliResult> {
|
||||
const timeout = opts.timeout ?? 30_000;
|
||||
try {
|
||||
const { stdout, stderr } = await exec('node', [MAIN, ...args], {
|
||||
cwd: ROOT,
|
||||
timeout,
|
||||
env: {
|
||||
...process.env,
|
||||
// Prevent chalk colors from polluting test assertions
|
||||
FORCE_COLOR: '0',
|
||||
NO_COLOR: '1',
|
||||
...opts.env,
|
||||
},
|
||||
});
|
||||
return { stdout, stderr, code: 0 };
|
||||
} catch (err: any) {
|
||||
return {
|
||||
stdout: err.stdout ?? '',
|
||||
stderr: err.stderr ?? '',
|
||||
code: err.code ?? 1,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse JSON output from a CLI command.
|
||||
* Throws a descriptive error if parsing fails.
|
||||
*/
|
||||
export function parseJsonOutput(stdout: string): any {
|
||||
try {
|
||||
return JSON.parse(stdout.trim());
|
||||
} catch {
|
||||
throw new Error(`Failed to parse CLI JSON output:\n${stdout.slice(0, 500)}`);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
/**
|
||||
* E2E tests for management/built-in commands.
|
||||
* These commands require no external network access (except verify --smoke).
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
describe('management commands E2E', () => {
|
||||
|
||||
// ── list ──
|
||||
it('list shows all registered commands', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
// Should have 50+ commands across 18 sites
|
||||
expect(data.length).toBeGreaterThan(50);
|
||||
// Each entry should have the standard fields
|
||||
expect(data[0]).toHaveProperty('command');
|
||||
expect(data[0]).toHaveProperty('site');
|
||||
expect(data[0]).toHaveProperty('name');
|
||||
expect(data[0]).toHaveProperty('strategy');
|
||||
expect(data[0]).toHaveProperty('browser');
|
||||
});
|
||||
|
||||
it('list default table format renders sites', async () => {
|
||||
const { stdout, code } = await runCli(['list']);
|
||||
expect(code).toBe(0);
|
||||
// Should contain site names
|
||||
expect(stdout).toContain('hackernews');
|
||||
expect(stdout).toContain('bilibili');
|
||||
expect(stdout).toContain('twitter');
|
||||
expect(stdout).toContain('commands across');
|
||||
});
|
||||
|
||||
it('list -f yaml produces valid yaml', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'yaml']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('command:');
|
||||
expect(stdout).toContain('site:');
|
||||
});
|
||||
|
||||
it('list -f csv produces valid csv', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'csv']);
|
||||
expect(code).toBe(0);
|
||||
const lines = stdout.trim().split('\n');
|
||||
expect(lines.length).toBeGreaterThan(50);
|
||||
});
|
||||
|
||||
it('list -f md produces markdown table', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'md']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('|');
|
||||
expect(stdout).toContain('command');
|
||||
});
|
||||
|
||||
// ── validate ──
|
||||
it('validate passes for all built-in adapters', async () => {
|
||||
const { stdout, code } = await runCli(['validate']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
expect(stdout).not.toContain('❌');
|
||||
});
|
||||
|
||||
it('validate works for specific site', async () => {
|
||||
const { stdout, code } = await runCli(['validate', 'hackernews']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
it('validate works for specific command', async () => {
|
||||
const { stdout, code } = await runCli(['validate', 'hackernews/top']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── verify ──
|
||||
it('verify runs validation without smoke tests', async () => {
|
||||
const { stdout, code } = await runCli(['verify']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── version ──
|
||||
it('--version shows version number', async () => {
|
||||
const { stdout, code } = await runCli(['--version']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout.trim()).toMatch(/^\d+\.\d+\.\d+$/);
|
||||
});
|
||||
|
||||
// ── help ──
|
||||
it('--help shows usage', async () => {
|
||||
const { stdout, code } = await runCli(['--help']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('opencli');
|
||||
expect(stdout).toContain('list');
|
||||
expect(stdout).toContain('validate');
|
||||
});
|
||||
|
||||
// ── unknown command ──
|
||||
it('unknown command shows error', async () => {
|
||||
const { stderr, code } = await runCli(['nonexistent-command-xyz']);
|
||||
expect(code).toBe(1);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,48 @@
|
||||
/**
|
||||
* E2E tests for output format rendering.
|
||||
* Uses hackernews (public, fast) as a stable data source.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
const FORMATS = ['json', 'yaml', 'csv', 'md'] as const;
|
||||
|
||||
describe('output formats E2E', () => {
|
||||
for (const fmt of FORMATS) {
|
||||
it(`hackernews top -f ${fmt} produces valid output`, async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '2', '-f', fmt]);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout.trim().length).toBeGreaterThan(0);
|
||||
|
||||
if (fmt === 'json') {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBe(2);
|
||||
}
|
||||
|
||||
if (fmt === 'yaml') {
|
||||
expect(stdout).toContain('title:');
|
||||
}
|
||||
|
||||
if (fmt === 'csv') {
|
||||
// CSV should have a header row + data rows
|
||||
const lines = stdout.trim().split('\n');
|
||||
expect(lines.length).toBeGreaterThanOrEqual(2);
|
||||
}
|
||||
|
||||
if (fmt === 'md') {
|
||||
// Markdown table should have pipe characters
|
||||
expect(stdout).toContain('|');
|
||||
}
|
||||
}, 30_000);
|
||||
}
|
||||
|
||||
it('list -f csv produces valid csv', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'csv']);
|
||||
expect(code).toBe(0);
|
||||
const lines = stdout.trim().split('\n');
|
||||
// Header + many data lines
|
||||
expect(lines.length).toBeGreaterThan(50);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,56 @@
|
||||
/**
|
||||
* E2E tests for public API commands (browser: false).
|
||||
* These commands use Node.js fetch directly — no browser needed.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from './helpers.js';
|
||||
|
||||
describe('public commands E2E', () => {
|
||||
// ── hackernews ──
|
||||
it('hackernews top returns structured data', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBe(3);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
expect(data[0]).toHaveProperty('score');
|
||||
expect(data[0]).toHaveProperty('rank');
|
||||
}, 30_000);
|
||||
|
||||
it('hackernews top respects --limit', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '1', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBe(1);
|
||||
}, 30_000);
|
||||
|
||||
// ── v2ex (public API, browser: false) ──
|
||||
it('v2ex hot returns topics', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'hot', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex latest returns topics', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'latest', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(Array.isArray(data)).toBe(true);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex topic returns topic detail', async () => {
|
||||
// Topic 1000001 is a well-known V2EX topic
|
||||
const { stdout, code } = await runCli(['v2ex', 'topic', '--id', '1000001', '-f', 'json']);
|
||||
// May fail if V2EX rate-limits, but should return structured data
|
||||
if (code === 0) {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
});
|
||||
@@ -0,0 +1,72 @@
|
||||
/**
|
||||
* Smoke tests for external API health.
|
||||
* Only run on schedule or manual dispatch — NOT on every push/PR.
|
||||
* These verify that external APIs haven't changed their structure.
|
||||
*/
|
||||
|
||||
import { describe, it, expect } from 'vitest';
|
||||
import { runCli, parseJsonOutput } from '../e2e/helpers.js';
|
||||
|
||||
describe('API health smoke tests', () => {
|
||||
|
||||
// ── Public API commands (should always work) ──
|
||||
it('hackernews API is responsive and returns expected structure', async () => {
|
||||
const { stdout, code } = await runCli(['hackernews', 'top', '--limit', '5', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBe(5);
|
||||
for (const item of data) {
|
||||
expect(item).toHaveProperty('title');
|
||||
expect(item).toHaveProperty('score');
|
||||
expect(item).toHaveProperty('author');
|
||||
expect(item).toHaveProperty('rank');
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex hot API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'hot', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
expect(data[0]).toHaveProperty('title');
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex latest API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'latest', '--limit', '3', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data.length).toBeGreaterThanOrEqual(1);
|
||||
}, 30_000);
|
||||
|
||||
it('v2ex topic API is responsive', async () => {
|
||||
const { stdout, code } = await runCli(['v2ex', 'topic', '--id', '1000001', '-f', 'json']);
|
||||
if (code === 0) {
|
||||
const data = parseJsonOutput(stdout);
|
||||
expect(data).toBeDefined();
|
||||
}
|
||||
}, 30_000);
|
||||
|
||||
// ── Validate all adapters ──
|
||||
it('all adapter definitions are valid', async () => {
|
||||
const { stdout, code } = await runCli(['validate']);
|
||||
expect(code).toBe(0);
|
||||
expect(stdout).toContain('PASS');
|
||||
});
|
||||
|
||||
// ── Command registry integrity ──
|
||||
it('all expected sites are registered', async () => {
|
||||
const { stdout, code } = await runCli(['list', '-f', 'json']);
|
||||
expect(code).toBe(0);
|
||||
const data = parseJsonOutput(stdout);
|
||||
const sites = new Set(data.map((d: any) => d.site));
|
||||
// Verify all 17 sites are present
|
||||
for (const expected of [
|
||||
'hackernews', 'bbc', 'bilibili', 'v2ex', 'weibo', 'zhihu',
|
||||
'twitter', 'reddit', 'xueqiu', 'reuters', 'youtube',
|
||||
'smzdm', 'boss', 'ctrip', 'coupang', 'xiaohongshu',
|
||||
'yahoo-finance',
|
||||
]) {
|
||||
expect(sites.has(expected)).toBe(true);
|
||||
}
|
||||
});
|
||||
});
|
||||
Reference in New Issue
Block a user