Compare commits
2 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 jackwener
|
de633ef754 | ||
|
jackwener
|
02c00e8595 |
@@ -3,7 +3,7 @@ name: Build Chrome Extension
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
tags: [ "ext-v*" ]
|
||||
tags: [ "v*.*.*" ]
|
||||
paths:
|
||||
- 'extension/**'
|
||||
- '.github/workflows/build-extension.yml'
|
||||
@@ -26,7 +26,7 @@ jobs:
|
||||
- name: Setup Node.js
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 22
|
||||
node-version: 20
|
||||
cache: 'npm'
|
||||
cache-dependency-path: extension/package-lock.json
|
||||
|
||||
@@ -44,22 +44,23 @@ jobs:
|
||||
|
||||
- name: Create Extension ZIP
|
||||
run: |
|
||||
EXT_VERSION=$(node -p "require('./extension/package.json').version")
|
||||
cd extension-package
|
||||
zip -r ../opencli-extension-v${EXT_VERSION}.zip .
|
||||
zip -r ../opencli-extension.zip .
|
||||
|
||||
- name: Upload Artifacts (Action Run)
|
||||
uses: actions/upload-artifact@v7
|
||||
with:
|
||||
name: opencli-extension-build
|
||||
path: opencli-extension-v*.zip
|
||||
path: |
|
||||
opencli-extension.zip
|
||||
retention-days: 7
|
||||
|
||||
- name: Attach to GitHub Release
|
||||
if: startsWith(github.ref, 'refs/tags/')
|
||||
uses: softprops/action-gh-release@v3.0.0
|
||||
uses: softprops/action-gh-release@v2.6.1
|
||||
with:
|
||||
files: opencli-extension-v*.zip
|
||||
files: |
|
||||
opencli-extension.zip
|
||||
draft: false
|
||||
prerelease: false
|
||||
env:
|
||||
|
||||
@@ -38,18 +38,6 @@ jobs:
|
||||
- name: Build
|
||||
run: npm run build
|
||||
|
||||
# Guard: committed cli-manifest.json must match the one build regenerates.
|
||||
# Prevents silent drift where unrelated adapter entries vanish or change
|
||||
# across PRs (agent hits unexpected manifest diff → surgical-merge churn).
|
||||
- name: Check cli-manifest.json is up-to-date
|
||||
if: runner.os == 'Linux'
|
||||
shell: bash
|
||||
run: |
|
||||
if ! git diff --exit-code -- cli-manifest.json; then
|
||||
echo "::error::cli-manifest.json is out of sync with the source. Run 'npm run build' and commit the result."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# ── Unit tests (vitest shard) ──
|
||||
# PR: ubuntu + Node 22 only (fast feedback, 2 jobs).
|
||||
# Push to main/dev: full matrix for cross-platform/cross-version coverage (12 jobs).
|
||||
@@ -59,7 +47,7 @@ jobs:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
os: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('["ubuntu-latest","macos-latest","windows-latest"]') || fromJSON('["ubuntu-latest"]') }}
|
||||
node-version: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('["22"]') || fromJSON('["22"]') }}
|
||||
node-version: ${{ (github.event_name == 'push' || github.event_name == 'schedule' || github.event_name == 'workflow_dispatch') && fromJSON('["20","22"]') || fromJSON('["22"]') }}
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- uses: actions/checkout@v6
|
||||
@@ -73,7 +61,7 @@ jobs:
|
||||
run: npm ci
|
||||
|
||||
- name: Run unit tests (Node ${{ matrix.node-version }}, shard ${{ matrix.shard }}/2)
|
||||
run: npx vitest run --project unit --project extension --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
run: npm test -- --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
|
||||
# ── Bun compatibility check ──
|
||||
bun-test:
|
||||
@@ -148,8 +136,12 @@ jobs:
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/smoke/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
|
||||
- name: Run smoke tests (macOS / Windows)
|
||||
if: runner.os != 'Linux'
|
||||
run: npx vitest run tests/smoke/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
timeout-minutes: 15
|
||||
|
||||
@@ -64,7 +64,11 @@ jobs:
|
||||
run: |
|
||||
xvfb-run --auto-servernum --server-args="-screen 0 1280x720x24" \
|
||||
npx vitest run tests/e2e/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
|
||||
- name: Run E2E tests (macOS / Windows)
|
||||
if: runner.os != 'Linux'
|
||||
run: npx vitest run tests/e2e/ --reporter=verbose
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
|
||||
@@ -26,41 +26,10 @@ jobs:
|
||||
- name: Type check
|
||||
run: npx tsc --noEmit
|
||||
|
||||
# Build before the manifest drift gate: adapter modules import
|
||||
# @jackwener/opencli/* through package exports, which resolve to dist/.
|
||||
# A fresh release checkout has no dist/ until the full build runs.
|
||||
- name: Build package and verify cli-manifest.json is up-to-date
|
||||
run: |
|
||||
npm run build
|
||||
if ! git diff --exit-code -- cli-manifest.json; then
|
||||
echo "::error::cli-manifest.json drift detected at release time. Run 'npm run build' locally and commit the result before tagging."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
- name: Install extension dependencies
|
||||
run: npm ci
|
||||
working-directory: extension
|
||||
|
||||
- name: Build extension
|
||||
run: npm run build
|
||||
working-directory: extension
|
||||
|
||||
- name: Package extension
|
||||
run: npm run package:release -- --out ../extension-package
|
||||
working-directory: extension
|
||||
|
||||
- name: Create extension ZIP
|
||||
run: |
|
||||
EXT_VERSION=$(jq -r .version extension/package.json)
|
||||
cd extension-package
|
||||
zip -r ../opencli-extension-v${EXT_VERSION}.zip .
|
||||
|
||||
- name: Create GitHub Release
|
||||
uses: softprops/action-gh-release@v3.0.0
|
||||
uses: softprops/action-gh-release@v2.6.1
|
||||
with:
|
||||
generate_release_notes: true
|
||||
files: |
|
||||
opencli-extension-v*.zip
|
||||
|
||||
- name: Publish to npm
|
||||
run: npm publish --provenance --access public
|
||||
|
||||
@@ -3,7 +3,6 @@ dist/
|
||||
!extension/dist/
|
||||
*.tsbuildinfo
|
||||
.opencli/
|
||||
.worktrees/
|
||||
.mcp.json
|
||||
*.log
|
||||
.DS_Store
|
||||
|
||||
-216
@@ -1,221 +1,5 @@
|
||||
# Changelog
|
||||
|
||||
## Unreleased
|
||||
|
||||
### Features
|
||||
|
||||
* **observation** — add trace artifact primitives, `browser console`, `browser network --since/--follow/--failed`, and adapter `--trace=retain-on-failure` for failure-retained browser evidence.
|
||||
* **autofix** — retire `OPENCLI_DIAGNOSTIC`; adapter repair now uses `--trace retain-on-failure`, trace `summary.md`, and error-envelope trace metadata.
|
||||
* **browser** — `bind` attaches `bound:*` workspaces to user-owned Chrome tabs without taking over window lifecycle; `sessions` reports `idleMsRemaining: null` for bound workspaces because they do not schedule idle close timers. ([#1169](https://github.com/jackwener/opencli/issues/1169), [#929](https://github.com/jackwener/opencli/issues/929))
|
||||
* **browser lifecycle** — owned browser workspaces now lease tabs inside a shared dedicated automation container instead of owning one Chrome window per workspace; lease state is persisted for MV3 service-worker reconciliation and idle cleanup is backed by alarms.
|
||||
* **web read** — make page extraction render-aware: same-origin iframe content is merged into the Markdown source, `--wait-for` can wait inside main/iframe documents, `--wait-until networkidle` waits for captured requests to settle, and `--diagnose` reports frames, empty containers, and API-like XHRs for shell/AJAX pages.
|
||||
|
||||
## [1.7.8](https://github.com/jackwener/opencli/compare/v1.7.7...v1.7.8) (2026-04-25)
|
||||
|
||||
### Features
|
||||
|
||||
* **powerchina** — procurement search adapter. ([#1155](https://github.com/jackwener/opencli/issues/1155))
|
||||
* **toutiao** — `articles` adapter for 头条号 creator dashboard. ([#1148](https://github.com/jackwener/opencli/issues/1148))
|
||||
* **weixin** — `create-draft` and `drafts` commands for Official Account. ([#1095](https://github.com/jackwener/opencli/issues/1095))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **chatgpt-app** — use AX send flow and support zh-CN generating state. ([#1135](https://github.com/jackwener/opencli/issues/1135))
|
||||
* **deepseek** — fix history titles and resume conversation on `ask`. ([#1153](https://github.com/jackwener/opencli/issues/1153))
|
||||
* **amazon** — fall back discussion to product page. ([#1154](https://github.com/jackwener/opencli/issues/1154))
|
||||
* **sinafinance** — match stock symbol in addition to name. ([#1158](https://github.com/jackwener/opencli/issues/1158))
|
||||
|
||||
### Chores
|
||||
|
||||
* **extension** — restore pre-1.6.8 neon terminal icons. ([#1177](https://github.com/jackwener/opencli/issues/1177))
|
||||
|
||||
## [1.7.7](https://github.com/jackwener/opencli/compare/v1.7.6...v1.7.7) (2026-04-23)
|
||||
|
||||
### Features
|
||||
|
||||
* **51job** — comprehensive adapter: `search`, `hot`, `detail`, `company`. ([#1132](https://github.com/jackwener/opencli/issues/1132))
|
||||
* **weread** — `ai-outline` command for AI-generated book outlines. ([#1141](https://github.com/jackwener/opencli/issues/1141))
|
||||
* **web/download** — video/audio/iframe download + `--stdout` streaming. ([#1146](https://github.com/jackwener/opencli/issues/1146))
|
||||
* **download** — hardened HTML→Markdown pipeline with better element handling. ([#1143](https://github.com/jackwener/opencli/issues/1143))
|
||||
* **verify** — fixture-based value validation + skill docs for COOKIE pitfalls. ([#1131](https://github.com/jackwener/opencli/issues/1131))
|
||||
* **agent-native retrospective** — analyze / verify guards / fixture content checks. ([#1133](https://github.com/jackwener/opencli/issues/1133))
|
||||
* **twitter** — expose `has_media` and `media_urls` columns. ([#1115](https://github.com/jackwener/opencli/issues/1115))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **core** — quality audit fixes: elapsed=0 display, daemon error handler state reset, cause chain truncation guard, download cookie expiry, launcher async kill, verbose error logging. ([#1151](https://github.com/jackwener/opencli/issues/1151))
|
||||
* **daemon** — allow extension ping CORS for reachability probing. ([#1150](https://github.com/jackwener/opencli/issues/1150))
|
||||
* **deepseek** — separate thinking process from response in `--think` mode. ([#1142](https://github.com/jackwener/opencli/issues/1142))
|
||||
* **deepseek** — use position-based model selection instead of text matching. ([#1123](https://github.com/jackwener/opencli/issues/1123))
|
||||
* **weread/book** — add fallback selectors for reader page without cover. ([#1138](https://github.com/jackwener/opencli/issues/1138))
|
||||
* **xiaoyuzhou** — correct podcast-episodes API endpoint. ([#1129](https://github.com/jackwener/opencli/issues/1129))
|
||||
* **bilibili** — resolve full video URLs and preserve full description. ([#1118](https://github.com/jackwener/opencli/issues/1118))
|
||||
|
||||
### Docs
|
||||
|
||||
* Fix stale references in READMEs and autofix skill doc. ([#1130](https://github.com/jackwener/opencli/issues/1130))
|
||||
* Restore and rewrite `opencli-usage` as orientation skill. ([#1128](https://github.com/jackwener/opencli/issues/1128))
|
||||
|
||||
## [1.7.6](https://github.com/jackwener/opencli/compare/v1.7.5...v1.7.6) (2026-04-21)
|
||||
|
||||
Extension bumped to 1.0.2 (body-truncation signal unified across raw / detail / fallback paths).
|
||||
|
||||
### Features
|
||||
|
||||
* **Window lifecycle flags** — `--live` (or `OPENCLI_LIVE=1`) keeps the automation window open after a command finishes; `--focus` (or `OPENCLI_WINDOW_FOCUSED=1`) brings the window to the foreground. Works on any subcommand. ([#1122](https://github.com/jackwener/opencli/issues/1122))
|
||||
* **Selector-first browser interactions** — `find` / `get` / `click` / `type` / `select` accept CSS selectors in addition to numeric refs; `--nth` disambiguates multiple matches. ([#1112](https://github.com/jackwener/opencli/issues/1112))
|
||||
* **Agent-native browser payload** — structured `network` bodies with truncation signal, `get html --as json` with `--depth` / `--children-max` / `--text-max` budgets, new `browser extract` command for long-form content with resume cursor. ([#1104](https://github.com/jackwener/opencli/issues/1104))
|
||||
* **`network --filter <fields>`** — filter captured requests by body-shape path segments for quick API discovery. ([#1103](https://github.com/jackwener/opencli/issues/1103))
|
||||
* **`get html --as json`** — structured HTML tree output; no more silent truncation on raw `--as html`. ([#1102](https://github.com/jackwener/opencli/issues/1102))
|
||||
* **`browser network` rewrite** — agent-native discovery with cache keys and shape preview. ([#1100](https://github.com/jackwener/opencli/issues/1100))
|
||||
* **Compound form fields** — date / select / file controls surface a `compound` envelope with format, options, `accept`. Cascading stale-ref recovery + bbox 0.99 dedup for tagged elements. ([#1116](https://github.com/jackwener/opencli/issues/1116))
|
||||
* **twitter `tweets`** — fetch a user's recent posts. ([#1098](https://github.com/jackwener/opencli/issues/1098))
|
||||
* **bilibili `video`** — new video command. ([#1110](https://github.com/jackwener/opencli/issues/1110))
|
||||
* **deepseek `--file`** — file upload support on `ask`. ([#1093](https://github.com/jackwener/opencli/issues/1093))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **twitter** — 5s timeout on `resolveTwitterQueryId` to prevent hang. ([#1106](https://github.com/jackwener/opencli/issues/1106))
|
||||
* **youtube** — fall back to Videos tab when Home has no videos. ([#1109](https://github.com/jackwener/opencli/issues/1109))
|
||||
* **jianyu** — keep accessible detail urls in search. ([#1099](https://github.com/jackwener/opencli/issues/1099))
|
||||
* **jianyu** — block inaccessible detail links and verification pages. ([#918](https://github.com/jackwener/opencli/issues/918))
|
||||
|
||||
### Docs
|
||||
|
||||
* **opencli-browser skill** — restored and upgraded for selector-first workflow. ([#1119](https://github.com/jackwener/opencli/issues/1119))
|
||||
* **Window lifecycle** — sync README + skill docs with `--live` / `--focus` behavior. ([#1125](https://github.com/jackwener/opencli/issues/1125))
|
||||
|
||||
### Extension (1.0.2)
|
||||
|
||||
* Unify body-truncation contract across raw / detail / fallback network paths; surface `body_truncated` / `body_full_size` / `body_truncation_reason`. ([#1104](https://github.com/jackwener/opencli/issues/1104))
|
||||
|
||||
## [1.7.5](https://github.com/jackwener/opencli/compare/v1.7.4...v1.7.5) (2026-04-20)
|
||||
|
||||
Extension bumped to 1.0.1 (multi-tab routing + cross-origin iframe).
|
||||
|
||||
### Features
|
||||
|
||||
* **DeepSeek adapter** — browser-based `ask` / `history` / `new` / `read` / `status` ([#1088](https://github.com/jackwener/opencli/issues/1088))
|
||||
* **Eastmoney adapters** — 13 finance adapters as Phase A oracle: `quote`, `rank`, `kline`, `sectors`, `etf`, `holders`, `money-flow`, `northbound`, `longhu`, `kuaixun`, `convertible`, `index-board`, `announcement` ([#1091](https://github.com/jackwener/opencli/issues/1091))
|
||||
* **Twitter GraphQL lists** — `list-tweets`, `list-add`, `list-remove` ([#1076](https://github.com/jackwener/opencli/issues/1076))
|
||||
* **nowcoder adapter** — 牛客网 with 16 commands ([#1036](https://github.com/jackwener/opencli/issues/1036))
|
||||
* **Chinese academic & policy adapters** — `baidu-scholar`, `google-scholar`, `wanfang`, `gov-law`, `gov-policy` ([#243](https://github.com/jackwener/opencli/issues/243))
|
||||
* **Download saved path** — `web read` and `weixin download` now show saved file location ([#1042](https://github.com/jackwener/opencli/issues/1042))
|
||||
* **Cross-origin iframe support** — CDP execution context for iframed content ([#1084](https://github.com/jackwener/opencli/issues/1084))
|
||||
|
||||
### Improvements
|
||||
|
||||
* **Multi-tab routing** — hardened target isolation and tab routing ([#1072](https://github.com/jackwener/opencli/issues/1072))
|
||||
* **Skill consolidation** — 6 skills merged into 3 (`opencli-adapter-author`, `opencli-autofix`, `smart-search`); removed mechanical commands `explore` / `synthesize` / `generate` / `cascade` / `record` ([#1094](https://github.com/jackwener/opencli/issues/1094))
|
||||
* **Browser docs rewrite** — docs reoriented for AI Agent use case ([#1080](https://github.com/jackwener/opencli/issues/1080))
|
||||
* **antigravity serve** — configurable timeout + auto-reconnect ([#859](https://github.com/jackwener/opencli/issues/859), [#1063](https://github.com/jackwener/opencli/issues/1063))
|
||||
* **Design debt cleanup** — deprecated APIs, arg validation, dead plugin code ([#1065](https://github.com/jackwener/opencli/issues/1065))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **xiaoyuzhou** — migrate from broken SSR to authenticated API ([#1059](https://github.com/jackwener/opencli/issues/1059)); accept `CONFIG_ERROR` in E2E guard ([#1066](https://github.com/jackwener/opencli/issues/1066))
|
||||
* **xiaohongshu** — detect draft save success ([#1060](https://github.com/jackwener/opencli/issues/1060)); verify title input sticks on publish ([#1050](https://github.com/jackwener/opencli/issues/1050))
|
||||
* **twitter** — repair lists scraping from detail pages ([#1053](https://github.com/jackwener/opencli/issues/1053))
|
||||
* **zsxq** — separate content from title, remove title truncation ([#1079](https://github.com/jackwener/opencli/issues/1079))
|
||||
* **extension** — per-workspace idle timeout for browser sessions ([#1064](https://github.com/jackwener/opencli/issues/1064))
|
||||
|
||||
### Revert
|
||||
|
||||
* Undo output renderer table-formatting patch ([#1085](https://github.com/jackwener/opencli/issues/1085), reverts [#1081](https://github.com/jackwener/opencli/issues/1081))
|
||||
|
||||
### Extension (1.0.1)
|
||||
|
||||
* Multi-tab routing support ([#1072](https://github.com/jackwener/opencli/issues/1072))
|
||||
* Cross-origin iframe CDP contexts ([#1084](https://github.com/jackwener/opencli/issues/1084))
|
||||
|
||||
## [1.7.0](https://github.com/jackwener/opencli/compare/v1.6.1...v1.7.0) (2026-04-11)
|
||||
|
||||
This is a major release with significant internal architecture changes.
|
||||
Adapter code, validation, and error handling have been modernized.
|
||||
|
||||
### ⚠ BREAKING CHANGES
|
||||
|
||||
* **Node.js >= 21 required** — `import.meta.dirname` is used in core modules; Node 20 and below will fail at startup.
|
||||
* **YAML adapters deprecated** — YAML-based `.yaml` adapters are no longer loaded. Existing YAML adapters must be converted to JS via `cli()` API. A deprecation warning is emitted if `.yaml` files are detected.
|
||||
* **`.ts` adapters no longer loaded at runtime** — The runtime only discovers `.js` files. If you have `.ts` adapters in `~/.opencli/clis/`, compile them to `.js` or rewrite using plain JS. A warning is printed when `.ts` files without a matching `.js` are found.
|
||||
* **Error output format changed** — All errors are now emitted as a structured YAML envelope to stderr. Scripts parsing stdout for `[{error, help}]` must switch to stderr / exit code. ([#923](https://github.com/jackwener/opencli/issues/923))
|
||||
* **`tabId` replaced by `targetId`** — Cross-layer page identity now uses `targetId`. Extensions and plugins referencing `tabId` must update. ([#899](https://github.com/jackwener/opencli/issues/899))
|
||||
* **`operate` renamed to `browser`** — All `opencli operate` commands are now `opencli browser`. ([#883](https://github.com/jackwener/opencli/issues/883))
|
||||
|
||||
### Features
|
||||
|
||||
* **auto-close adapter windows** — Browser tabs opened by adapters are automatically closed after execution; configurable via `OPENCLI_WINDOW_FOCUSED`. ([#915](https://github.com/jackwener/opencli/issues/915))
|
||||
* **Self-Repair protocol** — Automatic adapter fixing when commands fail. ([#866](https://github.com/jackwener/opencli/issues/866))
|
||||
* **EarlyHint callback** — Cost gating channel for generate pipeline. ([#882](https://github.com/jackwener/opencli/issues/882))
|
||||
* **verified generate pipeline** — Structured contract for AI-driven adapter generation. ([#878](https://github.com/jackwener/opencli/issues/878))
|
||||
* **structured diagnostic output** — AI-driven adapter repair gets structured diagnostics. ([#802](https://github.com/jackwener/opencli/issues/802))
|
||||
* **auto-downgrade to YAML in non-TTY** — Machine-readable output when piped. ([#737](https://github.com/jackwener/opencli/issues/737))
|
||||
* **Browser Use improvements** — Better click/type/state handling for browser automation. ([#707](https://github.com/jackwener/opencli/issues/707))
|
||||
* **CDP session-level network capture** — Full network capture support for CDPPage. ([#815](https://github.com/jackwener/opencli/issues/815), [#816](https://github.com/jackwener/opencli/issues/816))
|
||||
* **AutoResearch framework** — V2EX/Zhihu test suites (194 tasks). ([#731](https://github.com/jackwener/opencli/issues/731), [#717](https://github.com/jackwener/opencli/issues/717), [#741](https://github.com/jackwener/opencli/issues/741))
|
||||
* **new adapters:** Gitee ([#845](https://github.com/jackwener/opencli/issues/845)), 闲鱼 ([#696](https://github.com/jackwener/opencli/issues/696)), 1688 ([#650](https://github.com/jackwener/opencli/issues/650), [#820](https://github.com/jackwener/opencli/issues/820)), LessWrong ([#773](https://github.com/jackwener/opencli/issues/773)), 虎扑 ([#751](https://github.com/jackwener/opencli/issues/751)), 小鹅通 ([#617](https://github.com/jackwener/opencli/issues/617)), 元宝 ([#693](https://github.com/jackwener/opencli/issues/693)), 即梦 ([#897](https://github.com/jackwener/opencli/issues/897), [#895](https://github.com/jackwener/opencli/issues/895)), Quark Drive ([#858](https://github.com/jackwener/opencli/issues/858)), GitHub Trending/Binance/Weather ([#214](https://github.com/jackwener/opencli/issues/214))
|
||||
* **adapter enhancements:** Instagram post/reel/story/note ([#671](https://github.com/jackwener/opencli/issues/671)), Twitter image posts/replies ([#666](https://github.com/jackwener/opencli/issues/666), [#756](https://github.com/jackwener/opencli/issues/756)), 知乎 interactions ([#868](https://github.com/jackwener/opencli/issues/868)), Bilibili b23.tv short URL ([#740](https://github.com/jackwener/opencli/issues/740)), 雪球 kline/groups ([#809](https://github.com/jackwener/opencli/issues/809)), Amazon unified ranking ([#724](https://github.com/jackwener/opencli/issues/724)), Gemini deep-research ([#778](https://github.com/jackwener/opencli/issues/778)), 新浪财经热搜 ([#736](https://github.com/jackwener/opencli/issues/736)), linux-do topic split ([#821](https://github.com/jackwener/opencli/issues/821)), JD/淘宝/CNKI revived ([#248](https://github.com/jackwener/opencli/issues/248))
|
||||
|
||||
### Bug Fixes
|
||||
|
||||
* **security:** escape codegen strings and redact diagnostic body ([#930](https://github.com/jackwener/opencli/issues/930))
|
||||
* **bilibili:** add missing domain for following cli ([#947](https://github.com/jackwener/opencli/issues/947))
|
||||
* clean up stale `.ts` adapter files during upgrade ([#948](https://github.com/jackwener/opencli/issues/948))
|
||||
* clean up legacy shim files and stale tmp files on upgrade ([#934](https://github.com/jackwener/opencli/issues/934))
|
||||
* address deep review findings (security, correctness, consistency) ([#935](https://github.com/jackwener/opencli/issues/935))
|
||||
* batch quality improvements — dedupe completion, unify logging, fix docs ([#945](https://github.com/jackwener/opencli/issues/945))
|
||||
* graceful fallback when extension lacks network-capture support ([#865](https://github.com/jackwener/opencli/issues/865))
|
||||
* handle missing electron executable gracefully ([#747](https://github.com/jackwener/opencli/issues/747))
|
||||
* recover drifted tabs instead of abandoning them ([#715](https://github.com/jackwener/opencli/issues/715))
|
||||
* retry on "No window with id" CDP error ([#892](https://github.com/jackwener/opencli/issues/892))
|
||||
* **launcher:** graceful degradation and manual CDP override for Windows ([#744](https://github.com/jackwener/opencli/issues/744))
|
||||
* **xiaohongshu:** scope note interaction selectors, replace blind retry with MutationObserver ([#839](https://github.com/jackwener/opencli/issues/839), [#730](https://github.com/jackwener/opencli/issues/730))
|
||||
* **twitter:** relax reply composer timeout, use composer for text replies ([#862](https://github.com/jackwener/opencli/issues/862), [#860](https://github.com/jackwener/opencli/issues/860))
|
||||
* **doubao:** preserve image URLs, connect to correct CDP target ([#708](https://github.com/jackwener/opencli/issues/708), [#674](https://github.com/jackwener/opencli/issues/674))
|
||||
* **gemini:** stabilize ask reply state handling ([#735](https://github.com/jackwener/opencli/issues/735))
|
||||
* **douban:** fix marks pagination and improve subject data extraction ([#752](https://github.com/jackwener/opencli/issues/752))
|
||||
* **jianyu:** avoid early API bucket cutoff, stabilize search ([#916](https://github.com/jackwener/opencli/issues/916), [#912](https://github.com/jackwener/opencli/issues/912))
|
||||
* **xiaoe:** resolve missing episodes for long courses via auto-scroll ([#904](https://github.com/jackwener/opencli/issues/904))
|
||||
|
||||
### Refactoring
|
||||
|
||||
* **adapters:** convert adapter layer from TypeScript to JavaScript ([#928](https://github.com/jackwener/opencli/issues/928))
|
||||
* **adapters:** migrate all CLI adapters from YAML to TypeScript, then to JS ([#887](https://github.com/jackwener/opencli/issues/887), [#922](https://github.com/jackwener/opencli/issues/922))
|
||||
* **validate:** switch from YAML-file scanning to registry-based validation ([#943](https://github.com/jackwener/opencli/issues/943))
|
||||
* **strategy:** normalize strategy into runtime fields at registration time ([#941](https://github.com/jackwener/opencli/issues/941))
|
||||
* **errors:** unify error output as YAML envelope to stderr ([#923](https://github.com/jackwener/opencli/issues/923))
|
||||
* **daemon:** make daemon persistent, remove idle timeout ([#913](https://github.com/jackwener/opencli/issues/913))
|
||||
* **browser:** unify browser error classification and deduplicate retry logic ([#908](https://github.com/jackwener/opencli/issues/908))
|
||||
* **monorepo:** adapter separation — `clis/` at root ([#782](https://github.com/jackwener/opencli/issues/782))
|
||||
* rename `operate` to `browser` ([#883](https://github.com/jackwener/opencli/issues/883))
|
||||
* eliminate `any` types in core files ([#886](https://github.com/jackwener/opencli/issues/886))
|
||||
* migrate adapter imports to package exports ([#795](https://github.com/jackwener/opencli/issues/795))
|
||||
|
||||
### Performance
|
||||
|
||||
* **P0 optimizations** — faster startup, reduced overhead ([#944](https://github.com/jackwener/opencli/issues/944))
|
||||
* fast-path completion/version/shell-scripts to bypass full discovery ([#898](https://github.com/jackwener/opencli/issues/898))
|
||||
* optimize browser pipeline — tab query dedup, parallel stealth, incremental snapshots ([#713](https://github.com/jackwener/opencli/issues/713))
|
||||
* reduce round-trips in browser command hot path ([#712](https://github.com/jackwener/opencli/issues/712))
|
||||
* skip blank page on first browser command ([#710](https://github.com/jackwener/opencli/issues/710))
|
||||
|
||||
### Documentation
|
||||
|
||||
* restructure README narrative ([#885](https://github.com/jackwener/opencli/issues/885))
|
||||
* add Android Chrome usage guide ([#687](https://github.com/jackwener/opencli/issues/687))
|
||||
* add Electron app CLI quickstart guide
|
||||
* fix stale `.ts` references across skills and docs ([#954](https://github.com/jackwener/opencli/issues/954))
|
||||
* unify skill command references and merge opencli-generate into opencli-explorer ([#891](https://github.com/jackwener/opencli/issues/891), [#894](https://github.com/jackwener/opencli/issues/894))
|
||||
|
||||
### Upgrade Guide
|
||||
|
||||
1. **Update Node.js** to v21 or later (v22 LTS recommended).
|
||||
2. **Run `npm install -g @jackwener/opencli@latest`** — the preuninstall hook gracefully stops the old daemon; the first browser command after upgrade auto-restarts it.
|
||||
3. **If you have custom `.ts` adapters** in `~/.opencli/clis/`, rename or compile them to `.js`. A warning will be printed on startup if stale `.ts` files are detected.
|
||||
4. **If you have custom `.yaml` adapters**, convert them to JS using the `cli()` API (see `skills/opencli-adapter-author/references/adapter-template.md`).
|
||||
5. **If you parse error output from stdout**, switch to stderr. Errors are now structured YAML envelopes with typed exit codes.
|
||||
|
||||
|
||||
## [1.6.1](https://github.com/jackwener/opencli/compare/v1.6.0...v1.6.1) (2026-04-02)
|
||||
|
||||
|
||||
|
||||
+9
-8
@@ -18,6 +18,7 @@ npm run build
|
||||
# 4. Run a few checks
|
||||
npx tsc --noEmit
|
||||
npm test
|
||||
npm run test:adapter
|
||||
|
||||
# 5. Link globally (optional, for testing `opencli` command)
|
||||
npm link
|
||||
@@ -29,7 +30,7 @@ All adapters use TypeScript. Use the pipeline API for data-fetching commands, an
|
||||
|
||||
### Pipeline Adapter (Recommended for data-fetching commands)
|
||||
|
||||
Create a file like `clis/<site>/<command>.js`:
|
||||
Create a file like `clis/<site>/<command>.ts`:
|
||||
|
||||
```typescript
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
@@ -59,11 +60,11 @@ cli({
|
||||
});
|
||||
```
|
||||
|
||||
See [`hackernews/top.js`](clis/hackernews/top.js) for a real example.
|
||||
See [`hackernews/top.ts`](clis/hackernews/top.ts) for a real example.
|
||||
|
||||
### func() Adapter (For complex browser interactions)
|
||||
|
||||
Create a file like `clis/<site>/<command>.js`:
|
||||
Create a file like `clis/<site>/<command>.ts`:
|
||||
|
||||
```typescript
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
@@ -102,7 +103,7 @@ cli({
|
||||
});
|
||||
```
|
||||
|
||||
Install the [`opencli-adapter-author` skill](./skills/opencli-adapter-author/SKILL.md) if you need the full adapter workflow — recon → API discovery → field decoding → `opencli browser verify`.
|
||||
Use `opencli explore <url>` to discover APIs and see [opencli-explorer skill](./skills/opencli-explorer/SKILL.md) if you need the full adapter workflow.
|
||||
|
||||
### Validate Your Adapter
|
||||
|
||||
@@ -151,8 +152,8 @@ args: [
|
||||
See [TESTING.md](./TESTING.md) for the full guide and exact test locations.
|
||||
|
||||
```bash
|
||||
npm test # Default local gate: unit + extension + adapter tests
|
||||
npm run test:adapter # Adapter-only project (useful while iterating on adapters)
|
||||
npm test # Core unit tests (non-adapter)
|
||||
npm run test:adapter # Focused adapter tests: zhihu/twitter/reddit/bilibili
|
||||
npx vitest run tests/e2e/ # E2E tests
|
||||
npx vitest run # All tests
|
||||
```
|
||||
@@ -185,8 +186,8 @@ Common scopes: site name (`twitter`, `reddit`) or module name (`browser`, `pipel
|
||||
3. Run the checks that apply:
|
||||
```bash
|
||||
npx tsc --noEmit # Type check
|
||||
npm test # Default local gate: unit + extension + adapter
|
||||
npm run test:adapter # Adapter-only project (optional while iterating on adapters)
|
||||
npm test # Core unit tests
|
||||
npm run test:adapter # Focused adapter tests (if you touched adapter logic)
|
||||
opencli validate # Adapter validation
|
||||
```
|
||||
4. Commit using conventional commit format
|
||||
|
||||
@@ -11,22 +11,29 @@
|
||||
OpenCLI gives you one surface for three different kinds of automation:
|
||||
|
||||
- **Use built-in adapters** for sites like Bilibili, Zhihu, Xiaohongshu, Reddit, HackerNews, Twitter/X, and [many more](#built-in-commands).
|
||||
- **Let AI Agents operate any website** — install the `opencli-adapter-author` skill in your AI agent (Claude Code, Cursor, etc.), and it can navigate, click, type, extract, and inspect any page through your logged-in browser via `opencli browser` primitives.
|
||||
- **Write new adapters** end-to-end with `opencli browser` + the `opencli-adapter-author` skill, which guides from first recon through field decoding, code, and `opencli browser verify`.
|
||||
- **Drive a live browser directly** with `opencli browser` when an AI agent needs to click, type, extract, or inspect a page in real time.
|
||||
- **Generate new adapters** from real browser behavior with `explore`, `synthesize`, `generate`, and `cascade`.
|
||||
|
||||
It also works as a **CLI hub** for local tools such as `gh`, `docker`, and other binaries you register yourself, plus **desktop app adapters** for Electron apps like Cursor, Codex, Antigravity, ChatGPT, and Notion.
|
||||
|
||||
## Why OpenCLI
|
||||
|
||||
---
|
||||
|
||||
## Highlights
|
||||
|
||||
- **Desktop App Control** — Drive Electron apps (Cursor, Codex, ChatGPT, Notion, etc.) directly from the terminal via CDP.
|
||||
- **Browser Automation for AI Agents** — Install the `opencli-adapter-author` skill, and your AI agent can operate any website: navigate, click, type, extract, screenshot — all through your logged-in Chrome session.
|
||||
- **Multi-profile Browser Bridge** — Install the extension in each Chrome profile you want to use, then route commands with `--profile`, `OPENCLI_PROFILE`, or `opencli profile use`.
|
||||
- **Website → CLI** — Turn any website into a deterministic CLI: 100+ site surfaces are already registered, or write your own with the `opencli-adapter-author` skill + `opencli browser verify`.
|
||||
- **CLI All Electron** — CLI-ify apps like Antigravity Ultra! Now AI can control itself natively.
|
||||
- **Browser Automation** — `browser` gives AI agents direct browser control: click, type, extract, screenshot — any interaction, fully scriptable.
|
||||
- **Website → CLI** — Turn any website into a deterministic CLI: 70+ pre-built adapters, or crystallize your own with `opencli record`.
|
||||
- **Account-safe** — Reuses Chrome/Chromium logged-in state; your credentials never leave the browser.
|
||||
- **AI Agent ready** — One skill takes you from site recon through API discovery, field decoding, adapter writing, and verification.
|
||||
- **CLI Hub** — Discover, auto-install, and passthrough commands to any external CLI (gh, docker, obsidian, etc).
|
||||
- **Anti-detection built-in** — Patches `navigator.webdriver`, stubs `window.chrome`, fakes plugin lists, cleans ChromeDriver/Playwright globals, and strips CDP frames from Error stack traces. Extensive anti-fingerprinting and risk-control evasion measures baked in at every layer.
|
||||
- **AI Agent ready** — `explore` discovers APIs, `synthesize` generates adapters, `cascade` finds auth strategies, `browser` controls the browser directly.
|
||||
- **External CLI Hub** — Discover, auto-install, and passthrough commands to any external CLI (gh, obsidian, docker, etc). Zero setup.
|
||||
- **Self-healing setup** — `opencli doctor` diagnoses and auto-starts the daemon, extension, and live browser connectivity.
|
||||
- **Dynamic Loader** — Simply drop `.ts` adapters into the `clis/` folder for auto-registration.
|
||||
- **Zero LLM cost** — No tokens consumed at runtime. Run 10,000 times and pay nothing.
|
||||
- **Deterministic** — Same command, same output schema, every time. Pipeable, scriptable, CI-friendly.
|
||||
- **Broad coverage** — 79+ sites across global and Chinese platforms (Bilibili, Zhihu, Xiaohongshu, Reddit, HackerNews, and more), plus desktop Electron apps via CDP.
|
||||
|
||||
---
|
||||
|
||||
@@ -34,10 +41,7 @@ It also works as a **CLI hub** for local tools such as `gh`, `docker`, and other
|
||||
|
||||
### 1. Install OpenCLI
|
||||
|
||||
OpenCLI requires **Node.js >= 21**.
|
||||
|
||||
```bash
|
||||
node --version
|
||||
npm install -g @jackwener/opencli
|
||||
```
|
||||
|
||||
@@ -45,11 +49,7 @@ npm install -g @jackwener/opencli
|
||||
|
||||
OpenCLI connects to Chrome/Chromium through a lightweight Browser Bridge extension plus a small local daemon. The daemon auto-starts when needed.
|
||||
|
||||
**Option A — Chrome Web Store (recommended):**
|
||||
Install **OpenCLI** from the [Chrome Web Store](https://chromewebstore.google.com/detail/opencli/ildkmabpimmkaediidaifkhjpohdnifk).
|
||||
|
||||
**Option B — Manual install:**
|
||||
1. Download the latest `opencli-extension-v{version}.zip` from the GitHub [Releases page](https://github.com/jackwener/opencli/releases).
|
||||
1. Download the latest `opencli-extension.zip` from the GitHub [Releases page](https://github.com/jackwener/opencli/releases).
|
||||
2. Unzip it, open `chrome://extensions`, and enable **Developer mode**.
|
||||
3. Click **Load unpacked** and select the unzipped folder.
|
||||
|
||||
@@ -57,22 +57,10 @@ Install **OpenCLI** from the [Chrome Web Store](https://chromewebstore.google.co
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
opencli daemon status
|
||||
```
|
||||
|
||||
### 4. Optional: name your Chrome profile
|
||||
|
||||
Each Chrome profile runs its own OpenCLI extension instance. If you use multiple Chrome profiles, list the connected profiles and assign local aliases:
|
||||
|
||||
```bash
|
||||
opencli profile list
|
||||
opencli profile rename <contextId> work
|
||||
opencli profile use work
|
||||
opencli --profile work browser state
|
||||
```
|
||||
|
||||
With only one connected profile, OpenCLI uses it automatically. With multiple connected profiles and no default, OpenCLI asks you to choose instead of guessing.
|
||||
|
||||
### 5. Run your first commands
|
||||
### 4. Run your first commands
|
||||
|
||||
```bash
|
||||
opencli list
|
||||
@@ -86,26 +74,17 @@ Use OpenCLI directly when you want a reliable command instead of a live browser
|
||||
|
||||
- `opencli list` shows every registered command.
|
||||
- `opencli <site> <command>` runs a built-in or generated adapter.
|
||||
- `opencli external register mycli` exposes a local CLI through the same discovery surface.
|
||||
- `opencli doctor` helps diagnose browser connectivity.
|
||||
|
||||
## Extending OpenCLI
|
||||
|
||||
If you want to add your own commands, start with the [Extending OpenCLI guide](./docs/guide/extending-opencli.md). README keeps this short; the guide covers the directory layout, source-control model, and install commands.
|
||||
|
||||
| Need | Recommended path |
|
||||
|------|------------------|
|
||||
| Keep personal website commands in your own Git repo | `opencli plugin create` + `opencli plugin install file://...` |
|
||||
| Quickly draft a private local adapter | `opencli browser init <site>/<command>` in `~/.opencli/clis/` |
|
||||
| Modify an official adapter locally | `opencli adapter eject <site>` + `opencli adapter reset <site>` |
|
||||
| Publish or install third-party commands | `opencli plugin install github:user/repo` |
|
||||
| Wrap an existing local binary | `opencli external register <name>` |
|
||||
- `opencli register mycli` exposes a local CLI through the same discovery surface.
|
||||
- `opencli daemon status` and `opencli doctor` help diagnose browser connectivity.
|
||||
|
||||
## For AI Agents
|
||||
|
||||
OpenCLI's browser commands are designed to be used by AI Agents — not run manually. Install skills into your AI agent (Claude Code, Cursor, etc.), and the agent operates websites on your behalf using your logged-in Chrome session.
|
||||
Use two different entry points depending on the task:
|
||||
|
||||
### Install skills
|
||||
- [`skills/opencli-explorer/SKILL.md`](./skills/opencli-explorer/SKILL.md): the entry point for creating new adapters — supports both fully automated generation (`opencli generate <url>`) and manual exploration workflows.
|
||||
- [`skills/opencli-browser/SKILL.md`](./skills/opencli-browser/SKILL.md): the low-level control surface for live browsing, debugging, and manual intervention.
|
||||
|
||||
Install the packaged skills with:
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli
|
||||
@@ -114,68 +93,40 @@ npx skills add jackwener/opencli
|
||||
Or install only what you need:
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli --skill opencli-adapter-author
|
||||
npx skills add jackwener/opencli --skill opencli-autofix
|
||||
npx skills add jackwener/opencli --skill opencli-browser
|
||||
npx skills add jackwener/opencli --skill opencli-usage
|
||||
npx skills add jackwener/opencli --skill smart-search
|
||||
npx skills add jackwener/opencli --skill opencli-browser
|
||||
npx skills add jackwener/opencli --skill opencli-explorer
|
||||
npx skills add jackwener/opencli --skill opencli-oneshot
|
||||
```
|
||||
|
||||
### Which skill to use
|
||||
In practice:
|
||||
|
||||
| Skill | When to use | Example prompt to your AI agent |
|
||||
|-------|------------|-------------------------------|
|
||||
| **opencli-adapter-author** | Operate a site in real time, or write a reusable adapter for a new site | "Help me check my Xiaohongshu notifications" / "Write an adapter for douyin trending" / "Make a command that grabs the top posts from this page" |
|
||||
| **opencli-autofix** | Repair a broken adapter when a built-in command fails | "`opencli zhihu hot` is returning empty — fix it" |
|
||||
| **opencli-browser** | Browser automation reference for AI agents | "Use browser commands to scrape this page" |
|
||||
| **opencli-usage** | Quick reference for all OpenCLI commands and sites | "What commands does OpenCLI have for Twitter?" |
|
||||
| **smart-search** | Search across existing OpenCLI capabilities | "Find me a Bilibili trending adapter" |
|
||||
- start with `opencli-explorer` when the agent needs a reusable command for a site (it covers both automated and manual flows)
|
||||
- use `opencli-browser` when the agent needs to inspect or steer the page directly
|
||||
|
||||
### How it works
|
||||
|
||||
Once `opencli-adapter-author` is installed, your AI agent can:
|
||||
|
||||
1. **Navigate** to any URL using your logged-in browser
|
||||
2. **Read** page content via structured DOM snapshots (not screenshots)
|
||||
3. **Interact** — click buttons, fill forms, select options, press keys
|
||||
4. **Extract** data from the page or intercept network API responses
|
||||
5. **Wait** for elements, text, or page transitions
|
||||
|
||||
The agent handles all the `opencli browser` commands internally — you just describe what you want done in natural language.
|
||||
|
||||
**Skill references:**
|
||||
- [`skills/opencli-adapter-author/SKILL.md`](./skills/opencli-adapter-author/SKILL.md) — browser operation + adapter authoring, end-to-end
|
||||
- [`skills/opencli-autofix/SKILL.md`](./skills/opencli-autofix/SKILL.md) — repair broken adapters
|
||||
- [`skills/opencli-browser/SKILL.md`](./skills/opencli-browser/SKILL.md) — browser automation reference
|
||||
- [`skills/opencli-usage/SKILL.md`](./skills/opencli-usage/SKILL.md) — command and site reference
|
||||
- [`skills/smart-search/SKILL.md`](./skills/smart-search/SKILL.md) — capability search
|
||||
|
||||
Available browser commands include `open`, `state`, `click`, `type`, `select`, `keys`, `wait`, `get`, `find`, `extract`, `frames`, `screenshot`, `scroll`, `back`, `eval`, `network`, `tab list`, `tab new`, `tab select`, `tab close`, `init`, `verify`, and `close`.
|
||||
|
||||
`opencli browser open <url>` and `opencli browser tab new [url]` both return a target ID. Use `opencli browser tab list` to inspect the target IDs of tabs that already exist, then pass `--tab <targetId>` to route a command to a specific tab. `tab new` creates a new tab without changing the default browser target; only `tab select <targetId>` promotes that tab to the default target for later untargeted `opencli browser ...` commands.
|
||||
Available browser commands include `open`, `state`, `click`, `type`, `select`, `keys`, `wait`, `get`, `screenshot`, `scroll`, `back`, `eval`, `network`, `init`, `verify`, and `close`.
|
||||
|
||||
## Core Concepts
|
||||
|
||||
### `browser`: AI Agent browser control
|
||||
### `browser`: live control
|
||||
|
||||
`opencli browser` commands are the low-level primitives that AI Agents use to operate websites. You don't run these manually — instead, install the `opencli-adapter-author` skill into your AI agent, describe what you want in natural language, and the agent handles the browser operations.
|
||||
|
||||
For example, tell your agent: *"Help me check my Xiaohongshu notifications"* — the agent will use `opencli browser open`, `state`, `click`, etc. under the hood.
|
||||
Use `opencli browser` when the task is inherently interactive and the agent needs to operate the page directly.
|
||||
|
||||
### Built-in adapters: stable commands
|
||||
|
||||
Use site-specific commands such as `opencli hackernews top` or `opencli reddit hot` when the capability already exists. These are deterministic and work without browser — ideal for both humans and AI agents.
|
||||
Use site-specific commands such as `opencli hackernews top` or `opencli reddit hot` when the capability already exists and you want deterministic output.
|
||||
|
||||
### Writing a new adapter
|
||||
### `explore` / `synthesize` / `generate`: create new CLIs
|
||||
|
||||
When the site you need is not yet covered, use the `opencli-adapter-author` skill. It takes the agent end-to-end:
|
||||
Use these commands when the site you need is not covered yet:
|
||||
|
||||
1. Recon the site and classify its pattern (SPA / SSR / JSONP / Token / Streaming).
|
||||
2. Discover the right endpoint — network inspection, initial state, bundle search, token trace, or interceptor fallback.
|
||||
3. Decide the auth strategy — `PUBLIC` / `COOKIE` / `HEADER` / `INTERCEPT`.
|
||||
4. Decode response fields and design output columns.
|
||||
5. `opencli browser analyze <url>` for one-shot recon, then `opencli browser init <site>/<name>` → write adapter → `opencli browser verify <site>/<name>`.
|
||||
6. Persist site knowledge to `~/.opencli/sites/<site>/` so the next adapter for the same site is faster.
|
||||
- `explore` inspects the page, network activity, and capability surface.
|
||||
- `synthesize` turns exploration artifacts into evaluate-based YAML adapters.
|
||||
- `generate` runs the verified generation path and returns either a usable command or a structured explanation of why completion was blocked or needs human review.
|
||||
|
||||
### `cascade`: auth strategy discovery
|
||||
|
||||
Use `cascade` to probe fallback auth paths such as public endpoints, cookies, and custom headers before you commit to an adapter design.
|
||||
|
||||
### CLI Hub and desktop adapters
|
||||
|
||||
@@ -186,46 +137,15 @@ OpenCLI is not only for websites. It can also:
|
||||
|
||||
## Prerequisites
|
||||
|
||||
- **Node.js**: >= 21.0.0 (required for the standard npm install path)
|
||||
- **Bun**: >= 1.0 (optional alternative runtime)
|
||||
- **Node.js**: >= 20.0.0 (or **Bun** >= 1.0)
|
||||
- **Chrome or Chromium** running and logged into the target site for browser-backed commands
|
||||
|
||||
> **Important**: Browser-backed commands reuse your Chrome/Chromium login session. If you get empty data or permission-like failures, first confirm the site is already open and authenticated in Chrome/Chromium.
|
||||
|
||||
## Configuration
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `OPENCLI_DAEMON_PORT` | `19825` | HTTP port for the daemon-extension bridge |
|
||||
| `OPENCLI_PROFILE` | — | Browser Bridge profile alias/contextId to use when multiple Chrome profiles are connected |
|
||||
| `OPENCLI_WINDOW_FOCUSED` | `false` | Set to `1` to open the automation container in the foreground (useful for debugging). The `--focus` flag sets this. |
|
||||
| `OPENCLI_LIVE` | `false` | Set to `1` to keep the automation lease open after an adapter command finishes (useful for inspection). The `--live` flag sets this. |
|
||||
| `OPENCLI_BROWSER_CONNECT_TIMEOUT` | `30` | Seconds to wait for browser connection |
|
||||
| `OPENCLI_BROWSER_COMMAND_TIMEOUT` | `60` | Seconds to wait for a single browser command |
|
||||
| `OPENCLI_CDP_ENDPOINT` | — | Chrome DevTools Protocol endpoint for remote browser or Electron apps |
|
||||
| `OPENCLI_CDP_TARGET` | — | Filter CDP targets by URL substring (e.g. `detail.1688.com`) |
|
||||
| `OPENCLI_VERBOSE` | `false` | Enable verbose logging (`-v` flag also works) |
|
||||
| `DEBUG_SNAPSHOT` | — | Set to `1` for DOM snapshot debug output |
|
||||
|
||||
`--focus` works for both `opencli browser *` and browser-backed adapter commands. `--live` is mainly for adapter commands: browser subcommands already keep the automation lease open until you run `opencli browser close` or the idle timeout expires.
|
||||
|
||||
## Update
|
||||
|
||||
```bash
|
||||
npm install -g @jackwener/opencli@latest
|
||||
|
||||
# If you use the packaged OpenCLI skills, refresh them too
|
||||
npx skills add jackwener/opencli
|
||||
```
|
||||
|
||||
Or refresh only the skills you actually use:
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli --skill opencli-adapter-author
|
||||
npx skills add jackwener/opencli --skill opencli-autofix
|
||||
npx skills add jackwener/opencli --skill opencli-browser
|
||||
npx skills add jackwener/opencli --skill opencli-usage
|
||||
npx skills add jackwener/opencli --skill smart-search
|
||||
```
|
||||
|
||||
## For Developers
|
||||
@@ -250,36 +170,23 @@ To load the source Browser Bridge extension:
|
||||
| Site | Commands |
|
||||
|------|----------|
|
||||
| **xiaohongshu** | `search` `note` `comments` `feed` `user` `download` `publish` `notifications` `creator-notes` `creator-notes-summary` `creator-note-detail` `creator-profile` `creator-stats` |
|
||||
| **bilibili** | `hot` `search` `history` `feed` `ranking` `download` `comments` `dynamic` `favorite` `following` `me` `subtitle` `video` `user-videos` |
|
||||
| **bilibili** | `hot` `search` `history` `feed` `ranking` `download` `comments` `dynamic` `favorite` `following` `me` `subtitle` `user-videos` |
|
||||
| **tieba** | `hot` `posts` `search` `read` |
|
||||
| **hupu** | `hot` `search` `detail` `mentions` `reply` `like` `unlike` |
|
||||
| **twitter** | `trending` `search` `timeline` `tweets` `lists` `list-tweets` `list-add` `list-remove` `bookmarks` `post` `download` `profile` `article` `like` `likes` `notifications` `reply` `reply-dm` `thread` `follow` `unfollow` `followers` `following` `block` `unblock` `bookmark` `unbookmark` `delete` `hide-reply` `accept` |
|
||||
| **twitter** | `trending` `search` `timeline` `bookmarks` `post` `download` `profile` `article` `like` `likes` `notifications` `reply` `reply-dm` `thread` `follow` `unfollow` `followers` `following` `block` `unblock` `bookmark` `unbookmark` `delete` `hide-reply` `accept` |
|
||||
| **reddit** | `hot` `frontpage` `popular` `search` `subreddit` `read` `user` `user-posts` `user-comments` `upvote` `upvoted` `save` `saved` `comment` `subscribe` |
|
||||
| **zhihu** | `hot` `search` `question` `download` `follow` `like` `favorite` `comment` `answer` |
|
||||
| **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` `rankings` |
|
||||
| **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` |
|
||||
| **1688** | `search` `item` `assets` `download` `store` |
|
||||
| **gitee** | `trending` `search` `user` |
|
||||
| **gemini** | `new` `ask` `image` `deep-research` `deep-research-result` |
|
||||
| **claude** | `ask` `send` `new` `status` `read` `history` `detail` |
|
||||
| **yuanbao** | `new` `ask` |
|
||||
| **notebooklm** | `status` `list` `open` `current` `get` `history` `summary` `note-list` `notes-get` `source-list` `source-get` `source-fulltext` `source-guide` |
|
||||
| **spotify** | `auth` `status` `play` `pause` `next` `prev` `volume` `search` `queue` `shuffle` `repeat` |
|
||||
| **xianyu** | `search` `item` `chat` |
|
||||
| **xiaoe** | `courses` `detail` `catalog` `play-url` `content` |
|
||||
| **quark** | `ls` `mkdir` `mv` `rename` `rm` `save` `share-tree` |
|
||||
| **uiverse** | `code` `preview` |
|
||||
| **baidu-scholar** | `search` |
|
||||
| **google-scholar** | `search` `cite` `profile` |
|
||||
| **gov-law** | `search` `recent` |
|
||||
| **gov-policy** | `search` `recent` |
|
||||
| **nowcoder** | `hot` `trending` `topics` `recommend` `creators` `companies` `jobs` `search` `suggest` `experience` `referral` `salary` `papers` `practice` `notifications` `detail` |
|
||||
| **wanfang** | `search` |
|
||||
| **hackernews** | `top` `new` `best` `ask` `show` `jobs` `search` `user` |
|
||||
| **xiaoyuzhou** | `auth*` `podcast*` `podcast-episodes*` `episode*` `download*` `transcript*` |
|
||||
|
||||
100+ site surfaces in total — **[→ see all supported sites & commands](./docs/adapters/index.md)**
|
||||
|
||||
`*` `opencli xiaoyuzhou podcast`, `podcast-episodes`, `episode`, `download`, and `transcript` require local Xiaoyuzhou credentials in `~/.opencli/xiaoyuzhou.json`.
|
||||
79+ adapters in total — **[→ see all supported sites & commands](./docs/adapters/index.md)**
|
||||
|
||||
## CLI Hub
|
||||
|
||||
@@ -291,14 +198,14 @@ OpenCLI acts as a universal hub for your existing command-line tools — unified
|
||||
| **obsidian** | Obsidian vault management | `opencli obsidian search query="AI"` |
|
||||
| **docker** | Docker | `opencli docker ps` |
|
||||
| **lark-cli** | Lark/Feishu — messages, docs, calendar, tasks, 200+ commands | `opencli lark-cli calendar +agenda` |
|
||||
| **dws** | DingTalk — cross-platform CLI for DingTalk's full suite, designed for humans and AI agents | `opencli dws msg send --to user "hello"` |
|
||||
| **wecom-cli** | WeCom/企业微信 — CLI for WeCom open platform, for humans and AI agents | `opencli wecom-cli msg send --to user "hello"` |
|
||||
| **dingtalk** | DingTalk — cross-platform CLI for DingTalk's full suite, designed for humans and AI agents | `opencli dingtalk msg send --to user "hello"` |
|
||||
| **wecom** | WeCom/企业微信 — CLI for WeCom open platform, for humans and AI agents | `opencli wecom msg send --to user "hello"` |
|
||||
| **vercel** | Vercel — deploy projects, manage domains, env vars, logs | `opencli vercel deploy --prod` |
|
||||
|
||||
**Register your own** — add any local CLI so AI agents can discover it via `opencli list`:
|
||||
|
||||
```bash
|
||||
opencli external register mycli
|
||||
opencli register mycli
|
||||
```
|
||||
|
||||
### Desktop App Adapters
|
||||
@@ -310,7 +217,7 @@ Control Electron desktop apps directly from the terminal. Each adapter has its o
|
||||
| **Cursor** | Control Cursor IDE — Composer, chat, code extraction | [Doc](./docs/adapters/desktop/cursor.md) |
|
||||
| **Codex** | Drive OpenAI Codex CLI agent headlessly | [Doc](./docs/adapters/desktop/codex.md) |
|
||||
| **Antigravity** | Control Antigravity Ultra from terminal | [Doc](./docs/adapters/desktop/antigravity.md) |
|
||||
| **ChatGPT App** | Automate ChatGPT macOS desktop app | [Doc](./docs/adapters/desktop/chatgpt-app.md) |
|
||||
| **ChatGPT** | Automate ChatGPT macOS desktop app | [Doc](./docs/adapters/desktop/chatgpt.md) |
|
||||
| **ChatWise** | Multi-LLM client (GPT-4, Claude, Gemini) | [Doc](./docs/adapters/desktop/chatwise.md) |
|
||||
| **Notion** | Search, read, write Notion pages | [Doc](./docs/adapters/desktop/notion.md) |
|
||||
| **Discord** | Discord Desktop — messages, channels, servers | [Doc](./docs/adapters/desktop/discord.md) |
|
||||
@@ -330,24 +237,18 @@ OpenCLI supports downloading images, videos, and articles from supported platfor
|
||||
| **douban** | Images | Poster / still image lists |
|
||||
| **pixiv** | Images | Original-quality illustrations, multi-page |
|
||||
| **1688** | Images, Videos | Downloads page-visible product media from item pages |
|
||||
| **xiaoyuzhou** | Audio, Transcript | Downloads episode audio and transcript JSON/text with local credentials |
|
||||
| **zhihu** | Articles (Markdown) | Exports with optional image download |
|
||||
| **weixin** | Articles (Markdown) | WeChat Official Account articles |
|
||||
|
||||
For video downloads, install `yt-dlp` first: `brew install yt-dlp`
|
||||
|
||||
```bash
|
||||
opencli xiaohongshu download "https://www.xiaohongshu.com/search_result/<id>?xsec_token=..." --output ./xhs
|
||||
opencli xiaohongshu download "https://xhslink.com/..." --output ./xhs
|
||||
opencli xiaohongshu download abc123 --output ./xhs
|
||||
opencli bilibili download BV1xxx --output ./bilibili
|
||||
opencli twitter download elonmusk --limit 20 --output ./twitter
|
||||
opencli 1688 download 841141931191 --output ./1688-downloads
|
||||
opencli xiaoyuzhou download 69b3b675772ac2295bfc01d0 --output ./xiaoyuzhou
|
||||
opencli xiaoyuzhou transcript 69dd0c98e2c8be31551f6a33 --output ./xiaoyuzhou-transcripts
|
||||
```
|
||||
|
||||
`opencli xiaoyuzhou download` and `transcript` require local Xiaoyuzhou credentials in `~/.opencli/xiaoyuzhou.json`.
|
||||
|
||||
## Output Formats
|
||||
|
||||
All built-in commands support `--format` / `-f` with `table` (default), `json`, `yaml`, `md`, and `csv`.
|
||||
@@ -376,8 +277,8 @@ opencli follows Unix `sysexits.h` conventions so it integrates naturally with sh
|
||||
|
||||
```bash
|
||||
opencli spotify status || echo "exit $?" # 69 if browser not running
|
||||
opencli gh issue list 2>/dev/null
|
||||
[ $? -eq 77 ] && opencli gh auth login # auto-auth if not logged in
|
||||
opencli github issues 2>/dev/null
|
||||
[ $? -eq 77 ] && opencli github auth # auto-auth if not logged in
|
||||
```
|
||||
|
||||
## Plugins
|
||||
@@ -393,24 +294,25 @@ opencli plugin uninstall my-tool
|
||||
|
||||
| Plugin | Type | Description |
|
||||
|--------|------|-------------|
|
||||
| [opencli-plugin-github-trending](https://github.com/ByteYue/opencli-plugin-github-trending) | JS | GitHub Trending repositories |
|
||||
| [opencli-plugin-hot-digest](https://github.com/ByteYue/opencli-plugin-hot-digest) | JS | Multi-platform trending aggregator |
|
||||
| [opencli-plugin-juejin](https://github.com/Astro-Han/opencli-plugin-juejin) | JS | 稀土掘金 (Juejin) hot articles |
|
||||
| [opencli-plugin-vk](https://github.com/flobo3/opencli-plugin-vk) | JS | VK (VKontakte) wall, feed, and search |
|
||||
| [opencli-plugin-github-trending](https://github.com/ByteYue/opencli-plugin-github-trending) | TS | GitHub Trending repositories |
|
||||
| [opencli-plugin-hot-digest](https://github.com/ByteYue/opencli-plugin-hot-digest) | TS | Multi-platform trending aggregator |
|
||||
| [opencli-plugin-juejin](https://github.com/Astro-Han/opencli-plugin-juejin) | TS | 稀土掘金 (Juejin) hot articles |
|
||||
| [opencli-plugin-vk](https://github.com/flobo3/opencli-plugin-vk) | TS | VK (VKontakte) wall, feed, and search |
|
||||
|
||||
See [Plugins Guide](./docs/guide/plugins.md) for creating your own plugin.
|
||||
|
||||
## For AI Agents (Developer Guide)
|
||||
|
||||
Before writing any adapter code, read the [`opencli-adapter-author` skill](./skills/opencli-adapter-author/SKILL.md). It takes you end-to-end:
|
||||
> **Quick mode**: To generate a single command for a specific page URL, see [opencli-oneshot skill](./skills/opencli-oneshot/SKILL.md) — just a URL + one-line goal, 4 steps done.
|
||||
|
||||
- Recon the site and pick a pattern (SPA / SSR / JSONP / Token / Streaming).
|
||||
- Discover the right endpoint via `opencli browser network`, `eval`, or the interceptor fallback.
|
||||
- Decide auth strategy (`PUBLIC` / `COOKIE` / `HEADER` / `INTERCEPT`).
|
||||
- Run `opencli browser analyze <url>` for one-shot recon, decode response fields, design columns, scaffold with `opencli browser init`.
|
||||
- Verify with `opencli browser verify <site>/<name>` before shipping.
|
||||
> **Full mode**: Before writing any adapter code, read [opencli-explorer skill](./skills/opencli-explorer/SKILL.md). It contains the complete browser exploration workflow, the 5-tier authentication strategy decision tree, and debugging guide.
|
||||
|
||||
For long-lived personal commands that should live in your own Git repo, use a local plugin instead; see [Extending OpenCLI](./docs/guide/extending-opencli.md). Quick private adapters can still live at `~/.opencli/clis/<site>/<name>.js`. Site knowledge (endpoints, field maps, fixtures) accumulates in `~/.opencli/sites/<site>/` so the next adapter for the same site starts from context instead of zero.
|
||||
```bash
|
||||
opencli explore https://example.com --site mysite # Discover APIs + capabilities
|
||||
opencli synthesize mysite # Generate TS adapters
|
||||
opencli generate https://example.com --goal "hot" # One-shot: explore → synthesize → register
|
||||
opencli cascade https://api.example.com/data # Auto-probe: PUBLIC → COOKIE → HEADER
|
||||
```
|
||||
|
||||
## Testing
|
||||
|
||||
@@ -418,10 +320,10 @@ See **[TESTING.md](./TESTING.md)** for how to run and write tests.
|
||||
|
||||
## Troubleshooting
|
||||
|
||||
- **"Extension not connected"** — Ensure the Browser Bridge extension is installed from the [Chrome Web Store](https://chromewebstore.google.com/detail/opencli/ildkmabpimmkaediidaifkhjpohdnifk) and **enabled** in `chrome://extensions`.
|
||||
- **"Extension not connected"** — Ensure the Browser Bridge extension is installed and **enabled** in `chrome://extensions` in Chrome or Chromium.
|
||||
- **"attach failed: Cannot access a chrome-extension:// URL"** — Another extension may be interfering. Try disabling other extensions temporarily.
|
||||
- **Empty data or 'Unauthorized' error** — Your Chrome/Chromium login session may have expired. Navigate to the target site and log in again.
|
||||
- **Node API errors / missing `fetch` / startup crash on old Node** — OpenCLI requires **Node.js >= 21**. Run `node --version`, upgrade Node if needed, then retry.
|
||||
- **Node API errors** — Ensure Node.js >= 20. Some dependencies require modern Node APIs.
|
||||
- **Daemon issues** — Check status: `curl localhost:19825/status` · View logs: `curl localhost:19825/logs`
|
||||
|
||||
## Star History
|
||||
|
||||
+77
-162
@@ -10,31 +10,26 @@
|
||||
|
||||
OpenCLI 可以用同一套 CLI 做三类事情：
|
||||
|
||||
- **直接使用现成适配器**：B站、知乎、小红书、Twitter/X、Reddit、HackerNews 等 [100+ 站点](#内置命令) 开箱即用。
|
||||
- **让 AI Agent 操作任意网站**：在你的 AI Agent（Claude Code、Cursor 等）中安装 `opencli-adapter-author` skill，Agent 就能用你的已登录浏览器导航、点击、输入、提取任意网页内容。
|
||||
- **把新网站写成 CLI**：用 `opencli browser` 原语 + `opencli-adapter-author` skill，从站点侦察、API 发现、字段解码到 `opencli browser verify` 一条龙。
|
||||
- **直接使用现成适配器**：B站、知乎、小红书、Twitter/X、Reddit、HackerNews 等 [79+ 站点](#内置命令) 开箱即用。
|
||||
- **直接驱动浏览器**：用 `opencli browser` 让 AI Agent 实时点击、输入、提取、截图、检查页面状态。
|
||||
- **把新网站生成成 CLI**：通过 `explore`、`synthesize`、`generate`、`cascade` 从真实页面行为推导出新的适配器。
|
||||
|
||||
除了网站能力，OpenCLI 还是一个 **CLI 枢纽**：你可以把 `gh`、`docker` 等本地工具统一注册到 `opencli` 下，也可以通过桌面端适配器控制 Cursor、Codex、Antigravity、ChatGPT、Notion 等 Electron 应用。
|
||||
|
||||
## 亮点
|
||||
## 为什么是 OpenCLI
|
||||
|
||||
- **桌面应用控制** — 通过 CDP 直接在终端驱动 Electron 应用（Cursor、Codex、ChatGPT、Notion 等）。
|
||||
- **AI Agent 浏览器自动化** — 安装 `opencli-adapter-author` skill，你的 AI Agent 就能操作任意网站：导航、点击、输入、提取、截图——全部通过你的已登录 Chrome 会话完成。
|
||||
- **网站 → CLI** — 把任何网站变成确定性 CLI：100+ 站点能力已注册，或用 `opencli-adapter-author` skill + `opencli browser verify` 自己写。
|
||||
- **账号安全** — 复用 Chrome/Chromium 登录态，凭证永远不会离开浏览器。
|
||||
- **面向 AI Agent** — 一个 skill 带你走完站点侦察、API 发现、字段解码、适配器编写、验证的全流程。
|
||||
- **CLI 枢纽** — 统一发现、自动安装、纯透传任何外部 CLI（gh、docker、obsidian 等）。
|
||||
- **零 LLM 成本** — 运行时不消耗模型 token，跑 10,000 次也不花一分钱。
|
||||
- **确定性输出** — 相同命令，相同输出结构，每次一致。可管道、可脚本、CI 友好。
|
||||
- **同一个心智模型**：网站、浏览器自动化、Electron 应用、本地 CLI 都走同一个入口。
|
||||
- **复用真实会话**：浏览器命令直接使用你已经登录的 Chrome/Chromium，而不是重新造一套认证。
|
||||
- **输出稳定**：适配器命令返回固定结构，适合 shell、脚本、CI 和 AI Agent 工具调用。
|
||||
- **面向 AI Agent**：`browser` 负责实时操作，`explore` 负责探索接口，`synthesize` 负责生成适配器，`cascade` 负责探测认证路径。
|
||||
- **运行成本低**：已有命令运行时不消耗模型 token。
|
||||
- **天然可扩展**：既能用内置能力，也能注册本地 CLI，或直接往 `clis/` 丢 `.ts` 适配器。
|
||||
|
||||
## 快速开始
|
||||
|
||||
### 1. 安装 OpenCLI
|
||||
|
||||
OpenCLI 要求 **Node.js >= 21**。
|
||||
|
||||
```bash
|
||||
node --version
|
||||
npm install -g @jackwener/opencli
|
||||
```
|
||||
|
||||
@@ -42,11 +37,7 @@ npm install -g @jackwener/opencli
|
||||
|
||||
OpenCLI 通过轻量 Browser Bridge 扩展和本地微型 daemon 与 Chrome/Chromium 通信。daemon 会按需自动启动。
|
||||
|
||||
**方式 A — Chrome Web Store（推荐）：**
|
||||
在 [Chrome Web Store](https://chromewebstore.google.com/detail/opencli/ildkmabpimmkaediidaifkhjpohdnifk) 安装 **OpenCLI** 扩展。
|
||||
|
||||
**方式 B — 手动安装：**
|
||||
1. 到 GitHub [Releases 页面](https://github.com/jackwener/opencli/releases) 下载最新的 `opencli-extension-v{version}.zip`。
|
||||
1. 到 GitHub [Releases 页面](https://github.com/jackwener/opencli/releases) 下载最新的 `opencli-extension.zip`。
|
||||
2. 解压后打开 `chrome://extensions`，启用 **开发者模式**。
|
||||
3. 点击 **加载已解压的扩展程序**，选择解压后的目录。
|
||||
|
||||
@@ -54,6 +45,7 @@ OpenCLI 通过轻量 Browser Bridge 扩展和本地微型 daemon 与 Chrome/Chro
|
||||
|
||||
```bash
|
||||
opencli doctor
|
||||
opencli daemon status
|
||||
```
|
||||
|
||||
### 4. 跑第一个命令
|
||||
@@ -70,26 +62,17 @@ opencli bilibili hot --limit 5
|
||||
|
||||
- `opencli list` 查看当前所有命令
|
||||
- `opencli <site> <command>` 调用内置或生成好的适配器
|
||||
- `opencli external register mycli` 把本地 CLI 接入同一发现入口
|
||||
- `opencli doctor` 处理浏览器连通性问题
|
||||
|
||||
## 扩展 OpenCLI
|
||||
|
||||
如果你想新增自己的命令，先看 [扩展 OpenCLI](./docs/zh/guide/extending-opencli.md)。README 只保留入口；目录结构、源码管理方式和安装命令放在文档里。
|
||||
|
||||
| 需求 | 推荐路径 |
|
||||
|------|----------|
|
||||
| 把个人网站命令放在自己的 Git repo | `opencli plugin create` + `opencli plugin install file://...` |
|
||||
| 快速写一个本机私人 adapter | `opencli browser init <site>/<command>`，放在 `~/.opencli/clis/` |
|
||||
| 本地修改官方 adapter | `opencli adapter eject <site>` + `opencli adapter reset <site>` |
|
||||
| 发布或安装第三方命令 | `opencli plugin install github:user/repo` |
|
||||
| 包装已有本机 binary | `opencli external register <name>` |
|
||||
- `opencli register mycli` 把本地 CLI 接入同一发现入口
|
||||
- `opencli doctor` / `opencli daemon status` 处理浏览器连通性问题
|
||||
|
||||
## 给 AI Agent
|
||||
|
||||
OpenCLI 的 browser 命令是给 AI Agent 用的——不是手动执行的。把 skill 安装到你的 AI Agent（Claude Code、Cursor 等）中，Agent 就能用你的已登录 Chrome 会话替你操作网站。
|
||||
按任务类型，AI Agent 有两个不同入口：
|
||||
|
||||
### 安装 skill
|
||||
- [`skills/opencli-explorer/SKILL.md`](./skills/opencli-explorer/SKILL.md)：适配器创建入口，支持全自动生成（`opencli generate <url>`）和手动探索两种流程。
|
||||
- [`skills/opencli-browser/SKILL.md`](./skills/opencli-browser/SKILL.md)：底层控制入口，适合实时操作页面、debug 和人工介入。
|
||||
|
||||
安装全部 OpenCLI skills：
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli
|
||||
@@ -98,68 +81,40 @@ npx skills add jackwener/opencli
|
||||
或只装需要的 skill：
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli --skill opencli-adapter-author
|
||||
npx skills add jackwener/opencli --skill opencli-autofix
|
||||
npx skills add jackwener/opencli --skill opencli-browser
|
||||
npx skills add jackwener/opencli --skill opencli-usage
|
||||
npx skills add jackwener/opencli --skill smart-search
|
||||
npx skills add jackwener/opencli --skill opencli-browser
|
||||
npx skills add jackwener/opencli --skill opencli-explorer
|
||||
npx skills add jackwener/opencli --skill opencli-oneshot
|
||||
```
|
||||
|
||||
### 选择哪个 skill
|
||||
实际使用上：
|
||||
|
||||
| Skill | 适用场景 | 你对 AI Agent 说的话 |
|
||||
|-------|---------|-------------------|
|
||||
| **opencli-adapter-author** | 实时操作任意网站，或为新站点写可复用适配器 | "帮我看看小红书的通知" / "帮我做一个抖音热门的适配器" / "帮我做一个抓取这个页面热帖的命令" |
|
||||
| **opencli-autofix** | 内置命令失败时修复已有适配器 | "`opencli zhihu hot` 返回空了，修一下" |
|
||||
| **opencli-browser** | 浏览器自动化参考文档 | "用浏览器命令抓取这个页面" |
|
||||
| **opencli-usage** | 所有命令和站点的快速参考 | "OpenCLI 有哪些 Twitter 相关的命令？" |
|
||||
| **smart-search** | 在现有 OpenCLI 能力里搜索 | "帮我找个 B 站热门相关的适配器" |
|
||||
- 需要把某个站点收成可复用命令时，优先走 `opencli-explorer`（涵盖自动和手动两种路径）
|
||||
- 需要直接检查页面、操作页面时，再走 `opencli-browser`
|
||||
|
||||
### 工作原理
|
||||
|
||||
安装 `opencli-adapter-author` skill 后，你的 AI Agent 可以：
|
||||
|
||||
1. **导航**到任意 URL，使用你的已登录浏览器
|
||||
2. **读取**页面内容——通过结构化 DOM 快照（不是截图）
|
||||
3. **交互**——点击按钮、填写表单、选择选项、按键
|
||||
4. **提取**页面数据或拦截网络 API 响应
|
||||
5. **等待**元素、文本或页面跳转
|
||||
|
||||
Agent 在内部自动处理所有 `opencli browser` 命令——你只需用自然语言描述想做的事。
|
||||
|
||||
**Skill 参考文档：**
|
||||
- [`skills/opencli-adapter-author/SKILL.md`](./skills/opencli-adapter-author/SKILL.md) — 浏览器操作 + 适配器编写，全流程
|
||||
- [`skills/opencli-autofix/SKILL.md`](./skills/opencli-autofix/SKILL.md) — 修复已有适配器
|
||||
- [`skills/opencli-browser/SKILL.md`](./skills/opencli-browser/SKILL.md) — 浏览器自动化参考
|
||||
- [`skills/opencli-usage/SKILL.md`](./skills/opencli-usage/SKILL.md) — 命令和站点参考
|
||||
- [`skills/smart-search/SKILL.md`](./skills/smart-search/SKILL.md) — 能力搜索
|
||||
|
||||
`browser` 可用命令包括：`open`、`state`、`click`、`type`、`select`、`keys`、`wait`、`get`、`find`、`extract`、`frames`、`screenshot`、`scroll`、`back`、`eval`、`network`、`tab list`、`tab new`、`tab select`、`tab close`、`init`、`verify`、`close`。
|
||||
|
||||
`opencli browser open <url>` 和 `opencli browser tab new [url]` 都会返回 target ID。`opencli browser tab list` 用来查看当前已存在 tab 的 target ID，再通过 `--tab <targetId>` 把命令明确路由到某个 tab。`tab new` 只会新建 tab，不会改变默认浏览器目标；只有显式执行 `tab select <targetId>`，才会把该 tab 设为后续未指定 target 的 `opencli browser ...` 命令的默认目标。
|
||||
`browser` 可用命令包括：`open`、`state`、`click`、`type`、`select`、`keys`、`wait`、`get`、`screenshot`、`scroll`、`back`、`eval`、`network`、`init`、`verify`、`close`。
|
||||
|
||||
## 核心概念
|
||||
|
||||
### `browser`：AI Agent 的浏览器控制层
|
||||
### `browser`：实时操作
|
||||
|
||||
`opencli browser` 命令是 AI Agent 操作网站的底层原语。你不需要手动运行这些命令——把 `opencli-adapter-author` skill 安装到你的 AI Agent 中，用自然语言描述你想做的事，Agent 会自动处理浏览器操作。
|
||||
|
||||
比如你告诉 Agent：*"帮我看看小红书的通知"*——Agent 会在底层调用 `opencli browser open`、`state`、`click` 等命令。
|
||||
当任务本身就是交互式页面操作时，使用 `opencli browser` 直接驱动浏览器。
|
||||
|
||||
### 内置适配器：稳定命令
|
||||
|
||||
当某个站点能力已经存在时，优先使用 `opencli hackernews top`、`opencli reddit hot` 这类稳定命令。这些命令是确定性的，无需浏览器——人类和 AI Agent 都可以直接使用。
|
||||
当某个站点能力已经存在时，优先使用 `opencli hackernews top`、`opencli reddit hot` 这类稳定命令，而不是重新走一遍浏览器操作。
|
||||
|
||||
### 为新站点写适配器
|
||||
### `explore` / `synthesize` / `generate`：生成新的 CLI
|
||||
|
||||
当你需要的网站还没覆盖时，用 `opencli-adapter-author` skill，它会把 Agent 带到闭环：
|
||||
当你需要的网站还没覆盖时：
|
||||
|
||||
1. 侦察站点，分类 pattern（SPA / SSR / JSONP / Token / Streaming）
|
||||
2. 发现目标 endpoint——network 精读、initial state、bundle 搜索、token 溯源，或 interceptor 兜底
|
||||
3. 定认证策略——`PUBLIC` / `COOKIE` / `HEADER` / `INTERCEPT`
|
||||
4. 字段解码 + 设计输出列
|
||||
5. `opencli browser analyze <url>` 一步侦察，再 `opencli browser init <site>/<name>` → 写适配器 → `opencli browser verify <site>/<name>`
|
||||
6. 把站点知识沉到 `~/.opencli/sites/<site>/`，下次写同站点的其他命令直接吃缓存
|
||||
- `explore` 负责观察页面、网络请求和能力边界
|
||||
- `synthesize` 负责把探索结果转成 evaluate-based YAML 适配器
|
||||
- `generate` 负责跑通 verified generation 主链路，最后要么给出可直接使用的命令，要么返回结构化的阻塞原因 / 人工介入结果
|
||||
|
||||
### `cascade`：认证策略探测
|
||||
|
||||
用 `cascade` 去判断某个能力应该优先走公开接口、Cookie 还是自定义 Header，而不是一开始就把适配器写死。
|
||||
|
||||
### CLI 枢纽与桌面端适配器
|
||||
|
||||
@@ -170,43 +125,15 @@ OpenCLI 不只是网站 CLI，还可以：
|
||||
|
||||
## 前置要求
|
||||
|
||||
- **Node.js**: >= 21.0.0（标准 npm 安装路径要求）
|
||||
- **Bun**: >= 1.0（可选替代运行时）
|
||||
- **Node.js**: >= 20.0.0
|
||||
- 浏览器型命令需要 Chrome 或 Chromium 处于运行中，并已登录目标网站
|
||||
|
||||
> **重要**：浏览器型命令直接复用你的 Chrome/Chromium 登录态。如果拿到空数据或出现权限类失败，先确认目标站点已经在浏览器里打开并完成登录。
|
||||
|
||||
## 配置
|
||||
|
||||
| 变量 | 默认值 | 说明 |
|
||||
|------|--------|------|
|
||||
| `OPENCLI_DAEMON_PORT` | `19825` | daemon-extension 通信端口 |
|
||||
| `OPENCLI_WINDOW_FOCUSED` | `false` | 设为 `1` 时 automation 窗口在前台打开（适合调试）。`--focus` 标志会设置此变量 |
|
||||
| `OPENCLI_LIVE` | `false` | 设为 `1` 时 adapter 命令执行完后保留 automation 窗口不关闭（适合检查页面）。`--live` 标志会设置此变量 |
|
||||
| `OPENCLI_BROWSER_CONNECT_TIMEOUT` | `30` | 浏览器连接超时（秒） |
|
||||
| `OPENCLI_BROWSER_COMMAND_TIMEOUT` | `60` | 单个浏览器命令超时（秒） |
|
||||
| `OPENCLI_CDP_ENDPOINT` | — | Chrome DevTools Protocol 端点，用于远程浏览器或 Electron 应用 |
|
||||
| `OPENCLI_CDP_TARGET` | — | 按 URL 子串过滤 CDP target（如 `detail.1688.com`） |
|
||||
| `OPENCLI_VERBOSE` | `false` | 启用详细日志（`-v` 也可以） |
|
||||
| `DEBUG_SNAPSHOT` | — | 设为 `1` 输出 DOM 快照调试信息 |
|
||||
|
||||
`--focus` 同时适用于 `opencli browser *` 和浏览器型 adapter 命令。`--live` 主要是给 adapter 命令用的：`browser` 子命令本来就会一直保留 automation window，直到你手动执行 `opencli browser close` 或等空闲超时。
|
||||
|
||||
## 更新
|
||||
|
||||
```bash
|
||||
npm install -g @jackwener/opencli@latest
|
||||
|
||||
# 如果你在用打包发布的 OpenCLI skills，也一起刷新
|
||||
npx skills add jackwener/opencli
|
||||
```
|
||||
|
||||
如果你只装了部分 skill，也可以只刷新自己在用的：
|
||||
|
||||
```bash
|
||||
npx skills add jackwener/opencli --skill opencli-adapter-author
|
||||
npx skills add jackwener/opencli --skill opencli-autofix
|
||||
npx skills add jackwener/opencli --skill smart-search
|
||||
```
|
||||
|
||||
## 面向开发者
|
||||
@@ -232,12 +159,12 @@ npm link
|
||||
|
||||
| 站点 | 命令 | 模式 |
|
||||
|------|------|------|
|
||||
| **twitter** | `trending` `search` `timeline` `tweets` `lists` `list-tweets` `list-add` `list-remove` `bookmarks` `profile` `thread` `following` `followers` `notifications` `post` `reply` `delete` `like` `likes` `article` `follow` `unfollow` `bookmark` `unbookmark` `download` `accept` `reply-dm` `block` `unblock` `hide-reply` | 浏览器 |
|
||||
| **twitter** | `trending` `bookmarks` `profile` `search` `timeline` `thread` `following` `followers` `notifications` `post` `reply` `delete` `like` `article` `follow` `unfollow` `bookmark` `unbookmark` `download` `accept` `reply-dm` `block` `unblock` `hide-reply` | 浏览器 |
|
||||
| **reddit** | `hot` `frontpage` `popular` `search` `subreddit` `read` `user` `user-posts` `user-comments` `upvote` `save` `comment` `subscribe` `saved` `upvoted` | 浏览器 |
|
||||
| **tieba** | `hot` `posts` `search` `read` | 浏览器 |
|
||||
| **hupu** | `hot` `search` `detail` `mentions` `reply` `like` `unlike` | 浏览器 |
|
||||
| **cursor** | `status` `send` `read` `new` `dump` `composer` `model` `extract-code` `ask` `screenshot` `history` `export` | 桌面端 |
|
||||
| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `subtitle` `video` `comments` `dynamic` `ranking` `following` `user-videos` `download` | 浏览器 |
|
||||
| **bilibili** | `hot` `search` `me` `favorite` `history` `feed` `subtitle` `dynamic` `ranking` `following` `user-videos` `download` | 浏览器 |
|
||||
| **codex** | `status` `send` `read` `new` `dump` `extract-diff` `model` `ask` `screenshot` `history` `export` | 桌面端 |
|
||||
| **chatwise** | `status` `new` `send` `read` `ask` `model` `history` `export` `screenshot` | 桌面端 |
|
||||
| **doubao** | `status` `new` `send` `read` `ask` `history` `detail` `meeting-summary` `meeting-transcript` | 浏览器 |
|
||||
@@ -246,23 +173,16 @@ npm link
|
||||
| **discord-app** | `status` `send` `read` `channels` `servers` `search` `members` | 桌面端 |
|
||||
| **v2ex** | `hot` `latest` `topic` `node` `user` `member` `replies` `nodes` `daily` `me` `notifications` | 公开 / 浏览器 |
|
||||
| **xueqiu** | `feed` `hot-stock` `hot` `search` `stock` `comments` `watchlist` `earnings-date` `fund-holdings` `fund-snapshot` | 浏览器 |
|
||||
| **antigravity** | `status` `send` `read` `new` `dump` `extract-code` `model` `watch` `serve` | 桌面端 |
|
||||
| **chatgpt-app** | `status` `new` `send` `read` `ask` `model` | 桌面端 |
|
||||
| **xiaohongshu** | `search` `note` `comments` `notifications` `feed` `user` `download` `publish` `creator-notes` `creator-note-detail` `creator-notes-summary` `creator-profile` `creator-stats` | 浏览器 |
|
||||
| **antigravity** | `status` `send` `read` `new` `dump` `extract-code` `model` `watch` | 桌面端 |
|
||||
| **chatgpt** | `status` `new` `send` `read` `ask` `model` | 桌面端 |
|
||||
| **xiaohongshu** | `search` `notifications` `feed` `user` `download` `publish` `creator-notes` `creator-note-detail` `creator-notes-summary` `creator-profile` `creator-stats` | 浏览器 |
|
||||
| **xiaoe** | `courses` `detail` `catalog` `play-url` `content` | 浏览器 |
|
||||
| **quark** | `ls` `mkdir` `mv` `rename` `rm` `save` `share-tree` | 浏览器 |
|
||||
| **uiverse** | `code` `preview` | 浏览器 |
|
||||
| **apple-podcasts** | `search` `episodes` `top` | 公开 |
|
||||
| **baidu-scholar** | `search` | 公开 |
|
||||
| **google-scholar** | `search` `cite` `profile` | 公开 |
|
||||
| **gov-law** | `search` `recent` | 公开 |
|
||||
| **gov-policy** | `search` `recent` | 公开 |
|
||||
| **nowcoder** | `hot` `trending` `topics` `recommend` `creators` `companies` `jobs` `search` `suggest` `experience` `referral` `salary` `papers` `practice` `notifications` `detail` | 公开 / 浏览器 |
|
||||
| **wanfang** | `search` | 公开 |
|
||||
| **xiaoyuzhou** | `podcast*` `podcast-episodes*` `episode*` `download*` `transcript*` `auth` | 本地凭证 |
|
||||
| **xiaoyuzhou** | `podcast` `podcast-episodes` `episode` | 公开 |
|
||||
| **zhihu** | `hot` `search` `question` `download` `follow` `like` `favorite` `comment` `answer` | 浏览器 |
|
||||
| **weixin** | `download` | 浏览器 |
|
||||
| **youtube** | `search` `video` `transcript` `comments` `channel` `playlist` `feed` `history` `watch-later` `subscriptions` `like` `unlike` `subscribe` `unsubscribe` | 浏览器 |
|
||||
| **youtube** | `search` `video` `transcript` | 浏览器 |
|
||||
| **boss** | `search` `detail` `recommend` `joblist` `greet` `batchgreet` `send` `chatlist` `chatmsg` `invite` `mark` `exchange` `resume` `stats` | 浏览器 |
|
||||
| **coupang** | `search` `add-to-cart` | 浏览器 |
|
||||
| **bbc** | `news` | 公共 API |
|
||||
@@ -284,7 +204,7 @@ npm link
|
||||
| **sinafinance** | `news` | 🌐 公开 |
|
||||
| **barchart** | `quote` `options` `greeks` `flow` | 浏览器 |
|
||||
| **chaoxing** | `assignments` `exams` | 浏览器 |
|
||||
| **grok** | `ask` `image` | 浏览器 |
|
||||
| **grok** | `ask` | 浏览器 |
|
||||
| **hf** | `top` | 公开 |
|
||||
| **jike** | `feed` `search` `create` `like` `comment` `repost` `notifications` `post` `topic` `user` | 浏览器 |
|
||||
| **jimeng** | `generate` `history` | 浏览器 |
|
||||
@@ -296,11 +216,9 @@ npm link
|
||||
| **douban** | `search` `top250` `subject` `photos` `download` `marks` `reviews` `movie-hot` `book-hot` | 浏览器 |
|
||||
| **facebook** | `feed` `profile` `search` `friends` `groups` `events` `notifications` `memories` `add-friend` `join-group` | 浏览器 |
|
||||
| **google** | `news` `search` `suggest` `trends` | 公开 |
|
||||
| **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` `rankings` | 浏览器 |
|
||||
| **amazon** | `bestsellers` `search` `product` `offer` `discussion` `movers-shakers` `new-releases` | 浏览器 |
|
||||
| **1688** | `search` `item` `assets` `download` `store` | 浏览器 |
|
||||
| **gitee** | `trending` `search` `user` | 公开 / 浏览器 |
|
||||
| **gemini** | `new` `ask` `image` `deep-research` `deep-research-result` | 浏览器 |
|
||||
| **claude** | `ask` `send` `new` `status` `read` `history` `detail` | 浏览器 |
|
||||
| **spotify** | `auth` `status` `play` `pause` `next` `prev` `volume` `search` `queue` `shuffle` `repeat` | OAuth API |
|
||||
| **notebooklm** | `status` `list` `open` `current` `get` `history` `summary` `note-list` `notes-get` `source-list` `source-get` `source-fulltext` `source-guide` | 浏览器 |
|
||||
| **36kr** | `news` `hot` `search` `article` | 公开 / 浏览器 |
|
||||
@@ -318,9 +236,7 @@ npm link
|
||||
| **douyin** | `videos` `publish` `drafts` `draft` `delete` `stats` `profile` `update` `hashtag` `location` `activities` `collections` | 浏览器 |
|
||||
| **yuanbao** | `new` `ask` | 浏览器 |
|
||||
|
||||
100+ 站点能力 — **[→ 查看完整命令列表](./docs/adapters/index.md)**
|
||||
|
||||
`*` `opencli xiaoyuzhou podcast`、`podcast-episodes`、`episode`、`download`、`transcript` 需要本地小宇宙凭证：`~/.opencli/xiaoyuzhou.json`。
|
||||
79+ 适配器 — **[→ 查看完整命令列表](./docs/adapters/index.md)**
|
||||
|
||||
### 外部 CLI 枢纽
|
||||
|
||||
@@ -332,8 +248,8 @@ OpenCLI 也可以作为你现有命令行工具的统一入口，负责发现、
|
||||
| **obsidian** | Obsidian 仓库管理 | `opencli obsidian search query="AI"` |
|
||||
| **docker** | Docker 命令行工具 | `opencli docker ps` |
|
||||
| **lark-cli** | 飞书 CLI — 消息、文档、日历、任务，200+ 命令 | `opencli lark-cli calendar +agenda` |
|
||||
| **dws** | 钉钉 CLI — 钉钉全套产品能力的跨平台命令行工具，支持人类和 AI Agent 使用 | `opencli dws msg send --to user "hello"` |
|
||||
| **wecom-cli** | 企业微信 CLI — 企业微信开放平台命令行工具，支持人类和 AI Agent 使用 | `opencli wecom-cli msg send --to user "hello"` |
|
||||
| **dingtalk** | 钉钉 CLI — 钉钉全套产品能力的跨平台命令行工具，支持人类和 AI Agent 使用 | `opencli dingtalk msg send --to user "hello"` |
|
||||
| **wecom** | 企业微信 CLI — 企业微信开放平台命令行工具，支持人类和 AI Agent 使用 | `opencli wecom msg send --to user "hello"` |
|
||||
| **vercel** | Vercel — 部署项目、管理域名、环境变量、日志 | `opencli vercel deploy --prod` |
|
||||
|
||||
**零配置透传**：OpenCLI 会把你的输入原样转发给底层二进制，保留原生 stdout / stderr 行为。
|
||||
@@ -355,7 +271,7 @@ opencli register mycli
|
||||
| **Cursor** | 控制 Cursor IDE — Composer、对话、代码提取等 | [Doc](./docs/adapters/desktop/cursor.md) |
|
||||
| **Codex** | 在后台（无头）驱动 OpenAI Codex CLI Agent | [Doc](./docs/adapters/desktop/codex.md) |
|
||||
| **Antigravity** | 在终端直接控制 Antigravity Ultra | [Doc](./docs/adapters/desktop/antigravity.md) |
|
||||
| **ChatGPT App** | 自动化操作 ChatGPT macOS 桌面客户端 | [Doc](./docs/adapters/desktop/chatgpt-app.md) |
|
||||
| **ChatGPT** | 自动化操作 ChatGPT macOS 桌面客户端 | [Doc](./docs/adapters/desktop/chatgpt.md) |
|
||||
| **ChatWise** | 多 LLM 客户端（GPT-4、Claude、Gemini） | [Doc](./docs/adapters/desktop/chatwise.md) |
|
||||
| **Notion** | 搜索、读取、写入 Notion 页面 | [Doc](./docs/adapters/desktop/notion.md) |
|
||||
| **Discord** | Discord 桌面版 — 消息、频道、服务器 | [Doc](./docs/adapters/desktop/discord.md) |
|
||||
@@ -374,7 +290,6 @@ OpenCLI 支持从各平台下载图片、视频和文章。
|
||||
| **Twitter/X** | 图片、视频 | 从用户媒体页或单条推文下载 |
|
||||
| **Pixiv** | 图片 | 下载原始画质插画，支持多页作品 |
|
||||
| **1688** | 图片、视频 | 下载商品页中可见的商品素材 |
|
||||
| **小宇宙** | 音频、转录 | 使用本地凭证下载单集音频和转录 JSON / 文本 |
|
||||
| **知乎** | 文章（Markdown） | 导出文章，可选下载图片到本地 |
|
||||
| **微信公众号** | 文章（Markdown） | 导出微信公众号文章为 Markdown |
|
||||
| **豆瓣** | 图片 | 下载电影条目的海报 / 剧照图片 |
|
||||
@@ -394,8 +309,7 @@ brew install yt-dlp
|
||||
|
||||
```bash
|
||||
# 下载小红书笔记中的图片/视频
|
||||
opencli xiaohongshu download "https://www.xiaohongshu.com/search_result/<id>?xsec_token=..." --output ./xhs
|
||||
opencli xiaohongshu download "https://xhslink.com/..." --output ./xhs
|
||||
opencli xiaohongshu download abc123 --output ./xhs
|
||||
|
||||
# 下载B站视频（需要 yt-dlp）
|
||||
opencli bilibili download BV1xxx --output ./bilibili
|
||||
@@ -413,12 +327,6 @@ opencli douban download 30382501 --output ./douban
|
||||
# 下载 1688 商品页中的图片 / 视频素材
|
||||
opencli 1688 download 841141931191 --output ./1688-downloads
|
||||
|
||||
# 下载小宇宙单集音频
|
||||
opencli xiaoyuzhou download 69b3b675772ac2295bfc01d0 --output ./xiaoyuzhou
|
||||
|
||||
# 下载小宇宙单集转录
|
||||
opencli xiaoyuzhou transcript 69dd0c98e2c8be31551f6a33 --output ./xiaoyuzhou-transcripts
|
||||
|
||||
# 导出知乎文章为 Markdown
|
||||
opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --output ./zhihu
|
||||
|
||||
@@ -429,8 +337,6 @@ opencli zhihu download "https://zhuanlan.zhihu.com/p/xxx" --download-images
|
||||
opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||||
```
|
||||
|
||||
`opencli xiaoyuzhou download` 和 `transcript` 需要本地小宇宙凭证：`~/.opencli/xiaoyuzhou.json`。
|
||||
|
||||
|
||||
|
||||
## 输出格式
|
||||
@@ -475,7 +381,7 @@ esac
|
||||
|
||||
## 插件
|
||||
|
||||
通过社区贡献的插件扩展 OpenCLI。插件使用与内置命令相同的 JS 格式，启动时自动发现。
|
||||
通过社区贡献的插件扩展 OpenCLI。插件使用与内置命令相同的 YAML/TS 格式，启动时自动发现。
|
||||
|
||||
```bash
|
||||
opencli plugin install github:user/opencli-plugin-my-tool # 安装
|
||||
@@ -489,10 +395,9 @@ opencli plugin uninstall my-tool # 卸载
|
||||
|
||||
| 插件 | 类型 | 描述 |
|
||||
|------|------|------|
|
||||
| [opencli-plugin-github-trending](https://github.com/ByteYue/opencli-plugin-github-trending) | JS | GitHub Trending 仓库 |
|
||||
| [opencli-plugin-hot-digest](https://github.com/ByteYue/opencli-plugin-hot-digest) | JS | 多平台热榜聚合 |
|
||||
| [opencli-plugin-juejin](https://github.com/Astro-Han/opencli-plugin-juejin) | JS | 稀土掘金热门文章 |
|
||||
| [opencli-plugin-vk](https://github.com/flobo3/opencli-plugin-vk) | JS | VK (VKontakte) 动态、信息流和搜索 |
|
||||
| [opencli-plugin-github-trending](https://github.com/ByteYue/opencli-plugin-github-trending) | YAML | GitHub Trending 仓库 |
|
||||
| [opencli-plugin-hot-digest](https://github.com/ByteYue/opencli-plugin-hot-digest) | TS | 多平台热榜聚合 |
|
||||
| [opencli-plugin-juejin](https://github.com/Astro-Han/opencli-plugin-juejin) | YAML | 稀土掘金热门文章 |
|
||||
|
||||
详见 [插件指南](./docs/zh/guide/plugins.md) 了解如何创建自己的插件。
|
||||
|
||||
@@ -500,26 +405,36 @@ opencli plugin uninstall my-tool # 卸载
|
||||
|
||||
如果你是一个被要求查阅代码并编写新 `opencli` 适配器的 AI，请遵守以下工作流。
|
||||
|
||||
在动代码前，先读 [`opencli-adapter-author` skill](./skills/opencli-adapter-author/SKILL.md)。它把整个流程串起来：
|
||||
> **快速模式**：只想为某个页面快速生成一个命令？看 [opencli-oneshot skill](./skills/opencli-oneshot/SKILL.md) — 给一个 URL + 一句话描述，4 步搞定。
|
||||
|
||||
- 侦察站点，选定 pattern（SPA / SSR / JSONP / Token / Streaming）
|
||||
- 用 `opencli browser network`、`eval`、interceptor 等找到目标 endpoint
|
||||
- 定认证策略（`PUBLIC` / `COOKIE` / `HEADER` / `INTERCEPT`）
|
||||
- 先用 `opencli browser analyze <url>` 一步侦察，再字段解码、设计 columns、`opencli browser init` 生成骨架
|
||||
- 交付前用 `opencli browser verify <site>/<name>` 验证
|
||||
> **完整模式**：在编写任何新代码前，先阅读 [opencli-explorer skill](./skills/opencli-explorer/SKILL.md)。它包含完整的适配器探索开发指南、API 探测流程、5级认证策略以及常见陷阱。
|
||||
|
||||
在仓库外写的私有适配器放到 `~/.opencli/clis/<site>/<name>.js`；每个站点的 endpoint、字段映射、抓包样本会累积在 `~/.opencli/sites/<site>/`，下次写同站点的其他命令可以直接复用。
|
||||
```bash
|
||||
# 1. Deep Explore — 网络拦截 → 响应分析 → 能力推理 → 框架检测
|
||||
opencli explore https://example.com --site mysite
|
||||
|
||||
# 2. Synthesize — 从探索成果物生成 evaluate-based TS 适配器
|
||||
opencli synthesize mysite
|
||||
|
||||
# 3. Generate — 一键完成：探索 → 合成 → 注册
|
||||
opencli generate https://example.com --goal "hot"
|
||||
|
||||
# 4. Strategy Cascade — 自动降级探测：PUBLIC → COOKIE → HEADER
|
||||
opencli cascade https://api.example.com/data
|
||||
```
|
||||
|
||||
探索结果输出到 `.opencli/explore/<site>/`。
|
||||
|
||||
## 常见问题排查
|
||||
|
||||
- **"Extension not connected" 报错**
|
||||
- 确保你已从 [Chrome Web Store](https://chromewebstore.google.com/detail/opencli/ildkmabpimmkaediidaifkhjpohdnifk) 安装 OpenCLI 扩展，且在 `chrome://extensions` 中**已启用**。
|
||||
- 确保你当前的 Chrome 或 Chromium 已安装且**开启了** opencli Browser Bridge 扩展（在 `chrome://extensions` 中检查）。
|
||||
- **"attach failed: Cannot access a chrome-extension:// URL" 报错**
|
||||
- 其他 Chrome/Chromium 扩展（如 youmind、New Tab Override 或 AI 助手类扩展）可能产生冲突。请尝试**暂时禁用其他扩展**后重试。
|
||||
- **返回空数据，或者报错 "Unauthorized"**
|
||||
- Chrome/Chromium 里的登录态可能已经过期。请打开当前页面，在新标签页重新手工登录或刷新该页面。
|
||||
- **Node API 错误 / 缺少 `fetch` / 旧 Node 启动即崩**
|
||||
- OpenCLI 要求 **Node.js >= 21**。先执行 `node --version`，如果版本过低先升级，再重试命令。
|
||||
- **Node API 错误 (如 parseArgs, fs 等)**
|
||||
- 确保 Node.js 版本 `>= 20`。
|
||||
- **Daemon 问题**
|
||||
- 检查 daemon 状态：`curl localhost:19825/status`
|
||||
- 查看扩展日志：`curl localhost:19825/logs`
|
||||
|
||||
+17
-18
@@ -30,15 +30,12 @@ tests/
|
||||
├── smoke/
|
||||
│ └── api-health.test.ts # 外部 API、adapter 定义、命令注册健康检查
|
||||
src/
|
||||
├── **/*.test.ts # 单元测试（unit project）
|
||||
clis/
|
||||
└── **/*.test.{ts,js} # adapter 测试（adapter project）
|
||||
└── **/*.test.ts # 单元测试（当前 32 个文件）
|
||||
```
|
||||
|
||||
| 层 | 位置 | 当前文件数 | 运行方式 | 用途 |
|
||||
|---|---|---:|---|---|
|
||||
| 单元测试 | `src/**/*.test.ts` | 32 | `npm test` | 内部模块、pipeline、runtime |
|
||||
| Adapter 测试 | `clis/**/*.test.{ts,js}` | - | `npm test` / `npm run test:adapter` | adapter 命令与数据归一化 |
|
||||
| 单元测试 | `src/**/*.test.ts` | 32 | `npx vitest run src/` | 内部模块、pipeline、adapter 工具函数 |
|
||||
| E2E 测试 | `tests/e2e/*.test.ts` | 5 | `npx vitest run tests/e2e/` | 真实 CLI 命令执行 |
|
||||
| 烟雾测试 | `tests/smoke/*.test.ts` | 1 | `npx vitest run tests/smoke/` | 外部 API 与注册完整性 |
|
||||
|
||||
@@ -46,7 +43,7 @@ clis/
|
||||
|
||||
## 当前覆盖范围
|
||||
|
||||
### 单元测试与 Adapter 测试
|
||||
### 单元测试（32 个文件）
|
||||
|
||||
| 领域 | 文件 |
|
||||
|---|---|
|
||||
@@ -103,11 +100,8 @@ npm run build # 编译（E2E / smoke 测试需要 dist/src/main.js）
|
||||
### 运行命令
|
||||
|
||||
```bash
|
||||
# 默认本地测试口径（unit + extension + adapter）
|
||||
npm test
|
||||
|
||||
# 只跑 adapter project
|
||||
npm run test:adapter
|
||||
# 全部单元测试
|
||||
npx vitest run src/
|
||||
|
||||
# 全部 E2E 测试（会真实调用外部 API / 浏览器）
|
||||
npx vitest run tests/e2e/
|
||||
@@ -116,7 +110,7 @@ npx vitest run tests/e2e/
|
||||
npx vitest run tests/smoke/
|
||||
|
||||
# 单个测试文件
|
||||
npm test -- --run clis/apple-podcasts/commands.test.ts
|
||||
npx vitest run clis/apple-podcasts/commands.test.ts
|
||||
npx vitest run tests/e2e/management.test.ts
|
||||
|
||||
# 全部测试
|
||||
@@ -198,8 +192,7 @@ it('producthunt me fails gracefully without login', async () => {
|
||||
| Job | 触发条件 | 内容 |
|
||||
|---|---|---|
|
||||
| `build` | push/PR 到 `main`,`dev` | `tsc --noEmit` + `npm run build` |
|
||||
| `unit-test` | push/PR 到 `main`,`dev` | Node `22` 运行 `unit + extension`，按 `2` shard 并行 |
|
||||
| `adapter-test` | push/PR 到 `main`,`dev` | Node `22` 单独运行 `adapter` project |
|
||||
| `unit-test` | push/PR 到 `main`,`dev` | Node `20` 与 `22` 双版本运行 `src/` 单元测试，按 `2` shard 并行 |
|
||||
| `smoke-test` | `schedule` 或 `workflow_dispatch` | 安装真实 Chrome，`xvfb-run` 执行 `tests/smoke/` |
|
||||
|
||||
### `e2e-headed.yml`
|
||||
@@ -208,18 +201,19 @@ it('producthunt me fails gracefully without login', async () => {
|
||||
|---|---|---|
|
||||
| `e2e-headed` | push/PR 到 `main`,`dev`，或手动触发 | 安装真实 Chrome，`xvfb-run` 执行 `tests/e2e/` |
|
||||
|
||||
E2E 与 smoke 都使用 `./.github/actions/setup-chrome` 准备真实 Chrome。
|
||||
E2E 与 smoke 都使用 `./.github/actions/setup-chrome` 准备真实 Chrome，并通过 `OPENCLI_BROWSER_EXECUTABLE_PATH` 注入浏览器路径。
|
||||
|
||||
### Sharding
|
||||
|
||||
CI 里的 `unit-test` job 使用 vitest shard，只切 `unit + extension`，避免和独立的 `adapter-test` job 重复：
|
||||
单元测试使用 vitest 内置 shard，并在 Node `20` / `22` 两个版本上运行：
|
||||
|
||||
```yaml
|
||||
strategy:
|
||||
matrix:
|
||||
node-version: ['20', '22']
|
||||
shard: [1, 2]
|
||||
steps:
|
||||
- run: npx vitest run --project unit --project extension --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
- run: npx vitest run src/ --reporter=verbose --shard=${{ matrix.shard }}/2
|
||||
```
|
||||
|
||||
---
|
||||
@@ -233,7 +227,12 @@ opencli 通过 Browser Bridge 扩展连接浏览器：
|
||||
| 扩展已安装 / 已连接 | Extension 模式 | 本地用户，连接已登录的 Chrome |
|
||||
| 无扩展 token | CLI 自行拉起浏览器 | CI、无登录态或纯自动化场景 |
|
||||
|
||||
CI 通过 `./.github/actions/setup-chrome` 准备真实 Chrome，再直接执行测试。
|
||||
CI 中使用 `OPENCLI_BROWSER_EXECUTABLE_PATH` 指定真实 Chrome 路径：
|
||||
|
||||
```yaml
|
||||
env:
|
||||
OPENCLI_BROWSER_EXECUTABLE_PATH: ${{ steps.setup-chrome.outputs.chrome-path }}
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
/**
|
||||
* Layer 2: Claude Code Skill E2E Testing (LLM Judge)
|
||||
*
|
||||
* Spawns Claude Code with the opencli-adapter-author skill. Claude Code
|
||||
* Spawns Claude Code with the opencli-browser skill. Claude Code
|
||||
* completes the task using browse commands AND judges its own result.
|
||||
*
|
||||
* Task format: YAML with judge_context (multi-criteria, like Browser Use)
|
||||
@@ -19,7 +19,7 @@ import { fileURLToPath } from 'node:url';
|
||||
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
const RESULTS_DIR = join(__dirname, 'results');
|
||||
const SKILL_PATH = join(__dirname, '..', 'skills', 'opencli-adapter-author', 'SKILL.md');
|
||||
const SKILL_PATH = join(__dirname, '..', 'skills', 'opencli-browser', 'SKILL.md');
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────
|
||||
|
||||
|
||||
@@ -14,7 +14,7 @@ export const saveReliability: AutoResearchConfig = {
|
||||
'src/cli.ts',
|
||||
'src/discovery.ts',
|
||||
'src/registry.ts',
|
||||
'skills/opencli-adapter-author/SKILL.md',
|
||||
'skills/opencli-browser/SKILL.md',
|
||||
'autoresearch/save-tasks.json',
|
||||
'autoresearch/save-adapters/*.ts',
|
||||
],
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
/**
|
||||
* Preset: Skill E2E Quality
|
||||
*
|
||||
* Optimizes the opencli-adapter-author SKILL.md against the Layer 2 LLM E2E test suite.
|
||||
* Optimizes the opencli-browser SKILL.md against the Layer 2 LLM E2E test suite.
|
||||
* Metric: number of passing skill-tasks (out of 35).
|
||||
*/
|
||||
|
||||
@@ -10,7 +10,7 @@ import type { AutoResearchConfig } from '../config.js';
|
||||
export const skillQuality: AutoResearchConfig = {
|
||||
goal: 'Increase skill E2E pass rate to 35/35 (100%)',
|
||||
scope: [
|
||||
'skills/opencli-adapter-author/SKILL.md',
|
||||
'skills/opencli-browser/SKILL.md',
|
||||
],
|
||||
metric: 'pass_count',
|
||||
direction: 'higher',
|
||||
|
||||
-21423
File diff suppressed because it is too large
Load Diff
@@ -1,39 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './assets.js';
|
||||
import { __test__ as sharedTest } from './shared.js';
|
||||
describe('1688 assets normalization', () => {
|
||||
it('normalizes gallery and scanned assets into grouped media lists', () => {
|
||||
const result = __test__.normalizeAssets({
|
||||
href: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '测试商品 - 阿里巴巴',
|
||||
offerTitle: '测试商品',
|
||||
offerId: 887904326744,
|
||||
gallery: {
|
||||
mainImage: ['//img.example.com/main-1.jpg'],
|
||||
offerImgList: ['https://img.example.com/main-2.jpg'],
|
||||
wlImageInfos: [{ fullPathImageURI: 'https://img.example.com/main-3.jpg' }],
|
||||
},
|
||||
scannedAssets: [
|
||||
{ type: 'image', group: 'sku', url: 'https://img.example.com/sku-1.png', source: 'dom:.sku' },
|
||||
{ type: 'image', group: 'detail', url: 'https://img.example.com/detail-1.jpg', source: 'dom:.detail' },
|
||||
{ type: 'video', group: 'video', url: 'https://video.example.com/demo.mp4', source: 'script' },
|
||||
{ type: 'image', group: 'detail', url: 'blob:https://detail.1688.com/1', source: 'ignore' },
|
||||
],
|
||||
});
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.main_images).toEqual([
|
||||
'https://img.example.com/main-1.jpg',
|
||||
'https://img.example.com/main-2.jpg',
|
||||
'https://img.example.com/main-3.jpg',
|
||||
]);
|
||||
expect(result.sku_images).toEqual(['https://img.example.com/sku-1.png']);
|
||||
expect(result.detail_images).toEqual(['https://img.example.com/detail-1.jpg']);
|
||||
expect(result.videos).toEqual(['https://video.example.com/demo.mp4']);
|
||||
expect(result.main_count).toBe(3);
|
||||
expect(result.video_count).toBe(1);
|
||||
});
|
||||
it('normalizes media urls from style syntax and protocol-relative URLs', () => {
|
||||
expect(sharedTest.normalizeMediaUrl('url("//img.example.com/1.jpg")')).toBe('https://img.example.com/1.jpg');
|
||||
expect(sharedTest.normalizeMediaUrl('blob:https://detail.1688.com/1')).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,42 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './assets.js';
|
||||
import { __test__ as sharedTest } from './shared.js';
|
||||
|
||||
describe('1688 assets normalization', () => {
|
||||
it('normalizes gallery and scanned assets into grouped media lists', () => {
|
||||
const result = __test__.normalizeAssets({
|
||||
href: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '测试商品 - 阿里巴巴',
|
||||
offerTitle: '测试商品',
|
||||
offerId: 887904326744,
|
||||
gallery: {
|
||||
mainImage: ['//img.example.com/main-1.jpg'],
|
||||
offerImgList: ['https://img.example.com/main-2.jpg'],
|
||||
wlImageInfos: [{ fullPathImageURI: 'https://img.example.com/main-3.jpg' }],
|
||||
},
|
||||
scannedAssets: [
|
||||
{ type: 'image', group: 'sku', url: 'https://img.example.com/sku-1.png', source: 'dom:.sku' },
|
||||
{ type: 'image', group: 'detail', url: 'https://img.example.com/detail-1.jpg', source: 'dom:.detail' },
|
||||
{ type: 'video', group: 'video', url: 'https://video.example.com/demo.mp4', source: 'script' },
|
||||
{ type: 'image', group: 'detail', url: 'blob:https://detail.1688.com/1', source: 'ignore' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.main_images).toEqual([
|
||||
'https://img.example.com/main-1.jpg',
|
||||
'https://img.example.com/main-2.jpg',
|
||||
'https://img.example.com/main-3.jpg',
|
||||
]);
|
||||
expect(result.sku_images).toEqual(['https://img.example.com/sku-1.png']);
|
||||
expect(result.detail_images).toEqual(['https://img.example.com/detail-1.jpg']);
|
||||
expect(result.videos).toEqual(['https://video.example.com/demo.mp4']);
|
||||
expect(result.main_count).toBe(3);
|
||||
expect(result.video_count).toBe(1);
|
||||
});
|
||||
|
||||
it('normalizes media urls from style syntax and protocol-relative URLs', () => {
|
||||
expect(sharedTest.normalizeMediaUrl('url("//img.example.com/1.jpg")')).toBe('https://img.example.com/1.jpg');
|
||||
expect(sharedTest.normalizeMediaUrl('blob:https://detail.1688.com/1')).toBe('');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,52 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { assertAuthenticatedState, buildDetailUrl, buildProvenance, cleanText, extractOfferId, gotoAndReadState, uniqueMediaSources, } from './shared.js';
|
||||
function scriptToReadAssets() {
|
||||
return `
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
assertAuthenticatedState,
|
||||
buildDetailUrl,
|
||||
buildProvenance,
|
||||
cleanText,
|
||||
extractOfferId,
|
||||
gotoAndReadState,
|
||||
type MediaSource,
|
||||
uniqueMediaSources,
|
||||
} from './shared.js';
|
||||
|
||||
interface AssetBrowserPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
offerTitle?: string;
|
||||
offerId?: string | number;
|
||||
gallery?: {
|
||||
mainImage?: string[];
|
||||
offerImgList?: string[];
|
||||
wlImageInfos?: Array<{ fullPathImageURI?: string }>;
|
||||
[key: string]: unknown;
|
||||
};
|
||||
scannedAssets?: MediaSource[];
|
||||
}
|
||||
|
||||
export interface Normalized1688Assets {
|
||||
offer_id: string | null;
|
||||
title: string | null;
|
||||
item_url: string;
|
||||
main_images: string[];
|
||||
sku_images: string[];
|
||||
detail_images: string[];
|
||||
videos: string[];
|
||||
other_images: string[];
|
||||
raw_assets: MediaSource[];
|
||||
source: string[];
|
||||
main_count: number;
|
||||
sku_count: number;
|
||||
detail_count: number;
|
||||
video_count: number;
|
||||
source_url: string;
|
||||
fetched_at: string;
|
||||
strategy: string;
|
||||
}
|
||||
|
||||
function scriptToReadAssets(): string {
|
||||
return `
|
||||
(() => {
|
||||
const root = window.context ?? {};
|
||||
const model = root.result?.global?.globalData?.model ?? null;
|
||||
@@ -129,76 +174,84 @@ function scriptToReadAssets() {
|
||||
})()
|
||||
`;
|
||||
}
|
||||
function normalizeAssets(payload) {
|
||||
const offerId = cleanText(String(payload.offerId ?? '')) || extractOfferId(cleanText(payload.href)) || null;
|
||||
const itemUrl = offerId ? buildDetailUrl(offerId) : cleanText(payload.href);
|
||||
const seededAssets = [
|
||||
...((payload.gallery?.mainImage ?? []).map((url) => ({ type: 'image', group: 'main', url, source: 'page_state:mainImage' }))),
|
||||
...((payload.gallery?.offerImgList ?? []).map((url) => ({ type: 'image', group: 'main', url, source: 'page_state:offerImgList' }))),
|
||||
...((payload.gallery?.wlImageInfos ?? []).map((item) => ({
|
||||
type: 'image',
|
||||
group: 'main',
|
||||
url: item?.fullPathImageURI ?? '',
|
||||
source: 'page_state:wlImageInfos',
|
||||
}))),
|
||||
];
|
||||
const assets = uniqueMediaSources([...seededAssets, ...(payload.scannedAssets ?? [])]);
|
||||
const mainImages = assets.filter((item) => item.type === 'image' && item.group === 'main').map((item) => item.url);
|
||||
const skuImages = assets.filter((item) => item.type === 'image' && item.group === 'sku').map((item) => item.url);
|
||||
const detailImages = assets.filter((item) => item.type === 'image' && item.group === 'detail').map((item) => item.url);
|
||||
const videos = assets.filter((item) => item.type === 'video').map((item) => item.url);
|
||||
const otherImages = assets
|
||||
.filter((item) => item.type === 'image' && !['main', 'sku', 'detail'].includes(item.group))
|
||||
.map((item) => item.url);
|
||||
return {
|
||||
offer_id: offerId,
|
||||
title: cleanText(payload.offerTitle) || cleanText(payload.title) || null,
|
||||
item_url: itemUrl,
|
||||
main_images: mainImages,
|
||||
sku_images: skuImages,
|
||||
detail_images: detailImages,
|
||||
videos,
|
||||
other_images: otherImages,
|
||||
raw_assets: assets,
|
||||
source: [...new Set(assets.map((item) => cleanText(item.source)).filter(Boolean))],
|
||||
main_count: mainImages.length,
|
||||
sku_count: skuImages.length,
|
||||
detail_count: detailImages.length,
|
||||
video_count: videos.length,
|
||||
...buildProvenance(cleanText(payload.href) || itemUrl),
|
||||
};
|
||||
|
||||
function normalizeAssets(payload: AssetBrowserPayload): Normalized1688Assets {
|
||||
const offerId = cleanText(String(payload.offerId ?? '')) || extractOfferId(cleanText(payload.href)) || null;
|
||||
const itemUrl = offerId ? buildDetailUrl(offerId) : cleanText(payload.href);
|
||||
const seededAssets: MediaSource[] = [
|
||||
...((payload.gallery?.mainImage ?? []).map((url) => ({ type: 'image' as const, group: 'main' as const, url, source: 'page_state:mainImage' }))),
|
||||
...((payload.gallery?.offerImgList ?? []).map((url) => ({ type: 'image' as const, group: 'main' as const, url, source: 'page_state:offerImgList' }))),
|
||||
...((payload.gallery?.wlImageInfos ?? []).map((item) => ({
|
||||
type: 'image' as const,
|
||||
group: 'main' as const,
|
||||
url: item?.fullPathImageURI ?? '',
|
||||
source: 'page_state:wlImageInfos',
|
||||
}))),
|
||||
];
|
||||
|
||||
const assets = uniqueMediaSources([...seededAssets, ...(payload.scannedAssets ?? [])]);
|
||||
|
||||
const mainImages = assets.filter((item) => item.type === 'image' && item.group === 'main').map((item) => item.url);
|
||||
const skuImages = assets.filter((item) => item.type === 'image' && item.group === 'sku').map((item) => item.url);
|
||||
const detailImages = assets.filter((item) => item.type === 'image' && item.group === 'detail').map((item) => item.url);
|
||||
const videos = assets.filter((item) => item.type === 'video').map((item) => item.url);
|
||||
const otherImages = assets
|
||||
.filter((item) => item.type === 'image' && !['main', 'sku', 'detail'].includes(item.group))
|
||||
.map((item) => item.url);
|
||||
|
||||
return {
|
||||
offer_id: offerId,
|
||||
title: cleanText(payload.offerTitle) || cleanText(payload.title) || null,
|
||||
item_url: itemUrl,
|
||||
main_images: mainImages,
|
||||
sku_images: skuImages,
|
||||
detail_images: detailImages,
|
||||
videos,
|
||||
other_images: otherImages,
|
||||
raw_assets: assets,
|
||||
source: [...new Set(assets.map((item) => cleanText(item.source)).filter(Boolean))],
|
||||
main_count: mainImages.length,
|
||||
sku_count: skuImages.length,
|
||||
detail_count: detailImages.length,
|
||||
video_count: videos.length,
|
||||
...buildProvenance(cleanText(payload.href) || itemUrl),
|
||||
};
|
||||
}
|
||||
async function readAssetsPayload(page, itemUrl) {
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'assets');
|
||||
assertAuthenticatedState(state, 'assets');
|
||||
await page.autoScroll({ times: 3, delayMs: 400 });
|
||||
await page.wait(1);
|
||||
return await page.evaluate(scriptToReadAssets());
|
||||
|
||||
async function readAssetsPayload(page: IPage, itemUrl: string): Promise<AssetBrowserPayload> {
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'assets');
|
||||
assertAuthenticatedState(state, 'assets');
|
||||
await page.autoScroll({ times: 3, delayMs: 400 });
|
||||
await page.wait(1);
|
||||
return await page.evaluate(scriptToReadAssets()) as AssetBrowserPayload;
|
||||
}
|
||||
export async function extractAssetsForInput(page, input) {
|
||||
const itemUrl = buildDetailUrl(String(input ?? ''));
|
||||
const payload = await readAssetsPayload(page, itemUrl);
|
||||
return normalizeAssets(payload);
|
||||
|
||||
export async function extractAssetsForInput(page: IPage, input: string): Promise<Normalized1688Assets> {
|
||||
const itemUrl = buildDetailUrl(String(input ?? ''));
|
||||
const payload = await readAssetsPayload(page, itemUrl);
|
||||
return normalizeAssets(payload);
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'assets',
|
||||
description: '列出 1688 商品页可提取的图片/视频素材',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
],
|
||||
columns: ['offer_id', 'title', 'main_count', 'sku_count', 'detail_count', 'video_count'],
|
||||
func: async (page, kwargs) => {
|
||||
return [await extractAssetsForInput(page, String(kwargs.input ?? ''))];
|
||||
site: '1688',
|
||||
name: 'assets',
|
||||
description: '列出 1688 商品页可提取的图片/视频素材',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
],
|
||||
columns: ['offer_id', 'title', 'main_count', 'sku_count', 'detail_count', 'video_count'],
|
||||
func: async (page, kwargs) => {
|
||||
return [await extractAssetsForInput(page, String(kwargs.input ?? ''))];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeAssets,
|
||||
normalizeAssets,
|
||||
};
|
||||
@@ -1,76 +0,0 @@
|
||||
import * as path from 'node:path';
|
||||
import { formatCookieHeader } from '@jackwener/opencli/download';
|
||||
import { downloadMedia } from '@jackwener/opencli/download/media-download';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { cleanText } from './shared.js';
|
||||
import { extractAssetsForInput } from './assets.js';
|
||||
function extFromUrl(url, fallback) {
|
||||
try {
|
||||
const ext = path.extname(new URL(url).pathname).toLowerCase();
|
||||
if (ext && ext.length <= 8)
|
||||
return ext;
|
||||
}
|
||||
catch {
|
||||
// ignore
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
function toDownloadItems(offerId, assets) {
|
||||
const items = [];
|
||||
const pushImages = (urls, prefix) => {
|
||||
urls.forEach((url, index) => {
|
||||
items.push({
|
||||
type: 'image',
|
||||
url,
|
||||
filename: `${offerId}_${prefix}_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.jpg')}`,
|
||||
});
|
||||
});
|
||||
};
|
||||
pushImages(assets.main_images, 'main');
|
||||
pushImages(assets.sku_images, 'sku');
|
||||
pushImages(assets.detail_images, 'detail');
|
||||
pushImages(assets.other_images, 'other');
|
||||
assets.videos.forEach((url, index) => {
|
||||
items.push({
|
||||
type: 'video',
|
||||
url,
|
||||
filename: `${offerId}_video_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.mp4')}`,
|
||||
});
|
||||
});
|
||||
return items;
|
||||
}
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'download',
|
||||
description: '批量下载 1688 商品页可提取的图片和视频素材',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
{ name: 'output', default: './1688-downloads', help: '输出目录' },
|
||||
],
|
||||
columns: ['index', 'type', 'status', 'size'],
|
||||
func: async (page, kwargs) => {
|
||||
const assets = await extractAssetsForInput(page, String(kwargs.input ?? ''));
|
||||
const offerId = cleanText(assets.offer_id) || '1688';
|
||||
const items = toDownloadItems(offerId, assets);
|
||||
const browserCookies = await page.getCookies({ domain: '1688.com' });
|
||||
return downloadMedia(items, {
|
||||
output: String(kwargs.output || './1688-downloads'),
|
||||
subdir: offerId,
|
||||
cookies: formatCookieHeader(browserCookies),
|
||||
browserCookies,
|
||||
filenamePrefix: offerId,
|
||||
timeout: 60000,
|
||||
});
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
extFromUrl,
|
||||
toDownloadItems,
|
||||
};
|
||||
@@ -1,31 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './download.js';
|
||||
describe('1688 download helpers', () => {
|
||||
it('builds stable filenames for grouped assets', () => {
|
||||
const items = __test__.toDownloadItems('887904326744', {
|
||||
offer_id: '887904326744',
|
||||
title: '测试商品',
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
main_images: ['https://img.example.com/a.jpg'],
|
||||
sku_images: ['https://img.example.com/b.png'],
|
||||
detail_images: ['https://img.example.com/c.webp'],
|
||||
videos: ['https://video.example.com/d.mp4'],
|
||||
other_images: [],
|
||||
raw_assets: [],
|
||||
source: [],
|
||||
main_count: 1,
|
||||
sku_count: 1,
|
||||
detail_count: 1,
|
||||
video_count: 1,
|
||||
source_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: 'cookie',
|
||||
});
|
||||
expect(items.map((item) => item.filename)).toEqual([
|
||||
'887904326744_main_01.jpg',
|
||||
'887904326744_sku_01.png',
|
||||
'887904326744_detail_01.webp',
|
||||
'887904326744_video_01.mp4',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,33 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './download.js';
|
||||
|
||||
describe('1688 download helpers', () => {
|
||||
it('builds stable filenames for grouped assets', () => {
|
||||
const items = __test__.toDownloadItems('887904326744', {
|
||||
offer_id: '887904326744',
|
||||
title: '测试商品',
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
main_images: ['https://img.example.com/a.jpg'],
|
||||
sku_images: ['https://img.example.com/b.png'],
|
||||
detail_images: ['https://img.example.com/c.webp'],
|
||||
videos: ['https://video.example.com/d.mp4'],
|
||||
other_images: [],
|
||||
raw_assets: [],
|
||||
source: [],
|
||||
main_count: 1,
|
||||
sku_count: 1,
|
||||
detail_count: 1,
|
||||
video_count: 1,
|
||||
source_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: 'cookie',
|
||||
});
|
||||
|
||||
expect(items.map((item) => item.filename)).toEqual([
|
||||
'887904326744_main_01.jpg',
|
||||
'887904326744_sku_01.png',
|
||||
'887904326744_detail_01.webp',
|
||||
'887904326744_video_01.mp4',
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,83 @@
|
||||
import * as path from 'node:path';
|
||||
import { formatCookieHeader } from '@jackwener/opencli/download';
|
||||
import { downloadMedia, type MediaItem } from '@jackwener/opencli/download/media-download';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { cleanText } from './shared.js';
|
||||
import { extractAssetsForInput } from './assets.js';
|
||||
|
||||
function extFromUrl(url: string, fallback: string): string {
|
||||
try {
|
||||
const ext = path.extname(new URL(url).pathname).toLowerCase();
|
||||
if (ext && ext.length <= 8) return ext;
|
||||
} catch {
|
||||
// ignore
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
function toDownloadItems(offerId: string, assets: Awaited<ReturnType<typeof extractAssetsForInput>>): MediaItem[] {
|
||||
const items: MediaItem[] = [];
|
||||
|
||||
const pushImages = (urls: string[], prefix: string) => {
|
||||
urls.forEach((url, index) => {
|
||||
items.push({
|
||||
type: 'image',
|
||||
url,
|
||||
filename: `${offerId}_${prefix}_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.jpg')}`,
|
||||
});
|
||||
});
|
||||
};
|
||||
|
||||
pushImages(assets.main_images, 'main');
|
||||
pushImages(assets.sku_images, 'sku');
|
||||
pushImages(assets.detail_images, 'detail');
|
||||
pushImages(assets.other_images, 'other');
|
||||
|
||||
assets.videos.forEach((url, index) => {
|
||||
items.push({
|
||||
type: 'video',
|
||||
url,
|
||||
filename: `${offerId}_video_${String(index + 1).padStart(2, '0')}${extFromUrl(url, '.mp4')}`,
|
||||
});
|
||||
});
|
||||
|
||||
return items;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'download',
|
||||
description: '批量下载 1688 商品页可提取的图片和视频素材',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
{ name: 'output', default: './1688-downloads', help: '输出目录' },
|
||||
],
|
||||
columns: ['index', 'type', 'status', 'size'],
|
||||
func: async (page, kwargs) => {
|
||||
const assets = await extractAssetsForInput(page, String(kwargs.input ?? ''));
|
||||
const offerId = cleanText(assets.offer_id) || '1688';
|
||||
const items = toDownloadItems(offerId, assets);
|
||||
const browserCookies = await page.getCookies({ domain: '1688.com' });
|
||||
|
||||
return downloadMedia(items, {
|
||||
output: String(kwargs.output || './1688-downloads'),
|
||||
subdir: offerId,
|
||||
cookies: formatCookieHeader(browserCookies),
|
||||
browserCookies,
|
||||
filenamePrefix: offerId,
|
||||
timeout: 60000,
|
||||
});
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
extFromUrl,
|
||||
toDownloadItems,
|
||||
};
|
||||
@@ -1,187 +0,0 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { isRecord } from '@jackwener/opencli/utils';
|
||||
import { assertAuthenticatedState, buildDetailUrl, buildProvenance, canonicalizeSellerUrl, cleanMultilineText, cleanText, extractLocation, extractMemberId, extractOfferId, extractShopId, gotoAndReadState, normalizePriceTiers, parseMoqText, parsePriceText, toNumber, uniqueNonEmpty, } from './shared.js';
|
||||
function normalizeItemPayload(payload) {
|
||||
const href = cleanText(payload.href);
|
||||
const bodyText = cleanMultilineText(payload.bodyText);
|
||||
const sellerName = cleanText(payload.seller?.companyName);
|
||||
const sellerUrlRaw = cleanText(payload.seller?.winportUrl
|
||||
?? payload.seller?.sellerWinportUrlMap?.defaultUrl
|
||||
?? payload.seller?.sellerWinportUrlMap?.indexUrl);
|
||||
const sellerUrl = canonicalizeSellerUrl(sellerUrlRaw);
|
||||
const offerId = cleanText(String(payload.offerId ?? '')) || extractOfferId(href) || null;
|
||||
const memberId = cleanText(payload.seller?.memberId) || extractMemberId(sellerUrlRaw || href) || null;
|
||||
const shopId = extractShopId(sellerUrl ?? href);
|
||||
const unit = cleanText(payload.trade?.unit);
|
||||
const priceDisplay = cleanText(payload.trade?.priceDisplay);
|
||||
const priceRange = parsePriceText(priceDisplay ? `¥${priceDisplay}` : bodyText);
|
||||
const moqText = extractMoqText(bodyText, payload.trade?.beginAmount, unit);
|
||||
const moq = parseMoqText(moqText);
|
||||
const services = uniqueServices(payload);
|
||||
const serviceBadges = uniqueNonEmpty(services.map((service) => cleanText(service.serviceName)));
|
||||
const attributes = normalizeVisibleAttributes(payload.trade?.offerIDatacenterSellInfo);
|
||||
const priceTiers = normalizePriceTiers(payload.trade?.offerPriceModel?.currentPrices ?? [], unit || null);
|
||||
const images = uniqueNonEmpty([
|
||||
...(payload.gallery?.mainImage ?? []),
|
||||
...(payload.gallery?.offerImgList ?? []),
|
||||
...((payload.gallery?.wlImageInfos ?? []).map((item) => item.fullPathImageURI ?? '')),
|
||||
]);
|
||||
const detailUrl = offerId ? buildDetailUrl(offerId) : href;
|
||||
const provenance = buildProvenance(href || detailUrl);
|
||||
return {
|
||||
offer_id: offerId,
|
||||
member_id: memberId,
|
||||
shop_id: shopId,
|
||||
title: cleanText(payload.offerTitle) || stripAlibabaSuffix(payload.title) || firstNonEmptyLine(bodyText) || null,
|
||||
item_url: detailUrl,
|
||||
main_images: images,
|
||||
price_text: priceRange.price_text || null,
|
||||
price_tiers: priceTiers,
|
||||
currency: priceRange.currency,
|
||||
moq_text: moq.moq_text || null,
|
||||
moq_value: moq.moq_value,
|
||||
seller_name: sellerName || null,
|
||||
seller_url: sellerUrl,
|
||||
shop_name: sellerName || null,
|
||||
origin_place: extractLocation(bodyText),
|
||||
delivery_days_text: extractDeliveryDaysText(bodyText, services, payload.shipping),
|
||||
customization_text: extractKeywordLine(bodyText, ['来样定制', '来图定制', '支持定制', '可定制', '定制']),
|
||||
private_label_text: extractKeywordLine(bodyText, ['贴牌', '贴标', '定制logo', '打logo', 'OEM', 'ODM']),
|
||||
visible_attributes: attributes,
|
||||
sales_text: extractSalesText(bodyText),
|
||||
service_badges: serviceBadges,
|
||||
stock_quantity: extractStockQuantity(bodyText),
|
||||
...provenance,
|
||||
};
|
||||
}
|
||||
function normalizeVisibleAttributes(raw) {
|
||||
if (!isRecord(raw))
|
||||
return [];
|
||||
return Object.entries(raw)
|
||||
.filter(([key, value]) => key !== 'sellPointModel' && cleanText(key) && cleanText(String(value)))
|
||||
.map(([key, value]) => ({ key: cleanText(key), value: cleanText(String(value)) }));
|
||||
}
|
||||
function uniqueServices(payload) {
|
||||
const combined = [
|
||||
...(Array.isArray(payload.services) ? payload.services : []),
|
||||
...(Array.isArray(payload.shipping?.protectionInfos) ? payload.shipping.protectionInfos : []),
|
||||
...(Array.isArray(payload.shipping?.buyerProtectionModel) ? payload.shipping.buyerProtectionModel : []),
|
||||
];
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
for (const service of combined) {
|
||||
const key = cleanText(service.serviceName);
|
||||
if (!key || seen.has(key))
|
||||
continue;
|
||||
seen.add(key);
|
||||
result.push(service);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function stripAlibabaSuffix(title) {
|
||||
return cleanText(title).replace(/\s*-\s*阿里巴巴$/, '').trim();
|
||||
}
|
||||
function firstNonEmptyLine(text) {
|
||||
return text.split('\n').map((line) => cleanText(line)).find(Boolean) ?? '';
|
||||
}
|
||||
function extractMoqText(bodyText, beginAmount, unit) {
|
||||
const lineMatch = bodyText.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/);
|
||||
if (lineMatch)
|
||||
return lineMatch[0];
|
||||
const moqValue = toNumber(beginAmount);
|
||||
if (moqValue !== null) {
|
||||
return `${moqValue}${unit || ''}起批`;
|
||||
}
|
||||
return '';
|
||||
}
|
||||
function extractDeliveryDaysText(bodyText, services, shipping) {
|
||||
const shippingText = cleanText(shipping?.deliveryLimitText) || cleanText(shipping?.logisticsText);
|
||||
if (shippingText)
|
||||
return shippingText;
|
||||
const textMatch = bodyText.match(/\d+\s*(?:小时|天)(?:内)?发货/);
|
||||
if (textMatch)
|
||||
return textMatch[0];
|
||||
const hourMatch = services.find((service) => typeof service.agreeDeliveryHours === 'number');
|
||||
if (hourMatch && typeof hourMatch.agreeDeliveryHours === 'number') {
|
||||
return `${hourMatch.agreeDeliveryHours}小时内发货`;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function extractKeywordLine(bodyText, keywords) {
|
||||
const lines = bodyText.split('\n').map((line) => cleanText(line)).filter(Boolean);
|
||||
for (const line of lines) {
|
||||
if (keywords.some((keyword) => line.includes(keyword))) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function extractSalesText(bodyText) {
|
||||
const match = bodyText.match(/(?:全网销量|已售)\s*\d+(?:\.\d+)?\+?\s*[件套个单]?/);
|
||||
return match ? cleanText(match[0]) : null;
|
||||
}
|
||||
function extractStockQuantity(bodyText) {
|
||||
const match = bodyText.match(/库存\s*(\d+)/);
|
||||
return match ? Number.parseInt(match[1], 10) : null;
|
||||
}
|
||||
async function readItemPayload(page, itemUrl) {
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'item');
|
||||
assertAuthenticatedState(state, 'item');
|
||||
const payload = await page.evaluate(`
|
||||
(() => {
|
||||
const root = window.context ?? {};
|
||||
const model = root.result?.global?.globalData?.model ?? null;
|
||||
const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
|
||||
return {
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
offerTitle: model?.offerTitleModel?.subject ?? '',
|
||||
offerId: model?.tradeModel?.offerId ?? '',
|
||||
seller: toJson(model?.sellerModel),
|
||||
trade: toJson(model?.tradeModel),
|
||||
gallery: toJson(root.result?.data?.gallery?.fields ?? null),
|
||||
shipping: toJson(root.result?.data?.shippingServices?.fields ?? null),
|
||||
services: toJson(root.result?.data?.shippingServices?.fields?.protectionInfos ?? []),
|
||||
};
|
||||
})()
|
||||
`);
|
||||
const resolvedOfferId = cleanText(String(payload.offerId ?? '')) || extractOfferId(cleanText(payload.href));
|
||||
if (!resolvedOfferId) {
|
||||
throw new CommandExecutionError('1688 item page did not expose product context', '当前 tab 非商品详情上下文，请切到 detail.1688.com 商品页并重试');
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'item',
|
||||
description: '1688 商品详情（公开商品字段、价格阶梯、卖家基础信息）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
],
|
||||
columns: ['offer_id', 'title', 'price_text', 'moq_text', 'seller_name', 'origin_place'],
|
||||
func: async (page, kwargs) => {
|
||||
const itemUrl = buildDetailUrl(String(kwargs.input ?? ''));
|
||||
const payload = await readItemPayload(page, itemUrl);
|
||||
return [normalizeItemPayload(payload)];
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeItemPayload,
|
||||
normalizeVisibleAttributes,
|
||||
stripAlibabaSuffix,
|
||||
extractMoqText,
|
||||
extractDeliveryDaysText,
|
||||
extractKeywordLine,
|
||||
extractSalesText,
|
||||
extractStockQuantity,
|
||||
};
|
||||
@@ -1,67 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './item.js';
|
||||
describe('1688 item normalization', () => {
|
||||
it('normalizes public item payload into contract fields', () => {
|
||||
const result = __test__.normalizeItemPayload({
|
||||
href: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '法式春季长袖开衫连衣裙女新款大码女装碎花吊带裙套装142077 - 阿里巴巴',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
入驻13年
|
||||
主营：大码女装
|
||||
店铺回头率
|
||||
87%
|
||||
山东青岛
|
||||
3套起批
|
||||
已售1600+套
|
||||
支持定制logo
|
||||
`,
|
||||
offerTitle: '法式春季长袖开衫连衣裙女新款大码女装碎花吊带裙套装142077',
|
||||
offerId: 887904326744,
|
||||
seller: {
|
||||
companyName: '青岛沁澜衣品服装有限公司',
|
||||
memberId: 'b2b-1641351767',
|
||||
winportUrl: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a1',
|
||||
},
|
||||
trade: {
|
||||
beginAmount: 3,
|
||||
priceDisplay: '96.00-98.00',
|
||||
unit: '套',
|
||||
saleCount: 1655,
|
||||
offerIDatacenterSellInfo: {
|
||||
面料名称: '莫代尔',
|
||||
主面料成分: '莫代尔纤维',
|
||||
sellPointModel: '{"ignore":true}',
|
||||
},
|
||||
offerPriceModel: {
|
||||
currentPrices: [
|
||||
{ beginAmount: 3, price: '98.00' },
|
||||
{ beginAmount: 50, price: '97.00' },
|
||||
],
|
||||
},
|
||||
},
|
||||
gallery: {
|
||||
mainImage: ['https://example.com/1.jpg'],
|
||||
offerImgList: ['https://example.com/2.jpg'],
|
||||
wlImageInfos: [{ fullPathImageURI: 'https://example.com/3.jpg' }],
|
||||
},
|
||||
services: [
|
||||
{ serviceName: '延期必赔', agreeDeliveryHours: 360 },
|
||||
{ serviceName: '品质保障' },
|
||||
],
|
||||
});
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.member_id).toBe('b2b-1641351767');
|
||||
expect(result.shop_id).toBe('yinuoweierfushi');
|
||||
expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.price_text).toBe('¥96.00-98.00');
|
||||
expect(result.moq_text).toBe('3套起批');
|
||||
expect(result.origin_place).toBe('山东青岛');
|
||||
expect(result.delivery_days_text).toBe('360小时内发货');
|
||||
expect(result.private_label_text).toBe('支持定制logo');
|
||||
expect(result.visible_attributes).toEqual([
|
||||
{ key: '面料名称', value: '莫代尔' },
|
||||
{ key: '主面料成分', value: '莫代尔纤维' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './item.js';
|
||||
|
||||
describe('1688 item normalization', () => {
|
||||
it('normalizes public item payload into contract fields', () => {
|
||||
const result = __test__.normalizeItemPayload({
|
||||
href: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '法式春季长袖开衫连衣裙女新款大码女装碎花吊带裙套装142077 - 阿里巴巴',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
入驻13年
|
||||
主营：大码女装
|
||||
店铺回头率
|
||||
87%
|
||||
山东青岛
|
||||
3套起批
|
||||
已售1600+套
|
||||
支持定制logo
|
||||
`,
|
||||
offerTitle: '法式春季长袖开衫连衣裙女新款大码女装碎花吊带裙套装142077',
|
||||
offerId: 887904326744,
|
||||
seller: {
|
||||
companyName: '青岛沁澜衣品服装有限公司',
|
||||
memberId: 'b2b-1641351767',
|
||||
winportUrl: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a1',
|
||||
},
|
||||
trade: {
|
||||
beginAmount: 3,
|
||||
priceDisplay: '96.00-98.00',
|
||||
unit: '套',
|
||||
saleCount: 1655,
|
||||
offerIDatacenterSellInfo: {
|
||||
面料名称: '莫代尔',
|
||||
主面料成分: '莫代尔纤维',
|
||||
sellPointModel: '{"ignore":true}',
|
||||
},
|
||||
offerPriceModel: {
|
||||
currentPrices: [
|
||||
{ beginAmount: 3, price: '98.00' },
|
||||
{ beginAmount: 50, price: '97.00' },
|
||||
],
|
||||
},
|
||||
},
|
||||
gallery: {
|
||||
mainImage: ['https://example.com/1.jpg'],
|
||||
offerImgList: ['https://example.com/2.jpg'],
|
||||
wlImageInfos: [{ fullPathImageURI: 'https://example.com/3.jpg' }],
|
||||
},
|
||||
services: [
|
||||
{ serviceName: '延期必赔', agreeDeliveryHours: 360 },
|
||||
{ serviceName: '品质保障' },
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.member_id).toBe('b2b-1641351767');
|
||||
expect(result.shop_id).toBe('yinuoweierfushi');
|
||||
expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.price_text).toBe('¥96.00-98.00');
|
||||
expect(result.moq_text).toBe('3套起批');
|
||||
expect(result.origin_place).toBe('山东青岛');
|
||||
expect(result.delivery_days_text).toBe('360小时内发货');
|
||||
expect(result.private_label_text).toBe('支持定制logo');
|
||||
expect(result.visible_attributes).toEqual([
|
||||
{ key: '面料名称', value: '莫代尔' },
|
||||
{ key: '主面料成分', value: '莫代尔纤维' },
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,282 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import { isRecord } from '@jackwener/opencli/utils';
|
||||
import {
|
||||
assertAuthenticatedState,
|
||||
buildDetailUrl,
|
||||
buildProvenance,
|
||||
canonicalizeSellerUrl,
|
||||
cleanMultilineText,
|
||||
cleanText,
|
||||
extractLocation,
|
||||
extractMemberId,
|
||||
extractOfferId,
|
||||
extractShopId,
|
||||
gotoAndReadState,
|
||||
normalizePriceTiers,
|
||||
parseMoqText,
|
||||
parsePriceText,
|
||||
toNumber,
|
||||
uniqueNonEmpty,
|
||||
} from './shared.js';
|
||||
|
||||
interface BuyerProtectionModel {
|
||||
serviceName?: string;
|
||||
shortBuyerDesc?: string;
|
||||
packageBuyerDesc?: string;
|
||||
textDesc?: string;
|
||||
agreeDeliveryHours?: number;
|
||||
}
|
||||
|
||||
interface ItemBrowserPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
bodyText?: string;
|
||||
offerTitle?: string;
|
||||
offerId?: string | number;
|
||||
seller?: {
|
||||
companyName?: string;
|
||||
memberId?: string;
|
||||
winportUrl?: string;
|
||||
sellerWinportUrlMap?: Record<string, string>;
|
||||
};
|
||||
trade?: {
|
||||
beginAmount?: string | number;
|
||||
priceDisplay?: string;
|
||||
unit?: string;
|
||||
saleCount?: string | number;
|
||||
offerIDatacenterSellInfo?: Record<string, unknown>;
|
||||
offerPriceModel?: {
|
||||
currentPrices?: Array<{ beginAmount?: string | number; price?: string | number }>;
|
||||
};
|
||||
};
|
||||
gallery?: {
|
||||
mainImage?: string[];
|
||||
offerImgList?: string[];
|
||||
wlImageInfos?: Array<{ fullPathImageURI?: string }>;
|
||||
};
|
||||
shipping?: {
|
||||
deliveryLimitText?: string;
|
||||
logisticsText?: string;
|
||||
protectionInfos?: BuyerProtectionModel[];
|
||||
buyerProtectionModel?: BuyerProtectionModel[];
|
||||
};
|
||||
services?: BuyerProtectionModel[];
|
||||
}
|
||||
|
||||
interface VisibleAttribute {
|
||||
key: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
function normalizeItemPayload(payload: ItemBrowserPayload): Record<string, unknown> {
|
||||
const href = cleanText(payload.href);
|
||||
const bodyText = cleanMultilineText(payload.bodyText);
|
||||
const sellerName = cleanText(payload.seller?.companyName);
|
||||
const sellerUrlRaw = cleanText(
|
||||
payload.seller?.winportUrl
|
||||
?? payload.seller?.sellerWinportUrlMap?.defaultUrl
|
||||
?? payload.seller?.sellerWinportUrlMap?.indexUrl,
|
||||
);
|
||||
const sellerUrl = canonicalizeSellerUrl(sellerUrlRaw);
|
||||
const offerId = cleanText(String(payload.offerId ?? '')) || extractOfferId(href) || null;
|
||||
const memberId = cleanText(payload.seller?.memberId) || extractMemberId(sellerUrlRaw || href) || null;
|
||||
const shopId = extractShopId(sellerUrl ?? href);
|
||||
const unit = cleanText(payload.trade?.unit);
|
||||
const priceDisplay = cleanText(payload.trade?.priceDisplay);
|
||||
const priceRange = parsePriceText(priceDisplay ? `¥${priceDisplay}` : bodyText);
|
||||
const moqText = extractMoqText(bodyText, payload.trade?.beginAmount, unit);
|
||||
const moq = parseMoqText(moqText);
|
||||
const services = uniqueServices(payload);
|
||||
const serviceBadges = uniqueNonEmpty(services.map((service) => cleanText(service.serviceName)));
|
||||
const attributes = normalizeVisibleAttributes(payload.trade?.offerIDatacenterSellInfo);
|
||||
const priceTiers = normalizePriceTiers(payload.trade?.offerPriceModel?.currentPrices ?? [], unit || null);
|
||||
const images = uniqueNonEmpty([
|
||||
...(payload.gallery?.mainImage ?? []),
|
||||
...(payload.gallery?.offerImgList ?? []),
|
||||
...((payload.gallery?.wlImageInfos ?? []).map((item) => item.fullPathImageURI ?? '')),
|
||||
]);
|
||||
|
||||
const detailUrl = offerId ? buildDetailUrl(offerId) : href;
|
||||
const provenance = buildProvenance(href || detailUrl);
|
||||
|
||||
return {
|
||||
offer_id: offerId,
|
||||
member_id: memberId,
|
||||
shop_id: shopId,
|
||||
title: cleanText(payload.offerTitle) || stripAlibabaSuffix(payload.title) || firstNonEmptyLine(bodyText) || null,
|
||||
item_url: detailUrl,
|
||||
main_images: images,
|
||||
price_text: priceRange.price_text || null,
|
||||
price_tiers: priceTiers,
|
||||
currency: priceRange.currency,
|
||||
moq_text: moq.moq_text || null,
|
||||
moq_value: moq.moq_value,
|
||||
seller_name: sellerName || null,
|
||||
seller_url: sellerUrl,
|
||||
shop_name: sellerName || null,
|
||||
origin_place: extractLocation(bodyText),
|
||||
delivery_days_text: extractDeliveryDaysText(bodyText, services, payload.shipping),
|
||||
customization_text: extractKeywordLine(bodyText, ['来样定制', '来图定制', '支持定制', '可定制', '定制']),
|
||||
private_label_text: extractKeywordLine(bodyText, ['贴牌', '贴标', '定制logo', '打logo', 'OEM', 'ODM']),
|
||||
visible_attributes: attributes,
|
||||
sales_text: extractSalesText(bodyText),
|
||||
service_badges: serviceBadges,
|
||||
stock_quantity: extractStockQuantity(bodyText),
|
||||
...provenance,
|
||||
};
|
||||
}
|
||||
|
||||
function normalizeVisibleAttributes(raw: unknown): VisibleAttribute[] {
|
||||
if (!isRecord(raw)) return [];
|
||||
return Object.entries(raw)
|
||||
.filter(([key, value]) => key !== 'sellPointModel' && cleanText(key) && cleanText(String(value)))
|
||||
.map(([key, value]) => ({ key: cleanText(key), value: cleanText(String(value)) }));
|
||||
}
|
||||
|
||||
function uniqueServices(payload: ItemBrowserPayload): BuyerProtectionModel[] {
|
||||
const combined = [
|
||||
...(Array.isArray(payload.services) ? payload.services : []),
|
||||
...(Array.isArray(payload.shipping?.protectionInfos) ? payload.shipping.protectionInfos : []),
|
||||
...(Array.isArray(payload.shipping?.buyerProtectionModel) ? payload.shipping.buyerProtectionModel : []),
|
||||
];
|
||||
|
||||
const seen = new Set<string>();
|
||||
const result: BuyerProtectionModel[] = [];
|
||||
for (const service of combined) {
|
||||
const key = cleanText(service.serviceName);
|
||||
if (!key || seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
result.push(service);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function stripAlibabaSuffix(title: string | undefined): string {
|
||||
return cleanText(title).replace(/\s*-\s*阿里巴巴$/, '').trim();
|
||||
}
|
||||
|
||||
function firstNonEmptyLine(text: string): string {
|
||||
return text.split('\n').map((line) => cleanText(line)).find(Boolean) ?? '';
|
||||
}
|
||||
|
||||
function extractMoqText(bodyText: string, beginAmount: string | number | undefined, unit: string): string {
|
||||
const lineMatch = bodyText.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/);
|
||||
if (lineMatch) return lineMatch[0];
|
||||
|
||||
const moqValue = toNumber(beginAmount);
|
||||
if (moqValue !== null) {
|
||||
return `${moqValue}${unit || ''}起批`;
|
||||
}
|
||||
|
||||
return '';
|
||||
}
|
||||
|
||||
function extractDeliveryDaysText(
|
||||
bodyText: string,
|
||||
services: BuyerProtectionModel[],
|
||||
shipping: ItemBrowserPayload['shipping'],
|
||||
): string | null {
|
||||
const shippingText = cleanText(shipping?.deliveryLimitText) || cleanText(shipping?.logisticsText);
|
||||
if (shippingText) return shippingText;
|
||||
|
||||
const textMatch = bodyText.match(/\d+\s*(?:小时|天)(?:内)?发货/);
|
||||
if (textMatch) return textMatch[0];
|
||||
|
||||
const hourMatch = services.find((service) => typeof service.agreeDeliveryHours === 'number');
|
||||
if (hourMatch && typeof hourMatch.agreeDeliveryHours === 'number') {
|
||||
return `${hourMatch.agreeDeliveryHours}小时内发货`;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractKeywordLine(bodyText: string, keywords: string[]): string | null {
|
||||
const lines = bodyText.split('\n').map((line) => cleanText(line)).filter(Boolean);
|
||||
for (const line of lines) {
|
||||
if (keywords.some((keyword) => line.includes(keyword))) {
|
||||
return line;
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractSalesText(bodyText: string): string | null {
|
||||
const match = bodyText.match(/(?:全网销量|已售)\s*\d+(?:\.\d+)?\+?\s*[件套个单]?/);
|
||||
return match ? cleanText(match[0]) : null;
|
||||
}
|
||||
|
||||
function extractStockQuantity(bodyText: string): number | null {
|
||||
const match = bodyText.match(/库存\s*(\d+)/);
|
||||
return match ? Number.parseInt(match[1], 10) : null;
|
||||
}
|
||||
|
||||
async function readItemPayload(page: IPage, itemUrl: string): Promise<ItemBrowserPayload> {
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'item');
|
||||
assertAuthenticatedState(state, 'item');
|
||||
|
||||
const payload = await page.evaluate(`
|
||||
(() => {
|
||||
const root = window.context ?? {};
|
||||
const model = root.result?.global?.globalData?.model ?? null;
|
||||
const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
|
||||
return {
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
offerTitle: model?.offerTitleModel?.subject ?? '',
|
||||
offerId: model?.tradeModel?.offerId ?? '',
|
||||
seller: toJson(model?.sellerModel),
|
||||
trade: toJson(model?.tradeModel),
|
||||
gallery: toJson(root.result?.data?.gallery?.fields ?? null),
|
||||
shipping: toJson(root.result?.data?.shippingServices?.fields ?? null),
|
||||
services: toJson(root.result?.data?.shippingServices?.fields?.protectionInfos ?? []),
|
||||
};
|
||||
})()
|
||||
`) as ItemBrowserPayload;
|
||||
|
||||
const resolvedOfferId = cleanText(String(payload.offerId ?? '')) || extractOfferId(cleanText(payload.href));
|
||||
if (!resolvedOfferId) {
|
||||
throw new CommandExecutionError(
|
||||
'1688 item page did not expose product context',
|
||||
'当前 tab 非商品详情上下文，请切到 detail.1688.com 商品页并重试',
|
||||
);
|
||||
}
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'item',
|
||||
description: '1688 商品详情（公开商品字段、价格阶梯、卖家基础信息）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 商品 URL 或 offer ID（如 887904326744）',
|
||||
},
|
||||
],
|
||||
columns: ['offer_id', 'title', 'price_text', 'moq_text', 'seller_name', 'origin_place'],
|
||||
func: async (page, kwargs) => {
|
||||
const itemUrl = buildDetailUrl(String(kwargs.input ?? ''));
|
||||
const payload = await readItemPayload(page, itemUrl);
|
||||
return [normalizeItemPayload(payload)];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeItemPayload,
|
||||
normalizeVisibleAttributes,
|
||||
stripAlibabaSuffix,
|
||||
extractMoqText,
|
||||
extractDeliveryDaysText,
|
||||
extractKeywordLine,
|
||||
extractSalesText,
|
||||
extractStockQuantity,
|
||||
};
|
||||
@@ -1,309 +0,0 @@
|
||||
import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { FACTORY_BADGE_PATTERNS, SERVICE_BADGE_PATTERNS, assertAuthenticatedState, buildProvenance, buildSearchUrl, canonicalizeItemUrl, canonicalizeSellerUrl, cleanText, extractBadges, extractLocation, extractMemberId, extractOfferId, extractShopId, gotoAndReadState, parseMoqText, parsePriceText, SEARCH_LIMIT_DEFAULT, SEARCH_LIMIT_MAX, parseSearchLimit, uniqueNonEmpty, } from './shared.js';
|
||||
const SEARCH_ITEM_URL_PATTERNS = [
|
||||
'detail.1688.com/offer/',
|
||||
'detail.m.1688.com/page/index.html?offerId=',
|
||||
];
|
||||
const MAX_SEARCH_PAGES = 12;
|
||||
function normalizeSearchCandidate(candidate, sourceUrl) {
|
||||
const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
|
||||
const containerText = cleanText(candidate.container_text);
|
||||
const priceText = firstNonEmpty([
|
||||
normalizeInlineText(candidate.price_text),
|
||||
normalizeInlineText(extractPriceText(candidate.hover_price_text)),
|
||||
]);
|
||||
const priceRange = parsePriceText(priceText || containerText);
|
||||
const moq = parseMoqText(firstNonEmpty([
|
||||
normalizeInlineText(candidate.moq_text),
|
||||
normalizeInlineText(extractMoqText(containerText)),
|
||||
]));
|
||||
const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
|
||||
const evidenceText = uniqueNonEmpty([
|
||||
containerText,
|
||||
...(candidate.desc_rows ?? []),
|
||||
...(candidate.tag_items ?? []),
|
||||
...(candidate.hover_items ?? []),
|
||||
]).join('\n');
|
||||
const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
|
||||
const salesText = firstNonEmpty([
|
||||
extractSalesText(candidate.sales_text),
|
||||
extractSalesText(containerText),
|
||||
]);
|
||||
const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
return {
|
||||
rank: 0,
|
||||
offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
|
||||
member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
|
||||
shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
|
||||
title: cleanText(candidate.title) || firstWord(containerText) || null,
|
||||
item_url: canonicalItemUrl,
|
||||
seller_name: cleanText(candidate.seller_name) || null,
|
||||
seller_url: canonicalSellerUrl,
|
||||
price_text: priceRange.price_text || null,
|
||||
price_min: priceRange.price_min,
|
||||
price_max: priceRange.price_max,
|
||||
currency: priceRange.currency,
|
||||
moq_text: moq.moq_text || null,
|
||||
moq_value: moq.moq_value,
|
||||
location: extractLocation(containerText),
|
||||
badges,
|
||||
sales_text: salesText || null,
|
||||
return_rate_text: returnRateText,
|
||||
source_url: provenance.source_url,
|
||||
fetched_at: provenance.fetched_at,
|
||||
strategy: provenance.strategy,
|
||||
};
|
||||
}
|
||||
function extractMoqText(text) {
|
||||
const normalized = normalizeInlineText(text);
|
||||
return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
|
||||
?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
|
||||
?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
|
||||
?? '';
|
||||
}
|
||||
function extractPriceText(text) {
|
||||
const normalized = normalizeInlineText(text);
|
||||
return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
|
||||
}
|
||||
function extractSalesText(text) {
|
||||
const normalized = normalizeInlineText(text);
|
||||
if (!normalized)
|
||||
return '';
|
||||
if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
|
||||
return match ? cleanText(match[0]) : '';
|
||||
}
|
||||
function firstWord(text) {
|
||||
return text.split(/\s+/).find(Boolean) ?? '';
|
||||
}
|
||||
function firstNonEmpty(values) {
|
||||
return values.map((value) => cleanText(value)).find(Boolean) ?? '';
|
||||
}
|
||||
function normalizeInlineText(text) {
|
||||
return cleanText(text)
|
||||
.replace(/([¥$€])\s+(?=\d)/g, '$1')
|
||||
.replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
|
||||
.replace(/\s*([~-])\s*/g, '$1')
|
||||
.trim();
|
||||
}
|
||||
function extractReturnRateText(values) {
|
||||
return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
|
||||
.find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
|
||||
?? null;
|
||||
}
|
||||
function buildDedupeKey(row) {
|
||||
if (row.offer_id)
|
||||
return `offer:${row.offer_id}`;
|
||||
if (row.item_url)
|
||||
return `url:${row.item_url}`;
|
||||
return null;
|
||||
}
|
||||
async function readSearchPayload(page, url) {
|
||||
const state = await gotoAndReadState(page, url, 2500, 'search');
|
||||
assertAuthenticatedState(state, 'search');
|
||||
const payload = await page.evaluate(`
|
||||
(() => {
|
||||
const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
|
||||
const normalizeUrl = (href) => {
|
||||
if (!href) return '';
|
||||
try {
|
||||
return new URL(href, window.location.href).toString();
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
|
||||
.some((pattern) => (href || '').includes(pattern));
|
||||
const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
|
||||
const collectTexts = (root, selector) => uniqueTexts(
|
||||
Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
|
||||
);
|
||||
const firstText = (root, selectors) => {
|
||||
for (const selector of selectors) {
|
||||
const node = root.querySelector(selector);
|
||||
const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
|
||||
if (value) return value;
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const findMoqText = (values, priceText) => {
|
||||
const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
|
||||
return values.find((value) => moqPattern.test(value))
|
||||
|| normalizeText(priceText).match(moqPattern)?.[0]
|
||||
|| '';
|
||||
};
|
||||
const isSellerHref = (href) => {
|
||||
if (!href) return false;
|
||||
try {
|
||||
const url = new URL(href, window.location.href);
|
||||
const host = url.hostname || '';
|
||||
if (!host.endsWith('.1688.com')) return false;
|
||||
if (
|
||||
host === 's.1688.com'
|
||||
|| host === 'r.1688.com'
|
||||
|| host === 'air.1688.com'
|
||||
|| host === 'detail.1688.com'
|
||||
|| host === 'detail.m.1688.com'
|
||||
|| host === 'dj.1688.com'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const pickContainer = (anchor) => {
|
||||
let node = anchor;
|
||||
while (node && node !== document.body) {
|
||||
const text = normalizeText(node.innerText || node.textContent || '');
|
||||
if (text.length >= 40 && text.length <= 2000) {
|
||||
return node;
|
||||
}
|
||||
node = node.parentElement;
|
||||
}
|
||||
return anchor;
|
||||
};
|
||||
const collectCandidates = () => {
|
||||
const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
|
||||
const seen = new Set();
|
||||
const items = [];
|
||||
for (const anchor of anchors) {
|
||||
const href = anchor.href || '';
|
||||
if (!href || seen.has(href)) continue;
|
||||
seen.add(href);
|
||||
|
||||
const container = pickContainer(anchor);
|
||||
const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
|
||||
const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
|
||||
const sellerAnchor = Array.from(container.querySelectorAll('a'))
|
||||
.find((link) => isSellerHref(link.href || ''));
|
||||
const hoverPriceText = firstText(container, [
|
||||
'.offer-hover-wrapper .hover-price-item',
|
||||
'.offer-hover-wrapper .price-item',
|
||||
]);
|
||||
|
||||
items.push({
|
||||
item_url: href,
|
||||
title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
|
||||
|| normalizeText(anchor.innerText || anchor.textContent || ''),
|
||||
container_text: normalizeText(container.innerText || container.textContent || ''),
|
||||
desc_rows: collectTexts(container, '.offer-desc-row'),
|
||||
price_text: firstText(container, ['.offer-price-row .price-item']),
|
||||
sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
|
||||
hover_price_text: hoverPriceText,
|
||||
moq_text: findMoqText(hoverItems, hoverPriceText),
|
||||
tag_items: tagItems,
|
||||
hover_items: hoverItems,
|
||||
seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
|
||||
seller_url: sellerAnchor ? sellerAnchor.href : null,
|
||||
});
|
||||
}
|
||||
return items;
|
||||
};
|
||||
const findNextUrl = () => {
|
||||
const selectors = [
|
||||
'a.fui-next:not(.disabled)',
|
||||
'a.next-pagination-item:not(.disabled)',
|
||||
'a[rel="next"]:not(.disabled)',
|
||||
'a[data-role="next"]:not(.disabled)',
|
||||
];
|
||||
for (const selector of selectors) {
|
||||
const node = document.querySelector(selector);
|
||||
if (!node) continue;
|
||||
const href = normalizeUrl(node.getAttribute('href') || node.href || '');
|
||||
if (href) return href;
|
||||
}
|
||||
const textBased = Array.from(document.querySelectorAll('a'))
|
||||
.find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
|
||||
if (!textBased) return '';
|
||||
return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
|
||||
};
|
||||
|
||||
return {
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
next_url: findNextUrl(),
|
||||
candidates: collectCandidates(),
|
||||
};
|
||||
})()
|
||||
`);
|
||||
if (!payload || typeof payload !== 'object') {
|
||||
throw new CommandExecutionError('1688 search page did not return a readable payload', 'Open the same query in Chrome and verify the page is fully loaded before retrying.');
|
||||
}
|
||||
return payload;
|
||||
}
|
||||
async function collectSearchRows(page, query, limit) {
|
||||
const rowsByKey = new Map();
|
||||
const seenPages = new Set();
|
||||
let nextUrl = buildSearchUrl(query);
|
||||
let pageCount = 0;
|
||||
while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
|
||||
if (seenPages.has(nextUrl))
|
||||
break;
|
||||
seenPages.add(nextUrl);
|
||||
pageCount += 1;
|
||||
const payload = await readSearchPayload(page, nextUrl);
|
||||
const sourceUrl = cleanText(payload.href) || nextUrl;
|
||||
const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
|
||||
for (const candidate of candidates) {
|
||||
const row = normalizeSearchCandidate(candidate, sourceUrl);
|
||||
const dedupeKey = buildDedupeKey(row);
|
||||
if (!dedupeKey || rowsByKey.has(dedupeKey))
|
||||
continue;
|
||||
rowsByKey.set(dedupeKey, row);
|
||||
if (rowsByKey.size >= limit)
|
||||
break;
|
||||
}
|
||||
const candidateNextUrl = cleanText(payload.next_url);
|
||||
if (!candidateNextUrl || candidateNextUrl === sourceUrl)
|
||||
break;
|
||||
nextUrl = candidateNextUrl;
|
||||
}
|
||||
if (rowsByKey.size === 0) {
|
||||
throw new EmptyResultError('1688 search', 'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.');
|
||||
}
|
||||
return [...rowsByKey.values()]
|
||||
.slice(0, limit)
|
||||
.map((row, index) => ({ ...row, rank: index + 1 }));
|
||||
}
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'search',
|
||||
description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'query',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '搜索关键词，如 "置物架"',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: SEARCH_LIMIT_DEFAULT,
|
||||
help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'title', 'price_text', 'moq_text', 'seller_name', 'location'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query ?? '');
|
||||
const limit = parseSearchLimit(kwargs.limit);
|
||||
return collectSearchRows(page, query, limit);
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeSearchCandidate,
|
||||
extractMoqText,
|
||||
extractSalesText,
|
||||
firstWord,
|
||||
buildDedupeKey,
|
||||
};
|
||||
@@ -1,75 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './search.js';
|
||||
describe('1688 search normalization', () => {
|
||||
it('normalizes search candidates into structured result rows', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '宿舍置物架桌面加高架',
|
||||
container_text: '宿舍置物架桌面加高架 ¥56.00 2套起批 山东青岛 已售300+套',
|
||||
price_text: '¥ 56 .00',
|
||||
sales_text: '300+套',
|
||||
moq_text: '2套起批',
|
||||
tag_items: ['退货包运费', '回头率52%'],
|
||||
hover_items: ['验厂报告'],
|
||||
seller_name: '青岛沁澜衣品服装有限公司',
|
||||
seller_url: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a123',
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=置物架');
|
||||
expect(result.rank).toBe(0);
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.shop_id).toBe('yinuoweierfushi');
|
||||
expect(result.item_url).toBe('https://detail.1688.com/offer/887904326744.html');
|
||||
expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.price_text).toBe('¥56.00');
|
||||
expect(result.price_min).toBe(56);
|
||||
expect(result.price_max).toBe(56);
|
||||
expect(result.moq_value).toBe(2);
|
||||
expect(result.location).toBe('山东青岛');
|
||||
expect(result.sales_text).toBe('300+套');
|
||||
expect(result.badges).toEqual(expect.arrayContaining(['退货包运费', '验厂报告']));
|
||||
expect(result.return_rate_text).toBe('回头率52%');
|
||||
});
|
||||
it('does not use hover_price_text as MOQ source', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: 'test',
|
||||
container_text: 'test ¥56.00',
|
||||
price_text: '¥ 56 .00',
|
||||
hover_price_text: '¥56.00 3件起批',
|
||||
moq_text: null,
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=test');
|
||||
// hover_price_text should not be used for MOQ extraction
|
||||
expect(result.moq_text).toBeNull();
|
||||
expect(result.moq_value).toBeNull();
|
||||
});
|
||||
it('extracts offer id from mobile detail search links', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'http://detail.m.1688.com/page/index.html?offerId=910933345396&sortType=&pageId=',
|
||||
title: '',
|
||||
container_text: '桌面书桌办公室工位收纳展示新中式博古架多层茶具厨房摆放置物架 ¥24.3 已售20+件',
|
||||
price_text: '¥ 14 .28',
|
||||
sales_text: '1500+件',
|
||||
moq_text: '≥2个',
|
||||
seller_name: '泰商国际贸易（宁阳）有限公司',
|
||||
seller_url: 'http://tsgjmy.1688.com/',
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=桌面置物架');
|
||||
expect(result.offer_id).toBe('910933345396');
|
||||
expect(result.shop_id).toBe('tsgjmy');
|
||||
expect(result.item_url).toBe('https://detail.1688.com/offer/910933345396.html');
|
||||
expect(result.title).toContain('桌面书桌办公室工位收纳展示');
|
||||
expect(result.price_text).toBe('¥14.28');
|
||||
expect(result.sales_text).toBe('1500+件');
|
||||
expect(result.moq_text).toBe('≥2个');
|
||||
expect(result.moq_value).toBe(2);
|
||||
});
|
||||
it('prefers offer id and falls back to item url for dedupe key', () => {
|
||||
expect(__test__.buildDedupeKey({
|
||||
offer_id: '123456',
|
||||
item_url: 'https://detail.1688.com/offer/123456.html',
|
||||
})).toBe('offer:123456');
|
||||
expect(__test__.buildDedupeKey({
|
||||
offer_id: null,
|
||||
item_url: 'https://detail.1688.com/offer/123456.html',
|
||||
})).toBe('url:https://detail.1688.com/offer/123456.html');
|
||||
expect(__test__.buildDedupeKey({ offer_id: null, item_url: null })).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,81 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './search.js';
|
||||
|
||||
describe('1688 search normalization', () => {
|
||||
it('normalizes search candidates into structured result rows', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: '宿舍置物架桌面加高架',
|
||||
container_text: '宿舍置物架桌面加高架 ¥56.00 2套起批 山东青岛 已售300+套',
|
||||
price_text: '¥ 56 .00',
|
||||
sales_text: '300+套',
|
||||
moq_text: '2套起批',
|
||||
tag_items: ['退货包运费', '回头率52%'],
|
||||
hover_items: ['验厂报告'],
|
||||
seller_name: '青岛沁澜衣品服装有限公司',
|
||||
seller_url: 'https://yinuoweierfushi.1688.com/page/index.html?spm=a123',
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=置物架');
|
||||
|
||||
expect(result.rank).toBe(0);
|
||||
expect(result.offer_id).toBe('887904326744');
|
||||
expect(result.shop_id).toBe('yinuoweierfushi');
|
||||
expect(result.item_url).toBe('https://detail.1688.com/offer/887904326744.html');
|
||||
expect(result.seller_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.price_text).toBe('¥56.00');
|
||||
expect(result.price_min).toBe(56);
|
||||
expect(result.price_max).toBe(56);
|
||||
expect(result.moq_value).toBe(2);
|
||||
expect(result.location).toBe('山东青岛');
|
||||
expect(result.sales_text).toBe('300+套');
|
||||
expect(result.badges).toEqual(expect.arrayContaining(['退货包运费', '验厂报告']));
|
||||
expect(result.return_rate_text).toBe('回头率52%');
|
||||
});
|
||||
|
||||
it('does not use hover_price_text as MOQ source', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'https://detail.1688.com/offer/887904326744.html',
|
||||
title: 'test',
|
||||
container_text: 'test ¥56.00',
|
||||
price_text: '¥ 56 .00',
|
||||
hover_price_text: '¥56.00 3件起批',
|
||||
moq_text: null,
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=test');
|
||||
// hover_price_text should not be used for MOQ extraction
|
||||
expect(result.moq_text).toBeNull();
|
||||
expect(result.moq_value).toBeNull();
|
||||
});
|
||||
|
||||
it('extracts offer id from mobile detail search links', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
item_url: 'http://detail.m.1688.com/page/index.html?offerId=910933345396&sortType=&pageId=',
|
||||
title: '',
|
||||
container_text: '桌面书桌办公室工位收纳展示新中式博古架多层茶具厨房摆放置物架 ¥24.3 已售20+件',
|
||||
price_text: '¥ 14 .28',
|
||||
sales_text: '1500+件',
|
||||
moq_text: '≥2个',
|
||||
seller_name: '泰商国际贸易（宁阳）有限公司',
|
||||
seller_url: 'http://tsgjmy.1688.com/',
|
||||
}, 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=桌面置物架');
|
||||
|
||||
expect(result.offer_id).toBe('910933345396');
|
||||
expect(result.shop_id).toBe('tsgjmy');
|
||||
expect(result.item_url).toBe('https://detail.1688.com/offer/910933345396.html');
|
||||
expect(result.title).toContain('桌面书桌办公室工位收纳展示');
|
||||
expect(result.price_text).toBe('¥14.28');
|
||||
expect(result.sales_text).toBe('1500+件');
|
||||
expect(result.moq_text).toBe('≥2个');
|
||||
expect(result.moq_value).toBe(2);
|
||||
});
|
||||
|
||||
it('prefers offer id and falls back to item url for dedupe key', () => {
|
||||
expect(__test__.buildDedupeKey({
|
||||
offer_id: '123456',
|
||||
item_url: 'https://detail.1688.com/offer/123456.html',
|
||||
})).toBe('offer:123456');
|
||||
expect(__test__.buildDedupeKey({
|
||||
offer_id: null,
|
||||
item_url: 'https://detail.1688.com/offer/123456.html',
|
||||
})).toBe('url:https://detail.1688.com/offer/123456.html');
|
||||
expect(__test__.buildDedupeKey({ offer_id: null, item_url: null })).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,402 @@
|
||||
import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
FACTORY_BADGE_PATTERNS,
|
||||
SERVICE_BADGE_PATTERNS,
|
||||
assertAuthenticatedState,
|
||||
buildProvenance,
|
||||
buildSearchUrl,
|
||||
canonicalizeItemUrl,
|
||||
canonicalizeSellerUrl,
|
||||
cleanText,
|
||||
extractBadges,
|
||||
extractLocation,
|
||||
extractMemberId,
|
||||
extractOfferId,
|
||||
extractShopId,
|
||||
gotoAndReadState,
|
||||
parseMoqText,
|
||||
parsePriceText,
|
||||
SEARCH_LIMIT_DEFAULT,
|
||||
SEARCH_LIMIT_MAX,
|
||||
parseSearchLimit,
|
||||
uniqueNonEmpty,
|
||||
} from './shared.js';
|
||||
|
||||
interface SearchPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
bodyText?: string;
|
||||
next_url?: string;
|
||||
candidates?: Array<{
|
||||
item_url?: string;
|
||||
title?: string;
|
||||
container_text?: string;
|
||||
desc_rows?: string[];
|
||||
price_text?: string | null;
|
||||
sales_text?: string | null;
|
||||
hover_price_text?: string | null;
|
||||
moq_text?: string | null;
|
||||
tag_items?: string[];
|
||||
hover_items?: string[];
|
||||
seller_name?: string | null;
|
||||
seller_url?: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface SearchRow {
|
||||
rank: number;
|
||||
offer_id: string | null;
|
||||
member_id: string | null;
|
||||
shop_id: string | null;
|
||||
title: string | null;
|
||||
item_url: string | null;
|
||||
seller_name: string | null;
|
||||
seller_url: string | null;
|
||||
price_text: string | null;
|
||||
price_min: number | null;
|
||||
price_max: number | null;
|
||||
currency: string | null;
|
||||
moq_text: string | null;
|
||||
moq_value: number | null;
|
||||
location: string | null;
|
||||
badges: string[];
|
||||
sales_text: string | null;
|
||||
return_rate_text: string | null;
|
||||
source_url: string;
|
||||
fetched_at: string;
|
||||
strategy: string;
|
||||
}
|
||||
|
||||
const SEARCH_ITEM_URL_PATTERNS = [
|
||||
'detail.1688.com/offer/',
|
||||
'detail.m.1688.com/page/index.html?offerId=',
|
||||
];
|
||||
const MAX_SEARCH_PAGES = 12;
|
||||
|
||||
function normalizeSearchCandidate(
|
||||
candidate: NonNullable<SearchPayload['candidates']>[number],
|
||||
sourceUrl: string,
|
||||
): SearchRow {
|
||||
const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
|
||||
const containerText = cleanText(candidate.container_text);
|
||||
const priceText = firstNonEmpty([
|
||||
normalizeInlineText(candidate.price_text),
|
||||
normalizeInlineText(extractPriceText(candidate.hover_price_text)),
|
||||
]);
|
||||
const priceRange = parsePriceText(priceText || containerText);
|
||||
const moq = parseMoqText(firstNonEmpty([
|
||||
normalizeInlineText(candidate.moq_text),
|
||||
normalizeInlineText(extractMoqText(containerText)),
|
||||
]));
|
||||
const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
|
||||
const evidenceText = uniqueNonEmpty([
|
||||
containerText,
|
||||
...(candidate.desc_rows ?? []),
|
||||
...(candidate.tag_items ?? []),
|
||||
...(candidate.hover_items ?? []),
|
||||
]).join('\n');
|
||||
const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
|
||||
const salesText = firstNonEmpty([
|
||||
extractSalesText(candidate.sales_text),
|
||||
extractSalesText(containerText),
|
||||
]);
|
||||
const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
|
||||
return {
|
||||
rank: 0,
|
||||
offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
|
||||
member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
|
||||
shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
|
||||
title: cleanText(candidate.title) || firstWord(containerText) || null,
|
||||
item_url: canonicalItemUrl,
|
||||
seller_name: cleanText(candidate.seller_name) || null,
|
||||
seller_url: canonicalSellerUrl,
|
||||
price_text: priceRange.price_text || null,
|
||||
price_min: priceRange.price_min,
|
||||
price_max: priceRange.price_max,
|
||||
currency: priceRange.currency,
|
||||
moq_text: moq.moq_text || null,
|
||||
moq_value: moq.moq_value,
|
||||
location: extractLocation(containerText),
|
||||
badges,
|
||||
sales_text: salesText || null,
|
||||
return_rate_text: returnRateText,
|
||||
source_url: provenance.source_url,
|
||||
fetched_at: provenance.fetched_at,
|
||||
strategy: provenance.strategy,
|
||||
};
|
||||
}
|
||||
|
||||
function extractMoqText(text: string | null | undefined): string {
|
||||
const normalized = normalizeInlineText(text);
|
||||
return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
|
||||
?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
|
||||
?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
|
||||
?? '';
|
||||
}
|
||||
|
||||
function extractPriceText(text: string | null | undefined): string {
|
||||
const normalized = normalizeInlineText(text);
|
||||
return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
|
||||
}
|
||||
|
||||
function extractSalesText(text: string | null | undefined): string {
|
||||
const normalized = normalizeInlineText(text);
|
||||
if (!normalized) return '';
|
||||
if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
|
||||
return normalized;
|
||||
}
|
||||
const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
|
||||
return match ? cleanText(match[0]) : '';
|
||||
}
|
||||
|
||||
function firstWord(text: string): string {
|
||||
return text.split(/\s+/).find(Boolean) ?? '';
|
||||
}
|
||||
|
||||
function firstNonEmpty(values: Array<string | null | undefined>): string {
|
||||
return values.map((value) => cleanText(value)).find(Boolean) ?? '';
|
||||
}
|
||||
|
||||
function normalizeInlineText(text: string | null | undefined): string {
|
||||
return cleanText(text)
|
||||
.replace(/([¥$€])\s+(?=\d)/g, '$1')
|
||||
.replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
|
||||
.replace(/\s*([~-])\s*/g, '$1')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function extractReturnRateText(values: string[]): string | null {
|
||||
return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
|
||||
.find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
|
||||
?? null;
|
||||
}
|
||||
|
||||
function buildDedupeKey(row: Pick<SearchRow, 'offer_id' | 'item_url'>): string | null {
|
||||
if (row.offer_id) return `offer:${row.offer_id}`;
|
||||
if (row.item_url) return `url:${row.item_url}`;
|
||||
return null;
|
||||
}
|
||||
|
||||
async function readSearchPayload(page: IPage, url: string): Promise<SearchPayload> {
|
||||
const state = await gotoAndReadState(page, url, 2500, 'search');
|
||||
assertAuthenticatedState(state, 'search');
|
||||
|
||||
const payload = await page.evaluate(`
|
||||
(() => {
|
||||
const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
|
||||
const normalizeUrl = (href) => {
|
||||
if (!href) return '';
|
||||
try {
|
||||
return new URL(href, window.location.href).toString();
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
};
|
||||
const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
|
||||
.some((pattern) => (href || '').includes(pattern));
|
||||
const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
|
||||
const collectTexts = (root, selector) => uniqueTexts(
|
||||
Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
|
||||
);
|
||||
const firstText = (root, selectors) => {
|
||||
for (const selector of selectors) {
|
||||
const node = root.querySelector(selector);
|
||||
const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
|
||||
if (value) return value;
|
||||
}
|
||||
return '';
|
||||
};
|
||||
const findMoqText = (values, priceText) => {
|
||||
const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
|
||||
return values.find((value) => moqPattern.test(value))
|
||||
|| normalizeText(priceText).match(moqPattern)?.[0]
|
||||
|| '';
|
||||
};
|
||||
const isSellerHref = (href) => {
|
||||
if (!href) return false;
|
||||
try {
|
||||
const url = new URL(href, window.location.href);
|
||||
const host = url.hostname || '';
|
||||
if (!host.endsWith('.1688.com')) return false;
|
||||
if (
|
||||
host === 's.1688.com'
|
||||
|| host === 'r.1688.com'
|
||||
|| host === 'air.1688.com'
|
||||
|| host === 'detail.1688.com'
|
||||
|| host === 'detail.m.1688.com'
|
||||
|| host === 'dj.1688.com'
|
||||
) {
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
};
|
||||
const pickContainer = (anchor) => {
|
||||
let node = anchor;
|
||||
while (node && node !== document.body) {
|
||||
const text = normalizeText(node.innerText || node.textContent || '');
|
||||
if (text.length >= 40 && text.length <= 2000) {
|
||||
return node;
|
||||
}
|
||||
node = node.parentElement;
|
||||
}
|
||||
return anchor;
|
||||
};
|
||||
const collectCandidates = () => {
|
||||
const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
|
||||
const seen = new Set();
|
||||
const items = [];
|
||||
for (const anchor of anchors) {
|
||||
const href = anchor.href || '';
|
||||
if (!href || seen.has(href)) continue;
|
||||
seen.add(href);
|
||||
|
||||
const container = pickContainer(anchor);
|
||||
const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
|
||||
const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
|
||||
const sellerAnchor = Array.from(container.querySelectorAll('a'))
|
||||
.find((link) => isSellerHref(link.href || ''));
|
||||
const hoverPriceText = firstText(container, [
|
||||
'.offer-hover-wrapper .hover-price-item',
|
||||
'.offer-hover-wrapper .price-item',
|
||||
]);
|
||||
|
||||
items.push({
|
||||
item_url: href,
|
||||
title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
|
||||
|| normalizeText(anchor.innerText || anchor.textContent || ''),
|
||||
container_text: normalizeText(container.innerText || container.textContent || ''),
|
||||
desc_rows: collectTexts(container, '.offer-desc-row'),
|
||||
price_text: firstText(container, ['.offer-price-row .price-item']),
|
||||
sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
|
||||
hover_price_text: hoverPriceText,
|
||||
moq_text: findMoqText(hoverItems, hoverPriceText),
|
||||
tag_items: tagItems,
|
||||
hover_items: hoverItems,
|
||||
seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
|
||||
seller_url: sellerAnchor ? sellerAnchor.href : null,
|
||||
});
|
||||
}
|
||||
return items;
|
||||
};
|
||||
const findNextUrl = () => {
|
||||
const selectors = [
|
||||
'a.fui-next:not(.disabled)',
|
||||
'a.next-pagination-item:not(.disabled)',
|
||||
'a[rel="next"]:not(.disabled)',
|
||||
'a[data-role="next"]:not(.disabled)',
|
||||
];
|
||||
for (const selector of selectors) {
|
||||
const node = document.querySelector(selector);
|
||||
if (!node) continue;
|
||||
const href = normalizeUrl(node.getAttribute('href') || node.href || '');
|
||||
if (href) return href;
|
||||
}
|
||||
const textBased = Array.from(document.querySelectorAll('a'))
|
||||
.find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
|
||||
if (!textBased) return '';
|
||||
return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
|
||||
};
|
||||
|
||||
return {
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
next_url: findNextUrl(),
|
||||
candidates: collectCandidates(),
|
||||
};
|
||||
})()
|
||||
`) as SearchPayload;
|
||||
|
||||
if (!payload || typeof payload !== 'object') {
|
||||
throw new CommandExecutionError(
|
||||
'1688 search page did not return a readable payload',
|
||||
'Open the same query in Chrome and verify the page is fully loaded before retrying.',
|
||||
);
|
||||
}
|
||||
|
||||
return payload;
|
||||
}
|
||||
|
||||
async function collectSearchRows(page: IPage, query: string, limit: number): Promise<SearchRow[]> {
|
||||
const rowsByKey = new Map<string, SearchRow>();
|
||||
const seenPages = new Set<string>();
|
||||
let nextUrl = buildSearchUrl(query);
|
||||
let pageCount = 0;
|
||||
|
||||
while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
|
||||
if (seenPages.has(nextUrl)) break;
|
||||
seenPages.add(nextUrl);
|
||||
pageCount += 1;
|
||||
|
||||
const payload = await readSearchPayload(page, nextUrl);
|
||||
const sourceUrl = cleanText(payload.href) || nextUrl;
|
||||
const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
|
||||
|
||||
for (const candidate of candidates) {
|
||||
const row = normalizeSearchCandidate(candidate, sourceUrl);
|
||||
const dedupeKey = buildDedupeKey(row);
|
||||
if (!dedupeKey || rowsByKey.has(dedupeKey)) continue;
|
||||
rowsByKey.set(dedupeKey, row);
|
||||
if (rowsByKey.size >= limit) break;
|
||||
}
|
||||
|
||||
const candidateNextUrl = cleanText(payload.next_url);
|
||||
if (!candidateNextUrl || candidateNextUrl === sourceUrl) break;
|
||||
nextUrl = candidateNextUrl;
|
||||
}
|
||||
|
||||
if (rowsByKey.size === 0) {
|
||||
throw new EmptyResultError(
|
||||
'1688 search',
|
||||
'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.',
|
||||
);
|
||||
}
|
||||
|
||||
return [...rowsByKey.values()]
|
||||
.slice(0, limit)
|
||||
.map((row, index) => ({ ...row, rank: index + 1 }));
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'search',
|
||||
description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'query',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '搜索关键词，如 "置物架"',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: SEARCH_LIMIT_DEFAULT,
|
||||
help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'title', 'price_text', 'moq_text', 'seller_name', 'location'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query ?? '');
|
||||
const limit = parseSearchLimit(kwargs.limit);
|
||||
return collectSearchRows(page, query, limit);
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeSearchCandidate,
|
||||
extractMoqText,
|
||||
extractSalesText,
|
||||
firstWord,
|
||||
buildDedupeKey,
|
||||
};
|
||||
@@ -1,557 +0,0 @@
|
||||
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
export const SITE = '1688';
|
||||
export const HOME_URL = 'https://www.1688.com/';
|
||||
export const SEARCH_URL_PREFIX = 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=';
|
||||
export const DETAIL_URL_PREFIX = 'https://detail.1688.com/offer/';
|
||||
export const STORE_MOBILE_URL_PREFIX = 'https://winport.m.1688.com/page/index.html?memberId=';
|
||||
export const STRATEGY = 'cookie';
|
||||
export const SEARCH_LIMIT_DEFAULT = 20;
|
||||
export const SEARCH_LIMIT_MAX = 100;
|
||||
const STORE_GENERIC_HOSTS = new Set(['www', 'detail', 's', 'winport', 'work', 'air', 'dj']);
|
||||
const TRACKING_QUERY_KEYS = new Set([
|
||||
'spm',
|
||||
'tracelog',
|
||||
'clickid',
|
||||
'source',
|
||||
'scene',
|
||||
'from',
|
||||
'src',
|
||||
'ns',
|
||||
'cna',
|
||||
'pvid',
|
||||
]);
|
||||
const CAPTCHA_URL_MARKER = '/_____tmd_____/punish';
|
||||
const CAPTCHA_TEXT_PATTERNS = [
|
||||
'请拖动下方滑块完成验证',
|
||||
'请按住滑块，拖动到最右边',
|
||||
'通过验证以确保正常访问',
|
||||
'验证码拦截',
|
||||
'访问验证',
|
||||
'滑动验证',
|
||||
];
|
||||
const LOGIN_TEXT_PATTERNS = [
|
||||
'请登录',
|
||||
'登录后',
|
||||
'账号登录',
|
||||
'手机登录',
|
||||
'立即登录',
|
||||
'扫码登录',
|
||||
'请先完成登录',
|
||||
'请先登录后查看',
|
||||
];
|
||||
const LOGIN_URL_PATTERNS = ['/member/login', 'passport', 'login.taobao.com', 'account.1688.com'];
|
||||
export const FACTORY_BADGE_PATTERNS = [
|
||||
'源头工厂',
|
||||
'深度验厂',
|
||||
'实力工厂',
|
||||
'工厂档案',
|
||||
'加工专区',
|
||||
'验厂报告',
|
||||
'厂家直销',
|
||||
'生产厂家',
|
||||
'工厂直供',
|
||||
];
|
||||
export const SERVICE_BADGE_PATTERNS = [
|
||||
'延期必赔',
|
||||
'品质保障',
|
||||
'破损包赔',
|
||||
'退货包运费',
|
||||
'晚发必赔',
|
||||
'7*24小时响应',
|
||||
'48小时发货',
|
||||
'72小时发货',
|
||||
'后天达',
|
||||
'包邮',
|
||||
'闪电拿样',
|
||||
];
|
||||
const CHINA_LOCATIONS = [
|
||||
'北京',
|
||||
'天津',
|
||||
'上海',
|
||||
'重庆',
|
||||
'河北',
|
||||
'山西',
|
||||
'辽宁',
|
||||
'吉林',
|
||||
'黑龙江',
|
||||
'江苏',
|
||||
'浙江',
|
||||
'安徽',
|
||||
'福建',
|
||||
'江西',
|
||||
'山东',
|
||||
'河南',
|
||||
'湖北',
|
||||
'湖南',
|
||||
'广东',
|
||||
'海南',
|
||||
'四川',
|
||||
'贵州',
|
||||
'云南',
|
||||
'陕西',
|
||||
'甘肃',
|
||||
'青海',
|
||||
'台湾',
|
||||
'内蒙古',
|
||||
'广西',
|
||||
'西藏',
|
||||
'宁夏',
|
||||
'新疆',
|
||||
'香港',
|
||||
'澳门',
|
||||
];
|
||||
export function cleanText(value) {
|
||||
return typeof value === 'string'
|
||||
? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
|
||||
: '';
|
||||
}
|
||||
export function cleanMultilineText(value) {
|
||||
return typeof value === 'string'
|
||||
? value
|
||||
.replace(/\u00a0/g, ' ')
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: '';
|
||||
}
|
||||
export function uniqueNonEmpty(values) {
|
||||
return [...new Set(values.map((value) => cleanText(value)).filter(Boolean))];
|
||||
}
|
||||
export function parseSearchLimit(input) {
|
||||
const parsed = Number.parseInt(String(input ?? SEARCH_LIMIT_DEFAULT), 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new ArgumentError('1688 search --limit must be a positive integer', 'Example: opencli 1688 search "桌面置物架" --limit 20');
|
||||
}
|
||||
return Math.min(SEARCH_LIMIT_MAX, parsed);
|
||||
}
|
||||
export function buildSearchUrl(query) {
|
||||
const normalized = cleanText(query);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError('1688 search query cannot be empty', 'Example: opencli 1688 search "桌面置物架" --limit 20');
|
||||
}
|
||||
return `${SEARCH_URL_PREFIX}${encodeURIComponent(normalized)}`;
|
||||
}
|
||||
export function buildDetailUrl(input) {
|
||||
const offerId = extractOfferId(input);
|
||||
if (!offerId) {
|
||||
throw new ArgumentError('1688 item expects an offer URL or offer ID', 'Example: opencli 1688 item 887904326744');
|
||||
}
|
||||
return `${DETAIL_URL_PREFIX}${offerId}.html`;
|
||||
}
|
||||
export function resolveStoreUrl(input) {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError('1688 store expects a store URL or member ID', 'Example: opencli 1688 store https://yinuoweierfushi.1688.com/');
|
||||
}
|
||||
const memberId = extractMemberId(normalized);
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
if (/^https?:\/\//i.test(normalized)) {
|
||||
return canonicalizeStoreUrl(normalized);
|
||||
}
|
||||
if (normalized.endsWith('.1688.com')) {
|
||||
return canonicalizeStoreUrl(`https://${normalized}`);
|
||||
}
|
||||
if (/^[a-z0-9-]+$/i.test(normalized)) {
|
||||
return canonicalizeStoreUrl(`https://${normalized}.1688.com`);
|
||||
}
|
||||
throw new ArgumentError('1688 store expects a store URL or member ID', 'Example: opencli 1688 store b2b-22154705262941f196');
|
||||
}
|
||||
export function canonicalizeStoreUrl(input) {
|
||||
const url = parse1688Url(input);
|
||||
const memberId = extractMemberId(url.toString());
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host) {
|
||||
throw new ArgumentError('Invalid 1688 store URL', 'Example: opencli 1688 store https://yinuoweierfushi.1688.com/');
|
||||
}
|
||||
return `https://${host}`;
|
||||
}
|
||||
export function canonicalizeItemUrl(input) {
|
||||
const offerId = extractOfferId(input);
|
||||
if (offerId) {
|
||||
return `${DETAIL_URL_PREFIX}${offerId}.html`;
|
||||
}
|
||||
const url = parse1688UrlOrNull(input);
|
||||
if (!url)
|
||||
return null;
|
||||
stripTrackingParams(url);
|
||||
url.hash = '';
|
||||
return url.toString();
|
||||
}
|
||||
export function canonicalizeSellerUrl(input) {
|
||||
const memberId = extractMemberId(input);
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
const url = parse1688UrlOrNull(input);
|
||||
if (!url)
|
||||
return null;
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host)
|
||||
return null;
|
||||
return `https://${host}`;
|
||||
}
|
||||
export function extractOfferId(input) {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized)
|
||||
return null;
|
||||
const directId = normalized.match(/^\d{6,}$/)?.[0];
|
||||
if (directId)
|
||||
return directId;
|
||||
const detailMatch = normalized.match(/\/offer\/(\d{6,})\.html/i);
|
||||
if (detailMatch)
|
||||
return detailMatch[1];
|
||||
const queryMatch = normalized.match(/[?&]offerId=(\d{6,})/i);
|
||||
if (queryMatch)
|
||||
return queryMatch[1];
|
||||
return null;
|
||||
}
|
||||
export function extractMemberId(input) {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized)
|
||||
return null;
|
||||
const direct = normalized.match(/\bb2b-[a-z0-9]+\b/i)?.[0];
|
||||
if (direct)
|
||||
return direct;
|
||||
const queryMatch = normalized.match(/[?&]memberId=(b2b-[a-z0-9]+)/i);
|
||||
if (queryMatch)
|
||||
return queryMatch[1];
|
||||
const mobileMatch = normalized.match(/\/winport\/(b2b-[a-z0-9]+)\.html/i);
|
||||
if (mobileMatch)
|
||||
return mobileMatch[1];
|
||||
return null;
|
||||
}
|
||||
export function extractShopId(input) {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized)
|
||||
return null;
|
||||
try {
|
||||
const url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`);
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host)
|
||||
return null;
|
||||
return host.split('.')[0] ?? null;
|
||||
}
|
||||
catch {
|
||||
return /^[a-z0-9-]+$/i.test(normalized) ? normalized : null;
|
||||
}
|
||||
}
|
||||
export function buildProvenance(sourceUrl) {
|
||||
return {
|
||||
source_url: sourceUrl,
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: STRATEGY,
|
||||
};
|
||||
}
|
||||
export function parsePriceText(text) {
|
||||
const normalized = normalizeNumericText(cleanText(text));
|
||||
const matches = normalized.match(/\d+(?:,\d{3})*(?:\.\d+)?/g) ?? [];
|
||||
const values = matches
|
||||
.map((value) => Number.parseFloat(value.replace(/,/g, '')))
|
||||
.filter((value) => Number.isFinite(value));
|
||||
if (values.length === 0) {
|
||||
return {
|
||||
price_text: normalized,
|
||||
price_min: null,
|
||||
price_max: null,
|
||||
currency: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
price_text: normalized,
|
||||
price_min: values[0] ?? null,
|
||||
price_max: values[values.length - 1] ?? values[0] ?? null,
|
||||
currency: normalized.includes('¥') || normalized.includes('元') ? 'CNY' : null,
|
||||
};
|
||||
}
|
||||
export function normalizePriceTiers(rawTiers, unit) {
|
||||
return rawTiers
|
||||
.map((tier) => {
|
||||
const quantityMin = toNumber(tier.beginAmount);
|
||||
const priceText = cleanText(tier.price);
|
||||
const price = toNumber(tier.price);
|
||||
return {
|
||||
quantity_text: quantityMin !== null ? `${quantityMin}${unit ?? ''}` : '',
|
||||
quantity_min: quantityMin,
|
||||
price_text: priceText,
|
||||
price,
|
||||
currency: priceText ? 'CNY' : null,
|
||||
};
|
||||
})
|
||||
.filter((tier) => tier.price_text);
|
||||
}
|
||||
export function parseMoqText(text) {
|
||||
const normalized = normalizeNumericText(cleanText(text));
|
||||
const match = normalized.match(/(\d+(?:\.\d+)?)\s*(件|个|套|箱|包|双|台|把|只|pcs|piece|pieces)?\s*起批/i)
|
||||
?? normalized.match(/≥\s*(\d+(?:\.\d+)?)/);
|
||||
const rangeMatch = normalized.match(/(\d+(?:\.\d+)?)\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只|pcs|piece|pieces)/i);
|
||||
if (!match && !rangeMatch) {
|
||||
return {
|
||||
moq_text: normalized,
|
||||
moq_value: null,
|
||||
};
|
||||
}
|
||||
return {
|
||||
moq_text: normalized,
|
||||
moq_value: Number.parseFloat((match ?? rangeMatch)[1]),
|
||||
};
|
||||
}
|
||||
export function extractLocation(text) {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const primaryRegion = normalized.split(/送至|发往/)[0] ?? normalized;
|
||||
const lines = primaryRegion.split('\n');
|
||||
for (const line of lines) {
|
||||
const compact = cleanText(line);
|
||||
if (!compact || compact.length > 16)
|
||||
continue;
|
||||
if (CHINA_LOCATIONS.some((location) => compact.startsWith(location))) {
|
||||
return compact;
|
||||
}
|
||||
}
|
||||
const locationPattern = new RegExp(`(${CHINA_LOCATIONS.join('|')})[\\u4e00-\\u9fa5]{0,8}`);
|
||||
return primaryRegion.match(locationPattern)?.[0] ?? null;
|
||||
}
|
||||
export function extractAddress(text) {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const lineMatch = normalized.match(/地址[:：]\s*([^\n]+)/);
|
||||
if (lineMatch)
|
||||
return cleanText(lineMatch[1]);
|
||||
return normalized
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => line.includes('省') || line.includes('市') || line.includes('区') || line.includes('县'))
|
||||
?? null;
|
||||
}
|
||||
export function extractMetric(text, label) {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const direct = normalized.match(new RegExp(`(?:^|\\n)\\s*${escapeForRegex(label)}[:：]?\\s*([^\\n]+)`));
|
||||
if (direct)
|
||||
return cleanText(direct[1]);
|
||||
const lineBased = normalized.match(new RegExp(`(?:^|\\n)\\s*${escapeForRegex(label)}\\n([^\\n]+)`));
|
||||
return lineBased ? cleanText(lineBased[1]) : null;
|
||||
}
|
||||
export function extractYearsOnPlatform(text) {
|
||||
return text.match(/入驻\d+年/)?.[0] ?? null;
|
||||
}
|
||||
export function extractMainBusiness(text) {
|
||||
const value = extractMetric(text, '主营');
|
||||
return value ? value.replace(/^：/, '').trim() : null;
|
||||
}
|
||||
export function extractBadges(text, candidates) {
|
||||
return uniqueNonEmpty(candidates.filter((candidate) => cleanMultilineText(text).includes(candidate)));
|
||||
}
|
||||
export function guessTopCategories(text) {
|
||||
const mainBusiness = extractMainBusiness(text);
|
||||
if (!mainBusiness)
|
||||
return [];
|
||||
return uniqueNonEmpty(mainBusiness.split(/[、,/|]/).map((value) => value.trim()));
|
||||
}
|
||||
export function isCaptchaState(state) {
|
||||
const href = cleanText(state.href).toLowerCase();
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
if (href.includes(CAPTCHA_URL_MARKER))
|
||||
return true;
|
||||
return CAPTCHA_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
export function isLoginState(state) {
|
||||
const href = cleanText(state.href).toLowerCase();
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
if (LOGIN_URL_PATTERNS.some((pattern) => href.includes(pattern)))
|
||||
return true;
|
||||
return LOGIN_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
export function buildCaptchaHint(action) {
|
||||
return [
|
||||
`Open a clean 1688 ${action} page in the shared Chrome profile and finish any slider challenge first.`,
|
||||
'If you run opencli via CDP, set OPENCLI_CDP_TARGET=1688.com or a more specific 1688 host before retrying.',
|
||||
].join(' ');
|
||||
}
|
||||
export async function readPageState(page) {
|
||||
const result = await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
body_text: document.body ? document.body.innerText || '' : '',
|
||||
}))()
|
||||
`);
|
||||
return {
|
||||
href: cleanText(result.href),
|
||||
title: cleanText(result.title),
|
||||
body_text: cleanMultilineText(result.body_text),
|
||||
};
|
||||
}
|
||||
export async function gotoAndReadState(page, url, settleMs = 2500, action = 'page') {
|
||||
try {
|
||||
await page.goto(url, { settleMs });
|
||||
await page.wait(1.5);
|
||||
return readPageState(page);
|
||||
}
|
||||
catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes('Inspected target navigated or closed')
|
||||
|| message.includes('Cannot find context with specified id')
|
||||
|| message.includes('Target closed')) {
|
||||
throw new CommandExecutionError(`1688 ${action} navigation lost the current browser target`, `${buildCaptchaHint(action)} If CDP is attached to a stale or blocked tab, open a fresh 1688 tab and point OPENCLI_CDP_TARGET at that tab.`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
export async function ensure1688Session(page) {
|
||||
const state = await gotoAndReadState(page, HOME_URL, 1500, 'homepage');
|
||||
assertAuthenticatedState(state, 'homepage');
|
||||
}
|
||||
export function assertAuthenticatedState(state, action) {
|
||||
if (!isCaptchaState(state) && !isLoginState(state))
|
||||
return;
|
||||
throw new AuthRequiredError('1688.com', `请先在共享 Chrome 完成 1688 登录/验证，再重试（${action}）`);
|
||||
}
|
||||
export function assertNotCaptcha(state, action) {
|
||||
assertAuthenticatedState(state, action);
|
||||
}
|
||||
export function toNumber(value) {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const normalized = value.replace(/,/g, '').trim();
|
||||
if (!normalized)
|
||||
return null;
|
||||
const parsed = Number.parseFloat(normalized);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
export function limitCandidates(values, limit) {
|
||||
const normalizedLimit = Math.max(1, Math.trunc(limit) || 1);
|
||||
return values.slice(0, normalizedLimit);
|
||||
}
|
||||
export function normalizeMediaUrl(input) {
|
||||
const raw = cleanText(input);
|
||||
if (!raw)
|
||||
return '';
|
||||
let value = raw
|
||||
.replace(/^url\((.*)\)$/i, '$1')
|
||||
.replace(/^['"]|['"]$/g, '')
|
||||
.replace(/\\u002F/g, '/')
|
||||
.replace(/&/g, '&')
|
||||
.trim();
|
||||
if (!value || value.startsWith('data:') || value.startsWith('blob:'))
|
||||
return '';
|
||||
if (value.startsWith('//'))
|
||||
value = `https:${value}`;
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.toString();
|
||||
}
|
||||
catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
export function uniqueMediaSources(values) {
|
||||
const seen = new Set();
|
||||
const result = [];
|
||||
for (const value of values) {
|
||||
const url = normalizeMediaUrl(value.url);
|
||||
if (!url)
|
||||
continue;
|
||||
const key = `${value.type}:${url}`;
|
||||
if (seen.has(key))
|
||||
continue;
|
||||
seen.add(key);
|
||||
result.push({
|
||||
...value,
|
||||
url,
|
||||
source: cleanText(value.source) || undefined,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
function normalizeNumericText(value) {
|
||||
return value
|
||||
.replace(/([¥$€])\s+(?=\d)/g, '$1')
|
||||
.replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
|
||||
.replace(/\s*([~-])\s*/g, '$1')
|
||||
.trim();
|
||||
}
|
||||
function escapeForRegex(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
function parse1688Url(input) {
|
||||
const normalized = cleanText(input);
|
||||
try {
|
||||
const url = new URL(normalized);
|
||||
if (!url.hostname.endsWith('.1688.com') && url.hostname !== '1688.com' && url.hostname !== 'www.1688.com') {
|
||||
throw new Error('invalid-host');
|
||||
}
|
||||
stripTrackingParams(url);
|
||||
url.hash = '';
|
||||
return url;
|
||||
}
|
||||
catch {
|
||||
throw new ArgumentError('Invalid 1688 URL', 'Use a URL under 1688.com (for example: https://detail.1688.com/offer/887904326744.html)');
|
||||
}
|
||||
}
|
||||
function parse1688UrlOrNull(input) {
|
||||
try {
|
||||
return parse1688Url(input);
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function normalizeStoreHost(hostname) {
|
||||
const lower = cleanText(hostname).toLowerCase();
|
||||
if (!lower.endsWith('.1688.com'))
|
||||
return null;
|
||||
const [subdomain] = lower.split('.');
|
||||
if (!subdomain || STORE_GENERIC_HOSTS.has(subdomain))
|
||||
return null;
|
||||
return lower;
|
||||
}
|
||||
function stripTrackingParams(url) {
|
||||
const keys = [...url.searchParams.keys()];
|
||||
for (const key of keys) {
|
||||
if (TRACKING_QUERY_KEYS.has(key) || key.toLowerCase().startsWith('utm_')) {
|
||||
url.searchParams.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
export const __test__ = {
|
||||
SEARCH_LIMIT_DEFAULT,
|
||||
SEARCH_LIMIT_MAX,
|
||||
parseSearchLimit,
|
||||
buildSearchUrl,
|
||||
buildDetailUrl,
|
||||
resolveStoreUrl,
|
||||
canonicalizeStoreUrl,
|
||||
canonicalizeItemUrl,
|
||||
canonicalizeSellerUrl,
|
||||
extractOfferId,
|
||||
extractMemberId,
|
||||
extractShopId,
|
||||
parsePriceText,
|
||||
normalizePriceTiers,
|
||||
parseMoqText,
|
||||
extractLocation,
|
||||
extractAddress,
|
||||
extractMetric,
|
||||
extractYearsOnPlatform,
|
||||
extractMainBusiness,
|
||||
extractBadges,
|
||||
guessTopCategories,
|
||||
isCaptchaState,
|
||||
isLoginState,
|
||||
cleanText,
|
||||
cleanMultilineText,
|
||||
uniqueNonEmpty,
|
||||
normalizeMediaUrl,
|
||||
uniqueMediaSources,
|
||||
limitCandidates,
|
||||
};
|
||||
@@ -1,57 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './shared.js';
|
||||
describe('1688 shared helpers', () => {
|
||||
it('builds encoded search URLs and validates limit', () => {
|
||||
expect(__test__.buildSearchUrl('置物架')).toBe('https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=%E7%BD%AE%E7%89%A9%E6%9E%B6');
|
||||
expect(() => __test__.buildSearchUrl(' ')).toThrowError(/cannot be empty/i);
|
||||
expect(__test__.parseSearchLimit(3)).toBe(3);
|
||||
expect(__test__.parseSearchLimit('1000')).toBe(__test__.SEARCH_LIMIT_MAX);
|
||||
expect(() => __test__.parseSearchLimit('0')).toThrowError(/positive integer/i);
|
||||
});
|
||||
it('extracts IDs and canonicalizes urls', () => {
|
||||
expect(__test__.extractOfferId('887904326744')).toBe('887904326744');
|
||||
expect(__test__.extractOfferId('https://detail.1688.com/offer/887904326744.html')).toBe('887904326744');
|
||||
expect(__test__.extractMemberId('https://winport.m.1688.com/page/index.html?memberId=b2b-1641351767')).toBe('b2b-1641351767');
|
||||
expect(__test__.extractMemberId('b2b-22154705262941f196')).toBe('b2b-22154705262941f196');
|
||||
expect(__test__.resolveStoreUrl('b2b-22154705262941f196')).toBe('https://winport.m.1688.com/page/index.html?memberId=b2b-22154705262941f196');
|
||||
expect(__test__.canonicalizeStoreUrl('https://yinuoweierfushi.1688.com/page/index.html?spm=foo')).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(__test__.canonicalizeItemUrl('http://detail.m.1688.com/page/index.html?offerId=910933345396&spm=x')).toBe('https://detail.1688.com/offer/910933345396.html');
|
||||
expect(__test__.canonicalizeSellerUrl('https://yinuoweierfushi.1688.com/page/contactinfo.html?tracelog=1')).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(__test__.extractShopId('https://yinuoweierfushi.1688.com/page/index.html')).toBe('yinuoweierfushi');
|
||||
});
|
||||
it('parses price ranges and moq text', () => {
|
||||
expect(__test__.parsePriceText('¥96.00-98.00')).toEqual({
|
||||
price_text: '¥96.00-98.00',
|
||||
price_min: 96,
|
||||
price_max: 98,
|
||||
currency: 'CNY',
|
||||
});
|
||||
expect(__test__.parsePriceText('¥ 14 .28')).toEqual({
|
||||
price_text: '¥14.28',
|
||||
price_min: 14.28,
|
||||
price_max: 14.28,
|
||||
currency: 'CNY',
|
||||
});
|
||||
expect(__test__.parseMoqText('3套起批')).toEqual({
|
||||
moq_text: '3套起批',
|
||||
moq_value: 3,
|
||||
});
|
||||
expect(__test__.parseMoqText('2~999个')).toEqual({
|
||||
moq_text: '2~999个',
|
||||
moq_value: 2,
|
||||
});
|
||||
});
|
||||
it('detects captcha and login states', () => {
|
||||
expect(__test__.extractLocation('山东青岛 送至 江苏苏州')).toBe('山东青岛');
|
||||
expect(__test__.isCaptchaState({
|
||||
href: 'https://s.1688.com/_____tmd_____/punish',
|
||||
title: '验证码拦截',
|
||||
body_text: '请拖动下方滑块完成验证',
|
||||
})).toBe(true);
|
||||
expect(__test__.isLoginState({
|
||||
href: 'https://login.taobao.com/member/login.jhtml',
|
||||
title: '账号登录',
|
||||
body_text: '请登录后继续',
|
||||
})).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,75 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './shared.js';
|
||||
|
||||
describe('1688 shared helpers', () => {
|
||||
it('builds encoded search URLs and validates limit', () => {
|
||||
expect(__test__.buildSearchUrl('置物架')).toBe(
|
||||
'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=%E7%BD%AE%E7%89%A9%E6%9E%B6',
|
||||
);
|
||||
expect(() => __test__.buildSearchUrl(' ')).toThrowError(/cannot be empty/i);
|
||||
|
||||
expect(__test__.parseSearchLimit(3)).toBe(3);
|
||||
expect(__test__.parseSearchLimit('1000')).toBe(__test__.SEARCH_LIMIT_MAX);
|
||||
expect(() => __test__.parseSearchLimit('0')).toThrowError(/positive integer/i);
|
||||
});
|
||||
|
||||
it('extracts IDs and canonicalizes urls', () => {
|
||||
expect(__test__.extractOfferId('887904326744')).toBe('887904326744');
|
||||
expect(__test__.extractOfferId('https://detail.1688.com/offer/887904326744.html')).toBe('887904326744');
|
||||
expect(__test__.extractMemberId('https://winport.m.1688.com/page/index.html?memberId=b2b-1641351767')).toBe('b2b-1641351767');
|
||||
expect(__test__.extractMemberId('b2b-22154705262941f196')).toBe('b2b-22154705262941f196');
|
||||
expect(__test__.resolveStoreUrl('b2b-22154705262941f196')).toBe(
|
||||
'https://winport.m.1688.com/page/index.html?memberId=b2b-22154705262941f196',
|
||||
);
|
||||
expect(__test__.canonicalizeStoreUrl('https://yinuoweierfushi.1688.com/page/index.html?spm=foo')).toBe(
|
||||
'https://yinuoweierfushi.1688.com',
|
||||
);
|
||||
expect(__test__.canonicalizeItemUrl('http://detail.m.1688.com/page/index.html?offerId=910933345396&spm=x')).toBe(
|
||||
'https://detail.1688.com/offer/910933345396.html',
|
||||
);
|
||||
expect(__test__.canonicalizeSellerUrl('https://yinuoweierfushi.1688.com/page/contactinfo.html?tracelog=1')).toBe(
|
||||
'https://yinuoweierfushi.1688.com',
|
||||
);
|
||||
expect(__test__.extractShopId('https://yinuoweierfushi.1688.com/page/index.html')).toBe('yinuoweierfushi');
|
||||
});
|
||||
|
||||
it('parses price ranges and moq text', () => {
|
||||
expect(__test__.parsePriceText('¥96.00-98.00')).toEqual({
|
||||
price_text: '¥96.00-98.00',
|
||||
price_min: 96,
|
||||
price_max: 98,
|
||||
currency: 'CNY',
|
||||
});
|
||||
|
||||
expect(__test__.parsePriceText('¥ 14 .28')).toEqual({
|
||||
price_text: '¥14.28',
|
||||
price_min: 14.28,
|
||||
price_max: 14.28,
|
||||
currency: 'CNY',
|
||||
});
|
||||
|
||||
expect(__test__.parseMoqText('3套起批')).toEqual({
|
||||
moq_text: '3套起批',
|
||||
moq_value: 3,
|
||||
});
|
||||
|
||||
expect(__test__.parseMoqText('2~999个')).toEqual({
|
||||
moq_text: '2~999个',
|
||||
moq_value: 2,
|
||||
});
|
||||
});
|
||||
|
||||
it('detects captcha and login states', () => {
|
||||
expect(__test__.extractLocation('山东青岛 送至 江苏苏州')).toBe('山东青岛');
|
||||
expect(__test__.isCaptchaState({
|
||||
href: 'https://s.1688.com/_____tmd_____/punish',
|
||||
title: '验证码拦截',
|
||||
body_text: '请拖动下方滑块完成验证',
|
||||
})).toBe(true);
|
||||
expect(__test__.isLoginState({
|
||||
href: 'https://login.taobao.com/member/login.jhtml',
|
||||
title: '账号登录',
|
||||
body_text: '请登录后继续',
|
||||
})).toBe(true);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,672 @@
|
||||
import { ArgumentError, AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
|
||||
export const SITE = '1688';
|
||||
export const HOME_URL = 'https://www.1688.com/';
|
||||
export const SEARCH_URL_PREFIX = 'https://s.1688.com/selloffer/offer_search.htm?charset=utf8&keywords=';
|
||||
export const DETAIL_URL_PREFIX = 'https://detail.1688.com/offer/';
|
||||
export const STORE_MOBILE_URL_PREFIX = 'https://winport.m.1688.com/page/index.html?memberId=';
|
||||
export const STRATEGY = 'cookie';
|
||||
export const SEARCH_LIMIT_DEFAULT = 20;
|
||||
export const SEARCH_LIMIT_MAX = 100;
|
||||
|
||||
const STORE_GENERIC_HOSTS = new Set(['www', 'detail', 's', 'winport', 'work', 'air', 'dj']);
|
||||
const TRACKING_QUERY_KEYS = new Set([
|
||||
'spm',
|
||||
'tracelog',
|
||||
'clickid',
|
||||
'source',
|
||||
'scene',
|
||||
'from',
|
||||
'src',
|
||||
'ns',
|
||||
'cna',
|
||||
'pvid',
|
||||
]);
|
||||
const CAPTCHA_URL_MARKER = '/_____tmd_____/punish';
|
||||
const CAPTCHA_TEXT_PATTERNS = [
|
||||
'请拖动下方滑块完成验证',
|
||||
'请按住滑块，拖动到最右边',
|
||||
'通过验证以确保正常访问',
|
||||
'验证码拦截',
|
||||
'访问验证',
|
||||
'滑动验证',
|
||||
];
|
||||
const LOGIN_TEXT_PATTERNS = [
|
||||
'请登录',
|
||||
'登录后',
|
||||
'账号登录',
|
||||
'手机登录',
|
||||
'立即登录',
|
||||
'扫码登录',
|
||||
'请先完成登录',
|
||||
'请先登录后查看',
|
||||
];
|
||||
const LOGIN_URL_PATTERNS = ['/member/login', 'passport', 'login.taobao.com', 'account.1688.com'];
|
||||
|
||||
export const FACTORY_BADGE_PATTERNS = [
|
||||
'源头工厂',
|
||||
'深度验厂',
|
||||
'实力工厂',
|
||||
'工厂档案',
|
||||
'加工专区',
|
||||
'验厂报告',
|
||||
'厂家直销',
|
||||
'生产厂家',
|
||||
'工厂直供',
|
||||
];
|
||||
export const SERVICE_BADGE_PATTERNS = [
|
||||
'延期必赔',
|
||||
'品质保障',
|
||||
'破损包赔',
|
||||
'退货包运费',
|
||||
'晚发必赔',
|
||||
'7*24小时响应',
|
||||
'48小时发货',
|
||||
'72小时发货',
|
||||
'后天达',
|
||||
'包邮',
|
||||
'闪电拿样',
|
||||
];
|
||||
|
||||
const CHINA_LOCATIONS = [
|
||||
'北京',
|
||||
'天津',
|
||||
'上海',
|
||||
'重庆',
|
||||
'河北',
|
||||
'山西',
|
||||
'辽宁',
|
||||
'吉林',
|
||||
'黑龙江',
|
||||
'江苏',
|
||||
'浙江',
|
||||
'安徽',
|
||||
'福建',
|
||||
'江西',
|
||||
'山东',
|
||||
'河南',
|
||||
'湖北',
|
||||
'湖南',
|
||||
'广东',
|
||||
'海南',
|
||||
'四川',
|
||||
'贵州',
|
||||
'云南',
|
||||
'陕西',
|
||||
'甘肃',
|
||||
'青海',
|
||||
'台湾',
|
||||
'内蒙古',
|
||||
'广西',
|
||||
'西藏',
|
||||
'宁夏',
|
||||
'新疆',
|
||||
'香港',
|
||||
'澳门',
|
||||
];
|
||||
|
||||
export interface ProvenanceFields {
|
||||
source_url: string;
|
||||
fetched_at: string;
|
||||
strategy: string;
|
||||
}
|
||||
|
||||
export interface PageState {
|
||||
href: string;
|
||||
title: string;
|
||||
body_text: string;
|
||||
}
|
||||
|
||||
export interface PriceRange {
|
||||
price_text: string;
|
||||
price_min: number | null;
|
||||
price_max: number | null;
|
||||
currency: string | null;
|
||||
}
|
||||
|
||||
export interface MoqValue {
|
||||
moq_text: string;
|
||||
moq_value: number | null;
|
||||
}
|
||||
|
||||
export interface PriceTier {
|
||||
quantity_text: string;
|
||||
quantity_min: number | null;
|
||||
price_text: string;
|
||||
price: number | null;
|
||||
currency: string | null;
|
||||
}
|
||||
|
||||
export interface SearchCandidate {
|
||||
item_url: string;
|
||||
title: string;
|
||||
container_text: string;
|
||||
seller_name: string | null;
|
||||
seller_url: string | null;
|
||||
}
|
||||
|
||||
export interface MediaSource {
|
||||
type: 'image' | 'video';
|
||||
group: 'main' | 'sku' | 'detail' | 'video' | 'unknown';
|
||||
url: string;
|
||||
source?: string;
|
||||
}
|
||||
|
||||
export function cleanText(value: unknown): string {
|
||||
return typeof value === 'string'
|
||||
? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
|
||||
: '';
|
||||
}
|
||||
|
||||
export function cleanMultilineText(value: unknown): string {
|
||||
return typeof value === 'string'
|
||||
? value
|
||||
.replace(/\u00a0/g, ' ')
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: '';
|
||||
}
|
||||
|
||||
export function uniqueNonEmpty(values: Array<string | null | undefined>): string[] {
|
||||
return [...new Set(values.map((value) => cleanText(value)).filter(Boolean))];
|
||||
}
|
||||
|
||||
export function parseSearchLimit(input: unknown): number {
|
||||
const parsed = Number.parseInt(String(input ?? SEARCH_LIMIT_DEFAULT), 10);
|
||||
if (!Number.isFinite(parsed) || parsed < 1) {
|
||||
throw new ArgumentError(
|
||||
'1688 search --limit must be a positive integer',
|
||||
'Example: opencli 1688 search "桌面置物架" --limit 20',
|
||||
);
|
||||
}
|
||||
return Math.min(SEARCH_LIMIT_MAX, parsed);
|
||||
}
|
||||
|
||||
export function buildSearchUrl(query: string): string {
|
||||
const normalized = cleanText(query);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError(
|
||||
'1688 search query cannot be empty',
|
||||
'Example: opencli 1688 search "桌面置物架" --limit 20',
|
||||
);
|
||||
}
|
||||
return `${SEARCH_URL_PREFIX}${encodeURIComponent(normalized)}`;
|
||||
}
|
||||
|
||||
export function buildDetailUrl(input: string): string {
|
||||
const offerId = extractOfferId(input);
|
||||
if (!offerId) {
|
||||
throw new ArgumentError(
|
||||
'1688 item expects an offer URL or offer ID',
|
||||
'Example: opencli 1688 item 887904326744',
|
||||
);
|
||||
}
|
||||
return `${DETAIL_URL_PREFIX}${offerId}.html`;
|
||||
}
|
||||
|
||||
export function resolveStoreUrl(input: string): string {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError(
|
||||
'1688 store expects a store URL or member ID',
|
||||
'Example: opencli 1688 store https://yinuoweierfushi.1688.com/',
|
||||
);
|
||||
}
|
||||
|
||||
const memberId = extractMemberId(normalized);
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
|
||||
if (/^https?:\/\//i.test(normalized)) {
|
||||
return canonicalizeStoreUrl(normalized);
|
||||
}
|
||||
|
||||
if (normalized.endsWith('.1688.com')) {
|
||||
return canonicalizeStoreUrl(`https://${normalized}`);
|
||||
}
|
||||
|
||||
if (/^[a-z0-9-]+$/i.test(normalized)) {
|
||||
return canonicalizeStoreUrl(`https://${normalized}.1688.com`);
|
||||
}
|
||||
|
||||
throw new ArgumentError(
|
||||
'1688 store expects a store URL or member ID',
|
||||
'Example: opencli 1688 store b2b-22154705262941f196',
|
||||
);
|
||||
}
|
||||
|
||||
export function canonicalizeStoreUrl(input: string): string {
|
||||
const url = parse1688Url(input);
|
||||
const memberId = extractMemberId(url.toString());
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host) {
|
||||
throw new ArgumentError(
|
||||
'Invalid 1688 store URL',
|
||||
'Example: opencli 1688 store https://yinuoweierfushi.1688.com/',
|
||||
);
|
||||
}
|
||||
return `https://${host}`;
|
||||
}
|
||||
|
||||
export function canonicalizeItemUrl(input: string): string | null {
|
||||
const offerId = extractOfferId(input);
|
||||
if (offerId) {
|
||||
return `${DETAIL_URL_PREFIX}${offerId}.html`;
|
||||
}
|
||||
const url = parse1688UrlOrNull(input);
|
||||
if (!url) return null;
|
||||
stripTrackingParams(url);
|
||||
url.hash = '';
|
||||
return url.toString();
|
||||
}
|
||||
|
||||
export function canonicalizeSellerUrl(input: string): string | null {
|
||||
const memberId = extractMemberId(input);
|
||||
if (memberId) {
|
||||
return `${STORE_MOBILE_URL_PREFIX}${memberId}`;
|
||||
}
|
||||
const url = parse1688UrlOrNull(input);
|
||||
if (!url) return null;
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host) return null;
|
||||
return `https://${host}`;
|
||||
}
|
||||
|
||||
export function extractOfferId(input: string): string | null {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) return null;
|
||||
const directId = normalized.match(/^\d{6,}$/)?.[0];
|
||||
if (directId) return directId;
|
||||
const detailMatch = normalized.match(/\/offer\/(\d{6,})\.html/i);
|
||||
if (detailMatch) return detailMatch[1];
|
||||
const queryMatch = normalized.match(/[?&]offerId=(\d{6,})/i);
|
||||
if (queryMatch) return queryMatch[1];
|
||||
return null;
|
||||
}
|
||||
|
||||
export function extractMemberId(input: string): string | null {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) return null;
|
||||
const direct = normalized.match(/\bb2b-[a-z0-9]+\b/i)?.[0];
|
||||
if (direct) return direct;
|
||||
const queryMatch = normalized.match(/[?&]memberId=(b2b-[a-z0-9]+)/i);
|
||||
if (queryMatch) return queryMatch[1];
|
||||
const mobileMatch = normalized.match(/\/winport\/(b2b-[a-z0-9]+)\.html/i);
|
||||
if (mobileMatch) return mobileMatch[1];
|
||||
return null;
|
||||
}
|
||||
|
||||
export function extractShopId(input: string): string | null {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) return null;
|
||||
|
||||
try {
|
||||
const url = new URL(/^https?:\/\//i.test(normalized) ? normalized : `https://${normalized}`);
|
||||
const host = normalizeStoreHost(url.hostname);
|
||||
if (!host) return null;
|
||||
return host.split('.')[0] ?? null;
|
||||
} catch {
|
||||
return /^[a-z0-9-]+$/i.test(normalized) ? normalized : null;
|
||||
}
|
||||
}
|
||||
|
||||
export function buildProvenance(sourceUrl: string): ProvenanceFields {
|
||||
return {
|
||||
source_url: sourceUrl,
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: STRATEGY,
|
||||
};
|
||||
}
|
||||
|
||||
export function parsePriceText(text: string): PriceRange {
|
||||
const normalized = normalizeNumericText(cleanText(text));
|
||||
const matches = normalized.match(/\d+(?:,\d{3})*(?:\.\d+)?/g) ?? [];
|
||||
const values = matches
|
||||
.map((value) => Number.parseFloat(value.replace(/,/g, '')))
|
||||
.filter((value) => Number.isFinite(value));
|
||||
|
||||
if (values.length === 0) {
|
||||
return {
|
||||
price_text: normalized,
|
||||
price_min: null,
|
||||
price_max: null,
|
||||
currency: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
price_text: normalized,
|
||||
price_min: values[0] ?? null,
|
||||
price_max: values[values.length - 1] ?? values[0] ?? null,
|
||||
currency: normalized.includes('¥') || normalized.includes('元') ? 'CNY' : null,
|
||||
};
|
||||
}
|
||||
|
||||
export function normalizePriceTiers(
|
||||
rawTiers: Array<{ beginAmount?: unknown; price?: unknown }>,
|
||||
unit: string | null,
|
||||
): PriceTier[] {
|
||||
return rawTiers
|
||||
.map((tier) => {
|
||||
const quantityMin = toNumber(tier.beginAmount);
|
||||
const priceText = cleanText(tier.price);
|
||||
const price = toNumber(tier.price);
|
||||
return {
|
||||
quantity_text: quantityMin !== null ? `${quantityMin}${unit ?? ''}` : '',
|
||||
quantity_min: quantityMin,
|
||||
price_text: priceText,
|
||||
price,
|
||||
currency: priceText ? 'CNY' : null,
|
||||
};
|
||||
})
|
||||
.filter((tier) => tier.price_text);
|
||||
}
|
||||
|
||||
export function parseMoqText(text: string): MoqValue {
|
||||
const normalized = normalizeNumericText(cleanText(text));
|
||||
const match = normalized.match(/(\d+(?:\.\d+)?)\s*(件|个|套|箱|包|双|台|把|只|pcs|piece|pieces)?\s*起批/i)
|
||||
?? normalized.match(/≥\s*(\d+(?:\.\d+)?)/);
|
||||
const rangeMatch = normalized.match(
|
||||
/(\d+(?:\.\d+)?)\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只|pcs|piece|pieces)/i,
|
||||
);
|
||||
|
||||
if (!match && !rangeMatch) {
|
||||
return {
|
||||
moq_text: normalized,
|
||||
moq_value: null,
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
moq_text: normalized,
|
||||
moq_value: Number.parseFloat((match ?? rangeMatch)![1]),
|
||||
};
|
||||
}
|
||||
|
||||
export function extractLocation(text: string): string | null {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const primaryRegion = normalized.split(/送至|发往/)[0] ?? normalized;
|
||||
const lines = primaryRegion.split('\n');
|
||||
for (const line of lines) {
|
||||
const compact = cleanText(line);
|
||||
if (!compact || compact.length > 16) continue;
|
||||
if (CHINA_LOCATIONS.some((location) => compact.startsWith(location))) {
|
||||
return compact;
|
||||
}
|
||||
}
|
||||
|
||||
const locationPattern = new RegExp(`(${CHINA_LOCATIONS.join('|')})[\\u4e00-\\u9fa5]{0,8}`);
|
||||
return primaryRegion.match(locationPattern)?.[0] ?? null;
|
||||
}
|
||||
|
||||
export function extractAddress(text: string): string | null {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const lineMatch = normalized.match(/地址[:：]\s*([^\n]+)/);
|
||||
if (lineMatch) return cleanText(lineMatch[1]);
|
||||
return normalized
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => line.includes('省') || line.includes('市') || line.includes('区') || line.includes('县'))
|
||||
?? null;
|
||||
}
|
||||
|
||||
export function extractMetric(text: string, label: string): string | null {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const direct = normalized.match(new RegExp(`(?:^|\\n)\\s*${escapeForRegex(label)}[:：]?\\s*([^\\n]+)`));
|
||||
if (direct) return cleanText(direct[1]);
|
||||
|
||||
const lineBased = normalized.match(new RegExp(`(?:^|\\n)\\s*${escapeForRegex(label)}\\n([^\\n]+)`));
|
||||
return lineBased ? cleanText(lineBased[1]) : null;
|
||||
}
|
||||
|
||||
export function extractYearsOnPlatform(text: string): string | null {
|
||||
return text.match(/入驻\d+年/)?.[0] ?? null;
|
||||
}
|
||||
|
||||
export function extractMainBusiness(text: string): string | null {
|
||||
const value = extractMetric(text, '主营');
|
||||
return value ? value.replace(/^：/, '').trim() : null;
|
||||
}
|
||||
|
||||
export function extractBadges(text: string, candidates: string[]): string[] {
|
||||
return uniqueNonEmpty(candidates.filter((candidate) => cleanMultilineText(text).includes(candidate)));
|
||||
}
|
||||
|
||||
export function guessTopCategories(text: string): string[] {
|
||||
const mainBusiness = extractMainBusiness(text);
|
||||
if (!mainBusiness) return [];
|
||||
return uniqueNonEmpty(mainBusiness.split(/[、,/|]/).map((value) => value.trim()));
|
||||
}
|
||||
|
||||
export function isCaptchaState(state: Partial<PageState>): boolean {
|
||||
const href = cleanText(state.href).toLowerCase();
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
if (href.includes(CAPTCHA_URL_MARKER)) return true;
|
||||
return CAPTCHA_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
|
||||
export function isLoginState(state: Partial<PageState>): boolean {
|
||||
const href = cleanText(state.href).toLowerCase();
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
if (LOGIN_URL_PATTERNS.some((pattern) => href.includes(pattern))) return true;
|
||||
return LOGIN_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
|
||||
export function buildCaptchaHint(action: string): string {
|
||||
return [
|
||||
`Open a clean 1688 ${action} page in the shared Chrome profile and finish any slider challenge first.`,
|
||||
'If you run opencli via CDP, set OPENCLI_CDP_TARGET=1688.com or a more specific 1688 host before retrying.',
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
export async function readPageState(page: IPage): Promise<PageState> {
|
||||
const result = await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
body_text: document.body ? document.body.innerText || '' : '',
|
||||
}))()
|
||||
`) as Partial<PageState>;
|
||||
|
||||
return {
|
||||
href: cleanText(result.href),
|
||||
title: cleanText(result.title),
|
||||
body_text: cleanMultilineText(result.body_text),
|
||||
};
|
||||
}
|
||||
|
||||
export async function gotoAndReadState(
|
||||
page: IPage,
|
||||
url: string,
|
||||
settleMs: number = 2500,
|
||||
action: string = 'page',
|
||||
): Promise<PageState> {
|
||||
try {
|
||||
await page.goto(url, { settleMs });
|
||||
await page.wait(1.5);
|
||||
return readPageState(page);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (
|
||||
message.includes('Inspected target navigated or closed')
|
||||
|| message.includes('Cannot find context with specified id')
|
||||
|| message.includes('Target closed')
|
||||
) {
|
||||
throw new CommandExecutionError(
|
||||
`1688 ${action} navigation lost the current browser target`,
|
||||
`${buildCaptchaHint(action)} If CDP is attached to a stale or blocked tab, open a fresh 1688 tab and point OPENCLI_CDP_TARGET at that tab.`,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export async function ensure1688Session(page: IPage): Promise<void> {
|
||||
const state = await gotoAndReadState(page, HOME_URL, 1500, 'homepage');
|
||||
assertAuthenticatedState(state, 'homepage');
|
||||
}
|
||||
|
||||
export function assertAuthenticatedState(state: PageState, action: string): void {
|
||||
if (!isCaptchaState(state) && !isLoginState(state)) return;
|
||||
throw new AuthRequiredError('1688.com', `请先在共享 Chrome 完成 1688 登录/验证，再重试（${action}）`);
|
||||
}
|
||||
|
||||
export function assertNotCaptcha(state: PageState, action: string): void {
|
||||
assertAuthenticatedState(state, action);
|
||||
}
|
||||
|
||||
export function toNumber(value: unknown): number | null {
|
||||
if (typeof value === 'number' && Number.isFinite(value)) {
|
||||
return value;
|
||||
}
|
||||
if (typeof value === 'string') {
|
||||
const normalized = value.replace(/,/g, '').trim();
|
||||
if (!normalized) return null;
|
||||
const parsed = Number.parseFloat(normalized);
|
||||
return Number.isFinite(parsed) ? parsed : null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
export function limitCandidates<T>(values: T[], limit: number): T[] {
|
||||
const normalizedLimit = Math.max(1, Math.trunc(limit) || 1);
|
||||
return values.slice(0, normalizedLimit);
|
||||
}
|
||||
|
||||
export function normalizeMediaUrl(input: unknown): string {
|
||||
const raw = cleanText(input);
|
||||
if (!raw) return '';
|
||||
|
||||
let value = raw
|
||||
.replace(/^url\((.*)\)$/i, '$1')
|
||||
.replace(/^['"]|['"]$/g, '')
|
||||
.replace(/\\u002F/g, '/')
|
||||
.replace(/&/g, '&')
|
||||
.trim();
|
||||
|
||||
if (!value || value.startsWith('data:') || value.startsWith('blob:')) return '';
|
||||
if (value.startsWith('//')) value = `https:${value}`;
|
||||
|
||||
try {
|
||||
const url = new URL(value);
|
||||
return url.toString();
|
||||
} catch {
|
||||
return '';
|
||||
}
|
||||
}
|
||||
|
||||
export function uniqueMediaSources(values: MediaSource[]): MediaSource[] {
|
||||
const seen = new Set<string>();
|
||||
const result: MediaSource[] = [];
|
||||
for (const value of values) {
|
||||
const url = normalizeMediaUrl(value.url);
|
||||
if (!url) continue;
|
||||
const key = `${value.type}:${url}`;
|
||||
if (seen.has(key)) continue;
|
||||
seen.add(key);
|
||||
result.push({
|
||||
...value,
|
||||
url,
|
||||
source: cleanText(value.source) || undefined,
|
||||
});
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
function normalizeNumericText(value: string): string {
|
||||
return value
|
||||
.replace(/([¥$€])\s+(?=\d)/g, '$1')
|
||||
.replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
|
||||
.replace(/\s*([~-])\s*/g, '$1')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function escapeForRegex(value: string): string {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
||||
}
|
||||
|
||||
function parse1688Url(input: string): URL {
|
||||
const normalized = cleanText(input);
|
||||
try {
|
||||
const url = new URL(normalized);
|
||||
if (!url.hostname.endsWith('.1688.com') && url.hostname !== '1688.com' && url.hostname !== 'www.1688.com') {
|
||||
throw new Error('invalid-host');
|
||||
}
|
||||
stripTrackingParams(url);
|
||||
url.hash = '';
|
||||
return url;
|
||||
} catch {
|
||||
throw new ArgumentError(
|
||||
'Invalid 1688 URL',
|
||||
'Use a URL under 1688.com (for example: https://detail.1688.com/offer/887904326744.html)',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function parse1688UrlOrNull(input: string): URL | null {
|
||||
try {
|
||||
return parse1688Url(input);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function normalizeStoreHost(hostname: string): string | null {
|
||||
const lower = cleanText(hostname).toLowerCase();
|
||||
if (!lower.endsWith('.1688.com')) return null;
|
||||
const [subdomain] = lower.split('.');
|
||||
if (!subdomain || STORE_GENERIC_HOSTS.has(subdomain)) return null;
|
||||
return lower;
|
||||
}
|
||||
|
||||
function stripTrackingParams(url: URL): void {
|
||||
const keys = [...url.searchParams.keys()];
|
||||
for (const key of keys) {
|
||||
if (TRACKING_QUERY_KEYS.has(key) || key.toLowerCase().startsWith('utm_')) {
|
||||
url.searchParams.delete(key);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
SEARCH_LIMIT_DEFAULT,
|
||||
SEARCH_LIMIT_MAX,
|
||||
parseSearchLimit,
|
||||
buildSearchUrl,
|
||||
buildDetailUrl,
|
||||
resolveStoreUrl,
|
||||
canonicalizeStoreUrl,
|
||||
canonicalizeItemUrl,
|
||||
canonicalizeSellerUrl,
|
||||
extractOfferId,
|
||||
extractMemberId,
|
||||
extractShopId,
|
||||
parsePriceText,
|
||||
normalizePriceTiers,
|
||||
parseMoqText,
|
||||
extractLocation,
|
||||
extractAddress,
|
||||
extractMetric,
|
||||
extractYearsOnPlatform,
|
||||
extractMainBusiness,
|
||||
extractBadges,
|
||||
guessTopCategories,
|
||||
isCaptchaState,
|
||||
isLoginState,
|
||||
cleanText,
|
||||
cleanMultilineText,
|
||||
uniqueNonEmpty,
|
||||
normalizeMediaUrl,
|
||||
uniqueMediaSources,
|
||||
limitCandidates,
|
||||
};
|
||||
@@ -1,226 +0,0 @@
|
||||
import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { FACTORY_BADGE_PATTERNS, SERVICE_BADGE_PATTERNS, assertAuthenticatedState, buildDetailUrl, buildProvenance, canonicalizeSellerUrl, canonicalizeStoreUrl, cleanMultilineText, cleanText, extractAddress, extractBadges, extractMemberId, extractMetric, extractOfferId, extractShopId, extractYearsOnPlatform, gotoAndReadState, guessTopCategories, resolveStoreUrl, uniqueNonEmpty, } from './shared.js';
|
||||
function normalizeStorePayload(input) {
|
||||
const storePayload = input.storePayload;
|
||||
const contactPayload = input.contactPayload;
|
||||
const seed = input.seed;
|
||||
const contactText = cleanMultilineText(contactPayload?.bodyText);
|
||||
const storeText = cleanMultilineText(storePayload?.bodyText);
|
||||
const seedText = cleanMultilineText(seed?.bodyText);
|
||||
const combinedText = [contactText, storeText, seedText].filter(Boolean).join('\n');
|
||||
const sellerUrlRaw = cleanText(seed?.seller?.winportUrl
|
||||
?? seed?.seller?.sellerWinportUrlMap?.defaultUrl
|
||||
?? storePayload?.href
|
||||
?? input.resolvedUrl);
|
||||
const storeUrl = safeCanonicalStoreUrl(sellerUrlRaw || input.resolvedUrl) ?? input.resolvedUrl;
|
||||
const sellerUrl = canonicalizeSellerUrl(sellerUrlRaw) ?? storeUrl;
|
||||
const companyUrl = pickCompanyUrl(contactPayload?.href, storeUrl);
|
||||
const memberId = cleanText(seed?.seller?.memberId)
|
||||
|| input.explicitMemberId
|
||||
|| extractMemberId(input.resolvedUrl)
|
||||
|| extractMemberId(storePayload?.href ?? '')
|
||||
|| null;
|
||||
const shopId = extractShopId(sellerUrl) ?? extractShopId(storeUrl);
|
||||
const companyName = cleanText(seed?.seller?.companyName)
|
||||
|| firstNamedLine(contactText)
|
||||
|| firstNamedLine(storeText)
|
||||
|| null;
|
||||
const serviceBadges = uniqueNonEmpty([
|
||||
...extractBadges(combinedText, SERVICE_BADGE_PATTERNS),
|
||||
...((seed?.services ?? []).map((service) => cleanText(service.serviceName))),
|
||||
]);
|
||||
const factoryBadges = extractBadges(combinedText, FACTORY_BADGE_PATTERNS);
|
||||
return {
|
||||
member_id: memberId,
|
||||
shop_id: shopId,
|
||||
store_name: companyName,
|
||||
store_url: storeUrl,
|
||||
company_name: companyName,
|
||||
company_url: companyUrl,
|
||||
business_model_text: firstMetric(combinedText, ['经营模式', '生产加工', '主营产品']),
|
||||
years_on_platform_text: extractYearsOnPlatform(combinedText),
|
||||
location: extractAddress(contactText) ?? extractAddress(storeText),
|
||||
staff_size_text: firstMetric(combinedText, ['员工人数', '员工总数']),
|
||||
factory_badges: factoryBadges,
|
||||
service_badges: serviceBadges,
|
||||
response_rate_text: firstMetric(combinedText, ['响应率', '回复率', '响应速度']),
|
||||
return_rate_text: extractReturnRate(combinedText),
|
||||
top_categories: guessTopCategories(combinedText),
|
||||
phone_text: extractMetric(contactText, '电话'),
|
||||
mobile_text: extractMetric(contactText, '手机'),
|
||||
...buildProvenance(cleanText(contactPayload?.href) || cleanText(storePayload?.href) || input.resolvedUrl),
|
||||
};
|
||||
}
|
||||
function safeCanonicalStoreUrl(url) {
|
||||
try {
|
||||
return canonicalizeStoreUrl(url);
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function pickCompanyUrl(contactHref, storeUrl) {
|
||||
const fromPage = cleanText(contactHref);
|
||||
if (fromPage) {
|
||||
const normalized = buildContactUrl(fromPage);
|
||||
if (normalized)
|
||||
return normalized;
|
||||
}
|
||||
return buildContactUrl(storeUrl);
|
||||
}
|
||||
function buildContactUrl(storeUrl) {
|
||||
try {
|
||||
const parsed = new URL(storeUrl);
|
||||
if (!parsed.hostname.endsWith('.1688.com'))
|
||||
return null;
|
||||
return `${parsed.protocol}//${parsed.hostname}/page/contactinfo.html`;
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
function firstNamedLine(text) {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => line.includes('有限公司') || line.includes('商行') || line.includes('工厂'))
|
||||
?? null;
|
||||
}
|
||||
function firstMetric(text, labels) {
|
||||
for (const label of labels) {
|
||||
const value = extractMetric(text, label);
|
||||
if (value)
|
||||
return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function extractReturnRate(text) {
|
||||
const inline = text.match(/回头率\s*([0-9.]+%)/);
|
||||
if (inline)
|
||||
return cleanText(inline[0]);
|
||||
const multiline = text.match(/回头率\s*\n\s*([0-9.]+%)/);
|
||||
if (!multiline)
|
||||
return null;
|
||||
return `回头率${cleanText(multiline[1])}`;
|
||||
}
|
||||
function firstOfferId(links) {
|
||||
for (const link of links) {
|
||||
const offerId = extractOfferId(link);
|
||||
if (offerId)
|
||||
return offerId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
function firstContactUrl(links) {
|
||||
for (const link of links) {
|
||||
const url = buildContactUrl(link);
|
||||
if (url)
|
||||
return url;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
async function readStorePayload(page, url, action) {
|
||||
const state = await gotoAndReadState(page, url, 2500, action);
|
||||
assertAuthenticatedState(state, action);
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
offerLinks: Array.from(document.querySelectorAll('a[href*="detail.1688.com/offer/"], a[href*="offerId="]'))
|
||||
.map((anchor) => anchor.href)
|
||||
.filter(Boolean),
|
||||
contactLinks: Array.from(document.querySelectorAll('a[href*="contactinfo"]'))
|
||||
.map((anchor) => anchor.href)
|
||||
.filter(Boolean),
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
async function readItemSeed(page, offerId) {
|
||||
const itemUrl = buildDetailUrl(offerId);
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'store seed item');
|
||||
assertAuthenticatedState(state, 'store seed item');
|
||||
const seed = await page.evaluate(`
|
||||
(() => {
|
||||
const model = window.context?.result?.global?.globalData?.model ?? null;
|
||||
const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
|
||||
return {
|
||||
href: window.location.href,
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
seller: toJson(model?.sellerModel),
|
||||
services: toJson(model?.shippingServices?.fields?.buyerProtectionModel ?? []),
|
||||
};
|
||||
})()
|
||||
`);
|
||||
const hasSellerContext = !!cleanText(seed?.seller?.memberId) || !!cleanText(seed?.seller?.winportUrl);
|
||||
if (!hasSellerContext) {
|
||||
throw new CommandExecutionError('1688 store seed item did not expose seller context', '当前 tab 非商品详情上下文，请切到 detail.1688.com 商品页并重试');
|
||||
}
|
||||
return seed;
|
||||
}
|
||||
function hasAnyEvidence(storePayload, contactPayload, seed) {
|
||||
return !!cleanText(storePayload?.bodyText)
|
||||
|| !!cleanText(contactPayload?.bodyText)
|
||||
|| !!cleanText(seed?.bodyText);
|
||||
}
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'store',
|
||||
description: '1688 店铺/供应商公开信息（联系方式、主营、入驻年限、公开服务信号）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 店铺 URL 或 member ID（如 b2b-22154705262941f196）',
|
||||
},
|
||||
],
|
||||
columns: ['store_name', 'years_on_platform_text', 'location', 'return_rate_text'],
|
||||
func: async (page, kwargs) => {
|
||||
const rawInput = String(kwargs.input ?? '');
|
||||
const resolvedUrl = resolveStoreUrl(rawInput);
|
||||
const explicitMemberId = extractMemberId(rawInput);
|
||||
const storePayload = await readStorePayload(page, resolvedUrl, 'store');
|
||||
const contactUrl = firstContactUrl(storePayload.contactLinks ?? []) || buildContactUrl(storePayload.href || resolvedUrl);
|
||||
const contactPayload = contactUrl ? await readStorePayload(page, contactUrl, 'store contact') : null;
|
||||
const offerId = extractOfferId(rawInput)
|
||||
|| firstOfferId(storePayload.offerLinks ?? [])
|
||||
|| firstOfferId(contactPayload?.offerLinks ?? []);
|
||||
let seed = null;
|
||||
if (offerId) {
|
||||
try {
|
||||
seed = await readItemSeed(page, offerId);
|
||||
}
|
||||
catch (error) {
|
||||
if (!(error instanceof CommandExecutionError))
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
if (!hasAnyEvidence(storePayload, contactPayload, seed)) {
|
||||
throw new EmptyResultError('1688 store', 'Store page is reachable but no visible fields were extracted. Open the store page in Chrome and retry.');
|
||||
}
|
||||
return [
|
||||
normalizeStorePayload({
|
||||
resolvedUrl,
|
||||
storePayload,
|
||||
contactPayload,
|
||||
seed,
|
||||
explicitMemberId,
|
||||
}),
|
||||
];
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeStorePayload,
|
||||
safeCanonicalStoreUrl,
|
||||
buildContactUrl,
|
||||
firstNamedLine,
|
||||
firstMetric,
|
||||
extractReturnRate,
|
||||
firstOfferId,
|
||||
firstContactUrl,
|
||||
};
|
||||
@@ -1,62 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './store.js';
|
||||
describe('1688 store normalization', () => {
|
||||
it('merges store contact text with seller seed data', () => {
|
||||
const result = __test__.normalizeStorePayload({
|
||||
resolvedUrl: 'https://yinuoweierfushi.1688.com/?offerId=887904326744',
|
||||
explicitMemberId: null,
|
||||
storePayload: {
|
||||
href: 'https://yinuoweierfushi.1688.com/page/index.html',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
联系方式
|
||||
地址：山东省青岛市即墨区环秀街道办事处湘江二路97号甲
|
||||
`,
|
||||
offerLinks: ['https://detail.1688.com/offer/887904326744.html'],
|
||||
},
|
||||
contactPayload: {
|
||||
href: 'https://yinuoweierfushi.1688.com/page/contactinfo.html',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
电话：86 0532 86655366
|
||||
手机：15963238678
|
||||
地址：山东省青岛市即墨区环秀街道办事处湘江二路97号甲
|
||||
`,
|
||||
},
|
||||
seed: {
|
||||
bodyText: `
|
||||
入驻13年
|
||||
主营：大码女装
|
||||
店铺回头率
|
||||
87%
|
||||
延期必赔
|
||||
品质保障
|
||||
`,
|
||||
seller: {
|
||||
companyName: '青岛沁澜衣品服装有限公司',
|
||||
memberId: 'b2b-1641351767',
|
||||
winportUrl: 'https://yinuoweierfushi.1688.com/page/index.html?spm=abc',
|
||||
},
|
||||
services: [{ serviceName: '延期必赔' }, { serviceName: '品质保障' }],
|
||||
},
|
||||
});
|
||||
expect(result.member_id).toBe('b2b-1641351767');
|
||||
expect(result.store_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.company_url).toBe('https://yinuoweierfushi.1688.com/page/contactinfo.html');
|
||||
expect(result.years_on_platform_text).toBe('入驻13年');
|
||||
expect(result.location).toBe('山东省青岛市即墨区环秀街道办事处湘江二路97号甲');
|
||||
expect(result.return_rate_text).toContain('87%');
|
||||
expect(result.top_categories).toEqual(['大码女装']);
|
||||
expect(result.service_badges).toEqual(['延期必赔', '品质保障']);
|
||||
});
|
||||
it('builds contact urls and extracts offer ids', () => {
|
||||
expect(__test__.safeCanonicalStoreUrl('https://yinuoweierfushi.1688.com/page/index.html?spm=foo')).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(__test__.buildContactUrl('https://yinuoweierfushi.1688.com')).toBe('https://yinuoweierfushi.1688.com/page/contactinfo.html');
|
||||
expect(__test__.firstOfferId([
|
||||
'https://detail.1688.com/offer/887904326744.html',
|
||||
])).toBe('887904326744');
|
||||
expect(__test__.firstContactUrl([
|
||||
'https://yinuoweierfushi.1688.com/page/contactinfo.html?spm=1',
|
||||
])).toBe('https://yinuoweierfushi.1688.com/page/contactinfo.html');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,69 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './store.js';
|
||||
|
||||
describe('1688 store normalization', () => {
|
||||
it('merges store contact text with seller seed data', () => {
|
||||
const result = __test__.normalizeStorePayload({
|
||||
resolvedUrl: 'https://yinuoweierfushi.1688.com/?offerId=887904326744',
|
||||
explicitMemberId: null,
|
||||
storePayload: {
|
||||
href: 'https://yinuoweierfushi.1688.com/page/index.html',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
联系方式
|
||||
地址：山东省青岛市即墨区环秀街道办事处湘江二路97号甲
|
||||
`,
|
||||
offerLinks: ['https://detail.1688.com/offer/887904326744.html'],
|
||||
},
|
||||
contactPayload: {
|
||||
href: 'https://yinuoweierfushi.1688.com/page/contactinfo.html',
|
||||
bodyText: `
|
||||
青岛沁澜衣品服装有限公司
|
||||
电话：86 0532 86655366
|
||||
手机：15963238678
|
||||
地址：山东省青岛市即墨区环秀街道办事处湘江二路97号甲
|
||||
`,
|
||||
},
|
||||
seed: {
|
||||
bodyText: `
|
||||
入驻13年
|
||||
主营：大码女装
|
||||
店铺回头率
|
||||
87%
|
||||
延期必赔
|
||||
品质保障
|
||||
`,
|
||||
seller: {
|
||||
companyName: '青岛沁澜衣品服装有限公司',
|
||||
memberId: 'b2b-1641351767',
|
||||
winportUrl: 'https://yinuoweierfushi.1688.com/page/index.html?spm=abc',
|
||||
},
|
||||
services: [{ serviceName: '延期必赔' }, { serviceName: '品质保障' }],
|
||||
},
|
||||
});
|
||||
|
||||
expect(result.member_id).toBe('b2b-1641351767');
|
||||
expect(result.store_url).toBe('https://yinuoweierfushi.1688.com');
|
||||
expect(result.company_url).toBe('https://yinuoweierfushi.1688.com/page/contactinfo.html');
|
||||
expect(result.years_on_platform_text).toBe('入驻13年');
|
||||
expect(result.location).toBe('山东省青岛市即墨区环秀街道办事处湘江二路97号甲');
|
||||
expect(result.return_rate_text).toContain('87%');
|
||||
expect(result.top_categories).toEqual(['大码女装']);
|
||||
expect(result.service_badges).toEqual(['延期必赔', '品质保障']);
|
||||
});
|
||||
|
||||
it('builds contact urls and extracts offer ids', () => {
|
||||
expect(__test__.safeCanonicalStoreUrl('https://yinuoweierfushi.1688.com/page/index.html?spm=foo')).toBe(
|
||||
'https://yinuoweierfushi.1688.com',
|
||||
);
|
||||
expect(__test__.buildContactUrl('https://yinuoweierfushi.1688.com')).toBe(
|
||||
'https://yinuoweierfushi.1688.com/page/contactinfo.html',
|
||||
);
|
||||
expect(__test__.firstOfferId([
|
||||
'https://detail.1688.com/offer/887904326744.html',
|
||||
])).toBe('887904326744');
|
||||
expect(__test__.firstContactUrl([
|
||||
'https://yinuoweierfushi.1688.com/page/contactinfo.html?spm=1',
|
||||
])).toBe('https://yinuoweierfushi.1688.com/page/contactinfo.html');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,300 @@
|
||||
import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
FACTORY_BADGE_PATTERNS,
|
||||
SERVICE_BADGE_PATTERNS,
|
||||
assertAuthenticatedState,
|
||||
buildDetailUrl,
|
||||
buildProvenance,
|
||||
canonicalizeSellerUrl,
|
||||
canonicalizeStoreUrl,
|
||||
cleanMultilineText,
|
||||
cleanText,
|
||||
extractAddress,
|
||||
extractBadges,
|
||||
extractMemberId,
|
||||
extractMetric,
|
||||
extractOfferId,
|
||||
extractShopId,
|
||||
extractYearsOnPlatform,
|
||||
gotoAndReadState,
|
||||
guessTopCategories,
|
||||
resolveStoreUrl,
|
||||
uniqueNonEmpty,
|
||||
} from './shared.js';
|
||||
|
||||
interface StoreBrowserPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
bodyText?: string;
|
||||
offerLinks?: string[];
|
||||
contactLinks?: string[];
|
||||
}
|
||||
|
||||
interface StoreItemSeed {
|
||||
href?: string;
|
||||
bodyText?: string;
|
||||
seller?: {
|
||||
companyName?: string;
|
||||
memberId?: string;
|
||||
winportUrl?: string;
|
||||
sellerWinportUrlMap?: Record<string, string>;
|
||||
};
|
||||
services?: Array<{ serviceName?: string }>;
|
||||
}
|
||||
|
||||
function normalizeStorePayload(input: {
|
||||
resolvedUrl: string;
|
||||
storePayload: StoreBrowserPayload | null;
|
||||
contactPayload: StoreBrowserPayload | null;
|
||||
seed: StoreItemSeed | null;
|
||||
explicitMemberId: string | null;
|
||||
}): Record<string, unknown> {
|
||||
const storePayload = input.storePayload;
|
||||
const contactPayload = input.contactPayload;
|
||||
const seed = input.seed;
|
||||
|
||||
const contactText = cleanMultilineText(contactPayload?.bodyText);
|
||||
const storeText = cleanMultilineText(storePayload?.bodyText);
|
||||
const seedText = cleanMultilineText(seed?.bodyText);
|
||||
const combinedText = [contactText, storeText, seedText].filter(Boolean).join('\n');
|
||||
|
||||
const sellerUrlRaw = cleanText(
|
||||
seed?.seller?.winportUrl
|
||||
?? seed?.seller?.sellerWinportUrlMap?.defaultUrl
|
||||
?? storePayload?.href
|
||||
?? input.resolvedUrl,
|
||||
);
|
||||
const storeUrl = safeCanonicalStoreUrl(sellerUrlRaw || input.resolvedUrl) ?? input.resolvedUrl;
|
||||
const sellerUrl = canonicalizeSellerUrl(sellerUrlRaw) ?? storeUrl;
|
||||
const companyUrl = pickCompanyUrl(contactPayload?.href, storeUrl);
|
||||
const memberId = cleanText(seed?.seller?.memberId)
|
||||
|| input.explicitMemberId
|
||||
|| extractMemberId(input.resolvedUrl)
|
||||
|| extractMemberId(storePayload?.href ?? '')
|
||||
|| null;
|
||||
const shopId = extractShopId(sellerUrl) ?? extractShopId(storeUrl);
|
||||
const companyName = cleanText(seed?.seller?.companyName)
|
||||
|| firstNamedLine(contactText)
|
||||
|| firstNamedLine(storeText)
|
||||
|| null;
|
||||
const serviceBadges = uniqueNonEmpty([
|
||||
...extractBadges(combinedText, SERVICE_BADGE_PATTERNS),
|
||||
...((seed?.services ?? []).map((service) => cleanText(service.serviceName))),
|
||||
]);
|
||||
const factoryBadges = extractBadges(combinedText, FACTORY_BADGE_PATTERNS);
|
||||
|
||||
return {
|
||||
member_id: memberId,
|
||||
shop_id: shopId,
|
||||
store_name: companyName,
|
||||
store_url: storeUrl,
|
||||
company_name: companyName,
|
||||
company_url: companyUrl,
|
||||
business_model_text: firstMetric(combinedText, ['经营模式', '生产加工', '主营产品']),
|
||||
years_on_platform_text: extractYearsOnPlatform(combinedText),
|
||||
location: extractAddress(contactText) ?? extractAddress(storeText),
|
||||
staff_size_text: firstMetric(combinedText, ['员工人数', '员工总数']),
|
||||
factory_badges: factoryBadges,
|
||||
service_badges: serviceBadges,
|
||||
response_rate_text: firstMetric(combinedText, ['响应率', '回复率', '响应速度']),
|
||||
return_rate_text: extractReturnRate(combinedText),
|
||||
top_categories: guessTopCategories(combinedText),
|
||||
phone_text: extractMetric(contactText, '电话'),
|
||||
mobile_text: extractMetric(contactText, '手机'),
|
||||
...buildProvenance(cleanText(contactPayload?.href) || cleanText(storePayload?.href) || input.resolvedUrl),
|
||||
};
|
||||
}
|
||||
|
||||
function safeCanonicalStoreUrl(url: string): string | null {
|
||||
try {
|
||||
return canonicalizeStoreUrl(url);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function pickCompanyUrl(contactHref: string | undefined, storeUrl: string): string | null {
|
||||
const fromPage = cleanText(contactHref);
|
||||
if (fromPage) {
|
||||
const normalized = buildContactUrl(fromPage);
|
||||
if (normalized) return normalized;
|
||||
}
|
||||
return buildContactUrl(storeUrl);
|
||||
}
|
||||
|
||||
function buildContactUrl(storeUrl: string): string | null {
|
||||
try {
|
||||
const parsed = new URL(storeUrl);
|
||||
if (!parsed.hostname.endsWith('.1688.com')) return null;
|
||||
return `${parsed.protocol}//${parsed.hostname}/page/contactinfo.html`;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function firstNamedLine(text: string): string | null {
|
||||
return text
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => line.includes('有限公司') || line.includes('商行') || line.includes('工厂'))
|
||||
?? null;
|
||||
}
|
||||
|
||||
function firstMetric(text: string, labels: string[]): string | null {
|
||||
for (const label of labels) {
|
||||
const value = extractMetric(text, label);
|
||||
if (value) return value;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function extractReturnRate(text: string): string | null {
|
||||
const inline = text.match(/回头率\s*([0-9.]+%)/);
|
||||
if (inline) return cleanText(inline[0]);
|
||||
const multiline = text.match(/回头率\s*\n\s*([0-9.]+%)/);
|
||||
if (!multiline) return null;
|
||||
return `回头率${cleanText(multiline[1])}`;
|
||||
}
|
||||
|
||||
function firstOfferId(links: string[]): string | null {
|
||||
for (const link of links) {
|
||||
const offerId = extractOfferId(link);
|
||||
if (offerId) return offerId;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function firstContactUrl(links: string[]): string | null {
|
||||
for (const link of links) {
|
||||
const url = buildContactUrl(link);
|
||||
if (url) return url;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
async function readStorePayload(page: IPage, url: string, action: string): Promise<StoreBrowserPayload> {
|
||||
const state = await gotoAndReadState(page, url, 2500, action);
|
||||
assertAuthenticatedState(state, action);
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
offerLinks: Array.from(document.querySelectorAll('a[href*="detail.1688.com/offer/"], a[href*="offerId="]'))
|
||||
.map((anchor) => anchor.href)
|
||||
.filter(Boolean),
|
||||
contactLinks: Array.from(document.querySelectorAll('a[href*="contactinfo"]'))
|
||||
.map((anchor) => anchor.href)
|
||||
.filter(Boolean),
|
||||
}))()
|
||||
`) as StoreBrowserPayload;
|
||||
}
|
||||
|
||||
async function readItemSeed(page: IPage, offerId: string): Promise<StoreItemSeed> {
|
||||
const itemUrl = buildDetailUrl(offerId);
|
||||
const state = await gotoAndReadState(page, itemUrl, 2500, 'store seed item');
|
||||
assertAuthenticatedState(state, 'store seed item');
|
||||
|
||||
const seed = await page.evaluate(`
|
||||
(() => {
|
||||
const model = window.context?.result?.global?.globalData?.model ?? null;
|
||||
const toJson = (value) => JSON.parse(JSON.stringify(value ?? null));
|
||||
return {
|
||||
href: window.location.href,
|
||||
bodyText: document.body ? document.body.innerText || '' : '',
|
||||
seller: toJson(model?.sellerModel),
|
||||
services: toJson(model?.shippingServices?.fields?.buyerProtectionModel ?? []),
|
||||
};
|
||||
})()
|
||||
`) as StoreItemSeed;
|
||||
|
||||
const hasSellerContext = !!cleanText(seed?.seller?.memberId) || !!cleanText(seed?.seller?.winportUrl);
|
||||
if (!hasSellerContext) {
|
||||
throw new CommandExecutionError(
|
||||
'1688 store seed item did not expose seller context',
|
||||
'当前 tab 非商品详情上下文，请切到 detail.1688.com 商品页并重试',
|
||||
);
|
||||
}
|
||||
|
||||
return seed;
|
||||
}
|
||||
|
||||
function hasAnyEvidence(
|
||||
storePayload: StoreBrowserPayload | null,
|
||||
contactPayload: StoreBrowserPayload | null,
|
||||
seed: StoreItemSeed | null,
|
||||
): boolean {
|
||||
return !!cleanText(storePayload?.bodyText)
|
||||
|| !!cleanText(contactPayload?.bodyText)
|
||||
|| !!cleanText(seed?.bodyText);
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '1688',
|
||||
name: 'store',
|
||||
description: '1688 店铺/供应商公开信息（联系方式、主营、入驻年限、公开服务信号）',
|
||||
domain: 'www.1688.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: '1688 店铺 URL 或 member ID（如 b2b-22154705262941f196）',
|
||||
},
|
||||
],
|
||||
columns: ['store_name', 'years_on_platform_text', 'location', 'return_rate_text'],
|
||||
func: async (page, kwargs) => {
|
||||
const rawInput = String(kwargs.input ?? '');
|
||||
const resolvedUrl = resolveStoreUrl(rawInput);
|
||||
const explicitMemberId = extractMemberId(rawInput);
|
||||
|
||||
const storePayload = await readStorePayload(page, resolvedUrl, 'store');
|
||||
const contactUrl = firstContactUrl(storePayload.contactLinks ?? []) || buildContactUrl(storePayload.href || resolvedUrl);
|
||||
const contactPayload = contactUrl ? await readStorePayload(page, contactUrl, 'store contact') : null;
|
||||
const offerId = extractOfferId(rawInput)
|
||||
|| firstOfferId(storePayload.offerLinks ?? [])
|
||||
|| firstOfferId(contactPayload?.offerLinks ?? []);
|
||||
|
||||
let seed: StoreItemSeed | null = null;
|
||||
if (offerId) {
|
||||
try {
|
||||
seed = await readItemSeed(page, offerId);
|
||||
} catch (error) {
|
||||
if (!(error instanceof CommandExecutionError)) throw error;
|
||||
}
|
||||
}
|
||||
|
||||
if (!hasAnyEvidence(storePayload, contactPayload, seed)) {
|
||||
throw new EmptyResultError(
|
||||
'1688 store',
|
||||
'Store page is reachable but no visible fields were extracted. Open the store page in Chrome and retry.',
|
||||
);
|
||||
}
|
||||
|
||||
return [
|
||||
normalizeStorePayload({
|
||||
resolvedUrl,
|
||||
storePayload,
|
||||
contactPayload,
|
||||
seed,
|
||||
explicitMemberId,
|
||||
}),
|
||||
];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeStorePayload,
|
||||
safeCanonicalStoreUrl,
|
||||
buildContactUrl,
|
||||
firstNamedLine,
|
||||
firstMetric,
|
||||
extractReturnRate,
|
||||
firstOfferId,
|
||||
firstContactUrl,
|
||||
};
|
||||
@@ -5,30 +5,35 @@
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
|
||||
/** Extract article ID from a full URL or a bare numeric ID string */
|
||||
function parseArticleId(input) {
|
||||
const m = input.match(/\/p\/(\d+)/);
|
||||
return m ? m[1] : input.replace(/\D/g, '');
|
||||
function parseArticleId(input: string): string {
|
||||
const m = input.match(/\/p\/(\d+)/);
|
||||
return m ? m[1] : input.replace(/\D/g, '');
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'article',
|
||||
description: '获取36氪文章正文内容',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
args: [
|
||||
{ name: 'id', positional: true, required: true, help: 'Article ID or full 36kr article URL' },
|
||||
],
|
||||
columns: ['field', 'value'],
|
||||
func: async (page, args) => {
|
||||
const articleId = parseArticleId(String(args.id ?? ''));
|
||||
if (!articleId) {
|
||||
throw new CliError('INVALID_ARGUMENT', 'Invalid article ID or URL');
|
||||
}
|
||||
await page.installInterceptor('36kr.com/api');
|
||||
await page.goto(`https://www.36kr.com/p/${articleId}`);
|
||||
await page.wait(5);
|
||||
const data = await page.evaluate(`
|
||||
site: '36kr',
|
||||
name: 'article',
|
||||
description: '获取36氪文章正文内容',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.INTERCEPT,
|
||||
args: [
|
||||
{ name: 'id', positional: true, required: true, help: 'Article ID or full 36kr article URL' },
|
||||
],
|
||||
columns: ['field', 'value'],
|
||||
func: async (page: IPage, args) => {
|
||||
const articleId = parseArticleId(String(args.id ?? ''));
|
||||
if (!articleId) {
|
||||
throw new CliError('INVALID_ARGUMENT', 'Invalid article ID or URL');
|
||||
}
|
||||
|
||||
await page.installInterceptor('36kr.com/api');
|
||||
await page.goto(`https://www.36kr.com/p/${articleId}`);
|
||||
await page.wait(5);
|
||||
|
||||
const data: any = await page.evaluate(`
|
||||
(() => {
|
||||
// Title: 36kr uses class "article-title" on h1
|
||||
const title = document.querySelector('.article-title, h1')?.textContent?.trim() || '';
|
||||
@@ -48,15 +53,17 @@ cli({
|
||||
return { title, author, date, body };
|
||||
})()
|
||||
`);
|
||||
if (!data?.title) {
|
||||
throw new CliError('NOT_FOUND', 'Article not found or failed to load', 'Check the article ID');
|
||||
}
|
||||
return [
|
||||
{ field: 'title', value: data.title },
|
||||
{ field: 'author', value: data.author || '-' },
|
||||
{ field: 'date', value: data.date || '-' },
|
||||
{ field: 'url', value: `https://36kr.com/p/${articleId}` },
|
||||
{ field: 'body', value: data.body || '-' },
|
||||
];
|
||||
},
|
||||
|
||||
if (!data?.title) {
|
||||
throw new CliError('NOT_FOUND', 'Article not found or failed to load', 'Check the article ID');
|
||||
}
|
||||
|
||||
return [
|
||||
{ field: 'title', value: data.title },
|
||||
{ field: 'author', value: data.author || '-' },
|
||||
{ field: 'date', value: data.date || '-' },
|
||||
{ field: 'url', value: `https://36kr.com/p/${articleId}` },
|
||||
{ field: 'body', value: data.body || '-' },
|
||||
];
|
||||
},
|
||||
});
|
||||
@@ -1,86 +0,0 @@
|
||||
/**
|
||||
* 36kr hot-list — DOM scraping.
|
||||
*
|
||||
* Navigates to the 36kr hot-list page and scrapes rendered article links.
|
||||
* Supports category types: renqi (人气), zonghe (综合), shoucang (收藏), catalog (综合热门).
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
const TYPE_MAP = {
|
||||
renqi: '人气榜',
|
||||
zonghe: '综合榜',
|
||||
shoucang: '收藏榜',
|
||||
catalog: '热门资讯',
|
||||
};
|
||||
function getShanghaiDate(date = new Date()) {
|
||||
// Shanghai stays on UTC+8 year-round, so a fixed offset is sufficient here
|
||||
// and avoids the slow Intl timezone path that timed out on Windows CI.
|
||||
return new Date(date.getTime() + 8 * 60 * 60 * 1000).toISOString().slice(0, 10);
|
||||
}
|
||||
function buildHotListUrl(listType, date = new Date()) {
|
||||
if (listType === 'catalog') {
|
||||
return 'https://www.36kr.com/hot-list/catalog';
|
||||
}
|
||||
return `https://www.36kr.com/hot-list/${listType}/${getShanghaiDate(date)}/1`;
|
||||
}
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'hot',
|
||||
description: '36氪热榜 — trending articles (renqi/zonghe/shoucang/catalog)',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of items (max 50)' },
|
||||
{
|
||||
name: 'type',
|
||||
type: 'string',
|
||||
default: 'catalog',
|
||||
help: 'List type: renqi (人气), zonghe (综合), shoucang (收藏), catalog (热门资讯)',
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'title', 'url'],
|
||||
func: async (page, args) => {
|
||||
const count = Math.min(Number(args.limit) || 20, 50);
|
||||
const listType = String(args.type ?? 'catalog');
|
||||
if (!TYPE_MAP[listType]) {
|
||||
throw new CliError('INVALID_ARGUMENT', `Unknown type "${listType}". Valid types: ${Object.keys(TYPE_MAP).join(', ')}`);
|
||||
}
|
||||
const url = buildHotListUrl(listType);
|
||||
await page.goto(url);
|
||||
// Poll DOM until article links appear (36kr renders client-side)
|
||||
const deadline = Date.now() + 5000;
|
||||
while (Date.now() < deadline) {
|
||||
if (await page.evaluate('document.querySelectorAll("a[href*=\\"/p/\\"]").length'))
|
||||
break;
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
}
|
||||
// Scrape rendered article links from DOM (deduplicated)
|
||||
const domItems = await page.evaluate(`
|
||||
(() => {
|
||||
const seen = new Set();
|
||||
const results = [];
|
||||
const links = document.querySelectorAll('a[href*="/p/"]');
|
||||
for (const el of links) {
|
||||
const href = el.getAttribute('href') || '';
|
||||
const title = el.textContent?.trim() || '';
|
||||
if (!title || title.length < 5 || seen.has(href) || seen.has(title)) continue;
|
||||
seen.add(href);
|
||||
seen.add(title);
|
||||
results.push({ title, url: href.startsWith('http') ? href : 'https://36kr.com' + href });
|
||||
}
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
const items = Array.isArray(domItems) ? domItems : [];
|
||||
if (items.length === 0) {
|
||||
throw new CliError('NO_DATA', 'Could not retrieve 36kr hot list', '36kr may have changed its DOM structure');
|
||||
}
|
||||
return items.slice(0, count).map((item, i) => ({
|
||||
rank: i + 1,
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
}));
|
||||
},
|
||||
});
|
||||
export { buildHotListUrl, getShanghaiDate };
|
||||
@@ -1,15 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { buildHotListUrl, getShanghaiDate } from './hot.js';
|
||||
describe('36kr/hot date routing', () => {
|
||||
it('formats dates in Asia/Shanghai instead of UTC', () => {
|
||||
const date = new Date('2026-03-25T18:30:00.000Z');
|
||||
expect(getShanghaiDate(date)).toBe('2026-03-26');
|
||||
});
|
||||
it('builds dated hot-list routes with Shanghai-local date', () => {
|
||||
const date = new Date('2026-03-25T18:30:00.000Z');
|
||||
expect(buildHotListUrl('renqi', date)).toBe('https://www.36kr.com/hot-list/renqi/2026-03-26/1');
|
||||
});
|
||||
it('keeps catalog on the static route', () => {
|
||||
expect(buildHotListUrl('catalog')).toBe('https://www.36kr.com/hot-list/catalog');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,19 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
|
||||
import { buildHotListUrl, getShanghaiDate } from './hot.js';
|
||||
|
||||
describe('36kr/hot date routing', () => {
|
||||
it('formats dates in Asia/Shanghai instead of UTC', () => {
|
||||
const date = new Date('2026-03-25T18:30:00.000Z');
|
||||
expect(getShanghaiDate(date)).toBe('2026-03-26');
|
||||
});
|
||||
|
||||
it('builds dated hot-list routes with Shanghai-local date', () => {
|
||||
const date = new Date('2026-03-25T18:30:00.000Z');
|
||||
expect(buildHotListUrl('renqi', date)).toBe('https://www.36kr.com/hot-list/renqi/2026-03-26/1');
|
||||
});
|
||||
|
||||
it('keeps catalog on the static route', () => {
|
||||
expect(buildHotListUrl('catalog')).toBe('https://www.36kr.com/hot-list/catalog');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,105 @@
|
||||
/**
|
||||
* 36kr hot-list — DOM scraping.
|
||||
*
|
||||
* Navigates to the 36kr hot-list page and scrapes rendered article links.
|
||||
* Supports category types: renqi (人气), zonghe (综合), shoucang (收藏), catalog (综合热门).
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
|
||||
const TYPE_MAP: Record<string, string> = {
|
||||
renqi: '人气榜',
|
||||
zonghe: '综合榜',
|
||||
shoucang: '收藏榜',
|
||||
catalog: '热门资讯',
|
||||
};
|
||||
|
||||
function getShanghaiDate(date = new Date()): string {
|
||||
// Shanghai stays on UTC+8 year-round, so a fixed offset is sufficient here
|
||||
// and avoids the slow Intl timezone path that timed out on Windows CI.
|
||||
return new Date(date.getTime() + 8 * 60 * 60 * 1000).toISOString().slice(0, 10);
|
||||
}
|
||||
|
||||
function buildHotListUrl(listType: string, date = new Date()): string {
|
||||
if (listType === 'catalog') {
|
||||
return 'https://www.36kr.com/hot-list/catalog';
|
||||
}
|
||||
|
||||
return `https://www.36kr.com/hot-list/${listType}/${getShanghaiDate(date)}/1`;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'hot',
|
||||
description: '36氪热榜 — trending articles (renqi/zonghe/shoucang/catalog)',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of items (max 50)' },
|
||||
{
|
||||
name: 'type',
|
||||
type: 'string',
|
||||
default: 'catalog',
|
||||
help: 'List type: renqi (人气), zonghe (综合), shoucang (收藏), catalog (热门资讯)',
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'title', 'url'],
|
||||
func: async (page: IPage, args) => {
|
||||
const count = Math.min(Number(args.limit) || 20, 50);
|
||||
const listType = String(args.type ?? 'catalog');
|
||||
|
||||
if (!TYPE_MAP[listType]) {
|
||||
throw new CliError(
|
||||
'INVALID_ARGUMENT',
|
||||
`Unknown type "${listType}". Valid types: ${Object.keys(TYPE_MAP).join(', ')}`,
|
||||
);
|
||||
}
|
||||
|
||||
const url = buildHotListUrl(listType);
|
||||
|
||||
await page.goto(url);
|
||||
// Poll DOM until article links appear (36kr renders client-side)
|
||||
const deadline = Date.now() + 5000;
|
||||
while (Date.now() < deadline) {
|
||||
if (await page.evaluate('document.querySelectorAll("a[href*=\\"/p/\\"]").length')) break;
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
}
|
||||
|
||||
// Scrape rendered article links from DOM (deduplicated)
|
||||
const domItems: any = await page.evaluate(`
|
||||
(() => {
|
||||
const seen = new Set();
|
||||
const results = [];
|
||||
const links = document.querySelectorAll('a[href*="/p/"]');
|
||||
for (const el of links) {
|
||||
const href = el.getAttribute('href') || '';
|
||||
const title = el.textContent?.trim() || '';
|
||||
if (!title || title.length < 5 || seen.has(href) || seen.has(title)) continue;
|
||||
seen.add(href);
|
||||
seen.add(title);
|
||||
results.push({ title, url: href.startsWith('http') ? href : 'https://36kr.com' + href });
|
||||
}
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
|
||||
const items = Array.isArray(domItems) ? (domItems as any[]) : [];
|
||||
if (items.length === 0) {
|
||||
throw new CliError(
|
||||
'NO_DATA',
|
||||
'Could not retrieve 36kr hot list',
|
||||
'36kr may have changed its DOM structure',
|
||||
);
|
||||
}
|
||||
|
||||
return items.slice(0, count).map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
title: item.title,
|
||||
url: item.url,
|
||||
}));
|
||||
},
|
||||
});
|
||||
|
||||
export { buildHotListUrl, getShanghaiDate };
|
||||
@@ -1,51 +0,0 @@
|
||||
/**
|
||||
* 36kr latest news — public RSS feed, no browser needed.
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'news',
|
||||
description: 'Latest tech/startup news from 36kr (36氪)',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of articles (max 50)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'summary', 'date', 'url'],
|
||||
func: async (kwargs) => {
|
||||
const count = Math.min(kwargs.limit || 20, 50);
|
||||
const resp = await fetch('https://www.36kr.com/feed', {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; opencli/1.0)' },
|
||||
});
|
||||
if (!resp.ok)
|
||||
return [];
|
||||
const xml = await resp.text();
|
||||
const items = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < count) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url = block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ??
|
||||
block.match(/<link>(.*?)<\/link>/)?.[1] ??
|
||||
'';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
// Extract plain-text summary from HTML description (first ~120 chars)
|
||||
const rawDesc = block.match(/<description><!\[CDATA\[([\s\S]*?)\]\]>/)?.[1] ?? '';
|
||||
const summary = rawDesc
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 120);
|
||||
if (title) {
|
||||
items.push({ rank: items.length + 1, title, summary, date, url: url.trim() });
|
||||
}
|
||||
}
|
||||
return items;
|
||||
},
|
||||
});
|
||||
@@ -1,85 +0,0 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
const SAMPLE_RSS = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel><title>36氪</title>
|
||||
<item>
|
||||
<title>红杉中国领投AI公司「示例」，金额近2亿元</title>
|
||||
<link><![CDATA[https://36kr.com/p/1111111111111111?f=rss]]></link>
|
||||
<pubDate>2026-03-26 10:00:00 +0800</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>马斯克旗下xAI估值突破1000亿美元</title>
|
||||
<link><![CDATA[https://36kr.com/p/2222222222222222?f=rss]]></link>
|
||||
<pubDate>2026-03-26 09:00:00 +0800</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>OpenAI发布GPT-5，多模态能力大幅提升</title>
|
||||
<link><![CDATA[https://36kr.com/p/3333333333333333?f=rss]]></link>
|
||||
<pubDate>2026-03-25 20:00:00 +0800</pubDate>
|
||||
</item>
|
||||
</channel></rss>`;
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
describe('36kr/news RSS parsing', () => {
|
||||
it('parses RSS feed into ranked news items', async () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
text: async () => SAMPLE_RSS,
|
||||
});
|
||||
// Direct RSS parse test using the same regex logic as news.ts
|
||||
const xml = SAMPLE_RSS;
|
||||
const items = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < 10) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url = block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ??
|
||||
block.match(/<link>(.*?)<\/link>/)?.[1] ??
|
||||
'';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
if (title)
|
||||
items.push({ rank: items.length + 1, title, date, url: url.trim() });
|
||||
}
|
||||
expect(items).toHaveLength(3);
|
||||
expect(items[0].rank).toBe(1);
|
||||
expect(items[0].title).toBe('红杉中国领投AI公司「示例」，金额近2亿元');
|
||||
expect(items[0].date).toBe('2026-03-26');
|
||||
expect(items[0].url).toBe('https://36kr.com/p/1111111111111111?f=rss');
|
||||
});
|
||||
it('respects limit — returns at most N items', async () => {
|
||||
const xml = SAMPLE_RSS;
|
||||
const limit = 2;
|
||||
const items = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < limit) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url = block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ?? '';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
if (title)
|
||||
items.push({ rank: items.length + 1, title, date, url: url.trim() });
|
||||
}
|
||||
expect(items).toHaveLength(2);
|
||||
});
|
||||
it('skips items with empty title', async () => {
|
||||
const xml = `<rss><channel>
|
||||
<item><title></title><link>https://36kr.com/p/0</link><pubDate>2026-01-01</pubDate></item>
|
||||
<item><title>有标题的文章</title><link>https://36kr.com/p/1</link><pubDate>2026-01-01</pubDate></item>
|
||||
</channel></rss>`;
|
||||
const items = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml))) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
if (title)
|
||||
items.push({ title });
|
||||
}
|
||||
expect(items).toHaveLength(1);
|
||||
expect(items[0].title).toBe('有标题的文章');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,90 @@
|
||||
import { describe, it, expect, vi, afterEach } from 'vitest';
|
||||
|
||||
const SAMPLE_RSS = `<?xml version="1.0" encoding="UTF-8"?>
|
||||
<rss version="2.0"><channel><title>36氪</title>
|
||||
<item>
|
||||
<title>红杉中国领投AI公司「示例」，金额近2亿元</title>
|
||||
<link><![CDATA[https://36kr.com/p/1111111111111111?f=rss]]></link>
|
||||
<pubDate>2026-03-26 10:00:00 +0800</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>马斯克旗下xAI估值突破1000亿美元</title>
|
||||
<link><![CDATA[https://36kr.com/p/2222222222222222?f=rss]]></link>
|
||||
<pubDate>2026-03-26 09:00:00 +0800</pubDate>
|
||||
</item>
|
||||
<item>
|
||||
<title>OpenAI发布GPT-5，多模态能力大幅提升</title>
|
||||
<link><![CDATA[https://36kr.com/p/3333333333333333?f=rss]]></link>
|
||||
<pubDate>2026-03-25 20:00:00 +0800</pubDate>
|
||||
</item>
|
||||
</channel></rss>`;
|
||||
|
||||
afterEach(() => {
|
||||
vi.restoreAllMocks();
|
||||
});
|
||||
|
||||
describe('36kr/news RSS parsing', () => {
|
||||
it('parses RSS feed into ranked news items', async () => {
|
||||
vi.spyOn(globalThis, 'fetch').mockResolvedValue({
|
||||
ok: true,
|
||||
text: async () => SAMPLE_RSS,
|
||||
} as Response);
|
||||
|
||||
// Direct RSS parse test using the same regex logic as news.ts
|
||||
const xml = SAMPLE_RSS;
|
||||
const items: { rank: number; title: string; date: string; url: string }[] = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < 10) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url =
|
||||
block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ??
|
||||
block.match(/<link>(.*?)<\/link>/)?.[1] ??
|
||||
'';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
if (title) items.push({ rank: items.length + 1, title, date, url: url.trim() });
|
||||
}
|
||||
|
||||
expect(items).toHaveLength(3);
|
||||
expect(items[0].rank).toBe(1);
|
||||
expect(items[0].title).toBe('红杉中国领投AI公司「示例」，金额近2亿元');
|
||||
expect(items[0].date).toBe('2026-03-26');
|
||||
expect(items[0].url).toBe('https://36kr.com/p/1111111111111111?f=rss');
|
||||
});
|
||||
|
||||
it('respects limit — returns at most N items', async () => {
|
||||
const xml = SAMPLE_RSS;
|
||||
const limit = 2;
|
||||
const items: { rank: number; title: string; date: string; url: string }[] = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < limit) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url = block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ?? '';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
if (title) items.push({ rank: items.length + 1, title, date, url: url.trim() });
|
||||
}
|
||||
expect(items).toHaveLength(2);
|
||||
});
|
||||
|
||||
it('skips items with empty title', async () => {
|
||||
const xml = `<rss><channel>
|
||||
<item><title></title><link>https://36kr.com/p/0</link><pubDate>2026-01-01</pubDate></item>
|
||||
<item><title>有标题的文章</title><link>https://36kr.com/p/1</link><pubDate>2026-01-01</pubDate></item>
|
||||
</channel></rss>`;
|
||||
const items: any[] = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml))) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
if (title) items.push({ title });
|
||||
}
|
||||
expect(items).toHaveLength(1);
|
||||
expect(items[0].title).toBe('有标题的文章');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,54 @@
|
||||
/**
|
||||
* 36kr latest news — public RSS feed, no browser needed.
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'news',
|
||||
description: 'Latest tech/startup news from 36kr (36氪)',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
args: [
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of articles (max 50)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'summary', 'date', 'url'],
|
||||
func: async (_page, kwargs) => {
|
||||
const count = Math.min(kwargs.limit || 20, 50);
|
||||
const resp = await fetch('https://www.36kr.com/feed', {
|
||||
headers: { 'User-Agent': 'Mozilla/5.0 (compatible; opencli/1.0)' },
|
||||
});
|
||||
if (!resp.ok) return [];
|
||||
const xml = await resp.text();
|
||||
|
||||
const items: { rank: number; title: string; summary: string; date: string; url: string }[] = [];
|
||||
const itemRegex = /<item>([\s\S]*?)<\/item>/g;
|
||||
let match;
|
||||
while ((match = itemRegex.exec(xml)) && items.length < count) {
|
||||
const block = match[1];
|
||||
const title = block.match(/<title>([\s\S]*?)<\/title>/)?.[1]?.trim() ?? '';
|
||||
const url =
|
||||
block.match(/<link><!\[CDATA\[(.*?)\]\]>/)?.[1] ??
|
||||
block.match(/<link>(.*?)<\/link>/)?.[1] ??
|
||||
'';
|
||||
const pubDate = block.match(/<pubDate>(.*?)<\/pubDate>/)?.[1]?.trim() ?? '';
|
||||
const date = pubDate.slice(0, 10);
|
||||
// Extract plain-text summary from HTML description (first ~120 chars)
|
||||
const rawDesc = block.match(/<description><!\[CDATA\[([\s\S]*?)\]\]>/)?.[1] ?? '';
|
||||
const summary = rawDesc
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
.replace(/ /g, ' ')
|
||||
.replace(/&/g, '&')
|
||||
.replace(/</g, '<')
|
||||
.replace(/>/g, '>')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 120);
|
||||
|
||||
if (title) {
|
||||
items.push({ rank: items.length + 1, title, summary, date, url: url.trim() });
|
||||
}
|
||||
}
|
||||
return items;
|
||||
},
|
||||
});
|
||||
@@ -5,30 +5,33 @@
|
||||
*/
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
|
||||
cli({
|
||||
site: '36kr',
|
||||
name: 'search',
|
||||
description: '搜索36氪文章',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', positional: true, required: true, help: 'Search keyword (e.g. "AI", "OpenAI")' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of results (max 50)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'date', 'url'],
|
||||
func: async (page, args) => {
|
||||
const count = Math.min(Number(args.limit) || 20, 50);
|
||||
const query = encodeURIComponent(String(args.query ?? ''));
|
||||
await page.goto(`https://www.36kr.com/search/articles/${query}`);
|
||||
// Poll DOM until article links appear (36kr renders client-side)
|
||||
const deadline = Date.now() + 5000;
|
||||
while (Date.now() < deadline) {
|
||||
if (await page.evaluate('document.querySelectorAll("a[href*=\\"/p/\\"]").length'))
|
||||
break;
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
}
|
||||
const domItems = await page.evaluate(`
|
||||
site: '36kr',
|
||||
name: 'search',
|
||||
description: '搜索36氪文章',
|
||||
domain: 'www.36kr.com',
|
||||
strategy: Strategy.PUBLIC,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'query', positional: true, required: true, help: 'Search keyword (e.g. "AI", "OpenAI")' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: 'Number of results (max 50)' },
|
||||
],
|
||||
columns: ['rank', 'title', 'date', 'url'],
|
||||
func: async (page: IPage, args) => {
|
||||
const count = Math.min(Number(args.limit) || 20, 50);
|
||||
const query = encodeURIComponent(String(args.query ?? ''));
|
||||
|
||||
await page.goto(`https://www.36kr.com/search/articles/${query}`);
|
||||
// Poll DOM until article links appear (36kr renders client-side)
|
||||
const deadline = Date.now() + 5000;
|
||||
while (Date.now() < deadline) {
|
||||
if (await page.evaluate('document.querySelectorAll("a[href*=\\"/p/\\"]").length')) break;
|
||||
await new Promise(r => setTimeout(r, 300));
|
||||
}
|
||||
|
||||
const domItems: any = await page.evaluate(`
|
||||
(() => {
|
||||
const seen = new Set();
|
||||
const results = [];
|
||||
@@ -64,15 +67,17 @@ cli({
|
||||
return results;
|
||||
})()
|
||||
`);
|
||||
const items = Array.isArray(domItems) ? domItems : [];
|
||||
if (items.length === 0) {
|
||||
throw new CliError('NO_DATA', 'No results found', `Try a different query or check your keyword`);
|
||||
}
|
||||
return items.slice(0, count).map((item, i) => ({
|
||||
rank: i + 1,
|
||||
title: item.title,
|
||||
date: item.date,
|
||||
url: item.url,
|
||||
}));
|
||||
},
|
||||
|
||||
const items = Array.isArray(domItems) ? (domItems as any[]) : [];
|
||||
if (items.length === 0) {
|
||||
throw new CliError('NO_DATA', 'No results found', `Try a different query or check your keyword`);
|
||||
}
|
||||
|
||||
return items.slice(0, count).map((item: any, i: number) => ({
|
||||
rank: i + 1,
|
||||
title: item.title,
|
||||
date: item.date,
|
||||
url: item.url,
|
||||
}));
|
||||
},
|
||||
});
|
||||
@@ -1,125 +0,0 @@
|
||||
/**
|
||||
* 51job company jobs + basic info by encCoId.
|
||||
*
|
||||
* Navigates to `jobs.51job.com/all/co<encCoId>.html`. Each job card is an
|
||||
* `<a sensorsdata="…">` whose attribute is a JSON blob with jobId, title,
|
||||
* salary, area, year, degree — so parsing is just JSON, not DOM-text fragile.
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import { JOBS_ORIGIN, requirePage, navigateTo, parseCompanyJobCard } from './utils.js';
|
||||
|
||||
cli({
|
||||
site: '51job',
|
||||
name: 'company',
|
||||
description: '51job 公司简介 + 在招职位（按 encCoId）',
|
||||
domain: 'jobs.51job.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{ name: 'encCoId', type: 'string', required: true, positional: true, help: '加密公司 ID（search 返回的 encCoId）' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: '返回职位数（1-50）' },
|
||||
],
|
||||
columns: [
|
||||
'rank', 'jobId', 'title', 'salary', 'city', 'workYear', 'degree',
|
||||
'funcType', 'issueDate', 'url',
|
||||
'companyName', 'companyType', 'companySize', 'companyIndustry',
|
||||
'companyIntro', 'companyUrl',
|
||||
],
|
||||
func: async (page, kwargs) => {
|
||||
requirePage(page);
|
||||
const encCoId = String(kwargs.encCoId ?? '').trim();
|
||||
if (!encCoId) throw new CliError('INVALID_ARGUMENT', 'encCoId is required');
|
||||
if (!/^[A-Za-z0-9_]+$/.test(encCoId)) {
|
||||
throw new CliError('INVALID_ARGUMENT', `encCoId must be alphanumeric/underscore, got "${encCoId}"`);
|
||||
}
|
||||
const limit = Math.max(1, Math.min(Number(kwargs.limit) || 20, 50));
|
||||
|
||||
const url = `${JOBS_ORIGIN}/all/co${encCoId}.html`;
|
||||
await navigateTo(page, url, 2);
|
||||
|
||||
const script = `(() => {
|
||||
const sel = s => document.querySelector(s)?.innerText?.trim() || '';
|
||||
const bodyText = (document.body.innerText || '').slice(0, 400);
|
||||
if (/公司不存在|页面不存在|账号状态异常/.test(bodyText)) {
|
||||
return { error: 'NOT_FOUND', bodyText };
|
||||
}
|
||||
const companyName = sel('h1') || sel('.cname');
|
||||
// Company introduction block
|
||||
const introEl = document.querySelector('#companyIntroRef, .c-intro');
|
||||
const companyIntro = introEl ? (introEl.innerText || '').trim() : '';
|
||||
// Info sidebar (type / size / industry) — labels sit in .com-info dl or .coinfo
|
||||
const sidebarText = sel('.ci-content, .company-info, .coinfo, .com-info');
|
||||
const links = [...document.querySelectorAll('a[sensorsdata]')]
|
||||
.filter(a => /\\/\\d{6,}\\.html/.test(a.href || ''))
|
||||
.slice(0, 60)
|
||||
.map(a => {
|
||||
return {
|
||||
href: a.href,
|
||||
sensorsdata: a.getAttribute('sensorsdata') || '',
|
||||
text: (a.innerText || '').trim(),
|
||||
};
|
||||
});
|
||||
// Company meta is three inline spans under .c-info.ellipsis
|
||||
// (title/size/industry) — extract them by position.
|
||||
const cInfo = document.querySelector('.c-info.ellipsis');
|
||||
const cInfoParts = cInfo
|
||||
? [...cInfo.querySelectorAll('span')].map(s => (s.innerText || '').trim()).filter(Boolean)
|
||||
: [];
|
||||
return {
|
||||
companyName,
|
||||
companyIntro,
|
||||
links,
|
||||
cInfoParts,
|
||||
sidebarText: sidebarText.slice(0, 400),
|
||||
};
|
||||
})()`;
|
||||
const data = await page.evaluate(script);
|
||||
if (data.error === 'NOT_FOUND') {
|
||||
throw new CliError('NO_DATA', `Company ${encCoId} not found`);
|
||||
}
|
||||
if (!data.companyName) {
|
||||
throw new CliError('NO_DATA', `Could not parse company page ${encCoId}; layout may have changed`);
|
||||
}
|
||||
|
||||
const companyUrl = url;
|
||||
const [companyType = '', companySize = '', companyIndustry = ''] = data.cInfoParts || [];
|
||||
|
||||
const seen = new Set();
|
||||
const rows = [];
|
||||
for (const link of data.links || []) {
|
||||
const job = parseCompanyJobCard(link);
|
||||
if (!job) continue;
|
||||
if (seen.has(job.jobId)) continue;
|
||||
seen.add(job.jobId);
|
||||
rows.push({
|
||||
rank: rows.length + 1,
|
||||
...job,
|
||||
companyName: data.companyName,
|
||||
companyType,
|
||||
companySize,
|
||||
companyIndustry,
|
||||
companyIntro: data.companyIntro || '',
|
||||
companyUrl,
|
||||
});
|
||||
if (rows.length >= limit) break;
|
||||
}
|
||||
if (rows.length === 0) {
|
||||
// Still return a sentinel row with the company info so caller isn't left with [].
|
||||
return [{
|
||||
rank: 0,
|
||||
jobId: '',
|
||||
title: '(no active jobs)',
|
||||
salary: '', city: '', workYear: '', degree: '',
|
||||
funcType: '', issueDate: '', url: '',
|
||||
companyName: data.companyName,
|
||||
companyType, companySize, companyIndustry,
|
||||
companyIntro: data.companyIntro || '',
|
||||
companyUrl,
|
||||
}];
|
||||
}
|
||||
return rows;
|
||||
},
|
||||
});
|
||||
@@ -1,108 +0,0 @@
|
||||
/**
|
||||
* 51job job detail by jobId.
|
||||
*
|
||||
* Navigates to `jobs.51job.com/x/<jobId>.html` (SSR page — the generic `/x/`
|
||||
* area slug always resolves) and scrapes the structured blocks. No API
|
||||
* surface returns the full detail page, so DOM scraping is the only path.
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import { JOBS_ORIGIN, requirePage, navigateTo } from './utils.js';
|
||||
|
||||
cli({
|
||||
site: '51job',
|
||||
name: 'detail',
|
||||
description: '51job 职位详情（按 jobId）',
|
||||
domain: 'jobs.51job.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{ name: 'jobId', type: 'string', required: true, positional: true, help: '职位 ID（search 返回的 jobId）' },
|
||||
],
|
||||
columns: [
|
||||
'jobId', 'title', 'salary', 'location', 'workYear', 'degree',
|
||||
'category', 'address', 'ageRequirement',
|
||||
'description', 'welfare',
|
||||
'company', 'companyType', 'companySize', 'companyIndustry',
|
||||
'companyUrl', 'url',
|
||||
],
|
||||
func: async (page, kwargs) => {
|
||||
requirePage(page);
|
||||
const jobId = String(kwargs.jobId ?? '').trim();
|
||||
if (!jobId) throw new CliError('INVALID_ARGUMENT', 'jobId is required');
|
||||
if (!/^\d{6,12}$/.test(jobId)) throw new CliError('INVALID_ARGUMENT', `jobId must be a 6-12 digit number, got "${jobId}"`);
|
||||
|
||||
const url = `${JOBS_ORIGIN}/x/${jobId}.html`;
|
||||
await navigateTo(page, url, 2);
|
||||
|
||||
const script = `(() => {
|
||||
const sel = s => document.querySelector(s)?.innerText?.trim() || '';
|
||||
const all = s => [...document.querySelectorAll(s)].map(e => e.innerText.trim()).filter(Boolean);
|
||||
const finalUrl = window.location.href;
|
||||
const bodyText = (document.body.innerText || '').slice(0, 400);
|
||||
if (/职位已下线|该职位已删除|页面不存在/.test(bodyText)) {
|
||||
return { error: 'EXPIRED', bodyText };
|
||||
}
|
||||
const companyA = document.querySelector('.cname a, .tCompany_sidebar .com_msg a');
|
||||
const funcs = all('.bmsg .fp');
|
||||
const pick = (prefix) => {
|
||||
const row = funcs.find(f => f.startsWith(prefix));
|
||||
return row ? row.slice(prefix.length).replace(/^[:：\\s\\n]+/, '').trim() : '';
|
||||
};
|
||||
return {
|
||||
finalUrl,
|
||||
title: sel('h1') || sel('.cn .name'),
|
||||
salary: sel('.cn strong') || sel('strong'),
|
||||
meta: sel('.cn .msg.ltype') || sel('.msg.ltype'),
|
||||
description: (() => {
|
||||
const box = document.querySelector('.bmsg.job_msg') || document.querySelector('.job_msg');
|
||||
if (!box) return '';
|
||||
const clone = box.cloneNode(true);
|
||||
clone.querySelectorAll('.fp, .mt10, script, style').forEach(n => n.remove());
|
||||
return (clone.innerText || '').trim();
|
||||
})(),
|
||||
welfare: all('.t1 span, .jtag .t1 span'),
|
||||
category: pick('职能类别'),
|
||||
address: pick('上班地址'),
|
||||
ageRequirement: pick('年龄要求'),
|
||||
company: companyA?.innerText?.trim() || '',
|
||||
companyUrl: companyA?.href || '',
|
||||
companyTag: sel('.com_tag'),
|
||||
};
|
||||
})()`;
|
||||
const data = await page.evaluate(script);
|
||||
if (data.error === 'EXPIRED') {
|
||||
throw new CliError('NO_DATA', `Job ${jobId} is offline or removed`);
|
||||
}
|
||||
if (!data.title) {
|
||||
throw new CliError('NO_DATA', `Could not parse job detail for ${jobId}; page may have changed layout`);
|
||||
}
|
||||
|
||||
// meta looks like "北京-丰台区 | 3年及以上 | 本科"
|
||||
const [locRaw, workYear, degree] = (data.meta || '').split('|').map(s => s.trim());
|
||||
// companyTag looks like "国企\n\n150-500人\n\n电子技术/半导体/集成电路"
|
||||
const tagParts = (data.companyTag || '').split(/\n+/).map(s => s.trim()).filter(Boolean);
|
||||
|
||||
return [{
|
||||
jobId,
|
||||
title: data.title,
|
||||
salary: data.salary || '',
|
||||
location: locRaw || '',
|
||||
workYear: workYear || '',
|
||||
degree: degree || '',
|
||||
category: data.category || '',
|
||||
address: data.address || '',
|
||||
ageRequirement: data.ageRequirement || '',
|
||||
description: data.description || '',
|
||||
welfare: (data.welfare || []).join(','),
|
||||
company: data.company || '',
|
||||
companyType: tagParts[0] || '',
|
||||
companySize: tagParts[1] || '',
|
||||
companyIndustry: tagParts.slice(2).join(' / '),
|
||||
companyUrl: data.companyUrl || '',
|
||||
url: data.finalUrl || url,
|
||||
}];
|
||||
},
|
||||
});
|
||||
@@ -1,55 +0,0 @@
|
||||
/**
|
||||
* 51job hot / recommended feed.
|
||||
*
|
||||
* Same endpoint as `search`, but with empty keyword — 51job returns its
|
||||
* own ranked recommendation list (up to ~999 for most regions).
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import {
|
||||
WE_ORIGIN, SEARCH_COLUMNS, SORT_CODES,
|
||||
requirePage, navigateTo, pageFetchJson,
|
||||
buildSearchUrl, mapJobItem, resolveCity, resolveCode,
|
||||
} from './utils.js';
|
||||
|
||||
cli({
|
||||
site: '51job',
|
||||
name: 'hot',
|
||||
description: '51job 推荐职位（按城市/行业/排序浏览）',
|
||||
domain: 'we.51job.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{ name: 'area', type: 'string', default: '全国', help: '城市名或 6 位城市码（默认 "全国"）' },
|
||||
{ name: 'sort', type: 'string', default: '综合', help: '排序：综合 / 最新 / 薪资 / 距离' },
|
||||
{ name: 'page', type: 'int', default: 1, help: '页码（1-based）' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: '返回条数（1-50）' },
|
||||
],
|
||||
columns: SEARCH_COLUMNS,
|
||||
func: async (page, kwargs) => {
|
||||
requirePage(page);
|
||||
const limit = Math.max(1, Math.min(Number(kwargs.limit) || 20, 50));
|
||||
const pageNum = Math.max(1, Number(kwargs.page) || 1);
|
||||
const jobArea = resolveCity(kwargs.area);
|
||||
const sortType = resolveCode(kwargs.sort, SORT_CODES, '0');
|
||||
|
||||
const currentUrl = await page.evaluate(`(() => window.location.href)()`);
|
||||
if (!String(currentUrl).startsWith(WE_ORIGIN)) {
|
||||
await navigateTo(page, `${WE_ORIGIN}/pc/search?searchType=2`, 2);
|
||||
}
|
||||
|
||||
const url = buildSearchUrl({
|
||||
keyword: '', jobArea, sortType,
|
||||
pageNum, pageSize: Math.min(limit, 50),
|
||||
});
|
||||
const data = await pageFetchJson(page, url);
|
||||
if (data.status !== '1' && data.status !== 1) {
|
||||
throw new CliError('API_ERROR', `51job hot failed: ${data.message ?? 'unknown'}`);
|
||||
}
|
||||
const items = data?.resultbody?.job?.items ?? [];
|
||||
if (items.length === 0) throw new CliError('NO_DATA', 'No recommended jobs returned');
|
||||
return items.slice(0, limit).map((it, i) => mapJobItem(it, (pageNum - 1) * limit + i + 1));
|
||||
},
|
||||
});
|
||||
@@ -1,79 +0,0 @@
|
||||
/**
|
||||
* 51job keyword search.
|
||||
*
|
||||
* Backed by `we.51job.com/api/job/search-pc`, which returns a job list with
|
||||
* the full `jobDescribe` embedded. Needs the browser session because the
|
||||
* Aliyun WAF in front of `we.51job.com` challenges bare fetches; the
|
||||
* `pageFetchJson` helper runs inside the page so the WAF sees a real browser.
|
||||
*/
|
||||
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import {
|
||||
WE_ORIGIN, SEARCH_COLUMNS,
|
||||
SALARY_CODES, WORKYEAR_CODES, DEGREE_CODES,
|
||||
COMPANY_TYPE_CODES, COMPANY_SIZE_CODES, SORT_CODES,
|
||||
requirePage, navigateTo, pageFetchJson,
|
||||
buildSearchUrl, mapJobItem, resolveCity, resolveCode,
|
||||
} from './utils.js';
|
||||
|
||||
cli({
|
||||
site: '51job',
|
||||
name: 'search',
|
||||
description: '51job 前程无忧关键词职位搜索',
|
||||
domain: 'we.51job.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
browser: true,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{ name: 'keyword', type: 'string', required: true, positional: true, help: '搜索关键词（岗位名 / 技能 / 公司）' },
|
||||
{ name: 'area', type: 'string', default: '全国', help: '城市名或 6 位城市码（如 "杭州" / "080200" / "全国"）' },
|
||||
{ name: 'salary', type: 'string', default: '', help: '薪资区间（如 "10-15k" / "1-1.5万" / "20-30k"）' },
|
||||
{ name: 'experience', type: 'string', default: '', help: '工作年限（如 "应届" / "1-3年" / "3-5年" / "5-7年"）' },
|
||||
{ name: 'degree', type: 'string', default: '', help: '学历要求（如 "本科" / "大专" / "硕士"）' },
|
||||
{ name: 'companyType', type: 'string', default: '', help: '公司性质（如 "外资" / "国企" / "民营"）' },
|
||||
{ name: 'companySize', type: 'string', default: '', help: '公司规模（如 "50-150" / "1000-5000"）' },
|
||||
{ name: 'sort', type: 'string', default: '综合', help: '排序：综合 / 最新 / 薪资 / 距离' },
|
||||
{ name: 'page', type: 'int', default: 1, help: '页码（1-based）' },
|
||||
{ name: 'limit', type: 'int', default: 20, help: '返回条数（1-50）' },
|
||||
],
|
||||
columns: SEARCH_COLUMNS,
|
||||
func: async (page, kwargs) => {
|
||||
requirePage(page);
|
||||
const keyword = String(kwargs.keyword ?? '').trim();
|
||||
if (!keyword) throw new CliError('INVALID_ARGUMENT', 'keyword is required');
|
||||
const limit = Math.max(1, Math.min(Number(kwargs.limit) || 20, 50));
|
||||
const pageNum = Math.max(1, Number(kwargs.page) || 1);
|
||||
|
||||
const jobArea = resolveCity(kwargs.area);
|
||||
const salary = resolveCode(kwargs.salary, SALARY_CODES);
|
||||
const workYear = resolveCode(kwargs.experience, WORKYEAR_CODES);
|
||||
const degree = resolveCode(kwargs.degree, DEGREE_CODES);
|
||||
const companyType = resolveCode(kwargs.companyType, COMPANY_TYPE_CODES);
|
||||
const companySize = resolveCode(kwargs.companySize, COMPANY_SIZE_CODES);
|
||||
const sortType = resolveCode(kwargs.sort, SORT_CODES, '0');
|
||||
|
||||
// Establish WAF-clean origin. Reusing the same tab avoids the slider
|
||||
// challenge fire every call.
|
||||
const currentUrl = await page.evaluate(`(() => window.location.href)()`);
|
||||
if (!String(currentUrl).startsWith(WE_ORIGIN)) {
|
||||
await navigateTo(page, `${WE_ORIGIN}/pc/search?keyword=${encodeURIComponent(keyword)}&searchType=2`, 2);
|
||||
}
|
||||
|
||||
const url = buildSearchUrl({
|
||||
keyword, jobArea, salary, workYear, degree,
|
||||
companyType, companySize, sortType,
|
||||
pageNum, pageSize: Math.min(limit, 50),
|
||||
});
|
||||
|
||||
const data = await pageFetchJson(page, url);
|
||||
if (data.status !== '1' && data.status !== 1) {
|
||||
throw new CliError('API_ERROR', `51job search failed: ${data.message ?? 'unknown'}`);
|
||||
}
|
||||
const items = data?.resultbody?.job?.items ?? [];
|
||||
if (items.length === 0) {
|
||||
throw new CliError('NO_DATA', `No jobs matched "${keyword}"`);
|
||||
}
|
||||
return items.slice(0, limit).map((it, i) => mapJobItem(it, (pageNum - 1) * limit + i + 1));
|
||||
},
|
||||
});
|
||||
@@ -1,302 +0,0 @@
|
||||
/**
|
||||
* 51job shared utilities.
|
||||
*
|
||||
* Key design points:
|
||||
* - we.51job.com is protected by Aliyun WAF — bare `curl` / Node-side fetch
|
||||
* gets a slider CAPTCHA HTML page. Only browser-context fetch (page.evaluate)
|
||||
* with the session's cookies survives the challenge.
|
||||
* - `document.cookie` exposes the anti-bot cookies (`acw_sc__v2`, `ssxmod_itna`
|
||||
* etc.) — no HttpOnly/login needed for public pages.
|
||||
* - API (`we.51job.com/api/job/search-pc`) is same-origin when we've navigated
|
||||
* to `https://we.51job.com/...`, so fetch inside page.evaluate works.
|
||||
* - Detail / company pages live on `jobs.51job.com` and render data into the
|
||||
* DOM (SSR), so adapters for those navigate and scrape.
|
||||
*/
|
||||
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
|
||||
export const WE_ORIGIN = 'https://we.51job.com';
|
||||
export const JOBS_ORIGIN = 'https://jobs.51job.com';
|
||||
|
||||
/**
|
||||
* City name / alias → 6-digit jobArea code. `000000` is the national bucket.
|
||||
* Covers the 40 largest cities the search UI surfaces. Unknown input passed
|
||||
* as-is if it's already 6 digits; otherwise fall back to `000000` (all).
|
||||
*/
|
||||
export const CITY_CODES = {
|
||||
'全国': '000000', 'all': '000000',
|
||||
'北京': '010000', 'beijing': '010000',
|
||||
'上海': '020000', 'shanghai': '020000',
|
||||
'广州': '030200', 'guangzhou': '030200',
|
||||
'深圳': '040000', 'shenzhen': '040000',
|
||||
'武汉': '180200', 'wuhan': '180200',
|
||||
'西安': '200200', "xi'an": '200200', 'xian': '200200',
|
||||
'杭州': '080200', 'hangzhou': '080200',
|
||||
'南京': '070200', 'nanjing': '070200',
|
||||
'成都': '090200', 'chengdu': '090200',
|
||||
'苏州': '070300', 'suzhou': '070300',
|
||||
'重庆': '060000', 'chongqing': '060000',
|
||||
'天津': '050000', 'tianjin': '050000',
|
||||
'长沙': '190200', 'changsha': '190200',
|
||||
'郑州': '170200', 'zhengzhou': '170200',
|
||||
'青岛': '120300', 'qingdao': '120300',
|
||||
'合肥': '150200', 'hefei': '150200',
|
||||
'厦门': '110300', 'xiamen': '110300',
|
||||
'无锡': '070400', 'wuxi': '070400',
|
||||
'济南': '120200', 'jinan': '120200',
|
||||
'佛山': '030700', 'foshan': '030700',
|
||||
'东莞': '030800', 'dongguan': '030800',
|
||||
'宁波': '080300', 'ningbo': '080300',
|
||||
'福州': '110200', 'fuzhou': '110200',
|
||||
'昆明': '250200', 'kunming': '250200',
|
||||
'大连': '230300', 'dalian': '230300',
|
||||
'沈阳': '230200', 'shenyang': '230200',
|
||||
'哈尔滨': '220200', 'haerbin': '220200', 'harbin': '220200',
|
||||
'石家庄': '160200', 'shijiazhuang': '160200',
|
||||
'贵阳': '260200', 'guiyang': '260200',
|
||||
'南宁': '100200', 'nanning': '100200',
|
||||
'南昌': '130200', 'nanchang': '130200',
|
||||
'长春': '240200', 'changchun': '240200',
|
||||
'太原': '210200', 'taiyuan': '210200',
|
||||
'兰州': '280200', 'lanzhou': '280200',
|
||||
'乌鲁木齐': '310200', 'urumqi': '310200',
|
||||
'海口': '270200', 'haikou': '270200',
|
||||
'香港': '330000', 'hongkong': '330000', 'hk': '330000',
|
||||
};
|
||||
|
||||
/** Salary bucket code (matches 51job's `salary` filter). */
|
||||
export const SALARY_CODES = {
|
||||
'不限': '',
|
||||
'2千以下': '01', '2-3千': '02', '3-4.5千': '03',
|
||||
'4.5-6千': '04', '6-8千': '05', '8k-1万': '06', '8-10k': '06',
|
||||
'1-1.5万': '07', '10-15k': '07',
|
||||
'1.5-2万': '08', '15-20k': '08',
|
||||
'2-3万': '09', '20-30k': '09',
|
||||
'3-5万': '10', '30-50k': '10',
|
||||
'5万以上': '11', '50k以上': '11',
|
||||
};
|
||||
|
||||
/** Work experience bucket. */
|
||||
export const WORKYEAR_CODES = {
|
||||
'不限': '',
|
||||
'在校生': '01', '应届': '02', '1年以下': '03',
|
||||
'1-3年': '04', '3-5年': '05', '5-7年': '06',
|
||||
'7-10年': '07', '10年以上': '08',
|
||||
};
|
||||
|
||||
/** Degree bucket. */
|
||||
export const DEGREE_CODES = {
|
||||
'不限': '',
|
||||
'初中及以下': '01', '高中/中技/中专': '02', '高中': '02',
|
||||
'大专': '03', '本科': '04', '硕士': '05', '博士': '06',
|
||||
};
|
||||
|
||||
/** Company ownership type. */
|
||||
export const COMPANY_TYPE_CODES = {
|
||||
'不限': '',
|
||||
'外资': '01', '欧美': '0101', '日韩': '0102',
|
||||
'合资': '02', '国企': '03', '民营': '04',
|
||||
'上市公司': '05', '创业公司': '06', '事业单位': '07',
|
||||
'非营利': '08', '政府': '09',
|
||||
};
|
||||
|
||||
/** Company headcount bucket. */
|
||||
export const COMPANY_SIZE_CODES = {
|
||||
'不限': '',
|
||||
'少于50': '01', '50以下': '01',
|
||||
'50-150': '02', '150-500': '03',
|
||||
'500-1000': '04', '1000-5000': '05',
|
||||
'5000-10000': '06', '10000以上': '07',
|
||||
};
|
||||
|
||||
/** Sort strategy. */
|
||||
export const SORT_CODES = {
|
||||
'综合': '0', 'relevance': '0', 'default': '0',
|
||||
'最新': '1', 'new': '1', 'newest': '1',
|
||||
'薪资': '2', 'salary': '2', 'pay': '2',
|
||||
'距离': '9', 'distance': '9',
|
||||
};
|
||||
|
||||
export function resolveCity(input) {
|
||||
if (!input) return '000000';
|
||||
const s = String(input).trim();
|
||||
if (!s || s === '全国' || s.toLowerCase() === 'all') return '000000';
|
||||
if (/^\d{6}$/.test(s)) return s;
|
||||
const key = s.toLowerCase();
|
||||
if (CITY_CODES[s] !== undefined) return CITY_CODES[s];
|
||||
if (CITY_CODES[key] !== undefined) return CITY_CODES[key];
|
||||
for (const [name, code] of Object.entries(CITY_CODES)) {
|
||||
if (typeof name === 'string' && name.includes(s)) return code;
|
||||
}
|
||||
throw new CliError('INVALID_ARGUMENT', `Unknown city/area "${s}"`, 'Use a supported city name like "杭州" or a 6-digit city code');
|
||||
}
|
||||
|
||||
export function resolveCode(input, table, fallback = '') {
|
||||
if (input === undefined || input === null || input === '') return fallback;
|
||||
const s = String(input).trim();
|
||||
if (table[s] !== undefined) return table[s];
|
||||
const key = s.toLowerCase();
|
||||
if (table[key] !== undefined) return table[key];
|
||||
if (Object.values(table).includes(s)) return s;
|
||||
for (const [k, v] of Object.entries(table)) {
|
||||
if (typeof k === 'string' && k.includes(s)) return v;
|
||||
}
|
||||
return fallback;
|
||||
}
|
||||
|
||||
export function requirePage(page) {
|
||||
if (!page) throw new CliError('INTERNAL_ERROR', 'Browser page required (adapter must set browser: true)');
|
||||
}
|
||||
|
||||
/**
|
||||
* Navigate the page to a URL and give the SPA a moment to settle. Reuses
|
||||
* existing session cookies — first call on a fresh browser may trigger the
|
||||
* Aliyun WAF interstitial, which the headless Chromium solves automatically
|
||||
* because the JS that sets `acw_sc__v2` runs in the page.
|
||||
*/
|
||||
export async function navigateTo(page, url, waitSeconds = 2) {
|
||||
await page.goto(url);
|
||||
await page.wait({ time: waitSeconds });
|
||||
}
|
||||
|
||||
/**
|
||||
* Browser-context fetch: execute `fetch(url, { credentials: 'include' })`
|
||||
* inside the page so cookies apply and WAF sees a real browser. Returns
|
||||
* parsed JSON; throws on network / parse / status failure.
|
||||
*/
|
||||
export async function pageFetchJson(page, url, opts = {}) {
|
||||
const method = opts.method ?? 'GET';
|
||||
const body = opts.body ?? null;
|
||||
const timeout = opts.timeout ?? 15000;
|
||||
const headers = opts.headers ?? {};
|
||||
const script = `
|
||||
async () => {
|
||||
const ctrl = new AbortController();
|
||||
const timer = setTimeout(() => ctrl.abort(), ${timeout});
|
||||
try {
|
||||
const resp = await fetch(${JSON.stringify(url)}, {
|
||||
method: ${JSON.stringify(method)},
|
||||
credentials: 'include',
|
||||
headers: ${JSON.stringify({ Accept: 'application/json', ...headers })},
|
||||
${body !== null ? `body: ${JSON.stringify(body)},` : ''}
|
||||
signal: ctrl.signal,
|
||||
});
|
||||
const text = await resp.text();
|
||||
return { ok: resp.ok, status: resp.status, text };
|
||||
} catch (e) {
|
||||
return { ok: false, status: 0, text: '', error: String(e && e.message || e) };
|
||||
} finally {
|
||||
clearTimeout(timer);
|
||||
}
|
||||
}
|
||||
`;
|
||||
const res = await page.evaluate(script);
|
||||
if (res.error) throw new CliError('HTTP_ERROR', `51job fetch failed: ${res.error}`);
|
||||
if (!res.ok) throw new CliError('HTTP_ERROR', `51job HTTP ${res.status}`);
|
||||
if (res.text.trim().startsWith('<')) {
|
||||
throw new CliError('ANTI_BOT', '51job returned HTML (likely Aliyun WAF slider). Refresh browser session.');
|
||||
}
|
||||
try {
|
||||
return JSON.parse(res.text);
|
||||
} catch (e) {
|
||||
throw new CliError('API_ERROR', `51job invalid JSON: ${res.text.slice(0, 200)}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the canonical search-pc URL. All optional filters default to empty
|
||||
* (no constraint). `scene=7` + `source=1` match what the real SPA sends.
|
||||
*/
|
||||
export function buildSearchUrl(params) {
|
||||
const qs = new URLSearchParams();
|
||||
qs.set('api_key', '51job');
|
||||
qs.set('timestamp', String(Date.now()));
|
||||
qs.set('keyword', params.keyword ?? '');
|
||||
qs.set('searchType', '2');
|
||||
qs.set('function', params.function ?? '');
|
||||
qs.set('industry', params.industry ?? '');
|
||||
qs.set('jobArea', params.jobArea ?? '000000');
|
||||
qs.set('jobArea2', params.jobArea2 ?? '');
|
||||
qs.set('landmark', params.landmark ?? '');
|
||||
qs.set('metro', params.metro ?? '');
|
||||
qs.set('salary', params.salary ?? '');
|
||||
qs.set('workYear', params.workYear ?? '');
|
||||
qs.set('degree', params.degree ?? '');
|
||||
qs.set('companyType', params.companyType ?? '');
|
||||
qs.set('companySize', params.companySize ?? '');
|
||||
qs.set('jobType', params.jobType ?? '');
|
||||
qs.set('issueDate', params.issueDate ?? '');
|
||||
qs.set('sortType', params.sortType ?? '0');
|
||||
qs.set('pageNum', String(params.pageNum ?? 1));
|
||||
qs.set('pageSize', String(params.pageSize ?? 20));
|
||||
qs.set('source', '1');
|
||||
qs.set('scene', '7');
|
||||
return `${WE_ORIGIN}/api/job/search-pc?${qs.toString()}`;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map a raw search-pc `resultbody.job.items[i]` into the canonical row shape
|
||||
* we expose to the user. Kept here so `search` and `hot` stay aligned.
|
||||
*/
|
||||
export function mapJobItem(it, rank) {
|
||||
const area = it.jobAreaLevelDetail || {};
|
||||
return {
|
||||
rank,
|
||||
jobId: String(it.jobId ?? ''),
|
||||
title: it.jobName ?? '',
|
||||
salary: it.provideSalaryString ?? '',
|
||||
salaryMin: Number(it.jobSalaryMin ?? 0) || 0,
|
||||
salaryMax: Number(it.jobSalaryMax ?? 0) || 0,
|
||||
city: area.cityString ?? it.jobAreaString ?? '',
|
||||
district: area.districtString ?? '',
|
||||
workYear: it.workYearString ?? '',
|
||||
degree: it.degreeString ?? '',
|
||||
tags: Array.isArray(it.jobTags) ? it.jobTags.join(',') : '',
|
||||
company: it.companyName ?? '',
|
||||
companyFull: it.fullCompanyName ?? '',
|
||||
companyType: it.companyTypeString ?? '',
|
||||
companySize: it.companySizeString ?? '',
|
||||
industry: it.industryType1Str ?? '',
|
||||
hr: it.hrName ? `${it.hrName}·${it.hrPosition ?? ''}` : '',
|
||||
issueDate: it.issueDateString ?? '',
|
||||
url: it.jobHref ?? '',
|
||||
companyUrl: it.companyHref ?? '',
|
||||
encCoId: it.encCoId ?? '',
|
||||
};
|
||||
}
|
||||
|
||||
export const SEARCH_COLUMNS = [
|
||||
'rank', 'jobId', 'title', 'salary', 'salaryMin', 'salaryMax',
|
||||
'city', 'district', 'workYear', 'degree', 'tags',
|
||||
'company', 'companyFull', 'companyType', 'companySize', 'industry',
|
||||
'hr', 'issueDate', 'url', 'companyUrl', 'encCoId',
|
||||
];
|
||||
|
||||
/**
|
||||
* Parse a 51job company-page `<a sensorsdata="...">` payload into a stable
|
||||
* row fragment. Returns null when the attribute is absent or malformed.
|
||||
*/
|
||||
export function parseCompanyJobCard(raw) {
|
||||
if (!raw || typeof raw !== 'object') return null;
|
||||
const href = typeof raw.href === 'string' ? raw.href : '';
|
||||
const sensorsdata = typeof raw.sensorsdata === 'string' ? raw.sensorsdata : '';
|
||||
if (!href || !sensorsdata) return null;
|
||||
let data;
|
||||
try {
|
||||
data = JSON.parse(sensorsdata);
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
if (!data || !data.jobId) return null;
|
||||
return {
|
||||
jobId: String(data.jobId),
|
||||
title: data.jobTitle || '',
|
||||
salary: data.jobSalary || '',
|
||||
city: data.jobArea || '',
|
||||
workYear: data.jobYear || '',
|
||||
degree: data.jobDegree || '',
|
||||
funcType: data.funcType || '',
|
||||
issueDate: data.jobTime || '',
|
||||
url: href,
|
||||
};
|
||||
}
|
||||
@@ -1,69 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { CliError } from '@jackwener/opencli/errors';
|
||||
import { parseCompanyJobCard, pageFetchJson, resolveCity } from './utils.js';
|
||||
|
||||
describe('51job resolveCity', () => {
|
||||
it('maps known city names and explicit national scope', () => {
|
||||
expect(resolveCity('杭州')).toBe('080200');
|
||||
expect(resolveCity('all')).toBe('000000');
|
||||
expect(resolveCity('000000')).toBe('000000');
|
||||
});
|
||||
|
||||
it('rejects unknown non-empty inputs instead of silently widening to 全国', () => {
|
||||
expect(() => resolveCity('杭州z')).toThrowError(CliError);
|
||||
expect(() => resolveCity('杭州z')).toThrow(/Unknown city\/area/);
|
||||
});
|
||||
});
|
||||
|
||||
describe('51job pageFetchJson', () => {
|
||||
it('detects WAF challenge HTML and throws ANTI_BOT', async () => {
|
||||
const page = {
|
||||
evaluate: vi.fn().mockResolvedValue({
|
||||
ok: true,
|
||||
status: 200,
|
||||
text: '<html><title>slider</title></html>',
|
||||
}),
|
||||
};
|
||||
|
||||
await expect(pageFetchJson(page, 'https://we.51job.com/api/job/search-pc')).rejects.toMatchObject({
|
||||
code: 'ANTI_BOT',
|
||||
});
|
||||
});
|
||||
});
|
||||
|
||||
describe('51job parseCompanyJobCard', () => {
|
||||
it('parses sensorsdata JSON into a stable row fragment', () => {
|
||||
const row = parseCompanyJobCard({
|
||||
href: 'https://jobs.51job.com/shanghai/123456789.html',
|
||||
sensorsdata: JSON.stringify({
|
||||
jobId: '123456789',
|
||||
jobTitle: 'Senior Engineer',
|
||||
jobSalary: '20-30K',
|
||||
jobArea: '上海',
|
||||
jobYear: '3-5年',
|
||||
jobDegree: '本科',
|
||||
funcType: '后端开发',
|
||||
jobTime: '04-22',
|
||||
}),
|
||||
});
|
||||
|
||||
expect(row).toEqual({
|
||||
jobId: '123456789',
|
||||
title: 'Senior Engineer',
|
||||
salary: '20-30K',
|
||||
city: '上海',
|
||||
workYear: '3-5年',
|
||||
degree: '本科',
|
||||
funcType: '后端开发',
|
||||
issueDate: '04-22',
|
||||
url: 'https://jobs.51job.com/shanghai/123456789.html',
|
||||
});
|
||||
});
|
||||
|
||||
it('returns null on malformed sensorsdata', () => {
|
||||
expect(parseCompanyJobCard({
|
||||
href: 'https://jobs.51job.com/shanghai/123456789.html',
|
||||
sensorsdata: '{bad json}',
|
||||
})).toBeNull();
|
||||
});
|
||||
});
|
||||
@@ -1,32 +0,0 @@
|
||||
/**
|
||||
* Shared utilities for CLI adapters.
|
||||
*/
|
||||
import { ArgumentError } from '@jackwener/opencli/errors';
|
||||
/**
|
||||
* Clamp a numeric value to [min, max].
|
||||
* Matches the signature of lodash.clamp and Rust's clamp.
|
||||
*/
|
||||
export function clamp(value, min, max) {
|
||||
return Math.max(min, Math.min(value, max));
|
||||
}
|
||||
export function clampInt(raw, fallback, min, max) {
|
||||
const parsed = Number(raw);
|
||||
if (!Number.isFinite(parsed)) {
|
||||
return fallback;
|
||||
}
|
||||
return clamp(Math.floor(parsed), min, max);
|
||||
}
|
||||
export function normalizeNumericId(value, label, example) {
|
||||
const normalized = String(value ?? '').trim();
|
||||
if (!/^\d+$/.test(normalized)) {
|
||||
throw new ArgumentError(`${label} must be a numeric ID`, `Pass a numeric ${label}, for example: ${example}`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
export function requireNonEmptyQuery(value, label = 'query') {
|
||||
const normalized = String(value ?? '').trim();
|
||||
if (!normalized) {
|
||||
throw new ArgumentError(`${label} cannot be empty`);
|
||||
}
|
||||
return normalized;
|
||||
}
|
||||
@@ -0,0 +1,11 @@
|
||||
/**
|
||||
* Shared utilities for CLI adapters.
|
||||
*/
|
||||
|
||||
/**
|
||||
* Clamp a numeric value to [min, max].
|
||||
* Matches the signature of lodash.clamp and Rust's clamp.
|
||||
*/
|
||||
export function clamp(value: number, min: number, max: number): number {
|
||||
return Math.max(min, Math.min(value, max));
|
||||
}
|
||||
@@ -1,108 +0,0 @@
|
||||
/**
|
||||
* Shared command factories for Electron/desktop app adapters.
|
||||
* Eliminates duplicate screenshot/status/new/dump implementations
|
||||
* across cursor, codex, chatwise, etc.
|
||||
*/
|
||||
import * as fs from 'node:fs';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
/**
|
||||
* Factory: capture DOM HTML + accessibility snapshot.
|
||||
*/
|
||||
export function makeScreenshotCommand(site, displayName, extra = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'screenshot',
|
||||
description: `Capture a snapshot of the current ${label} window (DOM + Accessibility tree)`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'output', required: false, help: `Output file path (default: /tmp/${site}-snapshot.txt)` },
|
||||
],
|
||||
columns: ['Status', 'File'],
|
||||
func: async (page, kwargs) => {
|
||||
const outputPath = kwargs.output || `/tmp/${site}-snapshot.txt`;
|
||||
const snap = await page.snapshot({ compact: true });
|
||||
const html = await page.evaluate('document.documentElement.outerHTML');
|
||||
const htmlPath = outputPath.replace(/\.\w+$/, '') + '-dom.html';
|
||||
const snapPath = outputPath.replace(/\.\w+$/, '') + '-a11y.txt';
|
||||
fs.writeFileSync(htmlPath, html);
|
||||
fs.writeFileSync(snapPath, typeof snap === 'string' ? snap : JSON.stringify(snap, null, 2));
|
||||
return [
|
||||
{ Status: 'Success', File: htmlPath },
|
||||
{ Status: 'Success', File: snapPath },
|
||||
];
|
||||
},
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Factory: check CDP connection status.
|
||||
*/
|
||||
export function makeStatusCommand(site, displayName, extra = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'status',
|
||||
description: `Check active CDP connection to ${label}`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['Status', 'Url', 'Title'],
|
||||
func: async (page) => {
|
||||
const url = await page.evaluate('window.location.href');
|
||||
const title = await page.evaluate('document.title');
|
||||
return [{ Status: 'Connected', Url: url, Title: title }];
|
||||
},
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Factory: start a new session via Cmd/Ctrl+N.
|
||||
*/
|
||||
export function makeNewCommand(site, displayName, extra = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'new',
|
||||
description: `Start a new ${label} session`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['Status'],
|
||||
func: async (page) => {
|
||||
const isMac = process.platform === 'darwin';
|
||||
await page.pressKey(isMac ? 'Meta+N' : 'Control+N');
|
||||
await page.wait(1);
|
||||
return [{ Status: 'Success' }];
|
||||
},
|
||||
});
|
||||
}
|
||||
/**
|
||||
* Factory: dump DOM + snapshot for reverse-engineering.
|
||||
*/
|
||||
export function makeDumpCommand(site) {
|
||||
return cli({
|
||||
site,
|
||||
name: 'dump',
|
||||
description: `Dump the DOM and Accessibility tree of ${site} for reverse-engineering`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['action', 'files'],
|
||||
func: async (page) => {
|
||||
const dom = await page.evaluate('document.body.innerHTML');
|
||||
fs.writeFileSync(`/tmp/${site}-dom.html`, dom);
|
||||
const snap = await page.snapshot({ interactive: false });
|
||||
fs.writeFileSync(`/tmp/${site}-snapshot.json`, JSON.stringify(snap, null, 2));
|
||||
return [
|
||||
{
|
||||
action: 'Dom extraction finished',
|
||||
files: `/tmp/${site}-dom.html, /tmp/${site}-snapshot.json`,
|
||||
},
|
||||
];
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,121 @@
|
||||
/**
|
||||
* Shared command factories for Electron/desktop app adapters.
|
||||
* Eliminates duplicate screenshot/status/new/dump implementations
|
||||
* across cursor, codex, chatwise, etc.
|
||||
*/
|
||||
|
||||
import * as fs from 'node:fs';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import type { CliOptions } from '@jackwener/opencli/registry';
|
||||
|
||||
/**
|
||||
* Factory: capture DOM HTML + accessibility snapshot.
|
||||
*/
|
||||
export function makeScreenshotCommand(site: string, displayName?: string, extra: Partial<CliOptions> = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'screenshot',
|
||||
description: `Capture a snapshot of the current ${label} window (DOM + Accessibility tree)`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'output', required: false, help: `Output file path (default: /tmp/${site}-snapshot.txt)` },
|
||||
],
|
||||
columns: ['Status', 'File'],
|
||||
func: async (page: IPage, kwargs: any) => {
|
||||
const outputPath = (kwargs.output as string) || `/tmp/${site}-snapshot.txt`;
|
||||
|
||||
const snap = await page.snapshot({ compact: true });
|
||||
const html = await page.evaluate('document.documentElement.outerHTML');
|
||||
|
||||
const htmlPath = outputPath.replace(/\.\w+$/, '') + '-dom.html';
|
||||
const snapPath = outputPath.replace(/\.\w+$/, '') + '-a11y.txt';
|
||||
|
||||
fs.writeFileSync(htmlPath, html);
|
||||
fs.writeFileSync(snapPath, typeof snap === 'string' ? snap : JSON.stringify(snap, null, 2));
|
||||
|
||||
return [
|
||||
{ Status: 'Success', File: htmlPath },
|
||||
{ Status: 'Success', File: snapPath },
|
||||
];
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory: check CDP connection status.
|
||||
*/
|
||||
export function makeStatusCommand(site: string, displayName?: string, extra: Partial<CliOptions> = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'status',
|
||||
description: `Check active CDP connection to ${label}`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['Status', 'Url', 'Title'],
|
||||
func: async (page: IPage) => {
|
||||
const url = await page.evaluate('window.location.href');
|
||||
const title = await page.evaluate('document.title');
|
||||
return [{ Status: 'Connected', Url: url, Title: title }];
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory: start a new session via Cmd/Ctrl+N.
|
||||
*/
|
||||
export function makeNewCommand(site: string, displayName?: string, extra: Partial<CliOptions> = {}) {
|
||||
const label = displayName ?? site;
|
||||
return cli({
|
||||
...extra,
|
||||
site,
|
||||
name: 'new',
|
||||
description: `Start a new ${label} session`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['Status'],
|
||||
func: async (page: IPage) => {
|
||||
const isMac = process.platform === 'darwin';
|
||||
await page.pressKey(isMac ? 'Meta+N' : 'Control+N');
|
||||
await page.wait(1);
|
||||
return [{ Status: 'Success' }];
|
||||
},
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Factory: dump DOM + snapshot for reverse-engineering.
|
||||
*/
|
||||
export function makeDumpCommand(site: string) {
|
||||
return cli({
|
||||
site,
|
||||
name: 'dump',
|
||||
description: `Dump the DOM and Accessibility tree of ${site} for reverse-engineering`,
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
columns: ['action', 'files'],
|
||||
func: async (page: IPage) => {
|
||||
const dom = await page.evaluate('document.body.innerHTML');
|
||||
fs.writeFileSync(`/tmp/${site}-dom.html`, dom);
|
||||
|
||||
const snap = await page.snapshot({ interactive: false });
|
||||
fs.writeFileSync(`/tmp/${site}-snapshot.json`, JSON.stringify(snap, null, 2));
|
||||
|
||||
return [
|
||||
{
|
||||
action: 'Dom extraction finished',
|
||||
files: `/tmp/${site}-dom.html, /tmp/${site}-snapshot.json`,
|
||||
},
|
||||
];
|
||||
},
|
||||
});
|
||||
}
|
||||
@@ -1,29 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './rankings.js';
|
||||
describe('amazon bestsellers normalization', () => {
|
||||
it('normalizes bestseller cards and infers review counts from card text', () => {
|
||||
const result = __test__.normalizeRankingCandidate({
|
||||
asin: 'B0DR31GC3D',
|
||||
title: '',
|
||||
href: 'https://www.amazon.com/NUTIKAS-Shelves-Desktop-Orgnizer-Shlef/dp/B0DR31GC3D/ref=zg_bs',
|
||||
price_text: '$25.92',
|
||||
rating_text: '4.3 out of 5 stars',
|
||||
review_count_text: '',
|
||||
card_text: 'Desk Shelves Desktop Organizer Shlef\n4.3 out of 5 stars\n435\n$25.92',
|
||||
}, {
|
||||
listType: 'bestsellers',
|
||||
rankFallback: 2,
|
||||
listTitle: 'Amazon Best Sellers: Best Desktop & Off-Surface Shelves',
|
||||
sourceUrl: 'https://www.amazon.com/example',
|
||||
categoryTitle: null,
|
||||
categoryUrl: 'https://www.amazon.com/example',
|
||||
categoryPath: [],
|
||||
visibleCategoryLinks: [],
|
||||
});
|
||||
expect(result.rank).toBe(2);
|
||||
expect(result.asin).toBe('B0DR31GC3D');
|
||||
expect(result.title).toBe('Desk Shelves Desktop Organizer Shlef');
|
||||
expect(result.review_count).toBe(435);
|
||||
expect(result.list_title).toBe('Amazon Best Sellers: Best Desktop & Off-Surface Shelves');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,31 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './rankings.js';
|
||||
|
||||
describe('amazon bestsellers normalization', () => {
|
||||
it('normalizes bestseller cards and infers review counts from card text', () => {
|
||||
const result = __test__.normalizeRankingCandidate({
|
||||
asin: 'B0DR31GC3D',
|
||||
title: '',
|
||||
href: 'https://www.amazon.com/NUTIKAS-Shelves-Desktop-Orgnizer-Shlef/dp/B0DR31GC3D/ref=zg_bs',
|
||||
price_text: '$25.92',
|
||||
rating_text: '4.3 out of 5 stars',
|
||||
review_count_text: '',
|
||||
card_text: 'Desk Shelves Desktop Organizer Shlef\n4.3 out of 5 stars\n435\n$25.92',
|
||||
}, {
|
||||
listType: 'bestsellers',
|
||||
rankFallback: 2,
|
||||
listTitle: 'Amazon Best Sellers: Best Desktop & Off-Surface Shelves',
|
||||
sourceUrl: 'https://www.amazon.com/example',
|
||||
categoryTitle: null,
|
||||
categoryUrl: 'https://www.amazon.com/example',
|
||||
categoryPath: [],
|
||||
visibleCategoryLinks: [],
|
||||
});
|
||||
|
||||
expect(result.rank).toBe(2);
|
||||
expect(result.asin).toBe('B0DR31GC3D');
|
||||
expect(result.title).toBe('Desk Shelves Desktop Organizer Shlef');
|
||||
expect(result.review_count).toBe(435);
|
||||
expect(result.list_title).toBe('Amazon Best Sellers: Best Desktop & Off-Surface Shelves');
|
||||
});
|
||||
});
|
||||
@@ -1,7 +1,8 @@
|
||||
import { cli } from '@jackwener/opencli/registry';
|
||||
import { createRankingCliOptions } from './rankings.js';
|
||||
|
||||
cli(createRankingCliOptions({
|
||||
commandName: 'new-releases',
|
||||
listType: 'new_releases',
|
||||
description: 'Amazon New Releases pages for early momentum discovery',
|
||||
commandName: 'bestsellers',
|
||||
listType: 'bestsellers',
|
||||
description: 'Amazon Best Sellers pages for category candidate discovery',
|
||||
}));
|
||||
@@ -1,122 +0,0 @@
|
||||
import { AuthRequiredError, CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { buildProductUrl, buildDiscussionUrl, buildProvenance, cleanText, extractAsin, normalizeProductUrl, parseRatingValue, parseReviewCount, trimRatingPrefix, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js';
|
||||
function normalizeDiscussionPayload(payload) {
|
||||
const sourceUrl = cleanText(payload.href) || buildDiscussionUrl(payload.href ?? '');
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const averageRatingText = cleanText(payload.average_rating_text) || null;
|
||||
const totalReviewCountText = cleanText(payload.total_review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
return {
|
||||
asin,
|
||||
product_url: asin ? normalizeProductUrl(asin) : null,
|
||||
discussion_url: sourceUrl,
|
||||
...provenance,
|
||||
average_rating_text: averageRatingText,
|
||||
average_rating_value: parseRatingValue(averageRatingText),
|
||||
total_review_count_text: totalReviewCountText,
|
||||
total_review_count: parseReviewCount(totalReviewCountText),
|
||||
qa_urls: uniqueNonEmpty(payload.qa_links ?? []),
|
||||
review_samples: (payload.review_samples ?? []).map((sample) => ({
|
||||
title: trimRatingPrefix(sample.title) || null,
|
||||
rating_text: cleanText(sample.rating_text) || null,
|
||||
rating_value: parseRatingValue(sample.rating_text),
|
||||
author: cleanText(sample.author) || null,
|
||||
date_text: cleanText(sample.date_text) || null,
|
||||
body: cleanText(sample.body) || null,
|
||||
verified_purchase: sample.verified === true,
|
||||
})),
|
||||
};
|
||||
}
|
||||
function hasDiscussionSummary(payload) {
|
||||
return Boolean(cleanText(payload.average_rating_text) || cleanText(payload.total_review_count_text));
|
||||
}
|
||||
function isSignInState(state) {
|
||||
const href = cleanText(state.href).toLowerCase();
|
||||
const title = cleanText(state.title).toLowerCase();
|
||||
return href.includes('/ap/signin')
|
||||
|| title.includes('amazon sign-in');
|
||||
}
|
||||
async function readCurrentDiscussionPayload(page, limit) {
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
average_rating_text: document.querySelector('[data-hook="rating-out-of-text"]')?.textContent || '',
|
||||
total_review_count_text: document.querySelector('[data-hook="total-review-count"]')?.textContent || '',
|
||||
qa_links: Array.from(document.querySelectorAll('a[href*="ask/questions"]')).map((anchor) => anchor.href || ''),
|
||||
review_samples: Array.from(document.querySelectorAll('[data-hook="review"]')).slice(0, ${limit}).map((card) => ({
|
||||
title: card.querySelector('[data-hook="review-title"]')?.textContent || '',
|
||||
rating_text:
|
||||
card.querySelector('[data-hook="review-star-rating"]')?.textContent
|
||||
|| card.querySelector('[data-hook="cmps-review-star-rating"]')?.textContent
|
||||
|| '',
|
||||
author: card.querySelector('.a-profile-name')?.textContent || '',
|
||||
date_text: card.querySelector('[data-hook="review-date"]')?.textContent || '',
|
||||
body: card.querySelector('[data-hook="review-body"]')?.textContent || '',
|
||||
verified: !!card.querySelector('[data-hook="avp-badge"]'),
|
||||
})),
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
async function readDiscussionPayload(page, input, limit) {
|
||||
const reviewUrl = buildDiscussionUrl(input);
|
||||
const reviewState = await gotoAndReadState(page, reviewUrl, 2500, 'discussion');
|
||||
assertUsableState(reviewState, 'discussion');
|
||||
const reviewPayload = await readCurrentDiscussionPayload(page, limit);
|
||||
if (hasDiscussionSummary(reviewPayload)) {
|
||||
return reviewPayload;
|
||||
}
|
||||
const productUrl = buildProductUrl(input);
|
||||
const productState = await gotoAndReadState(page, productUrl, 2500, 'discussion');
|
||||
assertUsableState(productState, 'discussion');
|
||||
if (isSignInState(reviewState) && isSignInState(productState)) {
|
||||
throw new AuthRequiredError('amazon.com', 'Amazon review discussion requires an active signed-in Amazon session in the shared Chrome profile.');
|
||||
}
|
||||
const productPayload = await readCurrentDiscussionPayload(page, limit);
|
||||
if (hasDiscussionSummary(productPayload)) {
|
||||
return productPayload;
|
||||
}
|
||||
if (isSignInState(reviewState)) {
|
||||
throw new CommandExecutionError('amazon review page redirected to sign-in and product page fallback did not expose review summary', 'Open the product page in Chrome, verify reviews are visible, and retry.');
|
||||
}
|
||||
return reviewPayload;
|
||||
}
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'discussion',
|
||||
description: 'Amazon review summary and sample customer discussion from product review pages',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 10,
|
||||
help: 'Maximum number of review samples to return (default 10)',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'average_rating_value', 'total_review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 10);
|
||||
const payload = await readDiscussionPayload(page, input, limit);
|
||||
const normalized = normalizeDiscussionPayload(payload);
|
||||
if (!normalized.average_rating_text && !normalized.total_review_count_text) {
|
||||
throw new CommandExecutionError('amazon discussion page did not expose review summary', 'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.');
|
||||
}
|
||||
return [normalized];
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeDiscussionPayload,
|
||||
hasDiscussionSummary,
|
||||
isSignInState,
|
||||
};
|
||||
@@ -1,151 +0,0 @@
|
||||
import { describe, expect, it, vi } from 'vitest';
|
||||
import { AuthRequiredError } from '@jackwener/opencli/errors';
|
||||
import { getRegistry } from '@jackwener/opencli/registry';
|
||||
import { __test__ } from './discussion.js';
|
||||
import './discussion.js';
|
||||
|
||||
function createPageMock(evaluateResults) {
|
||||
const evaluate = vi.fn();
|
||||
for (const result of evaluateResults) {
|
||||
evaluate.mockResolvedValueOnce(result);
|
||||
}
|
||||
return {
|
||||
goto: vi.fn().mockResolvedValue(undefined),
|
||||
wait: vi.fn().mockResolvedValue(undefined),
|
||||
evaluate,
|
||||
snapshot: vi.fn().mockResolvedValue(undefined),
|
||||
click: vi.fn().mockResolvedValue(undefined),
|
||||
typeText: vi.fn().mockResolvedValue(undefined),
|
||||
pressKey: vi.fn().mockResolvedValue(undefined),
|
||||
scrollTo: vi.fn().mockResolvedValue(undefined),
|
||||
getFormState: vi.fn().mockResolvedValue({ forms: [], orphanFields: [] }),
|
||||
tabs: vi.fn().mockResolvedValue([]),
|
||||
selectTab: vi.fn().mockResolvedValue(undefined),
|
||||
networkRequests: vi.fn().mockResolvedValue([]),
|
||||
consoleMessages: vi.fn().mockResolvedValue([]),
|
||||
scroll: vi.fn().mockResolvedValue(undefined),
|
||||
autoScroll: vi.fn().mockResolvedValue(undefined),
|
||||
installInterceptor: vi.fn().mockResolvedValue(undefined),
|
||||
getInterceptedRequests: vi.fn().mockResolvedValue([]),
|
||||
getCookies: vi.fn().mockResolvedValue([]),
|
||||
screenshot: vi.fn().mockResolvedValue(''),
|
||||
waitForCapture: vi.fn().mockResolvedValue(undefined),
|
||||
};
|
||||
}
|
||||
|
||||
describe('amazon discussion normalization', () => {
|
||||
it('normalizes review summary and sample reviews', () => {
|
||||
const result = __test__.normalizeDiscussionPayload({
|
||||
href: 'https://www.amazon.com/product-reviews/B0FJS72893',
|
||||
average_rating_text: '3.9 out of 5',
|
||||
total_review_count_text: '27 global ratings',
|
||||
qa_links: [],
|
||||
review_samples: [
|
||||
{
|
||||
title: '5.0 out of 5 stars Great value and quality',
|
||||
rating_text: '5.0 out of 5 stars',
|
||||
author: 'GTreader2',
|
||||
date_text: 'Reviewed in the United States on February 21, 2026',
|
||||
body: 'Small but mighty.',
|
||||
verified: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.average_rating_value).toBe(3.9);
|
||||
expect(result.total_review_count).toBe(27);
|
||||
expect(result.review_samples).toEqual([
|
||||
{
|
||||
title: 'Great value and quality',
|
||||
rating_text: '5.0 out of 5 stars',
|
||||
rating_value: 5,
|
||||
author: 'GTreader2',
|
||||
date_text: 'Reviewed in the United States on February 21, 2026',
|
||||
body: 'Small but mighty.',
|
||||
verified_purchase: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
|
||||
it('falls back to the product page when the review page redirects to sign-in', async () => {
|
||||
const command = getRegistry().get('amazon/discussion');
|
||||
const page = createPageMock([
|
||||
{
|
||||
href: 'https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.amazon.com%2Fproduct-reviews%2FB09HKN2ZRT',
|
||||
title: 'Amazon Sign-In',
|
||||
body_text: 'Sign in Create account',
|
||||
},
|
||||
{
|
||||
href: 'https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.amazon.com%2Fproduct-reviews%2FB09HKN2ZRT',
|
||||
average_rating_text: '',
|
||||
total_review_count_text: '',
|
||||
review_samples: [],
|
||||
},
|
||||
{
|
||||
href: 'https://www.amazon.com/dp/B09HKN2ZRT',
|
||||
title: 'Amazon.com: Example product',
|
||||
body_text: 'Hello, zejia-wu Reviews',
|
||||
},
|
||||
{
|
||||
href: 'https://www.amazon.com/dp/B09HKN2ZRT',
|
||||
average_rating_text: '4.4 out of 5',
|
||||
total_review_count_text: '349 global ratings',
|
||||
review_samples: [
|
||||
{
|
||||
title: '5.0 out of 5 stars Perfect for the office',
|
||||
rating_text: '5.0 out of 5 stars',
|
||||
author: 'Ken',
|
||||
date_text: 'Reviewed in the United States on March 19, 2026',
|
||||
body: 'Good for the office, no complaints.',
|
||||
verified: true,
|
||||
},
|
||||
],
|
||||
},
|
||||
]);
|
||||
|
||||
const result = await command.func(page, { input: 'B09HKN2ZRT', limit: 1 });
|
||||
|
||||
expect(page.goto.mock.calls.map((call) => call[0])).toEqual([
|
||||
'https://www.amazon.com/product-reviews/B09HKN2ZRT',
|
||||
'https://www.amazon.com/dp/B09HKN2ZRT',
|
||||
]);
|
||||
expect(result).toEqual([
|
||||
expect.objectContaining({
|
||||
asin: 'B09HKN2ZRT',
|
||||
discussion_url: 'https://www.amazon.com/dp/B09HKN2ZRT',
|
||||
average_rating_value: 4.4,
|
||||
total_review_count: 349,
|
||||
}),
|
||||
]);
|
||||
});
|
||||
|
||||
it('throws AuthRequiredError when both review and product pages are gated', async () => {
|
||||
const command = getRegistry().get('amazon/discussion');
|
||||
const authState = {
|
||||
href: 'https://www.amazon.com/ap/signin?openid.return_to=https%3A%2F%2Fwww.amazon.com%2Fproduct-reviews%2FB09HKN2ZRT',
|
||||
title: 'Amazon Sign-In',
|
||||
body_text: 'Sign in Create account',
|
||||
};
|
||||
const page = createPageMock([
|
||||
authState,
|
||||
{
|
||||
href: authState.href,
|
||||
average_rating_text: '',
|
||||
total_review_count_text: '',
|
||||
review_samples: [],
|
||||
},
|
||||
authState,
|
||||
]);
|
||||
|
||||
await expect(command.func(page, { input: 'B09HKN2ZRT', limit: 1 })).rejects.toBeInstanceOf(AuthRequiredError);
|
||||
});
|
||||
|
||||
it('does not treat a public product page with sign-in copy as a gated page', () => {
|
||||
expect(__test__.isSignInState({
|
||||
href: 'https://www.amazon.com/dp/B09HKN2ZRT',
|
||||
title: 'Amazon.com: Example product',
|
||||
body_text: 'Hello, sign in Account & Lists Create account',
|
||||
})).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,38 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './discussion.js';
|
||||
|
||||
describe('amazon discussion normalization', () => {
|
||||
it('normalizes review summary and sample reviews', () => {
|
||||
const result = __test__.normalizeDiscussionPayload({
|
||||
href: 'https://www.amazon.com/product-reviews/B0FJS72893',
|
||||
average_rating_text: '3.9 out of 5',
|
||||
total_review_count_text: '27 global ratings',
|
||||
qa_links: [],
|
||||
review_samples: [
|
||||
{
|
||||
title: '5.0 out of 5 stars Great value and quality',
|
||||
rating_text: '5.0 out of 5 stars',
|
||||
author: 'GTreader2',
|
||||
date_text: 'Reviewed in the United States on February 21, 2026',
|
||||
body: 'Small but mighty.',
|
||||
verified: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.average_rating_value).toBe(3.9);
|
||||
expect(result.total_review_count).toBe(27);
|
||||
expect(result.review_samples).toEqual([
|
||||
{
|
||||
title: 'Great value and quality',
|
||||
rating_text: '5.0 out of 5 stars',
|
||||
rating_value: 5,
|
||||
author: 'GTreader2',
|
||||
date_text: 'Reviewed in the United States on February 21, 2026',
|
||||
body: 'Small but mighty.',
|
||||
verified_purchase: true,
|
||||
},
|
||||
]);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
buildDiscussionUrl,
|
||||
buildProvenance,
|
||||
cleanText,
|
||||
extractAsin,
|
||||
normalizeProductUrl,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
trimRatingPrefix,
|
||||
uniqueNonEmpty,
|
||||
assertUsableState,
|
||||
gotoAndReadState,
|
||||
} from './shared.js';
|
||||
|
||||
interface DiscussionPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
average_rating_text?: string | null;
|
||||
total_review_count_text?: string | null;
|
||||
qa_links?: string[];
|
||||
review_samples?: Array<{
|
||||
title?: string | null;
|
||||
rating_text?: string | null;
|
||||
author?: string | null;
|
||||
date_text?: string | null;
|
||||
body?: string | null;
|
||||
verified?: boolean;
|
||||
}>;
|
||||
}
|
||||
|
||||
function normalizeDiscussionPayload(payload: DiscussionPayload): Record<string, unknown> {
|
||||
const sourceUrl = cleanText(payload.href) || buildDiscussionUrl(payload.href ?? '');
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const averageRatingText = cleanText(payload.average_rating_text) || null;
|
||||
const totalReviewCountText = cleanText(payload.total_review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
|
||||
return {
|
||||
asin,
|
||||
product_url: asin ? normalizeProductUrl(asin) : null,
|
||||
discussion_url: sourceUrl,
|
||||
...provenance,
|
||||
average_rating_text: averageRatingText,
|
||||
average_rating_value: parseRatingValue(averageRatingText),
|
||||
total_review_count_text: totalReviewCountText,
|
||||
total_review_count: parseReviewCount(totalReviewCountText),
|
||||
qa_urls: uniqueNonEmpty(payload.qa_links ?? []),
|
||||
review_samples: (payload.review_samples ?? []).map((sample) => ({
|
||||
title: trimRatingPrefix(sample.title) || null,
|
||||
rating_text: cleanText(sample.rating_text) || null,
|
||||
rating_value: parseRatingValue(sample.rating_text),
|
||||
author: cleanText(sample.author) || null,
|
||||
date_text: cleanText(sample.date_text) || null,
|
||||
body: cleanText(sample.body) || null,
|
||||
verified_purchase: sample.verified === true,
|
||||
})),
|
||||
};
|
||||
}
|
||||
|
||||
async function readDiscussionPayload(page: IPage, input: string, limit: number): Promise<DiscussionPayload> {
|
||||
const url = buildDiscussionUrl(input);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'discussion');
|
||||
assertUsableState(state, 'discussion');
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
average_rating_text: document.querySelector('[data-hook="rating-out-of-text"]')?.textContent || '',
|
||||
total_review_count_text: document.querySelector('[data-hook="total-review-count"]')?.textContent || '',
|
||||
qa_links: Array.from(document.querySelectorAll('a[href*="ask/questions"]')).map((anchor) => anchor.href || ''),
|
||||
review_samples: Array.from(document.querySelectorAll('[data-hook="review"]')).slice(0, ${limit}).map((card) => ({
|
||||
title: card.querySelector('[data-hook="review-title"]')?.textContent || '',
|
||||
rating_text:
|
||||
card.querySelector('[data-hook="review-star-rating"]')?.textContent
|
||||
|| card.querySelector('[data-hook="cmps-review-star-rating"]')?.textContent
|
||||
|| '',
|
||||
author: card.querySelector('.a-profile-name')?.textContent || '',
|
||||
date_text: card.querySelector('[data-hook="review-date"]')?.textContent || '',
|
||||
body: card.querySelector('[data-hook="review-body"]')?.textContent || '',
|
||||
verified: !!card.querySelector('[data-hook="avp-badge"]'),
|
||||
})),
|
||||
}))()
|
||||
`) as DiscussionPayload;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'discussion',
|
||||
description: 'Amazon review summary and sample customer discussion from product review pages',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 10,
|
||||
help: 'Maximum number of review samples to return (default 10)',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'average_rating_value', 'total_review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 10);
|
||||
const payload = await readDiscussionPayload(page, input, limit);
|
||||
const normalized = normalizeDiscussionPayload(payload);
|
||||
|
||||
if (!normalized.average_rating_text && !normalized.total_review_count_text) {
|
||||
throw new CommandExecutionError(
|
||||
'amazon discussion page did not expose review summary',
|
||||
'The review page may have changed or hit a robot check. Open the review page in Chrome and retry.',
|
||||
);
|
||||
}
|
||||
|
||||
return [normalized];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeDiscussionPayload,
|
||||
};
|
||||
@@ -1,7 +0,0 @@
|
||||
import { cli } from '@jackwener/opencli/registry';
|
||||
import { createRankingCliOptions } from './rankings.js';
|
||||
cli(createRankingCliOptions({
|
||||
commandName: 'movers-shakers',
|
||||
listType: 'movers_shakers',
|
||||
description: 'Amazon Movers & Shakers pages for short-term growth signals',
|
||||
}));
|
||||
@@ -0,0 +1,8 @@
|
||||
import { cli } from '@jackwener/opencli/registry';
|
||||
import { createRankingCliOptions } from './rankings.js';
|
||||
|
||||
cli(createRankingCliOptions({
|
||||
commandName: 'movers-shakers',
|
||||
listType: 'movers_shakers',
|
||||
description: 'Amazon Movers & Shakers pages for short-term growth signals',
|
||||
}));
|
||||
@@ -1,7 +1,8 @@
|
||||
import { cli } from '@jackwener/opencli/registry';
|
||||
import { createRankingCliOptions } from './rankings.js';
|
||||
|
||||
cli(createRankingCliOptions({
|
||||
commandName: 'bestsellers',
|
||||
listType: 'bestsellers',
|
||||
description: 'Amazon Best Sellers pages for category candidate discovery',
|
||||
commandName: 'new-releases',
|
||||
listType: 'new_releases',
|
||||
description: 'Amazon New Releases pages for early momentum discovery',
|
||||
}));
|
||||
@@ -1,140 +0,0 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { buildProductUrl, buildProvenance, cleanText, extractAsin, isAmazonEntity, normalizeProductUrl, PRIMARY_PRICE_SELECTORS, parsePriceText, assertUsableState, gotoAndReadState, } from './shared.js';
|
||||
const OFFER_FACT_SELECTOR = [
|
||||
'#sellerProfileTriggerId',
|
||||
'#shipsFromSoldByInsideBuyBox_feature_div',
|
||||
'#fulfillerInfoFeature_feature_div',
|
||||
'#merchantInfoFeature_feature_div',
|
||||
'#tabular-buybox-container',
|
||||
'#merchant-info',
|
||||
].join(', ');
|
||||
function collapseAdjacentWords(text) {
|
||||
const parts = cleanText(text).split(' ').filter(Boolean);
|
||||
const deduped = [];
|
||||
for (const part of parts) {
|
||||
if (deduped[deduped.length - 1] === part)
|
||||
continue;
|
||||
deduped.push(part);
|
||||
}
|
||||
return deduped.join(' ');
|
||||
}
|
||||
function extractShipsFrom(text) {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/Ships from\s+(.+?)(?=Sold by|and Fulfilled by|$)/i);
|
||||
return match ? collapseAdjacentWords(match[1].replace(/Ships from/ig, '')) : null;
|
||||
}
|
||||
function extractSoldBy(text) {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/Sold by\s+(.+?)(?=and Fulfilled by|Ships from|$)/i);
|
||||
return match ? collapseAdjacentWords(match[1]) : null;
|
||||
}
|
||||
function isDeliveryLocationBlocked(text) {
|
||||
const normalized = cleanText(text).toLowerCase();
|
||||
return normalized.includes('cannot be shipped to your selected delivery location')
|
||||
|| normalized.includes('similar items shipping to')
|
||||
|| normalized.includes('deliver to hong kong');
|
||||
}
|
||||
function normalizeOfferPayload(payload) {
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const sourceUrl = cleanText(payload.href) || buildProductUrl(payload.href ?? '');
|
||||
const price = parsePriceText(payload.price_text);
|
||||
const merchantInfo = cleanText(payload.merchant_info) || null;
|
||||
const soldBy = cleanText(payload.sold_by)
|
||||
|| extractSoldBy(payload.ships_from_text ?? '')
|
||||
|| extractSoldBy(merchantInfo ?? '')
|
||||
|| null;
|
||||
const shipsFrom = extractShipsFrom(payload.ships_from_text ?? '')
|
||||
|| extractShipsFrom(merchantInfo ?? '')
|
||||
|| cleanText(payload.ships_from_text)
|
||||
|| null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
return {
|
||||
asin,
|
||||
product_url: normalizeProductUrl(payload.href),
|
||||
...provenance,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
merchant_info_text: merchantInfo,
|
||||
sold_by: soldBy,
|
||||
ships_from: shipsFrom,
|
||||
offer_listing_url: cleanText(payload.offer_link) || null,
|
||||
review_url: cleanText(payload.review_url) || null,
|
||||
qa_url: cleanText(payload.qa_url) || null,
|
||||
is_amazon_sold: isAmazonEntity(soldBy),
|
||||
is_amazon_fulfilled: isAmazonEntity(shipsFrom) || /fulfilled by amazon/i.test(merchantInfo ?? ''),
|
||||
};
|
||||
}
|
||||
async function readOfferPayload(page, input) {
|
||||
const url = buildProductUrl(input);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'offer');
|
||||
assertUsableState(state, 'offer');
|
||||
// Reconnecting to an existing Amazon target can surface the product page
|
||||
// before the buy-box / merchant blocks are reattached to the DOM.
|
||||
await page.wait({ selector: OFFER_FACT_SELECTOR, timeout: 6 }).catch(() => { });
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
price_text: (() => {
|
||||
const selectors = ${JSON.stringify(PRIMARY_PRICE_SELECTORS)};
|
||||
for (const selector of selectors) {
|
||||
const text = document.querySelector(selector)?.textContent || '';
|
||||
if (text.trim()) return text;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
merchant_info: document.querySelector('#merchant-info')?.textContent || '',
|
||||
sold_by: document.querySelector('#sellerProfileTriggerId')?.textContent || '',
|
||||
ships_from_text:
|
||||
document.querySelector('#shipsFromSoldByInsideBuyBox_feature_div')?.textContent
|
||||
|| document.querySelector('#fulfillerInfoFeature_feature_div')?.textContent
|
||||
|| document.querySelector('#merchantInfoFeature_feature_div')?.textContent
|
||||
|| document.querySelector('#tabular-buybox-container')?.textContent
|
||||
|| '',
|
||||
offer_link: document.querySelector('a[href*="/gp/offer-listing/"]')?.href || '',
|
||||
review_url: document.querySelector('a[href*="#customerReviews"]')?.href || '',
|
||||
qa_url: document.querySelector('a[href*="ask/questions"]')?.href || '',
|
||||
buybox_text:
|
||||
document.querySelector('#desktop_qualifiedBuyBox')?.textContent
|
||||
|| document.querySelector('#buybox')?.textContent
|
||||
|| '',
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'offer',
|
||||
description: 'Amazon seller, buy box, and fulfillment facts from the product page',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'price_text', 'sold_by', 'ships_from', 'is_amazon_sold', 'is_amazon_fulfilled'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const payload = await readOfferPayload(page, input);
|
||||
const normalized = normalizeOfferPayload(payload);
|
||||
if (!normalized.sold_by && !normalized.ships_from && !normalized.merchant_info_text) {
|
||||
if (isDeliveryLocationBlocked(payload.buybox_text)) {
|
||||
throw new CommandExecutionError('amazon offer buy box is blocked by the current delivery location', 'The shared Chrome profile is not set to the target US delivery address. Switch Amazon delivery location to the requested US destination, reopen the product page, and retry.');
|
||||
}
|
||||
throw new CommandExecutionError('amazon offer surface did not expose seller or fulfillment facts', 'The product page may have changed. Open the product page in Chrome, make sure the buy box is visible, and retry.');
|
||||
}
|
||||
return [normalized];
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
extractShipsFrom,
|
||||
extractSoldBy,
|
||||
isDeliveryLocationBlocked,
|
||||
normalizeOfferPayload,
|
||||
};
|
||||
@@ -1,29 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './offer.js';
|
||||
describe('amazon offer normalization', () => {
|
||||
it('extracts sold-by and fulfillment facts from product offer text', () => {
|
||||
const result = __test__.normalizeOfferPayload({
|
||||
href: 'https://www.amazon.com/dp/B0FJS72893',
|
||||
price_text: '$15.99',
|
||||
merchant_info: '',
|
||||
sold_by: 'KUATUDIRECT',
|
||||
ships_from_text: 'Ships from Amazon',
|
||||
offer_link: null,
|
||||
review_url: 'https://www.amazon.com/dp/B0FJS72893#customerReviews',
|
||||
qa_url: null,
|
||||
});
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.sold_by).toBe('KUATUDIRECT');
|
||||
expect(result.ships_from).toBe('Amazon');
|
||||
expect(result.is_amazon_sold).toBe(false);
|
||||
expect(result.is_amazon_fulfilled).toBe(true);
|
||||
});
|
||||
it('parses merchant info fallback text', () => {
|
||||
expect(__test__.extractSoldBy('Sold by Example Seller and Fulfilled by Amazon.')).toBe('Example Seller');
|
||||
expect(__test__.extractShipsFrom('Ships from Amazon')).toBe('Amazon');
|
||||
});
|
||||
it('detects delivery-location blocking in the buy box text', () => {
|
||||
expect(__test__.isDeliveryLocationBlocked('This item cannot be shipped to your selected delivery location. Similar items shipping to Hong Kong')).toBe(true);
|
||||
expect(__test__.isDeliveryLocationBlocked('Ships from Amazon')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,35 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './offer.js';
|
||||
|
||||
describe('amazon offer normalization', () => {
|
||||
it('extracts sold-by and fulfillment facts from product offer text', () => {
|
||||
const result = __test__.normalizeOfferPayload({
|
||||
href: 'https://www.amazon.com/dp/B0FJS72893',
|
||||
price_text: '$15.99',
|
||||
merchant_info: '',
|
||||
sold_by: 'KUATUDIRECT',
|
||||
ships_from_text: 'Ships from Amazon',
|
||||
offer_link: null,
|
||||
review_url: 'https://www.amazon.com/dp/B0FJS72893#customerReviews',
|
||||
qa_url: null,
|
||||
});
|
||||
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.sold_by).toBe('KUATUDIRECT');
|
||||
expect(result.ships_from).toBe('Amazon');
|
||||
expect(result.is_amazon_sold).toBe(false);
|
||||
expect(result.is_amazon_fulfilled).toBe(true);
|
||||
});
|
||||
|
||||
it('parses merchant info fallback text', () => {
|
||||
expect(__test__.extractSoldBy('Sold by Example Seller and Fulfilled by Amazon.')).toBe('Example Seller');
|
||||
expect(__test__.extractShipsFrom('Ships from Amazon')).toBe('Amazon');
|
||||
});
|
||||
|
||||
it('detects delivery-location blocking in the buy box text', () => {
|
||||
expect(__test__.isDeliveryLocationBlocked(
|
||||
'This item cannot be shipped to your selected delivery location. Similar items shipping to Hong Kong',
|
||||
)).toBe(true);
|
||||
expect(__test__.isDeliveryLocationBlocked('Ships from Amazon')).toBe(false);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,185 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
buildProductUrl,
|
||||
buildProvenance,
|
||||
cleanText,
|
||||
extractAsin,
|
||||
isAmazonEntity,
|
||||
normalizeProductUrl,
|
||||
PRIMARY_PRICE_SELECTORS,
|
||||
parsePriceText,
|
||||
assertUsableState,
|
||||
gotoAndReadState,
|
||||
} from './shared.js';
|
||||
|
||||
interface OfferPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
price_text?: string | null;
|
||||
merchant_info?: string | null;
|
||||
sold_by?: string | null;
|
||||
ships_from_text?: string | null;
|
||||
offer_link?: string | null;
|
||||
review_url?: string | null;
|
||||
qa_url?: string | null;
|
||||
buybox_text?: string | null;
|
||||
}
|
||||
|
||||
const OFFER_FACT_SELECTOR = [
|
||||
'#sellerProfileTriggerId',
|
||||
'#shipsFromSoldByInsideBuyBox_feature_div',
|
||||
'#fulfillerInfoFeature_feature_div',
|
||||
'#merchantInfoFeature_feature_div',
|
||||
'#tabular-buybox-container',
|
||||
'#merchant-info',
|
||||
].join(', ');
|
||||
|
||||
function collapseAdjacentWords(text: string): string {
|
||||
const parts = cleanText(text).split(' ').filter(Boolean);
|
||||
const deduped: string[] = [];
|
||||
for (const part of parts) {
|
||||
if (deduped[deduped.length - 1] === part) continue;
|
||||
deduped.push(part);
|
||||
}
|
||||
return deduped.join(' ');
|
||||
}
|
||||
|
||||
function extractShipsFrom(text: string): string | null {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/Ships from\s+(.+?)(?=Sold by|and Fulfilled by|$)/i);
|
||||
return match ? collapseAdjacentWords(match[1].replace(/Ships from/ig, '')) : null;
|
||||
}
|
||||
|
||||
function extractSoldBy(text: string): string | null {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/Sold by\s+(.+?)(?=and Fulfilled by|Ships from|$)/i);
|
||||
return match ? collapseAdjacentWords(match[1]) : null;
|
||||
}
|
||||
|
||||
function isDeliveryLocationBlocked(text: string | null | undefined): boolean {
|
||||
const normalized = cleanText(text).toLowerCase();
|
||||
return normalized.includes('cannot be shipped to your selected delivery location')
|
||||
|| normalized.includes('similar items shipping to')
|
||||
|| normalized.includes('deliver to hong kong');
|
||||
}
|
||||
|
||||
function normalizeOfferPayload(payload: OfferPayload): Record<string, unknown> {
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const sourceUrl = cleanText(payload.href) || buildProductUrl(payload.href ?? '');
|
||||
const price = parsePriceText(payload.price_text);
|
||||
const merchantInfo = cleanText(payload.merchant_info) || null;
|
||||
const soldBy = cleanText(payload.sold_by)
|
||||
|| extractSoldBy(payload.ships_from_text ?? '')
|
||||
|| extractSoldBy(merchantInfo ?? '')
|
||||
|| null;
|
||||
const shipsFrom = extractShipsFrom(payload.ships_from_text ?? '')
|
||||
|| extractShipsFrom(merchantInfo ?? '')
|
||||
|| cleanText(payload.ships_from_text)
|
||||
|| null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
|
||||
return {
|
||||
asin,
|
||||
product_url: normalizeProductUrl(payload.href),
|
||||
...provenance,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
merchant_info_text: merchantInfo,
|
||||
sold_by: soldBy,
|
||||
ships_from: shipsFrom,
|
||||
offer_listing_url: cleanText(payload.offer_link) || null,
|
||||
review_url: cleanText(payload.review_url) || null,
|
||||
qa_url: cleanText(payload.qa_url) || null,
|
||||
is_amazon_sold: isAmazonEntity(soldBy),
|
||||
is_amazon_fulfilled: isAmazonEntity(shipsFrom) || /fulfilled by amazon/i.test(merchantInfo ?? ''),
|
||||
};
|
||||
}
|
||||
|
||||
async function readOfferPayload(page: IPage, input: string): Promise<OfferPayload> {
|
||||
const url = buildProductUrl(input);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'offer');
|
||||
assertUsableState(state, 'offer');
|
||||
|
||||
// Reconnecting to an existing Amazon target can surface the product page
|
||||
// before the buy-box / merchant blocks are reattached to the DOM.
|
||||
await page.wait({ selector: OFFER_FACT_SELECTOR, timeout: 6 }).catch(() => {});
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
price_text: (() => {
|
||||
const selectors = ${JSON.stringify(PRIMARY_PRICE_SELECTORS)};
|
||||
for (const selector of selectors) {
|
||||
const text = document.querySelector(selector)?.textContent || '';
|
||||
if (text.trim()) return text;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
merchant_info: document.querySelector('#merchant-info')?.textContent || '',
|
||||
sold_by: document.querySelector('#sellerProfileTriggerId')?.textContent || '',
|
||||
ships_from_text:
|
||||
document.querySelector('#shipsFromSoldByInsideBuyBox_feature_div')?.textContent
|
||||
|| document.querySelector('#fulfillerInfoFeature_feature_div')?.textContent
|
||||
|| document.querySelector('#merchantInfoFeature_feature_div')?.textContent
|
||||
|| document.querySelector('#tabular-buybox-container')?.textContent
|
||||
|| '',
|
||||
offer_link: document.querySelector('a[href*="/gp/offer-listing/"]')?.href || '',
|
||||
review_url: document.querySelector('a[href*="#customerReviews"]')?.href || '',
|
||||
qa_url: document.querySelector('a[href*="ask/questions"]')?.href || '',
|
||||
buybox_text:
|
||||
document.querySelector('#desktop_qualifiedBuyBox')?.textContent
|
||||
|| document.querySelector('#buybox')?.textContent
|
||||
|| '',
|
||||
}))()
|
||||
`) as OfferPayload;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'offer',
|
||||
description: 'Amazon seller, buy box, and fulfillment facts from the product page',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'price_text', 'sold_by', 'ships_from', 'is_amazon_sold', 'is_amazon_fulfilled'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const payload = await readOfferPayload(page, input);
|
||||
const normalized = normalizeOfferPayload(payload);
|
||||
|
||||
if (!normalized.sold_by && !normalized.ships_from && !normalized.merchant_info_text) {
|
||||
if (isDeliveryLocationBlocked(payload.buybox_text)) {
|
||||
throw new CommandExecutionError(
|
||||
'amazon offer buy box is blocked by the current delivery location',
|
||||
'The shared Chrome profile is not set to the target US delivery address. Switch Amazon delivery location to the requested US destination, reopen the product page, and retry.',
|
||||
);
|
||||
}
|
||||
|
||||
throw new CommandExecutionError(
|
||||
'amazon offer surface did not expose seller or fulfillment facts',
|
||||
'The product page may have changed. Open the product page in Chrome, make sure the buy box is visible, and retry.',
|
||||
);
|
||||
}
|
||||
|
||||
return [normalized];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
extractShipsFrom,
|
||||
extractSoldBy,
|
||||
isDeliveryLocationBlocked,
|
||||
normalizeOfferPayload,
|
||||
};
|
||||
@@ -1,92 +0,0 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { buildProductUrl, buildProvenance, cleanText, extractAsin, PRIMARY_PRICE_SELECTORS, parsePriceText, parseRatingValue, parseReviewCount, normalizeProductUrl, uniqueNonEmpty, assertUsableState, gotoAndReadState, } from './shared.js';
|
||||
const PRODUCT_TITLE_SELECTOR = '#productTitle, #title span, [data-feature-name="title"] h1 span';
|
||||
const BYLINE_SELECTOR = '#bylineInfo, [data-feature-name="bylineInfo"] #bylineInfo';
|
||||
function normalizeProductPayload(payload) {
|
||||
const sourceUrl = cleanText(payload.href) || buildProductUrl(cleanText(payload.product_title) || cleanText(payload.href));
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const price = parsePriceText(payload.price_text);
|
||||
const ratingText = cleanText(payload.rating_text) || null;
|
||||
const reviewCountText = cleanText(payload.review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
return {
|
||||
asin,
|
||||
title: cleanText(payload.product_title) || cleanText(payload.title) || null,
|
||||
product_url: normalizeProductUrl(payload.href),
|
||||
...provenance,
|
||||
brand_text: cleanText(payload.byline) || null,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
review_url: cleanText(payload.review_url) || null,
|
||||
qa_url: cleanText(payload.qa_url) || null,
|
||||
breadcrumbs: uniqueNonEmpty(payload.breadcrumbs ?? []),
|
||||
bullet_points: uniqueNonEmpty(payload.bullets ?? []),
|
||||
};
|
||||
}
|
||||
async function readProductPayload(page, input) {
|
||||
const url = buildProductUrl(input);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'product');
|
||||
assertUsableState(state, 'product');
|
||||
// Amazon can report a "stable" DOM before the product title block hydrates,
|
||||
// especially when reconnecting to an existing shared CDP target.
|
||||
await page.wait({ selector: PRODUCT_TITLE_SELECTOR, timeout: 6 }).catch(() => { });
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
product_title: document.querySelector(${JSON.stringify(PRODUCT_TITLE_SELECTOR)})?.textContent || '',
|
||||
byline: document.querySelector(${JSON.stringify(BYLINE_SELECTOR)})?.textContent || '',
|
||||
price_text: (() => {
|
||||
const selectors = ${JSON.stringify(PRIMARY_PRICE_SELECTORS)};
|
||||
for (const selector of selectors) {
|
||||
const text = document.querySelector(selector)?.textContent || '';
|
||||
if (text.trim()) return text;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
rating_text:
|
||||
document.querySelector('#acrPopover')?.getAttribute('title')
|
||||
|| document.querySelector('#acrPopover')?.textContent
|
||||
|| '',
|
||||
review_count_text: document.querySelector('#acrCustomerReviewText')?.textContent || '',
|
||||
review_url: document.querySelector('a[href*="#customerReviews"]')?.href || '',
|
||||
qa_url: document.querySelector('a[href*="ask/questions"]')?.href || '',
|
||||
bullets: Array.from(document.querySelectorAll('#feature-bullets li .a-list-item')).map((node) => node.textContent || ''),
|
||||
breadcrumbs: Array.from(document.querySelectorAll('#wayfinding-breadcrumbs_feature_div a')).map((node) => node.textContent || ''),
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'product',
|
||||
description: 'Amazon product page facts for candidate validation',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const payload = await readProductPayload(page, input);
|
||||
if (!cleanText(payload.product_title)) {
|
||||
throw new CommandExecutionError('amazon product page did not expose product content', 'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.');
|
||||
}
|
||||
return [normalizeProductPayload(payload)];
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeProductPayload,
|
||||
};
|
||||
@@ -1,24 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './product.js';
|
||||
describe('amazon product normalization', () => {
|
||||
it('normalizes product facts from the product page', () => {
|
||||
const result = __test__.normalizeProductPayload({
|
||||
href: 'https://www.amazon.com/dp/B0FJS72893',
|
||||
title: 'Amazon.com: KVTUKIAIT Desktop Shelf Organizer',
|
||||
product_title: 'White Desktop Shelf Organizer for Top of Desk',
|
||||
byline: 'Visit the KVTUKIAIT Store',
|
||||
price_text: '$15.99',
|
||||
rating_text: '3.9 out of 5 stars',
|
||||
review_count_text: '27 ratings',
|
||||
review_url: 'https://www.amazon.com/dp/B0FJS72893#customerReviews',
|
||||
qa_url: null,
|
||||
bullets: ['SPACE-SAVING DESK SHELF ORGANIZER', 'SMALL AND STYLISH AESTHETIC DECOR'],
|
||||
breadcrumbs: ['Office Products', 'Desktop & Off-Surface Shelves'],
|
||||
});
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.price_value).toBe(15.99);
|
||||
expect(result.rating_value).toBe(3.9);
|
||||
expect(result.review_count).toBe(27);
|
||||
expect(result.breadcrumbs).toEqual(['Office Products', 'Desktop & Off-Surface Shelves']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,26 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './product.js';
|
||||
|
||||
describe('amazon product normalization', () => {
|
||||
it('normalizes product facts from the product page', () => {
|
||||
const result = __test__.normalizeProductPayload({
|
||||
href: 'https://www.amazon.com/dp/B0FJS72893',
|
||||
title: 'Amazon.com: KVTUKIAIT Desktop Shelf Organizer',
|
||||
product_title: 'White Desktop Shelf Organizer for Top of Desk',
|
||||
byline: 'Visit the KVTUKIAIT Store',
|
||||
price_text: '$15.99',
|
||||
rating_text: '3.9 out of 5 stars',
|
||||
review_count_text: '27 ratings',
|
||||
review_url: 'https://www.amazon.com/dp/B0FJS72893#customerReviews',
|
||||
qa_url: null,
|
||||
bullets: ['SPACE-SAVING DESK SHELF ORGANIZER', 'SMALL AND STYLISH AESTHETIC DECOR'],
|
||||
breadcrumbs: ['Office Products', 'Desktop & Off-Surface Shelves'],
|
||||
});
|
||||
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.price_value).toBe(15.99);
|
||||
expect(result.rating_value).toBe(3.9);
|
||||
expect(result.review_count).toBe(27);
|
||||
expect(result.breadcrumbs).toEqual(['Office Products', 'Desktop & Off-Surface Shelves']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,131 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
buildProductUrl,
|
||||
buildProvenance,
|
||||
cleanText,
|
||||
extractAsin,
|
||||
PRIMARY_PRICE_SELECTORS,
|
||||
parsePriceText,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
normalizeProductUrl,
|
||||
uniqueNonEmpty,
|
||||
assertUsableState,
|
||||
gotoAndReadState,
|
||||
} from './shared.js';
|
||||
|
||||
interface ProductPayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
product_title?: string | null;
|
||||
byline?: string | null;
|
||||
price_text?: string | null;
|
||||
rating_text?: string | null;
|
||||
review_count_text?: string | null;
|
||||
review_url?: string | null;
|
||||
qa_url?: string | null;
|
||||
bullets?: string[];
|
||||
breadcrumbs?: string[];
|
||||
}
|
||||
|
||||
const PRODUCT_TITLE_SELECTOR = '#productTitle, #title span, [data-feature-name="title"] h1 span';
|
||||
const BYLINE_SELECTOR = '#bylineInfo, [data-feature-name="bylineInfo"] #bylineInfo';
|
||||
|
||||
function normalizeProductPayload(payload: ProductPayload): Record<string, unknown> {
|
||||
const sourceUrl = cleanText(payload.href) || buildProductUrl(cleanText(payload.product_title) || cleanText(payload.href));
|
||||
const asin = extractAsin(payload.href ?? '') ?? null;
|
||||
const price = parsePriceText(payload.price_text);
|
||||
const ratingText = cleanText(payload.rating_text) || null;
|
||||
const reviewCountText = cleanText(payload.review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
|
||||
return {
|
||||
asin,
|
||||
title: cleanText(payload.product_title) || cleanText(payload.title) || null,
|
||||
product_url: normalizeProductUrl(payload.href),
|
||||
...provenance,
|
||||
brand_text: cleanText(payload.byline) || null,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
review_url: cleanText(payload.review_url) || null,
|
||||
qa_url: cleanText(payload.qa_url) || null,
|
||||
breadcrumbs: uniqueNonEmpty(payload.breadcrumbs ?? []),
|
||||
bullet_points: uniqueNonEmpty(payload.bullets ?? []),
|
||||
};
|
||||
}
|
||||
|
||||
async function readProductPayload(page: IPage, input: string): Promise<ProductPayload> {
|
||||
const url = buildProductUrl(input);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'product');
|
||||
assertUsableState(state, 'product');
|
||||
|
||||
// Amazon can report a "stable" DOM before the product title block hydrates,
|
||||
// especially when reconnecting to an existing shared CDP target.
|
||||
await page.wait({ selector: PRODUCT_TITLE_SELECTOR, timeout: 6 }).catch(() => {});
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
product_title: document.querySelector(${JSON.stringify(PRODUCT_TITLE_SELECTOR)})?.textContent || '',
|
||||
byline: document.querySelector(${JSON.stringify(BYLINE_SELECTOR)})?.textContent || '',
|
||||
price_text: (() => {
|
||||
const selectors = ${JSON.stringify(PRIMARY_PRICE_SELECTORS)};
|
||||
for (const selector of selectors) {
|
||||
const text = document.querySelector(selector)?.textContent || '';
|
||||
if (text.trim()) return text;
|
||||
}
|
||||
return '';
|
||||
})(),
|
||||
rating_text:
|
||||
document.querySelector('#acrPopover')?.getAttribute('title')
|
||||
|| document.querySelector('#acrPopover')?.textContent
|
||||
|| '',
|
||||
review_count_text: document.querySelector('#acrCustomerReviewText')?.textContent || '',
|
||||
review_url: document.querySelector('a[href*="#customerReviews"]')?.href || '',
|
||||
qa_url: document.querySelector('a[href*="ask/questions"]')?.href || '',
|
||||
bullets: Array.from(document.querySelectorAll('#feature-bullets li .a-list-item')).map((node) => node.textContent || ''),
|
||||
breadcrumbs: Array.from(document.querySelectorAll('#wayfinding-breadcrumbs_feature_div a')).map((node) => node.textContent || ''),
|
||||
}))()
|
||||
`) as ProductPayload;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'product',
|
||||
description: 'Amazon product page facts for candidate validation',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'ASIN or product URL, for example B0FJS72893',
|
||||
},
|
||||
],
|
||||
columns: ['asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const input = String(kwargs.input ?? '');
|
||||
const payload = await readProductPayload(page, input);
|
||||
if (!cleanText(payload.product_title)) {
|
||||
throw new CommandExecutionError(
|
||||
'amazon product page did not expose product content',
|
||||
'The product page may have changed or hit a robot check. Open the product page in Chrome and retry.',
|
||||
);
|
||||
}
|
||||
return [normalizeProductPayload(payload)];
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeProductPayload,
|
||||
};
|
||||
@@ -1,226 +0,0 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { Strategy } from '@jackwener/opencli/registry';
|
||||
import { assertUsableState, buildProvenance, cleanText, extractAsin, extractCategoryNodeId, extractReviewCountFromCardText, firstMeaningfulLine, gotoAndReadState, isRankingPaginationUrl, normalizeProductUrl, parsePriceText, parseRatingValue, parseReviewCount, resolveRankingUrl, toAbsoluteAmazonUrl, uniqueNonEmpty, } from './shared.js';
|
||||
function parseRank(rawRank, fallback) {
|
||||
const normalized = cleanText(rawRank);
|
||||
const match = normalized.match(/(\d{1,4})/);
|
||||
if (!match)
|
||||
return fallback;
|
||||
const parsed = Number.parseInt(match[1], 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
function normalizeVisibleCategoryLinks(links) {
|
||||
const normalized = (links ?? [])
|
||||
.map((entry) => ({
|
||||
title: cleanText(entry?.title),
|
||||
url: toAbsoluteAmazonUrl(entry?.url) ?? '',
|
||||
node_id: cleanText(entry?.node_id) || extractCategoryNodeId(entry?.url) || null,
|
||||
}))
|
||||
.filter((entry) => Boolean(entry.title) && Boolean(entry.url));
|
||||
const seen = new Set();
|
||||
const deduped = [];
|
||||
for (const entry of normalized) {
|
||||
if (seen.has(entry.url))
|
||||
continue;
|
||||
seen.add(entry.url);
|
||||
deduped.push(entry);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
export function normalizeRankingCandidate(candidate, context) {
|
||||
const productUrl = normalizeProductUrl(candidate.href);
|
||||
const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
|
||||
const title = cleanText(candidate.title) || firstMeaningfulLine(candidate.card_text);
|
||||
const price = parsePriceText(cleanText(candidate.price_text) || candidate.card_text);
|
||||
const ratingText = cleanText(candidate.rating_text) || null;
|
||||
const reviewCountText = cleanText(candidate.review_count_text)
|
||||
|| extractReviewCountFromCardText(candidate.card_text)
|
||||
|| null;
|
||||
const provenance = buildProvenance(context.sourceUrl);
|
||||
const categoryUrl = context.categoryUrl || context.sourceUrl;
|
||||
return {
|
||||
list_type: context.listType,
|
||||
rank: parseRank(candidate.rank_text, context.rankFallback),
|
||||
asin,
|
||||
title: title || null,
|
||||
product_url: productUrl,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
list_title: context.listTitle,
|
||||
category_title: context.categoryTitle,
|
||||
category_url: categoryUrl,
|
||||
category_node_id: extractCategoryNodeId(categoryUrl),
|
||||
category_path: context.categoryPath,
|
||||
visible_category_links: context.visibleCategoryLinks,
|
||||
...provenance,
|
||||
};
|
||||
}
|
||||
async function readRankingPage(page, listType, url) {
|
||||
const state = await gotoAndReadState(page, url, 2500, listType);
|
||||
assertUsableState(state, listType);
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
list_title:
|
||||
document.querySelector('#zg_banner_text')?.textContent
|
||||
|| document.querySelector('h1')?.textContent
|
||||
|| '',
|
||||
category_title:
|
||||
document.querySelector('#zg_browseRoot .zg_selected')?.textContent
|
||||
|| document.querySelector('#wayfinding-breadcrumbs_feature_div ul li:last-child')?.textContent
|
||||
|| document.querySelector('#wayfinding-breadcrumbs_container ul li:last-child')?.textContent
|
||||
|| '',
|
||||
category_path: Array.from(document.querySelectorAll(
|
||||
'#zg_browseRoot ul li a, #zg_browseRoot ul li span, ' +
|
||||
'#wayfinding-breadcrumbs_feature_div ul li a, #wayfinding-breadcrumbs_feature_div ul li span.a-list-item, ' +
|
||||
'#wayfinding-breadcrumbs_container ul li a, #wayfinding-breadcrumbs_container ul li span.a-list-item'
|
||||
))
|
||||
.map((entry) => (entry.textContent || '').trim())
|
||||
.filter(Boolean),
|
||||
cards: Array.from(document.querySelectorAll(
|
||||
'.p13n-sc-uncoverable-faceout, .zg-grid-general-faceout, [data-asin][class*="p13n"]'
|
||||
)).map((card) => ({
|
||||
rank_text:
|
||||
card.querySelector('.zg-bdg-text')?.textContent
|
||||
|| card.querySelector('[class*="rank"]')?.textContent
|
||||
|| '',
|
||||
asin:
|
||||
card.getAttribute('data-asin')
|
||||
|| card.getAttribute('id')
|
||||
|| '',
|
||||
title:
|
||||
card.querySelector('[class*="line-clamp"]')?.textContent
|
||||
|| card.querySelector('img')?.getAttribute('alt')
|
||||
|| card.querySelector('a[href*="/dp/"]')?.textContent
|
||||
|| '',
|
||||
href:
|
||||
card.querySelector('a[href*="/dp/"], a[href*="/gp/product/"]')?.href
|
||||
|| '',
|
||||
price_text:
|
||||
card.querySelector('.a-price .a-offscreen')?.textContent
|
||||
|| card.querySelector('.a-color-price')?.textContent
|
||||
|| '',
|
||||
rating_text:
|
||||
card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label')
|
||||
|| '',
|
||||
review_count_text:
|
||||
card.querySelector('a[href*="#customerReviews"]')?.textContent
|
||||
|| card.querySelector('.a-size-small')?.textContent
|
||||
|| '',
|
||||
card_text: card.innerText || '',
|
||||
})),
|
||||
page_links: Array.from(document.querySelectorAll('.a-pagination a[href], li.a-normal a[href], li.a-selected a[href]'))
|
||||
.map((anchor) => anchor.href || '')
|
||||
.filter(Boolean),
|
||||
visible_category_links: Array.from(document.querySelectorAll(
|
||||
'#zg_browseRoot a[href], #zg-left-col a[href], [class*="zg-browse"] a[href]'
|
||||
)).map((anchor) => ({
|
||||
title: (anchor.textContent || '').trim(),
|
||||
url: anchor.href || '',
|
||||
node_id:
|
||||
anchor.getAttribute('data-node-id')
|
||||
|| anchor.dataset?.nodeid
|
||||
|| '',
|
||||
}))
|
||||
.filter((entry) => entry.title && entry.url),
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
function createEmptyResultHint(commandName) {
|
||||
return [
|
||||
`Open the same Amazon ${commandName} page in shared Chrome and verify ranked items are visible.`,
|
||||
'If the page shows a robot check, clear it manually and retry.',
|
||||
].join(' ');
|
||||
}
|
||||
export function createRankingCliOptions(definition) {
|
||||
return {
|
||||
site: 'amazon',
|
||||
name: definition.commandName,
|
||||
description: definition.description,
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
positional: true,
|
||||
help: 'Ranking URL or supported Amazon path. Omit to use the list root.',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 100,
|
||||
help: 'Maximum number of ranked items to return (default 100)',
|
||||
},
|
||||
],
|
||||
columns: ['list_type', 'rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 100);
|
||||
const initialUrl = resolveRankingUrl(definition.listType, typeof kwargs.input === 'string' ? kwargs.input : undefined);
|
||||
const queue = [initialUrl];
|
||||
const visited = new Set();
|
||||
const seenEntityKeys = new Set();
|
||||
const results = [];
|
||||
let listTitle = null;
|
||||
while (queue.length > 0 && results.length < limit) {
|
||||
const nextUrl = queue.shift();
|
||||
if (visited.has(nextUrl))
|
||||
continue;
|
||||
visited.add(nextUrl);
|
||||
const payload = await readRankingPage(page, definition.listType, nextUrl);
|
||||
const sourceUrl = cleanText(payload.href) || nextUrl;
|
||||
listTitle = cleanText(payload.list_title) || cleanText(payload.title) || listTitle;
|
||||
const categoryPath = uniqueNonEmpty(payload.category_path ?? []);
|
||||
const categoryTitle = cleanText(payload.category_title)
|
||||
|| (categoryPath.length > 0 ? categoryPath[categoryPath.length - 1] : '');
|
||||
const visibleCategoryLinks = normalizeVisibleCategoryLinks(payload.visible_category_links);
|
||||
const cards = payload.cards ?? [];
|
||||
for (const card of cards) {
|
||||
const normalized = normalizeRankingCandidate(card, {
|
||||
listType: definition.listType,
|
||||
rankFallback: results.length + 1,
|
||||
listTitle,
|
||||
sourceUrl,
|
||||
categoryTitle: categoryTitle || null,
|
||||
categoryUrl: sourceUrl,
|
||||
categoryPath,
|
||||
visibleCategoryLinks,
|
||||
});
|
||||
const dedupeKey = cleanText(String(normalized.asin ?? ''))
|
||||
|| cleanText(String(normalized.product_url ?? ''));
|
||||
if (dedupeKey && seenEntityKeys.has(dedupeKey))
|
||||
continue;
|
||||
if (dedupeKey)
|
||||
seenEntityKeys.add(dedupeKey);
|
||||
results.push(normalized);
|
||||
if (results.length >= limit)
|
||||
break;
|
||||
}
|
||||
const pageLinks = uniqueNonEmpty(payload.page_links ?? []);
|
||||
for (const href of pageLinks) {
|
||||
const absolute = toAbsoluteAmazonUrl(href);
|
||||
if (!absolute || !isRankingPaginationUrl(definition.listType, absolute))
|
||||
continue;
|
||||
if (!visited.has(absolute) && !queue.includes(absolute)) {
|
||||
queue.push(absolute);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (results.length === 0) {
|
||||
throw new CommandExecutionError(`amazon ${definition.commandName} did not expose any ranked items`, createEmptyResultHint(definition.commandName));
|
||||
}
|
||||
return results.slice(0, limit);
|
||||
},
|
||||
};
|
||||
}
|
||||
export const __test__ = {
|
||||
parseRank,
|
||||
normalizeVisibleCategoryLinks,
|
||||
normalizeRankingCandidate,
|
||||
};
|
||||
@@ -1,41 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './rankings.js';
|
||||
describe('amazon rankings helpers', () => {
|
||||
it('normalizes ranking candidates with unified schema', () => {
|
||||
const result = __test__.normalizeRankingCandidate({
|
||||
rank_text: '#3',
|
||||
asin: 'B0DR31GC3D',
|
||||
title: 'Desk Shelves Desktop Organizer',
|
||||
href: 'https://www.amazon.com/dp/B0DR31GC3D/ref=zg_bs',
|
||||
price_text: '$25.92',
|
||||
rating_text: '4.3 out of 5 stars',
|
||||
review_count_text: '435',
|
||||
}, {
|
||||
listType: 'new_releases',
|
||||
rankFallback: 3,
|
||||
listTitle: 'Amazon New Releases',
|
||||
sourceUrl: 'https://www.amazon.com/gp/new-releases',
|
||||
categoryTitle: 'Home & Kitchen',
|
||||
categoryUrl: 'https://www.amazon.com/gp/new-releases/home-garden',
|
||||
categoryPath: ['Home & Kitchen'],
|
||||
visibleCategoryLinks: [{ title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null }],
|
||||
});
|
||||
expect(result.list_type).toBe('new_releases');
|
||||
expect(result.rank).toBe(3);
|
||||
expect(result.asin).toBe('B0DR31GC3D');
|
||||
expect(result.product_url).toBe('https://www.amazon.com/dp/B0DR31GC3D');
|
||||
expect(result.category_title).toBe('Home & Kitchen');
|
||||
expect(result.visible_category_links).toEqual([
|
||||
{ title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null },
|
||||
]);
|
||||
});
|
||||
it('deduplicates category links and parses rank fallback', () => {
|
||||
const links = __test__.normalizeVisibleCategoryLinks([
|
||||
{ title: 'Kitchen', url: '/gp/new-releases/home-garden' },
|
||||
{ title: 'Kitchen', url: 'https://www.amazon.com/gp/new-releases/home-garden' },
|
||||
{ title: 'Storage', url: '/gp/new-releases/storage', node_id: '1064954' },
|
||||
]);
|
||||
expect(links.length).toBe(2);
|
||||
expect(__test__.parseRank('N/A', 8)).toBe(8);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,47 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './rankings.js';
|
||||
|
||||
describe('amazon rankings helpers', () => {
|
||||
it('normalizes ranking candidates with unified schema', () => {
|
||||
const result = __test__.normalizeRankingCandidate(
|
||||
{
|
||||
rank_text: '#3',
|
||||
asin: 'B0DR31GC3D',
|
||||
title: 'Desk Shelves Desktop Organizer',
|
||||
href: 'https://www.amazon.com/dp/B0DR31GC3D/ref=zg_bs',
|
||||
price_text: '$25.92',
|
||||
rating_text: '4.3 out of 5 stars',
|
||||
review_count_text: '435',
|
||||
},
|
||||
{
|
||||
listType: 'new_releases',
|
||||
rankFallback: 3,
|
||||
listTitle: 'Amazon New Releases',
|
||||
sourceUrl: 'https://www.amazon.com/gp/new-releases',
|
||||
categoryTitle: 'Home & Kitchen',
|
||||
categoryUrl: 'https://www.amazon.com/gp/new-releases/home-garden',
|
||||
categoryPath: ['Home & Kitchen'],
|
||||
visibleCategoryLinks: [{ title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null }],
|
||||
},
|
||||
);
|
||||
|
||||
expect(result.list_type).toBe('new_releases');
|
||||
expect(result.rank).toBe(3);
|
||||
expect(result.asin).toBe('B0DR31GC3D');
|
||||
expect(result.product_url).toBe('https://www.amazon.com/dp/B0DR31GC3D');
|
||||
expect(result.category_title).toBe('Home & Kitchen');
|
||||
expect(result.visible_category_links).toEqual([
|
||||
{ title: 'Storage', url: 'https://www.amazon.com/gp/new-releases/storage', node_id: null },
|
||||
]);
|
||||
});
|
||||
|
||||
it('deduplicates category links and parses rank fallback', () => {
|
||||
const links = __test__.normalizeVisibleCategoryLinks([
|
||||
{ title: 'Kitchen', url: '/gp/new-releases/home-garden' },
|
||||
{ title: 'Kitchen', url: 'https://www.amazon.com/gp/new-releases/home-garden' },
|
||||
{ title: 'Storage', url: '/gp/new-releases/storage', node_id: '1064954' },
|
||||
]);
|
||||
expect(links.length).toBe(2);
|
||||
expect(__test__.parseRank('N/A', 8)).toBe(8);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,312 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { Strategy, type CliOptions } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
assertUsableState,
|
||||
buildProvenance,
|
||||
cleanText,
|
||||
extractAsin,
|
||||
extractCategoryNodeId,
|
||||
extractReviewCountFromCardText,
|
||||
firstMeaningfulLine,
|
||||
gotoAndReadState,
|
||||
isRankingPaginationUrl,
|
||||
normalizeProductUrl,
|
||||
parsePriceText,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
resolveRankingUrl,
|
||||
toAbsoluteAmazonUrl,
|
||||
uniqueNonEmpty,
|
||||
type AmazonRankingListType,
|
||||
} from './shared.js';
|
||||
|
||||
export interface RankingCardPayload {
|
||||
rank_text?: string | null;
|
||||
asin?: string | null;
|
||||
title?: string | null;
|
||||
href?: string | null;
|
||||
price_text?: string | null;
|
||||
rating_text?: string | null;
|
||||
review_count_text?: string | null;
|
||||
card_text?: string | null;
|
||||
}
|
||||
|
||||
interface RankingPagePayload {
|
||||
href?: string;
|
||||
title?: string;
|
||||
list_title?: string;
|
||||
category_title?: string;
|
||||
category_path?: string[];
|
||||
cards?: RankingCardPayload[];
|
||||
page_links?: string[];
|
||||
visible_category_links?: Array<{
|
||||
title?: string | null;
|
||||
url?: string | null;
|
||||
node_id?: string | null;
|
||||
}>;
|
||||
}
|
||||
|
||||
interface RankingCommandDefinition {
|
||||
commandName: string;
|
||||
listType: AmazonRankingListType;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface RankingNormalizeContext {
|
||||
listType: AmazonRankingListType;
|
||||
rankFallback: number;
|
||||
listTitle: string | null;
|
||||
sourceUrl: string;
|
||||
categoryTitle: string | null;
|
||||
categoryUrl: string | null;
|
||||
categoryPath: string[];
|
||||
visibleCategoryLinks: Array<{ title: string; url: string; node_id: string | null }>;
|
||||
}
|
||||
|
||||
function parseRank(rawRank: string | null | undefined, fallback: number): number {
|
||||
const normalized = cleanText(rawRank);
|
||||
const match = normalized.match(/(\d{1,4})/);
|
||||
if (!match) return fallback;
|
||||
const parsed = Number.parseInt(match[1], 10);
|
||||
return Number.isFinite(parsed) && parsed > 0 ? parsed : fallback;
|
||||
}
|
||||
|
||||
function normalizeVisibleCategoryLinks(
|
||||
links: RankingPagePayload['visible_category_links'],
|
||||
): Array<{ title: string; url: string; node_id: string | null }> {
|
||||
const normalized = (links ?? [])
|
||||
.map((entry) => ({
|
||||
title: cleanText(entry?.title),
|
||||
url: toAbsoluteAmazonUrl(entry?.url) ?? '',
|
||||
node_id: cleanText(entry?.node_id) || extractCategoryNodeId(entry?.url) || null,
|
||||
}))
|
||||
.filter((entry) => Boolean(entry.title) && Boolean(entry.url));
|
||||
|
||||
const seen = new Set<string>();
|
||||
const deduped: Array<{ title: string; url: string; node_id: string | null }> = [];
|
||||
for (const entry of normalized) {
|
||||
if (seen.has(entry.url)) continue;
|
||||
seen.add(entry.url);
|
||||
deduped.push(entry);
|
||||
}
|
||||
return deduped;
|
||||
}
|
||||
|
||||
export function normalizeRankingCandidate(
|
||||
candidate: RankingCardPayload,
|
||||
context: RankingNormalizeContext,
|
||||
): Record<string, unknown> {
|
||||
const productUrl = normalizeProductUrl(candidate.href);
|
||||
const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
|
||||
const title = cleanText(candidate.title) || firstMeaningfulLine(candidate.card_text);
|
||||
const price = parsePriceText(cleanText(candidate.price_text) || candidate.card_text);
|
||||
const ratingText = cleanText(candidate.rating_text) || null;
|
||||
const reviewCountText = cleanText(candidate.review_count_text)
|
||||
|| extractReviewCountFromCardText(candidate.card_text)
|
||||
|| null;
|
||||
const provenance = buildProvenance(context.sourceUrl);
|
||||
const categoryUrl = context.categoryUrl || context.sourceUrl;
|
||||
|
||||
return {
|
||||
list_type: context.listType,
|
||||
rank: parseRank(candidate.rank_text, context.rankFallback),
|
||||
asin,
|
||||
title: title || null,
|
||||
product_url: productUrl,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
list_title: context.listTitle,
|
||||
category_title: context.categoryTitle,
|
||||
category_url: categoryUrl,
|
||||
category_node_id: extractCategoryNodeId(categoryUrl),
|
||||
category_path: context.categoryPath,
|
||||
visible_category_links: context.visibleCategoryLinks,
|
||||
...provenance,
|
||||
};
|
||||
}
|
||||
|
||||
async function readRankingPage(
|
||||
page: IPage,
|
||||
listType: AmazonRankingListType,
|
||||
url: string,
|
||||
): Promise<RankingPagePayload> {
|
||||
const state = await gotoAndReadState(page, url, 2500, listType);
|
||||
assertUsableState(state, listType);
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
list_title:
|
||||
document.querySelector('#zg_banner_text')?.textContent
|
||||
|| document.querySelector('h1')?.textContent
|
||||
|| '',
|
||||
category_title:
|
||||
document.querySelector('#zg_browseRoot .zg_selected')?.textContent
|
||||
|| document.querySelector('#wayfinding-breadcrumbs_feature_div ul li:last-child')?.textContent
|
||||
|| document.querySelector('#wayfinding-breadcrumbs_container ul li:last-child')?.textContent
|
||||
|| '',
|
||||
category_path: Array.from(document.querySelectorAll(
|
||||
'#zg_browseRoot ul li a, #zg_browseRoot ul li span, ' +
|
||||
'#wayfinding-breadcrumbs_feature_div ul li a, #wayfinding-breadcrumbs_feature_div ul li span.a-list-item, ' +
|
||||
'#wayfinding-breadcrumbs_container ul li a, #wayfinding-breadcrumbs_container ul li span.a-list-item'
|
||||
))
|
||||
.map((entry) => (entry.textContent || '').trim())
|
||||
.filter(Boolean),
|
||||
cards: Array.from(document.querySelectorAll(
|
||||
'.p13n-sc-uncoverable-faceout, .zg-grid-general-faceout, [data-asin][class*="p13n"]'
|
||||
)).map((card) => ({
|
||||
rank_text:
|
||||
card.querySelector('.zg-bdg-text')?.textContent
|
||||
|| card.querySelector('[class*="rank"]')?.textContent
|
||||
|| '',
|
||||
asin:
|
||||
card.getAttribute('data-asin')
|
||||
|| card.getAttribute('id')
|
||||
|| '',
|
||||
title:
|
||||
card.querySelector('[class*="line-clamp"]')?.textContent
|
||||
|| card.querySelector('img')?.getAttribute('alt')
|
||||
|| card.querySelector('a[href*="/dp/"]')?.textContent
|
||||
|| '',
|
||||
href:
|
||||
card.querySelector('a[href*="/dp/"], a[href*="/gp/product/"]')?.href
|
||||
|| '',
|
||||
price_text:
|
||||
card.querySelector('.a-price .a-offscreen')?.textContent
|
||||
|| card.querySelector('.a-color-price')?.textContent
|
||||
|| '',
|
||||
rating_text:
|
||||
card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label')
|
||||
|| '',
|
||||
review_count_text:
|
||||
card.querySelector('a[href*="#customerReviews"]')?.textContent
|
||||
|| card.querySelector('.a-size-small')?.textContent
|
||||
|| '',
|
||||
card_text: card.innerText || '',
|
||||
})),
|
||||
page_links: Array.from(document.querySelectorAll('.a-pagination a[href], li.a-normal a[href], li.a-selected a[href]'))
|
||||
.map((anchor) => anchor.href || '')
|
||||
.filter(Boolean),
|
||||
visible_category_links: Array.from(document.querySelectorAll(
|
||||
'#zg_browseRoot a[href], #zg-left-col a[href], [class*="zg-browse"] a[href]'
|
||||
)).map((anchor) => ({
|
||||
title: (anchor.textContent || '').trim(),
|
||||
url: anchor.href || '',
|
||||
node_id:
|
||||
anchor.getAttribute('data-node-id')
|
||||
|| anchor.dataset?.nodeid
|
||||
|| '',
|
||||
}))
|
||||
.filter((entry) => entry.title && entry.url),
|
||||
}))()
|
||||
`) as RankingPagePayload;
|
||||
}
|
||||
|
||||
function createEmptyResultHint(commandName: string): string {
|
||||
return [
|
||||
`Open the same Amazon ${commandName} page in shared Chrome and verify ranked items are visible.`,
|
||||
'If the page shows a robot check, clear it manually and retry.',
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
export function createRankingCliOptions(definition: RankingCommandDefinition): CliOptions {
|
||||
return {
|
||||
site: 'amazon',
|
||||
name: definition.commandName,
|
||||
description: definition.description,
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'input',
|
||||
positional: true,
|
||||
help: 'Ranking URL or supported Amazon path. Omit to use the list root.',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 100,
|
||||
help: 'Maximum number of ranked items to return (default 100)',
|
||||
},
|
||||
],
|
||||
columns: ['list_type', 'rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 100);
|
||||
const initialUrl = resolveRankingUrl(definition.listType, typeof kwargs.input === 'string' ? kwargs.input : undefined);
|
||||
|
||||
const queue = [initialUrl];
|
||||
const visited = new Set<string>();
|
||||
const seenEntityKeys = new Set<string>();
|
||||
const results: Record<string, unknown>[] = [];
|
||||
let listTitle: string | null = null;
|
||||
|
||||
while (queue.length > 0 && results.length < limit) {
|
||||
const nextUrl = queue.shift()!;
|
||||
if (visited.has(nextUrl)) continue;
|
||||
visited.add(nextUrl);
|
||||
|
||||
const payload = await readRankingPage(page, definition.listType, nextUrl);
|
||||
const sourceUrl = cleanText(payload.href) || nextUrl;
|
||||
listTitle = cleanText(payload.list_title) || cleanText(payload.title) || listTitle;
|
||||
const categoryPath = uniqueNonEmpty(payload.category_path ?? []);
|
||||
const categoryTitle = cleanText(payload.category_title)
|
||||
|| (categoryPath.length > 0 ? categoryPath[categoryPath.length - 1] : '');
|
||||
const visibleCategoryLinks = normalizeVisibleCategoryLinks(payload.visible_category_links);
|
||||
const cards = payload.cards ?? [];
|
||||
|
||||
for (const card of cards) {
|
||||
const normalized = normalizeRankingCandidate(card, {
|
||||
listType: definition.listType,
|
||||
rankFallback: results.length + 1,
|
||||
listTitle,
|
||||
sourceUrl,
|
||||
categoryTitle: categoryTitle || null,
|
||||
categoryUrl: sourceUrl,
|
||||
categoryPath,
|
||||
visibleCategoryLinks,
|
||||
});
|
||||
|
||||
const dedupeKey = cleanText(String(normalized.asin ?? ''))
|
||||
|| cleanText(String(normalized.product_url ?? ''));
|
||||
if (dedupeKey && seenEntityKeys.has(dedupeKey)) continue;
|
||||
if (dedupeKey) seenEntityKeys.add(dedupeKey);
|
||||
|
||||
results.push(normalized);
|
||||
if (results.length >= limit) break;
|
||||
}
|
||||
|
||||
const pageLinks = uniqueNonEmpty(payload.page_links ?? []);
|
||||
for (const href of pageLinks) {
|
||||
const absolute = toAbsoluteAmazonUrl(href);
|
||||
if (!absolute || !isRankingPaginationUrl(definition.listType, absolute)) continue;
|
||||
if (!visited.has(absolute) && !queue.includes(absolute)) {
|
||||
queue.push(absolute);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (results.length === 0) {
|
||||
throw new CommandExecutionError(
|
||||
`amazon ${definition.commandName} did not expose any ranked items`,
|
||||
createEmptyResultHint(definition.commandName),
|
||||
);
|
||||
}
|
||||
|
||||
return results.slice(0, limit);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
parseRank,
|
||||
normalizeVisibleCategoryLinks,
|
||||
normalizeRankingCandidate,
|
||||
};
|
||||
@@ -1,87 +0,0 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import { buildProvenance, buildSearchUrl, cleanText, extractAsin, normalizeProductUrl, parsePriceText, parseRatingValue, parseReviewCount, assertUsableState, gotoAndReadState, } from './shared.js';
|
||||
function normalizeSearchCandidate(candidate, rank, sourceUrl) {
|
||||
const productUrl = normalizeProductUrl(candidate.href);
|
||||
const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
|
||||
const price = parsePriceText(candidate.price_text);
|
||||
const ratingText = cleanText(candidate.rating_text) || null;
|
||||
const reviewCountText = cleanText(candidate.review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
return {
|
||||
rank,
|
||||
asin,
|
||||
title: cleanText(candidate.title) || null,
|
||||
product_url: productUrl,
|
||||
...provenance,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
is_sponsored: candidate.sponsored === true,
|
||||
badges: (candidate.badge_texts ?? []).map((value) => cleanText(value)).filter(Boolean),
|
||||
};
|
||||
}
|
||||
async function readSearchPayload(page, query) {
|
||||
const url = buildSearchUrl(query);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'search');
|
||||
assertUsableState(state, 'search');
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
cards: Array.from(document.querySelectorAll('[data-component-type="s-search-result"]'))
|
||||
.map((card) => ({
|
||||
asin: card.getAttribute('data-asin') || '',
|
||||
title: card.querySelector('h2')?.textContent || '',
|
||||
href: card.querySelector('a.a-link-normal[href*="/dp/"]')?.href || '',
|
||||
price_text: card.querySelector('.a-price .a-offscreen')?.textContent || '',
|
||||
rating_text: card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label') || '',
|
||||
review_count_text: card.querySelector('a[href*="#customerReviews"]')?.textContent || '',
|
||||
sponsored: /sponsored/i.test(card.innerText || ''),
|
||||
badge_texts: Array.from(card.querySelectorAll('.a-badge-text')).map((node) => node.textContent || ''),
|
||||
})),
|
||||
}))()
|
||||
`);
|
||||
}
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'search',
|
||||
description: 'Amazon search results for product discovery and coarse filtering',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'query',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'Search query, for example "desk shelf organizer"',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 20,
|
||||
help: 'Maximum number of results to return (default 20)',
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query ?? '');
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 20);
|
||||
const payload = await readSearchPayload(page, query);
|
||||
const sourceUrl = cleanText(payload.href) || buildSearchUrl(query);
|
||||
const cards = (payload.cards ?? [])
|
||||
.filter((card) => cleanText(card.asin) && cleanText(card.title))
|
||||
.slice(0, limit);
|
||||
if (cards.length === 0) {
|
||||
throw new CommandExecutionError('amazon search did not expose any product cards', 'The search page may have changed or hit a robot check. Open the same query in Chrome, verify the page is visible, and retry.');
|
||||
}
|
||||
return cards.map((card, index) => normalizeSearchCandidate(card, index + 1, sourceUrl));
|
||||
},
|
||||
});
|
||||
export const __test__ = {
|
||||
normalizeSearchCandidate,
|
||||
};
|
||||
@@ -1,22 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './search.js';
|
||||
describe('amazon search normalization', () => {
|
||||
it('normalizes search cards into research-friendly fields', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
asin: 'B0FJS72893',
|
||||
title: 'White Desktop Shelf Organizer for Top of Desk',
|
||||
href: 'https://www.amazon.com/KVTUKIAIT-White-Desktop-Shelf-Organizer/dp/B0FJS72893/ref=sr_1_1',
|
||||
price_text: '$15.99',
|
||||
rating_text: '3.9 out of 5 stars, rating details',
|
||||
review_count_text: '(27)',
|
||||
sponsored: false,
|
||||
badge_texts: ['Limited time deal'],
|
||||
}, 1, 'https://www.amazon.com/s?k=desk+shelf+organizer');
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.product_url).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(result.price_value).toBe(15.99);
|
||||
expect(result.rating_value).toBe(3.9);
|
||||
expect(result.review_count).toBe(27);
|
||||
expect(result.badges).toEqual(['Limited time deal']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,24 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './search.js';
|
||||
|
||||
describe('amazon search normalization', () => {
|
||||
it('normalizes search cards into research-friendly fields', () => {
|
||||
const result = __test__.normalizeSearchCandidate({
|
||||
asin: 'B0FJS72893',
|
||||
title: 'White Desktop Shelf Organizer for Top of Desk',
|
||||
href: 'https://www.amazon.com/KVTUKIAIT-White-Desktop-Shelf-Organizer/dp/B0FJS72893/ref=sr_1_1',
|
||||
price_text: '$15.99',
|
||||
rating_text: '3.9 out of 5 stars, rating details',
|
||||
review_count_text: '(27)',
|
||||
sponsored: false,
|
||||
badge_texts: ['Limited time deal'],
|
||||
}, 1, 'https://www.amazon.com/s?k=desk+shelf+organizer');
|
||||
|
||||
expect(result.asin).toBe('B0FJS72893');
|
||||
expect(result.product_url).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(result.price_value).toBe(15.99);
|
||||
expect(result.rating_value).toBe(3.9);
|
||||
expect(result.review_count).toBe(27);
|
||||
expect(result.badges).toEqual(['Limited time deal']);
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,128 @@
|
||||
import { CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import {
|
||||
buildProvenance,
|
||||
buildSearchUrl,
|
||||
cleanText,
|
||||
extractAsin,
|
||||
normalizeProductUrl,
|
||||
parsePriceText,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
assertUsableState,
|
||||
gotoAndReadState,
|
||||
} from './shared.js';
|
||||
|
||||
interface SearchPayload {
|
||||
href?: string;
|
||||
cards?: Array<{
|
||||
asin?: string;
|
||||
title?: string;
|
||||
href?: string;
|
||||
price_text?: string | null;
|
||||
rating_text?: string | null;
|
||||
review_count_text?: string | null;
|
||||
sponsored?: boolean;
|
||||
badge_texts?: string[];
|
||||
}>;
|
||||
}
|
||||
|
||||
function normalizeSearchCandidate(
|
||||
candidate: NonNullable<SearchPayload['cards']>[number],
|
||||
rank: number,
|
||||
sourceUrl: string,
|
||||
): Record<string, unknown> {
|
||||
const productUrl = normalizeProductUrl(candidate.href);
|
||||
const asin = extractAsin(candidate.asin ?? '') ?? extractAsin(productUrl ?? '') ?? null;
|
||||
const price = parsePriceText(candidate.price_text);
|
||||
const ratingText = cleanText(candidate.rating_text) || null;
|
||||
const reviewCountText = cleanText(candidate.review_count_text) || null;
|
||||
const provenance = buildProvenance(sourceUrl);
|
||||
|
||||
return {
|
||||
rank,
|
||||
asin,
|
||||
title: cleanText(candidate.title) || null,
|
||||
product_url: productUrl,
|
||||
...provenance,
|
||||
price_text: price.price_text,
|
||||
price_value: price.price_value,
|
||||
currency: price.currency,
|
||||
rating_text: ratingText,
|
||||
rating_value: parseRatingValue(ratingText),
|
||||
review_count_text: reviewCountText,
|
||||
review_count: parseReviewCount(reviewCountText),
|
||||
is_sponsored: candidate.sponsored === true,
|
||||
badges: (candidate.badge_texts ?? []).map((value) => cleanText(value)).filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
async function readSearchPayload(page: IPage, query: string): Promise<SearchPayload> {
|
||||
const url = buildSearchUrl(query);
|
||||
const state = await gotoAndReadState(page, url, 2500, 'search');
|
||||
assertUsableState(state, 'search');
|
||||
|
||||
return await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
cards: Array.from(document.querySelectorAll('[data-component-type="s-search-result"]'))
|
||||
.map((card) => ({
|
||||
asin: card.getAttribute('data-asin') || '',
|
||||
title: card.querySelector('h2')?.textContent || '',
|
||||
href: card.querySelector('a.a-link-normal[href*="/dp/"]')?.href || '',
|
||||
price_text: card.querySelector('.a-price .a-offscreen')?.textContent || '',
|
||||
rating_text: card.querySelector('[aria-label*="out of 5 stars"]')?.getAttribute('aria-label') || '',
|
||||
review_count_text: card.querySelector('a[href*="#customerReviews"]')?.textContent || '',
|
||||
sponsored: /sponsored/i.test(card.innerText || ''),
|
||||
badge_texts: Array.from(card.querySelectorAll('.a-badge-text')).map((node) => node.textContent || ''),
|
||||
})),
|
||||
}))()
|
||||
`) as SearchPayload;
|
||||
}
|
||||
|
||||
cli({
|
||||
site: 'amazon',
|
||||
name: 'search',
|
||||
description: 'Amazon search results for product discovery and coarse filtering',
|
||||
domain: 'amazon.com',
|
||||
strategy: Strategy.COOKIE,
|
||||
navigateBefore: false,
|
||||
args: [
|
||||
{
|
||||
name: 'query',
|
||||
required: true,
|
||||
positional: true,
|
||||
help: 'Search query, for example "desk shelf organizer"',
|
||||
},
|
||||
{
|
||||
name: 'limit',
|
||||
type: 'int',
|
||||
default: 20,
|
||||
help: 'Maximum number of results to return (default 20)',
|
||||
},
|
||||
],
|
||||
columns: ['rank', 'asin', 'title', 'price_text', 'rating_value', 'review_count'],
|
||||
func: async (page, kwargs) => {
|
||||
const query = String(kwargs.query ?? '');
|
||||
const limit = Math.max(1, Number(kwargs.limit) || 20);
|
||||
const payload = await readSearchPayload(page, query);
|
||||
const sourceUrl = cleanText(payload.href) || buildSearchUrl(query);
|
||||
const cards = (payload.cards ?? [])
|
||||
.filter((card) => cleanText(card.asin) && cleanText(card.title))
|
||||
.slice(0, limit);
|
||||
|
||||
if (cards.length === 0) {
|
||||
throw new CommandExecutionError(
|
||||
'amazon search did not expose any product cards',
|
||||
'The search page may have changed or hit a robot check. Open the same query in Chrome, verify the page is visible, and retry.',
|
||||
);
|
||||
}
|
||||
|
||||
return cards.map((card, index) => normalizeSearchCandidate(card, index + 1, sourceUrl));
|
||||
},
|
||||
});
|
||||
|
||||
export const __test__ = {
|
||||
normalizeSearchCandidate,
|
||||
};
|
||||
@@ -1,365 +0,0 @@
|
||||
import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
export const SITE = 'amazon';
|
||||
export const DOMAIN = 'amazon.com';
|
||||
export const HOME_URL = 'https://www.amazon.com/';
|
||||
export const BESTSELLERS_URL = 'https://www.amazon.com/Best-Sellers/zgbs';
|
||||
export const NEW_RELEASES_URL = 'https://www.amazon.com/gp/new-releases';
|
||||
export const MOVERS_SHAKERS_URL = 'https://www.amazon.com/gp/movers-and-shakers';
|
||||
export const SEARCH_URL_PREFIX = 'https://www.amazon.com/s?k=';
|
||||
export const PRODUCT_URL_PREFIX = 'https://www.amazon.com/dp/';
|
||||
export const DISCUSSION_URL_PREFIX = 'https://www.amazon.com/product-reviews/';
|
||||
export const STRATEGY = 'cookie';
|
||||
export const PRIMARY_PRICE_SELECTORS = [
|
||||
'#corePrice_feature_div .a-offscreen',
|
||||
'#corePriceDisplay_desktop_feature_div .a-offscreen',
|
||||
'#corePrice_desktop .a-offscreen',
|
||||
'#apex_desktop .a-offscreen',
|
||||
'#newAccordionRow_0 .a-offscreen',
|
||||
'#price_inside_buybox',
|
||||
'#priceblock_ourprice',
|
||||
'#priceblock_dealprice',
|
||||
'#tp_price_block_total_price_ww',
|
||||
];
|
||||
const ROBOT_TEXT_PATTERNS = [
|
||||
'Sorry, we just need to make sure you\'re not a robot',
|
||||
'Enter the characters you see below',
|
||||
'Type the characters you see in this image',
|
||||
'To discuss automated access to Amazon data please contact',
|
||||
];
|
||||
const AMAZON_RANKING_SPECS = {
|
||||
bestsellers: {
|
||||
commandName: 'bestsellers',
|
||||
rootUrl: BESTSELLERS_URL,
|
||||
pathPattern: /(?:^|\/)zgbs(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon bestsellers expects a best sellers URL or /zgbs path',
|
||||
invalidInputHint: 'Example: opencli amazon bestsellers https://www.amazon.com/Best-Sellers/zgbs',
|
||||
},
|
||||
new_releases: {
|
||||
commandName: 'new-releases',
|
||||
rootUrl: NEW_RELEASES_URL,
|
||||
pathPattern: /\/gp\/new-releases(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon new-releases expects a new releases URL or /gp/new-releases path',
|
||||
invalidInputHint: 'Example: opencli amazon new-releases https://www.amazon.com/gp/new-releases',
|
||||
},
|
||||
movers_shakers: {
|
||||
commandName: 'movers-shakers',
|
||||
rootUrl: MOVERS_SHAKERS_URL,
|
||||
pathPattern: /\/gp\/movers-and-shakers(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon movers-shakers expects a movers-and-shakers URL or /gp/movers-and-shakers path',
|
||||
invalidInputHint: 'Example: opencli amazon movers-shakers https://www.amazon.com/gp/movers-and-shakers',
|
||||
},
|
||||
};
|
||||
export function cleanText(value) {
|
||||
return typeof value === 'string'
|
||||
? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
|
||||
: '';
|
||||
}
|
||||
export function cleanMultilineText(value) {
|
||||
return typeof value === 'string'
|
||||
? value
|
||||
.replace(/\u00a0/g, ' ')
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: '';
|
||||
}
|
||||
export function uniqueNonEmpty(values) {
|
||||
return [...new Set(values.map((value) => cleanText(value)).filter(Boolean))];
|
||||
}
|
||||
export function buildProvenance(sourceUrl) {
|
||||
return {
|
||||
source_url: sourceUrl,
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: STRATEGY,
|
||||
};
|
||||
}
|
||||
export function buildSearchUrl(query) {
|
||||
const normalized = cleanText(query);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError('amazon search query cannot be empty');
|
||||
}
|
||||
return `${SEARCH_URL_PREFIX}${encodeURIComponent(normalized)}`;
|
||||
}
|
||||
export function extractAsin(input) {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized)
|
||||
return null;
|
||||
if (/^[A-Z0-9]{10}$/i.test(normalized)) {
|
||||
return normalized.toUpperCase();
|
||||
}
|
||||
const match = normalized.match(/\/(?:dp|gp\/product|product-reviews)\/([A-Z0-9]{10})/i);
|
||||
return match ? match[1].toUpperCase() : null;
|
||||
}
|
||||
export function buildProductUrl(input) {
|
||||
const asin = extractAsin(input);
|
||||
if (!asin) {
|
||||
throw new ArgumentError('amazon product expects an ASIN or product URL', 'Example: opencli amazon product B0FJS72893');
|
||||
}
|
||||
return `${PRODUCT_URL_PREFIX}${asin}`;
|
||||
}
|
||||
export function buildDiscussionUrl(input) {
|
||||
const asin = extractAsin(input);
|
||||
if (!asin) {
|
||||
throw new ArgumentError('amazon discussion expects an ASIN or product URL', 'Example: opencli amazon discussion B0FJS72893');
|
||||
}
|
||||
return `${DISCUSSION_URL_PREFIX}${asin}`;
|
||||
}
|
||||
function getRankingSpec(listType) {
|
||||
return AMAZON_RANKING_SPECS[listType];
|
||||
}
|
||||
export function isSupportedRankingPath(listType, inputUrl) {
|
||||
try {
|
||||
const url = new URL(inputUrl);
|
||||
return getRankingSpec(listType).pathPattern.test(url.pathname);
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
export function resolveRankingUrl(listType, input) {
|
||||
const spec = getRankingSpec(listType);
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized || normalized === 'root')
|
||||
return spec.rootUrl;
|
||||
let candidateUrl;
|
||||
if (normalized.startsWith('/')) {
|
||||
candidateUrl = new URL(normalized, HOME_URL).toString();
|
||||
}
|
||||
else if (/^https?:\/\//i.test(normalized)) {
|
||||
candidateUrl = canonicalizeAmazonUrl(normalized);
|
||||
}
|
||||
else if (normalized.includes('amazon.') && normalized.includes('/')) {
|
||||
candidateUrl = canonicalizeAmazonUrl(`https://${normalized.replace(/^\/+/, '')}`);
|
||||
}
|
||||
else {
|
||||
throw new ArgumentError(spec.invalidInputMessage, spec.invalidInputHint);
|
||||
}
|
||||
if (!isSupportedRankingPath(listType, candidateUrl)) {
|
||||
throw new ArgumentError(spec.invalidInputMessage, spec.invalidInputHint);
|
||||
}
|
||||
return normalizeRankingInputUrl(candidateUrl);
|
||||
}
|
||||
function normalizeRankingInputUrl(inputUrl) {
|
||||
try {
|
||||
const url = new URL(inputUrl);
|
||||
const normalizedPathSegments = url.pathname
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.filter((segment) => !/^ref=/i.test(segment));
|
||||
url.pathname = `/${normalizedPathSegments.join('/')}`;
|
||||
url.hash = '';
|
||||
// Ranking pages are frequently shared with tracking refs that can land on unstable variants.
|
||||
// Dropping ref keeps the canonical ranking path while preserving useful params (for example pg=2).
|
||||
url.searchParams.delete('ref');
|
||||
return url.toString();
|
||||
}
|
||||
catch {
|
||||
return inputUrl;
|
||||
}
|
||||
}
|
||||
export function isRankingPaginationUrl(listType, inputUrl) {
|
||||
const absolute = toAbsoluteAmazonUrl(inputUrl);
|
||||
if (!absolute || !isSupportedRankingPath(listType, absolute))
|
||||
return false;
|
||||
try {
|
||||
const url = new URL(absolute);
|
||||
const ref = cleanText(url.searchParams.get('ref')).toLowerCase();
|
||||
// pg= query param is the most reliable pagination indicator across all ranking lists
|
||||
return url.searchParams.has('pg')
|
||||
|| /(?:^|_)pg(?:_|$)/.test(ref)
|
||||
// Amazon ranking pagination refs: zg_bs_pg_ (bestsellers), zg_bsnr_pg_ (new releases), zg_bsms_pg_ (movers & shakers)
|
||||
|| /zg_bs(?:nr|ms)?_pg_/.test(ref);
|
||||
}
|
||||
catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
export function extractCategoryNodeId(inputUrl) {
|
||||
const absolute = toAbsoluteAmazonUrl(inputUrl);
|
||||
if (!absolute)
|
||||
return null;
|
||||
try {
|
||||
const url = new URL(absolute);
|
||||
for (const key of ['node', 'nodeid', 'nodeId', 'browseNode']) {
|
||||
const value = cleanText(url.searchParams.get(key));
|
||||
if (/^\d{4,}$/.test(value))
|
||||
return value;
|
||||
}
|
||||
const rhValue = cleanText(url.searchParams.get('rh'));
|
||||
const rhMatch = decodeURIComponent(rhValue).match(/(?:^|,)\s*n:(\d{4,})(?:,|$)/i);
|
||||
if (rhMatch)
|
||||
return rhMatch[1];
|
||||
const pathMatches = [...url.pathname.matchAll(/\/(\d{4,})(?=\/|$)/g)];
|
||||
if (pathMatches.length > 0) {
|
||||
return pathMatches[pathMatches.length - 1][1];
|
||||
}
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
export function resolveBestsellersUrl(input) {
|
||||
return resolveRankingUrl('bestsellers', input);
|
||||
}
|
||||
export function canonicalizeAmazonUrl(input) {
|
||||
try {
|
||||
const url = new URL(input);
|
||||
if (!url.hostname.endsWith(DOMAIN)) {
|
||||
throw new Error('not-amazon');
|
||||
}
|
||||
return url.toString();
|
||||
}
|
||||
catch {
|
||||
throw new ArgumentError('Invalid Amazon URL');
|
||||
}
|
||||
}
|
||||
export function toAbsoluteAmazonUrl(value) {
|
||||
const normalized = cleanText(value);
|
||||
if (!normalized)
|
||||
return null;
|
||||
try {
|
||||
return new URL(normalized, HOME_URL).toString();
|
||||
}
|
||||
catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
export function normalizeProductUrl(value) {
|
||||
const normalized = cleanText(value);
|
||||
const asin = extractAsin(normalized);
|
||||
if (asin)
|
||||
return buildProductUrl(asin);
|
||||
return toAbsoluteAmazonUrl(normalized);
|
||||
}
|
||||
export function parsePriceText(text) {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/([$€£])\s*(\d+(?:,\d{3})*(?:\.\d+)?)/);
|
||||
if (!match) {
|
||||
return {
|
||||
price_text: normalized || null,
|
||||
price_value: null,
|
||||
currency: null,
|
||||
};
|
||||
}
|
||||
const currencyMap = {
|
||||
'$': 'USD',
|
||||
'€': 'EUR',
|
||||
'£': 'GBP',
|
||||
};
|
||||
return {
|
||||
price_text: `${match[1]}${match[2]}`,
|
||||
price_value: Number.parseFloat(match[2].replace(/,/g, '')),
|
||||
currency: currencyMap[match[1]] ?? null,
|
||||
};
|
||||
}
|
||||
export function parseRatingValue(text) {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/(\d+(?:\.\d+)?)\s*out of 5/i);
|
||||
return match ? Number.parseFloat(match[1]) : null;
|
||||
}
|
||||
export function parseReviewCount(text) {
|
||||
const normalized = cleanText(text);
|
||||
const compactMatch = normalized.match(/(\d+(?:\.\d+)?)\s*([kKmM])/);
|
||||
if (compactMatch) {
|
||||
const value = Number.parseFloat(compactMatch[1]);
|
||||
const multiplier = /m/i.test(compactMatch[2]) ? 1_000_000 : 1_000;
|
||||
return Number.isFinite(value) ? Math.round(value * multiplier) : null;
|
||||
}
|
||||
const match = normalized.match(/([\d,]+)/);
|
||||
return match ? Number.parseInt(match[1].replace(/,/g, ''), 10) : null;
|
||||
}
|
||||
export function extractReviewCountFromCardText(text) {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const match = normalized.match(/out of 5 stars(?:, rating details)?\s*([\d,]+)/i);
|
||||
if (match)
|
||||
return match[1];
|
||||
const numericLine = normalized
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => /^[\d,]+$/.test(line));
|
||||
return numericLine ?? null;
|
||||
}
|
||||
export function isAmazonEntity(text) {
|
||||
const normalized = cleanText(text).toLowerCase();
|
||||
return normalized.includes('amazon');
|
||||
}
|
||||
export function firstMeaningfulLine(text) {
|
||||
return cleanMultilineText(text)
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find(Boolean)
|
||||
?? '';
|
||||
}
|
||||
export function trimRatingPrefix(text) {
|
||||
const normalized = cleanText(text);
|
||||
if (!normalized)
|
||||
return null;
|
||||
return normalized.replace(/^\d+(?:\.\d+)?\s*out of 5 stars\s*/i, '').trim() || normalized;
|
||||
}
|
||||
export function isRobotState(state) {
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
return ROBOT_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
export function buildChallengeHint(action) {
|
||||
return [
|
||||
`Open a clean Amazon ${action} page in the shared Chrome profile and clear any robot check first.`,
|
||||
'If you are using CDP, set OPENCLI_CDP_TARGET=amazon.com and avoid parallel Amazon commands against the same browser target.',
|
||||
].join(' ');
|
||||
}
|
||||
export async function readPageState(page) {
|
||||
const result = await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
body_text: document.body ? document.body.innerText || '' : '',
|
||||
}))()
|
||||
`);
|
||||
return {
|
||||
href: cleanText(result.href),
|
||||
title: cleanText(result.title),
|
||||
body_text: cleanMultilineText(result.body_text),
|
||||
};
|
||||
}
|
||||
export async function gotoAndReadState(page, url, settleMs = 2500, action = 'page') {
|
||||
try {
|
||||
await page.goto(url, { settleMs });
|
||||
await page.wait(1.5);
|
||||
return await readPageState(page);
|
||||
}
|
||||
catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (message.includes('Inspected target navigated or closed')
|
||||
|| message.includes('Cannot find context with specified id')
|
||||
|| message.includes('Target closed')) {
|
||||
throw new CommandExecutionError(`amazon ${action} navigation lost the current browser target`, `${buildChallengeHint(action)} If CDP is attached to a stale tab, open a fresh Amazon tab and retry.`);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
export function assertUsableState(state, action) {
|
||||
if (!isRobotState(state))
|
||||
return;
|
||||
throw new CommandExecutionError(`amazon ${action} hit a robot check`, buildChallengeHint(action));
|
||||
}
|
||||
export const __test__ = {
|
||||
buildSearchUrl,
|
||||
extractAsin,
|
||||
buildProductUrl,
|
||||
buildDiscussionUrl,
|
||||
resolveBestsellersUrl,
|
||||
resolveRankingUrl,
|
||||
isSupportedRankingPath,
|
||||
isRankingPaginationUrl,
|
||||
extractCategoryNodeId,
|
||||
parsePriceText,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
extractReviewCountFromCardText,
|
||||
isAmazonEntity,
|
||||
trimRatingPrefix,
|
||||
isRobotState,
|
||||
PRIMARY_PRICE_SELECTORS,
|
||||
};
|
||||
@@ -1,44 +0,0 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './shared.js';
|
||||
describe('amazon shared helpers', () => {
|
||||
it('builds canonical product and discussion URLs from ASINs and product URLs', () => {
|
||||
expect(__test__.buildProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(__test__.buildProductUrl('https://www.amazon.com/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(__test__.buildDiscussionUrl('https://www.amazon.com/dp/B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893');
|
||||
});
|
||||
it('parses price, rating, and review-count text', () => {
|
||||
expect(__test__.parsePriceText('1 offer from $34.11')).toEqual({
|
||||
price_text: '$34.11',
|
||||
price_value: 34.11,
|
||||
currency: 'USD',
|
||||
});
|
||||
expect(__test__.parseRatingValue('3.9 out of 5 stars, rating details')).toBe(3.9);
|
||||
expect(__test__.parseReviewCount('27 global ratings')).toBe(27);
|
||||
expect(__test__.parseReviewCount('(2.9K)')).toBe(2900);
|
||||
expect(__test__.parseReviewCount('1.2M global ratings')).toBe(1200000);
|
||||
expect(__test__.extractReviewCountFromCardText('Desk Shelf\n4.3 out of 5 stars\n435\n$25.92')).toBe('435');
|
||||
});
|
||||
it('recognizes robot checks and Amazon-owned merchants', () => {
|
||||
expect(__test__.isAmazonEntity('Ships from Amazon')).toBe(true);
|
||||
expect(__test__.trimRatingPrefix('5.0 out of 5 stars Great value and quality')).toBe('Great value and quality');
|
||||
expect(__test__.isRobotState({
|
||||
title: 'Robot Check',
|
||||
body_text: 'Sorry, we just need to make sure you\'re not a robot',
|
||||
})).toBe(true);
|
||||
});
|
||||
it('requires a real best-sellers URL or path', () => {
|
||||
expect(__test__.resolveBestsellersUrl('/Best-Sellers/zgbs')).toBe('https://www.amazon.com/Best-Sellers/zgbs');
|
||||
expect(() => __test__.resolveBestsellersUrl('desk shelf organizer')).toThrow('amazon bestsellers expects a best sellers URL or /zgbs path');
|
||||
});
|
||||
it('resolves and validates all ranking list URLs', () => {
|
||||
expect(__test__.resolveRankingUrl('new_releases')).toBe('https://www.amazon.com/gp/new-releases');
|
||||
expect(__test__.resolveRankingUrl('movers_shakers')).toBe('https://www.amazon.com/gp/movers-and-shakers');
|
||||
expect(__test__.resolveRankingUrl('new_releases', '/gp/new-releases/kitchen')).toBe('https://www.amazon.com/gp/new-releases/kitchen');
|
||||
expect(__test__.resolveRankingUrl('bestsellers', 'https://www.amazon.com/Best-Sellers/zgbs/ref=zg_bsnr_tab_bs')).toBe('https://www.amazon.com/Best-Sellers/zgbs');
|
||||
expect(() => __test__.resolveRankingUrl('movers_shakers', 'https://example.com/gp/movers-and-shakers')).toThrow('Invalid Amazon URL');
|
||||
});
|
||||
it('extracts category node id from URL best effort', () => {
|
||||
expect(__test__.extractCategoryNodeId('https://www.amazon.com/Best-Sellers-Home-Kitchen/zgbs/home-garden/3744371')).toBe('3744371');
|
||||
expect(__test__.extractCategoryNodeId('https://www.amazon.com/s?k=desk+organizer&rh=n%3A1064954')).toBe('1064954');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,53 @@
|
||||
import { describe, expect, it } from 'vitest';
|
||||
import { __test__ } from './shared.js';
|
||||
|
||||
describe('amazon shared helpers', () => {
|
||||
it('builds canonical product and discussion URLs from ASINs and product URLs', () => {
|
||||
expect(__test__.buildProductUrl('B0FJS72893')).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(__test__.buildProductUrl('https://www.amazon.com/dp/B0FJS72893/ref=something')).toBe('https://www.amazon.com/dp/B0FJS72893');
|
||||
expect(__test__.buildDiscussionUrl('https://www.amazon.com/dp/B0FJS72893')).toBe('https://www.amazon.com/product-reviews/B0FJS72893');
|
||||
});
|
||||
|
||||
it('parses price, rating, and review-count text', () => {
|
||||
expect(__test__.parsePriceText('1 offer from $34.11')).toEqual({
|
||||
price_text: '$34.11',
|
||||
price_value: 34.11,
|
||||
currency: 'USD',
|
||||
});
|
||||
expect(__test__.parseRatingValue('3.9 out of 5 stars, rating details')).toBe(3.9);
|
||||
expect(__test__.parseReviewCount('27 global ratings')).toBe(27);
|
||||
expect(__test__.parseReviewCount('(2.9K)')).toBe(2900);
|
||||
expect(__test__.parseReviewCount('1.2M global ratings')).toBe(1200000);
|
||||
expect(__test__.extractReviewCountFromCardText('Desk Shelf\n4.3 out of 5 stars\n435\n$25.92')).toBe('435');
|
||||
});
|
||||
|
||||
it('recognizes robot checks and Amazon-owned merchants', () => {
|
||||
expect(__test__.isAmazonEntity('Ships from Amazon')).toBe(true);
|
||||
expect(__test__.trimRatingPrefix('5.0 out of 5 stars Great value and quality')).toBe('Great value and quality');
|
||||
expect(__test__.isRobotState({
|
||||
title: 'Robot Check',
|
||||
body_text: 'Sorry, we just need to make sure you\'re not a robot',
|
||||
})).toBe(true);
|
||||
});
|
||||
|
||||
it('requires a real best-sellers URL or path', () => {
|
||||
expect(__test__.resolveBestsellersUrl('/Best-Sellers/zgbs')).toBe('https://www.amazon.com/Best-Sellers/zgbs');
|
||||
expect(() => __test__.resolveBestsellersUrl('desk shelf organizer')).toThrow('amazon bestsellers expects a best sellers URL or /zgbs path');
|
||||
});
|
||||
|
||||
it('resolves and validates all ranking list URLs', () => {
|
||||
expect(__test__.resolveRankingUrl('new_releases')).toBe('https://www.amazon.com/gp/new-releases');
|
||||
expect(__test__.resolveRankingUrl('movers_shakers')).toBe('https://www.amazon.com/gp/movers-and-shakers');
|
||||
expect(__test__.resolveRankingUrl('new_releases', '/gp/new-releases/kitchen')).toBe('https://www.amazon.com/gp/new-releases/kitchen');
|
||||
expect(__test__.resolveRankingUrl(
|
||||
'bestsellers',
|
||||
'https://www.amazon.com/Best-Sellers/zgbs/ref=zg_bsnr_tab_bs',
|
||||
)).toBe('https://www.amazon.com/Best-Sellers/zgbs');
|
||||
expect(() => __test__.resolveRankingUrl('movers_shakers', 'https://example.com/gp/movers-and-shakers')).toThrow('Invalid Amazon URL');
|
||||
});
|
||||
|
||||
it('extracts category node id from URL best effort', () => {
|
||||
expect(__test__.extractCategoryNodeId('https://www.amazon.com/Best-Sellers-Home-Kitchen/zgbs/home-garden/3744371')).toBe('3744371');
|
||||
expect(__test__.extractCategoryNodeId('https://www.amazon.com/s?k=desk+organizer&rh=n%3A1064954')).toBe('1064954');
|
||||
});
|
||||
});
|
||||
@@ -0,0 +1,438 @@
|
||||
import { ArgumentError, CommandExecutionError } from '@jackwener/opencli/errors';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
|
||||
export const SITE = 'amazon';
|
||||
export const DOMAIN = 'amazon.com';
|
||||
export const HOME_URL = 'https://www.amazon.com/';
|
||||
export const BESTSELLERS_URL = 'https://www.amazon.com/Best-Sellers/zgbs';
|
||||
export const NEW_RELEASES_URL = 'https://www.amazon.com/gp/new-releases';
|
||||
export const MOVERS_SHAKERS_URL = 'https://www.amazon.com/gp/movers-and-shakers';
|
||||
export const SEARCH_URL_PREFIX = 'https://www.amazon.com/s?k=';
|
||||
export const PRODUCT_URL_PREFIX = 'https://www.amazon.com/dp/';
|
||||
export const DISCUSSION_URL_PREFIX = 'https://www.amazon.com/product-reviews/';
|
||||
export const STRATEGY = 'cookie';
|
||||
export const PRIMARY_PRICE_SELECTORS = [
|
||||
'#corePrice_feature_div .a-offscreen',
|
||||
'#corePriceDisplay_desktop_feature_div .a-offscreen',
|
||||
'#corePrice_desktop .a-offscreen',
|
||||
'#apex_desktop .a-offscreen',
|
||||
'#newAccordionRow_0 .a-offscreen',
|
||||
'#price_inside_buybox',
|
||||
'#priceblock_ourprice',
|
||||
'#priceblock_dealprice',
|
||||
'#tp_price_block_total_price_ww',
|
||||
];
|
||||
|
||||
const ROBOT_TEXT_PATTERNS = [
|
||||
'Sorry, we just need to make sure you\'re not a robot',
|
||||
'Enter the characters you see below',
|
||||
'Type the characters you see in this image',
|
||||
'To discuss automated access to Amazon data please contact',
|
||||
];
|
||||
|
||||
export type AmazonRankingListType = 'bestsellers' | 'new_releases' | 'movers_shakers';
|
||||
|
||||
interface AmazonRankingSpec {
|
||||
commandName: string;
|
||||
rootUrl: string;
|
||||
pathPattern: RegExp;
|
||||
invalidInputMessage: string;
|
||||
invalidInputHint: string;
|
||||
}
|
||||
|
||||
const AMAZON_RANKING_SPECS: Record<AmazonRankingListType, AmazonRankingSpec> = {
|
||||
bestsellers: {
|
||||
commandName: 'bestsellers',
|
||||
rootUrl: BESTSELLERS_URL,
|
||||
pathPattern: /(?:^|\/)zgbs(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon bestsellers expects a best sellers URL or /zgbs path',
|
||||
invalidInputHint: 'Example: opencli amazon bestsellers https://www.amazon.com/Best-Sellers/zgbs',
|
||||
},
|
||||
new_releases: {
|
||||
commandName: 'new-releases',
|
||||
rootUrl: NEW_RELEASES_URL,
|
||||
pathPattern: /\/gp\/new-releases(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon new-releases expects a new releases URL or /gp/new-releases path',
|
||||
invalidInputHint: 'Example: opencli amazon new-releases https://www.amazon.com/gp/new-releases',
|
||||
},
|
||||
movers_shakers: {
|
||||
commandName: 'movers-shakers',
|
||||
rootUrl: MOVERS_SHAKERS_URL,
|
||||
pathPattern: /\/gp\/movers-and-shakers(?:\/|$)/i,
|
||||
invalidInputMessage: 'amazon movers-shakers expects a movers-and-shakers URL or /gp/movers-and-shakers path',
|
||||
invalidInputHint: 'Example: opencli amazon movers-shakers https://www.amazon.com/gp/movers-and-shakers',
|
||||
},
|
||||
};
|
||||
|
||||
export interface ProvenanceFields {
|
||||
source_url: string;
|
||||
fetched_at: string;
|
||||
strategy: string;
|
||||
}
|
||||
|
||||
export interface PageState {
|
||||
href: string;
|
||||
title: string;
|
||||
body_text: string;
|
||||
}
|
||||
|
||||
export interface PriceValue {
|
||||
price_text: string | null;
|
||||
price_value: number | null;
|
||||
currency: string | null;
|
||||
}
|
||||
|
||||
export function cleanText(value: unknown): string {
|
||||
return typeof value === 'string'
|
||||
? value.replace(/\u00a0/g, ' ').replace(/\s+/g, ' ').trim()
|
||||
: '';
|
||||
}
|
||||
|
||||
export function cleanMultilineText(value: unknown): string {
|
||||
return typeof value === 'string'
|
||||
? value
|
||||
.replace(/\u00a0/g, ' ')
|
||||
.split('\n')
|
||||
.map((line) => line.replace(/\s+/g, ' ').trim())
|
||||
.filter(Boolean)
|
||||
.join('\n')
|
||||
: '';
|
||||
}
|
||||
|
||||
export function uniqueNonEmpty(values: Array<string | null | undefined>): string[] {
|
||||
return [...new Set(values.map((value) => cleanText(value)).filter(Boolean))];
|
||||
}
|
||||
|
||||
export function buildProvenance(sourceUrl: string): ProvenanceFields {
|
||||
return {
|
||||
source_url: sourceUrl,
|
||||
fetched_at: new Date().toISOString(),
|
||||
strategy: STRATEGY,
|
||||
};
|
||||
}
|
||||
|
||||
export function buildSearchUrl(query: string): string {
|
||||
const normalized = cleanText(query);
|
||||
if (!normalized) {
|
||||
throw new ArgumentError('amazon search query cannot be empty');
|
||||
}
|
||||
return `${SEARCH_URL_PREFIX}${encodeURIComponent(normalized)}`;
|
||||
}
|
||||
|
||||
export function extractAsin(input: string): string | null {
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized) return null;
|
||||
if (/^[A-Z0-9]{10}$/i.test(normalized)) {
|
||||
return normalized.toUpperCase();
|
||||
}
|
||||
const match = normalized.match(/\/(?:dp|gp\/product|product-reviews)\/([A-Z0-9]{10})/i);
|
||||
return match ? match[1].toUpperCase() : null;
|
||||
}
|
||||
|
||||
export function buildProductUrl(input: string): string {
|
||||
const asin = extractAsin(input);
|
||||
if (!asin) {
|
||||
throw new ArgumentError(
|
||||
'amazon product expects an ASIN or product URL',
|
||||
'Example: opencli amazon product B0FJS72893',
|
||||
);
|
||||
}
|
||||
return `${PRODUCT_URL_PREFIX}${asin}`;
|
||||
}
|
||||
|
||||
export function buildDiscussionUrl(input: string): string {
|
||||
const asin = extractAsin(input);
|
||||
if (!asin) {
|
||||
throw new ArgumentError(
|
||||
'amazon discussion expects an ASIN or product URL',
|
||||
'Example: opencli amazon discussion B0FJS72893',
|
||||
);
|
||||
}
|
||||
return `${DISCUSSION_URL_PREFIX}${asin}`;
|
||||
}
|
||||
|
||||
function getRankingSpec(listType: AmazonRankingListType): AmazonRankingSpec {
|
||||
return AMAZON_RANKING_SPECS[listType];
|
||||
}
|
||||
|
||||
export function isSupportedRankingPath(listType: AmazonRankingListType, inputUrl: string): boolean {
|
||||
try {
|
||||
const url = new URL(inputUrl);
|
||||
return getRankingSpec(listType).pathPattern.test(url.pathname);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function resolveRankingUrl(listType: AmazonRankingListType, input?: string): string {
|
||||
const spec = getRankingSpec(listType);
|
||||
const normalized = cleanText(input);
|
||||
if (!normalized || normalized === 'root') return spec.rootUrl;
|
||||
|
||||
let candidateUrl: string;
|
||||
if (normalized.startsWith('/')) {
|
||||
candidateUrl = new URL(normalized, HOME_URL).toString();
|
||||
} else if (/^https?:\/\//i.test(normalized)) {
|
||||
candidateUrl = canonicalizeAmazonUrl(normalized);
|
||||
} else if (normalized.includes('amazon.') && normalized.includes('/')) {
|
||||
candidateUrl = canonicalizeAmazonUrl(`https://${normalized.replace(/^\/+/, '')}`);
|
||||
} else {
|
||||
throw new ArgumentError(spec.invalidInputMessage, spec.invalidInputHint);
|
||||
}
|
||||
|
||||
if (!isSupportedRankingPath(listType, candidateUrl)) {
|
||||
throw new ArgumentError(spec.invalidInputMessage, spec.invalidInputHint);
|
||||
}
|
||||
return normalizeRankingInputUrl(candidateUrl);
|
||||
}
|
||||
|
||||
function normalizeRankingInputUrl(inputUrl: string): string {
|
||||
try {
|
||||
const url = new URL(inputUrl);
|
||||
const normalizedPathSegments = url.pathname
|
||||
.split('/')
|
||||
.filter(Boolean)
|
||||
.filter((segment) => !/^ref=/i.test(segment));
|
||||
url.pathname = `/${normalizedPathSegments.join('/')}`;
|
||||
url.hash = '';
|
||||
// Ranking pages are frequently shared with tracking refs that can land on unstable variants.
|
||||
// Dropping ref keeps the canonical ranking path while preserving useful params (for example pg=2).
|
||||
url.searchParams.delete('ref');
|
||||
return url.toString();
|
||||
} catch {
|
||||
return inputUrl;
|
||||
}
|
||||
}
|
||||
|
||||
export function isRankingPaginationUrl(listType: AmazonRankingListType, inputUrl: string): boolean {
|
||||
const absolute = toAbsoluteAmazonUrl(inputUrl);
|
||||
if (!absolute || !isSupportedRankingPath(listType, absolute)) return false;
|
||||
|
||||
try {
|
||||
const url = new URL(absolute);
|
||||
const ref = cleanText(url.searchParams.get('ref')).toLowerCase();
|
||||
// pg= query param is the most reliable pagination indicator across all ranking lists
|
||||
return url.searchParams.has('pg')
|
||||
|| /(?:^|_)pg(?:_|$)/.test(ref)
|
||||
// Amazon ranking pagination refs: zg_bs_pg_ (bestsellers), zg_bsnr_pg_ (new releases), zg_bsms_pg_ (movers & shakers)
|
||||
|| /zg_bs(?:nr|ms)?_pg_/.test(ref);
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
export function extractCategoryNodeId(inputUrl: string | null | undefined): string | null {
|
||||
const absolute = toAbsoluteAmazonUrl(inputUrl);
|
||||
if (!absolute) return null;
|
||||
|
||||
try {
|
||||
const url = new URL(absolute);
|
||||
|
||||
for (const key of ['node', 'nodeid', 'nodeId', 'browseNode']) {
|
||||
const value = cleanText(url.searchParams.get(key));
|
||||
if (/^\d{4,}$/.test(value)) return value;
|
||||
}
|
||||
|
||||
const rhValue = cleanText(url.searchParams.get('rh'));
|
||||
const rhMatch = decodeURIComponent(rhValue).match(/(?:^|,)\s*n:(\d{4,})(?:,|$)/i);
|
||||
if (rhMatch) return rhMatch[1];
|
||||
|
||||
const pathMatches = [...url.pathname.matchAll(/\/(\d{4,})(?=\/|$)/g)];
|
||||
if (pathMatches.length > 0) {
|
||||
return pathMatches[pathMatches.length - 1][1];
|
||||
}
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
export function resolveBestsellersUrl(input?: string): string {
|
||||
return resolveRankingUrl('bestsellers', input);
|
||||
}
|
||||
|
||||
export function canonicalizeAmazonUrl(input: string): string {
|
||||
try {
|
||||
const url = new URL(input);
|
||||
if (!url.hostname.endsWith(DOMAIN)) {
|
||||
throw new Error('not-amazon');
|
||||
}
|
||||
return url.toString();
|
||||
} catch {
|
||||
throw new ArgumentError('Invalid Amazon URL');
|
||||
}
|
||||
}
|
||||
|
||||
export function toAbsoluteAmazonUrl(value: string | null | undefined): string | null {
|
||||
const normalized = cleanText(value);
|
||||
if (!normalized) return null;
|
||||
try {
|
||||
return new URL(normalized, HOME_URL).toString();
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
export function normalizeProductUrl(value: string | null | undefined): string | null {
|
||||
const normalized = cleanText(value);
|
||||
const asin = extractAsin(normalized);
|
||||
if (asin) return buildProductUrl(asin);
|
||||
return toAbsoluteAmazonUrl(normalized);
|
||||
}
|
||||
|
||||
export function parsePriceText(text: string | null | undefined): PriceValue {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/([$€£])\s*(\d+(?:,\d{3})*(?:\.\d+)?)/);
|
||||
if (!match) {
|
||||
return {
|
||||
price_text: normalized || null,
|
||||
price_value: null,
|
||||
currency: null,
|
||||
};
|
||||
}
|
||||
|
||||
const currencyMap: Record<string, string> = {
|
||||
'$': 'USD',
|
||||
'€': 'EUR',
|
||||
'£': 'GBP',
|
||||
};
|
||||
|
||||
return {
|
||||
price_text: `${match[1]}${match[2]}`,
|
||||
price_value: Number.parseFloat(match[2].replace(/,/g, '')),
|
||||
currency: currencyMap[match[1]] ?? null,
|
||||
};
|
||||
}
|
||||
|
||||
export function parseRatingValue(text: string | null | undefined): number | null {
|
||||
const normalized = cleanText(text);
|
||||
const match = normalized.match(/(\d+(?:\.\d+)?)\s*out of 5/i);
|
||||
return match ? Number.parseFloat(match[1]) : null;
|
||||
}
|
||||
|
||||
export function parseReviewCount(text: string | null | undefined): number | null {
|
||||
const normalized = cleanText(text);
|
||||
const compactMatch = normalized.match(/(\d+(?:\.\d+)?)\s*([kKmM])/);
|
||||
if (compactMatch) {
|
||||
const value = Number.parseFloat(compactMatch[1]);
|
||||
const multiplier = /m/i.test(compactMatch[2]) ? 1_000_000 : 1_000;
|
||||
return Number.isFinite(value) ? Math.round(value * multiplier) : null;
|
||||
}
|
||||
const match = normalized.match(/([\d,]+)/);
|
||||
return match ? Number.parseInt(match[1].replace(/,/g, ''), 10) : null;
|
||||
}
|
||||
|
||||
export function extractReviewCountFromCardText(text: string | null | undefined): string | null {
|
||||
const normalized = cleanMultilineText(text);
|
||||
const match = normalized.match(/out of 5 stars(?:, rating details)?\s*([\d,]+)/i);
|
||||
if (match) return match[1];
|
||||
|
||||
const numericLine = normalized
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find((line) => /^[\d,]+$/.test(line));
|
||||
return numericLine ?? null;
|
||||
}
|
||||
|
||||
export function isAmazonEntity(text: string | null | undefined): boolean {
|
||||
const normalized = cleanText(text).toLowerCase();
|
||||
return normalized.includes('amazon');
|
||||
}
|
||||
|
||||
export function firstMeaningfulLine(text: string | null | undefined): string {
|
||||
return cleanMultilineText(text)
|
||||
.split('\n')
|
||||
.map((line) => cleanText(line))
|
||||
.find(Boolean)
|
||||
?? '';
|
||||
}
|
||||
|
||||
export function trimRatingPrefix(text: string | null | undefined): string | null {
|
||||
const normalized = cleanText(text);
|
||||
if (!normalized) return null;
|
||||
return normalized.replace(/^\d+(?:\.\d+)?\s*out of 5 stars\s*/i, '').trim() || normalized;
|
||||
}
|
||||
|
||||
export function isRobotState(state: Partial<PageState>): boolean {
|
||||
const title = cleanText(state.title);
|
||||
const bodyText = cleanMultilineText(state.body_text);
|
||||
return ROBOT_TEXT_PATTERNS.some((pattern) => title.includes(pattern) || bodyText.includes(pattern));
|
||||
}
|
||||
|
||||
export function buildChallengeHint(action: string): string {
|
||||
return [
|
||||
`Open a clean Amazon ${action} page in the shared Chrome profile and clear any robot check first.`,
|
||||
'If you are using CDP, set OPENCLI_CDP_TARGET=amazon.com and avoid parallel Amazon commands against the same browser target.',
|
||||
].join(' ');
|
||||
}
|
||||
|
||||
export async function readPageState(page: IPage): Promise<PageState> {
|
||||
const result = await page.evaluate(`
|
||||
(() => ({
|
||||
href: window.location.href,
|
||||
title: document.title || '',
|
||||
body_text: document.body ? document.body.innerText || '' : '',
|
||||
}))()
|
||||
`) as Partial<PageState>;
|
||||
|
||||
return {
|
||||
href: cleanText(result.href),
|
||||
title: cleanText(result.title),
|
||||
body_text: cleanMultilineText(result.body_text),
|
||||
};
|
||||
}
|
||||
|
||||
export async function gotoAndReadState(
|
||||
page: IPage,
|
||||
url: string,
|
||||
settleMs: number = 2500,
|
||||
action: string = 'page',
|
||||
): Promise<PageState> {
|
||||
try {
|
||||
await page.goto(url, { settleMs });
|
||||
await page.wait(1.5);
|
||||
return await readPageState(page);
|
||||
} catch (error) {
|
||||
const message = error instanceof Error ? error.message : String(error);
|
||||
if (
|
||||
message.includes('Inspected target navigated or closed')
|
||||
|| message.includes('Cannot find context with specified id')
|
||||
|| message.includes('Target closed')
|
||||
) {
|
||||
throw new CommandExecutionError(
|
||||
`amazon ${action} navigation lost the current browser target`,
|
||||
`${buildChallengeHint(action)} If CDP is attached to a stale tab, open a fresh Amazon tab and retry.`,
|
||||
);
|
||||
}
|
||||
throw error;
|
||||
}
|
||||
}
|
||||
|
||||
export function assertUsableState(state: PageState, action: string): void {
|
||||
if (!isRobotState(state)) return;
|
||||
throw new CommandExecutionError(
|
||||
`amazon ${action} hit a robot check`,
|
||||
buildChallengeHint(action),
|
||||
);
|
||||
}
|
||||
|
||||
export const __test__ = {
|
||||
buildSearchUrl,
|
||||
extractAsin,
|
||||
buildProductUrl,
|
||||
buildDiscussionUrl,
|
||||
resolveBestsellersUrl,
|
||||
resolveRankingUrl,
|
||||
isSupportedRankingPath,
|
||||
isRankingPaginationUrl,
|
||||
extractCategoryNodeId,
|
||||
parsePriceText,
|
||||
parseRatingValue,
|
||||
parseReviewCount,
|
||||
extractReviewCountFromCardText,
|
||||
isAmazonEntity,
|
||||
trimRatingPrefix,
|
||||
isRobotState,
|
||||
PRIMARY_PRICE_SELECTORS,
|
||||
};
|
||||
@@ -1,28 +0,0 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import * as fs from 'node:fs';
|
||||
export const dumpCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'dump',
|
||||
description: 'Dump the DOM to help AI understand the UI',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['htmlFile', 'snapFile'],
|
||||
func: async (page) => {
|
||||
// Extract HTML
|
||||
const html = await page.evaluate('document.body.innerHTML');
|
||||
fs.writeFileSync('/tmp/antigravity-dom.html', html);
|
||||
// Extract Snapshot
|
||||
let snapFile = '';
|
||||
try {
|
||||
const snap = await page.snapshot({ raw: true });
|
||||
snapFile = '/tmp/antigravity-snapshot.json';
|
||||
fs.writeFileSync(snapFile, JSON.stringify(snap, null, 2));
|
||||
}
|
||||
catch (e) {
|
||||
snapFile = 'Failed';
|
||||
}
|
||||
return [{ htmlFile: '/tmp/antigravity-dom.html', snapFile }];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
import * as fs from 'node:fs';
|
||||
|
||||
export const dumpCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'dump',
|
||||
description: 'Dump the DOM to help AI understand the UI',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['htmlFile', 'snapFile'],
|
||||
func: async (page) => {
|
||||
// Extract HTML
|
||||
const html = await page.evaluate('document.body.innerHTML');
|
||||
fs.writeFileSync('/tmp/antigravity-dom.html', html);
|
||||
|
||||
// Extract Snapshot
|
||||
let snapFile = '';
|
||||
try {
|
||||
const snap = await page.snapshot({ raw: true });
|
||||
snapFile = '/tmp/antigravity-snapshot.json';
|
||||
fs.writeFileSync(snapFile, JSON.stringify(snap, null, 2));
|
||||
} catch (e) {
|
||||
snapFile = 'Failed';
|
||||
}
|
||||
|
||||
return [{ htmlFile: '/tmp/antigravity-dom.html', snapFile }];
|
||||
},
|
||||
});
|
||||
@@ -1,15 +1,16 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
export const extractCodeCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'extract-code',
|
||||
description: 'Extract multi-line code blocks from the current Antigravity conversation',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['code'],
|
||||
func: async (page) => {
|
||||
const blocks = await page.evaluate(`
|
||||
site: 'antigravity',
|
||||
name: 'extract-code',
|
||||
description: 'Extract multi-line code blocks from the current Antigravity conversation',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['code'],
|
||||
func: async (page) => {
|
||||
const blocks = await page.evaluate(`
|
||||
async () => {
|
||||
// Find standard pre/code blocks
|
||||
let elements = Array.from(document.querySelectorAll('pre code'));
|
||||
@@ -27,6 +28,7 @@ export const extractCodeCommand = cli({
|
||||
return elements.map(el => el.innerText).filter(text => text.trim().length > 0);
|
||||
}
|
||||
`);
|
||||
return blocks.map((code) => ({ code }));
|
||||
},
|
||||
|
||||
return blocks.map((code: string) => ({ code }));
|
||||
},
|
||||
});
|
||||
@@ -1,18 +1,20 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
export const modelCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'model',
|
||||
description: 'Switch the active LLM model in Antigravity',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'name', help: 'Target model name (e.g. claude, gemini, o1)', required: true, positional: true }
|
||||
],
|
||||
columns: ['Status'],
|
||||
func: async (page, kwargs) => {
|
||||
const targetName = kwargs.name.toLowerCase();
|
||||
await page.evaluate(`
|
||||
site: 'antigravity',
|
||||
name: 'model',
|
||||
description: 'Switch the active LLM model in Antigravity',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'name', help: 'Target model name (e.g. claude, gemini, o1)', required: true, positional: true }
|
||||
],
|
||||
columns: ['Status'],
|
||||
func: async (page, kwargs) => {
|
||||
const targetName = kwargs.name.toLowerCase();
|
||||
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const targetModelName = ${JSON.stringify(targetName)};
|
||||
|
||||
@@ -38,7 +40,8 @@ export const modelCommand = cli({
|
||||
optionNode.click();
|
||||
}
|
||||
`);
|
||||
await page.wait(0.5);
|
||||
return [{ Status: `Model switched to: ${kwargs.name}` }];
|
||||
},
|
||||
|
||||
await page.wait(0.5);
|
||||
return [{ Status: `Model switched to: ${kwargs.name}` }];
|
||||
},
|
||||
});
|
||||
@@ -1,25 +0,0 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
export const newCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'new',
|
||||
description: 'Start a new conversation / clear context in Antigravity',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['status'],
|
||||
func: async (page) => {
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const btn = document.querySelector('[data-tooltip-id="new-conversation-tooltip"]');
|
||||
if (!btn) throw new Error('Could not find New Conversation button');
|
||||
|
||||
// In case it's disabled, we must check, but we'll try to click it anyway
|
||||
btn.click();
|
||||
}
|
||||
`);
|
||||
// Give it a moment to reset the UI
|
||||
await page.wait(0.5);
|
||||
return [{ status: 'Successfully started a new conversation' }];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,28 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
export const newCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'new',
|
||||
description: 'Start a new conversation / clear context in Antigravity',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['status'],
|
||||
func: async (page) => {
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const btn = document.querySelector('[data-tooltip-id="new-conversation-tooltip"]');
|
||||
if (!btn) throw new Error('Could not find New Conversation button');
|
||||
|
||||
// In case it's disabled, we must check, but we'll try to click it anyway
|
||||
btn.click();
|
||||
}
|
||||
`);
|
||||
|
||||
// Give it a moment to reset the UI
|
||||
await page.wait(0.5);
|
||||
|
||||
return [{ status: 'Successfully started a new conversation' }];
|
||||
},
|
||||
});
|
||||
@@ -1,34 +0,0 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
export const readCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'read',
|
||||
description: 'Read the latest chat messages from Antigravity AI',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'last', help: 'Number of recent messages to read (not fully implemented due to generic structure, currently returns full history text or latest chunk)' }
|
||||
],
|
||||
columns: ['role', 'content'],
|
||||
func: async (page, kwargs) => {
|
||||
// We execute a script inside Antigravity's Chromium environment to extract the text
|
||||
// of the entire conversation pane.
|
||||
const rawText = await page.evaluate(`
|
||||
async () => {
|
||||
const container = document.getElementById('conversation');
|
||||
if (!container) throw new Error('Could not find conversation container');
|
||||
|
||||
// Extract the full visible text of the conversation
|
||||
// In Electron/Chromium, innerText preserves basic visual line breaks nicely
|
||||
return container.innerText;
|
||||
}
|
||||
`);
|
||||
// We can do simple heuristic parsing based on typical visual markers if needed.
|
||||
// For now, we return the entire text blob, or just the last 2000 characters if it's too long.
|
||||
const cleanText = String(rawText).trim();
|
||||
return [{
|
||||
role: 'history',
|
||||
content: cleanText
|
||||
}];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,36 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
export const readCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'read',
|
||||
description: 'Read the latest chat messages from Antigravity AI',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'last', help: 'Number of recent messages to read (not fully implemented due to generic structure, currently returns full history text or latest chunk)' }
|
||||
],
|
||||
columns: ['role', 'content'],
|
||||
func: async (page, kwargs) => {
|
||||
// We execute a script inside Antigravity's Chromium environment to extract the text
|
||||
// of the entire conversation pane.
|
||||
const rawText = await page.evaluate(`
|
||||
async () => {
|
||||
const container = document.getElementById('conversation');
|
||||
if (!container) throw new Error('Could not find conversation container');
|
||||
|
||||
// Extract the full visible text of the conversation
|
||||
// In Electron/Chromium, innerText preserves basic visual line breaks nicely
|
||||
return container.innerText;
|
||||
}
|
||||
`);
|
||||
|
||||
// We can do simple heuristic parsing based on typical visual markers if needed.
|
||||
// For now, we return the entire text blob, or just the last 2000 characters if it's too long.
|
||||
const cleanText = String(rawText).trim();
|
||||
return [{
|
||||
role: 'history',
|
||||
content: cleanText
|
||||
}];
|
||||
},
|
||||
});
|
||||
@@ -1,35 +0,0 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
export const sendCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'send',
|
||||
description: 'Send a message to Antigravity AI via the internal Lexical editor',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'message', help: 'The message text to send', required: true, positional: true }
|
||||
],
|
||||
columns: ['Status', 'Message'],
|
||||
func: async (page, kwargs) => {
|
||||
const text = kwargs.message;
|
||||
// We use evaluate to focus and insert text because Lexical editors maintain
|
||||
// absolute control over their DOM and don't respond to raw node.textContent.
|
||||
// document.execCommand simulates a native paste/typing action perfectly.
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
if (!container) throw new Error('Could not find antigravity.agentSidePanelInputBox');
|
||||
const editor = container.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find Antigravity input box');
|
||||
|
||||
editor.focus();
|
||||
document.execCommand('insertText', false, ${JSON.stringify(text)});
|
||||
}
|
||||
`);
|
||||
// Wait for the React/Lexical state to flush the new input
|
||||
await page.wait(0.5);
|
||||
// Press Enter to submit the message
|
||||
await page.pressKey('Enter');
|
||||
return [{ Status: 'Sent successfully', Message: text }];
|
||||
},
|
||||
});
|
||||
@@ -0,0 +1,40 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
|
||||
export const sendCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'send',
|
||||
description: 'Send a message to Antigravity AI via the internal Lexical editor',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [
|
||||
{ name: 'message', help: 'The message text to send', required: true, positional: true }
|
||||
],
|
||||
columns: ['Status', 'Message'],
|
||||
func: async (page, kwargs) => {
|
||||
const text = kwargs.message;
|
||||
|
||||
// We use evaluate to focus and insert text because Lexical editors maintain
|
||||
// absolute control over their DOM and don't respond to raw node.textContent.
|
||||
// document.execCommand simulates a native paste/typing action perfectly.
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
if (!container) throw new Error('Could not find antigravity.agentSidePanelInputBox');
|
||||
const editor = container.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find Antigravity input box');
|
||||
|
||||
editor.focus();
|
||||
document.execCommand('insertText', false, ${JSON.stringify(text)});
|
||||
}
|
||||
`);
|
||||
|
||||
// Wait for the React/Lexical state to flush the new input
|
||||
await page.wait(0.5);
|
||||
|
||||
// Press Enter to submit the message
|
||||
await page.pressKey('Enter');
|
||||
|
||||
return [{ Status: 'Sent successfully', Message: text }];
|
||||
},
|
||||
});
|
||||
@@ -1,558 +0,0 @@
|
||||
/**
|
||||
* antigravity serve — Anthropic-compatible `/v1/messages` proxy server.
|
||||
*
|
||||
* Starts an HTTP server that accepts Anthropic Messages API requests,
|
||||
* forwards them to a running Antigravity app via CDP, polls for the response,
|
||||
* and returns it in Anthropic format.
|
||||
*
|
||||
* Usage:
|
||||
* opencli antigravity serve --port 8082
|
||||
* ANTHROPIC_BASE_URL=http://localhost:8082 claude
|
||||
*/
|
||||
import { createServer } from 'node:http';
|
||||
import { CDPBridge } from '@jackwener/opencli/browser/cdp';
|
||||
import { resolveElectronEndpoint } from '@jackwener/opencli/launcher';
|
||||
import { EXIT_CODES, getErrorMessage } from '@jackwener/opencli/errors';
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
function generateMsgId() {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
|
||||
let id = 'msg_';
|
||||
for (let i = 0; i < 24; i++)
|
||||
id += chars[Math.floor(Math.random() * chars.length)];
|
||||
return id;
|
||||
}
|
||||
function estimateTokens(text) {
|
||||
// Rough approximation: ~4 chars per token for English, ~2 for CJK
|
||||
return Math.max(1, Math.ceil(text.length / 3));
|
||||
}
|
||||
function extractTextContent(content) {
|
||||
if (typeof content === 'string')
|
||||
return content;
|
||||
return content
|
||||
.filter(b => b.type === 'text' && b.text)
|
||||
.map(b => b.text)
|
||||
.join('\n');
|
||||
}
|
||||
function readBody(req) {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks = [];
|
||||
req.on('data', (c) => chunks.push(c));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
function jsonResponse(res, status, data) {
|
||||
const body = JSON.stringify(data);
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, x-api-key, anthropic-version, Authorization',
|
||||
});
|
||||
res.end(body);
|
||||
}
|
||||
function sleep(ms) {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
function parseTimeoutValue(val, label, fallback) {
|
||||
if (val === undefined) {
|
||||
return fallback;
|
||||
}
|
||||
const parsed = typeof val === 'number' ? val : parseInt(String(val), 10);
|
||||
if (Number.isNaN(parsed) || parsed <= 0) {
|
||||
console.error(`[serve] Invalid ${label}="${val}", using default ${fallback}s`);
|
||||
return fallback;
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
function parseEnvTimeout(envVar, fallback) {
|
||||
return parseTimeoutValue(process.env[envVar], envVar, fallback);
|
||||
}
|
||||
// ─── DOM helpers ─────────────────────────────────────────────────────
|
||||
/**
|
||||
* Click the 'New Conversation' button to reset context.
|
||||
*/
|
||||
async function startNewConversation(page) {
|
||||
await page.evaluate(`
|
||||
(() => {
|
||||
const btn = document.querySelector('[data-tooltip-id="new-conversation-tooltip"]');
|
||||
if (btn) btn.click();
|
||||
})()
|
||||
`);
|
||||
await sleep(1000); // Give UI time to clear
|
||||
}
|
||||
/**
|
||||
* Switch the active model in Antigravity UI.
|
||||
*/
|
||||
async function switchModel(page, anthropicModelId) {
|
||||
// Map standard model IDs to Antigravity UI names based on actual UI
|
||||
let targetName = 'claude sonnet 4.6'; // Default fallback
|
||||
const id = anthropicModelId.toLowerCase();
|
||||
if (id.includes('sonnet')) {
|
||||
targetName = 'claude sonnet 4.6';
|
||||
}
|
||||
else if (id.includes('opus')) {
|
||||
targetName = 'claude opus 4.6';
|
||||
}
|
||||
else if (id.includes('gemini') && id.includes('pro')) {
|
||||
targetName = 'gemini 3.1 pro (high)';
|
||||
}
|
||||
else if (id.includes('gemini') && id.includes('flash')) {
|
||||
targetName = 'gemini 3 flash';
|
||||
}
|
||||
else if (id.includes('gpt')) {
|
||||
targetName = 'gpt-oss 120b';
|
||||
}
|
||||
try {
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const targetModelName = ${JSON.stringify(targetName)};
|
||||
const trigger = document.querySelector('div[aria-haspopup="dialog"] > div[tabindex="0"]');
|
||||
if (!trigger) return; // Silent fail if UI changed
|
||||
|
||||
// Open dropdown only if not already selected
|
||||
if (trigger.innerText.toLowerCase().includes(targetModelName)) return;
|
||||
|
||||
trigger.click();
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
|
||||
const spans = Array.from(document.querySelectorAll('[role="dialog"] span'));
|
||||
const target = spans.find(s => s.innerText.toLowerCase().includes(targetModelName));
|
||||
if (target) {
|
||||
const optionNode = target.closest('.cursor-pointer') || target;
|
||||
optionNode.click();
|
||||
} else {
|
||||
// Close if not found
|
||||
trigger.click();
|
||||
}
|
||||
}
|
||||
`);
|
||||
await sleep(500); // Wait for switch
|
||||
}
|
||||
catch (err) {
|
||||
console.error(`[serve] Warning: Could not switch to model ${targetName}:`, err);
|
||||
}
|
||||
}
|
||||
/**
|
||||
* Check if the Antigravity UI is currently generating a response
|
||||
* by looking for Stop/Cancel buttons or loading indicators.
|
||||
*/
|
||||
async function isGenerating(page) {
|
||||
const result = await page.evaluate(`
|
||||
(() => {
|
||||
// Look for a cancel/stop button in the UI
|
||||
const cancelBtn = document.querySelector('button[aria-label*="cancel" i], button[aria-label*="stop" i], button[title*="cancel" i], button[title*="stop" i]');
|
||||
return !!cancelBtn;
|
||||
})()
|
||||
`);
|
||||
return Boolean(result);
|
||||
}
|
||||
/**
|
||||
* Walk from the scroll container and find the deepest element that
|
||||
* has multiple non-empty children (our message container).
|
||||
*/
|
||||
function findMessageContainer(root, depth = 0) {
|
||||
if (!root || depth > 12)
|
||||
return null;
|
||||
const nonEmpty = Array.from(root.children).filter(c => c.innerText?.trim().length > 5);
|
||||
if (nonEmpty.length >= 2)
|
||||
return root;
|
||||
if (nonEmpty.length === 1)
|
||||
return findMessageContainer(nonEmpty[0], depth + 1);
|
||||
return root;
|
||||
}
|
||||
// ─── Antigravity CDP Operations ──────────────────────────────────────
|
||||
/**
|
||||
* Get the full chat text for change-detection polling.
|
||||
*/
|
||||
async function getConversationText(page) {
|
||||
const text = await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('conversation');
|
||||
if (!container) return '';
|
||||
// Read only the first child div (actual chat content),
|
||||
// skipping UI chrome like file change panels, model selectors, etc.
|
||||
const chatContent = container.children[0];
|
||||
return chatContent ? chatContent.innerText : container.innerText;
|
||||
})()
|
||||
`);
|
||||
return String(text ?? '');
|
||||
}
|
||||
/**
|
||||
* Get the text of the last assistant reply by navigating to the message container
|
||||
* and extracting the last non-empty message block.
|
||||
*/
|
||||
async function getLastAssistantReply(page, userText) {
|
||||
const text = await page.evaluate(`
|
||||
(() => {
|
||||
const conv = document.getElementById('conversation')?.children[0];
|
||||
const scroll = conv?.querySelector('.overflow-y-auto');
|
||||
|
||||
// Walk down until we find a container with multiple message siblings
|
||||
function findMsgContainer(el, depth) {
|
||||
if (!el || depth > 12) return null;
|
||||
const nonEmpty = Array.from(el.children).filter(c => c.innerText && c.innerText.trim().length > 5);
|
||||
if (nonEmpty.length >= 2) return el;
|
||||
if (nonEmpty.length === 1) return findMsgContainer(nonEmpty[0], depth + 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
const container = findMsgContainer(scroll || conv, 0);
|
||||
if (!container) return '';
|
||||
|
||||
// Get all non-empty children (skip trailing empty UI divs)
|
||||
const msgs = Array.from(container.children).filter(
|
||||
c => c.innerText && c.innerText.trim().length > 5
|
||||
);
|
||||
|
||||
if (msgs.length === 0) return '';
|
||||
|
||||
// The last element is the last assistant reply
|
||||
const last = msgs[msgs.length - 1];
|
||||
return last.innerText || '';
|
||||
})()
|
||||
`);
|
||||
let reply = String(text ?? '').trim();
|
||||
// Strip echoed user message from the top (Antigravity sometimes includes it)
|
||||
if (userText && reply.startsWith(userText)) {
|
||||
reply = reply.slice(userText.length).trim();
|
||||
}
|
||||
// Strip thinking block: "Thought for Xs\n..." at the start
|
||||
reply = reply.replace(/^Thought for[^\n]*\n+/i, '').trim();
|
||||
// Strip "Copy" button text at the end
|
||||
reply = reply.replace(/\s*\bCopy\b\s*$/m, '').trim();
|
||||
// De-duplicate trailing repeated content (e.g., "OK\n\nOK" → "OK")
|
||||
const half = Math.floor(reply.length / 2);
|
||||
const firstHalf = reply.slice(0, half).trim();
|
||||
const secondHalf = reply.slice(half).trim();
|
||||
if (firstHalf && firstHalf === secondHalf) {
|
||||
reply = firstHalf;
|
||||
}
|
||||
return reply;
|
||||
}
|
||||
async function sendMessage(page, message, bridge) {
|
||||
if (!bridge) {
|
||||
// Fallback: use JS-based approach
|
||||
await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
const editor = container?.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find input box');
|
||||
editor.focus();
|
||||
document.execCommand('insertText', false, ${JSON.stringify(message)});
|
||||
})()
|
||||
`);
|
||||
await sleep(500);
|
||||
await page.pressKey('Enter');
|
||||
return;
|
||||
}
|
||||
// Get the bounding box of the Lexical editor for a physical mouse click
|
||||
const rect = await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
if (!container) throw new Error('Could not find antigravity.agentSidePanelInputBox');
|
||||
const editor = container.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find Antigravity input box');
|
||||
const r = editor.getBoundingClientRect();
|
||||
return JSON.stringify({ x: r.left + r.width / 2, y: r.top + r.height / 2 });
|
||||
})()
|
||||
`);
|
||||
const { x, y } = JSON.parse(String(rect));
|
||||
// Physical mouse click to give the element real browser focus
|
||||
await bridge.send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
|
||||
await sleep(50);
|
||||
await bridge.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 });
|
||||
await sleep(200);
|
||||
// Inject text at the CDP level (no deprecated execCommand)
|
||||
await bridge.send('Input.insertText', { text: message });
|
||||
await sleep(300);
|
||||
// Send Enter via native CDP key event
|
||||
await bridge.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 });
|
||||
await sleep(50);
|
||||
await bridge.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 });
|
||||
}
|
||||
async function waitForReply(page, beforeText, opts = {}) {
|
||||
const timeout = opts.timeout ?? 120_000; // 2 minutes max
|
||||
const pollInterval = opts.pollInterval ?? 500; // 500ms polling
|
||||
const deadline = Date.now() + timeout;
|
||||
// Wait a bit to ensure the UI transitions to "generating" state after we hit Enter
|
||||
await sleep(1000);
|
||||
let hasStartedGenerating = false;
|
||||
let lastText = beforeText;
|
||||
let stableCount = 0;
|
||||
const stableThreshold = 4; // 4 * 500ms = 2s of stability fallback
|
||||
let reconnectCount = 0;
|
||||
while (Date.now() < deadline) {
|
||||
try {
|
||||
const generating = await isGenerating(page);
|
||||
const currentText = await getConversationText(page);
|
||||
const textChanged = currentText !== beforeText && currentText.length > 0;
|
||||
if (generating) {
|
||||
hasStartedGenerating = true;
|
||||
stableCount = 0; // Reset stability while generating
|
||||
}
|
||||
else {
|
||||
if (hasStartedGenerating) {
|
||||
// It actively generated and now it stopped -> DONE
|
||||
// Provide a small buffer to let React render the final message fully
|
||||
await sleep(500);
|
||||
return page;
|
||||
}
|
||||
// Fallback: If it never showed "Generating/Cancel", but text changed and is stable
|
||||
if (textChanged) {
|
||||
if (currentText === lastText) {
|
||||
stableCount++;
|
||||
if (stableCount >= stableThreshold) {
|
||||
return page; // Text has been stable for 2 seconds -> DONE
|
||||
}
|
||||
}
|
||||
else {
|
||||
stableCount = 0;
|
||||
lastText = currentText;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
catch (err) {
|
||||
const msg = err.message || String(err);
|
||||
const isSessionLoss = /closed|lost|not open|websocket/i.test(msg);
|
||||
if (opts.reconnect && isSessionLoss && reconnectCount < 2) {
|
||||
reconnectCount++;
|
||||
console.error(`[serve] CDP session loss detected (${msg}), attempting to reconnect (${reconnectCount}/2)...`);
|
||||
try {
|
||||
page = await opts.reconnect();
|
||||
// Reset stability tracking after reconnect
|
||||
stableCount = 0;
|
||||
lastText = beforeText;
|
||||
continue;
|
||||
}
|
||||
catch (reconnectErr) {
|
||||
console.error(`[serve] Reconnection failed: ${reconnectErr.message}`);
|
||||
throw err; // Throw original error if reconnection itself fails
|
||||
}
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
await sleep(pollInterval);
|
||||
}
|
||||
throw new Error(`Timeout waiting for Antigravity reply after ${timeout / 1000}s`);
|
||||
}
|
||||
// ─── Request Handlers ────────────────────────────────────────────────
|
||||
async function handleMessages(body, page, opts = {}) {
|
||||
const { bridge, timeout, reconnect } = opts;
|
||||
// Extract the last user message
|
||||
const userMessages = body.messages.filter(m => m.role === 'user');
|
||||
if (userMessages.length === 0) {
|
||||
throw new Error('No user message found in request');
|
||||
}
|
||||
const lastUserMsg = userMessages[userMessages.length - 1];
|
||||
const userText = extractTextContent(lastUserMsg.content);
|
||||
if (!userText.trim()) {
|
||||
throw new Error('Empty user message');
|
||||
}
|
||||
// Optimization 1: New conversation if this is the first message in the session
|
||||
if (body.messages.length === 1) {
|
||||
console.error(`[serve] New session detected (1 message). Starting new conversation in UI.`);
|
||||
await startNewConversation(page);
|
||||
}
|
||||
// Optimization 3: Switch model if requested
|
||||
if (body.model) {
|
||||
await switchModel(page, body.model);
|
||||
}
|
||||
// Get conversation state before sending
|
||||
const beforeText = await getConversationText(page);
|
||||
// Send the message
|
||||
console.error(`[serve] Sending: "${userText.slice(0, 80)}${userText.length > 80 ? '...' : ''}"`);
|
||||
await sendMessage(page, userText, bridge);
|
||||
// Poll for reply (change detection)
|
||||
console.error('[serve] Waiting for reply...');
|
||||
page = await waitForReply(page, beforeText, { timeout, reconnect });
|
||||
// Extract the actual reply text precisely from the DOM
|
||||
const replyText = await getLastAssistantReply(page, userText);
|
||||
console.error(`[serve] Got reply: "${replyText.slice(0, 80)}${replyText.length > 80 ? '...' : ''}"`);
|
||||
return {
|
||||
id: generateMsgId(),
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: replyText }],
|
||||
model: body.model ?? 'antigravity',
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: estimateTokens(userText),
|
||||
output_tokens: estimateTokens(replyText),
|
||||
},
|
||||
};
|
||||
}
|
||||
// ─── Server ──────────────────────────────────────────────────────────
|
||||
export async function startServe(opts = {}) {
|
||||
const port = opts.port ?? 8082;
|
||||
const envTimeoutSeconds = parseEnvTimeout('OPENCLI_ANTIGRAVITY_TIMEOUT', 120);
|
||||
const effectiveTimeoutSeconds = parseTimeoutValue(opts.timeout, '--timeout', envTimeoutSeconds);
|
||||
const effectiveTimeout = effectiveTimeoutSeconds * 1000;
|
||||
console.error(`[serve] Starting Antigravity API proxy on port ${port} (timeout: ${effectiveTimeout / 1000}s)`);
|
||||
// Lazy CDP connection — connect when first request comes in
|
||||
let cdp = null;
|
||||
let page = null;
|
||||
let requestInFlight = false;
|
||||
async function ensureConnected() {
|
||||
if (page) {
|
||||
try {
|
||||
await page.evaluate('1+1');
|
||||
return page;
|
||||
}
|
||||
catch {
|
||||
console.error('[serve] CDP connection lost, reconnecting...');
|
||||
cdp?.close().catch(() => { });
|
||||
cdp = null;
|
||||
page = null;
|
||||
}
|
||||
}
|
||||
const endpoint = await resolveElectronEndpoint('antigravity');
|
||||
// Note: Antigravity chat panel lives inside editor windows, not in Launchpad.
|
||||
// If multiple editor windows are open, set OPENCLI_CDP_TARGET to the window title.
|
||||
if (process.env.OPENCLI_CDP_TARGET) {
|
||||
console.error(`[serve] Using OPENCLI_CDP_TARGET=${process.env.OPENCLI_CDP_TARGET}`);
|
||||
}
|
||||
// List available targets for debugging
|
||||
try {
|
||||
const res = await fetch(`${endpoint.replace(/\/$/, '')}/json`);
|
||||
const targets = await res.json();
|
||||
const pages = targets.filter(t => t.type === 'page');
|
||||
console.error(`[serve] Available targets: ${pages.map(t => `"${t.title}"`).join(', ')}`);
|
||||
}
|
||||
catch { /* ignore */ }
|
||||
console.error(`[serve] Connecting via CDP (target pattern: "${process.env.OPENCLI_CDP_TARGET}")...`);
|
||||
cdp = new CDPBridge();
|
||||
try {
|
||||
page = await cdp.connect({ timeout: 15_000, cdpEndpoint: endpoint });
|
||||
}
|
||||
catch (err) {
|
||||
cdp = null;
|
||||
const errMsg = getErrorMessage(err);
|
||||
const cause = err instanceof Error ? err.cause : undefined;
|
||||
const isRefused = cause?.code === 'ECONNREFUSED' || errMsg.includes('ECONNREFUSED');
|
||||
throw new Error(isRefused
|
||||
? `Cannot connect to Antigravity at ${endpoint}.\n` +
|
||||
' 1. Make sure Antigravity is running\n' +
|
||||
' 2. Launch with: --remote-debugging-port=9234'
|
||||
: `CDP connection failed: ${errMsg}`);
|
||||
}
|
||||
console.error('[serve] ✅ CDP connected.');
|
||||
// Quick verification
|
||||
const hasUI = await page.evaluate(`
|
||||
(() => !!document.getElementById('conversation') || !!document.getElementById('antigravity.agentSidePanelInputBox'))()
|
||||
`);
|
||||
if (!hasUI) {
|
||||
console.error('[serve] ⚠️ Warning: chat UI elements not found in this target. Try setting OPENCLI_CDP_TARGET to the correct window title.');
|
||||
}
|
||||
return page;
|
||||
}
|
||||
const server = createServer(async (req, res) => {
|
||||
// CORS preflight
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204, {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, x-api-key, anthropic-version, Authorization',
|
||||
});
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
const url = req.url ?? '/';
|
||||
const pathname = url.split('?')[0];
|
||||
try {
|
||||
// GET /v1/models — return available models
|
||||
if (req.method === 'GET' && pathname === '/v1/models') {
|
||||
jsonResponse(res, 200, {
|
||||
data: [
|
||||
{
|
||||
id: 'antigravity',
|
||||
object: 'model',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
owned_by: 'antigravity',
|
||||
},
|
||||
],
|
||||
});
|
||||
return;
|
||||
}
|
||||
// POST /v1/messages — main endpoint
|
||||
if (req.method === 'POST' && pathname === '/v1/messages') {
|
||||
if (requestInFlight) {
|
||||
jsonResponse(res, 429, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'rate_limit_error',
|
||||
message: 'Another request is currently being processed. Antigravity can only handle one request at a time.',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
requestInFlight = true;
|
||||
try {
|
||||
const rawBody = await readBody(req);
|
||||
const body = JSON.parse(rawBody);
|
||||
if (body.stream) {
|
||||
jsonResponse(res, 400, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'invalid_request_error',
|
||||
message: 'Streaming is not supported. Set "stream": false.',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
// Lazy connect on first request
|
||||
const activePage = await ensureConnected();
|
||||
const response = await handleMessages(body, activePage, {
|
||||
bridge: cdp,
|
||||
timeout: effectiveTimeout,
|
||||
reconnect: ensureConnected,
|
||||
});
|
||||
jsonResponse(res, 200, response);
|
||||
}
|
||||
finally {
|
||||
requestInFlight = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
// Health check
|
||||
if (req.method === 'GET' && (pathname === '/' || pathname === '/health')) {
|
||||
jsonResponse(res, 200, { ok: true, cdpConnected: page !== null });
|
||||
return;
|
||||
}
|
||||
jsonResponse(res, 404, {
|
||||
type: 'error',
|
||||
error: { type: 'not_found_error', message: `Not found: ${pathname}` },
|
||||
});
|
||||
}
|
||||
catch (err) {
|
||||
console.error('[serve] Error:', err instanceof Error ? err.message : err);
|
||||
jsonResponse(res, 500, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'api_error',
|
||||
message: err instanceof Error ? err.message : 'Internal server error',
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
console.error(`\n[serve] ✅ Antigravity API proxy running at http://127.0.0.1:${port}`);
|
||||
console.error(`[serve] Compatible with Anthropic /v1/messages API`);
|
||||
console.error(`[serve] CDP connection will be established on first request.`);
|
||||
console.error(`\n[serve] Usage with Claude Code:`);
|
||||
console.error(` ANTHROPIC_BASE_URL=http://localhost:${port} claude\n`);
|
||||
});
|
||||
// Graceful shutdown
|
||||
const shutdown = () => {
|
||||
console.error('\n[serve] Shutting down...');
|
||||
cdp?.close().catch(() => { });
|
||||
server.close();
|
||||
process.exit(EXIT_CODES.SUCCESS);
|
||||
};
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
// Keep alive
|
||||
await new Promise(() => { });
|
||||
}
|
||||
@@ -0,0 +1,600 @@
|
||||
/**
|
||||
* antigravity serve — Anthropic-compatible `/v1/messages` proxy server.
|
||||
*
|
||||
* Starts an HTTP server that accepts Anthropic Messages API requests,
|
||||
* forwards them to a running Antigravity app via CDP, polls for the response,
|
||||
* and returns it in Anthropic format.
|
||||
*
|
||||
* Usage:
|
||||
* opencli antigravity serve --port 8082
|
||||
* ANTHROPIC_BASE_URL=http://localhost:8082 claude
|
||||
*/
|
||||
|
||||
import { createServer, type IncomingMessage, type ServerResponse } from 'node:http';
|
||||
import { CDPBridge } from '@jackwener/opencli/browser/cdp';
|
||||
import type { IPage } from '@jackwener/opencli/types';
|
||||
import { resolveElectronEndpoint } from '@jackwener/opencli/launcher';
|
||||
import { EXIT_CODES, getErrorMessage } from '@jackwener/opencli/errors';
|
||||
|
||||
// ─── Types ───────────────────────────────────────────────────────────
|
||||
|
||||
interface AnthropicRequest {
|
||||
model?: string;
|
||||
max_tokens?: number;
|
||||
system?: string | Array<{ type: string; text: string }>;
|
||||
messages: Array<{ role: string; content: string | Array<{ type: string; text?: string }> }>;
|
||||
stream?: boolean;
|
||||
}
|
||||
|
||||
interface AnthropicResponse {
|
||||
id: string;
|
||||
type: 'message';
|
||||
role: 'assistant';
|
||||
content: Array<{ type: 'text'; text: string }>;
|
||||
model: string;
|
||||
stop_reason: 'end_turn' | 'max_tokens' | 'stop_sequence';
|
||||
stop_sequence: null;
|
||||
usage: { input_tokens: number; output_tokens: number };
|
||||
}
|
||||
|
||||
// ─── Helpers ─────────────────────────────────────────────────────────
|
||||
|
||||
function generateMsgId(): string {
|
||||
const chars = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789';
|
||||
let id = 'msg_';
|
||||
for (let i = 0; i < 24; i++) id += chars[Math.floor(Math.random() * chars.length)];
|
||||
return id;
|
||||
}
|
||||
|
||||
function estimateTokens(text: string): number {
|
||||
// Rough approximation: ~4 chars per token for English, ~2 for CJK
|
||||
return Math.max(1, Math.ceil(text.length / 3));
|
||||
}
|
||||
|
||||
function extractTextContent(content: string | Array<{ type: string; text?: string }>): string {
|
||||
if (typeof content === 'string') return content;
|
||||
return content
|
||||
.filter(b => b.type === 'text' && b.text)
|
||||
.map(b => b.text!)
|
||||
.join('\n');
|
||||
}
|
||||
|
||||
function readBody(req: IncomingMessage): Promise<string> {
|
||||
return new Promise((resolve, reject) => {
|
||||
const chunks: Buffer[] = [];
|
||||
req.on('data', (c: Buffer) => chunks.push(c));
|
||||
req.on('end', () => resolve(Buffer.concat(chunks).toString('utf-8')));
|
||||
req.on('error', reject);
|
||||
});
|
||||
}
|
||||
|
||||
function jsonResponse(res: ServerResponse, status: number, data: unknown): void {
|
||||
const body = JSON.stringify(data);
|
||||
res.writeHead(status, {
|
||||
'Content-Type': 'application/json',
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, x-api-key, anthropic-version, Authorization',
|
||||
});
|
||||
res.end(body);
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise(resolve => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
// ─── DOM helpers ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Click the 'New Conversation' button to reset context.
|
||||
*/
|
||||
async function startNewConversation(page: IPage): Promise<void> {
|
||||
await page.evaluate(`
|
||||
(() => {
|
||||
const btn = document.querySelector('[data-tooltip-id="new-conversation-tooltip"]');
|
||||
if (btn) btn.click();
|
||||
})()
|
||||
`);
|
||||
await sleep(1000); // Give UI time to clear
|
||||
}
|
||||
|
||||
/**
|
||||
* Switch the active model in Antigravity UI.
|
||||
*/
|
||||
async function switchModel(page: IPage, anthropicModelId: string): Promise<void> {
|
||||
// Map standard model IDs to Antigravity UI names based on actual UI
|
||||
let targetName = 'claude sonnet 4.6'; // Default fallback
|
||||
const id = anthropicModelId.toLowerCase();
|
||||
|
||||
if (id.includes('sonnet')) {
|
||||
targetName = 'claude sonnet 4.6';
|
||||
} else if (id.includes('opus')) {
|
||||
targetName = 'claude opus 4.6';
|
||||
} else if (id.includes('gemini') && id.includes('pro')) {
|
||||
targetName = 'gemini 3.1 pro (high)';
|
||||
} else if (id.includes('gemini') && id.includes('flash')) {
|
||||
targetName = 'gemini 3 flash';
|
||||
} else if (id.includes('gpt')) {
|
||||
targetName = 'gpt-oss 120b';
|
||||
}
|
||||
|
||||
try {
|
||||
await page.evaluate(`
|
||||
async () => {
|
||||
const targetModelName = ${JSON.stringify(targetName)};
|
||||
const trigger = document.querySelector('div[aria-haspopup="dialog"] > div[tabindex="0"]');
|
||||
if (!trigger) return; // Silent fail if UI changed
|
||||
|
||||
// Open dropdown only if not already selected
|
||||
if (trigger.innerText.toLowerCase().includes(targetModelName)) return;
|
||||
|
||||
trigger.click();
|
||||
await new Promise(r => setTimeout(r, 200));
|
||||
|
||||
const spans = Array.from(document.querySelectorAll('[role="dialog"] span'));
|
||||
const target = spans.find(s => s.innerText.toLowerCase().includes(targetModelName));
|
||||
if (target) {
|
||||
const optionNode = target.closest('.cursor-pointer') || target;
|
||||
optionNode.click();
|
||||
} else {
|
||||
// Close if not found
|
||||
trigger.click();
|
||||
}
|
||||
}
|
||||
`);
|
||||
await sleep(500); // Wait for switch
|
||||
} catch (err) {
|
||||
console.error(`[serve] Warning: Could not switch to model ${targetName}:`, err);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Check if the Antigravity UI is currently generating a response
|
||||
* by looking for Stop/Cancel buttons or loading indicators.
|
||||
*/
|
||||
async function isGenerating(page: IPage): Promise<boolean> {
|
||||
const result = await page.evaluate(`
|
||||
(() => {
|
||||
// Look for a cancel/stop button in the UI
|
||||
const cancelBtn = document.querySelector('button[aria-label*="cancel" i], button[aria-label*="stop" i], button[title*="cancel" i], button[title*="stop" i]');
|
||||
return !!cancelBtn;
|
||||
})()
|
||||
`);
|
||||
return Boolean(result);
|
||||
}
|
||||
|
||||
/**
|
||||
* Walk from the scroll container and find the deepest element that
|
||||
* has multiple non-empty children (our message container).
|
||||
*/
|
||||
function findMessageContainer(root: Element | null, depth = 0): Element | null {
|
||||
if (!root || depth > 12) return null;
|
||||
const nonEmpty = Array.from(root.children).filter(
|
||||
c => (c as HTMLElement).innerText?.trim().length > 5
|
||||
);
|
||||
if (nonEmpty.length >= 2) return root;
|
||||
if (nonEmpty.length === 1) return findMessageContainer(nonEmpty[0], depth + 1);
|
||||
return root;
|
||||
}
|
||||
|
||||
// ─── Antigravity CDP Operations ──────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Get the full chat text for change-detection polling.
|
||||
*/
|
||||
async function getConversationText(page: IPage): Promise<string> {
|
||||
const text = await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('conversation');
|
||||
if (!container) return '';
|
||||
// Read only the first child div (actual chat content),
|
||||
// skipping UI chrome like file change panels, model selectors, etc.
|
||||
const chatContent = container.children[0];
|
||||
return chatContent ? chatContent.innerText : container.innerText;
|
||||
})()
|
||||
`);
|
||||
return String(text ?? '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Get the text of the last assistant reply by navigating to the message container
|
||||
* and extracting the last non-empty message block.
|
||||
*/
|
||||
async function getLastAssistantReply(page: IPage, userText?: string): Promise<string> {
|
||||
const text = await page.evaluate(`
|
||||
(() => {
|
||||
const conv = document.getElementById('conversation')?.children[0];
|
||||
const scroll = conv?.querySelector('.overflow-y-auto');
|
||||
|
||||
// Walk down until we find a container with multiple message siblings
|
||||
function findMsgContainer(el, depth) {
|
||||
if (!el || depth > 12) return null;
|
||||
const nonEmpty = Array.from(el.children).filter(c => c.innerText && c.innerText.trim().length > 5);
|
||||
if (nonEmpty.length >= 2) return el;
|
||||
if (nonEmpty.length === 1) return findMsgContainer(nonEmpty[0], depth + 1);
|
||||
return null;
|
||||
}
|
||||
|
||||
const container = findMsgContainer(scroll || conv, 0);
|
||||
if (!container) return '';
|
||||
|
||||
// Get all non-empty children (skip trailing empty UI divs)
|
||||
const msgs = Array.from(container.children).filter(
|
||||
c => c.innerText && c.innerText.trim().length > 5
|
||||
);
|
||||
|
||||
if (msgs.length === 0) return '';
|
||||
|
||||
// The last element is the last assistant reply
|
||||
const last = msgs[msgs.length - 1];
|
||||
return last.innerText || '';
|
||||
})()
|
||||
`);
|
||||
let reply = String(text ?? '').trim();
|
||||
|
||||
// Strip echoed user message from the top (Antigravity sometimes includes it)
|
||||
if (userText && reply.startsWith(userText)) {
|
||||
reply = reply.slice(userText.length).trim();
|
||||
}
|
||||
|
||||
// Strip thinking block: "Thought for Xs\n..." at the start
|
||||
reply = reply.replace(/^Thought for[^\n]*\n+/i, '').trim();
|
||||
|
||||
// Strip "Copy" button text at the end
|
||||
reply = reply.replace(/\s*\bCopy\b\s*$/m, '').trim();
|
||||
|
||||
// De-duplicate trailing repeated content (e.g., "OK\n\nOK" → "OK")
|
||||
const half = Math.floor(reply.length / 2);
|
||||
const firstHalf = reply.slice(0, half).trim();
|
||||
const secondHalf = reply.slice(half).trim();
|
||||
if (firstHalf && firstHalf === secondHalf) {
|
||||
reply = firstHalf;
|
||||
}
|
||||
|
||||
return reply;
|
||||
}
|
||||
|
||||
async function sendMessage(page: IPage, message: string, bridge?: CDPBridge): Promise<void> {
|
||||
if (!bridge) {
|
||||
// Fallback: use JS-based approach
|
||||
await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
const editor = container?.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find input box');
|
||||
editor.focus();
|
||||
document.execCommand('insertText', false, ${JSON.stringify(message)});
|
||||
})()
|
||||
`);
|
||||
await sleep(500);
|
||||
await page.pressKey('Enter');
|
||||
return;
|
||||
}
|
||||
|
||||
// Get the bounding box of the Lexical editor for a physical mouse click
|
||||
const rect = await page.evaluate(`
|
||||
(() => {
|
||||
const container = document.getElementById('antigravity.agentSidePanelInputBox');
|
||||
if (!container) throw new Error('Could not find antigravity.agentSidePanelInputBox');
|
||||
const editor = container.querySelector('[data-lexical-editor="true"]');
|
||||
if (!editor) throw new Error('Could not find Antigravity input box');
|
||||
const r = editor.getBoundingClientRect();
|
||||
return JSON.stringify({ x: r.left + r.width / 2, y: r.top + r.height / 2 });
|
||||
})()
|
||||
`);
|
||||
const { x, y } = JSON.parse(String(rect));
|
||||
|
||||
// Physical mouse click to give the element real browser focus
|
||||
await bridge.send('Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button: 'left', clickCount: 1 });
|
||||
await sleep(50);
|
||||
await bridge.send('Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button: 'left', clickCount: 1 });
|
||||
await sleep(200);
|
||||
|
||||
// Inject text at the CDP level (no deprecated execCommand)
|
||||
await bridge.send('Input.insertText', { text: message });
|
||||
await sleep(300);
|
||||
|
||||
// Send Enter via native CDP key event
|
||||
await bridge.send('Input.dispatchKeyEvent', { type: 'keyDown', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 });
|
||||
await sleep(50);
|
||||
await bridge.send('Input.dispatchKeyEvent', { type: 'keyUp', key: 'Enter', code: 'Enter', windowsVirtualKeyCode: 13, nativeVirtualKeyCode: 13 });
|
||||
}
|
||||
|
||||
async function waitForReply(
|
||||
page: IPage,
|
||||
beforeText: string,
|
||||
opts: { timeout?: number; pollInterval?: number } = {},
|
||||
): Promise<void> {
|
||||
const timeout = opts.timeout ?? 120_000; // 2 minutes max
|
||||
const pollInterval = opts.pollInterval ?? 500; // 500ms polling
|
||||
|
||||
const deadline = Date.now() + timeout;
|
||||
|
||||
// Wait a bit to ensure the UI transitions to "generating" state after we hit Enter
|
||||
await sleep(1000);
|
||||
|
||||
let hasStartedGenerating = false;
|
||||
let lastText = beforeText;
|
||||
let stableCount = 0;
|
||||
const stableThreshold = 4; // 4 * 500ms = 2s of stability fallback
|
||||
|
||||
while (Date.now() < deadline) {
|
||||
const generating = await isGenerating(page);
|
||||
const currentText = await getConversationText(page);
|
||||
const textChanged = currentText !== beforeText && currentText.length > 0;
|
||||
|
||||
if (generating) {
|
||||
hasStartedGenerating = true;
|
||||
stableCount = 0; // Reset stability while generating
|
||||
} else {
|
||||
if (hasStartedGenerating) {
|
||||
// It actively generated and now it stopped -> DONE
|
||||
// Provide a small buffer to let React render the final message fully
|
||||
await sleep(500);
|
||||
return;
|
||||
}
|
||||
|
||||
// Fallback: If it never showed "Generating/Cancel", but text changed and is stable
|
||||
if (textChanged) {
|
||||
if (currentText === lastText) {
|
||||
stableCount++;
|
||||
if (stableCount >= stableThreshold) {
|
||||
return; // Text has been stable for 2 seconds -> DONE
|
||||
}
|
||||
} else {
|
||||
stableCount = 0;
|
||||
lastText = currentText;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
await sleep(pollInterval);
|
||||
}
|
||||
|
||||
throw new Error('Timeout waiting for Antigravity reply');
|
||||
}
|
||||
|
||||
// ─── Request Handlers ────────────────────────────────────────────────
|
||||
|
||||
async function handleMessages(
|
||||
body: AnthropicRequest,
|
||||
page: IPage,
|
||||
bridge?: CDPBridge,
|
||||
): Promise<AnthropicResponse> {
|
||||
// Extract the last user message
|
||||
const userMessages = body.messages.filter(m => m.role === 'user');
|
||||
if (userMessages.length === 0) {
|
||||
throw new Error('No user message found in request');
|
||||
}
|
||||
const lastUserMsg = userMessages[userMessages.length - 1];
|
||||
const userText = extractTextContent(lastUserMsg.content);
|
||||
|
||||
if (!userText.trim()) {
|
||||
throw new Error('Empty user message');
|
||||
}
|
||||
|
||||
// Optimization 1: New conversation if this is the first message in the session
|
||||
if (body.messages.length === 1) {
|
||||
console.error(`[serve] New session detected (1 message). Starting new conversation in UI.`);
|
||||
await startNewConversation(page);
|
||||
}
|
||||
|
||||
// Optimization 3: Switch model if requested
|
||||
if (body.model) {
|
||||
await switchModel(page, body.model);
|
||||
}
|
||||
|
||||
// Get conversation state before sending
|
||||
const beforeText = await getConversationText(page);
|
||||
|
||||
// Send the message
|
||||
console.error(`[serve] Sending: "${userText.slice(0, 80)}${userText.length > 80 ? '...' : ''}"`);
|
||||
await sendMessage(page, userText, bridge);
|
||||
|
||||
// Poll for reply (change detection)
|
||||
console.error('[serve] Waiting for reply...');
|
||||
await waitForReply(page, beforeText);
|
||||
|
||||
// Extract the actual reply text precisely from the DOM
|
||||
const replyText = await getLastAssistantReply(page, userText);
|
||||
console.error(`[serve] Got reply: "${replyText.slice(0, 80)}${replyText.length > 80 ? '...' : ''}"`);
|
||||
|
||||
return {
|
||||
id: generateMsgId(),
|
||||
type: 'message',
|
||||
role: 'assistant',
|
||||
content: [{ type: 'text', text: replyText }],
|
||||
model: body.model ?? 'antigravity',
|
||||
stop_reason: 'end_turn',
|
||||
stop_sequence: null,
|
||||
usage: {
|
||||
input_tokens: estimateTokens(userText),
|
||||
output_tokens: estimateTokens(replyText),
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
// ─── Server ──────────────────────────────────────────────────────────
|
||||
|
||||
export async function startServe(opts: { port?: number } = {}): Promise<void> {
|
||||
const port = opts.port ?? 8082;
|
||||
|
||||
// Lazy CDP connection — connect when first request comes in
|
||||
let cdp: CDPBridge | null = null;
|
||||
let page: IPage | null = null;
|
||||
let requestInFlight = false;
|
||||
|
||||
async function ensureConnected(): Promise<IPage> {
|
||||
if (page) {
|
||||
try {
|
||||
await page.evaluate('1+1');
|
||||
return page;
|
||||
} catch {
|
||||
console.error('[serve] CDP connection lost, reconnecting...');
|
||||
cdp?.close().catch(() => {});
|
||||
cdp = null;
|
||||
page = null;
|
||||
}
|
||||
}
|
||||
|
||||
const endpoint = await resolveElectronEndpoint('antigravity');
|
||||
|
||||
// Note: Antigravity chat panel lives inside editor windows, not in Launchpad.
|
||||
// If multiple editor windows are open, set OPENCLI_CDP_TARGET to the window title.
|
||||
if (process.env.OPENCLI_CDP_TARGET) {
|
||||
console.error(`[serve] Using OPENCLI_CDP_TARGET=${process.env.OPENCLI_CDP_TARGET}`);
|
||||
}
|
||||
|
||||
// List available targets for debugging
|
||||
try {
|
||||
const res = await fetch(`${endpoint.replace(/\/$/, '')}/json`);
|
||||
const targets = await res.json() as Array<{ title?: string; type?: string }>;
|
||||
const pages = targets.filter(t => t.type === 'page');
|
||||
console.error(`[serve] Available targets: ${pages.map(t => `"${t.title}"`).join(', ')}`);
|
||||
} catch { /* ignore */ }
|
||||
|
||||
console.error(`[serve] Connecting via CDP (target pattern: "${process.env.OPENCLI_CDP_TARGET}")...`);
|
||||
cdp = new CDPBridge();
|
||||
try {
|
||||
page = await cdp.connect({ timeout: 15_000, cdpEndpoint: endpoint });
|
||||
} catch (err: unknown) {
|
||||
cdp = null;
|
||||
const errMsg = getErrorMessage(err);
|
||||
const cause = err instanceof Error ? (err.cause as Record<string, unknown> | undefined) : undefined;
|
||||
const isRefused = cause?.code === 'ECONNREFUSED' || errMsg.includes('ECONNREFUSED');
|
||||
throw new Error(
|
||||
isRefused
|
||||
? `Cannot connect to Antigravity at ${endpoint}.\n` +
|
||||
' 1. Make sure Antigravity is running\n' +
|
||||
' 2. Launch with: --remote-debugging-port=9234'
|
||||
: `CDP connection failed: ${errMsg}`
|
||||
);
|
||||
}
|
||||
|
||||
console.error('[serve] ✅ CDP connected.');
|
||||
|
||||
// Quick verification
|
||||
const hasUI = await page.evaluate(`
|
||||
(() => !!document.getElementById('conversation') || !!document.getElementById('antigravity.agentSidePanelInputBox'))()
|
||||
`);
|
||||
if (!hasUI) {
|
||||
console.error('[serve] ⚠️ Warning: chat UI elements not found in this target. Try setting OPENCLI_CDP_TARGET to the correct window title.');
|
||||
}
|
||||
|
||||
return page;
|
||||
}
|
||||
|
||||
const server = createServer(async (req, res) => {
|
||||
// CORS preflight
|
||||
if (req.method === 'OPTIONS') {
|
||||
res.writeHead(204, {
|
||||
'Access-Control-Allow-Origin': '*',
|
||||
'Access-Control-Allow-Methods': 'GET, POST, OPTIONS',
|
||||
'Access-Control-Allow-Headers': 'Content-Type, x-api-key, anthropic-version, Authorization',
|
||||
});
|
||||
res.end();
|
||||
return;
|
||||
}
|
||||
|
||||
const url = req.url ?? '/';
|
||||
const pathname = url.split('?')[0];
|
||||
|
||||
try {
|
||||
// GET /v1/models — return available models
|
||||
if (req.method === 'GET' && pathname === '/v1/models') {
|
||||
jsonResponse(res, 200, {
|
||||
data: [
|
||||
{
|
||||
id: 'antigravity',
|
||||
object: 'model',
|
||||
created: Math.floor(Date.now() / 1000),
|
||||
owned_by: 'antigravity',
|
||||
},
|
||||
],
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// POST /v1/messages — main endpoint
|
||||
if (req.method === 'POST' && pathname === '/v1/messages') {
|
||||
if (requestInFlight) {
|
||||
jsonResponse(res, 429, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'rate_limit_error',
|
||||
message: 'Another request is currently being processed. Antigravity can only handle one request at a time.',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
requestInFlight = true;
|
||||
try {
|
||||
const rawBody = await readBody(req);
|
||||
const body = JSON.parse(rawBody) as AnthropicRequest;
|
||||
|
||||
if (body.stream) {
|
||||
jsonResponse(res, 400, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'invalid_request_error',
|
||||
message: 'Streaming is not supported. Set "stream": false.',
|
||||
},
|
||||
});
|
||||
return;
|
||||
}
|
||||
|
||||
// Lazy connect on first request
|
||||
const activePage = await ensureConnected();
|
||||
const response = await handleMessages(body, activePage, cdp ?? undefined);
|
||||
jsonResponse(res, 200, response);
|
||||
} finally {
|
||||
requestInFlight = false;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
// Health check
|
||||
if (req.method === 'GET' && (pathname === '/' || pathname === '/health')) {
|
||||
jsonResponse(res, 200, { ok: true, cdpConnected: page !== null });
|
||||
return;
|
||||
}
|
||||
|
||||
jsonResponse(res, 404, {
|
||||
type: 'error',
|
||||
error: { type: 'not_found_error', message: `Not found: ${pathname}` },
|
||||
});
|
||||
} catch (err) {
|
||||
console.error('[serve] Error:', err instanceof Error ? err.message : err);
|
||||
jsonResponse(res, 500, {
|
||||
type: 'error',
|
||||
error: {
|
||||
type: 'api_error',
|
||||
message: err instanceof Error ? err.message : 'Internal server error',
|
||||
},
|
||||
});
|
||||
}
|
||||
});
|
||||
|
||||
server.listen(port, '127.0.0.1', () => {
|
||||
console.error(`\n[serve] ✅ Antigravity API proxy running at http://127.0.0.1:${port}`);
|
||||
console.error(`[serve] Compatible with Anthropic /v1/messages API`);
|
||||
console.error(`[serve] CDP connection will be established on first request.`);
|
||||
console.error(`\n[serve] Usage with Claude Code:`);
|
||||
console.error(` ANTHROPIC_BASE_URL=http://localhost:${port} claude\n`);
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
const shutdown = () => {
|
||||
console.error('\n[serve] Shutting down...');
|
||||
cdp?.close().catch(() => {});
|
||||
server.close();
|
||||
process.exit(EXIT_CODES.SUCCESS);
|
||||
};
|
||||
process.on('SIGTERM', shutdown);
|
||||
process.on('SIGINT', shutdown);
|
||||
|
||||
// Keep alive
|
||||
await new Promise(() => {});
|
||||
}
|
||||
|
||||
@@ -1,18 +0,0 @@
|
||||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||||
export const statusCommand = cli({
|
||||
site: 'antigravity',
|
||||
name: 'status',
|
||||
description: 'Check Antigravity CDP connection and get current page state',
|
||||
domain: 'localhost',
|
||||
strategy: Strategy.UI,
|
||||
browser: true,
|
||||
args: [],
|
||||
columns: ['status', 'url', 'title'],
|
||||
func: async (page) => {
|
||||
return {
|
||||
status: 'Connected',
|
||||
url: await page.evaluate('window.location.href'),
|
||||
title: await page.evaluate('document.title'),
|
||||
};
|
||||
},
|
||||
});
|
||||
Some files were not shown because too many files have changed in this diff Show More
Reference in New Issue
Block a user