Compare commits

...

1 Commits

Author SHA1 Message Date
jackwener 3b20ab48f2 refactor: deep CLI layer architecture improvements
CLI Layer:
1. execution.ts: auto-manages browser sessions, simplified signature
2. runtime.ts: add getBrowserFactory()
3. serialization.ts: new module for serialization helpers
4. cli.ts: format all built-in commands, extract inferHost()
5. commanderAdapter.ts: pure thin adapter

Src-wide cleanup:
6. download/index.ts: shared VIDEO_PLATFORM_DOMAINS, reuse isBinaryInstalled
7. Move site helpers: coupang/bilibili/chaoxing.ts -> clis/*/utils.ts
8. explore.ts: decompose into analyzeEndpoints, inferCapabilities, writeArtifacts

244 tests pass. 240 commands registered.
2026-03-21 02:17:41 +08:00
30 changed files with 687 additions and 487 deletions
+162 -124
View File
@@ -1,28 +1,41 @@
/**
* CLI entry point: registers built-in commands and wires up Commander.
*
* Built-in commands are registered inline here (list, validate, explore, etc.).
* Dynamic adapter commands are registered via commanderAdapter.ts.
*/
import { Command } from 'commander';
import chalk from 'chalk';
import { executeCommand } from './engine.js';
import { Strategy, type CliCommand, fullName, getRegistry, strategyLabel, serializeCommand, formatArgSummary, formatRegistryHelpText } from './registry.js';
import { type CliCommand, fullName, getRegistry, strategyLabel } from './registry.js';
import { serializeCommand, formatArgSummary } from './serialization.js';
import { render as renderOutput } from './output.js';
import { BrowserBridge, CDPBridge } from './browser/index.js';
import { browserSession, DEFAULT_BROWSER_COMMAND_TIMEOUT, runWithTimeout } from './runtime.js';
import { getBrowserFactory, browserSession } from './runtime.js';
import { PKG_VERSION } from './version.js';
import { printCompletionScript } from './completion.js';
import { CliError } from './errors.js';
import { shouldUseBrowserSession } from './capabilityRouting.js';
import { loadExternalClis, executeExternalCli, installExternalCli, registerExternalCli, isBinaryInstalled } from './external.js';
import { registerAllCommands } from './commanderAdapter.js';
export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
const program = new Command();
program.name('opencli').description('Make any website your CLI. Zero setup. AI-powered.').version(PKG_VERSION);
program
.name('opencli')
.description('Make any website your CLI. Zero setup. AI-powered.')
.version(PKG_VERSION);
// ── Built-in commands ──────────────────────────────────────────────────────
// ── Built-in: list ────────────────────────────────────────────────────────
program.command('list').description('List all available CLI commands').option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('--json', 'JSON output (deprecated)')
program
.command('list')
.description('List all available CLI commands')
.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table')
.option('--json', 'JSON output (deprecated)')
.action((opts) => {
const registry = getRegistry();
const commands = [...registry.values()].sort((a, b) => fullName(a).localeCompare(fullName(b)));
const fmt = opts.json && opts.format === 'table' ? 'json' : opts.format;
const isStructured = fmt === 'json' || fmt === 'yaml';
if (fmt !== 'table') {
const rows = isStructured
? commands.map(serializeCommand)
@@ -37,24 +50,39 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
}));
renderOutput(rows, {
fmt,
columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args', ...(isStructured ? ['columns', 'domain'] : [])],
columns: ['command', 'site', 'name', 'description', 'strategy', 'browser', 'args',
...(isStructured ? ['columns', 'domain'] : [])],
title: 'opencli/list',
source: 'opencli list',
});
return;
}
// Table (default) — grouped by site
const sites = new Map<string, CliCommand[]>();
for (const cmd of commands) { const g = sites.get(cmd.site) ?? []; g.push(cmd); sites.set(cmd.site, g); }
console.log(); console.log(chalk.bold(' opencli') + chalk.dim(' — available commands')); console.log();
for (const cmd of commands) {
const g = sites.get(cmd.site) ?? [];
g.push(cmd);
sites.set(cmd.site, g);
}
console.log();
console.log(chalk.bold(' opencli') + chalk.dim(' — available commands'));
console.log();
for (const [site, cmds] of sites) {
console.log(chalk.bold.cyan(` ${site}`));
for (const cmd of cmds) { const tag = strategyLabel(cmd) === 'public' ? chalk.green('[public]') : chalk.yellow(`[${strategyLabel(cmd)}]`); console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(`${cmd.description}`) : ''}`); }
for (const cmd of cmds) {
const tag = strategyLabel(cmd) === 'public'
? chalk.green('[public]')
: chalk.yellow(`[${strategyLabel(cmd)}]`);
console.log(` ${cmd.name} ${tag}${cmd.description ? chalk.dim(`${cmd.description}`) : ''}`);
}
console.log();
}
const externalClis = loadExternalClis();
if (externalClis.length > 0) {
console.log(chalk.bold.cyan(` external CLIs`));
console.log(chalk.bold.cyan(' external CLIs'));
for (const ext of externalClis) {
const isInstalled = isBinaryInstalled(ext.binary);
const tag = isInstalled ? chalk.green('[installed]') : chalk.yellow('[auto-install]');
@@ -62,17 +90,27 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
}
console.log();
}
console.log(chalk.dim(` ${commands.length} built-in commands across ${sites.size} sites, ${externalClis.length} external CLIs`)); console.log();
console.log(chalk.dim(` ${commands.length} built-in commands across ${sites.size} sites, ${externalClis.length} external CLIs`));
console.log();
});
program.command('validate').description('Validate CLI definitions').argument('[target]', 'site or site/name')
// ── Built-in: validate / verify ───────────────────────────────────────────
program
.command('validate')
.description('Validate CLI definitions')
.argument('[target]', 'site or site/name')
.action(async (target) => {
const { validateClisWithTarget, renderValidationReport } = await import('./validate.js');
console.log(renderValidationReport(validateClisWithTarget([BUILTIN_CLIS, USER_CLIS], target)));
});
program.command('verify').description('Validate + smoke test').argument('[target]').option('--smoke', 'Run smoke tests', false)
program
.command('verify')
.description('Validate + smoke test')
.argument('[target]')
.option('--smoke', 'Run smoke tests', false)
.action(async (target, opts) => {
const { verifyClis, renderVerifyReport } = await import('./verify.js');
const r = await verifyClis({ builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, target, smoke: opts.smoke });
@@ -80,28 +118,91 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
process.exitCode = r.ok ? 0 : 1;
});
program.command('explore').alias('probe').description('Explore a website: discover APIs, stores, and recommend strategies').argument('<url>').option('--site <name>').option('--goal <text>').option('--wait <s>', '', '3').option('--auto', 'Enable interactive fuzzing (simulate clicks to trigger lazy APIs)').option('--click <labels>', 'Comma-separated labels to click before fuzzing (e.g. "字幕,CC,评论")')
.action(async (url, opts) => { const { exploreUrl, renderExploreSummary } = await import('./explore.js'); const clickLabels = opts.click ? opts.click.split(',').map((s: string) => s.trim()) : undefined; const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; const workspace = `explore:${opts.site ?? (() => { try { return new URL(url).host; } catch { return 'default'; } })()}`; console.log(renderExploreSummary(await exploreUrl(url, { BrowserFactory: BrowserFactory as any, site: opts.site, goal: opts.goal, waitSeconds: parseFloat(opts.wait), auto: opts.auto, clickLabels, workspace }))); });
// ── Built-in: explore / synthesize / generate / cascade ───────────────────
program.command('synthesize').description('Synthesize CLIs from explore').argument('<target>').option('--top <n>', '', '3')
.action(async (target, opts) => { const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js'); console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) }))); });
program
.command('explore')
.alias('probe')
.description('Explore a website: discover APIs, stores, and recommend strategies')
.argument('<url>')
.option('--site <name>')
.option('--goal <text>')
.option('--wait <s>', '', '3')
.option('--auto', 'Enable interactive fuzzing')
.option('--click <labels>', 'Comma-separated labels to click before fuzzing')
.action(async (url, opts) => {
const { exploreUrl, renderExploreSummary } = await import('./explore.js');
const clickLabels = opts.click
? opts.click.split(',').map((s: string) => s.trim())
: undefined;
const workspace = `explore:${inferHost(url, opts.site)}`;
const result = await exploreUrl(url, {
BrowserFactory: getBrowserFactory() as any,
site: opts.site,
goal: opts.goal,
waitSeconds: parseFloat(opts.wait),
auto: opts.auto,
clickLabels,
workspace,
});
console.log(renderExploreSummary(result));
});
program.command('generate').description('One-shot: explore → synthesize → register').argument('<url>').option('--goal <text>').option('--site <name>')
.action(async (url, opts) => { const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js'); const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge; const workspace = `generate:${opts.site ?? (() => { try { return new URL(url).host; } catch { return 'default'; } })()}`; const r = await generateCliFromUrl({ url, BrowserFactory: BrowserFactory as any, builtinClis: BUILTIN_CLIS, userClis: USER_CLIS, goal: opts.goal, site: opts.site, workspace }); console.log(renderGenerateSummary(r)); process.exitCode = r.ok ? 0 : 1; });
program
.command('synthesize')
.description('Synthesize CLIs from explore')
.argument('<target>')
.option('--top <n>', '', '3')
.action(async (target, opts) => {
const { synthesizeFromExplore, renderSynthesizeSummary } = await import('./synthesize.js');
console.log(renderSynthesizeSummary(synthesizeFromExplore(target, { top: parseInt(opts.top) })));
});
program.command('cascade').description('Strategy cascade: find simplest working strategy').argument('<url>').option('--site <name>')
program
.command('generate')
.description('One-shot: explore → synthesize → register')
.argument('<url>')
.option('--goal <text>')
.option('--site <name>')
.action(async (url, opts) => {
const { generateCliFromUrl, renderGenerateSummary } = await import('./generate.js');
const workspace = `generate:${inferHost(url, opts.site)}`;
const r = await generateCliFromUrl({
url,
BrowserFactory: getBrowserFactory() as any,
builtinClis: BUILTIN_CLIS,
userClis: USER_CLIS,
goal: opts.goal,
site: opts.site,
workspace,
});
console.log(renderGenerateSummary(r));
process.exitCode = r.ok ? 0 : 1;
});
program
.command('cascade')
.description('Strategy cascade: find simplest working strategy')
.argument('<url>')
.option('--site <name>')
.action(async (url, opts) => {
const { cascadeProbe, renderCascadeResult } = await import('./cascade.js');
const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge;
const result = await browserSession(BrowserFactory as any, async (page) => {
// Navigate to the site first for cookie context
try { const siteUrl = new URL(url); await page.goto(`${siteUrl.protocol}//${siteUrl.host}`); await page.wait(2); } catch {}
const workspace = `cascade:${inferHost(url, opts.site)}`;
const result = await browserSession(getBrowserFactory(), async (page) => {
try {
const siteUrl = new URL(url);
await page.goto(`${siteUrl.protocol}//${siteUrl.host}`);
await page.wait(2);
} catch {}
return cascadeProbe(page, url);
}, { workspace: `cascade:${opts.site ?? (() => { try { return new URL(url).host; } catch { return 'default'; } })()}` });
}, { workspace });
console.log(renderCascadeResult(result));
});
program.command('doctor')
// ── Built-in: doctor / setup / completion ─────────────────────────────────
program
.command('doctor')
.description('Diagnose opencli browser bridge connectivity')
.option('--live', 'Test browser connectivity (requires Chrome running)', false)
.option('--sessions', 'Show active automation sessions', false)
@@ -111,36 +212,42 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
console.log(renderBrowserDoctorReport(report));
});
program.command('setup')
program
.command('setup')
.description('Interactive setup: verify browser bridge connectivity')
.action(async () => {
const { runSetup } = await import('./setup.js');
await runSetup({ cliVersion: PKG_VERSION });
});
program.command('completion')
program
.command('completion')
.description('Output shell completion script')
.argument('<shell>', 'Shell type: bash, zsh, or fish')
.action((shell) => {
printCompletionScript(shell);
});
// ── External CLIs ─────────────────────────────────────────────────────────
const externalClis = loadExternalClis();
program.command('install')
program
.command('install')
.description('Install an external CLI')
.argument('<name>', 'Name of the external CLI')
.action((name: string) => {
const ext = externalClis.find(e => e.name === name);
if (!ext) {
console.error(chalk.red(`External CLI '${name}' not found in registry.`));
process.exitCode = 1;
return;
console.error(chalk.red(`External CLI '${name}' not found in registry.`));
process.exitCode = 1;
return;
}
installExternalCli(ext);
});
program.command('register')
program
.command('register')
.description('Register an external CLI')
.argument('<name>', 'Name of the CLI')
.option('--binary <bin>', 'Binary name if different from name')
@@ -150,7 +257,6 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
registerExternalCli(name, { binary: opts.binary, install: opts.install, description: opts.desc });
});
// Helper: extract args from process.argv and passthrough to external CLI
function passthroughExternal(name: string) {
const idx = process.argv.indexOf(name);
const args = process.argv.slice(idx + 1);
@@ -164,17 +270,19 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
for (const ext of externalClis) {
if (program.commands.some(c => c.name() === ext.name)) continue;
program.command(ext.name)
program
.command(ext.name)
.description(`(External) ${ext.description || ext.name}`)
.allowUnknownOption()
.allowExcessArguments()
.action(() => passthroughExternal(ext.name));
}
// ── Antigravity serve (built-in, long-running) ──────────────────────────────
// ── Antigravity serve (long-running, special case) ────────────────────────
const antigravityCmd = program.command('antigravity').description('antigravity commands');
antigravityCmd.command('serve')
antigravityCmd
.command('serve')
.description('Start Anthropic-compatible API proxy for Antigravity')
.option('--port <port>', 'Server port (default: 8082)', '8082')
.action(async (opts) => {
@@ -182,92 +290,14 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
await startServe({ port: parseInt(opts.port) });
});
// ── Dynamic site commands ──────────────────────────────────────────────────
// ── Dynamic adapter commands ──────────────────────────────────────────────
const registry = getRegistry();
const siteGroups = new Map<string, Command>();
// Pre-seed with the antigravity command registered above to avoid duplicates
siteGroups.set('antigravity', antigravityCmd);
registerAllCommands(program, siteGroups);
for (const [, cmd] of registry) {
let siteCmd = siteGroups.get(cmd.site);
if (!siteCmd) { siteCmd = program.command(cmd.site).description(`${cmd.site} commands`); siteGroups.set(cmd.site, siteCmd); }
// Skip if this subcommand was already hardcoded (e.g. antigravity serve)
if (siteCmd.commands.some((c: Command) => c.name() === cmd.name)) continue;
const subCmd = siteCmd.command(cmd.name).description(cmd.description);
// ── Unknown command fallback ──────────────────────────────────────────────
// Register positional args first, then named options
const positionalArgs: typeof cmd.args = [];
for (const arg of cmd.args) {
if (arg.positional) {
const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`;
subCmd.argument(bracket, arg.help ?? '');
positionalArgs.push(arg);
} else {
const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
if (arg.required) subCmd.requiredOption(flag, arg.help ?? '');
else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
else subCmd.option(flag, arg.help ?? '');
}
}
subCmd.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table').option('-v, --verbose', 'Debug output', false);
subCmd.addHelpText('after', formatRegistryHelpText(cmd));
subCmd.action(async (...actionArgs: any[]) => {
// Commander passes positional args first, then options object, then the Command
const actionOpts = actionArgs[positionalArgs.length] ?? {};
const startTime = Date.now();
const kwargs: Record<string, any> = {};
// Collect positional args
for (let i = 0; i < positionalArgs.length; i++) {
const arg = positionalArgs[i];
const v = actionArgs[i];
if (v !== undefined) kwargs[arg.name] = v;
}
// Collect named options
for (const arg of cmd.args) {
if (arg.positional) continue;
const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase());
const v = actionOpts[arg.name] ?? actionOpts[camelName];
if (v !== undefined) kwargs[arg.name] = v;
}
try {
if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1';
let result: any;
if (shouldUseBrowserSession(cmd)) {
const BrowserFactory = process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge;
result = await browserSession(BrowserFactory as any, async (page) => {
// Cookie/header strategies require same-origin context for credentialed fetch.
if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
try { await page.goto(`https://${cmd.domain}`); await page.wait(2); } catch {}
}
return runWithTimeout(executeCommand(cmd, page, kwargs, actionOpts.verbose), { timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT, label: fullName(cmd) });
}, { workspace: `site:${cmd.site}` });
} else { result = await executeCommand(cmd, null, kwargs, actionOpts.verbose); }
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
console.error(chalk.yellow(`[Verbose] Warning: Command returned an empty result. If the website structural API changed or requires authentication, check the network or update the adapter.`));
}
const resolved = getRegistry().get(fullName(cmd)) ?? cmd;
renderOutput(result, { fmt: actionOpts.format, columns: resolved.columns, title: `${resolved.site}/${resolved.name}`, elapsed: (Date.now() - startTime) / 1000, source: fullName(resolved), footerExtra: resolved.footerExtra?.(kwargs) });
} catch (err: any) {
if (err instanceof CliError) {
console.error(chalk.red(`Error [${err.code}]: ${err.message}`));
if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`));
} else if (actionOpts.verbose && err.stack) {
console.error(chalk.red(err.stack));
} else {
console.error(chalk.red(`Error: ${err.message ?? err}`));
}
process.exitCode = 1;
}
});
}
// Dangerous system commands that should never be auto-registered
const DENY_LIST = new Set([
'rm', 'sudo', 'dd', 'mkfs', 'fdisk', 'shutdown', 'reboot',
'kill', 'killall', 'chmod', 'chown', 'passwd', 'su', 'mount',
@@ -294,3 +324,11 @@ export function runCli(BUILTIN_CLIS: string, USER_CLIS: string): void {
program.parse();
}
// ── Helpers ─────────────────────────────────────────────────────────────────
/** Infer a workspace-friendly hostname from a URL, with site override. */
function inferHost(url: string, site?: string): string {
if (site) return site;
try { return new URL(url).host; } catch { return 'default'; }
}
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet } from '../../bilibili.js';
import { apiGet } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, payloadData } from '../../bilibili.js';
import { apiGet, payloadData } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, payloadData, getSelfUid, stripHtml } from '../../bilibili.js';
import { apiGet, payloadData, getSelfUid, stripHtml } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,6 +1,6 @@
import { cli, Strategy } from '../../registry.js';
import type { IPage } from '../../types.js';
import { fetchJson, getSelfUid, resolveUid } from '../../bilibili.js';
import { fetchJson, getSelfUid, resolveUid } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, payloadData } from '../../bilibili.js';
import { apiGet, payloadData } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, getSelfUid } from '../../bilibili.js';
import { apiGet, getSelfUid } from './utils.js';
cli({
site: 'bilibili', name: 'me', description: 'My Bilibili profile info', domain: 'www.bilibili.com', strategy: Strategy.COOKIE,
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet } from '../../bilibili.js';
import { apiGet } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, stripHtml } from '../../bilibili.js';
import { apiGet, stripHtml } from './utils.js';
cli({
site: 'bilibili', name: 'search', description: 'Search Bilibili videos or users', domain: 'www.bilibili.com', strategy: Strategy.COOKIE,
+1 -1
View File
@@ -1,6 +1,6 @@
import { cli, Strategy } from '../../registry.js';
import type { IPage } from '../../types.js';
import { apiGet } from '../../bilibili.js';
import { apiGet } from './utils.js';
cli({
site: 'bilibili',
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { apiGet, payloadData, resolveUid } from '../../bilibili.js';
import { apiGet, payloadData, resolveUid } from './utils.js';
cli({
site: 'bilibili',
@@ -2,7 +2,7 @@
* Bilibili shared helpers: WBI signing, authenticated fetch, nav data, UID resolution.
*/
import type { IPage } from './types.js';
import type { IPage } from '../../types.js';
const MIXIN_KEY_ENC_TAB = [
46,47,18,2,53,8,23,32,15,50,10,31,58,3,45,35,27,43,5,49,
+1 -1
View File
@@ -3,7 +3,7 @@ import {
getCourses, initSession, enterCourse, getTabIframeUrl,
parseAssignmentsFromDom, sleep,
type AssignmentRow,
} from '../../chaoxing.js';
} from './utils.js';
cli({
site: 'chaoxing',
+1 -1
View File
@@ -3,7 +3,7 @@ import {
getCourses, initSession, enterCourse, getTabIframeUrl,
parseExamsFromDom, sleep,
type ExamRow,
} from '../../chaoxing.js';
} from './utils.js';
cli({
site: 'chaoxing',
@@ -1,5 +1,5 @@
import { describe, expect, it } from 'vitest';
import { formatTimestamp, workStatusLabel } from './chaoxing.js';
import { formatTimestamp, workStatusLabel } from './utils.js';
function localDatePrefixFromMillis(ts: number): string {
const d = new Date(ts);
@@ -6,7 +6,7 @@
* course pages loaded as iframes.
*/
import type { IPage } from './types.js';
import type { IPage } from '../../types.js';
// ── Utilities ────────────────────────────────────────────────────────
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { canonicalizeProductUrl, normalizeProductId } from '../../coupang.js';
import { canonicalizeProductUrl, normalizeProductId } from './utils.js';
function escapeJsString(value: string): string {
return JSON.stringify(value);
+1 -1
View File
@@ -1,5 +1,5 @@
import { cli, Strategy } from '../../registry.js';
import { mergeSearchItems, normalizeSearchItem, sanitizeSearchItems } from '../../coupang.js';
import { mergeSearchItems, normalizeSearchItem, sanitizeSearchItems } from './utils.js';
function escapeJsString(value: string): string {
return JSON.stringify(value);
@@ -5,7 +5,7 @@ import {
normalizeProductId,
normalizeSearchItem,
sanitizeSearchItems,
} from './coupang.js';
} from './utils.js';
describe('normalizeProductId', () => {
it('extracts product id from canonical path', () => {
+113
View File
@@ -0,0 +1,113 @@
/**
* Commander adapter: bridges Registry commands to Commander subcommands.
*
* This is a THIN adapter — it only handles:
* 1. Commander arg/option registration
* 2. Collecting kwargs from Commander's action args
* 3. Calling executeCommand (which handles browser sessions, validation, etc.)
* 4. Rendering output and errors
*
* All execution logic lives in execution.ts.
*/
import { Command } from 'commander';
import chalk from 'chalk';
import { type CliCommand, fullName, getRegistry } from './registry.js';
import { formatRegistryHelpText } from './serialization.js';
import { render as renderOutput } from './output.js';
import { executeCommand } from './execution.js';
import { CliError } from './errors.js';
/**
* Register a single CliCommand as a Commander subcommand.
*/
export function registerCommandToProgram(siteCmd: Command, cmd: CliCommand): void {
if (siteCmd.commands.some((c: Command) => c.name() === cmd.name)) return;
const subCmd = siteCmd.command(cmd.name).description(cmd.description);
// Register positional args first, then named options
const positionalArgs: typeof cmd.args = [];
for (const arg of cmd.args) {
if (arg.positional) {
const bracket = arg.required ? `<${arg.name}>` : `[${arg.name}]`;
subCmd.argument(bracket, arg.help ?? '');
positionalArgs.push(arg);
} else {
const flag = arg.required ? `--${arg.name} <value>` : `--${arg.name} [value]`;
if (arg.required) subCmd.requiredOption(flag, arg.help ?? '');
else if (arg.default != null) subCmd.option(flag, arg.help ?? '', String(arg.default));
else subCmd.option(flag, arg.help ?? '');
}
}
subCmd
.option('-f, --format <fmt>', 'Output format: table, json, yaml, md, csv', 'table')
.option('-v, --verbose', 'Debug output', false);
subCmd.addHelpText('after', formatRegistryHelpText(cmd));
subCmd.action(async (...actionArgs: any[]) => {
const actionOpts = actionArgs[positionalArgs.length] ?? {};
const startTime = Date.now();
// ── Collect kwargs ──────────────────────────────────────────────────
const kwargs: Record<string, any> = {};
for (let i = 0; i < positionalArgs.length; i++) {
const v = actionArgs[i];
if (v !== undefined) kwargs[positionalArgs[i].name] = v;
}
for (const arg of cmd.args) {
if (arg.positional) continue;
const camelName = arg.name.replace(/-([a-z])/g, (_m, ch: string) => ch.toUpperCase());
const v = actionOpts[arg.name] ?? actionOpts[camelName];
if (v !== undefined) kwargs[arg.name] = v;
}
// ── Execute + render ────────────────────────────────────────────────
try {
if (actionOpts.verbose) process.env.OPENCLI_VERBOSE = '1';
const result = await executeCommand(cmd, kwargs, actionOpts.verbose);
if (actionOpts.verbose && (!result || (Array.isArray(result) && result.length === 0))) {
console.error(chalk.yellow('[Verbose] Warning: Command returned an empty result.'));
}
const resolved = getRegistry().get(fullName(cmd)) ?? cmd;
renderOutput(result, {
fmt: actionOpts.format,
columns: resolved.columns,
title: `${resolved.site}/${resolved.name}`,
elapsed: (Date.now() - startTime) / 1000,
source: fullName(resolved),
footerExtra: resolved.footerExtra?.(kwargs),
});
} catch (err: any) {
if (err instanceof CliError) {
console.error(chalk.red(`Error [${err.code}]: ${err.message}`));
if (err.hint) console.error(chalk.yellow(`Hint: ${err.hint}`));
} else if (actionOpts.verbose && err.stack) {
console.error(chalk.red(err.stack));
} else {
console.error(chalk.red(`Error: ${err.message ?? err}`));
}
process.exitCode = 1;
}
});
}
/**
* Register all commands from the registry onto a Commander program.
*/
export function registerAllCommands(
program: Command,
siteGroups: Map<string, Command>,
): void {
for (const [, cmd] of getRegistry()) {
let siteCmd = siteGroups.get(cmd.site);
if (!siteCmd) {
siteCmd = program.command(cmd.site).description(`${cmd.site} commands`);
siteGroups.set(cmd.site, siteCmd);
}
registerCommandToProgram(siteCmd, cmd);
}
}
-108
View File
@@ -12,13 +12,7 @@ import * as fs from 'node:fs';
import * as path from 'node:path';
import yaml from 'js-yaml';
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, registerCommand } from './registry.js';
import type { IPage } from './types.js';
import { executePipeline } from './pipeline.js';
import { log } from './logger.js';
import { AdapterLoadError } from './errors.js';
/** Set of TS module paths that have been loaded */
const _loadedModules = new Set<string>();
/**
* Discover and register CLI commands.
@@ -171,105 +165,3 @@ async function registerYamlCli(filePath: string, defaultSite: string): Promise<v
log.warn(`Failed to load ${filePath}: ${err.message}`);
}
}
/**
* Validates and coerces arguments based on the command's Arg definitions.
*/
function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: Record<string, any>): Record<string, any> {
const result: Record<string, any> = { ...kwargs };
for (const argDef of cmdArgs) {
const val = result[argDef.name];
// 1. Check required
if (argDef.required && (val === undefined || val === null || val === '')) {
throw new Error(`Argument "${argDef.name}" is required.\n${argDef.help ? `Hint: ${argDef.help}` : ''}`);
}
if (val !== undefined && val !== null) {
// 2. Type coercion
if (argDef.type === 'int' || argDef.type === 'number') {
const num = Number(val);
if (Number.isNaN(num)) {
throw new Error(`Argument "${argDef.name}" must be a valid number. Received: "${val}"`);
}
result[argDef.name] = num;
} else if (argDef.type === 'boolean' || argDef.type === 'bool') {
if (typeof val === 'string') {
const lower = val.toLowerCase();
if (lower === 'true' || lower === '1') result[argDef.name] = true;
else if (lower === 'false' || lower === '0') result[argDef.name] = false;
else throw new Error(`Argument "${argDef.name}" must be a boolean (true/false). Received: "${val}"`);
} else {
result[argDef.name] = Boolean(val);
}
}
// 3. Choices validation
const coercedVal = result[argDef.name];
if (argDef.choices && argDef.choices.length > 0) {
// Only stringent check for string/number types against choices array
if (!argDef.choices.map(String).includes(String(coercedVal))) {
throw new Error(`Argument "${argDef.name}" must be one of: ${argDef.choices.join(', ')}. Received: "${coercedVal}"`);
}
}
} else if (argDef.default !== undefined) {
// Set default if value is missing
result[argDef.name] = argDef.default;
}
}
return result;
}
/**
* Execute a CLI command. Handles lazy-loading of TS modules.
*/
export async function executeCommand(
cmd: CliCommand,
page: IPage | null,
rawKwargs: Record<string, any>,
debug: boolean = false,
): Promise<any> {
let kwargs: Record<string, any>;
try {
kwargs = coerceAndValidateArgs(cmd.args, rawKwargs);
} catch (err: any) {
// Re-throw validation errors clearly
throw new Error(`[Argument Validation Error]\n${err.message}`);
}
// Lazy-load TS module on first execution
const internal = cmd as InternalCliCommand;
if (internal._lazy && internal._modulePath) {
const modulePath = internal._modulePath;
if (!_loadedModules.has(modulePath)) {
try {
await import(`file://${modulePath}`);
_loadedModules.add(modulePath);
} catch (err: any) {
throw new AdapterLoadError(
`Failed to load adapter module ${modulePath}: ${err.message}`,
'Check that the adapter file exists and has no syntax errors.',
);
}
}
// After loading, the module's cli() call will have updated the registry
// with the real func/pipeline. Re-fetch the command.
const { getRegistry, fullName } = await import('./registry.js');
const updated = getRegistry().get(fullName(cmd));
if (updated && updated.func) {
return updated.func(page!, kwargs, debug);
}
if (updated && updated.pipeline) {
return executePipeline(page, updated.pipeline, { args: kwargs, debug });
}
}
if (cmd.func) {
return cmd.func(page!, kwargs, debug);
}
if (cmd.pipeline) {
return executePipeline(page, cmd.pipeline, { args: kwargs, debug });
}
throw new Error(`Command ${cmd.site}/${cmd.name} has no func or pipeline`);
}
+21 -54
View File
@@ -2,7 +2,7 @@
* Download utilities: HTTP downloads, yt-dlp wrapper, format conversion.
*/
import { spawn, execSync } from 'node:child_process';
import { spawn } from 'node:child_process';
import * as fs from 'node:fs';
import * as path from 'node:path';
import * as https from 'node:https';
@@ -10,6 +10,7 @@ import * as http from 'node:http';
import * as os from 'node:os';
import { URL } from 'node:url';
import type { ProgressBar } from './progress.js';
import { isBinaryInstalled } from '../external.js';
export interface DownloadOptions {
cookies?: string;
@@ -36,68 +37,43 @@ export interface BrowserCookie {
expirationDate?: number;
}
/**
* Check if yt-dlp is available in PATH.
*/
/** Check if yt-dlp is available in PATH. */
export function checkYtdlp(): boolean {
try {
execSync('yt-dlp --version', { encoding: 'utf-8', stdio: 'pipe' });
return true;
} catch {
return false;
}
return isBinaryInstalled('yt-dlp');
}
/**
* Check if ffmpeg is available in PATH.
*/
/** Check if ffmpeg is available in PATH. */
export function checkFfmpeg(): boolean {
try {
execSync('ffmpeg -version', { encoding: 'utf-8', stdio: 'pipe' });
return true;
} catch {
return false;
}
return isBinaryInstalled('ffmpeg');
}
/** Domains that host video content and can be downloaded via yt-dlp. */
const VIDEO_PLATFORM_DOMAINS = [
'youtube.com', 'youtu.be', 'bilibili.com', 'twitter.com',
'x.com', 'tiktok.com', 'vimeo.com', 'twitch.tv',
];
const IMAGE_EXTENSIONS = new Set(['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif']);
const VIDEO_EXTENSIONS = new Set(['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts']);
const DOC_EXTENSIONS = new Set(['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown']);
/**
* Detect content type from URL and optional headers.
*/
export function detectContentType(url: string, contentType?: string): 'image' | 'video' | 'document' | 'binary' {
// Check content-type header first
if (contentType) {
if (contentType.startsWith('image/')) return 'image';
if (contentType.startsWith('video/')) return 'video';
if (contentType.startsWith('text/') || contentType.includes('json') || contentType.includes('xml')) return 'document';
}
// Detect from URL
const urlLower = url.toLowerCase();
const ext = path.extname(new URL(url).pathname).toLowerCase();
// Image extensions
if (['.jpg', '.jpeg', '.png', '.gif', '.webp', '.svg', '.ico', '.bmp', '.avif'].includes(ext)) {
return 'image';
}
// Video extensions
if (['.mp4', '.webm', '.avi', '.mov', '.mkv', '.flv', '.m3u8', '.ts'].includes(ext)) {
return 'video';
}
// Video platforms (need yt-dlp)
if (urlLower.includes('youtube.com') || urlLower.includes('youtu.be') ||
urlLower.includes('bilibili.com') || urlLower.includes('twitter.com') ||
urlLower.includes('x.com') || urlLower.includes('tiktok.com') ||
urlLower.includes('vimeo.com') || urlLower.includes('twitch.tv')) {
return 'video';
}
// Document extensions
if (['.html', '.htm', '.json', '.xml', '.txt', '.md', '.markdown'].includes(ext)) {
return 'document';
}
if (IMAGE_EXTENSIONS.has(ext)) return 'image';
if (VIDEO_EXTENSIONS.has(ext)) return 'video';
if (VIDEO_PLATFORM_DOMAINS.some(d => urlLower.includes(d))) return 'video';
if (DOC_EXTENSIONS.has(ext)) return 'document';
return 'binary';
}
@@ -106,16 +82,7 @@ export function detectContentType(url: string, contentType?: string): 'image' |
*/
export function requiresYtdlp(url: string): boolean {
const urlLower = url.toLowerCase();
return (
urlLower.includes('youtube.com') ||
urlLower.includes('youtu.be') ||
urlLower.includes('bilibili.com/video') ||
urlLower.includes('twitter.com') ||
urlLower.includes('x.com') ||
urlLower.includes('tiktok.com') ||
urlLower.includes('vimeo.com') ||
urlLower.includes('twitch.tv')
);
return VIDEO_PLATFORM_DOMAINS.some(d => urlLower.includes(d));
}
/**
+8 -7
View File
@@ -1,9 +1,10 @@
/**
* Tests for engine.ts: CLI discovery and command execution.
* Tests for discovery and execution modules.
*/
import { describe, it, expect, vi, beforeEach } from 'vitest';
import { discoverClis, executeCommand } from './engine.js';
import { discoverClis } from './discovery.js';
import { executeCommand } from './execution.js';
import { getRegistry, cli, Strategy } from './registry.js';
describe('discoverClis', () => {
@@ -27,7 +28,7 @@ describe('executeCommand', () => {
func: async (_page, kwargs) => [{ noteId: kwargs['note-id'] }],
});
const result = await executeCommand(cmd, null, { 'note-id': 'abc123' });
const result = await executeCommand(cmd, { 'note-id': 'abc123' });
expect(result).toEqual([{ noteId: 'abc123' }]);
});
@@ -43,7 +44,7 @@ describe('executeCommand', () => {
},
});
const result = await executeCommand(cmd, null, { query: 'hello' });
const result = await executeCommand(cmd, { query: 'hello' });
expect(result).toEqual([{ title: 'hello' }]);
});
@@ -61,7 +62,7 @@ describe('executeCommand', () => {
});
// Pipeline commands require page for evaluate step, so we'll test the error path
await expect(executeCommand(cmd, null, {})).rejects.toThrow();
await expect(executeCommand(cmd, {})).rejects.toThrow();
});
it('throws for command with no func or pipeline', async () => {
@@ -72,7 +73,7 @@ describe('executeCommand', () => {
browser: false,
});
await expect(executeCommand(cmd, null, {})).rejects.toThrow('has no func or pipeline');
await expect(executeCommand(cmd, {})).rejects.toThrow('has no func or pipeline');
});
it('passes debug flag to func', async () => {
@@ -88,7 +89,7 @@ describe('executeCommand', () => {
},
});
await executeCommand(cmd, null, {}, true);
await executeCommand(cmd, {}, true);
expect(receivedDebug).toBe(true);
});
});
+138
View File
@@ -0,0 +1,138 @@
/**
* Command execution: validates args, manages browser sessions, runs commands.
*
* This is the single entry point for executing any CLI command. It handles:
* 1. Argument validation and coercion
* 2. Browser session lifecycle (if needed)
* 3. Domain pre-navigation for cookie/header strategies
* 4. Timeout enforcement
* 5. Lazy-loading of TS modules from manifest
*/
import { type CliCommand, type InternalCliCommand, type Arg, Strategy, getRegistry, fullName } from './registry.js';
import type { IPage } from './types.js';
import { executePipeline } from './pipeline.js';
import { AdapterLoadError } from './errors.js';
import { shouldUseBrowserSession } from './capabilityRouting.js';
import { getBrowserFactory, browserSession, runWithTimeout, DEFAULT_BROWSER_COMMAND_TIMEOUT } from './runtime.js';
/** Set of TS module paths that have been loaded */
const _loadedModules = new Set<string>();
/**
* Validates and coerces arguments based on the command's Arg definitions.
*/
export function coerceAndValidateArgs(cmdArgs: Arg[], kwargs: Record<string, any>): Record<string, any> {
const result: Record<string, any> = { ...kwargs };
for (const argDef of cmdArgs) {
const val = result[argDef.name];
// 1. Check required
if (argDef.required && (val === undefined || val === null || val === '')) {
throw new Error(`Argument "${argDef.name}" is required.\n${argDef.help ? `Hint: ${argDef.help}` : ''}`);
}
if (val !== undefined && val !== null) {
// 2. Type coercion
if (argDef.type === 'int' || argDef.type === 'number') {
const num = Number(val);
if (Number.isNaN(num)) {
throw new Error(`Argument "${argDef.name}" must be a valid number. Received: "${val}"`);
}
result[argDef.name] = num;
} else if (argDef.type === 'boolean' || argDef.type === 'bool') {
if (typeof val === 'string') {
const lower = val.toLowerCase();
if (lower === 'true' || lower === '1') result[argDef.name] = true;
else if (lower === 'false' || lower === '0') result[argDef.name] = false;
else throw new Error(`Argument "${argDef.name}" must be a boolean (true/false). Received: "${val}"`);
} else {
result[argDef.name] = Boolean(val);
}
}
// 3. Choices validation
const coercedVal = result[argDef.name];
if (argDef.choices && argDef.choices.length > 0) {
if (!argDef.choices.map(String).includes(String(coercedVal))) {
throw new Error(`Argument "${argDef.name}" must be one of: ${argDef.choices.join(', ')}. Received: "${coercedVal}"`);
}
}
} else if (argDef.default !== undefined) {
result[argDef.name] = argDef.default;
}
}
return result;
}
/**
* Run a command's func or pipeline against a page.
*/
async function runCommand(
cmd: CliCommand,
page: IPage | null,
kwargs: Record<string, any>,
debug: boolean,
): Promise<any> {
// Lazy-load TS module on first execution (manifest fast-path)
const internal = cmd as InternalCliCommand;
if (internal._lazy && internal._modulePath) {
const modulePath = internal._modulePath;
if (!_loadedModules.has(modulePath)) {
try {
await import(`file://${modulePath}`);
_loadedModules.add(modulePath);
} catch (err: any) {
throw new AdapterLoadError(
`Failed to load adapter module ${modulePath}: ${err.message}`,
'Check that the adapter file exists and has no syntax errors.',
);
}
}
// After loading, the module's cli() call will have updated the registry.
const updated = getRegistry().get(fullName(cmd));
if (updated?.func) return updated.func(page!, kwargs, debug);
if (updated?.pipeline) return executePipeline(page, updated.pipeline, { args: kwargs, debug });
}
if (cmd.func) return cmd.func(page!, kwargs, debug);
if (cmd.pipeline) return executePipeline(page, cmd.pipeline, { args: kwargs, debug });
throw new Error(`Command ${fullName(cmd)} has no func or pipeline`);
}
/**
* Execute a CLI command. Automatically manages browser sessions when needed.
*
* This is the unified entry point — callers don't need to care about
* whether the command requires a browser or not.
*/
export async function executeCommand(
cmd: CliCommand,
rawKwargs: Record<string, any>,
debug: boolean = false,
): Promise<any> {
let kwargs: Record<string, any>;
try {
kwargs = coerceAndValidateArgs(cmd.args, rawKwargs);
} catch (err: any) {
throw new Error(`[Argument Validation Error]\n${err.message}`);
}
if (shouldUseBrowserSession(cmd)) {
const BrowserFactory = getBrowserFactory();
return browserSession(BrowserFactory, async (page) => {
// Cookie/header strategies require same-origin context for credentialed fetch.
if ((cmd.strategy === Strategy.COOKIE || cmd.strategy === Strategy.HEADER) && cmd.domain) {
try { await page.goto(`https://${cmd.domain}`); await page.wait(2); } catch {}
}
return runWithTimeout(runCommand(cmd, page, kwargs, debug), {
timeout: cmd.timeoutSeconds ?? DEFAULT_BROWSER_COMMAND_TIMEOUT,
label: fullName(cmd),
});
}, { workspace: `site:${cmd.site}` });
}
// Non-browser commands run directly
return runCommand(cmd, null, kwargs, debug);
}
+135 -109
View File
@@ -223,6 +223,132 @@ export interface DiscoveredStore {
const INTERACT_FUZZ_JS = interactFuzz.toString();
// ── Analysis helpers (extracted from exploreUrl) ───────────────────────────
/** Filter, deduplicate, and score network endpoints. */
function analyzeEndpoints(networkEntries: NetworkEntry[]): { analyzed: AnalyzedEndpoint[]; totalCount: number } {
const seen = new Map<string, AnalyzedEndpoint>();
for (const entry of networkEntries) {
if (!entry.url) continue;
const ct = entry.contentType.toLowerCase();
if (ct.includes('image/') || ct.includes('font/') || ct.includes('css') || ct.includes('javascript') || ct.includes('wasm')) continue;
if (entry.status && entry.status >= 400) continue;
const pattern = urlToPattern(entry.url);
const key = `${entry.method}:${pattern}`;
if (seen.has(key)) continue;
const qp: string[] = [];
try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
const ep: AnalyzedEndpoint = {
pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct,
queryParams: qp, hasSearchParam: qp.some(p => SEARCH_PARAMS.has(p)),
hasPaginationParam: qp.some(p => PAGINATION_PARAMS.has(p)),
hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)),
authIndicators: detectAuthIndicators(entry.requestHeaders),
responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
score: 0,
};
ep.score = scoreEndpoint(ep);
seen.set(key, ep);
}
const analyzed = [...seen.values()].filter(ep => ep.score >= 5).sort((a, b) => b.score - a.score);
return { analyzed, totalCount: seen.size };
}
/** Infer CLI capabilities from analyzed endpoints. */
function inferCapabilitiesFromEndpoints(
endpoints: AnalyzedEndpoint[],
stores: DiscoveredStore[],
opts: { site?: string; goal?: string; url: string },
): { capabilities: InferredCapability[]; topStrategy: string; authIndicators: string[] } {
const capabilities: InferredCapability[] = [];
const usedNames = new Set<string>();
for (const ep of endpoints.slice(0, 8)) {
let capName = inferCapabilityName(ep.url, opts.goal);
if (usedNames.has(capName)) {
const suffix = ep.pattern.split('/').filter(s => s && !s.startsWith('{') && !s.includes('.')).pop();
capName = suffix ? `${capName}_${suffix}` : `${capName}_${usedNames.size}`;
}
usedNames.add(capName);
const cols: string[] = [];
if (ep.responseAnalysis) {
for (const role of ['title', 'url', 'author', 'score', 'time']) {
if (ep.responseAnalysis.detectedFields[role]) cols.push(role);
}
}
const args: InferredCapability['recommendedArgs'] = [];
if (ep.hasSearchParam) args.push({ name: 'keyword', type: 'str', required: true });
args.push({ name: 'limit', type: 'int', required: false, default: 20 });
if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 });
const epStrategy = inferStrategy(ep.authIndicators);
let storeHint: { store: string; action: string } | undefined;
if ((epStrategy === 'intercept' || ep.authIndicators.includes('signature')) && stores.length > 0) {
for (const s of stores) {
const matchingAction = s.actions.find(a =>
capName.split('_').some(part => a.toLowerCase().includes(part)) ||
a.toLowerCase().includes('fetch') || a.toLowerCase().includes('get')
);
if (matchingAction) { storeHint = { store: s.id, action: matchingAction }; break; }
}
}
capabilities.push({
name: capName, description: `${opts.site ?? detectSiteName(opts.url)} ${capName}`,
strategy: storeHint ? 'store-action' : epStrategy,
confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern,
itemPath: ep.responseAnalysis?.itemPath ?? null,
recommendedColumns: cols.length ? cols : ['title', 'url'],
recommendedArgs: args,
...(storeHint ? { storeHint } : {}),
});
}
const allAuth = new Set(endpoints.flatMap(ep => ep.authIndicators));
const topStrategy = allAuth.has('signature') ? 'intercept'
: allAuth.has('bearer') || allAuth.has('csrf') ? 'header'
: allAuth.size === 0 ? 'public' : 'cookie';
return { capabilities, topStrategy, authIndicators: [...allAuth] };
}
/** Write explore artifacts (manifest, endpoints, capabilities, auth, stores) to disk. */
async function writeExploreArtifacts(
targetDir: string,
result: Record<string, any>,
analyzedEndpoints: AnalyzedEndpoint[],
stores: DiscoveredStore[],
): Promise<void> {
await fs.promises.mkdir(targetDir, { recursive: true });
const tasks = [
fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
site: result.site, target_url: result.target_url, final_url: result.final_url, title: result.title,
framework: result.framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
top_strategy: result.top_strategy, explored_at: new Date().toISOString(),
}, null, 2)),
fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
})), null, 2)),
fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(result.capabilities, null, 2)),
fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
top_strategy: result.top_strategy, indicators: result.auth_indicators, framework: result.framework,
}, null, 2)),
];
if (stores.length > 0) {
tasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
}
await Promise.all(tasks);
}
// ── Main explore function ──────────────────────────────────────────────────
export async function exploreUrl(
@@ -317,125 +443,25 @@ export async function exploreUrl(
} catch {}
}
// Step 7: Analyze endpoints
const seen = new Map<string, AnalyzedEndpoint>();
for (const entry of networkEntries) {
if (!entry.url) continue;
const ct = entry.contentType.toLowerCase();
if (ct.includes('image/') || ct.includes('font/') || ct.includes('css') || ct.includes('javascript') || ct.includes('wasm')) continue;
if (entry.status && entry.status >= 400) continue;
const pattern = urlToPattern(entry.url);
const key = `${entry.method}:${pattern}`;
if (seen.has(key)) continue;
const qp: string[] = [];
try { new URL(entry.url).searchParams.forEach((_v, k) => { if (!VOLATILE_PARAMS.has(k)) qp.push(k); }); } catch {}
const ep: AnalyzedEndpoint = {
pattern, method: entry.method, url: entry.url, status: entry.status, contentType: ct,
queryParams: qp, hasSearchParam: qp.some(p => SEARCH_PARAMS.has(p)),
hasPaginationParam: qp.some(p => PAGINATION_PARAMS.has(p)),
hasLimitParam: qp.some(p => LIMIT_PARAMS.has(p)),
authIndicators: detectAuthIndicators(entry.requestHeaders),
responseAnalysis: entry.responseBody ? analyzeResponseBody(entry.responseBody) : null,
score: 0,
};
ep.score = scoreEndpoint(ep);
seen.set(key, ep);
}
const analyzedEndpoints = [...seen.values()].filter(ep => ep.score >= 5).sort((a, b) => b.score - a.score);
// Step 8: Infer capabilities
const capabilities: InferredCapability[] = [];
const usedNames = new Set<string>();
for (const ep of analyzedEndpoints.slice(0, 8)) {
let capName = inferCapabilityName(ep.url, opts.goal);
if (usedNames.has(capName)) {
const suffix = ep.pattern.split('/').filter(s => s && !s.startsWith('{') && !s.includes('.')).pop();
capName = suffix ? `${capName}_${suffix}` : `${capName}_${usedNames.size}`;
}
usedNames.add(capName);
const cols: string[] = [];
if (ep.responseAnalysis) {
for (const role of ['title', 'url', 'author', 'score', 'time']) {
if (ep.responseAnalysis.detectedFields[role]) cols.push(role);
}
}
const args: InferredCapability['recommendedArgs'] = [];
if (ep.hasSearchParam) args.push({ name: 'keyword', type: 'str', required: true });
args.push({ name: 'limit', type: 'int', required: false, default: 20 });
if (ep.hasPaginationParam) args.push({ name: 'page', type: 'int', required: false, default: 1 });
// Link store actions to capabilities when store-action strategy is recommended
const epStrategy = inferStrategy(ep.authIndicators);
let storeHint: { store: string; action: string } | undefined;
if ((epStrategy === 'intercept' || ep.authIndicators.includes('signature')) && stores.length > 0) {
// Try to find a store/action that matches this endpoint's purpose
for (const s of stores) {
const matchingAction = s.actions.find(a =>
capName.split('_').some(part => a.toLowerCase().includes(part)) ||
a.toLowerCase().includes('fetch') || a.toLowerCase().includes('get')
);
if (matchingAction) {
storeHint = { store: s.id, action: matchingAction };
break;
}
}
}
capabilities.push({
name: capName, description: `${opts.site ?? detectSiteName(url)} ${capName}`,
strategy: storeHint ? 'store-action' : epStrategy,
confidence: Math.min(ep.score / 20, 1.0), endpoint: ep.pattern,
itemPath: ep.responseAnalysis?.itemPath ?? null,
recommendedColumns: cols.length ? cols : ['title', 'url'],
recommendedArgs: args,
...(storeHint ? { storeHint } : {}),
});
}
// Step 9: Determine overall auth strategy
const allAuth = new Set(analyzedEndpoints.flatMap(ep => ep.authIndicators));
const topStrategy = allAuth.has('signature') ? 'intercept' : allAuth.has('bearer') || allAuth.has('csrf') ? 'header' : allAuth.size === 0 ? 'public' : 'cookie';
// Step 7+8: Analyze endpoints and infer capabilities
const { analyzed: analyzedEndpoints, totalCount } = analyzeEndpoints(networkEntries);
const { capabilities, topStrategy, authIndicators } = inferCapabilitiesFromEndpoints(
analyzedEndpoints, stores, { site: opts.site, goal: opts.goal, url },
);
// Step 9: Assemble result and write artifacts
const siteName = opts.site ?? detectSiteName(metadata.url || url);
const targetDir = opts.outDir ?? path.join('.opencli', 'explore', siteName);
await fs.promises.mkdir(targetDir, { recursive: true });
const result = {
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
framework, stores, top_strategy: topStrategy,
endpoint_count: analyzedEndpoints.length + [...seen.values()].filter(ep => ep.score < 5).length,
endpoint_count: totalCount,
api_endpoint_count: analyzedEndpoints.length,
capabilities, auth_indicators: [...allAuth],
capabilities, auth_indicators: authIndicators,
};
// Write artifacts
const writeTasks = [];
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'manifest.json'), JSON.stringify({
site: siteName, target_url: url, final_url: metadata.url, title: metadata.title,
framework, stores: stores.map(s => ({ type: s.type, id: s.id, actions: s.actions })),
top_strategy: topStrategy, explored_at: new Date().toISOString(),
}, null, 2)));
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'endpoints.json'), JSON.stringify(analyzedEndpoints.map(ep => ({
pattern: ep.pattern, method: ep.method, url: ep.url, status: ep.status,
contentType: ep.contentType, score: ep.score, queryParams: ep.queryParams,
itemPath: ep.responseAnalysis?.itemPath ?? null, itemCount: ep.responseAnalysis?.itemCount ?? 0,
detectedFields: ep.responseAnalysis?.detectedFields ?? {}, authIndicators: ep.authIndicators,
})), null, 2)));
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'capabilities.json'), JSON.stringify(capabilities, null, 2)));
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'auth.json'), JSON.stringify({
top_strategy: topStrategy, indicators: [...allAuth], framework,
}, null, 2)));
if (stores.length > 0) {
writeTasks.push(fs.promises.writeFile(path.join(targetDir, 'stores.json'), JSON.stringify(stores, null, 2)));
}
await Promise.all(writeTasks);
await writeExploreArtifacts(targetDir, result, analyzedEndpoints, stores);
return { ...result, out_dir: targetDir };
})(), { timeout: exploreTimeout, label: `Explore ${url}` });
}, { workspace: opts.workspace });
+1 -1
View File
@@ -6,7 +6,7 @@
import * as os from 'node:os';
import * as path from 'node:path';
import { fileURLToPath } from 'node:url';
import { discoverClis } from './engine.js';
import { discoverClis } from './discovery.js';
import { getCompletions } from './completion.js';
import { runCli } from './cli.js';
+3 -66
View File
@@ -90,70 +90,7 @@ export function registerCommand(cmd: CliCommand): void {
_registry.set(fullName(cmd), cmd);
}
// ── Serialization helpers (shared by list, --help, manifest) ────────────────
// Re-export serialization helpers from their dedicated module
export { serializeArg, serializeCommand, formatArgSummary, formatRegistryHelpText } from './serialization.js';
export type { SerializedArg } from './serialization.js';
export type SerializedArg = {
name: string;
type: string;
required: boolean;
positional: boolean;
choices: string[];
default: unknown;
help: string;
};
/** Stable arg schema — every field is always present (no sparse objects). */
export function serializeArg(a: Arg): SerializedArg {
return {
name: a.name,
type: a.type ?? 'string',
required: !!a.required,
positional: !!a.positional,
choices: a.choices ?? [],
default: a.default ?? null,
help: a.help ?? '',
};
}
/** Full command metadata for structured output (json/yaml). */
export function serializeCommand(cmd: CliCommand) {
return {
command: fullName(cmd),
site: cmd.site,
name: cmd.name,
description: cmd.description,
strategy: strategyLabel(cmd),
browser: !!cmd.browser,
args: cmd.args.map(serializeArg),
columns: cmd.columns ?? [],
domain: cmd.domain ?? null,
};
}
/** Human-readable arg summary: `<required> [optional]` style. */
export function formatArgSummary(args: Arg[]): string {
return args
.map(a => {
if (a.positional) return a.required ? `<${a.name}>` : `[${a.name}]`;
return a.required ? `--${a.name}` : `[--${a.name}]`;
})
.join(' ');
}
/** Generate the --help appendix showing registry metadata not exposed by Commander. */
export function formatRegistryHelpText(cmd: CliCommand): string {
const lines: string[] = [];
const choicesArgs = cmd.args.filter(a => a.choices?.length);
for (const a of choicesArgs) {
const prefix = a.positional ? `<${a.name}>` : `--${a.name}`;
const def = a.default != null ? ` (default: ${a.default})` : '';
lines.push(` ${prefix}: ${a.choices!.join(', ')}${def}`);
}
const meta: string[] = [];
meta.push(`Strategy: ${strategyLabel(cmd)}`);
meta.push(`Browser: ${cmd.browser ? 'yes' : 'no'}`);
if (cmd.domain) meta.push(`Domain: ${cmd.domain}`);
lines.push(meta.join(' | '));
if (cmd.columns?.length) lines.push(`Output columns: ${cmd.columns.join(', ')}`);
return '\n' + lines.join('\n') + '\n';
}
+9
View File
@@ -1,5 +1,14 @@
import { BrowserBridge, CDPBridge } from './browser/index.js';
import type { IPage } from './types.js';
/**
* Returns the appropriate browser factory based on environment config.
* Uses CDPBridge when OPENCLI_CDP_ENDPOINT is set, otherwise BrowserBridge.
*/
export function getBrowserFactory(): new () => IBrowserFactory {
return (process.env.OPENCLI_CDP_ENDPOINT ? CDPBridge : BrowserBridge) as any;
}
export const DEFAULT_BROWSER_CONNECT_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_CONNECT_TIMEOUT ?? '30', 10);
export const DEFAULT_BROWSER_COMMAND_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_COMMAND_TIMEOUT ?? '60', 10);
export const DEFAULT_BROWSER_EXPLORE_TIMEOUT = parseInt(process.env.OPENCLI_BROWSER_EXPLORE_TIMEOUT ?? '120', 10);
+79
View File
@@ -0,0 +1,79 @@
/**
* Serialization and formatting helpers for CLI commands and args.
*
* Used by the `list` command, Commander --help, and build-manifest.
* Separated from registry.ts to keep the registry focused on types + registration.
*/
import type { Arg, CliCommand } from './registry.js';
import { fullName, strategyLabel } from './registry.js';
// ── Serialization ───────────────────────────────────────────────────────────
export type SerializedArg = {
name: string;
type: string;
required: boolean;
positional: boolean;
choices: string[];
default: unknown;
help: string;
};
/** Stable arg schema — every field is always present (no sparse objects). */
export function serializeArg(a: Arg): SerializedArg {
return {
name: a.name,
type: a.type ?? 'string',
required: !!a.required,
positional: !!a.positional,
choices: a.choices ?? [],
default: a.default ?? null,
help: a.help ?? '',
};
}
/** Full command metadata for structured output (json/yaml). */
export function serializeCommand(cmd: CliCommand) {
return {
command: fullName(cmd),
site: cmd.site,
name: cmd.name,
description: cmd.description,
strategy: strategyLabel(cmd),
browser: !!cmd.browser,
args: cmd.args.map(serializeArg),
columns: cmd.columns ?? [],
domain: cmd.domain ?? null,
};
}
// ── Formatting ──────────────────────────────────────────────────────────────
/** Human-readable arg summary: `<required> [optional]` style. */
export function formatArgSummary(args: Arg[]): string {
return args
.map(a => {
if (a.positional) return a.required ? `<${a.name}>` : `[${a.name}]`;
return a.required ? `--${a.name}` : `[--${a.name}]`;
})
.join(' ');
}
/** Generate the --help appendix showing registry metadata not exposed by Commander. */
export function formatRegistryHelpText(cmd: CliCommand): string {
const lines: string[] = [];
const choicesArgs = cmd.args.filter(a => a.choices?.length);
for (const a of choicesArgs) {
const prefix = a.positional ? `<${a.name}>` : `--${a.name}`;
const def = a.default != null ? ` (default: ${a.default})` : '';
lines.push(` ${prefix}: ${a.choices!.join(', ')}${def}`);
}
const meta: string[] = [];
meta.push(`Strategy: ${strategyLabel(cmd)}`);
meta.push(`Browser: ${cmd.browser ? 'yes' : 'no'}`);
if (cmd.domain) meta.push(`Domain: ${cmd.domain}`);
lines.push(meta.join(' | '));
if (cmd.columns?.length) lines.push(`Output columns: ${cmd.columns.join(', ')}`);
return '\n' + lines.join('\n') + '\n';
}