refactor: extract shared scoring + consolidate time utils (#823)

* refactor: extract shared scoring logic and consolidate time format utils

- Extract applyUrlScoreAdjustments() and scoreArrayResponse() to analysis.ts,
  eliminating duplicated endpoint scoring between explore.ts and record.ts
- Consolidate formatDuration/formatUptime into a single formatDuration(ms)
  in download/progress.ts, reused by commands/daemon.ts

* fix: preserve explore scoring semantics and round daemon uptime

- Revert explore.ts scoreEndpoint to original inline /api/ /x/ bonus
  without record's tracking/analytics penalty (blocker from review)
- Math.round uptime*1000 to avoid floating-point noise in daemon status
This commit is contained in:
jakevin
2026-04-06 02:50:37 +08:00
committed by GitHub
parent e9867dcab0
commit d61dd7be0e
5 changed files with 39 additions and 31 deletions
+26
View File
@@ -150,6 +150,32 @@ export function detectAuthFromContent(url: string, body: unknown): string[] {
return indicators;
}
// ── Shared scoring helpers ───────────────────────────────────────────────────
/** URL-based score adjustments shared by explore and record scoring. */
export function applyUrlScoreAdjustments(url: string, score: number): number {
let s = score;
if (url.includes('/api/') || url.includes('/x/')) s += 3;
if (url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) s -= 10;
if (url.match(/\/(ping|heartbeat|keep.?alive)/i)) s -= 10;
return s;
}
/** Score an array response based on item count and detected field roles. */
export function scoreArrayResponse(arrayResult: ArrayDiscovery | null): number {
if (!arrayResult) return 0;
let s = 10;
s += Math.min(arrayResult.items.length, 10);
const sample = arrayResult.items[0];
if (sample && typeof sample === 'object') {
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
for (const aliases of Object.values(FIELD_ROLES)) {
if (aliases.some(a => keys.includes(a))) s += 2;
}
}
return s;
}
// ── Query param classification ──────────────────────────────────────────────
/** Extract non-volatile query params and classify them. */
+2 -9
View File
@@ -7,14 +7,7 @@
import chalk from 'chalk';
import { fetchDaemonStatus, requestDaemonShutdown } from '../browser/daemon-client.js';
function formatUptime(seconds: number): string {
const h = Math.floor(seconds / 3600);
const m = Math.floor((seconds % 3600) / 60);
if (h > 0) return `${h}h ${m}m`;
if (m > 0) return `${m}m`;
return `${Math.floor(seconds)}s`;
}
import { formatDuration } from '../download/progress.js';
function formatTimeSince(timestampMs: number): string {
const seconds = (Date.now() - timestampMs) / 1000;
@@ -33,7 +26,7 @@ export async function daemonStatus(): Promise<void> {
}
console.log(`Daemon: ${chalk.green('running')} (PID ${status.pid})`);
console.log(`Uptime: ${formatUptime(status.uptime)}`);
console.log(`Uptime: ${formatDuration(Math.round(status.uptime * 1000))}`);
console.log(`Extension: ${status.extensionConnected ? chalk.green('connected') : chalk.yellow('disconnected')}`);
console.log(`Last CLI request: ${formatTimeSince(status.lastCliRequestTime)}`);
console.log(`Memory: ${status.memoryMB} MB`);
+7 -2
View File
@@ -29,8 +29,13 @@ export function formatDuration(ms: number): string {
const seconds = Math.floor(ms / 1000);
if (seconds < 60) return `${seconds}s`;
const minutes = Math.floor(seconds / 60);
const remainingSeconds = seconds % 60;
return `${minutes}m ${remainingSeconds}s`;
if (minutes < 60) {
const remainingSeconds = seconds % 60;
return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
}
const hours = Math.floor(minutes / 60);
const remainingMinutes = minutes % 60;
return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
}
/**
-1
View File
@@ -203,7 +203,6 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndp
if (ep.hasPaginationParam) s += 2;
if (ep.hasLimitParam) s += 2;
if (ep.status === 200) s += 2;
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3;
return s;
}
+4 -19
View File
@@ -27,6 +27,8 @@ import {
inferStrategy,
detectAuthFromContent,
classifyQueryParams,
applyUrlScoreAdjustments,
scoreArrayResponse,
} from './analysis.js';
// ── Types ──────────────────────────────────────────────────────────────────
@@ -82,11 +84,7 @@ function preferRecordedCandidate(current: RecordedCandidate, next: RecordedCandi
/** Apply shared endpoint score tweaks. */
function applyCommonEndpointScoreAdjustments(req: RecordedRequest, score: number): number {
let adjusted = score;
if (req.url.includes('/api/')) adjusted += 3;
if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) adjusted -= 10;
if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i)) adjusted -= 10;
return adjusted;
return applyUrlScoreAdjustments(req.url, score);
}
/** Build a candidate-level dedupe key. */
@@ -330,20 +328,7 @@ function generateReadRecordedJs(): string {
// ── Analysis helpers ───────────────────────────────────────────────────────
function scoreRequest(req: RecordedRequest, arrayResult: ReturnType<typeof findArrayPath> | null): number {
let s = 0;
if (arrayResult) {
s += 10;
s += Math.min(arrayResult.items.length, 10);
// Bonus for detected semantic fields
const sample = arrayResult.items[0];
if (sample && typeof sample === 'object') {
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
for (const aliases of Object.values(FIELD_ROLES)) {
if (aliases.some(a => keys.includes(a))) s += 2;
}
}
}
return applyCommonEndpointScoreAdjustments(req, s);
return applyCommonEndpointScoreAdjustments(req, scoreArrayResponse(arrayResult));
}
/** Check whether one recorded request is safe to treat as a write candidate. */