refactor: extract shared scoring + consolidate time utils (#823)
* refactor: extract shared scoring logic and consolidate time format utils - Extract applyUrlScoreAdjustments() and scoreArrayResponse() to analysis.ts, eliminating duplicated endpoint scoring between explore.ts and record.ts - Consolidate formatDuration/formatUptime into a single formatDuration(ms) in download/progress.ts, reused by commands/daemon.ts * fix: preserve explore scoring semantics and round daemon uptime - Revert explore.ts scoreEndpoint to original inline /api/ /x/ bonus without record's tracking/analytics penalty (blocker from review) - Math.round uptime*1000 to avoid floating-point noise in daemon status
This commit is contained in:
@@ -150,6 +150,32 @@ export function detectAuthFromContent(url: string, body: unknown): string[] {
|
||||
return indicators;
|
||||
}
|
||||
|
||||
// ── Shared scoring helpers ───────────────────────────────────────────────────
|
||||
|
||||
/** URL-based score adjustments shared by explore and record scoring. */
|
||||
export function applyUrlScoreAdjustments(url: string, score: number): number {
|
||||
let s = score;
|
||||
if (url.includes('/api/') || url.includes('/x/')) s += 3;
|
||||
if (url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) s -= 10;
|
||||
if (url.match(/\/(ping|heartbeat|keep.?alive)/i)) s -= 10;
|
||||
return s;
|
||||
}
|
||||
|
||||
/** Score an array response based on item count and detected field roles. */
|
||||
export function scoreArrayResponse(arrayResult: ArrayDiscovery | null): number {
|
||||
if (!arrayResult) return 0;
|
||||
let s = 10;
|
||||
s += Math.min(arrayResult.items.length, 10);
|
||||
const sample = arrayResult.items[0];
|
||||
if (sample && typeof sample === 'object') {
|
||||
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
|
||||
for (const aliases of Object.values(FIELD_ROLES)) {
|
||||
if (aliases.some(a => keys.includes(a))) s += 2;
|
||||
}
|
||||
}
|
||||
return s;
|
||||
}
|
||||
|
||||
// ── Query param classification ──────────────────────────────────────────────
|
||||
|
||||
/** Extract non-volatile query params and classify them. */
|
||||
|
||||
@@ -7,14 +7,7 @@
|
||||
|
||||
import chalk from 'chalk';
|
||||
import { fetchDaemonStatus, requestDaemonShutdown } from '../browser/daemon-client.js';
|
||||
|
||||
function formatUptime(seconds: number): string {
|
||||
const h = Math.floor(seconds / 3600);
|
||||
const m = Math.floor((seconds % 3600) / 60);
|
||||
if (h > 0) return `${h}h ${m}m`;
|
||||
if (m > 0) return `${m}m`;
|
||||
return `${Math.floor(seconds)}s`;
|
||||
}
|
||||
import { formatDuration } from '../download/progress.js';
|
||||
|
||||
function formatTimeSince(timestampMs: number): string {
|
||||
const seconds = (Date.now() - timestampMs) / 1000;
|
||||
@@ -33,7 +26,7 @@ export async function daemonStatus(): Promise<void> {
|
||||
}
|
||||
|
||||
console.log(`Daemon: ${chalk.green('running')} (PID ${status.pid})`);
|
||||
console.log(`Uptime: ${formatUptime(status.uptime)}`);
|
||||
console.log(`Uptime: ${formatDuration(Math.round(status.uptime * 1000))}`);
|
||||
console.log(`Extension: ${status.extensionConnected ? chalk.green('connected') : chalk.yellow('disconnected')}`);
|
||||
console.log(`Last CLI request: ${formatTimeSince(status.lastCliRequestTime)}`);
|
||||
console.log(`Memory: ${status.memoryMB} MB`);
|
||||
|
||||
@@ -29,8 +29,13 @@ export function formatDuration(ms: number): string {
|
||||
const seconds = Math.floor(ms / 1000);
|
||||
if (seconds < 60) return `${seconds}s`;
|
||||
const minutes = Math.floor(seconds / 60);
|
||||
const remainingSeconds = seconds % 60;
|
||||
return `${minutes}m ${remainingSeconds}s`;
|
||||
if (minutes < 60) {
|
||||
const remainingSeconds = seconds % 60;
|
||||
return remainingSeconds > 0 ? `${minutes}m ${remainingSeconds}s` : `${minutes}m`;
|
||||
}
|
||||
const hours = Math.floor(minutes / 60);
|
||||
const remainingMinutes = minutes % 60;
|
||||
return remainingMinutes > 0 ? `${hours}h ${remainingMinutes}m` : `${hours}h`;
|
||||
}
|
||||
|
||||
/**
|
||||
|
||||
@@ -203,7 +203,6 @@ function scoreEndpoint(ep: { contentType: string; responseAnalysis: AnalyzedEndp
|
||||
if (ep.hasPaginationParam) s += 2;
|
||||
if (ep.hasLimitParam) s += 2;
|
||||
if (ep.status === 200) s += 2;
|
||||
// Anti-Bot Empty Value Detection: penalize JSON endpoints returning empty data
|
||||
if (ep.responseAnalysis && ep.responseAnalysis.itemCount === 0 && ep.contentType.includes('json')) s -= 3;
|
||||
return s;
|
||||
}
|
||||
|
||||
+4
-19
@@ -27,6 +27,8 @@ import {
|
||||
inferStrategy,
|
||||
detectAuthFromContent,
|
||||
classifyQueryParams,
|
||||
applyUrlScoreAdjustments,
|
||||
scoreArrayResponse,
|
||||
} from './analysis.js';
|
||||
|
||||
// ── Types ──────────────────────────────────────────────────────────────────
|
||||
@@ -82,11 +84,7 @@ function preferRecordedCandidate(current: RecordedCandidate, next: RecordedCandi
|
||||
|
||||
/** Apply shared endpoint score tweaks. */
|
||||
function applyCommonEndpointScoreAdjustments(req: RecordedRequest, score: number): number {
|
||||
let adjusted = score;
|
||||
if (req.url.includes('/api/')) adjusted += 3;
|
||||
if (req.url.match(/\/(track|log|analytics|beacon|pixel|stats|metric)/i)) adjusted -= 10;
|
||||
if (req.url.match(/\/(ping|heartbeat|keep.?alive)/i)) adjusted -= 10;
|
||||
return adjusted;
|
||||
return applyUrlScoreAdjustments(req.url, score);
|
||||
}
|
||||
|
||||
/** Build a candidate-level dedupe key. */
|
||||
@@ -330,20 +328,7 @@ function generateReadRecordedJs(): string {
|
||||
// ── Analysis helpers ───────────────────────────────────────────────────────
|
||||
|
||||
function scoreRequest(req: RecordedRequest, arrayResult: ReturnType<typeof findArrayPath> | null): number {
|
||||
let s = 0;
|
||||
if (arrayResult) {
|
||||
s += 10;
|
||||
s += Math.min(arrayResult.items.length, 10);
|
||||
// Bonus for detected semantic fields
|
||||
const sample = arrayResult.items[0];
|
||||
if (sample && typeof sample === 'object') {
|
||||
const keys = Object.keys(sample as object).map(k => k.toLowerCase());
|
||||
for (const aliases of Object.values(FIELD_ROLES)) {
|
||||
if (aliases.some(a => keys.includes(a))) s += 2;
|
||||
}
|
||||
}
|
||||
}
|
||||
return applyCommonEndpointScoreAdjustments(req, s);
|
||||
return applyCommonEndpointScoreAdjustments(req, scoreArrayResponse(arrayResult));
|
||||
}
|
||||
|
||||
/** Check whether one recorded request is safe to treat as a write candidate. */
|
||||
|
||||
Reference in New Issue
Block a user