docs: Autogenerated public API documentation (#3798)
This commit is contained in:
Jan Buchar
committed by
Martin Adámek
Martin Adámek
parent
4f0c4c452d
commit
a72033d4c2
@@ -67,6 +67,37 @@ jobs:
|
||||
- name: Tests
|
||||
run: pnpm test
|
||||
|
||||
api_surface:
|
||||
name: Public API surface
|
||||
runs-on: ubuntu-22.04
|
||||
|
||||
steps:
|
||||
- name: Checkout repository
|
||||
uses: actions/checkout@v6
|
||||
|
||||
- name: Use Node.js 24
|
||||
uses: actions/setup-node@v6
|
||||
with:
|
||||
node-version: 24
|
||||
package-manager-cache: false
|
||||
|
||||
- name: Turbo cache
|
||||
id: turbo-cache
|
||||
uses: actions/cache@v5
|
||||
with:
|
||||
path: .turbo
|
||||
key: turbo-${{ github.job }}-${{ github.ref_name }}-${{ github.sha }}
|
||||
restore-keys: |
|
||||
turbo-${{ github.job }}-${{ github.ref_name }}-
|
||||
|
||||
- uses: apify/workflows/pnpm-install@main
|
||||
|
||||
- name: Build
|
||||
run: pnpm ci:build
|
||||
|
||||
- name: Check public API surface
|
||||
run: pnpm api:check
|
||||
|
||||
docs:
|
||||
name: Docs build
|
||||
if: (!contains(github.event.head_commit.message, '[skip ci]') && github.ref != 'refs/heads/master' && github.ref != 'refs/heads/v4')
|
||||
|
||||
@@ -33,3 +33,6 @@ test/e2e/**/packages
|
||||
|
||||
# Local vitest config overrides
|
||||
vitest.config.local.mts
|
||||
|
||||
# API Extractor intermediate reports (the committed reports live in docs/public-api/*.api.md)
|
||||
docs/public-api/temp
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
# Public API surface maps
|
||||
|
||||
Each `*.api.md` file in this folder is a generated **map of the public, type-level
|
||||
interface** of one publishable `@crawlee/*` package — every exported class, method,
|
||||
property, function, and type, with full signatures. These reports define **where we
|
||||
promise backwards compatibility**.
|
||||
|
||||
They are produced by [API Extractor](https://api-extractor.com/) from the built
|
||||
`dist/index.d.ts` of each package.
|
||||
|
||||
## Workflow
|
||||
|
||||
- After changing any package's public surface, regenerate the reports and commit them:
|
||||
|
||||
```sh
|
||||
pnpm build # the reports are generated from dist/
|
||||
pnpm api:extract
|
||||
```
|
||||
|
||||
- CI runs `pnpm api:check`, which fails if a committed report is out of date. A failing
|
||||
check means you changed the public API: either that change is intentional (commit the
|
||||
updated report — reviewers will see the surface diff) or it was accidental (fix it).
|
||||
|
||||
## Notes
|
||||
|
||||
- `docs/public-api/temp/` holds intermediate reports and is git-ignored.
|
||||
- `@crawlee/cli` and `@crawlee/templates` are deliberately excluded — they are tooling
|
||||
(a CLI binary and project scaffolding), not an importable API where we promise BC. The
|
||||
exclude list lives in `scripts/api-extractor/run.ts`.
|
||||
- The generator lives in `scripts/api-extractor/`. It temporarily strips the build's
|
||||
injected `// @ts-ignore` comment lines from the `.d.ts` files (restoring them
|
||||
afterwards) because API Extractor's AST walker trips over some of them; a small number
|
||||
of packages additionally need a sanitized-mirror fallback. See the comments in
|
||||
`scripts/api-extractor/run.ts` for details.
|
||||
- Symbols tagged `@internal` still show up here if they are exported. Shrinking these
|
||||
reports (hiding internals, e.g. via `@internal` + a trimmed rollup, or by not exporting
|
||||
them at all) is the goal tracked in issue #3109.
|
||||
@@ -0,0 +1,335 @@
|
||||
## API Report File for "@crawlee/basic"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { AddRequestsBatchedOptions } from '@crawlee/core';
|
||||
import type { AddRequestsBatchedResult } from '@crawlee/core';
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import { AutoscaledPool } from '@crawlee/core';
|
||||
import { AutoscaledPoolOptions } from '@crawlee/core';
|
||||
import type { Awaitable } from '@crawlee/types';
|
||||
import type { BaseHttpClient } from '@crawlee/types';
|
||||
import { BasePredicate } from 'ow';
|
||||
import type { BatchAddRequestsResult } from '@crawlee/types';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import { Cheerio } from '@crawlee/utils';
|
||||
import { CheerioAPI } from '@crawlee/utils';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import type { Configuration } from '@crawlee/core';
|
||||
import { ContextPipeline } from '@crawlee/core';
|
||||
import type { CrawleeLogger } from '@crawlee/core';
|
||||
import type { CrawlingContext } from '@crawlee/core';
|
||||
import { Dataset } from '@crawlee/core';
|
||||
import type { DatasetExportOptions } from '@crawlee/core';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import { Element as Element_2 } from '@crawlee/utils';
|
||||
import type { EnqueueLinksOptions } from '@crawlee/core';
|
||||
import type { EventManager } from '@crawlee/core';
|
||||
import type { FinalStatistics } from '@crawlee/core';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/core';
|
||||
import { IRequestLoader } from '@crawlee/core';
|
||||
import { IRequestManager } from '@crawlee/core';
|
||||
import type { ISession } from '@crawlee/types';
|
||||
import type { ISessionPool } from '@crawlee/types';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import { Predicate } from 'ow';
|
||||
import { ProxyConfiguration } from '@crawlee/core';
|
||||
import type { ProxyInfo } from '@crawlee/types';
|
||||
import type { ReadonlyDeep } from 'type-fest';
|
||||
import { Request as Request_2 } from '@crawlee/core';
|
||||
import { RequestQueue } from '@crawlee/core';
|
||||
import type { RequestsLike } from '@crawlee/core';
|
||||
import { RobotsTxtFile } from '@crawlee/utils';
|
||||
import type { RouterHandler } from '@crawlee/core';
|
||||
import type { RouterRoutes } from '@crawlee/core';
|
||||
import type { SetRequired } from 'type-fest';
|
||||
import type { SetStatusMessageOptions } from '@crawlee/types';
|
||||
import type { SkippedRequestCallback } from '@crawlee/core';
|
||||
import type { Source } from '@crawlee/core';
|
||||
import { Statistics } from '@crawlee/core';
|
||||
import type { StatisticsOptions } from '@crawlee/core';
|
||||
import type { StatisticState } from '@crawlee/core';
|
||||
import type { StorageClient } from '@crawlee/types';
|
||||
import type { StorageIdentifier } from '@crawlee/core';
|
||||
import { StringPredicate } from 'ow';
|
||||
import { TimeoutError } from '@apify/timeout';
|
||||
|
||||
// @public
|
||||
export class BasicCrawler<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
||||
constructor(options?: BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<CrawlingContext, Context>);
|
||||
// @internal
|
||||
protected addCrawlDepthRequestGenerator(requests: RequestsLike, newRequestDepth: number): AsyncGenerator<Source, void, undefined>;
|
||||
// (undocumented)
|
||||
protected additionalHttpErrorStatusCodes: Set<number>;
|
||||
addRequests(requests: ReadonlyDeep<RequestsLike>, options?: CrawlerAddRequestsOptions): Promise<CrawlerAddRequestsResult>;
|
||||
autoscaledPool?: AutoscaledPool;
|
||||
// (undocumented)
|
||||
protected autoscaledPoolOptions: AutoscaledPoolOptions;
|
||||
get basicContextPipeline(): ContextPipeline<{
|
||||
request: Request_2;
|
||||
}, CrawlingContext>;
|
||||
// (undocumented)
|
||||
protected blockedStatusCodes: Set<number>;
|
||||
protected buildBasicContextPipeline(): ContextPipeline<{
|
||||
request: Request_2;
|
||||
}, CrawlingContext>;
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext, CrawlingContext>;
|
||||
// (undocumented)
|
||||
protected calculateEnqueuedRequestLimit(explicitLimit?: number): Promise<number | undefined>;
|
||||
// (undocumented)
|
||||
protected _canRequestBeRetried(request: Request_2, error: Error): boolean;
|
||||
// (undocumented)
|
||||
get contextPipeline(): ContextPipeline<CrawlingContext, ExtendedContext>;
|
||||
// (undocumented)
|
||||
protected static readonly CRAWLEE_STATE_KEY = "CRAWLEE_STATE";
|
||||
protected _defaultIsFinishedFunction(): Promise<boolean>;
|
||||
protected delayRequest(request: Request_2, source: IRequestManager): boolean;
|
||||
// (undocumented)
|
||||
protected domainAccessedTime: Map<string, number>;
|
||||
// @internal
|
||||
protected enqueueLinksWithCrawlDepth(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: Request_2<Dictionary>, requestManager: IRequestManager): Promise<BatchAddRequestsResult>;
|
||||
// (undocumented)
|
||||
protected errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
||||
exportData<Data>(path: string, format?: 'json' | 'csv', options?: DatasetExportOptions): Promise<Data[]>;
|
||||
// (undocumented)
|
||||
protected failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
||||
protected _fetchNextRequest(): Promise<Request_2<Dictionary> | null>;
|
||||
// (undocumented)
|
||||
protected _getCookieHeaderFromRequest(request: Request_2): string;
|
||||
getData(...args: Parameters<Dataset['getData']>): ReturnType<Dataset['getData']>;
|
||||
getDataset(identifier?: string | StorageIdentifier): Promise<Dataset>;
|
||||
protected _getMessageFromError(error: Error, forceStack?: boolean): string | TimeoutError | undefined;
|
||||
// (undocumented)
|
||||
protected getPendingRequestCountApproximation(): Promise<number>;
|
||||
getRequestManager(): Promise<IRequestManager>;
|
||||
// @deprecated (undocumented)
|
||||
getRequestQueue(): Promise<IRequestManager>;
|
||||
// (undocumented)
|
||||
protected getRobotsTxtFileForUrl(url: string): Promise<RobotsTxtFile | undefined>;
|
||||
// (undocumented)
|
||||
protected handledRequestsCount: number;
|
||||
// (undocumented)
|
||||
protected _handleFailedRequestHandler(crawlingContext: CrawlingContext, error: Error): Promise<void>;
|
||||
protected handleRequest(crawlingContext: ExtendedContext, requestSource: IRequestManager, request: Request_2): Promise<void>;
|
||||
// (undocumented)
|
||||
protected handleSkippedRequest(options: Parameters<SkippedRequestCallback>[0]): Promise<void>;
|
||||
// (undocumented)
|
||||
hasFinishedBefore: boolean;
|
||||
// (undocumented)
|
||||
protected httpClient: BaseHttpClient;
|
||||
// (undocumented)
|
||||
protected ignoreHttpErrorStatusCodes: Set<number>;
|
||||
protected _init(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected internalTimeoutMillis: number;
|
||||
protected isErrorStatusCode(status: number): boolean;
|
||||
protected isProxyError(error: Error): boolean;
|
||||
protected _isTaskReadyFunction(): Promise<boolean>;
|
||||
protected _loadHandledRequestCount(): Promise<void>;
|
||||
// (undocumented)
|
||||
get log(): CrawleeLogger;
|
||||
// (undocumented)
|
||||
protected maxCrawlDepth?: number;
|
||||
// (undocumented)
|
||||
protected maxRequestRetries: number;
|
||||
// (undocumented)
|
||||
protected maxRequestsPerCrawl?: number;
|
||||
// (undocumented)
|
||||
protected onSkippedRequest?: SkippedRequestCallback;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
// (undocumented)
|
||||
protected _pauseOnMigration(): Promise<void>;
|
||||
proxyConfiguration?: ProxyConfiguration;
|
||||
pushData(data: Parameters<Dataset['pushData']>[0], datasetIdentifier?: string | StorageIdentifier): Promise<void>;
|
||||
protected _requestFunctionErrorHandler(error: Error, crawlingContext: CrawlingContext, request: Request_2, source: IRequestManager): Promise<void>;
|
||||
// (undocumented)
|
||||
protected requestHandler: RequestHandler<ExtendedContext>;
|
||||
// (undocumented)
|
||||
protected requestHandlerTimeoutMillis: number;
|
||||
protected requestManager?: IRequestManager;
|
||||
// (undocumented)
|
||||
protected respectRobotsTxtFile: boolean | {
|
||||
userAgent?: string;
|
||||
};
|
||||
// (undocumented)
|
||||
protected retryOnBlocked: boolean;
|
||||
readonly router: RouterHandler<Context>;
|
||||
run(requests?: RequestsLike, options?: CrawlerRunOptions): Promise<FinalStatistics>;
|
||||
// (undocumented)
|
||||
running: boolean;
|
||||
// (undocumented)
|
||||
protected runRequestHandler(crawlingContext: ExtendedContext): Promise<void>;
|
||||
// (undocumented)
|
||||
protected sameDomainDelayMillis: number;
|
||||
sessionPool: ISessionPool;
|
||||
setStatusMessage(message: string, options?: SetStatusMessageOptions): Promise<void>;
|
||||
readonly stats: Statistics;
|
||||
// (undocumented)
|
||||
protected statusMessageCallback?: StatusMessageCallback;
|
||||
// (undocumented)
|
||||
protected statusMessageLoggingInterval: number;
|
||||
stop(reason?: string): void;
|
||||
// (undocumented)
|
||||
protected _tagUserHandlerError<T>(cb: () => unknown): Promise<T>;
|
||||
teardown(): Promise<void>;
|
||||
protected _throwOnBlockedRequest(statusCode: number): void;
|
||||
protected _timeoutAndRetry<T>(handler: () => Promise<T>, timeout: number, error: Error | string, maxRetries?: number, retried?: number): Promise<T>;
|
||||
// (undocumented)
|
||||
protected unexpectedStop: boolean;
|
||||
// (undocumented)
|
||||
useState<State extends Dictionary = Dictionary>(defaultValue?: State): Promise<State>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BasicCrawlerOptions<Context extends CrawlingContext = CrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> {
|
||||
additionalHttpErrorStatusCodes?: number[];
|
||||
autoscaledPoolOptions?: AutoscaledPoolOptions;
|
||||
blockedStatusCodes?: number[];
|
||||
configuration?: Configuration;
|
||||
contextPipelineBuilder?: () => ContextPipeline<CrawlingContext, Context>;
|
||||
errorHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
||||
eventManager?: EventManager;
|
||||
extendContext?: (context: Context) => Awaitable<ContextExtension>;
|
||||
failedRequestHandler?: ErrorHandler<CrawlingContext, ExtendedContext>;
|
||||
httpClient?: BaseHttpClient;
|
||||
id?: string;
|
||||
ignoreHttpErrorStatusCodes?: number[];
|
||||
keepAlive?: boolean;
|
||||
logger?: CrawleeLogger;
|
||||
maxConcurrency?: number;
|
||||
maxCrawlDepth?: number;
|
||||
maxRequestRetries?: number;
|
||||
maxRequestsPerCrawl?: number;
|
||||
maxRequestsPerMinute?: number;
|
||||
minConcurrency?: number;
|
||||
onSkippedRequest?: SkippedRequestCallback;
|
||||
proxyConfiguration?: ProxyConfiguration;
|
||||
requestHandler?: RequestHandler<ExtendedContext>;
|
||||
requestHandlerTimeoutSecs?: number;
|
||||
// @deprecated
|
||||
requestList?: IRequestLoader;
|
||||
requestManager?: IRequestManager;
|
||||
// @deprecated
|
||||
requestQueue?: RequestQueue;
|
||||
respectRobotsTxtFile?: boolean | {
|
||||
userAgent?: string;
|
||||
};
|
||||
retryOnBlocked?: boolean;
|
||||
sameDomainDelaySecs?: number;
|
||||
sessionPool?: ISessionPool;
|
||||
statisticsOptions?: StatisticsOptions;
|
||||
statusMessageCallback?: StatusMessageCallback;
|
||||
statusMessageLoggingInterval?: number;
|
||||
storageClient?: StorageClient;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary> extends CrawlingContext<UserData> {
|
||||
}
|
||||
|
||||
export { Cheerio }
|
||||
|
||||
export { CheerioAPI }
|
||||
|
||||
export { CheerioRoot }
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CrawlerAddRequestsOptions extends AddRequestsBatchedOptions {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CrawlerAddRequestsResult extends AddRequestsBatchedResult {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CrawlerRunOptions extends CrawlerAddRequestsOptions {
|
||||
purgeRequestQueue?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export function createBasicRouter<Context extends BasicCrawlingContext = BasicCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CreateContextOptions {
|
||||
// (undocumented)
|
||||
proxyInfo?: ProxyInfo;
|
||||
// (undocumented)
|
||||
request: Request_2;
|
||||
// (undocumented)
|
||||
session: ISession;
|
||||
}
|
||||
|
||||
export { Element_2 as Element }
|
||||
|
||||
// @public (undocumented)
|
||||
export type ErrorHandler<Context extends CrawlingContext = CrawlingContext, ExtendedContext extends Context = Context> = (inputs: Context & Partial<ExtendedContext>, error: Error) => Awaitable<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type RequestHandler<Context extends CrawlingContext = CrawlingContext> = (inputs: Context) => Awaitable<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type RequireContextPipeline<DefaultContextType extends CrawlingContext, FinalContextType extends DefaultContextType> = DefaultContextType extends FinalContextType ? {} : {
|
||||
contextPipelineBuilder: () => ContextPipeline<CrawlingContext, FinalContextType>;
|
||||
};
|
||||
|
||||
// @public (undocumented)
|
||||
export type StatusMessageCallback<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> = (params: StatusMessageCallbackParams<Context, Crawler>) => Awaitable<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface StatusMessageCallbackParams<Context extends CrawlingContext = BasicCrawlingContext, Crawler extends BasicCrawler<any> = BasicCrawler<Context>> {
|
||||
// (undocumented)
|
||||
crawler: Crawler;
|
||||
// (undocumented)
|
||||
message: string;
|
||||
// (undocumented)
|
||||
previousState: StatisticState;
|
||||
// (undocumented)
|
||||
state: StatisticState;
|
||||
}
|
||||
|
||||
|
||||
export * from "@crawlee/core";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,536 @@
|
||||
## API Report File for "@crawlee/browser-pool"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { Browser } from 'playwright';
|
||||
import type { BrowserContext } from 'playwright';
|
||||
import type { BrowserFingerprintWithHeaders } from 'fingerprint-generator';
|
||||
import type { BrowserType } from 'playwright';
|
||||
import type { Cookie } from '@crawlee/types';
|
||||
import { CrawleeLogger } from '@crawlee/core';
|
||||
import { CriticalError } from '@crawlee/core';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import { EventEmitter } from 'node:events';
|
||||
import { FingerprintGenerator as FingerprintGenerator_2 } from 'fingerprint-generator';
|
||||
import type { FingerprintGeneratorOptions as FingerprintGeneratorOptions_2 } from 'fingerprint-generator';
|
||||
import { FingerprintInjector } from 'fingerprint-injector';
|
||||
import { IBrowserPool } from '@crawlee/types';
|
||||
import { NewPageOptions } from '@crawlee/types';
|
||||
import type { Page } from 'playwright';
|
||||
import type { PageState } from '@crawlee/types';
|
||||
import type Puppeteer from 'puppeteer';
|
||||
import type * as PuppeteerTypes from 'puppeteer';
|
||||
import QuickLRU from 'quick-lru';
|
||||
import { TypedEmitter } from 'tiny-typed-emitter';
|
||||
|
||||
// @internal (undocumented)
|
||||
export const anonymizeProxySugar: (proxyUrl?: string, username?: string, password?: string, options?: AnonymizeProxySugarOptions) => Promise<[string | undefined, PromiseVoid]>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface AnonymizeProxySugarOptions {
|
||||
// (undocumented)
|
||||
ignoreProxyCertificate?: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export const enum BROWSER_CONTROLLER_EVENTS {
|
||||
// (undocumented)
|
||||
BROWSER_CLOSED = "browserClosed"
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export const enum BROWSER_POOL_EVENTS {
|
||||
// (undocumented)
|
||||
BROWSER_CLOSED = "browserClosed",
|
||||
// (undocumented)
|
||||
BROWSER_LAUNCHED = "browserLaunched",
|
||||
// (undocumented)
|
||||
BROWSER_RETIRED = "browserRetired",
|
||||
// (undocumented)
|
||||
PAGE_CLOSED = "pageClosed",
|
||||
// (undocumented)
|
||||
PAGE_CREATED = "pageCreated"
|
||||
}
|
||||
|
||||
// @public
|
||||
export abstract class BrowserController<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends TypedEmitter<BrowserControllerEvents<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>> implements IBrowserController<NewPageResult> {
|
||||
constructor(browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
||||
activate(): void;
|
||||
// (undocumented)
|
||||
activePages: number;
|
||||
// (undocumented)
|
||||
assignBrowser(browser: LaunchResult, launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): void;
|
||||
browser: LaunchResult;
|
||||
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
close(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected abstract _close(): Promise<void>;
|
||||
// (undocumented)
|
||||
getCookies(page: NewPageResult): Promise<Cookie[]>;
|
||||
// (undocumented)
|
||||
protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
|
||||
// (undocumented)
|
||||
id: string;
|
||||
// (undocumented)
|
||||
isActive: boolean;
|
||||
kill(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected abstract _kill(): Promise<void>;
|
||||
// (undocumented)
|
||||
lastPageOpenedAt: number;
|
||||
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
// (undocumented)
|
||||
protected log: CrawleeLogger;
|
||||
newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
|
||||
// (undocumented)
|
||||
protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
|
||||
// (undocumented)
|
||||
abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
||||
proxyUrl?: string;
|
||||
// (undocumented)
|
||||
setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
|
||||
// (undocumented)
|
||||
protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
|
||||
// (undocumented)
|
||||
totalPages: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserControllerEvents<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
||||
// (undocumented)
|
||||
[BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class BrowserLaunchError extends CriticalError {
|
||||
constructor(...args: ConstructorParameters<typeof CriticalError>);
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export enum BrowserName {
|
||||
// (undocumented)
|
||||
chrome = "chrome",
|
||||
// (undocumented)
|
||||
edge = "edge",
|
||||
// (undocumented)
|
||||
firefox = "firefox",
|
||||
// (undocumented)
|
||||
safari = "safari"
|
||||
}
|
||||
|
||||
// @public
|
||||
export abstract class BrowserPlugin<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
||||
constructor(library: Library, options?: BrowserPluginOptions<LibraryOptions>);
|
||||
// (undocumented)
|
||||
protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void>;
|
||||
// (undocumented)
|
||||
browserPerProxy?: boolean;
|
||||
// (undocumented)
|
||||
abstract createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
createLaunchContext(options?: CreateLaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
// (undocumented)
|
||||
ignoreProxyCertificate?: boolean;
|
||||
// (undocumented)
|
||||
protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean;
|
||||
launch(launchContext?: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
||||
// (undocumented)
|
||||
protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult>;
|
||||
// (undocumented)
|
||||
launchOptions: LibraryOptions;
|
||||
// (undocumented)
|
||||
library: Library;
|
||||
// (undocumented)
|
||||
protected log: CrawleeLogger;
|
||||
// (undocumented)
|
||||
name: string;
|
||||
// (undocumented)
|
||||
proxyUrl?: string;
|
||||
// (undocumented)
|
||||
protected _throwAugmentedLaunchError(cause: unknown, executablePath: string | undefined, dockerImage: string, moduleInstallCommand: string): never;
|
||||
// (undocumented)
|
||||
useIncognitoPages: boolean;
|
||||
// (undocumented)
|
||||
userDataDir?: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPluginOptions<LibraryOptions> {
|
||||
browserPerProxy?: boolean;
|
||||
ignoreProxyCertificate?: boolean;
|
||||
launchOptions?: LibraryOptions;
|
||||
proxyUrl?: string;
|
||||
useIncognitoPages?: boolean;
|
||||
userDataDir?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class BrowserPool<Options extends BrowserPoolOptions = BrowserPoolOptions, BrowserPlugins extends BrowserPlugin[] = InferBrowserPluginArray<Options['browserPlugins']>, BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>, LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>, PageOptions = Parameters<BrowserControllerReturn['newPage']>[0], PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> implements IBrowserPool<PageReturn> {
|
||||
constructor(options: Options & BrowserPoolHooks<BrowserControllerReturn, LaunchContextReturn, PageReturn>);
|
||||
// (undocumented)
|
||||
activeBrowserControllers: Set<BrowserControllerReturn>;
|
||||
// (undocumented)
|
||||
browserPlugins: BrowserPlugins;
|
||||
closeAllBrowsers(): Promise<void>;
|
||||
// (undocumented)
|
||||
closeInactiveBrowserAfterMillis: number;
|
||||
closePage(page: PageReturn, options?: {
|
||||
error?: Error;
|
||||
}): Promise<void>;
|
||||
destroy(): Promise<void>;
|
||||
extractPageState(page: PageReturn): Promise<PageState>;
|
||||
// (undocumented)
|
||||
fingerprintCache?: QuickLRU<string, BrowserFingerprintWithHeaders>;
|
||||
// (undocumented)
|
||||
fingerprintGenerator?: FingerprintGenerator_2;
|
||||
// (undocumented)
|
||||
fingerprintInjector?: FingerprintInjector;
|
||||
// (undocumented)
|
||||
fingerprintOptions: FingerprintOptions;
|
||||
getBrowserControllerByPage(page: PageReturn): BrowserControllerReturn | undefined;
|
||||
getPage(id: string): PageReturn | undefined;
|
||||
getPageId(page: PageReturn): string | undefined;
|
||||
injectPageState(page: PageReturn, state: PageState): Promise<void>;
|
||||
// (undocumented)
|
||||
maxOpenPagesPerBrowser: number;
|
||||
newPage(options?: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
|
||||
newPageInNewBrowser(options?: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]>): Promise<PageReturn>;
|
||||
newPageWithEachPlugin(optionsList?: Omit<BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]>, 'browserPlugin'>[]): Promise<PageReturn[]>;
|
||||
// (undocumented)
|
||||
operationTimeoutMillis: number;
|
||||
// (undocumented)
|
||||
pageCounter: number;
|
||||
// (undocumented)
|
||||
pageIds: WeakMap<PageReturn, string>;
|
||||
// (undocumented)
|
||||
pages: Map<string, PageReturn>;
|
||||
// (undocumented)
|
||||
pageToBrowserController: WeakMap<PageReturn, BrowserControllerReturn>;
|
||||
// (undocumented)
|
||||
postLaunchHooks: PostLaunchHook<BrowserControllerReturn>[];
|
||||
// (undocumented)
|
||||
postPageCloseHooks: PostPageCloseHook<BrowserControllerReturn>[];
|
||||
// (undocumented)
|
||||
postPageCreateHooks: PostPageCreateHook<BrowserControllerReturn, PageReturn>[];
|
||||
// (undocumented)
|
||||
preLaunchHooks: PreLaunchHook<LaunchContextReturn>[];
|
||||
// (undocumented)
|
||||
prePageCloseHooks: PrePageCloseHook<BrowserControllerReturn, PageReturn>[];
|
||||
// (undocumented)
|
||||
prePageCreateHooks: PrePageCreateHook<BrowserControllerReturn, PageOptions>[];
|
||||
retireAllBrowsers(): void;
|
||||
// (undocumented)
|
||||
retireBrowserAfterPageCount: number;
|
||||
retireBrowserByPage(page: PageReturn): void;
|
||||
retireBrowserController(browserController: BrowserControllerReturn): void;
|
||||
// (undocumented)
|
||||
retiredBrowserControllers: Set<BrowserControllerReturn>;
|
||||
// (undocumented)
|
||||
startingBrowserControllers: Set<BrowserControllerReturn>;
|
||||
// (undocumented)
|
||||
useFingerprints?: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPoolEvents<BC extends BrowserController, Page> {
|
||||
// (undocumented)
|
||||
[BROWSER_POOL_EVENTS.BROWSER_LAUNCHED]: (browserController: BC) => void | Promise<void>;
|
||||
// (undocumented)
|
||||
[BROWSER_POOL_EVENTS.BROWSER_RETIRED]: (browserController: BC) => void | Promise<void>;
|
||||
// (undocumented)
|
||||
[BROWSER_POOL_EVENTS.PAGE_CLOSED]: (page: Page) => void | Promise<void>;
|
||||
// (undocumented)
|
||||
[BROWSER_POOL_EVENTS.PAGE_CREATED]: (page: Page) => void | Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPoolHooks<BC extends BrowserController, LC extends LaunchContext, PR extends UnwrapPromise<ReturnType<BC['newPage']>> = UnwrapPromise<ReturnType<BC['newPage']>>> {
|
||||
postLaunchHooks?: PostLaunchHook<BC>[];
|
||||
postPageCloseHooks?: PostPageCloseHook<BC>[];
|
||||
postPageCreateHooks?: PostPageCreateHook<BC, PR>[];
|
||||
preLaunchHooks?: PreLaunchHook<LC>[];
|
||||
prePageCloseHooks?: PrePageCloseHook<BC, PR>[];
|
||||
prePageCreateHooks?: PrePageCreateHook<BC>[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPoolNewPageInNewBrowserOptions<PageOptions, BP extends BrowserPlugin> {
|
||||
browserPlugin?: BP;
|
||||
id?: string;
|
||||
launchOptions?: BP['launchOptions'];
|
||||
pageOptions?: PageOptions;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPoolNewPageOptions<PageOptions, BP extends BrowserPlugin> extends NewPageOptions {
|
||||
browserPlugin?: BP;
|
||||
ignoreTlsErrors?: boolean;
|
||||
pageOptions?: PageOptions;
|
||||
proxyUrl?: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserPoolOptions<Plugin extends BrowserPlugin = BrowserPlugin> {
|
||||
browserPlugins: readonly Plugin[];
|
||||
closeInactiveBrowserAfterSecs?: number;
|
||||
// (undocumented)
|
||||
fingerprintOptions?: FingerprintOptions;
|
||||
maxOpenPagesPerBrowser?: number;
|
||||
operationTimeoutSecs?: number;
|
||||
retireBrowserAfterPageCount?: number;
|
||||
retireInactiveBrowserAfterSecs?: number;
|
||||
useFingerprints?: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserSpecification {
|
||||
httpVersion?: HttpVersion;
|
||||
maxVersion?: number;
|
||||
minVersion?: number;
|
||||
name: BrowserName;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface CommonLibrary {
|
||||
// (undocumented)
|
||||
launch(opts?: Dictionary): Promise<CommonBrowser>;
|
||||
// (undocumented)
|
||||
name?: () => string;
|
||||
// (undocumented)
|
||||
product?: string;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
export interface CommonPage {
|
||||
// (undocumented)
|
||||
close(...args: unknown[]): Promise<unknown>;
|
||||
// (undocumented)
|
||||
url(): string | Promise<string>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CreateLaunchContextOptions<Library extends CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {
|
||||
}
|
||||
|
||||
// @public
|
||||
export const DEFAULT_USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36";
|
||||
|
||||
// @public (undocumented)
|
||||
export const enum DeviceCategory {
|
||||
desktop = "desktop",
|
||||
mobile = "mobile"
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface FingerprintGenerator {
|
||||
// (undocumented)
|
||||
getFingerprint: (fingerprintGeneratorOptions?: FingerprintGeneratorOptions) => GetFingerprintReturn;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface FingerprintGeneratorOptions extends Partial<FingerprintGeneratorOptions_2> {
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface FingerprintOptions {
|
||||
fingerprintCacheSize?: number;
|
||||
fingerprintGeneratorOptions?: FingerprintGeneratorOptions;
|
||||
useFingerprintCache?: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface GetFingerprintReturn {
|
||||
// (undocumented)
|
||||
fingerprint: BrowserFingerprintWithHeaders;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface IBrowserController<Page = unknown> {
|
||||
readonly browser: unknown;
|
||||
close(): Promise<void>;
|
||||
getCookies(page: Page): Promise<Cookie[]>;
|
||||
readonly id: string;
|
||||
readonly launchContext: IBrowserLaunchContext;
|
||||
setCookies(page: Page, cookies: Cookie[]): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface IBrowserLaunchContext {
|
||||
fingerprint?: unknown;
|
||||
launchOptions?: Dictionary | undefined;
|
||||
proxyUrl?: string;
|
||||
useIncognitoPages?: boolean;
|
||||
}
|
||||
|
||||
export { IBrowserPool }
|
||||
|
||||
// @public (undocumented)
|
||||
export type InferBrowserPluginArray<Input extends readonly unknown[], Result extends BrowserPlugin[] = []> = Input extends readonly [infer FirstValue, ...infer Rest] | [infer FirstValue, ...infer Rest] ? FirstValue extends PlaywrightPlugin ? InferBrowserPluginArray<Rest, [...Result, PlaywrightPlugin]> : FirstValue extends PuppeteerPlugin ? InferBrowserPluginArray<Rest, [...Result, PuppeteerPlugin]> : never : Input extends [] ? Result : Input extends readonly (infer U)[] ? [
|
||||
U
|
||||
] extends [PuppeteerPlugin | PlaywrightPlugin] ? U[] : never : Result;
|
||||
|
||||
// @public (undocumented)
|
||||
export class LaunchContext<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
||||
constructor(options: LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>);
|
||||
// (undocumented)
|
||||
[K: PropertyKey]: unknown;
|
||||
// (undocumented)
|
||||
browserPerProxy?: boolean;
|
||||
// (undocumented)
|
||||
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
extend<T extends Record<PropertyKey, unknown>>(fields: T): void;
|
||||
// (undocumented)
|
||||
fingerprint?: BrowserFingerprintWithHeaders;
|
||||
// (undocumented)
|
||||
id?: string;
|
||||
// (undocumented)
|
||||
ignoreProxyCertificate?: boolean;
|
||||
// (undocumented)
|
||||
launchOptions: LibraryOptions;
|
||||
set proxyUrl(url: string | undefined);
|
||||
get proxyUrl(): string | undefined;
|
||||
// (undocumented)
|
||||
useIncognitoPages: boolean;
|
||||
// (undocumented)
|
||||
userDataDir: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface LaunchContextOptions<Library extends CommonLibrary = CommonLibrary, LibraryOptions extends Dictionary | undefined = Parameters<Library['launch']>[0], LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>, NewPageOptions = Parameters<LaunchResult['newPage']>[0], NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>> {
|
||||
browserPerProxy?: boolean;
|
||||
browserPlugin: BrowserPlugin<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>;
|
||||
id?: string;
|
||||
ignoreProxyCertificate?: boolean;
|
||||
launchOptions: LibraryOptions;
|
||||
// (undocumented)
|
||||
proxyUrl?: string;
|
||||
useIncognitoPages?: boolean;
|
||||
userDataDir?: string;
|
||||
}
|
||||
|
||||
export { NewPageOptions }
|
||||
|
||||
// @public (undocumented)
|
||||
export const enum OperatingSystemsName {
|
||||
android = "android",
|
||||
ios = "ios",
|
||||
// (undocumented)
|
||||
linux = "linux",
|
||||
// (undocumented)
|
||||
macos = "macos",
|
||||
// (undocumented)
|
||||
windows = "windows"
|
||||
}
|
||||
|
||||
// @public
|
||||
export class PlaywrightBrowser extends EventEmitter {
|
||||
// (undocumented)
|
||||
[Symbol.asyncDispose](): Promise<void>;
|
||||
constructor(options: BrowserOptions);
|
||||
// (undocumented)
|
||||
browserType(): BrowserType;
|
||||
// (undocumented)
|
||||
close(): Promise<void>;
|
||||
// (undocumented)
|
||||
contexts(): BrowserContext[];
|
||||
// (undocumented)
|
||||
isConnected(): boolean;
|
||||
// (undocumented)
|
||||
newBrowserCDPSession(): Promise<never>;
|
||||
// (undocumented)
|
||||
newContext(): Promise<never>;
|
||||
// (undocumented)
|
||||
newPage(...args: Parameters<BrowserContext['newPage']>): ReturnType<BrowserContext['newPage']>;
|
||||
// @internal (undocumented)
|
||||
_setBrowserType(browserType: BrowserType): void;
|
||||
// (undocumented)
|
||||
startTracing(): Promise<never>;
|
||||
// (undocumented)
|
||||
stopTracing(): Promise<never>;
|
||||
// (undocumented)
|
||||
version(): string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
|
||||
// (undocumented)
|
||||
protected _close(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected _getCookies(page: Page): Promise<Cookie[]>;
|
||||
// (undocumented)
|
||||
protected _kill(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page>;
|
||||
// (undocumented)
|
||||
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
||||
// (undocumented)
|
||||
protected _setCookies(page: Page, cookies: Cookie[]): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
|
||||
// (undocumented)
|
||||
protected _addProxyToLaunchOptions(launchContext: LaunchContext<BrowserType>): Promise<void>;
|
||||
// (undocumented)
|
||||
_containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;
|
||||
// (undocumented)
|
||||
createController(): PlaywrightController;
|
||||
// (undocumented)
|
||||
protected _isChromiumBasedBrowser(): boolean;
|
||||
// (undocumented)
|
||||
protected _launch(launchContext: LaunchContext<BrowserType>): Promise<Browser>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type PostLaunchHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
|
||||
|
||||
// @public
|
||||
export type PostPageCloseHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
|
||||
|
||||
// @public
|
||||
export type PostPageCreateHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (page: Page, browserController: BC) => void | Promise<void>;
|
||||
|
||||
// @public
|
||||
export type PreLaunchHook<LC extends LaunchContext> = (pageId: string, launchContext: LC) => void | Promise<void>;
|
||||
|
||||
// @public
|
||||
export type PrePageCloseHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (page: Page, browserController: BC) => void | Promise<void>;
|
||||
|
||||
// @public
|
||||
export type PrePageCreateHook<BC extends BrowserController, PO = Parameters<BC['newPage']>[0]> = (pageId: string, browserController: BC, pageOptions?: PO) => void | Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export class PuppeteerController extends BrowserController<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
||||
// (undocumented)
|
||||
protected _close(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected _getCookies(page: PuppeteerTypes.Page): Promise<Cookie[]>;
|
||||
// (undocumented)
|
||||
protected _kill(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page>;
|
||||
// (undocumented)
|
||||
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
|
||||
// (undocumented)
|
||||
protected _setCookies(page: PuppeteerTypes.Page, cookies: Cookie[]): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export class PuppeteerPlugin extends BrowserPlugin<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
|
||||
// (undocumented)
|
||||
protected _addProxyToLaunchOptions(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<void>;
|
||||
// (undocumented)
|
||||
createController(): PuppeteerController;
|
||||
// (undocumented)
|
||||
protected _isChromiumBasedBrowser(_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): boolean;
|
||||
// (undocumented)
|
||||
protected _launch(launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.LaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>): Promise<PuppeteerTypes.Browser>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type UnwrapPromise<T> = T extends PromiseLike<infer R> ? UnwrapPromise<R> : T;
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,14 @@
|
||||
## API Report File for "@crawlee/browser"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
|
||||
export * from "./internals/browser-crawler.js";
|
||||
export * from "./internals/browser-launcher.js";
|
||||
export * from "@crawlee/basic";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,81 @@
|
||||
## API Report File for "@crawlee/cheerio"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { BasicCrawlingContext } from '@crawlee/http';
|
||||
import type { BatchAddRequestsResult } from '@crawlee/types';
|
||||
import * as cheerio from 'cheerio';
|
||||
import type { CheerioAPI } from 'cheerio';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import { ContextPipeline } from '@crawlee/http';
|
||||
import { CrawlingContext } from '@crawlee/http';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type { EnqueueLinksOptions } from '@crawlee/http';
|
||||
import type { ErrorHandler } from '@crawlee/http';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/http';
|
||||
import { HttpCrawler } from '@crawlee/http';
|
||||
import type { HttpCrawlerOptions } from '@crawlee/http';
|
||||
import type { InternalHttpCrawlingContext } from '@crawlee/http';
|
||||
import type { InternalHttpHook } from '@crawlee/http';
|
||||
import { IRequestManager } from '@crawlee/http';
|
||||
import type { RequestHandler } from '@crawlee/http';
|
||||
import { RobotsTxtFile } from '@crawlee/utils';
|
||||
import { RouterHandler } from '@crawlee/http';
|
||||
import type { RouterRoutes } from '@crawlee/http';
|
||||
import type { SkippedRequestCallback } from '@crawlee/http';
|
||||
|
||||
// @public
|
||||
export class CheerioCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension> extends HttpCrawler<CheerioCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: CheerioCrawlerOptions<ContextExtension, ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
|
||||
readonly body: string;
|
||||
readonly $: CheerioAPI;
|
||||
} & {
|
||||
enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<BatchAddRequestsResult>;
|
||||
waitForSelector: (selector: string, _timeoutMs?: number) => Promise<void>;
|
||||
parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
|
||||
}>;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
export function cheerioCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CheerioCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends CheerioCrawlingContext = CheerioCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CheerioCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
||||
$: cheerio.CheerioAPI;
|
||||
body: string;
|
||||
enqueueLinks(options?: EnqueueLinksOptions): Promise<BatchAddRequestsResult>;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type CheerioErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type CheerioHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type CheerioRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public
|
||||
export function createCheerioRouter<Context extends CheerioCrawlingContext = CheerioCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
|
||||
export * from "@crawlee/http";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,56 @@
|
||||
## API Report File for "@crawlee/fs-storage"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { CrawleeLogger } from '@crawlee/types';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type * as storage from '@crawlee/types';
|
||||
|
||||
// @public (undocumented)
|
||||
export class FileSystemStorageClient implements storage.StorageClient {
|
||||
constructor(options?: FileSystemStorageOptions);
|
||||
// (undocumented)
|
||||
createDatasetClient(options?: storage.CreateDatasetClientOptions): Promise<storage.DatasetClient>;
|
||||
// (undocumented)
|
||||
createKeyValueStoreClient(options?: storage.CreateKeyValueStoreClientOptions): Promise<storage.KeyValueStoreClient>;
|
||||
// (undocumented)
|
||||
createRequestQueueClient(options?: storage.CreateRequestQueueClientOptions): Promise<storage.RequestQueueClient>;
|
||||
// (undocumented)
|
||||
readonly datasetClientCache: DatasetClient[];
|
||||
// (undocumented)
|
||||
readonly datasetsDirectory: string;
|
||||
getStorageClientCacheKey(): string;
|
||||
// (undocumented)
|
||||
readonly keyValueStoreCache: KeyValueStoreClient[];
|
||||
// (undocumented)
|
||||
readonly keyValueStoresDirectory: string;
|
||||
// (undocumented)
|
||||
readonly localDataDirectory: string;
|
||||
// (undocumented)
|
||||
readonly logger?: CrawleeLogger;
|
||||
purge(): Promise<void>;
|
||||
// (undocumented)
|
||||
readonly requestQueueCache: RequestQueueClient[];
|
||||
// (undocumented)
|
||||
readonly requestQueuesDirectory: string;
|
||||
// (undocumented)
|
||||
setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
|
||||
// (undocumented)
|
||||
storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
|
||||
teardown(): Promise<void>;
|
||||
// (undocumented)
|
||||
readonly writeMetadata: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface FileSystemStorageOptions {
|
||||
localDataDirectory?: string;
|
||||
logger?: CrawleeLogger;
|
||||
writeMetadata?: boolean;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,18 @@
|
||||
## API Report File for "@crawlee/got-scraping-client"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { BaseHttpClient } from '@crawlee/http-client';
|
||||
import { CustomFetchOptions } from '@crawlee/http-client';
|
||||
|
||||
// @public
|
||||
export class GotScrapingHttpClient extends BaseHttpClient {
|
||||
// (undocumented)
|
||||
fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,54 @@
|
||||
## API Report File for "@crawlee/http-client"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { BaseHttpClient as BaseHttpClient_2 } from '@crawlee/types';
|
||||
import { CookieJar } from 'tough-cookie';
|
||||
import type { CrawleeLogger } from '@crawlee/types';
|
||||
import type { SendRequestOptions } from '@crawlee/types';
|
||||
import type { SessionFingerprint } from '@crawlee/types';
|
||||
|
||||
// @public
|
||||
export abstract class BaseHttpClient implements BaseHttpClient_2 {
|
||||
constructor(options?: {
|
||||
logger?: CrawleeLogger;
|
||||
});
|
||||
protected abstract fetch(input: Request, init?: RequestInit & CustomFetchOptions): Promise<Response>;
|
||||
// (undocumented)
|
||||
protected log?: CrawleeLogger;
|
||||
sendRequest(initialRequest: Request, options?: SendRequestOptions): Promise<Response>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface CustomFetchOptions {
|
||||
cookieJar?: CookieJar;
|
||||
fingerprint?: SessionFingerprint;
|
||||
proxyUrl?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class FetchHttpClient extends BaseHttpClient {
|
||||
// (undocumented)
|
||||
fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface IResponseWithUrl extends Response {
|
||||
// (undocumented)
|
||||
url: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class ResponseWithUrl extends Response implements IResponseWithUrl {
|
||||
constructor(body: BodyInit | null, init: ResponseInit & {
|
||||
url?: string;
|
||||
});
|
||||
// (undocumented)
|
||||
url: string;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,250 @@
|
||||
## API Report File for "@crawlee/http"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { AllowedHttpMethods } from '@crawlee/types';
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import type { Awaitable } from '@crawlee/types';
|
||||
import { BasePredicate } from 'ow';
|
||||
import { BasicCrawler } from '@crawlee/basic';
|
||||
import type { BasicCrawlerOptions } from '@crawlee/basic';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import { ContextPipeline } from '@crawlee/basic';
|
||||
import type { CrawlingContext } from '@crawlee/basic';
|
||||
import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import { ErrorHandler } from '@crawlee/basic';
|
||||
import { GetUserDataFromRequest } from '@crawlee/basic';
|
||||
import type { ISession } from '@crawlee/types';
|
||||
import type { JsonValue } from 'type-fest';
|
||||
import { LoadedRequest } from '@crawlee/core';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import { Predicate } from 'ow';
|
||||
import { Request as Request_2 } from '@crawlee/basic';
|
||||
import type { Request as Request_3 } from '@crawlee/core';
|
||||
import { RequestHandler } from '@crawlee/basic';
|
||||
import type { RequestLike } from 'content-type';
|
||||
import type { RequireContextPipeline } from '@crawlee/basic';
|
||||
import type { ResponseLike } from 'content-type';
|
||||
import { ResponseWithUrl } from '@crawlee/http-client';
|
||||
import { RouterHandler } from '@crawlee/basic';
|
||||
import { RouterRoutes } from '@crawlee/basic';
|
||||
import { StringPredicate } from 'ow';
|
||||
import { Transform } from 'node:stream';
|
||||
|
||||
// @public
|
||||
export function ByteCounterStream(input: {
|
||||
logTransferredBytes: (transferredBytes: number) => void;
|
||||
loggingInterval?: number;
|
||||
}): Transform;
|
||||
|
||||
// @public
|
||||
export function createFileRouter<Context extends FileDownloadCrawlingContext = FileDownloadCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
export function createHttpRouter<Context extends HttpCrawlingContext = HttpCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
export class FileDownload extends BasicCrawler<FileDownloadCrawlingContext> {
|
||||
constructor(options?: BasicCrawlerOptions<FileDownloadCrawlingContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext_2<Dictionary>, CrawlingContext_2<Dictionary> & {
|
||||
request: LoadedRequest<Request_3>;
|
||||
response: ResponseWithUrl;
|
||||
contentType: {
|
||||
type: string;
|
||||
encoding: BufferEncoding;
|
||||
};
|
||||
[kBodyDrained]: Promise<void>;
|
||||
}>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface FileDownloadCrawlingContext<UserData extends Dictionary = any> extends CrawlingContext_2<UserData> {
|
||||
// (undocumented)
|
||||
contentType: {
|
||||
type: string;
|
||||
encoding: BufferEncoding;
|
||||
};
|
||||
// (undocumented)
|
||||
request: LoadedRequest<Request_3<UserData>>;
|
||||
// (undocumented)
|
||||
response: Response;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type FileDownloadErrorHandler<UserData extends Dictionary = any> = ErrorHandler<FileDownloadCrawlingContext<UserData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type FileDownloadHook<UserData extends Dictionary = any> = InternalHttpHook<FileDownloadCrawlingContext<UserData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type FileDownloadRequestHandler<UserData extends Dictionary = any> = RequestHandler<FileDownloadCrawlingContext<UserData>>;
|
||||
|
||||
// @public
|
||||
export class HttpCrawler<Context extends InternalHttpCrawlingContext<any, any> = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawler<Context, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: HttpCrawlerOptions<Context, ContextExtension, ExtendedContext> & RequireContextPipeline<InternalHttpCrawlingContext, Context>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext, InternalHttpCrawlingContext>;
|
||||
// (undocumented)
|
||||
protected _encodeResponse(request: Request_2, response: Response, encoding: BufferEncoding): {
|
||||
encoding: BufferEncoding;
|
||||
response: Response;
|
||||
};
|
||||
protected _extendSupportedMimeTypes(additionalMimeTypes: (string | RequestLike | ResponseLike)[]): void;
|
||||
// (undocumented)
|
||||
protected forceResponseEncoding?: string;
|
||||
protected _getRequestOptions(request: Request_2, session: ISession, proxyUrl?: string): {
|
||||
url: string;
|
||||
method: AllowedHttpMethods;
|
||||
proxyUrl: string | undefined;
|
||||
timeout: number;
|
||||
sessionToken: ISession;
|
||||
headers: Record<string, string> | undefined;
|
||||
https: {
|
||||
rejectUnauthorized: boolean;
|
||||
};
|
||||
body: string | undefined;
|
||||
};
|
||||
protected _handleRequestTimeout(session: ISession): void;
|
||||
// (undocumented)
|
||||
protected ignoreSslErrors: boolean;
|
||||
// (undocumented)
|
||||
protected isRequestBlocked(crawlingContext: InternalHttpCrawlingContext): Promise<string | false>;
|
||||
// (undocumented)
|
||||
protected navigationTimeoutMillis: number;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
ignoreSslErrors: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
additionalMimeTypes: ArrayPredicate<string>;
|
||||
suggestResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
|
||||
forceResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
|
||||
saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
protected _parseResponse(request: Request_2, response: Response): Promise<{
|
||||
response: Response;
|
||||
contentType: {
|
||||
type: string;
|
||||
encoding: BufferEncoding;
|
||||
};
|
||||
body: string;
|
||||
} | {
|
||||
body: Buffer<ArrayBuffer>;
|
||||
response: Response;
|
||||
contentType: {
|
||||
type: string;
|
||||
encoding: BufferEncoding;
|
||||
};
|
||||
}>;
|
||||
// (undocumented)
|
||||
protected postNavigationHooks: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
|
||||
// (undocumented)
|
||||
protected preNavigationHooks: InternalHttpHook<CrawlingContext>[];
|
||||
protected _requestFunction(input: RequestFunctionOptions): Promise<Response>;
|
||||
// (undocumented)
|
||||
protected saveResponseCookies: boolean;
|
||||
// (undocumented)
|
||||
protected suggestResponseEncoding?: string;
|
||||
// (undocumented)
|
||||
protected readonly supportedMimeTypes: Set<string>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface HttpCrawlerOptions<Context extends InternalHttpCrawlingContext = InternalHttpCrawlingContext, ContextExtension = Dictionary<never>, ExtendedContext extends Context = Context & ContextExtension> extends BasicCrawlerOptions<Context, ContextExtension, ExtendedContext> {
|
||||
additionalMimeTypes?: string[];
|
||||
forceResponseEncoding?: string;
|
||||
ignoreSslErrors?: boolean;
|
||||
navigationTimeoutSecs?: number;
|
||||
postNavigationHooks?: ((crawlingContext: CrawlingContextWithResponse) => Awaitable<void | Partial<CrawlingContextWithResponse>>)[];
|
||||
preNavigationHooks?: InternalHttpHook<CrawlingContext>[];
|
||||
saveResponseCookies?: boolean;
|
||||
suggestResponseEncoding?: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface HttpCrawlingContext<UserData extends Dictionary = any, JSONData extends JsonValue = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type HttpErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends JsonValue = any> = ErrorHandler<HttpCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type HttpHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends JsonValue = any> = InternalHttpHook<HttpCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type HttpRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends JsonValue = any> = RequestHandler<HttpCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @internal (undocumented)
|
||||
export interface InternalHttpCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends JsonValue = any> extends CrawlingContextWithResponse<UserData> {
|
||||
body: string | Buffer;
|
||||
contentType: {
|
||||
type: string;
|
||||
encoding: BufferEncoding;
|
||||
};
|
||||
json: JSONData;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
export type InternalHttpHook<Context> = (crawlingContext: Context) => Awaitable<void | Partial<Context>>;
|
||||
|
||||
// @public
|
||||
export function MinimumSpeedStream(input: {
|
||||
minSpeedKbps: number;
|
||||
historyLengthMs?: number;
|
||||
checkProgressInterval?: number;
|
||||
}): Transform;
|
||||
|
||||
|
||||
export * from "@crawlee/basic";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,29 @@
|
||||
## API Report File for "@crawlee/impit-client"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { BaseHttpClient } from '@crawlee/http-client';
|
||||
import type { CrawleeLogger } from '@crawlee/types';
|
||||
import type { CustomFetchOptions } from '@crawlee/http-client';
|
||||
import { ImpitOptions } from 'impit';
|
||||
|
||||
// @public (undocumented)
|
||||
export const Browser: {
|
||||
readonly Chrome: "chrome";
|
||||
readonly Firefox: "firefox";
|
||||
};
|
||||
|
||||
// @public
|
||||
export class ImpitHttpClient extends BaseHttpClient {
|
||||
constructor(options?: Omit<ImpitOptions, 'proxyUrl' | 'timeout'> & {
|
||||
logger?: CrawleeLogger;
|
||||
});
|
||||
// (undocumented)
|
||||
fetch(request: Request, options?: RequestInit & CustomFetchOptions): Promise<Response>;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,149 @@
|
||||
## API Report File for "@crawlee/jsdom"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import { BasePredicate } from 'ow';
|
||||
import type { BasicCrawlingContext } from '@crawlee/http';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import { ContextPipeline } from '@crawlee/http';
|
||||
import { CrawlingContext } from '@crawlee/http';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type { DOMWindow } from 'jsdom';
|
||||
import type { EnqueueLinksOptions } from '@crawlee/http';
|
||||
import type { ErrorHandler } from '@crawlee/http';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/http';
|
||||
import { HttpCrawler } from '@crawlee/http';
|
||||
import type { HttpCrawlerOptions } from '@crawlee/http';
|
||||
import type { InternalHttpCrawlingContext } from '@crawlee/http';
|
||||
import type { InternalHttpHook } from '@crawlee/http';
|
||||
import type { IRequestManager } from '@crawlee/http';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import { Predicate } from 'ow';
|
||||
import type { RequestHandler } from '@crawlee/http';
|
||||
import { RobotsTxtFile } from '@crawlee/utils';
|
||||
import { RouterHandler } from '@crawlee/http';
|
||||
import type { RouterRoutes } from '@crawlee/http';
|
||||
import type { SkippedRequestCallback } from '@crawlee/http';
|
||||
import { StringPredicate } from 'ow';
|
||||
import { VirtualConsole } from 'jsdom';
|
||||
|
||||
// @public
|
||||
export function createJSDOMRouter<Context extends JSDOMCrawlingContext = JSDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @internal (undocumented)
|
||||
export function domCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export class JSDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension> extends HttpCrawler<JSDOMCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: JSDOMCrawlerOptions<ContextExtension, ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
|
||||
readonly window: DOMWindow;
|
||||
readonly body: string;
|
||||
readonly document: Document;
|
||||
} & {
|
||||
enqueueLinks: (enqueueOptions?: EnqueueLinksOptions) => Promise<unknown>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
|
||||
}>;
|
||||
getVirtualConsole(): VirtualConsole;
|
||||
// (undocumented)
|
||||
protected hideInternalConsole: boolean;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
runScripts: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
hideInternalConsole: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
ignoreSslErrors: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
additionalMimeTypes: ArrayPredicate<string>;
|
||||
suggestResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
|
||||
forceResponseEncoding: StringPredicate & BasePredicate<string | undefined>;
|
||||
saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
// (undocumented)
|
||||
protected runScripts: boolean;
|
||||
// (undocumented)
|
||||
protected virtualConsole: VirtualConsole | null;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface JSDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends JSDOMCrawlingContext = JSDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends HttpCrawlerOptions<JSDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
|
||||
hideInternalConsole?: boolean;
|
||||
runScripts?: boolean;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface JSDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
||||
// (undocumented)
|
||||
body: string;
|
||||
// (undocumented)
|
||||
document: Document;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
// (undocumented)
|
||||
window: DOMWindow;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type JSDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = ErrorHandler<JSDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type JSDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = InternalHttpHook<JSDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type JSDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = RequestHandler<JSDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
|
||||
export * from "@crawlee/http";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,85 @@
|
||||
## API Report File for "@crawlee/linkedom"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { BasicCrawlingContext } from '@crawlee/http';
|
||||
import * as cheerio from 'cheerio';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import { ContextPipeline } from '@crawlee/http';
|
||||
import { CrawlingContext } from '@crawlee/http';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type { EnqueueLinksOptions } from '@crawlee/http';
|
||||
import type { ErrorHandler } from '@crawlee/http';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/http';
|
||||
import { HttpCrawler } from '@crawlee/http';
|
||||
import type { HttpCrawlerOptions } from '@crawlee/http';
|
||||
import type { InternalHttpCrawlingContext } from '@crawlee/http';
|
||||
import type { InternalHttpHook } from '@crawlee/http';
|
||||
import { IRequestManager } from '@crawlee/http';
|
||||
import type { RequestHandler } from '@crawlee/http';
|
||||
import { RobotsTxtFile } from '@crawlee/utils';
|
||||
import { RouterHandler } from '@crawlee/http';
|
||||
import type { RouterRoutes } from '@crawlee/http';
|
||||
import type { SkippedRequestCallback } from '@crawlee/http';
|
||||
|
||||
// @public
|
||||
export function createLinkeDOMRouter<Context extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
export class LinkeDOMCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension> extends HttpCrawler<LinkeDOMCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options: LinkeDOMCrawlerOptions<ContextExtension, ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, InternalHttpCrawlingContext<any, any> & {
|
||||
readonly window: Window;
|
||||
readonly body: string;
|
||||
readonly document: Document;
|
||||
} & {
|
||||
enqueueLinks: (enqueueOptions?: LinkeDOMCrawlerEnqueueLinksOptions) => Promise<unknown>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
parseWithCheerio(selector?: string, _timeoutMs?: number): Promise<cheerio.CheerioAPI>;
|
||||
}>;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
export function linkedomCrawlerEnqueueLinks(options: EnqueueLinksInternalOptions | BoundEnqueueLinksInternalOptions): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface LinkeDOMCrawlerEnqueueLinksOptions extends Omit<EnqueueLinksOptions, 'urls' | 'requestManager'> {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface LinkeDOMCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends LinkeDOMCrawlingContext = LinkeDOMCrawlingContext & ContextExtension, UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends HttpCrawlerOptions<LinkeDOMCrawlingContext<UserData, JSONData>, ContextExtension, ExtendedContext> {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface LinkeDOMCrawlingContext<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> extends InternalHttpCrawlingContext<UserData, JSONData> {
|
||||
// (undocumented)
|
||||
document: Document;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
// (undocumented)
|
||||
window: Window;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type LinkeDOMErrorHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = ErrorHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type LinkeDOMHook<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = InternalHttpHook<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type LinkeDOMRequestHandler<UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
|
||||
JSONData extends Dictionary = any> = RequestHandler<LinkeDOMCrawlingContext<UserData, JSONData>>;
|
||||
|
||||
|
||||
export * from "@crawlee/http";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,44 @@
|
||||
## API Report File for "@crawlee/memory-storage"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { CrawleeLogger } from '@crawlee/types';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type * as storage from '@crawlee/types';
|
||||
|
||||
// @public (undocumented)
|
||||
export class MemoryStorageClient implements storage.StorageClient {
|
||||
constructor(options?: MemoryStorageOptions);
|
||||
// (undocumented)
|
||||
createDatasetClient(options?: storage.CreateDatasetClientOptions): Promise<storage.DatasetClient>;
|
||||
// (undocumented)
|
||||
createKeyValueStoreClient(options?: storage.CreateKeyValueStoreClientOptions): Promise<storage.KeyValueStoreClient>;
|
||||
// (undocumented)
|
||||
createRequestQueueClient(options?: storage.CreateRequestQueueClientOptions): Promise<storage.RequestQueueClient>;
|
||||
// (undocumented)
|
||||
readonly datasetClientCache: DatasetClient[];
|
||||
getStorageClientCacheKey(): string;
|
||||
// (undocumented)
|
||||
readonly keyValueStoreCache: KeyValueStoreClient[];
|
||||
// (undocumented)
|
||||
readonly logger?: CrawleeLogger;
|
||||
purge(): Promise<void>;
|
||||
// (undocumented)
|
||||
readonly requestQueueCache: RequestQueueClient[];
|
||||
// (undocumented)
|
||||
setStatusMessage(message: string, options?: storage.SetStatusMessageOptions): Promise<void>;
|
||||
// (undocumented)
|
||||
storageExists(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
|
||||
teardown(): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface MemoryStorageOptions {
|
||||
logger?: CrawleeLogger;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,466 @@
|
||||
## API Report File for "@crawlee/playwright"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { AnyNode } from 'domhandler';
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import { BasePredicate } from 'ow';
|
||||
import { BasicCrawler } from '@crawlee/basic';
|
||||
import type { BasicCrawlerOptions } from '@crawlee/browser';
|
||||
import { BatchAddRequestsResult } from '@crawlee/types';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import type { Browser } from 'playwright';
|
||||
import { BrowserCrawler } from '@crawlee/browser';
|
||||
import type { BrowserCrawlerOptions } from '@crawlee/browser';
|
||||
import type { BrowserCrawlingContext } from '@crawlee/browser';
|
||||
import type { BrowserHook } from '@crawlee/browser';
|
||||
import type { BrowserLaunchContext } from '@crawlee/browser';
|
||||
import { BrowserLauncher } from '@crawlee/browser';
|
||||
import type { BrowserType } from 'playwright';
|
||||
import { Cheerio } from 'cheerio';
|
||||
import { CheerioAPI } from '@crawlee/browser';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import { Configuration } from '@crawlee/browser';
|
||||
import { ContextPipeline } from '@crawlee/browser';
|
||||
import type { ContextPipeline as ContextPipeline_2 } from '@crawlee/core';
|
||||
import { CrawlingContext } from '@crawlee/browser';
|
||||
import type { CrawlingContext as CrawlingContext_2 } from '@crawlee/core';
|
||||
import { Dictionary } from '@crawlee/utils';
|
||||
import type { Dictionary as Dictionary_2 } from '@crawlee/types';
|
||||
import type { EnqueueLinksOptions } from '@crawlee/core';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/browser';
|
||||
import type { GetUserDataFromRequest as GetUserDataFromRequest_2 } from '@crawlee/core';
|
||||
import type { GlobInput } from '@crawlee/browser';
|
||||
import { IRequestManager } from '@crawlee/browser';
|
||||
import type { LaunchOptions } from 'playwright';
|
||||
import type { LoadedRequest } from '@crawlee/browser';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import type { Page } from 'playwright';
|
||||
import { PlaywrightPlugin } from '@crawlee/browser-pool';
|
||||
import { Predicate } from 'ow';
|
||||
import type { PseudoUrlInput } from '@crawlee/browser';
|
||||
import type { RecoverableStatePersistenceOptions } from '@crawlee/core';
|
||||
import type { RegExpInput } from '@crawlee/browser';
|
||||
import type { Request as Request_2 } from '@crawlee/core';
|
||||
import { Request as Request_3 } from '@crawlee/browser';
|
||||
import type { RequestHandler } from '@crawlee/browser';
|
||||
import { RequestHandlerResult } from '@crawlee/core';
|
||||
import type { RequestTransform } from '@crawlee/browser';
|
||||
import type { Response as Response_2 } from 'playwright';
|
||||
import type { RestrictedCrawlingContext } from '@crawlee/core';
|
||||
import { RouterHandler } from '@crawlee/browser';
|
||||
import { RouterHandler as RouterHandler_2 } from '@crawlee/basic';
|
||||
import type { RouterRoutes } from '@crawlee/browser';
|
||||
import type { RouterRoutes as RouterRoutes_2 } from '@crawlee/core';
|
||||
import type { SetRequired } from 'type-fest';
|
||||
import type { SkippedRequestCallback } from '@crawlee/browser';
|
||||
import { Statistics } from '@crawlee/core';
|
||||
import type { StatisticsOptions } from '@crawlee/core';
|
||||
import type { StatisticState } from '@crawlee/core';
|
||||
import { StringPredicate } from 'ow';
|
||||
|
||||
// @public
|
||||
export class AdaptivePlaywrightCrawler<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends BasicCrawler<AdaptivePlaywrightCrawlerContext, ExtendedContext> {
|
||||
constructor(options?: AdaptivePlaywrightCrawlerOptions<ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected allowStorageAccess<R, TArgs extends any[]>(func: (...args: TArgs) => Promise<R>): (...args: TArgs) => Promise<R>;
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline_2<CrawlingContext_2<Dictionary_2>, CrawlingContext_2<Dictionary_2> & {
|
||||
readonly request: LoadedRequest<Request_3<Dictionary_2>>;
|
||||
readonly response: Response;
|
||||
readonly page: Page;
|
||||
readonly querySelector: AdaptivePlaywrightCrawlerContext["querySelector"];
|
||||
readonly waitForSelector: AdaptivePlaywrightCrawlerContext["waitForSelector"];
|
||||
readonly parseWithCheerio: AdaptivePlaywrightCrawlerContext["parseWithCheerio"];
|
||||
}>;
|
||||
// (undocumented)
|
||||
protected commitResult(crawlingContext: CrawlingContext_2, input: RequestHandlerResult): Promise<void>;
|
||||
// (undocumented)
|
||||
protected enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'urls'>, request: RestrictedCrawlingContext['request'], result: RequestHandlerResult): Promise<BatchAddRequestsResult>;
|
||||
protected getPendingRequestCountApproximation(): Promise<number>;
|
||||
// (undocumented)
|
||||
protected _init(): Promise<void>;
|
||||
// (undocumented)
|
||||
protected runRequestHandler(crawlingContext: CrawlingContext_2): Promise<void>;
|
||||
// (undocumented)
|
||||
readonly stats: AdaptivePlaywrightCrawlerStatistics;
|
||||
// (undocumented)
|
||||
teardown(): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface AdaptivePlaywrightCrawlerContext<UserData extends Dictionary_2 = Dictionary_2> extends CrawlingContext_2<UserData> {
|
||||
// (undocumented)
|
||||
enqueueLinks(options?: EnqueueLinksOptions): Promise<unknown>;
|
||||
page: Page;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
querySelector(selector: string, timeoutMs?: number): Promise<Cheerio<AnyNode>>;
|
||||
// (undocumented)
|
||||
request: LoadedRequest<Request_3<UserData>>;
|
||||
response: Response;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface AdaptivePlaywrightCrawlerOptions<ExtendedContext extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext> extends Omit<BasicCrawlerOptions<AdaptivePlaywrightCrawlerContext, ExtendedContext>, 'preNavigationHooks' | 'postNavigationHooks'> {
|
||||
postNavigationHooks?: AdaptivePostNavigationHook[];
|
||||
preNavigationHooks?: AdaptiveHook[];
|
||||
preventDirectStorageAccess?: boolean;
|
||||
renderingTypeDetectionRatio?: number;
|
||||
renderingTypePredictor?: Pick<RenderingTypePredictor, 'predict' | 'storeResult' | 'initialize'>;
|
||||
resultChecker?: (result: RequestHandlerResult) => boolean;
|
||||
resultComparator?: (resultA: RequestHandlerResult, resultB: RequestHandlerResult) => boolean | 'equal' | 'different' | 'inconclusive';
|
||||
}
|
||||
|
||||
// @public
|
||||
function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface BlockRequestsOptions {
|
||||
extraUrlPatterns?: string[];
|
||||
urlPatterns?: string[];
|
||||
}
|
||||
|
||||
// @public
|
||||
function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
|
||||
|
||||
// @public
|
||||
function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary_2[]>;
|
||||
|
||||
// @public (undocumented)
|
||||
function closeCookieModals(page: Page): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface CompiledScriptParams {
|
||||
// (undocumented)
|
||||
page: Page;
|
||||
// (undocumented)
|
||||
request: Request_3;
|
||||
}
|
||||
|
||||
// @public
|
||||
function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
|
||||
|
||||
// @public (undocumented)
|
||||
export function createAdaptivePlaywrightRouter<Context extends AdaptivePlaywrightCrawlerContext = AdaptivePlaywrightCrawlerContext, UserData extends Dictionary_2 = GetUserDataFromRequest_2<Context['request']>>(routes?: RouterRoutes_2<Context, UserData>): RouterHandler_2<Context>;
|
||||
|
||||
// @public
|
||||
export function createPlaywrightRouter<Context extends PlaywrightCrawlingContext = PlaywrightCrawlingContext, UserData extends Dictionary_2 = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface EnqueueLinksByClickingElementsOptions {
|
||||
clickOptions?: ClickOptions;
|
||||
exclude?: readonly (GlobInput | RegExpInput)[];
|
||||
forefront?: boolean;
|
||||
globs?: GlobInput[];
|
||||
label?: string;
|
||||
maxWaitForPageIdleSecs?: number;
|
||||
onSkippedRequest?: SkippedRequestCallback;
|
||||
page: Page;
|
||||
// @deprecated
|
||||
pseudoUrls?: PseudoUrlInput[];
|
||||
regexps?: RegExpInput[];
|
||||
requestManager: IRequestManager;
|
||||
selector: string;
|
||||
skipNavigation?: boolean;
|
||||
transformRequestFunction?: RequestTransform;
|
||||
userData?: Dictionary_2;
|
||||
waitForPageIdleSecs?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
function gotoExtended(page: Page, request: Request_3, gotoOptions?: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;
|
||||
|
||||
// @public
|
||||
export function handleCloudflareChallengeHook(options?: HandleCloudflareChallengeOptions): PlaywrightHook;
|
||||
|
||||
// @public (undocumented)
|
||||
interface HandleCloudflareChallengeOptions {
|
||||
clickCallback?: (page: Page, boundingBox: {
|
||||
x: number;
|
||||
y: number;
|
||||
}) => Promise<void>;
|
||||
clickPositionCallback?: (page: Page) => Promise<{
|
||||
x: number;
|
||||
y: number;
|
||||
} | null>;
|
||||
isBlockedCallback?: (page: Page) => Promise<boolean>;
|
||||
isChallengeCallback?: (page: Page) => Promise<boolean>;
|
||||
preChallengeSleepSecs?: number;
|
||||
sleepSecs?: number;
|
||||
verbose?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface InfiniteScrollOptions {
|
||||
buttonSelector?: string;
|
||||
maxScrollHeight?: number;
|
||||
scrollDownAndUp?: boolean;
|
||||
stopScrollCallback?: () => unknown | Promise<unknown>;
|
||||
timeoutSecs?: number;
|
||||
waitForSecs?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface InjectFileOptions {
|
||||
surviveNavigations?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
function injectJQuery(page: Page, options?: {
|
||||
surviveNavigations?: boolean;
|
||||
}): Promise<unknown>;
|
||||
|
||||
// @public
|
||||
export function launchPlaywright(launchContext?: PlaywrightLaunchContext, config?: Configuration): Promise<Browser>;
|
||||
|
||||
// @public
|
||||
function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
|
||||
|
||||
declare namespace playwrightClickElements {
|
||||
export {
|
||||
enqueueLinksByClickingElements,
|
||||
clickElementsAndInterceptNavigationRequests,
|
||||
clickElements,
|
||||
EnqueueLinksByClickingElementsOptions
|
||||
}
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
interface PlaywrightContextUtils {
|
||||
blockRequests(options?: BlockRequestsOptions): Promise<void>;
|
||||
closeCookieModals(): Promise<void>;
|
||||
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
||||
enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
|
||||
handleCloudflareChallenge(options?: HandleCloudflareChallengeOptions): Promise<Response_2 | undefined>;
|
||||
infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
|
||||
injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
|
||||
injectJQuery(): Promise<unknown>;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class PlaywrightCrawler<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawler<Page, Response_2, {
|
||||
browserPlugins: [PlaywrightPlugin];
|
||||
}, LaunchOptions, PlaywrightCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: PlaywrightCrawlerOptions<ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary_2>, BrowserCrawlingContext<Page, Response_2, Dictionary_2, Dictionary_2> & {
|
||||
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
|
||||
injectJQuery: () => Promise<void>;
|
||||
blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
|
||||
waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
|
||||
parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
|
||||
infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
|
||||
saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
|
||||
enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;
|
||||
compileScript: (scriptString: string, ctx?: Dictionary_2) => CompiledScriptFunction;
|
||||
closeCookieModals: () => Promise<void>;
|
||||
handleCloudflareChallenge: (options?: HandleCloudflareChallengeOptions) => Promise<Response_2 | undefined>;
|
||||
}>;
|
||||
// (undocumented)
|
||||
protected _navigationHandler(crawlingContext: PlaywrightCrawlingContext, gotoOptions: PlaywrightDirectNavigationOptions): Promise<Response_2 | null>;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
ignoreIframes: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
ignoreShadowRoots: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
headless: AnyPredicate<string | boolean>;
|
||||
browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PlaywrightCrawlerOptions<ContextExtension = Dictionary_2<never>, ExtendedContext extends PlaywrightCrawlingContext = PlaywrightCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, Response_2, PlaywrightCrawlingContext, ContextExtension, ExtendedContext, {
|
||||
browserPlugins: [PlaywrightPlugin];
|
||||
}> {
|
||||
launchContext?: PlaywrightLaunchContext;
|
||||
postNavigationHooks?: PlaywrightHook[];
|
||||
preNavigationHooks?: PlaywrightHook[];
|
||||
requestHandler?: RequestHandler<ExtendedContext>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PlaywrightCrawlingContext<UserData extends Dictionary_2 = Dictionary_2> extends BrowserCrawlingContext<Page, Response_2, UserData, PlaywrightGotoOptions>, PlaywrightContextUtils {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PlaywrightDirectNavigationOptions {
|
||||
referer?: string;
|
||||
timeout?: number;
|
||||
waitUntil?: 'domcontentloaded' | 'load' | 'networkidle';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type PlaywrightGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PlaywrightHook extends BrowserHook<PlaywrightCrawlingContext> {
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface PlaywrightLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
|
||||
launcher?: BrowserType;
|
||||
launchOptions?: LaunchOptions & Parameters<BrowserType['launchPersistentContext']>[1];
|
||||
proxyUrl?: string;
|
||||
useChrome?: boolean;
|
||||
useIncognitoPages?: boolean;
|
||||
userDataDir?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class PlaywrightLauncher extends BrowserLauncher<PlaywrightPlugin> {
|
||||
constructor(launchContext?: PlaywrightLaunchContext, config?: Configuration);
|
||||
// (undocumented)
|
||||
readonly config: Configuration;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
launchContextOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
proxyUrl: StringPredicate & BasePredicate<string | undefined>;
|
||||
useChrome: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
useIncognitoPages: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
browserPerProxy: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
ignoreProxyCertificate: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
userDataDir: StringPredicate & BasePredicate<string | undefined>;
|
||||
launchOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
userAgent: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
}
|
||||
|
||||
declare namespace playwrightUtils {
|
||||
export {
|
||||
injectFile,
|
||||
injectJQuery,
|
||||
gotoExtended,
|
||||
blockRequests,
|
||||
compileScript,
|
||||
infiniteScroll,
|
||||
saveSnapshot,
|
||||
parseWithCheerio,
|
||||
closeCookieModals,
|
||||
InjectFileOptions,
|
||||
BlockRequestsOptions,
|
||||
PlaywrightDirectNavigationOptions as DirectNavigationOptions,
|
||||
CompiledScriptParams,
|
||||
CompiledScriptFunction,
|
||||
InfiniteScrollOptions,
|
||||
SaveSnapshotOptions,
|
||||
HandleCloudflareChallengeOptions,
|
||||
PlaywrightContextUtils,
|
||||
enqueueLinksByClickingElements,
|
||||
playwrightUtils_2 as playwrightUtils
|
||||
}
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
const playwrightUtils_2: {
|
||||
injectFile: typeof injectFile;
|
||||
injectJQuery: typeof injectJQuery;
|
||||
gotoExtended: typeof gotoExtended;
|
||||
blockRequests: typeof blockRequests;
|
||||
enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
|
||||
parseWithCheerio: typeof parseWithCheerio;
|
||||
infiniteScroll: typeof infiniteScroll;
|
||||
saveSnapshot: typeof saveSnapshot;
|
||||
compileScript: typeof compileScript;
|
||||
closeCookieModals: typeof closeCookieModals;
|
||||
RenderingTypePredictor: typeof RenderingTypePredictor;
|
||||
handleCloudflareChallenge: typeof handleCloudflareChallenge;
|
||||
};
|
||||
|
||||
// @public (undocumented)
|
||||
export type RenderingType = 'clientOnly' | 'static';
|
||||
|
||||
// @public
|
||||
export class RenderingTypePredictor {
|
||||
constructor(input: RenderingTypePredictorOptions);
|
||||
// (undocumented)
|
||||
protected calculateFeatureVector(url: URLComponents, label: string | undefined): FeatureVector;
|
||||
initialize(): Promise<void>;
|
||||
predict(input: Request_2): {
|
||||
renderingType: RenderingType;
|
||||
detectionProbabilityRecommendation: number;
|
||||
};
|
||||
// (undocumented)
|
||||
protected retrain(): void;
|
||||
storeResult(requests: Request_2 | Request_2[], renderingType: RenderingType): void;
|
||||
}
|
||||
|
||||
// @public
|
||||
function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
interface SaveSnapshotOptions {
|
||||
config?: Configuration;
|
||||
key?: string;
|
||||
keyValueStoreName?: string | null;
|
||||
saveHtml?: boolean;
|
||||
saveScreenshot?: boolean;
|
||||
screenshotQuality?: number;
|
||||
}
|
||||
|
||||
|
||||
export * from "@crawlee/browser";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,387 @@
|
||||
## API Report File for "@crawlee/puppeteer"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import { BasePredicate } from 'ow';
|
||||
import { BatchAddRequestsResult } from '@crawlee/types';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import type { Browser } from 'puppeteer';
|
||||
import { BrowserCrawler } from '@crawlee/browser';
|
||||
import type { BrowserCrawlerOptions } from '@crawlee/browser';
|
||||
import type { BrowserCrawlingContext } from '@crawlee/browser';
|
||||
import type { BrowserHook } from '@crawlee/browser';
|
||||
import type { BrowserLaunchContext } from '@crawlee/browser';
|
||||
import { BrowserLauncher } from '@crawlee/browser';
|
||||
import { CheerioAPI } from '@crawlee/browser';
|
||||
import { CheerioRoot } from '@crawlee/utils';
|
||||
import type { ClickOptions } from 'puppeteer';
|
||||
import { Configuration } from '@crawlee/browser';
|
||||
import { ContextPipeline } from '@crawlee/browser';
|
||||
import { CrawlingContext } from '@crawlee/browser';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/browser';
|
||||
import type { GlobInput } from '@crawlee/browser';
|
||||
import type { HTTPRequest } from 'puppeteer';
|
||||
import type { HTTPResponse } from 'puppeteer';
|
||||
import { IRequestManager } from '@crawlee/browser';
|
||||
import type { LaunchOptions } from 'puppeteer';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import type { Page } from 'puppeteer';
|
||||
import { Predicate } from 'ow';
|
||||
import type { ProtocolMapping } from 'devtools-protocol/types/protocol-mapping.js';
|
||||
import type { PseudoUrlInput } from '@crawlee/browser';
|
||||
import { PuppeteerPlugin } from '@crawlee/browser-pool';
|
||||
import type { RegExpInput } from '@crawlee/browser';
|
||||
import { Request as Request_2 } from '@crawlee/browser';
|
||||
import type { RequestTransform } from '@crawlee/browser';
|
||||
import type { ResponseForRequest } from 'puppeteer';
|
||||
import { RouterHandler } from '@crawlee/browser';
|
||||
import type { RouterRoutes } from '@crawlee/browser';
|
||||
import type { SkippedRequestCallback } from '@crawlee/browser';
|
||||
import { StringPredicate } from 'ow';
|
||||
import type { Target } from 'puppeteer';
|
||||
|
||||
// @public
|
||||
function addInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
|
||||
|
||||
// @public
|
||||
function blockRequests(page: Page, options?: BlockRequestsOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BlockRequestsOptions {
|
||||
extraUrlPatterns?: string[];
|
||||
urlPatterns?: string[];
|
||||
}
|
||||
|
||||
// @public @deprecated
|
||||
const blockResources: (page: Page, resourceTypes?: string[]) => Promise<void>;
|
||||
|
||||
// @public @deprecated
|
||||
function cacheResponses(page: Page, cache: Dictionary<Partial<ResponseForRequest>>, responseUrlRules: (string | RegExp)[]): Promise<void>;
|
||||
|
||||
// @public
|
||||
function clickElements(page: Page, selector: string, clickOptions?: ClickOptions): Promise<void>;
|
||||
|
||||
// @public
|
||||
function clickElementsAndInterceptNavigationRequests(options: ClickElementsAndInterceptNavigationRequestsOptions): Promise<Dictionary[]>;
|
||||
|
||||
// @public (undocumented)
|
||||
function closeCookieModals(page: Page): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type CompiledScriptFunction = (params: CompiledScriptParams) => Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface CompiledScriptParams {
|
||||
// (undocumented)
|
||||
page: Page;
|
||||
// (undocumented)
|
||||
request: Request_2;
|
||||
}
|
||||
|
||||
// @public
|
||||
function compileScript(scriptString: string, context?: Dictionary): CompiledScriptFunction;
|
||||
|
||||
// @public
|
||||
export function createPuppeteerRouter<Context extends PuppeteerCrawlingContext = PuppeteerCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
function enqueueLinksByClickingElements(options: EnqueueLinksByClickingElementsOptions): Promise<BatchAddRequestsResult>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface EnqueueLinksByClickingElementsOptions {
|
||||
clickOptions?: ClickOptions;
|
||||
exclude?: readonly (GlobInput | RegExpInput)[];
|
||||
forefront?: boolean;
|
||||
globs?: GlobInput[];
|
||||
label?: string;
|
||||
maxWaitForPageIdleSecs?: number;
|
||||
onSkippedRequest?: SkippedRequestCallback;
|
||||
page: Page;
|
||||
// @deprecated
|
||||
pseudoUrls?: PseudoUrlInput[];
|
||||
regexps?: RegExpInput[];
|
||||
requestManager: IRequestManager;
|
||||
selector: string;
|
||||
skipNavigation?: boolean;
|
||||
transformRequestFunction?: RequestTransform;
|
||||
userData?: Dictionary;
|
||||
waitForPageIdleSecs?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
function gotoExtended(page: Page, request: Request_2, gotoOptions?: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;
|
||||
|
||||
// @public
|
||||
function infiniteScroll(page: Page, options?: InfiniteScrollOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface InfiniteScrollOptions {
|
||||
buttonSelector?: string;
|
||||
maxScrollHeight?: number;
|
||||
scrollDownAndUp?: boolean;
|
||||
stopScrollCallback?: () => unknown | Promise<unknown>;
|
||||
timeoutSecs?: number;
|
||||
waitForSecs?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
function injectFile(page: Page, filePath: string, options?: InjectFileOptions): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface InjectFileOptions {
|
||||
surviveNavigations?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
function injectJQuery(page: Page, options?: {
|
||||
surviveNavigations?: boolean;
|
||||
}): Promise<unknown>;
|
||||
|
||||
// @public (undocumented)
|
||||
export type InterceptHandler = (request: HTTPRequest) => unknown;
|
||||
|
||||
// @public
|
||||
function isTargetRelevant(page: Page, target: Target): boolean;
|
||||
|
||||
// @public
|
||||
export function launchPuppeteer(launchContext?: PuppeteerLaunchContext, config?: Configuration): Promise<Browser>;
|
||||
|
||||
// @public
|
||||
function parseWithCheerio(page: Page, ignoreShadowRoots?: boolean, ignoreIframes?: boolean): Promise<CheerioRoot>;
|
||||
|
||||
declare namespace puppeteerClickElements {
|
||||
export {
|
||||
enqueueLinksByClickingElements,
|
||||
clickElementsAndInterceptNavigationRequests,
|
||||
isTargetRelevant,
|
||||
clickElements,
|
||||
EnqueueLinksByClickingElementsOptions
|
||||
}
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
interface PuppeteerContextUtils {
|
||||
addInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
|
||||
blockRequests(options?: BlockRequestsOptions): Promise<void>;
|
||||
closeCookieModals(): Promise<void>;
|
||||
compileScript(scriptString: string, ctx?: Dictionary): CompiledScriptFunction;
|
||||
enqueueLinksByClickingElements(options: Omit<EnqueueLinksByClickingElementsOptions, 'page' | 'requestManager'>): Promise<BatchAddRequestsResult>;
|
||||
infiniteScroll(options?: InfiniteScrollOptions): Promise<void>;
|
||||
injectFile(filePath: string, options?: InjectFileOptions): Promise<unknown>;
|
||||
injectJQuery(): Promise<unknown>;
|
||||
parseWithCheerio(selector?: string, timeoutMs?: number): Promise<CheerioRoot>;
|
||||
removeInterceptRequestHandler(handler: InterceptHandler): Promise<void>;
|
||||
saveSnapshot(options?: SaveSnapshotOptions): Promise<void>;
|
||||
waitForSelector(selector: string, timeoutMs?: number): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class PuppeteerCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawler<Page, HTTPResponse, {
|
||||
browserPlugins: [PuppeteerPlugin];
|
||||
}, LaunchOptions, PuppeteerCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: PuppeteerCrawlerOptions<ContextExtension, ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext<Dictionary>, BrowserCrawlingContext<Page, HTTPResponse, Dictionary, Dictionary> & {
|
||||
injectFile: (filePath: string, options?: InjectFileOptions) => Promise<unknown>;
|
||||
injectJQuery: () => Promise<void>;
|
||||
waitForSelector: (selector: string, timeoutMs?: number) => Promise<void>;
|
||||
parseWithCheerio: (selector?: string, timeoutMs?: number) => Promise<CheerioAPI>;
|
||||
enqueueLinksByClickingElements: (options: Omit<EnqueueLinksByClickingElementsOptions, "page" | "requestManager">) => Promise<BatchAddRequestsResult>;
|
||||
blockRequests: (options?: BlockRequestsOptions) => Promise<void>;
|
||||
compileScript: (scriptString: string, ctx?: Dictionary) => CompiledScriptFunction;
|
||||
addInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
|
||||
removeInterceptRequestHandler: (handler: InterceptHandler) => Promise<void>;
|
||||
infiniteScroll: (options?: InfiniteScrollOptions) => Promise<void>;
|
||||
saveSnapshot: (options?: SaveSnapshotOptions) => Promise<void>;
|
||||
closeCookieModals: () => Promise<void>;
|
||||
}>;
|
||||
// (undocumented)
|
||||
protected _navigationHandler(crawlingContext: PuppeteerCrawlingContext, gotoOptions: PuppeteerDirectNavigationOptions): Promise<HTTPResponse | null>;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
headless: AnyPredicate<string | boolean>;
|
||||
browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PuppeteerCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends PuppeteerCrawlingContext = PuppeteerCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<Page, HTTPResponse, PuppeteerCrawlingContext, ContextExtension, ExtendedContext, {
|
||||
browserPlugins: [PuppeteerPlugin];
|
||||
}> {
|
||||
launchContext?: PuppeteerLaunchContext;
|
||||
postNavigationHooks?: PuppeteerHook[];
|
||||
preNavigationHooks?: PuppeteerHook[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PuppeteerCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<Page, HTTPResponse, UserData, PuppeteerGoToOptions>, PuppeteerContextUtils {
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PuppeteerDirectNavigationOptions {
|
||||
referer?: string;
|
||||
timeout?: number;
|
||||
waitUntil?: 'domcontentloaded' | 'load' | 'networkidle' | 'networkidle0' | 'networkidle2';
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type PuppeteerGoToOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PuppeteerHook extends BrowserHook<PuppeteerCrawlingContext> {
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface PuppeteerLaunchContext extends BrowserLaunchContext<PuppeteerPlugin['launchOptions'], unknown> {
|
||||
launcher?: unknown;
|
||||
launchOptions?: PuppeteerPlugin['launchOptions'];
|
||||
proxyUrl?: string;
|
||||
useChrome?: boolean;
|
||||
useIncognitoPages?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export class PuppeteerLauncher extends BrowserLauncher<PuppeteerPlugin, unknown> {
|
||||
constructor(launchContext?: PuppeteerLaunchContext, config?: Configuration);
|
||||
// (undocumented)
|
||||
readonly config: Configuration;
|
||||
// (undocumented)
|
||||
protected _getDefaultHeadlessOption(): boolean;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
launcher: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
proxyUrl: StringPredicate & BasePredicate<string | undefined>;
|
||||
useChrome: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
useIncognitoPages: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
browserPerProxy: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
ignoreProxyCertificate: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
userDataDir: StringPredicate & BasePredicate<string | undefined>;
|
||||
launchOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
userAgent: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
}
|
||||
|
||||
declare namespace puppeteerRequestInterception {
|
||||
export {
|
||||
addInterceptRequestHandler,
|
||||
removeInterceptRequestHandler,
|
||||
InterceptHandler
|
||||
}
|
||||
}
|
||||
|
||||
declare namespace puppeteerUtils {
|
||||
export {
|
||||
injectFile,
|
||||
injectJQuery,
|
||||
parseWithCheerio,
|
||||
blockRequests,
|
||||
sendCDPCommand,
|
||||
cacheResponses,
|
||||
compileScript,
|
||||
gotoExtended,
|
||||
infiniteScroll,
|
||||
saveSnapshot,
|
||||
closeCookieModals,
|
||||
PuppeteerDirectNavigationOptions as DirectNavigationOptions,
|
||||
InjectFileOptions,
|
||||
BlockRequestsOptions,
|
||||
CompiledScriptParams,
|
||||
CompiledScriptFunction,
|
||||
blockResources,
|
||||
InfiniteScrollOptions,
|
||||
SaveSnapshotOptions,
|
||||
PuppeteerContextUtils,
|
||||
enqueueLinksByClickingElements,
|
||||
addInterceptRequestHandler,
|
||||
removeInterceptRequestHandler,
|
||||
puppeteerUtils_2 as puppeteerUtils
|
||||
}
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
const puppeteerUtils_2: {
|
||||
injectFile: typeof injectFile;
|
||||
injectJQuery: typeof injectJQuery;
|
||||
enqueueLinksByClickingElements: typeof enqueueLinksByClickingElements;
|
||||
blockRequests: typeof blockRequests;
|
||||
compileScript: typeof compileScript;
|
||||
gotoExtended: typeof gotoExtended;
|
||||
addInterceptRequestHandler: typeof addInterceptRequestHandler;
|
||||
removeInterceptRequestHandler: typeof removeInterceptRequestHandler;
|
||||
infiniteScroll: typeof infiniteScroll;
|
||||
saveSnapshot: typeof saveSnapshot;
|
||||
parseWithCheerio: typeof parseWithCheerio;
|
||||
closeCookieModals: typeof closeCookieModals;
|
||||
};
|
||||
|
||||
// @public
|
||||
function removeInterceptRequestHandler(page: Page, handler: InterceptHandler): Promise<void>;
|
||||
|
||||
// @public
|
||||
function saveSnapshot(page: Page, options?: SaveSnapshotOptions): Promise<void>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface SaveSnapshotOptions {
|
||||
config?: Configuration;
|
||||
key?: string;
|
||||
keyValueStoreName?: string | null;
|
||||
saveHtml?: boolean;
|
||||
saveScreenshot?: boolean;
|
||||
screenshotQuality?: number;
|
||||
}
|
||||
|
||||
// @internal (undocumented)
|
||||
function sendCDPCommand<T extends keyof ProtocolMapping.Commands>(page: Page, command: T, ...args: ProtocolMapping.Commands[T]['paramsType']): Promise<ProtocolMapping.Commands[T]['returnType']>;
|
||||
|
||||
|
||||
export * from "@crawlee/browser";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,210 @@
|
||||
## API Report File for "@crawlee/stagehand"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { Action } from '@browserbasehq/stagehand';
|
||||
import { ActOptions } from '@browserbasehq/stagehand';
|
||||
import { ActResult } from '@browserbasehq/stagehand';
|
||||
import { AgentConfig } from '@browserbasehq/stagehand';
|
||||
import { AgentResult } from '@browserbasehq/stagehand';
|
||||
import { AnyPredicate } from 'ow';
|
||||
import { ArrayPredicate } from 'ow';
|
||||
import { BasePredicate } from 'ow';
|
||||
import { BooleanPredicate } from 'ow';
|
||||
import type { Browser } from 'playwright';
|
||||
import type { BrowserController } from '@crawlee/browser-pool';
|
||||
import { BrowserCrawler } from '@crawlee/browser';
|
||||
import type { BrowserCrawlerOptions } from '@crawlee/browser';
|
||||
import type { BrowserCrawlingContext } from '@crawlee/browser';
|
||||
import type { BrowserHook } from '@crawlee/browser';
|
||||
import type { BrowserLaunchContext } from '@crawlee/browser';
|
||||
import { BrowserPlugin } from '@crawlee/browser-pool';
|
||||
import type { BrowserPluginOptions } from '@crawlee/browser-pool';
|
||||
import type { BrowserType } from 'playwright';
|
||||
import type { ContextPipeline } from '@crawlee/browser';
|
||||
import type { CrawlingContext } from '@crawlee/browser';
|
||||
import type { Dictionary } from '@crawlee/types';
|
||||
import { ExtractOptions } from '@browserbasehq/stagehand';
|
||||
import type { GetUserDataFromRequest } from '@crawlee/browser';
|
||||
import type { LaunchContext } from '@crawlee/browser-pool';
|
||||
import type { LaunchOptions } from 'playwright';
|
||||
import type { LLMClient } from '@browserbasehq/stagehand';
|
||||
import type { LoadedContext } from '@crawlee/browser';
|
||||
import { ModelConfiguration } from '@browserbasehq/stagehand';
|
||||
import type { NonStreamingAgentInstance } from '@browserbasehq/stagehand';
|
||||
import { NumberPredicate } from 'ow';
|
||||
import { ObjectPredicate } from 'ow';
|
||||
import { ObserveOptions } from '@browserbasehq/stagehand';
|
||||
import type { Page } from 'playwright';
|
||||
import { Predicate } from 'ow';
|
||||
import type { RequestHandler } from '@crawlee/browser';
|
||||
import type { Response as Response_2 } from 'playwright';
|
||||
import { RouterHandler } from '@crawlee/browser';
|
||||
import type { RouterRoutes } from '@crawlee/browser';
|
||||
import { Stagehand } from '@browserbasehq/stagehand';
|
||||
import type { StreamingAgentInstance } from '@browserbasehq/stagehand';
|
||||
import { StringPredicate } from 'ow';
|
||||
import type { z } from 'zod';
|
||||
|
||||
export { Action }
|
||||
|
||||
export { ActOptions }
|
||||
|
||||
export { ActResult }
|
||||
|
||||
export { AgentConfig }
|
||||
|
||||
export { AgentResult }
|
||||
|
||||
// @public
|
||||
export function createStagehandRouter<Context extends StagehandCrawlingContext = StagehandCrawlingContext, UserData extends Dictionary = GetUserDataFromRequest<Context['request']>>(routes?: RouterRoutes<Context, UserData>): RouterHandler<Context>;
|
||||
|
||||
// @public
|
||||
function enhancePageWithStagehand(page: Page, stagehand: Stagehand): StagehandPage;
|
||||
|
||||
export { ExtractOptions }
|
||||
|
||||
export { ModelConfiguration }
|
||||
|
||||
export { ObserveOptions }
|
||||
|
||||
export { Stagehand }
|
||||
|
||||
// @public
|
||||
export class StagehandCrawler<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawler<StagehandPage, Response_2, {
|
||||
browserPlugins: [StagehandPlugin];
|
||||
}, LaunchOptions, StagehandCrawlingContext, ContextExtension, ExtendedContext> {
|
||||
constructor(options?: StagehandCrawlerOptions<ContextExtension, ExtendedContext>);
|
||||
// (undocumented)
|
||||
protected buildContextPipeline(): ContextPipeline<CrawlingContext, StagehandCrawlingContext>;
|
||||
protected _navigationHandler(crawlingContext: StagehandCrawlingContext, gotoOptions: StagehandGotoOptions): Promise<Response_2 | null>;
|
||||
// (undocumented)
|
||||
protected static optionsShape: {
|
||||
stagehandOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
browserPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
navigationTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
preNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
postNavigationHooks: ArrayPredicate<unknown> & BasePredicate<unknown[] | undefined>;
|
||||
launchContext: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
headless: AnyPredicate<string | boolean>;
|
||||
browserPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
saveResponseCookies: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
proxyConfiguration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
contextPipelineBuilder: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
extendContext: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestList: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestQueue: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
requestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
requestHandlerTimeoutSecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
errorHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
failedRequestHandler: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
maxRequestRetries: NumberPredicate & BasePredicate<number | undefined>;
|
||||
sameDomainDelaySecs: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerCrawl: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxCrawlDepth: NumberPredicate & BasePredicate<number | undefined>;
|
||||
autoscaledPoolOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
sessionPool: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
statusMessageLoggingInterval: NumberPredicate & BasePredicate<number | undefined>;
|
||||
statusMessageCallback: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
additionalHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
ignoreHttpErrorStatusCodes: ArrayPredicate<number>;
|
||||
blockedStatusCodes: ArrayPredicate<number>;
|
||||
retryOnBlocked: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
respectRobotsTxtFile: AnyPredicate<boolean | object>;
|
||||
onSkippedRequest: Predicate<Function> & BasePredicate<Function | undefined>;
|
||||
httpClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
configuration: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
storageClient: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
eventManager: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
logger: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
minConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxConcurrency: NumberPredicate & BasePredicate<number | undefined>;
|
||||
maxRequestsPerMinute: NumberPredicate & BasePredicate<number | undefined>;
|
||||
keepAlive: BooleanPredicate & BasePredicate<boolean | undefined>;
|
||||
statisticsOptions: ObjectPredicate<object> & BasePredicate<object | undefined>;
|
||||
id: StringPredicate & BasePredicate<string | undefined>;
|
||||
};
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StagehandCrawlerOptions<ContextExtension = Dictionary<never>, ExtendedContext extends StagehandCrawlingContext = StagehandCrawlingContext & ContextExtension> extends BrowserCrawlerOptions<StagehandPage, Response_2, StagehandCrawlingContext, ContextExtension, ExtendedContext, {
|
||||
browserPlugins: [StagehandPlugin];
|
||||
}> {
|
||||
launchContext?: StagehandLaunchContext;
|
||||
postNavigationHooks?: StagehandHook[];
|
||||
preNavigationHooks?: StagehandHook[];
|
||||
requestHandler?: StagehandRequestHandler;
|
||||
stagehandOptions?: StagehandOptions;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface StagehandCrawlingContext<UserData extends Dictionary = Dictionary> extends BrowserCrawlingContext<StagehandPage, Response_2, UserData, StagehandGotoOptions> {
|
||||
page: StagehandPage;
|
||||
stagehand: Stagehand;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type StagehandGotoOptions = NonNullable<Parameters<Page['goto']>[1]>;
|
||||
|
||||
// @public
|
||||
export interface StagehandHook extends BrowserHook<StagehandCrawlingContext> {
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StagehandLaunchContext extends BrowserLaunchContext<LaunchOptions, BrowserType> {
|
||||
launcher?: BrowserType;
|
||||
launchOptions?: LaunchOptions & Parameters<BrowserType['launchPersistentContext']>[1];
|
||||
proxyUrl?: string;
|
||||
stagehandOptions?: StagehandOptions;
|
||||
useChrome?: boolean;
|
||||
useIncognitoPages?: boolean;
|
||||
userDataDir?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StagehandOptions {
|
||||
apiKey?: string;
|
||||
cacheDir?: string;
|
||||
domSettleTimeout?: number;
|
||||
env?: 'LOCAL' | 'BROWSERBASE';
|
||||
llmClient?: LLMClient;
|
||||
logInferenceToFile?: boolean;
|
||||
model?: ModelConfiguration;
|
||||
projectId?: string;
|
||||
selfHeal?: boolean;
|
||||
systemPrompt?: string;
|
||||
verbose?: 0 | 1 | 2;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StagehandPage extends Page {
|
||||
act(instruction: string, options?: Omit<ActOptions, 'page'>): Promise<ActResult>;
|
||||
agent(config: AgentConfig & {
|
||||
stream: true;
|
||||
}): StreamingAgentInstance;
|
||||
// (undocumented)
|
||||
agent(config?: AgentConfig & {
|
||||
stream?: false;
|
||||
}): NonStreamingAgentInstance;
|
||||
extract<T>(instruction: string, schema: z.ZodType<T>, options?: Omit<ExtractOptions, 'page'>): Promise<T>;
|
||||
observe(options?: Omit<ObserveOptions, 'page'>): Promise<Action[]>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StagehandRequestHandler extends RequestHandler<LoadedContext<StagehandCrawlingContext>> {
|
||||
}
|
||||
|
||||
declare namespace stagehandUtils {
|
||||
export {
|
||||
enhancePageWithStagehand
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
export * from "@crawlee/browser";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,587 @@
|
||||
## API Report File for "@crawlee/types"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { CookieJar } from 'tough-cookie';
|
||||
import type { Readable } from 'node:stream';
|
||||
import type { SerializedCookieJar } from 'tough-cookie';
|
||||
|
||||
// @public (undocumented)
|
||||
export type AllowedHttpMethods = 'GET' | 'HEAD' | 'POST' | 'PUT' | 'DELETE' | 'TRACE' | 'OPTIONS' | 'CONNECT' | 'PATCH' | 'get' | 'head' | 'post' | 'put' | 'delete' | 'trace' | 'options' | 'connect' | 'patch';
|
||||
|
||||
// @public (undocumented)
|
||||
export type Awaitable<T> = T | PromiseLike<T>;
|
||||
|
||||
// @public
|
||||
export interface BaseHttpClient {
|
||||
sendRequest(request: Request, options?: SendRequestOptions): Promise<Response>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BatchAddRequestsResult {
|
||||
// (undocumented)
|
||||
processedRequests: ProcessedRequest[];
|
||||
// (undocumented)
|
||||
unprocessedRequests: UnprocessedRequest[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface BrowserLikeResponse {
|
||||
// (undocumented)
|
||||
headers(): Dictionary<string | string[]>;
|
||||
// (undocumented)
|
||||
url(): string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type Constructor<T = unknown> = new (...args: any[]) => T;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface Cookie {
|
||||
domain?: string;
|
||||
expires?: number;
|
||||
httpOnly?: boolean;
|
||||
name: string;
|
||||
path?: string;
|
||||
priority?: 'Low' | 'Medium' | 'High';
|
||||
sameParty?: boolean;
|
||||
sameSite?: 'Strict' | 'Lax' | 'None';
|
||||
secure?: boolean;
|
||||
sourcePort?: number;
|
||||
sourceScheme?: 'Unset' | 'NonSecure' | 'Secure';
|
||||
url?: string;
|
||||
value: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface CrawleeLogger {
|
||||
child(options: Partial<CrawleeLoggerOptions>): CrawleeLogger;
|
||||
debug(message: string, data?: Record<string, unknown>): void;
|
||||
deprecated(message: string): void;
|
||||
error(message: string, data?: Record<string, unknown>): void;
|
||||
exception(exception: Error, message: string, data?: Record<string, unknown>): void;
|
||||
getOptions(): CrawleeLoggerOptions;
|
||||
info(message: string, data?: Record<string, unknown>): void;
|
||||
logWithLevel(level: number, message: string, data?: Record<string, unknown>): void;
|
||||
perf(message: string, data?: Record<string, unknown>): void;
|
||||
setOptions(options: Partial<CrawleeLoggerOptions>): void;
|
||||
softFail(message: string, data?: Record<string, unknown>): void;
|
||||
warning(message: string, data?: Record<string, unknown>): void;
|
||||
warningOnce(message: string): void;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface CrawleeLoggerOptions {
|
||||
prefix?: string | null;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type CreateDatasetClientOptions = StorageIdentifier;
|
||||
|
||||
// @public
|
||||
export type CreateKeyValueStoreClientOptions = StorageIdentifier;
|
||||
|
||||
// @public
|
||||
export type CreateRequestQueueClientOptions = StorageIdentifier & {
|
||||
clientKey?: string;
|
||||
timeoutSecs?: number;
|
||||
};
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DatasetClient<Data extends Dictionary = Dictionary> {
|
||||
drop(): Promise<void>;
|
||||
getData(options?: DatasetClientListOptions): Promise<PaginatedList<Data>>;
|
||||
getMetadata(): Promise<DatasetInfo>;
|
||||
purge(): Promise<void>;
|
||||
pushData(items: Data[]): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DatasetClientListOptions {
|
||||
// (undocumented)
|
||||
desc?: boolean;
|
||||
// (undocumented)
|
||||
limit?: number;
|
||||
// (undocumented)
|
||||
offset?: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DatasetInfo {
|
||||
// (undocumented)
|
||||
accessedAt: Date;
|
||||
// (undocumented)
|
||||
actId?: string;
|
||||
// (undocumented)
|
||||
actRunId?: string;
|
||||
// (undocumented)
|
||||
createdAt: Date;
|
||||
// (undocumented)
|
||||
id: string;
|
||||
// (undocumented)
|
||||
itemCount: number;
|
||||
// (undocumented)
|
||||
modifiedAt: Date;
|
||||
// (undocumented)
|
||||
name?: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DatasetStats {
|
||||
// (undocumented)
|
||||
deleteCount?: number;
|
||||
// (undocumented)
|
||||
readCount?: number;
|
||||
// (undocumented)
|
||||
storageBytes?: number;
|
||||
// (undocumented)
|
||||
writeCount?: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type Dictionary<T = any> = Record<PropertyKey, T>;
|
||||
|
||||
// @public
|
||||
export interface HttpRequest {
|
||||
// (undocumented)
|
||||
body?: Readable;
|
||||
// (undocumented)
|
||||
cookieJar?: CookieJar;
|
||||
// (undocumented)
|
||||
encoding?: BufferEncoding;
|
||||
// (undocumented)
|
||||
followRedirect?: boolean | ((response: any) => boolean);
|
||||
// (undocumented)
|
||||
headerGenerator?: {
|
||||
getHeaders: (options: Record<string, unknown>) => Record<string, string>;
|
||||
};
|
||||
// (undocumented)
|
||||
headerGeneratorOptions?: Record<string, unknown>;
|
||||
// (undocumented)
|
||||
headers?: Headers;
|
||||
// (undocumented)
|
||||
insecureHTTPParser?: boolean;
|
||||
// (undocumented)
|
||||
maxRedirects?: number;
|
||||
// (undocumented)
|
||||
method?: AllowedHttpMethods;
|
||||
// (undocumented)
|
||||
proxyUrl?: string;
|
||||
// (undocumented)
|
||||
sessionToken?: object;
|
||||
// (undocumented)
|
||||
signal?: AbortSignal;
|
||||
// (undocumented)
|
||||
throwHttpErrors?: boolean;
|
||||
// (undocumented)
|
||||
timeout?: number;
|
||||
// (undocumented)
|
||||
url: string | URL;
|
||||
// (undocumented)
|
||||
useHeaderGenerator?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface HttpRequestOptions extends HttpRequest {
|
||||
form?: Record<string, string>;
|
||||
json?: unknown;
|
||||
password?: string;
|
||||
searchParams?: SearchParams;
|
||||
username?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface IBrowserPool<Page = unknown> {
|
||||
closePage(page: Page, options?: {
|
||||
error?: Error;
|
||||
}): Promise<void>;
|
||||
extractPageState(page: Page): Promise<PageState>;
|
||||
injectPageState(page: Page, state: PageState): Promise<void>;
|
||||
newPage(options?: NewPageOptions): Promise<Page>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface ISession {
|
||||
// (undocumented)
|
||||
cookieJar: CookieJar;
|
||||
// (undocumented)
|
||||
fingerprint?: SessionFingerprint;
|
||||
// (undocumented)
|
||||
readonly id: string;
|
||||
isUsable(): boolean;
|
||||
markBad(): void;
|
||||
markGood(): void;
|
||||
// (undocumented)
|
||||
proxyInfo?: ProxyInfo;
|
||||
retire(): void;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface ISessionPool {
|
||||
getSession(sessionId?: string): Promise<ISession | undefined>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface KeyValueStoreClient {
|
||||
deleteValue(key: string): Promise<void>;
|
||||
drop(): Promise<void>;
|
||||
getMetadata(): Promise<KeyValueStoreInfo>;
|
||||
getPublicUrl(key: string): Promise<string | undefined>;
|
||||
getValue(key: string): Promise<KeyValueStoreRecord | undefined>;
|
||||
listKeys(options?: KeyValueStoreListKeysOptions): Promise<KeyValueStoreItemData[]>;
|
||||
purge(): Promise<void>;
|
||||
recordExists(key: string): Promise<boolean>;
|
||||
setValue(record: KeyValueStoreInputRecord): Promise<void>;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface KeyValueStoreInfo {
|
||||
// (undocumented)
|
||||
accessedAt: Date;
|
||||
// (undocumented)
|
||||
actId?: string;
|
||||
// (undocumented)
|
||||
actRunId?: string;
|
||||
// (undocumented)
|
||||
createdAt: Date;
|
||||
// (undocumented)
|
||||
id: string;
|
||||
// (undocumented)
|
||||
modifiedAt: Date;
|
||||
// (undocumented)
|
||||
name?: string;
|
||||
// (undocumented)
|
||||
stats?: KeyValueStoreStats;
|
||||
// (undocumented)
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface KeyValueStoreInputRecord {
|
||||
// (undocumented)
|
||||
contentType?: string;
|
||||
// (undocumented)
|
||||
key: string;
|
||||
// (undocumented)
|
||||
value: KeyValueStoreRecordInputValue;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface KeyValueStoreItemData {
|
||||
// (undocumented)
|
||||
key: string;
|
||||
// (undocumented)
|
||||
size: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface KeyValueStoreListKeysOptions {
|
||||
exclusiveStartKey?: string;
|
||||
limit?: number;
|
||||
prefix?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface KeyValueStoreRecord {
|
||||
// (undocumented)
|
||||
contentType?: string;
|
||||
// (undocumented)
|
||||
key: string;
|
||||
// (undocumented)
|
||||
value: Buffer | ArrayBuffer;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type KeyValueStoreRecordInputValue = Buffer | ArrayBuffer | ArrayBufferView | string | NodeJS.ReadableStream | ReadableStream;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface KeyValueStoreStats {
|
||||
// (undocumented)
|
||||
deleteCount?: number;
|
||||
// (undocumented)
|
||||
listCount?: number;
|
||||
// (undocumented)
|
||||
readCount?: number;
|
||||
// (undocumented)
|
||||
storageBytes?: number;
|
||||
// (undocumented)
|
||||
writeCount?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface NewPageOptions {
|
||||
id?: string;
|
||||
session?: ISession;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface PageState {
|
||||
cookies: Cookie[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface PaginatedList<Data> {
|
||||
count: number;
|
||||
desc?: boolean;
|
||||
items: Data[];
|
||||
limit: number;
|
||||
offset: number;
|
||||
total: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ProcessedRequest {
|
||||
// (undocumented)
|
||||
requestId: string;
|
||||
// (undocumented)
|
||||
uniqueKey: string;
|
||||
// (undocumented)
|
||||
wasAlreadyHandled: boolean;
|
||||
// (undocumented)
|
||||
wasAlreadyPresent: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface ProxyInfo {
|
||||
hostname: string;
|
||||
ignoreTlsErrors?: boolean;
|
||||
password: string;
|
||||
port: number | string;
|
||||
url: string;
|
||||
username?: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface QueueOperationInfo {
|
||||
requestId: string;
|
||||
wasAlreadyHandled: boolean;
|
||||
wasAlreadyPresent: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type RedirectHandler = (redirectResponse: Response, updatedRequest: {
|
||||
url?: string | URL;
|
||||
headers: Headers;
|
||||
}) => void;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface RequestOptions {
|
||||
// (undocumented)
|
||||
[k: string]: unknown;
|
||||
// (undocumented)
|
||||
forefront?: boolean;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface RequestQueueClient {
|
||||
addBatchOfRequests(requests: RequestSchema[], options?: RequestOptions): Promise<BatchAddRequestsResult>;
|
||||
drop(): Promise<void>;
|
||||
fetchNextRequest(): Promise<RequestOptions | null>;
|
||||
getMetadata(): Promise<RequestQueueInfo>;
|
||||
getRequest(uniqueKey: string): Promise<RequestOptions | undefined>;
|
||||
isEmpty(): Promise<boolean>;
|
||||
isFinished(): Promise<boolean>;
|
||||
markRequestAsHandled(request: UpdateRequestSchema): Promise<QueueOperationInfo | null>;
|
||||
purge(): Promise<void>;
|
||||
reclaimRequest(request: UpdateRequestSchema, options?: RequestOptions): Promise<QueueOperationInfo | null>;
|
||||
setExpectedRequestProcessingTimeSecs?(secs: number): void;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface RequestQueueInfo {
|
||||
// (undocumented)
|
||||
accessedAt: Date;
|
||||
// (undocumented)
|
||||
actId?: string;
|
||||
// (undocumented)
|
||||
actRunId?: string;
|
||||
// (undocumented)
|
||||
createdAt: Date;
|
||||
// (undocumented)
|
||||
expireAt?: string;
|
||||
// (undocumented)
|
||||
hadMultipleClients?: boolean;
|
||||
// (undocumented)
|
||||
handledRequestCount: number;
|
||||
// (undocumented)
|
||||
id: string;
|
||||
// (undocumented)
|
||||
modifiedAt: Date;
|
||||
// (undocumented)
|
||||
name?: string;
|
||||
// (undocumented)
|
||||
pendingRequestCount: number;
|
||||
// (undocumented)
|
||||
stats?: RequestQueueStats;
|
||||
// (undocumented)
|
||||
totalRequestCount: number;
|
||||
// (undocumented)
|
||||
userId?: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface RequestQueueStats {
|
||||
// (undocumented)
|
||||
deleteCount?: number;
|
||||
// (undocumented)
|
||||
headItemReadCount?: number;
|
||||
// (undocumented)
|
||||
readCount?: number;
|
||||
// (undocumented)
|
||||
storageBytes?: number;
|
||||
// (undocumented)
|
||||
writeCount?: number;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface RequestSchema {
|
||||
// (undocumented)
|
||||
errorMessages?: string[];
|
||||
// (undocumented)
|
||||
handledAt?: string;
|
||||
// (undocumented)
|
||||
headers?: Dictionary<string>;
|
||||
// (undocumented)
|
||||
id?: string;
|
||||
// (undocumented)
|
||||
loadedUrl?: string;
|
||||
// (undocumented)
|
||||
method?: AllowedHttpMethods;
|
||||
// (undocumented)
|
||||
noRetry?: boolean;
|
||||
// (undocumented)
|
||||
payload?: string;
|
||||
// (undocumented)
|
||||
retryCount?: number;
|
||||
// (undocumented)
|
||||
uniqueKey: string;
|
||||
// (undocumented)
|
||||
url: string;
|
||||
// (undocumented)
|
||||
userData?: Dictionary;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type SearchParams = string | URLSearchParams | Record<string, string | number | boolean | null | undefined>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface SendRequestOptions {
|
||||
// (undocumented)
|
||||
cookieJar?: CookieJar;
|
||||
proxyUrl?: string;
|
||||
// (undocumented)
|
||||
session?: ISession;
|
||||
signal?: AbortSignal;
|
||||
timeoutMillis?: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface SessionFingerprint {
|
||||
browser?: 'chrome' | 'firefox' | 'safari' | 'edge';
|
||||
device?: 'desktop' | 'mobile';
|
||||
platform?: 'windows' | 'macos' | 'linux' | 'android' | 'ios';
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface SessionState {
|
||||
// (undocumented)
|
||||
cookieJar: SerializedCookieJar;
|
||||
// (undocumented)
|
||||
createdAt: string;
|
||||
// (undocumented)
|
||||
errorScore: number;
|
||||
// (undocumented)
|
||||
errorScoreDecrement: number;
|
||||
// (undocumented)
|
||||
expiresAt: string;
|
||||
// (undocumented)
|
||||
fingerprint?: SessionFingerprint;
|
||||
// (undocumented)
|
||||
id: string;
|
||||
// (undocumented)
|
||||
maxErrorScore: number;
|
||||
// (undocumented)
|
||||
maxUsageCount: number;
|
||||
// (undocumented)
|
||||
proxyInfo?: ProxyInfo;
|
||||
// (undocumented)
|
||||
retired: boolean;
|
||||
// (undocumented)
|
||||
usageCount: number;
|
||||
// (undocumented)
|
||||
userData: object;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface SetStatusMessageOptions {
|
||||
// (undocumented)
|
||||
isStatusMessageTerminal?: boolean;
|
||||
// (undocumented)
|
||||
level?: 'DEBUG' | 'INFO' | 'WARNING' | 'ERROR';
|
||||
}
|
||||
|
||||
// @public
|
||||
export interface StorageClient {
|
||||
createDatasetClient(options?: CreateDatasetClientOptions): Promise<DatasetClient>;
|
||||
createKeyValueStoreClient(options?: CreateKeyValueStoreClientOptions): Promise<KeyValueStoreClient>;
|
||||
createRequestQueueClient(options?: CreateRequestQueueClientOptions): Promise<RequestQueueClient>;
|
||||
getStorageClientCacheKey?(): string;
|
||||
// (undocumented)
|
||||
purge?(): Promise<void>;
|
||||
// (undocumented)
|
||||
setStatusMessage?(message: string, options?: SetStatusMessageOptions): Promise<void>;
|
||||
// (undocumented)
|
||||
stats?: {
|
||||
rateLimitErrors: number[];
|
||||
};
|
||||
storageExists?(id: string, type: 'Dataset' | 'KeyValueStore' | 'RequestQueue'): Promise<boolean>;
|
||||
// (undocumented)
|
||||
teardown?(): Promise<void>;
|
||||
}
|
||||
|
||||
// @public
|
||||
export type StorageIdentifier = {
|
||||
id: string;
|
||||
name?: never;
|
||||
alias?: never;
|
||||
} | {
|
||||
id?: never;
|
||||
name: string;
|
||||
alias?: never;
|
||||
} | {
|
||||
id?: never;
|
||||
name?: never;
|
||||
alias: string;
|
||||
} | {
|
||||
id?: never;
|
||||
name?: never;
|
||||
alias?: never;
|
||||
};
|
||||
|
||||
// @public (undocumented)
|
||||
export interface StreamOptions extends SendRequestOptions {
|
||||
// (undocumented)
|
||||
onRedirect?: RedirectHandler;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface UnprocessedRequest {
|
||||
// (undocumented)
|
||||
method?: AllowedHttpMethods;
|
||||
// (undocumented)
|
||||
uniqueKey: string;
|
||||
// (undocumented)
|
||||
url: string;
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export interface UpdateRequestSchema extends RequestSchema {
|
||||
// (undocumented)
|
||||
id: string;
|
||||
}
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,400 @@
|
||||
## API Report File for "@crawlee/utils"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import type { AllowedHttpMethods } from '@crawlee/types';
|
||||
import { Awaitable } from '@crawlee/types';
|
||||
import type { BaseHttpClient } from '@crawlee/types';
|
||||
import { Cheerio } from 'cheerio';
|
||||
import { CheerioAPI } from 'cheerio';
|
||||
import { Constructor } from '@crawlee/types';
|
||||
import type { CrawleeLogger } from '@crawlee/types';
|
||||
import { Dictionary } from '@crawlee/types';
|
||||
import { Element as Element_2 } from 'domhandler';
|
||||
import type { IncomingMessage } from 'node:http';
|
||||
import type { SearchParams } from '@crawlee/types';
|
||||
|
||||
// @internal
|
||||
export function applySearchParams(url: URL, searchParams: SearchParams | undefined): void;
|
||||
|
||||
// @internal
|
||||
export function asyncifyIterable<T>(iterable: Iterable<T> | AsyncIterable<T>): AsyncIterable<T>;
|
||||
|
||||
export { Awaitable }
|
||||
|
||||
export { Cheerio }
|
||||
|
||||
export { CheerioAPI }
|
||||
|
||||
// @public (undocumented)
|
||||
export type CheerioRoot = CheerioAPI;
|
||||
|
||||
// @public (undocumented)
|
||||
export function chunk<T>(array: readonly T[], chunkSize: number): T[][];
|
||||
|
||||
// @internal
|
||||
export function chunkedAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>, chunkSize: number): AsyncIterable<T[]>;
|
||||
|
||||
// @public (undocumented)
|
||||
export const CLOUDFLARE_RETRY_CSS_SELECTORS: string[];
|
||||
|
||||
export { Constructor }
|
||||
|
||||
// @public
|
||||
export interface CpuSample {
|
||||
// (undocumented)
|
||||
containerUsage: number;
|
||||
// (undocumented)
|
||||
systemUsage: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
export function createRequestDebugInfo(request: Request_2, response?: IncomingMessage | Partial<BrowserResponseLike>, additionalFields?: Dictionary): Dictionary;
|
||||
|
||||
export { Dictionary }
|
||||
|
||||
// @public
|
||||
const DISCORD_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const DISCORD_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export function discoverValidSitemaps(urls: string[], options?: {
|
||||
proxyUrl?: string;
|
||||
timeoutMillis?: number;
|
||||
signal?: AbortSignal;
|
||||
requestTimeoutMillis?: number;
|
||||
httpClient?: BaseHttpClient;
|
||||
logger?: CrawleeLogger;
|
||||
}): AsyncIterable<string>;
|
||||
|
||||
// @public
|
||||
export function downloadListOfUrls(options: DownloadListOfUrlsOptions): Promise<string[]>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface DownloadListOfUrlsOptions {
|
||||
encoding?: BufferEncoding;
|
||||
httpClient?: BaseHttpClient;
|
||||
proxyUrl?: string;
|
||||
url: string;
|
||||
urlRegExp?: RegExp;
|
||||
}
|
||||
|
||||
export { Element_2 as Element }
|
||||
|
||||
// @public
|
||||
const EMAIL_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const EMAIL_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
function emailsFromText(text: string): string[];
|
||||
|
||||
// @public
|
||||
function emailsFromUrls(urls: string[]): string[];
|
||||
|
||||
// @public (undocumented)
|
||||
export function entries<T extends {}>(obj: T): [keyof T, T[keyof T]][];
|
||||
|
||||
// @public
|
||||
export function expandShadowRoots(document: Document): string;
|
||||
|
||||
// @public
|
||||
export function extractUrls(options: ExtractUrlsOptions): string[];
|
||||
|
||||
// @public
|
||||
export function extractUrlsFromCheerio($: CheerioAPI, selector?: string, baseUrl?: string): string[];
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ExtractUrlsOptions {
|
||||
string: string;
|
||||
urlRegExp?: RegExp;
|
||||
}
|
||||
|
||||
// @public
|
||||
const FACEBOOK_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const FACEBOOK_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export function getCgroupsVersion(forceReset?: boolean): Promise<"V1" | "V2" | null>;
|
||||
|
||||
// @internal
|
||||
export function getCurrentCpuTicksV2(options?: {
|
||||
containerized?: boolean;
|
||||
logger?: CrawleeLogger;
|
||||
}): Promise<number>;
|
||||
|
||||
// @internal
|
||||
export function getMemoryInfo(options?: {
|
||||
containerized?: boolean;
|
||||
logger?: CrawleeLogger;
|
||||
}): Promise<MemoryInfo>;
|
||||
|
||||
// @public (undocumented)
|
||||
export function getObjectType(value: unknown): string;
|
||||
|
||||
// @public
|
||||
export function htmlToText(htmlOrCheerioElement: string | CheerioRoot): string;
|
||||
|
||||
// @internal
|
||||
export function inspectValue(value: unknown): string;
|
||||
|
||||
// @public
|
||||
const INSTAGRAM_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const INSTAGRAM_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @internal
|
||||
export function isAsyncIterable<T>(value: unknown): value is AsyncIterable<T>;
|
||||
|
||||
// @public
|
||||
export function isBuffer(value: unknown): value is Buffer | ArrayBuffer | ArrayBufferView;
|
||||
|
||||
// @public
|
||||
export function isContainerized(): Promise<boolean>;
|
||||
|
||||
// @public
|
||||
export function isDocker(forceReset?: boolean): Promise<boolean>;
|
||||
|
||||
// @internal
|
||||
export function isIterable<T>(value: unknown): value is Iterable<T>;
|
||||
|
||||
// @public (undocumented)
|
||||
export function isLambda(): boolean;
|
||||
|
||||
// @public
|
||||
export function isStream(value: unknown): value is NodeJS.ReadableStream | ReadableStream;
|
||||
|
||||
// @public (undocumented)
|
||||
export function keys<T extends {}>(obj: T): (keyof T)[];
|
||||
|
||||
// @public
|
||||
const LINKEDIN_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const LINKEDIN_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export interface MemoryInfo {
|
||||
childProcessesBytes: number;
|
||||
freeBytes: number;
|
||||
mainProcessBytes: number;
|
||||
totalBytes: number;
|
||||
usedBytes: number;
|
||||
}
|
||||
|
||||
// @public
|
||||
export function mergeAsyncIterables<T>(...iterables: AsyncIterable<T>[]): AsyncIterable<T>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface OpenGraphProperty {
|
||||
// (undocumented)
|
||||
children: OpenGraphProperty[];
|
||||
// (undocumented)
|
||||
name: string;
|
||||
// (undocumented)
|
||||
outputName: string;
|
||||
}
|
||||
|
||||
// @public
|
||||
function parseHandlesFromHtml(html: string, data?: Record<string, unknown> | null): SocialHandles;
|
||||
|
||||
// @public
|
||||
export function parseOpenGraph(raw: string, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
|
||||
|
||||
// @public (undocumented)
|
||||
export function parseOpenGraph($: CheerioAPI, additionalProperties?: OpenGraphProperty[]): Dictionary<OpenGraphResult>;
|
||||
|
||||
// @public (undocumented)
|
||||
export function parseSitemap<T extends ParseSitemapOptions>(initialSources: SitemapSource[], proxyUrl?: string, options?: T): AsyncIterable<T['emitNestedSitemaps'] extends true ? SitemapUrl | NestedSitemap : SitemapUrl>;
|
||||
|
||||
// @public (undocumented)
|
||||
export interface ParseSitemapOptions {
|
||||
emitNestedSitemaps?: true | false;
|
||||
httpClient?: BaseHttpClient;
|
||||
logger?: CrawleeLogger;
|
||||
maxDepth?: number;
|
||||
reportNetworkErrors?: boolean;
|
||||
sitemapRetries?: number;
|
||||
timeoutMillis?: number;
|
||||
}
|
||||
|
||||
// @internal
|
||||
export interface PeekableAsyncIterable<T> extends AsyncIterable<T> {
|
||||
// (undocumented)
|
||||
[Symbol.asyncIterator](): PeekableAsyncIterator<T>;
|
||||
}
|
||||
|
||||
// @internal
|
||||
export function peekableAsyncIterable<T>(iterable: AsyncIterable<T> | Iterable<T>): PeekableAsyncIterable<T>;
|
||||
|
||||
// @internal
|
||||
export interface PeekableAsyncIterator<T> extends AsyncIterator<T>, AsyncIterable<T> {
|
||||
peek(): Promise<T | undefined>;
|
||||
}
|
||||
|
||||
// @public
|
||||
function phonesFromText(text: string): string[];
|
||||
|
||||
// @public
|
||||
function phonesFromUrls(urls: string[]): string[];
|
||||
|
||||
// @public
|
||||
const PINTEREST_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const PINTEREST_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export const RETRY_CSS_SELECTORS: string[];
|
||||
|
||||
// @public
|
||||
class RobotsTxtFile {
|
||||
static find(url: string, options?: {
|
||||
signal?: AbortSignal;
|
||||
timeoutMillis?: number;
|
||||
proxyUrl?: string;
|
||||
httpClient?: BaseHttpClient;
|
||||
logger?: CrawleeLogger;
|
||||
}): Promise<RobotsTxtFile>;
|
||||
static from(url: string, content: string, proxyUrl?: string): RobotsTxtFile;
|
||||
getSitemaps(): string[];
|
||||
isAllowed(url: string, userAgent?: string): boolean;
|
||||
// (undocumented)
|
||||
protected static load(url: string, options?: {
|
||||
signal?: AbortSignal;
|
||||
timeoutMillis?: number;
|
||||
proxyUrl?: string;
|
||||
httpClient?: BaseHttpClient;
|
||||
logger?: CrawleeLogger;
|
||||
}): Promise<RobotsTxtFile>;
|
||||
parseSitemaps(): Promise<Sitemap>;
|
||||
parseUrlsFromSitemaps(): Promise<string[]>;
|
||||
}
|
||||
export { RobotsTxtFile as RobotsFile }
|
||||
export { RobotsTxtFile }
|
||||
|
||||
// @public
|
||||
export const ROTATE_PROXY_ERRORS: string[];
|
||||
|
||||
// @public
|
||||
export class Sitemap {
|
||||
constructor(urls: string[]);
|
||||
static fromXmlString(content: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
||||
static load(urls: string | string[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
||||
// (undocumented)
|
||||
protected static parse(sources: SitemapSource[], proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
||||
static tryCommonNames(url: string, proxyUrl?: string, parseSitemapOptions?: ParseSitemapOptions): Promise<Sitemap>;
|
||||
// (undocumented)
|
||||
readonly urls: string[];
|
||||
}
|
||||
|
||||
// @public (undocumented)
|
||||
export type SitemapUrl = SitemapUrlData & {
|
||||
originSitemapUrl: string;
|
||||
};
|
||||
|
||||
// @public
|
||||
export function sleep(millis?: number): Promise<void>;
|
||||
|
||||
// @public
|
||||
export function snakeCaseToCamelCase(snakeCaseStr: string): string;
|
||||
|
||||
declare namespace social {
|
||||
export {
|
||||
emailsFromText,
|
||||
emailsFromUrls,
|
||||
phonesFromText,
|
||||
phonesFromUrls,
|
||||
parseHandlesFromHtml,
|
||||
EMAIL_REGEX,
|
||||
EMAIL_REGEX_GLOBAL,
|
||||
SocialHandles,
|
||||
LINKEDIN_REGEX,
|
||||
LINKEDIN_REGEX_GLOBAL,
|
||||
INSTAGRAM_REGEX,
|
||||
INSTAGRAM_REGEX_GLOBAL,
|
||||
TWITTER_REGEX,
|
||||
TWITTER_REGEX_GLOBAL,
|
||||
FACEBOOK_REGEX,
|
||||
FACEBOOK_REGEX_GLOBAL,
|
||||
YOUTUBE_REGEX,
|
||||
YOUTUBE_REGEX_GLOBAL,
|
||||
TIKTOK_REGEX,
|
||||
TIKTOK_REGEX_GLOBAL,
|
||||
PINTEREST_REGEX,
|
||||
PINTEREST_REGEX_GLOBAL,
|
||||
DISCORD_REGEX,
|
||||
DISCORD_REGEX_GLOBAL
|
||||
}
|
||||
}
|
||||
|
||||
// @public
|
||||
interface SocialHandles {
|
||||
// (undocumented)
|
||||
discords: string[];
|
||||
// (undocumented)
|
||||
emails: string[];
|
||||
// (undocumented)
|
||||
facebooks: string[];
|
||||
// (undocumented)
|
||||
instagrams: string[];
|
||||
// (undocumented)
|
||||
linkedIns: string[];
|
||||
// (undocumented)
|
||||
phones: string[];
|
||||
// (undocumented)
|
||||
phonesUncertain: string[];
|
||||
// (undocumented)
|
||||
pinterests: string[];
|
||||
// (undocumented)
|
||||
tiktoks: string[];
|
||||
// (undocumented)
|
||||
twitters: string[];
|
||||
// (undocumented)
|
||||
youtubes: string[];
|
||||
}
|
||||
|
||||
// @public
|
||||
const TIKTOK_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const TIKTOK_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export function toBuffer(value: Buffer | ArrayBuffer | ArrayBufferView): Buffer;
|
||||
|
||||
// @public
|
||||
export function tryAbsoluteURL(href: string, baseUrl: string): string | undefined;
|
||||
|
||||
// @public
|
||||
const TWITTER_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const TWITTER_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// @public
|
||||
export const URL_NO_COMMAS_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
export const URL_WITH_COMMAS_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
export function weightedAvg(arrValues: number[], arrWeights: number[]): number;
|
||||
|
||||
// @public
|
||||
const YOUTUBE_REGEX: RegExp;
|
||||
|
||||
// @public
|
||||
const YOUTUBE_REGEX_GLOBAL: RegExp;
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -0,0 +1,45 @@
|
||||
## API Report File for "crawlee"
|
||||
|
||||
> Do not edit this file. It is a report generated by [API Extractor](https://api-extractor.com/).
|
||||
|
||||
```ts
|
||||
|
||||
import { downloadListOfUrls } from '@crawlee/utils';
|
||||
import { enqueueLinks } from '@crawlee/core';
|
||||
import { Log } from '@apify/log';
|
||||
import { parseOpenGraph } from '@crawlee/utils';
|
||||
import { playwrightUtils } from '@crawlee/playwright';
|
||||
import { puppeteerUtils } from '@crawlee/puppeteer';
|
||||
import { sleep } from '@crawlee/utils';
|
||||
import { social } from '@crawlee/utils';
|
||||
|
||||
// @public (undocumented)
|
||||
export const utils: {
|
||||
puppeteer: typeof puppeteerUtils;
|
||||
playwright: typeof playwrightUtils;
|
||||
log: Log;
|
||||
enqueueLinks: typeof enqueueLinks;
|
||||
social: typeof social;
|
||||
sleep: typeof sleep;
|
||||
downloadListOfUrls: typeof downloadListOfUrls;
|
||||
parseOpenGraph: typeof parseOpenGraph;
|
||||
};
|
||||
|
||||
|
||||
export * from "@crawlee/basic";
|
||||
export * from "@crawlee/browser";
|
||||
export * from "@crawlee/browser-pool";
|
||||
export * from "@crawlee/cheerio";
|
||||
export * from "@crawlee/core";
|
||||
export * from "@crawlee/fs-storage";
|
||||
export * from "@crawlee/http";
|
||||
export * from "@crawlee/jsdom";
|
||||
export * from "@crawlee/linkedom";
|
||||
export * from "@crawlee/memory-storage";
|
||||
export * from "@crawlee/playwright";
|
||||
export * from "@crawlee/puppeteer";
|
||||
export * from "@crawlee/utils";
|
||||
|
||||
// (No @packageDocumentation comment for this package)
|
||||
|
||||
```
|
||||
@@ -45,6 +45,8 @@
|
||||
"publish:prod": "lerna publish from-package --contents dist --force-publish --yes",
|
||||
"release:prod": "pnpm build && pnpm publish:prod",
|
||||
"release:pin-versions": "turbo run copy -- -- --pin-versions",
|
||||
"api:extract": "tsx scripts/api-extractor/run.ts",
|
||||
"api:check": "tsx scripts/api-extractor/run.ts --verify",
|
||||
"lint": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware",
|
||||
"lint:fix": "oxlint packages test docs --tsconfig=tsconfig.json --type-aware --fix",
|
||||
"format": "oxfmt packages test docs --write",
|
||||
@@ -56,6 +58,7 @@
|
||||
"@apify/oxlint-config": "^0.2.5",
|
||||
"@apify/tsconfig": "^0.2.0",
|
||||
"@commitlint/config-conventional": "^21.0.0",
|
||||
"@microsoft/api-extractor": "^7.58.9",
|
||||
"@playwright/browser-chromium": "1.61.1",
|
||||
"@playwright/browser-firefox": "1.61.1",
|
||||
"@playwright/browser-webkit": "1.61.1",
|
||||
|
||||
Generated
+148
@@ -30,6 +30,9 @@ importers:
|
||||
'@commitlint/config-conventional':
|
||||
specifier: ^20.0.0
|
||||
version: 20.5.0
|
||||
'@microsoft/api-extractor':
|
||||
specifier: ^7.58.9
|
||||
version: 7.58.9(@types/node@24.12.2)
|
||||
'@playwright/browser-chromium':
|
||||
specifier: 1.60.0
|
||||
version: 1.60.0
|
||||
@@ -3166,6 +3169,19 @@ packages:
|
||||
'@mermaid-js/parser@1.1.0':
|
||||
resolution: {integrity: sha512-gxK9ZX2+Fex5zu8LhRQoMeMPEHbc73UKZ0FQ54YrQtUxE1VVhMwzeNtKRPAu5aXks4FasbMe4xB4bWrmq6Jlxw==}
|
||||
|
||||
'@microsoft/api-extractor-model@7.33.8':
|
||||
resolution: {integrity: sha512-aIcoQggPyer3B6Ze3usz0YWC/oBwUHfRH5ETUsr+oT2BRA6SfTJl7IKPcPZkX4UR+PohowzW4uMxsvjrn8vm+w==}
|
||||
|
||||
'@microsoft/api-extractor@7.58.9':
|
||||
resolution: {integrity: sha512-S2UF4yza5GoxCmf7hJQNxJNZN9ltOVuOQv8Dy+Z21aol5ERoBNMdWcQHm4MJMPPItW4H/4rZD906iaf4mUojJA==}
|
||||
hasBin: true
|
||||
|
||||
'@microsoft/tsdoc-config@0.18.1':
|
||||
resolution: {integrity: sha512-9brPoVdfN9k9g0dcWkFeA7IH9bbcttzDJlXvkf8b2OBzd5MueR1V2wkKBL0abn0otvmkHJC6aapBOTJDDeMCZg==}
|
||||
|
||||
'@microsoft/tsdoc@0.16.0':
|
||||
resolution: {integrity: sha512-xgAyonlVVS+q7Vc7qLW0UrJU7rSFcETRWsqdXZtjzRU8dF+6CkozTK4V4y1LwOX7j8r/vHphjDeMeGI4tNGeGA==}
|
||||
|
||||
'@modelcontextprotocol/sdk@1.29.0':
|
||||
resolution: {integrity: sha512-zo37mZA9hJWpULgkRpowewez1y6ML5GsXJPY8FI0tBBCd77HEvza4jDqRKOXgHNn867PVGCyTdzqpz0izu5ZjQ==}
|
||||
engines: {node: '>=18'}
|
||||
@@ -3997,6 +4013,36 @@ packages:
|
||||
'@rtsao/scc@1.1.0':
|
||||
resolution: {integrity: sha512-zt6OdqaDoOnJ1ZYsCYGt9YmWzDXl4vQdKTyJev62gFhRGKdx7mcT54V9KIjg+d2wi9EXsPvAPKe7i7WjfVWB8g==}
|
||||
|
||||
'@rushstack/node-core-library@5.23.1':
|
||||
resolution: {integrity: sha512-wlKmIKIYCKuCASbITvOxLZXepPbwXvrv7S6ig6XNWFchSyhL/E2txmVXspHY49Wu2dzf7nI27a2k/yV5BA3EiA==}
|
||||
peerDependencies:
|
||||
'@types/node': '*'
|
||||
peerDependenciesMeta:
|
||||
'@types/node':
|
||||
optional: true
|
||||
|
||||
'@rushstack/problem-matcher@0.2.1':
|
||||
resolution: {integrity: sha512-gulfhBs6n+I5b7DvjKRfhMGyUejtSgOHTclF/eONr8hcgF1APEDjhxIsfdUYYMzC3rvLwGluqLjbwCFZ8nxrog==}
|
||||
peerDependencies:
|
||||
'@types/node': '*'
|
||||
peerDependenciesMeta:
|
||||
'@types/node':
|
||||
optional: true
|
||||
|
||||
'@rushstack/rig-package@0.7.3':
|
||||
resolution: {integrity: sha512-aAA518n6wxxjCfnTAOjQnm7ngNE0FVHxHAw2pxKlIhxrMn0XQjGcXKF0oKWpjBgJOmsaJpVob/v+zr3zxgPWuA==}
|
||||
|
||||
'@rushstack/terminal@0.24.0':
|
||||
resolution: {integrity: sha512-8ZQS4MMaGsv27EXCBiH7WMPkRZrffeDoIevs6z9TM5dzqiY6+Hn4evfK/G+gvgBTjfvfkHIZPQQmalmI2sM4TQ==}
|
||||
peerDependencies:
|
||||
'@types/node': '*'
|
||||
peerDependenciesMeta:
|
||||
'@types/node':
|
||||
optional: true
|
||||
|
||||
'@rushstack/ts-command-line@5.3.10':
|
||||
resolution: {integrity: sha512-fwI076HYknC0IrMXdY6UmjDv+PH7NHhNJX3/pY2UblSE5XrXgndXZPiOe/6ZtuFpn6DvVDVNhtkIzQ+Qu/MhVQ==}
|
||||
|
||||
'@sapphire/async-queue@1.5.5':
|
||||
resolution: {integrity: sha512-cvGzxbba6sav2zZkH8GPf2oGk9yYoD5qrNWdu9fRehifgnFZJMV+nuy2nON2roRO4yQQ+v7MK/Pktl/HgfsUXg==}
|
||||
engines: {node: '>=v14.0.0', npm: '>=7.0.0'}
|
||||
@@ -4436,6 +4482,9 @@ packages:
|
||||
'@tybys/wasm-util@0.9.0':
|
||||
resolution: {integrity: sha512-6+7nlbMVX/PVDCwaIQ8nTOPveOcFLSt8GcXdx8hD0bt39uWxYT88uXzqTd4fTvqta7oeUJqudepapKNt2DYJFw==}
|
||||
|
||||
'@types/argparse@1.0.38':
|
||||
resolution: {integrity: sha512-ebDJ9b0e702Yr7pWgB0jzm+CX4Srzz8RcXtLJDJB+BSccqMa36uyH/zUsSYao5+BD1ytv3k3rPYCq4mAE1hsXA==}
|
||||
|
||||
'@types/body-parser@1.19.6':
|
||||
resolution: {integrity: sha512-HLFeCYgz89uk22N5Qg3dvGvsv46B8GLvKKo1zKG4NybA8U2DiEO3w9lqGg29t/tfLRJpJ6iQxnVw4OnB7MoM9g==}
|
||||
|
||||
@@ -5135,6 +5184,14 @@ packages:
|
||||
peerDependencies:
|
||||
zod: ^3.25.76 || ^4.1.8
|
||||
|
||||
ajv-draft-04@1.0.0:
|
||||
resolution: {integrity: sha512-mv00Te6nmYbRp5DCwclxtt7yV/joXJPGS7nM+97GdxvuttCOfgI3K4U25zboyeX0O+myI8ERluxQe5wljMmVIw==}
|
||||
peerDependencies:
|
||||
ajv: ^8.5.0
|
||||
peerDependenciesMeta:
|
||||
ajv:
|
||||
optional: true
|
||||
|
||||
ajv-formats@2.1.1:
|
||||
resolution: {integrity: sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==}
|
||||
peerDependencies:
|
||||
@@ -6619,6 +6676,10 @@ packages:
|
||||
devtools-protocol@0.0.1612613:
|
||||
resolution: {integrity: sha512-hp32aOyalF3vGZiTPA3CiKCgpIH7QOj+S0dRVMmEReagtmWQYigiCQWp7OGWrgaW2IHNlSzCwfDneZe16DlBCQ==}
|
||||
|
||||
diff@8.0.4:
|
||||
resolution: {integrity: sha512-DPi0FmjiSU5EvQV0++GFDOJ9ASQUVFh5kD+OzOnYdi7n3Wpm9hWWGfB/O2blfHcMVTL5WkQXSnRiK9makhrcnw==}
|
||||
engines: {node: '>=0.3.1'}
|
||||
|
||||
diffie-hellman@5.0.3:
|
||||
resolution: {integrity: sha512-kqag/Nl+f3GwyK25fhUMYj81BUOrZ9IuJsjIcDE5icNM9FJHAVm3VcUDxdLPoQtTuUylWm6ZIknYJwwaPxsUzg==}
|
||||
|
||||
@@ -8495,6 +8556,9 @@ packages:
|
||||
resolution: {integrity: sha512-ekilCSN1jwRvIbgeg/57YFh8qQDNbwDb9xT/qu2DAHbFFZUicIl4ygVaAvzveMhMVr3LnpSKTNnwt8PoOfmKhQ==}
|
||||
hasBin: true
|
||||
|
||||
jju@1.4.0:
|
||||
resolution: {integrity: sha512-8wb9Yw966OSxApiCt0K3yNJL8pnNeIv+OEq2YMidz4FKP6nonSRoOXc80iXY4JaN2FC11B9qsNmDsm+ZOfMROA==}
|
||||
|
||||
joi@17.13.3:
|
||||
resolution: {integrity: sha512-otDA4ldcIx+ZXsKHWmp0YizCweVRZG96J10b0FevjfuncLO1oX59THoAmHkNubYJ+9gWsYsp5k8v4ib6oDv1fA==}
|
||||
|
||||
@@ -15881,6 +15945,41 @@ snapshots:
|
||||
dependencies:
|
||||
langium: 4.2.2
|
||||
|
||||
'@microsoft/api-extractor-model@7.33.8(@types/node@24.12.2)':
|
||||
dependencies:
|
||||
'@microsoft/tsdoc': 0.16.0
|
||||
'@microsoft/tsdoc-config': 0.18.1
|
||||
'@rushstack/node-core-library': 5.23.1(@types/node@24.12.2)
|
||||
transitivePeerDependencies:
|
||||
- '@types/node'
|
||||
|
||||
'@microsoft/api-extractor@7.58.9(@types/node@24.12.2)':
|
||||
dependencies:
|
||||
'@microsoft/api-extractor-model': 7.33.8(@types/node@24.12.2)
|
||||
'@microsoft/tsdoc': 0.16.0
|
||||
'@microsoft/tsdoc-config': 0.18.1
|
||||
'@rushstack/node-core-library': 5.23.1(@types/node@24.12.2)
|
||||
'@rushstack/rig-package': 0.7.3
|
||||
'@rushstack/terminal': 0.24.0(@types/node@24.12.2)
|
||||
'@rushstack/ts-command-line': 5.3.10(@types/node@24.12.2)
|
||||
diff: 8.0.4
|
||||
minimatch: 9.0.9
|
||||
resolve: 1.22.12
|
||||
semver: 7.7.4
|
||||
source-map: 0.6.1
|
||||
typescript: 5.9.3
|
||||
transitivePeerDependencies:
|
||||
- '@types/node'
|
||||
|
||||
'@microsoft/tsdoc-config@0.18.1':
|
||||
dependencies:
|
||||
'@microsoft/tsdoc': 0.16.0
|
||||
ajv: 8.18.0
|
||||
jju: 1.4.0
|
||||
resolve: 1.22.12
|
||||
|
||||
'@microsoft/tsdoc@0.16.0': {}
|
||||
|
||||
'@modelcontextprotocol/sdk@1.29.0(@cfworker/json-schema@4.1.1)(zod@4.3.6)':
|
||||
dependencies:
|
||||
'@hono/node-server': 1.19.14(hono@4.12.12)
|
||||
@@ -16635,6 +16734,45 @@ snapshots:
|
||||
|
||||
'@rtsao/scc@1.1.0': {}
|
||||
|
||||
'@rushstack/node-core-library@5.23.1(@types/node@24.12.2)':
|
||||
dependencies:
|
||||
ajv: 8.18.0
|
||||
ajv-draft-04: 1.0.0(ajv@8.18.0)
|
||||
ajv-formats: 3.0.1(ajv@8.18.0)
|
||||
fs-extra: 11.3.4
|
||||
import-lazy: 4.0.0
|
||||
jju: 1.4.0
|
||||
resolve: 1.22.12
|
||||
semver: 7.7.4
|
||||
optionalDependencies:
|
||||
'@types/node': 24.12.2
|
||||
|
||||
'@rushstack/problem-matcher@0.2.1(@types/node@24.12.2)':
|
||||
optionalDependencies:
|
||||
'@types/node': 24.12.2
|
||||
|
||||
'@rushstack/rig-package@0.7.3':
|
||||
dependencies:
|
||||
jju: 1.4.0
|
||||
resolve: 1.22.12
|
||||
|
||||
'@rushstack/terminal@0.24.0(@types/node@24.12.2)':
|
||||
dependencies:
|
||||
'@rushstack/node-core-library': 5.23.1(@types/node@24.12.2)
|
||||
'@rushstack/problem-matcher': 0.2.1(@types/node@24.12.2)
|
||||
supports-color: 8.1.1
|
||||
optionalDependencies:
|
||||
'@types/node': 24.12.2
|
||||
|
||||
'@rushstack/ts-command-line@5.3.10(@types/node@24.12.2)':
|
||||
dependencies:
|
||||
'@rushstack/terminal': 0.24.0(@types/node@24.12.2)
|
||||
'@types/argparse': 1.0.38
|
||||
argparse: 1.0.10
|
||||
string-argv: 0.3.2
|
||||
transitivePeerDependencies:
|
||||
- '@types/node'
|
||||
|
||||
'@sapphire/async-queue@1.5.5': {}
|
||||
|
||||
'@sapphire/shapeshift@3.9.7':
|
||||
@@ -17048,6 +17186,8 @@ snapshots:
|
||||
dependencies:
|
||||
tslib: 2.8.1
|
||||
|
||||
'@types/argparse@1.0.38': {}
|
||||
|
||||
'@types/body-parser@1.19.6':
|
||||
dependencies:
|
||||
'@types/connect': 3.4.38
|
||||
@@ -17819,6 +17959,10 @@ snapshots:
|
||||
'@opentelemetry/api': 1.9.0
|
||||
zod: 4.3.6
|
||||
|
||||
ajv-draft-04@1.0.0(ajv@8.18.0):
|
||||
optionalDependencies:
|
||||
ajv: 8.18.0
|
||||
|
||||
ajv-formats@2.1.1(ajv@8.18.0):
|
||||
optionalDependencies:
|
||||
ajv: 8.18.0
|
||||
@@ -19535,6 +19679,8 @@ snapshots:
|
||||
|
||||
devtools-protocol@0.0.1612613: {}
|
||||
|
||||
diff@8.0.4: {}
|
||||
|
||||
diffie-hellman@5.0.3:
|
||||
dependencies:
|
||||
bn.js: 4.12.3
|
||||
@@ -21859,6 +22005,8 @@ snapshots:
|
||||
|
||||
jiti@2.6.1: {}
|
||||
|
||||
jju@1.4.0: {}
|
||||
|
||||
joi@17.13.3:
|
||||
dependencies:
|
||||
'@hapi/hoek': 9.3.0
|
||||
|
||||
@@ -0,0 +1,54 @@
|
||||
{
|
||||
"$schema": "https://developer.microsoft.com/json-schemas/api-extractor/v7/api-extractor.schema.json",
|
||||
|
||||
"newlineKind": "lf",
|
||||
|
||||
"compiler": {
|
||||
"overrideTsconfig": {
|
||||
"compilerOptions": {
|
||||
"module": "NodeNext",
|
||||
"moduleResolution": "NodeNext",
|
||||
"target": "ESNext",
|
||||
"lib": ["DOM", "ES2023", "ES2024", "DOM.AsyncIterable"],
|
||||
"skipLibCheck": true,
|
||||
"strict": true
|
||||
}
|
||||
}
|
||||
},
|
||||
|
||||
"apiReport": {
|
||||
"enabled": true,
|
||||
"reportFolder": "<projectFolder>/../../docs/public-api",
|
||||
"reportTempFolder": "<projectFolder>/../../docs/public-api/temp"
|
||||
},
|
||||
|
||||
"docModel": {
|
||||
"enabled": false
|
||||
},
|
||||
|
||||
"dtsRollup": {
|
||||
"enabled": false
|
||||
},
|
||||
|
||||
"tsdocMetadata": {
|
||||
"enabled": false
|
||||
},
|
||||
|
||||
"messages": {
|
||||
"compilerMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "none"
|
||||
}
|
||||
},
|
||||
"extractorMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "none"
|
||||
}
|
||||
},
|
||||
"tsdocMessageReporting": {
|
||||
"default": {
|
||||
"logLevel": "none"
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
/* eslint-disable no-console */
|
||||
import { existsSync, mkdirSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
|
||||
import { dirname, relative, resolve } from 'node:path';
|
||||
|
||||
import { Extractor, ExtractorConfig, type IConfigFile } from '@microsoft/api-extractor';
|
||||
import { globbySync } from 'globby';
|
||||
|
||||
/**
|
||||
* Generates (`--verify` to check) a per-package map of the public type-level interface of
|
||||
* each publishable `@crawlee/*` package, committed to `docs/public-api/<package>.api.md`.
|
||||
* These reports define where we promise backwards compatibility; changes must be reviewed.
|
||||
*
|
||||
* The build (`scripts/typescript_fixes.mjs`) injects `// @ts-ignore` comment lines into the
|
||||
* `.d.ts` files that crash API Extractor's AST walker, so we strip them for the duration of
|
||||
* the run and restore them afterwards. A few packages re-export such a member across a
|
||||
* package boundary and crash anyway; those are retried against a sanitized mirror of the
|
||||
* dist tree with `@crawlee/*` deps remapped via tsconfig `paths`, which dodges the bug.
|
||||
*
|
||||
* When running under GitHub Actions (or with `--github`), failures are additionally emitted
|
||||
* as workflow commands (`::error::`) so they show up as inline annotations in the CI run.
|
||||
*/
|
||||
|
||||
const root = resolve(import.meta.dirname, '..', '..');
|
||||
const baseConfigPath = resolve(import.meta.dirname, 'api-extractor.base.json');
|
||||
const baseConfig = JSON.parse(readFileSync(baseConfigPath, 'utf8')) as IConfigFile;
|
||||
const reportFolder = resolve(root, 'docs', 'public-api');
|
||||
const mirrorRoot = resolve(root, 'node_modules', '.cache', 'api-extractor-dts');
|
||||
const verify = process.argv.includes('--verify');
|
||||
|
||||
// Emit GitHub Actions workflow commands (annotations) when running in CI, so out-of-date
|
||||
// reports and crashes surface as inline warnings/errors. Opt in with `--github` or force
|
||||
// off with `--no-github` (auto-detected via the runner-set GITHUB_ACTIONS env var otherwise).
|
||||
const github = process.argv.includes('--github')
|
||||
|| (process.env.GITHUB_ACTIONS === 'true' && !process.argv.includes('--no-github'));
|
||||
|
||||
// GitHub workflow commands must escape `%`, `\r` and `\n` in the message. See
|
||||
// https://docs.github.com/en/actions/reference/workflows-and-actions/workflow-commands
|
||||
const ghEscape = (message: string) => message.replace(/%/g, '%25').replace(/\r/g, '%0D').replace(/\n/g, '%0A');
|
||||
const ghCommand = (kind: 'error' | 'warning', message: string) => {
|
||||
if (github) console.log(`::${kind}::${ghEscape(message)}`);
|
||||
};
|
||||
|
||||
const TS_IGNORE_LINE = /^\s*\/\/ @ts-ignore optional peer dependency or compatibility with es2022\s*$/;
|
||||
// CLI binary and project scaffolding are tooling, not an importable API where we promise BC.
|
||||
const EXCLUDED = new Set(['@crawlee/cli', '@crawlee/templates']);
|
||||
|
||||
interface PackageManifest {
|
||||
name: string;
|
||||
private?: boolean;
|
||||
types?: string;
|
||||
exports?: Record<string, string | { types?: string }>;
|
||||
}
|
||||
|
||||
const packageJsonPaths = globbySync('packages/*/package.json', { cwd: root, absolute: true }).sort();
|
||||
|
||||
function manifest(pkgJsonPath: string): PackageManifest {
|
||||
return JSON.parse(readFileSync(pkgJsonPath, 'utf8')) as PackageManifest;
|
||||
}
|
||||
|
||||
function dtsEntry(pkgDir: string, pkg: PackageManifest): string | undefined {
|
||||
const dot = pkg.exports?.['.'];
|
||||
const candidate = (typeof dot === 'object' ? dot.types : undefined) ?? pkg.types ?? './dist/index.d.ts';
|
||||
const full = resolve(pkgDir, candidate);
|
||||
return existsSync(full) ? full : undefined;
|
||||
}
|
||||
|
||||
const stripTsIgnore = (content: string) =>
|
||||
content
|
||||
.split('\n')
|
||||
.filter((line) => !TS_IGNORE_LINE.test(line))
|
||||
.join('\n');
|
||||
|
||||
const reportFileName = (name: string) => `${name.replace('@', '').replace('/', '-')}.api.md`;
|
||||
|
||||
/** Lazily built sanitized mirror of the dist tree, with a `@crawlee/*` -> mirror paths map. */
|
||||
let mirror: { packages: string; paths: Record<string, string[]> } | undefined;
|
||||
function getMirror() {
|
||||
if (mirror) return mirror;
|
||||
rmSync(mirrorRoot, { recursive: true, force: true });
|
||||
for (const file of globbySync('packages/*/dist/**/*.d.ts', { cwd: root, absolute: true })) {
|
||||
const target = resolve(mirrorRoot, relative(root, file));
|
||||
mkdirSync(dirname(target), { recursive: true });
|
||||
writeFileSync(target, stripTsIgnore(readFileSync(file, 'utf8')));
|
||||
}
|
||||
const packages = resolve(mirrorRoot, 'packages');
|
||||
const paths: Record<string, string[]> = {};
|
||||
for (const pkgJsonPath of packageJsonPaths) {
|
||||
const dir = resolve(packages, relative(resolve(root, 'packages'), dirname(pkgJsonPath)));
|
||||
if (existsSync(resolve(dir, 'dist/index.d.ts'))) paths[manifest(pkgJsonPath).name] = [resolve(dir, 'dist/index.d.ts')];
|
||||
}
|
||||
mirror = { packages, paths };
|
||||
return mirror;
|
||||
}
|
||||
|
||||
function extract(pkgDir: string, pkgJsonPath: string, entry: string, paths?: Record<string, string[]>) {
|
||||
const config = ExtractorConfig.prepare({
|
||||
configObjectFullPath: baseConfigPath,
|
||||
packageJsonFullPath: pkgJsonPath,
|
||||
configObject: {
|
||||
...baseConfig,
|
||||
projectFolder: pkgDir,
|
||||
mainEntryPointFilePath: entry,
|
||||
compiler: paths
|
||||
? { overrideTsconfig: { compilerOptions: { baseUrl: root, paths } } }
|
||||
: baseConfig.compiler,
|
||||
apiReport: {
|
||||
enabled: true,
|
||||
reportFileName: reportFileName(manifest(pkgJsonPath).name),
|
||||
reportFolder,
|
||||
reportTempFolder: resolve(reportFolder, 'temp'),
|
||||
},
|
||||
},
|
||||
});
|
||||
return Extractor.invoke(config, { localBuild: !verify, showVerboseMessages: false });
|
||||
}
|
||||
|
||||
function main() {
|
||||
let failed = 0;
|
||||
|
||||
// The build injects `// @ts-ignore` lines into the `.d.ts` files that crash API
|
||||
// Extractor's AST walker, so strip them for the duration of the run and restore after.
|
||||
const originals = new Map<string, string>();
|
||||
for (const file of globbySync('packages/*/dist/**/*.d.ts', { cwd: root, absolute: true })) {
|
||||
const content = readFileSync(file, 'utf8');
|
||||
if (content.includes('@ts-ignore optional peer dependency')) {
|
||||
originals.set(file, content);
|
||||
writeFileSync(file, stripTsIgnore(content));
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
for (const pkgJsonPath of packageJsonPaths) {
|
||||
const pkg = manifest(pkgJsonPath);
|
||||
if (pkg.private || EXCLUDED.has(pkg.name)) continue;
|
||||
|
||||
const pkgDir = dirname(pkgJsonPath);
|
||||
const entry = dtsEntry(pkgDir, pkg);
|
||||
if (!entry) {
|
||||
const message = `${pkg.name}: no built dist/index.d.ts — run "pnpm build" first`;
|
||||
console.error(`✗ ${message}`);
|
||||
ghCommand('error', message);
|
||||
failed++;
|
||||
continue;
|
||||
}
|
||||
|
||||
// Up to date iff the committed report didn't change. Extractor warnings are
|
||||
// diagnostics, not BC-surface changes, so we key success on apiReportChanged.
|
||||
const ok = (result: { apiReportChanged: boolean }, via = '') => {
|
||||
if (verify && result.apiReportChanged) {
|
||||
const message = `${pkg.name}: report out of date${via} — run "pnpm api:extract" and commit the changes in docs/public-api/`;
|
||||
console.error(`✗ ${pkg.name}: report out of date${via}`);
|
||||
ghCommand('error', message);
|
||||
failed++;
|
||||
} else {
|
||||
console.log(`✓ ${pkg.name}${via}`);
|
||||
}
|
||||
};
|
||||
|
||||
try {
|
||||
ok(extract(pkgDir, pkgJsonPath, entry));
|
||||
} catch {
|
||||
// Fallback: retry against the sanitized mirror (dodges an API Extractor crash
|
||||
// on cross-package re-exports of comment-injected members, e.g. @crawlee/browser).
|
||||
try {
|
||||
const { packages, paths } = getMirror();
|
||||
const mirrorEntry = resolve(packages, relative(resolve(root, 'packages'), pkgDir), relative(pkgDir, entry));
|
||||
const { [pkg.name]: _self, ...deps } = paths;
|
||||
ok(extract(pkgDir, pkgJsonPath, mirrorEntry, deps), ' (via mirror)');
|
||||
} catch (err) {
|
||||
const message = `${pkg.name}: api-extractor crashed: ${(err as Error).message}`;
|
||||
console.error(`✗ ${message}`);
|
||||
ghCommand('error', message);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
}
|
||||
} finally {
|
||||
for (const [file, content] of originals) writeFileSync(file, content);
|
||||
rmSync(mirrorRoot, { recursive: true, force: true });
|
||||
}
|
||||
|
||||
if (failed > 0) {
|
||||
if (verify) console.error('\nRun "pnpm api:extract" and commit the changes in docs/public-api/.');
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user