github-featured/apify--crawlee
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: use biome for code formatting (#2301)

Browse Source 
This takes ~50ms on my machine 🤯 

- closes #2366 
- Replacing spaces with tabs won't be done right here, right now.
- eslint and biome are reconciled
- ~biome check fails because of typescript errors - we can either fix
those or find a way to ignore it~
This commit is contained in:
Jan Buchar
2024-05-22 11:00:43 +02:00
committed by GitHub
parent 43381bfe48
commit 5db533afdb
310 changed files with 6214 additions and 4548 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.editorconfig
-2
View File
@@ -7,6 +7,4 @@ charset = utf-8
trim_trailing_whitespace = true
insert_final_newline = true
end_of_line = lf
# editorconfig-tools is unable to ignore longs strings or urls
max_line_length = null
quote_type = single
.eslintrc.json
+69 -64
View File
@@ -1,66 +1,71 @@
{
"root": true,
"env": {
"browser": true,
"es2020": true,
"node": true
},
"extends": "@apify/eslint-config-ts",
"parserOptions": {
"project": "./tsconfig.eslint.json",
"ecmaVersion": 2022
},
"ignorePatterns": [
"node_modules",
"dist",
"coverage",
"**/*.d.ts"
],
"overrides": [
{
"plugins": [
"@typescript-eslint"
],
"files": [
"*.ts"
],
"rules": {
"@typescript-eslint/array-type": "error",
"@typescript-eslint/ban-ts-comment": 0,
"@typescript-eslint/consistent-type-imports": ["error", {
"disallowTypeAnnotations": false
}],
"@typescript-eslint/consistent-type-definitions": ["error", "interface"],
"@typescript-eslint/member-delimiter-style": ["error", {
"multiline": { "delimiter": "semi", "requireLast": true },
"singleline": { "delimiter": "semi", "requireLast": false }
}],
"@typescript-eslint/no-empty-interface": "off",
"no-empty-function": "off",
"@typescript-eslint/no-empty-function": "off",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-floating-promises": "error",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/comma-dangle": ["error", "always-multiline"]
}
}
],
"rules": {
"quote-props": ["error", "consistent"],
"import/no-extraneous-dependencies": "off",
"max-classes-per-file": 0,
"no-console": "error",
"no-underscore-dangle": 0,
"no-void": 0,
"max-len": ["error", {
"code": 160,
"ignoreUrls": true,
"ignoreComments": true
}],
"import/order": ["error", {
"groups": ["builtin", "external", ["parent", "sibling"], "index", "object"],
"alphabetize": { "order": "asc", "caseInsensitive": true },
"newlines-between": "always"
}]
}
"root": true,
"env": {
"browser": true,
"es2020": true,
"node": true
},
"extends": ["@apify/eslint-config-ts", "prettier"],
"parserOptions": {
"project": "./tsconfig.eslint.json",
"ecmaVersion": 2022
},
"ignorePatterns": ["node_modules", "dist", "coverage", "**/*.d.ts"],
"overrides": [
{
"plugins": ["@typescript-eslint"],
"files": ["*.ts"],
"rules": {
"@typescript-eslint/array-type": "error",
"@typescript-eslint/ban-ts-comment": 0,
"@typescript-eslint/consistent-type-imports": [
"error",
{
"disallowTypeAnnotations": false
}
],
"@typescript-eslint/consistent-type-definitions": [
"error",
"interface"
],
"@typescript-eslint/member-delimiter-style": [
"error",
{
"multiline": { "delimiter": "semi", "requireLast": true },
"singleline": { "delimiter": "semi", "requireLast": false }
}
],
"@typescript-eslint/no-empty-interface": "off",
"no-empty-function": "off",
"@typescript-eslint/no-empty-function": "off",
"@typescript-eslint/no-explicit-any": "off",
"@typescript-eslint/no-floating-promises": "error",
"@typescript-eslint/no-unused-vars": "off",
"@typescript-eslint/comma-dangle": ["error", "always-multiline"]
}
}
],
"rules": {
"quote-props": ["error", "consistent"],
"import/no-extraneous-dependencies": "off",
"max-classes-per-file": 0,
"no-console": "error",
"no-underscore-dangle": 0,
"no-void": 0,
"max-len": "off",
"import/order": [
"error",
{
"groups": [
"builtin",
"external",
["parent", "sibling"],
"index",
"object"
],
"alphabetize": { "order": "asc", "caseInsensitive": true },
"newlines-between": "always"
}
]
}
}
.github/workflows/test-ci.yml
+3
View File
@@ -150,6 +150,9 @@ jobs:
- name: ESLint
run: yarn lint
- name: Biome format
run: yarn format:check
release_next:
name: Release @next
if: github.event_name == 'push' && contains(github.event.ref, 'master') && (!contains(github.event.head_commit.message, '[skip ci]') && !contains(github.event.head_commit.message, 'docs:'))
biome.json
+18
View File
@@ -0,0 +1,18 @@
{
"formatter": {
"ignore": ["website/**", "packages/*/dist/**", "package.json"],
"formatWithErrors": true
},
"javascript": {
"formatter": {
"quoteStyle": "single",
"semicolons": "always",
"trailingComma": "all",
"lineWidth": 120,
"indentStyle": "space",
"indentWidth": 4,
"quoteProperties": "preserve",
"lineEnding": "lf"
}
}
}
docs/examples/.eslintrc.json
+11 -11
View File
@@ -1,13 +1,13 @@
{
"root": true,
"extends": "@apify/ts",
"parserOptions": {
"project": "./tsconfig.eslint.json",
"ecmaVersion": 2022
},
"rules": {
"import/extensions": 0,
"import/no-extraneous-dependencies": 0,
"no-console": "off"
}
"root": true,
"extends": "@apify/ts",
"parserOptions": {
"project": "./tsconfig.eslint.json",
"ecmaVersion": 2022
},
"rules": {
"import/extensions": 0,
"import/no-extraneous-dependencies": 0,
"no-console": "off"
}
}
docs/examples/cheerio_crawler.ts
+1 -3
View File
@@ -57,8 +57,6 @@ const crawler = new CheerioCrawler({
});
// Run the crawler and wait for it to finish.
await crawler.run([
'https://crawlee.dev',
]);
await crawler.run(['https://crawlee.dev']);
log.debug('Crawler finished.');
docs/examples/crawl_multiple_urls_cheerio.ts
+1 -5
View File
@@ -9,8 +9,4 @@ const crawler = new CheerioCrawler({
});
// Run the crawler with initial request
await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);
docs/examples/crawl_multiple_urls_playwright.ts
+1 -5
View File
@@ -9,8 +9,4 @@ const crawler = new PlaywrightCrawler({
});
// Run the crawler with initial request
await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);
docs/examples/crawl_multiple_urls_puppeteer.ts
+1 -5
View File
@@ -9,8 +9,4 @@ const crawler = new PuppeteerCrawler({
});
// Run the crawler with initial request
await crawler.run([
'http://www.example.com/page-1',
'http://www.example.com/page-2',
'http://www.example.com/page-3',
]);
await crawler.run(['http://www.example.com/page-1', 'http://www.example.com/page-2', 'http://www.example.com/page-3']);
docs/examples/forms.ts
+2 -2
View File
@@ -23,8 +23,8 @@ await Promise.all([
]);
// Obtain and print list of search results
const results = await page.$$eval('[data-testid="results-list"] div.search-title > a',
(nodes) => nodes.map((node) => ({
const results = await page.$$eval('[data-testid="results-list"] div.search-title > a', (nodes) =>
nodes.map((node) => ({
url: node.href,
name: node.innerText,
})),
docs/examples/http_crawler.ts
+2 -4
View File
@@ -36,7 +36,7 @@ const crawler = new HttpCrawler({
// the data will be stored as JSON files in ./storage/datasets/default
await Dataset.pushData({
url: request.url, // URL of the page
body, // HTML code of the page
body, // HTML code of the page
});
},
@@ -48,8 +48,6 @@ const crawler = new HttpCrawler({
// Run the crawler and wait for it to finish.
// It will crawl a list of URLs from an external file, load each URL using a plain HTTP request, and save HTML
await crawler.run([
'https://crawlee.dev',
]);
await crawler.run(['https://crawlee.dev']);
log.debug('Crawler finished.');
docs/examples/jsdom_crawler.ts
+1 -3
View File
@@ -57,8 +57,6 @@ const crawler = new JSDOMCrawler({
});
// Run the crawler and wait for it to finish.
await crawler.run([
'https://crawlee.dev',
]);
await crawler.run(['https://crawlee.dev']);
log.debug('Crawler finished.');
docs/examples/jsdom_crawler_react.ts
+5 -7
View File
@@ -1,9 +1,9 @@
import { JSDOMCrawler, log } from 'crawlee';
// Create an instance of the JSDOMCrawler class - crawler that automatically
// Create an instance of the JSDOMCrawler class - crawler that automatically
// loads the URLs and parses their HTML using the jsdom library.
const crawler = new JSDOMCrawler({
// Setting the `runScripts` option to `true` allows the crawler to execute client-side
// Setting the `runScripts` option to `true` allows the crawler to execute client-side
// JavaScript code on the page. This is required for some websites (such as the React application in this example), but may pose a security risk.
runScripts: true,
// This function will be called for each crawled URL.
@@ -18,15 +18,13 @@ const crawler = new JSDOMCrawler({
document.querySelectorAll('button')[18].click(); // =
const result = document.querySelectorAll('.component-display')[0].childNodes[0] as Element;
// The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,
// The result is passed to the console. Unlike with Playwright or Puppeteer crawlers,
// this console call goes to the Node.js console, not the browser console. All the code here runs right in Node.js!
log.info(result.innerHTML); // 2
},
});
// Run the crawler and wait for it to finish.
await crawler.run([
'https://ahfarmer.github.io/calculator/',
]);
await crawler.run(['https://ahfarmer.github.io/calculator/']);
log.debug('Crawler finished.');
log.debug('Crawler finished.');
docs/examples/map.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { Dataset, KeyValueStore } from 'crawlee';
const dataset = await Dataset.open<{
url: string,
headingCount: number,
url: string;
headingCount: number;
}>();
// Seeding the dataset with some data
docs/examples/reduce.ts
+2 -2
View File
@@ -1,8 +1,8 @@
import { Dataset, KeyValueStore } from 'crawlee';
const dataset = await Dataset.open<{
url: string,
headingCount: number,
url: string;
headingCount: number;
}>();
// Seeding the dataset with some data
docs/examples/tsconfig.json
+19 -19
View File
@@ -1,21 +1,21 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
"baseUrl": "../..",
"paths": {
"crawlee": ["packages/crawlee/src"],
"@crawlee/basic": ["packages/basic-crawler/src"],
"@crawlee/browser": ["packages/browser-crawler/src"],
"@crawlee/http": ["packages/http-crawler/src"],
"@crawlee/linkedom": ["packages/linkedom-crawler/src"],
"@crawlee/jsdom": ["packages/jsdom-crawler/src"],
"@crawlee/cheerio": ["packages/cheerio-crawler/src"],
"@crawlee/playwright": ["packages/playwright-crawler/src"],
"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],
"@crawlee/*": ["packages/*/src"]
},
"target": "ES2022",
"module": "ES2022",
"noUnusedParameters": false
}
"extends": "@apify/tsconfig",
"compilerOptions": {
"baseUrl": "../..",
"paths": {
"crawlee": ["packages/crawlee/src"],
"@crawlee/basic": ["packages/basic-crawler/src"],
"@crawlee/browser": ["packages/browser-crawler/src"],
"@crawlee/http": ["packages/http-crawler/src"],
"@crawlee/linkedom": ["packages/linkedom-crawler/src"],
"@crawlee/jsdom": ["packages/jsdom-crawler/src"],
"@crawlee/cheerio": ["packages/cheerio-crawler/src"],
"@crawlee/playwright": ["packages/playwright-crawler/src"],
"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],
"@crawlee/*": ["packages/*/src"]
},
"target": "ES2022",
"module": "ES2022",
"noUnusedParameters": false
}
}
docs/guides/avoid_blocking_playwright.ts
+7 -9
View File
@@ -6,16 +6,14 @@ const crawler = new PlaywrightCrawler({
useFingerprints: true, // this is the default
fingerprintOptions: {
fingerprintGeneratorOptions: {
browsers: [{
name: BrowserName.edge,
minVersion: 96,
}],
devices: [
DeviceCategory.desktop,
],
operatingSystems: [
OperatingSystemsName.windows,
browsers: [
{
name: BrowserName.edge,
minVersion: 96,
},
],
devices: [DeviceCategory.desktop],
operatingSystems: [OperatingSystemsName.windows],
},
},
},
docs/guides/avoid_blocking_puppeteer.ts
+3 -10
View File
@@ -6,16 +6,9 @@ const crawler = new PuppeteerCrawler({
useFingerprints: true, // this is the default
fingerprintOptions: {
fingerprintGeneratorOptions: {
browsers: [
BrowserName.chrome,
BrowserName.firefox,
],
devices: [
DeviceCategory.mobile,
],
locales: [
'en-US',
],
browsers: [BrowserName.chrome, BrowserName.firefox],
devices: [DeviceCategory.mobile],
locales: ['en-US'],
},
},
},
docs/guides/parallel-scraping/modified-detail-route.mjs
+1 -3
View File
@@ -5,9 +5,7 @@ router.addHandler('DETAIL', async ({ request, page, log }) => {
const manufacturer = urlPart[0].split('-')[0]; // 'sennheiser'
const title = await page.locator('.product-meta h1').textContent();
const sku = await page
.locator('span.product-meta__sku-number')
.textContent();
const sku = await page.locator('span.product-meta__sku-number').textContent();
const priceElement = page
.locator('span.price')
docs/guides/parallel-scraping/parallel-scraper.mjs
+26 -21
View File
@@ -50,12 +50,14 @@ if (!process.env.IN_WORKER_THREAD) {
await Dataset.pushData(data);
});
promises.push(new Promise((resolve) => {
proc.once('exit', (code, signal) => {
log.info(`Process ${i} exited with code ${code} and signal ${signal}`);
resolve();
});
}));
promises.push(
new Promise((resolve) => {
proc.once('exit', (code, signal) => {
log.info(`Process ${i} exited with code ${code} and signal ${signal}`);
resolve();
});
}),
);
}
await Promise.all(promises);
@@ -86,22 +88,25 @@ if (!process.env.IN_WORKER_THREAD) {
});
workerLogger.debug('Setting up crawler.');
const crawler = new PlaywrightCrawler({
log: workerLogger,
// Instead of the long requestHandler with
// if clauses we provide a router instance.
requestHandler: router,
// Enable the request locking experiment so that we can actually use the queue.
// highlight-start
experiments: {
requestLocking: true,
const crawler = new PlaywrightCrawler(
{
log: workerLogger,
// Instead of the long requestHandler with
// if clauses we provide a router instance.
requestHandler: router,
// Enable the request locking experiment so that we can actually use the queue.
// highlight-start
experiments: {
requestLocking: true,
},
// Provide the request queue we've pre-filled in previous steps
requestQueue,
// highlight-end
// Let's also limit the crawler's concurrency, we don't want to overload a single process 🐌
maxConcurrency: 5,
},
// Provide the request queue we've pre-filled in previous steps
requestQueue,
// highlight-end
// Let's also limit the crawler's concurrency, we don't want to overload a single process 🐌
maxConcurrency: 5,
}, config);
config,
);
await crawler.run();
}
docs/guides/proxy_management_inspection_cheerio.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new CheerioCrawler({
proxyConfiguration,
docs/guides/proxy_management_inspection_http.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { HttpCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new HttpCrawler({
proxyConfiguration,
docs/guides/proxy_management_inspection_jsdom.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new JSDOMCrawler({
proxyConfiguration,
docs/guides/proxy_management_inspection_playwright.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PlaywrightCrawler({
proxyConfiguration,
docs/guides/proxy_management_inspection_puppeteer.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PuppeteerCrawler({
proxyConfiguration,
docs/guides/proxy_management_integration_cheerio.ts
+1 -4
View File
@@ -1,10 +1,7 @@
import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: [
'http://proxy-1.com',
'http://proxy-2.com',
],
proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],
});
const crawler = new CheerioCrawler({
docs/guides/proxy_management_integration_http.ts
+1 -4
View File
@@ -1,10 +1,7 @@
import { HttpCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: [
'http://proxy-1.com',
'http://proxy-2.com',
],
proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],
});
const crawler = new HttpCrawler({
docs/guides/proxy_management_integration_jsdom.ts
+1 -4
View File
@@ -1,10 +1,7 @@
import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: [
'http://proxy-1.com',
'http://proxy-2.com',
],
proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],
});
const crawler = new JSDOMCrawler({
docs/guides/proxy_management_integration_playwright.ts
+1 -4
View File
@@ -1,10 +1,7 @@
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: [
'http://proxy-1.com',
'http://proxy-2.com',
],
proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],
});
const crawler = new PlaywrightCrawler({
docs/guides/proxy_management_integration_puppeteer.ts
+1 -4
View File
@@ -1,10 +1,7 @@
import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({
proxyUrls: [
'http://proxy-1.com',
'http://proxy-2.com',
],
proxyUrls: ['http://proxy-1.com', 'http://proxy-2.com'],
});
const crawler = new PuppeteerCrawler({
docs/guides/proxy_management_session_cheerio.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new CheerioCrawler({
useSessionPool: true,
docs/guides/proxy_management_session_http.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { HttpCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new HttpCrawler({
useSessionPool: true,
docs/guides/proxy_management_session_jsdom.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new JSDOMCrawler({
useSessionPool: true,
docs/guides/proxy_management_session_playwright.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PlaywrightCrawler({
useSessionPool: true,
docs/guides/proxy_management_session_puppeteer.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PuppeteerCrawler({
useSessionPool: true,
docs/guides/proxy_management_session_standalone.ts
+6 -2
View File
@@ -1,8 +1,12 @@
import { ProxyConfiguration, SessionPool } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const sessionPool = await SessionPool.open({ /* opts */ });
const sessionPool = await SessionPool.open({
/* opts */
});
const session = await sessionPool.getSession();
docs/guides/session_management_basic.ts
+3 -1
View File
@@ -1,7 +1,9 @@
import { BasicCrawler, ProxyConfiguration } from 'crawlee';
import { gotScraping } from 'got-scraping';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new BasicCrawler({
// Activates the Session pool (default is true).
docs/guides/session_management_cheerio.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { CheerioCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new CheerioCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
docs/guides/session_management_http.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { HttpCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new HttpCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
docs/guides/session_management_jsdom.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { JSDOMCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new JSDOMCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
docs/guides/session_management_playwright.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PlaywrightCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PlaywrightCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
docs/guides/session_management_puppeteer.ts
+3 -1
View File
@@ -1,6 +1,8 @@
import { PuppeteerCrawler, ProxyConfiguration } from 'crawlee';
const proxyConfiguration = new ProxyConfiguration({ /* opts */ });
const proxyConfiguration = new ProxyConfiguration({
/* opts */
});
const crawler = new PuppeteerCrawler({
// To use the proxy IP session rotation logic, you must turn the proxy usage on.
docs/introduction/03-filter-without-el.ts
+1 -3
View File
@@ -14,9 +14,7 @@ const crawler = new CheerioCrawler({
// Besides resolving the URLs, we now also need to
// grab their hostname for filtering.
const { hostname } = new URL(request.loadedUrl);
const absoluteUrls = links.map(
(link) => new URL(link, request.loadedUrl),
);
const absoluteUrls = links.map((link) => new URL(link, request.loadedUrl));
// We use the hostname to filter links that point
// to a different domain, even subdomain.
docs/introduction/03-find-without-el.ts
+1 -3
View File
@@ -17,9 +17,7 @@ const crawler = new CheerioCrawler({
// Then we need to resolve relative URLs,
// otherwise they would be unusable for crawling.
const absoluteUrls = links.map(
(link) => new URL(link, request.loadedUrl).href,
);
const absoluteUrls = links.map((link) => new URL(link, request.loadedUrl).href);
// Finally, we have to add the URLs to the queue
await crawler.addRequests(absoluteUrls);
docs/introduction/tsconfig.json
+19 -19
View File
@@ -1,21 +1,21 @@
{
"extends": "@apify/tsconfig",
"compilerOptions": {
"baseUrl": "../..",
"paths": {
"crawlee": ["packages/crawlee/src"],
"@crawlee/basic": ["packages/basic-crawler/src"],
"@crawlee/browser": ["packages/browser-crawler/src"],
"@crawlee/http": ["packages/http-crawler/src"],
"@crawlee/linkedom": ["packages/linkedom-crawler/src"],
"@crawlee/jsdom": ["packages/jsdom-crawler/src"],
"@crawlee/cheerio": ["packages/cheerio-crawler/src"],
"@crawlee/playwright": ["packages/playwright-crawler/src"],
"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],
"@crawlee/*": ["packages/*/src"]
},
"target": "ES2022",
"module": "ES2022",
"noUnusedParameters": false
}
"extends": "@apify/tsconfig",
"compilerOptions": {
"baseUrl": "../..",
"paths": {
"crawlee": ["packages/crawlee/src"],
"@crawlee/basic": ["packages/basic-crawler/src"],
"@crawlee/browser": ["packages/browser-crawler/src"],
"@crawlee/http": ["packages/http-crawler/src"],
"@crawlee/linkedom": ["packages/linkedom-crawler/src"],
"@crawlee/jsdom": ["packages/jsdom-crawler/src"],
"@crawlee/cheerio": ["packages/cheerio-crawler/src"],
"@crawlee/playwright": ["packages/playwright-crawler/src"],
"@crawlee/puppeteer": ["packages/puppeteer-crawler/src"],
"@crawlee/*": ["packages/*/src"]
},
"target": "ES2022",
"module": "ES2022",
"noUnusedParameters": false
}
}
lerna.json
+15 -20
View File
@@ -1,22 +1,17 @@
{
"packages": [
"packages/*"
],
"version": "3.10.0",
"command": {
"version": {
"conventionalCommits": true,
"createRelease": "github",
"message": "chore(release): %s"
},
"publish": {
"assets": []
}
},
"npmClient": "yarn",
"useNx": false,
"ignoreChanges": [
"**/test/**",
"**/*.md"
]
"packages": ["packages/*"],
"version": "3.10.0",
"command": {
"version": {
"conventionalCommits": true,
"createRelease": "github",
"message": "chore(release): %s"
},
"publish": {
"assets": []
}
},
"npmClient": "yarn",
"useNx": false,
"ignoreChanges": ["**/test/**", "**/*.md"]
}
package.json
+5 -1
View File
@@ -48,12 +48,15 @@
"release:prod": "yarn build && yarn publish:prod",
"release:pin-versions": "turbo run copy -- -- --pin-versions",
"lint": "eslint \"packages/**/*.ts\" \"test/**/*.ts\"",
"lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix"
"lint:fix": "eslint \"packages/**/*.ts\" \"test/**/*.ts\" --fix",
"format": "biome format --write .",
"format:check": "biome format ."
},
"devDependencies": {
"@apify/eslint-config-ts": "^0.4.0",
"@apify/log": "^2.4.0",
"@apify/tsconfig": "^0.1.0",
"@biomejs/biome": "^1.6.1",
"@commitlint/config-conventional": "^19.0.0",
"@playwright/browser-chromium": "1.44.0",
"@playwright/browser-firefox": "1.44.0",
@@ -88,6 +91,7 @@
"cross-env": "^7.0.3",
"deep-equal": "^2.0.5",
"eslint": "^8.52.0",
"eslint-config-prettier": "^9.1.0",
"express": "^4.18.1",
"fs-extra": "^11.0.0",
"gen-esm-wrapper": "^1.1.3",
packages/basic-crawler/src/internals/basic-crawler.ts
+188 -115
View File
@@ -59,9 +59,8 @@ import ow, { ArgumentError } from 'ow';
import { getDomain } from 'tldts';
import type { SetRequired } from 'type-fest';
export interface BasicCrawlingContext<
UserData extends Dictionary = Dictionary,
> extends CrawlingContext<BasicCrawler, UserData> {
export interface BasicCrawlingContext<UserData extends Dictionary = Dictionary>
extends CrawlingContext<BasicCrawler, UserData> {
/**
* This function automatically finds and enqueues links from the current page, adding them to the {@apilink RequestQueue}
* currently used by the crawler.
@@ -99,9 +98,14 @@ export interface BasicCrawlingContext<
*/
const SAFE_MIGRATION_WAIT_MILLIS = 20000;
export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context) => Awaitable<void>;
export type RequestHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
inputs: Context,
) => Awaitable<void>;
export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (inputs: Context, error: Error) => Awaitable<void>;
export type ErrorHandler<Context extends CrawlingContext = BasicCrawlingContext> = (
inputs: Context,
error: Error,
) => Awaitable<void>;
export interface StatusMessageCallbackParams<
Context extends CrawlingContext = BasicCrawlingContext,
@@ -542,7 +546,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
/**
* All `BasicCrawler` parameters are passed via an options object.
*/
constructor(options: BasicCrawlerOptions<Context> = {}, readonly config = Configuration.getGlobalConfig()) {
constructor(
options: BasicCrawlerOptions<Context> = {},
readonly config = Configuration.getGlobalConfig(),
) {
ow(options, 'BasicCrawlerOptions', ow.object.exactShape(BasicCrawler.optionsShape));
const {
@@ -643,7 +650,8 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const tryEnv = (val?: string) => (val == null ? null : +val);
// allow at least 5min for internal timeouts
this.internalTimeoutMillis = tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
this.internalTimeoutMillis =
tryEnv(process.env.CRAWLEE_INTERNAL_TIMEOUT) ?? Math.max(this.requestHandlerTimeoutMillis * 2, 300e3);
// override the default internal timeout of request queue to respect `requestHandlerTimeoutMillis`
if (this.requestQueue) {
@@ -657,7 +665,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
this.sameDomainDelayMillis = sameDomainDelaySecs * 1000;
this.maxSessionRotations = maxSessionRotations;
this.handledRequestsCount = 0;
this.stats = new Statistics({ logMessage: `${log.getOptions().prefix} request statistics:`, config, ...statisticsOptions });
this.stats = new Statistics({
logMessage: `${log.getOptions().prefix} request statistics:`,
config,
...statisticsOptions,
});
this.sessionPoolOptions = {
...sessionPoolOptions,
log,
@@ -665,8 +677,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (this.retryOnBlocked) {
this.sessionPoolOptions.blockedStatusCodes = sessionPoolOptions.blockedStatusCodes ?? [];
if (this.sessionPoolOptions.blockedStatusCodes.length !== 0) {
// eslint-disable-next-line max-len
log.warning(`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`);
log.warning(
`Both 'blockedStatusCodes' and 'retryOnBlocked' are set. Please note that the 'retryOnBlocked' feature might not work as expected.`,
);
}
}
this.useSessionPool = useSessionPool;
@@ -674,8 +687,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const maxSignedInteger = 2 ** 31 - 1;
if (this.requestHandlerTimeoutMillis > maxSignedInteger) {
log.warning(`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}`
+ ` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`);
log.warning(
`requestHandlerTimeoutMillis ${this.requestHandlerTimeoutMillis}` +
` does not fit a signed 32-bit integer. Limiting the value to ${maxSignedInteger}`,
);
this.requestHandlerTimeoutMillis = maxSignedInteger;
}
@@ -700,8 +715,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
isTaskReadyFunction: async () => {
if (isMaxPagesExceeded()) {
if (shouldLogMaxPagesExceeded) {
log.info('Crawler reached the maxRequestsPerCrawl limit of '
+ `${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`);
log.info(
'Crawler reached the maxRequestsPerCrawl limit of ' +
`${maxRequestsPerCrawl} requests and will shut down soon. Requests that are in progress will be allowed to finish.`,
);
shouldLogMaxPagesExceeded = false;
}
return false;
@@ -711,9 +728,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
},
isFinishedFunction: async () => {
if (isMaxPagesExceeded()) {
log.info(`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests `
+ 'and all requests that were in progress at that time have now finished. '
+ `In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`);
log.info(
`Earlier, the crawler reached the maxRequestsPerCrawl limit of ${maxRequestsPerCrawl} requests ` +
'and all requests that were in progress at that time have now finished. ' +
`In total, the crawler processed ${this.handledRequestsCount} requests and will shut down.`,
);
return true;
}
@@ -723,7 +742,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (isFinished) {
const reason = isFinishedFunction
? 'Crawler\'s custom isFinishedFunction() returned true, the crawler will shut down.'
? "Crawler's custom isFinishedFunction() returned true, the crawler will shut down."
: 'All requests from the queue have been processed, the crawler will shut down.';
log.info(reason);
}
@@ -759,8 +778,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* This method is periodically called by the crawler, every `statusMessageLoggingInterval` seconds.
*/
async setStatusMessage(message: string, options: SetStatusMessageOptions = {}) {
const data = options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
this.log.internal(LogLevel[options.level as 'DEBUG' ?? 'DEBUG'], message, data);
const data =
options.isStatusMessageTerminal != null ? { terminal: options.isStatusMessageTerminal } : undefined;
this.log.internal(LogLevel[(options.level as 'DEBUG') ?? 'DEBUG'], message, data);
const client = this.config.getStorageClient();
@@ -797,15 +817,23 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
let message: string;
if (operationMode === 'ERROR') {
// eslint-disable-next-line max-len
message = `Experiencing problems, ${this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
message = `Experiencing problems, ${
this.stats.state.requestsFailed - previousState.requestsFailed || this.stats.state.requestsFailed
} failed requests in the past ${this.statusMessageLoggingInterval} seconds.`;
} else {
const total = this.requestQueue?.getTotalCount() || this.requestList?.length();
message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${this.stats.state.requestsFailed} failed requests.`;
message = `Crawled ${this.stats.state.requestsFinished}${total ? `/${total}` : ''} pages, ${
this.stats.state.requestsFailed
} failed requests.`;
}
if (this.statusMessageCallback) {
return this.statusMessageCallback({ crawler: this as any, state: this.stats.state, previousState, message });
return this.statusMessageCallback({
crawler: this as any,
state: this.stats.state,
previousState,
message,
});
}
await this.setStatusMessage(message);
@@ -825,7 +853,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
*/
async run(requests?: (string | Request | RequestOptions)[], options?: CrawlerRunOptions): Promise<FinalStatistics> {
if (this.running) {
throw new Error('This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.');
throw new Error(
'This crawler instance is already running, you can add more requests to it via `crawler.addRequests()`.',
);
}
const purgeRequestQueue = options?.purgeRequestQueue ?? true;
@@ -859,7 +889,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await this.setStatusMessage('Starting the crawler.', { level: 'INFO' });
const sigintHandler = async () => {
this.log.warning('Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start');
this.log.warning(
'Pausing... Press CTRL+C again to force exit. To resume, do: CRAWLEE_PURGE_ON_START=0 npm start',
);
await this._pauseOnMigration();
await this.autoscaledPool!.abort();
};
@@ -914,8 +946,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
periodicLogger.stop();
// eslint-disable-next-line max-len
await this.setStatusMessage(`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${this.stats.state.requestsFinished} succeeded, ${this.stats.state.requestsFailed} failed.`, { isStatusMessageTerminal: true, level: 'INFO' });
await this.setStatusMessage(
`Finished! Total ${this.stats.state.requestsFinished + this.stats.state.requestsFailed} requests: ${
this.stats.state.requestsFinished
} succeeded, ${this.stats.state.requestsFailed} failed.`,
{ isStatusMessageTerminal: true, level: 'INFO' },
);
this.running = false;
this.hasFinishedBefore = true;
@@ -924,8 +960,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
async getRequestQueue() {
if (!this.requestQueue && this.requestList) {
// eslint-disable-next-line max-len
this.log.warningOnce('When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.');
this.log.warningOnce(
'When using RequestList and RequestQueue at the same time, you should instantiate both explicitly and provide them in the crawler options, to ensure correctly handled restarts of the crawler.',
);
}
this.requestQueue ??= await this._getRequestQueue();
@@ -947,7 +984,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* @param requests The requests to add
* @param options Options for the request queue
*/
async addRequests(requests: (string | Source)[], options: CrawlerAddRequestsOptions = {}): Promise<CrawlerAddRequestsResult> {
async addRequests(
requests: (string | Source)[],
options: CrawlerAddRequestsOptions = {},
): Promise<CrawlerAddRequestsResult> {
const requestQueue = await this.getRequestQueue();
return requestQueue.addRequestsBatched(requests, options);
}
@@ -987,7 +1027,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
if (!format) {
throw new Error(`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`);
throw new Error(
`Failed to infer format from the path: '${path}'. Supported formats: ${supportedFormats.join(', ')}`,
);
}
if (!supportedFormats.includes(format)) {
@@ -998,10 +1040,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const items = await dataset.export(options);
if (format === 'csv') {
const value = stringify([
Object.keys(items[0]),
...items.map((item) => Object.values(item)),
]);
const value = stringify([Object.keys(items[0]), ...items.map((item) => Object.values(item))]);
await ensureDir(dirname(path));
await writeFile(path, value);
this.log.info(`Export to ${path} finished!`);
@@ -1054,38 +1093,40 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected async _pauseOnMigration() {
if (this.autoscaledPool) {
// if run wasn't called, this is going to crash
await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS)
.catch((err) => {
if (err.message.includes('running tasks did not finish')) {
this.log.error('The crawler was paused due to migration to another host, '
+ 'but some requests did not finish in time. Those requests\' results may be duplicated.');
} else {
throw err;
}
});
await this.autoscaledPool.pause(SAFE_MIGRATION_WAIT_MILLIS).catch((err) => {
if (err.message.includes('running tasks did not finish')) {
this.log.error(
'The crawler was paused due to migration to another host, ' +
"but some requests did not finish in time. Those requests' results may be duplicated.",
);
} else {
throw err;
}
});
}
const requestListPersistPromise = (async () => {
if (this.requestList) {
if (await this.requestList.isFinished()) return;
await this.requestList.persistState()
.catch((err) => {
if (err.message.includes('Cannot persist state.')) {
this.log.error('The crawler attempted to persist its request list\'s state and failed due to missing or '
+ 'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList '
+ 'constructor to ensure your crawling state is persisted through host migrations and restarts.');
} else {
this.log.exception(err, 'An unexpected error occurred when the crawler '
+ 'attempted to persist its request list\'s state.');
}
});
await this.requestList.persistState().catch((err) => {
if (err.message.includes('Cannot persist state.')) {
this.log.error(
"The crawler attempted to persist its request list's state and failed due to missing or " +
'invalid config. Make sure to use either RequestList.open() or the "stateKeyPrefix" option of RequestList ' +
'constructor to ensure your crawling state is persisted through host migrations and restarts.',
);
} else {
this.log.exception(
err,
'An unexpected error occurred when the crawler ' +
"attempted to persist its request list's state.",
);
}
});
}
})();
await Promise.all([
requestListPersistPromise,
this.stats.persistState(),
]);
await Promise.all([requestListPersistPromise, this.stats.persistState()]);
}
/**
@@ -1103,7 +1144,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
} catch (err) {
// If requestQueue.addRequest() fails here then we must reclaim it back to
// the RequestList because probably it's not yet in the queue!
this.log.error('Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.', { request });
this.log.error(
'Adding of request from the RequestList to the RequestQueue failed, reclaiming request back to the list.',
{ request },
);
await this.requestList.reclaimRequest(request);
return null;
}
@@ -1115,9 +1159,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* Executed when `errorHandler` finishes or the request is successful.
* Can be used to clean up orphaned browser pages.
*/
protected async _cleanupContext(
_crawlingContext: Context,
) {}
protected async _cleanupContext(_crawlingContext: Context) {}
/**
* Delays processing of the request based on the `sameDomainDelaySecs` option,
@@ -1134,7 +1176,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const now = Date.now();
const lastAccessTime = this.domainAccessedTime.get(domain);
if (!lastAccessTime || (now - lastAccessTime) >= this.sameDomainDelayMillis) {
if (!lastAccessTime || now - lastAccessTime >= this.sameDomainDelayMillis) {
this.domainAccessedTime.set(domain, now);
return false;
}
@@ -1142,7 +1184,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// eslint-disable-next-line dot-notation
source['inProgress'].delete(request.id!);
const delay = lastAccessTime + this.sameDomainDelayMillis - now;
this.log.debug(`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`);
this.log.debug(
`Request ${request.url} (${request.id}) will be reclaimed after ${delay} milliseconds due to same domain delay`,
);
setTimeout(async () => {
this.log.debug(`Adding request ${request.url} (${request.id}) back to the queue`);
// eslint-disable-next-line dot-notation
@@ -1158,7 +1202,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* then retries them in a case of an error, etc.
*/
protected async _runTaskFunction() {
const source = this.requestQueue || this.requestList || await this.getRequestQueue();
const source = this.requestQueue || this.requestList || (await this.getRequestQueue());
let request: Request | null | undefined;
let session: Session | undefined;
@@ -1215,11 +1259,13 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
addRequests: this.addRequests.bind(this),
pushData: this.pushData.bind(this),
sendRequest: async (overrideOptions?: OptionsInit) => {
const cookieJar = session ? {
getCookieString: async (url: string) => session!.getCookieString(url),
setCookie: async (rawCookie: string, url: string) => session!.setCookie(rawCookie, url),
...overrideOptions?.cookieJar,
} : overrideOptions?.cookieJar;
const cookieJar = session
? {
getCookieString: async (url: string) => session!.getCookieString(url),
setCookie: async (rawCookie: string, url: string) => session!.setCookie(rawCookie, url),
...overrideOptions?.cookieJar,
}
: overrideOptions?.cookieJar;
return gotScraping({
url: request!.url,
@@ -1253,7 +1299,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await this._timeoutAndRetry(
async () => source.markRequestHandled(request!),
this.internalTimeoutMillis,
`Marking request ${request.url} (${request.id}) as handled timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
`Marking request ${request.url} (${request.id}) as handled timed out after ${
this.internalTimeoutMillis / 1e3
} seconds.`,
);
this.stats.finishJob(statisticsId);
@@ -1268,17 +1316,25 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
await addTimeoutToPromise(
async () => this._requestFunctionErrorHandler(err as Error, crawlingContext, source),
this.internalTimeoutMillis,
`Handling request failure of ${request.url} (${request.id}) timed out after ${this.internalTimeoutMillis / 1e3} seconds.`,
`Handling request failure of ${request.url} (${request.id}) timed out after ${
this.internalTimeoutMillis / 1e3
} seconds.`,
);
request.state = RequestState.DONE;
} catch (secondaryError: any) {
if (!secondaryError.triggeredFromUserHandler
if (
!secondaryError.triggeredFromUserHandler &&
// avoid reprinting the same critical error multiple times, as it will be printed by Nodejs at the end anyway
&& !(secondaryError instanceof CriticalError)) {
!(secondaryError instanceof CriticalError)
) {
const apifySpecific = process.env.APIFY_IS_AT_HOME
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.` : '';
this.log.exception(secondaryError as Error, 'An exception occurred during handling of failed request. '
+ `This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`);
? `This may have happened due to an internal error of Apify's API or due to a misconfigured crawler.`
: '';
this.log.exception(
secondaryError as Error,
'An exception occurred during handling of failed request. ' +
`This places the crawler and its underlying storages into an unknown state and crawling will be terminated. ${apifySpecific}`,
);
}
request.state = RequestState.ERROR;
throw secondaryError;
@@ -1296,11 +1352,18 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* Run async callback with given timeout and retry.
* @ignore
*/
protected async _timeoutAndRetry(handler: () => Promise<unknown>, timeout: number, error: Error | string, maxRetries = 3, retried = 1): Promise<void> {
protected async _timeoutAndRetry(
handler: () => Promise<unknown>,
timeout: number,
error: Error | string,
maxRetries = 3,
retried = 1,
): Promise<void> {
try {
await addTimeoutToPromise(handler, timeout, error);
} catch (e) {
if (retried <= maxRetries) { // we retry on any error, not just timeout
if (retried <= maxRetries) {
// we retry on any error, not just timeout
this.log.warning(`${(e as Error).message} (retrying ${retried}/${maxRetries})`);
return this._timeoutAndRetry(handler, timeout, error, maxRetries, retried + 1);
}
@@ -1314,7 +1377,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
*/
protected async _isTaskReadyFunction() {
// First check RequestList, since it's only in memory.
const isRequestListEmpty = this.requestList ? (await this.requestList.isEmpty()) : true;
const isRequestListEmpty = this.requestList ? await this.requestList.isEmpty() : true;
// If RequestList is not empty, task is ready, no reason to check RequestQueue.
if (!isRequestListEmpty) return true;
// If RequestQueue is not empty, task is ready, return true, otherwise false.
@@ -1325,10 +1388,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
* Returns true if both RequestList and RequestQueue have all requests finished.
*/
protected async _defaultIsFinishedFunction() {
const [
isRequestListFinished,
isRequestQueueFinished,
] = await Promise.all([
const [isRequestListFinished, isRequestQueueFinished] = await Promise.all([
this.requestList ? this.requestList.isFinished() : true,
this.requestQueue ? this.requestQueue.isFinished() : true,
]);
@@ -1367,7 +1427,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
if (error instanceof SessionError) {
await this._rotateSession(crawlingContext);
} else {
await this._tagUserHandlerError(() => this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
await this._tagUserHandlerError(() =>
this.errorHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),
);
}
if (!request.noRetry) {
@@ -1378,10 +1440,11 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
// We don't want to see the stack trace in the logs by default, when we are going to retry the request.
// Thus, we print the full stack trace only when CRAWLEE_VERBOSE_LOG environment variable is set to true.
const message = this._getMessageFromError(error);
this.log.warning(
`Reclaiming failed request back to the list or queue. ${message}`,
{ id, url, retryCount },
);
this.log.warning(`Reclaiming failed request back to the list or queue. ${message}`, {
id,
url,
retryCount,
});
await source.reclaimRequest(request, { forefront: request.userData?.__crawlee?.forefront });
return;
@@ -1410,7 +1473,7 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected async _tagUserHandlerError<T>(cb: () => unknown): Promise<T> {
try {
return await cb() as T;
return (await cb()) as T;
} catch (e: any) {
Object.defineProperty(e, 'triggeredFromUserHandler', { value: true });
throw e;
@@ -1422,13 +1485,12 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
const { id, url, method, uniqueKey } = crawlingContext.request;
const message = this._getMessageFromError(error, true);
this.log.error(
`Request failed and reached maximum retries. ${message}`,
{ id, url, method, uniqueKey },
);
this.log.error(`Request failed and reached maximum retries. ${message}`, { id, url, method, uniqueKey });
if (this.failedRequestHandler) {
await this._tagUserHandlerError(() => this.failedRequestHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error));
await this._tagUserHandlerError(() =>
this.failedRequestHandler?.(this._augmentContextWithDeprecatedError(crawlingContext, error), error),
);
}
}
@@ -1451,16 +1513,17 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
return process.env.CRAWLEE_VERBOSE_LOG ? error.stack : error.message || error; // stack in timeout errors does not really help
}
return (process.env.CRAWLEE_VERBOSE_LOG || forceStack)
? error.stack ?? ([error.message || error, ...stackLines].join('\n'))
return process.env.CRAWLEE_VERBOSE_LOG || forceStack
? error.stack ?? [error.message || error, ...stackLines].join('\n')
: [error.message || error, userLine].join('\n');
}
protected _canRequestBeRetried(request: Request, error: Error) {
// Request should never be retried, or the error encountered makes it not able to be retried, or the session rotation limit has been reached
if (request.noRetry
|| (error instanceof NonRetryableError)
|| (error instanceof SessionError && (this.maxSessionRotations <= (request.sessionRotationCount ?? 0)))
if (
request.noRetry ||
error instanceof NonRetryableError ||
(error instanceof SessionError && this.maxSessionRotations <= (request.sessionRotationCount ?? 0))
) {
return false;
}
@@ -1478,8 +1541,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected _augmentContextWithDeprecatedError(context: Context, error: Error) {
Object.defineProperty(context, 'error', {
get: () => {
// eslint-disable-next-line max-len
this.log.deprecated("The 'error' property of the crawling context is deprecated, and it is now passed as the second parameter in 'errorHandler' and 'failedRequestHandler'. Please update your code, as this property will be removed in a future version.");
this.log.deprecated(
"The 'error' property of the crawling context is deprecated, and it is now passed as the second parameter in 'errorHandler' and 'failedRequestHandler'. Please update your code, as this property will be removed in a future version.",
);
return error;
},
@@ -1505,7 +1569,10 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
}
}
protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(hooks: HookLike[], ...args: Parameters<HookLike>) {
protected async _executeHooks<HookLike extends (...args: any[]) => Awaitable<void>>(
hooks: HookLike[],
...args: Parameters<HookLike>
) {
if (Array.isArray(hooks) && hooks.length) {
for (const hook of hooks) {
await hook(...args);
@@ -1540,19 +1607,23 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
allowUndefined = false,
}: HandlePropertyNameChangeData<New, Old>) {
if (newProperty && oldProperty) {
this.log.warning([
`Both "${newName}" and "${oldName}" were provided in the crawler options.`,
`"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
`As such, "${newName}" will be used instead.`,
].join('\n'));
this.log.warning(
[
`Both "${newName}" and "${oldName}" were provided in the crawler options.`,
`"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
`As such, "${newName}" will be used instead.`,
].join('\n'),
);
// @ts-expect-error Assigning to possibly readonly properties
this[propertyKey] = newProperty;
} else if (oldProperty) {
this.log.warning([
`"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
`The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`,
].join('\n'));
this.log.warning(
[
`"${oldName}" has been renamed to "${newName}", and will be removed in a future version.`,
`The provided value will be used, but you should rename "${oldName}" to "${newName}" in your crawler options.`,
].join('\n'),
);
// @ts-expect-error Assigning to possibly readonly properties
this[propertyKey] = oldProperty;
@@ -1566,7 +1637,9 @@ export class BasicCrawler<Context extends CrawlingContext = BasicCrawlingContext
protected _getCookieHeaderFromRequest(request: Request) {
if (request.headers?.Cookie && request.headers?.cookie) {
this.log.warning(`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`);
this.log.warning(
`Encountered mixed casing for the cookie headers for request ${request.url} (${request.id}). Their values will be merged.`,
);
return mergeCookies(request.url, [request.headers.cookie, request.headers.Cookie]);
}
packages/basic-crawler/test/migration.test.ts
+39 -27
View File
@@ -36,11 +36,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "requestHandler" and "handleRequestFunction" were provided in the crawler options.`,
`"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "requestHandler" and "handleRequestFunction" were provided in the crawler options.`,
`"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandler']).toBe(newHandler);
@@ -56,10 +58,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleRequestFunction" to "requestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handleRequestFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleRequestFunction" to "requestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandler']).toBe(oldHandler);
@@ -96,11 +100,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(newHandler);
@@ -117,10 +123,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(oldHandler);
@@ -156,11 +164,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleRequestTimeoutSecs: 69,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "requestHandlerTimeoutSecs" and "handleRequestTimeoutSecs" were provided in the crawler options.`,
`"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,
`As such, "requestHandlerTimeoutSecs" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "requestHandlerTimeoutSecs" and "handleRequestTimeoutSecs" were provided in the crawler options.`,
`"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,
`As such, "requestHandlerTimeoutSecs" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandlerTimeoutMillis']).toEqual(420_000);
@@ -176,10 +186,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleRequestTimeoutSecs: 69,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleRequestTimeoutSecs" to "requestHandlerTimeoutSecs" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handleRequestTimeoutSecs" has been renamed to "requestHandlerTimeoutSecs", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleRequestTimeoutSecs" to "requestHandlerTimeoutSecs" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandlerTimeoutMillis']).toEqual(69_000);
packages/basic-crawler/test/tsconfig.json
+1 -4
View File
@@ -1,9 +1,6 @@
{
"extends": "../../../tsconfig.json",
"include": [
"**/*",
"../../**/*"
],
"include": ["**/*", "../../**/*"],
"compilerOptions": {
"types": ["vitest/globals"]
}
packages/basic-crawler/tsconfig.build.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
}
packages/basic-crawler/tsconfig.json
+2 -2
View File
@@ -1,4 +1,4 @@
{
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
}
packages/browser-crawler/src/internals/browser-crawler.ts
+103 -77
View File
@@ -56,14 +56,16 @@ export interface BrowserCrawlingContext<
response?: Response;
}
export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = RequestHandler<Context>;
export type BrowserRequestHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> =
RequestHandler<Context>;
export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> = ErrorHandler<Context>;
export type BrowserErrorHandler<Context extends BrowserCrawlingContext = BrowserCrawlingContext> =
ErrorHandler<Context>;
export type BrowserHook<
Context = BrowserCrawlingContext,
GoToOptions extends Dictionary | undefined = Dictionary,
> = (crawlingContext: Context, gotoOptions: GoToOptions) => Awaitable<void>;
export type BrowserHook<Context = BrowserCrawlingContext, GoToOptions extends Dictionary | undefined = Dictionary> = (
crawlingContext: Context,
gotoOptions: GoToOptions,
) => Awaitable<void>;
export interface BrowserCrawlerOptions<
Context extends BrowserCrawlingContext = BrowserCrawlingContext,
@@ -72,16 +74,14 @@ export interface BrowserCrawlerOptions<
__BrowserControllerReturn extends BrowserController = ReturnType<__BrowserPlugins[number]['createController']>,
__LaunchContextReturn extends LaunchContext = ReturnType<__BrowserPlugins[number]['createLaunchContext']>,
> extends Omit<
BasicCrawlerOptions,
// Overridden with browser context
| 'requestHandler'
| 'handleRequestFunction'
| 'failedRequestHandler'
| 'handleFailedRequestFunction'
| 'errorHandler'
> {
BasicCrawlerOptions,
// Overridden with browser context
| 'requestHandler'
| 'handleRequestFunction'
| 'failedRequestHandler'
| 'handleFailedRequestFunction'
| 'errorHandler'
> {
launchContext?: BrowserLaunchContext<any, any>;
/**
@@ -186,7 +186,8 @@ export interface BrowserCrawlerOptions<
* Custom options passed to the underlying {@apilink BrowserPool} constructor.
* We can tweak those to fine-tune browser management.
*/
browserPoolOptions?: Partial<BrowserPoolOptions> & Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
browserPoolOptions?: Partial<BrowserPoolOptions> &
Partial<BrowserPoolHooks<__BrowserControllerReturn, __LaunchContextReturn>>;
/**
* If set, the crawler will be configured for all connections to use
@@ -345,7 +346,10 @@ export abstract class BrowserCrawler<
/**
* All `BrowserCrawler` parameters are passed via an options object.
*/
protected constructor(options: BrowserCrawlerOptions<Context> = {}, override readonly config = Configuration.getGlobalConfig()) {
protected constructor(
options: BrowserCrawlerOptions<Context> = {},
override readonly config = Configuration.getGlobalConfig(),
) {
ow(options, 'BrowserCrawlerOptions', ow.object.exactShape(BrowserCrawler.optionsShape));
const {
navigationTimeoutSecs = 60,
@@ -368,11 +372,15 @@ export abstract class BrowserCrawler<
...basicCrawlerOptions
} = options;
super({
...basicCrawlerOptions,
requestHandler: async (...args) => this._runRequestHandler(...args),
requestHandlerTimeoutSecs: navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
}, config);
super(
{
...basicCrawlerOptions,
requestHandler: async (...args) => this._runRequestHandler(...args),
requestHandlerTimeoutSecs:
navigationTimeoutSecs + requestHandlerTimeoutSecs + BASIC_CRAWLER_TIMEOUT_BUFFER_SECS,
},
config,
);
this._handlePropertyNameChange({
newName: 'requestHandler',
@@ -420,22 +428,17 @@ export abstract class BrowserCrawler<
}
if (launchContext?.userAgent) {
if (browserPoolOptions.useFingerprints) this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
if (browserPoolOptions.useFingerprints)
this.log.info('Custom user agent provided, disabling automatic browser fingerprint injection!');
browserPoolOptions.useFingerprints = false;
}
const { preLaunchHooks = [], postLaunchHooks = [], ...rest } = browserPoolOptions;
this.browserPool = new BrowserPool<InternalBrowserPoolOptions>({
...rest as any,
preLaunchHooks: [
this._extendLaunchContext.bind(this),
...preLaunchHooks,
],
postLaunchHooks: [
this._maybeAddSessionRetiredListener.bind(this),
...postLaunchHooks,
],
...(rest as any),
preLaunchHooks: [this._extendLaunchContext.bind(this), ...preLaunchHooks],
postLaunchHooks: [this._maybeAddSessionRetiredListener.bind(this), ...postLaunchHooks],
});
}
@@ -449,9 +452,7 @@ export abstract class BrowserCrawler<
}
private async containsSelectors(page: CommonPage, selectors: string[]): Promise<string[] | null> {
const foundSelectors = (await Promise.all(
selectors.map((selector) => (page as any).$(selector)))
)
const foundSelectors = (await Promise.all(selectors.map((selector) => (page as any).$(selector))))
.map((x, i) => [x, selectors[i]] as [any, string])
.filter(([x]) => x !== null)
.map(([, selector]) => selector);
@@ -462,14 +463,15 @@ export abstract class BrowserCrawler<
protected override async isRequestBlocked(crawlingContext: Context): Promise<string | false> {
const { page, response } = crawlingContext;
// eslint-disable-next-line dot-notation
const blockedStatusCodes = ((this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0)
const blockedStatusCodes =
// eslint-disable-next-line dot-notation
? this.sessionPool!['blockedStatusCodes']
: DEFAULT_BLOCKED_STATUS_CODES;
(this.sessionPool?.['blockedStatusCodes'].length ?? 0) > 0
? // eslint-disable-next-line dot-notation
this.sessionPool!['blockedStatusCodes']
: DEFAULT_BLOCKED_STATUS_CODES;
// Cloudflare specific heuristic - wait 5 seconds if we get a 403 for the JS challenge to load / resolve.
if (await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS) && response?.status() === 403) {
if ((await this.containsSelectors(page, CLOUDFLARE_RETRY_CSS_SELECTORS)) && response?.status() === 403) {
await sleep(5000);
// here we cannot test for response code, because we only have the original response, not the possible Cloudflare redirect on passed challenge.
@@ -502,7 +504,9 @@ export abstract class BrowserCrawler<
if (this.proxyConfiguration) {
const { session } = crawlingContext;
const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, { request: crawlingContext.request });
const proxyInfo = await this.proxyConfiguration.newProxyInfo(session?.id, {
request: crawlingContext.request,
});
crawlingContext.proxyInfo = proxyInfo;
newPageOptions.proxyUrl = proxyInfo?.url;
@@ -510,16 +514,16 @@ export abstract class BrowserCrawler<
if (this.proxyConfiguration.isManInTheMiddle) {
/**
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
*/
* @see https://playwright.dev/docs/api/class-browser/#browser-new-context
* @see https://github.com/puppeteer/puppeteer/blob/main/docs/api.md
*/
newPageOptions.pageOptions = {
ignoreHTTPSErrors: true,
};
}
}
const page = await this.browserPool.newPage(newPageOptions) as CommonPage;
const page = (await this.browserPool.newPage(newPageOptions)) as CommonPage;
tryCancel();
this._enhanceCrawlingContextWithPageInfo(crawlingContext, page, useIncognitoPages || experimentalContainers);
@@ -547,7 +551,7 @@ export abstract class BrowserCrawler<
if (!this.requestMatchesEnqueueStrategy(request)) {
this.log.debug(
// eslint-disable-next-line max-len, dot-notation
// eslint-disable-next-line dot-notation
`Skipping request ${request.id} (starting url: ${request.url} -> loaded url: ${request.loadedUrl}) because it does not match the enqueue strategy (${request['enqueueStrategy']}).`,
);
@@ -580,14 +584,20 @@ export abstract class BrowserCrawler<
if (session) session.markGood();
}
protected _enhanceCrawlingContextWithPageInfo(crawlingContext: Context, page: CommonPage, createNewSession?: boolean): void {
protected _enhanceCrawlingContextWithPageInfo(
crawlingContext: Context,
page: CommonPage,
createNewSession?: boolean,
): void {
crawlingContext.page = page;
// This switch is because the crawlingContexts are created on per request basis.
// However, we need to add the proxy info and session from browser, which is created based on the browser-pool configuration.
// We would not have to do this switch if the proxy and configuration worked as in CheerioCrawler,
// which configures proxy and session for every new request
const browserControllerInstance = this.browserPool.getBrowserControllerByPage(page as any) as Context['browserController'];
const browserControllerInstance = this.browserPool.getBrowserControllerByPage(
page as any,
) as Context['browserController'];
crawlingContext.browserController = browserControllerInstance;
if (!createNewSession) {
@@ -623,7 +633,7 @@ export abstract class BrowserCrawler<
await this._applyCookies(crawlingContext, preNavigationHooksCookies, postNavigationHooksCookies);
try {
crawlingContext.response = await this._navigationHandler(crawlingContext, gotoOptions) ?? undefined;
crawlingContext.response = (await this._navigationHandler(crawlingContext, gotoOptions)) ?? undefined;
} catch (error) {
await this._handleNavigationTimeout(crawlingContext, error as Error);
@@ -638,18 +648,18 @@ export abstract class BrowserCrawler<
await this._executeHooks(this.postNavigationHooks, crawlingContext, gotoOptions);
}
protected async _applyCookies({ session, request, page, browserController }: Context, preHooksCookies: string, postHooksCookies: string) {
protected async _applyCookies(
{ session, request, page, browserController }: Context,
preHooksCookies: string,
postHooksCookies: string,
) {
const sessionCookie = session?.getCookies(request.url) ?? [];
const parsedPreHooksCookies = preHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
const parsedPostHooksCookies = postHooksCookies.split(/ *; */).map((c) => cookieStringToToughCookie(c));
await browserController.setCookies(
page,
[
...sessionCookie,
...parsedPreHooksCookies,
...parsedPostHooksCookies,
]
[...sessionCookie, ...parsedPreHooksCookies, ...parsedPostHooksCookies]
.filter((c): c is CookieObject => typeof c !== 'undefined' && c !== null)
.map((c) => ({ ...c, url: c.domain ? undefined : request.url })),
);
@@ -677,7 +687,10 @@ export abstract class BrowserCrawler<
}
}
protected abstract _navigationHandler(crawlingContext: Context, gotoOptions: GoToOptions): Promise<Context['response'] | null | undefined>;
protected abstract _navigationHandler(
crawlingContext: Context,
gotoOptions: GoToOptions,
): Promise<Context['response'] | null | undefined>;
/**
* Should be overridden in case of different automation library that does not support this response API.
@@ -710,10 +723,9 @@ export abstract class BrowserCrawler<
}
if (this.proxyConfiguration && !launchContext.proxyUrl) {
const proxyInfo = await this.proxyConfiguration.newProxyInfo(
launchContextExtends.session?.id,
{ proxyTier: (launchContext.proxyTier as number) ?? undefined },
);
const proxyInfo = await this.proxyConfiguration.newProxyInfo(launchContextExtends.session?.id, {
proxyTier: (launchContext.proxyTier as number) ?? undefined,
});
launchContext.proxyUrl = proxyInfo?.url;
launchContextExtends.proxyInfo = proxyInfo;
@@ -736,7 +748,9 @@ export abstract class BrowserCrawler<
const { launchContext } = browserController;
if (session.id === (launchContext.session as Session).id) {
this.browserPool.retireBrowserController(
browserController as Parameters<BrowserPool<InternalBrowserPoolOptions>['retireBrowserController']>[0],
browserController as Parameters<
BrowserPool<InternalBrowserPoolOptions>['retireBrowserController']
>[0],
);
}
};
@@ -782,7 +796,11 @@ export async function browserCrawlerEnqueueLinks({
userProvidedBaseUrl: options?.baseUrl,
});
const urls = await extractUrlsFromPage(page as any, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
const urls = await extractUrlsFromPage(
page as any,
options?.selector ?? 'a',
options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
return enqueueLinks({
requestQueue,
@@ -796,9 +814,16 @@ export async function browserCrawlerEnqueueLinks({
* Extracts URLs from a given page.
* @ignore
*/
// eslint-disable-next-line @typescript-eslint/ban-types
export async function extractUrlsFromPage(page: { $$eval: Function }, selector: string, baseUrl: string): Promise<string[]> {
const urls = await page.$$eval(selector, (linkEls: HTMLLinkElement[]) => linkEls.map((link) => link.getAttribute('href')).filter((href) => !!href)) ?? [];
export async function extractUrlsFromPage(
// eslint-disable-next-line @typescript-eslint/ban-types
page: { $$eval: Function },
selector: string,
baseUrl: string,
): Promise<string[]> {
const urls =
(await page.$$eval(selector, (linkEls: HTMLLinkElement[]) =>
linkEls.map((link) => link.getAttribute('href')).filter((href) => !!href),
)) ?? [];
const [base] = await page.$$eval('base', (els: HTMLLinkElement[]) => els.map((el) => el.getAttribute('href')));
const absoluteBaseUrl = base && tryAbsoluteURL(base, baseUrl);
@@ -806,17 +831,18 @@ export async function extractUrlsFromPage(page: { $$eval: Function }, selector:
baseUrl = absoluteBaseUrl;
}
return urls.map((href: string) => {
// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
if (!isHrefAbsolute && !baseUrl) {
throw new Error(`An extracted URL: ${href} is relative and options.baseUrl is not set. `
+ 'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.');
}
return urls
.map((href: string) => {
// Throw a meaningful error when only a relative URL would be extracted instead of waiting for the Request to fail later.
const isHrefAbsolute = /^[a-z][a-z0-9+.-]*:/.test(href); // Grabbed this in 'is-absolute-url' package.
if (!isHrefAbsolute && !baseUrl) {
throw new Error(
`An extracted URL: ${href} is relative and options.baseUrl is not set. ` +
'Use options.baseUrl in enqueueLinks() to automatically resolve relative URLs.',
);
}
return baseUrl
? tryAbsoluteURL(href, baseUrl)
: href;
})
return baseUrl ? tryAbsoluteURL(href, baseUrl) : href;
})
.filter((href: string | undefined) => !!href);
}
packages/browser-crawler/src/internals/browser-launcher.ts
+15 -11
View File
@@ -40,10 +40,10 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
browserPerProxy?: boolean;
/**
* With this option selected, all pages will be opened in a new incognito browser context.
* This means they will not share cookies nor cache and their resources will not be throttled by one another.
* @default false
*/
* With this option selected, all pages will be opened in a new incognito browser context.
* This means they will not share cookies nor cache and their resources will not be throttled by one another.
* @default false
*/
useIncognitoPages?: boolean;
/**
@@ -54,10 +54,10 @@ export interface BrowserLaunchContext<TOptions, Launcher> extends BrowserPluginO
experimentalContainers?: boolean;
/**
* Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.
* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.
* If not specified, a temporary directory is used instead.
*/
* Sets the [User Data Directory](https://chromium.googlesource.com/chromium/src/+/master/docs/user_data_dir.md) path.
* The user data directory contains profile data such as history, bookmarks, and cookies, as well as other per-installation local state.
* If not specified, a temporary directory is used instead.
*/
userDataDir?: string;
/**
@@ -119,8 +119,9 @@ export abstract class BrowserLauncher<
} catch (err) {
const e = err as Error & { code: string };
if (e.code === 'MODULE_NOT_FOUND') {
const msg = `Cannot find module '${launcher}'. Did you you install the '${launcher}' package?\n`
+ `Make sure you have '${launcher}' in your package.json dependencies and in your package-lock.json, if you use it.`;
const msg =
`Cannot find module '${launcher}'. Did you you install the '${launcher}' package?\n` +
`Make sure you have '${launcher}' in your package.json dependencies and in your package-lock.json, if you use it.`;
if (process.env.APIFY_IS_AT_HOME) {
e.message = `${msg}\nOn the Apify platform, '${launcher}' can only be used with the ${apifyImageName} Docker image.`;
}
@@ -133,7 +134,10 @@ export abstract class BrowserLauncher<
/**
* All `BrowserLauncher` parameters are passed via an launchContext object.
*/
constructor(launchContext: BrowserLaunchContext<LaunchOptions, Launcher>, readonly config = Configuration.getGlobalConfig()) {
constructor(
launchContext: BrowserLaunchContext<LaunchOptions, Launcher>,
readonly config = Configuration.getGlobalConfig(),
) {
const {
launcher,
proxyUrl,
packages/browser-crawler/test/migration.test.ts
+26 -18
View File
@@ -44,11 +44,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handlePageFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['userProvidedRequestHandler']).toBe(newHandler);
@@ -70,10 +72,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handlePageFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['userProvidedRequestHandler']).toBe(oldHandler);
@@ -122,11 +126,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(newHandler);
@@ -149,10 +155,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(oldHandler);
packages/browser-crawler/test/tsconfig.json
+1 -4
View File
@@ -1,9 +1,6 @@
{
"extends": "../../../tsconfig.json",
"include": [
"**/*",
"../../**/*"
],
"include": ["**/*", "../../**/*"],
"compilerOptions": {
"types": ["vitest/globals"]
}
packages/browser-crawler/tsconfig.build.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
}
packages/browser-crawler/tsconfig.json
+2 -2
View File
@@ -1,4 +1,4 @@
{
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
}
packages/browser-pool/src/abstract-classes/browser-controller.ts
+19 -36
View File
@@ -4,7 +4,6 @@ import { nanoid } from 'nanoid';
import { TypedEmitter } from 'tiny-typed-emitter';
import type { BrowserPlugin, CommonBrowser, CommonLibrary } from './browser-plugin';
import { throwImplementationNeeded } from './utils';
import { BROWSER_CONTROLLER_EVENTS } from '../events';
import type { LaunchContext } from '../launch-context';
import { log } from '../logger';
@@ -19,8 +18,9 @@ export interface BrowserControllerEvents<
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
> {
[BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]:
(controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>) => void;
[BROWSER_CONTROLLER_EVENTS.BROWSER_CLOSED]: (
controller: BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
) => void;
}
/**
@@ -109,7 +109,10 @@ export abstract class BrowserController<
/**
* @ignore
*/
assignBrowser(browser: LaunchResult, launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): void {
assignBrowser(
browser: LaunchResult,
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): void {
if (this.browser) {
throw new Error('BrowserController already has a browser instance assigned.');
}
@@ -181,48 +184,28 @@ export abstract class BrowserController<
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract async _close(): Promise<void> {
throwImplementationNeeded('_close');
}
protected abstract _close(): Promise<void>;
/**
* @private
*/
protected abstract _kill(): Promise<void>;
/**
* @private
*/
protected abstract _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult>;
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract async _kill(): Promise<void> {
throwImplementationNeeded('_kill');
}
protected abstract _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void>;
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract async _newPage(pageOptions?: NewPageOptions): Promise<NewPageResult> {
throwImplementationNeeded('_newPage');
}
protected abstract _getCookies(page: NewPageResult): Promise<Cookie[]>;
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract async _setCookies(page: NewPageResult, cookies: Cookie[]): Promise<void> {
throwImplementationNeeded('_setCookies');
}
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract async _getCookies(page: NewPageResult): Promise<Cookie[]> {
throwImplementationNeeded('_getCookies');
}
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown> {
throwImplementationNeeded('_normalizeProxyOptions');
}
abstract normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown>;
}
packages/browser-pool/src/abstract-classes/browser-plugin.ts
+36 -22
View File
@@ -3,7 +3,6 @@ import type { Dictionary } from '@crawlee/types';
import merge from 'lodash.merge';
import type { BrowserController } from './browser-controller';
import { throwImplementationNeeded } from './utils';
import type { LaunchContextOptions } from '../launch-context';
import { LaunchContext } from '../launch-context';
import type { UnwrapPromise } from '../utils';
@@ -18,7 +17,8 @@ import type { UnwrapPromise } from '../utils';
*
* After you update it here, please update it also in jsdom-crawler.ts
*/
export const DEFAULT_USER_AGENT = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
export const DEFAULT_USER_AGENT =
'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/107.0.0.0 Safari/537.36';
/**
* Each plugin expects an instance of the object with the `.launch()` property.
@@ -90,7 +90,12 @@ export interface CreateLaunchContextOptions<
LaunchResult extends CommonBrowser = UnwrapPromise<ReturnType<Library['launch']>>,
NewPageOptions = Parameters<LaunchResult['newPage']>[0],
NewPageResult = UnwrapPromise<ReturnType<LaunchResult['newPage']>>,
> extends Partial<Omit<LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>, 'browserPlugin'>> {}
> extends Partial<
Omit<
LaunchContextOptions<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
'browserPlugin'
>
> {}
/**
* The `BrowserPlugin` serves two purposes. First, it is the base class that
@@ -181,7 +186,13 @@ export abstract class BrowserPlugin<
* Launches the browser using provided launch context.
*/
async launch(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> = this.createLaunchContext(),
launchContext: LaunchContext<
Library,
LibraryOptions,
LaunchResult,
NewPageOptions,
NewPageResult
> = this.createLaunchContext(),
): Promise<LaunchResult> {
launchContext.launchOptions ??= {} as LibraryOptions;
@@ -240,7 +251,11 @@ export abstract class BrowserPlugin<
errorMessage.push(`- ${moduleInstallCommand}`);
errorMessage.push('', 'The original error is available in the `cause` property. Below is the error received when trying to launch a browser:', '');
errorMessage.push(
'',
'The original error is available in the `cause` property. Below is the error received when trying to launch a browser:',
'',
);
// Add in a zero-width space so we can remove it later when printing the error stack
throw new BrowserLaunchError(`${errorMessage.join('\n')}\u200b`, { cause });
@@ -249,32 +264,31 @@ export abstract class BrowserPlugin<
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
// eslint-disable-next-line max-len
protected abstract _addProxyToLaunchOptions(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<void> {
throwImplementationNeeded('_addProxyToLaunchOptions');
}
protected abstract _addProxyToLaunchOptions(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): Promise<void>;
// @ts-expect-error Give runtime error as well as compile time
protected abstract _isChromiumBasedBrowser(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): boolean {
throwImplementationNeeded('_isChromiumBasedBrowser');
}
protected abstract _isChromiumBasedBrowser(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): boolean;
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract _launch(launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>): Promise<LaunchResult> {
throwImplementationNeeded('_launch');
}
protected abstract _launch(
launchContext: LaunchContext<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult>,
): Promise<LaunchResult>;
/**
* @private
*/
// @ts-expect-error Give runtime error as well as compile time
protected abstract _createController(): BrowserController<Library, LibraryOptions, LaunchResult, NewPageOptions, NewPageResult> {
throwImplementationNeeded('_createController');
}
protected abstract _createController(): BrowserController<
Library,
LibraryOptions,
LaunchResult,
NewPageOptions,
NewPageResult
>;
}
export class BrowserLaunchError extends CriticalError {
packages/browser-pool/src/abstract-classes/utils.ts
-3
View File
@@ -1,3 +0,0 @@
export function throwImplementationNeeded(methodName: string): never {
throw new Error(`You need to implement method ${methodName}.`);
}
packages/browser-pool/src/anonymize-proxy.ts
+2 -8
View File
@@ -26,14 +26,8 @@ export const anonymizeProxySugar = async (
];
}
return [
undefined,
async () => {},
];
return [undefined, async () => {}];
}
return [
undefined,
async () => {},
];
return [undefined, async () => {}];
};
packages/browser-pool/src/browser-pool.ts
+101 -102
View File
@@ -12,7 +12,11 @@ import { TypedEmitter } from 'tiny-typed-emitter';
import type { BrowserController } from './abstract-classes/browser-controller';
import type { BrowserPlugin } from './abstract-classes/browser-plugin';
import { BROWSER_POOL_EVENTS } from './events';
import { createFingerprintPreLaunchHook, createPrePageCreateHook, createPostPageCreateHook } from './fingerprinting/hooks';
import {
createFingerprintPreLaunchHook,
createPrePageCreateHook,
createPostPageCreateHook,
} from './fingerprinting/hooks';
import type { FingerprintGeneratorOptions } from './fingerprinting/types';
import type { LaunchContext } from './launch-context';
import { log } from './logger';
@@ -45,11 +49,11 @@ export interface FingerprintOptions {
*/
useFingerprintCache?: boolean;
/**
* The maximum number of fingerprints that can be stored in the cache.
*
* Only relevant if `useFingerprintCache` is set to `true`.
* @default 10000
*/
* The maximum number of fingerprints that can be stored in the cache.
*
* Only relevant if `useFingerprintCache` is set to `true`.
* @default 10000
*/
fingerprintCacheSize?: number;
}
@@ -128,7 +132,10 @@ export type PreLaunchHook<LC extends LaunchContext> = (pageId: string, launchCon
* hooks complete. If you attempt to call `await browserController.close()` from
* a post-launch hook, it will deadlock the process. This API is subject to change.
*/
export type PostLaunchHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
export type PostLaunchHook<BC extends BrowserController> = (
pageId: string,
browserController: BC,
) => void | Promise<void>;
/**
* Pre-page-create hooks are executed just before a new page is created. They
@@ -139,10 +146,11 @@ export type PostLaunchHook<BC extends BrowserController> = (pageId: string, brow
* So far, new page options are only supported by `PlaywrightController` in incognito contexts.
* If the page options are not supported by `BrowserController` the `pageOptions` argument is `undefined`.
*/
export type PrePageCreateHook<
BC extends BrowserController,
PO = Parameters<BC['newPage']>[0],
> = (pageId: string, browserController: BC, pageOptions?: PO) => void | Promise<void>;
export type PrePageCreateHook<BC extends BrowserController, PO = Parameters<BC['newPage']>[0]> = (
pageId: string,
browserController: BC,
pageOptions?: PO,
) => void | Promise<void>;
/**
* Post-page-create hooks are called right after a new page is created
@@ -152,10 +160,10 @@ export type PrePageCreateHook<
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
export type PostPageCreateHook<
BC extends BrowserController,
Page = UnwrapPromise<ReturnType<BC['newPage']>>,
> = (page: Page, browserController: BC) => void | Promise<void>;
export type PostPageCreateHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (
page: Page,
browserController: BC,
) => void | Promise<void>;
/**
* Pre-page-close hooks give you the opportunity to make last second changes
@@ -164,17 +172,20 @@ export type PostPageCreateHook<
* The hooks are called with two arguments:
* `page`: `Page` and `browserController`: {@apilink BrowserController}
*/
export type PrePageCloseHook<
BC extends BrowserController,
Page = UnwrapPromise<ReturnType<BC['newPage']>>,
> = (page: Page, browserController: BC) => void | Promise<void>;
export type PrePageCloseHook<BC extends BrowserController, Page = UnwrapPromise<ReturnType<BC['newPage']>>> = (
page: Page,
browserController: BC,
) => void | Promise<void>;
/**
* Post-page-close hooks allow you to do page related clean up.
* The hooks are called with two arguments:
* `pageId`: `string` and `browserController`: {@apilink BrowserController}
*/
export type PostPageCloseHook<BC extends BrowserController> = (pageId: string, browserController: BC) => void | Promise<void>;
export type PostPageCloseHook<BC extends BrowserController> = (
pageId: string,
browserController: BC,
) => void | Promise<void>;
export interface BrowserPoolHooks<
BC extends BrowserController,
@@ -286,7 +297,9 @@ export class BrowserPool<
BrowserControllerReturn extends BrowserController = ReturnType<BrowserPlugins[number]['createController']>,
LaunchContextReturn extends LaunchContext = ReturnType<BrowserPlugins[number]['createLaunchContext']>,
PageOptions = Parameters<BrowserControllerReturn['newPage']>[0],
PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>>,
PageReturn extends UnwrapPromise<ReturnType<BrowserControllerReturn['newPage']>> = UnwrapPromise<
ReturnType<BrowserControllerReturn['newPage']>
>,
> extends TypedEmitter<BrowserPoolEvents<BrowserControllerReturn, PageReturn>> {
browserPlugins: BrowserPlugins;
maxOpenPagesPerBrowser: number;
@@ -325,22 +338,25 @@ export class BrowserPool<
this.browserKillerInterval!.unref();
ow(options, ow.object.exactShape({
browserPlugins: ow.array.minLength(1),
maxOpenPagesPerBrowser: ow.optional.number,
retireBrowserAfterPageCount: ow.optional.number,
operationTimeoutSecs: ow.optional.number,
closeInactiveBrowserAfterSecs: ow.optional.number,
retireInactiveBrowserAfterSecs: ow.optional.number,
preLaunchHooks: ow.optional.array,
postLaunchHooks: ow.optional.array,
prePageCreateHooks: ow.optional.array,
postPageCreateHooks: ow.optional.array,
prePageCloseHooks: ow.optional.array,
postPageCloseHooks: ow.optional.array,
useFingerprints: ow.optional.boolean,
fingerprintOptions: ow.optional.object,
}));
ow(
options,
ow.object.exactShape({
browserPlugins: ow.array.minLength(1),
maxOpenPagesPerBrowser: ow.optional.number,
retireBrowserAfterPageCount: ow.optional.number,
operationTimeoutSecs: ow.optional.number,
closeInactiveBrowserAfterSecs: ow.optional.number,
retireInactiveBrowserAfterSecs: ow.optional.number,
preLaunchHooks: ow.optional.array,
postLaunchHooks: ow.optional.array,
prePageCreateHooks: ow.optional.array,
postPageCreateHooks: ow.optional.array,
prePageCloseHooks: ow.optional.array,
postPageCloseHooks: ow.optional.array,
useFingerprints: ow.optional.boolean,
fingerprintOptions: ow.optional.object,
}),
);
const {
browserPlugins,
@@ -368,8 +384,9 @@ export class BrowserPool<
const firstPluginName = firstPluginConstructor.name;
const providedPluginName = (providedPlugin as BrowserPlugin).constructor.name;
// eslint-disable-next-line max-len
throw new Error(`Browser plugin at index ${i} (${providedPluginName}) is not an instance of the same plugin as the first plugin provided (${firstPluginName}).`);
throw new Error(
`Browser plugin at index ${i} (${providedPluginName}) is not an instance of the same plugin as the first plugin provided (${firstPluginName}).`,
);
}
}
@@ -382,14 +399,17 @@ export class BrowserPool<
this.fingerprintOptions = fingerprintOptions;
this.browserRetireInterval = setInterval(
async () => this.activeBrowserControllers.forEach((controller) => {
if (
controller.activePages === 0
&& controller.lastPageOpenedAt < (Date.now() - retireInactiveBrowserAfterSecs * 1000)
) {
this.retireBrowserController(controller);
}
}), retireInactiveBrowserAfterSecs * 1000);
async () =>
this.activeBrowserControllers.forEach((controller) => {
if (
controller.activePages === 0 &&
controller.lastPageOpenedAt < Date.now() - retireInactiveBrowserAfterSecs * 1000
) {
this.retireBrowserController(controller);
}
}),
retireInactiveBrowserAfterSecs * 1000,
);
this.browserRetireInterval!.unref();
@@ -413,13 +433,7 @@ export class BrowserPool<
* or their page limits have been exceeded.
*/
async newPage(options: BrowserPoolNewPageOptions<PageOptions, BrowserPlugins[number]> = {}): Promise<PageReturn> {
const {
id = nanoid(),
pageOptions,
browserPlugin = this._pickBrowserPlugin(),
proxyUrl,
proxyTier,
} = options;
const { id = nanoid(), pageOptions, browserPlugin = this._pickBrowserPlugin(), proxyUrl, proxyTier } = options;
if (this.pages.has(id)) {
throw new Error(`Page with ID: ${id} already exists.`);
@@ -433,7 +447,8 @@ export class BrowserPool<
return this.limiter(async () => {
let browserController = this._pickBrowserWithFreeCapacity(browserPlugin, { proxyTier, proxyUrl });
if (!browserController) browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
if (!browserController)
browserController = await this._launchBrowser(id, { browserPlugin, proxyTier, proxyUrl });
tryCancel();
return this._createPageForBrowser(id, browserController, pageOptions, proxyUrl);
@@ -445,13 +460,10 @@ export class BrowserPool<
* browser to open the page in. Use the `launchOptions` option to
* configure the new browser.
*/
async newPageInNewBrowser(options: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]> = {}): Promise<PageReturn> {
const {
id = nanoid(),
pageOptions,
launchOptions,
browserPlugin = this._pickBrowserPlugin(),
} = options;
async newPageInNewBrowser(
options: BrowserPoolNewPageInNewBrowserOptions<PageOptions, BrowserPlugins[number]> = {},
): Promise<PageReturn> {
const { id = nanoid(), pageOptions, launchOptions, browserPlugin = this._pickBrowserPlugin() } = options;
if (this.pages.has(id)) {
throw new Error(`Page with ID: ${id} already exists.`);
@@ -544,9 +556,10 @@ export class BrowserPool<
await browserController['isActivePromise'];
tryCancel();
const finalPageOptions = (browserController.launchContext.useIncognitoPages || browserController.launchContext.experimentalContainers)
? pageOptions
: undefined;
const finalPageOptions =
browserController.launchContext.useIncognitoPages || browserController.launchContext.experimentalContainers
? pageOptions
: undefined;
if (finalPageOptions) {
Object.assign(finalPageOptions, browserController.normalizeProxyOptions(proxyUrl, pageOptions));
@@ -558,11 +571,11 @@ export class BrowserPool<
let page: PageReturn;
try {
page = await addTimeoutToPromise(
page = (await addTimeoutToPromise(
async () => browserController.newPage(finalPageOptions),
this.operationTimeoutMillis,
'browserController.newPage() timed out.',
) as PageReturn;
)) as PageReturn;
tryCancel();
this.pages.set(pageId, page);
@@ -577,7 +590,9 @@ export class BrowserPool<
this._overridePageClose(page);
} catch (err) {
this.retireBrowserController(browserController);
throw new Error(`browserController.newPage() failed: ${browserController.id}\nCause:${(err as Error).message}.`);
throw new Error(
`browserController.newPage() failed: ${browserController.id}\nCause:${(err as Error).message}.`,
);
}
await this._executeHooks(this.postPageCreateHooks, page, browserController);
@@ -660,12 +675,7 @@ export class BrowserPool<
}
private async _launchBrowser(pageId: string, options: InternalLaunchBrowserOptions<BrowserPlugins[number]>) {
const {
browserPlugin,
launchOptions,
proxyTier,
proxyUrl,
} = options;
const { browserPlugin, launchOptions, proxyTier, proxyUrl } = options;
const browserController = browserPlugin.createController() as BrowserControllerReturn;
this.activeBrowserControllers.add(browserController);
@@ -701,10 +711,9 @@ export class BrowserPool<
} catch (err) {
this.activeBrowserControllers.delete(browserController);
browserController.close().catch((closeErr) => {
log.error(
`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`,
{ id: browserController.id },
);
log.error(`Could not close browser whose post-launch hooks failed.\nCause:${closeErr.message}`, {
id: browserController.id,
});
});
throw err;
}
@@ -727,24 +736,21 @@ export class BrowserPool<
return this.browserPlugins[pluginIndex];
}
private _pickBrowserWithFreeCapacity(
browserPlugin: BrowserPlugin,
options?: Partial<TieredProxy>,
) {
private _pickBrowserWithFreeCapacity(browserPlugin: BrowserPlugin, options?: Partial<TieredProxy>) {
return [...this.activeBrowserControllers].find((controller) => {
const hasCapacity = controller.activePages < this.maxOpenPagesPerBrowser;
const isCorrectPlugin = controller.browserPlugin === browserPlugin;
const isSameProxyUrl = controller.proxyUrl === options?.proxyUrl;
const isCorrectProxyTier = controller.proxyTier === options?.proxyTier;
return isCorrectPlugin
&& hasCapacity
&& (
(!controller.launchContext.browserPerProxy && !options?.proxyTier)
|| (options?.proxyTier && isCorrectProxyTier)
|| (options?.proxyUrl && isSameProxyUrl)
|| (!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier)
);
return (
isCorrectPlugin &&
hasCapacity &&
((!controller.launchContext.browserPerProxy && !options?.proxyTier) ||
(options?.proxyTier && isCorrectProxyTier) ||
(options?.proxyUrl && isSameProxyUrl) ||
(!options?.proxyUrl && !options?.proxyTier && !controller.proxyUrl && !controller.proxyTier))
);
});
}
@@ -781,10 +787,9 @@ export class BrowserPool<
page.close = async (...args: unknown[]) => {
await this._executeHooks(this.prePageCloseHooks, page, browserController);
await originalPageClose.apply(page, args)
.catch((err: Error) => {
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
});
await originalPageClose.apply(page, args).catch((err: Error) => {
log.debug(`Could not close page.\nCause:${err.message}`, { id: browserController.id });
});
await this._executeHooks(this.postPageCloseHooks, pageId, browserController);
@@ -833,14 +838,8 @@ export class BrowserPool<
// It is usual to generate proxy per browser and we want to know the proxyUrl for the caching.
createFingerprintPreLaunchHook(this),
];
this.prePageCreateHooks = [
createPrePageCreateHook(),
...this.prePageCreateHooks,
];
this.postPageCreateHooks = [
createPostPageCreateHook(this.fingerprintInjector!),
...this.postPageCreateHooks,
];
this.prePageCreateHooks = [createPrePageCreateHook(), ...this.prePageCreateHooks];
this.postPageCreateHooks = [createPostPageCreateHook(this.fingerprintInjector!), ...this.postPageCreateHooks];
}
}
packages/browser-pool/src/fingerprinting/hooks.ts
+8 -6
View File
@@ -15,9 +15,7 @@ export function createFingerprintPreLaunchHook(browserPool: BrowserPool<any, any
const {
fingerprintGenerator,
fingerprintCache,
fingerprintOptions: {
fingerprintGeneratorOptions,
},
fingerprintOptions: { fingerprintGeneratorOptions },
} = browserPool;
return (_pageId: string, launchContext: LaunchContext) => {
@@ -26,8 +24,9 @@ export function createFingerprintPreLaunchHook(browserPool: BrowserPool<any, any
const { launchOptions }: { launchOptions: any } = launchContext;
// If no options are passed we try to pass best default options as possible to match browser and OS.
const fingerprintGeneratorFinalOptions = fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
let fingerprint : BrowserFingerprintWithHeaders;
const fingerprintGeneratorFinalOptions =
fingerprintGeneratorOptions || getGeneratorDefaultOptions(launchContext);
let fingerprint: BrowserFingerprintWithHeaders;
if (cacheKey && fingerprintCache?.has(cacheKey)) {
fingerprint = fingerprintCache.get(cacheKey)!;
@@ -43,7 +42,10 @@ export function createFingerprintPreLaunchHook(browserPool: BrowserPool<any, any
if (useIncognitoPages) {
return;
}
const { navigator: { userAgent }, screen } = fingerprint.fingerprint!;
const {
navigator: { userAgent },
screen,
} = fingerprint.fingerprint!;
launchOptions.userAgent = userAgent;
packages/browser-pool/src/fingerprinting/types.ts
+29 -29
View File
@@ -10,36 +10,36 @@ export interface GetFingerprintReturn {
export interface FingerprintGeneratorOptions {
/**
* List of `BrowserSpecification` objects
* or one of `chrome`, `edge`, `firefox` and `safari`.
*/
* List of `BrowserSpecification` objects
* or one of `chrome`, `edge`, `firefox` and `safari`.
*/
browsers?: BrowserSpecification[] | BrowserName[];
/**
* Browser generation query based on the real world data.
* For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).
*
* > Note: If `browserListQuery` is passed, the `browsers` array is ignored.
*/
* Browser generation query based on the real world data.
* For more info see the [query docs](https://github.com/browserslist/browserslist#full-list).
*
* > Note: If `browserListQuery` is passed, the `browsers` array is ignored.
*/
browserListQuery?: string;
/**
* List of operating systems to generate the headers for.
*/
* List of operating systems to generate the headers for.
*/
operatingSystems?: OperatingSystemsName[];
/**
* List of device types to generate the fingerprints for.
*/
* List of device types to generate the fingerprints for.
*/
devices?: DeviceCategory[];
/**
* List of at most 10 languages to include in the
* [Accept-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header
* in the language format accepted by that header, for example `en`, `en-US` or `de`.
*/
* List of at most 10 languages to include in the
* [Accept-Language](https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Accept-Language) request header
* in the language format accepted by that header, for example `en`, `en-US` or `de`.
*/
locales?: string[];
/**
* Http version to be used to generate headers (the headers differ depending on the version).
*
* Can be either 1 or 2. Default value is 2.
*/
* Http version to be used to generate headers (the headers differ depending on the version).
*
* Can be either 1 or 2. Default value is 2.
*/
httpVersion?: HttpVersion;
/**
* Defines the screen dimensions of the generated fingerprint.
@@ -59,7 +59,7 @@ const SUPPORTED_HTTP_VERSIONS = ['1', '2'] as const;
/**
* String specifying the HTTP version to use.
*/
type HttpVersion = typeof SUPPORTED_HTTP_VERSIONS[number];
type HttpVersion = (typeof SUPPORTED_HTTP_VERSIONS)[number];
export enum BrowserName {
chrome = 'chrome',
@@ -70,20 +70,20 @@ export enum BrowserName {
export interface BrowserSpecification {
/**
* String representing the browser name.
*/
* String representing the browser name.
*/
name: BrowserName;
/**
* Minimum version of browser used.
*/
* Minimum version of browser used.
*/
minVersion?: number;
/**
* Maximum version of browser used.
*/
* Maximum version of browser used.
*/
maxVersion?: number;
/**
* HTTP version to be used for header generation (the headers differ depending on the version).
*/
* HTTP version to be used for header generation (the headers differ depending on the version).
*/
httpVersion?: HttpVersion;
}
packages/browser-pool/src/fingerprinting/utils.ts
+2 -1
View File
@@ -24,7 +24,8 @@ const getBrowserName = (browserPlugin: BrowserPlugin, launchOptions: any): Brows
if (browserPlugin instanceof PlaywrightPlugin) {
browserName = library.name!();
} if (browserPlugin instanceof PuppeteerPlugin) {
}
if (browserPlugin instanceof PuppeteerPlugin) {
browserName = launchOptions.product || library.product;
}
packages/browser-pool/src/playwright/playwright-controller.ts
+19 -5
View File
@@ -10,7 +10,11 @@ import type { SafeParameters } from '../utils';
const tabIds = new WeakMap<Page, number>();
const keyFromTabId = (tabId: string | number) => `.${tabId}.`;
export class PlaywrightController extends BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], Browser> {
export class PlaywrightController extends BrowserController<
BrowserType,
SafeParameters<BrowserType['launch']>[0],
Browser
> {
normalizeProxyOptions(proxyUrl: string | undefined, pageOptions: any): Record<string, unknown> {
if (!proxyUrl) {
return {};
@@ -31,8 +35,14 @@ export class PlaywrightController extends BrowserController<BrowserType, SafePar
}
protected async _newPage(contextOptions?: SafeParameters<Browser['newPage']>[0]): Promise<Page> {
if (contextOptions !== undefined && !this.launchContext.useIncognitoPages && !this.launchContext.experimentalContainers) {
throw new Error('A new page can be created with provided context only when using incognito pages or experimental containers.');
if (
contextOptions !== undefined &&
!this.launchContext.useIncognitoPages &&
!this.launchContext.experimentalContainers
) {
throw new Error(
'A new page can be created with provided context only when using incognito pages or experimental containers.',
);
}
let close = async () => {};
@@ -66,7 +76,9 @@ export class PlaywrightController extends BrowserController<BrowserType, SafePar
if (this.launchContext.experimentalContainers) {
await page.goto('data:text/plain,tabid');
await page.waitForNavigation();
const { tabid, proxyip }: { tabid: number; proxyip: string } = JSON.parse(decodeURIComponent(page.url().slice('about:blank#'.length)));
const { tabid, proxyip }: { tabid: number; proxyip: string } = JSON.parse(
decodeURIComponent(page.url().slice('about:blank#'.length)),
);
if (contextOptions?.proxy) {
const url = new URL(contextOptions.proxy.server);
@@ -96,7 +108,9 @@ export class PlaywrightController extends BrowserController<BrowserType, SafePar
const { remoteIPAddress } = response;
if (remoteIPAddress && remoteIPAddress !== proxyip) {
// eslint-disable-next-line no-console
console.warn(`Request to ${response.url} was through ${remoteIPAddress} instead of ${proxyip}`);
console.warn(
`Request to ${response.url} was through ${remoteIPAddress} instead of ${proxyip}`,
);
}
});
}
packages/browser-pool/src/playwright/playwright-plugin.ts
+29 -21
View File
@@ -19,10 +19,13 @@ import type { SafeParameters } from '../utils';
const getFreePort = async () => {
return new Promise<number>((resolve, reject) => {
const server = net.createServer().once('error', reject).listen(() => {
resolve((server.address() as net.AddressInfo).port);
server.close();
});
const server = net
.createServer()
.once('error', reject)
.listen(() => {
resolve((server.address() as net.AddressInfo).port);
server.close();
});
});
};
@@ -30,20 +33,18 @@ const getFreePort = async () => {
// taacPath = browser-pool/dist/tab-as-a-container
const taacPath = path.join(__dirname, '..', 'tab-as-a-container');
export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
export class PlaywrightPlugin extends BrowserPlugin<
BrowserType,
SafeParameters<BrowserType['launch']>[0],
PlaywrightBrowser
> {
private _browserVersion?: string;
_containerProxyServer?: Awaited<ReturnType<typeof createProxyServerForContainers>>;
protected async _launch(launchContext: LaunchContext<BrowserType>): Promise<PlaywrightBrowser> {
const {
launchOptions,
useIncognitoPages,
proxyUrl,
} = launchContext;
const { launchOptions, useIncognitoPages, proxyUrl } = launchContext;
let {
userDataDir,
} = launchContext;
let { userDataDir } = launchContext;
let browser: PlaywrightBrowser;
@@ -82,9 +83,7 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
let firefoxPort: number | undefined;
if (experimentalContainers) {
launchOptions!.args = [
...(launchOptions!.args ?? []),
];
launchOptions!.args = [...(launchOptions!.args ?? [])];
// Use native headless mode so we can load an extension
if (launchOptions!.headless && this.library.name() === 'chromium') {
@@ -92,7 +91,10 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
}
if (this.library.name() === 'chromium') {
launchOptions!.args.push(`--disable-extensions-except=${taacPath}`, `--load-extension=${taacPath}`);
launchOptions!.args.push(
`--disable-extensions-except=${taacPath}`,
`--load-extension=${taacPath}`,
);
} else if (this.library.name() === 'firefox') {
firefoxPort = await getFreePort();
@@ -115,9 +117,11 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
}
}
const browserContext = await this.library.launchPersistentContext(userDataDir, launchOptions).catch((error) => {
return this._throwOnFailedLaunch(launchContext, error);
});
const browserContext = await this.library
.launchPersistentContext(userDataDir, launchOptions)
.catch((error) => {
return this._throwOnFailedLaunch(launchContext, error);
});
browserContext.once('close', () => {
if (userDataDir.includes('apify-playwright-firefox-taac-')) {
@@ -192,7 +196,11 @@ export class PlaywrightPlugin extends BrowserPlugin<BrowserType, SafeParameters<
);
}
protected _createController(): BrowserController<BrowserType, SafeParameters<BrowserType['launch']>[0], PlaywrightBrowser> {
protected _createController(): BrowserController<
BrowserType,
SafeParameters<BrowserType['launch']>[0],
PlaywrightBrowser
> {
return new PlaywrightController(this);
}
packages/browser-pool/src/puppeteer/puppeteer-controller.ts
+4 -2
View File
@@ -40,7 +40,9 @@ export class PuppeteerController extends BrowserController<
protected async _newPage(contextOptions?: PuppeteerNewPageOptions): Promise<PuppeteerTypes.Page> {
if (contextOptions !== undefined) {
if (!this.launchContext.useIncognitoPages) {
throw new Error('A new page can be created with provided context only when using incognito pages or experimental containers.');
throw new Error(
'A new page can be created with provided context only when using incognito pages or experimental containers.',
);
}
let close = async () => {};
@@ -65,7 +67,7 @@ export class PuppeteerController extends BrowserController<
const { CdpBrowser } = await import('puppeteer');
const oldPuppeteerVersion = !CdpBrowser || 'createIncognitoBrowserContext' in CdpBrowser.prototype;
const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';
const context = await (this.browser as any)[method](contextOptions) as PuppeteerTypes.BrowserContext;
const context = (await (this.browser as any)[method](contextOptions)) as PuppeteerTypes.BrowserContext;
tryCancel();
const page = await context.newPage();
tryCancel();
packages/browser-pool/src/puppeteer/puppeteer-plugin.ts
+44 -20
View File
@@ -22,7 +22,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
PuppeteerNewPageOptions
> {
protected async _launch(
launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,
launchContext: LaunchContext<
typeof Puppeteer,
PuppeteerTypes.PuppeteerLaunchOptions,
PuppeteerTypes.Browser,
PuppeteerNewPageOptions
>,
): Promise<PuppeteerTypes.Browser> {
let oldPuppeteerVersion = false;
@@ -34,13 +39,7 @@ export class PuppeteerPlugin extends BrowserPlugin<
} catch {
// ignore
}
const {
launchOptions,
userDataDir,
useIncognitoPages,
experimentalContainers,
proxyUrl,
} = launchContext;
const { launchOptions, userDataDir, useIncognitoPages, experimentalContainers, proxyUrl } = launchContext;
if (experimentalContainers) {
throw new Error('Experimental containers are only available with Playwright');
@@ -110,26 +109,36 @@ export class PuppeteerPlugin extends BrowserPlugin<
}
});
const boundMethods = (['newPage', 'close', 'userAgent', 'createIncognitoBrowserContext', 'createBrowserContext', 'version', 'on', 'process'] as const)
.reduce((map, method) => {
map[method] = browser[method as 'close']?.bind(browser);
return map;
}, {} as Dictionary);
const boundMethods = (
[
'newPage',
'close',
'userAgent',
'createIncognitoBrowserContext',
'createBrowserContext',
'version',
'on',
'process',
] as const
).reduce((map, method) => {
map[method] = browser[method as 'close']?.bind(browser);
return map;
}, {} as Dictionary);
const method = oldPuppeteerVersion ? 'createIncognitoBrowserContext' : 'createBrowserContext';
browser = new Proxy(browser, {
get: (target, property: keyof typeof browser, receiver) => {
if (property === 'newPage') {
return (async (...args: Parameters<PuppeteerTypes.BrowserContext['newPage']>) => {
return async (...args: Parameters<PuppeteerTypes.BrowserContext['newPage']>) => {
let page: PuppeteerTypes.Page;
if (useIncognitoPages) {
const [anonymizedProxyUrl, close] = await anonymizeProxySugar(proxyUrl);
try {
const context = await (browser as any)[method]({
const context = (await (browser as any)[method]({
proxyServer: anonymizedProxyUrl ?? proxyUrl,
}) as PuppeteerTypes.BrowserContext;
})) as PuppeteerTypes.BrowserContext;
page = await context.newPage(...args);
@@ -165,7 +174,7 @@ export class PuppeteerPlugin extends BrowserPlugin<
*/
return page;
});
};
}
if (property in boundMethods) {
@@ -179,12 +188,22 @@ export class PuppeteerPlugin extends BrowserPlugin<
return browser;
}
protected _createController(): BrowserController<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions> {
protected _createController(): BrowserController<
typeof Puppeteer,
PuppeteerTypes.PuppeteerLaunchOptions,
PuppeteerTypes.Browser,
PuppeteerNewPageOptions
> {
return new PuppeteerController(this);
}
protected async _addProxyToLaunchOptions(
_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,
_launchContext: LaunchContext<
typeof Puppeteer,
PuppeteerTypes.PuppeteerLaunchOptions,
PuppeteerTypes.Browser,
PuppeteerNewPageOptions
>,
): Promise<void> {
/*
// DO NOT USE YET! DOING SO DISABLES CACHE WHICH IS 50% PERFORMANCE HIT!
@@ -214,7 +233,12 @@ export class PuppeteerPlugin extends BrowserPlugin<
}
protected _isChromiumBasedBrowser(
_launchContext: LaunchContext<typeof Puppeteer, PuppeteerTypes.PuppeteerLaunchOptions, PuppeteerTypes.Browser, PuppeteerNewPageOptions>,
_launchContext: LaunchContext<
typeof Puppeteer,
PuppeteerTypes.PuppeteerLaunchOptions,
PuppeteerTypes.Browser,
PuppeteerNewPageOptions
>,
): boolean {
return true;
}
packages/browser-pool/src/utils.ts
+25 -27
View File
@@ -17,30 +17,28 @@ export type InferBrowserPluginArray<
Input extends readonly unknown[],
// The results of this type
Result extends BrowserPlugin[] = [],
> =
// If the input is a tuple or a readonly array (`[] as const`), get the first and the rest of the values
Input extends readonly [infer FirstValue, ...infer Rest] | [infer FirstValue, ...infer Rest]
// If the first value is a PlaywrightPlugin
? FirstValue extends PlaywrightPlugin
// Add it to the result, and continue parsing
? InferBrowserPluginArray<Rest, [...Result, PlaywrightPlugin]>
// Else if the first value is a PuppeteerPlugin
: FirstValue extends PuppeteerPlugin
// Add it to the result, and continue parsing
? InferBrowserPluginArray<Rest, [...Result, PuppeteerPlugin]>
// Return never as it isn't a valid type
: never
// If there's no more inputs to parse
: Input extends []
// Return the results
? Result
// If the input is a general array of elements (not a tuple), infer it's values type
: Input extends readonly (infer U)[]
// If the values are a union of the plugins
? [U] extends [PuppeteerPlugin | PlaywrightPlugin]
// Return an array of the union
? U[]
// Return never as it isn't a valid type
: never
// Return the result
: Result;
> = Input extends readonly [infer FirstValue, ...infer Rest] | [infer FirstValue, ...infer Rest] // If the input is a tuple or a readonly array (`[] as const`), get the first and the rest of the values
? // If the first value is a PlaywrightPlugin
FirstValue extends PlaywrightPlugin
? // Add it to the result, and continue parsing
InferBrowserPluginArray<Rest, [...Result, PlaywrightPlugin]>
: // Else if the first value is a PuppeteerPlugin
FirstValue extends PuppeteerPlugin
? // Add it to the result, and continue parsing
InferBrowserPluginArray<Rest, [...Result, PuppeteerPlugin]>
: // Return never as it isn't a valid type
never
: // If there's no more inputs to parse
Input extends []
? // Return the results
Result
: // If the input is a general array of elements (not a tuple), infer it's values type
Input extends readonly (infer U)[]
? // If the values are a union of the plugins
[U] extends [PuppeteerPlugin | PlaywrightPlugin]
? // Return an array of the union
U[]
: // Return never as it isn't a valid type
never
: // Return the result
Result;
packages/browser-pool/tab-as-a-container/background.js
+37 -21
View File
@@ -95,14 +95,18 @@ chrome.webRequest.onBeforeSendHeaders.addListener(
if (header.name.toLowerCase() === 'cookie') {
const id = keyFromTabId(getOpenerId(details.tabId));
const fixedCookies = header.value.split('; ').filter((x) => x.startsWith(id)).map((x) => x.slice(id.length)).join('; ');
const fixedCookies = header.value
.split('; ')
.filter((x) => x.startsWith(id))
.map((x) => x.slice(id.length))
.join('; ');
header.value = fixedCookies;
}
// Sometimes Chrome makes a request on a ghost tab.
// We don't want these in order to prevent cluttering cookies.
// Yes, `webNavigation.onComitted` is emitted and `webNavigation.onCreatedNavigationTarget` is not.
if (header.name.toLowerCase() === 'purpose' && header.value === 'prefetch' && !(counter.has(details.tabId))) {
if (header.name.toLowerCase() === 'purpose' && header.value === 'prefetch' && !counter.has(details.tabId)) {
// eslint-disable-next-line no-console
console.log(details);
return {
@@ -111,7 +115,7 @@ chrome.webRequest.onBeforeSendHeaders.addListener(
}
// This one is for Firefox
if (header.name.toLowerCase() === 'x-moz' && header.value === 'prefetch' && !(counter.has(details.tabId))) {
if (header.name.toLowerCase() === 'x-moz' && header.value === 'prefetch' && !counter.has(details.tabId)) {
// eslint-disable-next-line no-console
console.log(details);
return {
@@ -134,7 +138,9 @@ chrome.webRequest.onBeforeSendHeaders.addListener(
}
return {
requestHeaders: details.requestHeaders.filter((header) => header.name.toLowerCase() !== 'cookie' || header.value !== ''),
requestHeaders: details.requestHeaders.filter(
(header) => header.name.toLowerCase() !== 'cookie' || header.value !== '',
),
};
},
{ urls: ['<all_urls>'] },
@@ -152,13 +158,15 @@ chrome.webRequest.onHeadersReceived.addListener(
const openerId = getOpenerId(details.tabId);
header.value = parts.map((part) => {
const equalsIndex = part.indexOf('=');
if (equalsIndex === -1) {
return `${keyFromTabId(openerId)}=${part.trimStart()}`;
}
return keyFromTabId(openerId) + part.trimStart();
}).join('\n');
header.value = parts
.map((part) => {
const equalsIndex = part.indexOf('=');
if (equalsIndex === -1) {
return `${keyFromTabId(openerId)}=${part.trimStart()}`;
}
return keyFromTabId(openerId) + part.trimStart();
})
.join('\n');
}
}
@@ -187,13 +195,17 @@ chrome.tabs.onRemoved.addListener(async (tabId) => {
const id = keyFromTabId(opener);
chrome.cookies.getAll({}, async (cookies) => {
await Promise.allSettled(cookies.filter((cookie) => cookie.name.startsWith(id)).map((cookie) => {
return chrome.cookies.remove({
name: cookie.name,
url: getCookieURL(cookie),
storeId: cookie.storeId,
});
}));
await Promise.allSettled(
cookies
.filter((cookie) => cookie.name.startsWith(id))
.map((cookie) => {
return chrome.cookies.remove({
name: cookie.name,
url: getCookieURL(cookie),
storeId: cookie.storeId,
});
}),
);
});
});
@@ -246,7 +258,7 @@ const getNextLocalhostIp = (openerId) => {
}
// [127.0.0.1 - 127.255.255.254] = 1 * 255 * 255 * 254 = 16 516 350
while (localhostIpCache.length >= (1 * 255 * 255 * 254)) {
while (localhostIpCache.length >= 1 * 255 * 255 * 254) {
localhostIpCache.delete(localhostIpCache.keys().next().value);
}
@@ -341,7 +353,9 @@ const onCompleted = async (details) => {
// Different protocols are required, otherwise `onCompleted` won't be emitted.
const result = await routes[route](details, body);
if (result !== undefined) {
await chrome.tabs.update(details.tabId, { url: `about:blank#${encodeURIComponent(JSON.stringify(result))}` });
await chrome.tabs.update(details.tabId, {
url: `about:blank#${encodeURIComponent(JSON.stringify(result))}`,
});
}
}
} catch {
@@ -400,7 +414,9 @@ chrome.webNavigation.onCompleted.addListener(onCompleted);
window.totallyRandomString = true;
const code = "'use strict'; const tabId = '${getOpenerId(details.tabId)}'; (() => {\\n" + ${JSON.stringify(contentText)} + "\\n})();\\n";
const code = "'use strict'; const tabId = '${getOpenerId(
details.tabId,
)}'; (() => {\\n" + ${JSON.stringify(contentText)} + "\\n})();\\n";
${executeCodeInPageContext}
})();
`,
packages/browser-pool/tab-as-a-container/content.js
+37 -18
View File
@@ -177,7 +177,9 @@ const FakeStorage = class Storage {
}
if (arguments.length === 0) {
throw fixStack(new TypeError(`Failed to execute 'key' on 'Storage': 1 argument required, but only 0 present.`));
throw fixStack(
new TypeError(`Failed to execute 'key' on 'Storage': 1 argument required, but only 0 present.`),
);
}
index = NumberIsFinite(index) ? index : 0;
@@ -208,7 +210,9 @@ const FakeStorage = class Storage {
}
if (arguments.length === 0) {
throw fixStack(new TypeError(`Failed to execute 'getItem' on 'Storage': 1 argument required, but only 0 present.`));
throw fixStack(
new TypeError(`Failed to execute 'getItem' on 'Storage': 1 argument required, but only 0 present.`),
);
}
return StoragePrototype.getItem.call(priv.storage, priv.prefix + key);
@@ -221,7 +225,9 @@ const FakeStorage = class Storage {
}
if (arguments.length === 0) {
throw fixStack(new TypeError(`Failed to execute 'removeItem' on 'Storage': 1 argument required, but only 0 present.`));
throw fixStack(
new TypeError(`Failed to execute 'removeItem' on 'Storage': 1 argument required, but only 0 present.`),
);
}
StoragePrototype.removeItem.call(priv.storage, priv.prefix + key);
@@ -234,7 +240,11 @@ const FakeStorage = class Storage {
}
if (arguments.length === 0 || arguments.length === 1) {
throw fixStack(new TypeError(`Failed to execute 'setItem' on 'Storage': 2 arguments required, but only ${arguments.length} present.`));
throw fixStack(
new TypeError(
`Failed to execute 'setItem' on 'Storage': 2 arguments required, but only ${arguments.length} present.`,
),
);
}
StoragePrototype.setItem.call(priv.storage, priv.prefix + key, value);
@@ -257,7 +267,9 @@ const createStorage = ({ storage, prefix }) => {
// getPrototypeOf: (target) => {},
defineProperty: (target, key, descriptor) => {
if ('set' in descriptor || 'get' in descriptor) {
throw fixStack(new TypeError(`Failed to set a named property on 'Storage': Accessor properties are not allowed.`));
throw fixStack(
new TypeError(`Failed to set a named property on 'Storage': Accessor properties are not allowed.`),
);
}
FakeStoragePrototype.setItem.call(target, key, descriptor.value);
@@ -363,17 +375,18 @@ const createStorage = ({ storage, prefix }) => {
const toHide = new WeakMap();
for (const Type of [Function, Object, Array]) {
const create = (fallback) => function () {
if (this instanceof FakeStorage) {
return '[object Storage]';
}
const create = (fallback) =>
function () {
if (this instanceof FakeStorage) {
return '[object Storage]';
}
if (WeakMapPrototype.has.call(toHide, this)) {
return `function ${WeakMapPrototype.get.call(toHide, this)}() { [native code] }`;
}
if (WeakMapPrototype.has.call(toHide, this)) {
return `function ${WeakMapPrototype.get.call(toHide, this)}() { [native code] }`;
}
return fallback.call(this);
};
return fallback.call(this);
};
const toString = create(Type.prototype.toString);
const toLocaleString = create(Type.prototype.toLocaleString);
@@ -400,8 +413,12 @@ try {
const fakeLocalStorage = createStorage({ storage: sessionStorage, prefix: 'l.' });
const fakeSessionStorage = createStorage({ storage: sessionStorage, prefix: 's.' });
const getLocalStorage = function localStorage() { return fakeLocalStorage; };
const getSessionStorage = function sessionStorage() { return fakeSessionStorage; };
const getLocalStorage = function localStorage() {
return fakeLocalStorage;
};
const getSessionStorage = function sessionStorage() {
return fakeSessionStorage;
};
WeakMapPrototype.set.call(toHide, FakeStorage, 'Storage');
WeakMapPrototype.set.call(toHide, FakeStoragePrototype.key, 'key');
@@ -450,7 +467,9 @@ try {
const getCookie = function cookie() {
try {
const cookies = StringSplitSafe(realGetCookie.call(this), '; ');
const filtered = ArrayPrototype.filter.call(cookies, (cookie) => StringPrototype.startsWith.call(cookie, tabPrefix));
const filtered = ArrayPrototype.filter.call(cookies, (cookie) =>
StringPrototype.startsWith.call(cookie, tabPrefix),
);
const mapped = ArrayPrototype.map.call(filtered, (cookie) => {
const result = StringPrototype.slice.call(cookie, tabPrefix.length);
@@ -472,7 +491,7 @@ try {
const delimiterIndex = StringPrototype.indexOf.call(cookieString, ';');
const equalsIndex = StringPrototype.indexOf.call(cookieString, '=');
if ((equalsIndex === -1) || ((delimiterIndex !== -1) && (equalsIndex > delimiterIndex))) {
if (equalsIndex === -1 || (delimiterIndex !== -1 && equalsIndex > delimiterIndex)) {
cookieString = `=${cookieString}`;
}
packages/browser-pool/tab-as-a-container/manifest.json
+2 -6
View File
@@ -3,9 +3,7 @@
"name": "Tab as a Container",
"version": "1.0.0",
"background": {
"scripts": [
"background.js"
],
"scripts": ["background.js"],
"persistent": true
},
"permissions": [
@@ -18,8 +16,6 @@
"proxy",
"<all_urls>"
],
"web_accessible_resources": [
"content.js"
],
"web_accessible_resources": ["content.js"],
"incognito": "not_allowed"
}
packages/browser-pool/test/changing-page-options.test.ts
+19 -12
View File
@@ -12,17 +12,20 @@ import { createProxyServer } from '../../../test/browser-pool/browser-plugins/cr
describe.each([
['Puppeteer', new PuppeteerPlugin(puppeteer, { useIncognitoPages: true })],
['Playwright', new PlaywrightPlugin(playwright.chromium, {
useIncognitoPages: true,
launchOptions: {
args: [
// Exclude loopback interface from proxy bypass list,
// so the request to localhost goes through proxy.
// This way there's no need for a 3rd party server.
'--proxy-bypass-list=<-loopback>',
],
},
})], // Chromium is faster than firefox and webkit
[
'Playwright',
new PlaywrightPlugin(playwright.chromium, {
useIncognitoPages: true,
launchOptions: {
args: [
// Exclude loopback interface from proxy bypass list,
// so the request to localhost goes through proxy.
// This way there's no need for a 3rd party server.
'--proxy-bypass-list=<-loopback>',
],
},
}),
], // Chromium is faster than firefox and webkit
])('BrowserPool - %s - prePageCreateHooks > should allow changing pageOptions', (_, plugin) => {
let target: http.Server;
let protectedProxy: ProxyChainServer;
@@ -44,7 +47,11 @@ describe.each([
});
test('should allow changing pageOptions', async () => {
const hook: PrePageCreateHook<PlaywrightController | PuppeteerController> = (_pageId, _controller, pageOptions) => {
const hook: PrePageCreateHook<PlaywrightController | PuppeteerController> = (
_pageId,
_controller,
pageOptions,
) => {
if (!pageOptions) {
expect(false).toBe(true);
return;
packages/browser-pool/test/multiple-plugins.test.ts
+2 -8
View File
@@ -13,10 +13,7 @@ describe('BrowserPool - Using multiple plugins', () => {
beforeEach(async () => {
vitest.clearAllMocks();
browserPool = new BrowserPool({
browserPlugins: [
chromePlugin,
firefoxPlugin,
],
browserPlugins: [chromePlugin, firefoxPlugin],
closeInactiveBrowserAfterSecs: 2,
retireInactiveBrowserAfterSecs: 30,
});
@@ -36,10 +33,7 @@ describe('BrowserPool - Using multiple plugins', () => {
});
test('should loop through plugins round-robin', async () => {
const correctPluginOrder = [
chromePlugin,
firefoxPlugin,
];
const correctPluginOrder = [chromePlugin, firefoxPlugin];
const pagePromises = correctPluginOrder.map(async () => browserPool.newPage());
packages/browser-pool/test/no-hybrid-plugins.test.ts
+14 -6
View File
@@ -4,15 +4,23 @@ import puppeteer from 'puppeteer';
describe('Hybrid BrowserPool plugins should not be allowed', () => {
test('mixing Puppeteer with Playwright should throw an error', () => {
expect(() => new BrowserPool({
browserPlugins: [new PuppeteerPlugin(puppeteer), new PlaywrightPlugin(playwright.chromium)],
}),
expect(
() =>
new BrowserPool({
browserPlugins: [new PuppeteerPlugin(puppeteer), new PlaywrightPlugin(playwright.chromium)],
}),
).toThrowError();
});
test('providing multiple different Playwright plugins should not throw an error', () => {
expect(() => new BrowserPool({
browserPlugins: [new PlaywrightPlugin(playwright.chromium), new PlaywrightPlugin(playwright.firefox)],
})).not.toThrowError();
expect(
() =>
new BrowserPool({
browserPlugins: [
new PlaywrightPlugin(playwright.chromium),
new PlaywrightPlugin(playwright.firefox),
],
}),
).not.toThrowError();
});
});
packages/browser-pool/tsconfig.build.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
}
packages/browser-pool/tsconfig.json
+2 -2
View File
@@ -1,4 +1,4 @@
{
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
}
packages/cheerio-crawler/src/internals/cheerio-crawler.ts
+41 -28
View File
@@ -12,12 +12,7 @@ import type {
Configuration,
RequestProvider,
} from '@crawlee/http';
import {
HttpCrawler,
enqueueLinks,
Router,
resolveBaseUrlForEnqueueLinksFiltering,
} from '@crawlee/http';
import { HttpCrawler, enqueueLinks, Router, resolveBaseUrlForEnqueueLinksFiltering } from '@crawlee/http';
import type { Dictionary } from '@crawlee/types';
import { extractUrlsFromCheerio } from '@crawlee/utils';
import type { CheerioOptions } from 'cheerio';
@@ -28,22 +23,22 @@ import { WritableStream } from 'htmlparser2/lib/WritableStream';
export type CheerioErrorHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
> = ErrorHandler<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlerOptions<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>> {}
> extends HttpCrawlerOptions<CheerioCrawlingContext<UserData, JSONData>> {}
export type CheerioHook<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
> = InternalHttpHook<CheerioCrawlingContext<UserData, JSONData>>;
export interface CheerioCrawlingContext<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> extends InternalHttpCrawlingContext<UserData, JSONData, CheerioCrawler> {
> extends InternalHttpCrawlingContext<UserData, JSONData, CheerioCrawler> {
/**
* The [Cheerio](https://cheerio.js.org/) object with parsed HTML.
* Cheerio is available only for HTML and XML content types.
@@ -69,7 +64,7 @@ export interface CheerioCrawlingContext<
export type CheerioRequestHandler<
UserData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
JSONData extends Dictionary = any, // with default to Dictionary we cant use a typed router in untyped crawler
> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
> = RequestHandler<CheerioCrawlingContext<UserData, JSONData>>;
/**
* Provides a framework for the parallel crawling of web pages using plain HTTP requests and
@@ -157,16 +152,23 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
super(options, config);
}
protected override async _parseHTML(response: IncomingMessage, isXml: boolean, crawlingContext: CheerioCrawlingContext) {
protected override async _parseHTML(
response: IncomingMessage,
isXml: boolean,
crawlingContext: CheerioCrawlingContext,
) {
const dom = await this._parseHtmlToDom(response, isXml);
const $ = cheerio.load(dom as string, {
xmlMode: isXml,
// Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
// and not good for scraping. It also does not have a great streaming interface.
// Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
_useHtmlParser2: true,
} as CheerioOptions);
const $ = cheerio.load(
dom as string,
{
xmlMode: isXml,
// Recent versions of cheerio use parse5 as the HTML parser/serializer. It's more strict than htmlparser2
// and not good for scraping. It also does not have a great streaming interface.
// Here we tell cheerio to use htmlparser2 for serialization, otherwise the conflict produces weird errors.
_useHtmlParser2: true,
} as CheerioOptions,
);
return {
dom,
@@ -188,15 +190,16 @@ export class CheerioCrawler extends HttpCrawler<CheerioCrawlingContext> {
protected async _parseHtmlToDom(response: IncomingMessage, isXml: boolean) {
return new Promise((resolve, reject) => {
const domHandler = new DomHandler((err, dom) => {
if (err) reject(err);
else resolve(dom);
}, { xmlMode: isXml });
const domHandler = new DomHandler(
(err, dom) => {
if (err) reject(err);
else resolve(dom);
},
{ xmlMode: isXml },
);
const parser = new WritableStream(domHandler, { decodeEntities: true, xmlMode: isXml });
parser.on('error', reject);
response
.on('error', reject)
.pipe(parser);
response.on('error', reject).pipe(parser);
});
}
@@ -215,7 +218,13 @@ interface EnqueueLinksInternalOptions {
}
/** @internal */
export async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, originalRequestUrl, finalRequestUrl }: EnqueueLinksInternalOptions) {
export async function cheerioCrawlerEnqueueLinks({
options,
$,
requestQueue,
originalRequestUrl,
finalRequestUrl,
}: EnqueueLinksInternalOptions) {
if (!$) {
throw new Error('Cannot enqueue links because the DOM is not available.');
}
@@ -227,7 +236,11 @@ export async function cheerioCrawlerEnqueueLinks({ options, $, requestQueue, ori
userProvidedBaseUrl: options?.baseUrl,
});
const urls = extractUrlsFromCheerio($, options?.selector ?? 'a', options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl);
const urls = extractUrlsFromCheerio(
$,
options?.selector ?? 'a',
options?.baseUrl ?? finalRequestUrl ?? originalRequestUrl,
);
return enqueueLinks({
requestQueue,
packages/cheerio-crawler/test/migration.test.ts
+26 -18
View File
@@ -36,11 +36,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handlePageFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "requestHandler" and "handlePageFunction" were provided in the crawler options.`,
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`As such, "requestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandler']).toBe(newHandler);
@@ -56,10 +58,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handlePageFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handlePageFunction" has been renamed to "requestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handlePageFunction" to "requestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['requestHandler']).toBe(oldHandler);
@@ -96,11 +100,13 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`Both "failedRequestHandler" and "handleFailedRequestFunction" were provided in the crawler options.`,
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`As such, "failedRequestHandler" will be used instead.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(newHandler);
@@ -117,10 +123,12 @@ describe('Moving from handleRequest* to requestHandler*', () => {
handleFailedRequestFunction: oldHandler,
});
expect(warningSpy).toHaveBeenCalledWith<[string]>([
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'));
expect(warningSpy).toHaveBeenCalledWith<[string]>(
[
`"handleFailedRequestFunction" has been renamed to "failedRequestHandler", and will be removed in a future version.`,
`The provided value will be used, but you should rename "handleFailedRequestFunction" to "failedRequestHandler" in your crawler options.`,
].join('\n'),
);
// eslint-disable-next-line dot-notation -- accessing private property
expect(crawler['failedRequestHandler']).toBe(oldHandler);
packages/cheerio-crawler/test/tsconfig.json
+1 -4
View File
@@ -1,9 +1,6 @@
{
"extends": "../../../tsconfig.json",
"include": [
"**/*",
"../../**/*"
],
"include": ["**/*", "../../**/*"],
"compilerOptions": {
"types": ["vitest/globals"]
}
packages/cheerio-crawler/tsconfig.build.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
}
packages/cheerio-crawler/tsconfig.json
+2 -2
View File
@@ -1,4 +1,4 @@
{
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
}
packages/cli/src/commands/CreateProjectCommand.ts
+54 -34
View File
@@ -35,42 +35,54 @@ async function rewrite(path: string, replacer: (from: string) => string) {
}
}
async function withRetries<F extends (...args: unknown[]) => unknown>(func: F, retries: number, label: string): Promise<Awaited<ReturnType<F>>> {
async function withRetries<F extends (...args: unknown[]) => unknown>(
func: F,
retries: number,
label: string,
): Promise<Awaited<ReturnType<F>>> {
let attempt = 0;
let lastError: any;
while (attempt < retries) {
try {
return await func() as Awaited<ReturnType<F>>;
return (await func()) as Awaited<ReturnType<F>>;
} catch (error: any) {
attempt++;
lastError = error;
if (attempt < retries) {
console.warn(`${colors.yellow(`[${label}]`)}: Attempt ${attempt + 1} of ${retries} failed, and will be retried`, error.message || error);
console.warn(
`${colors.yellow(`[${label}]`)}: Attempt ${attempt + 1} of ${retries} failed, and will be retried`,
error.message || error,
);
}
// Wait 2500ms + (2500 * retries) before giving up to give it some time between retries
await setTimeout(2500 + (2500 * attempt));
await setTimeout(2500 + 2500 * attempt);
}
}
throw new Error(`${colors.red(`[${label}]`)}: All ${retries} attempts failed, and will not be retried\n\n${lastError.stack || lastError}`);
throw new Error(
`${colors.red(`[${label}]`)}: All ${retries} attempts failed, and will not be retried\n\n${
lastError.stack || lastError
}`,
);
}
async function downloadTemplateFilesToDisk(template: Template, destinationDirectory: string) {
const promises: Promise<void>[] = [];
for (const file of template.files) {
const promise = async () => downloadFile(file.url).then(async (buffer) => {
// Make sure the folder for the file exists
const fileDirName = dirname(file.path);
const fileFolder = resolve(destinationDirectory, fileDirName);
await ensureDir(fileFolder);
const promise = async () =>
downloadFile(file.url).then(async (buffer) => {
// Make sure the folder for the file exists
const fileDirName = dirname(file.path);
const fileFolder = resolve(destinationDirectory, fileDirName);
await ensureDir(fileFolder);
// Write the actual file
await writeFile(resolve(destinationDirectory, file.path), buffer);
});
// Write the actual file
await writeFile(resolve(destinationDirectory, file.path), buffer);
});
promises.push(withRetries(promise, 3, `Template: ${template.name}, file: ${file.path}`));
}
@@ -127,19 +139,21 @@ export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectAr
// Check proper format of projectName
if (!projectName) {
const projectNamePrompt = await prompt([{
name: 'projectName',
message: 'Name of the new project folder:',
type: 'input',
validate: (promptText) => {
try {
validateProjectName(promptText);
} catch (err: any) {
return err.message;
}
return true;
const projectNamePrompt = await prompt([
{
name: 'projectName',
message: 'Name of the new project folder:',
type: 'input',
validate: (promptText) => {
try {
validateProjectName(promptText);
} catch (err: any) {
return err.message;
}
return true;
},
},
}]);
]);
({ projectName } = projectNamePrompt);
} else {
validateProjectName(projectName);
@@ -152,13 +166,15 @@ export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectAr
}));
if (!template) {
const answer = await prompt([{
type: 'list',
name: 'template',
message: 'Please select the template for your new Crawlee project',
default: choices[0],
choices,
}]);
const answer = await prompt([
{
type: 'list',
name: 'template',
message: 'Please select the template for your new Crawlee project',
default: choices[0],
choices,
},
]);
template = answer.template;
}
@@ -178,12 +194,16 @@ export class CreateProjectCommand<T> implements CommandModule<T, CreateProjectAr
const templateData = manifest.templates.find((item) => item.name === template)!;
await downloadTemplateFilesToDisk(templateData, projectDir);
await rewrite(resolve(projectDir, 'package.json'), (pkg) => pkg.replace(/"name": "[\w-]+"/, `"name": "${projectName}"`));
await rewrite(resolve(projectDir, 'package.json'), (pkg) =>
pkg.replace(/"name": "[\w-]+"/, `"name": "${projectName}"`),
);
// Run npm install in project dir.
const npm = /^win/.test(process.platform) ? 'npm.cmd' : 'npm';
execSync(`${npm} install`, { cwd: projectDir, stdio: 'inherit' });
console.log(colors.green(`Project ${projectName} was created. To run it, run "cd ${projectName}" and "npm start".`));
console.log(
colors.green(`Project ${projectName} was created. To run it, run "cd ${projectName}" and "npm start".`),
);
}
}
packages/cli/src/commands/InstallPlaywrightBrowsersCommand.ts
+7 -2
View File
@@ -20,7 +20,8 @@ export class InstallPlaywrightBrowsersCommand<T> implements CommandModule<T, Ins
alias: 'f',
default: false,
type: 'boolean',
describe: 'Use `--force` to force installation of browsers even if the environment is marked as having them.',
describe:
'Use `--force` to force installation of browsers even if the environment is marked as having them.',
});
return args as Argv<InstallPlaywrightBrowsersArgs>;
@@ -33,7 +34,11 @@ export class InstallPlaywrightBrowsersCommand<T> implements CommandModule<T, Ins
return;
}
console.warn(ansiColors.yellow('Installing Playwright browsers in an environment where browsers have already been installed...'));
console.warn(
ansiColors.yellow(
'Installing Playwright browsers in an environment where browsers have already been installed...',
),
);
} else {
console.log(ansiColors.green('Installing Playwright browsers...'));
}
packages/cli/src/index.ts
+8 -4
View File
@@ -1,7 +1,7 @@
#!/usr/bin/env node
// eslint-disable-next-line @typescript-eslint/no-var-requires
require('yargonaut')
require('yargonaut') //
.style('blue')
.style('yellow', 'required')
.helpStyle('green')
@@ -29,10 +29,14 @@ function getCLIVersion(): string {
}
}
const cli = yargs.scriptName('crawlee')
const cli = yargs
.scriptName('crawlee')
.version(getCLIVersion())
.usage('Usage: $0 <command> [options]')
.example('$0 run --no-purge', 'Runs the project in current working directory and disables automatic purging of default storages')
.example(
'$0 run --no-purge',
'Runs the project in current working directory and disables automatic purging of default storages',
)
.alias('v', 'version')
.alias('h', 'help')
.command(new CreateProjectCommand())
@@ -42,7 +46,7 @@ const cli = yargs.scriptName('crawlee')
.strict();
void (async () => {
const args = await cli.parse(process.argv.slice(2)) as { _: string[] };
const args = (await cli.parse(process.argv.slice(2))) as { _: string[] };
if (args._.length === 0) {
yargs.showHelp();
packages/cli/tsconfig.build.json
+5 -5
View File
@@ -1,7 +1,7 @@
{
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
"extends": "../../tsconfig.build.json",
"compilerOptions": {
"outDir": "./dist"
},
"include": ["src/**/*"]
}
packages/cli/tsconfig.json
+2 -2
View File
@@ -1,4 +1,4 @@
{
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
"extends": "../../tsconfig.json",
"include": ["src/**/*"]
}
packages/core/src/autoscaling/autoscaled_pool.ts
+27 -22
View File
@@ -214,25 +214,28 @@ export class AutoscaledPool {
options: AutoscaledPoolOptions,
private readonly config = Configuration.getGlobalConfig(),
) {
ow(options, ow.object.exactShape({
runTaskFunction: ow.function,
isFinishedFunction: ow.function,
isTaskReadyFunction: ow.function,
maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
minConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
desiredConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
desiredConcurrencyRatio: ow.optional.number.greaterThan(0).lessThan(1),
scaleUpStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
scaleDownStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
maybeRunIntervalSecs: ow.optional.number.greaterThan(0),
loggingIntervalSecs: ow.any(ow.number.greaterThan(0), ow.nullOrUndefined),
autoscaleIntervalSecs: ow.optional.number.greaterThan(0),
taskTimeoutSecs: ow.optional.number.greaterThanOrEqual(0),
systemStatusOptions: ow.optional.object,
snapshotterOptions: ow.optional.object,
log: ow.optional.object,
maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
}));
ow(
options,
ow.object.exactShape({
runTaskFunction: ow.function,
isFinishedFunction: ow.function,
isTaskReadyFunction: ow.function,
maxConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
minConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
desiredConcurrency: ow.optional.number.integer.greaterThanOrEqual(1),
desiredConcurrencyRatio: ow.optional.number.greaterThan(0).lessThan(1),
scaleUpStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
scaleDownStepRatio: ow.optional.number.greaterThan(0).lessThan(1),
maybeRunIntervalSecs: ow.optional.number.greaterThan(0),
loggingIntervalSecs: ow.any(ow.number.greaterThan(0), ow.nullOrUndefined),
autoscaleIntervalSecs: ow.optional.number.greaterThan(0),
taskTimeoutSecs: ow.optional.number.greaterThanOrEqual(0),
systemStatusOptions: ow.optional.object,
snapshotterOptions: ow.optional.object,
log: ow.optional.object,
maxTasksPerMinute: ow.optional.number.integerOrInfinite.greaterThanOrEqual(1),
}),
);
const {
runTaskFunction,
@@ -241,7 +244,7 @@ export class AutoscaledPool {
maxConcurrency = 200,
minConcurrency = 1,
desiredConcurrency,
desiredConcurrencyRatio = 0.90,
desiredConcurrencyRatio = 0.9,
scaleUpStepRatio = 0.1,
scaleDownStepRatio = 0.1,
maybeRunIntervalSecs = 0.5,
@@ -421,8 +424,10 @@ export class AutoscaledPool {
let timeout: NodeJS.Timeout;
if (timeoutSecs) {
timeout = setTimeout(() => {
const err = new Error('The pool\'s running tasks did not finish'
+ `in ${timeoutSecs} secs after pool.pause() invocation.`);
const err = new Error(
"The pool's running tasks did not finish" +
`in ${timeoutSecs} secs after pool.pause() invocation.`,
);
reject(err);
}, timeoutSecs);
}
packages/core/src/autoscaling/snapshotter.ts
+59 -23
View File
@@ -67,10 +67,27 @@ export interface SnapshotterOptions {
config?: Configuration;
}
interface MemorySnapshot { createdAt: Date; isOverloaded: boolean; usedBytes?: number }
interface CpuSnapshot { createdAt: Date; isOverloaded: boolean; usedRatio: number; ticks?: { idle: number; total: number } }
interface EventLoopSnapshot { createdAt: Date; isOverloaded: boolean; exceededMillis: number }
interface ClientSnapshot { createdAt: Date; isOverloaded: boolean; rateLimitErrorCount: number }
interface MemorySnapshot {
createdAt: Date;
isOverloaded: boolean;
usedBytes?: number;
}
interface CpuSnapshot {
createdAt: Date;
isOverloaded: boolean;
usedRatio: number;
ticks?: { idle: number; total: number };
}
interface EventLoopSnapshot {
createdAt: Date;
isOverloaded: boolean;
exceededMillis: number;
}
interface ClientSnapshot {
createdAt: Date;
isOverloaded: boolean;
rateLimitErrorCount: number;
}
/**
* Creates snapshots of system resources at given intervals and marks the resource
@@ -124,17 +141,20 @@ export class Snapshotter {
* @param [options] All `Snapshotter` configuration options.
*/
constructor(options: SnapshotterOptions = {}) {
ow(options, ow.object.exactShape({
eventLoopSnapshotIntervalSecs: ow.optional.number,
clientSnapshotIntervalSecs: ow.optional.number,
snapshotHistorySecs: ow.optional.number,
maxBlockedMillis: ow.optional.number,
maxUsedMemoryRatio: ow.optional.number,
maxClientErrors: ow.optional.number,
log: ow.optional.object,
client: ow.optional.object,
config: ow.optional.object,
}));
ow(
options,
ow.object.exactShape({
eventLoopSnapshotIntervalSecs: ow.optional.number,
clientSnapshotIntervalSecs: ow.optional.number,
snapshotHistorySecs: ow.optional.number,
maxBlockedMillis: ow.optional.number,
maxUsedMemoryRatio: ow.optional.number,
maxClientErrors: ow.optional.number,
log: ow.optional.object,
client: ow.optional.object,
config: ow.optional.object,
}),
);
const {
eventLoopSnapshotIntervalSecs = 0.5,
@@ -176,12 +196,17 @@ export class Snapshotter {
} else {
const { totalBytes } = await this._getMemoryInfo();
this.maxMemoryBytes = Math.ceil(totalBytes * this.config.get('availableMemoryRatio')!);
this.log.debug(`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. `
+ 'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.');
this.log.debug(
`Setting max memory of this run to ${Math.round(this.maxMemoryBytes / 1024 / 1024)} MB. ` +
'Use the CRAWLEE_MEMORY_MBYTES or CRAWLEE_AVAILABLE_MEMORY_RATIO environment variable to override it.',
);
}
// Start snapshotting.
this.eventLoopInterval = betterSetInterval(this._snapshotEventLoop.bind(this), this.eventLoopSnapshotIntervalMillis);
this.eventLoopInterval = betterSetInterval(
this._snapshotEventLoop.bind(this),
this.eventLoopSnapshotIntervalMillis,
);
this.clientInterval = betterSetInterval(this._snapshotClient.bind(this), this.clientSnapshotIntervalMillis);
this.events.on(EventType.SYSTEM_INFO, this._snapshotCpu);
this.events.on(EventType.SYSTEM_INFO, this._snapshotMemory);
@@ -278,7 +303,11 @@ export class Snapshotter {
protected _memoryOverloadWarning(systemInfo: SystemInfo) {
const { memCurrentBytes } = systemInfo;
const createdAt = systemInfo.createdAt ? new Date(systemInfo.createdAt) : new Date();
if (this.lastLoggedCriticalMemoryOverloadAt && +createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS) return;
if (
this.lastLoggedCriticalMemoryOverloadAt &&
+createdAt < +this.lastLoggedCriticalMemoryOverloadAt + CRITICAL_OVERLOAD_RATE_LIMIT_MILLIS
)
return;
const maxDesiredMemoryBytes = this.maxUsedMemoryRatio * this.maxMemoryBytes!;
const reserveMemory = this.maxMemoryBytes! * (1 - this.maxUsedMemoryRatio) * RESERVE_MEMORY_RATIO;
@@ -287,9 +316,13 @@ export class Snapshotter {
if (isCriticalOverload) {
const usedPercentage = Math.round((memCurrentBytes! / this.maxMemoryBytes!) * 100);
const toMb = (bytes: number) => Math.round(bytes / (1024 ** 2));
this.log.warning('Memory is critically overloaded. '
+ `Using ${toMb(memCurrentBytes!)} MB of ${toMb(this.maxMemoryBytes!)} MB (${usedPercentage}%). Consider increasing available memory.`);
const toMb = (bytes: number) => Math.round(bytes / 1024 ** 2);
this.log.warning(
'Memory is critically overloaded. ' +
`Using ${toMb(memCurrentBytes!)} MB of ${toMb(
this.maxMemoryBytes!,
)} MB (${usedPercentage}%). Consider increasing available memory.`,
);
this.lastLoggedCriticalMemoryOverloadAt = createdAt;
}
}
@@ -371,7 +404,10 @@ export class Snapshotter {
* Removes snapshots that are older than the snapshotHistorySecs option
* from the array (destructively - in place).
*/
protected _pruneSnapshots(snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[], now: Date) {
protected _pruneSnapshots(
snapshots: MemorySnapshot[] | CpuSnapshot[] | EventLoopSnapshot[] | ClientSnapshot[],
now: Date,
) {
let oldCount = 0;
for (let i = 0; i < snapshots.length; i++) {
const { createdAt } = snapshots[i];
packages/core/src/autoscaling/system_status.ts
+21 -11
View File
@@ -126,15 +126,18 @@ export class SystemStatus {
private readonly snapshotter: Snapshotter;
constructor(options: SystemStatusOptions = {}) {
ow(options, ow.object.exactShape({
currentHistorySecs: ow.optional.number,
maxMemoryOverloadedRatio: ow.optional.number,
maxEventLoopOverloadedRatio: ow.optional.number,
maxCpuOverloadedRatio: ow.optional.number,
maxClientOverloadedRatio: ow.optional.number,
snapshotter: ow.optional.object,
config: ow.optional.object,
}));
ow(
options,
ow.object.exactShape({
currentHistorySecs: ow.optional.number,
maxMemoryOverloadedRatio: ow.optional.number,
maxEventLoopOverloadedRatio: ow.optional.number,
maxCpuOverloadedRatio: ow.optional.number,
maxClientOverloadedRatio: ow.optional.number,
snapshotter: ow.optional.object,
config: ow.optional.object,
}),
);
const {
currentHistorySecs = 5,
@@ -203,7 +206,11 @@ export class SystemStatus {
const cpuInfo = this._isCpuOverloaded(sampleDurationMillis);
const clientInfo = this._isClientOverloaded(sampleDurationMillis);
return {
isSystemIdle: !memInfo.isOverloaded && !eventLoopInfo.isOverloaded && !cpuInfo.isOverloaded && !clientInfo.isOverloaded,
isSystemIdle:
!memInfo.isOverloaded &&
!eventLoopInfo.isOverloaded &&
!cpuInfo.isOverloaded &&
!clientInfo.isOverloaded,
memInfo,
eventLoopInfo,
cpuInfo,
@@ -251,7 +258,10 @@ export class SystemStatus {
* Returns an object with sample information and an isOverloaded property
* set to true if at least the ratio of snapshots in the sample are overloaded.
*/
protected _isSampleOverloaded<T extends { createdAt: Date; isOverloaded: boolean }>(sample: T[], ratio: number): ClientInfo {
protected _isSampleOverloaded<T extends { createdAt: Date; isOverloaded: boolean }>(
sample: T[],
ratio: number,
): ClientInfo {
if (sample.length === 0) {
return {
isOverloaded: false,
packages/core/src/configuration.ts
+3 -1
View File
@@ -295,7 +295,9 @@ export class Configuration {
const logLevel = this.get('logLevel');
if (logLevel) {
const level = Number.isFinite(+logLevel) ? +logLevel : LogLevel[String(logLevel).toUpperCase() as unknown as LogLevel];
const level = Number.isFinite(+logLevel)
? +logLevel
: LogLevel[String(logLevel).toUpperCase() as unknown as LogLevel];
log.setLevel(level as LogLevel);
}
}
packages/core/src/cookie_utils.ts
+10 -4
View File
@@ -9,7 +9,9 @@ import { CookieParseError } from './session_pool/errors';
/**
* @internal
*/
export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> }): Cookie[] {
export function getCookiesFromResponse(
response: IncomingMessage | BrowserLikeResponse | { headers: Dictionary<string | string[]> },
): Cookie[] {
const headers = typeof response.headers === 'function' ? response.headers() : response.headers;
const cookieHeader = headers['set-cookie'] || '';
@@ -29,7 +31,7 @@ export function getCookiesFromResponse(response: IncomingMessage | BrowserLikeRe
* @internal
*/
export function getDefaultCookieExpirationDate(maxAgeSecs: number) {
return new Date(Date.now() + (maxAgeSecs * 1000));
return new Date(Date.now() + maxAgeSecs * 1000);
}
/**
@@ -59,7 +61,9 @@ export function toughCookieToBrowserPoolCookie(toughCookie: Cookie): CookieObjec
*/
export function browserPoolCookieToToughCookie(cookieObject: CookieObject, maxAgeSecs: number) {
const isExpiresValid = cookieObject.expires && typeof cookieObject.expires === 'number' && cookieObject.expires > 0;
const expires = isExpiresValid ? new Date(cookieObject.expires! * 1000) : getDefaultCookieExpirationDate(maxAgeSecs);
const expires = isExpiresValid
? new Date(cookieObject.expires! * 1000)
: getDefaultCookieExpirationDate(maxAgeSecs);
const domainHasLeadingDot = cookieObject.domain?.startsWith?.('.');
const domain = domainHasLeadingDot ? cookieObject.domain?.slice?.(1) : cookieObject.domain;
return new Cookie({
@@ -114,7 +118,9 @@ export function mergeCookies(url: string, sourceCookies: string[]): string {
});
if (similarKeyCookie) {
log.deprecated(`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`);
log.deprecated(
`Found cookies with similar name during cookie merging: '${cookie.key}' and '${similarKeyCookie.key}'`,
);
}
jar.setCookieSync(cookie, url);
packages/core/src/crawlers/crawler_commons.ts
+51 -19
View File
@@ -12,9 +12,10 @@ import type { Session } from '../session_pool/session';
import type { RequestQueueOperationOptions, Dataset, RecordOptions } from '../storages';
import { KeyValueStore } from '../storages';
// we need `Record<string & {}, unknown>` here, otherwise `Omit<Context>` is resolved badly
// eslint-disable-next-line
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary> extends Record<string & {}, unknown> {
export interface RestrictedCrawlingContext<UserData extends Dictionary = Dictionary>
// we need `Record<string & {}, unknown>` here, otherwise `Omit<Context>` is resolved badly
// eslint-disable-next-line
extends Record<string & {}, unknown> {
/**
* The original {@apilink Request} object.
*/
@@ -74,7 +75,9 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
/**
* Get a key-value store with given name or id, or the default one for the crawler.
*/
getKeyValueStore: (idOrName?: string) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue'>>;
getKeyValueStore: (
idOrName?: string,
) => Promise<Pick<KeyValueStore, 'id' | 'name' | 'getValue' | 'getAutoSavedValue' | 'setValue'>>;
/**
* A preconfigured logger for the request handler.
@@ -82,7 +85,8 @@ export interface RestrictedCrawlingContext<UserData extends Dictionary = Diction
log: Log;
}
export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary> extends RestrictedCrawlingContext<UserData> {
export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary = Dictionary>
extends RestrictedCrawlingContext<UserData> {
id: string;
session?: Session;
@@ -120,7 +124,7 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
*/
enqueueLinks(
options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>
options?: ReadonlyDeep<Omit<EnqueueLinksOptions, 'requestQueue'>> & Pick<EnqueueLinksOptions, 'requestQueue'>,
): Promise<BatchAddRequestsResult>;
/**
@@ -154,12 +158,19 @@ export interface CrawlingContext<Crawler = unknown, UserData extends Dictionary
* @experimental
*/
export class RequestHandlerResult {
private _keyValueStoreChanges: Record<string, Record<string, { changedValue: unknown; options?: RecordOptions }>> = {};
private _keyValueStoreChanges: Record<string, Record<string, { changedValue: unknown; options?: RecordOptions }>> =
{};
private pushDataCalls: Parameters<RestrictedCrawlingContext['pushData']>[] = [];
private addRequestsCalls: Parameters<RestrictedCrawlingContext['addRequests']>[] = [];
private enqueueLinksCalls: Parameters<RestrictedCrawlingContext['enqueueLinks']>[] = [];
constructor(private config: Configuration, private crawleeStateKey: string) {}
constructor(
private config: Configuration,
private crawleeStateKey: string,
) {}
/**
* A record of calls to {@apilink RestrictedCrawlingContext.pushData}, {@apilink RestrictedCrawlingContext.addRequests}, {@apilink RestrictedCrawlingContext.enqueueLinks} made by a request handler.
@@ -169,13 +180,19 @@ export class RequestHandlerResult {
addRequests: Parameters<RestrictedCrawlingContext['addRequests']>[];
enqueueLinks: Parameters<RestrictedCrawlingContext['enqueueLinks']>[];
}> {
return { pushData: this.pushDataCalls, addRequests: this.addRequestsCalls, enqueueLinks: this.enqueueLinksCalls };
return {
pushData: this.pushDataCalls,
addRequests: this.addRequestsCalls,
enqueueLinks: this.enqueueLinksCalls,
};
}
/**
* A record of changes made to key-value stores by a request handler.
*/
get keyValueStoreChanges(): ReadonlyDeep<Record<string, Record<string, { changedValue: unknown; options?: RecordOptions }>>> {
get keyValueStoreChanges(): ReadonlyDeep<
Record<string, Record<string, { changedValue: unknown; options?: RecordOptions }>>
> {
return this._keyValueStoreChanges;
}
@@ -183,14 +200,16 @@ export class RequestHandlerResult {
* Items added to datasets by a request handler.
*/
get datasetItems(): ReadonlyDeep<{ item: Dictionary; datasetIdOrName?: string }[]> {
return this.pushDataCalls.flatMap(([data, datasetIdOrName]) => (Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })));
return this.pushDataCalls.flatMap(([data, datasetIdOrName]) =>
(Array.isArray(data) ? data : [data]).map((item) => ({ item, datasetIdOrName })),
);
}
/**
* URLs enqueued to the request queue by a request handler, either via {@apilink RestrictedCrawlingContext.addRequests} or {@apilink RestrictedCrawlingContext.enqueueLinks}
*/
get enqueuedUrls(): ReadonlyDeep<{ url: string; label?: string }[]> {
const result: {url: string; label? : string}[] = [];
const result: { url: string; label?: string }[] = [];
for (const [options] of this.enqueueLinksCalls) {
result.push(...(options?.urls?.map((url) => ({ url, label: options?.label })) ?? []));
@@ -198,7 +217,11 @@ export class RequestHandlerResult {
for (const [requests] of this.addRequestsCalls) {
for (const request of requests) {
if (typeof request === 'object' && (!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) && request.url !== undefined) {
if (
typeof request === 'object' &&
(!('requestsFromUrl' in request) || request.requestsFromUrl !== undefined) &&
request.url !== undefined
) {
result.push({ url: request.url, label: request.label });
} else if (typeof request === 'string') {
result.push({ url: request });
@@ -212,12 +235,16 @@ export class RequestHandlerResult {
/**
* URL lists enqueued to the request queue by a request handler via {@apilink RestrictedCrawlingContext.addRequests} using the `requestsFromUrl` option.
*/
get enqueuedUrlLists(): ReadonlyDeep<{ listUrl: string; label? : string }[]> {
const result: {listUrl: string; label? : string}[] = [];
get enqueuedUrlLists(): ReadonlyDeep<{ listUrl: string; label?: string }[]> {
const result: { listUrl: string; label?: string }[] = [];
for (const [requests] of this.addRequestsCalls) {
for (const request of requests) {
if (typeof request === 'object' && 'requestsFromUrl' in request && request.requestsFromUrl !== undefined) {
if (
typeof request === 'object' &&
'requestsFromUrl' in request &&
request.requestsFromUrl !== undefined
) {
result.push({ listUrl: request.requestsFromUrl, label: request.label });
}
}
@@ -249,11 +276,11 @@ export class RequestHandlerResult {
return {
id: this.idOrDefault(idOrName),
name: idOrName,
getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? await store.getValue(key),
getValue: async (key) => this.getKeyValueStoreChangedValue(idOrName, key) ?? (await store.getValue(key)),
getAutoSavedValue: async <T extends Dictionary = Dictionary>(key: string, defaultValue: T = {} as T) => {
let value = this.getKeyValueStoreChangedValue(idOrName, key);
if (value === null) {
value = await store.getValue(key) ?? defaultValue;
value = (await store.getValue(key)) ?? defaultValue;
this.setKeyValueStoreChangedValue(idOrName, key, value);
}
@@ -273,7 +300,12 @@ export class RequestHandlerResult {
return this.keyValueStoreChanges[id][key]?.changedValue ?? null;
};
private setKeyValueStoreChangedValue = (idOrName: string | undefined, key: string, changedValue: unknown, options?: RecordOptions) => {
private setKeyValueStoreChangedValue = (
idOrName: string | undefined,
key: string,
changedValue: unknown,
options?: RecordOptions,
) => {
const id = this.idOrDefault(idOrName);
this._keyValueStoreChanges[id] ??= {};
this._keyValueStoreChanges[id][key] = { changedValue, options };
packages/core/src/crawlers/error_snapshotter.ts
+13 -4
View File
@@ -81,7 +81,8 @@ export class ErrorSnapshotter {
const html = await page.content();
htmlFileName = html ? await this.saveHTMLSnapshot(html, keyValueStore, fileName) : undefined;
}
} else if (typeof body === 'string') { // for non-browser contexts
} else if (typeof body === 'string') {
// for non-browser contexts
htmlFileName = await this.saveHTMLSnapshot(body, keyValueStore, fileName);
}
@@ -101,7 +102,10 @@ export class ErrorSnapshotter {
* This function is applicable for browser contexts only.
* Returns an object containing the filenames of the screenshot and HTML file.
*/
async contextCaptureSnapshot(context: BrowserCrawlingContext, fileName: string): Promise<SnapshotResult | undefined> {
async contextCaptureSnapshot(
context: BrowserCrawlingContext,
fileName: string,
): Promise<SnapshotResult | undefined> {
try {
await context.saveSnapshot({ key: fileName });
return {
@@ -129,9 +133,14 @@ export class ErrorSnapshotter {
* Generate a unique fileName for each error snapshot.
*/
generateFilename(error: ErrnoException): string {
const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } = ErrorSnapshotter;
const { SNAPSHOT_PREFIX, BASE_MESSAGE, MAX_HASH_LENGTH, MAX_ERROR_CHARACTERS, MAX_FILENAME_LENGTH } =
ErrorSnapshotter;
// Create a hash of the error stack trace
const errorStackHash = crypto.createHash('sha1').update(error.stack || error.message || '').digest('hex').slice(0, MAX_HASH_LENGTH);
const errorStackHash = crypto
.createHash('sha1')
.update(error.stack || error.message || '')
.digest('hex')
.slice(0, MAX_HASH_LENGTH);
const errorMessagePrefix = (error.message || BASE_MESSAGE).slice(0, MAX_ERROR_CHARACTERS).trim();
/**
packages/core/src/crawlers/error_tracker.ts
+17 -14
View File
@@ -43,11 +43,7 @@ const getPathFromStackTrace = (stack: string[]) => {
for (const line of stack) {
const path = extractPathFromStackTraceLine(line);
if (
path.startsWith('node:')
|| path.includes('/node_modules/')
|| path.includes('\\node_modules\\')
) {
if (path.startsWith('node:') || path.includes('/node_modules/') || path.includes('\\node_modules\\')) {
continue;
}
@@ -73,7 +69,12 @@ const getStackTraceGroup = (error: ErrnoException, storage: Record<string, unkno
let normalizedStackTrace = null;
if (sliceAt !== -1) {
normalizedStackTrace = showFullStack ? stack!.slice(sliceAt).map((x) => x.trim()).join('\n') : getPathFromStackTrace(stack!.slice(sliceAt));
normalizedStackTrace = showFullStack
? stack!
.slice(sliceAt)
.map((x) => x.trim())
.join('\n')
: getPathFromStackTrace(stack!.slice(sliceAt));
}
if (!normalizedStackTrace) {
@@ -188,7 +189,7 @@ const normalizedCalculatePlaceholder = (a: string[], b: string[]) => {
const output = calculatePlaceholder(a, b);
// We can't be too general
if ((arrayCount(output, '_') / output.length) >= 0.5) {
if (arrayCount(output, '_') / output.length >= 0.5) {
return ['_'];
}
@@ -197,10 +198,7 @@ const normalizedCalculatePlaceholder = (a: string[], b: string[]) => {
// Merge A (missing placeholders) into B (can contain placeholders but does not have to)
const mergeMessages = (a: string, b: string, storage: Record<string, unknown>) => {
const placeholder = normalizedCalculatePlaceholder(
a.split(' '),
b.split(' '),
).join(' ');
const placeholder = normalizedCalculatePlaceholder(a.split(' '), b.split(' ')).join(' ');
if (placeholder === '_') {
return undefined;
@@ -227,9 +225,14 @@ const getErrorMessageGroup = (error: ErrnoException, storage: Record<string, unk
if (!message) {
try {
message = typeof error === 'string' ? error : `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
message =
typeof error === 'string'
? error
: `Unknown error message. Received non-error object: ${JSON.stringify(error)}`;
} catch {
message = `Unknown error message. Received non-error object, and could not stringify it: ${inspect(error, { depth: 0 })}`;
message = `Unknown error message. Received non-error object, and could not stringify it: ${inspect(error, {
depth: 0,
})}`;
}
}
@@ -354,7 +357,7 @@ export class ErrorTracker {
// Capture a snapshot (screenshot and HTML) on the first occurrence of an error
if (group.count === 1 && context) {
await this.captureSnapshot(group, error, context).catch(() => { });
await this.captureSnapshot(group, error, context).catch(() => {});
}
if (typeof error.cause === 'object' && error.cause !== null) {
packages/core/src/crawlers/statistics.ts
+20 -12
View File
@@ -109,14 +109,17 @@ export class Statistics {
* @internal
*/
constructor(options: StatisticsOptions = {}) {
ow(options, ow.object.exactShape({
logIntervalSecs: ow.optional.number,
logMessage: ow.optional.string,
keyValueStore: ow.optional.object,
config: ow.optional.object,
persistenceOptions: ow.optional.object,
saveErrorSnapshots: ow.optional.boolean,
}));
ow(
options,
ow.object.exactShape({
logIntervalSecs: ow.optional.number,
logMessage: ow.optional.string,
keyValueStore: ow.optional.object,
config: ow.optional.object,
persistenceOptions: ow.optional.object,
saveErrorSnapshots: ow.optional.boolean,
}),
);
const {
logIntervalSecs = 60,
@@ -226,8 +229,10 @@ export class Statistics {
this.state.requestsFinished++;
this.state.requestTotalFinishedDurationMillis += jobDurationMillis;
this._saveRetryCountForJob(job);
if (jobDurationMillis < this.state.requestMinDurationMillis) this.state.requestMinDurationMillis = jobDurationMillis;
if (jobDurationMillis > this.state.requestMaxDurationMillis) this.state.requestMaxDurationMillis = jobDurationMillis;
if (jobDurationMillis < this.state.requestMinDurationMillis)
this.state.requestMinDurationMillis = jobDurationMillis;
if (jobDurationMillis > this.state.requestMaxDurationMillis)
this.state.requestMaxDurationMillis = jobDurationMillis;
this.requestsInProgress.delete(id);
}
@@ -259,7 +264,8 @@ export class Statistics {
return {
requestAvgFailedDurationMillis: Math.round(requestTotalFailedDurationMillis / requestsFailed) || Infinity,
requestAvgFinishedDurationMillis: Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
requestAvgFinishedDurationMillis:
Math.round(requestTotalFinishedDurationMillis / requestsFinished) || Infinity,
requestsFinishedPerMinute: Math.round(requestsFinished / totalMinutes) || 0,
requestsFailedPerMinute: Math.floor(requestsFailed / totalMinutes) || 0,
requestTotalDurationMillis: requestTotalFinishedDurationMillis + requestTotalFailedDurationMillis,
@@ -396,7 +402,9 @@ export class Statistics {
const result = {
...this.state,
crawlerLastStartTimestamp: this.instanceStart,
crawlerFinishedAt: this.state.crawlerFinishedAt ? new Date(this.state.crawlerFinishedAt).toISOString() : null,
crawlerFinishedAt: this.state.crawlerFinishedAt
? new Date(this.state.crawlerFinishedAt).toISOString()
: null,
crawlerStartedAt: this.state.crawlerStartedAt ? new Date(this.state.crawlerStartedAt).toISOString() : null,
requestRetryHistogram: this.requestRetryHistogram,
statsId: this.id,
packages/core/src/enqueue_links/enqueue_links.ts
+42 -50
View File
@@ -232,58 +232,43 @@ export enum EnqueueStrategy {
* @param options All `enqueueLinks()` parameters are passed via an options object.
* @returns Promise that resolves to {@apilink BatchAddRequestsResult} object.
*/
export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>): Promise<BatchAddRequestsResult> {
export async function enqueueLinks(
options: SetRequired<EnqueueLinksOptions, 'requestQueue' | 'urls'>,
): Promise<BatchAddRequestsResult> {
if (!options || Object.keys(options).length === 0) {
throw new RangeError([
// eslint-disable-next-line max-len
'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/docs/examples/crawl-relative-links',
].join('\n'));
throw new RangeError(
[
'enqueueLinks() was called without the required options. You can only do that when you use the `crawlingContext.enqueueLinks()` method in request handlers.',
'Check out our guide on how to use enqueueLinks() here: https://crawlee.dev/docs/examples/crawl-relative-links',
].join('\n'),
);
}
ow(options, ow.object.exactShape({
urls: ow.array.ofType(ow.string),
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
forefront: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
limit: ow.optional.number,
selector: ow.optional.string,
baseUrl: ow.optional.string,
userData: ow.optional.object,
label: ow.optional.string,
pseudoUrls: ow.optional.array.ofType(ow.any(
ow.string,
ow.object.hasKeys('purl'),
)),
globs: ow.optional.array.ofType(ow.any(
ow.string,
ow.object.hasKeys('glob'),
)),
exclude: ow.optional.array.ofType(ow.any(
ow.string,
ow.regExp,
ow.object.hasKeys('glob'),
ow.object.hasKeys('regexp'),
)),
regexps: ow.optional.array.ofType(ow.any(
ow.regExp,
ow.object.hasKeys('regexp'),
)),
transformRequestFunction: ow.optional.function,
strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
}));
ow(
options,
ow.object.exactShape({
urls: ow.array.ofType(ow.string),
requestQueue: ow.object.hasKeys('fetchNextRequest', 'addRequest'),
forefront: ow.optional.boolean,
skipNavigation: ow.optional.boolean,
limit: ow.optional.number,
selector: ow.optional.string,
baseUrl: ow.optional.string,
userData: ow.optional.object,
label: ow.optional.string,
pseudoUrls: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('purl'))),
globs: ow.optional.array.ofType(ow.any(ow.string, ow.object.hasKeys('glob'))),
exclude: ow.optional.array.ofType(
ow.any(ow.string, ow.regExp, ow.object.hasKeys('glob'), ow.object.hasKeys('regexp')),
),
regexps: ow.optional.array.ofType(ow.any(ow.regExp, ow.object.hasKeys('regexp'))),
transformRequestFunction: ow.optional.function,
strategy: ow.optional.string.oneOf(Object.values(EnqueueStrategy)),
}),
);
const {
requestQueue,
limit,
urls,
pseudoUrls,
exclude,
globs,
regexps,
transformRequestFunction,
forefront,
} = options;
const { requestQueue, limit, urls, pseudoUrls, exclude, globs, regexps, transformRequestFunction, forefront } =
options;
const urlExcludePatternObjects: UrlPatternObject[] = [];
const urlPatternObjects: UrlPatternObject[] = [];
@@ -361,7 +346,9 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
let requestOptions = createRequestOptions(urls, options);
if (transformRequestFunction) {
requestOptions = requestOptions.map((request) => transformRequestFunction(request)).filter((r) => !!r) as RequestOptions[];
requestOptions = requestOptions
.map((request) => transformRequestFunction(request))
.filter((r) => !!r) as RequestOptions[];
}
function createFilteredRequests() {
@@ -371,7 +358,12 @@ export async function enqueueLinks(options: SetRequired<EnqueueLinksOptions, 're
}
// Generate requests based on the user patterns first
const generatedRequestsFromUserFilters = createRequests(requestOptions, urlPatternObjects, urlExcludePatternObjects, options.strategy);
const generatedRequestsFromUserFilters = createRequests(
requestOptions,
urlPatternObjects,
urlExcludePatternObjects,
options.strategy,
);
// ...then filter them by the enqueue links strategy (making this an AND check)
return filterRequestsByPatterns(generatedRequestsFromUserFilters, enqueueStrategyPatterns);
}
packages/core/src/enqueue_links/shared.ts
+30 -25
View File
@@ -24,22 +24,34 @@ export type UrlPatternObject = {
regexp?: RegExp;
} & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
export type PseudoUrlObject = { purl: string } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
export type PseudoUrlObject = { purl: string } & Pick<
RequestOptions,
'method' | 'payload' | 'label' | 'userData' | 'headers'
>;
export type PseudoUrlInput = string | PseudoUrlObject;
export type GlobObject = { glob: string } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
export type GlobObject = { glob: string } & Pick<
RequestOptions,
'method' | 'payload' | 'label' | 'userData' | 'headers'
>;
export type GlobInput = string | GlobObject;
export type RegExpObject = { regexp: RegExp } & Pick<RequestOptions, 'method' | 'payload' | 'label' | 'userData' | 'headers'>;
export type RegExpObject = { regexp: RegExp } & Pick<
RequestOptions,
'method' | 'payload' | 'label' | 'userData' | 'headers'
>;
export type RegExpInput = RegExp | RegExpObject;
/**
* @ignore
*/
export function updateEnqueueLinksPatternCache(item: GlobInput | RegExpInput | PseudoUrlInput, pattern: RegExpObject | GlobObject): void {
export function updateEnqueueLinksPatternCache(
item: GlobInput | RegExpInput | PseudoUrlInput,
pattern: RegExpObject | GlobObject,
): void {
enqueueLinksPatternCache.set(item, pattern);
if (enqueueLinksPatternCache.size > MAX_ENQUEUE_LINKS_CACHE_SIZE) {
const key = enqueueLinksPatternCache.keys().next().value;
@@ -95,7 +107,7 @@ export function constructGlobObjectsFromGlobs(globs: Readonly<GlobInput[]>): Glo
return false;
})
.map((item) => {
// Get glob object from cache.
// Get glob object from cache.
let globObject = enqueueLinksPatternCache.get(item);
if (globObject) return globObject;
@@ -117,7 +129,8 @@ export function constructGlobObjectsFromGlobs(globs: Readonly<GlobInput[]>): Glo
*/
export function validateGlobPattern(glob: string): string {
const globTrimmed = glob.trim();
if (globTrimmed.length === 0) throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
if (globTrimmed.length === 0)
throw new Error(`Cannot parse Glob pattern '${globTrimmed}': it must be an non-empty string`);
return globTrimmed;
}
@@ -158,10 +171,7 @@ export function createRequests(
.filter(({ url }) => {
return !excludePatternObjects.some((excludePatternObject) => {
const { regexp, glob } = excludePatternObject;
return (
(regexp && url.match(regexp)) || // eslint-disable-line
(glob && minimatch(url, glob, { nocase: true }))
);
return (regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }));
});
})
.map(({ url, opts }) => {
@@ -171,13 +181,11 @@ export function createRequests(
for (const urlPatternObject of urlPatternObjects) {
const { regexp, glob, ...requestRegExpOptions } = urlPatternObject;
if (
(regexp && url.match(regexp)) || // eslint-disable-line
(glob && minimatch(url, glob, { nocase: true }))
) {
const request = typeof opts === 'string'
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
: { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
if ((regexp && url.match(regexp)) || (glob && minimatch(url, glob, { nocase: true }))) {
const request =
typeof opts === 'string'
? { url: opts, ...requestRegExpOptions, enqueueStrategy: strategy }
: { ...opts, ...requestRegExpOptions, enqueueStrategy: strategy };
return new Request(request);
}
@@ -200,10 +208,7 @@ export function filterRequestsByPatterns(requests: Request[], patterns?: UrlPatt
for (const urlPatternObject of patterns) {
const { regexp, glob } = urlPatternObject;
if (
(regexp && request.url.match(regexp)) || // eslint-disable-line
(glob && minimatch(request.url, glob, { nocase: true }))
) {
if ((regexp && request.url.match(regexp)) || (glob && minimatch(request.url, glob, { nocase: true }))) {
filtered.push(request);
// Break the pattern loop, as we already matched this request once
break;
@@ -222,10 +227,10 @@ export function createRequestOptions(
options: Pick<EnqueueLinksOptions, 'label' | 'userData' | 'baseUrl' | 'skipNavigation' | 'strategy'> = {},
): RequestOptions[] {
return sources
.map(
(src) => (
typeof src === 'string' ? { url: src, enqueueStrategy: options.strategy } : { ...src, enqueueStrategy: options.strategy } as RequestOptions
),
.map((src) =>
typeof src === 'string'
? { url: src, enqueueStrategy: options.strategy }
: ({ ...src, enqueueStrategy: options.strategy } as RequestOptions),
)
.filter(({ url }) => {
try {

Some files were not shown because too many files have changed in this diff Show More