ntohidi
057b45b585
Relax lxml~=5.3 (>=5.3,<6) to >=5.3,<7 so crawl4ai can co-install with packages requiring lxml 6.x (e.g. scrapling). Crawl4ai only uses stable lxml APIs (etree, html, fromstring, CSSSelector) unchanged in 6.x.
101 lines
2.5 KiB
TOML
101 lines
2.5 KiB
TOML
[build-system]
|
|
requires = ["setuptools>=64.0.0", "wheel"]
|
|
build-backend = "setuptools.build_meta"
|
|
|
|
[project]
|
|
name = "Crawl4AI"
|
|
dynamic = ["version"]
|
|
description = "🚀🤖 Crawl4AI: Open-source LLM Friendly Web Crawler & scraper"
|
|
readme = "README.md"
|
|
requires-python = ">=3.10"
|
|
license = "Apache-2.0"
|
|
authors = [
|
|
{name = "Unclecode", email = "unclecode@kidocode.com"}
|
|
]
|
|
dependencies = [
|
|
"aiofiles>=24.1.0",
|
|
"aiohttp>=3.11.11",
|
|
"aiosqlite~=0.20",
|
|
"anyio>=4.0.0",
|
|
"lxml>=5.3,<7",
|
|
"unclecode-litellm==1.81.13",
|
|
"numpy>=1.26.0,<3",
|
|
"pillow>=10.4",
|
|
"playwright>=1.49.0",
|
|
"patchright>=1.49.0",
|
|
"python-dotenv~=1.0",
|
|
"requests~=2.26",
|
|
"beautifulsoup4~=4.12",
|
|
"playwright-stealth>=2.0.0",
|
|
"xxhash~=3.4",
|
|
"rank-bm25~=0.2",
|
|
"snowballstemmer~=2.2",
|
|
"pydantic>=2.10",
|
|
"pyOpenSSL>=25.3.0",
|
|
"psutil>=6.1.1",
|
|
"PyYAML>=6.0",
|
|
"nltk>=3.9.1",
|
|
"rich>=13.9.4",
|
|
"cssselect>=1.2.0",
|
|
"httpx>=0.27.2",
|
|
"httpx[http2]>=0.27.2",
|
|
"fake-useragent>=2.0.3",
|
|
"click>=8.1.7",
|
|
"chardet>=5.2.0",
|
|
"brotli>=1.1.0",
|
|
"humanize>=4.10.0",
|
|
"lark>=1.2.2",
|
|
"alphashape>=1.3.1",
|
|
"shapely>=2.0.0"
|
|
]
|
|
classifiers = [
|
|
"Development Status :: 4 - Beta",
|
|
"Intended Audience :: Developers",
|
|
"Programming Language :: Python :: 3",
|
|
"Programming Language :: Python :: 3.10",
|
|
"Programming Language :: Python :: 3.11",
|
|
"Programming Language :: Python :: 3.12",
|
|
"Programming Language :: Python :: 3.13",
|
|
]
|
|
|
|
[project.optional-dependencies]
|
|
pdf = ["pypdf"]
|
|
torch = ["torch", "nltk", "scikit-learn"]
|
|
transformer = ["transformers", "tokenizers", "sentence-transformers"]
|
|
cosine = ["torch", "transformers", "nltk", "sentence-transformers"]
|
|
sync = ["selenium"]
|
|
all = [
|
|
"pypdf",
|
|
"torch",
|
|
"nltk",
|
|
"scikit-learn",
|
|
"transformers",
|
|
"tokenizers",
|
|
"sentence-transformers",
|
|
"selenium"
|
|
]
|
|
|
|
[project.scripts]
|
|
crawl4ai-download-models = "crawl4ai.model_loader:main"
|
|
crawl4ai-migrate = "crawl4ai.migrations:main"
|
|
crawl4ai-setup = "crawl4ai.install:post_install"
|
|
crawl4ai-doctor = "crawl4ai.install:doctor"
|
|
crwl = "crawl4ai.cli:main"
|
|
|
|
[tool.setuptools]
|
|
packages = {find = {where = ["."], include = ["crawl4ai*"]}}
|
|
|
|
[tool.setuptools.package-data]
|
|
crawl4ai = ["js_snippet/*.js", "crawlers/google_search/*.js"]
|
|
|
|
[tool.setuptools.dynamic]
|
|
version = {attr = "crawl4ai.__version__.__version__"}
|
|
|
|
[tool.uv.sources]
|
|
crawl4ai = { workspace = true }
|
|
|
|
[dependency-groups]
|
|
dev = [
|
|
"crawl4ai",
|
|
]
|