Coverage for haystack/components/converters/pypdf.py: 94%
103 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-21 13:53 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5import io
6import os
7from enum import Enum
8from pathlib import Path
9from typing import Any
11from haystack import Document, component, default_from_dict, default_to_dict, logging
12from haystack.components.converters.utils import LinkFormat, get_bytestream_from_source, normalize_metadata
13from haystack.dataclasses import ByteStream
14from haystack.lazy_imports import LazyImport
16with LazyImport("Run 'pip install pypdf'") as pypdf_import:
17 from pypdf import PdfReader
20logger = logging.getLogger(__name__)
23class PyPDFExtractionMode(Enum):
24 """
25 The mode to use for extracting text from a PDF.
26 """
28 PLAIN = "plain"
29 LAYOUT = "layout"
31 def __str__(self) -> str:
32 """
33 Convert a PyPDFExtractionMode enum to a string.
34 """
35 return self.value
37 @staticmethod
38 def from_str(string: str) -> "PyPDFExtractionMode":
39 """
40 Convert a string to a PyPDFExtractionMode enum.
41 """
42 enum_map = {e.value: e for e in PyPDFExtractionMode}
43 mode = enum_map.get(string)
44 if mode is None:
45 msg = f"Unknown extraction mode '{string}'. Supported modes are: {list(enum_map.keys())}"
46 raise ValueError(msg)
47 return mode
50@component
51class PyPDFToDocument:
52 """
53 Converts PDF files to documents your pipeline can query.
55 This component uses the PyPDF library.
56 You can attach metadata to the resulting documents.
58 ### Usage example
60 ```python
61 from haystack.components.converters.pypdf import PyPDFToDocument
62 from datetime import datetime
64 converter = PyPDFToDocument()
65 results = converter.run(
66 sources=["test/test_files/pdf/sample_pdf_1.pdf"], meta={"date_added": datetime.now().isoformat()}
67 )
68 documents = results["documents"]
70 print(documents[0].content)
71 # >> 'This is a text from the PDF file.'
72 ```
73 """
75 def __init__(
76 self,
77 *,
78 extraction_mode: str | PyPDFExtractionMode = PyPDFExtractionMode.PLAIN,
79 plain_mode_orientations: tuple = (0, 90, 180, 270),
80 plain_mode_space_width: float = 200.0,
81 layout_mode_space_vertically: bool = True,
82 layout_mode_scale_weight: float = 1.25,
83 layout_mode_strip_rotated: bool = True,
84 layout_mode_font_height_weight: float = 1.0,
85 store_full_path: bool = False,
86 link_format: str | LinkFormat = LinkFormat.NONE,
87 ) -> None:
88 """
89 Create an PyPDFToDocument component.
91 :param extraction_mode:
92 The mode to use for extracting text from a PDF.
93 Layout mode is an experimental mode that adheres to the rendered layout of the PDF.
94 :param plain_mode_orientations:
95 Tuple of orientations to look for when extracting text from a PDF in plain mode.
96 Ignored if `extraction_mode` is `PyPDFExtractionMode.LAYOUT`.
97 :param plain_mode_space_width:
98 Forces default space width if not extracted from font.
99 Ignored if `extraction_mode` is `PyPDFExtractionMode.LAYOUT`.
100 :param layout_mode_space_vertically:
101 Whether to include blank lines inferred from y distance + font height.
102 Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
103 :param layout_mode_scale_weight:
104 Multiplier for string length when calculating weighted average character width.
105 Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
106 :param layout_mode_strip_rotated:
107 Layout mode does not support rotated text. Set to `False` to include rotated text anyway.
108 If rotated text is discovered, layout will be degraded and a warning will be logged.
109 Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
110 :param layout_mode_font_height_weight:
111 Multiplier for font height when calculating blank line height.
112 Ignored if `extraction_mode` is `PyPDFExtractionMode.PLAIN`.
113 :param store_full_path:
114 If True, the full path of the file is stored in the metadata of the document.
115 If False, only the file name is stored.
116 :param link_format:
117 The format used for the hyperlinks found in the PDF link annotations.
118 The links of a page are appended at the end of that page's text, one per line. PDF link annotations
119 carry no anchor text, so the address is used as the link text as well. Can be either:
120 `LinkFormat.MARKDOWN` or `"markdown"` to get `[address](address)`,
121 `LinkFormat.PLAIN` or `"plain"` to get `address (address)`,
122 `LinkFormat.NONE` or `"none"` to get text without links.
123 """
124 pypdf_import.check()
126 self.store_full_path = store_full_path
128 if isinstance(extraction_mode, str):
129 extraction_mode = PyPDFExtractionMode.from_str(extraction_mode)
130 self.extraction_mode = extraction_mode
131 self.plain_mode_orientations = plain_mode_orientations
132 self.plain_mode_space_width = plain_mode_space_width
133 self.layout_mode_space_vertically = layout_mode_space_vertically
134 self.layout_mode_scale_weight = layout_mode_scale_weight
135 self.layout_mode_strip_rotated = layout_mode_strip_rotated
136 self.layout_mode_font_height_weight = layout_mode_font_height_weight
137 self.link_format = LinkFormat.from_str(link_format) if isinstance(link_format, str) else link_format
139 def to_dict(self) -> dict[str, Any]:
140 """
141 Serializes the component to a dictionary.
143 :returns:
144 Dictionary with serialized data.
145 """
146 return default_to_dict(
147 self,
148 extraction_mode=str(self.extraction_mode),
149 plain_mode_orientations=self.plain_mode_orientations,
150 plain_mode_space_width=self.plain_mode_space_width,
151 layout_mode_space_vertically=self.layout_mode_space_vertically,
152 layout_mode_scale_weight=self.layout_mode_scale_weight,
153 layout_mode_strip_rotated=self.layout_mode_strip_rotated,
154 layout_mode_font_height_weight=self.layout_mode_font_height_weight,
155 store_full_path=self.store_full_path,
156 link_format=str(self.link_format),
157 )
159 @classmethod
160 def from_dict(cls, data: dict[str, Any]) -> "PyPDFToDocument":
161 """
162 Deserializes the component from a dictionary.
164 :param data:
165 Dictionary with serialized data.
167 :returns:
168 Deserialized component.
169 """
170 if "link_format" in data.get("init_parameters", {}):
171 data["init_parameters"]["link_format"] = LinkFormat.from_str(data["init_parameters"]["link_format"])
172 return default_from_dict(cls, data)
174 def _extract_links(self, page: Any, page_number: int) -> list[str]:
175 """
176 Extracts the hyperlinks from the link annotations of a single page.
178 Annotations that cannot be read are skipped, so that a single malformed annotation does not
179 prevent the page - and with it the whole document - from being converted.
181 :param page:
182 The page to extract the links from.
183 :param page_number:
184 The 1-based number of the page, only used for logging.
186 :returns:
187 The formatted links found on the page.
188 """
189 page_links: list[str] = []
190 try:
191 if "/Annots" not in page:
192 return page_links
193 # Indexing the page resolves indirect references to the annotation array, `get` would not.
194 for annot in page["/Annots"] or []:
195 try:
196 annot_obj = annot.get_object()
197 if annot_obj.get("/Subtype") == "/Link":
198 a = annot_obj.get("/A")
199 if a and a.get("/S") == "/URI":
200 uri = a.get("/URI")
201 if uri:
202 uri_str = uri.decode("utf-8") if isinstance(uri, bytes) else str(uri)
203 if self.link_format == LinkFormat.MARKDOWN:
204 page_links.append(f"[{uri_str}]({uri_str})")
205 else: # PLAIN
206 page_links.append(f"{uri_str} ({uri_str})")
207 except Exception:
208 logger.debug("Skipping malformed annotation in page {page}", page=page_number)
209 except Exception:
210 # For example when /Annots is an unresolvable reference: keep the page text instead of losing it.
211 logger.debug("Could not read the annotations of page {page}. Skipping its links.", page=page_number)
212 return page_links
214 def _default_convert(self, reader: "PdfReader") -> str:
215 texts = []
216 for page_number, page in enumerate(reader.pages, start=1):
217 extracted_text = page.extract_text(
218 orientations=self.plain_mode_orientations,
219 extraction_mode=self.extraction_mode.value,
220 space_width=self.plain_mode_space_width,
221 layout_mode_space_vertically=self.layout_mode_space_vertically,
222 layout_mode_scale_weight=self.layout_mode_scale_weight,
223 layout_mode_strip_rotated=self.layout_mode_strip_rotated,
224 layout_mode_font_height_weight=self.layout_mode_font_height_weight,
225 )
227 if self.link_format != LinkFormat.NONE:
228 page_links = self._extract_links(page, page_number)
229 if page_links:
230 extracted_text += "\n\n" + "\n".join(page_links)
232 texts.append(extracted_text)
233 return "\f".join(texts)
235 @component.output_types(documents=list[Document])
236 def run(
237 self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
238 ) -> dict[str, list[Document]]:
239 """
240 Converts PDF files to documents.
242 :param sources:
243 List of file paths or ByteStream objects to convert.
244 :param meta:
245 Optional metadata to attach to the documents.
246 This value can be a list of dictionaries or a single dictionary.
247 If it's a single dictionary, its content is added to the metadata of all produced documents.
248 If it's a list, its length must match the number of sources, as they are zipped together.
249 For ByteStream objects, their `meta` is added to the output documents.
251 :returns:
252 A dictionary with the following keys:
253 - `documents`: A list of converted documents.
254 """
255 documents = []
256 meta_list = normalize_metadata(meta, sources_count=len(sources))
258 for source, metadata in zip(sources, meta_list, strict=True):
259 try:
260 bytestream = get_bytestream_from_source(source)
261 except Exception as e:
262 logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
263 continue
264 try:
265 pdf_reader = PdfReader(io.BytesIO(bytestream.data))
266 text = self._default_convert(pdf_reader)
267 except Exception as e:
268 logger.warning(
269 "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
270 )
271 continue
273 if text is None or text.strip() == "":
274 logger.warning(
275 "PyPDFToDocument could not extract text from the file {source}. Returning an empty document.",
276 source=source,
277 )
279 merged_metadata = {**bytestream.meta, **metadata}
281 if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
282 merged_metadata["file_path"] = os.path.basename(file_path)
283 document = Document(content=text, meta=merged_metadata)
284 documents.append(document)
286 return {"documents": documents}