Coverage for haystack/components/converters/pdfminer.py: 100%
105 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-21 13:53 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5import io
6import os
7import re
8from collections.abc import Iterator
9from pathlib import Path
10from typing import Any
12from haystack import Document, component, default_from_dict, default_to_dict, logging
13from haystack.components.converters.utils import LinkFormat, get_bytestream_from_source, normalize_metadata
14from haystack.dataclasses import ByteStream
15from haystack.lazy_imports import LazyImport
17with LazyImport("Run 'pip install pdfminer.six'") as pdfminer_import:
18 from pdfminer.converter import PDFPageAggregator
19 from pdfminer.layout import LAParams, LTTextContainer
20 from pdfminer.pdfinterp import PDFPageInterpreter, PDFResourceManager
21 from pdfminer.pdfpage import PDFPage
22 from pdfminer.pdftypes import resolve1
24logger = logging.getLogger(__name__)
26CID_PATTERN = r"\(cid:\d+\)" # regex pattern to detect CID characters
29@component
30class PDFMinerToDocument:
31 """
32 Converts PDF files to Documents.
34 Uses `pdfminer` compatible converters to convert PDF files to Documents. https://pdfminersix.readthedocs.io/en/latest/
36 Usage example:
38 ```python
39 from haystack.components.converters.pdfminer import PDFMinerToDocument
40 from datetime import datetime
42 converter = PDFMinerToDocument()
43 results = converter.run(
44 sources=["test/test_files/pdf/sample_pdf_1.pdf"], meta={"date_added": datetime.now().isoformat()}
45 )
47 print(results["documents"][0].content)
48 # >> 'This is a text from the PDF file.'
49 ```
50 """
52 def __init__(
53 self,
54 line_overlap: float = 0.5,
55 char_margin: float = 2.0,
56 line_margin: float = 0.5,
57 word_margin: float = 0.1,
58 boxes_flow: float | None = 0.5,
59 detect_vertical: bool = True,
60 all_texts: bool = False,
61 store_full_path: bool = False,
62 link_format: str | LinkFormat = LinkFormat.NONE,
63 ) -> None:
64 """
65 Create a PDFMinerToDocument component.
67 :param line_overlap:
68 This parameter determines whether two characters are considered to be on
69 the same line based on the amount of overlap between them.
70 The overlap is calculated relative to the minimum height of both characters.
71 :param char_margin:
72 Determines whether two characters are part of the same line based on the distance between them.
73 If the distance is less than the margin specified, the characters are considered to be on the same line.
74 The margin is calculated relative to the width of the character.
75 :param word_margin:
76 Determines whether two characters on the same line are part of the same word
77 based on the distance between them. If the distance is greater than the margin specified,
78 an intermediate space will be added between them to make the text more readable.
79 The margin is calculated relative to the width of the character.
80 :param line_margin:
81 This parameter determines whether two lines are part of the same paragraph based on
82 the distance between them. If the distance is less than the margin specified,
83 the lines are considered to be part of the same paragraph.
84 The margin is calculated relative to the height of a line.
85 :param boxes_flow:
86 This parameter determines the importance of horizontal and vertical position when
87 determining the order of text boxes. A value between -1.0 and +1.0 can be set,
88 with -1.0 indicating that only horizontal position matters and +1.0 indicating
89 that only vertical position matters. Setting the value to 'None' will disable advanced
90 layout analysis, and text boxes will be ordered based on the position of their bottom left corner.
91 :param detect_vertical:
92 This parameter determines whether vertical text should be considered during layout analysis.
93 :param all_texts:
94 If layout analysis should be performed on text in figures.
95 :param store_full_path:
96 If True, the full path of the file is stored in the metadata of the document.
97 If False, only the file name is stored.
98 :param link_format:
99 The format used for the hyperlinks found in the PDF link annotations.
100 The links of a page are appended at the end of that page's text, one per line. PDF link annotations
101 carry no anchor text, so the address is used as the link text as well. Can be either:
102 `LinkFormat.MARKDOWN` or `"markdown"` to get `[address](address)`,
103 `LinkFormat.PLAIN` or `"plain"` to get `address (address)`,
104 `LinkFormat.NONE` or `"none"` to get text without links.
105 """
107 pdfminer_import.check()
109 self.layout_params = LAParams(
110 line_overlap=line_overlap,
111 char_margin=char_margin,
112 line_margin=line_margin,
113 word_margin=word_margin,
114 boxes_flow=boxes_flow,
115 detect_vertical=detect_vertical,
116 all_texts=all_texts,
117 )
118 self.store_full_path = store_full_path
119 self.link_format = LinkFormat.from_str(link_format) if isinstance(link_format, str) else link_format
120 self.cid_pattern = re.compile(CID_PATTERN)
122 def to_dict(self) -> dict[str, Any]:
123 """
124 Serializes the component to a dictionary.
126 :returns:
127 Dictionary with serialized data.
128 """
129 return default_to_dict(
130 self,
131 line_overlap=self.layout_params.line_overlap,
132 char_margin=self.layout_params.char_margin,
133 line_margin=self.layout_params.line_margin,
134 word_margin=self.layout_params.word_margin,
135 boxes_flow=self.layout_params.boxes_flow,
136 detect_vertical=self.layout_params.detect_vertical,
137 all_texts=self.layout_params.all_texts,
138 store_full_path=self.store_full_path,
139 link_format=str(self.link_format),
140 )
142 @classmethod
143 def from_dict(cls, data: dict[str, Any]) -> "PDFMinerToDocument":
144 """
145 Deserializes the component from a dictionary.
147 :param data:
148 Dictionary with serialized data.
150 :returns:
151 Deserialized component.
152 """
153 if "link_format" in data.get("init_parameters", {}):
154 data["init_parameters"]["link_format"] = LinkFormat.from_str(data["init_parameters"]["link_format"])
155 return default_from_dict(cls, data)
157 def _iter_pages(self, fp: io.BytesIO) -> Iterator[tuple[Any, Any]]:
158 """
159 Lazily yields the pages of a PDF as `(LTPage, PDFPage)` pairs.
161 The pages are processed one at a time so that only a single page layout is kept in memory.
163 :param fp:
164 The PDF file to read.
166 :returns:
167 An iterator over `(page layout object, page object)` pairs.
168 """
169 rsrcmgr = PDFResourceManager(caching=True)
170 device = PDFPageAggregator(rsrcmgr, laparams=self.layout_params)
171 interpreter = PDFPageInterpreter(rsrcmgr, device)
172 for pdf_page in PDFPage.get_pages(fp, caching=True):
173 interpreter.process_page(pdf_page)
174 yield device.get_result(), pdf_page
176 def _convert_page(self, lt_page: Any, pdf_page: Any) -> str:
177 """
178 Extracts text from a single PDF page.
180 :param lt_page:
181 PDF page layout object (LTPage).
182 :param pdf_page:
183 PDF page (PDFPage).
185 :returns:
186 PDF text from the single page converted to str
187 """
188 text = ""
189 for container in lt_page:
190 # Keep text only
191 if isinstance(container, LTTextContainer):
192 container_text = container.get_text()
193 if container_text:
194 text += "\n\n"
195 text += container_text
197 if self.link_format != LinkFormat.NONE and getattr(pdf_page, "annots", None):
198 page_links = []
199 annots = resolve1(pdf_page.annots)
200 if annots:
201 for annot in annots:
202 try:
203 annot_obj = resolve1(annot)
204 if (
205 isinstance(annot_obj, dict)
206 and annot_obj.get("Subtype")
207 and resolve1(annot_obj.get("Subtype")).name == "Link"
208 ):
209 a = annot_obj.get("A")
210 if a:
211 a_obj = resolve1(a)
212 if (
213 isinstance(a_obj, dict)
214 and a_obj.get("S")
215 and resolve1(a_obj.get("S")).name == "URI"
216 ):
217 uri = a_obj.get("URI")
218 if uri:
219 # Decode bytes if needed (pdfminer sometimes returns bytes for strings)
220 uri_str = uri.decode("utf-8") if isinstance(uri, bytes) else str(uri)
221 if self.link_format == LinkFormat.MARKDOWN:
222 page_links.append(f"[{uri_str}]({uri_str})")
223 else: # PLAIN
224 page_links.append(f"{uri_str} ({uri_str})")
225 except Exception:
226 logger.debug("Skipping malformed annotation")
227 continue
228 if page_links:
229 text += "\n\n" + "\n".join(page_links)
231 return text
233 def detect_undecoded_cid_characters(self, text: str) -> dict[str, Any]:
234 """
235 Look for character sequences of CID, i.e.: characters that haven't been properly decoded from their CID format.
237 This is useful to detect if the text extractor is not able to extract the text correctly, e.g. if the PDF uses
238 non-standard fonts.
240 A PDF font may include a ToUnicode map (mapping from character code to Unicode) to support operations like
241 searching strings or copy & paste in a PDF viewer. This map immediately provides the mapping the text extractor
242 needs. If that map is not available the text extractor cannot decode the CID characters and will return them
243 as is.
245 see: https://pdfminersix.readthedocs.io/en/latest/faq.html#why-are-there-cid-x-values-in-the-textual-output
247 :param text: The text to check for undecoded CID characters
248 :returns:
249 A dictionary containing detection results
250 """
252 matches = re.findall(self.cid_pattern, text)
253 total_chars = len(text)
254 cid_chars = sum(len(match) for match in matches)
255 percentage = (cid_chars / total_chars * 100) if total_chars > 0 else 0
257 return {"total_chars": total_chars, "cid_chars": cid_chars, "percentage": round(percentage, 2)}
259 @component.output_types(documents=list[Document])
260 def run(
261 self, sources: list[str | Path | ByteStream], meta: dict[str, Any] | list[dict[str, Any]] | None = None
262 ) -> dict[str, Any]:
263 """
264 Converts PDF files to Documents.
266 :param sources:
267 List of PDF file paths or ByteStream objects.
268 :param meta:
269 Optional metadata to attach to the Documents.
270 This value can be either a list of dictionaries or a single dictionary.
271 If it's a single dictionary, its content is added to the metadata of all produced Documents.
272 If it's a list, the length of the list must match the number of sources, because the two lists will
273 be zipped.
274 If `sources` contains ByteStream objects, their `meta` will be added to the output Documents.
276 :returns:
277 A dictionary with the following keys:
278 - `documents`: Created Documents
279 """
280 documents = []
282 meta_list = normalize_metadata(meta, sources_count=len(sources))
284 for source, metadata in zip(sources, meta_list, strict=True):
285 try:
286 bytestream = get_bytestream_from_source(source)
287 except Exception as e:
288 logger.warning("Could not read {source}. Skipping it. Error: {error}", source=source, error=e)
289 continue
290 try:
291 fp = io.BytesIO(bytestream.data)
292 text = "\f".join(self._convert_page(lt_page, pdf_page) for lt_page, pdf_page in self._iter_pages(fp))
293 except Exception as e:
294 logger.warning(
295 "Could not read {source} and convert it to Document, skipping. {error}", source=source, error=e
296 )
297 continue
299 if text is None or text.strip() == "":
300 logger.warning(
301 "PDFMinerToDocument could not extract text from the file {source}. Returning an empty document.",
302 source=source,
303 )
305 merged_metadata = {**bytestream.meta, **metadata}
307 if not self.store_full_path and (file_path := bytestream.meta.get("file_path")):
308 merged_metadata["file_path"] = os.path.basename(file_path)
310 analysis = self.detect_undecoded_cid_characters(text)
312 if analysis["percentage"] > 0:
313 logger.warning(
314 "Detected {cid_chars} undecoded CID characters in {total_chars} characters"
315 " ({percentage}%) in {source}.",
316 cid_chars=analysis["cid_chars"],
317 total_chars=analysis["total_chars"],
318 percentage=analysis["percentage"],
319 source=source,
320 )
322 document = Document(content=text, meta=merged_metadata)
323 documents.append(document)
325 return {"documents": documents}