Coverage for haystack/components/evaluators/document_recall.py: 97%
73 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-21 13:53 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5from enum import Enum
6from typing import Any
8from haystack import component, default_to_dict, logging
9from haystack.dataclasses import Document
11logger = logging.getLogger(__name__)
14class RecallMode(Enum):
15 """
16 Enum for the mode to use for calculating the recall score.
17 """
19 # Score is based on whether any document is retrieved.
20 SINGLE_HIT = "single_hit"
21 # Score is based on how many documents were retrieved.
22 MULTI_HIT = "multi_hit"
24 def __str__(self) -> str:
25 return self.value
27 @staticmethod
28 def from_str(string: str) -> "RecallMode":
29 """
30 Convert a string to a RecallMode enum.
31 """
32 enum_map = {e.value: e for e in RecallMode}
33 mode = enum_map.get(string)
34 if mode is None:
35 msg = f"Unknown recall mode '{string}'. Supported modes are: {list(enum_map.keys())}"
36 raise ValueError(msg)
37 return mode
40@component
41class DocumentRecallEvaluator:
42 """
43 Evaluator that calculates the Recall score for a list of documents.
45 Returns both a list of scores for each question and the average.
46 There can be multiple ground truth documents and multiple predicted documents as input.
48 Usage example:
49 ```python
50 from haystack import Document
51 from haystack.components.evaluators import DocumentRecallEvaluator
53 evaluator = DocumentRecallEvaluator()
54 result = evaluator.run(
55 ground_truth_documents=[
56 [Document(content="France")],
57 [Document(content="9th century"), Document(content="9th")],
58 ],
59 retrieved_documents=[
60 [Document(content="France")],
61 [Document(content="9th century"), Document(content="10th century"), Document(content="9th")],
62 ],
63 )
64 print(result["individual_scores"])
65 # [1.0, 1.0]
66 print(result["score"])
67 # 1.0
68 ```
69 """
71 def __init__(
72 self, mode: str | RecallMode = RecallMode.SINGLE_HIT, document_comparison_field: str = "content"
73 ) -> None:
74 """
75 Create a DocumentRecallEvaluator component.
77 :param mode:
78 Mode to use for calculating the recall score.
79 :param document_comparison_field:
80 The Document field to use for comparison. Possible options:
81 - `"content"`: uses `doc.content`
82 - `"id"`: uses `doc.id`
83 - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
84 (e.g. `"meta.file_id"`, `"meta.page_number"`)
85 Nested keys are supported (e.g. `"meta.source.url"`).
86 """
87 if isinstance(mode, str):
88 mode = RecallMode.from_str(mode)
90 self.mode = mode
91 self.document_comparison_field = document_comparison_field
93 def _get_comparison_value(self, doc: Document) -> Any:
94 """
95 Extract the comparison value from a document based on the configured field.
96 """
97 if self.document_comparison_field == "content":
98 return doc.content
99 if self.document_comparison_field == "id":
100 return doc.id
101 if self.document_comparison_field.startswith("meta."):
102 parts = self.document_comparison_field[5:].split(".")
103 value = doc.meta
104 for part in parts:
105 if not isinstance(value, dict) or part not in value:
106 return None
107 value = value[part]
108 return value
109 msg = (
110 f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
111 "Use 'content', 'id', or 'meta.<key>'."
112 )
113 raise ValueError(msg)
115 def _unique_comparison_values(self, documents: list[Document]) -> set:
116 """
117 Collect the unique comparison values of the documents, ignoring missing or empty ones.
118 """
119 return {value for doc in documents if (value := self._get_comparison_value(doc)) not in ("", None)}
121 def _comparison_sets(
122 self, ground_truth_documents: list[Document], retrieved_documents: list[Document]
123 ) -> tuple[set, set]:
124 """
125 Collect the unique comparison values of both sides, warning if either has no value to match on.
126 """
127 unique_truths = self._unique_comparison_values(ground_truth_documents)
128 if not unique_truths:
129 logger.warning(
130 "There are no ground truth documents or none of them contain a valid comparison value. "
131 "Score will be set to 0."
132 )
134 unique_retrievals = self._unique_comparison_values(retrieved_documents)
135 if not unique_retrievals:
136 logger.warning(
137 "There are no retrieved documents or none of them contain a valid comparison value. "
138 "Score will be set to 0."
139 )
141 return unique_truths, unique_retrievals
143 def _recall_single_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
144 unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents)
145 if not unique_truths or not unique_retrievals:
146 return 0.0
148 return float(len(unique_truths.intersection(unique_retrievals)) > 0)
150 def _recall_multi_hit(self, ground_truth_documents: list[Document], retrieved_documents: list[Document]) -> float:
151 unique_truths, unique_retrievals = self._comparison_sets(ground_truth_documents, retrieved_documents)
152 if not unique_truths or not unique_retrievals:
153 return 0.0
155 return len(unique_truths.intersection(unique_retrievals)) / len(unique_truths)
157 @component.output_types(score=float, individual_scores=list[float])
158 def run(
159 self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
160 ) -> dict[str, Any]:
161 """
162 Run the DocumentRecallEvaluator on the given inputs.
164 `ground_truth_documents` and `retrieved_documents` must have the same length.
166 :param ground_truth_documents:
167 A list of expected documents for each question.
168 :param retrieved_documents:
169 A list of retrieved documents for each question.
170 A dictionary with the following outputs:
171 - `score` - The average of calculated scores.
172 - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the proportion of matching
173 documents retrieved. If the mode is `single_hit`, the individual scores are 0 or 1.
174 """
175 if len(ground_truth_documents) != len(retrieved_documents):
176 msg = "The length of ground_truth_documents and retrieved_documents must be the same."
177 raise ValueError(msg)
179 if self.mode == RecallMode.SINGLE_HIT:
180 mode_function = self._recall_single_hit
181 elif self.mode == RecallMode.MULTI_HIT:
182 mode_function = self._recall_multi_hit
184 scores = [mode_function(gt, ret) for gt, ret in zip(ground_truth_documents, retrieved_documents, strict=True)]
186 return {"score": sum(scores) / len(retrieved_documents), "individual_scores": scores}
188 def to_dict(self) -> dict[str, Any]:
189 """
190 Serializes the component to a dictionary.
192 :returns:
193 Dictionary with serialized data.
194 """
195 return default_to_dict(self, mode=str(self.mode), document_comparison_field=self.document_comparison_field)