Coverage for haystack/components/evaluators/document_ndcg.py: 100%
70 statements
« prev ^ index » next coverage.py v7.15.4, created at 2026-08-21 13:53 +0000
1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai>
2#
3# SPDX-License-Identifier: Apache-2.0
5from math import log2
6from typing import Any
8from haystack import Document, component, default_to_dict
11@component
12class DocumentNDCGEvaluator:
13 """
14 Evaluator that calculates the normalized discounted cumulative gain (NDCG) of retrieved documents.
16 Each question can have multiple ground truth documents and multiple retrieved documents.
17 If the ground truth documents have relevance scores, the NDCG calculation uses these scores.
18 Otherwise, it assumes binary relevance of all ground truth documents.
20 Usage example:
21 ```python
22 from haystack import Document
23 from haystack.components.evaluators import DocumentNDCGEvaluator
25 evaluator = DocumentNDCGEvaluator()
26 result = evaluator.run(
27 ground_truth_documents=[[Document(content="France", score=1.0), Document(content="Paris", score=0.5)]],
28 retrieved_documents=[[Document(content="France"), Document(content="Germany"), Document(content="Paris")]],
29 )
30 print(result["individual_scores"])
31 # [0.8869]
32 print(result["score"])
33 # 0.8869
34 ```
35 """
37 def __init__(self, document_comparison_field: str = "content") -> None:
38 """
39 Create a DocumentNDCGEvaluator component.
41 :param document_comparison_field:
42 The Document field to use for comparison. Possible options:
43 - `"content"`: uses `doc.content`
44 - `"id"`: uses `doc.id`
45 - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]`
46 (e.g. `"meta.file_id"`, `"meta.page_number"`)
47 Nested keys are supported (e.g. `"meta.source.url"`).
48 """
49 self.document_comparison_field = document_comparison_field
51 def _get_comparison_value(self, doc: Document) -> Any:
52 """
53 Extract the comparison value from a document based on the configured field.
54 """
55 if self.document_comparison_field == "content":
56 return doc.content
57 if self.document_comparison_field == "id":
58 return doc.id
59 if self.document_comparison_field.startswith("meta."):
60 parts = self.document_comparison_field[5:].split(".")
61 value = doc.meta
62 for part in parts:
63 if not isinstance(value, dict) or part not in value:
64 return None
65 value = value[part]
66 return value
67 msg = (
68 f"Unsupported document_comparison_field: '{self.document_comparison_field}'. "
69 "Use 'content', 'id', or 'meta.<key>'."
70 )
71 raise ValueError(msg)
73 def _build_relevance_map(self, gt_docs: list[Document]) -> dict[Any, float]:
74 """
75 Map each ground truth comparison value to its relevance score.
77 Documents whose comparison value cannot be determined (e.g. missing meta key) are skipped,
78 since they can never be matched during retrieval either. Documents that share a comparison
79 value are collapsed to a single entry keeping the highest relevance, so `calculate_dcg` and
80 `calculate_idcg` credit the same unique relevant set with the same scores.
81 """
82 relevant_value_to_score: dict[Any, float] = {}
83 for doc in gt_docs:
84 value = self._get_comparison_value(doc)
85 if value is None:
86 continue
87 relevance = doc.score if doc.score is not None else 1.0
88 relevant_value_to_score[value] = max(relevant_value_to_score.get(value, relevance), relevance)
89 return relevant_value_to_score
91 def to_dict(self) -> dict[str, Any]:
92 """
93 Serializes the component to a dictionary.
95 :returns:
96 Dictionary with serialized data.
97 """
98 return default_to_dict(self, document_comparison_field=self.document_comparison_field)
100 @component.output_types(score=float, individual_scores=list[float])
101 def run(
102 self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]]
103 ) -> dict[str, Any]:
104 """
105 Run the DocumentNDCGEvaluator on the given inputs.
107 `ground_truth_documents` and `retrieved_documents` must have the same length.
108 The list items within `ground_truth_documents` and `retrieved_documents` can differ in length.
110 :param ground_truth_documents:
111 Lists of expected documents, one list per question. Binary relevance is used if documents have no scores.
112 :param retrieved_documents:
113 Lists of retrieved documents, one list per question.
114 :returns:
115 A dictionary with the following outputs:
116 - `score` - The average of calculated scores.
117 - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents the NDCG for each question.
118 """
119 self.validate_inputs(ground_truth_documents, retrieved_documents)
121 individual_scores = []
123 for gt_docs, ret_docs in zip(ground_truth_documents, retrieved_documents, strict=True):
124 dcg = self.calculate_dcg(gt_docs, ret_docs)
125 idcg = self.calculate_idcg(gt_docs)
126 ndcg = dcg / idcg if idcg > 0 else 0
127 individual_scores.append(ndcg)
129 score = sum(individual_scores) / len(ground_truth_documents)
131 return {"score": score, "individual_scores": individual_scores}
133 @staticmethod
134 def validate_inputs(gt_docs: list[list[Document]], ret_docs: list[list[Document]]) -> None:
135 """
136 Validate the input parameters.
138 :param gt_docs:
139 The ground_truth_documents to validate.
140 :param ret_docs:
141 The retrieved_documents to validate.
143 :raises ValueError:
144 If the ground_truth_documents or the retrieved_documents are an empty list.
145 If the length of ground_truth_documents and retrieved_documents differs.
146 If any list of documents in ground_truth_documents contains a mix of documents with and without a score.
147 """
148 if len(gt_docs) == 0 or len(ret_docs) == 0:
149 msg = "ground_truth_documents and retrieved_documents must be provided."
150 raise ValueError(msg)
152 if len(gt_docs) != len(ret_docs):
153 msg = "The length of ground_truth_documents and retrieved_documents must be the same."
154 raise ValueError(msg)
156 for docs in gt_docs:
157 if any(doc.score is not None for doc in docs) and any(doc.score is None for doc in docs):
158 msg = "Either none or all documents in each list of ground_truth_documents must have a score."
159 raise ValueError(msg)
161 def calculate_dcg(self, gt_docs: list[Document], ret_docs: list[Document]) -> float:
162 """
163 Calculate the discounted cumulative gain (DCG) of the retrieved documents.
165 :param gt_docs:
166 The ground truth documents.
167 :param ret_docs:
168 The retrieved documents.
169 :returns:
170 The discounted cumulative gain (DCG) of the retrieved
171 documents based on the ground truth documents.
172 """
173 dcg = 0.0
174 relevant_value_to_score = self._build_relevance_map(gt_docs)
176 # Credit each relevant value at most once by popping it when first matched. A duplicate
177 # retrieval of the same document then finds nothing, so it cannot inflate DCG past IDCG.
178 for i, doc in enumerate(ret_docs):
179 value = self._get_comparison_value(doc)
180 if value is not None and value in relevant_value_to_score:
181 dcg += relevant_value_to_score.pop(value) / log2(i + 2) # i + 2 because i is 0-indexed
182 return dcg
184 def calculate_idcg(self, gt_docs: list[Document]) -> float:
185 """
186 Calculate the ideal discounted cumulative gain (IDCG) of the ground truth documents.
188 Ground truth documents whose comparison value cannot be determined (e.g. missing meta key)
189 are excluded, since they can never be matched in `calculate_dcg` either. Documents that share
190 a comparison value are collapsed to a single relevant item, mirroring `calculate_dcg`, which
191 credits each relevant value at most once. Including duplicates or unmatchable documents here
192 would inflate the IDCG and make it impossible for NDCG to reach 1.0 for a perfect retrieval.
194 :param gt_docs:
195 The ground truth documents.
196 :returns:
197 The ideal discounted cumulative gain (IDCG) of the ground truth documents.
198 """
199 relevant_value_to_score = self._build_relevance_map(gt_docs)
201 idcg = 0.0
202 for i, relevance in enumerate(sorted(relevant_value_to_score.values(), reverse=True)):
203 idcg += relevance / log2(i + 2) # i + 2 because i is 0-indexed
204 return idcg