Coverage for haystack/components/evaluators/document_mrr.py: 90%

1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai> 

2# 

3# SPDX-License-Identifier: Apache-2.0 

4 

5from typing import Any 

6 

7from haystack import Document, component, default_to_dict 

8 

9 

10@component 

11class DocumentMRREvaluator: 

12 """ 

13 Evaluator that calculates the mean reciprocal rank of the retrieved documents. 

14 

15 MRR measures how high the first retrieved document is ranked. 

16 Each question can have multiple ground truth documents and multiple retrieved documents. 

17 

18 `DocumentMRREvaluator` doesn't normalize its inputs, the `DocumentCleaner` component 

19 should be used to clean and normalize the documents before passing them to this evaluator. 

20 

21 Usage example: 

22 ```python 

23 from haystack import Document 

24 from haystack.components.evaluators import DocumentMRREvaluator 

25 

26 evaluator = DocumentMRREvaluator() 

27 result = evaluator.run( 

28 ground_truth_documents=[ 

29 [Document(content="France")], 

30 [Document(content="9th century"), Document(content="9th")], 

31 ], 

32 retrieved_documents=[ 

33 [Document(content="France")], 

34 [Document(content="9th century"), Document(content="10th century"), Document(content="9th")], 

35 ], 

36 ) 

37 print(result["individual_scores"]) 

38 # [1.0, 1.0] 

39 print(result["score"]) 

40 # 1.0 

41 ``` 

42 """ 

43 

44 def __init__(self, document_comparison_field: str = "content") -> None: 

45 """ 

46 Create a DocumentMRREvaluator component. 

47 

48 :param document_comparison_field: 

49 The Document field to use for comparison. Possible options: 

50 - `"content"`: uses `doc.content` 

51 - `"id"`: uses `doc.id` 

52 - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]` 

53 (e.g. `"meta.file_id"`, `"meta.page_number"`) 

54 Nested keys are supported (e.g. `"meta.source.url"`). 

55 """ 

56 self.document_comparison_field = document_comparison_field 

57 

58 def _get_comparison_value(self, doc: Document) -> Any: 

59 """ 

60 Extract the comparison value from a document based on the configured field. 

61 """ 

62 if self.document_comparison_field == "content": 

63 return doc.content 

64 if self.document_comparison_field == "id": 

65 return doc.id 

66 if self.document_comparison_field.startswith("meta."): 

67 parts = self.document_comparison_field[5:].split(".") 

68 value = doc.meta 

69 for part in parts: 

70 if not isinstance(value, dict) or part not in value: 

71 return None 

72 value = value[part] 

73 return value 

74 msg = ( 

75 f"Unsupported document_comparison_field: '{self.document_comparison_field}'. " 

76 "Use 'content', 'id', or 'meta.<key>'." 

77 ) 

78 raise ValueError(msg) 

79 

80 def to_dict(self) -> dict[str, Any]: 

81 """ 

82 Serializes the component to a dictionary. 

83 

84 :returns: 

85 Dictionary with serialized data. 

86 """ 

87 return default_to_dict(self, document_comparison_field=self.document_comparison_field) 

88 

89 # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm. 

90 @component.output_types(score=float, individual_scores=list[float]) 

91 def run( 

92 self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]] 

93 ) -> dict[str, Any]: 

94 """ 

95 Run the DocumentMRREvaluator on the given inputs. 

96 

97 `ground_truth_documents` and `retrieved_documents` must have the same length. 

98 

99 :param ground_truth_documents: 

100 A list of expected documents for each question. 

101 :param retrieved_documents: 

102 A list of retrieved documents for each question. 

103 :returns: 

104 A dictionary with the following outputs: 

105 - `score` - The average of calculated scores. 

106 - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high the first retrieved 

107 document is ranked. 

108 """ 

109 if len(ground_truth_documents) != len(retrieved_documents): 

110 msg = "The length of ground_truth_documents and retrieved_documents must be the same." 

111 raise ValueError(msg) 

112 

113 individual_scores = [] 

114 

115 for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True): 

116 reciprocal_rank = 0.0 

117 

118 ground_truth_values = [val for doc in ground_truth if (val := self._get_comparison_value(doc)) is not None] 

119 for rank, retrieved_document in enumerate(retrieved): 

120 retrieved_value = self._get_comparison_value(retrieved_document) 

121 if retrieved_value is None: 

122 continue 

123 if retrieved_value in ground_truth_values: 

124 reciprocal_rank = 1 / (rank + 1) 

125 break 

126 individual_scores.append(reciprocal_rank) 

127 

128 score = sum(individual_scores) / len(ground_truth_documents) 

129 

130 return {"score": score, "individual_scores": individual_scores}