Coverage for haystack/components/evaluators/document_map.py: 92%

1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai> 

2# 

3# SPDX-License-Identifier: Apache-2.0 

4 

5from typing import Any 

6 

7from haystack import Document, component, default_to_dict 

8 

9 

10@component 

11class DocumentMAPEvaluator: 

12 """ 

13 A Mean Average Precision (MAP) evaluator for documents. 

14 

15 Evaluator that calculates the mean average precision of the retrieved documents, a metric 

16 that measures how high retrieved documents are ranked. 

17 Each question can have multiple ground truth documents and multiple retrieved documents. 

18 

19 `DocumentMAPEvaluator` doesn't normalize its inputs, the `DocumentCleaner` component 

20 should be used to clean and normalize the documents before passing them to this evaluator. 

21 

22 Usage example: 

23 ```python 

24 from haystack import Document 

25 from haystack.components.evaluators import DocumentMAPEvaluator 

26 

27 evaluator = DocumentMAPEvaluator() 

28 result = evaluator.run( 

29 ground_truth_documents=[ 

30 [Document(content="France")], 

31 [Document(content="9th century"), Document(content="9th")], 

32 ], 

33 retrieved_documents=[ 

34 [Document(content="France")], 

35 [Document(content="9th century"), Document(content="10th century"), Document(content="9th")], 

36 ], 

37 ) 

38 

39 print(result["individual_scores"]) 

40 # [1.0, 0.8333333333333333] 

41 print(result["score"]) 

42 # 0.9166666666666666 

43 ``` 

44 """ 

45 

46 def __init__(self, document_comparison_field: str = "content") -> None: 

47 """ 

48 Create a DocumentMAPEvaluator component. 

49 

50 :param document_comparison_field: 

51 The Document field to use for comparison. Possible options: 

52 - `"content"`: uses `doc.content` 

53 - `"id"`: uses `doc.id` 

54 - A `meta.` prefix followed by a key name: uses `doc.meta["<key>"]` 

55 (e.g. `"meta.file_id"`, `"meta.page_number"`) 

56 Nested keys are supported (e.g. `"meta.source.url"`). 

57 """ 

58 self.document_comparison_field = document_comparison_field 

59 

60 def _get_comparison_value(self, doc: Document) -> Any: 

61 """ 

62 Extract the comparison value from a document based on the configured field. 

63 """ 

64 if self.document_comparison_field == "content": 

65 return doc.content 

66 if self.document_comparison_field == "id": 

67 return doc.id 

68 if self.document_comparison_field.startswith("meta."): 

69 parts = self.document_comparison_field[5:].split(".") 

70 value = doc.meta 

71 for part in parts: 

72 if not isinstance(value, dict) or part not in value: 

73 return None 

74 value = value[part] 

75 return value 

76 msg = ( 

77 f"Unsupported document_comparison_field: '{self.document_comparison_field}'. " 

78 "Use 'content', 'id', or 'meta.<key>'." 

79 ) 

80 raise ValueError(msg) 

81 

82 def to_dict(self) -> dict[str, Any]: 

83 """ 

84 Serializes the component to a dictionary. 

85 

86 :returns: 

87 Dictionary with serialized data. 

88 """ 

89 return default_to_dict(self, document_comparison_field=self.document_comparison_field) 

90 

91 # Refer to https://www.pinecone.io/learn/offline-evaluation/ for the algorithm. 

92 @component.output_types(score=float, individual_scores=list[float]) 

93 def run( 

94 self, ground_truth_documents: list[list[Document]], retrieved_documents: list[list[Document]] 

95 ) -> dict[str, Any]: 

96 """ 

97 Run the DocumentMAPEvaluator on the given inputs. 

98 

99 All lists must have the same length. 

100 

101 :param ground_truth_documents: 

102 A list of expected documents for each question. 

103 :param retrieved_documents: 

104 A list of retrieved documents for each question. 

105 :returns: 

106 A dictionary with the following outputs: 

107 - `score` - The average of calculated scores. 

108 - `individual_scores` - A list of numbers from 0.0 to 1.0 that represents how high retrieved documents 

109 are ranked. 

110 """ 

111 if len(ground_truth_documents) != len(retrieved_documents): 

112 msg = "The length of ground_truth_documents and retrieved_documents must be the same." 

113 raise ValueError(msg) 

114 

115 individual_scores = [] 

116 

117 for ground_truth, retrieved in zip(ground_truth_documents, retrieved_documents, strict=True): 

118 average_precision = 0.0 

119 average_precision_numerator = 0.0 

120 retrieved_relevant_documents = 0 

121 

122 # A list keeps the deduplication working for unhashable comparison values, for example when 

123 # document_comparison_field points to a meta key holding a list. 

124 uncredited_ground_truth_values: list[Any] = [] 

125 for doc in ground_truth: 

126 value = self._get_comparison_value(doc) 

127 if value is not None and value not in uncredited_ground_truth_values: 

128 uncredited_ground_truth_values.append(value) 

129 

130 total_relevant_documents = len(uncredited_ground_truth_values) 

131 for rank, retrieved_document in enumerate(retrieved): 

132 retrieved_value = self._get_comparison_value(retrieved_document) 

133 if retrieved_value is None: 

134 continue 

135 

136 if retrieved_value in uncredited_ground_truth_values: 

137 uncredited_ground_truth_values.remove(retrieved_value) 

138 retrieved_relevant_documents += 1 

139 average_precision_numerator += retrieved_relevant_documents / (rank + 1) 

140 if total_relevant_documents: 

141 average_precision = average_precision_numerator / total_relevant_documents 

142 individual_scores.append(average_precision) 

143 

144 score = sum(individual_scores) / len(ground_truth_documents) 

145 return {"score": score, "individual_scores": individual_scores}