Coverage for haystack/components/evaluators/answer_exact_match.py: 100%

1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai> 

2# 

3# SPDX-License-Identifier: Apache-2.0 

4 

5from typing import Any 

6 

7from haystack.core.component import component 

8 

9 

10@component 

11class AnswerExactMatchEvaluator: 

12 """ 

13 An answer exact match evaluator class. 

14 

15 The evaluator that checks if the predicted answers matches any of the ground truth answers exactly. 

16 The result is a number from 0.0 to 1.0, it represents the proportion of predicted answers 

17 that matched one of the ground truth answers. 

18 There can be multiple ground truth answers and multiple predicted answers as input. 

19 

20 

21 Usage example: 

22 ```python 

23 from haystack.components.evaluators import AnswerExactMatchEvaluator 

24 

25 evaluator = AnswerExactMatchEvaluator() 

26 result = evaluator.run( 

27 ground_truth_answers=["Berlin", "Paris"], 

28 predicted_answers=["Berlin", "Lyon"], 

29 ) 

30 

31 print(result["individual_scores"]) 

32 # [1, 0] 

33 print(result["score"]) 

34 # 0.5 

35 ``` 

36 """ 

37 

38 @component.output_types(individual_scores=list[int], score=float) 

39 def run(self, ground_truth_answers: list[str], predicted_answers: list[str]) -> dict[str, Any]: 

40 """ 

41 Run the AnswerExactMatchEvaluator on the given inputs. 

42 

43 The `ground_truth_answers` and `retrieved_answers` must have the same length. 

44 

45 :param ground_truth_answers: 

46 A list of expected answers. 

47 :param predicted_answers: 

48 A list of predicted answers. 

49 :returns: 

50 A dictionary with the following outputs: 

51 - `individual_scores` - A list of 0s and 1s, where 1 means that the predicted answer matched one of the 

52 ground truth. 

53 - `score` - A number from 0.0 to 1.0 that represents the proportion of questions where any predicted 

54 answer matched one of the ground truth answers. 

55 """ 

56 if not len(ground_truth_answers) == len(predicted_answers): 

57 raise ValueError("The length of ground_truth_answers and predicted_answers must be the same.") 

58 

59 matches = [] 

60 for truth, extracted in zip(ground_truth_answers, predicted_answers, strict=True): 

61 if truth == extracted: 

62 matches.append(1) 

63 else: 

64 matches.append(0) 

65 

66 # The proportion of questions where any predicted answer matched one of the ground truth answers 

67 average = sum(matches) / len(predicted_answers) 

68 

69 return {"individual_scores": matches, "score": average}