Coverage for haystack/utils/filters.py: 100%

1# SPDX-FileCopyrightText: 2022-present deepset GmbH <info@deepset.ai> 

2# 

3# SPDX-License-Identifier: Apache-2.0 

4 

5from dataclasses import fields 

6from datetime import datetime 

7from typing import Any 

8 

9import dateutil.parser 

10 

11from haystack.dataclasses import ByteStream, Document 

12from haystack.errors import FilterError 

13 

14 

15def document_matches_filter( 

16 filters: dict[str, Any], document: Document | ByteStream, *, strict_datetime_comparison: bool = False 

17) -> bool: 

18 """ 

19 Return whether `filters` match the Document or the ByteStream. 

20 

21 For a detailed specification of the filters, refer to the 

22 `DocumentStore.filter_documents()` protocol documentation. 

23 

24 :param strict_datetime_comparison: 

25 If `True`, timezone-naive and timezone-aware datetimes never match each other. 

26 If `False` (the default), the timezone from the aware datetime is copied to the naive one before comparing. 

27 """ 

28 if "field" in filters: 

29 return _comparison_condition( 

30 condition=filters, document=document, strict_datetime_comparison=strict_datetime_comparison 

31 ) 

32 return _logic_condition(condition=filters, document=document, strict_datetime_comparison=strict_datetime_comparison) 

33 

34 

35def _and(document: Document | ByteStream, conditions: list[dict[str, Any]], strict_datetime_comparison: bool) -> bool: 

36 return all( 

37 _comparison_condition( 

38 condition=condition, document=document, strict_datetime_comparison=strict_datetime_comparison 

39 ) 

40 for condition in conditions 

41 ) 

42 

43 

44def _or(document: Document | ByteStream, conditions: list[dict[str, Any]], strict_datetime_comparison: bool) -> bool: 

45 return any( 

46 _comparison_condition( 

47 condition=condition, document=document, strict_datetime_comparison=strict_datetime_comparison 

48 ) 

49 for condition in conditions 

50 ) 

51 

52 

53def _not(document: Document | ByteStream, conditions: list[dict[str, Any]], strict_datetime_comparison: bool) -> bool: 

54 return not _and(document=document, conditions=conditions, strict_datetime_comparison=strict_datetime_comparison) 

55 

56 

57LOGICAL_OPERATORS = {"NOT": _not, "OR": _or, "AND": _and} 

58 

59 

60def _equal(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

61 if value == filter_value: 

62 return True 

63 return _dates_are_equal(value=value, filter_value=filter_value, strict=strict_datetime_comparison) 

64 

65 

66def _not_equal(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

67 return not _equal(value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison) 

68 

69 

70def _looks_like_iso_date(value: Any) -> bool: 

71 """ 

72 Cheaply reject values that can't be a full ISO 8601 date, before paying for a real parse. 

73 

74 Deliberately conservative: a complete `YYYY-MM-DD` prefix is required, so partial dates such as 

75 "2025" are not read as dates, and the ISO 8601 basic format ("20250203") is not recognised. 

76 """ 

77 return ( 

78 isinstance(value, str) 

79 and len(value) >= 10 

80 and value[:4].isdigit() 

81 and value[4] == "-" 

82 and value[5:7].isdigit() 

83 and value[7] == "-" 

84 and value[8:10].isdigit() 

85 and (len(value) == 10 or value[10] in {"T", "t", " "}) 

86 ) 

87 

88 

89def _parse_iso_date(value: str) -> datetime | None: 

90 """Parse a strict ISO 8601 string, returning None if it isn't one.""" 

91 try: 

92 return datetime.fromisoformat(value) 

93 except ValueError: 

94 # Python 3.10's fromisoformat rejects valid ISO 8601 spellings that later versions accept, 

95 # most notably a trailing "Z". 

96 try: 

97 return dateutil.parser.isoparse(value) 

98 except (ValueError, OverflowError): 

99 return None 

100 

101 

102def _dates_are_equal(value: Any, filter_value: Any, strict: bool) -> bool: 

103 """ 

104 Return whether both values are ISO 8601 datetimes denoting the same point in time. 

105 

106 Only strict ISO 8601 strings and `datetime` objects are considered. Mixed-awareness datetimes are 

107 reconciled unless strict mode is enabled. Returns False for anything that isn't a pair of comparable 

108 datetimes, so that `==` stays total. 

109 """ 

110 parsed: list[datetime] = [] 

111 for candidate in (value, filter_value): 

112 if isinstance(candidate, datetime): 

113 parsed.append(candidate) 

114 continue 

115 if not _looks_like_iso_date(candidate): 

116 return False 

117 date = _parse_iso_date(candidate) 

118 if date is None: 

119 return False 

120 parsed.append(date) 

121 

122 first, second = parsed 

123 if strict and (first.tzinfo is None) != (second.tzinfo is None): 

124 return False 

125 first, second = _ensure_both_dates_naive_or_aware(first, second) 

126 return first == second 

127 

128 

129def _prepare_ordering_comparison( 

130 value: Any, filter_value: Any, strict_datetime_comparison: bool 

131) -> tuple[Any, Any, bool]: 

132 """ 

133 Normalize both values for ordering comparisons, parsing strings as dates. 

134 

135 :returns: 

136 A tuple containing the normalized value, normalized filter value, and whether the values are comparable. 

137 The boolean is `False` when strict datetime comparison is enabled and one datetime is timezone-naive while 

138 the other is timezone-aware; otherwise it is `True`. 

139 """ 

140 if isinstance(value, str) or isinstance(filter_value, str): 

141 if not isinstance(value, datetime): 

142 value = _parse_date(value) 

143 if not isinstance(filter_value, datetime): 

144 filter_value = _parse_date(filter_value) 

145 

146 if isinstance(value, datetime) and isinstance(filter_value, datetime): 

147 if strict_datetime_comparison and (value.tzinfo is None) != (filter_value.tzinfo is None): 

148 return value, filter_value, False 

149 value, filter_value = _ensure_both_dates_naive_or_aware(value, filter_value) 

150 

151 if isinstance(filter_value, list): 

152 msg = f"Filter value can't be of type {type(filter_value)} using operators '>', '>=', '<', '<='" 

153 raise FilterError(msg) 

154 return value, filter_value, True 

155 

156 

157def _greater_than(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

158 if value is None or filter_value is None: 

159 # We can't compare None values reliably using operators '>', '>=', '<', '<=' 

160 return False 

161 

162 value, filter_value, comparable = _prepare_ordering_comparison( 

163 value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison 

164 ) 

165 if not comparable: 

166 return False 

167 return value > filter_value 

168 

169 

170def _parse_date(value: str) -> datetime: 

171 """Try parsing the value as an ISO format date, then fall back to dateutil.parser.""" 

172 try: 

173 return datetime.fromisoformat(value) 

174 except (ValueError, TypeError): 

175 try: 

176 return dateutil.parser.parse(value) 

177 except (ValueError, TypeError) as exc: 

178 msg = ( 

179 "Can't compare strings using operators '>', '>=', '<', '<='. " 

180 "Strings are only comparable if they are ISO formatted dates." 

181 ) 

182 raise FilterError(msg) from exc 

183 

184 

185def _ensure_both_dates_naive_or_aware(date1: datetime, date2: datetime) -> tuple[datetime, datetime]: 

186 """Ensure that both dates are either naive or aware.""" 

187 # Both naive 

188 if date1.tzinfo is None and date2.tzinfo is None: 

189 return date1, date2 

190 

191 # Both aware 

192 if date1.tzinfo is not None and date2.tzinfo is not None: 

193 return date1, date2 

194 

195 # One naive, one aware 

196 if date1.tzinfo is None: 

197 date1 = date1.replace(tzinfo=date2.tzinfo) 

198 else: 

199 date2 = date2.replace(tzinfo=date1.tzinfo) 

200 return date1, date2 

201 

202 

203def _greater_than_equal(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

204 if value is None or filter_value is None: 

205 # We can't compare None values reliably using operators '>', '>=', '<', '<=' 

206 return False 

207 

208 value, filter_value, comparable = _prepare_ordering_comparison( 

209 value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison 

210 ) 

211 if not comparable: 

212 return False 

213 return value >= filter_value 

214 

215 

216def _less_than(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

217 if value is None or filter_value is None: 

218 # We can't compare None values reliably using operators '>', '>=', '<', '<=' 

219 return False 

220 

221 value, filter_value, comparable = _prepare_ordering_comparison( 

222 value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison 

223 ) 

224 if not comparable: 

225 return False 

226 return value < filter_value 

227 

228 

229def _less_than_equal(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

230 if value is None or filter_value is None: 

231 # We can't compare None values reliably using operators '>', '>=', '<', '<=' 

232 return False 

233 

234 value, filter_value, comparable = _prepare_ordering_comparison( 

235 value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison 

236 ) 

237 if not comparable: 

238 return False 

239 return value <= filter_value 

240 

241 

242def _in(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

243 if not isinstance(filter_value, list): 

244 msg = ( 

245 f"Filter value must be a `list` when using operator 'in' or 'not in', received type '{type(filter_value)}'" 

246 ) 

247 raise FilterError(msg) 

248 return any(_equal(e, value, strict_datetime_comparison) for e in filter_value) 

249 

250 

251def _not_in(value: Any, filter_value: Any, strict_datetime_comparison: bool) -> bool: 

252 return not _in(value=value, filter_value=filter_value, strict_datetime_comparison=strict_datetime_comparison) 

253 

254 

255COMPARISON_OPERATORS = { 

256 "==": _equal, 

257 "!=": _not_equal, 

258 ">": _greater_than, 

259 ">=": _greater_than_equal, 

260 "<": _less_than, 

261 "<=": _less_than_equal, 

262 "in": _in, 

263 "not in": _not_in, 

264} 

265 

266 

267def _logic_condition( 

268 condition: dict[str, Any], document: Document | ByteStream, strict_datetime_comparison: bool 

269) -> bool: 

270 if "operator" not in condition: 

271 msg = f"'operator' key missing in {condition}" 

272 raise FilterError(msg) 

273 if "conditions" not in condition: 

274 msg = f"'conditions' key missing in {condition}" 

275 raise FilterError(msg) 

276 operator: str = condition["operator"] 

277 if operator not in LOGICAL_OPERATORS: 

278 msg = f"Unknown logical operator '{operator}'. Valid operators are: {sorted(LOGICAL_OPERATORS)}" 

279 raise FilterError(msg) 

280 conditions: list[dict[str, Any]] = condition["conditions"] 

281 return LOGICAL_OPERATORS[operator]( 

282 document=document, conditions=conditions, strict_datetime_comparison=strict_datetime_comparison 

283 ) 

284 

285 

286def _comparison_condition( 

287 condition: dict[str, Any], document: Document | ByteStream, strict_datetime_comparison: bool 

288) -> bool: 

289 if "field" not in condition: 

290 # 'field' key is only found in comparison dictionaries. 

291 # We assume this is a logic dictionary since it's not present. 

292 return _logic_condition( 

293 condition=condition, document=document, strict_datetime_comparison=strict_datetime_comparison 

294 ) 

295 field: str = condition["field"] 

296 

297 if "operator" not in condition: 

298 msg = f"'operator' key missing in {condition}" 

299 raise FilterError(msg) 

300 if "value" not in condition: 

301 msg = f"'value' key missing in {condition}" 

302 raise FilterError(msg) 

303 

304 if "." in field: 

305 # Handles fields formatted like so: 

306 # 'meta.person.name' 

307 parts = field.split(".") 

308 document_value = getattr(document, parts[0]) 

309 for part in parts[1:]: 

310 if not isinstance(document_value, dict) or part not in document_value: 

311 # If a field is not found (or an intermediate value is not a dict, 

312 # e.g. None) we treat it as None 

313 document_value = None 

314 break 

315 document_value = document_value[part] 

316 elif field not in [f.name for f in fields(document)]: 

317 # Converted legacy filters don't add the `meta.` prefix, so we assume 

318 # that all filter fields that are not actual fields in Document are converted 

319 # filters. 

320 # 

321 # We handle this to avoid breaking compatibility with converted legacy filters. 

322 # This will be removed as soon as we stop supporting legacy filters. 

323 document_value = document.meta.get(field) 

324 else: 

325 document_value = getattr(document, field) 

326 operator: str = condition["operator"] 

327 if operator not in COMPARISON_OPERATORS: 

328 msg = f"Unknown comparison operator '{operator}'. Valid operators are: {sorted(COMPARISON_OPERATORS)}" 

329 raise FilterError(msg) 

330 filter_value: Any = condition["value"] 

331 return COMPARISON_OPERATORS[operator]( 

332 filter_value=filter_value, value=document_value, strict_datetime_comparison=strict_datetime_comparison 

333 )