github-featured/deepset-ai--haystack
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
python-coverage-comment-action-data
deepset-ai--haystack/htmlcov/z_5084ce8bb9f0af0b_misc_py.html
T
github-actions 76ef5e8a59 ci: Update coverage data
2026-08-21 13:53:45 +00:00

356 lines
75 KiB
HTML
Raw Permalink Blame History

<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Coverage for haystack/utils/misc.py: 95%</title>
<link rel="icon" sizes="32x32" href="favicon_32_cb_c827f16f.png">
<link rel="stylesheet" href="style_cb_4667309f.css" type="text/css">
<script src="coverage_html_cb_dd2e7eb5.js" defer></script>
</head>
<body class="pyfile">
<header>
<div class="content">
<h1>
<span class="text">Coverage for </span><b>haystack<span class="sep">/</span>utils<span class="sep">/</span>misc.py</b>:
<span class="pc_cov">95%</span>
</h1>
<aside id="help_panel_wrapper">
<input id="help_panel_state" type="checkbox">
<label for="help_panel_state">
<img id="keyboard_icon" src="keybd_closed_cb_900cfef5.png" alt="Show/hide keyboard shortcuts">
</label>
<div id="help_panel">
<p class="legend">Shortcuts on this page</p>
<div class="keyhelp">
<p>
<kbd>r</kbd>
<kbd>m</kbd>
<kbd>x</kbd>
&nbsp; toggle line displays
</p>
<p>
<kbd>j</kbd>
<kbd>k</kbd>
&nbsp; next/prev highlighted chunk
</p>
<p>
<kbd>0</kbd> &nbsp; (zero) top of page
</p>
<p>
<kbd>1</kbd> &nbsp; (one) first highlighted chunk
</p>
<p>
<kbd>[</kbd>
<kbd>]</kbd>
&nbsp; prev/next file
</p>
<p>
<kbd>u</kbd> &nbsp; up to the index
</p>
<p>
<kbd>?</kbd> &nbsp; show/hide this help
</p>
</div>
</div>
</aside>
<h2>
<span class="text">96 statements &nbsp;</span>
<button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">91<span class="text"> run</span></button>
<button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">5<span class="text"> missing</span></button>
<button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">18<span class="text"> excluded</span></button>
</h2>
<p class="text">
<a id="prevFileLink" class="nav" href="z_5084ce8bb9f0af0b_jupyter_py.html">&#xab; prev</a> &nbsp; &nbsp;
<a id="indexLink" class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a id="nextFileLink" class="nav" href="z_5084ce8bb9f0af0b_requests_utils_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
<aside class="hidden">
<button type="button" class="button_next_chunk" data-shortcut="j"></button>
<button type="button" class="button_prev_chunk" data-shortcut="k"></button>
<button type="button" class="button_top_of_page" data-shortcut="0"></button>
<button type="button" class="button_first_chunk" data-shortcut="1"></button>
<button type="button" class="button_prev_file" data-shortcut="["></button>
<button type="button" class="button_next_file" data-shortcut="]"></button>
<button type="button" class="button_to_index" data-shortcut="u"></button>
<button type="button" class="button_show_hide_help" data-shortcut="?"></button>
</aside>
</div>
</header>
<main id="source">
<p class="pln"><span class="n"><a id="t1" href="#t1">1</a></span><span class="t"><span class="com"># SPDX-FileCopyrightText: 2022-present deepset GmbH &lt;info@deepset.ai&gt;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t2" href="#t2">2</a></span><span class="t"><span class="com">#</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t3" href="#t3">3</a></span><span class="t"><span class="com"># SPDX-License-Identifier: Apache-2.0</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t4" href="#t4">4</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t"><span class="key">import</span> <span class="nam">json</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t"><span class="key">import</span> <span class="nam">mimetypes</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">tempfile</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">from</span> <span class="nam">collections</span> <span class="key">import</span> <span class="nam">defaultdict</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">from</span> <span class="nam">dataclasses</span> <span class="key">import</span> <span class="nam">replace</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t"><span class="key">from</span> <span class="nam">math</span> <span class="key">import</span> <span class="nam">inf</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">from</span> <span class="nam">pathlib</span> <span class="key">import</span> <span class="nam">Path</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t"><span class="key">from</span> <span class="nam">typing</span> <span class="key">import</span> <span class="nam">TYPE_CHECKING</span><span class="op">,</span> <span class="nam">Any</span><span class="op">,</span> <span class="nam">Literal</span><span class="op">,</span> <span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t"><span class="key">from</span> <span class="nam">numpy</span> <span class="key">import</span> <span class="nam">exp</span><span class="op">,</span> <span class="nam">ndarray</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span> <span class="key">import</span> <span class="nam">logging</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t"><span class="key">if</span> <span class="nam">TYPE_CHECKING</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t"> <span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">dataclasses</span> <span class="key">import</span> <span class="nam">Document</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t"><span class="nam">CUSTOM_MIMETYPES</span> <span class="op">=</span> <span class="op">{</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t"> <span class="com"># we add markdown because it is not added by the mimetypes module</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"> <span class="com"># see https://github.com/python/cpython/pull/17995</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t"> <span class="str">&quot;.md&quot;</span><span class="op">:</span> <span class="str">&quot;text/markdown&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t"> <span class="str">&quot;.markdown&quot;</span><span class="op">:</span> <span class="str">&quot;text/markdown&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t"> <span class="com"># we add msg because it is not added by the mimetypes module</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"> <span class="str">&quot;.msg&quot;</span><span class="op">:</span> <span class="str">&quot;application/vnd.ms-outlook&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t"><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="nam">logger</span> <span class="op">=</span> <span class="nam">logging</span><span class="op">.</span><span class="nam">getLogger</span><span class="op">(</span><span class="nam">__name__</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t"><span class="key">def</span> <span class="nam">expand_page_range</span><span class="op">(</span><span class="nam">page_range</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span> <span class="op">|</span> <span class="nam">int</span><span class="op">]</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">list</span><span class="op">[</span><span class="nam">int</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="str"> Takes a list of page numbers and ranges and expands them into a list of page numbers.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str"> For example, given a page_range=[&#x27;1-3&#x27;, &#x27;5&#x27;, &#x27;8&#x27;, &#x27;10-12&#x27;] the function will return [1, 2, 3, 5, 8, 10, 11, 12]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str"> :param page_range: List of page numbers and ranges</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str"> :returns:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="str"> An expanded list of page integers</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="str"> :raises ValueError:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t"><span class="str"> If any element is not a valid integer or a range string in the format `&#x27;start-end&#x27;`.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"> <span class="nam">expanded_page_range</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"> <span class="key">for</span> <span class="nam">page</span> <span class="key">in</span> <span class="nam">page_range</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t"> <span class="key">if</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">page</span><span class="op">,</span> <span class="nam">int</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"> <span class="com"># check if it&#x27;s a range wrongly passed as an integer expression</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"> <span class="key">if</span> <span class="str">&quot;-&quot;</span> <span class="key">in</span> <span class="nam">str</span><span class="op">(</span><span class="nam">page</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"> <span class="nam">msg</span> <span class="op">=</span> <span class="str">&quot;range must be a string in the format &#x27;start-end&#x27;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">Invalid page range: </span><span class="op">{</span><span class="nam">page</span><span class="op">}</span><span class="fst"> - </span><span class="op">{</span><span class="nam">msg</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"> <span class="nam">expanded_page_range</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">page</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"> <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">page</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span> <span class="key">and</span> <span class="nam">page</span><span class="op">.</span><span class="nam">isdigit</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"> <span class="nam">expanded_page_range</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">page</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"> <span class="key">elif</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">page</span><span class="op">,</span> <span class="nam">str</span><span class="op">)</span> <span class="key">and</span> <span class="str">&quot;-&quot;</span> <span class="key">in</span> <span class="nam">page</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"> <span class="nam">parts</span> <span class="op">=</span> <span class="nam">page</span><span class="op">.</span><span class="nam">split</span><span class="op">(</span><span class="str">&quot;-&quot;</span><span class="op">,</span> <span class="nam">maxsplit</span><span class="op">=</span><span class="num">1</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">parts</span><span class="op">[</span><span class="num">0</span><span class="op">]</span><span class="op">.</span><span class="nam">isdigit</span><span class="op">(</span><span class="op">)</span> <span class="key">or</span> <span class="key">not</span> <span class="nam">parts</span><span class="op">[</span><span class="num">1</span><span class="op">]</span><span class="op">.</span><span class="nam">isdigit</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"> <span class="nam">msg</span> <span class="op">=</span> <span class="str">&quot;range must be a string in the format &#x27;start-end&#x27;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">Invalid page range: </span><span class="op">{</span><span class="nam">page</span><span class="op">}</span><span class="fst"> - </span><span class="op">{</span><span class="nam">msg</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t"> <span class="nam">start</span><span class="op">,</span> <span class="nam">end</span> <span class="op">=</span> <span class="nam">parts</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t"> <span class="nam">expanded_page_range</span><span class="op">.</span><span class="nam">extend</span><span class="op">(</span><span class="nam">range</span><span class="op">(</span><span class="nam">int</span><span class="op">(</span><span class="nam">start</span><span class="op">)</span><span class="op">,</span> <span class="nam">int</span><span class="op">(</span><span class="nam">end</span><span class="op">)</span> <span class="op">+</span> <span class="num">1</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"> <span class="key">else</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"> <span class="nam">msg</span> <span class="op">=</span> <span class="str">&quot;range must be a string in the format &#x27;start-end&#x27; or an integer&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">Invalid page range: </span><span class="op">{</span><span class="nam">page</span><span class="op">}</span><span class="fst"> - </span><span class="op">{</span><span class="nam">msg</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">expanded_page_range</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">&quot;No valid page numbers or ranges found in the input list&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"> <span class="key">return</span> <span class="nam">expanded_page_range</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t"><span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t"><span class="key">def</span> <span class="nam">expit</span><span class="op">(</span><span class="nam">x</span><span class="op">:</span> <span class="nam">float</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">float</span><span class="op">:</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t"><span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"><span class="key">def</span> <span class="nam">expit</span><span class="op">(</span><span class="nam">x</span><span class="op">:</span> <span class="nam">ndarray</span><span class="op">[</span><span class="nam">Any</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">ndarray</span><span class="op">[</span><span class="nam">Any</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">:</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"><span class="key">def</span> <span class="nam">expit</span><span class="op">(</span><span class="nam">x</span><span class="op">:</span> <span class="nam">float</span> <span class="op">|</span> <span class="nam">ndarray</span><span class="op">[</span><span class="nam">Any</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">float</span> <span class="op">|</span> <span class="nam">ndarray</span><span class="op">[</span><span class="nam">Any</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t"><span class="str"> Compute logistic sigmoid function. Maps input values to a range between 0 and 1</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t"><span class="str"> :param x: input value. Can be a scalar or a numpy array.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t"> <span class="key">return</span> <span class="num">1</span> <span class="op">/</span> <span class="op">(</span><span class="num">1</span> <span class="op">+</span> <span class="nam">exp</span><span class="op">(</span><span class="op">-</span><span class="nam">x</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t"><span class="key">def</span> <span class="nam">_guess_mime_type</span><span class="op">(</span><span class="nam">path</span><span class="op">:</span> <span class="nam">Path</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">str</span> <span class="op">|</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t"><span class="str"> Guess the MIME type of the provided file path.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t"><span class="str"> :param path: The file path to get the MIME type for.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t"><span class="str"> :returns: The MIME type of the provided file path, or `None` if the MIME type cannot be determined.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"> <span class="nam">extension</span> <span class="op">=</span> <span class="nam">path</span><span class="op">.</span><span class="nam">suffix</span><span class="op">.</span><span class="nam">lower</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t"> <span class="nam">mime_type</span> <span class="op">=</span> <span class="nam">mimetypes</span><span class="op">.</span><span class="nam">guess_type</span><span class="op">(</span><span class="nam">path</span><span class="op">.</span><span class="nam">as_posix</span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">[</span><span class="num">0</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t"> <span class="com"># lookup custom mappings if the mime type is not found</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"> <span class="key">return</span> <span class="nam">CUSTOM_MIMETYPES</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">extension</span><span class="op">,</span> <span class="nam">mime_type</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="key">def</span> <span class="nam">_get_output_dir</span><span class="op">(</span><span class="nam">out_dir</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str"> Find or create a writable directory for saving status files.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t"><span class="str"> Tries in the following order:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t"><span class="str"> 1. ~/.haystack/{out_dir}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t"><span class="str"> 2. {tempdir}/haystack/{out_dir}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t"><span class="str"> 3. ./.haystack/{out_dir}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t"><span class="str"> :raises RuntimeError: If no directory could be created.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t"><span class="str"> :returns:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t"><span class="str"> The path to the created directory.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t"> <span class="nam">candidates</span> <span class="op">=</span> <span class="op">[</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t"> <span class="nam">Path</span><span class="op">.</span><span class="nam">home</span><span class="op">(</span><span class="op">)</span> <span class="op">/</span> <span class="str">&quot;.haystack&quot;</span> <span class="op">/</span> <span class="nam">out_dir</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t"> <span class="nam">Path</span><span class="op">(</span><span class="nam">tempfile</span><span class="op">.</span><span class="nam">gettempdir</span><span class="op">(</span><span class="op">)</span><span class="op">)</span> <span class="op">/</span> <span class="str">&quot;haystack&quot;</span> <span class="op">/</span> <span class="nam">out_dir</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t"> <span class="nam">Path</span><span class="op">.</span><span class="nam">cwd</span><span class="op">(</span><span class="op">)</span> <span class="op">/</span> <span class="str">&quot;.haystack&quot;</span> <span class="op">/</span> <span class="nam">out_dir</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t"> <span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t"> <span class="key">for</span> <span class="nam">candidate</span> <span class="key">in</span> <span class="nam">candidates</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"> <span class="nam">candidate</span><span class="op">.</span><span class="nam">mkdir</span><span class="op">(</span><span class="nam">parents</span><span class="op">=</span><span class="key">True</span><span class="op">,</span> <span class="nam">exist_ok</span><span class="op">=</span><span class="key">True</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"> <span class="key">return</span> <span class="nam">str</span><span class="op">(</span><span class="nam">candidate</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t"> <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t"> <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="mis mis2 show_mis"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t"> <span class="fst">f&quot;</span><span class="fst">Could not create a writable directory for output files in any of the following locations: </span><span class="op">{</span><span class="nam">candidates</span><span class="op">}</span><span class="fst">&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="mis mis2 show_mis"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t"><span class="key">def</span> <span class="nam">_deduplicate_documents</span><span class="op">(</span><span class="nam">documents</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="str">&quot;Document&quot;</span><span class="op">]</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">list</span><span class="op">[</span><span class="str">&quot;Document&quot;</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t"><span class="str"> Deduplicate a list of documents by their id keeping the duplicate with the highest score if a score is present.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t"><span class="str"> :param documents: List of documents to deduplicate.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t"><span class="str"> :returns: List of deduplicated documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t"> <span class="com"># Keep for each Document id the one with the highest score</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t"> <span class="nam">highest_scoring_docs</span><span class="op">:</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="str">&quot;Document&quot;</span><span class="op">]</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t"> <span class="key">for</span> <span class="nam">doc</span> <span class="key">in</span> <span class="nam">documents</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t"> <span class="nam">score</span> <span class="op">=</span> <span class="nam">doc</span><span class="op">.</span><span class="nam">score</span> <span class="key">if</span> <span class="nam">doc</span><span class="op">.</span><span class="nam">score</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">else</span> <span class="op">-</span><span class="nam">inf</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t"> <span class="nam">best</span> <span class="op">=</span> <span class="nam">highest_scoring_docs</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">doc</span><span class="op">.</span><span class="nam">id</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t"> <span class="key">if</span> <span class="nam">best</span> <span class="key">is</span> <span class="key">None</span> <span class="key">or</span> <span class="nam">score</span> <span class="op">&gt;</span> <span class="op">(</span><span class="nam">best</span><span class="op">.</span><span class="nam">score</span> <span class="key">if</span> <span class="nam">best</span><span class="op">.</span><span class="nam">score</span> <span class="key">is</span> <span class="key">not</span> <span class="key">None</span> <span class="key">else</span> <span class="op">-</span><span class="nam">inf</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t"> <span class="nam">highest_scoring_docs</span><span class="op">[</span><span class="nam">doc</span><span class="op">.</span><span class="nam">id</span><span class="op">]</span> <span class="op">=</span> <span class="nam">doc</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t"> <span class="key">return</span> <span class="nam">list</span><span class="op">(</span><span class="nam">highest_scoring_docs</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t"><span class="key">def</span> <span class="nam">_reciprocal_rank_fusion</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t"> <span class="nam">document_lists</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">list</span><span class="op">[</span><span class="str">&quot;Document&quot;</span><span class="op">]</span><span class="op">]</span><span class="op">,</span> <span class="nam">weights</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">float</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t"><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">list</span><span class="op">[</span><span class="str">&quot;Document&quot;</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t"><span class="str"> Merge multiple ranked lists of Documents using Reciprocal Rank Fusion, deduplicating across lists.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t"><span class="str"> See the original paper: https://plg.uwaterloo.ca/~gvcormac/cormacksigir09-rrf.pdf</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t"><span class="str"> The constant k is set to 61 (60 was suggested by the original paper, plus 1 as python lists are 0-based and the</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t"><span class="str"> paper used 1-based ranking).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t"><span class="str"> :param document_lists: A list of ranked document lists to fuse.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t"><span class="str"> :param weights: Optional per-list weights. Defaults to equal weights.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t"><span class="str"> :returns:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t"><span class="str"> Deduplicated list of documents with updated RRF scores.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">document_lists</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t"> <span class="key">return</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t"> <span class="nam">k</span> <span class="op">=</span> <span class="num">61</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t"> <span class="nam">scores_map</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="nam">defaultdict</span><span class="op">(</span><span class="nam">int</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t"> <span class="nam">documents_map</span><span class="op">:</span> <span class="nam">dict</span> <span class="op">=</span> <span class="op">{</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t"> <span class="nam">resolved_weights</span> <span class="op">=</span> <span class="nam">weights</span> <span class="key">if</span> <span class="nam">weights</span> <span class="key">else</span> <span class="op">[</span><span class="num">1</span> <span class="op">/</span> <span class="nam">len</span><span class="op">(</span><span class="nam">document_lists</span><span class="op">)</span><span class="op">]</span> <span class="op">*</span> <span class="nam">len</span><span class="op">(</span><span class="nam">document_lists</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t"> <span class="key">for</span> <span class="nam">documents</span><span class="op">,</span> <span class="nam">weight</span> <span class="key">in</span> <span class="nam">zip</span><span class="op">(</span><span class="nam">document_lists</span><span class="op">,</span> <span class="nam">resolved_weights</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t"> <span class="key">for</span> <span class="nam">rank</span><span class="op">,</span> <span class="nam">doc</span> <span class="key">in</span> <span class="nam">enumerate</span><span class="op">(</span><span class="nam">documents</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t"> <span class="nam">scores_map</span><span class="op">[</span><span class="nam">doc</span><span class="op">.</span><span class="nam">id</span><span class="op">]</span> <span class="op">+=</span> <span class="op">(</span><span class="nam">weight</span> <span class="op">*</span> <span class="nam">len</span><span class="op">(</span><span class="nam">document_lists</span><span class="op">)</span><span class="op">)</span> <span class="op">/</span> <span class="op">(</span><span class="nam">k</span> <span class="op">+</span> <span class="nam">rank</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t"> <span class="nam">documents_map</span><span class="op">[</span><span class="nam">doc</span><span class="op">.</span><span class="nam">id</span><span class="op">]</span> <span class="op">=</span> <span class="nam">doc</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t"> <span class="key">for</span> <span class="nam">_id</span> <span class="key">in</span> <span class="nam">scores_map</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t"> <span class="nam">scores_map</span><span class="op">[</span><span class="nam">_id</span><span class="op">]</span> <span class="op">/=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">document_lists</span><span class="op">)</span> <span class="op">/</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t"> <span class="key">return</span> <span class="op">[</span><span class="nam">replace</span><span class="op">(</span><span class="nam">doc</span><span class="op">,</span> <span class="nam">score</span><span class="op">=</span><span class="nam">scores_map</span><span class="op">[</span><span class="nam">doc</span><span class="op">.</span><span class="nam">id</span><span class="op">]</span><span class="op">)</span> <span class="key">for</span> <span class="nam">doc</span> <span class="key">in</span> <span class="nam">documents_map</span><span class="op">.</span><span class="nam">values</span><span class="op">(</span><span class="op">)</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t"><span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t"><span class="key">def</span> <span class="nam">_parse_dict_from_json</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t"> <span class="nam">text</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">expected_keys</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="op">...</span><span class="op">,</span> <span class="nam">raise_on_failure</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">True</span><span class="op">]</span> <span class="op">=</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t"><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">:</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t"><span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t"><span class="key">def</span> <span class="nam">_parse_dict_from_json</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t"> <span class="nam">text</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">expected_keys</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="op">...</span><span class="op">,</span> <span class="nam">raise_on_failure</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="key">False</span><span class="op">]</span> <span class="op">=</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t"><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span><span class="op">:</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t"><span class="op">@</span><span class="nam">overload</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t"><span class="key">def</span> <span class="nam">_parse_dict_from_json</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t"> <span class="nam">text</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">expected_keys</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="op">...</span><span class="op">,</span> <span class="nam">raise_on_failure</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="exc show_exc"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t"><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span><span class="op">:</span> <span class="op">...</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t"><span class="key">def</span> <span class="nam">_parse_dict_from_json</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t"> <span class="nam">text</span><span class="op">:</span> <span class="nam">str</span><span class="op">,</span> <span class="nam">expected_keys</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span> <span class="nam">raise_on_failure</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">True</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t"><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t"><span class="str"> Parses a JSON string containing a dictionary.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t"><span class="str"> :param text: The string to parse.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t"><span class="str"> :param expected_keys: A list of keys that must be present in the parsed dictionary.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t"><span class="str"> :param raise_on_failure: If True, raises an exception on failure. If False, logs a warning and returns None.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t"><span class="str"> :return: The parsed dictionary, or None if parsing fails and raise_on_failure is False.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t"><span class="str"> :raises json.JSONDecodeError: If the text is not valid JSON and raise_on_failure is True.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t"><span class="str"> :raises ValueError: If the parsed object is not a dictionary or has missing expected keys,</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t"><span class="str"> and `raise_on_failure` is True.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t"> <span class="nam">cleaned_text</span> <span class="op">=</span> <span class="nam">text</span><span class="op">.</span><span class="nam">strip</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t"> <span class="nam">parsed_json</span> <span class="op">=</span> <span class="nam">json</span><span class="op">.</span><span class="nam">loads</span><span class="op">(</span><span class="nam">cleaned_text</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t"> <span class="key">except</span> <span class="nam">json</span><span class="op">.</span><span class="nam">JSONDecodeError</span> <span class="key">as</span> <span class="nam">e</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t"> <span class="key">if</span> <span class="nam">raise_on_failure</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t"> <span class="key">raise</span> <span class="nam">e</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">&quot;Failed to parse JSON from text: {text}. Error: {error}&quot;</span><span class="op">,</span> <span class="nam">text</span><span class="op">=</span><span class="nam">text</span><span class="op">,</span> <span class="nam">error</span><span class="op">=</span><span class="nam">e</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t"> <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">isinstance</span><span class="op">(</span><span class="nam">parsed_json</span><span class="op">,</span> <span class="nam">dict</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t"> <span class="key">if</span> <span class="nam">raise_on_failure</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">Expected a JSON object containing a dictionary but got </span><span class="op">{</span><span class="nam">type</span><span class="op">(</span><span class="nam">parsed_json</span><span class="op">)</span><span class="op">.</span><span class="nam">__name__</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t"> <span class="str">&quot;Expected a JSON object containing a dictionary but got {type}. Returning None&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t"> <span class="nam">type</span><span class="op">=</span><span class="nam">type</span><span class="op">(</span><span class="nam">parsed_json</span><span class="op">)</span><span class="op">.</span><span class="nam">__name__</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t"> <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">expected_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t"> <span class="key">return</span> <span class="nam">parsed_json</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t239" href="#t239">239</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t240" href="#t240">240</a></span><span class="t"> <span class="nam">missing_keys</span> <span class="op">=</span> <span class="op">[</span><span class="nam">key</span> <span class="key">for</span> <span class="nam">key</span> <span class="key">in</span> <span class="nam">expected_keys</span> <span class="key">if</span> <span class="nam">key</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">parsed_json</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t241" href="#t241">241</a></span><span class="t"> <span class="key">if</span> <span class="nam">missing_keys</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t242" href="#t242">242</a></span><span class="t"> <span class="key">if</span> <span class="nam">raise_on_failure</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t243" href="#t243">243</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">Missing expected keys in JSON: </span><span class="op">{</span><span class="nam">missing_keys</span><span class="op">}</span><span class="fst">. Got keys: </span><span class="op">{</span><span class="nam">list</span><span class="op">(</span><span class="nam">parsed_json</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t244" href="#t244">244</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t245" href="#t245">245</a></span><span class="t"> <span class="str">&quot;Missing expected keys in JSON: {missing_keys}. Got keys: {keys}&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t246" href="#t246">246</a></span><span class="t"> <span class="nam">missing_keys</span><span class="op">=</span><span class="nam">missing_keys</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t247" href="#t247">247</a></span><span class="t"> <span class="nam">keys</span><span class="op">=</span><span class="nam">list</span><span class="op">(</span><span class="nam">parsed_json</span><span class="op">.</span><span class="nam">keys</span><span class="op">(</span><span class="op">)</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t248" href="#t248">248</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t249" href="#t249">249</a></span><span class="t"> <span class="key">return</span> <span class="key">None</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t250" href="#t250">250</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t251" href="#t251">251</a></span><span class="t"> <span class="key">return</span> <span class="nam">parsed_json</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t252" href="#t252">252</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t253" href="#t253">253</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t254" href="#t254">254</a></span><span class="t"><span class="key">def</span> <span class="nam">_normalize_metadata_field_name</span><span class="op">(</span><span class="nam">metadata_field</span><span class="op">:</span> <span class="nam">str</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t255" href="#t255">255</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t256" href="#t256">256</a></span><span class="t"><span class="str"> Normalizes a metadata field name by removing the &quot;meta.&quot; prefix if present.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t257" href="#t257">257</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t258" href="#t258">258</a></span><span class="t"> <span class="key">return</span> <span class="nam">metadata_field</span><span class="op">[</span><span class="num">5</span><span class="op">:</span><span class="op">]</span> <span class="key">if</span> <span class="nam">metadata_field</span><span class="op">.</span><span class="nam">startswith</span><span class="op">(</span><span class="str">&quot;meta.&quot;</span><span class="op">)</span> <span class="key">else</span> <span class="nam">metadata_field</span>&nbsp;</span><span class="r"></span></p>
</main>
<footer>
<div class="content">
<p>
<a class="nav" href="z_5084ce8bb9f0af0b_jupyter_py.html">&#xab; prev</a> &nbsp; &nbsp;
<a class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a class="nav" href="z_5084ce8bb9f0af0b_requests_utils_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
</div>
</footer>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink