github-featured/deepset-ai--haystack
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
python-coverage-comment-action-data
deepset-ai--haystack/htmlcov/z_138edecadec193a7_approximate_counter_py.html
T
github-actions 76ef5e8a59 ci: Update coverage data
2026-08-21 13:53:45 +00:00

175 lines
24 KiB
HTML
Raw Permalink Blame History

<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Coverage for haystack/token_counters/approximate_counter.py: 100%</title>
<link rel="icon" sizes="32x32" href="favicon_32_cb_c827f16f.png">
<link rel="stylesheet" href="style_cb_4667309f.css" type="text/css">
<script src="coverage_html_cb_dd2e7eb5.js" defer></script>
</head>
<body class="pyfile">
<header>
<div class="content">
<h1>
<span class="text">Coverage for </span><b>haystack<span class="sep">/</span>token_counters<span class="sep">/</span>approximate_counter.py</b>:
<span class="pc_cov">100%</span>
</h1>
<aside id="help_panel_wrapper">
<input id="help_panel_state" type="checkbox">
<label for="help_panel_state">
<img id="keyboard_icon" src="keybd_closed_cb_900cfef5.png" alt="Show/hide keyboard shortcuts">
</label>
<div id="help_panel">
<p class="legend">Shortcuts on this page</p>
<div class="keyhelp">
<p>
<kbd>r</kbd>
<kbd>m</kbd>
<kbd>x</kbd>
&nbsp; toggle line displays
</p>
<p>
<kbd>j</kbd>
<kbd>k</kbd>
&nbsp; next/prev highlighted chunk
</p>
<p>
<kbd>0</kbd> &nbsp; (zero) top of page
</p>
<p>
<kbd>1</kbd> &nbsp; (one) first highlighted chunk
</p>
<p>
<kbd>[</kbd>
<kbd>]</kbd>
&nbsp; prev/next file
</p>
<p>
<kbd>u</kbd> &nbsp; up to the index
</p>
<p>
<kbd>?</kbd> &nbsp; show/hide this help
</p>
</div>
</div>
</aside>
<h2>
<span class="text">21 statements &nbsp;</span>
<button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">21<span class="text"> run</span></button>
<button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">0<span class="text"> missing</span></button>
<button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
</h2>
<p class="text">
<a id="prevFileLink" class="nav" href="z_138edecadec193a7___init___py.html">&#xab; prev</a> &nbsp; &nbsp;
<a id="indexLink" class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a id="nextFileLink" class="nav" href="z_138edecadec193a7_openai_counter_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
<aside class="hidden">
<button type="button" class="button_next_chunk" data-shortcut="j"></button>
<button type="button" class="button_prev_chunk" data-shortcut="k"></button>
<button type="button" class="button_top_of_page" data-shortcut="0"></button>
<button type="button" class="button_first_chunk" data-shortcut="1"></button>
<button type="button" class="button_prev_file" data-shortcut="["></button>
<button type="button" class="button_next_file" data-shortcut="]"></button>
<button type="button" class="button_to_index" data-shortcut="u"></button>
<button type="button" class="button_show_hide_help" data-shortcut="?"></button>
</aside>
</div>
</header>
<main id="source">
<p class="pln"><span class="n"><a id="t1" href="#t1">1</a></span><span class="t"><span class="com"># SPDX-FileCopyrightText: 2022-present deepset GmbH &lt;info@deepset.ai&gt;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t2" href="#t2">2</a></span><span class="t"><span class="com">#</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t3" href="#t3">3</a></span><span class="t"><span class="com"># SPDX-License-Identifier: Apache-2.0</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t4" href="#t4">4</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t"><span class="key">from</span> <span class="nam">typing</span> <span class="key">import</span> <span class="nam">Any</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">core</span><span class="op">.</span><span class="nam">serialization</span> <span class="key">import</span> <span class="nam">default_to_dict</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">dataclasses</span> <span class="key">import</span> <span class="nam">ChatMessage</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">token_counters</span><span class="op">.</span><span class="nam">types</span> <span class="key">import</span> <span class="nam">TokenCounter</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">token_counters</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="nam">_non_text_tokens</span><span class="op">,</span> <span class="nam">_rendered_conversation</span><span class="op">,</span> <span class="nam">_rendered_tools</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">tools</span> <span class="key">import</span> <span class="nam">ToolsType</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t"><span class="key">class</span> <span class="nam">ApproximateTokenCounter</span><span class="op">(</span><span class="nam">TokenCounter</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t"><span class="str"> Estimates tokens from text length using a flat ratio of characters to tokens.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t"><span class="str"> ## Usage Example:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t"><span class="str"> ```python</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t"><span class="str"> from haystack.dataclasses import ChatMessage</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t"><span class="str"> from haystack.token_counters import ApproximateTokenCounter</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"><span class="str"> counter = ApproximateTokenCounter(chars_per_token=4.0)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t"><span class="str"> messages = [</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t"><span class="str"> ChatMessage.from_user(&quot;Hello, how are you?&quot;),</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t"><span class="str"> ChatMessage.from_assistant(&quot;I&#x27;m good, thank you! How can I assist you today?&quot;)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"><span class="str"> ]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t"><span class="str"> token_count = counter.count(messages)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t"><span class="str"> print(f&quot;Estimated token count: {token_count}&quot;)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="str"> ```</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t"> <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">chars_per_token</span><span class="op">:</span> <span class="nam">float</span> <span class="op">=</span> <span class="num">4.0</span><span class="op">,</span> <span class="nam">tokens_per_image</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">85</span><span class="op">,</span> <span class="nam">tokens_per_file</span><span class="op">:</span> <span class="nam">int</span> <span class="op">=</span> <span class="num">1000</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="str"> Initialize the counter.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str"> :param chars_per_token: How many characters to treat as one token.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str"> :param tokens_per_image: Tokens to charge per image, which has no text to measure. The default is what</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t"><span class="str"> OpenAI charges for a small image; raise it if you send large ones.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str"> :param tokens_per_file: Tokens to charge per file. A rough stand-in for a short document, since the real</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="str"> cost depends on the page count; raise it if you send long ones.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="str"> :raises ValueError: If `chars_per_token` is not positive.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t"> <span class="key">if</span> <span class="nam">chars_per_token</span> <span class="op">&lt;=</span> <span class="num">0</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">`chars_per_token` must be greater than 0, got </span><span class="op">{</span><span class="nam">chars_per_token</span><span class="op">}</span><span class="fst">.</span><span class="fst">&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">chars_per_token</span> <span class="op">=</span> <span class="nam">chars_per_token</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_image</span> <span class="op">=</span> <span class="nam">tokens_per_image</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_file</span> <span class="op">=</span> <span class="nam">tokens_per_file</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"> <span class="key">def</span> <span class="nam">count</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">messages</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">ChatMessage</span><span class="op">]</span><span class="op">,</span> <span class="nam">tools</span><span class="op">:</span> <span class="nam">ToolsType</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="key">None</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">int</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"><span class="str"> Return the estimated number of tokens the given messages occupy.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"><span class="str"> :param messages: The messages to measure.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="str"> :param tools: Tools whose schemas are sent alongside the messages, and so consume tokens too.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t"><span class="str"> :returns: The estimated token count, or `0` when there is nothing to measure.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">messages</span> <span class="key">and</span> <span class="key">not</span> <span class="nam">tools</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"> <span class="key">return</span> <span class="num">0</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"> <span class="nam">text</span> <span class="op">=</span> <span class="nam">_rendered_conversation</span><span class="op">(</span><span class="nam">messages</span><span class="op">)</span> <span class="op">+</span> <span class="nam">_rendered_tools</span><span class="op">(</span><span class="nam">tools</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"> <span class="nam">text_tokens</span> <span class="op">=</span> <span class="nam">int</span><span class="op">(</span><span class="nam">len</span><span class="op">(</span><span class="nam">text</span><span class="op">)</span> <span class="op">/</span> <span class="nam">self</span><span class="op">.</span><span class="nam">chars_per_token</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"> <span class="key">return</span> <span class="nam">text_tokens</span> <span class="op">+</span> <span class="nam">_non_text_tokens</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"> <span class="nam">messages</span><span class="op">=</span><span class="nam">messages</span><span class="op">,</span> <span class="nam">tokens_per_image</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_image</span><span class="op">,</span> <span class="nam">tokens_per_file</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_file</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"> <span class="key">def</span> <span class="nam">to_dict</span><span class="op">(</span><span class="nam">self</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="str"> Serialize the counter.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str"> :returns: A dictionary representation of the counter.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"> <span class="key">return</span> <span class="nam">default_to_dict</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"> <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"> <span class="nam">chars_per_token</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">chars_per_token</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"> <span class="nam">tokens_per_image</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_image</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t"> <span class="nam">tokens_per_file</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">tokens_per_file</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
</main>
<footer>
<div class="content">
<p>
<a class="nav" href="z_138edecadec193a7___init___py.html">&#xab; prev</a> &nbsp; &nbsp;
<a class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a class="nav" href="z_138edecadec193a7_openai_counter_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
</div>
</footer>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink