github-featured/deepset-ai--haystack
2
0
Fork 0
Code Issues Pull Requests Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
python-coverage-comment-action-data
deepset-ai--haystack/htmlcov/z_6d562fb38bfc5ee6_csv_py.html
T
github-actions 76ef5e8a59 ci: Update coverage data
2026-08-21 13:53:45 +00:00

336 lines
66 KiB
HTML
Raw Permalink Blame History

<!DOCTYPE html>
<html lang="en">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
<title>Coverage for haystack/components/converters/csv.py: 95%</title>
<link rel="icon" sizes="32x32" href="favicon_32_cb_c827f16f.png">
<link rel="stylesheet" href="style_cb_4667309f.css" type="text/css">
<script src="coverage_html_cb_dd2e7eb5.js" defer></script>
</head>
<body class="pyfile">
<header>
<div class="content">
<h1>
<span class="text">Coverage for </span><b>haystack<span class="sep">/</span>components<span class="sep">/</span>converters<span class="sep">/</span>csv.py</b>:
<span class="pc_cov">95%</span>
</h1>
<aside id="help_panel_wrapper">
<input id="help_panel_state" type="checkbox">
<label for="help_panel_state">
<img id="keyboard_icon" src="keybd_closed_cb_900cfef5.png" alt="Show/hide keyboard shortcuts">
</label>
<div id="help_panel">
<p class="legend">Shortcuts on this page</p>
<div class="keyhelp">
<p>
<kbd>r</kbd>
<kbd>m</kbd>
<kbd>x</kbd>
&nbsp; toggle line displays
</p>
<p>
<kbd>j</kbd>
<kbd>k</kbd>
&nbsp; next/prev highlighted chunk
</p>
<p>
<kbd>0</kbd> &nbsp; (zero) top of page
</p>
<p>
<kbd>1</kbd> &nbsp; (one) first highlighted chunk
</p>
<p>
<kbd>[</kbd>
<kbd>]</kbd>
&nbsp; prev/next file
</p>
<p>
<kbd>u</kbd> &nbsp; up to the index
</p>
<p>
<kbd>?</kbd> &nbsp; show/hide this help
</p>
</div>
</div>
</aside>
<h2>
<span class="text">88 statements &nbsp;</span>
<button type="button" class="run button_toggle_run" value="run" data-shortcut="r" title="Toggle lines run">84<span class="text"> run</span></button>
<button type="button" class="mis show_mis button_toggle_mis" value="mis" data-shortcut="m" title="Toggle lines missing">4<span class="text"> missing</span></button>
<button type="button" class="exc show_exc button_toggle_exc" value="exc" data-shortcut="x" title="Toggle lines excluded">0<span class="text"> excluded</span></button>
</h2>
<p class="text">
<a id="prevFileLink" class="nav" href="z_6d562fb38bfc5ee6___init___py.html">&#xab; prev</a> &nbsp; &nbsp;
<a id="indexLink" class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a id="nextFileLink" class="nav" href="z_6d562fb38bfc5ee6_docx_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
<aside class="hidden">
<button type="button" class="button_next_chunk" data-shortcut="j"></button>
<button type="button" class="button_prev_chunk" data-shortcut="k"></button>
<button type="button" class="button_top_of_page" data-shortcut="0"></button>
<button type="button" class="button_first_chunk" data-shortcut="1"></button>
<button type="button" class="button_prev_file" data-shortcut="["></button>
<button type="button" class="button_next_file" data-shortcut="]"></button>
<button type="button" class="button_to_index" data-shortcut="u"></button>
<button type="button" class="button_show_hide_help" data-shortcut="?"></button>
</aside>
</div>
</header>
<main id="source">
<p class="pln"><span class="n"><a id="t1" href="#t1">1</a></span><span class="t"><span class="com"># SPDX-FileCopyrightText: 2022-present deepset GmbH &lt;info@deepset.ai&gt;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t2" href="#t2">2</a></span><span class="t"><span class="com">#</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t3" href="#t3">3</a></span><span class="t"><span class="com"># SPDX-License-Identifier: Apache-2.0</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t4" href="#t4">4</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t5" href="#t5">5</a></span><span class="t"><span class="key">import</span> <span class="nam">csv</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t6" href="#t6">6</a></span><span class="t"><span class="key">import</span> <span class="nam">io</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t7" href="#t7">7</a></span><span class="t"><span class="key">import</span> <span class="nam">os</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t8" href="#t8">8</a></span><span class="t"><span class="key">from</span> <span class="nam">pathlib</span> <span class="key">import</span> <span class="nam">Path</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t9" href="#t9">9</a></span><span class="t"><span class="key">from</span> <span class="nam">typing</span> <span class="key">import</span> <span class="nam">Any</span><span class="op">,</span> <span class="nam">Literal</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t10" href="#t10">10</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t11" href="#t11">11</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span> <span class="key">import</span> <span class="nam">Document</span><span class="op">,</span> <span class="nam">component</span><span class="op">,</span> <span class="nam">logging</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t12" href="#t12">12</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">components</span><span class="op">.</span><span class="nam">converters</span><span class="op">.</span><span class="nam">utils</span> <span class="key">import</span> <span class="nam">get_bytestream_from_source</span><span class="op">,</span> <span class="nam">normalize_metadata</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t13" href="#t13">13</a></span><span class="t"><span class="key">from</span> <span class="nam">haystack</span><span class="op">.</span><span class="nam">dataclasses</span> <span class="key">import</span> <span class="nam">ByteStream</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t14" href="#t14">14</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t15" href="#t15">15</a></span><span class="t"><span class="nam">logger</span> <span class="op">=</span> <span class="nam">logging</span><span class="op">.</span><span class="nam">getLogger</span><span class="op">(</span><span class="nam">__name__</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t16" href="#t16">16</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t17" href="#t17">17</a></span><span class="t"><span class="nam">_ROW_MODE_SIZE_WARN_BYTES</span> <span class="op">=</span> <span class="num">5</span> <span class="op">*</span> <span class="num">1024</span> <span class="op">*</span> <span class="num">1024</span> <span class="com"># ~5MB; warn when parsing rows might be memory-heavy</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t18" href="#t18">18</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t19" href="#t19">19</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t20" href="#t20">20</a></span><span class="t"><span class="op">@</span><span class="nam">component</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t21" href="#t21">21</a></span><span class="t"><span class="key">class</span> <span class="nam">CSVToDocument</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t22" href="#t22">22</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t23" href="#t23">23</a></span><span class="t"><span class="str"> Converts CSV files to Documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t24" href="#t24">24</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t25" href="#t25">25</a></span><span class="t"><span class="str"> By default, it uses UTF-8 encoding when converting files but</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t26" href="#t26">26</a></span><span class="t"><span class="str"> you can also set a custom encoding.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t27" href="#t27">27</a></span><span class="t"><span class="str"> It can attach metadata to the resulting documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t28" href="#t28">28</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t29" href="#t29">29</a></span><span class="t"><span class="str"> ### Usage example</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t30" href="#t30">30</a></span><span class="t"><span class="str"> ```python</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t31" href="#t31">31</a></span><span class="t"><span class="str"> from haystack.components.converters.csv import CSVToDocument</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t32" href="#t32">32</a></span><span class="t"><span class="str"> from datetime import datetime</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t33" href="#t33">33</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t34" href="#t34">34</a></span><span class="t"><span class="str"> converter = CSVToDocument()</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t35" href="#t35">35</a></span><span class="t"><span class="str"> results = converter.run(</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t36" href="#t36">36</a></span><span class="t"><span class="str"> sources=[&quot;test/test_files/csv/sample_1.csv&quot;], meta={&quot;date_added&quot;: datetime.now().isoformat()}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t37" href="#t37">37</a></span><span class="t"><span class="str"> )</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t38" href="#t38">38</a></span><span class="t"><span class="str"> documents = results[&quot;documents&quot;]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t39" href="#t39">39</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t40" href="#t40">40</a></span><span class="t"><span class="str"> print(documents[0].content)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t41" href="#t41">41</a></span><span class="t"><span class="str"> # &gt;&gt; &#x27;col1,col2\\nrow1,row1\\nrow2,row2\\n&#x27;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t42" href="#t42">42</a></span><span class="t"><span class="str"> ```</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t43" href="#t43">43</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t44" href="#t44">44</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t45" href="#t45">45</a></span><span class="t"> <span class="key">def</span> <span class="nam">__init__</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t46" href="#t46">46</a></span><span class="t"> <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t47" href="#t47">47</a></span><span class="t"> <span class="nam">encoding</span><span class="op">:</span> <span class="nam">str</span> <span class="op">=</span> <span class="str">&quot;utf-8&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t48" href="#t48">48</a></span><span class="t"> <span class="nam">store_full_path</span><span class="op">:</span> <span class="nam">bool</span> <span class="op">=</span> <span class="key">False</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t49" href="#t49">49</a></span><span class="t"> <span class="op">*</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t50" href="#t50">50</a></span><span class="t"> <span class="nam">conversion_mode</span><span class="op">:</span> <span class="nam">Literal</span><span class="op">[</span><span class="str">&quot;file&quot;</span><span class="op">,</span> <span class="str">&quot;row&quot;</span><span class="op">]</span> <span class="op">=</span> <span class="str">&quot;file&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t51" href="#t51">51</a></span><span class="t"> <span class="nam">delimiter</span><span class="op">:</span> <span class="nam">str</span> <span class="op">=</span> <span class="str">&quot;,&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t52" href="#t52">52</a></span><span class="t"> <span class="nam">quotechar</span><span class="op">:</span> <span class="nam">str</span> <span class="op">=</span> <span class="str">&#x27;&quot;&#x27;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t53" href="#t53">53</a></span><span class="t"> <span class="op">)</span> <span class="op">-&gt;</span> <span class="key">None</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t54" href="#t54">54</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t55" href="#t55">55</a></span><span class="t"><span class="str"> Creates a CSVToDocument component.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t56" href="#t56">56</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t57" href="#t57">57</a></span><span class="t"><span class="str"> :param encoding:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t58" href="#t58">58</a></span><span class="t"><span class="str"> The encoding of the csv files to convert.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t59" href="#t59">59</a></span><span class="t"><span class="str"> If the encoding is specified in the metadata of a source ByteStream,</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t60" href="#t60">60</a></span><span class="t"><span class="str"> it overrides this value.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t61" href="#t61">61</a></span><span class="t"><span class="str"> :param store_full_path:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t62" href="#t62">62</a></span><span class="t"><span class="str"> If True, the full path of the file is stored in the metadata of the document.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t63" href="#t63">63</a></span><span class="t"><span class="str"> If False, only the file name is stored.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t64" href="#t64">64</a></span><span class="t"><span class="str"> :param conversion_mode:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t65" href="#t65">65</a></span><span class="t"><span class="str"> - &quot;file&quot; (default): one Document per CSV file whose content is the raw CSV text.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t66" href="#t66">66</a></span><span class="t"><span class="str"> - &quot;row&quot;: convert each CSV row to its own Document (requires `content_column` in `run()`).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t67" href="#t67">67</a></span><span class="t"><span class="str"> :param delimiter:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t68" href="#t68">68</a></span><span class="t"><span class="str"> CSV delimiter used when parsing in row mode (passed to ``csv.DictReader``).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t69" href="#t69">69</a></span><span class="t"><span class="str"> :param quotechar:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t70" href="#t70">70</a></span><span class="t"><span class="str"> CSV quote character used when parsing in row mode (passed to ``csv.DictReader``).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t71" href="#t71">71</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t72" href="#t72">72</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">encoding</span> <span class="op">=</span> <span class="nam">encoding</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t73" href="#t73">73</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">store_full_path</span> <span class="op">=</span> <span class="nam">store_full_path</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t74" href="#t74">74</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">conversion_mode</span> <span class="op">=</span> <span class="nam">conversion_mode</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t75" href="#t75">75</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">delimiter</span> <span class="op">=</span> <span class="nam">delimiter</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t76" href="#t76">76</a></span><span class="t"> <span class="nam">self</span><span class="op">.</span><span class="nam">quotechar</span> <span class="op">=</span> <span class="nam">quotechar</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t77" href="#t77">77</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t78" href="#t78">78</a></span><span class="t"> <span class="com"># Basic validation</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t79" href="#t79">79</a></span><span class="t"> <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">delimiter</span><span class="op">)</span> <span class="op">!=</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t80" href="#t80">80</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">&quot;CSVToDocument: delimiter must be a single character.&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t81" href="#t81">81</a></span><span class="t"> <span class="key">if</span> <span class="nam">len</span><span class="op">(</span><span class="nam">self</span><span class="op">.</span><span class="nam">quotechar</span><span class="op">)</span> <span class="op">!=</span> <span class="num">1</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t82" href="#t82">82</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span><span class="str">&quot;CSVToDocument: quotechar must be a single character.&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t83" href="#t83">83</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t84" href="#t84">84</a></span><span class="t"> <span class="op">@</span><span class="nam">component</span><span class="op">.</span><span class="nam">output_types</span><span class="op">(</span><span class="nam">documents</span><span class="op">=</span><span class="nam">list</span><span class="op">[</span><span class="nam">Document</span><span class="op">]</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t85" href="#t85">85</a></span><span class="t"> <span class="key">def</span> <span class="nam">run</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t86" href="#t86">86</a></span><span class="t"> <span class="nam">self</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t87" href="#t87">87</a></span><span class="t"> <span class="nam">sources</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">str</span> <span class="op">|</span> <span class="nam">Path</span> <span class="op">|</span> <span class="nam">ByteStream</span><span class="op">]</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t88" href="#t88">88</a></span><span class="t"> <span class="op">*</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t89" href="#t89">89</a></span><span class="t"> <span class="nam">content_column</span><span class="op">:</span> <span class="nam">str</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t90" href="#t90">90</a></span><span class="t"> <span class="nam">meta</span><span class="op">:</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span> <span class="op">|</span> <span class="nam">list</span><span class="op">[</span><span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">]</span> <span class="op">|</span> <span class="key">None</span> <span class="op">=</span> <span class="key">None</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t91" href="#t91">91</a></span><span class="t"> <span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t92" href="#t92">92</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t93" href="#t93">93</a></span><span class="t"><span class="str"> Converts CSV files to a Document (file mode) or to one Document per row (row mode).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t94" href="#t94">94</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t95" href="#t95">95</a></span><span class="t"><span class="str"> :param sources:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t96" href="#t96">96</a></span><span class="t"><span class="str"> List of file paths or ByteStream objects.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t97" href="#t97">97</a></span><span class="t"><span class="str"> :param content_column:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t98" href="#t98">98</a></span><span class="t"><span class="str"> **Required when** ``conversion_mode=&quot;row&quot;``.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t99" href="#t99">99</a></span><span class="t"><span class="str"> The column name whose values become ``Document.content`` for each row.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t100" href="#t100">100</a></span><span class="t"><span class="str"> The column must exist in the CSV header.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t101" href="#t101">101</a></span><span class="t"><span class="str"> :param meta:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t102" href="#t102">102</a></span><span class="t"><span class="str"> Optional metadata to attach to the documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t103" href="#t103">103</a></span><span class="t"><span class="str"> This value can be either a list of dictionaries or a single dictionary.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t104" href="#t104">104</a></span><span class="t"><span class="str"> If it&#x27;s a single dictionary, its content is added to the metadata of all produced documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t105" href="#t105">105</a></span><span class="t"><span class="str"> If it&#x27;s a list, the length of the list must match the number of sources, because the two lists will</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t106" href="#t106">106</a></span><span class="t"><span class="str"> be zipped.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t107" href="#t107">107</a></span><span class="t"><span class="str"> If `sources` contains ByteStream objects, their `meta` will be added to the output documents.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t108" href="#t108">108</a></span><span class="t"><span class="str"> :returns:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t109" href="#t109">109</a></span><span class="t"><span class="str"> A dictionary with the following keys:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t110" href="#t110">110</a></span><span class="t"><span class="str"> - `documents`: Created documents</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t111" href="#t111">111</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t112" href="#t112">112</a></span><span class="t"> <span class="nam">documents</span><span class="op">:</span> <span class="nam">list</span><span class="op">[</span><span class="nam">Document</span><span class="op">]</span> <span class="op">=</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t113" href="#t113">113</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t114" href="#t114">114</a></span><span class="t"> <span class="nam">meta_list</span> <span class="op">=</span> <span class="nam">normalize_metadata</span><span class="op">(</span><span class="nam">meta</span><span class="op">,</span> <span class="nam">sources_count</span><span class="op">=</span><span class="nam">len</span><span class="op">(</span><span class="nam">sources</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t115" href="#t115">115</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t116" href="#t116">116</a></span><span class="t"> <span class="key">for</span> <span class="nam">source</span><span class="op">,</span> <span class="nam">metadata</span> <span class="key">in</span> <span class="nam">zip</span><span class="op">(</span><span class="nam">sources</span><span class="op">,</span> <span class="nam">meta_list</span><span class="op">,</span> <span class="nam">strict</span><span class="op">=</span><span class="key">True</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t117" href="#t117">117</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t118" href="#t118">118</a></span><span class="t"> <span class="nam">bytestream</span> <span class="op">=</span> <span class="nam">get_bytestream_from_source</span><span class="op">(</span><span class="nam">source</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t119" href="#t119">119</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span> <span class="key">as</span> <span class="nam">e</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t120" href="#t120">120</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span><span class="str">&quot;Could not read {source}. Skipping it. Error: {error}&quot;</span><span class="op">,</span> <span class="nam">source</span><span class="op">=</span><span class="nam">source</span><span class="op">,</span> <span class="nam">error</span><span class="op">=</span><span class="nam">e</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t121" href="#t121">121</a></span><span class="t"> <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t122" href="#t122">122</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t123" href="#t123">123</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t124" href="#t124">124</a></span><span class="t"> <span class="nam">encoding</span> <span class="op">=</span> <span class="nam">bytestream</span><span class="op">.</span><span class="nam">meta</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">&quot;encoding&quot;</span><span class="op">,</span> <span class="nam">self</span><span class="op">.</span><span class="nam">encoding</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t125" href="#t125">125</a></span><span class="t"> <span class="nam">raw</span> <span class="op">=</span> <span class="nam">io</span><span class="op">.</span><span class="nam">BytesIO</span><span class="op">(</span><span class="nam">bytestream</span><span class="op">.</span><span class="nam">data</span><span class="op">)</span><span class="op">.</span><span class="nam">getvalue</span><span class="op">(</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t126" href="#t126">126</a></span><span class="t"> <span class="nam">data</span> <span class="op">=</span> <span class="nam">raw</span><span class="op">.</span><span class="nam">decode</span><span class="op">(</span><span class="nam">encoding</span><span class="op">=</span><span class="nam">encoding</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t127" href="#t127">127</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span> <span class="key">as</span> <span class="nam">e</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t128" href="#t128">128</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t129" href="#t129">129</a></span><span class="t"> <span class="str">&quot;Could not convert file {source}. Skipping it. Error message: {error}&quot;</span><span class="op">,</span> <span class="nam">source</span><span class="op">=</span><span class="nam">source</span><span class="op">,</span> <span class="nam">error</span><span class="op">=</span><span class="nam">e</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t130" href="#t130">130</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t131" href="#t131">131</a></span><span class="t"> <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t132" href="#t132">132</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t133" href="#t133">133</a></span><span class="t"> <span class="nam">merged_metadata</span> <span class="op">=</span> <span class="op">{</span><span class="op">**</span><span class="nam">bytestream</span><span class="op">.</span><span class="nam">meta</span><span class="op">,</span> <span class="op">**</span><span class="nam">metadata</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t134" href="#t134">134</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t135" href="#t135">135</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">self</span><span class="op">.</span><span class="nam">store_full_path</span> <span class="key">and</span> <span class="str">&quot;file_path&quot;</span> <span class="key">in</span> <span class="nam">bytestream</span><span class="op">.</span><span class="nam">meta</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t136" href="#t136">136</a></span><span class="t"> <span class="nam">file_path</span> <span class="op">=</span> <span class="nam">bytestream</span><span class="op">.</span><span class="nam">meta</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="str">&quot;file_path&quot;</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t137" href="#t137">137</a></span><span class="t"> <span class="key">if</span> <span class="nam">file_path</span><span class="op">:</span> <span class="com"># Ensure the value is not None for mypy</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t138" href="#t138">138</a></span><span class="t"> <span class="nam">merged_metadata</span><span class="op">[</span><span class="str">&quot;file_path&quot;</span><span class="op">]</span> <span class="op">=</span> <span class="nam">os</span><span class="op">.</span><span class="nam">path</span><span class="op">.</span><span class="nam">basename</span><span class="op">(</span><span class="nam">file_path</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t139" href="#t139">139</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t140" href="#t140">140</a></span><span class="t"> <span class="com"># Mode: file (backward-compatible default) -&gt; one Document per file</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t141" href="#t141">141</a></span><span class="t"> <span class="key">if</span> <span class="nam">self</span><span class="op">.</span><span class="nam">conversion_mode</span> <span class="op">==</span> <span class="str">&quot;file&quot;</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t142" href="#t142">142</a></span><span class="t"> <span class="nam">documents</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">Document</span><span class="op">(</span><span class="nam">content</span><span class="op">=</span><span class="nam">data</span><span class="op">,</span> <span class="nam">meta</span><span class="op">=</span><span class="nam">merged_metadata</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t143" href="#t143">143</a></span><span class="t"> <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t144" href="#t144">144</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t145" href="#t145">145</a></span><span class="t"> <span class="com"># --- ROW MODE (strict) ---</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t146" href="#t146">146</a></span><span class="t"> <span class="com"># Require content_column in run(); no fallback</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t147" href="#t147">147</a></span><span class="t"> <span class="key">if</span> <span class="key">not</span> <span class="nam">content_column</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t148" href="#t148">148</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t149" href="#t149">149</a></span><span class="t"> <span class="str">&quot;CSVToDocument(row): &#x27;content_column&#x27; is required in run() when conversion_mode=&#x27;row&#x27;.&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t150" href="#t150">150</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t151" href="#t151">151</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t152" href="#t152">152</a></span><span class="t"> <span class="com"># Warn for large CSVs in row mode (memory consideration)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t153" href="#t153">153</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t154" href="#t154">154</a></span><span class="t"> <span class="nam">size_bytes</span> <span class="op">=</span> <span class="nam">len</span><span class="op">(</span><span class="nam">raw</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t155" href="#t155">155</a></span><span class="t"> <span class="key">if</span> <span class="nam">size_bytes</span> <span class="op">&gt;</span> <span class="nam">_ROW_MODE_SIZE_WARN_BYTES</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t156" href="#t156">156</a></span><span class="t"> <span class="nam">logger</span><span class="op">.</span><span class="nam">warning</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t157" href="#t157">157</a></span><span class="t"> <span class="str">&quot;CSVToDocument(row): parsing a large CSV (~{mb:.1f} MB). &quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t158" href="#t158">158</a></span><span class="t"> <span class="str">&quot;Consider chunking/streaming if you hit memory issues.&quot;</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t159" href="#t159">159</a></span><span class="t"> <span class="nam">mb</span><span class="op">=</span><span class="nam">size_bytes</span> <span class="op">/</span> <span class="op">(</span><span class="num">1024</span> <span class="op">*</span> <span class="num">1024</span><span class="op">)</span><span class="op">,</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t160" href="#t160">160</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t161" href="#t161">161</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t162" href="#t162">162</a></span><span class="t"> <span class="key">pass</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t163" href="#t163">163</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t164" href="#t164">164</a></span><span class="t"> <span class="com"># Create DictReader; if this fails, raise (no fallback)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t165" href="#t165">165</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t166" href="#t166">166</a></span><span class="t"> <span class="com"># ``restkey`` ensures surplus fields on ragged rows (rows with more values than the</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t167" href="#t167">167</a></span><span class="t"> <span class="com"># header, e.g. an unquoted comma inside a value) land under an explicit string key</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t168" href="#t168">168</a></span><span class="t"> <span class="com"># instead of the default ``None`` key, which would break ``Document`` id generation.</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t169" href="#t169">169</a></span><span class="t"> <span class="nam">reader</span> <span class="op">=</span> <span class="nam">csv</span><span class="op">.</span><span class="nam">DictReader</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t170" href="#t170">170</a></span><span class="t"> <span class="nam">io</span><span class="op">.</span><span class="nam">StringIO</span><span class="op">(</span><span class="nam">data</span><span class="op">)</span><span class="op">,</span> <span class="nam">delimiter</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">delimiter</span><span class="op">,</span> <span class="nam">quotechar</span><span class="op">=</span><span class="nam">self</span><span class="op">.</span><span class="nam">quotechar</span><span class="op">,</span> <span class="nam">restkey</span><span class="op">=</span><span class="str">&quot;extra_columns&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t171" href="#t171">171</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t172" href="#t172">172</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span> <span class="key">as</span> <span class="nam">e</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t173" href="#t173">173</a></span><span class="t"> <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">CSVToDocument(row): could not parse CSV rows for </span><span class="op">{</span><span class="nam">source</span><span class="op">}</span><span class="fst">: </span><span class="op">{</span><span class="nam">e</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span> <span class="key">from</span> <span class="nam">e</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t174" href="#t174">174</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t175" href="#t175">175</a></span><span class="t"> <span class="com"># Validate header contains content_column; strict error if missing</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t176" href="#t176">176</a></span><span class="t"> <span class="nam">header</span> <span class="op">=</span> <span class="nam">reader</span><span class="op">.</span><span class="nam">fieldnames</span> <span class="key">or</span> <span class="op">[</span><span class="op">]</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t177" href="#t177">177</a></span><span class="t"> <span class="key">if</span> <span class="nam">content_column</span> <span class="key">not</span> <span class="key">in</span> <span class="nam">header</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t178" href="#t178">178</a></span><span class="t"> <span class="key">raise</span> <span class="nam">ValueError</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t179" href="#t179">179</a></span><span class="t"> <span class="fst">f&quot;</span><span class="fst">CSVToDocument(row): content_column=&#x27;</span><span class="op">{</span><span class="nam">content_column</span><span class="op">}</span><span class="fst">&#x27; not found in header </span><span class="fst">&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t180" href="#t180">180</a></span><span class="t"> <span class="fst">f&quot;</span><span class="fst">for </span><span class="op">{</span><span class="nam">source</span><span class="op">}</span><span class="fst">. Available columns: </span><span class="op">{</span><span class="nam">header</span><span class="op">}</span><span class="fst">&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t181" href="#t181">181</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t182" href="#t182">182</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t183" href="#t183">183</a></span><span class="t"> <span class="com"># Build documents; if a row processing fails, raise immediately (no skip)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t184" href="#t184">184</a></span><span class="t"> <span class="key">for</span> <span class="nam">i</span><span class="op">,</span> <span class="nam">row</span> <span class="key">in</span> <span class="nam">enumerate</span><span class="op">(</span><span class="nam">reader</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t185" href="#t185">185</a></span><span class="t"> <span class="key">try</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t186" href="#t186">186</a></span><span class="t"> <span class="nam">doc</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_build_document_from_row</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t187" href="#t187">187</a></span><span class="t"> <span class="nam">row</span><span class="op">=</span><span class="nam">row</span><span class="op">,</span> <span class="nam">base_meta</span><span class="op">=</span><span class="nam">merged_metadata</span><span class="op">,</span> <span class="nam">row_index</span><span class="op">=</span><span class="nam">i</span><span class="op">,</span> <span class="nam">content_column</span><span class="op">=</span><span class="nam">content_column</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t188" href="#t188">188</a></span><span class="t"> <span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t189" href="#t189">189</a></span><span class="t"> <span class="key">except</span> <span class="nam">Exception</span> <span class="key">as</span> <span class="nam">e</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="mis show_mis"><span class="n"><a id="t190" href="#t190">190</a></span><span class="t"> <span class="key">raise</span> <span class="nam">RuntimeError</span><span class="op">(</span><span class="fst">f&quot;</span><span class="fst">CSVToDocument(row): failed to process row </span><span class="op">{</span><span class="nam">i</span><span class="op">}</span><span class="fst"> for </span><span class="op">{</span><span class="nam">source</span><span class="op">}</span><span class="fst">: </span><span class="op">{</span><span class="nam">e</span><span class="op">}</span><span class="fst">&quot;</span><span class="op">)</span> <span class="key">from</span> <span class="nam">e</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t191" href="#t191">191</a></span><span class="t"> <span class="nam">documents</span><span class="op">.</span><span class="nam">append</span><span class="op">(</span><span class="nam">doc</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t192" href="#t192">192</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t193" href="#t193">193</a></span><span class="t"> <span class="key">return</span> <span class="op">{</span><span class="str">&quot;documents&quot;</span><span class="op">:</span> <span class="nam">documents</span><span class="op">}</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t194" href="#t194">194</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t195" href="#t195">195</a></span><span class="t"> <span class="com"># ----- helpers -----</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t196" href="#t196">196</a></span><span class="t"> <span class="key">def</span> <span class="nam">_safe_value</span><span class="op">(</span><span class="nam">self</span><span class="op">,</span> <span class="nam">value</span><span class="op">:</span> <span class="nam">Any</span><span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">str</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t197" href="#t197">197</a></span><span class="t"> <span class="str">&quot;&quot;&quot;Normalize CSV cell values: None -&gt; &#x27;&#x27;, everything -&gt; str.&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t198" href="#t198">198</a></span><span class="t"> <span class="key">return</span> <span class="str">&quot;&quot;</span> <span class="key">if</span> <span class="nam">value</span> <span class="key">is</span> <span class="key">None</span> <span class="key">else</span> <span class="nam">str</span><span class="op">(</span><span class="nam">value</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t199" href="#t199">199</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t200" href="#t200">200</a></span><span class="t"> <span class="key">def</span> <span class="nam">_build_document_from_row</span><span class="op">(</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t201" href="#t201">201</a></span><span class="t"> <span class="nam">self</span><span class="op">,</span> <span class="nam">row</span><span class="op">:</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">,</span> <span class="nam">base_meta</span><span class="op">:</span> <span class="nam">dict</span><span class="op">[</span><span class="nam">str</span><span class="op">,</span> <span class="nam">Any</span><span class="op">]</span><span class="op">,</span> <span class="nam">row_index</span><span class="op">:</span> <span class="nam">int</span><span class="op">,</span> <span class="nam">content_column</span><span class="op">:</span> <span class="nam">str</span>&nbsp;</span><span class="r"></span></p>
<p class="run run2"><span class="n"><a id="t202" href="#t202">202</a></span><span class="t"> <span class="op">)</span> <span class="op">-&gt;</span> <span class="nam">Document</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t203" href="#t203">203</a></span><span class="t"> <span class="str">&quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t204" href="#t204">204</a></span><span class="t"><span class="str"> Build a ``Document`` from one parsed CSV row.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t205" href="#t205">205</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t206" href="#t206">206</a></span><span class="t"><span class="str"> :param row: Mapping of column name to cell value for the current row</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t207" href="#t207">207</a></span><span class="t"><span class="str"> (as produced by ``csv.DictReader``).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t208" href="#t208">208</a></span><span class="t"><span class="str"> :param base_meta: File-level and user-provided metadata to start from</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t209" href="#t209">209</a></span><span class="t"><span class="str"> (for example: ``file_path``, ``encoding``).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t210" href="#t210">210</a></span><span class="t"><span class="str"> :param row_index: Zero-based row index in the CSV; stored as</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t211" href="#t211">211</a></span><span class="t"><span class="str"> ``row_number`` in the output document&#x27;s metadata.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t212" href="#t212">212</a></span><span class="t"><span class="str"> :param content_column: Column name to use for ``Document.content``.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t213" href="#t213">213</a></span><span class="t"><span class="str"> :returns: A ``Document`` with chosen content and merged metadata.</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t214" href="#t214">214</a></span><span class="t"><span class="str"> Remaining row columns are added to ``meta`` with collision-safe</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t215" href="#t215">215</a></span><span class="t"><span class="str"> keys (prefixed with ``csv_`` if needed).</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t216" href="#t216">216</a></span><span class="t"><span class="str"> &quot;&quot;&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t217" href="#t217">217</a></span><span class="t"> <span class="nam">row_meta</span> <span class="op">=</span> <span class="nam">dict</span><span class="op">(</span><span class="nam">base_meta</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t218" href="#t218">218</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t219" href="#t219">219</a></span><span class="t"> <span class="com"># content (strict: content_column must exist; validated by caller)</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t220" href="#t220">220</a></span><span class="t"> <span class="nam">content</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_safe_value</span><span class="op">(</span><span class="nam">row</span><span class="op">.</span><span class="nam">get</span><span class="op">(</span><span class="nam">content_column</span><span class="op">)</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t221" href="#t221">221</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t222" href="#t222">222</a></span><span class="t"> <span class="com"># merge remaining columns into meta with collision handling</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t223" href="#t223">223</a></span><span class="t"> <span class="key">for</span> <span class="nam">k</span><span class="op">,</span> <span class="nam">v</span> <span class="key">in</span> <span class="nam">row</span><span class="op">.</span><span class="nam">items</span><span class="op">(</span><span class="op">)</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t224" href="#t224">224</a></span><span class="t"> <span class="key">if</span> <span class="nam">k</span> <span class="op">==</span> <span class="nam">content_column</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t225" href="#t225">225</a></span><span class="t"> <span class="key">continue</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t226" href="#t226">226</a></span><span class="t"> <span class="nam">key_to_use</span> <span class="op">=</span> <span class="nam">k</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t227" href="#t227">227</a></span><span class="t"> <span class="key">if</span> <span class="nam">key_to_use</span> <span class="key">in</span> <span class="nam">row_meta</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t228" href="#t228">228</a></span><span class="t"> <span class="com"># Avoid clobbering existing meta like file_path/encoding; prefix and de-dupe</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t229" href="#t229">229</a></span><span class="t"> <span class="nam">base_key</span> <span class="op">=</span> <span class="fst">f&quot;</span><span class="fst">csv_</span><span class="op">{</span><span class="nam">key_to_use</span><span class="op">}</span><span class="fst">&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t230" href="#t230">230</a></span><span class="t"> <span class="nam">key_to_use</span> <span class="op">=</span> <span class="nam">base_key</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t231" href="#t231">231</a></span><span class="t"> <span class="nam">suffix</span> <span class="op">=</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t232" href="#t232">232</a></span><span class="t"> <span class="key">while</span> <span class="nam">key_to_use</span> <span class="key">in</span> <span class="nam">row_meta</span><span class="op">:</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t233" href="#t233">233</a></span><span class="t"> <span class="nam">key_to_use</span> <span class="op">=</span> <span class="fst">f&quot;</span><span class="op">{</span><span class="nam">base_key</span><span class="op">}</span><span class="fst">_</span><span class="op">{</span><span class="nam">suffix</span><span class="op">}</span><span class="fst">&quot;</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t234" href="#t234">234</a></span><span class="t"> <span class="nam">suffix</span> <span class="op">+=</span> <span class="num">1</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t235" href="#t235">235</a></span><span class="t"> <span class="nam">row_meta</span><span class="op">[</span><span class="nam">key_to_use</span><span class="op">]</span> <span class="op">=</span> <span class="nam">self</span><span class="op">.</span><span class="nam">_safe_value</span><span class="op">(</span><span class="nam">v</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
<p class="pln"><span class="n"><a id="t236" href="#t236">236</a></span><span class="t">&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t237" href="#t237">237</a></span><span class="t"> <span class="nam">row_meta</span><span class="op">[</span><span class="str">&quot;row_number&quot;</span><span class="op">]</span> <span class="op">=</span> <span class="nam">row_index</span>&nbsp;</span><span class="r"></span></p>
<p class="run"><span class="n"><a id="t238" href="#t238">238</a></span><span class="t"> <span class="key">return</span> <span class="nam">Document</span><span class="op">(</span><span class="nam">content</span><span class="op">=</span><span class="nam">content</span><span class="op">,</span> <span class="nam">meta</span><span class="op">=</span><span class="nam">row_meta</span><span class="op">)</span>&nbsp;</span><span class="r"></span></p>
</main>
<footer>
<div class="content">
<p>
<a class="nav" href="z_6d562fb38bfc5ee6___init___py.html">&#xab; prev</a> &nbsp; &nbsp;
<a class="nav" href="index.html">&Hat; index</a> &nbsp; &nbsp;
<a class="nav" href="z_6d562fb38bfc5ee6_docx_py.html">&#xbb; next</a>
&nbsp; &nbsp; &nbsp;
<a class="nav" href="https://coverage.readthedocs.io/en/7.15.4">coverage.py v7.15.4</a>,
created at 2026-08-21 13:53 +0000
</p>
</div>
</footer>
</body>
</html>
Reference in New Issue View Git Blame Copy Permalink