unstructured
1009 строк · 30.7 Кб
1# pyright: reportPrivateUsage=false2)21from unstructured.documents.html import (22HEADING_TAGS,23LIST_ITEM_TAGS,24SECTION_TAGS,25TABLE_TAGS,26TEXT_TAGS,27HTMLDocument,28HTMLNarrativeText,29HTMLTable,30HTMLTitle,31TagsMixin,32_parse_HTMLTable_from_table_elem,33)34)52)58tag62for tag in TAGS63if tag not in (INCLUDED_TAGS + TABLE_TAGS + VOID_TAGS + ["html", "head", "body"])64]65# -- table-extraction behaviors ------------------------------------------------------------------68The presence of a `<thead>` element in the original also determines whether a `<thead>` element104appears in `.text_as_html` or whether the first row of cells is simply in the body.105"""106html_str = (107"<html>\n"108"<body>\n"109" <table>\n"110" <thead>\n"111" <tr><th>Lorem</th><th>Ipsum</th></tr>\n"112" </thead>\n"113" <tbody>\n"114" <tr><th>Lorem ipsum</th><td>dolor sit amet nulla</td></tr>\n"115" <tr><th>Ut enim non</th><td>ad minim\nveniam quis</td></tr>\n"116" </tbody>\n"117" <tfoot>\n"118" <tr><th>Dolor</th><td>Equis</td></tr>\n"119" </tfoot>\n"120" </table>\n"121"</body>\n"122"</html>"123)124# -- element-suppression behaviors ---------------------------------------------------------------218# ------------------------------------------------------------------------------------------------243<body>248<table>249<tbody>250<tr>251<td><p>Hi there!</p></td>252</tr>253</tbody>254</table>255</body>256</html>"""257doc = HTMLDocument.from_string(raw_html)258el = doc.elements[0]259assert el.ancestortags + (el.tag,) == ("html", "body", "table")260<body>284<table>285<tbody>286<tr>287<td><p>Hi there! I am Matt!</p></td>288</tr>289</tbody>290</table>291</body>292</html>"""293document = HTMLDocument.from_string(doc).doc_after_cleaners(skip_table=False)294assert document.pages[0].elements[0] == Table(text="Hi there! I am Matt!")295)307def test_construct_text(doc, expected):308document_tree = etree.fromstring(doc, etree.HTMLParser())309para = document_tree.find(".//p")310text = html._construct_text(para)311assert text == expected312)345def test_get_emphasized_texts_from_tag(doc: str, root: str, expected: List[Dict[str, str]]):346document_tree = etree.fromstring(doc, etree.HTMLParser())347el = document_tree.find(f".//{root}")348assert el is not None349<div>413<p>●</p>414<p>●</p>415</div>416<div>417<p>An excellent point!</p>418</div>419"""421document_tree = etree.fromstring(doc, etree.HTMLParser())422el = document_tree.find(".//div")423parsed_el, _ = html._process_list_item(el, max_predecessor_len=10)424assert parsed_el == ListItem(text="An excellent point!")425<div>443<p>●</p>444<p>●</p>445</div>446"""448document_tree = etree.fromstring(doc, etree.HTMLParser())449el = document_tree.find(".//div")450parsed_el, _ = html._process_list_item(el)451assert parsed_el is None452<div>457<p>●</p>458<p>●</p>459</div>460<div>461</div>462"""463document_tree = etree.fromstring(doc, etree.HTMLParser())464el = document_tree.find(".//div")465assert html.is_list_item_tag(el) is True466parsed_el, _ = html._process_list_item(el)467assert parsed_el is None468<div>473<p>●</p>474<p>●</p>475<p>●</p>476<p>●</p>477<p>●</p>478</div>479<div>480<p>An excellent point!</p>481</div>482"""484document_tree = etree.fromstring(doc, etree.HTMLParser())485el = document_tree.find(".//div")486parsed_el, _ = html._process_list_item(el, max_predecessor_len=2)487assert parsed_el is None488<body>501<header>502<p>Here is a header. We want to ignore anything that is in this section.</p>503</header>504<h1>{TITLE1}</h1>505<p>{SECTION1}</p>506<p></p>507<p>{TITLE2}</p>508<p><b>{SECTION2}</b></p>509<table>510<tbody>511<tr>512<td><p>{TABLE_SECTION}</p></td>513</tr>514</tbody>515</table>516<hr>517<h2>{TITLE3}</h2>518<div>{SECTION3}</div>519<footer>520<p>Here is a footer. We want to ignore anything that is in this section</p>521</footer>522<div>523<p>Let's ignore anything after the footer too since it's probably garbage.</p>524</div>525</body>526</html>"""527filename = os.path.join(tmpdir.dirname, "sample-doc.html")528with open(filename, "w") as f:529f.write(doc)530<body>564<p>Lots preamble stuff yada yada yada</p>565<main>566<article>567<section>568<h2>A Wonderful Section!</h2>569<p>Look at this amazing section!</p>570</section>571<section>572<h2>Another Wonderful Section!</h2>573<p>Look at this other amazing section!</p>574</section>575</article>576</main>577</body>"""578html_document = HTMLDocument.from_string(html_str)579document_tree = html_document.document_tree580main_tag = html._find_main(document_tree)581assert main_tag.tag == "main"582<body>587<p>Lots preamble stuff yada yada yada</p>588<article>589<section>590<h2>A Wonderful Section!</h2>591<p>Look at this amazing section!</p>592</section>593<section>594<h2>Another Wonderful Section!</h2>595<p>Look at this other amazing section!</p>596</section>597</article>598</body>"""599html_document = HTMLDocument.from_string(html_str)600document_tree = html_document.document_tree601root = html._find_main(document_tree)602assert root.tag == "html"603<body>608<p>Lots preamble stuff yada yada yada</p>609<article>610<h2>A Wonderful Section!</h2>611<p>Look at this amazing section!</p>612</article>613<article>614<h2>Another Wonderful Section!</h2>615<p>Look at this other amazing section!</p>616</article>617</body>"""618html_document = HTMLDocument.from_string(html_str)619document_tree = html_document.document_tree620articles = html._find_articles(document_tree)621assert len(articles) == 2622<body>627<p>Lots preamble stuff yada yada yada</p>628<section>629<h2>A Wonderful Section!</h2>630<p>Look at this amazing section!</p>631</section>632<section>633<h2>Another Wonderful Section!</h2>634<p>Look at this other amazing section!</p>635</section>636</body>"""637html_document = HTMLDocument.from_string(html_str)638document_tree = html_document.document_tree639articles = html._find_articles(document_tree)640assert len(articles) == 1641<body>657<{tag}>658There is some text here.659</{tag}>660</body>661"""662html_document = HTMLDocument.from_string(html_str)663assert len(html_document.pages[0].elements) == 1664<body>670<{tag}>671There is some text here.672</{tag}>673</body>674"""675html_document = HTMLDocument.from_string(html_str)676assert len(html_document.pages) == 0677<body>688<{tag1}>689<{tag2}>690There is some text here.691</{tag2}>692</{tag1}>693</body>694"""695html_document = HTMLDocument.from_string(html_str).doc_after_cleaners(skip_table=False)696assert len(html_document.pages[0].elements) == 1697<div>Get excited for our first annual family day!</div>702<div>Best.<br clear=3D"all">703<div><br></div>704-- <br>705<div dir=3D"ltr">706<div dir=3D"ltr">Dino the Datasaur<div>Unstructured Technologies<br><div>Data Scientist707</div>708<div>Doylestown, PA 18901</div>709<div><br></div>710</div>711</div>712</div>713</div>714</div>"""715html_document = HTMLDocument.from_string(html_str)716<body>731<ol>732<li>Happy Groundhog's day!</li>733<li>Looks like six more weeks of winter ...</li>734</ol>735</body>736</html>"""737html_document = HTMLDocument.from_string(html_str)738<body>748<p>749<span>• Happy Groundhog's day!</span>750</p>751<p>752<span>• Looks like six more weeks of winter ...</span>753</p>754</body>755</html>"""756html_document = HTMLDocument.from_string(html_str)757<body>767<table>768<tbody>769<tr>770<td>•</td>771<td><p>Happy Groundhog's day!</p></td>772</tr>773<tr>774<td>•</td>775<td><p>Looks like six more weeks of winter ...</p></td>776</tr>777</tbody>778</table>779</body>780</html>"""781document_tree = etree.fromstring(doc, etree.HTMLParser())782table = document_tree.find(".//table")783bulleted_text = html._bulleted_text_from_table(table)784assert bulleted_text == [785ListItem(text="Happy Groundhog's day!"),786ListItem(text="Looks like six more weeks of winter ..."),787]788<body>793<table>794<tbody>795<tr>796<td>•</td>797<td><p>Happy Groundhog's day!</p></td>798</tr>799<tr>800<td>•</td>801<td><p>Looks like six more weeks of winter ...</p></td>802</tr>803</tbody>804</table>805</body>806</html>"""807html_document = HTMLDocument.from_string(html_str)808<table><tbody><tr><td>A table thing.</td></tr></tbody></table>833<p>A non-table thing</p>834"""835doc = HTMLDocument.from_string(html_doc)836assert len(doc.elements) == 2837doc.doc_after_cleaners(skip_table=True, inplace=True)838assert len(doc.elements) == 1839<html charset="unicode">851<p>Hello again 😀</p>852</html>853"""854doc = HTMLDocument.from_string(raw_html)855# NOTE(robinson) - unclear why right now, but the output is the emoji on the test runners856# and the byte string representation when running locally on mac857assert doc.elements[0].text in ["Hello again ð\x9f\x98\x80", "Hello again 😀"]858# -- unit-level tests ----------------------------------------------------------------------------889nested_table_html979)980# -- module-level fixtures -----------------------------------------------------------------------991