services/parser/tei.py - angular - boris00043/semantic-papers

services/parser/tei.py
95 строк3 KB
boris
parser service
13 окт 2025, 15:26
13 окт 2025, 15:264ad4e28
from functools import cached_property

import bs4
from pydantic import BaseModel


class TEIAuthor(BaseModel):
    forename: str
    surname: str


class ArticlePart(BaseModel):
    title: str
    content: list[str]


class Article(BaseModel):
    title: str | None
    authors: list[TEIAuthor]
    keywords: list[str]
    body: list[ArticlePart]


class TEIArticleProcessor:
    tei: str

    def __init__(self, tei: str, title: str | None = None) -> None:
        self.tei = tei
        self.soup = bs4.BeautifulSoup(self.tei, "xml")

    def keywords(self) -> list[str]:
        if kws := self.soup.find("keywords"):
            return [i.text for i in kws.children if i.text != "\n"]
        return []

    def title(self) -> str:
        title_tag = self.soup.find("title")
        if title_tag is None:
            return "<<UNKNOWN>>"
        return title_tag.text

    def authors(self) -> list[TEIAuthor]:
        result: list[TEIAuthor] = []
        source_desc = self.soup.find("sourceDesc")
        if source_desc is None:
            return []
        for author in source_desc.find_all("author"):
            persName = author.persName
            if persName is None:
                continue
            forenameTag = persName.find("forename")
            surnameTag = persName.find("surname")
            forename = forenameTag.text if forenameTag else ""
            surname = surnameTag.text if surnameTag else ""
            result.append(TEIAuthor(forename=forename, surname=surname))

        return result

    def body(self) -> list[ArticlePart]:
        body = self.soup.find("body")
        result: list[ArticlePart] = []
        cur = None

        if body is None:
            return []

        for ch0 in body.children:
            if isinstance(ch0, str) and str.isspace(ch0):
                continue
            assert isinstance(ch0, bs4.element.Tag)
            if ch0.name == "figure":
                continue
            for ch1 in ch0.children:
                assert isinstance(ch1, bs4.element.Tag)
                if ch1.name == "head":
                    cur = ArticlePart(title=ch1.text, content=[])
                    result.append(cur)
                else:
                    if cur is None:
                        cur = ArticlePart(title="", content=[])
                        result.append(cur)

                    cur.content.append(ch1.text)

        return result


def tei_to_article(tei: str) -> Article:
    processor = TEIArticleProcessor(tei)
    return Article(
        title=processor.title(),
        authors=processor.authors(),
        keywords=processor.keywords(),
        body=processor.body(),
    )